Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2018 Intel Corporation
4 */
5
6#include "gem/i915_gem_internal.h"
7#include "gem/i915_gem_pm.h"
8#include "gt/intel_engine_user.h"
9#include "gt/intel_gt.h"
10#include "i915_selftest.h"
11#include "intel_reset.h"
12
13#include "selftests/igt_flush_test.h"
14#include "selftests/igt_reset.h"
15#include "selftests/igt_spinner.h"
16#include "selftests/intel_scheduler_helpers.h"
17#include "selftests/mock_drm.h"
18
19#include "gem/selftests/igt_gem_utils.h"
20#include "gem/selftests/mock_context.h"
21
22static const struct wo_register {
23 enum intel_platform platform;
24 u32 reg;
25} wo_registers[] = {
26 { INTEL_GEMINILAKE, 0x731c }
27};
28
29struct wa_lists {
30 struct i915_wa_list gt_wa_list;
31 struct {
32 struct i915_wa_list wa_list;
33 struct i915_wa_list ctx_wa_list;
34 } engine[I915_NUM_ENGINES];
35};
36
37static int request_add_sync(struct i915_request *rq, int err)
38{
39 i915_request_get(rq);
40 i915_request_add(rq);
41 if (i915_request_wait(rq, 0, HZ / 5) < 0)
42 err = -EIO;
43 i915_request_put(rq);
44
45 return err;
46}
47
48static int request_add_spin(struct i915_request *rq, struct igt_spinner *spin)
49{
50 int err = 0;
51
52 i915_request_get(rq);
53 i915_request_add(rq);
54 if (spin && !igt_wait_for_spinner(spin, rq))
55 err = -ETIMEDOUT;
56 i915_request_put(rq);
57
58 return err;
59}
60
61static void
62reference_lists_init(struct intel_gt *gt, struct wa_lists *lists)
63{
64 struct intel_engine_cs *engine;
65 enum intel_engine_id id;
66
67 memset(lists, 0, sizeof(*lists));
68
69 wa_init_start(&lists->gt_wa_list, gt, "GT_REF", "global");
70 gt_init_workarounds(gt, &lists->gt_wa_list);
71 wa_init_finish(&lists->gt_wa_list);
72
73 for_each_engine(engine, gt, id) {
74 struct i915_wa_list *wal = &lists->engine[id].wa_list;
75
76 wa_init_start(wal, gt, "REF", engine->name);
77 engine_init_workarounds(engine, wal);
78 wa_init_finish(wal);
79
80 __intel_engine_init_ctx_wa(engine,
81 &lists->engine[id].ctx_wa_list,
82 "CTX_REF");
83 }
84}
85
86static void
87reference_lists_fini(struct intel_gt *gt, struct wa_lists *lists)
88{
89 struct intel_engine_cs *engine;
90 enum intel_engine_id id;
91
92 for_each_engine(engine, gt, id)
93 intel_wa_list_free(&lists->engine[id].wa_list);
94
95 intel_wa_list_free(&lists->gt_wa_list);
96}
97
98static struct drm_i915_gem_object *
99read_nonprivs(struct intel_context *ce)
100{
101 struct intel_engine_cs *engine = ce->engine;
102 const u32 base = engine->mmio_base;
103 struct drm_i915_gem_object *result;
104 struct i915_request *rq;
105 struct i915_vma *vma;
106 u32 srm, *cs;
107 int err;
108 int i;
109
110 result = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
111 if (IS_ERR(result))
112 return result;
113
114 i915_gem_object_set_cache_coherency(result, I915_CACHE_LLC);
115
116 cs = i915_gem_object_pin_map_unlocked(result, I915_MAP_WB);
117 if (IS_ERR(cs)) {
118 err = PTR_ERR(cs);
119 goto err_obj;
120 }
121 memset(cs, 0xc5, PAGE_SIZE);
122 i915_gem_object_flush_map(result);
123 i915_gem_object_unpin_map(result);
124
125 vma = i915_vma_instance(result, &engine->gt->ggtt->vm, NULL);
126 if (IS_ERR(vma)) {
127 err = PTR_ERR(vma);
128 goto err_obj;
129 }
130
131 err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
132 if (err)
133 goto err_obj;
134
135 rq = intel_context_create_request(ce);
136 if (IS_ERR(rq)) {
137 err = PTR_ERR(rq);
138 goto err_pin;
139 }
140
141 i915_vma_lock(vma);
142 err = i915_vma_move_to_active(vma, rq, EXEC_OBJECT_WRITE);
143 i915_vma_unlock(vma);
144 if (err)
145 goto err_req;
146
147 srm = MI_STORE_REGISTER_MEM | MI_SRM_LRM_GLOBAL_GTT;
148 if (GRAPHICS_VER(engine->i915) >= 8)
149 srm++;
150
151 cs = intel_ring_begin(rq, 4 * RING_MAX_NONPRIV_SLOTS);
152 if (IS_ERR(cs)) {
153 err = PTR_ERR(cs);
154 goto err_req;
155 }
156
157 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
158 *cs++ = srm;
159 *cs++ = i915_mmio_reg_offset(RING_FORCE_TO_NONPRIV(base, i));
160 *cs++ = i915_ggtt_offset(vma) + sizeof(u32) * i;
161 *cs++ = 0;
162 }
163 intel_ring_advance(rq, cs);
164
165 i915_request_add(rq);
166 i915_vma_unpin(vma);
167
168 return result;
169
170err_req:
171 i915_request_add(rq);
172err_pin:
173 i915_vma_unpin(vma);
174err_obj:
175 i915_gem_object_put(result);
176 return ERR_PTR(err);
177}
178
179static u32
180get_whitelist_reg(const struct intel_engine_cs *engine, unsigned int i)
181{
182 i915_reg_t reg = i < engine->whitelist.count ?
183 engine->whitelist.list[i].reg :
184 RING_NOPID(engine->mmio_base);
185
186 return i915_mmio_reg_offset(reg);
187}
188
189static void
190print_results(const struct intel_engine_cs *engine, const u32 *results)
191{
192 unsigned int i;
193
194 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
195 u32 expected = get_whitelist_reg(engine, i);
196 u32 actual = results[i];
197
198 pr_info("RING_NONPRIV[%d]: expected 0x%08x, found 0x%08x\n",
199 i, expected, actual);
200 }
201}
202
203static int check_whitelist(struct intel_context *ce)
204{
205 struct intel_engine_cs *engine = ce->engine;
206 struct drm_i915_gem_object *results;
207 struct intel_wedge_me wedge;
208 u32 *vaddr;
209 int err;
210 int i;
211
212 results = read_nonprivs(ce);
213 if (IS_ERR(results))
214 return PTR_ERR(results);
215
216 err = 0;
217 i915_gem_object_lock(results, NULL);
218 intel_wedge_on_timeout(&wedge, engine->gt, HZ / 5) /* safety net! */
219 err = i915_gem_object_set_to_cpu_domain(results, false);
220
221 if (intel_gt_is_wedged(engine->gt))
222 err = -EIO;
223 if (err)
224 goto out_put;
225
226 vaddr = i915_gem_object_pin_map(results, I915_MAP_WB);
227 if (IS_ERR(vaddr)) {
228 err = PTR_ERR(vaddr);
229 goto out_put;
230 }
231
232 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++) {
233 u32 expected = get_whitelist_reg(engine, i);
234 u32 actual = vaddr[i];
235
236 if (expected != actual) {
237 print_results(engine, vaddr);
238 pr_err("Invalid RING_NONPRIV[%d], expected 0x%08x, found 0x%08x\n",
239 i, expected, actual);
240
241 err = -EINVAL;
242 break;
243 }
244 }
245
246 i915_gem_object_unpin_map(results);
247out_put:
248 i915_gem_object_unlock(results);
249 i915_gem_object_put(results);
250 return err;
251}
252
253static int do_device_reset(struct intel_engine_cs *engine)
254{
255 intel_gt_reset(engine->gt, engine->mask, "live_workarounds");
256 return 0;
257}
258
259static int do_engine_reset(struct intel_engine_cs *engine)
260{
261 return intel_engine_reset(engine, "live_workarounds");
262}
263
264static int do_guc_reset(struct intel_engine_cs *engine)
265{
266 /* Currently a no-op as the reset is handled by GuC */
267 return 0;
268}
269
270static int
271switch_to_scratch_context(struct intel_engine_cs *engine,
272 struct igt_spinner *spin,
273 struct i915_request **rq)
274{
275 struct intel_context *ce;
276 int err = 0;
277
278 ce = intel_context_create(engine);
279 if (IS_ERR(ce))
280 return PTR_ERR(ce);
281
282 *rq = igt_spinner_create_request(spin, ce, MI_NOOP);
283 intel_context_put(ce);
284
285 if (IS_ERR(*rq)) {
286 spin = NULL;
287 err = PTR_ERR(*rq);
288 goto err;
289 }
290
291 err = request_add_spin(*rq, spin);
292err:
293 if (err && spin)
294 igt_spinner_end(spin);
295
296 return err;
297}
298
299static int check_whitelist_across_reset(struct intel_engine_cs *engine,
300 int (*reset)(struct intel_engine_cs *),
301 const char *name)
302{
303 struct intel_context *ce, *tmp;
304 struct igt_spinner spin;
305 struct i915_request *rq;
306 intel_wakeref_t wakeref;
307 int err;
308
309 pr_info("Checking %d whitelisted registers on %s (RING_NONPRIV) [%s]\n",
310 engine->whitelist.count, engine->name, name);
311
312 ce = intel_context_create(engine);
313 if (IS_ERR(ce))
314 return PTR_ERR(ce);
315
316 err = igt_spinner_init(&spin, engine->gt);
317 if (err)
318 goto out_ctx;
319
320 err = check_whitelist(ce);
321 if (err) {
322 pr_err("Invalid whitelist *before* %s reset!\n", name);
323 goto out_spin;
324 }
325
326 err = switch_to_scratch_context(engine, &spin, &rq);
327 if (err)
328 goto out_spin;
329
330 /* Ensure the spinner hasn't aborted */
331 if (i915_request_completed(rq)) {
332 pr_err("%s spinner failed to start\n", name);
333 err = -ETIMEDOUT;
334 goto out_spin;
335 }
336
337 with_intel_runtime_pm(engine->uncore->rpm, wakeref)
338 err = reset(engine);
339
340 /* Ensure the reset happens and kills the engine */
341 if (err == 0)
342 err = intel_selftest_wait_for_rq(rq);
343
344 igt_spinner_end(&spin);
345
346 if (err) {
347 pr_err("%s reset failed\n", name);
348 goto out_spin;
349 }
350
351 err = check_whitelist(ce);
352 if (err) {
353 pr_err("Whitelist not preserved in context across %s reset!\n",
354 name);
355 goto out_spin;
356 }
357
358 tmp = intel_context_create(engine);
359 if (IS_ERR(tmp)) {
360 err = PTR_ERR(tmp);
361 goto out_spin;
362 }
363 intel_context_put(ce);
364 ce = tmp;
365
366 err = check_whitelist(ce);
367 if (err) {
368 pr_err("Invalid whitelist *after* %s reset in fresh context!\n",
369 name);
370 goto out_spin;
371 }
372
373out_spin:
374 igt_spinner_fini(&spin);
375out_ctx:
376 intel_context_put(ce);
377 return err;
378}
379
380static struct i915_vma *create_batch(struct i915_address_space *vm)
381{
382 struct drm_i915_gem_object *obj;
383 struct i915_vma *vma;
384 int err;
385
386 obj = i915_gem_object_create_internal(vm->i915, 16 * PAGE_SIZE);
387 if (IS_ERR(obj))
388 return ERR_CAST(obj);
389
390 vma = i915_vma_instance(obj, vm, NULL);
391 if (IS_ERR(vma)) {
392 err = PTR_ERR(vma);
393 goto err_obj;
394 }
395
396 err = i915_vma_pin(vma, 0, 0, PIN_USER);
397 if (err)
398 goto err_obj;
399
400 return vma;
401
402err_obj:
403 i915_gem_object_put(obj);
404 return ERR_PTR(err);
405}
406
407static u32 reg_write(u32 old, u32 new, u32 rsvd)
408{
409 if (rsvd == 0x0000ffff) {
410 old &= ~(new >> 16);
411 old |= new & (new >> 16);
412 } else {
413 old &= ~rsvd;
414 old |= new & rsvd;
415 }
416
417 return old;
418}
419
420static bool wo_register(struct intel_engine_cs *engine, u32 reg)
421{
422 enum intel_platform platform = INTEL_INFO(engine->i915)->platform;
423 int i;
424
425 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
426 RING_FORCE_TO_NONPRIV_ACCESS_WR)
427 return true;
428
429 for (i = 0; i < ARRAY_SIZE(wo_registers); i++) {
430 if (wo_registers[i].platform == platform &&
431 wo_registers[i].reg == reg)
432 return true;
433 }
434
435 return false;
436}
437
438static bool timestamp(const struct intel_engine_cs *engine, u32 reg)
439{
440 reg = (reg - engine->mmio_base) & ~RING_FORCE_TO_NONPRIV_ACCESS_MASK;
441 switch (reg) {
442 case 0x358:
443 case 0x35c:
444 case 0x3a8:
445 return true;
446
447 default:
448 return false;
449 }
450}
451
452static bool ro_register(u32 reg)
453{
454 if ((reg & RING_FORCE_TO_NONPRIV_ACCESS_MASK) ==
455 RING_FORCE_TO_NONPRIV_ACCESS_RD)
456 return true;
457
458 return false;
459}
460
461static int whitelist_writable_count(struct intel_engine_cs *engine)
462{
463 int count = engine->whitelist.count;
464 int i;
465
466 for (i = 0; i < engine->whitelist.count; i++) {
467 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
468
469 if (ro_register(reg))
470 count--;
471 }
472
473 return count;
474}
475
476static int check_dirty_whitelist(struct intel_context *ce)
477{
478 const u32 values[] = {
479 0x00000000,
480 0x01010101,
481 0x10100101,
482 0x03030303,
483 0x30300303,
484 0x05050505,
485 0x50500505,
486 0x0f0f0f0f,
487 0xf00ff00f,
488 0x10101010,
489 0xf0f01010,
490 0x30303030,
491 0xa0a03030,
492 0x50505050,
493 0xc0c05050,
494 0xf0f0f0f0,
495 0x11111111,
496 0x33333333,
497 0x55555555,
498 0x0000ffff,
499 0x00ff00ff,
500 0xff0000ff,
501 0xffff00ff,
502 0xffffffff,
503 };
504 struct intel_engine_cs *engine = ce->engine;
505 struct i915_vma *scratch;
506 struct i915_vma *batch;
507 int err = 0, i, v, sz;
508 u32 *cs, *results;
509
510 sz = (2 * ARRAY_SIZE(values) + 1) * sizeof(u32);
511 scratch = __vm_create_scratch_for_read_pinned(ce->vm, sz);
512 if (IS_ERR(scratch))
513 return PTR_ERR(scratch);
514
515 batch = create_batch(ce->vm);
516 if (IS_ERR(batch)) {
517 err = PTR_ERR(batch);
518 goto out_scratch;
519 }
520
521 for (i = 0; i < engine->whitelist.count; i++) {
522 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
523 struct i915_gem_ww_ctx ww;
524 u64 addr = scratch->node.start;
525 struct i915_request *rq;
526 u32 srm, lrm, rsvd;
527 u32 expect;
528 int idx;
529 bool ro_reg;
530
531 if (wo_register(engine, reg))
532 continue;
533
534 if (timestamp(engine, reg))
535 continue; /* timestamps are expected to autoincrement */
536
537 ro_reg = ro_register(reg);
538
539 i915_gem_ww_ctx_init(&ww, false);
540retry:
541 cs = NULL;
542 err = i915_gem_object_lock(scratch->obj, &ww);
543 if (!err)
544 err = i915_gem_object_lock(batch->obj, &ww);
545 if (!err)
546 err = intel_context_pin_ww(ce, &ww);
547 if (err)
548 goto out;
549
550 cs = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
551 if (IS_ERR(cs)) {
552 err = PTR_ERR(cs);
553 goto out_ctx;
554 }
555
556 results = i915_gem_object_pin_map(scratch->obj, I915_MAP_WB);
557 if (IS_ERR(results)) {
558 err = PTR_ERR(results);
559 goto out_unmap_batch;
560 }
561
562 /* Clear non priv flags */
563 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
564
565 srm = MI_STORE_REGISTER_MEM;
566 lrm = MI_LOAD_REGISTER_MEM;
567 if (GRAPHICS_VER(engine->i915) >= 8)
568 lrm++, srm++;
569
570 pr_debug("%s: Writing garbage to %x\n",
571 engine->name, reg);
572
573 /* SRM original */
574 *cs++ = srm;
575 *cs++ = reg;
576 *cs++ = lower_32_bits(addr);
577 *cs++ = upper_32_bits(addr);
578
579 idx = 1;
580 for (v = 0; v < ARRAY_SIZE(values); v++) {
581 /* LRI garbage */
582 *cs++ = MI_LOAD_REGISTER_IMM(1);
583 *cs++ = reg;
584 *cs++ = values[v];
585
586 /* SRM result */
587 *cs++ = srm;
588 *cs++ = reg;
589 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
590 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
591 idx++;
592 }
593 for (v = 0; v < ARRAY_SIZE(values); v++) {
594 /* LRI garbage */
595 *cs++ = MI_LOAD_REGISTER_IMM(1);
596 *cs++ = reg;
597 *cs++ = ~values[v];
598
599 /* SRM result */
600 *cs++ = srm;
601 *cs++ = reg;
602 *cs++ = lower_32_bits(addr + sizeof(u32) * idx);
603 *cs++ = upper_32_bits(addr + sizeof(u32) * idx);
604 idx++;
605 }
606 GEM_BUG_ON(idx * sizeof(u32) > scratch->size);
607
608 /* LRM original -- don't leave garbage in the context! */
609 *cs++ = lrm;
610 *cs++ = reg;
611 *cs++ = lower_32_bits(addr);
612 *cs++ = upper_32_bits(addr);
613
614 *cs++ = MI_BATCH_BUFFER_END;
615
616 i915_gem_object_flush_map(batch->obj);
617 i915_gem_object_unpin_map(batch->obj);
618 intel_gt_chipset_flush(engine->gt);
619 cs = NULL;
620
621 rq = i915_request_create(ce);
622 if (IS_ERR(rq)) {
623 err = PTR_ERR(rq);
624 goto out_unmap_scratch;
625 }
626
627 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
628 err = engine->emit_init_breadcrumb(rq);
629 if (err)
630 goto err_request;
631 }
632
633 err = i915_vma_move_to_active(batch, rq, 0);
634 if (err)
635 goto err_request;
636
637 err = i915_vma_move_to_active(scratch, rq,
638 EXEC_OBJECT_WRITE);
639 if (err)
640 goto err_request;
641
642 err = engine->emit_bb_start(rq,
643 batch->node.start, PAGE_SIZE,
644 0);
645 if (err)
646 goto err_request;
647
648err_request:
649 err = request_add_sync(rq, err);
650 if (err) {
651 pr_err("%s: Futzing %x timedout; cancelling test\n",
652 engine->name, reg);
653 intel_gt_set_wedged(engine->gt);
654 goto out_unmap_scratch;
655 }
656
657 GEM_BUG_ON(values[ARRAY_SIZE(values) - 1] != 0xffffffff);
658 if (!ro_reg) {
659 /* detect write masking */
660 rsvd = results[ARRAY_SIZE(values)];
661 if (!rsvd) {
662 pr_err("%s: Unable to write to whitelisted register %x\n",
663 engine->name, reg);
664 err = -EINVAL;
665 goto out_unmap_scratch;
666 }
667 } else {
668 rsvd = 0;
669 }
670
671 expect = results[0];
672 idx = 1;
673 for (v = 0; v < ARRAY_SIZE(values); v++) {
674 if (ro_reg)
675 expect = results[0];
676 else
677 expect = reg_write(expect, values[v], rsvd);
678
679 if (results[idx] != expect)
680 err++;
681 idx++;
682 }
683 for (v = 0; v < ARRAY_SIZE(values); v++) {
684 if (ro_reg)
685 expect = results[0];
686 else
687 expect = reg_write(expect, ~values[v], rsvd);
688
689 if (results[idx] != expect)
690 err++;
691 idx++;
692 }
693 if (err) {
694 pr_err("%s: %d mismatch between values written to whitelisted register [%x], and values read back!\n",
695 engine->name, err, reg);
696
697 if (ro_reg)
698 pr_info("%s: Whitelisted read-only register: %x, original value %08x\n",
699 engine->name, reg, results[0]);
700 else
701 pr_info("%s: Whitelisted register: %x, original value %08x, rsvd %08x\n",
702 engine->name, reg, results[0], rsvd);
703
704 expect = results[0];
705 idx = 1;
706 for (v = 0; v < ARRAY_SIZE(values); v++) {
707 u32 w = values[v];
708
709 if (ro_reg)
710 expect = results[0];
711 else
712 expect = reg_write(expect, w, rsvd);
713 pr_info("Wrote %08x, read %08x, expect %08x\n",
714 w, results[idx], expect);
715 idx++;
716 }
717 for (v = 0; v < ARRAY_SIZE(values); v++) {
718 u32 w = ~values[v];
719
720 if (ro_reg)
721 expect = results[0];
722 else
723 expect = reg_write(expect, w, rsvd);
724 pr_info("Wrote %08x, read %08x, expect %08x\n",
725 w, results[idx], expect);
726 idx++;
727 }
728
729 err = -EINVAL;
730 }
731out_unmap_scratch:
732 i915_gem_object_unpin_map(scratch->obj);
733out_unmap_batch:
734 if (cs)
735 i915_gem_object_unpin_map(batch->obj);
736out_ctx:
737 intel_context_unpin(ce);
738out:
739 if (err == -EDEADLK) {
740 err = i915_gem_ww_ctx_backoff(&ww);
741 if (!err)
742 goto retry;
743 }
744 i915_gem_ww_ctx_fini(&ww);
745 if (err)
746 break;
747 }
748
749 if (igt_flush_test(engine->i915))
750 err = -EIO;
751
752 i915_vma_unpin_and_release(&batch, 0);
753out_scratch:
754 i915_vma_unpin_and_release(&scratch, 0);
755 return err;
756}
757
758static int live_dirty_whitelist(void *arg)
759{
760 struct intel_gt *gt = arg;
761 struct intel_engine_cs *engine;
762 enum intel_engine_id id;
763
764 /* Can the user write to the whitelisted registers? */
765
766 if (GRAPHICS_VER(gt->i915) < 7) /* minimum requirement for LRI, SRM, LRM */
767 return 0;
768
769 for_each_engine(engine, gt, id) {
770 struct intel_context *ce;
771 int err;
772
773 if (engine->whitelist.count == 0)
774 continue;
775
776 ce = intel_context_create(engine);
777 if (IS_ERR(ce))
778 return PTR_ERR(ce);
779
780 err = check_dirty_whitelist(ce);
781 intel_context_put(ce);
782 if (err)
783 return err;
784 }
785
786 return 0;
787}
788
789static int live_reset_whitelist(void *arg)
790{
791 struct intel_gt *gt = arg;
792 struct intel_engine_cs *engine;
793 enum intel_engine_id id;
794 int err = 0;
795
796 /* If we reset the gpu, we should not lose the RING_NONPRIV */
797 igt_global_reset_lock(gt);
798
799 for_each_engine(engine, gt, id) {
800 if (engine->whitelist.count == 0)
801 continue;
802
803 if (intel_has_reset_engine(gt)) {
804 if (intel_engine_uses_guc(engine)) {
805 struct intel_selftest_saved_policy saved;
806 int err2;
807
808 err = intel_selftest_modify_policy(engine, &saved,
809 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
810 if (err)
811 goto out;
812
813 err = check_whitelist_across_reset(engine,
814 do_guc_reset,
815 "guc");
816
817 err2 = intel_selftest_restore_policy(engine, &saved);
818 if (err == 0)
819 err = err2;
820 } else {
821 err = check_whitelist_across_reset(engine,
822 do_engine_reset,
823 "engine");
824 }
825
826 if (err)
827 goto out;
828 }
829
830 if (intel_has_gpu_reset(gt)) {
831 err = check_whitelist_across_reset(engine,
832 do_device_reset,
833 "device");
834 if (err)
835 goto out;
836 }
837 }
838
839out:
840 igt_global_reset_unlock(gt);
841 return err;
842}
843
844static int read_whitelisted_registers(struct intel_context *ce,
845 struct i915_vma *results)
846{
847 struct intel_engine_cs *engine = ce->engine;
848 struct i915_request *rq;
849 int i, err = 0;
850 u32 srm, *cs;
851
852 rq = intel_context_create_request(ce);
853 if (IS_ERR(rq))
854 return PTR_ERR(rq);
855
856 i915_vma_lock(results);
857 err = i915_vma_move_to_active(results, rq, EXEC_OBJECT_WRITE);
858 i915_vma_unlock(results);
859 if (err)
860 goto err_req;
861
862 srm = MI_STORE_REGISTER_MEM;
863 if (GRAPHICS_VER(engine->i915) >= 8)
864 srm++;
865
866 cs = intel_ring_begin(rq, 4 * engine->whitelist.count);
867 if (IS_ERR(cs)) {
868 err = PTR_ERR(cs);
869 goto err_req;
870 }
871
872 for (i = 0; i < engine->whitelist.count; i++) {
873 u64 offset = results->node.start + sizeof(u32) * i;
874 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
875
876 /* Clear non priv flags */
877 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
878
879 *cs++ = srm;
880 *cs++ = reg;
881 *cs++ = lower_32_bits(offset);
882 *cs++ = upper_32_bits(offset);
883 }
884 intel_ring_advance(rq, cs);
885
886err_req:
887 return request_add_sync(rq, err);
888}
889
890static int scrub_whitelisted_registers(struct intel_context *ce)
891{
892 struct intel_engine_cs *engine = ce->engine;
893 struct i915_request *rq;
894 struct i915_vma *batch;
895 int i, err = 0;
896 u32 *cs;
897
898 batch = create_batch(ce->vm);
899 if (IS_ERR(batch))
900 return PTR_ERR(batch);
901
902 cs = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
903 if (IS_ERR(cs)) {
904 err = PTR_ERR(cs);
905 goto err_batch;
906 }
907
908 *cs++ = MI_LOAD_REGISTER_IMM(whitelist_writable_count(engine));
909 for (i = 0; i < engine->whitelist.count; i++) {
910 u32 reg = i915_mmio_reg_offset(engine->whitelist.list[i].reg);
911
912 if (ro_register(reg))
913 continue;
914
915 /* Clear non priv flags */
916 reg &= RING_FORCE_TO_NONPRIV_ADDRESS_MASK;
917
918 *cs++ = reg;
919 *cs++ = 0xffffffff;
920 }
921 *cs++ = MI_BATCH_BUFFER_END;
922
923 i915_gem_object_flush_map(batch->obj);
924 intel_gt_chipset_flush(engine->gt);
925
926 rq = intel_context_create_request(ce);
927 if (IS_ERR(rq)) {
928 err = PTR_ERR(rq);
929 goto err_unpin;
930 }
931
932 if (engine->emit_init_breadcrumb) { /* Be nice if we hang */
933 err = engine->emit_init_breadcrumb(rq);
934 if (err)
935 goto err_request;
936 }
937
938 i915_vma_lock(batch);
939 err = i915_vma_move_to_active(batch, rq, 0);
940 i915_vma_unlock(batch);
941 if (err)
942 goto err_request;
943
944 /* Perform the writes from an unprivileged "user" batch */
945 err = engine->emit_bb_start(rq, batch->node.start, 0, 0);
946
947err_request:
948 err = request_add_sync(rq, err);
949
950err_unpin:
951 i915_gem_object_unpin_map(batch->obj);
952err_batch:
953 i915_vma_unpin_and_release(&batch, 0);
954 return err;
955}
956
957struct regmask {
958 i915_reg_t reg;
959 u8 graphics_ver;
960};
961
962static bool find_reg(struct drm_i915_private *i915,
963 i915_reg_t reg,
964 const struct regmask *tbl,
965 unsigned long count)
966{
967 u32 offset = i915_mmio_reg_offset(reg);
968
969 while (count--) {
970 if (GRAPHICS_VER(i915) == tbl->graphics_ver &&
971 i915_mmio_reg_offset(tbl->reg) == offset)
972 return true;
973 tbl++;
974 }
975
976 return false;
977}
978
979static bool pardon_reg(struct drm_i915_private *i915, i915_reg_t reg)
980{
981 /* Alas, we must pardon some whitelists. Mistakes already made */
982 static const struct regmask pardon[] = {
983 { GEN9_CTX_PREEMPT_REG, 9 },
984 { _MMIO(0xb118), 9 }, /* GEN8_L3SQCREG4 */
985 };
986
987 return find_reg(i915, reg, pardon, ARRAY_SIZE(pardon));
988}
989
990static bool result_eq(struct intel_engine_cs *engine,
991 u32 a, u32 b, i915_reg_t reg)
992{
993 if (a != b && !pardon_reg(engine->i915, reg)) {
994 pr_err("Whitelisted register 0x%4x not context saved: A=%08x, B=%08x\n",
995 i915_mmio_reg_offset(reg), a, b);
996 return false;
997 }
998
999 return true;
1000}
1001
1002static bool writeonly_reg(struct drm_i915_private *i915, i915_reg_t reg)
1003{
1004 /* Some registers do not seem to behave and our writes unreadable */
1005 static const struct regmask wo[] = {
1006 { GEN9_SLICE_COMMON_ECO_CHICKEN1, 9 },
1007 };
1008
1009 return find_reg(i915, reg, wo, ARRAY_SIZE(wo));
1010}
1011
1012static bool result_neq(struct intel_engine_cs *engine,
1013 u32 a, u32 b, i915_reg_t reg)
1014{
1015 if (a == b && !writeonly_reg(engine->i915, reg)) {
1016 pr_err("Whitelist register 0x%4x:%08x was unwritable\n",
1017 i915_mmio_reg_offset(reg), a);
1018 return false;
1019 }
1020
1021 return true;
1022}
1023
1024static int
1025check_whitelisted_registers(struct intel_engine_cs *engine,
1026 struct i915_vma *A,
1027 struct i915_vma *B,
1028 bool (*fn)(struct intel_engine_cs *engine,
1029 u32 a, u32 b,
1030 i915_reg_t reg))
1031{
1032 u32 *a, *b;
1033 int i, err;
1034
1035 a = i915_gem_object_pin_map_unlocked(A->obj, I915_MAP_WB);
1036 if (IS_ERR(a))
1037 return PTR_ERR(a);
1038
1039 b = i915_gem_object_pin_map_unlocked(B->obj, I915_MAP_WB);
1040 if (IS_ERR(b)) {
1041 err = PTR_ERR(b);
1042 goto err_a;
1043 }
1044
1045 err = 0;
1046 for (i = 0; i < engine->whitelist.count; i++) {
1047 const struct i915_wa *wa = &engine->whitelist.list[i];
1048
1049 if (i915_mmio_reg_offset(wa->reg) &
1050 RING_FORCE_TO_NONPRIV_ACCESS_RD)
1051 continue;
1052
1053 if (!fn(engine, a[i], b[i], wa->reg))
1054 err = -EINVAL;
1055 }
1056
1057 i915_gem_object_unpin_map(B->obj);
1058err_a:
1059 i915_gem_object_unpin_map(A->obj);
1060 return err;
1061}
1062
1063static int live_isolated_whitelist(void *arg)
1064{
1065 struct intel_gt *gt = arg;
1066 struct {
1067 struct i915_vma *scratch[2];
1068 } client[2] = {};
1069 struct intel_engine_cs *engine;
1070 enum intel_engine_id id;
1071 int i, err = 0;
1072
1073 /*
1074 * Check that a write into a whitelist register works, but
1075 * invisible to a second context.
1076 */
1077
1078 if (!intel_engines_has_context_isolation(gt->i915))
1079 return 0;
1080
1081 for (i = 0; i < ARRAY_SIZE(client); i++) {
1082 client[i].scratch[0] =
1083 __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1084 if (IS_ERR(client[i].scratch[0])) {
1085 err = PTR_ERR(client[i].scratch[0]);
1086 goto err;
1087 }
1088
1089 client[i].scratch[1] =
1090 __vm_create_scratch_for_read_pinned(gt->vm, 4096);
1091 if (IS_ERR(client[i].scratch[1])) {
1092 err = PTR_ERR(client[i].scratch[1]);
1093 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1094 goto err;
1095 }
1096 }
1097
1098 for_each_engine(engine, gt, id) {
1099 struct intel_context *ce[2];
1100
1101 if (!engine->kernel_context->vm)
1102 continue;
1103
1104 if (!whitelist_writable_count(engine))
1105 continue;
1106
1107 ce[0] = intel_context_create(engine);
1108 if (IS_ERR(ce[0])) {
1109 err = PTR_ERR(ce[0]);
1110 break;
1111 }
1112 ce[1] = intel_context_create(engine);
1113 if (IS_ERR(ce[1])) {
1114 err = PTR_ERR(ce[1]);
1115 intel_context_put(ce[0]);
1116 break;
1117 }
1118
1119 /* Read default values */
1120 err = read_whitelisted_registers(ce[0], client[0].scratch[0]);
1121 if (err)
1122 goto err_ce;
1123
1124 /* Try to overwrite registers (should only affect ctx0) */
1125 err = scrub_whitelisted_registers(ce[0]);
1126 if (err)
1127 goto err_ce;
1128
1129 /* Read values from ctx1, we expect these to be defaults */
1130 err = read_whitelisted_registers(ce[1], client[1].scratch[0]);
1131 if (err)
1132 goto err_ce;
1133
1134 /* Verify that both reads return the same default values */
1135 err = check_whitelisted_registers(engine,
1136 client[0].scratch[0],
1137 client[1].scratch[0],
1138 result_eq);
1139 if (err)
1140 goto err_ce;
1141
1142 /* Read back the updated values in ctx0 */
1143 err = read_whitelisted_registers(ce[0], client[0].scratch[1]);
1144 if (err)
1145 goto err_ce;
1146
1147 /* User should be granted privilege to overwhite regs */
1148 err = check_whitelisted_registers(engine,
1149 client[0].scratch[0],
1150 client[0].scratch[1],
1151 result_neq);
1152err_ce:
1153 intel_context_put(ce[1]);
1154 intel_context_put(ce[0]);
1155 if (err)
1156 break;
1157 }
1158
1159err:
1160 for (i = 0; i < ARRAY_SIZE(client); i++) {
1161 i915_vma_unpin_and_release(&client[i].scratch[1], 0);
1162 i915_vma_unpin_and_release(&client[i].scratch[0], 0);
1163 }
1164
1165 if (igt_flush_test(gt->i915))
1166 err = -EIO;
1167
1168 return err;
1169}
1170
1171static bool
1172verify_wa_lists(struct intel_gt *gt, struct wa_lists *lists,
1173 const char *str)
1174{
1175 struct intel_engine_cs *engine;
1176 enum intel_engine_id id;
1177 bool ok = true;
1178
1179 ok &= wa_list_verify(gt, &lists->gt_wa_list, str);
1180
1181 for_each_engine(engine, gt, id) {
1182 struct intel_context *ce;
1183
1184 ce = intel_context_create(engine);
1185 if (IS_ERR(ce))
1186 return false;
1187
1188 ok &= engine_wa_list_verify(ce,
1189 &lists->engine[id].wa_list,
1190 str) == 0;
1191
1192 ok &= engine_wa_list_verify(ce,
1193 &lists->engine[id].ctx_wa_list,
1194 str) == 0;
1195
1196 intel_context_put(ce);
1197 }
1198
1199 return ok;
1200}
1201
1202static int
1203live_gpu_reset_workarounds(void *arg)
1204{
1205 struct intel_gt *gt = arg;
1206 intel_wakeref_t wakeref;
1207 struct wa_lists *lists;
1208 bool ok;
1209
1210 if (!intel_has_gpu_reset(gt))
1211 return 0;
1212
1213 lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1214 if (!lists)
1215 return -ENOMEM;
1216
1217 pr_info("Verifying after GPU reset...\n");
1218
1219 igt_global_reset_lock(gt);
1220 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1221
1222 reference_lists_init(gt, lists);
1223
1224 ok = verify_wa_lists(gt, lists, "before reset");
1225 if (!ok)
1226 goto out;
1227
1228 intel_gt_reset(gt, ALL_ENGINES, "live_workarounds");
1229
1230 ok = verify_wa_lists(gt, lists, "after reset");
1231
1232out:
1233 reference_lists_fini(gt, lists);
1234 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1235 igt_global_reset_unlock(gt);
1236 kfree(lists);
1237
1238 return ok ? 0 : -ESRCH;
1239}
1240
1241static int
1242live_engine_reset_workarounds(void *arg)
1243{
1244 struct intel_gt *gt = arg;
1245 struct intel_engine_cs *engine;
1246 enum intel_engine_id id;
1247 struct intel_context *ce;
1248 struct igt_spinner spin;
1249 struct i915_request *rq;
1250 intel_wakeref_t wakeref;
1251 struct wa_lists *lists;
1252 int ret = 0;
1253
1254 if (!intel_has_reset_engine(gt))
1255 return 0;
1256
1257 lists = kzalloc(sizeof(*lists), GFP_KERNEL);
1258 if (!lists)
1259 return -ENOMEM;
1260
1261 igt_global_reset_lock(gt);
1262 wakeref = intel_runtime_pm_get(gt->uncore->rpm);
1263
1264 reference_lists_init(gt, lists);
1265
1266 for_each_engine(engine, gt, id) {
1267 struct intel_selftest_saved_policy saved;
1268 bool using_guc = intel_engine_uses_guc(engine);
1269 bool ok;
1270 int ret2;
1271
1272 pr_info("Verifying after %s reset...\n", engine->name);
1273 ret = intel_selftest_modify_policy(engine, &saved,
1274 SELFTEST_SCHEDULER_MODIFY_FAST_RESET);
1275 if (ret)
1276 break;
1277
1278 ce = intel_context_create(engine);
1279 if (IS_ERR(ce)) {
1280 ret = PTR_ERR(ce);
1281 goto restore;
1282 }
1283
1284 if (!using_guc) {
1285 ok = verify_wa_lists(gt, lists, "before reset");
1286 if (!ok) {
1287 ret = -ESRCH;
1288 goto err;
1289 }
1290
1291 ret = intel_engine_reset(engine, "live_workarounds:idle");
1292 if (ret) {
1293 pr_err("%s: Reset failed while idle\n", engine->name);
1294 goto err;
1295 }
1296
1297 ok = verify_wa_lists(gt, lists, "after idle reset");
1298 if (!ok) {
1299 ret = -ESRCH;
1300 goto err;
1301 }
1302 }
1303
1304 ret = igt_spinner_init(&spin, engine->gt);
1305 if (ret)
1306 goto err;
1307
1308 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
1309 if (IS_ERR(rq)) {
1310 ret = PTR_ERR(rq);
1311 igt_spinner_fini(&spin);
1312 goto err;
1313 }
1314
1315 ret = request_add_spin(rq, &spin);
1316 if (ret) {
1317 pr_err("%s: Spinner failed to start\n", engine->name);
1318 igt_spinner_fini(&spin);
1319 goto err;
1320 }
1321
1322 /* Ensure the spinner hasn't aborted */
1323 if (i915_request_completed(rq)) {
1324 ret = -ETIMEDOUT;
1325 goto skip;
1326 }
1327
1328 if (!using_guc) {
1329 ret = intel_engine_reset(engine, "live_workarounds:active");
1330 if (ret) {
1331 pr_err("%s: Reset failed on an active spinner\n",
1332 engine->name);
1333 igt_spinner_fini(&spin);
1334 goto err;
1335 }
1336 }
1337
1338 /* Ensure the reset happens and kills the engine */
1339 if (ret == 0)
1340 ret = intel_selftest_wait_for_rq(rq);
1341
1342skip:
1343 igt_spinner_end(&spin);
1344 igt_spinner_fini(&spin);
1345
1346 ok = verify_wa_lists(gt, lists, "after busy reset");
1347 if (!ok)
1348 ret = -ESRCH;
1349
1350err:
1351 intel_context_put(ce);
1352
1353restore:
1354 ret2 = intel_selftest_restore_policy(engine, &saved);
1355 if (ret == 0)
1356 ret = ret2;
1357 if (ret)
1358 break;
1359 }
1360
1361 reference_lists_fini(gt, lists);
1362 intel_runtime_pm_put(gt->uncore->rpm, wakeref);
1363 igt_global_reset_unlock(gt);
1364 kfree(lists);
1365
1366 igt_flush_test(gt->i915);
1367
1368 return ret;
1369}
1370
1371int intel_workarounds_live_selftests(struct drm_i915_private *i915)
1372{
1373 static const struct i915_subtest tests[] = {
1374 SUBTEST(live_dirty_whitelist),
1375 SUBTEST(live_reset_whitelist),
1376 SUBTEST(live_isolated_whitelist),
1377 SUBTEST(live_gpu_reset_workarounds),
1378 SUBTEST(live_engine_reset_workarounds),
1379 };
1380
1381 if (intel_gt_is_wedged(to_gt(i915)))
1382 return 0;
1383
1384 return intel_gt_live_subtests(tests, to_gt(i915));
1385}