Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2019 Intel Corporation
4 */
5
6#include "debugfs_gt.h"
7#include "i915_drv.h"
8#include "intel_context.h"
9#include "intel_gt.h"
10#include "intel_gt_buffer_pool.h"
11#include "intel_gt_clock_utils.h"
12#include "intel_gt_pm.h"
13#include "intel_gt_requests.h"
14#include "intel_mocs.h"
15#include "intel_rc6.h"
16#include "intel_renderstate.h"
17#include "intel_rps.h"
18#include "intel_uncore.h"
19#include "intel_pm.h"
20#include "shmem_utils.h"
21
22void intel_gt_init_early(struct intel_gt *gt, struct drm_i915_private *i915)
23{
24 gt->i915 = i915;
25 gt->uncore = &i915->uncore;
26
27 spin_lock_init(>->irq_lock);
28
29 INIT_LIST_HEAD(>->closed_vma);
30 spin_lock_init(>->closed_lock);
31
32 intel_gt_init_buffer_pool(gt);
33 intel_gt_init_reset(gt);
34 intel_gt_init_requests(gt);
35 intel_gt_init_timelines(gt);
36 intel_gt_pm_init_early(gt);
37
38 intel_rps_init_early(>->rps);
39 intel_uc_init_early(>->uc);
40}
41
42void intel_gt_init_hw_early(struct intel_gt *gt, struct i915_ggtt *ggtt)
43{
44 gt->ggtt = ggtt;
45}
46
47int intel_gt_init_mmio(struct intel_gt *gt)
48{
49 intel_uc_init_mmio(>->uc);
50 intel_sseu_info_init(gt);
51
52 return intel_engines_init_mmio(gt);
53}
54
55static void init_unused_ring(struct intel_gt *gt, u32 base)
56{
57 struct intel_uncore *uncore = gt->uncore;
58
59 intel_uncore_write(uncore, RING_CTL(base), 0);
60 intel_uncore_write(uncore, RING_HEAD(base), 0);
61 intel_uncore_write(uncore, RING_TAIL(base), 0);
62 intel_uncore_write(uncore, RING_START(base), 0);
63}
64
65static void init_unused_rings(struct intel_gt *gt)
66{
67 struct drm_i915_private *i915 = gt->i915;
68
69 if (IS_I830(i915)) {
70 init_unused_ring(gt, PRB1_BASE);
71 init_unused_ring(gt, SRB0_BASE);
72 init_unused_ring(gt, SRB1_BASE);
73 init_unused_ring(gt, SRB2_BASE);
74 init_unused_ring(gt, SRB3_BASE);
75 } else if (IS_GEN(i915, 2)) {
76 init_unused_ring(gt, SRB0_BASE);
77 init_unused_ring(gt, SRB1_BASE);
78 } else if (IS_GEN(i915, 3)) {
79 init_unused_ring(gt, PRB1_BASE);
80 init_unused_ring(gt, PRB2_BASE);
81 }
82}
83
84int intel_gt_init_hw(struct intel_gt *gt)
85{
86 struct drm_i915_private *i915 = gt->i915;
87 struct intel_uncore *uncore = gt->uncore;
88 int ret;
89
90 gt->last_init_time = ktime_get();
91
92 /* Double layer security blanket, see i915_gem_init() */
93 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
94
95 if (HAS_EDRAM(i915) && INTEL_GEN(i915) < 9)
96 intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));
97
98 if (IS_HASWELL(i915))
99 intel_uncore_write(uncore,
100 MI_PREDICATE_RESULT_2,
101 IS_HSW_GT3(i915) ?
102 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
103
104 /* Apply the GT workarounds... */
105 intel_gt_apply_workarounds(gt);
106 /* ...and determine whether they are sticking. */
107 intel_gt_verify_workarounds(gt, "init");
108
109 intel_gt_init_swizzling(gt);
110
111 /*
112 * At least 830 can leave some of the unused rings
113 * "active" (ie. head != tail) after resume which
114 * will prevent c3 entry. Makes sure all unused rings
115 * are totally idle.
116 */
117 init_unused_rings(gt);
118
119 ret = i915_ppgtt_init_hw(gt);
120 if (ret) {
121 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
122 goto out;
123 }
124
125 /* We can't enable contexts until all firmware is loaded */
126 ret = intel_uc_init_hw(>->uc);
127 if (ret) {
128 i915_probe_error(i915, "Enabling uc failed (%d)\n", ret);
129 goto out;
130 }
131
132 intel_mocs_init(gt);
133
134out:
135 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
136 return ret;
137}
138
139static void rmw_set(struct intel_uncore *uncore, i915_reg_t reg, u32 set)
140{
141 intel_uncore_rmw(uncore, reg, 0, set);
142}
143
144static void rmw_clear(struct intel_uncore *uncore, i915_reg_t reg, u32 clr)
145{
146 intel_uncore_rmw(uncore, reg, clr, 0);
147}
148
149static void clear_register(struct intel_uncore *uncore, i915_reg_t reg)
150{
151 intel_uncore_rmw(uncore, reg, 0, 0);
152}
153
154static void gen8_clear_engine_error_register(struct intel_engine_cs *engine)
155{
156 GEN6_RING_FAULT_REG_RMW(engine, RING_FAULT_VALID, 0);
157 GEN6_RING_FAULT_REG_POSTING_READ(engine);
158}
159
160void
161intel_gt_clear_error_registers(struct intel_gt *gt,
162 intel_engine_mask_t engine_mask)
163{
164 struct drm_i915_private *i915 = gt->i915;
165 struct intel_uncore *uncore = gt->uncore;
166 u32 eir;
167
168 if (!IS_GEN(i915, 2))
169 clear_register(uncore, PGTBL_ER);
170
171 if (INTEL_GEN(i915) < 4)
172 clear_register(uncore, IPEIR(RENDER_RING_BASE));
173 else
174 clear_register(uncore, IPEIR_I965);
175
176 clear_register(uncore, EIR);
177 eir = intel_uncore_read(uncore, EIR);
178 if (eir) {
179 /*
180 * some errors might have become stuck,
181 * mask them.
182 */
183 DRM_DEBUG_DRIVER("EIR stuck: 0x%08x, masking\n", eir);
184 rmw_set(uncore, EMR, eir);
185 intel_uncore_write(uncore, GEN2_IIR,
186 I915_MASTER_ERROR_INTERRUPT);
187 }
188
189 if (INTEL_GEN(i915) >= 12) {
190 rmw_clear(uncore, GEN12_RING_FAULT_REG, RING_FAULT_VALID);
191 intel_uncore_posting_read(uncore, GEN12_RING_FAULT_REG);
192 } else if (INTEL_GEN(i915) >= 8) {
193 rmw_clear(uncore, GEN8_RING_FAULT_REG, RING_FAULT_VALID);
194 intel_uncore_posting_read(uncore, GEN8_RING_FAULT_REG);
195 } else if (INTEL_GEN(i915) >= 6) {
196 struct intel_engine_cs *engine;
197 enum intel_engine_id id;
198
199 for_each_engine_masked(engine, gt, engine_mask, id)
200 gen8_clear_engine_error_register(engine);
201 }
202}
203
204static void gen6_check_faults(struct intel_gt *gt)
205{
206 struct intel_engine_cs *engine;
207 enum intel_engine_id id;
208 u32 fault;
209
210 for_each_engine(engine, gt, id) {
211 fault = GEN6_RING_FAULT_REG_READ(engine);
212 if (fault & RING_FAULT_VALID) {
213 drm_dbg(&engine->i915->drm, "Unexpected fault\n"
214 "\tAddr: 0x%08lx\n"
215 "\tAddress space: %s\n"
216 "\tSource ID: %d\n"
217 "\tType: %d\n",
218 fault & PAGE_MASK,
219 fault & RING_FAULT_GTTSEL_MASK ?
220 "GGTT" : "PPGTT",
221 RING_FAULT_SRCID(fault),
222 RING_FAULT_FAULT_TYPE(fault));
223 }
224 }
225}
226
227static void gen8_check_faults(struct intel_gt *gt)
228{
229 struct intel_uncore *uncore = gt->uncore;
230 i915_reg_t fault_reg, fault_data0_reg, fault_data1_reg;
231 u32 fault;
232
233 if (INTEL_GEN(gt->i915) >= 12) {
234 fault_reg = GEN12_RING_FAULT_REG;
235 fault_data0_reg = GEN12_FAULT_TLB_DATA0;
236 fault_data1_reg = GEN12_FAULT_TLB_DATA1;
237 } else {
238 fault_reg = GEN8_RING_FAULT_REG;
239 fault_data0_reg = GEN8_FAULT_TLB_DATA0;
240 fault_data1_reg = GEN8_FAULT_TLB_DATA1;
241 }
242
243 fault = intel_uncore_read(uncore, fault_reg);
244 if (fault & RING_FAULT_VALID) {
245 u32 fault_data0, fault_data1;
246 u64 fault_addr;
247
248 fault_data0 = intel_uncore_read(uncore, fault_data0_reg);
249 fault_data1 = intel_uncore_read(uncore, fault_data1_reg);
250
251 fault_addr = ((u64)(fault_data1 & FAULT_VA_HIGH_BITS) << 44) |
252 ((u64)fault_data0 << 12);
253
254 drm_dbg(&uncore->i915->drm, "Unexpected fault\n"
255 "\tAddr: 0x%08x_%08x\n"
256 "\tAddress space: %s\n"
257 "\tEngine ID: %d\n"
258 "\tSource ID: %d\n"
259 "\tType: %d\n",
260 upper_32_bits(fault_addr), lower_32_bits(fault_addr),
261 fault_data1 & FAULT_GTT_SEL ? "GGTT" : "PPGTT",
262 GEN8_RING_FAULT_ENGINE_ID(fault),
263 RING_FAULT_SRCID(fault),
264 RING_FAULT_FAULT_TYPE(fault));
265 }
266}
267
268void intel_gt_check_and_clear_faults(struct intel_gt *gt)
269{
270 struct drm_i915_private *i915 = gt->i915;
271
272 /* From GEN8 onwards we only have one 'All Engine Fault Register' */
273 if (INTEL_GEN(i915) >= 8)
274 gen8_check_faults(gt);
275 else if (INTEL_GEN(i915) >= 6)
276 gen6_check_faults(gt);
277 else
278 return;
279
280 intel_gt_clear_error_registers(gt, ALL_ENGINES);
281}
282
283void intel_gt_flush_ggtt_writes(struct intel_gt *gt)
284{
285 struct intel_uncore *uncore = gt->uncore;
286 intel_wakeref_t wakeref;
287
288 /*
289 * No actual flushing is required for the GTT write domain for reads
290 * from the GTT domain. Writes to it "immediately" go to main memory
291 * as far as we know, so there's no chipset flush. It also doesn't
292 * land in the GPU render cache.
293 *
294 * However, we do have to enforce the order so that all writes through
295 * the GTT land before any writes to the device, such as updates to
296 * the GATT itself.
297 *
298 * We also have to wait a bit for the writes to land from the GTT.
299 * An uncached read (i.e. mmio) seems to be ideal for the round-trip
300 * timing. This issue has only been observed when switching quickly
301 * between GTT writes and CPU reads from inside the kernel on recent hw,
302 * and it appears to only affect discrete GTT blocks (i.e. on LLC
303 * system agents we cannot reproduce this behaviour, until Cannonlake
304 * that was!).
305 */
306
307 wmb();
308
309 if (INTEL_INFO(gt->i915)->has_coherent_ggtt)
310 return;
311
312 intel_gt_chipset_flush(gt);
313
314 with_intel_runtime_pm_if_in_use(uncore->rpm, wakeref) {
315 unsigned long flags;
316
317 spin_lock_irqsave(&uncore->lock, flags);
318 intel_uncore_posting_read_fw(uncore,
319 RING_HEAD(RENDER_RING_BASE));
320 spin_unlock_irqrestore(&uncore->lock, flags);
321 }
322}
323
324void intel_gt_chipset_flush(struct intel_gt *gt)
325{
326 wmb();
327 if (INTEL_GEN(gt->i915) < 6)
328 intel_gtt_chipset_flush();
329}
330
331void intel_gt_driver_register(struct intel_gt *gt)
332{
333 intel_rps_driver_register(>->rps);
334
335 debugfs_gt_register(gt);
336}
337
338static int intel_gt_init_scratch(struct intel_gt *gt, unsigned int size)
339{
340 struct drm_i915_private *i915 = gt->i915;
341 struct drm_i915_gem_object *obj;
342 struct i915_vma *vma;
343 int ret;
344
345 obj = i915_gem_object_create_stolen(i915, size);
346 if (IS_ERR(obj))
347 obj = i915_gem_object_create_internal(i915, size);
348 if (IS_ERR(obj)) {
349 DRM_ERROR("Failed to allocate scratch page\n");
350 return PTR_ERR(obj);
351 }
352
353 vma = i915_vma_instance(obj, >->ggtt->vm, NULL);
354 if (IS_ERR(vma)) {
355 ret = PTR_ERR(vma);
356 goto err_unref;
357 }
358
359 ret = i915_ggtt_pin(vma, 0, PIN_HIGH);
360 if (ret)
361 goto err_unref;
362
363 gt->scratch = i915_vma_make_unshrinkable(vma);
364
365 return 0;
366
367err_unref:
368 i915_gem_object_put(obj);
369 return ret;
370}
371
372static void intel_gt_fini_scratch(struct intel_gt *gt)
373{
374 i915_vma_unpin_and_release(>->scratch, 0);
375}
376
377static struct i915_address_space *kernel_vm(struct intel_gt *gt)
378{
379 if (INTEL_PPGTT(gt->i915) > INTEL_PPGTT_ALIASING)
380 return &i915_ppgtt_create(gt)->vm;
381 else
382 return i915_vm_get(>->ggtt->vm);
383}
384
385static int __engines_record_defaults(struct intel_gt *gt)
386{
387 struct i915_request *requests[I915_NUM_ENGINES] = {};
388 struct intel_engine_cs *engine;
389 enum intel_engine_id id;
390 int err = 0;
391
392 /*
393 * As we reset the gpu during very early sanitisation, the current
394 * register state on the GPU should reflect its defaults values.
395 * We load a context onto the hw (with restore-inhibit), then switch
396 * over to a second context to save that default register state. We
397 * can then prime every new context with that state so they all start
398 * from the same default HW values.
399 */
400
401 for_each_engine(engine, gt, id) {
402 struct intel_renderstate so;
403 struct intel_context *ce;
404 struct i915_request *rq;
405
406 /* We must be able to switch to something! */
407 GEM_BUG_ON(!engine->kernel_context);
408
409 err = intel_renderstate_init(&so, engine);
410 if (err)
411 goto out;
412
413 ce = intel_context_create(engine);
414 if (IS_ERR(ce)) {
415 err = PTR_ERR(ce);
416 goto out;
417 }
418
419 rq = intel_context_create_request(ce);
420 if (IS_ERR(rq)) {
421 err = PTR_ERR(rq);
422 intel_context_put(ce);
423 goto out;
424 }
425
426 err = intel_engine_emit_ctx_wa(rq);
427 if (err)
428 goto err_rq;
429
430 err = intel_renderstate_emit(&so, rq);
431 if (err)
432 goto err_rq;
433
434err_rq:
435 requests[id] = i915_request_get(rq);
436 i915_request_add(rq);
437 intel_renderstate_fini(&so);
438 if (err)
439 goto out;
440 }
441
442 /* Flush the default context image to memory, and enable powersaving. */
443 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME) {
444 err = -EIO;
445 goto out;
446 }
447
448 for (id = 0; id < ARRAY_SIZE(requests); id++) {
449 struct i915_request *rq;
450 struct file *state;
451
452 rq = requests[id];
453 if (!rq)
454 continue;
455
456 if (rq->fence.error) {
457 err = -EIO;
458 goto out;
459 }
460
461 GEM_BUG_ON(!test_bit(CONTEXT_ALLOC_BIT, &rq->context->flags));
462 if (!rq->context->state)
463 continue;
464
465 /* Keep a copy of the state's backing pages; free the obj */
466 state = shmem_create_from_object(rq->context->state->obj);
467 if (IS_ERR(state)) {
468 err = PTR_ERR(state);
469 goto out;
470 }
471 rq->engine->default_state = state;
472 }
473
474out:
475 /*
476 * If we have to abandon now, we expect the engines to be idle
477 * and ready to be torn-down. The quickest way we can accomplish
478 * this is by declaring ourselves wedged.
479 */
480 if (err)
481 intel_gt_set_wedged(gt);
482
483 for (id = 0; id < ARRAY_SIZE(requests); id++) {
484 struct intel_context *ce;
485 struct i915_request *rq;
486
487 rq = requests[id];
488 if (!rq)
489 continue;
490
491 ce = rq->context;
492 i915_request_put(rq);
493 intel_context_put(ce);
494 }
495 return err;
496}
497
498static int __engines_verify_workarounds(struct intel_gt *gt)
499{
500 struct intel_engine_cs *engine;
501 enum intel_engine_id id;
502 int err = 0;
503
504 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
505 return 0;
506
507 for_each_engine(engine, gt, id) {
508 if (intel_engine_verify_workarounds(engine, "load"))
509 err = -EIO;
510 }
511
512 /* Flush and restore the kernel context for safety */
513 if (intel_gt_wait_for_idle(gt, I915_GEM_IDLE_TIMEOUT) == -ETIME)
514 err = -EIO;
515
516 return err;
517}
518
519static void __intel_gt_disable(struct intel_gt *gt)
520{
521 intel_gt_set_wedged_on_fini(gt);
522
523 intel_gt_suspend_prepare(gt);
524 intel_gt_suspend_late(gt);
525
526 GEM_BUG_ON(intel_gt_pm_is_awake(gt));
527}
528
529int intel_gt_init(struct intel_gt *gt)
530{
531 int err;
532
533 err = i915_inject_probe_error(gt->i915, -ENODEV);
534 if (err)
535 return err;
536
537 /*
538 * This is just a security blanket to placate dragons.
539 * On some systems, we very sporadically observe that the first TLBs
540 * used by the CS may be stale, despite us poking the TLB reset. If
541 * we hold the forcewake during initialisation these problems
542 * just magically go away.
543 */
544 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
545
546 intel_gt_init_clock_frequency(gt);
547
548 err = intel_gt_init_scratch(gt, IS_GEN(gt->i915, 2) ? SZ_256K : SZ_4K);
549 if (err)
550 goto out_fw;
551
552 intel_gt_pm_init(gt);
553
554 gt->vm = kernel_vm(gt);
555 if (!gt->vm) {
556 err = -ENOMEM;
557 goto err_pm;
558 }
559
560 err = intel_engines_init(gt);
561 if (err)
562 goto err_engines;
563
564 err = intel_uc_init(>->uc);
565 if (err)
566 goto err_engines;
567
568 err = intel_gt_resume(gt);
569 if (err)
570 goto err_uc_init;
571
572 err = __engines_record_defaults(gt);
573 if (err)
574 goto err_gt;
575
576 err = __engines_verify_workarounds(gt);
577 if (err)
578 goto err_gt;
579
580 err = i915_inject_probe_error(gt->i915, -EIO);
581 if (err)
582 goto err_gt;
583
584 goto out_fw;
585err_gt:
586 __intel_gt_disable(gt);
587 intel_uc_fini_hw(>->uc);
588err_uc_init:
589 intel_uc_fini(>->uc);
590err_engines:
591 intel_engines_release(gt);
592 i915_vm_put(fetch_and_zero(>->vm));
593err_pm:
594 intel_gt_pm_fini(gt);
595 intel_gt_fini_scratch(gt);
596out_fw:
597 if (err)
598 intel_gt_set_wedged_on_init(gt);
599 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
600 return err;
601}
602
603void intel_gt_driver_remove(struct intel_gt *gt)
604{
605 __intel_gt_disable(gt);
606
607 intel_uc_driver_remove(>->uc);
608
609 intel_engines_release(gt);
610}
611
612void intel_gt_driver_unregister(struct intel_gt *gt)
613{
614 intel_rps_driver_unregister(>->rps);
615
616 /*
617 * Upon unregistering the device to prevent any new users, cancel
618 * all in-flight requests so that we can quickly unbind the active
619 * resources.
620 */
621 intel_gt_set_wedged(gt);
622}
623
624void intel_gt_driver_release(struct intel_gt *gt)
625{
626 struct i915_address_space *vm;
627 intel_wakeref_t wakeref;
628
629 /* Scrub all HW state upon release */
630 with_intel_runtime_pm(gt->uncore->rpm, wakeref)
631 __intel_gt_reset(gt, ALL_ENGINES);
632
633 vm = fetch_and_zero(>->vm);
634 if (vm) /* FIXME being called twice on error paths :( */
635 i915_vm_put(vm);
636
637 intel_gt_pm_fini(gt);
638 intel_gt_fini_scratch(gt);
639 intel_gt_fini_buffer_pool(gt);
640}
641
642void intel_gt_driver_late_release(struct intel_gt *gt)
643{
644 /* We need to wait for inflight RCU frees to release their grip */
645 rcu_barrier();
646
647 intel_uc_driver_late_release(>->uc);
648 intel_gt_fini_requests(gt);
649 intel_gt_fini_reset(gt);
650 intel_gt_fini_timelines(gt);
651 intel_engines_free(gt);
652}
653
654void intel_gt_info_print(const struct intel_gt_info *info,
655 struct drm_printer *p)
656{
657 drm_printf(p, "available engines: %x\n", info->engine_mask);
658
659 intel_sseu_dump(&info->sseu, p);
660}