Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2016 Intel Corporation
4 */
5
6#include <linux/string_helpers.h>
7
8#include <drm/drm_print.h>
9
10#include "gem/i915_gem_context.h"
11#include "gem/i915_gem_internal.h"
12#include "gt/intel_gt_regs.h"
13
14#include "i915_cmd_parser.h"
15#include "i915_drv.h"
16#include "i915_irq.h"
17#include "i915_reg.h"
18#include "intel_breadcrumbs.h"
19#include "intel_context.h"
20#include "intel_engine.h"
21#include "intel_engine_pm.h"
22#include "intel_engine_regs.h"
23#include "intel_engine_user.h"
24#include "intel_execlists_submission.h"
25#include "intel_gt.h"
26#include "intel_gt_mcr.h"
27#include "intel_gt_pm.h"
28#include "intel_gt_requests.h"
29#include "intel_lrc.h"
30#include "intel_lrc_reg.h"
31#include "intel_reset.h"
32#include "intel_ring.h"
33#include "uc/intel_guc_submission.h"
34
35/* Haswell does have the CXT_SIZE register however it does not appear to be
36 * valid. Now, docs explain in dwords what is in the context object. The full
37 * size is 70720 bytes, however, the power context and execlist context will
38 * never be saved (power context is stored elsewhere, and execlists don't work
39 * on HSW) - so the final size, including the extra state required for the
40 * Resource Streamer, is 66944 bytes, which rounds to 17 pages.
41 */
42#define HSW_CXT_TOTAL_SIZE (17 * PAGE_SIZE)
43
44#define DEFAULT_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
45#define GEN8_LR_CONTEXT_RENDER_SIZE (20 * PAGE_SIZE)
46#define GEN9_LR_CONTEXT_RENDER_SIZE (22 * PAGE_SIZE)
47#define GEN11_LR_CONTEXT_RENDER_SIZE (14 * PAGE_SIZE)
48
49#define GEN8_LR_CONTEXT_OTHER_SIZE ( 2 * PAGE_SIZE)
50
51#define MAX_MMIO_BASES 3
52struct engine_info {
53 u8 class;
54 u8 instance;
55 /* mmio bases table *must* be sorted in reverse graphics_ver order */
56 struct engine_mmio_base {
57 u32 graphics_ver : 8;
58 u32 base : 24;
59 } mmio_bases[MAX_MMIO_BASES];
60};
61
62static const struct engine_info intel_engines[] = {
63 [RCS0] = {
64 .class = RENDER_CLASS,
65 .instance = 0,
66 .mmio_bases = {
67 { .graphics_ver = 1, .base = RENDER_RING_BASE }
68 },
69 },
70 [BCS0] = {
71 .class = COPY_ENGINE_CLASS,
72 .instance = 0,
73 .mmio_bases = {
74 { .graphics_ver = 6, .base = BLT_RING_BASE }
75 },
76 },
77 [BCS1] = {
78 .class = COPY_ENGINE_CLASS,
79 .instance = 1,
80 .mmio_bases = {
81 { .graphics_ver = 12, .base = XEHPC_BCS1_RING_BASE }
82 },
83 },
84 [BCS2] = {
85 .class = COPY_ENGINE_CLASS,
86 .instance = 2,
87 .mmio_bases = {
88 { .graphics_ver = 12, .base = XEHPC_BCS2_RING_BASE }
89 },
90 },
91 [BCS3] = {
92 .class = COPY_ENGINE_CLASS,
93 .instance = 3,
94 .mmio_bases = {
95 { .graphics_ver = 12, .base = XEHPC_BCS3_RING_BASE }
96 },
97 },
98 [BCS4] = {
99 .class = COPY_ENGINE_CLASS,
100 .instance = 4,
101 .mmio_bases = {
102 { .graphics_ver = 12, .base = XEHPC_BCS4_RING_BASE }
103 },
104 },
105 [BCS5] = {
106 .class = COPY_ENGINE_CLASS,
107 .instance = 5,
108 .mmio_bases = {
109 { .graphics_ver = 12, .base = XEHPC_BCS5_RING_BASE }
110 },
111 },
112 [BCS6] = {
113 .class = COPY_ENGINE_CLASS,
114 .instance = 6,
115 .mmio_bases = {
116 { .graphics_ver = 12, .base = XEHPC_BCS6_RING_BASE }
117 },
118 },
119 [BCS7] = {
120 .class = COPY_ENGINE_CLASS,
121 .instance = 7,
122 .mmio_bases = {
123 { .graphics_ver = 12, .base = XEHPC_BCS7_RING_BASE }
124 },
125 },
126 [BCS8] = {
127 .class = COPY_ENGINE_CLASS,
128 .instance = 8,
129 .mmio_bases = {
130 { .graphics_ver = 12, .base = XEHPC_BCS8_RING_BASE }
131 },
132 },
133 [VCS0] = {
134 .class = VIDEO_DECODE_CLASS,
135 .instance = 0,
136 .mmio_bases = {
137 { .graphics_ver = 11, .base = GEN11_BSD_RING_BASE },
138 { .graphics_ver = 6, .base = GEN6_BSD_RING_BASE },
139 { .graphics_ver = 4, .base = BSD_RING_BASE }
140 },
141 },
142 [VCS1] = {
143 .class = VIDEO_DECODE_CLASS,
144 .instance = 1,
145 .mmio_bases = {
146 { .graphics_ver = 11, .base = GEN11_BSD2_RING_BASE },
147 { .graphics_ver = 8, .base = GEN8_BSD2_RING_BASE }
148 },
149 },
150 [VCS2] = {
151 .class = VIDEO_DECODE_CLASS,
152 .instance = 2,
153 .mmio_bases = {
154 { .graphics_ver = 11, .base = GEN11_BSD3_RING_BASE }
155 },
156 },
157 [VCS3] = {
158 .class = VIDEO_DECODE_CLASS,
159 .instance = 3,
160 .mmio_bases = {
161 { .graphics_ver = 11, .base = GEN11_BSD4_RING_BASE }
162 },
163 },
164 [VCS4] = {
165 .class = VIDEO_DECODE_CLASS,
166 .instance = 4,
167 .mmio_bases = {
168 { .graphics_ver = 12, .base = XEHP_BSD5_RING_BASE }
169 },
170 },
171 [VCS5] = {
172 .class = VIDEO_DECODE_CLASS,
173 .instance = 5,
174 .mmio_bases = {
175 { .graphics_ver = 12, .base = XEHP_BSD6_RING_BASE }
176 },
177 },
178 [VCS6] = {
179 .class = VIDEO_DECODE_CLASS,
180 .instance = 6,
181 .mmio_bases = {
182 { .graphics_ver = 12, .base = XEHP_BSD7_RING_BASE }
183 },
184 },
185 [VCS7] = {
186 .class = VIDEO_DECODE_CLASS,
187 .instance = 7,
188 .mmio_bases = {
189 { .graphics_ver = 12, .base = XEHP_BSD8_RING_BASE }
190 },
191 },
192 [VECS0] = {
193 .class = VIDEO_ENHANCEMENT_CLASS,
194 .instance = 0,
195 .mmio_bases = {
196 { .graphics_ver = 11, .base = GEN11_VEBOX_RING_BASE },
197 { .graphics_ver = 7, .base = VEBOX_RING_BASE }
198 },
199 },
200 [VECS1] = {
201 .class = VIDEO_ENHANCEMENT_CLASS,
202 .instance = 1,
203 .mmio_bases = {
204 { .graphics_ver = 11, .base = GEN11_VEBOX2_RING_BASE }
205 },
206 },
207 [VECS2] = {
208 .class = VIDEO_ENHANCEMENT_CLASS,
209 .instance = 2,
210 .mmio_bases = {
211 { .graphics_ver = 12, .base = XEHP_VEBOX3_RING_BASE }
212 },
213 },
214 [VECS3] = {
215 .class = VIDEO_ENHANCEMENT_CLASS,
216 .instance = 3,
217 .mmio_bases = {
218 { .graphics_ver = 12, .base = XEHP_VEBOX4_RING_BASE }
219 },
220 },
221 [CCS0] = {
222 .class = COMPUTE_CLASS,
223 .instance = 0,
224 .mmio_bases = {
225 { .graphics_ver = 12, .base = GEN12_COMPUTE0_RING_BASE }
226 }
227 },
228 [CCS1] = {
229 .class = COMPUTE_CLASS,
230 .instance = 1,
231 .mmio_bases = {
232 { .graphics_ver = 12, .base = GEN12_COMPUTE1_RING_BASE }
233 }
234 },
235 [CCS2] = {
236 .class = COMPUTE_CLASS,
237 .instance = 2,
238 .mmio_bases = {
239 { .graphics_ver = 12, .base = GEN12_COMPUTE2_RING_BASE }
240 }
241 },
242 [CCS3] = {
243 .class = COMPUTE_CLASS,
244 .instance = 3,
245 .mmio_bases = {
246 { .graphics_ver = 12, .base = GEN12_COMPUTE3_RING_BASE }
247 }
248 },
249 [GSC0] = {
250 .class = OTHER_CLASS,
251 .instance = OTHER_GSC_INSTANCE,
252 .mmio_bases = {
253 { .graphics_ver = 12, .base = MTL_GSC_RING_BASE }
254 }
255 },
256};
257
258/**
259 * intel_engine_context_size() - return the size of the context for an engine
260 * @gt: the gt
261 * @class: engine class
262 *
263 * Each engine class may require a different amount of space for a context
264 * image.
265 *
266 * Return: size (in bytes) of an engine class specific context image
267 *
268 * Note: this size includes the HWSP, which is part of the context image
269 * in LRC mode, but does not include the "shared data page" used with
270 * GuC submission. The caller should account for this if using the GuC.
271 */
272u32 intel_engine_context_size(struct intel_gt *gt, u8 class)
273{
274 struct intel_uncore *uncore = gt->uncore;
275 u32 cxt_size;
276
277 BUILD_BUG_ON(I915_GTT_PAGE_SIZE != PAGE_SIZE);
278
279 switch (class) {
280 case COMPUTE_CLASS:
281 fallthrough;
282 case RENDER_CLASS:
283 switch (GRAPHICS_VER(gt->i915)) {
284 default:
285 MISSING_CASE(GRAPHICS_VER(gt->i915));
286 return DEFAULT_LR_CONTEXT_RENDER_SIZE;
287 case 12:
288 case 11:
289 return GEN11_LR_CONTEXT_RENDER_SIZE;
290 case 9:
291 return GEN9_LR_CONTEXT_RENDER_SIZE;
292 case 8:
293 return GEN8_LR_CONTEXT_RENDER_SIZE;
294 case 7:
295 if (IS_HASWELL(gt->i915))
296 return HSW_CXT_TOTAL_SIZE;
297
298 cxt_size = intel_uncore_read(uncore, GEN7_CXT_SIZE);
299 return round_up(GEN7_CXT_TOTAL_SIZE(cxt_size) * 64,
300 PAGE_SIZE);
301 case 6:
302 cxt_size = intel_uncore_read(uncore, CXT_SIZE);
303 return round_up(GEN6_CXT_TOTAL_SIZE(cxt_size) * 64,
304 PAGE_SIZE);
305 case 5:
306 case 4:
307 /*
308 * There is a discrepancy here between the size reported
309 * by the register and the size of the context layout
310 * in the docs. Both are described as authorative!
311 *
312 * The discrepancy is on the order of a few cachelines,
313 * but the total is under one page (4k), which is our
314 * minimum allocation anyway so it should all come
315 * out in the wash.
316 */
317 cxt_size = intel_uncore_read(uncore, CXT_SIZE) + 1;
318 drm_dbg(>->i915->drm,
319 "graphics_ver = %d CXT_SIZE = %d bytes [0x%08x]\n",
320 GRAPHICS_VER(gt->i915), cxt_size * 64,
321 cxt_size - 1);
322 return round_up(cxt_size * 64, PAGE_SIZE);
323 case 3:
324 case 2:
325 /* For the special day when i810 gets merged. */
326 case 1:
327 return 0;
328 }
329 break;
330 default:
331 MISSING_CASE(class);
332 fallthrough;
333 case VIDEO_DECODE_CLASS:
334 case VIDEO_ENHANCEMENT_CLASS:
335 case COPY_ENGINE_CLASS:
336 case OTHER_CLASS:
337 if (GRAPHICS_VER(gt->i915) < 8)
338 return 0;
339 return GEN8_LR_CONTEXT_OTHER_SIZE;
340 }
341}
342
343static u32 __engine_mmio_base(struct drm_i915_private *i915,
344 const struct engine_mmio_base *bases)
345{
346 int i;
347
348 for (i = 0; i < MAX_MMIO_BASES; i++)
349 if (GRAPHICS_VER(i915) >= bases[i].graphics_ver)
350 break;
351
352 GEM_BUG_ON(i == MAX_MMIO_BASES);
353 GEM_BUG_ON(!bases[i].base);
354
355 return bases[i].base;
356}
357
358static void __sprint_engine_name(struct intel_engine_cs *engine)
359{
360 /*
361 * Before we know what the uABI name for this engine will be,
362 * we still would like to keep track of this engine in the debug logs.
363 * We throw in a ' here as a reminder that this isn't its final name.
364 */
365 GEM_WARN_ON(snprintf(engine->name, sizeof(engine->name), "%s'%u",
366 intel_engine_class_repr(engine->class),
367 engine->instance) >= sizeof(engine->name));
368}
369
370void intel_engine_set_hwsp_writemask(struct intel_engine_cs *engine, u32 mask)
371{
372 /*
373 * Though they added more rings on g4x/ilk, they did not add
374 * per-engine HWSTAM until gen6.
375 */
376 if (GRAPHICS_VER(engine->i915) < 6 && engine->class != RENDER_CLASS)
377 return;
378
379 if (GRAPHICS_VER(engine->i915) >= 3)
380 ENGINE_WRITE(engine, RING_HWSTAM, mask);
381 else
382 ENGINE_WRITE16(engine, RING_HWSTAM, mask);
383}
384
385static void intel_engine_sanitize_mmio(struct intel_engine_cs *engine)
386{
387 /* Mask off all writes into the unknown HWSP */
388 intel_engine_set_hwsp_writemask(engine, ~0u);
389}
390
391static void nop_irq_handler(struct intel_engine_cs *engine, u16 iir)
392{
393 GEM_DEBUG_WARN_ON(iir);
394}
395
396static u32 get_reset_domain(u8 ver, enum intel_engine_id id)
397{
398 u32 reset_domain;
399
400 if (ver >= 11) {
401 static const u32 engine_reset_domains[] = {
402 [RCS0] = GEN11_GRDOM_RENDER,
403 [BCS0] = GEN11_GRDOM_BLT,
404 [BCS1] = XEHPC_GRDOM_BLT1,
405 [BCS2] = XEHPC_GRDOM_BLT2,
406 [BCS3] = XEHPC_GRDOM_BLT3,
407 [BCS4] = XEHPC_GRDOM_BLT4,
408 [BCS5] = XEHPC_GRDOM_BLT5,
409 [BCS6] = XEHPC_GRDOM_BLT6,
410 [BCS7] = XEHPC_GRDOM_BLT7,
411 [BCS8] = XEHPC_GRDOM_BLT8,
412 [VCS0] = GEN11_GRDOM_MEDIA,
413 [VCS1] = GEN11_GRDOM_MEDIA2,
414 [VCS2] = GEN11_GRDOM_MEDIA3,
415 [VCS3] = GEN11_GRDOM_MEDIA4,
416 [VCS4] = GEN11_GRDOM_MEDIA5,
417 [VCS5] = GEN11_GRDOM_MEDIA6,
418 [VCS6] = GEN11_GRDOM_MEDIA7,
419 [VCS7] = GEN11_GRDOM_MEDIA8,
420 [VECS0] = GEN11_GRDOM_VECS,
421 [VECS1] = GEN11_GRDOM_VECS2,
422 [VECS2] = GEN11_GRDOM_VECS3,
423 [VECS3] = GEN11_GRDOM_VECS4,
424 [CCS0] = GEN11_GRDOM_RENDER,
425 [CCS1] = GEN11_GRDOM_RENDER,
426 [CCS2] = GEN11_GRDOM_RENDER,
427 [CCS3] = GEN11_GRDOM_RENDER,
428 [GSC0] = GEN12_GRDOM_GSC,
429 };
430 GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) ||
431 !engine_reset_domains[id]);
432 reset_domain = engine_reset_domains[id];
433 } else {
434 static const u32 engine_reset_domains[] = {
435 [RCS0] = GEN6_GRDOM_RENDER,
436 [BCS0] = GEN6_GRDOM_BLT,
437 [VCS0] = GEN6_GRDOM_MEDIA,
438 [VCS1] = GEN8_GRDOM_MEDIA2,
439 [VECS0] = GEN6_GRDOM_VECS,
440 };
441 GEM_BUG_ON(id >= ARRAY_SIZE(engine_reset_domains) ||
442 !engine_reset_domains[id]);
443 reset_domain = engine_reset_domains[id];
444 }
445
446 return reset_domain;
447}
448
449static int intel_engine_setup(struct intel_gt *gt, enum intel_engine_id id,
450 u8 logical_instance)
451{
452 const struct engine_info *info = &intel_engines[id];
453 struct drm_i915_private *i915 = gt->i915;
454 struct intel_engine_cs *engine;
455 u8 guc_class;
456
457 BUILD_BUG_ON(MAX_ENGINE_CLASS >= BIT(GEN11_ENGINE_CLASS_WIDTH));
458 BUILD_BUG_ON(MAX_ENGINE_INSTANCE >= BIT(GEN11_ENGINE_INSTANCE_WIDTH));
459 BUILD_BUG_ON(I915_MAX_VCS > (MAX_ENGINE_INSTANCE + 1));
460 BUILD_BUG_ON(I915_MAX_VECS > (MAX_ENGINE_INSTANCE + 1));
461
462 if (GEM_DEBUG_WARN_ON(id >= ARRAY_SIZE(gt->engine)))
463 return -EINVAL;
464
465 if (GEM_DEBUG_WARN_ON(info->class > MAX_ENGINE_CLASS))
466 return -EINVAL;
467
468 if (GEM_DEBUG_WARN_ON(info->instance > MAX_ENGINE_INSTANCE))
469 return -EINVAL;
470
471 if (GEM_DEBUG_WARN_ON(gt->engine_class[info->class][info->instance]))
472 return -EINVAL;
473
474 engine = kzalloc(sizeof(*engine), GFP_KERNEL);
475 if (!engine)
476 return -ENOMEM;
477
478 BUILD_BUG_ON(BITS_PER_TYPE(engine->mask) < I915_NUM_ENGINES);
479
480 INIT_LIST_HEAD(&engine->pinned_contexts_list);
481 engine->id = id;
482 engine->legacy_idx = INVALID_ENGINE;
483 engine->mask = BIT(id);
484 engine->reset_domain = get_reset_domain(GRAPHICS_VER(gt->i915),
485 id);
486 engine->i915 = i915;
487 engine->gt = gt;
488 engine->uncore = gt->uncore;
489 guc_class = engine_class_to_guc_class(info->class);
490 engine->guc_id = MAKE_GUC_ID(guc_class, info->instance);
491 engine->mmio_base = __engine_mmio_base(i915, info->mmio_bases);
492
493 engine->irq_handler = nop_irq_handler;
494
495 engine->class = info->class;
496 engine->instance = info->instance;
497 engine->logical_mask = BIT(logical_instance);
498 __sprint_engine_name(engine);
499
500 if ((engine->class == COMPUTE_CLASS && !RCS_MASK(engine->gt) &&
501 __ffs(CCS_MASK(engine->gt)) == engine->instance) ||
502 engine->class == RENDER_CLASS)
503 engine->flags |= I915_ENGINE_FIRST_RENDER_COMPUTE;
504
505 /* features common between engines sharing EUs */
506 if (engine->class == RENDER_CLASS || engine->class == COMPUTE_CLASS) {
507 engine->flags |= I915_ENGINE_HAS_RCS_REG_STATE;
508 engine->flags |= I915_ENGINE_HAS_EU_PRIORITY;
509 }
510
511 engine->props.heartbeat_interval_ms =
512 CONFIG_DRM_I915_HEARTBEAT_INTERVAL;
513 engine->props.max_busywait_duration_ns =
514 CONFIG_DRM_I915_MAX_REQUEST_BUSYWAIT;
515 engine->props.preempt_timeout_ms =
516 CONFIG_DRM_I915_PREEMPT_TIMEOUT;
517 engine->props.stop_timeout_ms =
518 CONFIG_DRM_I915_STOP_TIMEOUT;
519 engine->props.timeslice_duration_ms =
520 CONFIG_DRM_I915_TIMESLICE_DURATION;
521
522 /*
523 * Mid-thread pre-emption is not available in Gen12. Unfortunately,
524 * some compute workloads run quite long threads. That means they get
525 * reset due to not pre-empting in a timely manner. So, bump the
526 * pre-emption timeout value to be much higher for compute engines.
527 */
528 if (GRAPHICS_VER(i915) == 12 && (engine->flags & I915_ENGINE_HAS_RCS_REG_STATE))
529 engine->props.preempt_timeout_ms = CONFIG_DRM_I915_PREEMPT_TIMEOUT_COMPUTE;
530
531 /* Cap properties according to any system limits */
532#define CLAMP_PROP(field) \
533 do { \
534 u64 clamp = intel_clamp_##field(engine, engine->props.field); \
535 if (clamp != engine->props.field) { \
536 drm_notice(&engine->i915->drm, \
537 "Warning, clamping %s to %lld to prevent overflow\n", \
538 #field, clamp); \
539 engine->props.field = clamp; \
540 } \
541 } while (0)
542
543 CLAMP_PROP(heartbeat_interval_ms);
544 CLAMP_PROP(max_busywait_duration_ns);
545 CLAMP_PROP(preempt_timeout_ms);
546 CLAMP_PROP(stop_timeout_ms);
547 CLAMP_PROP(timeslice_duration_ms);
548
549#undef CLAMP_PROP
550
551 engine->defaults = engine->props; /* never to change again */
552
553 engine->context_size = intel_engine_context_size(gt, engine->class);
554 if (WARN_ON(engine->context_size > BIT(20)))
555 engine->context_size = 0;
556 if (engine->context_size)
557 DRIVER_CAPS(i915)->has_logical_contexts = true;
558
559 ewma__engine_latency_init(&engine->latency);
560 seqcount_init(&engine->stats.execlists.lock);
561
562 ATOMIC_INIT_NOTIFIER_HEAD(&engine->context_status_notifier);
563
564 /* Scrub mmio state on takeover */
565 intel_engine_sanitize_mmio(engine);
566
567 gt->engine_class[info->class][info->instance] = engine;
568 gt->engine[id] = engine;
569
570 return 0;
571}
572
573u64 intel_clamp_heartbeat_interval_ms(struct intel_engine_cs *engine, u64 value)
574{
575 value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT));
576
577 return value;
578}
579
580u64 intel_clamp_max_busywait_duration_ns(struct intel_engine_cs *engine, u64 value)
581{
582 value = min(value, jiffies_to_nsecs(2));
583
584 return value;
585}
586
587u64 intel_clamp_preempt_timeout_ms(struct intel_engine_cs *engine, u64 value)
588{
589 /*
590 * NB: The GuC API only supports 32bit values. However, the limit is further
591 * reduced due to internal calculations which would otherwise overflow.
592 */
593 if (intel_guc_submission_is_wanted(&engine->gt->uc.guc))
594 value = min_t(u64, value, guc_policy_max_preempt_timeout_ms());
595
596 value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT));
597
598 return value;
599}
600
601u64 intel_clamp_stop_timeout_ms(struct intel_engine_cs *engine, u64 value)
602{
603 value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT));
604
605 return value;
606}
607
608u64 intel_clamp_timeslice_duration_ms(struct intel_engine_cs *engine, u64 value)
609{
610 /*
611 * NB: The GuC API only supports 32bit values. However, the limit is further
612 * reduced due to internal calculations which would otherwise overflow.
613 */
614 if (intel_guc_submission_is_wanted(&engine->gt->uc.guc))
615 value = min_t(u64, value, guc_policy_max_exec_quantum_ms());
616
617 value = min_t(u64, value, jiffies_to_msecs(MAX_SCHEDULE_TIMEOUT));
618
619 return value;
620}
621
622static void __setup_engine_capabilities(struct intel_engine_cs *engine)
623{
624 struct drm_i915_private *i915 = engine->i915;
625
626 if (engine->class == VIDEO_DECODE_CLASS) {
627 /*
628 * HEVC support is present on first engine instance
629 * before Gen11 and on all instances afterwards.
630 */
631 if (GRAPHICS_VER(i915) >= 11 ||
632 (GRAPHICS_VER(i915) >= 9 && engine->instance == 0))
633 engine->uabi_capabilities |=
634 I915_VIDEO_CLASS_CAPABILITY_HEVC;
635
636 /*
637 * SFC block is present only on even logical engine
638 * instances.
639 */
640 if ((GRAPHICS_VER(i915) >= 11 &&
641 (engine->gt->info.vdbox_sfc_access &
642 BIT(engine->instance))) ||
643 (GRAPHICS_VER(i915) >= 9 && engine->instance == 0))
644 engine->uabi_capabilities |=
645 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
646 } else if (engine->class == VIDEO_ENHANCEMENT_CLASS) {
647 if (GRAPHICS_VER(i915) >= 9 &&
648 engine->gt->info.sfc_mask & BIT(engine->instance))
649 engine->uabi_capabilities |=
650 I915_VIDEO_AND_ENHANCE_CLASS_CAPABILITY_SFC;
651 }
652}
653
654static void intel_setup_engine_capabilities(struct intel_gt *gt)
655{
656 struct intel_engine_cs *engine;
657 enum intel_engine_id id;
658
659 for_each_engine(engine, gt, id)
660 __setup_engine_capabilities(engine);
661}
662
663/**
664 * intel_engines_release() - free the resources allocated for Command Streamers
665 * @gt: pointer to struct intel_gt
666 */
667void intel_engines_release(struct intel_gt *gt)
668{
669 struct intel_engine_cs *engine;
670 enum intel_engine_id id;
671
672 /*
673 * Before we release the resources held by engine, we must be certain
674 * that the HW is no longer accessing them -- having the GPU scribble
675 * to or read from a page being used for something else causes no end
676 * of fun.
677 *
678 * The GPU should be reset by this point, but assume the worst just
679 * in case we aborted before completely initialising the engines.
680 */
681 GEM_BUG_ON(intel_gt_pm_is_awake(gt));
682 if (!INTEL_INFO(gt->i915)->gpu_reset_clobbers_display)
683 __intel_gt_reset(gt, ALL_ENGINES);
684
685 /* Decouple the backend; but keep the layout for late GPU resets */
686 for_each_engine(engine, gt, id) {
687 if (!engine->release)
688 continue;
689
690 intel_wakeref_wait_for_idle(&engine->wakeref);
691 GEM_BUG_ON(intel_engine_pm_is_awake(engine));
692
693 engine->release(engine);
694 engine->release = NULL;
695
696 memset(&engine->reset, 0, sizeof(engine->reset));
697 }
698}
699
700void intel_engine_free_request_pool(struct intel_engine_cs *engine)
701{
702 if (!engine->request_pool)
703 return;
704
705 kmem_cache_free(i915_request_slab_cache(), engine->request_pool);
706}
707
708void intel_engines_free(struct intel_gt *gt)
709{
710 struct intel_engine_cs *engine;
711 enum intel_engine_id id;
712
713 /* Free the requests! dma-resv keeps fences around for an eternity */
714 rcu_barrier();
715
716 for_each_engine(engine, gt, id) {
717 intel_engine_free_request_pool(engine);
718 kfree(engine);
719 gt->engine[id] = NULL;
720 }
721}
722
723static
724bool gen11_vdbox_has_sfc(struct intel_gt *gt,
725 unsigned int physical_vdbox,
726 unsigned int logical_vdbox, u16 vdbox_mask)
727{
728 struct drm_i915_private *i915 = gt->i915;
729
730 /*
731 * In Gen11, only even numbered logical VDBOXes are hooked
732 * up to an SFC (Scaler & Format Converter) unit.
733 * In Gen12, Even numbered physical instance always are connected
734 * to an SFC. Odd numbered physical instances have SFC only if
735 * previous even instance is fused off.
736 *
737 * Starting with Xe_HP, there's also a dedicated SFC_ENABLE field
738 * in the fuse register that tells us whether a specific SFC is present.
739 */
740 if ((gt->info.sfc_mask & BIT(physical_vdbox / 2)) == 0)
741 return false;
742 else if (MEDIA_VER(i915) >= 12)
743 return (physical_vdbox % 2 == 0) ||
744 !(BIT(physical_vdbox - 1) & vdbox_mask);
745 else if (MEDIA_VER(i915) == 11)
746 return logical_vdbox % 2 == 0;
747
748 return false;
749}
750
751static void engine_mask_apply_media_fuses(struct intel_gt *gt)
752{
753 struct drm_i915_private *i915 = gt->i915;
754 unsigned int logical_vdbox = 0;
755 unsigned int i;
756 u32 media_fuse, fuse1;
757 u16 vdbox_mask;
758 u16 vebox_mask;
759
760 if (MEDIA_VER(gt->i915) < 11)
761 return;
762
763 /*
764 * On newer platforms the fusing register is called 'enable' and has
765 * enable semantics, while on older platforms it is called 'disable'
766 * and bits have disable semantices.
767 */
768 media_fuse = intel_uncore_read(gt->uncore, GEN11_GT_VEBOX_VDBOX_DISABLE);
769 if (MEDIA_VER_FULL(i915) < IP_VER(12, 50))
770 media_fuse = ~media_fuse;
771
772 vdbox_mask = media_fuse & GEN11_GT_VDBOX_DISABLE_MASK;
773 vebox_mask = (media_fuse & GEN11_GT_VEBOX_DISABLE_MASK) >>
774 GEN11_GT_VEBOX_DISABLE_SHIFT;
775
776 if (MEDIA_VER_FULL(i915) >= IP_VER(12, 50)) {
777 fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1);
778 gt->info.sfc_mask = REG_FIELD_GET(XEHP_SFC_ENABLE_MASK, fuse1);
779 } else {
780 gt->info.sfc_mask = ~0;
781 }
782
783 for (i = 0; i < I915_MAX_VCS; i++) {
784 if (!HAS_ENGINE(gt, _VCS(i))) {
785 vdbox_mask &= ~BIT(i);
786 continue;
787 }
788
789 if (!(BIT(i) & vdbox_mask)) {
790 gt->info.engine_mask &= ~BIT(_VCS(i));
791 drm_dbg(&i915->drm, "vcs%u fused off\n", i);
792 continue;
793 }
794
795 if (gen11_vdbox_has_sfc(gt, i, logical_vdbox, vdbox_mask))
796 gt->info.vdbox_sfc_access |= BIT(i);
797 logical_vdbox++;
798 }
799 drm_dbg(&i915->drm, "vdbox enable: %04x, instances: %04lx\n",
800 vdbox_mask, VDBOX_MASK(gt));
801 GEM_BUG_ON(vdbox_mask != VDBOX_MASK(gt));
802
803 for (i = 0; i < I915_MAX_VECS; i++) {
804 if (!HAS_ENGINE(gt, _VECS(i))) {
805 vebox_mask &= ~BIT(i);
806 continue;
807 }
808
809 if (!(BIT(i) & vebox_mask)) {
810 gt->info.engine_mask &= ~BIT(_VECS(i));
811 drm_dbg(&i915->drm, "vecs%u fused off\n", i);
812 }
813 }
814 drm_dbg(&i915->drm, "vebox enable: %04x, instances: %04lx\n",
815 vebox_mask, VEBOX_MASK(gt));
816 GEM_BUG_ON(vebox_mask != VEBOX_MASK(gt));
817}
818
819static void engine_mask_apply_compute_fuses(struct intel_gt *gt)
820{
821 struct drm_i915_private *i915 = gt->i915;
822 struct intel_gt_info *info = >->info;
823 int ss_per_ccs = info->sseu.max_subslices / I915_MAX_CCS;
824 unsigned long ccs_mask;
825 unsigned int i;
826
827 if (GRAPHICS_VER(i915) < 11)
828 return;
829
830 if (hweight32(CCS_MASK(gt)) <= 1)
831 return;
832
833 ccs_mask = intel_slicemask_from_xehp_dssmask(info->sseu.compute_subslice_mask,
834 ss_per_ccs);
835 /*
836 * If all DSS in a quadrant are fused off, the corresponding CCS
837 * engine is not available for use.
838 */
839 for_each_clear_bit(i, &ccs_mask, I915_MAX_CCS) {
840 info->engine_mask &= ~BIT(_CCS(i));
841 drm_dbg(&i915->drm, "ccs%u fused off\n", i);
842 }
843}
844
845static void engine_mask_apply_copy_fuses(struct intel_gt *gt)
846{
847 struct drm_i915_private *i915 = gt->i915;
848 struct intel_gt_info *info = >->info;
849 unsigned long meml3_mask;
850 unsigned long quad;
851
852 if (!(GRAPHICS_VER_FULL(i915) >= IP_VER(12, 60) &&
853 GRAPHICS_VER_FULL(i915) < IP_VER(12, 70)))
854 return;
855
856 meml3_mask = intel_uncore_read(gt->uncore, GEN10_MIRROR_FUSE3);
857 meml3_mask = REG_FIELD_GET(GEN12_MEML3_EN_MASK, meml3_mask);
858
859 /*
860 * Link Copy engines may be fused off according to meml3_mask. Each
861 * bit is a quad that houses 2 Link Copy and two Sub Copy engines.
862 */
863 for_each_clear_bit(quad, &meml3_mask, GEN12_MAX_MSLICES) {
864 unsigned int instance = quad * 2 + 1;
865 intel_engine_mask_t mask = GENMASK(_BCS(instance + 1),
866 _BCS(instance));
867
868 if (mask & info->engine_mask) {
869 drm_dbg(&i915->drm, "bcs%u fused off\n", instance);
870 drm_dbg(&i915->drm, "bcs%u fused off\n", instance + 1);
871
872 info->engine_mask &= ~mask;
873 }
874 }
875}
876
877/*
878 * Determine which engines are fused off in our particular hardware.
879 * Note that we have a catch-22 situation where we need to be able to access
880 * the blitter forcewake domain to read the engine fuses, but at the same time
881 * we need to know which engines are available on the system to know which
882 * forcewake domains are present. We solve this by intializing the forcewake
883 * domains based on the full engine mask in the platform capabilities before
884 * calling this function and pruning the domains for fused-off engines
885 * afterwards.
886 */
887static intel_engine_mask_t init_engine_mask(struct intel_gt *gt)
888{
889 struct intel_gt_info *info = >->info;
890
891 GEM_BUG_ON(!info->engine_mask);
892
893 engine_mask_apply_media_fuses(gt);
894 engine_mask_apply_compute_fuses(gt);
895 engine_mask_apply_copy_fuses(gt);
896
897 return info->engine_mask;
898}
899
900static void populate_logical_ids(struct intel_gt *gt, u8 *logical_ids,
901 u8 class, const u8 *map, u8 num_instances)
902{
903 int i, j;
904 u8 current_logical_id = 0;
905
906 for (j = 0; j < num_instances; ++j) {
907 for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) {
908 if (!HAS_ENGINE(gt, i) ||
909 intel_engines[i].class != class)
910 continue;
911
912 if (intel_engines[i].instance == map[j]) {
913 logical_ids[intel_engines[i].instance] =
914 current_logical_id++;
915 break;
916 }
917 }
918 }
919}
920
921static void setup_logical_ids(struct intel_gt *gt, u8 *logical_ids, u8 class)
922{
923 /*
924 * Logical to physical mapping is needed for proper support
925 * to split-frame feature.
926 */
927 if (MEDIA_VER(gt->i915) >= 11 && class == VIDEO_DECODE_CLASS) {
928 const u8 map[] = { 0, 2, 4, 6, 1, 3, 5, 7 };
929
930 populate_logical_ids(gt, logical_ids, class,
931 map, ARRAY_SIZE(map));
932 } else {
933 int i;
934 u8 map[MAX_ENGINE_INSTANCE + 1];
935
936 for (i = 0; i < MAX_ENGINE_INSTANCE + 1; ++i)
937 map[i] = i;
938 populate_logical_ids(gt, logical_ids, class,
939 map, ARRAY_SIZE(map));
940 }
941}
942
943/**
944 * intel_engines_init_mmio() - allocate and prepare the Engine Command Streamers
945 * @gt: pointer to struct intel_gt
946 *
947 * Return: non-zero if the initialization failed.
948 */
949int intel_engines_init_mmio(struct intel_gt *gt)
950{
951 struct drm_i915_private *i915 = gt->i915;
952 const unsigned int engine_mask = init_engine_mask(gt);
953 unsigned int mask = 0;
954 unsigned int i, class;
955 u8 logical_ids[MAX_ENGINE_INSTANCE + 1];
956 int err;
957
958 drm_WARN_ON(&i915->drm, engine_mask == 0);
959 drm_WARN_ON(&i915->drm, engine_mask &
960 GENMASK(BITS_PER_TYPE(mask) - 1, I915_NUM_ENGINES));
961
962 if (i915_inject_probe_failure(i915))
963 return -ENODEV;
964
965 for (class = 0; class < MAX_ENGINE_CLASS + 1; ++class) {
966 setup_logical_ids(gt, logical_ids, class);
967
968 for (i = 0; i < ARRAY_SIZE(intel_engines); ++i) {
969 u8 instance = intel_engines[i].instance;
970
971 if (intel_engines[i].class != class ||
972 !HAS_ENGINE(gt, i))
973 continue;
974
975 err = intel_engine_setup(gt, i,
976 logical_ids[instance]);
977 if (err)
978 goto cleanup;
979
980 mask |= BIT(i);
981 }
982 }
983
984 /*
985 * Catch failures to update intel_engines table when the new engines
986 * are added to the driver by a warning and disabling the forgotten
987 * engines.
988 */
989 if (drm_WARN_ON(&i915->drm, mask != engine_mask))
990 gt->info.engine_mask = mask;
991
992 gt->info.num_engines = hweight32(mask);
993
994 intel_gt_check_and_clear_faults(gt);
995
996 intel_setup_engine_capabilities(gt);
997
998 intel_uncore_prune_engine_fw_domains(gt->uncore, gt);
999
1000 return 0;
1001
1002cleanup:
1003 intel_engines_free(gt);
1004 return err;
1005}
1006
1007void intel_engine_init_execlists(struct intel_engine_cs *engine)
1008{
1009 struct intel_engine_execlists * const execlists = &engine->execlists;
1010
1011 execlists->port_mask = 1;
1012 GEM_BUG_ON(!is_power_of_2(execlists_num_ports(execlists)));
1013 GEM_BUG_ON(execlists_num_ports(execlists) > EXECLIST_MAX_PORTS);
1014
1015 memset(execlists->pending, 0, sizeof(execlists->pending));
1016 execlists->active =
1017 memset(execlists->inflight, 0, sizeof(execlists->inflight));
1018}
1019
1020static void cleanup_status_page(struct intel_engine_cs *engine)
1021{
1022 struct i915_vma *vma;
1023
1024 /* Prevent writes into HWSP after returning the page to the system */
1025 intel_engine_set_hwsp_writemask(engine, ~0u);
1026
1027 vma = fetch_and_zero(&engine->status_page.vma);
1028 if (!vma)
1029 return;
1030
1031 if (!HWS_NEEDS_PHYSICAL(engine->i915))
1032 i915_vma_unpin(vma);
1033
1034 i915_gem_object_unpin_map(vma->obj);
1035 i915_gem_object_put(vma->obj);
1036}
1037
1038static int pin_ggtt_status_page(struct intel_engine_cs *engine,
1039 struct i915_gem_ww_ctx *ww,
1040 struct i915_vma *vma)
1041{
1042 unsigned int flags;
1043
1044 if (!HAS_LLC(engine->i915) && i915_ggtt_has_aperture(engine->gt->ggtt))
1045 /*
1046 * On g33, we cannot place HWS above 256MiB, so
1047 * restrict its pinning to the low mappable arena.
1048 * Though this restriction is not documented for
1049 * gen4, gen5, or byt, they also behave similarly
1050 * and hang if the HWS is placed at the top of the
1051 * GTT. To generalise, it appears that all !llc
1052 * platforms have issues with us placing the HWS
1053 * above the mappable region (even though we never
1054 * actually map it).
1055 */
1056 flags = PIN_MAPPABLE;
1057 else
1058 flags = PIN_HIGH;
1059
1060 return i915_ggtt_pin(vma, ww, 0, flags);
1061}
1062
1063static int init_status_page(struct intel_engine_cs *engine)
1064{
1065 struct drm_i915_gem_object *obj;
1066 struct i915_gem_ww_ctx ww;
1067 struct i915_vma *vma;
1068 void *vaddr;
1069 int ret;
1070
1071 INIT_LIST_HEAD(&engine->status_page.timelines);
1072
1073 /*
1074 * Though the HWS register does support 36bit addresses, historically
1075 * we have had hangs and corruption reported due to wild writes if
1076 * the HWS is placed above 4G. We only allow objects to be allocated
1077 * in GFP_DMA32 for i965, and no earlier physical address users had
1078 * access to more than 4G.
1079 */
1080 obj = i915_gem_object_create_internal(engine->i915, PAGE_SIZE);
1081 if (IS_ERR(obj)) {
1082 drm_err(&engine->i915->drm,
1083 "Failed to allocate status page\n");
1084 return PTR_ERR(obj);
1085 }
1086
1087 i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
1088
1089 vma = i915_vma_instance(obj, &engine->gt->ggtt->vm, NULL);
1090 if (IS_ERR(vma)) {
1091 ret = PTR_ERR(vma);
1092 goto err_put;
1093 }
1094
1095 i915_gem_ww_ctx_init(&ww, true);
1096retry:
1097 ret = i915_gem_object_lock(obj, &ww);
1098 if (!ret && !HWS_NEEDS_PHYSICAL(engine->i915))
1099 ret = pin_ggtt_status_page(engine, &ww, vma);
1100 if (ret)
1101 goto err;
1102
1103 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WB);
1104 if (IS_ERR(vaddr)) {
1105 ret = PTR_ERR(vaddr);
1106 goto err_unpin;
1107 }
1108
1109 engine->status_page.addr = memset(vaddr, 0, PAGE_SIZE);
1110 engine->status_page.vma = vma;
1111
1112err_unpin:
1113 if (ret)
1114 i915_vma_unpin(vma);
1115err:
1116 if (ret == -EDEADLK) {
1117 ret = i915_gem_ww_ctx_backoff(&ww);
1118 if (!ret)
1119 goto retry;
1120 }
1121 i915_gem_ww_ctx_fini(&ww);
1122err_put:
1123 if (ret)
1124 i915_gem_object_put(obj);
1125 return ret;
1126}
1127
1128static int engine_setup_common(struct intel_engine_cs *engine)
1129{
1130 int err;
1131
1132 init_llist_head(&engine->barrier_tasks);
1133
1134 err = init_status_page(engine);
1135 if (err)
1136 return err;
1137
1138 engine->breadcrumbs = intel_breadcrumbs_create(engine);
1139 if (!engine->breadcrumbs) {
1140 err = -ENOMEM;
1141 goto err_status;
1142 }
1143
1144 engine->sched_engine = i915_sched_engine_create(ENGINE_PHYSICAL);
1145 if (!engine->sched_engine) {
1146 err = -ENOMEM;
1147 goto err_sched_engine;
1148 }
1149 engine->sched_engine->private_data = engine;
1150
1151 err = intel_engine_init_cmd_parser(engine);
1152 if (err)
1153 goto err_cmd_parser;
1154
1155 intel_engine_init_execlists(engine);
1156 intel_engine_init__pm(engine);
1157 intel_engine_init_retire(engine);
1158
1159 /* Use the whole device by default */
1160 engine->sseu =
1161 intel_sseu_from_device_info(&engine->gt->info.sseu);
1162
1163 intel_engine_init_workarounds(engine);
1164 intel_engine_init_whitelist(engine);
1165 intel_engine_init_ctx_wa(engine);
1166
1167 if (GRAPHICS_VER(engine->i915) >= 12)
1168 engine->flags |= I915_ENGINE_HAS_RELATIVE_MMIO;
1169
1170 return 0;
1171
1172err_cmd_parser:
1173 i915_sched_engine_put(engine->sched_engine);
1174err_sched_engine:
1175 intel_breadcrumbs_put(engine->breadcrumbs);
1176err_status:
1177 cleanup_status_page(engine);
1178 return err;
1179}
1180
1181struct measure_breadcrumb {
1182 struct i915_request rq;
1183 struct intel_ring ring;
1184 u32 cs[2048];
1185};
1186
1187static int measure_breadcrumb_dw(struct intel_context *ce)
1188{
1189 struct intel_engine_cs *engine = ce->engine;
1190 struct measure_breadcrumb *frame;
1191 int dw;
1192
1193 GEM_BUG_ON(!engine->gt->scratch);
1194
1195 frame = kzalloc(sizeof(*frame), GFP_KERNEL);
1196 if (!frame)
1197 return -ENOMEM;
1198
1199 frame->rq.engine = engine;
1200 frame->rq.context = ce;
1201 rcu_assign_pointer(frame->rq.timeline, ce->timeline);
1202 frame->rq.hwsp_seqno = ce->timeline->hwsp_seqno;
1203
1204 frame->ring.vaddr = frame->cs;
1205 frame->ring.size = sizeof(frame->cs);
1206 frame->ring.wrap =
1207 BITS_PER_TYPE(frame->ring.size) - ilog2(frame->ring.size);
1208 frame->ring.effective_size = frame->ring.size;
1209 intel_ring_update_space(&frame->ring);
1210 frame->rq.ring = &frame->ring;
1211
1212 mutex_lock(&ce->timeline->mutex);
1213 spin_lock_irq(&engine->sched_engine->lock);
1214
1215 dw = engine->emit_fini_breadcrumb(&frame->rq, frame->cs) - frame->cs;
1216
1217 spin_unlock_irq(&engine->sched_engine->lock);
1218 mutex_unlock(&ce->timeline->mutex);
1219
1220 GEM_BUG_ON(dw & 1); /* RING_TAIL must be qword aligned */
1221
1222 kfree(frame);
1223 return dw;
1224}
1225
1226struct intel_context *
1227intel_engine_create_pinned_context(struct intel_engine_cs *engine,
1228 struct i915_address_space *vm,
1229 unsigned int ring_size,
1230 unsigned int hwsp,
1231 struct lock_class_key *key,
1232 const char *name)
1233{
1234 struct intel_context *ce;
1235 int err;
1236
1237 ce = intel_context_create(engine);
1238 if (IS_ERR(ce))
1239 return ce;
1240
1241 __set_bit(CONTEXT_BARRIER_BIT, &ce->flags);
1242 ce->timeline = page_pack_bits(NULL, hwsp);
1243 ce->ring = NULL;
1244 ce->ring_size = ring_size;
1245
1246 i915_vm_put(ce->vm);
1247 ce->vm = i915_vm_get(vm);
1248
1249 err = intel_context_pin(ce); /* perma-pin so it is always available */
1250 if (err) {
1251 intel_context_put(ce);
1252 return ERR_PTR(err);
1253 }
1254
1255 list_add_tail(&ce->pinned_contexts_link, &engine->pinned_contexts_list);
1256
1257 /*
1258 * Give our perma-pinned kernel timelines a separate lockdep class,
1259 * so that we can use them from within the normal user timelines
1260 * should we need to inject GPU operations during their request
1261 * construction.
1262 */
1263 lockdep_set_class_and_name(&ce->timeline->mutex, key, name);
1264
1265 return ce;
1266}
1267
1268void intel_engine_destroy_pinned_context(struct intel_context *ce)
1269{
1270 struct intel_engine_cs *engine = ce->engine;
1271 struct i915_vma *hwsp = engine->status_page.vma;
1272
1273 GEM_BUG_ON(ce->timeline->hwsp_ggtt != hwsp);
1274
1275 mutex_lock(&hwsp->vm->mutex);
1276 list_del(&ce->timeline->engine_link);
1277 mutex_unlock(&hwsp->vm->mutex);
1278
1279 list_del(&ce->pinned_contexts_link);
1280 intel_context_unpin(ce);
1281 intel_context_put(ce);
1282}
1283
1284static struct intel_context *
1285create_kernel_context(struct intel_engine_cs *engine)
1286{
1287 static struct lock_class_key kernel;
1288
1289 return intel_engine_create_pinned_context(engine, engine->gt->vm, SZ_4K,
1290 I915_GEM_HWS_SEQNO_ADDR,
1291 &kernel, "kernel_context");
1292}
1293
1294/**
1295 * intel_engines_init_common - initialize cengine state which might require hw access
1296 * @engine: Engine to initialize.
1297 *
1298 * Initializes @engine@ structure members shared between legacy and execlists
1299 * submission modes which do require hardware access.
1300 *
1301 * Typcally done at later stages of submission mode specific engine setup.
1302 *
1303 * Returns zero on success or an error code on failure.
1304 */
1305static int engine_init_common(struct intel_engine_cs *engine)
1306{
1307 struct intel_context *ce;
1308 int ret;
1309
1310 engine->set_default_submission(engine);
1311
1312 /*
1313 * We may need to do things with the shrinker which
1314 * require us to immediately switch back to the default
1315 * context. This can cause a problem as pinning the
1316 * default context also requires GTT space which may not
1317 * be available. To avoid this we always pin the default
1318 * context.
1319 */
1320 ce = create_kernel_context(engine);
1321 if (IS_ERR(ce))
1322 return PTR_ERR(ce);
1323
1324 ret = measure_breadcrumb_dw(ce);
1325 if (ret < 0)
1326 goto err_context;
1327
1328 engine->emit_fini_breadcrumb_dw = ret;
1329 engine->kernel_context = ce;
1330
1331 return 0;
1332
1333err_context:
1334 intel_engine_destroy_pinned_context(ce);
1335 return ret;
1336}
1337
1338int intel_engines_init(struct intel_gt *gt)
1339{
1340 int (*setup)(struct intel_engine_cs *engine);
1341 struct intel_engine_cs *engine;
1342 enum intel_engine_id id;
1343 int err;
1344
1345 if (intel_uc_uses_guc_submission(>->uc)) {
1346 gt->submission_method = INTEL_SUBMISSION_GUC;
1347 setup = intel_guc_submission_setup;
1348 } else if (HAS_EXECLISTS(gt->i915)) {
1349 gt->submission_method = INTEL_SUBMISSION_ELSP;
1350 setup = intel_execlists_submission_setup;
1351 } else {
1352 gt->submission_method = INTEL_SUBMISSION_RING;
1353 setup = intel_ring_submission_setup;
1354 }
1355
1356 for_each_engine(engine, gt, id) {
1357 err = engine_setup_common(engine);
1358 if (err)
1359 return err;
1360
1361 err = setup(engine);
1362 if (err) {
1363 intel_engine_cleanup_common(engine);
1364 return err;
1365 }
1366
1367 /* The backend should now be responsible for cleanup */
1368 GEM_BUG_ON(engine->release == NULL);
1369
1370 err = engine_init_common(engine);
1371 if (err)
1372 return err;
1373
1374 intel_engine_add_user(engine);
1375 }
1376
1377 return 0;
1378}
1379
1380/**
1381 * intel_engines_cleanup_common - cleans up the engine state created by
1382 * the common initiailizers.
1383 * @engine: Engine to cleanup.
1384 *
1385 * This cleans up everything created by the common helpers.
1386 */
1387void intel_engine_cleanup_common(struct intel_engine_cs *engine)
1388{
1389 GEM_BUG_ON(!list_empty(&engine->sched_engine->requests));
1390
1391 i915_sched_engine_put(engine->sched_engine);
1392 intel_breadcrumbs_put(engine->breadcrumbs);
1393
1394 intel_engine_fini_retire(engine);
1395 intel_engine_cleanup_cmd_parser(engine);
1396
1397 if (engine->default_state)
1398 fput(engine->default_state);
1399
1400 if (engine->kernel_context)
1401 intel_engine_destroy_pinned_context(engine->kernel_context);
1402
1403 GEM_BUG_ON(!llist_empty(&engine->barrier_tasks));
1404 cleanup_status_page(engine);
1405
1406 intel_wa_list_free(&engine->ctx_wa_list);
1407 intel_wa_list_free(&engine->wa_list);
1408 intel_wa_list_free(&engine->whitelist);
1409}
1410
1411/**
1412 * intel_engine_resume - re-initializes the HW state of the engine
1413 * @engine: Engine to resume.
1414 *
1415 * Returns zero on success or an error code on failure.
1416 */
1417int intel_engine_resume(struct intel_engine_cs *engine)
1418{
1419 intel_engine_apply_workarounds(engine);
1420 intel_engine_apply_whitelist(engine);
1421
1422 return engine->resume(engine);
1423}
1424
1425u64 intel_engine_get_active_head(const struct intel_engine_cs *engine)
1426{
1427 struct drm_i915_private *i915 = engine->i915;
1428
1429 u64 acthd;
1430
1431 if (GRAPHICS_VER(i915) >= 8)
1432 acthd = ENGINE_READ64(engine, RING_ACTHD, RING_ACTHD_UDW);
1433 else if (GRAPHICS_VER(i915) >= 4)
1434 acthd = ENGINE_READ(engine, RING_ACTHD);
1435 else
1436 acthd = ENGINE_READ(engine, ACTHD);
1437
1438 return acthd;
1439}
1440
1441u64 intel_engine_get_last_batch_head(const struct intel_engine_cs *engine)
1442{
1443 u64 bbaddr;
1444
1445 if (GRAPHICS_VER(engine->i915) >= 8)
1446 bbaddr = ENGINE_READ64(engine, RING_BBADDR, RING_BBADDR_UDW);
1447 else
1448 bbaddr = ENGINE_READ(engine, RING_BBADDR);
1449
1450 return bbaddr;
1451}
1452
1453static unsigned long stop_timeout(const struct intel_engine_cs *engine)
1454{
1455 if (in_atomic() || irqs_disabled()) /* inside atomic preempt-reset? */
1456 return 0;
1457
1458 /*
1459 * If we are doing a normal GPU reset, we can take our time and allow
1460 * the engine to quiesce. We've stopped submission to the engine, and
1461 * if we wait long enough an innocent context should complete and
1462 * leave the engine idle. So they should not be caught unaware by
1463 * the forthcoming GPU reset (which usually follows the stop_cs)!
1464 */
1465 return READ_ONCE(engine->props.stop_timeout_ms);
1466}
1467
1468static int __intel_engine_stop_cs(struct intel_engine_cs *engine,
1469 int fast_timeout_us,
1470 int slow_timeout_ms)
1471{
1472 struct intel_uncore *uncore = engine->uncore;
1473 const i915_reg_t mode = RING_MI_MODE(engine->mmio_base);
1474 int err;
1475
1476 intel_uncore_write_fw(uncore, mode, _MASKED_BIT_ENABLE(STOP_RING));
1477
1478 /*
1479 * Wa_22011802037 : gen11, gen12, Prior to doing a reset, ensure CS is
1480 * stopped, set ring stop bit and prefetch disable bit to halt CS
1481 */
1482 if (IS_GRAPHICS_VER(engine->i915, 11, 12))
1483 intel_uncore_write_fw(uncore, RING_MODE_GEN7(engine->mmio_base),
1484 _MASKED_BIT_ENABLE(GEN12_GFX_PREFETCH_DISABLE));
1485
1486 err = __intel_wait_for_register_fw(engine->uncore, mode,
1487 MODE_IDLE, MODE_IDLE,
1488 fast_timeout_us,
1489 slow_timeout_ms,
1490 NULL);
1491
1492 /* A final mmio read to let GPU writes be hopefully flushed to memory */
1493 intel_uncore_posting_read_fw(uncore, mode);
1494 return err;
1495}
1496
1497int intel_engine_stop_cs(struct intel_engine_cs *engine)
1498{
1499 int err = 0;
1500
1501 if (GRAPHICS_VER(engine->i915) < 3)
1502 return -ENODEV;
1503
1504 ENGINE_TRACE(engine, "\n");
1505 /*
1506 * TODO: Find out why occasionally stopping the CS times out. Seen
1507 * especially with gem_eio tests.
1508 *
1509 * Occasionally trying to stop the cs times out, but does not adversely
1510 * affect functionality. The timeout is set as a config parameter that
1511 * defaults to 100ms. In most cases the follow up operation is to wait
1512 * for pending MI_FORCE_WAKES. The assumption is that this timeout is
1513 * sufficient for any pending MI_FORCEWAKEs to complete. Once root
1514 * caused, the caller must check and handle the return from this
1515 * function.
1516 */
1517 if (__intel_engine_stop_cs(engine, 1000, stop_timeout(engine))) {
1518 ENGINE_TRACE(engine,
1519 "timed out on STOP_RING -> IDLE; HEAD:%04x, TAIL:%04x\n",
1520 ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR,
1521 ENGINE_READ_FW(engine, RING_TAIL) & TAIL_ADDR);
1522
1523 /*
1524 * Sometimes we observe that the idle flag is not
1525 * set even though the ring is empty. So double
1526 * check before giving up.
1527 */
1528 if ((ENGINE_READ_FW(engine, RING_HEAD) & HEAD_ADDR) !=
1529 (ENGINE_READ_FW(engine, RING_TAIL) & TAIL_ADDR))
1530 err = -ETIMEDOUT;
1531 }
1532
1533 return err;
1534}
1535
1536void intel_engine_cancel_stop_cs(struct intel_engine_cs *engine)
1537{
1538 ENGINE_TRACE(engine, "\n");
1539
1540 ENGINE_WRITE_FW(engine, RING_MI_MODE, _MASKED_BIT_DISABLE(STOP_RING));
1541}
1542
1543static u32 __cs_pending_mi_force_wakes(struct intel_engine_cs *engine)
1544{
1545 static const i915_reg_t _reg[I915_NUM_ENGINES] = {
1546 [RCS0] = MSG_IDLE_CS,
1547 [BCS0] = MSG_IDLE_BCS,
1548 [VCS0] = MSG_IDLE_VCS0,
1549 [VCS1] = MSG_IDLE_VCS1,
1550 [VCS2] = MSG_IDLE_VCS2,
1551 [VCS3] = MSG_IDLE_VCS3,
1552 [VCS4] = MSG_IDLE_VCS4,
1553 [VCS5] = MSG_IDLE_VCS5,
1554 [VCS6] = MSG_IDLE_VCS6,
1555 [VCS7] = MSG_IDLE_VCS7,
1556 [VECS0] = MSG_IDLE_VECS0,
1557 [VECS1] = MSG_IDLE_VECS1,
1558 [VECS2] = MSG_IDLE_VECS2,
1559 [VECS3] = MSG_IDLE_VECS3,
1560 [CCS0] = MSG_IDLE_CS,
1561 [CCS1] = MSG_IDLE_CS,
1562 [CCS2] = MSG_IDLE_CS,
1563 [CCS3] = MSG_IDLE_CS,
1564 };
1565 u32 val;
1566
1567 if (!_reg[engine->id].reg) {
1568 drm_err(&engine->i915->drm,
1569 "MSG IDLE undefined for engine id %u\n", engine->id);
1570 return 0;
1571 }
1572
1573 val = intel_uncore_read(engine->uncore, _reg[engine->id]);
1574
1575 /* bits[29:25] & bits[13:9] >> shift */
1576 return (val & (val >> 16) & MSG_IDLE_FW_MASK) >> MSG_IDLE_FW_SHIFT;
1577}
1578
1579static void __gpm_wait_for_fw_complete(struct intel_gt *gt, u32 fw_mask)
1580{
1581 int ret;
1582
1583 /* Ensure GPM receives fw up/down after CS is stopped */
1584 udelay(1);
1585
1586 /* Wait for forcewake request to complete in GPM */
1587 ret = __intel_wait_for_register_fw(gt->uncore,
1588 GEN9_PWRGT_DOMAIN_STATUS,
1589 fw_mask, fw_mask, 5000, 0, NULL);
1590
1591 /* Ensure CS receives fw ack from GPM */
1592 udelay(1);
1593
1594 if (ret)
1595 GT_TRACE(gt, "Failed to complete pending forcewake %d\n", ret);
1596}
1597
1598/*
1599 * Wa_22011802037:gen12: In addition to stopping the cs, we need to wait for any
1600 * pending MI_FORCE_WAKEUP requests that the CS has initiated to complete. The
1601 * pending status is indicated by bits[13:9] (masked by bits[29:25]) in the
1602 * MSG_IDLE register. There's one MSG_IDLE register per reset domain. Since we
1603 * are concerned only with the gt reset here, we use a logical OR of pending
1604 * forcewakeups from all reset domains and then wait for them to complete by
1605 * querying PWRGT_DOMAIN_STATUS.
1606 */
1607void intel_engine_wait_for_pending_mi_fw(struct intel_engine_cs *engine)
1608{
1609 u32 fw_pending = __cs_pending_mi_force_wakes(engine);
1610
1611 if (fw_pending)
1612 __gpm_wait_for_fw_complete(engine->gt, fw_pending);
1613}
1614
1615/* NB: please notice the memset */
1616void intel_engine_get_instdone(const struct intel_engine_cs *engine,
1617 struct intel_instdone *instdone)
1618{
1619 struct drm_i915_private *i915 = engine->i915;
1620 struct intel_uncore *uncore = engine->uncore;
1621 u32 mmio_base = engine->mmio_base;
1622 int slice;
1623 int subslice;
1624 int iter;
1625
1626 memset(instdone, 0, sizeof(*instdone));
1627
1628 if (GRAPHICS_VER(i915) >= 8) {
1629 instdone->instdone =
1630 intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1631
1632 if (engine->id != RCS0)
1633 return;
1634
1635 instdone->slice_common =
1636 intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1637 if (GRAPHICS_VER(i915) >= 12) {
1638 instdone->slice_common_extra[0] =
1639 intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA);
1640 instdone->slice_common_extra[1] =
1641 intel_uncore_read(uncore, GEN12_SC_INSTDONE_EXTRA2);
1642 }
1643
1644 for_each_ss_steering(iter, engine->gt, slice, subslice) {
1645 instdone->sampler[slice][subslice] =
1646 intel_gt_mcr_read(engine->gt,
1647 GEN8_SAMPLER_INSTDONE,
1648 slice, subslice);
1649 instdone->row[slice][subslice] =
1650 intel_gt_mcr_read(engine->gt,
1651 GEN8_ROW_INSTDONE,
1652 slice, subslice);
1653 }
1654
1655 if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55)) {
1656 for_each_ss_steering(iter, engine->gt, slice, subslice)
1657 instdone->geom_svg[slice][subslice] =
1658 intel_gt_mcr_read(engine->gt,
1659 XEHPG_INSTDONE_GEOM_SVG,
1660 slice, subslice);
1661 }
1662 } else if (GRAPHICS_VER(i915) >= 7) {
1663 instdone->instdone =
1664 intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1665
1666 if (engine->id != RCS0)
1667 return;
1668
1669 instdone->slice_common =
1670 intel_uncore_read(uncore, GEN7_SC_INSTDONE);
1671 instdone->sampler[0][0] =
1672 intel_uncore_read(uncore, GEN7_SAMPLER_INSTDONE);
1673 instdone->row[0][0] =
1674 intel_uncore_read(uncore, GEN7_ROW_INSTDONE);
1675 } else if (GRAPHICS_VER(i915) >= 4) {
1676 instdone->instdone =
1677 intel_uncore_read(uncore, RING_INSTDONE(mmio_base));
1678 if (engine->id == RCS0)
1679 /* HACK: Using the wrong struct member */
1680 instdone->slice_common =
1681 intel_uncore_read(uncore, GEN4_INSTDONE1);
1682 } else {
1683 instdone->instdone = intel_uncore_read(uncore, GEN2_INSTDONE);
1684 }
1685}
1686
1687static bool ring_is_idle(struct intel_engine_cs *engine)
1688{
1689 bool idle = true;
1690
1691 if (I915_SELFTEST_ONLY(!engine->mmio_base))
1692 return true;
1693
1694 if (!intel_engine_pm_get_if_awake(engine))
1695 return true;
1696
1697 /* First check that no commands are left in the ring */
1698 if ((ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR) !=
1699 (ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR))
1700 idle = false;
1701
1702 /* No bit for gen2, so assume the CS parser is idle */
1703 if (GRAPHICS_VER(engine->i915) > 2 &&
1704 !(ENGINE_READ(engine, RING_MI_MODE) & MODE_IDLE))
1705 idle = false;
1706
1707 intel_engine_pm_put(engine);
1708
1709 return idle;
1710}
1711
1712void __intel_engine_flush_submission(struct intel_engine_cs *engine, bool sync)
1713{
1714 struct tasklet_struct *t = &engine->sched_engine->tasklet;
1715
1716 if (!t->callback)
1717 return;
1718
1719 local_bh_disable();
1720 if (tasklet_trylock(t)) {
1721 /* Must wait for any GPU reset in progress. */
1722 if (__tasklet_is_enabled(t))
1723 t->callback(t);
1724 tasklet_unlock(t);
1725 }
1726 local_bh_enable();
1727
1728 /* Synchronise and wait for the tasklet on another CPU */
1729 if (sync)
1730 tasklet_unlock_wait(t);
1731}
1732
1733/**
1734 * intel_engine_is_idle() - Report if the engine has finished process all work
1735 * @engine: the intel_engine_cs
1736 *
1737 * Return true if there are no requests pending, nothing left to be submitted
1738 * to hardware, and that the engine is idle.
1739 */
1740bool intel_engine_is_idle(struct intel_engine_cs *engine)
1741{
1742 /* More white lies, if wedged, hw state is inconsistent */
1743 if (intel_gt_is_wedged(engine->gt))
1744 return true;
1745
1746 if (!intel_engine_pm_is_awake(engine))
1747 return true;
1748
1749 /* Waiting to drain ELSP? */
1750 intel_synchronize_hardirq(engine->i915);
1751 intel_engine_flush_submission(engine);
1752
1753 /* ELSP is empty, but there are ready requests? E.g. after reset */
1754 if (!i915_sched_engine_is_empty(engine->sched_engine))
1755 return false;
1756
1757 /* Ring stopped? */
1758 return ring_is_idle(engine);
1759}
1760
1761bool intel_engines_are_idle(struct intel_gt *gt)
1762{
1763 struct intel_engine_cs *engine;
1764 enum intel_engine_id id;
1765
1766 /*
1767 * If the driver is wedged, HW state may be very inconsistent and
1768 * report that it is still busy, even though we have stopped using it.
1769 */
1770 if (intel_gt_is_wedged(gt))
1771 return true;
1772
1773 /* Already parked (and passed an idleness test); must still be idle */
1774 if (!READ_ONCE(gt->awake))
1775 return true;
1776
1777 for_each_engine(engine, gt, id) {
1778 if (!intel_engine_is_idle(engine))
1779 return false;
1780 }
1781
1782 return true;
1783}
1784
1785bool intel_engine_irq_enable(struct intel_engine_cs *engine)
1786{
1787 if (!engine->irq_enable)
1788 return false;
1789
1790 /* Caller disables interrupts */
1791 spin_lock(engine->gt->irq_lock);
1792 engine->irq_enable(engine);
1793 spin_unlock(engine->gt->irq_lock);
1794
1795 return true;
1796}
1797
1798void intel_engine_irq_disable(struct intel_engine_cs *engine)
1799{
1800 if (!engine->irq_disable)
1801 return;
1802
1803 /* Caller disables interrupts */
1804 spin_lock(engine->gt->irq_lock);
1805 engine->irq_disable(engine);
1806 spin_unlock(engine->gt->irq_lock);
1807}
1808
1809void intel_engines_reset_default_submission(struct intel_gt *gt)
1810{
1811 struct intel_engine_cs *engine;
1812 enum intel_engine_id id;
1813
1814 for_each_engine(engine, gt, id) {
1815 if (engine->sanitize)
1816 engine->sanitize(engine);
1817
1818 engine->set_default_submission(engine);
1819 }
1820}
1821
1822bool intel_engine_can_store_dword(struct intel_engine_cs *engine)
1823{
1824 switch (GRAPHICS_VER(engine->i915)) {
1825 case 2:
1826 return false; /* uses physical not virtual addresses */
1827 case 3:
1828 /* maybe only uses physical not virtual addresses */
1829 return !(IS_I915G(engine->i915) || IS_I915GM(engine->i915));
1830 case 4:
1831 return !IS_I965G(engine->i915); /* who knows! */
1832 case 6:
1833 return engine->class != VIDEO_DECODE_CLASS; /* b0rked */
1834 default:
1835 return true;
1836 }
1837}
1838
1839static struct intel_timeline *get_timeline(struct i915_request *rq)
1840{
1841 struct intel_timeline *tl;
1842
1843 /*
1844 * Even though we are holding the engine->sched_engine->lock here, there
1845 * is no control over the submission queue per-se and we are
1846 * inspecting the active state at a random point in time, with an
1847 * unknown queue. Play safe and make sure the timeline remains valid.
1848 * (Only being used for pretty printing, one extra kref shouldn't
1849 * cause a camel stampede!)
1850 */
1851 rcu_read_lock();
1852 tl = rcu_dereference(rq->timeline);
1853 if (!kref_get_unless_zero(&tl->kref))
1854 tl = NULL;
1855 rcu_read_unlock();
1856
1857 return tl;
1858}
1859
1860static int print_ring(char *buf, int sz, struct i915_request *rq)
1861{
1862 int len = 0;
1863
1864 if (!i915_request_signaled(rq)) {
1865 struct intel_timeline *tl = get_timeline(rq);
1866
1867 len = scnprintf(buf, sz,
1868 "ring:{start:%08x, hwsp:%08x, seqno:%08x, runtime:%llums}, ",
1869 i915_ggtt_offset(rq->ring->vma),
1870 tl ? tl->hwsp_offset : 0,
1871 hwsp_seqno(rq),
1872 DIV_ROUND_CLOSEST_ULL(intel_context_get_total_runtime_ns(rq->context),
1873 1000 * 1000));
1874
1875 if (tl)
1876 intel_timeline_put(tl);
1877 }
1878
1879 return len;
1880}
1881
1882static void hexdump(struct drm_printer *m, const void *buf, size_t len)
1883{
1884 const size_t rowsize = 8 * sizeof(u32);
1885 const void *prev = NULL;
1886 bool skip = false;
1887 size_t pos;
1888
1889 for (pos = 0; pos < len; pos += rowsize) {
1890 char line[128];
1891
1892 if (prev && !memcmp(prev, buf + pos, rowsize)) {
1893 if (!skip) {
1894 drm_printf(m, "*\n");
1895 skip = true;
1896 }
1897 continue;
1898 }
1899
1900 WARN_ON_ONCE(hex_dump_to_buffer(buf + pos, len - pos,
1901 rowsize, sizeof(u32),
1902 line, sizeof(line),
1903 false) >= sizeof(line));
1904 drm_printf(m, "[%04zx] %s\n", pos, line);
1905
1906 prev = buf + pos;
1907 skip = false;
1908 }
1909}
1910
1911static const char *repr_timer(const struct timer_list *t)
1912{
1913 if (!READ_ONCE(t->expires))
1914 return "inactive";
1915
1916 if (timer_pending(t))
1917 return "active";
1918
1919 return "expired";
1920}
1921
1922static void intel_engine_print_registers(struct intel_engine_cs *engine,
1923 struct drm_printer *m)
1924{
1925 struct drm_i915_private *dev_priv = engine->i915;
1926 struct intel_engine_execlists * const execlists = &engine->execlists;
1927 u64 addr;
1928
1929 if (engine->id == RENDER_CLASS && IS_GRAPHICS_VER(dev_priv, 4, 7))
1930 drm_printf(m, "\tCCID: 0x%08x\n", ENGINE_READ(engine, CCID));
1931 if (HAS_EXECLISTS(dev_priv)) {
1932 drm_printf(m, "\tEL_STAT_HI: 0x%08x\n",
1933 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI));
1934 drm_printf(m, "\tEL_STAT_LO: 0x%08x\n",
1935 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO));
1936 }
1937 drm_printf(m, "\tRING_START: 0x%08x\n",
1938 ENGINE_READ(engine, RING_START));
1939 drm_printf(m, "\tRING_HEAD: 0x%08x\n",
1940 ENGINE_READ(engine, RING_HEAD) & HEAD_ADDR);
1941 drm_printf(m, "\tRING_TAIL: 0x%08x\n",
1942 ENGINE_READ(engine, RING_TAIL) & TAIL_ADDR);
1943 drm_printf(m, "\tRING_CTL: 0x%08x%s\n",
1944 ENGINE_READ(engine, RING_CTL),
1945 ENGINE_READ(engine, RING_CTL) & (RING_WAIT | RING_WAIT_SEMAPHORE) ? " [waiting]" : "");
1946 if (GRAPHICS_VER(engine->i915) > 2) {
1947 drm_printf(m, "\tRING_MODE: 0x%08x%s\n",
1948 ENGINE_READ(engine, RING_MI_MODE),
1949 ENGINE_READ(engine, RING_MI_MODE) & (MODE_IDLE) ? " [idle]" : "");
1950 }
1951
1952 if (GRAPHICS_VER(dev_priv) >= 6) {
1953 drm_printf(m, "\tRING_IMR: 0x%08x\n",
1954 ENGINE_READ(engine, RING_IMR));
1955 drm_printf(m, "\tRING_ESR: 0x%08x\n",
1956 ENGINE_READ(engine, RING_ESR));
1957 drm_printf(m, "\tRING_EMR: 0x%08x\n",
1958 ENGINE_READ(engine, RING_EMR));
1959 drm_printf(m, "\tRING_EIR: 0x%08x\n",
1960 ENGINE_READ(engine, RING_EIR));
1961 }
1962
1963 addr = intel_engine_get_active_head(engine);
1964 drm_printf(m, "\tACTHD: 0x%08x_%08x\n",
1965 upper_32_bits(addr), lower_32_bits(addr));
1966 addr = intel_engine_get_last_batch_head(engine);
1967 drm_printf(m, "\tBBADDR: 0x%08x_%08x\n",
1968 upper_32_bits(addr), lower_32_bits(addr));
1969 if (GRAPHICS_VER(dev_priv) >= 8)
1970 addr = ENGINE_READ64(engine, RING_DMA_FADD, RING_DMA_FADD_UDW);
1971 else if (GRAPHICS_VER(dev_priv) >= 4)
1972 addr = ENGINE_READ(engine, RING_DMA_FADD);
1973 else
1974 addr = ENGINE_READ(engine, DMA_FADD_I8XX);
1975 drm_printf(m, "\tDMA_FADDR: 0x%08x_%08x\n",
1976 upper_32_bits(addr), lower_32_bits(addr));
1977 if (GRAPHICS_VER(dev_priv) >= 4) {
1978 drm_printf(m, "\tIPEIR: 0x%08x\n",
1979 ENGINE_READ(engine, RING_IPEIR));
1980 drm_printf(m, "\tIPEHR: 0x%08x\n",
1981 ENGINE_READ(engine, RING_IPEHR));
1982 } else {
1983 drm_printf(m, "\tIPEIR: 0x%08x\n", ENGINE_READ(engine, IPEIR));
1984 drm_printf(m, "\tIPEHR: 0x%08x\n", ENGINE_READ(engine, IPEHR));
1985 }
1986
1987 if (HAS_EXECLISTS(dev_priv) && !intel_engine_uses_guc(engine)) {
1988 struct i915_request * const *port, *rq;
1989 const u32 *hws =
1990 &engine->status_page.addr[I915_HWS_CSB_BUF0_INDEX];
1991 const u8 num_entries = execlists->csb_size;
1992 unsigned int idx;
1993 u8 read, write;
1994
1995 drm_printf(m, "\tExeclist tasklet queued? %s (%s), preempt? %s, timeslice? %s\n",
1996 str_yes_no(test_bit(TASKLET_STATE_SCHED, &engine->sched_engine->tasklet.state)),
1997 str_enabled_disabled(!atomic_read(&engine->sched_engine->tasklet.count)),
1998 repr_timer(&engine->execlists.preempt),
1999 repr_timer(&engine->execlists.timer));
2000
2001 read = execlists->csb_head;
2002 write = READ_ONCE(*execlists->csb_write);
2003
2004 drm_printf(m, "\tExeclist status: 0x%08x %08x; CSB read:%d, write:%d, entries:%d\n",
2005 ENGINE_READ(engine, RING_EXECLIST_STATUS_LO),
2006 ENGINE_READ(engine, RING_EXECLIST_STATUS_HI),
2007 read, write, num_entries);
2008
2009 if (read >= num_entries)
2010 read = 0;
2011 if (write >= num_entries)
2012 write = 0;
2013 if (read > write)
2014 write += num_entries;
2015 while (read < write) {
2016 idx = ++read % num_entries;
2017 drm_printf(m, "\tExeclist CSB[%d]: 0x%08x, context: %d\n",
2018 idx, hws[idx * 2], hws[idx * 2 + 1]);
2019 }
2020
2021 i915_sched_engine_active_lock_bh(engine->sched_engine);
2022 rcu_read_lock();
2023 for (port = execlists->active; (rq = *port); port++) {
2024 char hdr[160];
2025 int len;
2026
2027 len = scnprintf(hdr, sizeof(hdr),
2028 "\t\tActive[%d]: ccid:%08x%s%s, ",
2029 (int)(port - execlists->active),
2030 rq->context->lrc.ccid,
2031 intel_context_is_closed(rq->context) ? "!" : "",
2032 intel_context_is_banned(rq->context) ? "*" : "");
2033 len += print_ring(hdr + len, sizeof(hdr) - len, rq);
2034 scnprintf(hdr + len, sizeof(hdr) - len, "rq: ");
2035 i915_request_show(m, rq, hdr, 0);
2036 }
2037 for (port = execlists->pending; (rq = *port); port++) {
2038 char hdr[160];
2039 int len;
2040
2041 len = scnprintf(hdr, sizeof(hdr),
2042 "\t\tPending[%d]: ccid:%08x%s%s, ",
2043 (int)(port - execlists->pending),
2044 rq->context->lrc.ccid,
2045 intel_context_is_closed(rq->context) ? "!" : "",
2046 intel_context_is_banned(rq->context) ? "*" : "");
2047 len += print_ring(hdr + len, sizeof(hdr) - len, rq);
2048 scnprintf(hdr + len, sizeof(hdr) - len, "rq: ");
2049 i915_request_show(m, rq, hdr, 0);
2050 }
2051 rcu_read_unlock();
2052 i915_sched_engine_active_unlock_bh(engine->sched_engine);
2053 } else if (GRAPHICS_VER(dev_priv) > 6) {
2054 drm_printf(m, "\tPP_DIR_BASE: 0x%08x\n",
2055 ENGINE_READ(engine, RING_PP_DIR_BASE));
2056 drm_printf(m, "\tPP_DIR_BASE_READ: 0x%08x\n",
2057 ENGINE_READ(engine, RING_PP_DIR_BASE_READ));
2058 drm_printf(m, "\tPP_DIR_DCLV: 0x%08x\n",
2059 ENGINE_READ(engine, RING_PP_DIR_DCLV));
2060 }
2061}
2062
2063static void print_request_ring(struct drm_printer *m, struct i915_request *rq)
2064{
2065 struct i915_vma_resource *vma_res = rq->batch_res;
2066 void *ring;
2067 int size;
2068
2069 drm_printf(m,
2070 "[head %04x, postfix %04x, tail %04x, batch 0x%08x_%08x]:\n",
2071 rq->head, rq->postfix, rq->tail,
2072 vma_res ? upper_32_bits(vma_res->start) : ~0u,
2073 vma_res ? lower_32_bits(vma_res->start) : ~0u);
2074
2075 size = rq->tail - rq->head;
2076 if (rq->tail < rq->head)
2077 size += rq->ring->size;
2078
2079 ring = kmalloc(size, GFP_ATOMIC);
2080 if (ring) {
2081 const void *vaddr = rq->ring->vaddr;
2082 unsigned int head = rq->head;
2083 unsigned int len = 0;
2084
2085 if (rq->tail < head) {
2086 len = rq->ring->size - head;
2087 memcpy(ring, vaddr + head, len);
2088 head = 0;
2089 }
2090 memcpy(ring + len, vaddr + head, size - len);
2091
2092 hexdump(m, ring, size);
2093 kfree(ring);
2094 }
2095}
2096
2097static unsigned long read_ul(void *p, size_t x)
2098{
2099 return *(unsigned long *)(p + x);
2100}
2101
2102static void print_properties(struct intel_engine_cs *engine,
2103 struct drm_printer *m)
2104{
2105 static const struct pmap {
2106 size_t offset;
2107 const char *name;
2108 } props[] = {
2109#define P(x) { \
2110 .offset = offsetof(typeof(engine->props), x), \
2111 .name = #x \
2112}
2113 P(heartbeat_interval_ms),
2114 P(max_busywait_duration_ns),
2115 P(preempt_timeout_ms),
2116 P(stop_timeout_ms),
2117 P(timeslice_duration_ms),
2118
2119 {},
2120#undef P
2121 };
2122 const struct pmap *p;
2123
2124 drm_printf(m, "\tProperties:\n");
2125 for (p = props; p->name; p++)
2126 drm_printf(m, "\t\t%s: %lu [default %lu]\n",
2127 p->name,
2128 read_ul(&engine->props, p->offset),
2129 read_ul(&engine->defaults, p->offset));
2130}
2131
2132static void engine_dump_request(struct i915_request *rq, struct drm_printer *m, const char *msg)
2133{
2134 struct intel_timeline *tl = get_timeline(rq);
2135
2136 i915_request_show(m, rq, msg, 0);
2137
2138 drm_printf(m, "\t\tring->start: 0x%08x\n",
2139 i915_ggtt_offset(rq->ring->vma));
2140 drm_printf(m, "\t\tring->head: 0x%08x\n",
2141 rq->ring->head);
2142 drm_printf(m, "\t\tring->tail: 0x%08x\n",
2143 rq->ring->tail);
2144 drm_printf(m, "\t\tring->emit: 0x%08x\n",
2145 rq->ring->emit);
2146 drm_printf(m, "\t\tring->space: 0x%08x\n",
2147 rq->ring->space);
2148
2149 if (tl) {
2150 drm_printf(m, "\t\tring->hwsp: 0x%08x\n",
2151 tl->hwsp_offset);
2152 intel_timeline_put(tl);
2153 }
2154
2155 print_request_ring(m, rq);
2156
2157 if (rq->context->lrc_reg_state) {
2158 drm_printf(m, "Logical Ring Context:\n");
2159 hexdump(m, rq->context->lrc_reg_state, PAGE_SIZE);
2160 }
2161}
2162
2163void intel_engine_dump_active_requests(struct list_head *requests,
2164 struct i915_request *hung_rq,
2165 struct drm_printer *m)
2166{
2167 struct i915_request *rq;
2168 const char *msg;
2169 enum i915_request_state state;
2170
2171 list_for_each_entry(rq, requests, sched.link) {
2172 if (rq == hung_rq)
2173 continue;
2174
2175 state = i915_test_request_state(rq);
2176 if (state < I915_REQUEST_QUEUED)
2177 continue;
2178
2179 if (state == I915_REQUEST_ACTIVE)
2180 msg = "\t\tactive on engine";
2181 else
2182 msg = "\t\tactive in queue";
2183
2184 engine_dump_request(rq, m, msg);
2185 }
2186}
2187
2188static void engine_dump_active_requests(struct intel_engine_cs *engine,
2189 struct drm_printer *m)
2190{
2191 struct intel_context *hung_ce = NULL;
2192 struct i915_request *hung_rq = NULL;
2193
2194 /*
2195 * No need for an engine->irq_seqno_barrier() before the seqno reads.
2196 * The GPU is still running so requests are still executing and any
2197 * hardware reads will be out of date by the time they are reported.
2198 * But the intention here is just to report an instantaneous snapshot
2199 * so that's fine.
2200 */
2201 intel_engine_get_hung_entity(engine, &hung_ce, &hung_rq);
2202
2203 drm_printf(m, "\tRequests:\n");
2204
2205 if (hung_rq)
2206 engine_dump_request(hung_rq, m, "\t\thung");
2207 else if (hung_ce)
2208 drm_printf(m, "\t\tGot hung ce but no hung rq!\n");
2209
2210 if (intel_uc_uses_guc_submission(&engine->gt->uc))
2211 intel_guc_dump_active_requests(engine, hung_rq, m);
2212 else
2213 intel_execlists_dump_active_requests(engine, hung_rq, m);
2214
2215 if (hung_rq)
2216 i915_request_put(hung_rq);
2217}
2218
2219void intel_engine_dump(struct intel_engine_cs *engine,
2220 struct drm_printer *m,
2221 const char *header, ...)
2222{
2223 struct i915_gpu_error * const error = &engine->i915->gpu_error;
2224 struct i915_request *rq;
2225 intel_wakeref_t wakeref;
2226 ktime_t dummy;
2227
2228 if (header) {
2229 va_list ap;
2230
2231 va_start(ap, header);
2232 drm_vprintf(m, header, &ap);
2233 va_end(ap);
2234 }
2235
2236 if (intel_gt_is_wedged(engine->gt))
2237 drm_printf(m, "*** WEDGED ***\n");
2238
2239 drm_printf(m, "\tAwake? %d\n", atomic_read(&engine->wakeref.count));
2240 drm_printf(m, "\tBarriers?: %s\n",
2241 str_yes_no(!llist_empty(&engine->barrier_tasks)));
2242 drm_printf(m, "\tLatency: %luus\n",
2243 ewma__engine_latency_read(&engine->latency));
2244 if (intel_engine_supports_stats(engine))
2245 drm_printf(m, "\tRuntime: %llums\n",
2246 ktime_to_ms(intel_engine_get_busy_time(engine,
2247 &dummy)));
2248 drm_printf(m, "\tForcewake: %x domains, %d active\n",
2249 engine->fw_domain, READ_ONCE(engine->fw_active));
2250
2251 rcu_read_lock();
2252 rq = READ_ONCE(engine->heartbeat.systole);
2253 if (rq)
2254 drm_printf(m, "\tHeartbeat: %d ms ago\n",
2255 jiffies_to_msecs(jiffies - rq->emitted_jiffies));
2256 rcu_read_unlock();
2257 drm_printf(m, "\tReset count: %d (global %d)\n",
2258 i915_reset_engine_count(error, engine),
2259 i915_reset_count(error));
2260 print_properties(engine, m);
2261
2262 engine_dump_active_requests(engine, m);
2263
2264 drm_printf(m, "\tMMIO base: 0x%08x\n", engine->mmio_base);
2265 wakeref = intel_runtime_pm_get_if_in_use(engine->uncore->rpm);
2266 if (wakeref) {
2267 intel_engine_print_registers(engine, m);
2268 intel_runtime_pm_put(engine->uncore->rpm, wakeref);
2269 } else {
2270 drm_printf(m, "\tDevice is asleep; skipping register dump\n");
2271 }
2272
2273 intel_execlists_show_requests(engine, m, i915_request_show, 8);
2274
2275 drm_printf(m, "HWSP:\n");
2276 hexdump(m, engine->status_page.addr, PAGE_SIZE);
2277
2278 drm_printf(m, "Idle? %s\n", str_yes_no(intel_engine_is_idle(engine)));
2279
2280 intel_engine_print_breadcrumbs(engine, m);
2281}
2282
2283/**
2284 * intel_engine_get_busy_time() - Return current accumulated engine busyness
2285 * @engine: engine to report on
2286 * @now: monotonic timestamp of sampling
2287 *
2288 * Returns accumulated time @engine was busy since engine stats were enabled.
2289 */
2290ktime_t intel_engine_get_busy_time(struct intel_engine_cs *engine, ktime_t *now)
2291{
2292 return engine->busyness(engine, now);
2293}
2294
2295struct intel_context *
2296intel_engine_create_virtual(struct intel_engine_cs **siblings,
2297 unsigned int count, unsigned long flags)
2298{
2299 if (count == 0)
2300 return ERR_PTR(-EINVAL);
2301
2302 if (count == 1 && !(flags & FORCE_VIRTUAL))
2303 return intel_context_create(siblings[0]);
2304
2305 GEM_BUG_ON(!siblings[0]->cops->create_virtual);
2306 return siblings[0]->cops->create_virtual(siblings, count, flags);
2307}
2308
2309static struct i915_request *engine_execlist_find_hung_request(struct intel_engine_cs *engine)
2310{
2311 struct i915_request *request, *active = NULL;
2312
2313 /*
2314 * This search does not work in GuC submission mode. However, the GuC
2315 * will report the hanging context directly to the driver itself. So
2316 * the driver should never get here when in GuC mode.
2317 */
2318 GEM_BUG_ON(intel_uc_uses_guc_submission(&engine->gt->uc));
2319
2320 /*
2321 * We are called by the error capture, reset and to dump engine
2322 * state at random points in time. In particular, note that neither is
2323 * crucially ordered with an interrupt. After a hang, the GPU is dead
2324 * and we assume that no more writes can happen (we waited long enough
2325 * for all writes that were in transaction to be flushed) - adding an
2326 * extra delay for a recent interrupt is pointless. Hence, we do
2327 * not need an engine->irq_seqno_barrier() before the seqno reads.
2328 * At all other times, we must assume the GPU is still running, but
2329 * we only care about the snapshot of this moment.
2330 */
2331 lockdep_assert_held(&engine->sched_engine->lock);
2332
2333 rcu_read_lock();
2334 request = execlists_active(&engine->execlists);
2335 if (request) {
2336 struct intel_timeline *tl = request->context->timeline;
2337
2338 list_for_each_entry_from_reverse(request, &tl->requests, link) {
2339 if (__i915_request_is_complete(request))
2340 break;
2341
2342 active = request;
2343 }
2344 }
2345 rcu_read_unlock();
2346 if (active)
2347 return active;
2348
2349 list_for_each_entry(request, &engine->sched_engine->requests,
2350 sched.link) {
2351 if (i915_test_request_state(request) != I915_REQUEST_ACTIVE)
2352 continue;
2353
2354 active = request;
2355 break;
2356 }
2357
2358 return active;
2359}
2360
2361void intel_engine_get_hung_entity(struct intel_engine_cs *engine,
2362 struct intel_context **ce, struct i915_request **rq)
2363{
2364 unsigned long flags;
2365
2366 *ce = intel_engine_get_hung_context(engine);
2367 if (*ce) {
2368 intel_engine_clear_hung_context(engine);
2369
2370 *rq = intel_context_get_active_request(*ce);
2371 return;
2372 }
2373
2374 /*
2375 * Getting here with GuC enabled means it is a forced error capture
2376 * with no actual hang. So, no need to attempt the execlist search.
2377 */
2378 if (intel_uc_uses_guc_submission(&engine->gt->uc))
2379 return;
2380
2381 spin_lock_irqsave(&engine->sched_engine->lock, flags);
2382 *rq = engine_execlist_find_hung_request(engine);
2383 if (*rq)
2384 *rq = i915_request_get_rcu(*rq);
2385 spin_unlock_irqrestore(&engine->sched_engine->lock, flags);
2386}
2387
2388void xehp_enable_ccs_engines(struct intel_engine_cs *engine)
2389{
2390 /*
2391 * If there are any non-fused-off CCS engines, we need to enable CCS
2392 * support in the RCU_MODE register. This only needs to be done once,
2393 * so for simplicity we'll take care of this in the RCS engine's
2394 * resume handler; since the RCS and all CCS engines belong to the
2395 * same reset domain and are reset together, this will also take care
2396 * of re-applying the setting after i915-triggered resets.
2397 */
2398 if (!CCS_MASK(engine->gt))
2399 return;
2400
2401 intel_uncore_write(engine->uncore, GEN12_RCU_MODE,
2402 _MASKED_BIT_ENABLE(GEN12_RCU_MODE_CCS_ENABLE));
2403}
2404
2405#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
2406#include "mock_engine.c"
2407#include "selftest_engine.c"
2408#include "selftest_engine_cs.c"
2409#endif