Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2014-2019 Intel Corporation
4 */
5
6#include <linux/bsearch.h>
7
8#include "gem/i915_gem_lmem.h"
9#include "gt/intel_engine_regs.h"
10#include "gt/intel_gt.h"
11#include "gt/intel_gt_mcr.h"
12#include "gt/intel_gt_regs.h"
13#include "gt/intel_lrc.h"
14#include "gt/shmem_utils.h"
15#include "intel_guc_ads.h"
16#include "intel_guc_capture.h"
17#include "intel_guc_fwif.h"
18#include "intel_guc_print.h"
19#include "intel_uc.h"
20#include "i915_drv.h"
21
22/*
23 * The Additional Data Struct (ADS) has pointers for different buffers used by
24 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
25 * all the extra buffers indirectly linked via the ADS struct's entries.
26 *
27 * Layout of the ADS blob allocated for the GuC:
28 *
29 * +---------------------------------------+ <== base
30 * | guc_ads |
31 * +---------------------------------------+
32 * | guc_policies |
33 * +---------------------------------------+
34 * | guc_gt_system_info |
35 * +---------------------------------------+
36 * | guc_engine_usage |
37 * +---------------------------------------+ <== static
38 * | guc_mmio_reg[countA] (engine 0.0) |
39 * | guc_mmio_reg[countB] (engine 0.1) |
40 * | guc_mmio_reg[countC] (engine 1.0) |
41 * | ... |
42 * +---------------------------------------+ <== dynamic
43 * | padding |
44 * +---------------------------------------+ <== 4K aligned
45 * | golden contexts |
46 * +---------------------------------------+
47 * | padding |
48 * +---------------------------------------+ <== 4K aligned
49 * | capture lists |
50 * +---------------------------------------+
51 * | padding |
52 * +---------------------------------------+ <== 4K aligned
53 * | private data |
54 * +---------------------------------------+
55 * | padding |
56 * +---------------------------------------+ <== 4K aligned
57 */
58struct __guc_ads_blob {
59 struct guc_ads ads;
60 struct guc_policies policies;
61 struct guc_gt_system_info system_info;
62 struct guc_engine_usage engine_usage;
63 /* From here on, location is dynamic! Refer to above diagram. */
64 struct guc_mmio_reg regset[];
65} __packed;
66
67#define ads_blob_read(guc_, field_) \
68 iosys_map_rd_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, field_)
69
70#define ads_blob_write(guc_, field_, val_) \
71 iosys_map_wr_field(&(guc_)->ads_map, 0, struct __guc_ads_blob, \
72 field_, val_)
73
74#define info_map_write(map_, field_, val_) \
75 iosys_map_wr_field(map_, 0, struct guc_gt_system_info, field_, val_)
76
77#define info_map_read(map_, field_) \
78 iosys_map_rd_field(map_, 0, struct guc_gt_system_info, field_)
79
80static u32 guc_ads_regset_size(struct intel_guc *guc)
81{
82 GEM_BUG_ON(!guc->ads_regset_size);
83 return guc->ads_regset_size;
84}
85
86static u32 guc_ads_golden_ctxt_size(struct intel_guc *guc)
87{
88 return PAGE_ALIGN(guc->ads_golden_ctxt_size);
89}
90
91static u32 guc_ads_capture_size(struct intel_guc *guc)
92{
93 return PAGE_ALIGN(guc->ads_capture_size);
94}
95
96static u32 guc_ads_private_data_size(struct intel_guc *guc)
97{
98 return PAGE_ALIGN(guc->fw.private_data_size);
99}
100
101static u32 guc_ads_regset_offset(struct intel_guc *guc)
102{
103 return offsetof(struct __guc_ads_blob, regset);
104}
105
106static u32 guc_ads_golden_ctxt_offset(struct intel_guc *guc)
107{
108 u32 offset;
109
110 offset = guc_ads_regset_offset(guc) +
111 guc_ads_regset_size(guc);
112
113 return PAGE_ALIGN(offset);
114}
115
116static u32 guc_ads_capture_offset(struct intel_guc *guc)
117{
118 u32 offset;
119
120 offset = guc_ads_golden_ctxt_offset(guc) +
121 guc_ads_golden_ctxt_size(guc);
122
123 return PAGE_ALIGN(offset);
124}
125
126static u32 guc_ads_private_data_offset(struct intel_guc *guc)
127{
128 u32 offset;
129
130 offset = guc_ads_capture_offset(guc) +
131 guc_ads_capture_size(guc);
132
133 return PAGE_ALIGN(offset);
134}
135
136static u32 guc_ads_blob_size(struct intel_guc *guc)
137{
138 return guc_ads_private_data_offset(guc) +
139 guc_ads_private_data_size(guc);
140}
141
142static void guc_policies_init(struct intel_guc *guc)
143{
144 struct intel_gt *gt = guc_to_gt(guc);
145 struct drm_i915_private *i915 = gt->i915;
146 u32 global_flags = 0;
147
148 ads_blob_write(guc, policies.dpc_promote_time,
149 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
150 ads_blob_write(guc, policies.max_num_work_items,
151 GLOBAL_POLICY_MAX_NUM_WI);
152
153 if (i915->params.reset < 2)
154 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
155
156 ads_blob_write(guc, policies.global_flags, global_flags);
157 ads_blob_write(guc, policies.is_valid, 1);
158}
159
160void intel_guc_ads_print_policy_info(struct intel_guc *guc,
161 struct drm_printer *dp)
162{
163 if (unlikely(iosys_map_is_null(&guc->ads_map)))
164 return;
165
166 drm_printf(dp, "Global scheduling policies:\n");
167 drm_printf(dp, " DPC promote time = %u\n",
168 ads_blob_read(guc, policies.dpc_promote_time));
169 drm_printf(dp, " Max num work items = %u\n",
170 ads_blob_read(guc, policies.max_num_work_items));
171 drm_printf(dp, " Flags = %u\n",
172 ads_blob_read(guc, policies.global_flags));
173}
174
175static int guc_action_policies_update(struct intel_guc *guc, u32 policy_offset)
176{
177 u32 action[] = {
178 INTEL_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
179 policy_offset
180 };
181
182 return intel_guc_send_busy_loop(guc, action, ARRAY_SIZE(action), 0, true);
183}
184
185int intel_guc_global_policies_update(struct intel_guc *guc)
186{
187 struct intel_gt *gt = guc_to_gt(guc);
188 u32 scheduler_policies;
189 intel_wakeref_t wakeref;
190 int ret;
191
192 if (iosys_map_is_null(&guc->ads_map))
193 return -EOPNOTSUPP;
194
195 scheduler_policies = ads_blob_read(guc, ads.scheduler_policies);
196 GEM_BUG_ON(!scheduler_policies);
197
198 guc_policies_init(guc);
199
200 if (!intel_guc_is_ready(guc))
201 return 0;
202
203 with_intel_runtime_pm(>->i915->runtime_pm, wakeref)
204 ret = guc_action_policies_update(guc, scheduler_policies);
205
206 return ret;
207}
208
209static void guc_mapping_table_init(struct intel_gt *gt,
210 struct iosys_map *info_map)
211{
212 unsigned int i, j;
213 struct intel_engine_cs *engine;
214 enum intel_engine_id id;
215
216 /* Table must be set to invalid values for entries not used */
217 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
218 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
219 info_map_write(info_map, mapping_table[i][j],
220 GUC_MAX_INSTANCES_PER_CLASS);
221
222 for_each_engine(engine, gt, id) {
223 u8 guc_class = engine_class_to_guc_class(engine->class);
224
225 info_map_write(info_map, mapping_table[guc_class][ilog2(engine->logical_mask)],
226 engine->instance);
227 }
228}
229
230/*
231 * The save/restore register list must be pre-calculated to a temporary
232 * buffer before it can be copied inside the ADS.
233 */
234struct temp_regset {
235 /*
236 * ptr to the section of the storage for the engine currently being
237 * worked on
238 */
239 struct guc_mmio_reg *registers;
240 /* ptr to the base of the allocated storage for all engines */
241 struct guc_mmio_reg *storage;
242 u32 storage_used;
243 u32 storage_max;
244};
245
246static int guc_mmio_reg_cmp(const void *a, const void *b)
247{
248 const struct guc_mmio_reg *ra = a;
249 const struct guc_mmio_reg *rb = b;
250
251 return (int)ra->offset - (int)rb->offset;
252}
253
254static struct guc_mmio_reg * __must_check
255__mmio_reg_add(struct temp_regset *regset, struct guc_mmio_reg *reg)
256{
257 u32 pos = regset->storage_used;
258 struct guc_mmio_reg *slot;
259
260 if (pos >= regset->storage_max) {
261 size_t size = ALIGN((pos + 1) * sizeof(*slot), PAGE_SIZE);
262 struct guc_mmio_reg *r = krealloc(regset->storage,
263 size, GFP_KERNEL);
264 if (!r) {
265 WARN_ONCE(1, "Incomplete regset list: can't add register (%d)\n",
266 -ENOMEM);
267 return ERR_PTR(-ENOMEM);
268 }
269
270 regset->registers = r + (regset->registers - regset->storage);
271 regset->storage = r;
272 regset->storage_max = size / sizeof(*slot);
273 }
274
275 slot = ®set->storage[pos];
276 regset->storage_used++;
277 *slot = *reg;
278
279 return slot;
280}
281
282static long __must_check guc_mmio_reg_add(struct intel_gt *gt,
283 struct temp_regset *regset,
284 u32 offset, u32 flags)
285{
286 u32 count = regset->storage_used - (regset->registers - regset->storage);
287 struct guc_mmio_reg entry = {
288 .offset = offset,
289 .flags = flags,
290 };
291 struct guc_mmio_reg *slot;
292
293 /*
294 * The mmio list is built using separate lists within the driver.
295 * It's possible that at some point we may attempt to add the same
296 * register more than once. Do not consider this an error; silently
297 * move on if the register is already in the list.
298 */
299 if (bsearch(&entry, regset->registers, count,
300 sizeof(entry), guc_mmio_reg_cmp))
301 return 0;
302
303 slot = __mmio_reg_add(regset, &entry);
304 if (IS_ERR(slot))
305 return PTR_ERR(slot);
306
307 while (slot-- > regset->registers) {
308 GEM_BUG_ON(slot[0].offset == slot[1].offset);
309 if (slot[1].offset > slot[0].offset)
310 break;
311
312 swap(slot[1], slot[0]);
313 }
314
315 return 0;
316}
317
318#define GUC_MMIO_REG_ADD(gt, regset, reg, masked) \
319 guc_mmio_reg_add(gt, \
320 regset, \
321 i915_mmio_reg_offset(reg), \
322 (masked) ? GUC_REGSET_MASKED : 0)
323
324#define GUC_REGSET_STEERING(group, instance) ( \
325 FIELD_PREP(GUC_REGSET_STEERING_GROUP, (group)) | \
326 FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, (instance)) | \
327 GUC_REGSET_NEEDS_STEERING \
328)
329
330static long __must_check guc_mcr_reg_add(struct intel_gt *gt,
331 struct temp_regset *regset,
332 i915_mcr_reg_t reg, u32 flags)
333{
334 u8 group, inst;
335
336 /*
337 * The GuC doesn't have a default steering, so we need to explicitly
338 * steer all registers that need steering. However, we do not keep track
339 * of all the steering ranges, only of those that have a chance of using
340 * a non-default steering from the i915 pov. Instead of adding such
341 * tracking, it is easier to just program the default steering for all
342 * regs that don't need a non-default one.
343 */
344 intel_gt_mcr_get_nonterminated_steering(gt, reg, &group, &inst);
345 flags |= GUC_REGSET_STEERING(group, inst);
346
347 return guc_mmio_reg_add(gt, regset, i915_mmio_reg_offset(reg), flags);
348}
349
350#define GUC_MCR_REG_ADD(gt, regset, reg, masked) \
351 guc_mcr_reg_add(gt, \
352 regset, \
353 (reg), \
354 (masked) ? GUC_REGSET_MASKED : 0)
355
356static int guc_mmio_regset_init(struct temp_regset *regset,
357 struct intel_engine_cs *engine)
358{
359 struct intel_gt *gt = engine->gt;
360 const u32 base = engine->mmio_base;
361 struct i915_wa_list *wal = &engine->wa_list;
362 struct i915_wa *wa;
363 unsigned int i;
364 int ret = 0;
365
366 /*
367 * Each engine's registers point to a new start relative to
368 * storage
369 */
370 regset->registers = regset->storage + regset->storage_used;
371
372 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_MODE_GEN7(base), true);
373 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_HWS_PGA(base), false);
374 ret |= GUC_MMIO_REG_ADD(gt, regset, RING_IMR(base), false);
375
376 if ((engine->flags & I915_ENGINE_FIRST_RENDER_COMPUTE) &&
377 CCS_MASK(engine->gt))
378 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN12_RCU_MODE, true);
379
380 /*
381 * some of the WA registers are MCR registers. As it is safe to
382 * use MCR form for non-MCR registers, for code simplicity, all
383 * WA registers are added with MCR form.
384 */
385 for (i = 0, wa = wal->list; i < wal->count; i++, wa++)
386 ret |= GUC_MCR_REG_ADD(gt, regset, wa->mcr_reg, wa->masked_reg);
387
388 /* Be extra paranoid and include all whitelist registers. */
389 for (i = 0; i < RING_MAX_NONPRIV_SLOTS; i++)
390 ret |= GUC_MMIO_REG_ADD(gt, regset,
391 RING_FORCE_TO_NONPRIV(base, i),
392 false);
393
394 /* add in local MOCS registers */
395 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++)
396 if (GRAPHICS_VER_FULL(engine->i915) >= IP_VER(12, 50))
397 ret |= GUC_MCR_REG_ADD(gt, regset, XEHP_LNCFCMOCS(i), false);
398 else
399 ret |= GUC_MMIO_REG_ADD(gt, regset, GEN9_LNCFCMOCS(i), false);
400
401 if (GRAPHICS_VER(engine->i915) >= 12) {
402 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL0)), false);
403 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL1)), false);
404 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL2)), false);
405 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL3)), false);
406 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL4)), false);
407 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL5)), false);
408 ret |= GUC_MCR_REG_ADD(gt, regset, MCR_REG(i915_mmio_reg_offset(EU_PERF_CNTL6)), false);
409 }
410
411 return ret ? -1 : 0;
412}
413
414static long guc_mmio_reg_state_create(struct intel_guc *guc)
415{
416 struct intel_gt *gt = guc_to_gt(guc);
417 struct intel_engine_cs *engine;
418 enum intel_engine_id id;
419 struct temp_regset temp_set = {};
420 long total = 0;
421 long ret;
422
423 for_each_engine(engine, gt, id) {
424 u32 used = temp_set.storage_used;
425
426 ret = guc_mmio_regset_init(&temp_set, engine);
427 if (ret < 0)
428 goto fail_regset_init;
429
430 guc->ads_regset_count[id] = temp_set.storage_used - used;
431 total += guc->ads_regset_count[id];
432 }
433
434 guc->ads_regset = temp_set.storage;
435
436 guc_dbg(guc, "Used %zu KB for temporary ADS regset\n",
437 (temp_set.storage_max * sizeof(struct guc_mmio_reg)) >> 10);
438
439 return total * sizeof(struct guc_mmio_reg);
440
441fail_regset_init:
442 kfree(temp_set.storage);
443 return ret;
444}
445
446static void guc_mmio_reg_state_init(struct intel_guc *guc)
447{
448 struct intel_gt *gt = guc_to_gt(guc);
449 struct intel_engine_cs *engine;
450 enum intel_engine_id id;
451 u32 addr_ggtt, offset;
452
453 offset = guc_ads_regset_offset(guc);
454 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
455
456 iosys_map_memcpy_to(&guc->ads_map, offset, guc->ads_regset,
457 guc->ads_regset_size);
458
459 for_each_engine(engine, gt, id) {
460 u32 count = guc->ads_regset_count[id];
461 u8 guc_class;
462
463 /* Class index is checked in class converter */
464 GEM_BUG_ON(engine->instance >= GUC_MAX_INSTANCES_PER_CLASS);
465
466 guc_class = engine_class_to_guc_class(engine->class);
467
468 if (!count) {
469 ads_blob_write(guc,
470 ads.reg_state_list[guc_class][engine->instance].address,
471 0);
472 ads_blob_write(guc,
473 ads.reg_state_list[guc_class][engine->instance].count,
474 0);
475 continue;
476 }
477
478 ads_blob_write(guc,
479 ads.reg_state_list[guc_class][engine->instance].address,
480 addr_ggtt);
481 ads_blob_write(guc,
482 ads.reg_state_list[guc_class][engine->instance].count,
483 count);
484
485 addr_ggtt += count * sizeof(struct guc_mmio_reg);
486 }
487}
488
489static void fill_engine_enable_masks(struct intel_gt *gt,
490 struct iosys_map *info_map)
491{
492 info_map_write(info_map, engine_enabled_masks[GUC_RENDER_CLASS], RCS_MASK(gt));
493 info_map_write(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS], CCS_MASK(gt));
494 info_map_write(info_map, engine_enabled_masks[GUC_BLITTER_CLASS], BCS_MASK(gt));
495 info_map_write(info_map, engine_enabled_masks[GUC_VIDEO_CLASS], VDBOX_MASK(gt));
496 info_map_write(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS], VEBOX_MASK(gt));
497
498 /* The GSC engine is an instance (6) of OTHER_CLASS */
499 if (gt->engine[GSC0])
500 info_map_write(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS],
501 BIT(gt->engine[GSC0]->instance));
502}
503
504#define LR_HW_CONTEXT_SIZE (80 * sizeof(u32))
505#define XEHP_LR_HW_CONTEXT_SIZE (96 * sizeof(u32))
506#define LR_HW_CONTEXT_SZ(i915) (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 50) ? \
507 XEHP_LR_HW_CONTEXT_SIZE : \
508 LR_HW_CONTEXT_SIZE)
509#define LRC_SKIP_SIZE(i915) (LRC_PPHWSP_SZ * PAGE_SIZE + LR_HW_CONTEXT_SZ(i915))
510static int guc_prep_golden_context(struct intel_guc *guc)
511{
512 struct intel_gt *gt = guc_to_gt(guc);
513 u32 addr_ggtt, offset;
514 u32 total_size = 0, alloc_size, real_size;
515 u8 engine_class, guc_class;
516 struct guc_gt_system_info local_info;
517 struct iosys_map info_map;
518
519 /*
520 * Reserve the memory for the golden contexts and point GuC at it but
521 * leave it empty for now. The context data will be filled in later
522 * once there is something available to put there.
523 *
524 * Note that the HWSP and ring context are not included.
525 *
526 * Note also that the storage must be pinned in the GGTT, so that the
527 * address won't change after GuC has been told where to find it. The
528 * GuC will also validate that the LRC base + size fall within the
529 * allowed GGTT range.
530 */
531 if (!iosys_map_is_null(&guc->ads_map)) {
532 offset = guc_ads_golden_ctxt_offset(guc);
533 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
534 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
535 offsetof(struct __guc_ads_blob, system_info));
536 } else {
537 memset(&local_info, 0, sizeof(local_info));
538 iosys_map_set_vaddr(&info_map, &local_info);
539 fill_engine_enable_masks(gt, &info_map);
540 }
541
542 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
543 guc_class = engine_class_to_guc_class(engine_class);
544
545 if (!info_map_read(&info_map, engine_enabled_masks[guc_class]))
546 continue;
547
548 real_size = intel_engine_context_size(gt, engine_class);
549 alloc_size = PAGE_ALIGN(real_size);
550 total_size += alloc_size;
551
552 if (iosys_map_is_null(&guc->ads_map))
553 continue;
554
555 /*
556 * This interface is slightly confusing. We need to pass the
557 * base address of the full golden context and the size of just
558 * the engine state, which is the section of the context image
559 * that starts after the execlists context. This is required to
560 * allow the GuC to restore just the engine state when a
561 * watchdog reset occurs.
562 * We calculate the engine state size by removing the size of
563 * what comes before it in the context image (which is identical
564 * on all engines).
565 */
566 ads_blob_write(guc, ads.eng_state_size[guc_class],
567 real_size - LRC_SKIP_SIZE(gt->i915));
568 ads_blob_write(guc, ads.golden_context_lrca[guc_class],
569 addr_ggtt);
570
571 addr_ggtt += alloc_size;
572 }
573
574 /* Make sure current size matches what we calculated previously */
575 if (guc->ads_golden_ctxt_size)
576 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
577
578 return total_size;
579}
580
581static struct intel_engine_cs *find_engine_state(struct intel_gt *gt, u8 engine_class)
582{
583 struct intel_engine_cs *engine;
584 enum intel_engine_id id;
585
586 for_each_engine(engine, gt, id) {
587 if (engine->class != engine_class)
588 continue;
589
590 if (!engine->default_state)
591 continue;
592
593 return engine;
594 }
595
596 return NULL;
597}
598
599static void guc_init_golden_context(struct intel_guc *guc)
600{
601 struct intel_engine_cs *engine;
602 struct intel_gt *gt = guc_to_gt(guc);
603 unsigned long offset;
604 u32 addr_ggtt, total_size = 0, alloc_size, real_size;
605 u8 engine_class, guc_class;
606
607 if (!intel_uc_uses_guc_submission(>->uc))
608 return;
609
610 GEM_BUG_ON(iosys_map_is_null(&guc->ads_map));
611
612 /*
613 * Go back and fill in the golden context data now that it is
614 * available.
615 */
616 offset = guc_ads_golden_ctxt_offset(guc);
617 addr_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma) + offset;
618
619 for (engine_class = 0; engine_class <= MAX_ENGINE_CLASS; ++engine_class) {
620 guc_class = engine_class_to_guc_class(engine_class);
621 if (!ads_blob_read(guc, system_info.engine_enabled_masks[guc_class]))
622 continue;
623
624 real_size = intel_engine_context_size(gt, engine_class);
625 alloc_size = PAGE_ALIGN(real_size);
626 total_size += alloc_size;
627
628 engine = find_engine_state(gt, engine_class);
629 if (!engine) {
630 guc_err(guc, "No engine state recorded for class %d!\n",
631 engine_class);
632 ads_blob_write(guc, ads.eng_state_size[guc_class], 0);
633 ads_blob_write(guc, ads.golden_context_lrca[guc_class], 0);
634 continue;
635 }
636
637 GEM_BUG_ON(ads_blob_read(guc, ads.eng_state_size[guc_class]) !=
638 real_size - LRC_SKIP_SIZE(gt->i915));
639 GEM_BUG_ON(ads_blob_read(guc, ads.golden_context_lrca[guc_class]) != addr_ggtt);
640
641 addr_ggtt += alloc_size;
642
643 shmem_read_to_iosys_map(engine->default_state, 0, &guc->ads_map,
644 offset, real_size);
645 offset += alloc_size;
646 }
647
648 GEM_BUG_ON(guc->ads_golden_ctxt_size != total_size);
649}
650
651static u32 guc_get_capture_engine_mask(struct iosys_map *info_map, u32 capture_class)
652{
653 u32 mask;
654
655 switch (capture_class) {
656 case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE:
657 mask = info_map_read(info_map, engine_enabled_masks[GUC_RENDER_CLASS]);
658 mask |= info_map_read(info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]);
659 break;
660
661 case GUC_CAPTURE_LIST_CLASS_VIDEO:
662 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEO_CLASS]);
663 break;
664
665 case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE:
666 mask = info_map_read(info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]);
667 break;
668
669 case GUC_CAPTURE_LIST_CLASS_BLITTER:
670 mask = info_map_read(info_map, engine_enabled_masks[GUC_BLITTER_CLASS]);
671 break;
672
673 case GUC_CAPTURE_LIST_CLASS_GSC_OTHER:
674 mask = info_map_read(info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]);
675 break;
676
677 default:
678 mask = 0;
679 }
680
681 return mask;
682}
683
684static int
685guc_capture_prep_lists(struct intel_guc *guc)
686{
687 struct intel_gt *gt = guc_to_gt(guc);
688 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0;
689 struct guc_gt_system_info local_info;
690 struct iosys_map info_map;
691 bool ads_is_mapped;
692 size_t size = 0;
693 void *ptr;
694 int i, j;
695
696 ads_is_mapped = !iosys_map_is_null(&guc->ads_map);
697 if (ads_is_mapped) {
698 capture_offset = guc_ads_capture_offset(guc);
699 ads_ggtt = intel_guc_ggtt_offset(guc, guc->ads_vma);
700 info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
701 offsetof(struct __guc_ads_blob, system_info));
702 } else {
703 memset(&local_info, 0, sizeof(local_info));
704 iosys_map_set_vaddr(&info_map, &local_info);
705 fill_engine_enable_masks(gt, &info_map);
706 }
707
708 /* first, set aside the first page for a capture_list with zero descriptors */
709 total_size = PAGE_SIZE;
710 if (ads_is_mapped) {
711 if (!intel_guc_capture_getnullheader(guc, &ptr, &size))
712 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
713 null_ggtt = ads_ggtt + capture_offset;
714 capture_offset += PAGE_SIZE;
715 }
716
717 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
718 for (j = 0; j < GUC_MAX_ENGINE_CLASSES; j++) {
719 u32 engine_mask = guc_get_capture_engine_mask(&info_map, j);
720
721 /* null list if we dont have said engine or list */
722 if (!engine_mask) {
723 if (ads_is_mapped) {
724 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
725 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
726 }
727 continue;
728 }
729 if (intel_guc_capture_getlistsize(guc, i,
730 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
731 j, &size)) {
732 if (ads_is_mapped)
733 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
734 goto engine_instance_list;
735 }
736 total_size += size;
737 if (ads_is_mapped) {
738 if (total_size > guc->ads_capture_size ||
739 intel_guc_capture_getlist(guc, i,
740 GUC_CAPTURE_LIST_TYPE_ENGINE_CLASS,
741 j, &ptr)) {
742 ads_blob_write(guc, ads.capture_class[i][j], null_ggtt);
743 continue;
744 }
745 ads_blob_write(guc, ads.capture_class[i][j], ads_ggtt +
746 capture_offset);
747 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
748 capture_offset += size;
749 }
750engine_instance_list:
751 if (intel_guc_capture_getlistsize(guc, i,
752 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
753 j, &size)) {
754 if (ads_is_mapped)
755 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
756 continue;
757 }
758 total_size += size;
759 if (ads_is_mapped) {
760 if (total_size > guc->ads_capture_size ||
761 intel_guc_capture_getlist(guc, i,
762 GUC_CAPTURE_LIST_TYPE_ENGINE_INSTANCE,
763 j, &ptr)) {
764 ads_blob_write(guc, ads.capture_instance[i][j], null_ggtt);
765 continue;
766 }
767 ads_blob_write(guc, ads.capture_instance[i][j], ads_ggtt +
768 capture_offset);
769 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
770 capture_offset += size;
771 }
772 }
773 if (intel_guc_capture_getlistsize(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0, &size)) {
774 if (ads_is_mapped)
775 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
776 continue;
777 }
778 total_size += size;
779 if (ads_is_mapped) {
780 if (total_size > guc->ads_capture_size ||
781 intel_guc_capture_getlist(guc, i, GUC_CAPTURE_LIST_TYPE_GLOBAL, 0,
782 &ptr)) {
783 ads_blob_write(guc, ads.capture_global[i], null_ggtt);
784 continue;
785 }
786 ads_blob_write(guc, ads.capture_global[i], ads_ggtt + capture_offset);
787 iosys_map_memcpy_to(&guc->ads_map, capture_offset, ptr, size);
788 capture_offset += size;
789 }
790 }
791
792 if (guc->ads_capture_size && guc->ads_capture_size != PAGE_ALIGN(total_size))
793 guc_warn(guc, "ADS capture alloc size changed from %d to %d\n",
794 guc->ads_capture_size, PAGE_ALIGN(total_size));
795
796 return PAGE_ALIGN(total_size);
797}
798
799static void __guc_ads_init(struct intel_guc *guc)
800{
801 struct intel_gt *gt = guc_to_gt(guc);
802 struct drm_i915_private *i915 = gt->i915;
803 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(&guc->ads_map,
804 offsetof(struct __guc_ads_blob, system_info));
805 u32 base;
806
807 /* GuC scheduling policies */
808 guc_policies_init(guc);
809
810 /* System info */
811 fill_engine_enable_masks(gt, &info_map);
812
813 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_SLICE_ENABLED],
814 hweight8(gt->info.sseu.slice_mask));
815 ads_blob_write(guc, system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_VDBOX_SFC_SUPPORT_MASK],
816 gt->info.vdbox_sfc_access);
817
818 if (GRAPHICS_VER(i915) >= 12 && !IS_DGFX(i915)) {
819 u32 distdbreg = intel_uncore_read(gt->uncore,
820 GEN12_DIST_DBS_POPULATED);
821 ads_blob_write(guc,
822 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
823 ((distdbreg >> GEN12_DOORBELLS_PER_SQIDI_SHIFT)
824 & GEN12_DOORBELLS_PER_SQIDI) + 1);
825 }
826
827 /* Golden contexts for re-initialising after a watchdog reset */
828 guc_prep_golden_context(guc);
829
830 guc_mapping_table_init(guc_to_gt(guc), &info_map);
831
832 base = intel_guc_ggtt_offset(guc, guc->ads_vma);
833
834 /* Lists for error capture debug */
835 guc_capture_prep_lists(guc);
836
837 /* ADS */
838 ads_blob_write(guc, ads.scheduler_policies, base +
839 offsetof(struct __guc_ads_blob, policies));
840 ads_blob_write(guc, ads.gt_system_info, base +
841 offsetof(struct __guc_ads_blob, system_info));
842
843 /* MMIO save/restore list */
844 guc_mmio_reg_state_init(guc);
845
846 /* Private Data */
847 ads_blob_write(guc, ads.private_data, base +
848 guc_ads_private_data_offset(guc));
849
850 i915_gem_object_flush_map(guc->ads_vma->obj);
851}
852
853/**
854 * intel_guc_ads_create() - allocates and initializes GuC ADS.
855 * @guc: intel_guc struct
856 *
857 * GuC needs memory block (Additional Data Struct), where it will store
858 * some data. Allocate and initialize such memory block for GuC use.
859 */
860int intel_guc_ads_create(struct intel_guc *guc)
861{
862 void *ads_blob;
863 u32 size;
864 int ret;
865
866 GEM_BUG_ON(guc->ads_vma);
867
868 /*
869 * Create reg state size dynamically on system memory to be copied to
870 * the final ads blob on gt init/reset
871 */
872 ret = guc_mmio_reg_state_create(guc);
873 if (ret < 0)
874 return ret;
875 guc->ads_regset_size = ret;
876
877 /* Likewise the golden contexts: */
878 ret = guc_prep_golden_context(guc);
879 if (ret < 0)
880 return ret;
881 guc->ads_golden_ctxt_size = ret;
882
883 /* Likewise the capture lists: */
884 ret = guc_capture_prep_lists(guc);
885 if (ret < 0)
886 return ret;
887 guc->ads_capture_size = ret;
888
889 /* Now the total size can be determined: */
890 size = guc_ads_blob_size(guc);
891
892 ret = intel_guc_allocate_and_map_vma(guc, size, &guc->ads_vma,
893 &ads_blob);
894 if (ret)
895 return ret;
896
897 if (i915_gem_object_is_lmem(guc->ads_vma->obj))
898 iosys_map_set_vaddr_iomem(&guc->ads_map, (void __iomem *)ads_blob);
899 else
900 iosys_map_set_vaddr(&guc->ads_map, ads_blob);
901
902 __guc_ads_init(guc);
903
904 return 0;
905}
906
907void intel_guc_ads_init_late(struct intel_guc *guc)
908{
909 /*
910 * The golden context setup requires the saved engine state from
911 * __engines_record_defaults(). However, that requires engines to be
912 * operational which means the ADS must already have been configured.
913 * Fortunately, the golden context state is not needed until a hang
914 * occurs, so it can be filled in during this late init phase.
915 */
916 guc_init_golden_context(guc);
917}
918
919void intel_guc_ads_destroy(struct intel_guc *guc)
920{
921 i915_vma_unpin_and_release(&guc->ads_vma, I915_VMA_RELEASE_MAP);
922 iosys_map_clear(&guc->ads_map);
923 kfree(guc->ads_regset);
924}
925
926static void guc_ads_private_data_reset(struct intel_guc *guc)
927{
928 u32 size;
929
930 size = guc_ads_private_data_size(guc);
931 if (!size)
932 return;
933
934 iosys_map_memset(&guc->ads_map, guc_ads_private_data_offset(guc),
935 0, size);
936}
937
938/**
939 * intel_guc_ads_reset() - prepares GuC Additional Data Struct for reuse
940 * @guc: intel_guc struct
941 *
942 * GuC stores some data in ADS, which might be stale after a reset.
943 * Reinitialize whole ADS in case any part of it was corrupted during
944 * previous GuC run.
945 */
946void intel_guc_ads_reset(struct intel_guc *guc)
947{
948 if (!guc->ads_vma)
949 return;
950
951 __guc_ads_init(guc);
952
953 guc_ads_private_data_reset(guc);
954}
955
956u32 intel_guc_engine_usage_offset(struct intel_guc *guc)
957{
958 return intel_guc_ggtt_offset(guc, guc->ads_vma) +
959 offsetof(struct __guc_ads_blob, engine_usage);
960}
961
962struct iosys_map intel_guc_engine_usage_record_map(struct intel_engine_cs *engine)
963{
964 struct intel_guc *guc = &engine->gt->uc.guc;
965 u8 guc_class = engine_class_to_guc_class(engine->class);
966 size_t offset = offsetof(struct __guc_ads_blob,
967 engine_usage.engines[guc_class][ilog2(engine->logical_mask)]);
968
969 return IOSYS_MAP_INIT_OFFSET(&guc->ads_map, offset);
970}