Loading...
Note: File does not exist in v6.2.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_guc_ads.h"
7
8#include <linux/fault-inject.h>
9
10#include <drm/drm_managed.h>
11
12#include <generated/xe_wa_oob.h>
13
14#include "abi/guc_actions_abi.h"
15#include "regs/xe_engine_regs.h"
16#include "regs/xe_gt_regs.h"
17#include "regs/xe_guc_regs.h"
18#include "xe_bo.h"
19#include "xe_gt.h"
20#include "xe_gt_ccs_mode.h"
21#include "xe_gt_printk.h"
22#include "xe_guc.h"
23#include "xe_guc_capture.h"
24#include "xe_guc_ct.h"
25#include "xe_hw_engine.h"
26#include "xe_lrc.h"
27#include "xe_map.h"
28#include "xe_mmio.h"
29#include "xe_platform_types.h"
30#include "xe_uc_fw.h"
31#include "xe_wa.h"
32#include "xe_gt_mcr.h"
33
34/* Slack of a few additional entries per engine */
35#define ADS_REGSET_EXTRA_MAX 8
36
37static struct xe_guc *
38ads_to_guc(struct xe_guc_ads *ads)
39{
40 return container_of(ads, struct xe_guc, ads);
41}
42
43static struct xe_gt *
44ads_to_gt(struct xe_guc_ads *ads)
45{
46 return container_of(ads, struct xe_gt, uc.guc.ads);
47}
48
49static struct xe_device *
50ads_to_xe(struct xe_guc_ads *ads)
51{
52 return gt_to_xe(ads_to_gt(ads));
53}
54
55static struct iosys_map *
56ads_to_map(struct xe_guc_ads *ads)
57{
58 return &ads->bo->vmap;
59}
60
61/* UM Queue parameters: */
62#define GUC_UM_QUEUE_SIZE (SZ_64K)
63#define GUC_PAGE_RES_TIMEOUT_US (-1)
64
65/*
66 * The Additional Data Struct (ADS) has pointers for different buffers used by
67 * the GuC. One single gem object contains the ADS struct itself (guc_ads) and
68 * all the extra buffers indirectly linked via the ADS struct's entries.
69 *
70 * Layout of the ADS blob allocated for the GuC:
71 *
72 * +---------------------------------------+ <== base
73 * | guc_ads |
74 * +---------------------------------------+
75 * | guc_policies |
76 * +---------------------------------------+
77 * | guc_gt_system_info |
78 * +---------------------------------------+
79 * | guc_engine_usage |
80 * +---------------------------------------+
81 * | guc_um_init_params |
82 * +---------------------------------------+ <== static
83 * | guc_mmio_reg[countA] (engine 0.0) |
84 * | guc_mmio_reg[countB] (engine 0.1) |
85 * | guc_mmio_reg[countC] (engine 1.0) |
86 * | ... |
87 * +---------------------------------------+ <== dynamic
88 * | padding |
89 * +---------------------------------------+ <== 4K aligned
90 * | golden contexts |
91 * +---------------------------------------+
92 * | padding |
93 * +---------------------------------------+ <== 4K aligned
94 * | w/a KLVs |
95 * +---------------------------------------+
96 * | padding |
97 * +---------------------------------------+ <== 4K aligned
98 * | capture lists |
99 * +---------------------------------------+
100 * | padding |
101 * +---------------------------------------+ <== 4K aligned
102 * | UM queues |
103 * +---------------------------------------+
104 * | padding |
105 * +---------------------------------------+ <== 4K aligned
106 * | private data |
107 * +---------------------------------------+
108 * | padding |
109 * +---------------------------------------+ <== 4K aligned
110 */
111struct __guc_ads_blob {
112 struct guc_ads ads;
113 struct guc_policies policies;
114 struct guc_gt_system_info system_info;
115 struct guc_engine_usage engine_usage;
116 struct guc_um_init_params um_init_params;
117 /* From here on, location is dynamic! Refer to above diagram. */
118 struct guc_mmio_reg regset[];
119} __packed;
120
121#define ads_blob_read(ads_, field_) \
122 xe_map_rd_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \
123 struct __guc_ads_blob, field_)
124
125#define ads_blob_write(ads_, field_, val_) \
126 xe_map_wr_field(ads_to_xe(ads_), ads_to_map(ads_), 0, \
127 struct __guc_ads_blob, field_, val_)
128
129#define info_map_write(xe_, map_, field_, val_) \
130 xe_map_wr_field(xe_, map_, 0, struct guc_gt_system_info, field_, val_)
131
132#define info_map_read(xe_, map_, field_) \
133 xe_map_rd_field(xe_, map_, 0, struct guc_gt_system_info, field_)
134
135static size_t guc_ads_regset_size(struct xe_guc_ads *ads)
136{
137 struct xe_device *xe = ads_to_xe(ads);
138
139 xe_assert(xe, ads->regset_size);
140
141 return ads->regset_size;
142}
143
144static size_t guc_ads_golden_lrc_size(struct xe_guc_ads *ads)
145{
146 return PAGE_ALIGN(ads->golden_lrc_size);
147}
148
149static u32 guc_ads_waklv_size(struct xe_guc_ads *ads)
150{
151 return PAGE_ALIGN(ads->ads_waklv_size);
152}
153
154static size_t guc_ads_capture_size(struct xe_guc_ads *ads)
155{
156 return PAGE_ALIGN(ads->capture_size);
157}
158
159static size_t guc_ads_um_queues_size(struct xe_guc_ads *ads)
160{
161 struct xe_device *xe = ads_to_xe(ads);
162
163 if (!xe->info.has_usm)
164 return 0;
165
166 return GUC_UM_QUEUE_SIZE * GUC_UM_HW_QUEUE_MAX;
167}
168
169static size_t guc_ads_private_data_size(struct xe_guc_ads *ads)
170{
171 return PAGE_ALIGN(ads_to_guc(ads)->fw.private_data_size);
172}
173
174static size_t guc_ads_regset_offset(struct xe_guc_ads *ads)
175{
176 return offsetof(struct __guc_ads_blob, regset);
177}
178
179static size_t guc_ads_golden_lrc_offset(struct xe_guc_ads *ads)
180{
181 size_t offset;
182
183 offset = guc_ads_regset_offset(ads) +
184 guc_ads_regset_size(ads);
185
186 return PAGE_ALIGN(offset);
187}
188
189static size_t guc_ads_waklv_offset(struct xe_guc_ads *ads)
190{
191 u32 offset;
192
193 offset = guc_ads_golden_lrc_offset(ads) +
194 guc_ads_golden_lrc_size(ads);
195
196 return PAGE_ALIGN(offset);
197}
198
199static size_t guc_ads_capture_offset(struct xe_guc_ads *ads)
200{
201 size_t offset;
202
203 offset = guc_ads_waklv_offset(ads) +
204 guc_ads_waklv_size(ads);
205
206 return PAGE_ALIGN(offset);
207}
208
209static size_t guc_ads_um_queues_offset(struct xe_guc_ads *ads)
210{
211 u32 offset;
212
213 offset = guc_ads_capture_offset(ads) +
214 guc_ads_capture_size(ads);
215
216 return PAGE_ALIGN(offset);
217}
218
219static size_t guc_ads_private_data_offset(struct xe_guc_ads *ads)
220{
221 size_t offset;
222
223 offset = guc_ads_um_queues_offset(ads) +
224 guc_ads_um_queues_size(ads);
225
226 return PAGE_ALIGN(offset);
227}
228
229static size_t guc_ads_size(struct xe_guc_ads *ads)
230{
231 return guc_ads_private_data_offset(ads) +
232 guc_ads_private_data_size(ads);
233}
234
235static bool needs_wa_1607983814(struct xe_device *xe)
236{
237 return GRAPHICS_VERx100(xe) < 1250;
238}
239
240static size_t calculate_regset_size(struct xe_gt *gt)
241{
242 struct xe_reg_sr_entry *sr_entry;
243 unsigned long sr_idx;
244 struct xe_hw_engine *hwe;
245 enum xe_hw_engine_id id;
246 unsigned int count = 0;
247
248 for_each_hw_engine(hwe, gt, id)
249 xa_for_each(&hwe->reg_sr.xa, sr_idx, sr_entry)
250 count++;
251
252 count += ADS_REGSET_EXTRA_MAX * XE_NUM_HW_ENGINES;
253
254 if (needs_wa_1607983814(gt_to_xe(gt)))
255 count += LNCFCMOCS_REG_COUNT;
256
257 return count * sizeof(struct guc_mmio_reg);
258}
259
260static u32 engine_enable_mask(struct xe_gt *gt, enum xe_engine_class class)
261{
262 struct xe_hw_engine *hwe;
263 enum xe_hw_engine_id id;
264 u32 mask = 0;
265
266 for_each_hw_engine(hwe, gt, id)
267 if (hwe->class == class)
268 mask |= BIT(hwe->instance);
269
270 return mask;
271}
272
273static size_t calculate_golden_lrc_size(struct xe_guc_ads *ads)
274{
275 struct xe_gt *gt = ads_to_gt(ads);
276 size_t total_size = 0, alloc_size, real_size;
277 int class;
278
279 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
280 if (!engine_enable_mask(gt, class))
281 continue;
282
283 real_size = xe_gt_lrc_size(gt, class);
284 alloc_size = PAGE_ALIGN(real_size);
285 total_size += alloc_size;
286 }
287
288 return total_size;
289}
290
291static void guc_waklv_enable_one_word(struct xe_guc_ads *ads,
292 enum xe_guc_klv_ids klv_id,
293 u32 value,
294 u32 *offset, u32 *remain)
295{
296 u32 size;
297 u32 klv_entry[] = {
298 /* 16:16 key/length */
299 FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
300 FIELD_PREP(GUC_KLV_0_LEN, 1),
301 value,
302 /* 1 dword data */
303 };
304
305 size = sizeof(klv_entry);
306
307 if (*remain < size) {
308 drm_warn(&ads_to_xe(ads)->drm,
309 "w/a klv buffer too small to add klv id %d\n", klv_id);
310 } else {
311 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset,
312 klv_entry, size);
313 *offset += size;
314 *remain -= size;
315 }
316}
317
318static void guc_waklv_enable_simple(struct xe_guc_ads *ads,
319 enum xe_guc_klv_ids klv_id, u32 *offset, u32 *remain)
320{
321 u32 klv_entry[] = {
322 /* 16:16 key/length */
323 FIELD_PREP(GUC_KLV_0_KEY, klv_id) |
324 FIELD_PREP(GUC_KLV_0_LEN, 0),
325 /* 0 dwords data */
326 };
327 u32 size;
328
329 size = sizeof(klv_entry);
330
331 if (xe_gt_WARN(ads_to_gt(ads), *remain < size,
332 "w/a klv buffer too small to add klv id %d\n", klv_id))
333 return;
334
335 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), *offset,
336 klv_entry, size);
337 *offset += size;
338 *remain -= size;
339}
340
341static void guc_waklv_init(struct xe_guc_ads *ads)
342{
343 struct xe_gt *gt = ads_to_gt(ads);
344 u64 addr_ggtt;
345 u32 offset, remain, size;
346
347 offset = guc_ads_waklv_offset(ads);
348 remain = guc_ads_waklv_size(ads);
349
350 if (XE_WA(gt, 14019882105))
351 guc_waklv_enable_simple(ads,
352 GUC_WORKAROUND_KLV_BLOCK_INTERRUPTS_WHEN_MGSR_BLOCKED,
353 &offset, &remain);
354 if (XE_WA(gt, 18024947630))
355 guc_waklv_enable_simple(ads,
356 GUC_WORKAROUND_KLV_ID_GAM_PFQ_SHADOW_TAIL_POLLING,
357 &offset, &remain);
358 if (XE_WA(gt, 16022287689))
359 guc_waklv_enable_simple(ads,
360 GUC_WORKAROUND_KLV_ID_DISABLE_MTP_DURING_ASYNC_COMPUTE,
361 &offset, &remain);
362
363 if (XE_WA(gt, 14022866841))
364 guc_waklv_enable_simple(ads,
365 GUC_WA_KLV_WAKE_POWER_DOMAINS_FOR_OUTBOUND_MMIO,
366 &offset, &remain);
367
368 /*
369 * On RC6 exit, GuC will write register 0xB04 with the default value provided. As of now,
370 * the default value for this register is determined to be 0xC40. This could change in the
371 * future, so GuC depends on KMD to send it the correct value.
372 */
373 if (XE_WA(gt, 13011645652))
374 guc_waklv_enable_one_word(ads,
375 GUC_WA_KLV_NP_RD_WRITE_TO_CLEAR_RCSM_AT_CGP_LATE_RESTORE,
376 0xC40,
377 &offset, &remain);
378
379 if (XE_WA(gt, 14022293748) || XE_WA(gt, 22019794406))
380 guc_waklv_enable_simple(ads,
381 GUC_WORKAROUND_KLV_ID_BACK_TO_BACK_RCS_ENGINE_RESET,
382 &offset, &remain);
383
384 size = guc_ads_waklv_size(ads) - remain;
385 if (!size)
386 return;
387
388 offset = guc_ads_waklv_offset(ads);
389 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset;
390
391 ads_blob_write(ads, ads.wa_klv_addr_lo, lower_32_bits(addr_ggtt));
392 ads_blob_write(ads, ads.wa_klv_addr_hi, upper_32_bits(addr_ggtt));
393 ads_blob_write(ads, ads.wa_klv_size, size);
394}
395
396static int calculate_waklv_size(struct xe_guc_ads *ads)
397{
398 /*
399 * A single page is both the minimum size possible and
400 * is sufficiently large enough for all current platforms.
401 */
402 return SZ_4K;
403}
404
405#define MAX_GOLDEN_LRC_SIZE (SZ_4K * 64)
406
407int xe_guc_ads_init(struct xe_guc_ads *ads)
408{
409 struct xe_device *xe = ads_to_xe(ads);
410 struct xe_gt *gt = ads_to_gt(ads);
411 struct xe_tile *tile = gt_to_tile(gt);
412 struct xe_bo *bo;
413
414 ads->golden_lrc_size = calculate_golden_lrc_size(ads);
415 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads));
416 ads->regset_size = calculate_regset_size(gt);
417 ads->ads_waklv_size = calculate_waklv_size(ads);
418
419 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ads_size(ads) + MAX_GOLDEN_LRC_SIZE,
420 XE_BO_FLAG_SYSTEM |
421 XE_BO_FLAG_GGTT |
422 XE_BO_FLAG_GGTT_INVALIDATE);
423 if (IS_ERR(bo))
424 return PTR_ERR(bo);
425
426 ads->bo = bo;
427
428 return 0;
429}
430ALLOW_ERROR_INJECTION(xe_guc_ads_init, ERRNO); /* See xe_pci_probe() */
431
432/**
433 * xe_guc_ads_init_post_hwconfig - initialize ADS post hwconfig load
434 * @ads: Additional data structures object
435 *
436 * Recalculate golden_lrc_size, capture_size and regset_size as the number
437 * hardware engines may have changed after the hwconfig was loaded. Also verify
438 * the new sizes fit in the already allocated ADS buffer object.
439 *
440 * Return: 0 on success, negative error code on error.
441 */
442int xe_guc_ads_init_post_hwconfig(struct xe_guc_ads *ads)
443{
444 struct xe_gt *gt = ads_to_gt(ads);
445 u32 prev_regset_size = ads->regset_size;
446
447 xe_gt_assert(gt, ads->bo);
448
449 ads->golden_lrc_size = calculate_golden_lrc_size(ads);
450 /* Calculate Capture size with worst size */
451 ads->capture_size = xe_guc_capture_ads_input_worst_size(ads_to_guc(ads));
452 ads->regset_size = calculate_regset_size(gt);
453
454 xe_gt_assert(gt, ads->golden_lrc_size +
455 (ads->regset_size - prev_regset_size) <=
456 MAX_GOLDEN_LRC_SIZE);
457
458 return 0;
459}
460
461static void guc_policies_init(struct xe_guc_ads *ads)
462{
463 struct xe_device *xe = ads_to_xe(ads);
464 u32 global_flags = 0;
465
466 ads_blob_write(ads, policies.dpc_promote_time,
467 GLOBAL_POLICY_DEFAULT_DPC_PROMOTE_TIME_US);
468 ads_blob_write(ads, policies.max_num_work_items,
469 GLOBAL_POLICY_MAX_NUM_WI);
470
471 if (xe->wedged.mode == 2)
472 global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
473
474 ads_blob_write(ads, policies.global_flags, global_flags);
475 ads_blob_write(ads, policies.is_valid, 1);
476}
477
478static void fill_engine_enable_masks(struct xe_gt *gt,
479 struct iosys_map *info_map)
480{
481 struct xe_device *xe = gt_to_xe(gt);
482
483 info_map_write(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS],
484 engine_enable_mask(gt, XE_ENGINE_CLASS_RENDER));
485 info_map_write(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS],
486 engine_enable_mask(gt, XE_ENGINE_CLASS_COPY));
487 info_map_write(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS],
488 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_DECODE));
489 info_map_write(xe, info_map,
490 engine_enabled_masks[GUC_VIDEOENHANCE_CLASS],
491 engine_enable_mask(gt, XE_ENGINE_CLASS_VIDEO_ENHANCE));
492 info_map_write(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS],
493 engine_enable_mask(gt, XE_ENGINE_CLASS_COMPUTE));
494 info_map_write(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS],
495 engine_enable_mask(gt, XE_ENGINE_CLASS_OTHER));
496}
497
498static void guc_prep_golden_lrc_null(struct xe_guc_ads *ads)
499{
500 struct xe_device *xe = ads_to_xe(ads);
501 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
502 offsetof(struct __guc_ads_blob, system_info));
503 u8 guc_class;
504
505 for (guc_class = 0; guc_class <= GUC_MAX_ENGINE_CLASSES; ++guc_class) {
506 if (!info_map_read(xe, &info_map,
507 engine_enabled_masks[guc_class]))
508 continue;
509
510 ads_blob_write(ads, ads.eng_state_size[guc_class],
511 guc_ads_golden_lrc_size(ads) -
512 xe_lrc_skip_size(xe));
513 ads_blob_write(ads, ads.golden_context_lrca[guc_class],
514 xe_bo_ggtt_addr(ads->bo) +
515 guc_ads_golden_lrc_offset(ads));
516 }
517}
518
519static void guc_mapping_table_init_invalid(struct xe_gt *gt,
520 struct iosys_map *info_map)
521{
522 struct xe_device *xe = gt_to_xe(gt);
523 unsigned int i, j;
524
525 /* Table must be set to invalid values for entries not used */
526 for (i = 0; i < GUC_MAX_ENGINE_CLASSES; ++i)
527 for (j = 0; j < GUC_MAX_INSTANCES_PER_CLASS; ++j)
528 info_map_write(xe, info_map, mapping_table[i][j],
529 GUC_MAX_INSTANCES_PER_CLASS);
530}
531
532static void guc_mapping_table_init(struct xe_gt *gt,
533 struct iosys_map *info_map)
534{
535 struct xe_device *xe = gt_to_xe(gt);
536 struct xe_hw_engine *hwe;
537 enum xe_hw_engine_id id;
538
539 guc_mapping_table_init_invalid(gt, info_map);
540
541 for_each_hw_engine(hwe, gt, id) {
542 u8 guc_class;
543
544 guc_class = xe_engine_class_to_guc_class(hwe->class);
545 info_map_write(xe, info_map,
546 mapping_table[guc_class][hwe->logical_instance],
547 hwe->instance);
548 }
549}
550
551static u32 guc_get_capture_engine_mask(struct xe_gt *gt, struct iosys_map *info_map,
552 enum guc_capture_list_class_type capture_class)
553{
554 struct xe_device *xe = gt_to_xe(gt);
555 u32 mask;
556
557 switch (capture_class) {
558 case GUC_CAPTURE_LIST_CLASS_RENDER_COMPUTE:
559 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_RENDER_CLASS]);
560 mask |= info_map_read(xe, info_map, engine_enabled_masks[GUC_COMPUTE_CLASS]);
561 break;
562 case GUC_CAPTURE_LIST_CLASS_VIDEO:
563 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEO_CLASS]);
564 break;
565 case GUC_CAPTURE_LIST_CLASS_VIDEOENHANCE:
566 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_VIDEOENHANCE_CLASS]);
567 break;
568 case GUC_CAPTURE_LIST_CLASS_BLITTER:
569 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_BLITTER_CLASS]);
570 break;
571 case GUC_CAPTURE_LIST_CLASS_GSC_OTHER:
572 mask = info_map_read(xe, info_map, engine_enabled_masks[GUC_GSC_OTHER_CLASS]);
573 break;
574 default:
575 mask = 0;
576 }
577
578 return mask;
579}
580
581static inline bool get_capture_list(struct xe_guc_ads *ads, struct xe_guc *guc, struct xe_gt *gt,
582 int owner, int type, int class, u32 *total_size, size_t *size,
583 void **pptr)
584{
585 *size = 0;
586
587 if (!xe_guc_capture_getlistsize(guc, owner, type, class, size)) {
588 if (*total_size + *size > ads->capture_size)
589 xe_gt_dbg(gt, "Capture size overflow :%zu vs %d\n",
590 *total_size + *size, ads->capture_size);
591 else if (!xe_guc_capture_getlist(guc, owner, type, class, pptr))
592 return false;
593 }
594
595 return true;
596}
597
598static int guc_capture_prep_lists(struct xe_guc_ads *ads)
599{
600 struct xe_guc *guc = ads_to_guc(ads);
601 struct xe_gt *gt = ads_to_gt(ads);
602 u32 ads_ggtt, capture_offset, null_ggtt, total_size = 0;
603 struct iosys_map info_map;
604 size_t size = 0;
605 void *ptr;
606 int i, j;
607
608 /*
609 * GuC Capture's steered reg-list needs to be allocated and initialized
610 * after the GuC-hwconfig is available which guaranteed from here.
611 */
612 xe_guc_capture_steered_list_init(ads_to_guc(ads));
613
614 capture_offset = guc_ads_capture_offset(ads);
615 ads_ggtt = xe_bo_ggtt_addr(ads->bo);
616 info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
617 offsetof(struct __guc_ads_blob, system_info));
618
619 /* first, set aside the first page for a capture_list with zero descriptors */
620 total_size = PAGE_SIZE;
621 if (!xe_guc_capture_getnullheader(guc, &ptr, &size))
622 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr, size);
623
624 null_ggtt = ads_ggtt + capture_offset;
625 capture_offset += PAGE_SIZE;
626
627 /*
628 * Populate capture list : at this point adps is already allocated and
629 * mapped to worst case size
630 */
631 for (i = 0; i < GUC_CAPTURE_LIST_INDEX_MAX; i++) {
632 bool write_empty_list;
633
634 for (j = 0; j < GUC_CAPTURE_LIST_CLASS_MAX; j++) {
635 u32 engine_mask = guc_get_capture_engine_mask(gt, &info_map, j);
636 /* null list if we dont have said engine or list */
637 if (!engine_mask) {
638 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt);
639 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt);
640 continue;
641 }
642
643 /* engine exists: start with engine-class registers */
644 write_empty_list = get_capture_list(ads, guc, gt, i,
645 GUC_STATE_CAPTURE_TYPE_ENGINE_CLASS,
646 j, &total_size, &size, &ptr);
647 if (!write_empty_list) {
648 ads_blob_write(ads, ads.capture_class[i][j],
649 ads_ggtt + capture_offset);
650 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset,
651 ptr, size);
652 total_size += size;
653 capture_offset += size;
654 } else {
655 ads_blob_write(ads, ads.capture_class[i][j], null_ggtt);
656 }
657
658 /* engine exists: next, engine-instance registers */
659 write_empty_list = get_capture_list(ads, guc, gt, i,
660 GUC_STATE_CAPTURE_TYPE_ENGINE_INSTANCE,
661 j, &total_size, &size, &ptr);
662 if (!write_empty_list) {
663 ads_blob_write(ads, ads.capture_instance[i][j],
664 ads_ggtt + capture_offset);
665 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset,
666 ptr, size);
667 total_size += size;
668 capture_offset += size;
669 } else {
670 ads_blob_write(ads, ads.capture_instance[i][j], null_ggtt);
671 }
672 }
673
674 /* global registers is last in our PF/VF loops */
675 write_empty_list = get_capture_list(ads, guc, gt, i,
676 GUC_STATE_CAPTURE_TYPE_GLOBAL,
677 0, &total_size, &size, &ptr);
678 if (!write_empty_list) {
679 ads_blob_write(ads, ads.capture_global[i], ads_ggtt + capture_offset);
680 xe_map_memcpy_to(ads_to_xe(ads), ads_to_map(ads), capture_offset, ptr,
681 size);
682 total_size += size;
683 capture_offset += size;
684 } else {
685 ads_blob_write(ads, ads.capture_global[i], null_ggtt);
686 }
687 }
688
689 if (ads->capture_size != PAGE_ALIGN(total_size))
690 xe_gt_dbg(gt, "ADS capture alloc size changed from %d to %d\n",
691 ads->capture_size, PAGE_ALIGN(total_size));
692 return PAGE_ALIGN(total_size);
693}
694
695static void guc_mmio_regset_write_one(struct xe_guc_ads *ads,
696 struct iosys_map *regset_map,
697 struct xe_reg reg,
698 unsigned int n_entry)
699{
700 struct guc_mmio_reg entry = {
701 .offset = reg.addr,
702 .flags = reg.masked ? GUC_REGSET_MASKED : 0,
703 };
704
705 if (reg.mcr) {
706 struct xe_reg_mcr mcr_reg = XE_REG_MCR(reg.addr);
707 u8 group, instance;
708
709 bool steer = xe_gt_mcr_get_nonterminated_steering(ads_to_gt(ads), mcr_reg,
710 &group, &instance);
711
712 if (steer) {
713 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_GROUP, group);
714 entry.flags |= FIELD_PREP(GUC_REGSET_STEERING_INSTANCE, instance);
715 entry.flags |= GUC_REGSET_STEERING_NEEDED;
716 }
717 }
718
719 xe_map_memcpy_to(ads_to_xe(ads), regset_map, n_entry * sizeof(entry),
720 &entry, sizeof(entry));
721}
722
723static unsigned int guc_mmio_regset_write(struct xe_guc_ads *ads,
724 struct iosys_map *regset_map,
725 struct xe_hw_engine *hwe)
726{
727 struct xe_device *xe = ads_to_xe(ads);
728 struct xe_hw_engine *hwe_rcs_reset_domain =
729 xe_gt_any_hw_engine_by_reset_domain(hwe->gt, XE_ENGINE_CLASS_RENDER);
730 struct xe_reg_sr_entry *entry;
731 unsigned long idx;
732 unsigned int count = 0;
733 const struct {
734 struct xe_reg reg;
735 bool skip;
736 } *e, extra_regs[] = {
737 { .reg = RING_MODE(hwe->mmio_base), },
738 { .reg = RING_HWS_PGA(hwe->mmio_base), },
739 { .reg = RING_IMR(hwe->mmio_base), },
740 { .reg = RCU_MODE, .skip = hwe != hwe_rcs_reset_domain },
741 { .reg = CCS_MODE,
742 .skip = hwe != hwe_rcs_reset_domain || !xe_gt_ccs_mode_enabled(hwe->gt) },
743 };
744 u32 i;
745
746 BUILD_BUG_ON(ARRAY_SIZE(extra_regs) > ADS_REGSET_EXTRA_MAX);
747
748 xa_for_each(&hwe->reg_sr.xa, idx, entry)
749 guc_mmio_regset_write_one(ads, regset_map, entry->reg, count++);
750
751 for (e = extra_regs; e < extra_regs + ARRAY_SIZE(extra_regs); e++) {
752 if (e->skip)
753 continue;
754
755 guc_mmio_regset_write_one(ads, regset_map, e->reg, count++);
756 }
757
758 /* Wa_1607983814 */
759 if (needs_wa_1607983814(xe) && hwe->class == XE_ENGINE_CLASS_RENDER) {
760 for (i = 0; i < LNCFCMOCS_REG_COUNT; i++) {
761 guc_mmio_regset_write_one(ads, regset_map,
762 XELP_LNCFCMOCS(i), count++);
763 }
764 }
765
766 return count;
767}
768
769static void guc_mmio_reg_state_init(struct xe_guc_ads *ads)
770{
771 size_t regset_offset = guc_ads_regset_offset(ads);
772 struct xe_gt *gt = ads_to_gt(ads);
773 struct xe_hw_engine *hwe;
774 enum xe_hw_engine_id id;
775 u32 addr = xe_bo_ggtt_addr(ads->bo) + regset_offset;
776 struct iosys_map regset_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
777 regset_offset);
778 unsigned int regset_used = 0;
779
780 for_each_hw_engine(hwe, gt, id) {
781 unsigned int count;
782 u8 gc;
783
784 /*
785 * 1. Write all MMIO entries for this exec queue to the table. No
786 * need to worry about fused-off engines and when there are
787 * entries in the regset: the reg_state_list has been zero'ed
788 * by xe_guc_ads_populate()
789 */
790 count = guc_mmio_regset_write(ads, ®set_map, hwe);
791 if (!count)
792 continue;
793
794 /*
795 * 2. Record in the header (ads.reg_state_list) the address
796 * location and number of entries
797 */
798 gc = xe_engine_class_to_guc_class(hwe->class);
799 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].address, addr);
800 ads_blob_write(ads, ads.reg_state_list[gc][hwe->instance].count, count);
801
802 addr += count * sizeof(struct guc_mmio_reg);
803 iosys_map_incr(®set_map, count * sizeof(struct guc_mmio_reg));
804
805 regset_used += count * sizeof(struct guc_mmio_reg);
806 }
807
808 xe_gt_assert(gt, regset_used <= ads->regset_size);
809}
810
811static void guc_um_init_params(struct xe_guc_ads *ads)
812{
813 u32 um_queue_offset = guc_ads_um_queues_offset(ads);
814 u64 base_dpa;
815 u32 base_ggtt;
816 int i;
817
818 base_ggtt = xe_bo_ggtt_addr(ads->bo) + um_queue_offset;
819 base_dpa = xe_bo_main_addr(ads->bo, PAGE_SIZE) + um_queue_offset;
820
821 for (i = 0; i < GUC_UM_HW_QUEUE_MAX; ++i) {
822 ads_blob_write(ads, um_init_params.queue_params[i].base_dpa,
823 base_dpa + (i * GUC_UM_QUEUE_SIZE));
824 ads_blob_write(ads, um_init_params.queue_params[i].base_ggtt_address,
825 base_ggtt + (i * GUC_UM_QUEUE_SIZE));
826 ads_blob_write(ads, um_init_params.queue_params[i].size_in_bytes,
827 GUC_UM_QUEUE_SIZE);
828 }
829
830 ads_blob_write(ads, um_init_params.page_response_timeout_in_us,
831 GUC_PAGE_RES_TIMEOUT_US);
832}
833
834static void guc_doorbell_init(struct xe_guc_ads *ads)
835{
836 struct xe_device *xe = ads_to_xe(ads);
837 struct xe_gt *gt = ads_to_gt(ads);
838
839 if (GRAPHICS_VER(xe) >= 12 && !IS_DGFX(xe)) {
840 u32 distdbreg =
841 xe_mmio_read32(>->mmio, DIST_DBS_POPULATED);
842
843 ads_blob_write(ads,
844 system_info.generic_gt_sysinfo[GUC_GENERIC_GT_SYSINFO_DOORBELL_COUNT_PER_SQIDI],
845 REG_FIELD_GET(DOORBELLS_PER_SQIDI_MASK, distdbreg) + 1);
846 }
847}
848
849/**
850 * xe_guc_ads_populate_minimal - populate minimal ADS
851 * @ads: Additional data structures object
852 *
853 * This function populates a minimal ADS that does not support submissions but
854 * enough so the GuC can load and the hwconfig table can be read.
855 */
856void xe_guc_ads_populate_minimal(struct xe_guc_ads *ads)
857{
858 struct xe_gt *gt = ads_to_gt(ads);
859 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
860 offsetof(struct __guc_ads_blob, system_info));
861 u32 base = xe_bo_ggtt_addr(ads->bo);
862
863 xe_gt_assert(gt, ads->bo);
864
865 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, ads->bo->size);
866 guc_policies_init(ads);
867 guc_prep_golden_lrc_null(ads);
868 guc_mapping_table_init_invalid(gt, &info_map);
869 guc_doorbell_init(ads);
870
871 ads_blob_write(ads, ads.scheduler_policies, base +
872 offsetof(struct __guc_ads_blob, policies));
873 ads_blob_write(ads, ads.gt_system_info, base +
874 offsetof(struct __guc_ads_blob, system_info));
875 ads_blob_write(ads, ads.private_data, base +
876 guc_ads_private_data_offset(ads));
877}
878
879void xe_guc_ads_populate(struct xe_guc_ads *ads)
880{
881 struct xe_device *xe = ads_to_xe(ads);
882 struct xe_gt *gt = ads_to_gt(ads);
883 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
884 offsetof(struct __guc_ads_blob, system_info));
885 u32 base = xe_bo_ggtt_addr(ads->bo);
886
887 xe_gt_assert(gt, ads->bo);
888
889 xe_map_memset(ads_to_xe(ads), ads_to_map(ads), 0, 0, ads->bo->size);
890 guc_policies_init(ads);
891 fill_engine_enable_masks(gt, &info_map);
892 guc_mmio_reg_state_init(ads);
893 guc_prep_golden_lrc_null(ads);
894 guc_mapping_table_init(gt, &info_map);
895 guc_capture_prep_lists(ads);
896 guc_doorbell_init(ads);
897 guc_waklv_init(ads);
898
899 if (xe->info.has_usm) {
900 guc_um_init_params(ads);
901 ads_blob_write(ads, ads.um_init_data, base +
902 offsetof(struct __guc_ads_blob, um_init_params));
903 }
904
905 ads_blob_write(ads, ads.scheduler_policies, base +
906 offsetof(struct __guc_ads_blob, policies));
907 ads_blob_write(ads, ads.gt_system_info, base +
908 offsetof(struct __guc_ads_blob, system_info));
909 ads_blob_write(ads, ads.private_data, base +
910 guc_ads_private_data_offset(ads));
911}
912
913static void guc_populate_golden_lrc(struct xe_guc_ads *ads)
914{
915 struct xe_device *xe = ads_to_xe(ads);
916 struct xe_gt *gt = ads_to_gt(ads);
917 struct iosys_map info_map = IOSYS_MAP_INIT_OFFSET(ads_to_map(ads),
918 offsetof(struct __guc_ads_blob, system_info));
919 size_t total_size = 0, alloc_size, real_size;
920 u32 addr_ggtt, offset;
921 int class;
922
923 offset = guc_ads_golden_lrc_offset(ads);
924 addr_ggtt = xe_bo_ggtt_addr(ads->bo) + offset;
925
926 for (class = 0; class < XE_ENGINE_CLASS_MAX; ++class) {
927 u8 guc_class;
928
929 guc_class = xe_engine_class_to_guc_class(class);
930
931 if (!info_map_read(xe, &info_map,
932 engine_enabled_masks[guc_class]))
933 continue;
934
935 xe_gt_assert(gt, gt->default_lrc[class]);
936
937 real_size = xe_gt_lrc_size(gt, class);
938 alloc_size = PAGE_ALIGN(real_size);
939 total_size += alloc_size;
940
941 /*
942 * This interface is slightly confusing. We need to pass the
943 * base address of the full golden context and the size of just
944 * the engine state, which is the section of the context image
945 * that starts after the execlists LRC registers. This is
946 * required to allow the GuC to restore just the engine state
947 * when a watchdog reset occurs.
948 * We calculate the engine state size by removing the size of
949 * what comes before it in the context image (which is identical
950 * on all engines).
951 */
952 ads_blob_write(ads, ads.eng_state_size[guc_class],
953 real_size - xe_lrc_skip_size(xe));
954 ads_blob_write(ads, ads.golden_context_lrca[guc_class],
955 addr_ggtt);
956
957 xe_map_memcpy_to(xe, ads_to_map(ads), offset,
958 gt->default_lrc[class], real_size);
959
960 addr_ggtt += alloc_size;
961 offset += alloc_size;
962 }
963
964 xe_gt_assert(gt, total_size == ads->golden_lrc_size);
965}
966
967void xe_guc_ads_populate_post_load(struct xe_guc_ads *ads)
968{
969 guc_populate_golden_lrc(ads);
970}
971
972static int guc_ads_action_update_policies(struct xe_guc_ads *ads, u32 policy_offset)
973{
974 struct xe_guc_ct *ct = &ads_to_guc(ads)->ct;
975 u32 action[] = {
976 XE_GUC_ACTION_GLOBAL_SCHED_POLICY_CHANGE,
977 policy_offset
978 };
979
980 return xe_guc_ct_send(ct, action, ARRAY_SIZE(action), 0, 0);
981}
982
983/**
984 * xe_guc_ads_scheduler_policy_toggle_reset - Toggle reset policy
985 * @ads: Additional data structures object
986 *
987 * This function update the GuC's engine reset policy based on wedged.mode.
988 *
989 * Return: 0 on success, and negative error code otherwise.
990 */
991int xe_guc_ads_scheduler_policy_toggle_reset(struct xe_guc_ads *ads)
992{
993 struct xe_device *xe = ads_to_xe(ads);
994 struct xe_gt *gt = ads_to_gt(ads);
995 struct xe_tile *tile = gt_to_tile(gt);
996 struct guc_policies *policies;
997 struct xe_bo *bo;
998 int ret = 0;
999
1000 policies = kmalloc(sizeof(*policies), GFP_KERNEL);
1001 if (!policies)
1002 return -ENOMEM;
1003
1004 policies->dpc_promote_time = ads_blob_read(ads, policies.dpc_promote_time);
1005 policies->max_num_work_items = ads_blob_read(ads, policies.max_num_work_items);
1006 policies->is_valid = 1;
1007 if (xe->wedged.mode == 2)
1008 policies->global_flags |= GLOBAL_POLICY_DISABLE_ENGINE_RESET;
1009 else
1010 policies->global_flags &= ~GLOBAL_POLICY_DISABLE_ENGINE_RESET;
1011
1012 bo = xe_managed_bo_create_from_data(xe, tile, policies, sizeof(struct guc_policies),
1013 XE_BO_FLAG_VRAM_IF_DGFX(tile) |
1014 XE_BO_FLAG_GGTT);
1015 if (IS_ERR(bo)) {
1016 ret = PTR_ERR(bo);
1017 goto out;
1018 }
1019
1020 ret = guc_ads_action_update_policies(ads, xe_bo_ggtt_addr(bo));
1021out:
1022 kfree(policies);
1023 return ret;
1024}