Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include "xe_guc_ct.h"
7
8#include <linux/bitfield.h>
9#include <linux/circ_buf.h>
10#include <linux/delay.h>
11
12#include <drm/drm_managed.h>
13
14#include "abi/guc_actions_abi.h"
15#include "abi/guc_klvs_abi.h"
16#include "xe_bo.h"
17#include "xe_device.h"
18#include "xe_gt.h"
19#include "xe_gt_pagefault.h"
20#include "xe_gt_tlb_invalidation.h"
21#include "xe_guc.h"
22#include "xe_guc_submit.h"
23#include "xe_map.h"
24#include "xe_pm.h"
25#include "xe_trace.h"
26
27/* Used when a CT send wants to block and / or receive data */
28struct g2h_fence {
29 u32 *response_buffer;
30 u32 seqno;
31 u16 response_len;
32 u16 error;
33 u16 hint;
34 u16 reason;
35 bool retry;
36 bool fail;
37 bool done;
38};
39
40static void g2h_fence_init(struct g2h_fence *g2h_fence, u32 *response_buffer)
41{
42 g2h_fence->response_buffer = response_buffer;
43 g2h_fence->response_len = 0;
44 g2h_fence->fail = false;
45 g2h_fence->retry = false;
46 g2h_fence->done = false;
47 g2h_fence->seqno = ~0x0;
48}
49
50static bool g2h_fence_needs_alloc(struct g2h_fence *g2h_fence)
51{
52 return g2h_fence->seqno == ~0x0;
53}
54
55static struct xe_guc *
56ct_to_guc(struct xe_guc_ct *ct)
57{
58 return container_of(ct, struct xe_guc, ct);
59}
60
61static struct xe_gt *
62ct_to_gt(struct xe_guc_ct *ct)
63{
64 return container_of(ct, struct xe_gt, uc.guc.ct);
65}
66
67static struct xe_device *
68ct_to_xe(struct xe_guc_ct *ct)
69{
70 return gt_to_xe(ct_to_gt(ct));
71}
72
73/**
74 * DOC: GuC CTB Blob
75 *
76 * We allocate single blob to hold both CTB descriptors and buffers:
77 *
78 * +--------+-----------------------------------------------+------+
79 * | offset | contents | size |
80 * +========+===============================================+======+
81 * | 0x0000 | H2G CTB Descriptor (send) | |
82 * +--------+-----------------------------------------------+ 4K |
83 * | 0x0800 | G2H CTB Descriptor (g2h) | |
84 * +--------+-----------------------------------------------+------+
85 * | 0x1000 | H2G CT Buffer (send) | n*4K |
86 * | | | |
87 * +--------+-----------------------------------------------+------+
88 * | 0x1000 | G2H CT Buffer (g2h) | m*4K |
89 * | + n*4K | | |
90 * +--------+-----------------------------------------------+------+
91 *
92 * Size of each ``CT Buffer`` must be multiple of 4K.
93 * We don't expect too many messages in flight at any time, unless we are
94 * using the GuC submission. In that case each request requires a minimum
95 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
96 * enough space to avoid backpressure on the driver. We increase the size
97 * of the receive buffer (relative to the send) to ensure a G2H response
98 * CTB has a landing spot.
99 */
100
101#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
102#define CTB_H2G_BUFFER_SIZE (SZ_4K)
103#define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE)
104#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4)
105
106static size_t guc_ct_size(void)
107{
108 return 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE +
109 CTB_G2H_BUFFER_SIZE;
110}
111
112static void guc_ct_fini(struct drm_device *drm, void *arg)
113{
114 struct xe_guc_ct *ct = arg;
115
116 xa_destroy(&ct->fence_lookup);
117}
118
119static void g2h_worker_func(struct work_struct *w);
120
121static void primelockdep(struct xe_guc_ct *ct)
122{
123 if (!IS_ENABLED(CONFIG_LOCKDEP))
124 return;
125
126 fs_reclaim_acquire(GFP_KERNEL);
127 might_lock(&ct->lock);
128 fs_reclaim_release(GFP_KERNEL);
129}
130
131int xe_guc_ct_init(struct xe_guc_ct *ct)
132{
133 struct xe_device *xe = ct_to_xe(ct);
134 struct xe_gt *gt = ct_to_gt(ct);
135 struct xe_tile *tile = gt_to_tile(gt);
136 struct xe_bo *bo;
137 int err;
138
139 xe_assert(xe, !(guc_ct_size() % PAGE_SIZE));
140
141 drmm_mutex_init(&xe->drm, &ct->lock);
142 spin_lock_init(&ct->fast_lock);
143 xa_init(&ct->fence_lookup);
144 INIT_WORK(&ct->g2h_worker, g2h_worker_func);
145 init_waitqueue_head(&ct->wq);
146 init_waitqueue_head(&ct->g2h_fence_wq);
147
148 primelockdep(ct);
149
150 bo = xe_managed_bo_create_pin_map(xe, tile, guc_ct_size(),
151 XE_BO_CREATE_VRAM_IF_DGFX(tile) |
152 XE_BO_CREATE_GGTT_BIT);
153 if (IS_ERR(bo))
154 return PTR_ERR(bo);
155
156 ct->bo = bo;
157
158 err = drmm_add_action_or_reset(&xe->drm, guc_ct_fini, ct);
159 if (err)
160 return err;
161
162 return 0;
163}
164
165#define desc_read(xe_, guc_ctb__, field_) \
166 xe_map_rd_field(xe_, &guc_ctb__->desc, 0, \
167 struct guc_ct_buffer_desc, field_)
168
169#define desc_write(xe_, guc_ctb__, field_, val_) \
170 xe_map_wr_field(xe_, &guc_ctb__->desc, 0, \
171 struct guc_ct_buffer_desc, field_, val_)
172
173static void guc_ct_ctb_h2g_init(struct xe_device *xe, struct guc_ctb *h2g,
174 struct iosys_map *map)
175{
176 h2g->info.size = CTB_H2G_BUFFER_SIZE / sizeof(u32);
177 h2g->info.resv_space = 0;
178 h2g->info.tail = 0;
179 h2g->info.head = 0;
180 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
181 h2g->info.size) -
182 h2g->info.resv_space;
183 h2g->info.broken = false;
184
185 h2g->desc = *map;
186 xe_map_memset(xe, &h2g->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
187
188 h2g->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2);
189}
190
191static void guc_ct_ctb_g2h_init(struct xe_device *xe, struct guc_ctb *g2h,
192 struct iosys_map *map)
193{
194 g2h->info.size = CTB_G2H_BUFFER_SIZE / sizeof(u32);
195 g2h->info.resv_space = G2H_ROOM_BUFFER_SIZE / sizeof(u32);
196 g2h->info.head = 0;
197 g2h->info.tail = 0;
198 g2h->info.space = CIRC_SPACE(g2h->info.tail, g2h->info.head,
199 g2h->info.size) -
200 g2h->info.resv_space;
201 g2h->info.broken = false;
202
203 g2h->desc = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE);
204 xe_map_memset(xe, &g2h->desc, 0, 0, sizeof(struct guc_ct_buffer_desc));
205
206 g2h->cmds = IOSYS_MAP_INIT_OFFSET(map, CTB_DESC_SIZE * 2 +
207 CTB_H2G_BUFFER_SIZE);
208}
209
210static int guc_ct_ctb_h2g_register(struct xe_guc_ct *ct)
211{
212 struct xe_guc *guc = ct_to_guc(ct);
213 u32 desc_addr, ctb_addr, size;
214 int err;
215
216 desc_addr = xe_bo_ggtt_addr(ct->bo);
217 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2;
218 size = ct->ctbs.h2g.info.size * sizeof(u32);
219
220 err = xe_guc_self_cfg64(guc,
221 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY,
222 desc_addr);
223 if (err)
224 return err;
225
226 err = xe_guc_self_cfg64(guc,
227 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY,
228 ctb_addr);
229 if (err)
230 return err;
231
232 return xe_guc_self_cfg32(guc,
233 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY,
234 size);
235}
236
237static int guc_ct_ctb_g2h_register(struct xe_guc_ct *ct)
238{
239 struct xe_guc *guc = ct_to_guc(ct);
240 u32 desc_addr, ctb_addr, size;
241 int err;
242
243 desc_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE;
244 ctb_addr = xe_bo_ggtt_addr(ct->bo) + CTB_DESC_SIZE * 2 +
245 CTB_H2G_BUFFER_SIZE;
246 size = ct->ctbs.g2h.info.size * sizeof(u32);
247
248 err = xe_guc_self_cfg64(guc,
249 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
250 desc_addr);
251 if (err)
252 return err;
253
254 err = xe_guc_self_cfg64(guc,
255 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
256 ctb_addr);
257 if (err)
258 return err;
259
260 return xe_guc_self_cfg32(guc,
261 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
262 size);
263}
264
265static int guc_ct_control_toggle(struct xe_guc_ct *ct, bool enable)
266{
267 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
268 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
269 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
270 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION,
271 GUC_ACTION_HOST2GUC_CONTROL_CTB),
272 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL,
273 enable ? GUC_CTB_CONTROL_ENABLE :
274 GUC_CTB_CONTROL_DISABLE),
275 };
276 int ret = xe_guc_mmio_send(ct_to_guc(ct), request, ARRAY_SIZE(request));
277
278 return ret > 0 ? -EPROTO : ret;
279}
280
281int xe_guc_ct_enable(struct xe_guc_ct *ct)
282{
283 struct xe_device *xe = ct_to_xe(ct);
284 int err;
285
286 xe_assert(xe, !ct->enabled);
287
288 guc_ct_ctb_h2g_init(xe, &ct->ctbs.h2g, &ct->bo->vmap);
289 guc_ct_ctb_g2h_init(xe, &ct->ctbs.g2h, &ct->bo->vmap);
290
291 err = guc_ct_ctb_h2g_register(ct);
292 if (err)
293 goto err_out;
294
295 err = guc_ct_ctb_g2h_register(ct);
296 if (err)
297 goto err_out;
298
299 err = guc_ct_control_toggle(ct, true);
300 if (err)
301 goto err_out;
302
303 mutex_lock(&ct->lock);
304 spin_lock_irq(&ct->fast_lock);
305 ct->g2h_outstanding = 0;
306 ct->enabled = true;
307 spin_unlock_irq(&ct->fast_lock);
308 mutex_unlock(&ct->lock);
309
310 smp_mb();
311 wake_up_all(&ct->wq);
312 drm_dbg(&xe->drm, "GuC CT communication channel enabled\n");
313
314 return 0;
315
316err_out:
317 drm_err(&xe->drm, "Failed to enable CT (%d)\n", err);
318
319 return err;
320}
321
322void xe_guc_ct_disable(struct xe_guc_ct *ct)
323{
324 mutex_lock(&ct->lock); /* Serialise dequeue_one_g2h() */
325 spin_lock_irq(&ct->fast_lock); /* Serialise CT fast-path */
326 ct->enabled = false; /* Finally disable CT communication */
327 spin_unlock_irq(&ct->fast_lock);
328 mutex_unlock(&ct->lock);
329
330 xa_destroy(&ct->fence_lookup);
331}
332
333static bool h2g_has_room(struct xe_guc_ct *ct, u32 cmd_len)
334{
335 struct guc_ctb *h2g = &ct->ctbs.h2g;
336
337 lockdep_assert_held(&ct->lock);
338
339 if (cmd_len > h2g->info.space) {
340 h2g->info.head = desc_read(ct_to_xe(ct), h2g, head);
341 h2g->info.space = CIRC_SPACE(h2g->info.tail, h2g->info.head,
342 h2g->info.size) -
343 h2g->info.resv_space;
344 if (cmd_len > h2g->info.space)
345 return false;
346 }
347
348 return true;
349}
350
351static bool g2h_has_room(struct xe_guc_ct *ct, u32 g2h_len)
352{
353 if (!g2h_len)
354 return true;
355
356 lockdep_assert_held(&ct->fast_lock);
357
358 return ct->ctbs.g2h.info.space > g2h_len;
359}
360
361static int has_room(struct xe_guc_ct *ct, u32 cmd_len, u32 g2h_len)
362{
363 lockdep_assert_held(&ct->lock);
364
365 if (!g2h_has_room(ct, g2h_len) || !h2g_has_room(ct, cmd_len))
366 return -EBUSY;
367
368 return 0;
369}
370
371static void h2g_reserve_space(struct xe_guc_ct *ct, u32 cmd_len)
372{
373 lockdep_assert_held(&ct->lock);
374 ct->ctbs.h2g.info.space -= cmd_len;
375}
376
377static void __g2h_reserve_space(struct xe_guc_ct *ct, u32 g2h_len, u32 num_g2h)
378{
379 xe_assert(ct_to_xe(ct), g2h_len <= ct->ctbs.g2h.info.space);
380
381 if (g2h_len) {
382 lockdep_assert_held(&ct->fast_lock);
383
384 ct->ctbs.g2h.info.space -= g2h_len;
385 ct->g2h_outstanding += num_g2h;
386 }
387}
388
389static void __g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
390{
391 lockdep_assert_held(&ct->fast_lock);
392 xe_assert(ct_to_xe(ct), ct->ctbs.g2h.info.space + g2h_len <=
393 ct->ctbs.g2h.info.size - ct->ctbs.g2h.info.resv_space);
394
395 ct->ctbs.g2h.info.space += g2h_len;
396 --ct->g2h_outstanding;
397}
398
399static void g2h_release_space(struct xe_guc_ct *ct, u32 g2h_len)
400{
401 spin_lock_irq(&ct->fast_lock);
402 __g2h_release_space(ct, g2h_len);
403 spin_unlock_irq(&ct->fast_lock);
404}
405
406#define H2G_CT_HEADERS (GUC_CTB_HDR_LEN + 1) /* one DW CTB header and one DW HxG header */
407
408static int h2g_write(struct xe_guc_ct *ct, const u32 *action, u32 len,
409 u32 ct_fence_value, bool want_response)
410{
411 struct xe_device *xe = ct_to_xe(ct);
412 struct guc_ctb *h2g = &ct->ctbs.h2g;
413 u32 cmd[H2G_CT_HEADERS];
414 u32 tail = h2g->info.tail;
415 u32 full_len;
416 struct iosys_map map = IOSYS_MAP_INIT_OFFSET(&h2g->cmds,
417 tail * sizeof(u32));
418
419 full_len = len + GUC_CTB_HDR_LEN;
420
421 lockdep_assert_held(&ct->lock);
422 xe_assert(xe, full_len <= GUC_CTB_MSG_MAX_LEN);
423 xe_assert(xe, tail <= h2g->info.size);
424
425 /* Command will wrap, zero fill (NOPs), return and check credits again */
426 if (tail + full_len > h2g->info.size) {
427 xe_map_memset(xe, &map, 0, 0,
428 (h2g->info.size - tail) * sizeof(u32));
429 h2g_reserve_space(ct, (h2g->info.size - tail));
430 h2g->info.tail = 0;
431 desc_write(xe, h2g, tail, h2g->info.tail);
432
433 return -EAGAIN;
434 }
435
436 /*
437 * dw0: CT header (including fence)
438 * dw1: HXG header (including action code)
439 * dw2+: action data
440 */
441 cmd[0] = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
442 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
443 FIELD_PREP(GUC_CTB_MSG_0_FENCE, ct_fence_value);
444 if (want_response) {
445 cmd[1] =
446 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
447 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
448 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
449 } else {
450 cmd[1] =
451 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_EVENT) |
452 FIELD_PREP(GUC_HXG_EVENT_MSG_0_ACTION |
453 GUC_HXG_EVENT_MSG_0_DATA0, action[0]);
454 }
455
456 /* H2G header in cmd[1] replaces action[0] so: */
457 --len;
458 ++action;
459
460 /* Write H2G ensuring visable before descriptor update */
461 xe_map_memcpy_to(xe, &map, 0, cmd, H2G_CT_HEADERS * sizeof(u32));
462 xe_map_memcpy_to(xe, &map, H2G_CT_HEADERS * sizeof(u32), action, len * sizeof(u32));
463 xe_device_wmb(xe);
464
465 /* Update local copies */
466 h2g->info.tail = (tail + full_len) % h2g->info.size;
467 h2g_reserve_space(ct, full_len);
468
469 /* Update descriptor */
470 desc_write(xe, h2g, tail, h2g->info.tail);
471
472 trace_xe_guc_ctb_h2g(ct_to_gt(ct)->info.id, *(action - 1), full_len,
473 desc_read(xe, h2g, head), h2g->info.tail);
474
475 return 0;
476}
477
478static int __guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action,
479 u32 len, u32 g2h_len, u32 num_g2h,
480 struct g2h_fence *g2h_fence)
481{
482 struct xe_device *xe = ct_to_xe(ct);
483 int ret;
484
485 xe_assert(xe, !g2h_len || !g2h_fence);
486 xe_assert(xe, !num_g2h || !g2h_fence);
487 xe_assert(xe, !g2h_len || num_g2h);
488 xe_assert(xe, g2h_len || !num_g2h);
489 lockdep_assert_held(&ct->lock);
490
491 if (unlikely(ct->ctbs.h2g.info.broken)) {
492 ret = -EPIPE;
493 goto out;
494 }
495
496 if (unlikely(!ct->enabled)) {
497 ret = -ENODEV;
498 goto out;
499 }
500
501 if (g2h_fence) {
502 g2h_len = GUC_CTB_HXG_MSG_MAX_LEN;
503 num_g2h = 1;
504
505 if (g2h_fence_needs_alloc(g2h_fence)) {
506 void *ptr;
507
508 g2h_fence->seqno = (ct->fence_seqno++ & 0xffff);
509 ptr = xa_store(&ct->fence_lookup,
510 g2h_fence->seqno,
511 g2h_fence, GFP_ATOMIC);
512 if (IS_ERR(ptr)) {
513 ret = PTR_ERR(ptr);
514 goto out;
515 }
516 }
517 }
518
519 if (g2h_len)
520 spin_lock_irq(&ct->fast_lock);
521retry:
522 ret = has_room(ct, len + GUC_CTB_HDR_LEN, g2h_len);
523 if (unlikely(ret))
524 goto out_unlock;
525
526 ret = h2g_write(ct, action, len, g2h_fence ? g2h_fence->seqno : 0,
527 !!g2h_fence);
528 if (unlikely(ret)) {
529 if (ret == -EAGAIN)
530 goto retry;
531 goto out_unlock;
532 }
533
534 __g2h_reserve_space(ct, g2h_len, num_g2h);
535 xe_guc_notify(ct_to_guc(ct));
536out_unlock:
537 if (g2h_len)
538 spin_unlock_irq(&ct->fast_lock);
539out:
540 return ret;
541}
542
543static void kick_reset(struct xe_guc_ct *ct)
544{
545 xe_gt_reset_async(ct_to_gt(ct));
546}
547
548static int dequeue_one_g2h(struct xe_guc_ct *ct);
549
550static int guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
551 u32 g2h_len, u32 num_g2h,
552 struct g2h_fence *g2h_fence)
553{
554 struct drm_device *drm = &ct_to_xe(ct)->drm;
555 struct drm_printer p = drm_info_printer(drm->dev);
556 unsigned int sleep_period_ms = 1;
557 int ret;
558
559 xe_assert(ct_to_xe(ct), !g2h_len || !g2h_fence);
560 lockdep_assert_held(&ct->lock);
561 xe_device_assert_mem_access(ct_to_xe(ct));
562
563try_again:
564 ret = __guc_ct_send_locked(ct, action, len, g2h_len, num_g2h,
565 g2h_fence);
566
567 /*
568 * We wait to try to restore credits for about 1 second before bailing.
569 * In the case of H2G credits we have no choice but just to wait for the
570 * GuC to consume H2Gs in the channel so we use a wait / sleep loop. In
571 * the case of G2H we process any G2H in the channel, hopefully freeing
572 * credits as we consume the G2H messages.
573 */
574 if (unlikely(ret == -EBUSY &&
575 !h2g_has_room(ct, len + GUC_CTB_HDR_LEN))) {
576 struct guc_ctb *h2g = &ct->ctbs.h2g;
577
578 if (sleep_period_ms == 1024)
579 goto broken;
580
581 trace_xe_guc_ct_h2g_flow_control(h2g->info.head, h2g->info.tail,
582 h2g->info.size,
583 h2g->info.space,
584 len + GUC_CTB_HDR_LEN);
585 msleep(sleep_period_ms);
586 sleep_period_ms <<= 1;
587
588 goto try_again;
589 } else if (unlikely(ret == -EBUSY)) {
590 struct xe_device *xe = ct_to_xe(ct);
591 struct guc_ctb *g2h = &ct->ctbs.g2h;
592
593 trace_xe_guc_ct_g2h_flow_control(g2h->info.head,
594 desc_read(xe, g2h, tail),
595 g2h->info.size,
596 g2h->info.space,
597 g2h_fence ?
598 GUC_CTB_HXG_MSG_MAX_LEN :
599 g2h_len);
600
601#define g2h_avail(ct) \
602 (desc_read(ct_to_xe(ct), (&ct->ctbs.g2h), tail) != ct->ctbs.g2h.info.head)
603 if (!wait_event_timeout(ct->wq, !ct->g2h_outstanding ||
604 g2h_avail(ct), HZ))
605 goto broken;
606#undef g2h_avail
607
608 if (dequeue_one_g2h(ct) < 0)
609 goto broken;
610
611 goto try_again;
612 }
613
614 return ret;
615
616broken:
617 drm_err(drm, "No forward process on H2G, reset required");
618 xe_guc_ct_print(ct, &p, true);
619 ct->ctbs.h2g.info.broken = true;
620
621 return -EDEADLK;
622}
623
624static int guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
625 u32 g2h_len, u32 num_g2h, struct g2h_fence *g2h_fence)
626{
627 int ret;
628
629 xe_assert(ct_to_xe(ct), !g2h_len || !g2h_fence);
630
631 mutex_lock(&ct->lock);
632 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, g2h_fence);
633 mutex_unlock(&ct->lock);
634
635 return ret;
636}
637
638int xe_guc_ct_send(struct xe_guc_ct *ct, const u32 *action, u32 len,
639 u32 g2h_len, u32 num_g2h)
640{
641 int ret;
642
643 ret = guc_ct_send(ct, action, len, g2h_len, num_g2h, NULL);
644 if (ret == -EDEADLK)
645 kick_reset(ct);
646
647 return ret;
648}
649
650int xe_guc_ct_send_locked(struct xe_guc_ct *ct, const u32 *action, u32 len,
651 u32 g2h_len, u32 num_g2h)
652{
653 int ret;
654
655 ret = guc_ct_send_locked(ct, action, len, g2h_len, num_g2h, NULL);
656 if (ret == -EDEADLK)
657 kick_reset(ct);
658
659 return ret;
660}
661
662int xe_guc_ct_send_g2h_handler(struct xe_guc_ct *ct, const u32 *action, u32 len)
663{
664 int ret;
665
666 lockdep_assert_held(&ct->lock);
667
668 ret = guc_ct_send_locked(ct, action, len, 0, 0, NULL);
669 if (ret == -EDEADLK)
670 kick_reset(ct);
671
672 return ret;
673}
674
675/*
676 * Check if a GT reset is in progress or will occur and if GT reset brought the
677 * CT back up. Randomly picking 5 seconds for an upper limit to do a GT a reset.
678 */
679static bool retry_failure(struct xe_guc_ct *ct, int ret)
680{
681 if (!(ret == -EDEADLK || ret == -EPIPE || ret == -ENODEV))
682 return false;
683
684#define ct_alive(ct) \
685 (ct->enabled && !ct->ctbs.h2g.info.broken && !ct->ctbs.g2h.info.broken)
686 if (!wait_event_interruptible_timeout(ct->wq, ct_alive(ct), HZ * 5))
687 return false;
688#undef ct_alive
689
690 return true;
691}
692
693static int guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
694 u32 *response_buffer, bool no_fail)
695{
696 struct xe_device *xe = ct_to_xe(ct);
697 struct g2h_fence g2h_fence;
698 int ret = 0;
699
700 /*
701 * We use a fence to implement blocking sends / receiving response data.
702 * The seqno of the fence is sent in the H2G, returned in the G2H, and
703 * an xarray is used as storage media with the seqno being to key.
704 * Fields in the fence hold success, failure, retry status and the
705 * response data. Safe to allocate on the stack as the xarray is the
706 * only reference and it cannot be present after this function exits.
707 */
708retry:
709 g2h_fence_init(&g2h_fence, response_buffer);
710retry_same_fence:
711 ret = guc_ct_send(ct, action, len, 0, 0, &g2h_fence);
712 if (unlikely(ret == -ENOMEM)) {
713 void *ptr;
714
715 /* Retry allocation /w GFP_KERNEL */
716 ptr = xa_store(&ct->fence_lookup,
717 g2h_fence.seqno,
718 &g2h_fence, GFP_KERNEL);
719 if (IS_ERR(ptr))
720 return PTR_ERR(ptr);
721
722 goto retry_same_fence;
723 } else if (unlikely(ret)) {
724 if (ret == -EDEADLK)
725 kick_reset(ct);
726
727 if (no_fail && retry_failure(ct, ret))
728 goto retry_same_fence;
729
730 if (!g2h_fence_needs_alloc(&g2h_fence))
731 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);
732
733 return ret;
734 }
735
736 ret = wait_event_timeout(ct->g2h_fence_wq, g2h_fence.done, HZ);
737 if (!ret) {
738 drm_err(&xe->drm, "Timed out wait for G2H, fence %u, action %04x",
739 g2h_fence.seqno, action[0]);
740 xa_erase_irq(&ct->fence_lookup, g2h_fence.seqno);
741 return -ETIME;
742 }
743
744 if (g2h_fence.retry) {
745 drm_warn(&xe->drm, "Send retry, action 0x%04x, reason %d",
746 action[0], g2h_fence.reason);
747 goto retry;
748 }
749 if (g2h_fence.fail) {
750 drm_err(&xe->drm, "Send failed, action 0x%04x, error %d, hint %d",
751 action[0], g2h_fence.error, g2h_fence.hint);
752 ret = -EIO;
753 }
754
755 return ret > 0 ? 0 : ret;
756}
757
758int xe_guc_ct_send_recv(struct xe_guc_ct *ct, const u32 *action, u32 len,
759 u32 *response_buffer)
760{
761 return guc_ct_send_recv(ct, action, len, response_buffer, false);
762}
763
764int xe_guc_ct_send_recv_no_fail(struct xe_guc_ct *ct, const u32 *action,
765 u32 len, u32 *response_buffer)
766{
767 return guc_ct_send_recv(ct, action, len, response_buffer, true);
768}
769
770static int parse_g2h_event(struct xe_guc_ct *ct, u32 *msg, u32 len)
771{
772 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]);
773
774 lockdep_assert_held(&ct->lock);
775
776 switch (action) {
777 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
778 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
779 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
780 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
781 g2h_release_space(ct, len);
782 }
783
784 return 0;
785}
786
787static int parse_g2h_response(struct xe_guc_ct *ct, u32 *msg, u32 len)
788{
789 struct xe_device *xe = ct_to_xe(ct);
790 u32 response_len = len - GUC_CTB_MSG_MIN_LEN;
791 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, msg[0]);
792 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]);
793 struct g2h_fence *g2h_fence;
794
795 lockdep_assert_held(&ct->lock);
796
797 g2h_fence = xa_erase(&ct->fence_lookup, fence);
798 if (unlikely(!g2h_fence)) {
799 /* Don't tear down channel, as send could've timed out */
800 drm_warn(&xe->drm, "G2H fence (%u) not found!\n", fence);
801 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
802 return 0;
803 }
804
805 xe_assert(xe, fence == g2h_fence->seqno);
806
807 if (type == GUC_HXG_TYPE_RESPONSE_FAILURE) {
808 g2h_fence->fail = true;
809 g2h_fence->error =
810 FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, msg[1]);
811 g2h_fence->hint =
812 FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, msg[1]);
813 } else if (type == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
814 g2h_fence->retry = true;
815 g2h_fence->reason =
816 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, msg[1]);
817 } else if (g2h_fence->response_buffer) {
818 g2h_fence->response_len = response_len;
819 memcpy(g2h_fence->response_buffer, msg + GUC_CTB_MSG_MIN_LEN,
820 response_len * sizeof(u32));
821 }
822
823 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
824
825 g2h_fence->done = true;
826 smp_mb();
827
828 wake_up_all(&ct->g2h_fence_wq);
829
830 return 0;
831}
832
833static int parse_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
834{
835 struct xe_device *xe = ct_to_xe(ct);
836 u32 hxg, origin, type;
837 int ret;
838
839 lockdep_assert_held(&ct->lock);
840
841 hxg = msg[1];
842
843 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg);
844 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
845 drm_err(&xe->drm,
846 "G2H channel broken on read, origin=%d, reset required\n",
847 origin);
848 ct->ctbs.g2h.info.broken = true;
849
850 return -EPROTO;
851 }
852
853 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg);
854 switch (type) {
855 case GUC_HXG_TYPE_EVENT:
856 ret = parse_g2h_event(ct, msg, len);
857 break;
858 case GUC_HXG_TYPE_RESPONSE_SUCCESS:
859 case GUC_HXG_TYPE_RESPONSE_FAILURE:
860 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
861 ret = parse_g2h_response(ct, msg, len);
862 break;
863 default:
864 drm_err(&xe->drm,
865 "G2H channel broken on read, type=%d, reset required\n",
866 type);
867 ct->ctbs.g2h.info.broken = true;
868
869 ret = -EOPNOTSUPP;
870 }
871
872 return ret;
873}
874
875static int process_g2h_msg(struct xe_guc_ct *ct, u32 *msg, u32 len)
876{
877 struct xe_device *xe = ct_to_xe(ct);
878 struct xe_guc *guc = ct_to_guc(ct);
879 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]);
880 u32 *payload = msg + GUC_CTB_HXG_MSG_MIN_LEN;
881 u32 adj_len = len - GUC_CTB_HXG_MSG_MIN_LEN;
882 int ret = 0;
883
884 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]) != GUC_HXG_TYPE_EVENT)
885 return 0;
886
887 switch (action) {
888 case XE_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
889 ret = xe_guc_sched_done_handler(guc, payload, adj_len);
890 break;
891 case XE_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
892 ret = xe_guc_deregister_done_handler(guc, payload, adj_len);
893 break;
894 case XE_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
895 ret = xe_guc_exec_queue_reset_handler(guc, payload, adj_len);
896 break;
897 case XE_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
898 ret = xe_guc_exec_queue_reset_failure_handler(guc, payload,
899 adj_len);
900 break;
901 case XE_GUC_ACTION_SCHED_ENGINE_MODE_DONE:
902 /* Selftest only at the moment */
903 break;
904 case XE_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
905 case XE_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
906 /* FIXME: Handle this */
907 break;
908 case XE_GUC_ACTION_NOTIFY_MEMORY_CAT_ERROR:
909 ret = xe_guc_exec_queue_memory_cat_error_handler(guc, payload,
910 adj_len);
911 break;
912 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
913 ret = xe_guc_pagefault_handler(guc, payload, adj_len);
914 break;
915 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
916 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
917 adj_len);
918 break;
919 case XE_GUC_ACTION_ACCESS_COUNTER_NOTIFY:
920 ret = xe_guc_access_counter_notify_handler(guc, payload,
921 adj_len);
922 break;
923 default:
924 drm_err(&xe->drm, "unexpected action 0x%04x\n", action);
925 }
926
927 if (ret)
928 drm_err(&xe->drm, "action 0x%04x failed processing, ret=%d\n",
929 action, ret);
930
931 return 0;
932}
933
934static int g2h_read(struct xe_guc_ct *ct, u32 *msg, bool fast_path)
935{
936 struct xe_device *xe = ct_to_xe(ct);
937 struct guc_ctb *g2h = &ct->ctbs.g2h;
938 u32 tail, head, len;
939 s32 avail;
940 u32 action;
941
942 lockdep_assert_held(&ct->fast_lock);
943
944 if (!ct->enabled)
945 return -ENODEV;
946
947 if (g2h->info.broken)
948 return -EPIPE;
949
950 /* Calculate DW available to read */
951 tail = desc_read(xe, g2h, tail);
952 avail = tail - g2h->info.head;
953 if (unlikely(avail == 0))
954 return 0;
955
956 if (avail < 0)
957 avail += g2h->info.size;
958
959 /* Read header */
960 xe_map_memcpy_from(xe, msg, &g2h->cmds, sizeof(u32) * g2h->info.head,
961 sizeof(u32));
962 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, msg[0]) + GUC_CTB_MSG_MIN_LEN;
963 if (len > avail) {
964 drm_err(&xe->drm,
965 "G2H channel broken on read, avail=%d, len=%d, reset required\n",
966 avail, len);
967 g2h->info.broken = true;
968
969 return -EPROTO;
970 }
971
972 head = (g2h->info.head + 1) % g2h->info.size;
973 avail = len - 1;
974
975 /* Read G2H message */
976 if (avail + head > g2h->info.size) {
977 u32 avail_til_wrap = g2h->info.size - head;
978
979 xe_map_memcpy_from(xe, msg + 1,
980 &g2h->cmds, sizeof(u32) * head,
981 avail_til_wrap * sizeof(u32));
982 xe_map_memcpy_from(xe, msg + 1 + avail_til_wrap,
983 &g2h->cmds, 0,
984 (avail - avail_til_wrap) * sizeof(u32));
985 } else {
986 xe_map_memcpy_from(xe, msg + 1,
987 &g2h->cmds, sizeof(u32) * head,
988 avail * sizeof(u32));
989 }
990
991 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]);
992
993 if (fast_path) {
994 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[1]) != GUC_HXG_TYPE_EVENT)
995 return 0;
996
997 switch (action) {
998 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
999 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1000 break; /* Process these in fast-path */
1001 default:
1002 return 0;
1003 }
1004 }
1005
1006 /* Update local / descriptor header */
1007 g2h->info.head = (head + avail) % g2h->info.size;
1008 desc_write(xe, g2h, head, g2h->info.head);
1009
1010 trace_xe_guc_ctb_g2h(ct_to_gt(ct)->info.id, action, len,
1011 g2h->info.head, tail);
1012
1013 return len;
1014}
1015
1016static void g2h_fast_path(struct xe_guc_ct *ct, u32 *msg, u32 len)
1017{
1018 struct xe_device *xe = ct_to_xe(ct);
1019 struct xe_guc *guc = ct_to_guc(ct);
1020 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[1]);
1021 u32 *payload = msg + GUC_CTB_HXG_MSG_MIN_LEN;
1022 u32 adj_len = len - GUC_CTB_HXG_MSG_MIN_LEN;
1023 int ret = 0;
1024
1025 switch (action) {
1026 case XE_GUC_ACTION_REPORT_PAGE_FAULT_REQ_DESC:
1027 ret = xe_guc_pagefault_handler(guc, payload, adj_len);
1028 break;
1029 case XE_GUC_ACTION_TLB_INVALIDATION_DONE:
1030 __g2h_release_space(ct, len);
1031 ret = xe_guc_tlb_invalidation_done_handler(guc, payload,
1032 adj_len);
1033 break;
1034 default:
1035 drm_warn(&xe->drm, "NOT_POSSIBLE");
1036 }
1037
1038 if (ret)
1039 drm_err(&xe->drm, "action 0x%04x failed processing, ret=%d\n",
1040 action, ret);
1041}
1042
1043/**
1044 * xe_guc_ct_fast_path - process critical G2H in the IRQ handler
1045 * @ct: GuC CT object
1046 *
1047 * Anything related to page faults is critical for performance, process these
1048 * critical G2H in the IRQ. This is safe as these handlers either just wake up
1049 * waiters or queue another worker.
1050 */
1051void xe_guc_ct_fast_path(struct xe_guc_ct *ct)
1052{
1053 struct xe_device *xe = ct_to_xe(ct);
1054 bool ongoing;
1055 int len;
1056
1057 ongoing = xe_device_mem_access_get_if_ongoing(ct_to_xe(ct));
1058 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1059 return;
1060
1061 spin_lock(&ct->fast_lock);
1062 do {
1063 len = g2h_read(ct, ct->fast_msg, true);
1064 if (len > 0)
1065 g2h_fast_path(ct, ct->fast_msg, len);
1066 } while (len > 0);
1067 spin_unlock(&ct->fast_lock);
1068
1069 if (ongoing)
1070 xe_device_mem_access_put(xe);
1071}
1072
1073/* Returns less than zero on error, 0 on done, 1 on more available */
1074static int dequeue_one_g2h(struct xe_guc_ct *ct)
1075{
1076 int len;
1077 int ret;
1078
1079 lockdep_assert_held(&ct->lock);
1080
1081 spin_lock_irq(&ct->fast_lock);
1082 len = g2h_read(ct, ct->msg, false);
1083 spin_unlock_irq(&ct->fast_lock);
1084 if (len <= 0)
1085 return len;
1086
1087 ret = parse_g2h_msg(ct, ct->msg, len);
1088 if (unlikely(ret < 0))
1089 return ret;
1090
1091 ret = process_g2h_msg(ct, ct->msg, len);
1092 if (unlikely(ret < 0))
1093 return ret;
1094
1095 return 1;
1096}
1097
1098static void g2h_worker_func(struct work_struct *w)
1099{
1100 struct xe_guc_ct *ct = container_of(w, struct xe_guc_ct, g2h_worker);
1101 bool ongoing;
1102 int ret;
1103
1104 /*
1105 * Normal users must always hold mem_access.ref around CT calls. However
1106 * during the runtime pm callbacks we rely on CT to talk to the GuC, but
1107 * at this stage we can't rely on mem_access.ref and even the
1108 * callback_task will be different than current. For such cases we just
1109 * need to ensure we always process the responses from any blocking
1110 * ct_send requests or where we otherwise expect some response when
1111 * initiated from those callbacks (which will need to wait for the below
1112 * dequeue_one_g2h()). The dequeue_one_g2h() will gracefully fail if
1113 * the device has suspended to the point that the CT communication has
1114 * been disabled.
1115 *
1116 * If we are inside the runtime pm callback, we can be the only task
1117 * still issuing CT requests (since that requires having the
1118 * mem_access.ref). It seems like it might in theory be possible to
1119 * receive unsolicited events from the GuC just as we are
1120 * suspending-resuming, but those will currently anyway be lost when
1121 * eventually exiting from suspend, hence no need to wake up the device
1122 * here. If we ever need something stronger than get_if_ongoing() then
1123 * we need to be careful with blocking the pm callbacks from getting CT
1124 * responses, if the worker here is blocked on those callbacks
1125 * completing, creating a deadlock.
1126 */
1127 ongoing = xe_device_mem_access_get_if_ongoing(ct_to_xe(ct));
1128 if (!ongoing && xe_pm_read_callback_task(ct_to_xe(ct)) == NULL)
1129 return;
1130
1131 do {
1132 mutex_lock(&ct->lock);
1133 ret = dequeue_one_g2h(ct);
1134 mutex_unlock(&ct->lock);
1135
1136 if (unlikely(ret == -EPROTO || ret == -EOPNOTSUPP)) {
1137 struct drm_device *drm = &ct_to_xe(ct)->drm;
1138 struct drm_printer p = drm_info_printer(drm->dev);
1139
1140 xe_guc_ct_print(ct, &p, false);
1141 kick_reset(ct);
1142 }
1143 } while (ret == 1);
1144
1145 if (ongoing)
1146 xe_device_mem_access_put(ct_to_xe(ct));
1147}
1148
1149static void guc_ctb_snapshot_capture(struct xe_device *xe, struct guc_ctb *ctb,
1150 struct guc_ctb_snapshot *snapshot,
1151 bool atomic)
1152{
1153 u32 head, tail;
1154
1155 xe_map_memcpy_from(xe, &snapshot->desc, &ctb->desc, 0,
1156 sizeof(struct guc_ct_buffer_desc));
1157 memcpy(&snapshot->info, &ctb->info, sizeof(struct guc_ctb_info));
1158
1159 snapshot->cmds = kmalloc_array(ctb->info.size, sizeof(u32),
1160 atomic ? GFP_ATOMIC : GFP_KERNEL);
1161
1162 if (!snapshot->cmds) {
1163 drm_err(&xe->drm, "Skipping CTB commands snapshot. Only CTB info will be available.\n");
1164 return;
1165 }
1166
1167 head = snapshot->desc.head;
1168 tail = snapshot->desc.tail;
1169
1170 if (head != tail) {
1171 struct iosys_map map =
1172 IOSYS_MAP_INIT_OFFSET(&ctb->cmds, head * sizeof(u32));
1173
1174 while (head != tail) {
1175 snapshot->cmds[head] = xe_map_rd(xe, &map, 0, u32);
1176 ++head;
1177 if (head == ctb->info.size) {
1178 head = 0;
1179 map = ctb->cmds;
1180 } else {
1181 iosys_map_incr(&map, sizeof(u32));
1182 }
1183 }
1184 }
1185}
1186
1187static void guc_ctb_snapshot_print(struct guc_ctb_snapshot *snapshot,
1188 struct drm_printer *p)
1189{
1190 u32 head, tail;
1191
1192 drm_printf(p, "\tsize: %d\n", snapshot->info.size);
1193 drm_printf(p, "\tresv_space: %d\n", snapshot->info.resv_space);
1194 drm_printf(p, "\thead: %d\n", snapshot->info.head);
1195 drm_printf(p, "\ttail: %d\n", snapshot->info.tail);
1196 drm_printf(p, "\tspace: %d\n", snapshot->info.space);
1197 drm_printf(p, "\tbroken: %d\n", snapshot->info.broken);
1198 drm_printf(p, "\thead (memory): %d\n", snapshot->desc.head);
1199 drm_printf(p, "\ttail (memory): %d\n", snapshot->desc.tail);
1200 drm_printf(p, "\tstatus (memory): 0x%x\n", snapshot->desc.status);
1201
1202 if (!snapshot->cmds)
1203 return;
1204
1205 head = snapshot->desc.head;
1206 tail = snapshot->desc.tail;
1207
1208 while (head != tail) {
1209 drm_printf(p, "\tcmd[%d]: 0x%08x\n", head,
1210 snapshot->cmds[head]);
1211 ++head;
1212 if (head == snapshot->info.size)
1213 head = 0;
1214 }
1215}
1216
1217static void guc_ctb_snapshot_free(struct guc_ctb_snapshot *snapshot)
1218{
1219 kfree(snapshot->cmds);
1220}
1221
1222/**
1223 * xe_guc_ct_snapshot_capture - Take a quick snapshot of the CT state.
1224 * @ct: GuC CT object.
1225 * @atomic: Boolean to indicate if this is called from atomic context like
1226 * reset or CTB handler or from some regular path like debugfs.
1227 *
1228 * This can be printed out in a later stage like during dev_coredump
1229 * analysis.
1230 *
1231 * Returns: a GuC CT snapshot object that must be freed by the caller
1232 * by using `xe_guc_ct_snapshot_free`.
1233 */
1234struct xe_guc_ct_snapshot *xe_guc_ct_snapshot_capture(struct xe_guc_ct *ct,
1235 bool atomic)
1236{
1237 struct xe_device *xe = ct_to_xe(ct);
1238 struct xe_guc_ct_snapshot *snapshot;
1239
1240 snapshot = kzalloc(sizeof(*snapshot),
1241 atomic ? GFP_ATOMIC : GFP_KERNEL);
1242
1243 if (!snapshot) {
1244 drm_err(&xe->drm, "Skipping CTB snapshot entirely.\n");
1245 return NULL;
1246 }
1247
1248 if (ct->enabled) {
1249 snapshot->ct_enabled = true;
1250 snapshot->g2h_outstanding = READ_ONCE(ct->g2h_outstanding);
1251 guc_ctb_snapshot_capture(xe, &ct->ctbs.h2g,
1252 &snapshot->h2g, atomic);
1253 guc_ctb_snapshot_capture(xe, &ct->ctbs.g2h,
1254 &snapshot->g2h, atomic);
1255 }
1256
1257 return snapshot;
1258}
1259
1260/**
1261 * xe_guc_ct_snapshot_print - Print out a given GuC CT snapshot.
1262 * @snapshot: GuC CT snapshot object.
1263 * @p: drm_printer where it will be printed out.
1264 *
1265 * This function prints out a given GuC CT snapshot object.
1266 */
1267void xe_guc_ct_snapshot_print(struct xe_guc_ct_snapshot *snapshot,
1268 struct drm_printer *p)
1269{
1270 if (!snapshot)
1271 return;
1272
1273 if (snapshot->ct_enabled) {
1274 drm_puts(p, "\nH2G CTB (all sizes in DW):\n");
1275 guc_ctb_snapshot_print(&snapshot->h2g, p);
1276
1277 drm_puts(p, "\nG2H CTB (all sizes in DW):\n");
1278 guc_ctb_snapshot_print(&snapshot->g2h, p);
1279
1280 drm_printf(p, "\tg2h outstanding: %d\n",
1281 snapshot->g2h_outstanding);
1282 } else {
1283 drm_puts(p, "\nCT disabled\n");
1284 }
1285}
1286
1287/**
1288 * xe_guc_ct_snapshot_free - Free all allocated objects for a given snapshot.
1289 * @snapshot: GuC CT snapshot object.
1290 *
1291 * This function free all the memory that needed to be allocated at capture
1292 * time.
1293 */
1294void xe_guc_ct_snapshot_free(struct xe_guc_ct_snapshot *snapshot)
1295{
1296 if (!snapshot)
1297 return;
1298
1299 guc_ctb_snapshot_free(&snapshot->h2g);
1300 guc_ctb_snapshot_free(&snapshot->g2h);
1301 kfree(snapshot);
1302}
1303
1304/**
1305 * xe_guc_ct_print - GuC CT Print.
1306 * @ct: GuC CT.
1307 * @p: drm_printer where it will be printed out.
1308 * @atomic: Boolean to indicate if this is called from atomic context like
1309 * reset or CTB handler or from some regular path like debugfs.
1310 *
1311 * This function quickly capture a snapshot and immediately print it out.
1312 */
1313void xe_guc_ct_print(struct xe_guc_ct *ct, struct drm_printer *p, bool atomic)
1314{
1315 struct xe_guc_ct_snapshot *snapshot;
1316
1317 snapshot = xe_guc_ct_snapshot_capture(ct, atomic);
1318 xe_guc_ct_snapshot_print(snapshot, p);
1319 xe_guc_ct_snapshot_free(snapshot);
1320}