Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2016-2019 Intel Corporation
4 */
5
6#include <linux/circ_buf.h>
7#include <linux/ktime.h>
8#include <linux/time64.h>
9#include <linux/string_helpers.h>
10#include <linux/timekeeping.h>
11
12#include "i915_drv.h"
13#include "intel_guc_ct.h"
14#include "intel_guc_print.h"
15
16#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
17enum {
18 CT_DEAD_ALIVE = 0,
19 CT_DEAD_SETUP,
20 CT_DEAD_WRITE,
21 CT_DEAD_DEADLOCK,
22 CT_DEAD_H2G_HAS_ROOM,
23 CT_DEAD_READ,
24 CT_DEAD_PROCESS_FAILED,
25};
26
27static void ct_dead_ct_worker_func(struct work_struct *w);
28
29#define CT_DEAD(ct, reason) \
30 do { \
31 if (!(ct)->dead_ct_reported) { \
32 (ct)->dead_ct_reason |= 1 << CT_DEAD_##reason; \
33 queue_work(system_unbound_wq, &(ct)->dead_ct_worker); \
34 } \
35 } while (0)
36#else
37#define CT_DEAD(ct, reason) do { } while (0)
38#endif
39
40static inline struct intel_guc *ct_to_guc(struct intel_guc_ct *ct)
41{
42 return container_of(ct, struct intel_guc, ct);
43}
44
45#define CT_ERROR(_ct, _fmt, ...) \
46 guc_err(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
47#ifdef CONFIG_DRM_I915_DEBUG_GUC
48#define CT_DEBUG(_ct, _fmt, ...) \
49 guc_dbg(ct_to_guc(_ct), "CT: " _fmt, ##__VA_ARGS__)
50#else
51#define CT_DEBUG(...) do { } while (0)
52#endif
53#define CT_PROBE_ERROR(_ct, _fmt, ...) \
54 guc_probe_error(ct_to_guc(ct), "CT: " _fmt, ##__VA_ARGS__)
55
56/**
57 * DOC: CTB Blob
58 *
59 * We allocate single blob to hold both CTB descriptors and buffers:
60 *
61 * +--------+-----------------------------------------------+------+
62 * | offset | contents | size |
63 * +========+===============================================+======+
64 * | 0x0000 | H2G `CTB Descriptor`_ (send) | |
65 * +--------+-----------------------------------------------+ 4K |
66 * | 0x0800 | G2H `CTB Descriptor`_ (recv) | |
67 * +--------+-----------------------------------------------+------+
68 * | 0x1000 | H2G `CT Buffer`_ (send) | n*4K |
69 * | | | |
70 * +--------+-----------------------------------------------+------+
71 * | 0x1000 | G2H `CT Buffer`_ (recv) | m*4K |
72 * | + n*4K | | |
73 * +--------+-----------------------------------------------+------+
74 *
75 * Size of each `CT Buffer`_ must be multiple of 4K.
76 * We don't expect too many messages in flight at any time, unless we are
77 * using the GuC submission. In that case each request requires a minimum
78 * 2 dwords which gives us a maximum 256 queue'd requests. Hopefully this
79 * enough space to avoid backpressure on the driver. We increase the size
80 * of the receive buffer (relative to the send) to ensure a G2H response
81 * CTB has a landing spot.
82 */
83#define CTB_DESC_SIZE ALIGN(sizeof(struct guc_ct_buffer_desc), SZ_2K)
84#define CTB_H2G_BUFFER_SIZE (SZ_4K)
85#define CTB_G2H_BUFFER_SIZE (4 * CTB_H2G_BUFFER_SIZE)
86#define G2H_ROOM_BUFFER_SIZE (CTB_G2H_BUFFER_SIZE / 4)
87
88struct ct_request {
89 struct list_head link;
90 u32 fence;
91 u32 status;
92 u32 response_len;
93 u32 *response_buf;
94};
95
96struct ct_incoming_msg {
97 struct list_head link;
98 u32 size;
99 u32 msg[] __counted_by(size);
100};
101
102enum { CTB_SEND = 0, CTB_RECV = 1 };
103
104enum { CTB_OWNER_HOST = 0 };
105
106/*
107 * Some H2G commands involve a synchronous response that the driver needs
108 * to wait for. In such cases, a timeout is required to prevent the driver
109 * from waiting forever in the case of an error (either no error response
110 * is defined in the protocol or something has died and requires a reset).
111 * The specific command may be defined as having a time bound response but
112 * the CT is a queue and that time guarantee only starts from the point
113 * when the command reaches the head of the queue and is processed by GuC.
114 *
115 * Ideally there would be a helper to report the progress of a given
116 * command through the CT. However, that would require a significant
117 * amount of work in the CT layer. In the meantime, provide a reasonable
118 * estimation of the worst case latency it should take for the entire
119 * queue to drain. And therefore, how long a caller should wait before
120 * giving up on their request. The current estimate is based on empirical
121 * measurement of a test that fills the buffer with context creation and
122 * destruction requests as they seem to be the slowest operation.
123 */
124long intel_guc_ct_max_queue_time_jiffies(void)
125{
126 /*
127 * A 4KB buffer full of context destroy commands takes a little
128 * over a second to process so bump that to 2s to be super safe.
129 */
130 return (CTB_H2G_BUFFER_SIZE * HZ) / SZ_2K;
131}
132
133static void ct_receive_tasklet_func(struct tasklet_struct *t);
134static void ct_incoming_request_worker_func(struct work_struct *w);
135
136/**
137 * intel_guc_ct_init_early - Initialize CT state without requiring device access
138 * @ct: pointer to CT struct
139 */
140void intel_guc_ct_init_early(struct intel_guc_ct *ct)
141{
142 spin_lock_init(&ct->ctbs.send.lock);
143 spin_lock_init(&ct->ctbs.recv.lock);
144 spin_lock_init(&ct->requests.lock);
145 INIT_LIST_HEAD(&ct->requests.pending);
146 INIT_LIST_HEAD(&ct->requests.incoming);
147#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
148 INIT_WORK(&ct->dead_ct_worker, ct_dead_ct_worker_func);
149#endif
150 INIT_WORK(&ct->requests.worker, ct_incoming_request_worker_func);
151 tasklet_setup(&ct->receive_tasklet, ct_receive_tasklet_func);
152 init_waitqueue_head(&ct->wq);
153}
154
155static void guc_ct_buffer_desc_init(struct guc_ct_buffer_desc *desc)
156{
157 memset(desc, 0, sizeof(*desc));
158}
159
160static void guc_ct_buffer_reset(struct intel_guc_ct_buffer *ctb)
161{
162 u32 space;
163
164 ctb->broken = false;
165 ctb->tail = 0;
166 ctb->head = 0;
167 space = CIRC_SPACE(ctb->tail, ctb->head, ctb->size) - ctb->resv_space;
168 atomic_set(&ctb->space, space);
169
170 guc_ct_buffer_desc_init(ctb->desc);
171}
172
173static void guc_ct_buffer_init(struct intel_guc_ct_buffer *ctb,
174 struct guc_ct_buffer_desc *desc,
175 u32 *cmds, u32 size_in_bytes, u32 resv_space)
176{
177 GEM_BUG_ON(size_in_bytes % 4);
178
179 ctb->desc = desc;
180 ctb->cmds = cmds;
181 ctb->size = size_in_bytes / 4;
182 ctb->resv_space = resv_space / 4;
183
184 guc_ct_buffer_reset(ctb);
185}
186
187static int guc_action_control_ctb(struct intel_guc *guc, u32 control)
188{
189 u32 request[HOST2GUC_CONTROL_CTB_REQUEST_MSG_LEN] = {
190 FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
191 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
192 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, GUC_ACTION_HOST2GUC_CONTROL_CTB),
193 FIELD_PREP(HOST2GUC_CONTROL_CTB_REQUEST_MSG_1_CONTROL, control),
194 };
195 int ret;
196
197 GEM_BUG_ON(control != GUC_CTB_CONTROL_DISABLE && control != GUC_CTB_CONTROL_ENABLE);
198
199 /* CT control must go over MMIO */
200 ret = intel_guc_send_mmio(guc, request, ARRAY_SIZE(request), NULL, 0);
201
202 return ret > 0 ? -EPROTO : ret;
203}
204
205static int ct_control_enable(struct intel_guc_ct *ct, bool enable)
206{
207 int err;
208
209 err = guc_action_control_ctb(ct_to_guc(ct), enable ?
210 GUC_CTB_CONTROL_ENABLE : GUC_CTB_CONTROL_DISABLE);
211 if (unlikely(err))
212 CT_PROBE_ERROR(ct, "Failed to control/%s CTB (%pe)\n",
213 str_enable_disable(enable), ERR_PTR(err));
214
215 return err;
216}
217
218static int ct_register_buffer(struct intel_guc_ct *ct, bool send,
219 u32 desc_addr, u32 buff_addr, u32 size)
220{
221 int err;
222
223 err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
224 GUC_KLV_SELF_CFG_H2G_CTB_DESCRIPTOR_ADDR_KEY :
225 GUC_KLV_SELF_CFG_G2H_CTB_DESCRIPTOR_ADDR_KEY,
226 desc_addr);
227 if (unlikely(err))
228 goto failed;
229
230 err = intel_guc_self_cfg64(ct_to_guc(ct), send ?
231 GUC_KLV_SELF_CFG_H2G_CTB_ADDR_KEY :
232 GUC_KLV_SELF_CFG_G2H_CTB_ADDR_KEY,
233 buff_addr);
234 if (unlikely(err))
235 goto failed;
236
237 err = intel_guc_self_cfg32(ct_to_guc(ct), send ?
238 GUC_KLV_SELF_CFG_H2G_CTB_SIZE_KEY :
239 GUC_KLV_SELF_CFG_G2H_CTB_SIZE_KEY,
240 size);
241 if (unlikely(err))
242failed:
243 CT_PROBE_ERROR(ct, "Failed to register %s buffer (%pe)\n",
244 send ? "SEND" : "RECV", ERR_PTR(err));
245
246 return err;
247}
248
249/**
250 * intel_guc_ct_init - Init buffer-based communication
251 * @ct: pointer to CT struct
252 *
253 * Allocate memory required for buffer-based communication.
254 *
255 * Return: 0 on success, a negative errno code on failure.
256 */
257int intel_guc_ct_init(struct intel_guc_ct *ct)
258{
259 struct intel_guc *guc = ct_to_guc(ct);
260 struct guc_ct_buffer_desc *desc;
261 u32 blob_size;
262 u32 cmds_size;
263 u32 resv_space;
264 void *blob;
265 u32 *cmds;
266 int err;
267
268 err = i915_inject_probe_error(guc_to_i915(guc), -ENXIO);
269 if (err)
270 return err;
271
272 GEM_BUG_ON(ct->vma);
273
274 blob_size = 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE + CTB_G2H_BUFFER_SIZE;
275 err = intel_guc_allocate_and_map_vma(guc, blob_size, &ct->vma, &blob);
276 if (unlikely(err)) {
277 CT_PROBE_ERROR(ct, "Failed to allocate %u for CTB data (%pe)\n",
278 blob_size, ERR_PTR(err));
279 return err;
280 }
281
282 CT_DEBUG(ct, "base=%#x size=%u\n", intel_guc_ggtt_offset(guc, ct->vma), blob_size);
283
284 /* store pointers to desc and cmds for send ctb */
285 desc = blob;
286 cmds = blob + 2 * CTB_DESC_SIZE;
287 cmds_size = CTB_H2G_BUFFER_SIZE;
288 resv_space = 0;
289 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "send",
290 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
291 resv_space);
292
293 guc_ct_buffer_init(&ct->ctbs.send, desc, cmds, cmds_size, resv_space);
294
295 /* store pointers to desc and cmds for recv ctb */
296 desc = blob + CTB_DESC_SIZE;
297 cmds = blob + 2 * CTB_DESC_SIZE + CTB_H2G_BUFFER_SIZE;
298 cmds_size = CTB_G2H_BUFFER_SIZE;
299 resv_space = G2H_ROOM_BUFFER_SIZE;
300 CT_DEBUG(ct, "%s desc %#tx cmds %#tx size %u/%u\n", "recv",
301 ptrdiff(desc, blob), ptrdiff(cmds, blob), cmds_size,
302 resv_space);
303
304 guc_ct_buffer_init(&ct->ctbs.recv, desc, cmds, cmds_size, resv_space);
305
306 return 0;
307}
308
309/**
310 * intel_guc_ct_fini - Fini buffer-based communication
311 * @ct: pointer to CT struct
312 *
313 * Deallocate memory required for buffer-based communication.
314 */
315void intel_guc_ct_fini(struct intel_guc_ct *ct)
316{
317 GEM_BUG_ON(ct->enabled);
318
319 tasklet_kill(&ct->receive_tasklet);
320 i915_vma_unpin_and_release(&ct->vma, I915_VMA_RELEASE_MAP);
321 memset(ct, 0, sizeof(*ct));
322}
323
324/**
325 * intel_guc_ct_enable - Enable buffer based command transport.
326 * @ct: pointer to CT struct
327 *
328 * Return: 0 on success, a negative errno code on failure.
329 */
330int intel_guc_ct_enable(struct intel_guc_ct *ct)
331{
332 struct intel_guc *guc = ct_to_guc(ct);
333 u32 base, desc, cmds, size;
334 void *blob;
335 int err;
336
337 GEM_BUG_ON(ct->enabled);
338
339 /* vma should be already allocated and map'ed */
340 GEM_BUG_ON(!ct->vma);
341 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(ct->vma->obj));
342 base = intel_guc_ggtt_offset(guc, ct->vma);
343
344 /* blob should start with send descriptor */
345 blob = __px_vaddr(ct->vma->obj);
346 GEM_BUG_ON(blob != ct->ctbs.send.desc);
347
348 /* (re)initialize descriptors */
349 guc_ct_buffer_reset(&ct->ctbs.send);
350 guc_ct_buffer_reset(&ct->ctbs.recv);
351
352 /*
353 * Register both CT buffers starting with RECV buffer.
354 * Descriptors are in first half of the blob.
355 */
356 desc = base + ptrdiff(ct->ctbs.recv.desc, blob);
357 cmds = base + ptrdiff(ct->ctbs.recv.cmds, blob);
358 size = ct->ctbs.recv.size * 4;
359 err = ct_register_buffer(ct, false, desc, cmds, size);
360 if (unlikely(err))
361 goto err_out;
362
363 desc = base + ptrdiff(ct->ctbs.send.desc, blob);
364 cmds = base + ptrdiff(ct->ctbs.send.cmds, blob);
365 size = ct->ctbs.send.size * 4;
366 err = ct_register_buffer(ct, true, desc, cmds, size);
367 if (unlikely(err))
368 goto err_out;
369
370 err = ct_control_enable(ct, true);
371 if (unlikely(err))
372 goto err_out;
373
374 ct->enabled = true;
375 ct->stall_time = KTIME_MAX;
376#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
377 ct->dead_ct_reported = false;
378 ct->dead_ct_reason = CT_DEAD_ALIVE;
379#endif
380
381 return 0;
382
383err_out:
384 CT_PROBE_ERROR(ct, "Failed to enable CTB (%pe)\n", ERR_PTR(err));
385 CT_DEAD(ct, SETUP);
386 return err;
387}
388
389/**
390 * intel_guc_ct_disable - Disable buffer based command transport.
391 * @ct: pointer to CT struct
392 */
393void intel_guc_ct_disable(struct intel_guc_ct *ct)
394{
395 struct intel_guc *guc = ct_to_guc(ct);
396
397 GEM_BUG_ON(!ct->enabled);
398
399 ct->enabled = false;
400
401 if (intel_guc_is_fw_running(guc)) {
402 ct_control_enable(ct, false);
403 }
404}
405
406#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
407static void ct_track_lost_and_found(struct intel_guc_ct *ct, u32 fence, u32 action)
408{
409 unsigned int lost = fence % ARRAY_SIZE(ct->requests.lost_and_found);
410#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
411 unsigned long entries[SZ_32];
412 unsigned int n;
413
414 n = stack_trace_save(entries, ARRAY_SIZE(entries), 1);
415
416 /* May be called under spinlock, so avoid sleeping */
417 ct->requests.lost_and_found[lost].stack = stack_depot_save(entries, n, GFP_NOWAIT);
418#endif
419 ct->requests.lost_and_found[lost].fence = fence;
420 ct->requests.lost_and_found[lost].action = action;
421}
422#endif
423
424static u32 ct_get_next_fence(struct intel_guc_ct *ct)
425{
426 /* For now it's trivial */
427 return ++ct->requests.last_fence;
428}
429
430static int ct_write(struct intel_guc_ct *ct,
431 const u32 *action,
432 u32 len /* in dwords */,
433 u32 fence, u32 flags)
434{
435 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
436 struct guc_ct_buffer_desc *desc = ctb->desc;
437 u32 tail = ctb->tail;
438 u32 size = ctb->size;
439 u32 header;
440 u32 hxg;
441 u32 type;
442 u32 *cmds = ctb->cmds;
443 unsigned int i;
444
445 if (unlikely(desc->status))
446 goto corrupted;
447
448 GEM_BUG_ON(tail > size);
449
450#ifdef CONFIG_DRM_I915_DEBUG_GUC
451 if (unlikely(tail != READ_ONCE(desc->tail))) {
452 CT_ERROR(ct, "Tail was modified %u != %u\n",
453 desc->tail, tail);
454 desc->status |= GUC_CTB_STATUS_MISMATCH;
455 goto corrupted;
456 }
457 if (unlikely(READ_ONCE(desc->head) >= size)) {
458 CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
459 desc->head, size);
460 desc->status |= GUC_CTB_STATUS_OVERFLOW;
461 goto corrupted;
462 }
463#endif
464
465 /*
466 * dw0: CT header (including fence)
467 * dw1: HXG header (including action code)
468 * dw2+: action data
469 */
470 header = FIELD_PREP(GUC_CTB_MSG_0_FORMAT, GUC_CTB_FORMAT_HXG) |
471 FIELD_PREP(GUC_CTB_MSG_0_NUM_DWORDS, len) |
472 FIELD_PREP(GUC_CTB_MSG_0_FENCE, fence);
473
474 type = (flags & INTEL_GUC_CT_SEND_NB) ? GUC_HXG_TYPE_FAST_REQUEST :
475 GUC_HXG_TYPE_REQUEST;
476 hxg = FIELD_PREP(GUC_HXG_MSG_0_TYPE, type) |
477 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION |
478 GUC_HXG_REQUEST_MSG_0_DATA0, action[0]);
479
480 CT_DEBUG(ct, "writing (tail %u) %*ph %*ph %*ph\n",
481 tail, 4, &header, 4, &hxg, 4 * (len - 1), &action[1]);
482
483 cmds[tail] = header;
484 tail = (tail + 1) % size;
485
486 cmds[tail] = hxg;
487 tail = (tail + 1) % size;
488
489 for (i = 1; i < len; i++) {
490 cmds[tail] = action[i];
491 tail = (tail + 1) % size;
492 }
493 GEM_BUG_ON(tail > size);
494
495#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
496 ct_track_lost_and_found(ct, fence,
497 FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, action[0]));
498#endif
499
500 /*
501 * make sure H2G buffer update and LRC tail update (if this triggering a
502 * submission) are visible before updating the descriptor tail
503 */
504 intel_guc_write_barrier(ct_to_guc(ct));
505
506 /* update local copies */
507 ctb->tail = tail;
508 GEM_BUG_ON(atomic_read(&ctb->space) < len + GUC_CTB_HDR_LEN);
509 atomic_sub(len + GUC_CTB_HDR_LEN, &ctb->space);
510
511 /* now update descriptor */
512 WRITE_ONCE(desc->tail, tail);
513
514 return 0;
515
516corrupted:
517 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
518 desc->head, desc->tail, desc->status);
519 CT_DEAD(ct, WRITE);
520 ctb->broken = true;
521 return -EPIPE;
522}
523
524/**
525 * wait_for_ct_request_update - Wait for CT request state update.
526 * @ct: pointer to CT
527 * @req: pointer to pending request
528 * @status: placeholder for status
529 *
530 * For each sent request, GuC shall send back CT response message.
531 * Our message handler will update status of tracked request once
532 * response message with given fence is received. Wait here and
533 * check for valid response status value.
534 *
535 * Return:
536 * * 0 response received (status is valid)
537 * * -ETIMEDOUT no response within hardcoded timeout
538 */
539static int wait_for_ct_request_update(struct intel_guc_ct *ct, struct ct_request *req, u32 *status)
540{
541 int err;
542 bool ct_enabled;
543
544 /*
545 * Fast commands should complete in less than 10us, so sample quickly
546 * up to that length of time, then switch to a slower sleep-wait loop.
547 * No GuC command should ever take longer than 10ms but many GuC
548 * commands can be inflight at time, so use a 1s timeout on the slower
549 * sleep-wait loop.
550 */
551#define GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS 10
552#define GUC_CTB_RESPONSE_TIMEOUT_LONG_MS 1000
553#define done \
554 (!(ct_enabled = intel_guc_ct_enabled(ct)) || \
555 FIELD_GET(GUC_HXG_MSG_0_ORIGIN, READ_ONCE(req->status)) == \
556 GUC_HXG_ORIGIN_GUC)
557 err = wait_for_us(done, GUC_CTB_RESPONSE_TIMEOUT_SHORT_MS);
558 if (err)
559 err = wait_for(done, GUC_CTB_RESPONSE_TIMEOUT_LONG_MS);
560#undef done
561 if (!ct_enabled)
562 err = -ENODEV;
563
564 *status = req->status;
565 return err;
566}
567
568#define GUC_CTB_TIMEOUT_MS 1500
569static inline bool ct_deadlocked(struct intel_guc_ct *ct)
570{
571 long timeout = GUC_CTB_TIMEOUT_MS;
572 bool ret = ktime_ms_delta(ktime_get(), ct->stall_time) > timeout;
573
574 if (unlikely(ret)) {
575 struct guc_ct_buffer_desc *send = ct->ctbs.send.desc;
576 struct guc_ct_buffer_desc *recv = ct->ctbs.send.desc;
577
578 CT_ERROR(ct, "Communication stalled for %lld ms, desc status=%#x,%#x\n",
579 ktime_ms_delta(ktime_get(), ct->stall_time),
580 send->status, recv->status);
581 CT_ERROR(ct, "H2G Space: %u (Bytes)\n",
582 atomic_read(&ct->ctbs.send.space) * 4);
583 CT_ERROR(ct, "Head: %u (Dwords)\n", ct->ctbs.send.desc->head);
584 CT_ERROR(ct, "Tail: %u (Dwords)\n", ct->ctbs.send.desc->tail);
585 CT_ERROR(ct, "G2H Space: %u (Bytes)\n",
586 atomic_read(&ct->ctbs.recv.space) * 4);
587 CT_ERROR(ct, "Head: %u\n (Dwords)", ct->ctbs.recv.desc->head);
588 CT_ERROR(ct, "Tail: %u\n (Dwords)", ct->ctbs.recv.desc->tail);
589
590 CT_DEAD(ct, DEADLOCK);
591 ct->ctbs.send.broken = true;
592 }
593
594 return ret;
595}
596
597static inline bool g2h_has_room(struct intel_guc_ct *ct, u32 g2h_len_dw)
598{
599 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
600
601 /*
602 * We leave a certain amount of space in the G2H CTB buffer for
603 * unexpected G2H CTBs (e.g. logging, engine hang, etc...)
604 */
605 return !g2h_len_dw || atomic_read(&ctb->space) >= g2h_len_dw;
606}
607
608static inline void g2h_reserve_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
609{
610 lockdep_assert_held(&ct->ctbs.send.lock);
611
612 GEM_BUG_ON(!g2h_has_room(ct, g2h_len_dw));
613
614 if (g2h_len_dw)
615 atomic_sub(g2h_len_dw, &ct->ctbs.recv.space);
616}
617
618static inline void g2h_release_space(struct intel_guc_ct *ct, u32 g2h_len_dw)
619{
620 atomic_add(g2h_len_dw, &ct->ctbs.recv.space);
621}
622
623static inline bool h2g_has_room(struct intel_guc_ct *ct, u32 len_dw)
624{
625 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
626 struct guc_ct_buffer_desc *desc = ctb->desc;
627 u32 head;
628 u32 space;
629
630 if (atomic_read(&ctb->space) >= len_dw)
631 return true;
632
633 head = READ_ONCE(desc->head);
634 if (unlikely(head > ctb->size)) {
635 CT_ERROR(ct, "Invalid head offset %u >= %u)\n",
636 head, ctb->size);
637 desc->status |= GUC_CTB_STATUS_OVERFLOW;
638 ctb->broken = true;
639 CT_DEAD(ct, H2G_HAS_ROOM);
640 return false;
641 }
642
643 space = CIRC_SPACE(ctb->tail, head, ctb->size);
644 atomic_set(&ctb->space, space);
645
646 return space >= len_dw;
647}
648
649static int has_room_nb(struct intel_guc_ct *ct, u32 h2g_dw, u32 g2h_dw)
650{
651 bool h2g = h2g_has_room(ct, h2g_dw);
652 bool g2h = g2h_has_room(ct, g2h_dw);
653
654 lockdep_assert_held(&ct->ctbs.send.lock);
655
656 if (unlikely(!h2g || !g2h)) {
657 if (ct->stall_time == KTIME_MAX)
658 ct->stall_time = ktime_get();
659
660 /* Be paranoid and kick G2H tasklet to free credits */
661 if (!g2h)
662 tasklet_hi_schedule(&ct->receive_tasklet);
663
664 if (unlikely(ct_deadlocked(ct)))
665 return -EPIPE;
666 else
667 return -EBUSY;
668 }
669
670 ct->stall_time = KTIME_MAX;
671 return 0;
672}
673
674#define G2H_LEN_DW(f) ({ \
675 typeof(f) f_ = (f); \
676 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) ? \
677 FIELD_GET(INTEL_GUC_CT_SEND_G2H_DW_MASK, f_) + \
678 GUC_CTB_HXG_MSG_MIN_LEN : 0; \
679})
680static int ct_send_nb(struct intel_guc_ct *ct,
681 const u32 *action,
682 u32 len,
683 u32 flags)
684{
685 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
686 unsigned long spin_flags;
687 u32 g2h_len_dw = G2H_LEN_DW(flags);
688 u32 fence;
689 int ret;
690
691 spin_lock_irqsave(&ctb->lock, spin_flags);
692
693 ret = has_room_nb(ct, len + GUC_CTB_HDR_LEN, g2h_len_dw);
694 if (unlikely(ret))
695 goto out;
696
697 fence = ct_get_next_fence(ct);
698 ret = ct_write(ct, action, len, fence, flags);
699 if (unlikely(ret))
700 goto out;
701
702 g2h_reserve_space(ct, g2h_len_dw);
703 intel_guc_notify(ct_to_guc(ct));
704
705out:
706 spin_unlock_irqrestore(&ctb->lock, spin_flags);
707
708 return ret;
709}
710
711static int ct_send(struct intel_guc_ct *ct,
712 const u32 *action,
713 u32 len,
714 u32 *response_buf,
715 u32 response_buf_size,
716 u32 *status)
717{
718 struct intel_guc_ct_buffer *ctb = &ct->ctbs.send;
719 struct ct_request request;
720 unsigned long flags;
721 unsigned int sleep_period_ms = 1;
722 bool send_again;
723 u32 fence;
724 int err;
725
726 GEM_BUG_ON(!ct->enabled);
727 GEM_BUG_ON(!len);
728 GEM_BUG_ON(len > GUC_CTB_HXG_MSG_MAX_LEN - GUC_CTB_HDR_LEN);
729 GEM_BUG_ON(!response_buf && response_buf_size);
730 might_sleep();
731
732resend:
733 send_again = false;
734
735 /*
736 * We use a lazy spin wait loop here as we believe that if the CT
737 * buffers are sized correctly the flow control condition should be
738 * rare. Reserving the maximum size in the G2H credits as we don't know
739 * how big the response is going to be.
740 */
741retry:
742 spin_lock_irqsave(&ctb->lock, flags);
743 if (unlikely(!h2g_has_room(ct, len + GUC_CTB_HDR_LEN) ||
744 !g2h_has_room(ct, GUC_CTB_HXG_MSG_MAX_LEN))) {
745 if (ct->stall_time == KTIME_MAX)
746 ct->stall_time = ktime_get();
747 spin_unlock_irqrestore(&ctb->lock, flags);
748
749 if (unlikely(ct_deadlocked(ct)))
750 return -EPIPE;
751
752 if (msleep_interruptible(sleep_period_ms))
753 return -EINTR;
754 sleep_period_ms = sleep_period_ms << 1;
755
756 goto retry;
757 }
758
759 ct->stall_time = KTIME_MAX;
760
761 fence = ct_get_next_fence(ct);
762 request.fence = fence;
763 request.status = 0;
764 request.response_len = response_buf_size;
765 request.response_buf = response_buf;
766
767 spin_lock(&ct->requests.lock);
768 list_add_tail(&request.link, &ct->requests.pending);
769 spin_unlock(&ct->requests.lock);
770
771 err = ct_write(ct, action, len, fence, 0);
772 g2h_reserve_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
773
774 spin_unlock_irqrestore(&ctb->lock, flags);
775
776 if (unlikely(err))
777 goto unlink;
778
779 intel_guc_notify(ct_to_guc(ct));
780
781 err = wait_for_ct_request_update(ct, &request, status);
782 g2h_release_space(ct, GUC_CTB_HXG_MSG_MAX_LEN);
783 if (unlikely(err)) {
784 if (err == -ENODEV)
785 /* wait_for_ct_request_update returns -ENODEV on reset/suspend in progress.
786 * In this case, output is debug rather than error info
787 */
788 CT_DEBUG(ct, "Request %#x (fence %u) cancelled as CTB is disabled\n",
789 action[0], request.fence);
790 else
791 CT_ERROR(ct, "No response for request %#x (fence %u)\n",
792 action[0], request.fence);
793 goto unlink;
794 }
795
796 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) == GUC_HXG_TYPE_NO_RESPONSE_RETRY) {
797 CT_DEBUG(ct, "retrying request %#x (%u)\n", *action,
798 FIELD_GET(GUC_HXG_RETRY_MSG_0_REASON, *status));
799 send_again = true;
800 goto unlink;
801 }
802
803 if (FIELD_GET(GUC_HXG_MSG_0_TYPE, *status) != GUC_HXG_TYPE_RESPONSE_SUCCESS) {
804 err = -EIO;
805 goto unlink;
806 }
807
808 if (response_buf) {
809 /* There shall be no data in the status */
810 WARN_ON(FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, request.status));
811 /* Return actual response len */
812 err = request.response_len;
813 } else {
814 /* There shall be no response payload */
815 WARN_ON(request.response_len);
816 /* Return data decoded from the status dword */
817 err = FIELD_GET(GUC_HXG_RESPONSE_MSG_0_DATA0, *status);
818 }
819
820unlink:
821 spin_lock_irqsave(&ct->requests.lock, flags);
822 list_del(&request.link);
823 spin_unlock_irqrestore(&ct->requests.lock, flags);
824
825 if (unlikely(send_again))
826 goto resend;
827
828 return err;
829}
830
831/*
832 * Command Transport (CT) buffer based GuC send function.
833 */
834int intel_guc_ct_send(struct intel_guc_ct *ct, const u32 *action, u32 len,
835 u32 *response_buf, u32 response_buf_size, u32 flags)
836{
837 u32 status = ~0; /* undefined */
838 int ret;
839
840 if (unlikely(!ct->enabled)) {
841 struct intel_guc *guc = ct_to_guc(ct);
842 struct intel_uc *uc = container_of(guc, struct intel_uc, guc);
843
844 WARN(!uc->reset_in_progress, "Unexpected send: action=%#x\n", *action);
845 return -ENODEV;
846 }
847
848 if (unlikely(ct->ctbs.send.broken))
849 return -EPIPE;
850
851 if (flags & INTEL_GUC_CT_SEND_NB)
852 return ct_send_nb(ct, action, len, flags);
853
854 ret = ct_send(ct, action, len, response_buf, response_buf_size, &status);
855 if (unlikely(ret < 0)) {
856 if (ret != -ENODEV)
857 CT_ERROR(ct, "Sending action %#x failed (%pe) status=%#X\n",
858 action[0], ERR_PTR(ret), status);
859 } else if (unlikely(ret)) {
860 CT_DEBUG(ct, "send action %#x returned %d (%#x)\n",
861 action[0], ret, ret);
862 }
863
864 return ret;
865}
866
867static struct ct_incoming_msg *ct_alloc_msg(u32 num_dwords)
868{
869 struct ct_incoming_msg *msg;
870
871 msg = kmalloc(struct_size(msg, msg, num_dwords), GFP_ATOMIC);
872 if (msg)
873 msg->size = num_dwords;
874 return msg;
875}
876
877static void ct_free_msg(struct ct_incoming_msg *msg)
878{
879 kfree(msg);
880}
881
882/*
883 * Return: number available remaining dwords to read (0 if empty)
884 * or a negative error code on failure
885 */
886static int ct_read(struct intel_guc_ct *ct, struct ct_incoming_msg **msg)
887{
888 struct intel_guc_ct_buffer *ctb = &ct->ctbs.recv;
889 struct guc_ct_buffer_desc *desc = ctb->desc;
890 u32 head = ctb->head;
891 u32 tail = READ_ONCE(desc->tail);
892 u32 size = ctb->size;
893 u32 *cmds = ctb->cmds;
894 s32 available;
895 unsigned int len;
896 unsigned int i;
897 u32 header;
898
899 if (unlikely(ctb->broken))
900 return -EPIPE;
901
902 if (unlikely(desc->status)) {
903 u32 status = desc->status;
904
905 if (status & GUC_CTB_STATUS_UNUSED) {
906 /*
907 * Potentially valid if a CLIENT_RESET request resulted in
908 * contexts/engines being reset. But should never happen as
909 * no contexts should be active when CLIENT_RESET is sent.
910 */
911 CT_ERROR(ct, "Unexpected G2H after GuC has stopped!\n");
912 status &= ~GUC_CTB_STATUS_UNUSED;
913 }
914
915 if (status)
916 goto corrupted;
917 }
918
919 GEM_BUG_ON(head > size);
920
921#ifdef CONFIG_DRM_I915_DEBUG_GUC
922 if (unlikely(head != READ_ONCE(desc->head))) {
923 CT_ERROR(ct, "Head was modified %u != %u\n",
924 desc->head, head);
925 desc->status |= GUC_CTB_STATUS_MISMATCH;
926 goto corrupted;
927 }
928#endif
929 if (unlikely(tail >= size)) {
930 CT_ERROR(ct, "Invalid tail offset %u >= %u)\n",
931 tail, size);
932 desc->status |= GUC_CTB_STATUS_OVERFLOW;
933 goto corrupted;
934 }
935
936 /* tail == head condition indicates empty */
937 available = tail - head;
938 if (unlikely(available == 0)) {
939 *msg = NULL;
940 return 0;
941 }
942
943 /* beware of buffer wrap case */
944 if (unlikely(available < 0))
945 available += size;
946 CT_DEBUG(ct, "available %d (%u:%u:%u)\n", available, head, tail, size);
947 GEM_BUG_ON(available < 0);
948
949 header = cmds[head];
950 head = (head + 1) % size;
951
952 /* message len with header */
953 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, header) + GUC_CTB_MSG_MIN_LEN;
954 if (unlikely(len > (u32)available)) {
955 CT_ERROR(ct, "Incomplete message %*ph %*ph %*ph\n",
956 4, &header,
957 4 * (head + available - 1 > size ?
958 size - head : available - 1), &cmds[head],
959 4 * (head + available - 1 > size ?
960 available - 1 - size + head : 0), &cmds[0]);
961 desc->status |= GUC_CTB_STATUS_UNDERFLOW;
962 goto corrupted;
963 }
964
965 *msg = ct_alloc_msg(len);
966 if (!*msg) {
967 CT_ERROR(ct, "No memory for message %*ph %*ph %*ph\n",
968 4, &header,
969 4 * (head + available - 1 > size ?
970 size - head : available - 1), &cmds[head],
971 4 * (head + available - 1 > size ?
972 available - 1 - size + head : 0), &cmds[0]);
973 return available;
974 }
975
976 (*msg)->msg[0] = header;
977
978 for (i = 1; i < len; i++) {
979 (*msg)->msg[i] = cmds[head];
980 head = (head + 1) % size;
981 }
982 CT_DEBUG(ct, "received %*ph\n", 4 * len, (*msg)->msg);
983
984 /* update local copies */
985 ctb->head = head;
986
987 /* now update descriptor */
988 WRITE_ONCE(desc->head, head);
989
990 intel_guc_write_barrier(ct_to_guc(ct));
991
992 return available - len;
993
994corrupted:
995 CT_ERROR(ct, "Corrupted descriptor head=%u tail=%u status=%#x\n",
996 desc->head, desc->tail, desc->status);
997 ctb->broken = true;
998 CT_DEAD(ct, READ);
999 return -EPIPE;
1000}
1001
1002#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
1003static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1004{
1005 unsigned int n;
1006 char *buf = NULL;
1007 bool found = false;
1008
1009 lockdep_assert_held(&ct->requests.lock);
1010
1011 for (n = 0; n < ARRAY_SIZE(ct->requests.lost_and_found); n++) {
1012 if (ct->requests.lost_and_found[n].fence != fence)
1013 continue;
1014 found = true;
1015
1016#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
1017 buf = kmalloc(SZ_4K, GFP_NOWAIT);
1018 if (buf && stack_depot_snprint(ct->requests.lost_and_found[n].stack,
1019 buf, SZ_4K, 0)) {
1020 CT_ERROR(ct, "Fence %u was used by action %#04x sent at\n%s",
1021 fence, ct->requests.lost_and_found[n].action, buf);
1022 break;
1023 }
1024#endif
1025 CT_ERROR(ct, "Fence %u was used by action %#04x\n",
1026 fence, ct->requests.lost_and_found[n].action);
1027 break;
1028 }
1029 kfree(buf);
1030 return found;
1031}
1032#else
1033static bool ct_check_lost_and_found(struct intel_guc_ct *ct, u32 fence)
1034{
1035 return false;
1036}
1037#endif
1038
1039static int ct_handle_response(struct intel_guc_ct *ct, struct ct_incoming_msg *response)
1040{
1041 u32 len = FIELD_GET(GUC_CTB_MSG_0_NUM_DWORDS, response->msg[0]);
1042 u32 fence = FIELD_GET(GUC_CTB_MSG_0_FENCE, response->msg[0]);
1043 const u32 *hxg = &response->msg[GUC_CTB_MSG_MIN_LEN];
1044 const u32 *data = &hxg[GUC_HXG_MSG_MIN_LEN];
1045 u32 datalen = len - GUC_HXG_MSG_MIN_LEN;
1046 struct ct_request *req;
1047 unsigned long flags;
1048 bool found = false;
1049 int err = 0;
1050
1051 GEM_BUG_ON(len < GUC_HXG_MSG_MIN_LEN);
1052 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]) != GUC_HXG_ORIGIN_GUC);
1053 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_SUCCESS &&
1054 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_NO_RESPONSE_RETRY &&
1055 FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_RESPONSE_FAILURE);
1056
1057 CT_DEBUG(ct, "response fence %u status %#x\n", fence, hxg[0]);
1058
1059 spin_lock_irqsave(&ct->requests.lock, flags);
1060 list_for_each_entry(req, &ct->requests.pending, link) {
1061 if (unlikely(fence != req->fence)) {
1062 CT_DEBUG(ct, "request %u awaits response\n",
1063 req->fence);
1064 continue;
1065 }
1066 if (unlikely(datalen > req->response_len)) {
1067 CT_ERROR(ct, "Response %u too long (datalen %u > %u)\n",
1068 req->fence, datalen, req->response_len);
1069 datalen = min(datalen, req->response_len);
1070 err = -EMSGSIZE;
1071 }
1072 if (datalen)
1073 memcpy(req->response_buf, data, 4 * datalen);
1074 req->response_len = datalen;
1075 WRITE_ONCE(req->status, hxg[0]);
1076 found = true;
1077 break;
1078 }
1079
1080#ifdef CONFIG_DRM_I915_SELFTEST
1081 if (!found && ct_to_guc(ct)->fast_response_selftest) {
1082 CT_DEBUG(ct, "Assuming unsolicited response due to FAST_REQUEST selftest\n");
1083 ct_to_guc(ct)->fast_response_selftest++;
1084 found = true;
1085 }
1086#endif
1087
1088 if (!found) {
1089 CT_ERROR(ct, "Unsolicited response message: len %u, data %#x (fence %u, last %u)\n",
1090 len, hxg[0], fence, ct->requests.last_fence);
1091 if (!ct_check_lost_and_found(ct, fence)) {
1092 list_for_each_entry(req, &ct->requests.pending, link)
1093 CT_ERROR(ct, "request %u awaits response\n",
1094 req->fence);
1095 }
1096 err = -ENOKEY;
1097 }
1098 spin_unlock_irqrestore(&ct->requests.lock, flags);
1099
1100 if (unlikely(err))
1101 return err;
1102
1103 ct_free_msg(response);
1104 return 0;
1105}
1106
1107static int ct_process_request(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1108{
1109 struct intel_guc *guc = ct_to_guc(ct);
1110 const u32 *hxg;
1111 const u32 *payload;
1112 u32 hxg_len, action, len;
1113 int ret;
1114
1115 hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1116 hxg_len = request->size - GUC_CTB_MSG_MIN_LEN;
1117 payload = &hxg[GUC_HXG_MSG_MIN_LEN];
1118 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1119 len = hxg_len - GUC_HXG_MSG_MIN_LEN;
1120
1121 CT_DEBUG(ct, "request %x %*ph\n", action, 4 * len, payload);
1122
1123 switch (action) {
1124 case INTEL_GUC_ACTION_DEFAULT:
1125 ret = intel_guc_to_host_process_recv_msg(guc, payload, len);
1126 break;
1127 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1128 ret = intel_guc_deregister_done_process_msg(guc, payload,
1129 len);
1130 break;
1131 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1132 ret = intel_guc_sched_done_process_msg(guc, payload, len);
1133 break;
1134 case INTEL_GUC_ACTION_CONTEXT_RESET_NOTIFICATION:
1135 ret = intel_guc_context_reset_process_msg(guc, payload, len);
1136 break;
1137 case INTEL_GUC_ACTION_STATE_CAPTURE_NOTIFICATION:
1138 ret = intel_guc_error_capture_process_msg(guc, payload, len);
1139 if (unlikely(ret))
1140 CT_ERROR(ct, "error capture notification failed %x %*ph\n",
1141 action, 4 * len, payload);
1142 break;
1143 case INTEL_GUC_ACTION_ENGINE_FAILURE_NOTIFICATION:
1144 ret = intel_guc_engine_failure_process_msg(guc, payload, len);
1145 break;
1146 case INTEL_GUC_ACTION_NOTIFY_FLUSH_LOG_BUFFER_TO_FILE:
1147 intel_guc_log_handle_flush_event(&guc->log);
1148 ret = 0;
1149 break;
1150 case INTEL_GUC_ACTION_NOTIFY_CRASH_DUMP_POSTED:
1151 case INTEL_GUC_ACTION_NOTIFY_EXCEPTION:
1152 ret = intel_guc_crash_process_msg(guc, action);
1153 break;
1154 case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
1155 ret = intel_guc_tlb_invalidation_done(guc, payload, len);
1156 break;
1157 default:
1158 ret = -EOPNOTSUPP;
1159 break;
1160 }
1161
1162 if (unlikely(ret)) {
1163 CT_ERROR(ct, "Failed to process request %04x (%pe)\n",
1164 action, ERR_PTR(ret));
1165 return ret;
1166 }
1167
1168 ct_free_msg(request);
1169 return 0;
1170}
1171
1172static bool ct_process_incoming_requests(struct intel_guc_ct *ct)
1173{
1174 unsigned long flags;
1175 struct ct_incoming_msg *request;
1176 bool done;
1177 int err;
1178
1179 spin_lock_irqsave(&ct->requests.lock, flags);
1180 request = list_first_entry_or_null(&ct->requests.incoming,
1181 struct ct_incoming_msg, link);
1182 if (request)
1183 list_del(&request->link);
1184 done = !!list_empty(&ct->requests.incoming);
1185 spin_unlock_irqrestore(&ct->requests.lock, flags);
1186
1187 if (!request)
1188 return true;
1189
1190 err = ct_process_request(ct, request);
1191 if (unlikely(err)) {
1192 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1193 ERR_PTR(err), 4 * request->size, request->msg);
1194 CT_DEAD(ct, PROCESS_FAILED);
1195 ct_free_msg(request);
1196 }
1197
1198 return done;
1199}
1200
1201static void ct_incoming_request_worker_func(struct work_struct *w)
1202{
1203 struct intel_guc_ct *ct =
1204 container_of(w, struct intel_guc_ct, requests.worker);
1205 bool done;
1206
1207 do {
1208 done = ct_process_incoming_requests(ct);
1209 } while (!done);
1210}
1211
1212static int ct_handle_event(struct intel_guc_ct *ct, struct ct_incoming_msg *request)
1213{
1214 const u32 *hxg = &request->msg[GUC_CTB_MSG_MIN_LEN];
1215 u32 action = FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, hxg[0]);
1216 unsigned long flags;
1217
1218 GEM_BUG_ON(FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]) != GUC_HXG_TYPE_EVENT);
1219
1220 /*
1221 * Adjusting the space must be done in IRQ or deadlock can occur as the
1222 * CTB processing in the below workqueue can send CTBs which creates a
1223 * circular dependency if the space was returned there.
1224 */
1225 switch (action) {
1226 case INTEL_GUC_ACTION_SCHED_CONTEXT_MODE_DONE:
1227 case INTEL_GUC_ACTION_DEREGISTER_CONTEXT_DONE:
1228 case INTEL_GUC_ACTION_TLB_INVALIDATION_DONE:
1229 g2h_release_space(ct, request->size);
1230 }
1231
1232 /*
1233 * TLB invalidation responses must be handled immediately as processing
1234 * of other G2H notifications may be blocked by an invalidation request.
1235 */
1236 if (action == INTEL_GUC_ACTION_TLB_INVALIDATION_DONE)
1237 return ct_process_request(ct, request);
1238
1239 spin_lock_irqsave(&ct->requests.lock, flags);
1240 list_add_tail(&request->link, &ct->requests.incoming);
1241 spin_unlock_irqrestore(&ct->requests.lock, flags);
1242
1243 queue_work(system_unbound_wq, &ct->requests.worker);
1244 return 0;
1245}
1246
1247static int ct_handle_hxg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1248{
1249 u32 origin, type;
1250 u32 *hxg;
1251 int err;
1252
1253 if (unlikely(msg->size < GUC_CTB_HXG_MSG_MIN_LEN))
1254 return -EBADMSG;
1255
1256 hxg = &msg->msg[GUC_CTB_MSG_MIN_LEN];
1257
1258 origin = FIELD_GET(GUC_HXG_MSG_0_ORIGIN, hxg[0]);
1259 if (unlikely(origin != GUC_HXG_ORIGIN_GUC)) {
1260 err = -EPROTO;
1261 goto failed;
1262 }
1263
1264 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, hxg[0]);
1265 switch (type) {
1266 case GUC_HXG_TYPE_EVENT:
1267 err = ct_handle_event(ct, msg);
1268 break;
1269 case GUC_HXG_TYPE_RESPONSE_SUCCESS:
1270 case GUC_HXG_TYPE_RESPONSE_FAILURE:
1271 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
1272 err = ct_handle_response(ct, msg);
1273 break;
1274 default:
1275 err = -EOPNOTSUPP;
1276 }
1277
1278 if (unlikely(err)) {
1279failed:
1280 CT_ERROR(ct, "Failed to handle HXG message (%pe) %*ph\n",
1281 ERR_PTR(err), 4 * GUC_HXG_MSG_MIN_LEN, hxg);
1282 }
1283 return err;
1284}
1285
1286static void ct_handle_msg(struct intel_guc_ct *ct, struct ct_incoming_msg *msg)
1287{
1288 u32 format = FIELD_GET(GUC_CTB_MSG_0_FORMAT, msg->msg[0]);
1289 int err;
1290
1291 if (format == GUC_CTB_FORMAT_HXG)
1292 err = ct_handle_hxg(ct, msg);
1293 else
1294 err = -EOPNOTSUPP;
1295
1296 if (unlikely(err)) {
1297 CT_ERROR(ct, "Failed to process CT message (%pe) %*ph\n",
1298 ERR_PTR(err), 4 * msg->size, msg->msg);
1299 ct_free_msg(msg);
1300 }
1301}
1302
1303/*
1304 * Return: number available remaining dwords to read (0 if empty)
1305 * or a negative error code on failure
1306 */
1307static int ct_receive(struct intel_guc_ct *ct)
1308{
1309 struct ct_incoming_msg *msg = NULL;
1310 unsigned long flags;
1311 int ret;
1312
1313 spin_lock_irqsave(&ct->ctbs.recv.lock, flags);
1314 ret = ct_read(ct, &msg);
1315 spin_unlock_irqrestore(&ct->ctbs.recv.lock, flags);
1316 if (ret < 0)
1317 return ret;
1318
1319 if (msg)
1320 ct_handle_msg(ct, msg);
1321
1322 return ret;
1323}
1324
1325static void ct_try_receive_message(struct intel_guc_ct *ct)
1326{
1327 int ret;
1328
1329 if (GEM_WARN_ON(!ct->enabled))
1330 return;
1331
1332 ret = ct_receive(ct);
1333 if (ret > 0)
1334 tasklet_hi_schedule(&ct->receive_tasklet);
1335}
1336
1337static void ct_receive_tasklet_func(struct tasklet_struct *t)
1338{
1339 struct intel_guc_ct *ct = from_tasklet(ct, t, receive_tasklet);
1340
1341 ct_try_receive_message(ct);
1342}
1343
1344/*
1345 * When we're communicating with the GuC over CT, GuC uses events
1346 * to notify us about new messages being posted on the RECV buffer.
1347 */
1348void intel_guc_ct_event_handler(struct intel_guc_ct *ct)
1349{
1350 if (unlikely(!ct->enabled)) {
1351 WARN(1, "Unexpected GuC event received while CT disabled!\n");
1352 return;
1353 }
1354
1355 ct_try_receive_message(ct);
1356}
1357
1358void intel_guc_ct_print_info(struct intel_guc_ct *ct,
1359 struct drm_printer *p)
1360{
1361 drm_printf(p, "CT %s\n", str_enabled_disabled(ct->enabled));
1362
1363 if (!ct->enabled)
1364 return;
1365
1366 drm_printf(p, "H2G Space: %u\n",
1367 atomic_read(&ct->ctbs.send.space) * 4);
1368 drm_printf(p, "Head: %u\n",
1369 ct->ctbs.send.desc->head);
1370 drm_printf(p, "Tail: %u\n",
1371 ct->ctbs.send.desc->tail);
1372 drm_printf(p, "G2H Space: %u\n",
1373 atomic_read(&ct->ctbs.recv.space) * 4);
1374 drm_printf(p, "Head: %u\n",
1375 ct->ctbs.recv.desc->head);
1376 drm_printf(p, "Tail: %u\n",
1377 ct->ctbs.recv.desc->tail);
1378}
1379
1380#if IS_ENABLED(CONFIG_DRM_I915_DEBUG_GUC)
1381static void ct_dead_ct_worker_func(struct work_struct *w)
1382{
1383 struct intel_guc_ct *ct = container_of(w, struct intel_guc_ct, dead_ct_worker);
1384 struct intel_guc *guc = ct_to_guc(ct);
1385
1386 if (ct->dead_ct_reported)
1387 return;
1388
1389 ct->dead_ct_reported = true;
1390
1391 guc_info(guc, "CTB is dead - reason=0x%X\n", ct->dead_ct_reason);
1392 intel_klog_error_capture(guc_to_gt(guc), (intel_engine_mask_t)~0U);
1393}
1394#endif