Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2023 Intel Corporation
4 */
5
6#include <linux/bitfield.h>
7#include <linux/delay.h>
8#include <linux/fault-inject.h>
9
10#include <drm/drm_managed.h>
11
12#include <kunit/static_stub.h>
13#include <kunit/test-bug.h>
14
15#include "abi/guc_actions_sriov_abi.h"
16#include "abi/guc_relay_actions_abi.h"
17#include "abi/guc_relay_communication_abi.h"
18
19#include "xe_assert.h"
20#include "xe_device.h"
21#include "xe_gt.h"
22#include "xe_gt_sriov_printk.h"
23#include "xe_gt_sriov_pf_service.h"
24#include "xe_guc.h"
25#include "xe_guc_ct.h"
26#include "xe_guc_hxg_helpers.h"
27#include "xe_guc_relay.h"
28#include "xe_guc_relay_types.h"
29#include "xe_sriov.h"
30
31/*
32 * How long should we wait for the response?
33 * XXX this value is subject for the profiling.
34 */
35#define RELAY_TIMEOUT_MSEC (2500)
36
37static void relays_worker_fn(struct work_struct *w);
38
39static struct xe_guc *relay_to_guc(struct xe_guc_relay *relay)
40{
41 return container_of(relay, struct xe_guc, relay);
42}
43
44static struct xe_guc_ct *relay_to_ct(struct xe_guc_relay *relay)
45{
46 return &relay_to_guc(relay)->ct;
47}
48
49static struct xe_gt *relay_to_gt(struct xe_guc_relay *relay)
50{
51 return guc_to_gt(relay_to_guc(relay));
52}
53
54static struct xe_device *relay_to_xe(struct xe_guc_relay *relay)
55{
56 return gt_to_xe(relay_to_gt(relay));
57}
58
59#define relay_assert(relay, condition) xe_gt_assert(relay_to_gt(relay), condition)
60#define relay_notice(relay, msg...) xe_gt_sriov_notice(relay_to_gt(relay), "relay: " msg)
61#define relay_debug(relay, msg...) xe_gt_sriov_dbg_verbose(relay_to_gt(relay), "relay: " msg)
62
63static int relay_get_totalvfs(struct xe_guc_relay *relay)
64{
65 struct xe_device *xe = relay_to_xe(relay);
66 struct pci_dev *pdev = to_pci_dev(xe->drm.dev);
67
68 KUNIT_STATIC_STUB_REDIRECT(relay_get_totalvfs, relay);
69 return IS_SRIOV_VF(xe) ? 0 : pci_sriov_get_totalvfs(pdev);
70}
71
72static bool relay_is_ready(struct xe_guc_relay *relay)
73{
74 return mempool_initialized(&relay->pool);
75}
76
77static u32 relay_get_next_rid(struct xe_guc_relay *relay)
78{
79 u32 rid;
80
81 spin_lock(&relay->lock);
82 rid = ++relay->last_rid;
83 spin_unlock(&relay->lock);
84
85 return rid;
86}
87
88/**
89 * struct relay_transaction - internal data used to handle transactions
90 *
91 * Relation between struct relay_transaction members::
92 *
93 * <-------------------- GUC_CTB_MAX_DWORDS -------------->
94 * <-------- GUC_RELAY_MSG_MAX_LEN --->
95 * <--- offset ---> <--- request_len ------->
96 * +----------------+-------------------------+----------+--+
97 * | | | | |
98 * +----------------+-------------------------+----------+--+
99 * ^ ^
100 * / /
101 * request_buf request
102 *
103 * <-------------------- GUC_CTB_MAX_DWORDS -------------->
104 * <-------- GUC_RELAY_MSG_MAX_LEN --->
105 * <--- offset ---> <--- response_len --->
106 * +----------------+----------------------+-------------+--+
107 * | | | | |
108 * +----------------+----------------------+-------------+--+
109 * ^ ^
110 * / /
111 * response_buf response
112 */
113struct relay_transaction {
114 /**
115 * @incoming: indicates whether this transaction represents an incoming
116 * request from the remote VF/PF or this transaction
117 * represents outgoing request to the remote VF/PF.
118 */
119 bool incoming;
120
121 /**
122 * @remote: PF/VF identifier of the origin (or target) of the relay
123 * request message.
124 */
125 u32 remote;
126
127 /** @rid: identifier of the VF/PF relay message. */
128 u32 rid;
129
130 /**
131 * @request: points to the inner VF/PF request message, copied to the
132 * #response_buf starting at #offset.
133 */
134 u32 *request;
135
136 /** @request_len: length of the inner VF/PF request message. */
137 u32 request_len;
138
139 /**
140 * @response: points to the placeholder buffer where inner VF/PF
141 * response will be located, for outgoing transaction
142 * this could be caller's buffer (if provided) otherwise
143 * it points to the #response_buf starting at #offset.
144 */
145 u32 *response;
146
147 /**
148 * @response_len: length of the inner VF/PF response message (only
149 * if #status is 0), initially set to the size of the
150 * placeholder buffer where response message will be
151 * copied.
152 */
153 u32 response_len;
154
155 /**
156 * @offset: offset to the start of the inner VF/PF relay message inside
157 * buffers; this offset is equal the length of the outer GuC
158 * relay header message.
159 */
160 u32 offset;
161
162 /**
163 * @request_buf: buffer with VF/PF request message including outer
164 * transport message.
165 */
166 u32 request_buf[GUC_CTB_MAX_DWORDS];
167
168 /**
169 * @response_buf: buffer with VF/PF response message including outer
170 * transport message.
171 */
172 u32 response_buf[GUC_CTB_MAX_DWORDS];
173
174 /**
175 * @reply: status of the reply, 0 means that data pointed by the
176 * #response is valid.
177 */
178 int reply;
179
180 /** @done: completion of the outgoing transaction. */
181 struct completion done;
182
183 /** @link: transaction list link */
184 struct list_head link;
185};
186
187static u32 prepare_pf2guc(u32 *msg, u32 target, u32 rid)
188{
189 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
190 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
191 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_PF2GUC_RELAY_TO_VF);
192 msg[1] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_1_VFID, target);
193 msg[2] = FIELD_PREP(PF2GUC_RELAY_TO_VF_REQUEST_MSG_2_RELAY_ID, rid);
194
195 return PF2GUC_RELAY_TO_VF_REQUEST_MSG_MIN_LEN;
196}
197
198static u32 prepare_vf2guc(u32 *msg, u32 rid)
199{
200 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
201 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_REQUEST) |
202 FIELD_PREP(GUC_HXG_REQUEST_MSG_0_ACTION, XE_GUC_ACTION_VF2GUC_RELAY_TO_PF);
203 msg[1] = FIELD_PREP(VF2GUC_RELAY_TO_PF_REQUEST_MSG_1_RELAY_ID, rid);
204
205 return VF2GUC_RELAY_TO_PF_REQUEST_MSG_MIN_LEN;
206}
207
208static struct relay_transaction *
209__relay_get_transaction(struct xe_guc_relay *relay, bool incoming, u32 remote, u32 rid,
210 const u32 *action, u32 action_len, u32 *resp, u32 resp_size)
211{
212 struct relay_transaction *txn;
213
214 relay_assert(relay, action_len >= GUC_RELAY_MSG_MIN_LEN);
215 relay_assert(relay, action_len <= GUC_RELAY_MSG_MAX_LEN);
216 relay_assert(relay, !(!!resp ^ !!resp_size));
217 relay_assert(relay, resp_size <= GUC_RELAY_MSG_MAX_LEN);
218 relay_assert(relay, resp_size == 0 || resp_size >= GUC_RELAY_MSG_MIN_LEN);
219
220 if (unlikely(!relay_is_ready(relay)))
221 return ERR_PTR(-ENODEV);
222
223 /*
224 * For incoming requests we can't use GFP_KERNEL as those are delivered
225 * with CTB lock held which is marked as used in the reclaim path.
226 * Btw, that's one of the reason why we use mempool here!
227 */
228 txn = mempool_alloc(&relay->pool, incoming ? GFP_ATOMIC : GFP_KERNEL);
229 if (!txn)
230 return ERR_PTR(-ENOMEM);
231
232 txn->incoming = incoming;
233 txn->remote = remote;
234 txn->rid = rid;
235 txn->offset = remote ?
236 prepare_pf2guc(incoming ? txn->response_buf : txn->request_buf, remote, rid) :
237 prepare_vf2guc(incoming ? txn->response_buf : txn->request_buf, rid);
238
239 relay_assert(relay, txn->offset);
240 relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->request_buf));
241 relay_assert(relay, txn->offset + GUC_RELAY_MSG_MAX_LEN <= ARRAY_SIZE(txn->response_buf));
242
243 txn->request = txn->request_buf + txn->offset;
244 memcpy(&txn->request_buf[txn->offset], action, sizeof(u32) * action_len);
245 txn->request_len = action_len;
246
247 txn->response = resp ?: txn->response_buf + txn->offset;
248 txn->response_len = resp_size ?: GUC_RELAY_MSG_MAX_LEN;
249 txn->reply = -ENOMSG;
250 INIT_LIST_HEAD(&txn->link);
251 init_completion(&txn->done);
252
253 return txn;
254}
255
256static struct relay_transaction *
257relay_new_transaction(struct xe_guc_relay *relay, u32 target, const u32 *action, u32 len,
258 u32 *resp, u32 resp_size)
259{
260 u32 rid = relay_get_next_rid(relay);
261
262 return __relay_get_transaction(relay, false, target, rid, action, len, resp, resp_size);
263}
264
265static struct relay_transaction *
266relay_new_incoming_transaction(struct xe_guc_relay *relay, u32 origin, u32 rid,
267 const u32 *action, u32 len)
268{
269 return __relay_get_transaction(relay, true, origin, rid, action, len, NULL, 0);
270}
271
272static void relay_release_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn)
273{
274 relay_assert(relay, list_empty(&txn->link));
275
276 txn->offset = 0;
277 txn->response = NULL;
278 txn->reply = -ESTALE;
279 mempool_free(txn, &relay->pool);
280}
281
282static int relay_send_transaction(struct xe_guc_relay *relay, struct relay_transaction *txn)
283{
284 u32 len = txn->incoming ? txn->response_len : txn->request_len;
285 u32 *buf = txn->incoming ? txn->response_buf : txn->request_buf;
286 u32 *msg = buf + txn->offset;
287 int ret;
288
289 relay_assert(relay, txn->offset);
290 relay_assert(relay, txn->offset + len <= GUC_CTB_MAX_DWORDS);
291 relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN);
292 relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN);
293
294 relay_debug(relay, "sending %s.%u to %u = %*ph\n",
295 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])),
296 txn->rid, txn->remote, (int)sizeof(u32) * len, msg);
297
298 ret = xe_guc_ct_send_block(relay_to_ct(relay), buf, len + txn->offset);
299
300 if (unlikely(ret > 0)) {
301 relay_notice(relay, "Unexpected data=%d from GuC, wrong ABI?\n", ret);
302 ret = -EPROTO;
303 }
304 if (unlikely(ret < 0)) {
305 relay_notice(relay, "Failed to send %s.%x to GuC (%pe) %*ph ...\n",
306 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, buf[0])),
307 FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, buf[0]),
308 ERR_PTR(ret), (int)sizeof(u32) * txn->offset, buf);
309 relay_notice(relay, "Failed to send %s.%u to %u (%pe) %*ph\n",
310 guc_hxg_type_to_string(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])),
311 txn->rid, txn->remote, ERR_PTR(ret), (int)sizeof(u32) * len, msg);
312 }
313
314 return ret;
315}
316
317static void __fini_relay(struct drm_device *drm, void *arg)
318{
319 struct xe_guc_relay *relay = arg;
320
321 mempool_exit(&relay->pool);
322}
323
324/**
325 * xe_guc_relay_init - Initialize a &xe_guc_relay
326 * @relay: the &xe_guc_relay to initialize
327 *
328 * Initialize remaining members of &xe_guc_relay that may depend
329 * on the SR-IOV mode.
330 *
331 * Return: 0 on success or a negative error code on failure.
332 */
333int xe_guc_relay_init(struct xe_guc_relay *relay)
334{
335 const int XE_RELAY_MEMPOOL_MIN_NUM = 1;
336 struct xe_device *xe = relay_to_xe(relay);
337 int err;
338
339 relay_assert(relay, !relay_is_ready(relay));
340
341 if (!IS_SRIOV(xe))
342 return 0;
343
344 spin_lock_init(&relay->lock);
345 INIT_WORK(&relay->worker, relays_worker_fn);
346 INIT_LIST_HEAD(&relay->pending_relays);
347 INIT_LIST_HEAD(&relay->incoming_actions);
348
349 err = mempool_init_kmalloc_pool(&relay->pool, XE_RELAY_MEMPOOL_MIN_NUM +
350 relay_get_totalvfs(relay),
351 sizeof(struct relay_transaction));
352 if (err)
353 return err;
354
355 relay_debug(relay, "using mempool with %d elements\n", relay->pool.min_nr);
356
357 return drmm_add_action_or_reset(&xe->drm, __fini_relay, relay);
358}
359ALLOW_ERROR_INJECTION(xe_guc_relay_init, ERRNO); /* See xe_pci_probe() */
360
361static u32 to_relay_error(int err)
362{
363 /* XXX: assume that relay errors match errno codes */
364 return err < 0 ? -err : GUC_RELAY_ERROR_UNDISCLOSED;
365}
366
367static int from_relay_error(u32 error)
368{
369 /* XXX: assume that relay errors match errno codes */
370 return error ? -error : -ENODATA;
371}
372
373static u32 sanitize_relay_error(u32 error)
374{
375 /* XXX TBD if generic error codes will be allowed */
376 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG))
377 error = GUC_RELAY_ERROR_UNDISCLOSED;
378 return error;
379}
380
381static u32 sanitize_relay_error_hint(u32 hint)
382{
383 /* XXX TBD if generic error codes will be allowed */
384 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG))
385 hint = 0;
386 return hint;
387}
388
389static u32 prepare_error_reply(u32 *msg, u32 error, u32 hint)
390{
391 msg[0] = FIELD_PREP(GUC_HXG_MSG_0_ORIGIN, GUC_HXG_ORIGIN_HOST) |
392 FIELD_PREP(GUC_HXG_MSG_0_TYPE, GUC_HXG_TYPE_RESPONSE_FAILURE) |
393 FIELD_PREP(GUC_HXG_FAILURE_MSG_0_HINT, hint) |
394 FIELD_PREP(GUC_HXG_FAILURE_MSG_0_ERROR, error);
395
396 XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_ERROR, error));
397 XE_WARN_ON(!FIELD_FIT(GUC_HXG_FAILURE_MSG_0_HINT, hint));
398
399 return GUC_HXG_FAILURE_MSG_LEN;
400}
401
402static void relay_testonly_nop(struct xe_guc_relay *relay)
403{
404 KUNIT_STATIC_STUB_REDIRECT(relay_testonly_nop, relay);
405}
406
407static int relay_send_message_and_wait(struct xe_guc_relay *relay,
408 struct relay_transaction *txn,
409 u32 *buf, u32 buf_size)
410{
411 unsigned long timeout = msecs_to_jiffies(RELAY_TIMEOUT_MSEC);
412 u32 *msg = &txn->request_buf[txn->offset];
413 u32 len = txn->request_len;
414 u32 type, action, data0;
415 int ret;
416 long n;
417
418 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);
419 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]);
420 data0 = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]);
421
422 relay_debug(relay, "%s.%u to %u action %#x:%u\n",
423 guc_hxg_type_to_string(type),
424 txn->rid, txn->remote, action, data0);
425
426 /* list ordering does not need to match RID ordering */
427 spin_lock(&relay->lock);
428 list_add_tail(&txn->link, &relay->pending_relays);
429 spin_unlock(&relay->lock);
430
431resend:
432 ret = relay_send_transaction(relay, txn);
433 if (unlikely(ret < 0))
434 goto unlink;
435
436wait:
437 n = wait_for_completion_timeout(&txn->done, timeout);
438 if (unlikely(n == 0 && txn->reply)) {
439 ret = -ETIME;
440 goto unlink;
441 }
442
443 relay_debug(relay, "%u.%u reply %d after %u msec\n",
444 txn->remote, txn->rid, txn->reply, jiffies_to_msecs(timeout - n));
445 if (unlikely(txn->reply)) {
446 reinit_completion(&txn->done);
447 if (txn->reply == -EAGAIN)
448 goto resend;
449 if (txn->reply == -EBUSY) {
450 relay_testonly_nop(relay);
451 goto wait;
452 }
453 if (txn->reply > 0)
454 ret = from_relay_error(txn->reply);
455 else
456 ret = txn->reply;
457 goto unlink;
458 }
459
460 relay_debug(relay, "%u.%u response %*ph\n", txn->remote, txn->rid,
461 (int)sizeof(u32) * txn->response_len, txn->response);
462 relay_assert(relay, txn->response_len >= GUC_RELAY_MSG_MIN_LEN);
463 ret = txn->response_len;
464
465unlink:
466 spin_lock(&relay->lock);
467 list_del_init(&txn->link);
468 spin_unlock(&relay->lock);
469
470 if (unlikely(ret < 0)) {
471 relay_notice(relay, "Unsuccessful %s.%u %#x:%u to %u (%pe) %*ph\n",
472 guc_hxg_type_to_string(type), txn->rid,
473 action, data0, txn->remote, ERR_PTR(ret),
474 (int)sizeof(u32) * len, msg);
475 }
476
477 return ret;
478}
479
480static int relay_send_to(struct xe_guc_relay *relay, u32 target,
481 const u32 *msg, u32 len, u32 *buf, u32 buf_size)
482{
483 struct relay_transaction *txn;
484 int ret;
485
486 relay_assert(relay, len >= GUC_RELAY_MSG_MIN_LEN);
487 relay_assert(relay, len <= GUC_RELAY_MSG_MAX_LEN);
488 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_HOST);
489 relay_assert(relay, guc_hxg_type_is_action(FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])));
490
491 if (unlikely(!relay_is_ready(relay)))
492 return -ENODEV;
493
494 txn = relay_new_transaction(relay, target, msg, len, buf, buf_size);
495 if (IS_ERR(txn))
496 return PTR_ERR(txn);
497
498 switch (FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0])) {
499 case GUC_HXG_TYPE_REQUEST:
500 ret = relay_send_message_and_wait(relay, txn, buf, buf_size);
501 break;
502 case GUC_HXG_TYPE_FAST_REQUEST:
503 relay_assert(relay, !GUC_HXG_TYPE_FAST_REQUEST);
504 fallthrough;
505 case GUC_HXG_TYPE_EVENT:
506 ret = relay_send_transaction(relay, txn);
507 break;
508 default:
509 ret = -EINVAL;
510 break;
511 }
512
513 relay_release_transaction(relay, txn);
514 return ret;
515}
516
517#ifdef CONFIG_PCI_IOV
518/**
519 * xe_guc_relay_send_to_vf - Send a message to the VF.
520 * @relay: the &xe_guc_relay which will send the message
521 * @target: target VF number
522 * @msg: request message to be sent
523 * @len: length of the request message (in dwords, can't be 0)
524 * @buf: placeholder for the response message
525 * @buf_size: size of the response message placeholder (in dwords)
526 *
527 * This function can only be used by the driver running in the SR-IOV PF mode.
528 *
529 * Return: Non-negative response length (in dwords) or
530 * a negative error code on failure.
531 */
532int xe_guc_relay_send_to_vf(struct xe_guc_relay *relay, u32 target,
533 const u32 *msg, u32 len, u32 *buf, u32 buf_size)
534{
535 relay_assert(relay, IS_SRIOV_PF(relay_to_xe(relay)));
536
537 return relay_send_to(relay, target, msg, len, buf, buf_size);
538}
539#endif
540
541/**
542 * xe_guc_relay_send_to_pf - Send a message to the PF.
543 * @relay: the &xe_guc_relay which will send the message
544 * @msg: request message to be sent
545 * @len: length of the message (in dwords, can't be 0)
546 * @buf: placeholder for the response message
547 * @buf_size: size of the response message placeholder (in dwords)
548 *
549 * This function can only be used by driver running in SR-IOV VF mode.
550 *
551 * Return: Non-negative response length (in dwords) or
552 * a negative error code on failure.
553 */
554int xe_guc_relay_send_to_pf(struct xe_guc_relay *relay,
555 const u32 *msg, u32 len, u32 *buf, u32 buf_size)
556{
557 relay_assert(relay, IS_SRIOV_VF(relay_to_xe(relay)));
558
559 return relay_send_to(relay, PFID, msg, len, buf, buf_size);
560}
561
562static int relay_handle_reply(struct xe_guc_relay *relay, u32 origin,
563 u32 rid, int reply, const u32 *msg, u32 len)
564{
565 struct relay_transaction *pending;
566 int err = -ESRCH;
567
568 spin_lock(&relay->lock);
569 list_for_each_entry(pending, &relay->pending_relays, link) {
570 if (pending->remote != origin || pending->rid != rid) {
571 relay_debug(relay, "%u.%u still awaits response\n",
572 pending->remote, pending->rid);
573 continue;
574 }
575 err = 0; /* found! */
576 if (reply == 0) {
577 if (len > pending->response_len) {
578 reply = -ENOBUFS;
579 err = -ENOBUFS;
580 } else {
581 memcpy(pending->response, msg, 4 * len);
582 pending->response_len = len;
583 }
584 }
585 pending->reply = reply;
586 complete_all(&pending->done);
587 break;
588 }
589 spin_unlock(&relay->lock);
590
591 return err;
592}
593
594static int relay_handle_failure(struct xe_guc_relay *relay, u32 origin,
595 u32 rid, const u32 *msg, u32 len)
596{
597 int error = FIELD_GET(GUC_HXG_FAILURE_MSG_0_ERROR, msg[0]);
598 u32 hint __maybe_unused = FIELD_GET(GUC_HXG_FAILURE_MSG_0_HINT, msg[0]);
599
600 relay_assert(relay, len);
601 relay_debug(relay, "%u.%u error %#x (%pe) hint %u debug %*ph\n",
602 origin, rid, error, ERR_PTR(-error), hint, 4 * (len - 1), msg + 1);
603
604 return relay_handle_reply(relay, origin, rid, error ?: -EREMOTEIO, NULL, 0);
605}
606
607static int relay_testloop_action_handler(struct xe_guc_relay *relay, u32 origin,
608 const u32 *msg, u32 len, u32 *response, u32 size)
609{
610 static ktime_t last_reply = 0;
611 u32 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);
612 u32 action = FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]);
613 u32 opcode = FIELD_GET(GUC_HXG_REQUEST_MSG_0_DATA0, msg[0]);
614 ktime_t now = ktime_get();
615 bool busy;
616 int ret;
617
618 relay_assert(relay, guc_hxg_type_is_action(type));
619 relay_assert(relay, action == GUC_RELAY_ACTION_VFXPF_TESTLOOP);
620
621 if (!IS_ENABLED(CONFIG_DRM_XE_DEBUG_SRIOV))
622 return -ECONNREFUSED;
623
624 if (!last_reply)
625 last_reply = now;
626 busy = ktime_before(now, ktime_add_ms(last_reply, 2 * RELAY_TIMEOUT_MSEC));
627 if (!busy)
628 last_reply = now;
629
630 switch (opcode) {
631 case VFXPF_TESTLOOP_OPCODE_NOP:
632 if (type == GUC_HXG_TYPE_EVENT)
633 return 0;
634 return guc_hxg_msg_encode_success(response, 0);
635 case VFXPF_TESTLOOP_OPCODE_BUSY:
636 if (type == GUC_HXG_TYPE_EVENT)
637 return -EPROTO;
638 msleep(RELAY_TIMEOUT_MSEC / 8);
639 if (busy)
640 return -EINPROGRESS;
641 return guc_hxg_msg_encode_success(response, 0);
642 case VFXPF_TESTLOOP_OPCODE_RETRY:
643 if (type == GUC_HXG_TYPE_EVENT)
644 return -EPROTO;
645 msleep(RELAY_TIMEOUT_MSEC / 8);
646 if (busy)
647 return guc_hxg_msg_encode_retry(response, 0);
648 return guc_hxg_msg_encode_success(response, 0);
649 case VFXPF_TESTLOOP_OPCODE_ECHO:
650 if (type == GUC_HXG_TYPE_EVENT)
651 return -EPROTO;
652 if (size < len)
653 return -ENOBUFS;
654 ret = guc_hxg_msg_encode_success(response, len);
655 memcpy(response + ret, msg + ret, (len - ret) * sizeof(u32));
656 return len;
657 case VFXPF_TESTLOOP_OPCODE_FAIL:
658 return -EHWPOISON;
659 default:
660 break;
661 }
662
663 relay_notice(relay, "Unexpected action %#x opcode %#x\n", action, opcode);
664 return -EBADRQC;
665}
666
667static int relay_action_handler(struct xe_guc_relay *relay, u32 origin,
668 const u32 *msg, u32 len, u32 *response, u32 size)
669{
670 struct xe_gt *gt = relay_to_gt(relay);
671 u32 type;
672 int ret;
673
674 relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN);
675
676 if (FIELD_GET(GUC_HXG_REQUEST_MSG_0_ACTION, msg[0]) == GUC_RELAY_ACTION_VFXPF_TESTLOOP)
677 return relay_testloop_action_handler(relay, origin, msg, len, response, size);
678
679 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);
680
681 if (IS_SRIOV_PF(relay_to_xe(relay)))
682 ret = xe_gt_sriov_pf_service_process_request(gt, origin, msg, len, response, size);
683 else
684 ret = -EOPNOTSUPP;
685
686 if (type == GUC_HXG_TYPE_EVENT)
687 relay_assert(relay, ret <= 0);
688
689 return ret;
690}
691
692static struct relay_transaction *relay_dequeue_transaction(struct xe_guc_relay *relay)
693{
694 struct relay_transaction *txn;
695
696 spin_lock(&relay->lock);
697 txn = list_first_entry_or_null(&relay->incoming_actions, struct relay_transaction, link);
698 if (txn)
699 list_del_init(&txn->link);
700 spin_unlock(&relay->lock);
701
702 return txn;
703}
704
705static void relay_process_incoming_action(struct xe_guc_relay *relay)
706{
707 struct relay_transaction *txn;
708 bool again = false;
709 u32 type;
710 int ret;
711
712 txn = relay_dequeue_transaction(relay);
713 if (!txn)
714 return;
715
716 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, txn->request_buf[txn->offset]);
717
718 ret = relay_action_handler(relay, txn->remote,
719 txn->request_buf + txn->offset, txn->request_len,
720 txn->response_buf + txn->offset,
721 ARRAY_SIZE(txn->response_buf) - txn->offset);
722
723 if (ret == -EINPROGRESS) {
724 again = true;
725 ret = guc_hxg_msg_encode_busy(txn->response_buf + txn->offset, 0);
726 }
727
728 if (ret > 0) {
729 txn->response_len = ret;
730 ret = relay_send_transaction(relay, txn);
731 }
732
733 if (ret < 0) {
734 u32 error = to_relay_error(ret);
735
736 relay_notice(relay, "Failed to handle %s.%u from %u (%pe) %*ph\n",
737 guc_hxg_type_to_string(type), txn->rid, txn->remote,
738 ERR_PTR(ret), 4 * txn->request_len, txn->request_buf + txn->offset);
739
740 txn->response_len = prepare_error_reply(txn->response_buf + txn->offset,
741 txn->remote ?
742 sanitize_relay_error(error) : error,
743 txn->remote ?
744 sanitize_relay_error_hint(-ret) : -ret);
745 ret = relay_send_transaction(relay, txn);
746 again = false;
747 }
748
749 if (again) {
750 spin_lock(&relay->lock);
751 list_add(&txn->link, &relay->incoming_actions);
752 spin_unlock(&relay->lock);
753 return;
754 }
755
756 if (unlikely(ret < 0))
757 relay_notice(relay, "Failed to process action.%u (%pe) %*ph\n",
758 txn->rid, ERR_PTR(ret), 4 * txn->request_len,
759 txn->request_buf + txn->offset);
760
761 relay_release_transaction(relay, txn);
762}
763
764static bool relay_needs_worker(struct xe_guc_relay *relay)
765{
766 bool is_empty;
767
768 spin_lock(&relay->lock);
769 is_empty = list_empty(&relay->incoming_actions);
770 spin_unlock(&relay->lock);
771
772 return !is_empty;
773
774}
775
776static void relay_kick_worker(struct xe_guc_relay *relay)
777{
778 KUNIT_STATIC_STUB_REDIRECT(relay_kick_worker, relay);
779 queue_work(relay_to_xe(relay)->sriov.wq, &relay->worker);
780}
781
782static void relays_worker_fn(struct work_struct *w)
783{
784 struct xe_guc_relay *relay = container_of(w, struct xe_guc_relay, worker);
785
786 relay_process_incoming_action(relay);
787
788 if (relay_needs_worker(relay))
789 relay_kick_worker(relay);
790}
791
792static int relay_queue_action_msg(struct xe_guc_relay *relay, u32 origin, u32 rid,
793 const u32 *msg, u32 len)
794{
795 struct relay_transaction *txn;
796
797 txn = relay_new_incoming_transaction(relay, origin, rid, msg, len);
798 if (IS_ERR(txn))
799 return PTR_ERR(txn);
800
801 spin_lock(&relay->lock);
802 list_add_tail(&txn->link, &relay->incoming_actions);
803 spin_unlock(&relay->lock);
804
805 relay_kick_worker(relay);
806 return 0;
807}
808
809static int relay_process_msg(struct xe_guc_relay *relay, u32 origin, u32 rid,
810 const u32 *msg, u32 len)
811{
812 u32 type;
813 int err;
814
815 if (unlikely(len < GUC_HXG_MSG_MIN_LEN))
816 return -EPROTO;
817
818 if (FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) != GUC_HXG_ORIGIN_HOST)
819 return -EPROTO;
820
821 type = FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]);
822 relay_debug(relay, "received %s.%u from %u = %*ph\n",
823 guc_hxg_type_to_string(type), rid, origin, 4 * len, msg);
824
825 switch (type) {
826 case GUC_HXG_TYPE_REQUEST:
827 case GUC_HXG_TYPE_FAST_REQUEST:
828 case GUC_HXG_TYPE_EVENT:
829 err = relay_queue_action_msg(relay, origin, rid, msg, len);
830 break;
831 case GUC_HXG_TYPE_RESPONSE_SUCCESS:
832 err = relay_handle_reply(relay, origin, rid, 0, msg, len);
833 break;
834 case GUC_HXG_TYPE_NO_RESPONSE_BUSY:
835 err = relay_handle_reply(relay, origin, rid, -EBUSY, NULL, 0);
836 break;
837 case GUC_HXG_TYPE_NO_RESPONSE_RETRY:
838 err = relay_handle_reply(relay, origin, rid, -EAGAIN, NULL, 0);
839 break;
840 case GUC_HXG_TYPE_RESPONSE_FAILURE:
841 err = relay_handle_failure(relay, origin, rid, msg, len);
842 break;
843 default:
844 err = -EBADRQC;
845 }
846
847 if (unlikely(err))
848 relay_notice(relay, "Failed to process %s.%u from %u (%pe) %*ph\n",
849 guc_hxg_type_to_string(type), rid, origin,
850 ERR_PTR(err), 4 * len, msg);
851
852 return err;
853}
854
855/**
856 * xe_guc_relay_process_guc2vf - Handle relay notification message from the GuC.
857 * @relay: the &xe_guc_relay which will handle the message
858 * @msg: message to be handled
859 * @len: length of the message (in dwords)
860 *
861 * This function will handle relay messages received from the GuC.
862 *
863 * This function is can only be used if driver is running in SR-IOV mode.
864 *
865 * Return: 0 on success or a negative error code on failure.
866 */
867int xe_guc_relay_process_guc2vf(struct xe_guc_relay *relay, const u32 *msg, u32 len)
868{
869 u32 rid;
870
871 relay_assert(relay, len >= GUC_HXG_MSG_MIN_LEN);
872 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);
873 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);
874 relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==
875 XE_GUC_ACTION_GUC2VF_RELAY_FROM_PF);
876
877 if (unlikely(!IS_SRIOV_VF(relay_to_xe(relay)) && !kunit_get_current_test()))
878 return -EPERM;
879
880 if (unlikely(!relay_is_ready(relay)))
881 return -ENODEV;
882
883 if (unlikely(len < GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN))
884 return -EPROTO;
885
886 if (unlikely(len > GUC2VF_RELAY_FROM_PF_EVENT_MSG_MAX_LEN))
887 return -EMSGSIZE;
888
889 if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0])))
890 return -EPFNOSUPPORT;
891
892 rid = FIELD_GET(GUC2VF_RELAY_FROM_PF_EVENT_MSG_1_RELAY_ID, msg[1]);
893
894 return relay_process_msg(relay, PFID, rid,
895 msg + GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN,
896 len - GUC2VF_RELAY_FROM_PF_EVENT_MSG_MIN_LEN);
897}
898
899#ifdef CONFIG_PCI_IOV
900/**
901 * xe_guc_relay_process_guc2pf - Handle relay notification message from the GuC.
902 * @relay: the &xe_guc_relay which will handle the message
903 * @msg: message to be handled
904 * @len: length of the message (in dwords)
905 *
906 * This function will handle relay messages received from the GuC.
907 *
908 * This function can only be used if driver is running in SR-IOV PF mode.
909 *
910 * Return: 0 on success or a negative error code on failure.
911 */
912int xe_guc_relay_process_guc2pf(struct xe_guc_relay *relay, const u32 *msg, u32 len)
913{
914 u32 origin, rid;
915 int err;
916
917 relay_assert(relay, len >= GUC_HXG_EVENT_MSG_MIN_LEN);
918 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_ORIGIN, msg[0]) == GUC_HXG_ORIGIN_GUC);
919 relay_assert(relay, FIELD_GET(GUC_HXG_MSG_0_TYPE, msg[0]) == GUC_HXG_TYPE_EVENT);
920 relay_assert(relay, FIELD_GET(GUC_HXG_EVENT_MSG_0_ACTION, msg[0]) ==
921 XE_GUC_ACTION_GUC2PF_RELAY_FROM_VF);
922
923 if (unlikely(!IS_SRIOV_PF(relay_to_xe(relay)) && !kunit_get_current_test()))
924 return -EPERM;
925
926 if (unlikely(!relay_is_ready(relay)))
927 return -ENODEV;
928
929 if (unlikely(len < GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN))
930 return -EPROTO;
931
932 if (unlikely(len > GUC2PF_RELAY_FROM_VF_EVENT_MSG_MAX_LEN))
933 return -EMSGSIZE;
934
935 if (unlikely(FIELD_GET(GUC_HXG_EVENT_MSG_0_DATA0, msg[0])))
936 return -EPFNOSUPPORT;
937
938 origin = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_1_VFID, msg[1]);
939 rid = FIELD_GET(GUC2PF_RELAY_FROM_VF_EVENT_MSG_2_RELAY_ID, msg[2]);
940
941 if (unlikely(origin > relay_get_totalvfs(relay)))
942 return -ENOENT;
943
944 err = relay_process_msg(relay, origin, rid,
945 msg + GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN,
946 len - GUC2PF_RELAY_FROM_VF_EVENT_MSG_MIN_LEN);
947
948 return err;
949}
950#endif
951
952#if IS_BUILTIN(CONFIG_DRM_XE_KUNIT_TEST)
953#include "tests/xe_guc_relay_test.c"
954#endif