Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Management-Controller-to-Driver Interface
4 *
5 * Copyright 2008-2013 Solarflare Communications Inc.
6 * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
7 */
8#include <linux/delay.h>
9#include <linux/slab.h>
10#include <linux/io.h>
11#include <linux/spinlock.h>
12#include <linux/netdevice.h>
13#include <linux/etherdevice.h>
14#include <linux/ethtool.h>
15#include <linux/if_vlan.h>
16#include <linux/timer.h>
17#include <linux/list.h>
18#include <linux/pci.h>
19#include <linux/device.h>
20#include <linux/rwsem.h>
21#include <linux/vmalloc.h>
22#include <net/netevent.h>
23#include <linux/log2.h>
24#include <linux/net_tstamp.h>
25#include <linux/wait.h>
26
27#include "bitfield.h"
28#include "mcdi.h"
29
30static void cdx_mcdi_cancel_cmd(struct cdx_mcdi *cdx, struct cdx_mcdi_cmd *cmd);
31static void cdx_mcdi_wait_for_cleanup(struct cdx_mcdi *cdx);
32static int cdx_mcdi_rpc_async_internal(struct cdx_mcdi *cdx,
33 struct cdx_mcdi_cmd *cmd,
34 unsigned int *handle);
35static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,
36 bool allow_retry);
37static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,
38 struct cdx_mcdi_cmd *cmd);
39static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,
40 struct cdx_mcdi_cmd *cmd,
41 struct cdx_dword *outbuf,
42 int len,
43 struct list_head *cleanup_list);
44static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,
45 struct cdx_mcdi_cmd *cmd,
46 struct list_head *cleanup_list);
47static void cdx_mcdi_cmd_work(struct work_struct *context);
48static void cdx_mcdi_mode_fail(struct cdx_mcdi *cdx, struct list_head *cleanup_list);
49static void _cdx_mcdi_display_error(struct cdx_mcdi *cdx, unsigned int cmd,
50 size_t inlen, int raw, int arg, int err_no);
51
52static bool cdx_cmd_cancelled(struct cdx_mcdi_cmd *cmd)
53{
54 return cmd->state == MCDI_STATE_RUNNING_CANCELLED;
55}
56
57static void cdx_mcdi_cmd_release(struct kref *ref)
58{
59 kfree(container_of(ref, struct cdx_mcdi_cmd, ref));
60}
61
62static unsigned int cdx_mcdi_cmd_handle(struct cdx_mcdi_cmd *cmd)
63{
64 return cmd->handle;
65}
66
67static void _cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,
68 struct cdx_mcdi_cmd *cmd,
69 struct list_head *cleanup_list)
70{
71 /* if cancelled, the completers have already been called */
72 if (cdx_cmd_cancelled(cmd))
73 return;
74
75 if (cmd->completer) {
76 list_add_tail(&cmd->cleanup_list, cleanup_list);
77 ++mcdi->outstanding_cleanups;
78 kref_get(&cmd->ref);
79 }
80}
81
82static void cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,
83 struct cdx_mcdi_cmd *cmd,
84 struct list_head *cleanup_list)
85{
86 list_del(&cmd->list);
87 _cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
88 cmd->state = MCDI_STATE_FINISHED;
89 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
90 if (list_empty(&mcdi->cmd_list))
91 wake_up(&mcdi->cmd_complete_wq);
92}
93
94static unsigned long cdx_mcdi_rpc_timeout(struct cdx_mcdi *cdx, unsigned int cmd)
95{
96 if (!cdx->mcdi_ops->mcdi_rpc_timeout)
97 return MCDI_RPC_TIMEOUT;
98 else
99 return cdx->mcdi_ops->mcdi_rpc_timeout(cdx, cmd);
100}
101
102int cdx_mcdi_init(struct cdx_mcdi *cdx)
103{
104 struct cdx_mcdi_iface *mcdi;
105 int rc = -ENOMEM;
106
107 cdx->mcdi = kzalloc(sizeof(*cdx->mcdi), GFP_KERNEL);
108 if (!cdx->mcdi)
109 goto fail;
110
111 mcdi = cdx_mcdi_if(cdx);
112 mcdi->cdx = cdx;
113
114 mcdi->workqueue = alloc_ordered_workqueue("mcdi_wq", 0);
115 if (!mcdi->workqueue)
116 goto fail2;
117 mutex_init(&mcdi->iface_lock);
118 mcdi->mode = MCDI_MODE_EVENTS;
119 INIT_LIST_HEAD(&mcdi->cmd_list);
120 init_waitqueue_head(&mcdi->cmd_complete_wq);
121
122 mcdi->new_epoch = true;
123
124 return 0;
125fail2:
126 kfree(cdx->mcdi);
127 cdx->mcdi = NULL;
128fail:
129 return rc;
130}
131
132void cdx_mcdi_finish(struct cdx_mcdi *cdx)
133{
134 struct cdx_mcdi_iface *mcdi;
135
136 mcdi = cdx_mcdi_if(cdx);
137 if (!mcdi)
138 return;
139
140 cdx_mcdi_wait_for_cleanup(cdx);
141
142 destroy_workqueue(mcdi->workqueue);
143 kfree(cdx->mcdi);
144 cdx->mcdi = NULL;
145}
146
147static bool cdx_mcdi_flushed(struct cdx_mcdi_iface *mcdi, bool ignore_cleanups)
148{
149 bool flushed;
150
151 mutex_lock(&mcdi->iface_lock);
152 flushed = list_empty(&mcdi->cmd_list) &&
153 (ignore_cleanups || !mcdi->outstanding_cleanups);
154 mutex_unlock(&mcdi->iface_lock);
155 return flushed;
156}
157
158/* Wait for outstanding MCDI commands to complete. */
159static void cdx_mcdi_wait_for_cleanup(struct cdx_mcdi *cdx)
160{
161 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
162
163 if (!mcdi)
164 return;
165
166 wait_event(mcdi->cmd_complete_wq,
167 cdx_mcdi_flushed(mcdi, false));
168}
169
170int cdx_mcdi_wait_for_quiescence(struct cdx_mcdi *cdx,
171 unsigned int timeout_jiffies)
172{
173 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
174 DEFINE_WAIT_FUNC(wait, woken_wake_function);
175 int rc = 0;
176
177 if (!mcdi)
178 return -EINVAL;
179
180 flush_workqueue(mcdi->workqueue);
181
182 add_wait_queue(&mcdi->cmd_complete_wq, &wait);
183
184 while (!cdx_mcdi_flushed(mcdi, true)) {
185 rc = wait_woken(&wait, TASK_IDLE, timeout_jiffies);
186 if (rc)
187 continue;
188 break;
189 }
190
191 remove_wait_queue(&mcdi->cmd_complete_wq, &wait);
192
193 if (rc > 0)
194 rc = 0;
195 else if (rc == 0)
196 rc = -ETIMEDOUT;
197
198 return rc;
199}
200
201static u8 cdx_mcdi_payload_csum(const struct cdx_dword *hdr, size_t hdr_len,
202 const struct cdx_dword *sdu, size_t sdu_len)
203{
204 u8 *p = (u8 *)hdr;
205 u8 csum = 0;
206 int i;
207
208 for (i = 0; i < hdr_len; i++)
209 csum += p[i];
210
211 p = (u8 *)sdu;
212 for (i = 0; i < sdu_len; i++)
213 csum += p[i];
214
215 return ~csum & 0xff;
216}
217
218static void cdx_mcdi_send_request(struct cdx_mcdi *cdx,
219 struct cdx_mcdi_cmd *cmd)
220{
221 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
222 const struct cdx_dword *inbuf = cmd->inbuf;
223 size_t inlen = cmd->inlen;
224 struct cdx_dword hdr[2];
225 size_t hdr_len;
226 bool not_epoch;
227 u32 xflags;
228
229 if (!mcdi)
230 return;
231
232 mcdi->prev_seq = cmd->seq;
233 mcdi->seq_held_by[cmd->seq] = cmd;
234 mcdi->db_held_by = cmd;
235 cmd->started = jiffies;
236
237 not_epoch = !mcdi->new_epoch;
238 xflags = 0;
239
240 /* MCDI v2 */
241 WARN_ON(inlen > MCDI_CTL_SDU_LEN_MAX_V2);
242 CDX_POPULATE_DWORD_7(hdr[0],
243 MCDI_HEADER_RESPONSE, 0,
244 MCDI_HEADER_RESYNC, 1,
245 MCDI_HEADER_CODE, MC_CMD_V2_EXTN,
246 MCDI_HEADER_DATALEN, 0,
247 MCDI_HEADER_SEQ, cmd->seq,
248 MCDI_HEADER_XFLAGS, xflags,
249 MCDI_HEADER_NOT_EPOCH, not_epoch);
250 CDX_POPULATE_DWORD_3(hdr[1],
251 MC_CMD_V2_EXTN_IN_EXTENDED_CMD, cmd->cmd,
252 MC_CMD_V2_EXTN_IN_ACTUAL_LEN, inlen,
253 MC_CMD_V2_EXTN_IN_MESSAGE_TYPE,
254 MC_CMD_V2_EXTN_IN_MCDI_MESSAGE_TYPE_PLATFORM);
255 hdr_len = 8;
256
257 hdr[0].cdx_u32 |= (__force __le32)(cdx_mcdi_payload_csum(hdr, hdr_len, inbuf, inlen) <<
258 MCDI_HEADER_XFLAGS_LBN);
259
260 print_hex_dump_debug("MCDI REQ HEADER: ", DUMP_PREFIX_NONE, 32, 4, hdr, hdr_len, false);
261 print_hex_dump_debug("MCDI REQ PAYLOAD: ", DUMP_PREFIX_NONE, 32, 4, inbuf, inlen, false);
262
263 cdx->mcdi_ops->mcdi_request(cdx, hdr, hdr_len, inbuf, inlen);
264
265 mcdi->new_epoch = false;
266}
267
268static int cdx_mcdi_errno(struct cdx_mcdi *cdx, unsigned int mcdi_err)
269{
270 switch (mcdi_err) {
271 case 0:
272 case MC_CMD_ERR_QUEUE_FULL:
273 return mcdi_err;
274 case MC_CMD_ERR_EPERM:
275 return -EPERM;
276 case MC_CMD_ERR_ENOENT:
277 return -ENOENT;
278 case MC_CMD_ERR_EINTR:
279 return -EINTR;
280 case MC_CMD_ERR_EAGAIN:
281 return -EAGAIN;
282 case MC_CMD_ERR_EACCES:
283 return -EACCES;
284 case MC_CMD_ERR_EBUSY:
285 return -EBUSY;
286 case MC_CMD_ERR_EINVAL:
287 return -EINVAL;
288 case MC_CMD_ERR_ERANGE:
289 return -ERANGE;
290 case MC_CMD_ERR_EDEADLK:
291 return -EDEADLK;
292 case MC_CMD_ERR_ENOSYS:
293 return -EOPNOTSUPP;
294 case MC_CMD_ERR_ETIME:
295 return -ETIME;
296 case MC_CMD_ERR_EALREADY:
297 return -EALREADY;
298 case MC_CMD_ERR_ENOSPC:
299 return -ENOSPC;
300 case MC_CMD_ERR_ENOMEM:
301 return -ENOMEM;
302 case MC_CMD_ERR_ENOTSUP:
303 return -EOPNOTSUPP;
304 case MC_CMD_ERR_ALLOC_FAIL:
305 return -ENOBUFS;
306 case MC_CMD_ERR_MAC_EXIST:
307 return -EADDRINUSE;
308 case MC_CMD_ERR_NO_EVB_PORT:
309 return -EAGAIN;
310 default:
311 return -EPROTO;
312 }
313}
314
315static void cdx_mcdi_process_cleanup_list(struct cdx_mcdi *cdx,
316 struct list_head *cleanup_list)
317{
318 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
319 unsigned int cleanups = 0;
320
321 if (!mcdi)
322 return;
323
324 while (!list_empty(cleanup_list)) {
325 struct cdx_mcdi_cmd *cmd =
326 list_first_entry(cleanup_list,
327 struct cdx_mcdi_cmd, cleanup_list);
328 cmd->completer(cdx, cmd->cookie, cmd->rc,
329 cmd->outbuf, cmd->outlen);
330 list_del(&cmd->cleanup_list);
331 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
332 ++cleanups;
333 }
334
335 if (cleanups) {
336 bool all_done;
337
338 mutex_lock(&mcdi->iface_lock);
339 CDX_WARN_ON_PARANOID(cleanups > mcdi->outstanding_cleanups);
340 all_done = (mcdi->outstanding_cleanups -= cleanups) == 0;
341 mutex_unlock(&mcdi->iface_lock);
342 if (all_done)
343 wake_up(&mcdi->cmd_complete_wq);
344 }
345}
346
347static void _cdx_mcdi_cancel_cmd(struct cdx_mcdi_iface *mcdi,
348 unsigned int handle,
349 struct list_head *cleanup_list)
350{
351 struct cdx_mcdi_cmd *cmd;
352
353 list_for_each_entry(cmd, &mcdi->cmd_list, list)
354 if (cdx_mcdi_cmd_handle(cmd) == handle) {
355 switch (cmd->state) {
356 case MCDI_STATE_QUEUED:
357 case MCDI_STATE_RETRY:
358 pr_debug("command %#x inlen %zu cancelled in queue\n",
359 cmd->cmd, cmd->inlen);
360 /* if not yet running, properly cancel it */
361 cmd->rc = -EPIPE;
362 cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
363 break;
364 case MCDI_STATE_RUNNING:
365 case MCDI_STATE_RUNNING_CANCELLED:
366 case MCDI_STATE_FINISHED:
367 default:
368 /* invalid state? */
369 WARN_ON(1);
370 }
371 break;
372 }
373}
374
375static void cdx_mcdi_cancel_cmd(struct cdx_mcdi *cdx, struct cdx_mcdi_cmd *cmd)
376{
377 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
378 LIST_HEAD(cleanup_list);
379
380 if (!mcdi)
381 return;
382
383 mutex_lock(&mcdi->iface_lock);
384 cdx_mcdi_timeout_cmd(mcdi, cmd, &cleanup_list);
385 mutex_unlock(&mcdi->iface_lock);
386 cdx_mcdi_process_cleanup_list(cdx, &cleanup_list);
387}
388
389struct cdx_mcdi_blocking_data {
390 struct kref ref;
391 bool done;
392 wait_queue_head_t wq;
393 int rc;
394 struct cdx_dword *outbuf;
395 size_t outlen;
396 size_t outlen_actual;
397};
398
399static void cdx_mcdi_blocking_data_release(struct kref *ref)
400{
401 kfree(container_of(ref, struct cdx_mcdi_blocking_data, ref));
402}
403
404static void cdx_mcdi_rpc_completer(struct cdx_mcdi *cdx, unsigned long cookie,
405 int rc, struct cdx_dword *outbuf,
406 size_t outlen_actual)
407{
408 struct cdx_mcdi_blocking_data *wait_data =
409 (struct cdx_mcdi_blocking_data *)cookie;
410
411 wait_data->rc = rc;
412 memcpy(wait_data->outbuf, outbuf,
413 min(outlen_actual, wait_data->outlen));
414 wait_data->outlen_actual = outlen_actual;
415 /* memory barrier */
416 smp_wmb();
417 wait_data->done = true;
418 wake_up(&wait_data->wq);
419 kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
420}
421
422static int cdx_mcdi_rpc_sync(struct cdx_mcdi *cdx, unsigned int cmd,
423 const struct cdx_dword *inbuf, size_t inlen,
424 struct cdx_dword *outbuf, size_t outlen,
425 size_t *outlen_actual, bool quiet)
426{
427 struct cdx_mcdi_blocking_data *wait_data;
428 struct cdx_mcdi_cmd *cmd_item;
429 unsigned int handle;
430 int rc;
431
432 if (outlen_actual)
433 *outlen_actual = 0;
434
435 wait_data = kmalloc(sizeof(*wait_data), GFP_KERNEL);
436 if (!wait_data)
437 return -ENOMEM;
438
439 cmd_item = kmalloc(sizeof(*cmd_item), GFP_KERNEL);
440 if (!cmd_item) {
441 kfree(wait_data);
442 return -ENOMEM;
443 }
444
445 kref_init(&wait_data->ref);
446 wait_data->done = false;
447 init_waitqueue_head(&wait_data->wq);
448 wait_data->outbuf = outbuf;
449 wait_data->outlen = outlen;
450
451 kref_init(&cmd_item->ref);
452 cmd_item->quiet = quiet;
453 cmd_item->cookie = (unsigned long)wait_data;
454 cmd_item->completer = &cdx_mcdi_rpc_completer;
455 cmd_item->cmd = cmd;
456 cmd_item->inlen = inlen;
457 cmd_item->inbuf = inbuf;
458
459 /* Claim an extra reference for the completer to put. */
460 kref_get(&wait_data->ref);
461 rc = cdx_mcdi_rpc_async_internal(cdx, cmd_item, &handle);
462 if (rc) {
463 kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
464 goto out;
465 }
466
467 if (!wait_event_timeout(wait_data->wq, wait_data->done,
468 cdx_mcdi_rpc_timeout(cdx, cmd)) &&
469 !wait_data->done) {
470 pr_err("MC command 0x%x inlen %zu timed out (sync)\n",
471 cmd, inlen);
472
473 cdx_mcdi_cancel_cmd(cdx, cmd_item);
474
475 wait_data->rc = -ETIMEDOUT;
476 wait_data->outlen_actual = 0;
477 }
478
479 if (outlen_actual)
480 *outlen_actual = wait_data->outlen_actual;
481 rc = wait_data->rc;
482
483out:
484 kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
485
486 return rc;
487}
488
489static bool cdx_mcdi_get_seq(struct cdx_mcdi_iface *mcdi, unsigned char *seq)
490{
491 *seq = mcdi->prev_seq;
492 do {
493 *seq = (*seq + 1) % ARRAY_SIZE(mcdi->seq_held_by);
494 } while (mcdi->seq_held_by[*seq] && *seq != mcdi->prev_seq);
495 return !mcdi->seq_held_by[*seq];
496}
497
498static int cdx_mcdi_rpc_async_internal(struct cdx_mcdi *cdx,
499 struct cdx_mcdi_cmd *cmd,
500 unsigned int *handle)
501{
502 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
503 LIST_HEAD(cleanup_list);
504
505 if (!mcdi) {
506 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
507 return -ENETDOWN;
508 }
509
510 if (mcdi->mode == MCDI_MODE_FAIL) {
511 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
512 return -ENETDOWN;
513 }
514
515 cmd->mcdi = mcdi;
516 INIT_WORK(&cmd->work, cdx_mcdi_cmd_work);
517 INIT_LIST_HEAD(&cmd->list);
518 INIT_LIST_HEAD(&cmd->cleanup_list);
519 cmd->rc = 0;
520 cmd->outbuf = NULL;
521 cmd->outlen = 0;
522
523 queue_work(mcdi->workqueue, &cmd->work);
524 return 0;
525}
526
527static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,
528 struct cdx_mcdi_cmd *cmd)
529{
530 struct cdx_mcdi *cdx = mcdi->cdx;
531 u8 seq;
532
533 if (!mcdi->db_held_by &&
534 cdx_mcdi_get_seq(mcdi, &seq)) {
535 cmd->seq = seq;
536 cmd->reboot_seen = false;
537 cdx_mcdi_send_request(cdx, cmd);
538 cmd->state = MCDI_STATE_RUNNING;
539 } else {
540 cmd->state = MCDI_STATE_QUEUED;
541 }
542}
543
544/* try to advance other commands */
545static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,
546 bool allow_retry)
547{
548 struct cdx_mcdi_cmd *cmd, *tmp;
549
550 list_for_each_entry_safe(cmd, tmp, &mcdi->cmd_list, list)
551 if (cmd->state == MCDI_STATE_QUEUED ||
552 (cmd->state == MCDI_STATE_RETRY && allow_retry))
553 cdx_mcdi_cmd_start_or_queue(mcdi, cmd);
554}
555
556void cdx_mcdi_process_cmd(struct cdx_mcdi *cdx, struct cdx_dword *outbuf, int len)
557{
558 struct cdx_mcdi_iface *mcdi;
559 struct cdx_mcdi_cmd *cmd;
560 LIST_HEAD(cleanup_list);
561 unsigned int respseq;
562
563 if (!len || !outbuf) {
564 pr_err("Got empty MC response\n");
565 return;
566 }
567
568 mcdi = cdx_mcdi_if(cdx);
569 if (!mcdi)
570 return;
571
572 respseq = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_SEQ);
573
574 mutex_lock(&mcdi->iface_lock);
575 cmd = mcdi->seq_held_by[respseq];
576
577 if (cmd) {
578 if (cmd->state == MCDI_STATE_FINISHED) {
579 mutex_unlock(&mcdi->iface_lock);
580 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
581 return;
582 }
583
584 cdx_mcdi_complete_cmd(mcdi, cmd, outbuf, len, &cleanup_list);
585 } else {
586 pr_err("MC response unexpected for seq : %0X\n", respseq);
587 }
588
589 mutex_unlock(&mcdi->iface_lock);
590
591 cdx_mcdi_process_cleanup_list(mcdi->cdx, &cleanup_list);
592}
593
594static void cdx_mcdi_cmd_work(struct work_struct *context)
595{
596 struct cdx_mcdi_cmd *cmd =
597 container_of(context, struct cdx_mcdi_cmd, work);
598 struct cdx_mcdi_iface *mcdi = cmd->mcdi;
599
600 mutex_lock(&mcdi->iface_lock);
601
602 cmd->handle = mcdi->prev_handle++;
603 list_add_tail(&cmd->list, &mcdi->cmd_list);
604 cdx_mcdi_cmd_start_or_queue(mcdi, cmd);
605
606 mutex_unlock(&mcdi->iface_lock);
607}
608
609/*
610 * Returns true if the MCDI module is finished with the command.
611 * (examples of false would be if the command was proxied, or it was
612 * rejected by the MC due to lack of resources and requeued).
613 */
614static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,
615 struct cdx_mcdi_cmd *cmd,
616 struct cdx_dword *outbuf,
617 int len,
618 struct list_head *cleanup_list)
619{
620 size_t resp_hdr_len, resp_data_len;
621 struct cdx_mcdi *cdx = mcdi->cdx;
622 unsigned int respcmd, error;
623 bool completed = false;
624 int rc;
625
626 /* ensure the command can't go away before this function returns */
627 kref_get(&cmd->ref);
628
629 respcmd = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_CODE);
630 error = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_ERROR);
631
632 if (respcmd != MC_CMD_V2_EXTN) {
633 resp_hdr_len = 4;
634 resp_data_len = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_DATALEN);
635 } else {
636 resp_data_len = 0;
637 resp_hdr_len = 8;
638 if (len >= 8)
639 resp_data_len =
640 CDX_DWORD_FIELD(outbuf[1], MC_CMD_V2_EXTN_IN_ACTUAL_LEN);
641 }
642
643 if ((resp_hdr_len + resp_data_len) > len) {
644 pr_warn("Incomplete MCDI response received %d. Expected %zu\n",
645 len, (resp_hdr_len + resp_data_len));
646 resp_data_len = 0;
647 }
648
649 print_hex_dump_debug("MCDI RESP HEADER: ", DUMP_PREFIX_NONE, 32, 4,
650 outbuf, resp_hdr_len, false);
651 print_hex_dump_debug("MCDI RESP PAYLOAD: ", DUMP_PREFIX_NONE, 32, 4,
652 outbuf + (resp_hdr_len / 4), resp_data_len, false);
653
654 if (error && resp_data_len == 0) {
655 /* MC rebooted during command */
656 rc = -EIO;
657 } else {
658 if (WARN_ON_ONCE(error && resp_data_len < 4))
659 resp_data_len = 4;
660 if (error) {
661 rc = CDX_DWORD_FIELD(outbuf[resp_hdr_len / 4], CDX_DWORD);
662 if (!cmd->quiet) {
663 int err_arg = 0;
664
665 if (resp_data_len >= MC_CMD_ERR_ARG_OFST + 4) {
666 int offset = (resp_hdr_len + MC_CMD_ERR_ARG_OFST) / 4;
667
668 err_arg = CDX_DWORD_VAL(outbuf[offset]);
669 }
670
671 _cdx_mcdi_display_error(cdx, cmd->cmd,
672 cmd->inlen, rc, err_arg,
673 cdx_mcdi_errno(cdx, rc));
674 }
675 rc = cdx_mcdi_errno(cdx, rc);
676 } else {
677 rc = 0;
678 }
679 }
680
681 /* free doorbell */
682 if (mcdi->db_held_by == cmd)
683 mcdi->db_held_by = NULL;
684
685 if (cdx_cmd_cancelled(cmd)) {
686 list_del(&cmd->list);
687 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
688 completed = true;
689 } else if (rc == MC_CMD_ERR_QUEUE_FULL) {
690 cmd->state = MCDI_STATE_RETRY;
691 } else {
692 cmd->rc = rc;
693 cmd->outbuf = outbuf + DIV_ROUND_UP(resp_hdr_len, 4);
694 cmd->outlen = resp_data_len;
695 cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
696 completed = true;
697 }
698
699 /* free sequence number and buffer */
700 mcdi->seq_held_by[cmd->seq] = NULL;
701
702 cdx_mcdi_start_or_queue(mcdi, rc != MC_CMD_ERR_QUEUE_FULL);
703
704 /* wake up anyone waiting for flush */
705 wake_up(&mcdi->cmd_complete_wq);
706
707 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
708
709 return completed;
710}
711
712static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,
713 struct cdx_mcdi_cmd *cmd,
714 struct list_head *cleanup_list)
715{
716 struct cdx_mcdi *cdx = mcdi->cdx;
717
718 pr_err("MC command 0x%x inlen %zu state %d timed out after %u ms\n",
719 cmd->cmd, cmd->inlen, cmd->state,
720 jiffies_to_msecs(jiffies - cmd->started));
721
722 cmd->rc = -ETIMEDOUT;
723 cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
724
725 cdx_mcdi_mode_fail(cdx, cleanup_list);
726}
727
728/**
729 * cdx_mcdi_rpc - Issue an MCDI command and wait for completion
730 * @cdx: NIC through which to issue the command
731 * @cmd: Command type number
732 * @inbuf: Command parameters
733 * @inlen: Length of command parameters, in bytes. Must be a multiple
734 * of 4 and no greater than %MCDI_CTL_SDU_LEN_MAX_V1.
735 * @outbuf: Response buffer. May be %NULL if @outlen is 0.
736 * @outlen: Length of response buffer, in bytes. If the actual
737 * response is longer than @outlen & ~3, it will be truncated
738 * to that length.
739 * @outlen_actual: Pointer through which to return the actual response
740 * length. May be %NULL if this is not needed.
741 *
742 * This function may sleep and therefore must be called in process
743 * context.
744 *
745 * Return: A negative error code, or zero if successful. The error
746 * code may come from the MCDI response or may indicate a failure
747 * to communicate with the MC. In the former case, the response
748 * will still be copied to @outbuf and *@outlen_actual will be
749 * set accordingly. In the latter case, *@outlen_actual will be
750 * set to zero.
751 */
752int cdx_mcdi_rpc(struct cdx_mcdi *cdx, unsigned int cmd,
753 const struct cdx_dword *inbuf, size_t inlen,
754 struct cdx_dword *outbuf, size_t outlen,
755 size_t *outlen_actual)
756{
757 return cdx_mcdi_rpc_sync(cdx, cmd, inbuf, inlen, outbuf, outlen,
758 outlen_actual, false);
759}
760
761/**
762 * cdx_mcdi_rpc_async - Schedule an MCDI command to run asynchronously
763 * @cdx: NIC through which to issue the command
764 * @cmd: Command type number
765 * @inbuf: Command parameters
766 * @inlen: Length of command parameters, in bytes
767 * @complete: Function to be called on completion or cancellation.
768 * @cookie: Arbitrary value to be passed to @complete.
769 *
770 * This function does not sleep and therefore may be called in atomic
771 * context. It will fail if event queues are disabled or if MCDI
772 * event completions have been disabled due to an error.
773 *
774 * If it succeeds, the @complete function will be called exactly once
775 * in process context, when one of the following occurs:
776 * (a) the completion event is received (in process context)
777 * (b) event queues are disabled (in the process that disables them)
778 */
779int
780cdx_mcdi_rpc_async(struct cdx_mcdi *cdx, unsigned int cmd,
781 const struct cdx_dword *inbuf, size_t inlen,
782 cdx_mcdi_async_completer *complete, unsigned long cookie)
783{
784 struct cdx_mcdi_cmd *cmd_item =
785 kmalloc(sizeof(struct cdx_mcdi_cmd) + inlen, GFP_ATOMIC);
786
787 if (!cmd_item)
788 return -ENOMEM;
789
790 kref_init(&cmd_item->ref);
791 cmd_item->quiet = true;
792 cmd_item->cookie = cookie;
793 cmd_item->completer = complete;
794 cmd_item->cmd = cmd;
795 cmd_item->inlen = inlen;
796 /* inbuf is probably not valid after return, so take a copy */
797 cmd_item->inbuf = (struct cdx_dword *)(cmd_item + 1);
798 memcpy(cmd_item + 1, inbuf, inlen);
799
800 return cdx_mcdi_rpc_async_internal(cdx, cmd_item, NULL);
801}
802
803static void _cdx_mcdi_display_error(struct cdx_mcdi *cdx, unsigned int cmd,
804 size_t inlen, int raw, int arg, int err_no)
805{
806 pr_err("MC command 0x%x inlen %d failed err_no=%d (raw=%d) arg=%d\n",
807 cmd, (int)inlen, err_no, raw, arg);
808}
809
810/*
811 * Set MCDI mode to fail to prevent any new commands, then cancel any
812 * outstanding commands.
813 * Caller must hold the mcdi iface_lock.
814 */
815static void cdx_mcdi_mode_fail(struct cdx_mcdi *cdx, struct list_head *cleanup_list)
816{
817 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
818
819 if (!mcdi)
820 return;
821
822 mcdi->mode = MCDI_MODE_FAIL;
823
824 while (!list_empty(&mcdi->cmd_list)) {
825 struct cdx_mcdi_cmd *cmd;
826
827 cmd = list_first_entry(&mcdi->cmd_list, struct cdx_mcdi_cmd,
828 list);
829 _cdx_mcdi_cancel_cmd(mcdi, cdx_mcdi_cmd_handle(cmd), cleanup_list);
830 }
831}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Management-Controller-to-Driver Interface
4 *
5 * Copyright 2008-2013 Solarflare Communications Inc.
6 * Copyright (C) 2022-2023, Advanced Micro Devices, Inc.
7 */
8#include <linux/delay.h>
9#include <linux/slab.h>
10#include <linux/io.h>
11#include <linux/spinlock.h>
12#include <linux/netdevice.h>
13#include <linux/etherdevice.h>
14#include <linux/ethtool.h>
15#include <linux/if_vlan.h>
16#include <linux/timer.h>
17#include <linux/list.h>
18#include <linux/pci.h>
19#include <linux/device.h>
20#include <linux/rwsem.h>
21#include <linux/vmalloc.h>
22#include <net/netevent.h>
23#include <linux/log2.h>
24#include <linux/net_tstamp.h>
25#include <linux/wait.h>
26
27#include "bitfield.h"
28#include "mcdi.h"
29
30struct cdx_mcdi_copy_buffer {
31 struct cdx_dword buffer[DIV_ROUND_UP(MCDI_CTL_SDU_LEN_MAX, 4)];
32};
33
34static void cdx_mcdi_cancel_cmd(struct cdx_mcdi *cdx, struct cdx_mcdi_cmd *cmd);
35static void cdx_mcdi_wait_for_cleanup(struct cdx_mcdi *cdx);
36static int cdx_mcdi_rpc_async_internal(struct cdx_mcdi *cdx,
37 struct cdx_mcdi_cmd *cmd,
38 unsigned int *handle);
39static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,
40 bool allow_retry);
41static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,
42 struct cdx_mcdi_cmd *cmd);
43static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,
44 struct cdx_mcdi_cmd *cmd,
45 struct cdx_dword *outbuf,
46 int len,
47 struct list_head *cleanup_list);
48static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,
49 struct cdx_mcdi_cmd *cmd,
50 struct list_head *cleanup_list);
51static void cdx_mcdi_cmd_work(struct work_struct *context);
52static void cdx_mcdi_mode_fail(struct cdx_mcdi *cdx, struct list_head *cleanup_list);
53static void _cdx_mcdi_display_error(struct cdx_mcdi *cdx, unsigned int cmd,
54 size_t inlen, int raw, int arg, int err_no);
55
56static bool cdx_cmd_cancelled(struct cdx_mcdi_cmd *cmd)
57{
58 return cmd->state == MCDI_STATE_RUNNING_CANCELLED;
59}
60
61static void cdx_mcdi_cmd_release(struct kref *ref)
62{
63 kfree(container_of(ref, struct cdx_mcdi_cmd, ref));
64}
65
66static unsigned int cdx_mcdi_cmd_handle(struct cdx_mcdi_cmd *cmd)
67{
68 return cmd->handle;
69}
70
71static void _cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,
72 struct cdx_mcdi_cmd *cmd,
73 struct list_head *cleanup_list)
74{
75 /* if cancelled, the completers have already been called */
76 if (cdx_cmd_cancelled(cmd))
77 return;
78
79 if (cmd->completer) {
80 list_add_tail(&cmd->cleanup_list, cleanup_list);
81 ++mcdi->outstanding_cleanups;
82 kref_get(&cmd->ref);
83 }
84}
85
86static void cdx_mcdi_remove_cmd(struct cdx_mcdi_iface *mcdi,
87 struct cdx_mcdi_cmd *cmd,
88 struct list_head *cleanup_list)
89{
90 list_del(&cmd->list);
91 _cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
92 cmd->state = MCDI_STATE_FINISHED;
93 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
94 if (list_empty(&mcdi->cmd_list))
95 wake_up(&mcdi->cmd_complete_wq);
96}
97
98static unsigned long cdx_mcdi_rpc_timeout(struct cdx_mcdi *cdx, unsigned int cmd)
99{
100 if (!cdx->mcdi_ops->mcdi_rpc_timeout)
101 return MCDI_RPC_TIMEOUT;
102 else
103 return cdx->mcdi_ops->mcdi_rpc_timeout(cdx, cmd);
104}
105
106int cdx_mcdi_init(struct cdx_mcdi *cdx)
107{
108 struct cdx_mcdi_iface *mcdi;
109 int rc = -ENOMEM;
110
111 cdx->mcdi = kzalloc(sizeof(*cdx->mcdi), GFP_KERNEL);
112 if (!cdx->mcdi)
113 goto fail;
114
115 mcdi = cdx_mcdi_if(cdx);
116 mcdi->cdx = cdx;
117
118 mcdi->workqueue = alloc_ordered_workqueue("mcdi_wq", 0);
119 if (!mcdi->workqueue)
120 goto fail2;
121 mutex_init(&mcdi->iface_lock);
122 mcdi->mode = MCDI_MODE_EVENTS;
123 INIT_LIST_HEAD(&mcdi->cmd_list);
124 init_waitqueue_head(&mcdi->cmd_complete_wq);
125
126 mcdi->new_epoch = true;
127
128 return 0;
129fail2:
130 kfree(cdx->mcdi);
131 cdx->mcdi = NULL;
132fail:
133 return rc;
134}
135
136void cdx_mcdi_finish(struct cdx_mcdi *cdx)
137{
138 struct cdx_mcdi_iface *mcdi;
139
140 mcdi = cdx_mcdi_if(cdx);
141 if (!mcdi)
142 return;
143
144 cdx_mcdi_wait_for_cleanup(cdx);
145
146 destroy_workqueue(mcdi->workqueue);
147 kfree(cdx->mcdi);
148 cdx->mcdi = NULL;
149}
150
151static bool cdx_mcdi_flushed(struct cdx_mcdi_iface *mcdi, bool ignore_cleanups)
152{
153 bool flushed;
154
155 mutex_lock(&mcdi->iface_lock);
156 flushed = list_empty(&mcdi->cmd_list) &&
157 (ignore_cleanups || !mcdi->outstanding_cleanups);
158 mutex_unlock(&mcdi->iface_lock);
159 return flushed;
160}
161
162/* Wait for outstanding MCDI commands to complete. */
163static void cdx_mcdi_wait_for_cleanup(struct cdx_mcdi *cdx)
164{
165 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
166
167 if (!mcdi)
168 return;
169
170 wait_event(mcdi->cmd_complete_wq,
171 cdx_mcdi_flushed(mcdi, false));
172}
173
174int cdx_mcdi_wait_for_quiescence(struct cdx_mcdi *cdx,
175 unsigned int timeout_jiffies)
176{
177 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
178 DEFINE_WAIT_FUNC(wait, woken_wake_function);
179 int rc = 0;
180
181 if (!mcdi)
182 return -EINVAL;
183
184 flush_workqueue(mcdi->workqueue);
185
186 add_wait_queue(&mcdi->cmd_complete_wq, &wait);
187
188 while (!cdx_mcdi_flushed(mcdi, true)) {
189 rc = wait_woken(&wait, TASK_IDLE, timeout_jiffies);
190 if (rc)
191 continue;
192 break;
193 }
194
195 remove_wait_queue(&mcdi->cmd_complete_wq, &wait);
196
197 if (rc > 0)
198 rc = 0;
199 else if (rc == 0)
200 rc = -ETIMEDOUT;
201
202 return rc;
203}
204
205static u8 cdx_mcdi_payload_csum(const struct cdx_dword *hdr, size_t hdr_len,
206 const struct cdx_dword *sdu, size_t sdu_len)
207{
208 u8 *p = (u8 *)hdr;
209 u8 csum = 0;
210 int i;
211
212 for (i = 0; i < hdr_len; i++)
213 csum += p[i];
214
215 p = (u8 *)sdu;
216 for (i = 0; i < sdu_len; i++)
217 csum += p[i];
218
219 return ~csum & 0xff;
220}
221
222static void cdx_mcdi_send_request(struct cdx_mcdi *cdx,
223 struct cdx_mcdi_cmd *cmd)
224{
225 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
226 const struct cdx_dword *inbuf = cmd->inbuf;
227 size_t inlen = cmd->inlen;
228 struct cdx_dword hdr[2];
229 size_t hdr_len;
230 bool not_epoch;
231 u32 xflags;
232
233 if (!mcdi)
234 return;
235
236 mcdi->prev_seq = cmd->seq;
237 mcdi->seq_held_by[cmd->seq] = cmd;
238 mcdi->db_held_by = cmd;
239 cmd->started = jiffies;
240
241 not_epoch = !mcdi->new_epoch;
242 xflags = 0;
243
244 /* MCDI v2 */
245 WARN_ON(inlen > MCDI_CTL_SDU_LEN_MAX_V2);
246 CDX_POPULATE_DWORD_7(hdr[0],
247 MCDI_HEADER_RESPONSE, 0,
248 MCDI_HEADER_RESYNC, 1,
249 MCDI_HEADER_CODE, MC_CMD_V2_EXTN,
250 MCDI_HEADER_DATALEN, 0,
251 MCDI_HEADER_SEQ, cmd->seq,
252 MCDI_HEADER_XFLAGS, xflags,
253 MCDI_HEADER_NOT_EPOCH, not_epoch);
254 CDX_POPULATE_DWORD_3(hdr[1],
255 MC_CMD_V2_EXTN_IN_EXTENDED_CMD, cmd->cmd,
256 MC_CMD_V2_EXTN_IN_ACTUAL_LEN, inlen,
257 MC_CMD_V2_EXTN_IN_MESSAGE_TYPE,
258 MC_CMD_V2_EXTN_IN_MCDI_MESSAGE_TYPE_PLATFORM);
259 hdr_len = 8;
260
261 hdr[0].cdx_u32 |= (__force __le32)(cdx_mcdi_payload_csum(hdr, hdr_len, inbuf, inlen) <<
262 MCDI_HEADER_XFLAGS_LBN);
263
264 print_hex_dump_debug("MCDI REQ HEADER: ", DUMP_PREFIX_NONE, 32, 4, hdr, hdr_len, false);
265 print_hex_dump_debug("MCDI REQ PAYLOAD: ", DUMP_PREFIX_NONE, 32, 4, inbuf, inlen, false);
266
267 cdx->mcdi_ops->mcdi_request(cdx, hdr, hdr_len, inbuf, inlen);
268
269 mcdi->new_epoch = false;
270}
271
272static int cdx_mcdi_errno(struct cdx_mcdi *cdx, unsigned int mcdi_err)
273{
274 switch (mcdi_err) {
275 case 0:
276 case MC_CMD_ERR_QUEUE_FULL:
277 return mcdi_err;
278 case MC_CMD_ERR_EPERM:
279 return -EPERM;
280 case MC_CMD_ERR_ENOENT:
281 return -ENOENT;
282 case MC_CMD_ERR_EINTR:
283 return -EINTR;
284 case MC_CMD_ERR_EAGAIN:
285 return -EAGAIN;
286 case MC_CMD_ERR_EACCES:
287 return -EACCES;
288 case MC_CMD_ERR_EBUSY:
289 return -EBUSY;
290 case MC_CMD_ERR_EINVAL:
291 return -EINVAL;
292 case MC_CMD_ERR_ERANGE:
293 return -ERANGE;
294 case MC_CMD_ERR_EDEADLK:
295 return -EDEADLK;
296 case MC_CMD_ERR_ENOSYS:
297 return -EOPNOTSUPP;
298 case MC_CMD_ERR_ETIME:
299 return -ETIME;
300 case MC_CMD_ERR_EALREADY:
301 return -EALREADY;
302 case MC_CMD_ERR_ENOSPC:
303 return -ENOSPC;
304 case MC_CMD_ERR_ENOMEM:
305 return -ENOMEM;
306 case MC_CMD_ERR_ENOTSUP:
307 return -EOPNOTSUPP;
308 case MC_CMD_ERR_ALLOC_FAIL:
309 return -ENOBUFS;
310 case MC_CMD_ERR_MAC_EXIST:
311 return -EADDRINUSE;
312 case MC_CMD_ERR_NO_EVB_PORT:
313 return -EAGAIN;
314 default:
315 return -EPROTO;
316 }
317}
318
319static void cdx_mcdi_process_cleanup_list(struct cdx_mcdi *cdx,
320 struct list_head *cleanup_list)
321{
322 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
323 unsigned int cleanups = 0;
324
325 if (!mcdi)
326 return;
327
328 while (!list_empty(cleanup_list)) {
329 struct cdx_mcdi_cmd *cmd =
330 list_first_entry(cleanup_list,
331 struct cdx_mcdi_cmd, cleanup_list);
332 cmd->completer(cdx, cmd->cookie, cmd->rc,
333 cmd->outbuf, cmd->outlen);
334 list_del(&cmd->cleanup_list);
335 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
336 ++cleanups;
337 }
338
339 if (cleanups) {
340 bool all_done;
341
342 mutex_lock(&mcdi->iface_lock);
343 CDX_WARN_ON_PARANOID(cleanups > mcdi->outstanding_cleanups);
344 all_done = (mcdi->outstanding_cleanups -= cleanups) == 0;
345 mutex_unlock(&mcdi->iface_lock);
346 if (all_done)
347 wake_up(&mcdi->cmd_complete_wq);
348 }
349}
350
351static void _cdx_mcdi_cancel_cmd(struct cdx_mcdi_iface *mcdi,
352 unsigned int handle,
353 struct list_head *cleanup_list)
354{
355 struct cdx_mcdi_cmd *cmd;
356
357 list_for_each_entry(cmd, &mcdi->cmd_list, list)
358 if (cdx_mcdi_cmd_handle(cmd) == handle) {
359 switch (cmd->state) {
360 case MCDI_STATE_QUEUED:
361 case MCDI_STATE_RETRY:
362 pr_debug("command %#x inlen %zu cancelled in queue\n",
363 cmd->cmd, cmd->inlen);
364 /* if not yet running, properly cancel it */
365 cmd->rc = -EPIPE;
366 cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
367 break;
368 case MCDI_STATE_RUNNING:
369 case MCDI_STATE_RUNNING_CANCELLED:
370 case MCDI_STATE_FINISHED:
371 default:
372 /* invalid state? */
373 WARN_ON(1);
374 }
375 break;
376 }
377}
378
379static void cdx_mcdi_cancel_cmd(struct cdx_mcdi *cdx, struct cdx_mcdi_cmd *cmd)
380{
381 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
382 LIST_HEAD(cleanup_list);
383
384 if (!mcdi)
385 return;
386
387 mutex_lock(&mcdi->iface_lock);
388 cdx_mcdi_timeout_cmd(mcdi, cmd, &cleanup_list);
389 mutex_unlock(&mcdi->iface_lock);
390 cdx_mcdi_process_cleanup_list(cdx, &cleanup_list);
391}
392
393struct cdx_mcdi_blocking_data {
394 struct kref ref;
395 bool done;
396 wait_queue_head_t wq;
397 int rc;
398 struct cdx_dword *outbuf;
399 size_t outlen;
400 size_t outlen_actual;
401};
402
403static void cdx_mcdi_blocking_data_release(struct kref *ref)
404{
405 kfree(container_of(ref, struct cdx_mcdi_blocking_data, ref));
406}
407
408static void cdx_mcdi_rpc_completer(struct cdx_mcdi *cdx, unsigned long cookie,
409 int rc, struct cdx_dword *outbuf,
410 size_t outlen_actual)
411{
412 struct cdx_mcdi_blocking_data *wait_data =
413 (struct cdx_mcdi_blocking_data *)cookie;
414
415 wait_data->rc = rc;
416 memcpy(wait_data->outbuf, outbuf,
417 min(outlen_actual, wait_data->outlen));
418 wait_data->outlen_actual = outlen_actual;
419 /* memory barrier */
420 smp_wmb();
421 wait_data->done = true;
422 wake_up(&wait_data->wq);
423 kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
424}
425
426static int cdx_mcdi_rpc_sync(struct cdx_mcdi *cdx, unsigned int cmd,
427 const struct cdx_dword *inbuf, size_t inlen,
428 struct cdx_dword *outbuf, size_t outlen,
429 size_t *outlen_actual, bool quiet)
430{
431 struct cdx_mcdi_blocking_data *wait_data;
432 struct cdx_mcdi_cmd *cmd_item;
433 unsigned int handle;
434 int rc;
435
436 if (outlen_actual)
437 *outlen_actual = 0;
438
439 wait_data = kmalloc(sizeof(*wait_data), GFP_KERNEL);
440 if (!wait_data)
441 return -ENOMEM;
442
443 cmd_item = kmalloc(sizeof(*cmd_item), GFP_KERNEL);
444 if (!cmd_item) {
445 kfree(wait_data);
446 return -ENOMEM;
447 }
448
449 kref_init(&wait_data->ref);
450 wait_data->done = false;
451 init_waitqueue_head(&wait_data->wq);
452 wait_data->outbuf = outbuf;
453 wait_data->outlen = outlen;
454
455 kref_init(&cmd_item->ref);
456 cmd_item->quiet = quiet;
457 cmd_item->cookie = (unsigned long)wait_data;
458 cmd_item->completer = &cdx_mcdi_rpc_completer;
459 cmd_item->cmd = cmd;
460 cmd_item->inlen = inlen;
461 cmd_item->inbuf = inbuf;
462
463 /* Claim an extra reference for the completer to put. */
464 kref_get(&wait_data->ref);
465 rc = cdx_mcdi_rpc_async_internal(cdx, cmd_item, &handle);
466 if (rc) {
467 kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
468 goto out;
469 }
470
471 if (!wait_event_timeout(wait_data->wq, wait_data->done,
472 cdx_mcdi_rpc_timeout(cdx, cmd)) &&
473 !wait_data->done) {
474 pr_err("MC command 0x%x inlen %zu timed out (sync)\n",
475 cmd, inlen);
476
477 cdx_mcdi_cancel_cmd(cdx, cmd_item);
478
479 wait_data->rc = -ETIMEDOUT;
480 wait_data->outlen_actual = 0;
481 }
482
483 if (outlen_actual)
484 *outlen_actual = wait_data->outlen_actual;
485 rc = wait_data->rc;
486
487out:
488 kref_put(&wait_data->ref, cdx_mcdi_blocking_data_release);
489
490 return rc;
491}
492
493static bool cdx_mcdi_get_seq(struct cdx_mcdi_iface *mcdi, unsigned char *seq)
494{
495 *seq = mcdi->prev_seq;
496 do {
497 *seq = (*seq + 1) % ARRAY_SIZE(mcdi->seq_held_by);
498 } while (mcdi->seq_held_by[*seq] && *seq != mcdi->prev_seq);
499 return !mcdi->seq_held_by[*seq];
500}
501
502static int cdx_mcdi_rpc_async_internal(struct cdx_mcdi *cdx,
503 struct cdx_mcdi_cmd *cmd,
504 unsigned int *handle)
505{
506 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
507 LIST_HEAD(cleanup_list);
508
509 if (!mcdi) {
510 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
511 return -ENETDOWN;
512 }
513
514 if (mcdi->mode == MCDI_MODE_FAIL) {
515 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
516 return -ENETDOWN;
517 }
518
519 cmd->mcdi = mcdi;
520 INIT_WORK(&cmd->work, cdx_mcdi_cmd_work);
521 INIT_LIST_HEAD(&cmd->list);
522 INIT_LIST_HEAD(&cmd->cleanup_list);
523 cmd->rc = 0;
524 cmd->outbuf = NULL;
525 cmd->outlen = 0;
526
527 queue_work(mcdi->workqueue, &cmd->work);
528 return 0;
529}
530
531static void cdx_mcdi_cmd_start_or_queue(struct cdx_mcdi_iface *mcdi,
532 struct cdx_mcdi_cmd *cmd)
533{
534 struct cdx_mcdi *cdx = mcdi->cdx;
535 u8 seq;
536
537 if (!mcdi->db_held_by &&
538 cdx_mcdi_get_seq(mcdi, &seq)) {
539 cmd->seq = seq;
540 cmd->reboot_seen = false;
541 cdx_mcdi_send_request(cdx, cmd);
542 cmd->state = MCDI_STATE_RUNNING;
543 } else {
544 cmd->state = MCDI_STATE_QUEUED;
545 }
546}
547
548/* try to advance other commands */
549static void cdx_mcdi_start_or_queue(struct cdx_mcdi_iface *mcdi,
550 bool allow_retry)
551{
552 struct cdx_mcdi_cmd *cmd, *tmp;
553
554 list_for_each_entry_safe(cmd, tmp, &mcdi->cmd_list, list)
555 if (cmd->state == MCDI_STATE_QUEUED ||
556 (cmd->state == MCDI_STATE_RETRY && allow_retry))
557 cdx_mcdi_cmd_start_or_queue(mcdi, cmd);
558}
559
560void cdx_mcdi_process_cmd(struct cdx_mcdi *cdx, struct cdx_dword *outbuf, int len)
561{
562 struct cdx_mcdi_iface *mcdi;
563 struct cdx_mcdi_cmd *cmd;
564 LIST_HEAD(cleanup_list);
565 unsigned int respseq;
566
567 if (!len || !outbuf) {
568 pr_err("Got empty MC response\n");
569 return;
570 }
571
572 mcdi = cdx_mcdi_if(cdx);
573 if (!mcdi)
574 return;
575
576 respseq = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_SEQ);
577
578 mutex_lock(&mcdi->iface_lock);
579 cmd = mcdi->seq_held_by[respseq];
580
581 if (cmd) {
582 if (cmd->state == MCDI_STATE_FINISHED) {
583 mutex_unlock(&mcdi->iface_lock);
584 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
585 return;
586 }
587
588 cdx_mcdi_complete_cmd(mcdi, cmd, outbuf, len, &cleanup_list);
589 } else {
590 pr_err("MC response unexpected for seq : %0X\n", respseq);
591 }
592
593 mutex_unlock(&mcdi->iface_lock);
594
595 cdx_mcdi_process_cleanup_list(mcdi->cdx, &cleanup_list);
596}
597
598static void cdx_mcdi_cmd_work(struct work_struct *context)
599{
600 struct cdx_mcdi_cmd *cmd =
601 container_of(context, struct cdx_mcdi_cmd, work);
602 struct cdx_mcdi_iface *mcdi = cmd->mcdi;
603
604 mutex_lock(&mcdi->iface_lock);
605
606 cmd->handle = mcdi->prev_handle++;
607 list_add_tail(&cmd->list, &mcdi->cmd_list);
608 cdx_mcdi_cmd_start_or_queue(mcdi, cmd);
609
610 mutex_unlock(&mcdi->iface_lock);
611}
612
613/*
614 * Returns true if the MCDI module is finished with the command.
615 * (examples of false would be if the command was proxied, or it was
616 * rejected by the MC due to lack of resources and requeued).
617 */
618static bool cdx_mcdi_complete_cmd(struct cdx_mcdi_iface *mcdi,
619 struct cdx_mcdi_cmd *cmd,
620 struct cdx_dword *outbuf,
621 int len,
622 struct list_head *cleanup_list)
623{
624 size_t resp_hdr_len, resp_data_len;
625 struct cdx_mcdi *cdx = mcdi->cdx;
626 unsigned int respcmd, error;
627 bool completed = false;
628 int rc;
629
630 /* ensure the command can't go away before this function returns */
631 kref_get(&cmd->ref);
632
633 respcmd = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_CODE);
634 error = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_ERROR);
635
636 if (respcmd != MC_CMD_V2_EXTN) {
637 resp_hdr_len = 4;
638 resp_data_len = CDX_DWORD_FIELD(outbuf[0], MCDI_HEADER_DATALEN);
639 } else {
640 resp_data_len = 0;
641 resp_hdr_len = 8;
642 if (len >= 8)
643 resp_data_len =
644 CDX_DWORD_FIELD(outbuf[1], MC_CMD_V2_EXTN_IN_ACTUAL_LEN);
645 }
646
647 if ((resp_hdr_len + resp_data_len) > len) {
648 pr_warn("Incomplete MCDI response received %d. Expected %zu\n",
649 len, (resp_hdr_len + resp_data_len));
650 resp_data_len = 0;
651 }
652
653 print_hex_dump_debug("MCDI RESP HEADER: ", DUMP_PREFIX_NONE, 32, 4,
654 outbuf, resp_hdr_len, false);
655 print_hex_dump_debug("MCDI RESP PAYLOAD: ", DUMP_PREFIX_NONE, 32, 4,
656 outbuf + (resp_hdr_len / 4), resp_data_len, false);
657
658 if (error && resp_data_len == 0) {
659 /* MC rebooted during command */
660 rc = -EIO;
661 } else {
662 if (WARN_ON_ONCE(error && resp_data_len < 4))
663 resp_data_len = 4;
664 if (error) {
665 rc = CDX_DWORD_FIELD(outbuf[resp_hdr_len / 4], CDX_DWORD);
666 if (!cmd->quiet) {
667 int err_arg = 0;
668
669 if (resp_data_len >= MC_CMD_ERR_ARG_OFST + 4) {
670 int offset = (resp_hdr_len + MC_CMD_ERR_ARG_OFST) / 4;
671
672 err_arg = CDX_DWORD_VAL(outbuf[offset]);
673 }
674
675 _cdx_mcdi_display_error(cdx, cmd->cmd,
676 cmd->inlen, rc, err_arg,
677 cdx_mcdi_errno(cdx, rc));
678 }
679 rc = cdx_mcdi_errno(cdx, rc);
680 } else {
681 rc = 0;
682 }
683 }
684
685 /* free doorbell */
686 if (mcdi->db_held_by == cmd)
687 mcdi->db_held_by = NULL;
688
689 if (cdx_cmd_cancelled(cmd)) {
690 list_del(&cmd->list);
691 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
692 completed = true;
693 } else if (rc == MC_CMD_ERR_QUEUE_FULL) {
694 cmd->state = MCDI_STATE_RETRY;
695 } else {
696 cmd->rc = rc;
697 cmd->outbuf = outbuf + DIV_ROUND_UP(resp_hdr_len, 4);
698 cmd->outlen = resp_data_len;
699 cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
700 completed = true;
701 }
702
703 /* free sequence number and buffer */
704 mcdi->seq_held_by[cmd->seq] = NULL;
705
706 cdx_mcdi_start_or_queue(mcdi, rc != MC_CMD_ERR_QUEUE_FULL);
707
708 /* wake up anyone waiting for flush */
709 wake_up(&mcdi->cmd_complete_wq);
710
711 kref_put(&cmd->ref, cdx_mcdi_cmd_release);
712
713 return completed;
714}
715
716static void cdx_mcdi_timeout_cmd(struct cdx_mcdi_iface *mcdi,
717 struct cdx_mcdi_cmd *cmd,
718 struct list_head *cleanup_list)
719{
720 struct cdx_mcdi *cdx = mcdi->cdx;
721
722 pr_err("MC command 0x%x inlen %zu state %d timed out after %u ms\n",
723 cmd->cmd, cmd->inlen, cmd->state,
724 jiffies_to_msecs(jiffies - cmd->started));
725
726 cmd->rc = -ETIMEDOUT;
727 cdx_mcdi_remove_cmd(mcdi, cmd, cleanup_list);
728
729 cdx_mcdi_mode_fail(cdx, cleanup_list);
730}
731
732/**
733 * cdx_mcdi_rpc - Issue an MCDI command and wait for completion
734 * @cdx: NIC through which to issue the command
735 * @cmd: Command type number
736 * @inbuf: Command parameters
737 * @inlen: Length of command parameters, in bytes. Must be a multiple
738 * of 4 and no greater than %MCDI_CTL_SDU_LEN_MAX_V1.
739 * @outbuf: Response buffer. May be %NULL if @outlen is 0.
740 * @outlen: Length of response buffer, in bytes. If the actual
741 * response is longer than @outlen & ~3, it will be truncated
742 * to that length.
743 * @outlen_actual: Pointer through which to return the actual response
744 * length. May be %NULL if this is not needed.
745 *
746 * This function may sleep and therefore must be called in process
747 * context.
748 *
749 * Return: A negative error code, or zero if successful. The error
750 * code may come from the MCDI response or may indicate a failure
751 * to communicate with the MC. In the former case, the response
752 * will still be copied to @outbuf and *@outlen_actual will be
753 * set accordingly. In the latter case, *@outlen_actual will be
754 * set to zero.
755 */
756int cdx_mcdi_rpc(struct cdx_mcdi *cdx, unsigned int cmd,
757 const struct cdx_dword *inbuf, size_t inlen,
758 struct cdx_dword *outbuf, size_t outlen,
759 size_t *outlen_actual)
760{
761 return cdx_mcdi_rpc_sync(cdx, cmd, inbuf, inlen, outbuf, outlen,
762 outlen_actual, false);
763}
764
765/**
766 * cdx_mcdi_rpc_async - Schedule an MCDI command to run asynchronously
767 * @cdx: NIC through which to issue the command
768 * @cmd: Command type number
769 * @inbuf: Command parameters
770 * @inlen: Length of command parameters, in bytes
771 * @complete: Function to be called on completion or cancellation.
772 * @cookie: Arbitrary value to be passed to @complete.
773 *
774 * This function does not sleep and therefore may be called in atomic
775 * context. It will fail if event queues are disabled or if MCDI
776 * event completions have been disabled due to an error.
777 *
778 * If it succeeds, the @complete function will be called exactly once
779 * in process context, when one of the following occurs:
780 * (a) the completion event is received (in process context)
781 * (b) event queues are disabled (in the process that disables them)
782 */
783int
784cdx_mcdi_rpc_async(struct cdx_mcdi *cdx, unsigned int cmd,
785 const struct cdx_dword *inbuf, size_t inlen,
786 cdx_mcdi_async_completer *complete, unsigned long cookie)
787{
788 struct cdx_mcdi_cmd *cmd_item =
789 kmalloc(sizeof(struct cdx_mcdi_cmd) + inlen, GFP_ATOMIC);
790
791 if (!cmd_item)
792 return -ENOMEM;
793
794 kref_init(&cmd_item->ref);
795 cmd_item->quiet = true;
796 cmd_item->cookie = cookie;
797 cmd_item->completer = complete;
798 cmd_item->cmd = cmd;
799 cmd_item->inlen = inlen;
800 /* inbuf is probably not valid after return, so take a copy */
801 cmd_item->inbuf = (struct cdx_dword *)(cmd_item + 1);
802 memcpy(cmd_item + 1, inbuf, inlen);
803
804 return cdx_mcdi_rpc_async_internal(cdx, cmd_item, NULL);
805}
806
807static void _cdx_mcdi_display_error(struct cdx_mcdi *cdx, unsigned int cmd,
808 size_t inlen, int raw, int arg, int err_no)
809{
810 pr_err("MC command 0x%x inlen %d failed err_no=%d (raw=%d) arg=%d\n",
811 cmd, (int)inlen, err_no, raw, arg);
812}
813
814/*
815 * Set MCDI mode to fail to prevent any new commands, then cancel any
816 * outstanding commands.
817 * Caller must hold the mcdi iface_lock.
818 */
819static void cdx_mcdi_mode_fail(struct cdx_mcdi *cdx, struct list_head *cleanup_list)
820{
821 struct cdx_mcdi_iface *mcdi = cdx_mcdi_if(cdx);
822
823 if (!mcdi)
824 return;
825
826 mcdi->mode = MCDI_MODE_FAIL;
827
828 while (!list_empty(&mcdi->cmd_list)) {
829 struct cdx_mcdi_cmd *cmd;
830
831 cmd = list_first_entry(&mcdi->cmd_list, struct cdx_mcdi_cmd,
832 list);
833 _cdx_mcdi_cancel_cmd(mcdi, cdx_mcdi_cmd_handle(cmd), cleanup_list);
834 }
835}