Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (C) 2023 Intel Corporation */
3
4#include "idpf_controlq.h"
5
6/**
7 * idpf_ctlq_setup_regs - initialize control queue registers
8 * @cq: pointer to the specific control queue
9 * @q_create_info: structs containing info for each queue to be initialized
10 */
11static void idpf_ctlq_setup_regs(struct idpf_ctlq_info *cq,
12 struct idpf_ctlq_create_info *q_create_info)
13{
14 /* set control queue registers in our local struct */
15 cq->reg.head = q_create_info->reg.head;
16 cq->reg.tail = q_create_info->reg.tail;
17 cq->reg.len = q_create_info->reg.len;
18 cq->reg.bah = q_create_info->reg.bah;
19 cq->reg.bal = q_create_info->reg.bal;
20 cq->reg.len_mask = q_create_info->reg.len_mask;
21 cq->reg.len_ena_mask = q_create_info->reg.len_ena_mask;
22 cq->reg.head_mask = q_create_info->reg.head_mask;
23}
24
25/**
26 * idpf_ctlq_init_regs - Initialize control queue registers
27 * @hw: pointer to hw struct
28 * @cq: pointer to the specific Control queue
29 * @is_rxq: true if receive control queue, false otherwise
30 *
31 * Initialize registers. The caller is expected to have already initialized the
32 * descriptor ring memory and buffer memory
33 */
34static void idpf_ctlq_init_regs(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
35 bool is_rxq)
36{
37 /* Update tail to post pre-allocated buffers for rx queues */
38 if (is_rxq)
39 wr32(hw, cq->reg.tail, (u32)(cq->ring_size - 1));
40
41 /* For non-Mailbox control queues only TAIL need to be set */
42 if (cq->q_id != -1)
43 return;
44
45 /* Clear Head for both send or receive */
46 wr32(hw, cq->reg.head, 0);
47
48 /* set starting point */
49 wr32(hw, cq->reg.bal, lower_32_bits(cq->desc_ring.pa));
50 wr32(hw, cq->reg.bah, upper_32_bits(cq->desc_ring.pa));
51 wr32(hw, cq->reg.len, (cq->ring_size | cq->reg.len_ena_mask));
52}
53
54/**
55 * idpf_ctlq_init_rxq_bufs - populate receive queue descriptors with buf
56 * @cq: pointer to the specific Control queue
57 *
58 * Record the address of the receive queue DMA buffers in the descriptors.
59 * The buffers must have been previously allocated.
60 */
61static void idpf_ctlq_init_rxq_bufs(struct idpf_ctlq_info *cq)
62{
63 int i;
64
65 for (i = 0; i < cq->ring_size; i++) {
66 struct idpf_ctlq_desc *desc = IDPF_CTLQ_DESC(cq, i);
67 struct idpf_dma_mem *bi = cq->bi.rx_buff[i];
68
69 /* No buffer to post to descriptor, continue */
70 if (!bi)
71 continue;
72
73 desc->flags =
74 cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD);
75 desc->opcode = 0;
76 desc->datalen = cpu_to_le16(bi->size);
77 desc->ret_val = 0;
78 desc->v_opcode_dtype = 0;
79 desc->v_retval = 0;
80 desc->params.indirect.addr_high =
81 cpu_to_le32(upper_32_bits(bi->pa));
82 desc->params.indirect.addr_low =
83 cpu_to_le32(lower_32_bits(bi->pa));
84 desc->params.indirect.param0 = 0;
85 desc->params.indirect.sw_cookie = 0;
86 desc->params.indirect.v_flags = 0;
87 }
88}
89
90/**
91 * idpf_ctlq_shutdown - shutdown the CQ
92 * @hw: pointer to hw struct
93 * @cq: pointer to the specific Control queue
94 *
95 * The main shutdown routine for any controq queue
96 */
97static void idpf_ctlq_shutdown(struct idpf_hw *hw, struct idpf_ctlq_info *cq)
98{
99 mutex_lock(&cq->cq_lock);
100
101 /* free ring buffers and the ring itself */
102 idpf_ctlq_dealloc_ring_res(hw, cq);
103
104 /* Set ring_size to 0 to indicate uninitialized queue */
105 cq->ring_size = 0;
106
107 mutex_unlock(&cq->cq_lock);
108 mutex_destroy(&cq->cq_lock);
109}
110
111/**
112 * idpf_ctlq_add - add one control queue
113 * @hw: pointer to hardware struct
114 * @qinfo: info for queue to be created
115 * @cq_out: (output) double pointer to control queue to be created
116 *
117 * Allocate and initialize a control queue and add it to the control queue list.
118 * The cq parameter will be allocated/initialized and passed back to the caller
119 * if no errors occur.
120 *
121 * Note: idpf_ctlq_init must be called prior to any calls to idpf_ctlq_add
122 */
123int idpf_ctlq_add(struct idpf_hw *hw,
124 struct idpf_ctlq_create_info *qinfo,
125 struct idpf_ctlq_info **cq_out)
126{
127 struct idpf_ctlq_info *cq;
128 bool is_rxq = false;
129 int err;
130
131 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
132 if (!cq)
133 return -ENOMEM;
134
135 cq->cq_type = qinfo->type;
136 cq->q_id = qinfo->id;
137 cq->buf_size = qinfo->buf_size;
138 cq->ring_size = qinfo->len;
139
140 cq->next_to_use = 0;
141 cq->next_to_clean = 0;
142 cq->next_to_post = cq->ring_size - 1;
143
144 switch (qinfo->type) {
145 case IDPF_CTLQ_TYPE_MAILBOX_RX:
146 is_rxq = true;
147 fallthrough;
148 case IDPF_CTLQ_TYPE_MAILBOX_TX:
149 err = idpf_ctlq_alloc_ring_res(hw, cq);
150 break;
151 default:
152 err = -EBADR;
153 break;
154 }
155
156 if (err)
157 goto init_free_q;
158
159 if (is_rxq) {
160 idpf_ctlq_init_rxq_bufs(cq);
161 } else {
162 /* Allocate the array of msg pointers for TX queues */
163 cq->bi.tx_msg = kcalloc(qinfo->len,
164 sizeof(struct idpf_ctlq_msg *),
165 GFP_KERNEL);
166 if (!cq->bi.tx_msg) {
167 err = -ENOMEM;
168 goto init_dealloc_q_mem;
169 }
170 }
171
172 idpf_ctlq_setup_regs(cq, qinfo);
173
174 idpf_ctlq_init_regs(hw, cq, is_rxq);
175
176 mutex_init(&cq->cq_lock);
177
178 list_add(&cq->cq_list, &hw->cq_list_head);
179
180 *cq_out = cq;
181
182 return 0;
183
184init_dealloc_q_mem:
185 /* free ring buffers and the ring itself */
186 idpf_ctlq_dealloc_ring_res(hw, cq);
187init_free_q:
188 kfree(cq);
189
190 return err;
191}
192
193/**
194 * idpf_ctlq_remove - deallocate and remove specified control queue
195 * @hw: pointer to hardware struct
196 * @cq: pointer to control queue to be removed
197 */
198void idpf_ctlq_remove(struct idpf_hw *hw,
199 struct idpf_ctlq_info *cq)
200{
201 list_del(&cq->cq_list);
202 idpf_ctlq_shutdown(hw, cq);
203 kfree(cq);
204}
205
206/**
207 * idpf_ctlq_init - main initialization routine for all control queues
208 * @hw: pointer to hardware struct
209 * @num_q: number of queues to initialize
210 * @q_info: array of structs containing info for each queue to be initialized
211 *
212 * This initializes any number and any type of control queues. This is an all
213 * or nothing routine; if one fails, all previously allocated queues will be
214 * destroyed. This must be called prior to using the individual add/remove
215 * APIs.
216 */
217int idpf_ctlq_init(struct idpf_hw *hw, u8 num_q,
218 struct idpf_ctlq_create_info *q_info)
219{
220 struct idpf_ctlq_info *cq, *tmp;
221 int err;
222 int i;
223
224 INIT_LIST_HEAD(&hw->cq_list_head);
225
226 for (i = 0; i < num_q; i++) {
227 struct idpf_ctlq_create_info *qinfo = q_info + i;
228
229 err = idpf_ctlq_add(hw, qinfo, &cq);
230 if (err)
231 goto init_destroy_qs;
232 }
233
234 return 0;
235
236init_destroy_qs:
237 list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)
238 idpf_ctlq_remove(hw, cq);
239
240 return err;
241}
242
243/**
244 * idpf_ctlq_deinit - destroy all control queues
245 * @hw: pointer to hw struct
246 */
247void idpf_ctlq_deinit(struct idpf_hw *hw)
248{
249 struct idpf_ctlq_info *cq, *tmp;
250
251 list_for_each_entry_safe(cq, tmp, &hw->cq_list_head, cq_list)
252 idpf_ctlq_remove(hw, cq);
253}
254
255/**
256 * idpf_ctlq_send - send command to Control Queue (CTQ)
257 * @hw: pointer to hw struct
258 * @cq: handle to control queue struct to send on
259 * @num_q_msg: number of messages to send on control queue
260 * @q_msg: pointer to array of queue messages to be sent
261 *
262 * The caller is expected to allocate DMAable buffers and pass them to the
263 * send routine via the q_msg struct / control queue specific data struct.
264 * The control queue will hold a reference to each send message until
265 * the completion for that message has been cleaned.
266 */
267int idpf_ctlq_send(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
268 u16 num_q_msg, struct idpf_ctlq_msg q_msg[])
269{
270 struct idpf_ctlq_desc *desc;
271 int num_desc_avail;
272 int err = 0;
273 int i;
274
275 mutex_lock(&cq->cq_lock);
276
277 /* Ensure there are enough descriptors to send all messages */
278 num_desc_avail = IDPF_CTLQ_DESC_UNUSED(cq);
279 if (num_desc_avail == 0 || num_desc_avail < num_q_msg) {
280 err = -ENOSPC;
281 goto err_unlock;
282 }
283
284 for (i = 0; i < num_q_msg; i++) {
285 struct idpf_ctlq_msg *msg = &q_msg[i];
286
287 desc = IDPF_CTLQ_DESC(cq, cq->next_to_use);
288
289 desc->opcode = cpu_to_le16(msg->opcode);
290 desc->pfid_vfid = cpu_to_le16(msg->func_id);
291
292 desc->v_opcode_dtype = cpu_to_le32(msg->cookie.mbx.chnl_opcode);
293 desc->v_retval = cpu_to_le32(msg->cookie.mbx.chnl_retval);
294
295 desc->flags = cpu_to_le16((msg->host_id & IDPF_HOST_ID_MASK) <<
296 IDPF_CTLQ_FLAG_HOST_ID_S);
297 if (msg->data_len) {
298 struct idpf_dma_mem *buff = msg->ctx.indirect.payload;
299
300 desc->datalen |= cpu_to_le16(msg->data_len);
301 desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_BUF);
302 desc->flags |= cpu_to_le16(IDPF_CTLQ_FLAG_RD);
303
304 /* Update the address values in the desc with the pa
305 * value for respective buffer
306 */
307 desc->params.indirect.addr_high =
308 cpu_to_le32(upper_32_bits(buff->pa));
309 desc->params.indirect.addr_low =
310 cpu_to_le32(lower_32_bits(buff->pa));
311
312 memcpy(&desc->params, msg->ctx.indirect.context,
313 IDPF_INDIRECT_CTX_SIZE);
314 } else {
315 memcpy(&desc->params, msg->ctx.direct,
316 IDPF_DIRECT_CTX_SIZE);
317 }
318
319 /* Store buffer info */
320 cq->bi.tx_msg[cq->next_to_use] = msg;
321
322 (cq->next_to_use)++;
323 if (cq->next_to_use == cq->ring_size)
324 cq->next_to_use = 0;
325 }
326
327 /* Force memory write to complete before letting hardware
328 * know that there are new descriptors to fetch.
329 */
330 dma_wmb();
331
332 wr32(hw, cq->reg.tail, cq->next_to_use);
333
334err_unlock:
335 mutex_unlock(&cq->cq_lock);
336
337 return err;
338}
339
340/**
341 * idpf_ctlq_clean_sq - reclaim send descriptors on HW write back for the
342 * requested queue
343 * @cq: pointer to the specific Control queue
344 * @clean_count: (input|output) number of descriptors to clean as input, and
345 * number of descriptors actually cleaned as output
346 * @msg_status: (output) pointer to msg pointer array to be populated; needs
347 * to be allocated by caller
348 *
349 * Returns an array of message pointers associated with the cleaned
350 * descriptors. The pointers are to the original ctlq_msgs sent on the cleaned
351 * descriptors. The status will be returned for each; any messages that failed
352 * to send will have a non-zero status. The caller is expected to free original
353 * ctlq_msgs and free or reuse the DMA buffers.
354 */
355int idpf_ctlq_clean_sq(struct idpf_ctlq_info *cq, u16 *clean_count,
356 struct idpf_ctlq_msg *msg_status[])
357{
358 struct idpf_ctlq_desc *desc;
359 u16 i, num_to_clean;
360 u16 ntc, desc_err;
361
362 if (*clean_count == 0)
363 return 0;
364 if (*clean_count > cq->ring_size)
365 return -EBADR;
366
367 mutex_lock(&cq->cq_lock);
368
369 ntc = cq->next_to_clean;
370
371 num_to_clean = *clean_count;
372
373 for (i = 0; i < num_to_clean; i++) {
374 /* Fetch next descriptor and check if marked as done */
375 desc = IDPF_CTLQ_DESC(cq, ntc);
376 if (!(le16_to_cpu(desc->flags) & IDPF_CTLQ_FLAG_DD))
377 break;
378
379 /* strip off FW internal code */
380 desc_err = le16_to_cpu(desc->ret_val) & 0xff;
381
382 msg_status[i] = cq->bi.tx_msg[ntc];
383 msg_status[i]->status = desc_err;
384
385 cq->bi.tx_msg[ntc] = NULL;
386
387 /* Zero out any stale data */
388 memset(desc, 0, sizeof(*desc));
389
390 ntc++;
391 if (ntc == cq->ring_size)
392 ntc = 0;
393 }
394
395 cq->next_to_clean = ntc;
396
397 mutex_unlock(&cq->cq_lock);
398
399 /* Return number of descriptors actually cleaned */
400 *clean_count = i;
401
402 return 0;
403}
404
405/**
406 * idpf_ctlq_post_rx_buffs - post buffers to descriptor ring
407 * @hw: pointer to hw struct
408 * @cq: pointer to control queue handle
409 * @buff_count: (input|output) input is number of buffers caller is trying to
410 * return; output is number of buffers that were not posted
411 * @buffs: array of pointers to dma mem structs to be given to hardware
412 *
413 * Caller uses this function to return DMA buffers to the descriptor ring after
414 * consuming them; buff_count will be the number of buffers.
415 *
416 * Note: this function needs to be called after a receive call even
417 * if there are no DMA buffers to be returned, i.e. buff_count = 0,
418 * buffs = NULL to support direct commands
419 */
420int idpf_ctlq_post_rx_buffs(struct idpf_hw *hw, struct idpf_ctlq_info *cq,
421 u16 *buff_count, struct idpf_dma_mem **buffs)
422{
423 struct idpf_ctlq_desc *desc;
424 u16 ntp = cq->next_to_post;
425 bool buffs_avail = false;
426 u16 tbp = ntp + 1;
427 int i = 0;
428
429 if (*buff_count > cq->ring_size)
430 return -EBADR;
431
432 if (*buff_count > 0)
433 buffs_avail = true;
434
435 mutex_lock(&cq->cq_lock);
436
437 if (tbp >= cq->ring_size)
438 tbp = 0;
439
440 if (tbp == cq->next_to_clean)
441 /* Nothing to do */
442 goto post_buffs_out;
443
444 /* Post buffers for as many as provided or up until the last one used */
445 while (ntp != cq->next_to_clean) {
446 desc = IDPF_CTLQ_DESC(cq, ntp);
447
448 if (cq->bi.rx_buff[ntp])
449 goto fill_desc;
450 if (!buffs_avail) {
451 /* If the caller hasn't given us any buffers or
452 * there are none left, search the ring itself
453 * for an available buffer to move to this
454 * entry starting at the next entry in the ring
455 */
456 tbp = ntp + 1;
457
458 /* Wrap ring if necessary */
459 if (tbp >= cq->ring_size)
460 tbp = 0;
461
462 while (tbp != cq->next_to_clean) {
463 if (cq->bi.rx_buff[tbp]) {
464 cq->bi.rx_buff[ntp] =
465 cq->bi.rx_buff[tbp];
466 cq->bi.rx_buff[tbp] = NULL;
467
468 /* Found a buffer, no need to
469 * search anymore
470 */
471 break;
472 }
473
474 /* Wrap ring if necessary */
475 tbp++;
476 if (tbp >= cq->ring_size)
477 tbp = 0;
478 }
479
480 if (tbp == cq->next_to_clean)
481 goto post_buffs_out;
482 } else {
483 /* Give back pointer to DMA buffer */
484 cq->bi.rx_buff[ntp] = buffs[i];
485 i++;
486
487 if (i >= *buff_count)
488 buffs_avail = false;
489 }
490
491fill_desc:
492 desc->flags =
493 cpu_to_le16(IDPF_CTLQ_FLAG_BUF | IDPF_CTLQ_FLAG_RD);
494
495 /* Post buffers to descriptor */
496 desc->datalen = cpu_to_le16(cq->bi.rx_buff[ntp]->size);
497 desc->params.indirect.addr_high =
498 cpu_to_le32(upper_32_bits(cq->bi.rx_buff[ntp]->pa));
499 desc->params.indirect.addr_low =
500 cpu_to_le32(lower_32_bits(cq->bi.rx_buff[ntp]->pa));
501
502 ntp++;
503 if (ntp == cq->ring_size)
504 ntp = 0;
505 }
506
507post_buffs_out:
508 /* Only update tail if buffers were actually posted */
509 if (cq->next_to_post != ntp) {
510 if (ntp)
511 /* Update next_to_post to ntp - 1 since current ntp
512 * will not have a buffer
513 */
514 cq->next_to_post = ntp - 1;
515 else
516 /* Wrap to end of end ring since current ntp is 0 */
517 cq->next_to_post = cq->ring_size - 1;
518
519 dma_wmb();
520
521 wr32(hw, cq->reg.tail, cq->next_to_post);
522 }
523
524 mutex_unlock(&cq->cq_lock);
525
526 /* return the number of buffers that were not posted */
527 *buff_count = *buff_count - i;
528
529 return 0;
530}
531
532/**
533 * idpf_ctlq_recv - receive control queue message call back
534 * @cq: pointer to control queue handle to receive on
535 * @num_q_msg: (input|output) input number of messages that should be received;
536 * output number of messages actually received
537 * @q_msg: (output) array of received control queue messages on this q;
538 * needs to be pre-allocated by caller for as many messages as requested
539 *
540 * Called by interrupt handler or polling mechanism. Caller is expected
541 * to free buffers
542 */
543int idpf_ctlq_recv(struct idpf_ctlq_info *cq, u16 *num_q_msg,
544 struct idpf_ctlq_msg *q_msg)
545{
546 u16 num_to_clean, ntc, flags;
547 struct idpf_ctlq_desc *desc;
548 int err = 0;
549 u16 i;
550
551 /* take the lock before we start messing with the ring */
552 mutex_lock(&cq->cq_lock);
553
554 ntc = cq->next_to_clean;
555
556 num_to_clean = *num_q_msg;
557
558 for (i = 0; i < num_to_clean; i++) {
559 /* Fetch next descriptor and check if marked as done */
560 desc = IDPF_CTLQ_DESC(cq, ntc);
561 flags = le16_to_cpu(desc->flags);
562
563 if (!(flags & IDPF_CTLQ_FLAG_DD))
564 break;
565
566 q_msg[i].vmvf_type = (flags &
567 (IDPF_CTLQ_FLAG_FTYPE_VM |
568 IDPF_CTLQ_FLAG_FTYPE_PF)) >>
569 IDPF_CTLQ_FLAG_FTYPE_S;
570
571 if (flags & IDPF_CTLQ_FLAG_ERR)
572 err = -EBADMSG;
573
574 q_msg[i].cookie.mbx.chnl_opcode =
575 le32_to_cpu(desc->v_opcode_dtype);
576 q_msg[i].cookie.mbx.chnl_retval =
577 le32_to_cpu(desc->v_retval);
578
579 q_msg[i].opcode = le16_to_cpu(desc->opcode);
580 q_msg[i].data_len = le16_to_cpu(desc->datalen);
581 q_msg[i].status = le16_to_cpu(desc->ret_val);
582
583 if (desc->datalen) {
584 memcpy(q_msg[i].ctx.indirect.context,
585 &desc->params.indirect, IDPF_INDIRECT_CTX_SIZE);
586
587 /* Assign pointer to dma buffer to ctlq_msg array
588 * to be given to upper layer
589 */
590 q_msg[i].ctx.indirect.payload = cq->bi.rx_buff[ntc];
591
592 /* Zero out pointer to DMA buffer info;
593 * will be repopulated by post buffers API
594 */
595 cq->bi.rx_buff[ntc] = NULL;
596 } else {
597 memcpy(q_msg[i].ctx.direct, desc->params.raw,
598 IDPF_DIRECT_CTX_SIZE);
599 }
600
601 /* Zero out stale data in descriptor */
602 memset(desc, 0, sizeof(struct idpf_ctlq_desc));
603
604 ntc++;
605 if (ntc == cq->ring_size)
606 ntc = 0;
607 }
608
609 cq->next_to_clean = ntc;
610
611 mutex_unlock(&cq->cq_lock);
612
613 *num_q_msg = i;
614 if (*num_q_msg == 0)
615 err = -ENOMSG;
616
617 return err;
618}