Loading...
Note: File does not exist in v4.17.
1// SPDX-License-Identifier: GPL-2.0-only
2/* Copyright (C) 2023 Intel Corporation */
3
4#include "idpf.h"
5#include "idpf_virtchnl.h"
6
7/**
8 * idpf_buf_lifo_push - push a buffer pointer onto stack
9 * @stack: pointer to stack struct
10 * @buf: pointer to buf to push
11 *
12 * Returns 0 on success, negative on failure
13 **/
14static int idpf_buf_lifo_push(struct idpf_buf_lifo *stack,
15 struct idpf_tx_stash *buf)
16{
17 if (unlikely(stack->top == stack->size))
18 return -ENOSPC;
19
20 stack->bufs[stack->top++] = buf;
21
22 return 0;
23}
24
25/**
26 * idpf_buf_lifo_pop - pop a buffer pointer from stack
27 * @stack: pointer to stack struct
28 **/
29static struct idpf_tx_stash *idpf_buf_lifo_pop(struct idpf_buf_lifo *stack)
30{
31 if (unlikely(!stack->top))
32 return NULL;
33
34 return stack->bufs[--stack->top];
35}
36
37/**
38 * idpf_tx_timeout - Respond to a Tx Hang
39 * @netdev: network interface device structure
40 * @txqueue: TX queue
41 */
42void idpf_tx_timeout(struct net_device *netdev, unsigned int txqueue)
43{
44 struct idpf_adapter *adapter = idpf_netdev_to_adapter(netdev);
45
46 adapter->tx_timeout_count++;
47
48 netdev_err(netdev, "Detected Tx timeout: Count %d, Queue %d\n",
49 adapter->tx_timeout_count, txqueue);
50 if (!idpf_is_reset_in_prog(adapter)) {
51 set_bit(IDPF_HR_FUNC_RESET, adapter->flags);
52 queue_delayed_work(adapter->vc_event_wq,
53 &adapter->vc_event_task,
54 msecs_to_jiffies(10));
55 }
56}
57
58/**
59 * idpf_tx_buf_rel - Release a Tx buffer
60 * @tx_q: the queue that owns the buffer
61 * @tx_buf: the buffer to free
62 */
63static void idpf_tx_buf_rel(struct idpf_queue *tx_q, struct idpf_tx_buf *tx_buf)
64{
65 if (tx_buf->skb) {
66 if (dma_unmap_len(tx_buf, len))
67 dma_unmap_single(tx_q->dev,
68 dma_unmap_addr(tx_buf, dma),
69 dma_unmap_len(tx_buf, len),
70 DMA_TO_DEVICE);
71 dev_kfree_skb_any(tx_buf->skb);
72 } else if (dma_unmap_len(tx_buf, len)) {
73 dma_unmap_page(tx_q->dev,
74 dma_unmap_addr(tx_buf, dma),
75 dma_unmap_len(tx_buf, len),
76 DMA_TO_DEVICE);
77 }
78
79 tx_buf->next_to_watch = NULL;
80 tx_buf->skb = NULL;
81 tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
82 dma_unmap_len_set(tx_buf, len, 0);
83}
84
85/**
86 * idpf_tx_buf_rel_all - Free any empty Tx buffers
87 * @txq: queue to be cleaned
88 */
89static void idpf_tx_buf_rel_all(struct idpf_queue *txq)
90{
91 u16 i;
92
93 /* Buffers already cleared, nothing to do */
94 if (!txq->tx_buf)
95 return;
96
97 /* Free all the Tx buffer sk_buffs */
98 for (i = 0; i < txq->desc_count; i++)
99 idpf_tx_buf_rel(txq, &txq->tx_buf[i]);
100
101 kfree(txq->tx_buf);
102 txq->tx_buf = NULL;
103
104 if (!txq->buf_stack.bufs)
105 return;
106
107 for (i = 0; i < txq->buf_stack.size; i++)
108 kfree(txq->buf_stack.bufs[i]);
109
110 kfree(txq->buf_stack.bufs);
111 txq->buf_stack.bufs = NULL;
112}
113
114/**
115 * idpf_tx_desc_rel - Free Tx resources per queue
116 * @txq: Tx descriptor ring for a specific queue
117 * @bufq: buffer q or completion q
118 *
119 * Free all transmit software resources
120 */
121static void idpf_tx_desc_rel(struct idpf_queue *txq, bool bufq)
122{
123 if (bufq)
124 idpf_tx_buf_rel_all(txq);
125
126 if (!txq->desc_ring)
127 return;
128
129 dmam_free_coherent(txq->dev, txq->size, txq->desc_ring, txq->dma);
130 txq->desc_ring = NULL;
131 txq->next_to_alloc = 0;
132 txq->next_to_use = 0;
133 txq->next_to_clean = 0;
134}
135
136/**
137 * idpf_tx_desc_rel_all - Free Tx Resources for All Queues
138 * @vport: virtual port structure
139 *
140 * Free all transmit software resources
141 */
142static void idpf_tx_desc_rel_all(struct idpf_vport *vport)
143{
144 int i, j;
145
146 if (!vport->txq_grps)
147 return;
148
149 for (i = 0; i < vport->num_txq_grp; i++) {
150 struct idpf_txq_group *txq_grp = &vport->txq_grps[i];
151
152 for (j = 0; j < txq_grp->num_txq; j++)
153 idpf_tx_desc_rel(txq_grp->txqs[j], true);
154
155 if (idpf_is_queue_model_split(vport->txq_model))
156 idpf_tx_desc_rel(txq_grp->complq, false);
157 }
158}
159
160/**
161 * idpf_tx_buf_alloc_all - Allocate memory for all buffer resources
162 * @tx_q: queue for which the buffers are allocated
163 *
164 * Returns 0 on success, negative on failure
165 */
166static int idpf_tx_buf_alloc_all(struct idpf_queue *tx_q)
167{
168 int buf_size;
169 int i;
170
171 /* Allocate book keeping buffers only. Buffers to be supplied to HW
172 * are allocated by kernel network stack and received as part of skb
173 */
174 buf_size = sizeof(struct idpf_tx_buf) * tx_q->desc_count;
175 tx_q->tx_buf = kzalloc(buf_size, GFP_KERNEL);
176 if (!tx_q->tx_buf)
177 return -ENOMEM;
178
179 /* Initialize tx_bufs with invalid completion tags */
180 for (i = 0; i < tx_q->desc_count; i++)
181 tx_q->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
182
183 /* Initialize tx buf stack for out-of-order completions if
184 * flow scheduling offload is enabled
185 */
186 tx_q->buf_stack.bufs =
187 kcalloc(tx_q->desc_count, sizeof(struct idpf_tx_stash *),
188 GFP_KERNEL);
189 if (!tx_q->buf_stack.bufs)
190 return -ENOMEM;
191
192 tx_q->buf_stack.size = tx_q->desc_count;
193 tx_q->buf_stack.top = tx_q->desc_count;
194
195 for (i = 0; i < tx_q->desc_count; i++) {
196 tx_q->buf_stack.bufs[i] = kzalloc(sizeof(*tx_q->buf_stack.bufs[i]),
197 GFP_KERNEL);
198 if (!tx_q->buf_stack.bufs[i])
199 return -ENOMEM;
200 }
201
202 return 0;
203}
204
205/**
206 * idpf_tx_desc_alloc - Allocate the Tx descriptors
207 * @tx_q: the tx ring to set up
208 * @bufq: buffer or completion queue
209 *
210 * Returns 0 on success, negative on failure
211 */
212static int idpf_tx_desc_alloc(struct idpf_queue *tx_q, bool bufq)
213{
214 struct device *dev = tx_q->dev;
215 u32 desc_sz;
216 int err;
217
218 if (bufq) {
219 err = idpf_tx_buf_alloc_all(tx_q);
220 if (err)
221 goto err_alloc;
222
223 desc_sz = sizeof(struct idpf_base_tx_desc);
224 } else {
225 desc_sz = sizeof(struct idpf_splitq_tx_compl_desc);
226 }
227
228 tx_q->size = tx_q->desc_count * desc_sz;
229
230 /* Allocate descriptors also round up to nearest 4K */
231 tx_q->size = ALIGN(tx_q->size, 4096);
232 tx_q->desc_ring = dmam_alloc_coherent(dev, tx_q->size, &tx_q->dma,
233 GFP_KERNEL);
234 if (!tx_q->desc_ring) {
235 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
236 tx_q->size);
237 err = -ENOMEM;
238 goto err_alloc;
239 }
240
241 tx_q->next_to_alloc = 0;
242 tx_q->next_to_use = 0;
243 tx_q->next_to_clean = 0;
244 set_bit(__IDPF_Q_GEN_CHK, tx_q->flags);
245
246 return 0;
247
248err_alloc:
249 idpf_tx_desc_rel(tx_q, bufq);
250
251 return err;
252}
253
254/**
255 * idpf_tx_desc_alloc_all - allocate all queues Tx resources
256 * @vport: virtual port private structure
257 *
258 * Returns 0 on success, negative on failure
259 */
260static int idpf_tx_desc_alloc_all(struct idpf_vport *vport)
261{
262 struct device *dev = &vport->adapter->pdev->dev;
263 int err = 0;
264 int i, j;
265
266 /* Setup buffer queues. In single queue model buffer queues and
267 * completion queues will be same
268 */
269 for (i = 0; i < vport->num_txq_grp; i++) {
270 for (j = 0; j < vport->txq_grps[i].num_txq; j++) {
271 struct idpf_queue *txq = vport->txq_grps[i].txqs[j];
272 u8 gen_bits = 0;
273 u16 bufidx_mask;
274
275 err = idpf_tx_desc_alloc(txq, true);
276 if (err) {
277 dev_err(dev, "Allocation for Tx Queue %u failed\n",
278 i);
279 goto err_out;
280 }
281
282 if (!idpf_is_queue_model_split(vport->txq_model))
283 continue;
284
285 txq->compl_tag_cur_gen = 0;
286
287 /* Determine the number of bits in the bufid
288 * mask and add one to get the start of the
289 * generation bits
290 */
291 bufidx_mask = txq->desc_count - 1;
292 while (bufidx_mask >> 1) {
293 txq->compl_tag_gen_s++;
294 bufidx_mask = bufidx_mask >> 1;
295 }
296 txq->compl_tag_gen_s++;
297
298 gen_bits = IDPF_TX_SPLITQ_COMPL_TAG_WIDTH -
299 txq->compl_tag_gen_s;
300 txq->compl_tag_gen_max = GETMAXVAL(gen_bits);
301
302 /* Set bufid mask based on location of first
303 * gen bit; it cannot simply be the descriptor
304 * ring size-1 since we can have size values
305 * where not all of those bits are set.
306 */
307 txq->compl_tag_bufid_m =
308 GETMAXVAL(txq->compl_tag_gen_s);
309 }
310
311 if (!idpf_is_queue_model_split(vport->txq_model))
312 continue;
313
314 /* Setup completion queues */
315 err = idpf_tx_desc_alloc(vport->txq_grps[i].complq, false);
316 if (err) {
317 dev_err(dev, "Allocation for Tx Completion Queue %u failed\n",
318 i);
319 goto err_out;
320 }
321 }
322
323err_out:
324 if (err)
325 idpf_tx_desc_rel_all(vport);
326
327 return err;
328}
329
330/**
331 * idpf_rx_page_rel - Release an rx buffer page
332 * @rxq: the queue that owns the buffer
333 * @rx_buf: the buffer to free
334 */
335static void idpf_rx_page_rel(struct idpf_queue *rxq, struct idpf_rx_buf *rx_buf)
336{
337 if (unlikely(!rx_buf->page))
338 return;
339
340 page_pool_put_full_page(rxq->pp, rx_buf->page, false);
341
342 rx_buf->page = NULL;
343 rx_buf->page_offset = 0;
344}
345
346/**
347 * idpf_rx_hdr_buf_rel_all - Release header buffer memory
348 * @rxq: queue to use
349 */
350static void idpf_rx_hdr_buf_rel_all(struct idpf_queue *rxq)
351{
352 struct idpf_adapter *adapter = rxq->vport->adapter;
353
354 dma_free_coherent(&adapter->pdev->dev,
355 rxq->desc_count * IDPF_HDR_BUF_SIZE,
356 rxq->rx_buf.hdr_buf_va,
357 rxq->rx_buf.hdr_buf_pa);
358 rxq->rx_buf.hdr_buf_va = NULL;
359}
360
361/**
362 * idpf_rx_buf_rel_all - Free all Rx buffer resources for a queue
363 * @rxq: queue to be cleaned
364 */
365static void idpf_rx_buf_rel_all(struct idpf_queue *rxq)
366{
367 u16 i;
368
369 /* queue already cleared, nothing to do */
370 if (!rxq->rx_buf.buf)
371 return;
372
373 /* Free all the bufs allocated and given to hw on Rx queue */
374 for (i = 0; i < rxq->desc_count; i++)
375 idpf_rx_page_rel(rxq, &rxq->rx_buf.buf[i]);
376
377 if (rxq->rx_hsplit_en)
378 idpf_rx_hdr_buf_rel_all(rxq);
379
380 page_pool_destroy(rxq->pp);
381 rxq->pp = NULL;
382
383 kfree(rxq->rx_buf.buf);
384 rxq->rx_buf.buf = NULL;
385}
386
387/**
388 * idpf_rx_desc_rel - Free a specific Rx q resources
389 * @rxq: queue to clean the resources from
390 * @bufq: buffer q or completion q
391 * @q_model: single or split q model
392 *
393 * Free a specific rx queue resources
394 */
395static void idpf_rx_desc_rel(struct idpf_queue *rxq, bool bufq, s32 q_model)
396{
397 if (!rxq)
398 return;
399
400 if (rxq->skb) {
401 dev_kfree_skb_any(rxq->skb);
402 rxq->skb = NULL;
403 }
404
405 if (bufq || !idpf_is_queue_model_split(q_model))
406 idpf_rx_buf_rel_all(rxq);
407
408 rxq->next_to_alloc = 0;
409 rxq->next_to_clean = 0;
410 rxq->next_to_use = 0;
411 if (!rxq->desc_ring)
412 return;
413
414 dmam_free_coherent(rxq->dev, rxq->size, rxq->desc_ring, rxq->dma);
415 rxq->desc_ring = NULL;
416}
417
418/**
419 * idpf_rx_desc_rel_all - Free Rx Resources for All Queues
420 * @vport: virtual port structure
421 *
422 * Free all rx queues resources
423 */
424static void idpf_rx_desc_rel_all(struct idpf_vport *vport)
425{
426 struct idpf_rxq_group *rx_qgrp;
427 u16 num_rxq;
428 int i, j;
429
430 if (!vport->rxq_grps)
431 return;
432
433 for (i = 0; i < vport->num_rxq_grp; i++) {
434 rx_qgrp = &vport->rxq_grps[i];
435
436 if (!idpf_is_queue_model_split(vport->rxq_model)) {
437 for (j = 0; j < rx_qgrp->singleq.num_rxq; j++)
438 idpf_rx_desc_rel(rx_qgrp->singleq.rxqs[j],
439 false, vport->rxq_model);
440 continue;
441 }
442
443 num_rxq = rx_qgrp->splitq.num_rxq_sets;
444 for (j = 0; j < num_rxq; j++)
445 idpf_rx_desc_rel(&rx_qgrp->splitq.rxq_sets[j]->rxq,
446 false, vport->rxq_model);
447
448 if (!rx_qgrp->splitq.bufq_sets)
449 continue;
450
451 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
452 struct idpf_bufq_set *bufq_set =
453 &rx_qgrp->splitq.bufq_sets[j];
454
455 idpf_rx_desc_rel(&bufq_set->bufq, true,
456 vport->rxq_model);
457 }
458 }
459}
460
461/**
462 * idpf_rx_buf_hw_update - Store the new tail and head values
463 * @rxq: queue to bump
464 * @val: new head index
465 */
466void idpf_rx_buf_hw_update(struct idpf_queue *rxq, u32 val)
467{
468 rxq->next_to_use = val;
469
470 if (unlikely(!rxq->tail))
471 return;
472
473 /* writel has an implicit memory barrier */
474 writel(val, rxq->tail);
475}
476
477/**
478 * idpf_rx_hdr_buf_alloc_all - Allocate memory for header buffers
479 * @rxq: ring to use
480 *
481 * Returns 0 on success, negative on failure.
482 */
483static int idpf_rx_hdr_buf_alloc_all(struct idpf_queue *rxq)
484{
485 struct idpf_adapter *adapter = rxq->vport->adapter;
486
487 rxq->rx_buf.hdr_buf_va =
488 dma_alloc_coherent(&adapter->pdev->dev,
489 IDPF_HDR_BUF_SIZE * rxq->desc_count,
490 &rxq->rx_buf.hdr_buf_pa,
491 GFP_KERNEL);
492 if (!rxq->rx_buf.hdr_buf_va)
493 return -ENOMEM;
494
495 return 0;
496}
497
498/**
499 * idpf_rx_post_buf_refill - Post buffer id to refill queue
500 * @refillq: refill queue to post to
501 * @buf_id: buffer id to post
502 */
503static void idpf_rx_post_buf_refill(struct idpf_sw_queue *refillq, u16 buf_id)
504{
505 u16 nta = refillq->next_to_alloc;
506
507 /* store the buffer ID and the SW maintained GEN bit to the refillq */
508 refillq->ring[nta] =
509 FIELD_PREP(IDPF_RX_BI_BUFID_M, buf_id) |
510 FIELD_PREP(IDPF_RX_BI_GEN_M,
511 test_bit(__IDPF_Q_GEN_CHK, refillq->flags));
512
513 if (unlikely(++nta == refillq->desc_count)) {
514 nta = 0;
515 change_bit(__IDPF_Q_GEN_CHK, refillq->flags);
516 }
517 refillq->next_to_alloc = nta;
518}
519
520/**
521 * idpf_rx_post_buf_desc - Post buffer to bufq descriptor ring
522 * @bufq: buffer queue to post to
523 * @buf_id: buffer id to post
524 *
525 * Returns false if buffer could not be allocated, true otherwise.
526 */
527static bool idpf_rx_post_buf_desc(struct idpf_queue *bufq, u16 buf_id)
528{
529 struct virtchnl2_splitq_rx_buf_desc *splitq_rx_desc = NULL;
530 u16 nta = bufq->next_to_alloc;
531 struct idpf_rx_buf *buf;
532 dma_addr_t addr;
533
534 splitq_rx_desc = IDPF_SPLITQ_RX_BUF_DESC(bufq, nta);
535 buf = &bufq->rx_buf.buf[buf_id];
536
537 if (bufq->rx_hsplit_en) {
538 splitq_rx_desc->hdr_addr =
539 cpu_to_le64(bufq->rx_buf.hdr_buf_pa +
540 (u32)buf_id * IDPF_HDR_BUF_SIZE);
541 }
542
543 addr = idpf_alloc_page(bufq->pp, buf, bufq->rx_buf_size);
544 if (unlikely(addr == DMA_MAPPING_ERROR))
545 return false;
546
547 splitq_rx_desc->pkt_addr = cpu_to_le64(addr);
548 splitq_rx_desc->qword0.buf_id = cpu_to_le16(buf_id);
549
550 nta++;
551 if (unlikely(nta == bufq->desc_count))
552 nta = 0;
553 bufq->next_to_alloc = nta;
554
555 return true;
556}
557
558/**
559 * idpf_rx_post_init_bufs - Post initial buffers to bufq
560 * @bufq: buffer queue to post working set to
561 * @working_set: number of buffers to put in working set
562 *
563 * Returns true if @working_set bufs were posted successfully, false otherwise.
564 */
565static bool idpf_rx_post_init_bufs(struct idpf_queue *bufq, u16 working_set)
566{
567 int i;
568
569 for (i = 0; i < working_set; i++) {
570 if (!idpf_rx_post_buf_desc(bufq, i))
571 return false;
572 }
573
574 idpf_rx_buf_hw_update(bufq,
575 bufq->next_to_alloc & ~(bufq->rx_buf_stride - 1));
576
577 return true;
578}
579
580/**
581 * idpf_rx_create_page_pool - Create a page pool
582 * @rxbufq: RX queue to create page pool for
583 *
584 * Returns &page_pool on success, casted -errno on failure
585 */
586static struct page_pool *idpf_rx_create_page_pool(struct idpf_queue *rxbufq)
587{
588 struct page_pool_params pp = {
589 .flags = PP_FLAG_DMA_MAP | PP_FLAG_DMA_SYNC_DEV,
590 .order = 0,
591 .pool_size = rxbufq->desc_count,
592 .nid = NUMA_NO_NODE,
593 .dev = rxbufq->vport->netdev->dev.parent,
594 .max_len = PAGE_SIZE,
595 .dma_dir = DMA_FROM_DEVICE,
596 .offset = 0,
597 };
598
599 return page_pool_create(&pp);
600}
601
602/**
603 * idpf_rx_buf_alloc_all - Allocate memory for all buffer resources
604 * @rxbufq: queue for which the buffers are allocated; equivalent to
605 * rxq when operating in singleq mode
606 *
607 * Returns 0 on success, negative on failure
608 */
609static int idpf_rx_buf_alloc_all(struct idpf_queue *rxbufq)
610{
611 int err = 0;
612
613 /* Allocate book keeping buffers */
614 rxbufq->rx_buf.buf = kcalloc(rxbufq->desc_count,
615 sizeof(struct idpf_rx_buf), GFP_KERNEL);
616 if (!rxbufq->rx_buf.buf) {
617 err = -ENOMEM;
618 goto rx_buf_alloc_all_out;
619 }
620
621 if (rxbufq->rx_hsplit_en) {
622 err = idpf_rx_hdr_buf_alloc_all(rxbufq);
623 if (err)
624 goto rx_buf_alloc_all_out;
625 }
626
627 /* Allocate buffers to be given to HW. */
628 if (idpf_is_queue_model_split(rxbufq->vport->rxq_model)) {
629 int working_set = IDPF_RX_BUFQ_WORKING_SET(rxbufq);
630
631 if (!idpf_rx_post_init_bufs(rxbufq, working_set))
632 err = -ENOMEM;
633 } else {
634 if (idpf_rx_singleq_buf_hw_alloc_all(rxbufq,
635 rxbufq->desc_count - 1))
636 err = -ENOMEM;
637 }
638
639rx_buf_alloc_all_out:
640 if (err)
641 idpf_rx_buf_rel_all(rxbufq);
642
643 return err;
644}
645
646/**
647 * idpf_rx_bufs_init - Initialize page pool, allocate rx bufs, and post to HW
648 * @rxbufq: RX queue to create page pool for
649 *
650 * Returns 0 on success, negative on failure
651 */
652static int idpf_rx_bufs_init(struct idpf_queue *rxbufq)
653{
654 struct page_pool *pool;
655
656 pool = idpf_rx_create_page_pool(rxbufq);
657 if (IS_ERR(pool))
658 return PTR_ERR(pool);
659
660 rxbufq->pp = pool;
661
662 return idpf_rx_buf_alloc_all(rxbufq);
663}
664
665/**
666 * idpf_rx_bufs_init_all - Initialize all RX bufs
667 * @vport: virtual port struct
668 *
669 * Returns 0 on success, negative on failure
670 */
671int idpf_rx_bufs_init_all(struct idpf_vport *vport)
672{
673 struct idpf_rxq_group *rx_qgrp;
674 struct idpf_queue *q;
675 int i, j, err;
676
677 for (i = 0; i < vport->num_rxq_grp; i++) {
678 rx_qgrp = &vport->rxq_grps[i];
679
680 /* Allocate bufs for the rxq itself in singleq */
681 if (!idpf_is_queue_model_split(vport->rxq_model)) {
682 int num_rxq = rx_qgrp->singleq.num_rxq;
683
684 for (j = 0; j < num_rxq; j++) {
685 q = rx_qgrp->singleq.rxqs[j];
686 err = idpf_rx_bufs_init(q);
687 if (err)
688 return err;
689 }
690
691 continue;
692 }
693
694 /* Otherwise, allocate bufs for the buffer queues */
695 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
696 q = &rx_qgrp->splitq.bufq_sets[j].bufq;
697 err = idpf_rx_bufs_init(q);
698 if (err)
699 return err;
700 }
701 }
702
703 return 0;
704}
705
706/**
707 * idpf_rx_desc_alloc - Allocate queue Rx resources
708 * @rxq: Rx queue for which the resources are setup
709 * @bufq: buffer or completion queue
710 * @q_model: single or split queue model
711 *
712 * Returns 0 on success, negative on failure
713 */
714static int idpf_rx_desc_alloc(struct idpf_queue *rxq, bool bufq, s32 q_model)
715{
716 struct device *dev = rxq->dev;
717
718 if (bufq)
719 rxq->size = rxq->desc_count *
720 sizeof(struct virtchnl2_splitq_rx_buf_desc);
721 else
722 rxq->size = rxq->desc_count *
723 sizeof(union virtchnl2_rx_desc);
724
725 /* Allocate descriptors and also round up to nearest 4K */
726 rxq->size = ALIGN(rxq->size, 4096);
727 rxq->desc_ring = dmam_alloc_coherent(dev, rxq->size,
728 &rxq->dma, GFP_KERNEL);
729 if (!rxq->desc_ring) {
730 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
731 rxq->size);
732 return -ENOMEM;
733 }
734
735 rxq->next_to_alloc = 0;
736 rxq->next_to_clean = 0;
737 rxq->next_to_use = 0;
738 set_bit(__IDPF_Q_GEN_CHK, rxq->flags);
739
740 return 0;
741}
742
743/**
744 * idpf_rx_desc_alloc_all - allocate all RX queues resources
745 * @vport: virtual port structure
746 *
747 * Returns 0 on success, negative on failure
748 */
749static int idpf_rx_desc_alloc_all(struct idpf_vport *vport)
750{
751 struct device *dev = &vport->adapter->pdev->dev;
752 struct idpf_rxq_group *rx_qgrp;
753 struct idpf_queue *q;
754 int i, j, err;
755 u16 num_rxq;
756
757 for (i = 0; i < vport->num_rxq_grp; i++) {
758 rx_qgrp = &vport->rxq_grps[i];
759 if (idpf_is_queue_model_split(vport->rxq_model))
760 num_rxq = rx_qgrp->splitq.num_rxq_sets;
761 else
762 num_rxq = rx_qgrp->singleq.num_rxq;
763
764 for (j = 0; j < num_rxq; j++) {
765 if (idpf_is_queue_model_split(vport->rxq_model))
766 q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
767 else
768 q = rx_qgrp->singleq.rxqs[j];
769 err = idpf_rx_desc_alloc(q, false, vport->rxq_model);
770 if (err) {
771 dev_err(dev, "Memory allocation for Rx Queue %u failed\n",
772 i);
773 goto err_out;
774 }
775 }
776
777 if (!idpf_is_queue_model_split(vport->rxq_model))
778 continue;
779
780 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
781 q = &rx_qgrp->splitq.bufq_sets[j].bufq;
782 err = idpf_rx_desc_alloc(q, true, vport->rxq_model);
783 if (err) {
784 dev_err(dev, "Memory allocation for Rx Buffer Queue %u failed\n",
785 i);
786 goto err_out;
787 }
788 }
789 }
790
791 return 0;
792
793err_out:
794 idpf_rx_desc_rel_all(vport);
795
796 return err;
797}
798
799/**
800 * idpf_txq_group_rel - Release all resources for txq groups
801 * @vport: vport to release txq groups on
802 */
803static void idpf_txq_group_rel(struct idpf_vport *vport)
804{
805 int i, j;
806
807 if (!vport->txq_grps)
808 return;
809
810 for (i = 0; i < vport->num_txq_grp; i++) {
811 struct idpf_txq_group *txq_grp = &vport->txq_grps[i];
812
813 for (j = 0; j < txq_grp->num_txq; j++) {
814 kfree(txq_grp->txqs[j]);
815 txq_grp->txqs[j] = NULL;
816 }
817 kfree(txq_grp->complq);
818 txq_grp->complq = NULL;
819 }
820 kfree(vport->txq_grps);
821 vport->txq_grps = NULL;
822}
823
824/**
825 * idpf_rxq_sw_queue_rel - Release software queue resources
826 * @rx_qgrp: rx queue group with software queues
827 */
828static void idpf_rxq_sw_queue_rel(struct idpf_rxq_group *rx_qgrp)
829{
830 int i, j;
831
832 for (i = 0; i < rx_qgrp->vport->num_bufqs_per_qgrp; i++) {
833 struct idpf_bufq_set *bufq_set = &rx_qgrp->splitq.bufq_sets[i];
834
835 for (j = 0; j < bufq_set->num_refillqs; j++) {
836 kfree(bufq_set->refillqs[j].ring);
837 bufq_set->refillqs[j].ring = NULL;
838 }
839 kfree(bufq_set->refillqs);
840 bufq_set->refillqs = NULL;
841 }
842}
843
844/**
845 * idpf_rxq_group_rel - Release all resources for rxq groups
846 * @vport: vport to release rxq groups on
847 */
848static void idpf_rxq_group_rel(struct idpf_vport *vport)
849{
850 int i;
851
852 if (!vport->rxq_grps)
853 return;
854
855 for (i = 0; i < vport->num_rxq_grp; i++) {
856 struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
857 u16 num_rxq;
858 int j;
859
860 if (idpf_is_queue_model_split(vport->rxq_model)) {
861 num_rxq = rx_qgrp->splitq.num_rxq_sets;
862 for (j = 0; j < num_rxq; j++) {
863 kfree(rx_qgrp->splitq.rxq_sets[j]);
864 rx_qgrp->splitq.rxq_sets[j] = NULL;
865 }
866
867 idpf_rxq_sw_queue_rel(rx_qgrp);
868 kfree(rx_qgrp->splitq.bufq_sets);
869 rx_qgrp->splitq.bufq_sets = NULL;
870 } else {
871 num_rxq = rx_qgrp->singleq.num_rxq;
872 for (j = 0; j < num_rxq; j++) {
873 kfree(rx_qgrp->singleq.rxqs[j]);
874 rx_qgrp->singleq.rxqs[j] = NULL;
875 }
876 }
877 }
878 kfree(vport->rxq_grps);
879 vport->rxq_grps = NULL;
880}
881
882/**
883 * idpf_vport_queue_grp_rel_all - Release all queue groups
884 * @vport: vport to release queue groups for
885 */
886static void idpf_vport_queue_grp_rel_all(struct idpf_vport *vport)
887{
888 idpf_txq_group_rel(vport);
889 idpf_rxq_group_rel(vport);
890}
891
892/**
893 * idpf_vport_queues_rel - Free memory for all queues
894 * @vport: virtual port
895 *
896 * Free the memory allocated for queues associated to a vport
897 */
898void idpf_vport_queues_rel(struct idpf_vport *vport)
899{
900 idpf_tx_desc_rel_all(vport);
901 idpf_rx_desc_rel_all(vport);
902 idpf_vport_queue_grp_rel_all(vport);
903
904 kfree(vport->txqs);
905 vport->txqs = NULL;
906}
907
908/**
909 * idpf_vport_init_fast_path_txqs - Initialize fast path txq array
910 * @vport: vport to init txqs on
911 *
912 * We get a queue index from skb->queue_mapping and we need a fast way to
913 * dereference the queue from queue groups. This allows us to quickly pull a
914 * txq based on a queue index.
915 *
916 * Returns 0 on success, negative on failure
917 */
918static int idpf_vport_init_fast_path_txqs(struct idpf_vport *vport)
919{
920 int i, j, k = 0;
921
922 vport->txqs = kcalloc(vport->num_txq, sizeof(struct idpf_queue *),
923 GFP_KERNEL);
924
925 if (!vport->txqs)
926 return -ENOMEM;
927
928 for (i = 0; i < vport->num_txq_grp; i++) {
929 struct idpf_txq_group *tx_grp = &vport->txq_grps[i];
930
931 for (j = 0; j < tx_grp->num_txq; j++, k++) {
932 vport->txqs[k] = tx_grp->txqs[j];
933 vport->txqs[k]->idx = k;
934 }
935 }
936
937 return 0;
938}
939
940/**
941 * idpf_vport_init_num_qs - Initialize number of queues
942 * @vport: vport to initialize queues
943 * @vport_msg: data to be filled into vport
944 */
945void idpf_vport_init_num_qs(struct idpf_vport *vport,
946 struct virtchnl2_create_vport *vport_msg)
947{
948 struct idpf_vport_user_config_data *config_data;
949 u16 idx = vport->idx;
950
951 config_data = &vport->adapter->vport_config[idx]->user_config;
952 vport->num_txq = le16_to_cpu(vport_msg->num_tx_q);
953 vport->num_rxq = le16_to_cpu(vport_msg->num_rx_q);
954 /* number of txqs and rxqs in config data will be zeros only in the
955 * driver load path and we dont update them there after
956 */
957 if (!config_data->num_req_tx_qs && !config_data->num_req_rx_qs) {
958 config_data->num_req_tx_qs = le16_to_cpu(vport_msg->num_tx_q);
959 config_data->num_req_rx_qs = le16_to_cpu(vport_msg->num_rx_q);
960 }
961
962 if (idpf_is_queue_model_split(vport->txq_model))
963 vport->num_complq = le16_to_cpu(vport_msg->num_tx_complq);
964 if (idpf_is_queue_model_split(vport->rxq_model))
965 vport->num_bufq = le16_to_cpu(vport_msg->num_rx_bufq);
966
967 /* Adjust number of buffer queues per Rx queue group. */
968 if (!idpf_is_queue_model_split(vport->rxq_model)) {
969 vport->num_bufqs_per_qgrp = 0;
970 vport->bufq_size[0] = IDPF_RX_BUF_2048;
971
972 return;
973 }
974
975 vport->num_bufqs_per_qgrp = IDPF_MAX_BUFQS_PER_RXQ_GRP;
976 /* Bufq[0] default buffer size is 4K
977 * Bufq[1] default buffer size is 2K
978 */
979 vport->bufq_size[0] = IDPF_RX_BUF_4096;
980 vport->bufq_size[1] = IDPF_RX_BUF_2048;
981}
982
983/**
984 * idpf_vport_calc_num_q_desc - Calculate number of queue groups
985 * @vport: vport to calculate q groups for
986 */
987void idpf_vport_calc_num_q_desc(struct idpf_vport *vport)
988{
989 struct idpf_vport_user_config_data *config_data;
990 int num_bufqs = vport->num_bufqs_per_qgrp;
991 u32 num_req_txq_desc, num_req_rxq_desc;
992 u16 idx = vport->idx;
993 int i;
994
995 config_data = &vport->adapter->vport_config[idx]->user_config;
996 num_req_txq_desc = config_data->num_req_txq_desc;
997 num_req_rxq_desc = config_data->num_req_rxq_desc;
998
999 vport->complq_desc_count = 0;
1000 if (num_req_txq_desc) {
1001 vport->txq_desc_count = num_req_txq_desc;
1002 if (idpf_is_queue_model_split(vport->txq_model)) {
1003 vport->complq_desc_count = num_req_txq_desc;
1004 if (vport->complq_desc_count < IDPF_MIN_TXQ_COMPLQ_DESC)
1005 vport->complq_desc_count =
1006 IDPF_MIN_TXQ_COMPLQ_DESC;
1007 }
1008 } else {
1009 vport->txq_desc_count = IDPF_DFLT_TX_Q_DESC_COUNT;
1010 if (idpf_is_queue_model_split(vport->txq_model))
1011 vport->complq_desc_count =
1012 IDPF_DFLT_TX_COMPLQ_DESC_COUNT;
1013 }
1014
1015 if (num_req_rxq_desc)
1016 vport->rxq_desc_count = num_req_rxq_desc;
1017 else
1018 vport->rxq_desc_count = IDPF_DFLT_RX_Q_DESC_COUNT;
1019
1020 for (i = 0; i < num_bufqs; i++) {
1021 if (!vport->bufq_desc_count[i])
1022 vport->bufq_desc_count[i] =
1023 IDPF_RX_BUFQ_DESC_COUNT(vport->rxq_desc_count,
1024 num_bufqs);
1025 }
1026}
1027
1028/**
1029 * idpf_vport_calc_total_qs - Calculate total number of queues
1030 * @adapter: private data struct
1031 * @vport_idx: vport idx to retrieve vport pointer
1032 * @vport_msg: message to fill with data
1033 * @max_q: vport max queue info
1034 *
1035 * Return 0 on success, error value on failure.
1036 */
1037int idpf_vport_calc_total_qs(struct idpf_adapter *adapter, u16 vport_idx,
1038 struct virtchnl2_create_vport *vport_msg,
1039 struct idpf_vport_max_q *max_q)
1040{
1041 int dflt_splitq_txq_grps = 0, dflt_singleq_txqs = 0;
1042 int dflt_splitq_rxq_grps = 0, dflt_singleq_rxqs = 0;
1043 u16 num_req_tx_qs = 0, num_req_rx_qs = 0;
1044 struct idpf_vport_config *vport_config;
1045 u16 num_txq_grps, num_rxq_grps;
1046 u32 num_qs;
1047
1048 vport_config = adapter->vport_config[vport_idx];
1049 if (vport_config) {
1050 num_req_tx_qs = vport_config->user_config.num_req_tx_qs;
1051 num_req_rx_qs = vport_config->user_config.num_req_rx_qs;
1052 } else {
1053 int num_cpus;
1054
1055 /* Restrict num of queues to cpus online as a default
1056 * configuration to give best performance. User can always
1057 * override to a max number of queues via ethtool.
1058 */
1059 num_cpus = num_online_cpus();
1060
1061 dflt_splitq_txq_grps = min_t(int, max_q->max_txq, num_cpus);
1062 dflt_singleq_txqs = min_t(int, max_q->max_txq, num_cpus);
1063 dflt_splitq_rxq_grps = min_t(int, max_q->max_rxq, num_cpus);
1064 dflt_singleq_rxqs = min_t(int, max_q->max_rxq, num_cpus);
1065 }
1066
1067 if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->txq_model))) {
1068 num_txq_grps = num_req_tx_qs ? num_req_tx_qs : dflt_splitq_txq_grps;
1069 vport_msg->num_tx_complq = cpu_to_le16(num_txq_grps *
1070 IDPF_COMPLQ_PER_GROUP);
1071 vport_msg->num_tx_q = cpu_to_le16(num_txq_grps *
1072 IDPF_DFLT_SPLITQ_TXQ_PER_GROUP);
1073 } else {
1074 num_txq_grps = IDPF_DFLT_SINGLEQ_TX_Q_GROUPS;
1075 num_qs = num_txq_grps * (num_req_tx_qs ? num_req_tx_qs :
1076 dflt_singleq_txqs);
1077 vport_msg->num_tx_q = cpu_to_le16(num_qs);
1078 vport_msg->num_tx_complq = 0;
1079 }
1080 if (idpf_is_queue_model_split(le16_to_cpu(vport_msg->rxq_model))) {
1081 num_rxq_grps = num_req_rx_qs ? num_req_rx_qs : dflt_splitq_rxq_grps;
1082 vport_msg->num_rx_bufq = cpu_to_le16(num_rxq_grps *
1083 IDPF_MAX_BUFQS_PER_RXQ_GRP);
1084 vport_msg->num_rx_q = cpu_to_le16(num_rxq_grps *
1085 IDPF_DFLT_SPLITQ_RXQ_PER_GROUP);
1086 } else {
1087 num_rxq_grps = IDPF_DFLT_SINGLEQ_RX_Q_GROUPS;
1088 num_qs = num_rxq_grps * (num_req_rx_qs ? num_req_rx_qs :
1089 dflt_singleq_rxqs);
1090 vport_msg->num_rx_q = cpu_to_le16(num_qs);
1091 vport_msg->num_rx_bufq = 0;
1092 }
1093
1094 return 0;
1095}
1096
1097/**
1098 * idpf_vport_calc_num_q_groups - Calculate number of queue groups
1099 * @vport: vport to calculate q groups for
1100 */
1101void idpf_vport_calc_num_q_groups(struct idpf_vport *vport)
1102{
1103 if (idpf_is_queue_model_split(vport->txq_model))
1104 vport->num_txq_grp = vport->num_txq;
1105 else
1106 vport->num_txq_grp = IDPF_DFLT_SINGLEQ_TX_Q_GROUPS;
1107
1108 if (idpf_is_queue_model_split(vport->rxq_model))
1109 vport->num_rxq_grp = vport->num_rxq;
1110 else
1111 vport->num_rxq_grp = IDPF_DFLT_SINGLEQ_RX_Q_GROUPS;
1112}
1113
1114/**
1115 * idpf_vport_calc_numq_per_grp - Calculate number of queues per group
1116 * @vport: vport to calculate queues for
1117 * @num_txq: return parameter for number of TX queues
1118 * @num_rxq: return parameter for number of RX queues
1119 */
1120static void idpf_vport_calc_numq_per_grp(struct idpf_vport *vport,
1121 u16 *num_txq, u16 *num_rxq)
1122{
1123 if (idpf_is_queue_model_split(vport->txq_model))
1124 *num_txq = IDPF_DFLT_SPLITQ_TXQ_PER_GROUP;
1125 else
1126 *num_txq = vport->num_txq;
1127
1128 if (idpf_is_queue_model_split(vport->rxq_model))
1129 *num_rxq = IDPF_DFLT_SPLITQ_RXQ_PER_GROUP;
1130 else
1131 *num_rxq = vport->num_rxq;
1132}
1133
1134/**
1135 * idpf_rxq_set_descids - set the descids supported by this queue
1136 * @vport: virtual port data structure
1137 * @q: rx queue for which descids are set
1138 *
1139 */
1140static void idpf_rxq_set_descids(struct idpf_vport *vport, struct idpf_queue *q)
1141{
1142 if (vport->rxq_model == VIRTCHNL2_QUEUE_MODEL_SPLIT) {
1143 q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SPLITQ_M;
1144 } else {
1145 if (vport->base_rxd)
1146 q->rxdids = VIRTCHNL2_RXDID_1_32B_BASE_M;
1147 else
1148 q->rxdids = VIRTCHNL2_RXDID_2_FLEX_SQ_NIC_M;
1149 }
1150}
1151
1152/**
1153 * idpf_txq_group_alloc - Allocate all txq group resources
1154 * @vport: vport to allocate txq groups for
1155 * @num_txq: number of txqs to allocate for each group
1156 *
1157 * Returns 0 on success, negative on failure
1158 */
1159static int idpf_txq_group_alloc(struct idpf_vport *vport, u16 num_txq)
1160{
1161 bool flow_sch_en;
1162 int err, i;
1163
1164 vport->txq_grps = kcalloc(vport->num_txq_grp,
1165 sizeof(*vport->txq_grps), GFP_KERNEL);
1166 if (!vport->txq_grps)
1167 return -ENOMEM;
1168
1169 flow_sch_en = !idpf_is_cap_ena(vport->adapter, IDPF_OTHER_CAPS,
1170 VIRTCHNL2_CAP_SPLITQ_QSCHED);
1171
1172 for (i = 0; i < vport->num_txq_grp; i++) {
1173 struct idpf_txq_group *tx_qgrp = &vport->txq_grps[i];
1174 struct idpf_adapter *adapter = vport->adapter;
1175 int j;
1176
1177 tx_qgrp->vport = vport;
1178 tx_qgrp->num_txq = num_txq;
1179
1180 for (j = 0; j < tx_qgrp->num_txq; j++) {
1181 tx_qgrp->txqs[j] = kzalloc(sizeof(*tx_qgrp->txqs[j]),
1182 GFP_KERNEL);
1183 if (!tx_qgrp->txqs[j]) {
1184 err = -ENOMEM;
1185 goto err_alloc;
1186 }
1187 }
1188
1189 for (j = 0; j < tx_qgrp->num_txq; j++) {
1190 struct idpf_queue *q = tx_qgrp->txqs[j];
1191
1192 q->dev = &adapter->pdev->dev;
1193 q->desc_count = vport->txq_desc_count;
1194 q->tx_max_bufs = idpf_get_max_tx_bufs(adapter);
1195 q->tx_min_pkt_len = idpf_get_min_tx_pkt_len(adapter);
1196 q->vport = vport;
1197 q->txq_grp = tx_qgrp;
1198 hash_init(q->sched_buf_hash);
1199
1200 if (flow_sch_en)
1201 set_bit(__IDPF_Q_FLOW_SCH_EN, q->flags);
1202 }
1203
1204 if (!idpf_is_queue_model_split(vport->txq_model))
1205 continue;
1206
1207 tx_qgrp->complq = kcalloc(IDPF_COMPLQ_PER_GROUP,
1208 sizeof(*tx_qgrp->complq),
1209 GFP_KERNEL);
1210 if (!tx_qgrp->complq) {
1211 err = -ENOMEM;
1212 goto err_alloc;
1213 }
1214
1215 tx_qgrp->complq->dev = &adapter->pdev->dev;
1216 tx_qgrp->complq->desc_count = vport->complq_desc_count;
1217 tx_qgrp->complq->vport = vport;
1218 tx_qgrp->complq->txq_grp = tx_qgrp;
1219
1220 if (flow_sch_en)
1221 __set_bit(__IDPF_Q_FLOW_SCH_EN, tx_qgrp->complq->flags);
1222 }
1223
1224 return 0;
1225
1226err_alloc:
1227 idpf_txq_group_rel(vport);
1228
1229 return err;
1230}
1231
1232/**
1233 * idpf_rxq_group_alloc - Allocate all rxq group resources
1234 * @vport: vport to allocate rxq groups for
1235 * @num_rxq: number of rxqs to allocate for each group
1236 *
1237 * Returns 0 on success, negative on failure
1238 */
1239static int idpf_rxq_group_alloc(struct idpf_vport *vport, u16 num_rxq)
1240{
1241 struct idpf_adapter *adapter = vport->adapter;
1242 struct idpf_queue *q;
1243 int i, k, err = 0;
1244 bool hs;
1245
1246 vport->rxq_grps = kcalloc(vport->num_rxq_grp,
1247 sizeof(struct idpf_rxq_group), GFP_KERNEL);
1248 if (!vport->rxq_grps)
1249 return -ENOMEM;
1250
1251 hs = idpf_vport_get_hsplit(vport) == ETHTOOL_TCP_DATA_SPLIT_ENABLED;
1252
1253 for (i = 0; i < vport->num_rxq_grp; i++) {
1254 struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
1255 int j;
1256
1257 rx_qgrp->vport = vport;
1258 if (!idpf_is_queue_model_split(vport->rxq_model)) {
1259 rx_qgrp->singleq.num_rxq = num_rxq;
1260 for (j = 0; j < num_rxq; j++) {
1261 rx_qgrp->singleq.rxqs[j] =
1262 kzalloc(sizeof(*rx_qgrp->singleq.rxqs[j]),
1263 GFP_KERNEL);
1264 if (!rx_qgrp->singleq.rxqs[j]) {
1265 err = -ENOMEM;
1266 goto err_alloc;
1267 }
1268 }
1269 goto skip_splitq_rx_init;
1270 }
1271 rx_qgrp->splitq.num_rxq_sets = num_rxq;
1272
1273 for (j = 0; j < num_rxq; j++) {
1274 rx_qgrp->splitq.rxq_sets[j] =
1275 kzalloc(sizeof(struct idpf_rxq_set),
1276 GFP_KERNEL);
1277 if (!rx_qgrp->splitq.rxq_sets[j]) {
1278 err = -ENOMEM;
1279 goto err_alloc;
1280 }
1281 }
1282
1283 rx_qgrp->splitq.bufq_sets = kcalloc(vport->num_bufqs_per_qgrp,
1284 sizeof(struct idpf_bufq_set),
1285 GFP_KERNEL);
1286 if (!rx_qgrp->splitq.bufq_sets) {
1287 err = -ENOMEM;
1288 goto err_alloc;
1289 }
1290
1291 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
1292 struct idpf_bufq_set *bufq_set =
1293 &rx_qgrp->splitq.bufq_sets[j];
1294 int swq_size = sizeof(struct idpf_sw_queue);
1295
1296 q = &rx_qgrp->splitq.bufq_sets[j].bufq;
1297 q->dev = &adapter->pdev->dev;
1298 q->desc_count = vport->bufq_desc_count[j];
1299 q->vport = vport;
1300 q->rxq_grp = rx_qgrp;
1301 q->idx = j;
1302 q->rx_buf_size = vport->bufq_size[j];
1303 q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK;
1304 q->rx_buf_stride = IDPF_RX_BUF_STRIDE;
1305
1306 if (hs) {
1307 q->rx_hsplit_en = true;
1308 q->rx_hbuf_size = IDPF_HDR_BUF_SIZE;
1309 }
1310
1311 bufq_set->num_refillqs = num_rxq;
1312 bufq_set->refillqs = kcalloc(num_rxq, swq_size,
1313 GFP_KERNEL);
1314 if (!bufq_set->refillqs) {
1315 err = -ENOMEM;
1316 goto err_alloc;
1317 }
1318 for (k = 0; k < bufq_set->num_refillqs; k++) {
1319 struct idpf_sw_queue *refillq =
1320 &bufq_set->refillqs[k];
1321
1322 refillq->dev = &vport->adapter->pdev->dev;
1323 refillq->desc_count =
1324 vport->bufq_desc_count[j];
1325 set_bit(__IDPF_Q_GEN_CHK, refillq->flags);
1326 set_bit(__IDPF_RFLQ_GEN_CHK, refillq->flags);
1327 refillq->ring = kcalloc(refillq->desc_count,
1328 sizeof(u16),
1329 GFP_KERNEL);
1330 if (!refillq->ring) {
1331 err = -ENOMEM;
1332 goto err_alloc;
1333 }
1334 }
1335 }
1336
1337skip_splitq_rx_init:
1338 for (j = 0; j < num_rxq; j++) {
1339 if (!idpf_is_queue_model_split(vport->rxq_model)) {
1340 q = rx_qgrp->singleq.rxqs[j];
1341 goto setup_rxq;
1342 }
1343 q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
1344 rx_qgrp->splitq.rxq_sets[j]->refillq0 =
1345 &rx_qgrp->splitq.bufq_sets[0].refillqs[j];
1346 if (vport->num_bufqs_per_qgrp > IDPF_SINGLE_BUFQ_PER_RXQ_GRP)
1347 rx_qgrp->splitq.rxq_sets[j]->refillq1 =
1348 &rx_qgrp->splitq.bufq_sets[1].refillqs[j];
1349
1350 if (hs) {
1351 q->rx_hsplit_en = true;
1352 q->rx_hbuf_size = IDPF_HDR_BUF_SIZE;
1353 }
1354
1355setup_rxq:
1356 q->dev = &adapter->pdev->dev;
1357 q->desc_count = vport->rxq_desc_count;
1358 q->vport = vport;
1359 q->rxq_grp = rx_qgrp;
1360 q->idx = (i * num_rxq) + j;
1361 /* In splitq mode, RXQ buffer size should be
1362 * set to that of the first buffer queue
1363 * associated with this RXQ
1364 */
1365 q->rx_buf_size = vport->bufq_size[0];
1366 q->rx_buffer_low_watermark = IDPF_LOW_WATERMARK;
1367 q->rx_max_pkt_size = vport->netdev->mtu +
1368 IDPF_PACKET_HDR_PAD;
1369 idpf_rxq_set_descids(vport, q);
1370 }
1371 }
1372
1373err_alloc:
1374 if (err)
1375 idpf_rxq_group_rel(vport);
1376
1377 return err;
1378}
1379
1380/**
1381 * idpf_vport_queue_grp_alloc_all - Allocate all queue groups/resources
1382 * @vport: vport with qgrps to allocate
1383 *
1384 * Returns 0 on success, negative on failure
1385 */
1386static int idpf_vport_queue_grp_alloc_all(struct idpf_vport *vport)
1387{
1388 u16 num_txq, num_rxq;
1389 int err;
1390
1391 idpf_vport_calc_numq_per_grp(vport, &num_txq, &num_rxq);
1392
1393 err = idpf_txq_group_alloc(vport, num_txq);
1394 if (err)
1395 goto err_out;
1396
1397 err = idpf_rxq_group_alloc(vport, num_rxq);
1398 if (err)
1399 goto err_out;
1400
1401 return 0;
1402
1403err_out:
1404 idpf_vport_queue_grp_rel_all(vport);
1405
1406 return err;
1407}
1408
1409/**
1410 * idpf_vport_queues_alloc - Allocate memory for all queues
1411 * @vport: virtual port
1412 *
1413 * Allocate memory for queues associated with a vport. Returns 0 on success,
1414 * negative on failure.
1415 */
1416int idpf_vport_queues_alloc(struct idpf_vport *vport)
1417{
1418 int err;
1419
1420 err = idpf_vport_queue_grp_alloc_all(vport);
1421 if (err)
1422 goto err_out;
1423
1424 err = idpf_tx_desc_alloc_all(vport);
1425 if (err)
1426 goto err_out;
1427
1428 err = idpf_rx_desc_alloc_all(vport);
1429 if (err)
1430 goto err_out;
1431
1432 err = idpf_vport_init_fast_path_txqs(vport);
1433 if (err)
1434 goto err_out;
1435
1436 return 0;
1437
1438err_out:
1439 idpf_vport_queues_rel(vport);
1440
1441 return err;
1442}
1443
1444/**
1445 * idpf_tx_handle_sw_marker - Handle queue marker packet
1446 * @tx_q: tx queue to handle software marker
1447 */
1448static void idpf_tx_handle_sw_marker(struct idpf_queue *tx_q)
1449{
1450 struct idpf_vport *vport = tx_q->vport;
1451 int i;
1452
1453 clear_bit(__IDPF_Q_SW_MARKER, tx_q->flags);
1454 /* Hardware must write marker packets to all queues associated with
1455 * completion queues. So check if all queues received marker packets
1456 */
1457 for (i = 0; i < vport->num_txq; i++)
1458 /* If we're still waiting on any other TXQ marker completions,
1459 * just return now since we cannot wake up the marker_wq yet.
1460 */
1461 if (test_bit(__IDPF_Q_SW_MARKER, vport->txqs[i]->flags))
1462 return;
1463
1464 /* Drain complete */
1465 set_bit(IDPF_VPORT_SW_MARKER, vport->flags);
1466 wake_up(&vport->sw_marker_wq);
1467}
1468
1469/**
1470 * idpf_tx_splitq_clean_hdr - Clean TX buffer resources for header portion of
1471 * packet
1472 * @tx_q: tx queue to clean buffer from
1473 * @tx_buf: buffer to be cleaned
1474 * @cleaned: pointer to stats struct to track cleaned packets/bytes
1475 * @napi_budget: Used to determine if we are in netpoll
1476 */
1477static void idpf_tx_splitq_clean_hdr(struct idpf_queue *tx_q,
1478 struct idpf_tx_buf *tx_buf,
1479 struct idpf_cleaned_stats *cleaned,
1480 int napi_budget)
1481{
1482 napi_consume_skb(tx_buf->skb, napi_budget);
1483
1484 if (dma_unmap_len(tx_buf, len)) {
1485 dma_unmap_single(tx_q->dev,
1486 dma_unmap_addr(tx_buf, dma),
1487 dma_unmap_len(tx_buf, len),
1488 DMA_TO_DEVICE);
1489
1490 dma_unmap_len_set(tx_buf, len, 0);
1491 }
1492
1493 /* clear tx_buf data */
1494 tx_buf->skb = NULL;
1495
1496 cleaned->bytes += tx_buf->bytecount;
1497 cleaned->packets += tx_buf->gso_segs;
1498}
1499
1500/**
1501 * idpf_tx_clean_stashed_bufs - clean bufs that were stored for
1502 * out of order completions
1503 * @txq: queue to clean
1504 * @compl_tag: completion tag of packet to clean (from completion descriptor)
1505 * @cleaned: pointer to stats struct to track cleaned packets/bytes
1506 * @budget: Used to determine if we are in netpoll
1507 */
1508static void idpf_tx_clean_stashed_bufs(struct idpf_queue *txq, u16 compl_tag,
1509 struct idpf_cleaned_stats *cleaned,
1510 int budget)
1511{
1512 struct idpf_tx_stash *stash;
1513 struct hlist_node *tmp_buf;
1514
1515 /* Buffer completion */
1516 hash_for_each_possible_safe(txq->sched_buf_hash, stash, tmp_buf,
1517 hlist, compl_tag) {
1518 if (unlikely(stash->buf.compl_tag != (int)compl_tag))
1519 continue;
1520
1521 if (stash->buf.skb) {
1522 idpf_tx_splitq_clean_hdr(txq, &stash->buf, cleaned,
1523 budget);
1524 } else if (dma_unmap_len(&stash->buf, len)) {
1525 dma_unmap_page(txq->dev,
1526 dma_unmap_addr(&stash->buf, dma),
1527 dma_unmap_len(&stash->buf, len),
1528 DMA_TO_DEVICE);
1529 dma_unmap_len_set(&stash->buf, len, 0);
1530 }
1531
1532 /* Push shadow buf back onto stack */
1533 idpf_buf_lifo_push(&txq->buf_stack, stash);
1534
1535 hash_del(&stash->hlist);
1536 }
1537}
1538
1539/**
1540 * idpf_stash_flow_sch_buffers - store buffer parameters info to be freed at a
1541 * later time (only relevant for flow scheduling mode)
1542 * @txq: Tx queue to clean
1543 * @tx_buf: buffer to store
1544 */
1545static int idpf_stash_flow_sch_buffers(struct idpf_queue *txq,
1546 struct idpf_tx_buf *tx_buf)
1547{
1548 struct idpf_tx_stash *stash;
1549
1550 if (unlikely(!dma_unmap_addr(tx_buf, dma) &&
1551 !dma_unmap_len(tx_buf, len)))
1552 return 0;
1553
1554 stash = idpf_buf_lifo_pop(&txq->buf_stack);
1555 if (unlikely(!stash)) {
1556 net_err_ratelimited("%s: No out-of-order TX buffers left!\n",
1557 txq->vport->netdev->name);
1558
1559 return -ENOMEM;
1560 }
1561
1562 /* Store buffer params in shadow buffer */
1563 stash->buf.skb = tx_buf->skb;
1564 stash->buf.bytecount = tx_buf->bytecount;
1565 stash->buf.gso_segs = tx_buf->gso_segs;
1566 dma_unmap_addr_set(&stash->buf, dma, dma_unmap_addr(tx_buf, dma));
1567 dma_unmap_len_set(&stash->buf, len, dma_unmap_len(tx_buf, len));
1568 stash->buf.compl_tag = tx_buf->compl_tag;
1569
1570 /* Add buffer to buf_hash table to be freed later */
1571 hash_add(txq->sched_buf_hash, &stash->hlist, stash->buf.compl_tag);
1572
1573 memset(tx_buf, 0, sizeof(struct idpf_tx_buf));
1574
1575 /* Reinitialize buf_id portion of tag */
1576 tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
1577
1578 return 0;
1579}
1580
1581#define idpf_tx_splitq_clean_bump_ntc(txq, ntc, desc, buf) \
1582do { \
1583 (ntc)++; \
1584 if (unlikely(!(ntc))) { \
1585 ntc -= (txq)->desc_count; \
1586 buf = (txq)->tx_buf; \
1587 desc = IDPF_FLEX_TX_DESC(txq, 0); \
1588 } else { \
1589 (buf)++; \
1590 (desc)++; \
1591 } \
1592} while (0)
1593
1594/**
1595 * idpf_tx_splitq_clean - Reclaim resources from buffer queue
1596 * @tx_q: Tx queue to clean
1597 * @end: queue index until which it should be cleaned
1598 * @napi_budget: Used to determine if we are in netpoll
1599 * @cleaned: pointer to stats struct to track cleaned packets/bytes
1600 * @descs_only: true if queue is using flow-based scheduling and should
1601 * not clean buffers at this time
1602 *
1603 * Cleans the queue descriptor ring. If the queue is using queue-based
1604 * scheduling, the buffers will be cleaned as well. If the queue is using
1605 * flow-based scheduling, only the descriptors are cleaned at this time.
1606 * Separate packet completion events will be reported on the completion queue,
1607 * and the buffers will be cleaned separately. The stats are not updated from
1608 * this function when using flow-based scheduling.
1609 */
1610static void idpf_tx_splitq_clean(struct idpf_queue *tx_q, u16 end,
1611 int napi_budget,
1612 struct idpf_cleaned_stats *cleaned,
1613 bool descs_only)
1614{
1615 union idpf_tx_flex_desc *next_pending_desc = NULL;
1616 union idpf_tx_flex_desc *tx_desc;
1617 s16 ntc = tx_q->next_to_clean;
1618 struct idpf_tx_buf *tx_buf;
1619
1620 tx_desc = IDPF_FLEX_TX_DESC(tx_q, ntc);
1621 next_pending_desc = IDPF_FLEX_TX_DESC(tx_q, end);
1622 tx_buf = &tx_q->tx_buf[ntc];
1623 ntc -= tx_q->desc_count;
1624
1625 while (tx_desc != next_pending_desc) {
1626 union idpf_tx_flex_desc *eop_desc;
1627
1628 /* If this entry in the ring was used as a context descriptor,
1629 * it's corresponding entry in the buffer ring will have an
1630 * invalid completion tag since no buffer was used. We can
1631 * skip this descriptor since there is no buffer to clean.
1632 */
1633 if (unlikely(tx_buf->compl_tag == IDPF_SPLITQ_TX_INVAL_COMPL_TAG))
1634 goto fetch_next_txq_desc;
1635
1636 eop_desc = (union idpf_tx_flex_desc *)tx_buf->next_to_watch;
1637
1638 /* clear next_to_watch to prevent false hangs */
1639 tx_buf->next_to_watch = NULL;
1640
1641 if (descs_only) {
1642 if (idpf_stash_flow_sch_buffers(tx_q, tx_buf))
1643 goto tx_splitq_clean_out;
1644
1645 while (tx_desc != eop_desc) {
1646 idpf_tx_splitq_clean_bump_ntc(tx_q, ntc,
1647 tx_desc, tx_buf);
1648
1649 if (dma_unmap_len(tx_buf, len)) {
1650 if (idpf_stash_flow_sch_buffers(tx_q,
1651 tx_buf))
1652 goto tx_splitq_clean_out;
1653 }
1654 }
1655 } else {
1656 idpf_tx_splitq_clean_hdr(tx_q, tx_buf, cleaned,
1657 napi_budget);
1658
1659 /* unmap remaining buffers */
1660 while (tx_desc != eop_desc) {
1661 idpf_tx_splitq_clean_bump_ntc(tx_q, ntc,
1662 tx_desc, tx_buf);
1663
1664 /* unmap any remaining paged data */
1665 if (dma_unmap_len(tx_buf, len)) {
1666 dma_unmap_page(tx_q->dev,
1667 dma_unmap_addr(tx_buf, dma),
1668 dma_unmap_len(tx_buf, len),
1669 DMA_TO_DEVICE);
1670 dma_unmap_len_set(tx_buf, len, 0);
1671 }
1672 }
1673 }
1674
1675fetch_next_txq_desc:
1676 idpf_tx_splitq_clean_bump_ntc(tx_q, ntc, tx_desc, tx_buf);
1677 }
1678
1679tx_splitq_clean_out:
1680 ntc += tx_q->desc_count;
1681 tx_q->next_to_clean = ntc;
1682}
1683
1684#define idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, buf) \
1685do { \
1686 (buf)++; \
1687 (ntc)++; \
1688 if (unlikely((ntc) == (txq)->desc_count)) { \
1689 buf = (txq)->tx_buf; \
1690 ntc = 0; \
1691 } \
1692} while (0)
1693
1694/**
1695 * idpf_tx_clean_buf_ring - clean flow scheduling TX queue buffers
1696 * @txq: queue to clean
1697 * @compl_tag: completion tag of packet to clean (from completion descriptor)
1698 * @cleaned: pointer to stats struct to track cleaned packets/bytes
1699 * @budget: Used to determine if we are in netpoll
1700 *
1701 * Cleans all buffers associated with the input completion tag either from the
1702 * TX buffer ring or from the hash table if the buffers were previously
1703 * stashed. Returns the byte/segment count for the cleaned packet associated
1704 * this completion tag.
1705 */
1706static bool idpf_tx_clean_buf_ring(struct idpf_queue *txq, u16 compl_tag,
1707 struct idpf_cleaned_stats *cleaned,
1708 int budget)
1709{
1710 u16 idx = compl_tag & txq->compl_tag_bufid_m;
1711 struct idpf_tx_buf *tx_buf = NULL;
1712 u16 ntc = txq->next_to_clean;
1713 u16 num_descs_cleaned = 0;
1714 u16 orig_idx = idx;
1715
1716 tx_buf = &txq->tx_buf[idx];
1717
1718 while (tx_buf->compl_tag == (int)compl_tag) {
1719 if (tx_buf->skb) {
1720 idpf_tx_splitq_clean_hdr(txq, tx_buf, cleaned, budget);
1721 } else if (dma_unmap_len(tx_buf, len)) {
1722 dma_unmap_page(txq->dev,
1723 dma_unmap_addr(tx_buf, dma),
1724 dma_unmap_len(tx_buf, len),
1725 DMA_TO_DEVICE);
1726 dma_unmap_len_set(tx_buf, len, 0);
1727 }
1728
1729 memset(tx_buf, 0, sizeof(struct idpf_tx_buf));
1730 tx_buf->compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
1731
1732 num_descs_cleaned++;
1733 idpf_tx_clean_buf_ring_bump_ntc(txq, idx, tx_buf);
1734 }
1735
1736 /* If we didn't clean anything on the ring for this completion, there's
1737 * nothing more to do.
1738 */
1739 if (unlikely(!num_descs_cleaned))
1740 return false;
1741
1742 /* Otherwise, if we did clean a packet on the ring directly, it's safe
1743 * to assume that the descriptors starting from the original
1744 * next_to_clean up until the previously cleaned packet can be reused.
1745 * Therefore, we will go back in the ring and stash any buffers still
1746 * in the ring into the hash table to be cleaned later.
1747 */
1748 tx_buf = &txq->tx_buf[ntc];
1749 while (tx_buf != &txq->tx_buf[orig_idx]) {
1750 idpf_stash_flow_sch_buffers(txq, tx_buf);
1751 idpf_tx_clean_buf_ring_bump_ntc(txq, ntc, tx_buf);
1752 }
1753
1754 /* Finally, update next_to_clean to reflect the work that was just done
1755 * on the ring, if any. If the packet was only cleaned from the hash
1756 * table, the ring will not be impacted, therefore we should not touch
1757 * next_to_clean. The updated idx is used here
1758 */
1759 txq->next_to_clean = idx;
1760
1761 return true;
1762}
1763
1764/**
1765 * idpf_tx_handle_rs_completion - clean a single packet and all of its buffers
1766 * whether on the buffer ring or in the hash table
1767 * @txq: Tx ring to clean
1768 * @desc: pointer to completion queue descriptor to extract completion
1769 * information from
1770 * @cleaned: pointer to stats struct to track cleaned packets/bytes
1771 * @budget: Used to determine if we are in netpoll
1772 *
1773 * Returns bytes/packets cleaned
1774 */
1775static void idpf_tx_handle_rs_completion(struct idpf_queue *txq,
1776 struct idpf_splitq_tx_compl_desc *desc,
1777 struct idpf_cleaned_stats *cleaned,
1778 int budget)
1779{
1780 u16 compl_tag;
1781
1782 if (!test_bit(__IDPF_Q_FLOW_SCH_EN, txq->flags)) {
1783 u16 head = le16_to_cpu(desc->q_head_compl_tag.q_head);
1784
1785 return idpf_tx_splitq_clean(txq, head, budget, cleaned, false);
1786 }
1787
1788 compl_tag = le16_to_cpu(desc->q_head_compl_tag.compl_tag);
1789
1790 /* If we didn't clean anything on the ring, this packet must be
1791 * in the hash table. Go clean it there.
1792 */
1793 if (!idpf_tx_clean_buf_ring(txq, compl_tag, cleaned, budget))
1794 idpf_tx_clean_stashed_bufs(txq, compl_tag, cleaned, budget);
1795}
1796
1797/**
1798 * idpf_tx_clean_complq - Reclaim resources on completion queue
1799 * @complq: Tx ring to clean
1800 * @budget: Used to determine if we are in netpoll
1801 * @cleaned: returns number of packets cleaned
1802 *
1803 * Returns true if there's any budget left (e.g. the clean is finished)
1804 */
1805static bool idpf_tx_clean_complq(struct idpf_queue *complq, int budget,
1806 int *cleaned)
1807{
1808 struct idpf_splitq_tx_compl_desc *tx_desc;
1809 struct idpf_vport *vport = complq->vport;
1810 s16 ntc = complq->next_to_clean;
1811 struct idpf_netdev_priv *np;
1812 unsigned int complq_budget;
1813 bool complq_ok = true;
1814 int i;
1815
1816 complq_budget = vport->compln_clean_budget;
1817 tx_desc = IDPF_SPLITQ_TX_COMPLQ_DESC(complq, ntc);
1818 ntc -= complq->desc_count;
1819
1820 do {
1821 struct idpf_cleaned_stats cleaned_stats = { };
1822 struct idpf_queue *tx_q;
1823 int rel_tx_qid;
1824 u16 hw_head;
1825 u8 ctype; /* completion type */
1826 u16 gen;
1827
1828 /* if the descriptor isn't done, no work yet to do */
1829 gen = le16_get_bits(tx_desc->qid_comptype_gen,
1830 IDPF_TXD_COMPLQ_GEN_M);
1831 if (test_bit(__IDPF_Q_GEN_CHK, complq->flags) != gen)
1832 break;
1833
1834 /* Find necessary info of TX queue to clean buffers */
1835 rel_tx_qid = le16_get_bits(tx_desc->qid_comptype_gen,
1836 IDPF_TXD_COMPLQ_QID_M);
1837 if (rel_tx_qid >= complq->txq_grp->num_txq ||
1838 !complq->txq_grp->txqs[rel_tx_qid]) {
1839 dev_err(&complq->vport->adapter->pdev->dev,
1840 "TxQ not found\n");
1841 goto fetch_next_desc;
1842 }
1843 tx_q = complq->txq_grp->txqs[rel_tx_qid];
1844
1845 /* Determine completion type */
1846 ctype = le16_get_bits(tx_desc->qid_comptype_gen,
1847 IDPF_TXD_COMPLQ_COMPL_TYPE_M);
1848 switch (ctype) {
1849 case IDPF_TXD_COMPLT_RE:
1850 hw_head = le16_to_cpu(tx_desc->q_head_compl_tag.q_head);
1851
1852 idpf_tx_splitq_clean(tx_q, hw_head, budget,
1853 &cleaned_stats, true);
1854 break;
1855 case IDPF_TXD_COMPLT_RS:
1856 idpf_tx_handle_rs_completion(tx_q, tx_desc,
1857 &cleaned_stats, budget);
1858 break;
1859 case IDPF_TXD_COMPLT_SW_MARKER:
1860 idpf_tx_handle_sw_marker(tx_q);
1861 break;
1862 default:
1863 dev_err(&tx_q->vport->adapter->pdev->dev,
1864 "Unknown TX completion type: %d\n",
1865 ctype);
1866 goto fetch_next_desc;
1867 }
1868
1869 u64_stats_update_begin(&tx_q->stats_sync);
1870 u64_stats_add(&tx_q->q_stats.tx.packets, cleaned_stats.packets);
1871 u64_stats_add(&tx_q->q_stats.tx.bytes, cleaned_stats.bytes);
1872 tx_q->cleaned_pkts += cleaned_stats.packets;
1873 tx_q->cleaned_bytes += cleaned_stats.bytes;
1874 complq->num_completions++;
1875 u64_stats_update_end(&tx_q->stats_sync);
1876
1877fetch_next_desc:
1878 tx_desc++;
1879 ntc++;
1880 if (unlikely(!ntc)) {
1881 ntc -= complq->desc_count;
1882 tx_desc = IDPF_SPLITQ_TX_COMPLQ_DESC(complq, 0);
1883 change_bit(__IDPF_Q_GEN_CHK, complq->flags);
1884 }
1885
1886 prefetch(tx_desc);
1887
1888 /* update budget accounting */
1889 complq_budget--;
1890 } while (likely(complq_budget));
1891
1892 /* Store the state of the complq to be used later in deciding if a
1893 * TXQ can be started again
1894 */
1895 if (unlikely(IDPF_TX_COMPLQ_PENDING(complq->txq_grp) >
1896 IDPF_TX_COMPLQ_OVERFLOW_THRESH(complq)))
1897 complq_ok = false;
1898
1899 np = netdev_priv(complq->vport->netdev);
1900 for (i = 0; i < complq->txq_grp->num_txq; ++i) {
1901 struct idpf_queue *tx_q = complq->txq_grp->txqs[i];
1902 struct netdev_queue *nq;
1903 bool dont_wake;
1904
1905 /* We didn't clean anything on this queue, move along */
1906 if (!tx_q->cleaned_bytes)
1907 continue;
1908
1909 *cleaned += tx_q->cleaned_pkts;
1910
1911 /* Update BQL */
1912 nq = netdev_get_tx_queue(tx_q->vport->netdev, tx_q->idx);
1913
1914 dont_wake = !complq_ok || IDPF_TX_BUF_RSV_LOW(tx_q) ||
1915 np->state != __IDPF_VPORT_UP ||
1916 !netif_carrier_ok(tx_q->vport->netdev);
1917 /* Check if the TXQ needs to and can be restarted */
1918 __netif_txq_completed_wake(nq, tx_q->cleaned_pkts, tx_q->cleaned_bytes,
1919 IDPF_DESC_UNUSED(tx_q), IDPF_TX_WAKE_THRESH,
1920 dont_wake);
1921
1922 /* Reset cleaned stats for the next time this queue is
1923 * cleaned
1924 */
1925 tx_q->cleaned_bytes = 0;
1926 tx_q->cleaned_pkts = 0;
1927 }
1928
1929 ntc += complq->desc_count;
1930 complq->next_to_clean = ntc;
1931
1932 return !!complq_budget;
1933}
1934
1935/**
1936 * idpf_tx_splitq_build_ctb - populate command tag and size for queue
1937 * based scheduling descriptors
1938 * @desc: descriptor to populate
1939 * @params: pointer to tx params struct
1940 * @td_cmd: command to be filled in desc
1941 * @size: size of buffer
1942 */
1943void idpf_tx_splitq_build_ctb(union idpf_tx_flex_desc *desc,
1944 struct idpf_tx_splitq_params *params,
1945 u16 td_cmd, u16 size)
1946{
1947 desc->q.qw1.cmd_dtype =
1948 le16_encode_bits(params->dtype, IDPF_FLEX_TXD_QW1_DTYPE_M);
1949 desc->q.qw1.cmd_dtype |=
1950 le16_encode_bits(td_cmd, IDPF_FLEX_TXD_QW1_CMD_M);
1951 desc->q.qw1.buf_size = cpu_to_le16(size);
1952 desc->q.qw1.l2tags.l2tag1 = cpu_to_le16(params->td_tag);
1953}
1954
1955/**
1956 * idpf_tx_splitq_build_flow_desc - populate command tag and size for flow
1957 * scheduling descriptors
1958 * @desc: descriptor to populate
1959 * @params: pointer to tx params struct
1960 * @td_cmd: command to be filled in desc
1961 * @size: size of buffer
1962 */
1963void idpf_tx_splitq_build_flow_desc(union idpf_tx_flex_desc *desc,
1964 struct idpf_tx_splitq_params *params,
1965 u16 td_cmd, u16 size)
1966{
1967 desc->flow.qw1.cmd_dtype = (u16)params->dtype | td_cmd;
1968 desc->flow.qw1.rxr_bufsize = cpu_to_le16((u16)size);
1969 desc->flow.qw1.compl_tag = cpu_to_le16(params->compl_tag);
1970}
1971
1972/**
1973 * idpf_tx_maybe_stop_common - 1st level check for common Tx stop conditions
1974 * @tx_q: the queue to be checked
1975 * @size: number of descriptors we want to assure is available
1976 *
1977 * Returns 0 if stop is not needed
1978 */
1979int idpf_tx_maybe_stop_common(struct idpf_queue *tx_q, unsigned int size)
1980{
1981 struct netdev_queue *nq;
1982
1983 if (likely(IDPF_DESC_UNUSED(tx_q) >= size))
1984 return 0;
1985
1986 u64_stats_update_begin(&tx_q->stats_sync);
1987 u64_stats_inc(&tx_q->q_stats.tx.q_busy);
1988 u64_stats_update_end(&tx_q->stats_sync);
1989
1990 nq = netdev_get_tx_queue(tx_q->vport->netdev, tx_q->idx);
1991
1992 return netif_txq_maybe_stop(nq, IDPF_DESC_UNUSED(tx_q), size, size);
1993}
1994
1995/**
1996 * idpf_tx_maybe_stop_splitq - 1st level check for Tx splitq stop conditions
1997 * @tx_q: the queue to be checked
1998 * @descs_needed: number of descriptors required for this packet
1999 *
2000 * Returns 0 if stop is not needed
2001 */
2002static int idpf_tx_maybe_stop_splitq(struct idpf_queue *tx_q,
2003 unsigned int descs_needed)
2004{
2005 if (idpf_tx_maybe_stop_common(tx_q, descs_needed))
2006 goto splitq_stop;
2007
2008 /* If there are too many outstanding completions expected on the
2009 * completion queue, stop the TX queue to give the device some time to
2010 * catch up
2011 */
2012 if (unlikely(IDPF_TX_COMPLQ_PENDING(tx_q->txq_grp) >
2013 IDPF_TX_COMPLQ_OVERFLOW_THRESH(tx_q->txq_grp->complq)))
2014 goto splitq_stop;
2015
2016 /* Also check for available book keeping buffers; if we are low, stop
2017 * the queue to wait for more completions
2018 */
2019 if (unlikely(IDPF_TX_BUF_RSV_LOW(tx_q)))
2020 goto splitq_stop;
2021
2022 return 0;
2023
2024splitq_stop:
2025 u64_stats_update_begin(&tx_q->stats_sync);
2026 u64_stats_inc(&tx_q->q_stats.tx.q_busy);
2027 u64_stats_update_end(&tx_q->stats_sync);
2028 netif_stop_subqueue(tx_q->vport->netdev, tx_q->idx);
2029
2030 return -EBUSY;
2031}
2032
2033/**
2034 * idpf_tx_buf_hw_update - Store the new tail value
2035 * @tx_q: queue to bump
2036 * @val: new tail index
2037 * @xmit_more: more skb's pending
2038 *
2039 * The naming here is special in that 'hw' signals that this function is about
2040 * to do a register write to update our queue status. We know this can only
2041 * mean tail here as HW should be owning head for TX.
2042 */
2043void idpf_tx_buf_hw_update(struct idpf_queue *tx_q, u32 val,
2044 bool xmit_more)
2045{
2046 struct netdev_queue *nq;
2047
2048 nq = netdev_get_tx_queue(tx_q->vport->netdev, tx_q->idx);
2049 tx_q->next_to_use = val;
2050
2051 idpf_tx_maybe_stop_common(tx_q, IDPF_TX_DESC_NEEDED);
2052
2053 /* Force memory writes to complete before letting h/w
2054 * know there are new descriptors to fetch. (Only
2055 * applicable for weak-ordered memory model archs,
2056 * such as IA-64).
2057 */
2058 wmb();
2059
2060 /* notify HW of packet */
2061 if (netif_xmit_stopped(nq) || !xmit_more)
2062 writel(val, tx_q->tail);
2063}
2064
2065/**
2066 * idpf_tx_desc_count_required - calculate number of Tx descriptors needed
2067 * @txq: queue to send buffer on
2068 * @skb: send buffer
2069 *
2070 * Returns number of data descriptors needed for this skb.
2071 */
2072unsigned int idpf_tx_desc_count_required(struct idpf_queue *txq,
2073 struct sk_buff *skb)
2074{
2075 const struct skb_shared_info *shinfo;
2076 unsigned int count = 0, i;
2077
2078 count += !!skb_headlen(skb);
2079
2080 if (!skb_is_nonlinear(skb))
2081 return count;
2082
2083 shinfo = skb_shinfo(skb);
2084 for (i = 0; i < shinfo->nr_frags; i++) {
2085 unsigned int size;
2086
2087 size = skb_frag_size(&shinfo->frags[i]);
2088
2089 /* We only need to use the idpf_size_to_txd_count check if the
2090 * fragment is going to span multiple descriptors,
2091 * i.e. size >= 16K.
2092 */
2093 if (size >= SZ_16K)
2094 count += idpf_size_to_txd_count(size);
2095 else
2096 count++;
2097 }
2098
2099 if (idpf_chk_linearize(skb, txq->tx_max_bufs, count)) {
2100 if (__skb_linearize(skb))
2101 return 0;
2102
2103 count = idpf_size_to_txd_count(skb->len);
2104 u64_stats_update_begin(&txq->stats_sync);
2105 u64_stats_inc(&txq->q_stats.tx.linearize);
2106 u64_stats_update_end(&txq->stats_sync);
2107 }
2108
2109 return count;
2110}
2111
2112/**
2113 * idpf_tx_dma_map_error - handle TX DMA map errors
2114 * @txq: queue to send buffer on
2115 * @skb: send buffer
2116 * @first: original first buffer info buffer for packet
2117 * @idx: starting point on ring to unwind
2118 */
2119void idpf_tx_dma_map_error(struct idpf_queue *txq, struct sk_buff *skb,
2120 struct idpf_tx_buf *first, u16 idx)
2121{
2122 u64_stats_update_begin(&txq->stats_sync);
2123 u64_stats_inc(&txq->q_stats.tx.dma_map_errs);
2124 u64_stats_update_end(&txq->stats_sync);
2125
2126 /* clear dma mappings for failed tx_buf map */
2127 for (;;) {
2128 struct idpf_tx_buf *tx_buf;
2129
2130 tx_buf = &txq->tx_buf[idx];
2131 idpf_tx_buf_rel(txq, tx_buf);
2132 if (tx_buf == first)
2133 break;
2134 if (idx == 0)
2135 idx = txq->desc_count;
2136 idx--;
2137 }
2138
2139 if (skb_is_gso(skb)) {
2140 union idpf_tx_flex_desc *tx_desc;
2141
2142 /* If we failed a DMA mapping for a TSO packet, we will have
2143 * used one additional descriptor for a context
2144 * descriptor. Reset that here.
2145 */
2146 tx_desc = IDPF_FLEX_TX_DESC(txq, idx);
2147 memset(tx_desc, 0, sizeof(struct idpf_flex_tx_ctx_desc));
2148 if (idx == 0)
2149 idx = txq->desc_count;
2150 idx--;
2151 }
2152
2153 /* Update tail in case netdev_xmit_more was previously true */
2154 idpf_tx_buf_hw_update(txq, idx, false);
2155}
2156
2157/**
2158 * idpf_tx_splitq_bump_ntu - adjust NTU and generation
2159 * @txq: the tx ring to wrap
2160 * @ntu: ring index to bump
2161 */
2162static unsigned int idpf_tx_splitq_bump_ntu(struct idpf_queue *txq, u16 ntu)
2163{
2164 ntu++;
2165
2166 if (ntu == txq->desc_count) {
2167 ntu = 0;
2168 txq->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(txq);
2169 }
2170
2171 return ntu;
2172}
2173
2174/**
2175 * idpf_tx_splitq_map - Build the Tx flex descriptor
2176 * @tx_q: queue to send buffer on
2177 * @params: pointer to splitq params struct
2178 * @first: first buffer info buffer to use
2179 *
2180 * This function loops over the skb data pointed to by *first
2181 * and gets a physical address for each memory location and programs
2182 * it and the length into the transmit flex descriptor.
2183 */
2184static void idpf_tx_splitq_map(struct idpf_queue *tx_q,
2185 struct idpf_tx_splitq_params *params,
2186 struct idpf_tx_buf *first)
2187{
2188 union idpf_tx_flex_desc *tx_desc;
2189 unsigned int data_len, size;
2190 struct idpf_tx_buf *tx_buf;
2191 u16 i = tx_q->next_to_use;
2192 struct netdev_queue *nq;
2193 struct sk_buff *skb;
2194 skb_frag_t *frag;
2195 u16 td_cmd = 0;
2196 dma_addr_t dma;
2197
2198 skb = first->skb;
2199
2200 td_cmd = params->offload.td_cmd;
2201
2202 data_len = skb->data_len;
2203 size = skb_headlen(skb);
2204
2205 tx_desc = IDPF_FLEX_TX_DESC(tx_q, i);
2206
2207 dma = dma_map_single(tx_q->dev, skb->data, size, DMA_TO_DEVICE);
2208
2209 tx_buf = first;
2210
2211 params->compl_tag =
2212 (tx_q->compl_tag_cur_gen << tx_q->compl_tag_gen_s) | i;
2213
2214 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2215 unsigned int max_data = IDPF_TX_MAX_DESC_DATA_ALIGNED;
2216
2217 if (dma_mapping_error(tx_q->dev, dma))
2218 return idpf_tx_dma_map_error(tx_q, skb, first, i);
2219
2220 tx_buf->compl_tag = params->compl_tag;
2221
2222 /* record length, and DMA address */
2223 dma_unmap_len_set(tx_buf, len, size);
2224 dma_unmap_addr_set(tx_buf, dma, dma);
2225
2226 /* buf_addr is in same location for both desc types */
2227 tx_desc->q.buf_addr = cpu_to_le64(dma);
2228
2229 /* The stack can send us fragments that are too large for a
2230 * single descriptor i.e. frag size > 16K-1. We will need to
2231 * split the fragment across multiple descriptors in this case.
2232 * To adhere to HW alignment restrictions, the fragment needs
2233 * to be split such that the first chunk ends on a 4K boundary
2234 * and all subsequent chunks start on a 4K boundary. We still
2235 * want to send as much data as possible though, so our
2236 * intermediate descriptor chunk size will be 12K.
2237 *
2238 * For example, consider a 32K fragment mapped to DMA addr 2600.
2239 * ------------------------------------------------------------
2240 * | frag_size = 32K |
2241 * ------------------------------------------------------------
2242 * |2600 |16384 |28672
2243 *
2244 * 3 descriptors will be used for this fragment. The HW expects
2245 * the descriptors to contain the following:
2246 * ------------------------------------------------------------
2247 * | size = 13784 | size = 12K | size = 6696 |
2248 * | dma = 2600 | dma = 16384 | dma = 28672 |
2249 * ------------------------------------------------------------
2250 *
2251 * We need to first adjust the max_data for the first chunk so
2252 * that it ends on a 4K boundary. By negating the value of the
2253 * DMA address and taking only the low order bits, we're
2254 * effectively calculating
2255 * 4K - (DMA addr lower order bits) =
2256 * bytes to next boundary.
2257 *
2258 * Add that to our base aligned max_data (12K) and we have
2259 * our first chunk size. In the example above,
2260 * 13784 = 12K + (4096-2600)
2261 *
2262 * After guaranteeing the first chunk ends on a 4K boundary, we
2263 * will give the intermediate descriptors 12K chunks and
2264 * whatever is left to the final descriptor. This ensures that
2265 * all descriptors used for the remaining chunks of the
2266 * fragment start on a 4K boundary and we use as few
2267 * descriptors as possible.
2268 */
2269 max_data += -dma & (IDPF_TX_MAX_READ_REQ_SIZE - 1);
2270 while (unlikely(size > IDPF_TX_MAX_DESC_DATA)) {
2271 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd,
2272 max_data);
2273
2274 tx_desc++;
2275 i++;
2276
2277 if (i == tx_q->desc_count) {
2278 tx_desc = IDPF_FLEX_TX_DESC(tx_q, 0);
2279 i = 0;
2280 tx_q->compl_tag_cur_gen =
2281 IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q);
2282 }
2283
2284 /* Since this packet has a buffer that is going to span
2285 * multiple descriptors, it's going to leave holes in
2286 * to the TX buffer ring. To ensure these holes do not
2287 * cause issues in the cleaning routines, we will clear
2288 * them of any stale data and assign them the same
2289 * completion tag as the current packet. Then when the
2290 * packet is being cleaned, the cleaning routines will
2291 * simply pass over these holes and finish cleaning the
2292 * rest of the packet.
2293 */
2294 memset(&tx_q->tx_buf[i], 0, sizeof(struct idpf_tx_buf));
2295 tx_q->tx_buf[i].compl_tag = params->compl_tag;
2296
2297 /* Adjust the DMA offset and the remaining size of the
2298 * fragment. On the first iteration of this loop,
2299 * max_data will be >= 12K and <= 16K-1. On any
2300 * subsequent iteration of this loop, max_data will
2301 * always be 12K.
2302 */
2303 dma += max_data;
2304 size -= max_data;
2305
2306 /* Reset max_data since remaining chunks will be 12K
2307 * at most
2308 */
2309 max_data = IDPF_TX_MAX_DESC_DATA_ALIGNED;
2310
2311 /* buf_addr is in same location for both desc types */
2312 tx_desc->q.buf_addr = cpu_to_le64(dma);
2313 }
2314
2315 if (!data_len)
2316 break;
2317
2318 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size);
2319 tx_desc++;
2320 i++;
2321
2322 if (i == tx_q->desc_count) {
2323 tx_desc = IDPF_FLEX_TX_DESC(tx_q, 0);
2324 i = 0;
2325 tx_q->compl_tag_cur_gen = IDPF_TX_ADJ_COMPL_TAG_GEN(tx_q);
2326 }
2327
2328 size = skb_frag_size(frag);
2329 data_len -= size;
2330
2331 dma = skb_frag_dma_map(tx_q->dev, frag, 0, size,
2332 DMA_TO_DEVICE);
2333
2334 tx_buf = &tx_q->tx_buf[i];
2335 }
2336
2337 /* record SW timestamp if HW timestamp is not available */
2338 skb_tx_timestamp(skb);
2339
2340 /* write last descriptor with RS and EOP bits */
2341 td_cmd |= params->eop_cmd;
2342 idpf_tx_splitq_build_desc(tx_desc, params, td_cmd, size);
2343 i = idpf_tx_splitq_bump_ntu(tx_q, i);
2344
2345 /* set next_to_watch value indicating a packet is present */
2346 first->next_to_watch = tx_desc;
2347
2348 tx_q->txq_grp->num_completions_pending++;
2349
2350 /* record bytecount for BQL */
2351 nq = netdev_get_tx_queue(tx_q->vport->netdev, tx_q->idx);
2352 netdev_tx_sent_queue(nq, first->bytecount);
2353
2354 idpf_tx_buf_hw_update(tx_q, i, netdev_xmit_more());
2355}
2356
2357/**
2358 * idpf_tso - computes mss and TSO length to prepare for TSO
2359 * @skb: pointer to skb
2360 * @off: pointer to struct that holds offload parameters
2361 *
2362 * Returns error (negative) if TSO was requested but cannot be applied to the
2363 * given skb, 0 if TSO does not apply to the given skb, or 1 otherwise.
2364 */
2365int idpf_tso(struct sk_buff *skb, struct idpf_tx_offload_params *off)
2366{
2367 const struct skb_shared_info *shinfo;
2368 union {
2369 struct iphdr *v4;
2370 struct ipv6hdr *v6;
2371 unsigned char *hdr;
2372 } ip;
2373 union {
2374 struct tcphdr *tcp;
2375 struct udphdr *udp;
2376 unsigned char *hdr;
2377 } l4;
2378 u32 paylen, l4_start;
2379 int err;
2380
2381 if (!skb_is_gso(skb))
2382 return 0;
2383
2384 err = skb_cow_head(skb, 0);
2385 if (err < 0)
2386 return err;
2387
2388 shinfo = skb_shinfo(skb);
2389
2390 ip.hdr = skb_network_header(skb);
2391 l4.hdr = skb_transport_header(skb);
2392
2393 /* initialize outer IP header fields */
2394 if (ip.v4->version == 4) {
2395 ip.v4->tot_len = 0;
2396 ip.v4->check = 0;
2397 } else if (ip.v6->version == 6) {
2398 ip.v6->payload_len = 0;
2399 }
2400
2401 l4_start = skb_transport_offset(skb);
2402
2403 /* remove payload length from checksum */
2404 paylen = skb->len - l4_start;
2405
2406 switch (shinfo->gso_type & ~SKB_GSO_DODGY) {
2407 case SKB_GSO_TCPV4:
2408 case SKB_GSO_TCPV6:
2409 csum_replace_by_diff(&l4.tcp->check,
2410 (__force __wsum)htonl(paylen));
2411 off->tso_hdr_len = __tcp_hdrlen(l4.tcp) + l4_start;
2412 break;
2413 case SKB_GSO_UDP_L4:
2414 csum_replace_by_diff(&l4.udp->check,
2415 (__force __wsum)htonl(paylen));
2416 /* compute length of segmentation header */
2417 off->tso_hdr_len = sizeof(struct udphdr) + l4_start;
2418 l4.udp->len = htons(shinfo->gso_size + sizeof(struct udphdr));
2419 break;
2420 default:
2421 return -EINVAL;
2422 }
2423
2424 off->tso_len = skb->len - off->tso_hdr_len;
2425 off->mss = shinfo->gso_size;
2426 off->tso_segs = shinfo->gso_segs;
2427
2428 off->tx_flags |= IDPF_TX_FLAGS_TSO;
2429
2430 return 1;
2431}
2432
2433/**
2434 * __idpf_chk_linearize - Check skb is not using too many buffers
2435 * @skb: send buffer
2436 * @max_bufs: maximum number of buffers
2437 *
2438 * For TSO we need to count the TSO header and segment payload separately. As
2439 * such we need to check cases where we have max_bufs-1 fragments or more as we
2440 * can potentially require max_bufs+1 DMA transactions, 1 for the TSO header, 1
2441 * for the segment payload in the first descriptor, and another max_buf-1 for
2442 * the fragments.
2443 */
2444static bool __idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs)
2445{
2446 const struct skb_shared_info *shinfo = skb_shinfo(skb);
2447 const skb_frag_t *frag, *stale;
2448 int nr_frags, sum;
2449
2450 /* no need to check if number of frags is less than max_bufs - 1 */
2451 nr_frags = shinfo->nr_frags;
2452 if (nr_frags < (max_bufs - 1))
2453 return false;
2454
2455 /* We need to walk through the list and validate that each group
2456 * of max_bufs-2 fragments totals at least gso_size.
2457 */
2458 nr_frags -= max_bufs - 2;
2459 frag = &shinfo->frags[0];
2460
2461 /* Initialize size to the negative value of gso_size minus 1. We use
2462 * this as the worst case scenario in which the frag ahead of us only
2463 * provides one byte which is why we are limited to max_bufs-2
2464 * descriptors for a single transmit as the header and previous
2465 * fragment are already consuming 2 descriptors.
2466 */
2467 sum = 1 - shinfo->gso_size;
2468
2469 /* Add size of frags 0 through 4 to create our initial sum */
2470 sum += skb_frag_size(frag++);
2471 sum += skb_frag_size(frag++);
2472 sum += skb_frag_size(frag++);
2473 sum += skb_frag_size(frag++);
2474 sum += skb_frag_size(frag++);
2475
2476 /* Walk through fragments adding latest fragment, testing it, and
2477 * then removing stale fragments from the sum.
2478 */
2479 for (stale = &shinfo->frags[0];; stale++) {
2480 int stale_size = skb_frag_size(stale);
2481
2482 sum += skb_frag_size(frag++);
2483
2484 /* The stale fragment may present us with a smaller
2485 * descriptor than the actual fragment size. To account
2486 * for that we need to remove all the data on the front and
2487 * figure out what the remainder would be in the last
2488 * descriptor associated with the fragment.
2489 */
2490 if (stale_size > IDPF_TX_MAX_DESC_DATA) {
2491 int align_pad = -(skb_frag_off(stale)) &
2492 (IDPF_TX_MAX_READ_REQ_SIZE - 1);
2493
2494 sum -= align_pad;
2495 stale_size -= align_pad;
2496
2497 do {
2498 sum -= IDPF_TX_MAX_DESC_DATA_ALIGNED;
2499 stale_size -= IDPF_TX_MAX_DESC_DATA_ALIGNED;
2500 } while (stale_size > IDPF_TX_MAX_DESC_DATA);
2501 }
2502
2503 /* if sum is negative we failed to make sufficient progress */
2504 if (sum < 0)
2505 return true;
2506
2507 if (!nr_frags--)
2508 break;
2509
2510 sum -= stale_size;
2511 }
2512
2513 return false;
2514}
2515
2516/**
2517 * idpf_chk_linearize - Check if skb exceeds max descriptors per packet
2518 * @skb: send buffer
2519 * @max_bufs: maximum scatter gather buffers for single packet
2520 * @count: number of buffers this packet needs
2521 *
2522 * Make sure we don't exceed maximum scatter gather buffers for a single
2523 * packet. We have to do some special checking around the boundary (max_bufs-1)
2524 * if TSO is on since we need count the TSO header and payload separately.
2525 * E.g.: a packet with 7 fragments can require 9 DMA transactions; 1 for TSO
2526 * header, 1 for segment payload, and then 7 for the fragments.
2527 */
2528bool idpf_chk_linearize(struct sk_buff *skb, unsigned int max_bufs,
2529 unsigned int count)
2530{
2531 if (likely(count < max_bufs))
2532 return false;
2533 if (skb_is_gso(skb))
2534 return __idpf_chk_linearize(skb, max_bufs);
2535
2536 return count > max_bufs;
2537}
2538
2539/**
2540 * idpf_tx_splitq_get_ctx_desc - grab next desc and update buffer ring
2541 * @txq: queue to put context descriptor on
2542 *
2543 * Since the TX buffer rings mimics the descriptor ring, update the tx buffer
2544 * ring entry to reflect that this index is a context descriptor
2545 */
2546static struct idpf_flex_tx_ctx_desc *
2547idpf_tx_splitq_get_ctx_desc(struct idpf_queue *txq)
2548{
2549 struct idpf_flex_tx_ctx_desc *desc;
2550 int i = txq->next_to_use;
2551
2552 memset(&txq->tx_buf[i], 0, sizeof(struct idpf_tx_buf));
2553 txq->tx_buf[i].compl_tag = IDPF_SPLITQ_TX_INVAL_COMPL_TAG;
2554
2555 /* grab the next descriptor */
2556 desc = IDPF_FLEX_TX_CTX_DESC(txq, i);
2557 txq->next_to_use = idpf_tx_splitq_bump_ntu(txq, i);
2558
2559 return desc;
2560}
2561
2562/**
2563 * idpf_tx_drop_skb - free the SKB and bump tail if necessary
2564 * @tx_q: queue to send buffer on
2565 * @skb: pointer to skb
2566 */
2567netdev_tx_t idpf_tx_drop_skb(struct idpf_queue *tx_q, struct sk_buff *skb)
2568{
2569 u64_stats_update_begin(&tx_q->stats_sync);
2570 u64_stats_inc(&tx_q->q_stats.tx.skb_drops);
2571 u64_stats_update_end(&tx_q->stats_sync);
2572
2573 idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false);
2574
2575 dev_kfree_skb(skb);
2576
2577 return NETDEV_TX_OK;
2578}
2579
2580/**
2581 * idpf_tx_splitq_frame - Sends buffer on Tx ring using flex descriptors
2582 * @skb: send buffer
2583 * @tx_q: queue to send buffer on
2584 *
2585 * Returns NETDEV_TX_OK if sent, else an error code
2586 */
2587static netdev_tx_t idpf_tx_splitq_frame(struct sk_buff *skb,
2588 struct idpf_queue *tx_q)
2589{
2590 struct idpf_tx_splitq_params tx_params = { };
2591 struct idpf_tx_buf *first;
2592 unsigned int count;
2593 int tso;
2594
2595 count = idpf_tx_desc_count_required(tx_q, skb);
2596 if (unlikely(!count))
2597 return idpf_tx_drop_skb(tx_q, skb);
2598
2599 tso = idpf_tso(skb, &tx_params.offload);
2600 if (unlikely(tso < 0))
2601 return idpf_tx_drop_skb(tx_q, skb);
2602
2603 /* Check for splitq specific TX resources */
2604 count += (IDPF_TX_DESCS_PER_CACHE_LINE + tso);
2605 if (idpf_tx_maybe_stop_splitq(tx_q, count)) {
2606 idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false);
2607
2608 return NETDEV_TX_BUSY;
2609 }
2610
2611 if (tso) {
2612 /* If tso is needed, set up context desc */
2613 struct idpf_flex_tx_ctx_desc *ctx_desc =
2614 idpf_tx_splitq_get_ctx_desc(tx_q);
2615
2616 ctx_desc->tso.qw1.cmd_dtype =
2617 cpu_to_le16(IDPF_TX_DESC_DTYPE_FLEX_TSO_CTX |
2618 IDPF_TX_FLEX_CTX_DESC_CMD_TSO);
2619 ctx_desc->tso.qw0.flex_tlen =
2620 cpu_to_le32(tx_params.offload.tso_len &
2621 IDPF_TXD_FLEX_CTX_TLEN_M);
2622 ctx_desc->tso.qw0.mss_rt =
2623 cpu_to_le16(tx_params.offload.mss &
2624 IDPF_TXD_FLEX_CTX_MSS_RT_M);
2625 ctx_desc->tso.qw0.hdr_len = tx_params.offload.tso_hdr_len;
2626
2627 u64_stats_update_begin(&tx_q->stats_sync);
2628 u64_stats_inc(&tx_q->q_stats.tx.lso_pkts);
2629 u64_stats_update_end(&tx_q->stats_sync);
2630 }
2631
2632 /* record the location of the first descriptor for this packet */
2633 first = &tx_q->tx_buf[tx_q->next_to_use];
2634 first->skb = skb;
2635
2636 if (tso) {
2637 first->gso_segs = tx_params.offload.tso_segs;
2638 first->bytecount = skb->len +
2639 ((first->gso_segs - 1) * tx_params.offload.tso_hdr_len);
2640 } else {
2641 first->gso_segs = 1;
2642 first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN);
2643 }
2644
2645 if (test_bit(__IDPF_Q_FLOW_SCH_EN, tx_q->flags)) {
2646 tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_FLOW_SCHE;
2647 tx_params.eop_cmd = IDPF_TXD_FLEX_FLOW_CMD_EOP;
2648 /* Set the RE bit to catch any packets that may have not been
2649 * stashed during RS completion cleaning. MIN_GAP is set to
2650 * MIN_RING size to ensure it will be set at least once each
2651 * time around the ring.
2652 */
2653 if (!(tx_q->next_to_use % IDPF_TX_SPLITQ_RE_MIN_GAP)) {
2654 tx_params.eop_cmd |= IDPF_TXD_FLEX_FLOW_CMD_RE;
2655 tx_q->txq_grp->num_completions_pending++;
2656 }
2657
2658 if (skb->ip_summed == CHECKSUM_PARTIAL)
2659 tx_params.offload.td_cmd |= IDPF_TXD_FLEX_FLOW_CMD_CS_EN;
2660
2661 } else {
2662 tx_params.dtype = IDPF_TX_DESC_DTYPE_FLEX_L2TAG1_L2TAG2;
2663 tx_params.eop_cmd = IDPF_TXD_LAST_DESC_CMD;
2664
2665 if (skb->ip_summed == CHECKSUM_PARTIAL)
2666 tx_params.offload.td_cmd |= IDPF_TX_FLEX_DESC_CMD_CS_EN;
2667 }
2668
2669 idpf_tx_splitq_map(tx_q, &tx_params, first);
2670
2671 return NETDEV_TX_OK;
2672}
2673
2674/**
2675 * idpf_tx_splitq_start - Selects the right Tx queue to send buffer
2676 * @skb: send buffer
2677 * @netdev: network interface device structure
2678 *
2679 * Returns NETDEV_TX_OK if sent, else an error code
2680 */
2681netdev_tx_t idpf_tx_splitq_start(struct sk_buff *skb,
2682 struct net_device *netdev)
2683{
2684 struct idpf_vport *vport = idpf_netdev_to_vport(netdev);
2685 struct idpf_queue *tx_q;
2686
2687 if (unlikely(skb_get_queue_mapping(skb) >= vport->num_txq)) {
2688 dev_kfree_skb_any(skb);
2689
2690 return NETDEV_TX_OK;
2691 }
2692
2693 tx_q = vport->txqs[skb_get_queue_mapping(skb)];
2694
2695 /* hardware can't handle really short frames, hardware padding works
2696 * beyond this point
2697 */
2698 if (skb_put_padto(skb, tx_q->tx_min_pkt_len)) {
2699 idpf_tx_buf_hw_update(tx_q, tx_q->next_to_use, false);
2700
2701 return NETDEV_TX_OK;
2702 }
2703
2704 return idpf_tx_splitq_frame(skb, tx_q);
2705}
2706
2707/**
2708 * idpf_ptype_to_htype - get a hash type
2709 * @decoded: Decoded Rx packet type related fields
2710 *
2711 * Returns appropriate hash type (such as PKT_HASH_TYPE_L2/L3/L4) to be used by
2712 * skb_set_hash based on PTYPE as parsed by HW Rx pipeline and is part of
2713 * Rx desc.
2714 */
2715enum pkt_hash_types idpf_ptype_to_htype(const struct idpf_rx_ptype_decoded *decoded)
2716{
2717 if (!decoded->known)
2718 return PKT_HASH_TYPE_NONE;
2719 if (decoded->payload_layer == IDPF_RX_PTYPE_PAYLOAD_LAYER_PAY2 &&
2720 decoded->inner_prot)
2721 return PKT_HASH_TYPE_L4;
2722 if (decoded->payload_layer == IDPF_RX_PTYPE_PAYLOAD_LAYER_PAY2 &&
2723 decoded->outer_ip)
2724 return PKT_HASH_TYPE_L3;
2725 if (decoded->outer_ip == IDPF_RX_PTYPE_OUTER_L2)
2726 return PKT_HASH_TYPE_L2;
2727
2728 return PKT_HASH_TYPE_NONE;
2729}
2730
2731/**
2732 * idpf_rx_hash - set the hash value in the skb
2733 * @rxq: Rx descriptor ring packet is being transacted on
2734 * @skb: pointer to current skb being populated
2735 * @rx_desc: Receive descriptor
2736 * @decoded: Decoded Rx packet type related fields
2737 */
2738static void idpf_rx_hash(struct idpf_queue *rxq, struct sk_buff *skb,
2739 struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
2740 struct idpf_rx_ptype_decoded *decoded)
2741{
2742 u32 hash;
2743
2744 if (unlikely(!idpf_is_feature_ena(rxq->vport, NETIF_F_RXHASH)))
2745 return;
2746
2747 hash = le16_to_cpu(rx_desc->hash1) |
2748 (rx_desc->ff2_mirrid_hash2.hash2 << 16) |
2749 (rx_desc->hash3 << 24);
2750
2751 skb_set_hash(skb, hash, idpf_ptype_to_htype(decoded));
2752}
2753
2754/**
2755 * idpf_rx_csum - Indicate in skb if checksum is good
2756 * @rxq: Rx descriptor ring packet is being transacted on
2757 * @skb: pointer to current skb being populated
2758 * @csum_bits: checksum fields extracted from the descriptor
2759 * @decoded: Decoded Rx packet type related fields
2760 *
2761 * skb->protocol must be set before this function is called
2762 */
2763static void idpf_rx_csum(struct idpf_queue *rxq, struct sk_buff *skb,
2764 struct idpf_rx_csum_decoded *csum_bits,
2765 struct idpf_rx_ptype_decoded *decoded)
2766{
2767 bool ipv4, ipv6;
2768
2769 /* check if Rx checksum is enabled */
2770 if (unlikely(!idpf_is_feature_ena(rxq->vport, NETIF_F_RXCSUM)))
2771 return;
2772
2773 /* check if HW has decoded the packet and checksum */
2774 if (!(csum_bits->l3l4p))
2775 return;
2776
2777 ipv4 = IDPF_RX_PTYPE_TO_IPV(decoded, IDPF_RX_PTYPE_OUTER_IPV4);
2778 ipv6 = IDPF_RX_PTYPE_TO_IPV(decoded, IDPF_RX_PTYPE_OUTER_IPV6);
2779
2780 if (ipv4 && (csum_bits->ipe || csum_bits->eipe))
2781 goto checksum_fail;
2782
2783 if (ipv6 && csum_bits->ipv6exadd)
2784 return;
2785
2786 /* check for L4 errors and handle packets that were not able to be
2787 * checksummed
2788 */
2789 if (csum_bits->l4e)
2790 goto checksum_fail;
2791
2792 /* Only report checksum unnecessary for ICMP, TCP, UDP, or SCTP */
2793 switch (decoded->inner_prot) {
2794 case IDPF_RX_PTYPE_INNER_PROT_ICMP:
2795 case IDPF_RX_PTYPE_INNER_PROT_TCP:
2796 case IDPF_RX_PTYPE_INNER_PROT_UDP:
2797 if (!csum_bits->raw_csum_inv) {
2798 u16 csum = csum_bits->raw_csum;
2799
2800 skb->csum = csum_unfold((__force __sum16)~swab16(csum));
2801 skb->ip_summed = CHECKSUM_COMPLETE;
2802 } else {
2803 skb->ip_summed = CHECKSUM_UNNECESSARY;
2804 }
2805 break;
2806 case IDPF_RX_PTYPE_INNER_PROT_SCTP:
2807 skb->ip_summed = CHECKSUM_UNNECESSARY;
2808 break;
2809 default:
2810 break;
2811 }
2812
2813 return;
2814
2815checksum_fail:
2816 u64_stats_update_begin(&rxq->stats_sync);
2817 u64_stats_inc(&rxq->q_stats.rx.hw_csum_err);
2818 u64_stats_update_end(&rxq->stats_sync);
2819}
2820
2821/**
2822 * idpf_rx_splitq_extract_csum_bits - Extract checksum bits from descriptor
2823 * @rx_desc: receive descriptor
2824 * @csum: structure to extract checksum fields
2825 *
2826 **/
2827static void idpf_rx_splitq_extract_csum_bits(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
2828 struct idpf_rx_csum_decoded *csum)
2829{
2830 u8 qword0, qword1;
2831
2832 qword0 = rx_desc->status_err0_qw0;
2833 qword1 = rx_desc->status_err0_qw1;
2834
2835 csum->ipe = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_IPE_M,
2836 qword1);
2837 csum->eipe = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_EIPE_M,
2838 qword1);
2839 csum->l4e = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_XSUM_L4E_M,
2840 qword1);
2841 csum->l3l4p = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_L3L4P_M,
2842 qword1);
2843 csum->ipv6exadd = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_IPV6EXADD_M,
2844 qword0);
2845 csum->raw_csum_inv =
2846 le16_get_bits(rx_desc->ptype_err_fflags0,
2847 VIRTCHNL2_RX_FLEX_DESC_ADV_RAW_CSUM_INV_M);
2848 csum->raw_csum = le16_to_cpu(rx_desc->misc.raw_cs);
2849}
2850
2851/**
2852 * idpf_rx_rsc - Set the RSC fields in the skb
2853 * @rxq : Rx descriptor ring packet is being transacted on
2854 * @skb : pointer to current skb being populated
2855 * @rx_desc: Receive descriptor
2856 * @decoded: Decoded Rx packet type related fields
2857 *
2858 * Return 0 on success and error code on failure
2859 *
2860 * Populate the skb fields with the total number of RSC segments, RSC payload
2861 * length and packet type.
2862 */
2863static int idpf_rx_rsc(struct idpf_queue *rxq, struct sk_buff *skb,
2864 struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc,
2865 struct idpf_rx_ptype_decoded *decoded)
2866{
2867 u16 rsc_segments, rsc_seg_len;
2868 bool ipv4, ipv6;
2869 int len;
2870
2871 if (unlikely(!decoded->outer_ip))
2872 return -EINVAL;
2873
2874 rsc_seg_len = le16_to_cpu(rx_desc->misc.rscseglen);
2875 if (unlikely(!rsc_seg_len))
2876 return -EINVAL;
2877
2878 ipv4 = IDPF_RX_PTYPE_TO_IPV(decoded, IDPF_RX_PTYPE_OUTER_IPV4);
2879 ipv6 = IDPF_RX_PTYPE_TO_IPV(decoded, IDPF_RX_PTYPE_OUTER_IPV6);
2880
2881 if (unlikely(!(ipv4 ^ ipv6)))
2882 return -EINVAL;
2883
2884 rsc_segments = DIV_ROUND_UP(skb->data_len, rsc_seg_len);
2885 if (unlikely(rsc_segments == 1))
2886 return 0;
2887
2888 NAPI_GRO_CB(skb)->count = rsc_segments;
2889 skb_shinfo(skb)->gso_size = rsc_seg_len;
2890
2891 skb_reset_network_header(skb);
2892 len = skb->len - skb_transport_offset(skb);
2893
2894 if (ipv4) {
2895 struct iphdr *ipv4h = ip_hdr(skb);
2896
2897 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4;
2898
2899 /* Reset and set transport header offset in skb */
2900 skb_set_transport_header(skb, sizeof(struct iphdr));
2901
2902 /* Compute the TCP pseudo header checksum*/
2903 tcp_hdr(skb)->check =
2904 ~tcp_v4_check(len, ipv4h->saddr, ipv4h->daddr, 0);
2905 } else {
2906 struct ipv6hdr *ipv6h = ipv6_hdr(skb);
2907
2908 skb_shinfo(skb)->gso_type = SKB_GSO_TCPV6;
2909 skb_set_transport_header(skb, sizeof(struct ipv6hdr));
2910 tcp_hdr(skb)->check =
2911 ~tcp_v6_check(len, &ipv6h->saddr, &ipv6h->daddr, 0);
2912 }
2913
2914 tcp_gro_complete(skb);
2915
2916 u64_stats_update_begin(&rxq->stats_sync);
2917 u64_stats_inc(&rxq->q_stats.rx.rsc_pkts);
2918 u64_stats_update_end(&rxq->stats_sync);
2919
2920 return 0;
2921}
2922
2923/**
2924 * idpf_rx_process_skb_fields - Populate skb header fields from Rx descriptor
2925 * @rxq: Rx descriptor ring packet is being transacted on
2926 * @skb: pointer to current skb being populated
2927 * @rx_desc: Receive descriptor
2928 *
2929 * This function checks the ring, descriptor, and packet information in
2930 * order to populate the hash, checksum, protocol, and
2931 * other fields within the skb.
2932 */
2933static int idpf_rx_process_skb_fields(struct idpf_queue *rxq,
2934 struct sk_buff *skb,
2935 struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
2936{
2937 struct idpf_rx_csum_decoded csum_bits = { };
2938 struct idpf_rx_ptype_decoded decoded;
2939 u16 rx_ptype;
2940
2941 rx_ptype = le16_get_bits(rx_desc->ptype_err_fflags0,
2942 VIRTCHNL2_RX_FLEX_DESC_ADV_PTYPE_M);
2943
2944 skb->protocol = eth_type_trans(skb, rxq->vport->netdev);
2945
2946 decoded = rxq->vport->rx_ptype_lkup[rx_ptype];
2947 /* If we don't know the ptype we can't do anything else with it. Just
2948 * pass it up the stack as-is.
2949 */
2950 if (!decoded.known)
2951 return 0;
2952
2953 /* process RSS/hash */
2954 idpf_rx_hash(rxq, skb, rx_desc, &decoded);
2955
2956 if (le16_get_bits(rx_desc->hdrlen_flags,
2957 VIRTCHNL2_RX_FLEX_DESC_ADV_RSC_M))
2958 return idpf_rx_rsc(rxq, skb, rx_desc, &decoded);
2959
2960 idpf_rx_splitq_extract_csum_bits(rx_desc, &csum_bits);
2961 idpf_rx_csum(rxq, skb, &csum_bits, &decoded);
2962
2963 return 0;
2964}
2965
2966/**
2967 * idpf_rx_add_frag - Add contents of Rx buffer to sk_buff as a frag
2968 * @rx_buf: buffer containing page to add
2969 * @skb: sk_buff to place the data into
2970 * @size: packet length from rx_desc
2971 *
2972 * This function will add the data contained in rx_buf->page to the skb.
2973 * It will just attach the page as a frag to the skb.
2974 * The function will then update the page offset.
2975 */
2976void idpf_rx_add_frag(struct idpf_rx_buf *rx_buf, struct sk_buff *skb,
2977 unsigned int size)
2978{
2979 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buf->page,
2980 rx_buf->page_offset, size, rx_buf->truesize);
2981
2982 rx_buf->page = NULL;
2983}
2984
2985/**
2986 * idpf_rx_construct_skb - Allocate skb and populate it
2987 * @rxq: Rx descriptor queue
2988 * @rx_buf: Rx buffer to pull data from
2989 * @size: the length of the packet
2990 *
2991 * This function allocates an skb. It then populates it with the page
2992 * data from the current receive descriptor, taking care to set up the
2993 * skb correctly.
2994 */
2995struct sk_buff *idpf_rx_construct_skb(struct idpf_queue *rxq,
2996 struct idpf_rx_buf *rx_buf,
2997 unsigned int size)
2998{
2999 unsigned int headlen;
3000 struct sk_buff *skb;
3001 void *va;
3002
3003 va = page_address(rx_buf->page) + rx_buf->page_offset;
3004
3005 /* prefetch first cache line of first page */
3006 net_prefetch(va);
3007 /* allocate a skb to store the frags */
3008 skb = __napi_alloc_skb(&rxq->q_vector->napi, IDPF_RX_HDR_SIZE,
3009 GFP_ATOMIC);
3010 if (unlikely(!skb)) {
3011 idpf_rx_put_page(rx_buf);
3012
3013 return NULL;
3014 }
3015
3016 skb_record_rx_queue(skb, rxq->idx);
3017 skb_mark_for_recycle(skb);
3018
3019 /* Determine available headroom for copy */
3020 headlen = size;
3021 if (headlen > IDPF_RX_HDR_SIZE)
3022 headlen = eth_get_headlen(skb->dev, va, IDPF_RX_HDR_SIZE);
3023
3024 /* align pull length to size of long to optimize memcpy performance */
3025 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
3026
3027 /* if we exhaust the linear part then add what is left as a frag */
3028 size -= headlen;
3029 if (!size) {
3030 idpf_rx_put_page(rx_buf);
3031
3032 return skb;
3033 }
3034
3035 skb_add_rx_frag(skb, 0, rx_buf->page, rx_buf->page_offset + headlen,
3036 size, rx_buf->truesize);
3037
3038 /* Since we're giving the page to the stack, clear our reference to it.
3039 * We'll get a new one during buffer posting.
3040 */
3041 rx_buf->page = NULL;
3042
3043 return skb;
3044}
3045
3046/**
3047 * idpf_rx_hdr_construct_skb - Allocate skb and populate it from header buffer
3048 * @rxq: Rx descriptor queue
3049 * @va: Rx buffer to pull data from
3050 * @size: the length of the packet
3051 *
3052 * This function allocates an skb. It then populates it with the page data from
3053 * the current receive descriptor, taking care to set up the skb correctly.
3054 * This specifically uses a header buffer to start building the skb.
3055 */
3056static struct sk_buff *idpf_rx_hdr_construct_skb(struct idpf_queue *rxq,
3057 const void *va,
3058 unsigned int size)
3059{
3060 struct sk_buff *skb;
3061
3062 /* allocate a skb to store the frags */
3063 skb = __napi_alloc_skb(&rxq->q_vector->napi, size, GFP_ATOMIC);
3064 if (unlikely(!skb))
3065 return NULL;
3066
3067 skb_record_rx_queue(skb, rxq->idx);
3068
3069 memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
3070
3071 /* More than likely, a payload fragment, which will use a page from
3072 * page_pool will be added to the SKB so mark it for recycle
3073 * preemptively. And if not, it's inconsequential.
3074 */
3075 skb_mark_for_recycle(skb);
3076
3077 return skb;
3078}
3079
3080/**
3081 * idpf_rx_splitq_test_staterr - tests bits in Rx descriptor
3082 * status and error fields
3083 * @stat_err_field: field from descriptor to test bits in
3084 * @stat_err_bits: value to mask
3085 *
3086 */
3087static bool idpf_rx_splitq_test_staterr(const u8 stat_err_field,
3088 const u8 stat_err_bits)
3089{
3090 return !!(stat_err_field & stat_err_bits);
3091}
3092
3093/**
3094 * idpf_rx_splitq_is_eop - process handling of EOP buffers
3095 * @rx_desc: Rx descriptor for current buffer
3096 *
3097 * If the buffer is an EOP buffer, this function exits returning true,
3098 * otherwise return false indicating that this is in fact a non-EOP buffer.
3099 */
3100static bool idpf_rx_splitq_is_eop(struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc)
3101{
3102 /* if we are the last buffer then there is nothing else to do */
3103 return likely(idpf_rx_splitq_test_staterr(rx_desc->status_err0_qw1,
3104 IDPF_RXD_EOF_SPLITQ));
3105}
3106
3107/**
3108 * idpf_rx_splitq_clean - Clean completed descriptors from Rx queue
3109 * @rxq: Rx descriptor queue to retrieve receive buffer queue
3110 * @budget: Total limit on number of packets to process
3111 *
3112 * This function provides a "bounce buffer" approach to Rx interrupt
3113 * processing. The advantage to this is that on systems that have
3114 * expensive overhead for IOMMU access this provides a means of avoiding
3115 * it by maintaining the mapping of the page to the system.
3116 *
3117 * Returns amount of work completed
3118 */
3119static int idpf_rx_splitq_clean(struct idpf_queue *rxq, int budget)
3120{
3121 int total_rx_bytes = 0, total_rx_pkts = 0;
3122 struct idpf_queue *rx_bufq = NULL;
3123 struct sk_buff *skb = rxq->skb;
3124 u16 ntc = rxq->next_to_clean;
3125
3126 /* Process Rx packets bounded by budget */
3127 while (likely(total_rx_pkts < budget)) {
3128 struct virtchnl2_rx_flex_desc_adv_nic_3 *rx_desc;
3129 struct idpf_sw_queue *refillq = NULL;
3130 struct idpf_rxq_set *rxq_set = NULL;
3131 struct idpf_rx_buf *rx_buf = NULL;
3132 union virtchnl2_rx_desc *desc;
3133 unsigned int pkt_len = 0;
3134 unsigned int hdr_len = 0;
3135 u16 gen_id, buf_id = 0;
3136 /* Header buffer overflow only valid for header split */
3137 bool hbo = false;
3138 int bufq_id;
3139 u8 rxdid;
3140
3141 /* get the Rx desc from Rx queue based on 'next_to_clean' */
3142 desc = IDPF_RX_DESC(rxq, ntc);
3143 rx_desc = (struct virtchnl2_rx_flex_desc_adv_nic_3 *)desc;
3144
3145 /* This memory barrier is needed to keep us from reading
3146 * any other fields out of the rx_desc
3147 */
3148 dma_rmb();
3149
3150 /* if the descriptor isn't done, no work yet to do */
3151 gen_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id,
3152 VIRTCHNL2_RX_FLEX_DESC_ADV_GEN_M);
3153
3154 if (test_bit(__IDPF_Q_GEN_CHK, rxq->flags) != gen_id)
3155 break;
3156
3157 rxdid = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_RXDID_M,
3158 rx_desc->rxdid_ucast);
3159 if (rxdid != VIRTCHNL2_RXDID_2_FLEX_SPLITQ) {
3160 IDPF_RX_BUMP_NTC(rxq, ntc);
3161 u64_stats_update_begin(&rxq->stats_sync);
3162 u64_stats_inc(&rxq->q_stats.rx.bad_descs);
3163 u64_stats_update_end(&rxq->stats_sync);
3164 continue;
3165 }
3166
3167 pkt_len = le16_get_bits(rx_desc->pktlen_gen_bufq_id,
3168 VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_PBUF_M);
3169
3170 hbo = FIELD_GET(VIRTCHNL2_RX_FLEX_DESC_ADV_STATUS0_HBO_M,
3171 rx_desc->status_err0_qw1);
3172
3173 if (unlikely(hbo)) {
3174 /* If a header buffer overflow, occurs, i.e. header is
3175 * too large to fit in the header split buffer, HW will
3176 * put the entire packet, including headers, in the
3177 * data/payload buffer.
3178 */
3179 u64_stats_update_begin(&rxq->stats_sync);
3180 u64_stats_inc(&rxq->q_stats.rx.hsplit_buf_ovf);
3181 u64_stats_update_end(&rxq->stats_sync);
3182 goto bypass_hsplit;
3183 }
3184
3185 hdr_len = le16_get_bits(rx_desc->hdrlen_flags,
3186 VIRTCHNL2_RX_FLEX_DESC_ADV_LEN_HDR_M);
3187
3188bypass_hsplit:
3189 bufq_id = le16_get_bits(rx_desc->pktlen_gen_bufq_id,
3190 VIRTCHNL2_RX_FLEX_DESC_ADV_BUFQ_ID_M);
3191
3192 rxq_set = container_of(rxq, struct idpf_rxq_set, rxq);
3193 if (!bufq_id)
3194 refillq = rxq_set->refillq0;
3195 else
3196 refillq = rxq_set->refillq1;
3197
3198 /* retrieve buffer from the rxq */
3199 rx_bufq = &rxq->rxq_grp->splitq.bufq_sets[bufq_id].bufq;
3200
3201 buf_id = le16_to_cpu(rx_desc->buf_id);
3202
3203 rx_buf = &rx_bufq->rx_buf.buf[buf_id];
3204
3205 if (hdr_len) {
3206 const void *va = (u8 *)rx_bufq->rx_buf.hdr_buf_va +
3207 (u32)buf_id * IDPF_HDR_BUF_SIZE;
3208
3209 skb = idpf_rx_hdr_construct_skb(rxq, va, hdr_len);
3210 u64_stats_update_begin(&rxq->stats_sync);
3211 u64_stats_inc(&rxq->q_stats.rx.hsplit_pkts);
3212 u64_stats_update_end(&rxq->stats_sync);
3213 }
3214
3215 if (pkt_len) {
3216 idpf_rx_sync_for_cpu(rx_buf, pkt_len);
3217 if (skb)
3218 idpf_rx_add_frag(rx_buf, skb, pkt_len);
3219 else
3220 skb = idpf_rx_construct_skb(rxq, rx_buf,
3221 pkt_len);
3222 } else {
3223 idpf_rx_put_page(rx_buf);
3224 }
3225
3226 /* exit if we failed to retrieve a buffer */
3227 if (!skb)
3228 break;
3229
3230 idpf_rx_post_buf_refill(refillq, buf_id);
3231
3232 IDPF_RX_BUMP_NTC(rxq, ntc);
3233 /* skip if it is non EOP desc */
3234 if (!idpf_rx_splitq_is_eop(rx_desc))
3235 continue;
3236
3237 /* pad skb if needed (to make valid ethernet frame) */
3238 if (eth_skb_pad(skb)) {
3239 skb = NULL;
3240 continue;
3241 }
3242
3243 /* probably a little skewed due to removing CRC */
3244 total_rx_bytes += skb->len;
3245
3246 /* protocol */
3247 if (unlikely(idpf_rx_process_skb_fields(rxq, skb, rx_desc))) {
3248 dev_kfree_skb_any(skb);
3249 skb = NULL;
3250 continue;
3251 }
3252
3253 /* send completed skb up the stack */
3254 napi_gro_receive(&rxq->q_vector->napi, skb);
3255 skb = NULL;
3256
3257 /* update budget accounting */
3258 total_rx_pkts++;
3259 }
3260
3261 rxq->next_to_clean = ntc;
3262
3263 rxq->skb = skb;
3264 u64_stats_update_begin(&rxq->stats_sync);
3265 u64_stats_add(&rxq->q_stats.rx.packets, total_rx_pkts);
3266 u64_stats_add(&rxq->q_stats.rx.bytes, total_rx_bytes);
3267 u64_stats_update_end(&rxq->stats_sync);
3268
3269 /* guarantee a trip back through this routine if there was a failure */
3270 return total_rx_pkts;
3271}
3272
3273/**
3274 * idpf_rx_update_bufq_desc - Update buffer queue descriptor
3275 * @bufq: Pointer to the buffer queue
3276 * @refill_desc: SW Refill queue descriptor containing buffer ID
3277 * @buf_desc: Buffer queue descriptor
3278 *
3279 * Return 0 on success and negative on failure.
3280 */
3281static int idpf_rx_update_bufq_desc(struct idpf_queue *bufq, u16 refill_desc,
3282 struct virtchnl2_splitq_rx_buf_desc *buf_desc)
3283{
3284 struct idpf_rx_buf *buf;
3285 dma_addr_t addr;
3286 u16 buf_id;
3287
3288 buf_id = FIELD_GET(IDPF_RX_BI_BUFID_M, refill_desc);
3289
3290 buf = &bufq->rx_buf.buf[buf_id];
3291
3292 addr = idpf_alloc_page(bufq->pp, buf, bufq->rx_buf_size);
3293 if (unlikely(addr == DMA_MAPPING_ERROR))
3294 return -ENOMEM;
3295
3296 buf_desc->pkt_addr = cpu_to_le64(addr);
3297 buf_desc->qword0.buf_id = cpu_to_le16(buf_id);
3298
3299 if (!bufq->rx_hsplit_en)
3300 return 0;
3301
3302 buf_desc->hdr_addr = cpu_to_le64(bufq->rx_buf.hdr_buf_pa +
3303 (u32)buf_id * IDPF_HDR_BUF_SIZE);
3304
3305 return 0;
3306}
3307
3308/**
3309 * idpf_rx_clean_refillq - Clean refill queue buffers
3310 * @bufq: buffer queue to post buffers back to
3311 * @refillq: refill queue to clean
3312 *
3313 * This function takes care of the buffer refill management
3314 */
3315static void idpf_rx_clean_refillq(struct idpf_queue *bufq,
3316 struct idpf_sw_queue *refillq)
3317{
3318 struct virtchnl2_splitq_rx_buf_desc *buf_desc;
3319 u16 bufq_nta = bufq->next_to_alloc;
3320 u16 ntc = refillq->next_to_clean;
3321 int cleaned = 0;
3322 u16 gen;
3323
3324 buf_desc = IDPF_SPLITQ_RX_BUF_DESC(bufq, bufq_nta);
3325
3326 /* make sure we stop at ring wrap in the unlikely case ring is full */
3327 while (likely(cleaned < refillq->desc_count)) {
3328 u16 refill_desc = IDPF_SPLITQ_RX_BI_DESC(refillq, ntc);
3329 bool failure;
3330
3331 gen = FIELD_GET(IDPF_RX_BI_GEN_M, refill_desc);
3332 if (test_bit(__IDPF_RFLQ_GEN_CHK, refillq->flags) != gen)
3333 break;
3334
3335 failure = idpf_rx_update_bufq_desc(bufq, refill_desc,
3336 buf_desc);
3337 if (failure)
3338 break;
3339
3340 if (unlikely(++ntc == refillq->desc_count)) {
3341 change_bit(__IDPF_RFLQ_GEN_CHK, refillq->flags);
3342 ntc = 0;
3343 }
3344
3345 if (unlikely(++bufq_nta == bufq->desc_count)) {
3346 buf_desc = IDPF_SPLITQ_RX_BUF_DESC(bufq, 0);
3347 bufq_nta = 0;
3348 } else {
3349 buf_desc++;
3350 }
3351
3352 cleaned++;
3353 }
3354
3355 if (!cleaned)
3356 return;
3357
3358 /* We want to limit how many transactions on the bus we trigger with
3359 * tail writes so we only do it in strides. It's also important we
3360 * align the write to a multiple of 8 as required by HW.
3361 */
3362 if (((bufq->next_to_use <= bufq_nta ? 0 : bufq->desc_count) +
3363 bufq_nta - bufq->next_to_use) >= IDPF_RX_BUF_POST_STRIDE)
3364 idpf_rx_buf_hw_update(bufq, ALIGN_DOWN(bufq_nta,
3365 IDPF_RX_BUF_POST_STRIDE));
3366
3367 /* update next to alloc since we have filled the ring */
3368 refillq->next_to_clean = ntc;
3369 bufq->next_to_alloc = bufq_nta;
3370}
3371
3372/**
3373 * idpf_rx_clean_refillq_all - Clean all refill queues
3374 * @bufq: buffer queue with refill queues
3375 *
3376 * Iterates through all refill queues assigned to the buffer queue assigned to
3377 * this vector. Returns true if clean is complete within budget, false
3378 * otherwise.
3379 */
3380static void idpf_rx_clean_refillq_all(struct idpf_queue *bufq)
3381{
3382 struct idpf_bufq_set *bufq_set;
3383 int i;
3384
3385 bufq_set = container_of(bufq, struct idpf_bufq_set, bufq);
3386 for (i = 0; i < bufq_set->num_refillqs; i++)
3387 idpf_rx_clean_refillq(bufq, &bufq_set->refillqs[i]);
3388}
3389
3390/**
3391 * idpf_vport_intr_clean_queues - MSIX mode Interrupt Handler
3392 * @irq: interrupt number
3393 * @data: pointer to a q_vector
3394 *
3395 */
3396static irqreturn_t idpf_vport_intr_clean_queues(int __always_unused irq,
3397 void *data)
3398{
3399 struct idpf_q_vector *q_vector = (struct idpf_q_vector *)data;
3400
3401 q_vector->total_events++;
3402 napi_schedule(&q_vector->napi);
3403
3404 return IRQ_HANDLED;
3405}
3406
3407/**
3408 * idpf_vport_intr_napi_del_all - Unregister napi for all q_vectors in vport
3409 * @vport: virtual port structure
3410 *
3411 */
3412static void idpf_vport_intr_napi_del_all(struct idpf_vport *vport)
3413{
3414 u16 v_idx;
3415
3416 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++)
3417 netif_napi_del(&vport->q_vectors[v_idx].napi);
3418}
3419
3420/**
3421 * idpf_vport_intr_napi_dis_all - Disable NAPI for all q_vectors in the vport
3422 * @vport: main vport structure
3423 */
3424static void idpf_vport_intr_napi_dis_all(struct idpf_vport *vport)
3425{
3426 int v_idx;
3427
3428 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++)
3429 napi_disable(&vport->q_vectors[v_idx].napi);
3430}
3431
3432/**
3433 * idpf_vport_intr_rel - Free memory allocated for interrupt vectors
3434 * @vport: virtual port
3435 *
3436 * Free the memory allocated for interrupt vectors associated to a vport
3437 */
3438void idpf_vport_intr_rel(struct idpf_vport *vport)
3439{
3440 int i, j, v_idx;
3441
3442 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
3443 struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx];
3444
3445 kfree(q_vector->bufq);
3446 q_vector->bufq = NULL;
3447 kfree(q_vector->tx);
3448 q_vector->tx = NULL;
3449 kfree(q_vector->rx);
3450 q_vector->rx = NULL;
3451 }
3452
3453 /* Clean up the mapping of queues to vectors */
3454 for (i = 0; i < vport->num_rxq_grp; i++) {
3455 struct idpf_rxq_group *rx_qgrp = &vport->rxq_grps[i];
3456
3457 if (idpf_is_queue_model_split(vport->rxq_model))
3458 for (j = 0; j < rx_qgrp->splitq.num_rxq_sets; j++)
3459 rx_qgrp->splitq.rxq_sets[j]->rxq.q_vector = NULL;
3460 else
3461 for (j = 0; j < rx_qgrp->singleq.num_rxq; j++)
3462 rx_qgrp->singleq.rxqs[j]->q_vector = NULL;
3463 }
3464
3465 if (idpf_is_queue_model_split(vport->txq_model))
3466 for (i = 0; i < vport->num_txq_grp; i++)
3467 vport->txq_grps[i].complq->q_vector = NULL;
3468 else
3469 for (i = 0; i < vport->num_txq_grp; i++)
3470 for (j = 0; j < vport->txq_grps[i].num_txq; j++)
3471 vport->txq_grps[i].txqs[j]->q_vector = NULL;
3472
3473 kfree(vport->q_vectors);
3474 vport->q_vectors = NULL;
3475}
3476
3477/**
3478 * idpf_vport_intr_rel_irq - Free the IRQ association with the OS
3479 * @vport: main vport structure
3480 */
3481static void idpf_vport_intr_rel_irq(struct idpf_vport *vport)
3482{
3483 struct idpf_adapter *adapter = vport->adapter;
3484 int vector;
3485
3486 for (vector = 0; vector < vport->num_q_vectors; vector++) {
3487 struct idpf_q_vector *q_vector = &vport->q_vectors[vector];
3488 int irq_num, vidx;
3489
3490 /* free only the irqs that were actually requested */
3491 if (!q_vector)
3492 continue;
3493
3494 vidx = vport->q_vector_idxs[vector];
3495 irq_num = adapter->msix_entries[vidx].vector;
3496
3497 /* clear the affinity_mask in the IRQ descriptor */
3498 irq_set_affinity_hint(irq_num, NULL);
3499 free_irq(irq_num, q_vector);
3500 }
3501}
3502
3503/**
3504 * idpf_vport_intr_dis_irq_all - Disable all interrupt
3505 * @vport: main vport structure
3506 */
3507static void idpf_vport_intr_dis_irq_all(struct idpf_vport *vport)
3508{
3509 struct idpf_q_vector *q_vector = vport->q_vectors;
3510 int q_idx;
3511
3512 for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++)
3513 writel(0, q_vector[q_idx].intr_reg.dyn_ctl);
3514}
3515
3516/**
3517 * idpf_vport_intr_buildreg_itr - Enable default interrupt generation settings
3518 * @q_vector: pointer to q_vector
3519 * @type: itr index
3520 * @itr: itr value
3521 */
3522static u32 idpf_vport_intr_buildreg_itr(struct idpf_q_vector *q_vector,
3523 const int type, u16 itr)
3524{
3525 u32 itr_val;
3526
3527 itr &= IDPF_ITR_MASK;
3528 /* Don't clear PBA because that can cause lost interrupts that
3529 * came in while we were cleaning/polling
3530 */
3531 itr_val = q_vector->intr_reg.dyn_ctl_intena_m |
3532 (type << q_vector->intr_reg.dyn_ctl_itridx_s) |
3533 (itr << (q_vector->intr_reg.dyn_ctl_intrvl_s - 1));
3534
3535 return itr_val;
3536}
3537
3538/**
3539 * idpf_update_dim_sample - Update dim sample with packets and bytes
3540 * @q_vector: the vector associated with the interrupt
3541 * @dim_sample: dim sample to update
3542 * @dim: dim instance structure
3543 * @packets: total packets
3544 * @bytes: total bytes
3545 *
3546 * Update the dim sample with the packets and bytes which are passed to this
3547 * function. Set the dim state appropriately if the dim settings gets stale.
3548 */
3549static void idpf_update_dim_sample(struct idpf_q_vector *q_vector,
3550 struct dim_sample *dim_sample,
3551 struct dim *dim, u64 packets, u64 bytes)
3552{
3553 dim_update_sample(q_vector->total_events, packets, bytes, dim_sample);
3554 dim_sample->comp_ctr = 0;
3555
3556 /* if dim settings get stale, like when not updated for 1 second or
3557 * longer, force it to start again. This addresses the frequent case
3558 * of an idle queue being switched to by the scheduler.
3559 */
3560 if (ktime_ms_delta(dim_sample->time, dim->start_sample.time) >= HZ)
3561 dim->state = DIM_START_MEASURE;
3562}
3563
3564/**
3565 * idpf_net_dim - Update net DIM algorithm
3566 * @q_vector: the vector associated with the interrupt
3567 *
3568 * Create a DIM sample and notify net_dim() so that it can possibly decide
3569 * a new ITR value based on incoming packets, bytes, and interrupts.
3570 *
3571 * This function is a no-op if the queue is not configured to dynamic ITR.
3572 */
3573static void idpf_net_dim(struct idpf_q_vector *q_vector)
3574{
3575 struct dim_sample dim_sample = { };
3576 u64 packets, bytes;
3577 u32 i;
3578
3579 if (!IDPF_ITR_IS_DYNAMIC(q_vector->tx_intr_mode))
3580 goto check_rx_itr;
3581
3582 for (i = 0, packets = 0, bytes = 0; i < q_vector->num_txq; i++) {
3583 struct idpf_queue *txq = q_vector->tx[i];
3584 unsigned int start;
3585
3586 do {
3587 start = u64_stats_fetch_begin(&txq->stats_sync);
3588 packets += u64_stats_read(&txq->q_stats.tx.packets);
3589 bytes += u64_stats_read(&txq->q_stats.tx.bytes);
3590 } while (u64_stats_fetch_retry(&txq->stats_sync, start));
3591 }
3592
3593 idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->tx_dim,
3594 packets, bytes);
3595 net_dim(&q_vector->tx_dim, dim_sample);
3596
3597check_rx_itr:
3598 if (!IDPF_ITR_IS_DYNAMIC(q_vector->rx_intr_mode))
3599 return;
3600
3601 for (i = 0, packets = 0, bytes = 0; i < q_vector->num_rxq; i++) {
3602 struct idpf_queue *rxq = q_vector->rx[i];
3603 unsigned int start;
3604
3605 do {
3606 start = u64_stats_fetch_begin(&rxq->stats_sync);
3607 packets += u64_stats_read(&rxq->q_stats.rx.packets);
3608 bytes += u64_stats_read(&rxq->q_stats.rx.bytes);
3609 } while (u64_stats_fetch_retry(&rxq->stats_sync, start));
3610 }
3611
3612 idpf_update_dim_sample(q_vector, &dim_sample, &q_vector->rx_dim,
3613 packets, bytes);
3614 net_dim(&q_vector->rx_dim, dim_sample);
3615}
3616
3617/**
3618 * idpf_vport_intr_update_itr_ena_irq - Update itr and re-enable MSIX interrupt
3619 * @q_vector: q_vector for which itr is being updated and interrupt enabled
3620 *
3621 * Update the net_dim() algorithm and re-enable the interrupt associated with
3622 * this vector.
3623 */
3624void idpf_vport_intr_update_itr_ena_irq(struct idpf_q_vector *q_vector)
3625{
3626 u32 intval;
3627
3628 /* net_dim() updates ITR out-of-band using a work item */
3629 idpf_net_dim(q_vector);
3630
3631 intval = idpf_vport_intr_buildreg_itr(q_vector,
3632 IDPF_NO_ITR_UPDATE_IDX, 0);
3633
3634 writel(intval, q_vector->intr_reg.dyn_ctl);
3635}
3636
3637/**
3638 * idpf_vport_intr_req_irq - get MSI-X vectors from the OS for the vport
3639 * @vport: main vport structure
3640 * @basename: name for the vector
3641 */
3642static int idpf_vport_intr_req_irq(struct idpf_vport *vport, char *basename)
3643{
3644 struct idpf_adapter *adapter = vport->adapter;
3645 int vector, err, irq_num, vidx;
3646 const char *vec_name;
3647
3648 for (vector = 0; vector < vport->num_q_vectors; vector++) {
3649 struct idpf_q_vector *q_vector = &vport->q_vectors[vector];
3650
3651 vidx = vport->q_vector_idxs[vector];
3652 irq_num = adapter->msix_entries[vidx].vector;
3653
3654 if (q_vector->num_rxq && q_vector->num_txq)
3655 vec_name = "TxRx";
3656 else if (q_vector->num_rxq)
3657 vec_name = "Rx";
3658 else if (q_vector->num_txq)
3659 vec_name = "Tx";
3660 else
3661 continue;
3662
3663 q_vector->name = kasprintf(GFP_KERNEL, "%s-%s-%d",
3664 basename, vec_name, vidx);
3665
3666 err = request_irq(irq_num, idpf_vport_intr_clean_queues, 0,
3667 q_vector->name, q_vector);
3668 if (err) {
3669 netdev_err(vport->netdev,
3670 "Request_irq failed, error: %d\n", err);
3671 goto free_q_irqs;
3672 }
3673 /* assign the mask for this irq */
3674 irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
3675 }
3676
3677 return 0;
3678
3679free_q_irqs:
3680 while (--vector >= 0) {
3681 vidx = vport->q_vector_idxs[vector];
3682 irq_num = adapter->msix_entries[vidx].vector;
3683 free_irq(irq_num, &vport->q_vectors[vector]);
3684 }
3685
3686 return err;
3687}
3688
3689/**
3690 * idpf_vport_intr_write_itr - Write ITR value to the ITR register
3691 * @q_vector: q_vector structure
3692 * @itr: Interrupt throttling rate
3693 * @tx: Tx or Rx ITR
3694 */
3695void idpf_vport_intr_write_itr(struct idpf_q_vector *q_vector, u16 itr, bool tx)
3696{
3697 struct idpf_intr_reg *intr_reg;
3698
3699 if (tx && !q_vector->tx)
3700 return;
3701 else if (!tx && !q_vector->rx)
3702 return;
3703
3704 intr_reg = &q_vector->intr_reg;
3705 writel(ITR_REG_ALIGN(itr) >> IDPF_ITR_GRAN_S,
3706 tx ? intr_reg->tx_itr : intr_reg->rx_itr);
3707}
3708
3709/**
3710 * idpf_vport_intr_ena_irq_all - Enable IRQ for the given vport
3711 * @vport: main vport structure
3712 */
3713static void idpf_vport_intr_ena_irq_all(struct idpf_vport *vport)
3714{
3715 bool dynamic;
3716 int q_idx;
3717 u16 itr;
3718
3719 for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) {
3720 struct idpf_q_vector *qv = &vport->q_vectors[q_idx];
3721
3722 /* Set the initial ITR values */
3723 if (qv->num_txq) {
3724 dynamic = IDPF_ITR_IS_DYNAMIC(qv->tx_intr_mode);
3725 itr = vport->tx_itr_profile[qv->tx_dim.profile_ix];
3726 idpf_vport_intr_write_itr(qv, dynamic ?
3727 itr : qv->tx_itr_value,
3728 true);
3729 }
3730
3731 if (qv->num_rxq) {
3732 dynamic = IDPF_ITR_IS_DYNAMIC(qv->rx_intr_mode);
3733 itr = vport->rx_itr_profile[qv->rx_dim.profile_ix];
3734 idpf_vport_intr_write_itr(qv, dynamic ?
3735 itr : qv->rx_itr_value,
3736 false);
3737 }
3738
3739 if (qv->num_txq || qv->num_rxq)
3740 idpf_vport_intr_update_itr_ena_irq(qv);
3741 }
3742}
3743
3744/**
3745 * idpf_vport_intr_deinit - Release all vector associations for the vport
3746 * @vport: main vport structure
3747 */
3748void idpf_vport_intr_deinit(struct idpf_vport *vport)
3749{
3750 idpf_vport_intr_dis_irq_all(vport);
3751 idpf_vport_intr_napi_dis_all(vport);
3752 idpf_vport_intr_napi_del_all(vport);
3753 idpf_vport_intr_rel_irq(vport);
3754}
3755
3756/**
3757 * idpf_tx_dim_work - Call back from the stack
3758 * @work: work queue structure
3759 */
3760static void idpf_tx_dim_work(struct work_struct *work)
3761{
3762 struct idpf_q_vector *q_vector;
3763 struct idpf_vport *vport;
3764 struct dim *dim;
3765 u16 itr;
3766
3767 dim = container_of(work, struct dim, work);
3768 q_vector = container_of(dim, struct idpf_q_vector, tx_dim);
3769 vport = q_vector->vport;
3770
3771 if (dim->profile_ix >= ARRAY_SIZE(vport->tx_itr_profile))
3772 dim->profile_ix = ARRAY_SIZE(vport->tx_itr_profile) - 1;
3773
3774 /* look up the values in our local table */
3775 itr = vport->tx_itr_profile[dim->profile_ix];
3776
3777 idpf_vport_intr_write_itr(q_vector, itr, true);
3778
3779 dim->state = DIM_START_MEASURE;
3780}
3781
3782/**
3783 * idpf_rx_dim_work - Call back from the stack
3784 * @work: work queue structure
3785 */
3786static void idpf_rx_dim_work(struct work_struct *work)
3787{
3788 struct idpf_q_vector *q_vector;
3789 struct idpf_vport *vport;
3790 struct dim *dim;
3791 u16 itr;
3792
3793 dim = container_of(work, struct dim, work);
3794 q_vector = container_of(dim, struct idpf_q_vector, rx_dim);
3795 vport = q_vector->vport;
3796
3797 if (dim->profile_ix >= ARRAY_SIZE(vport->rx_itr_profile))
3798 dim->profile_ix = ARRAY_SIZE(vport->rx_itr_profile) - 1;
3799
3800 /* look up the values in our local table */
3801 itr = vport->rx_itr_profile[dim->profile_ix];
3802
3803 idpf_vport_intr_write_itr(q_vector, itr, false);
3804
3805 dim->state = DIM_START_MEASURE;
3806}
3807
3808/**
3809 * idpf_init_dim - Set up dynamic interrupt moderation
3810 * @qv: q_vector structure
3811 */
3812static void idpf_init_dim(struct idpf_q_vector *qv)
3813{
3814 INIT_WORK(&qv->tx_dim.work, idpf_tx_dim_work);
3815 qv->tx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
3816 qv->tx_dim.profile_ix = IDPF_DIM_DEFAULT_PROFILE_IX;
3817
3818 INIT_WORK(&qv->rx_dim.work, idpf_rx_dim_work);
3819 qv->rx_dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
3820 qv->rx_dim.profile_ix = IDPF_DIM_DEFAULT_PROFILE_IX;
3821}
3822
3823/**
3824 * idpf_vport_intr_napi_ena_all - Enable NAPI for all q_vectors in the vport
3825 * @vport: main vport structure
3826 */
3827static void idpf_vport_intr_napi_ena_all(struct idpf_vport *vport)
3828{
3829 int q_idx;
3830
3831 for (q_idx = 0; q_idx < vport->num_q_vectors; q_idx++) {
3832 struct idpf_q_vector *q_vector = &vport->q_vectors[q_idx];
3833
3834 idpf_init_dim(q_vector);
3835 napi_enable(&q_vector->napi);
3836 }
3837}
3838
3839/**
3840 * idpf_tx_splitq_clean_all- Clean completion queues
3841 * @q_vec: queue vector
3842 * @budget: Used to determine if we are in netpoll
3843 * @cleaned: returns number of packets cleaned
3844 *
3845 * Returns false if clean is not complete else returns true
3846 */
3847static bool idpf_tx_splitq_clean_all(struct idpf_q_vector *q_vec,
3848 int budget, int *cleaned)
3849{
3850 u16 num_txq = q_vec->num_txq;
3851 bool clean_complete = true;
3852 int i, budget_per_q;
3853
3854 if (unlikely(!num_txq))
3855 return true;
3856
3857 budget_per_q = DIV_ROUND_UP(budget, num_txq);
3858 for (i = 0; i < num_txq; i++)
3859 clean_complete &= idpf_tx_clean_complq(q_vec->tx[i],
3860 budget_per_q, cleaned);
3861
3862 return clean_complete;
3863}
3864
3865/**
3866 * idpf_rx_splitq_clean_all- Clean completion queues
3867 * @q_vec: queue vector
3868 * @budget: Used to determine if we are in netpoll
3869 * @cleaned: returns number of packets cleaned
3870 *
3871 * Returns false if clean is not complete else returns true
3872 */
3873static bool idpf_rx_splitq_clean_all(struct idpf_q_vector *q_vec, int budget,
3874 int *cleaned)
3875{
3876 u16 num_rxq = q_vec->num_rxq;
3877 bool clean_complete = true;
3878 int pkts_cleaned = 0;
3879 int i, budget_per_q;
3880
3881 /* We attempt to distribute budget to each Rx queue fairly, but don't
3882 * allow the budget to go below 1 because that would exit polling early.
3883 */
3884 budget_per_q = num_rxq ? max(budget / num_rxq, 1) : 0;
3885 for (i = 0; i < num_rxq; i++) {
3886 struct idpf_queue *rxq = q_vec->rx[i];
3887 int pkts_cleaned_per_q;
3888
3889 pkts_cleaned_per_q = idpf_rx_splitq_clean(rxq, budget_per_q);
3890 /* if we clean as many as budgeted, we must not be done */
3891 if (pkts_cleaned_per_q >= budget_per_q)
3892 clean_complete = false;
3893 pkts_cleaned += pkts_cleaned_per_q;
3894 }
3895 *cleaned = pkts_cleaned;
3896
3897 for (i = 0; i < q_vec->num_bufq; i++)
3898 idpf_rx_clean_refillq_all(q_vec->bufq[i]);
3899
3900 return clean_complete;
3901}
3902
3903/**
3904 * idpf_vport_splitq_napi_poll - NAPI handler
3905 * @napi: struct from which you get q_vector
3906 * @budget: budget provided by stack
3907 */
3908static int idpf_vport_splitq_napi_poll(struct napi_struct *napi, int budget)
3909{
3910 struct idpf_q_vector *q_vector =
3911 container_of(napi, struct idpf_q_vector, napi);
3912 bool clean_complete;
3913 int work_done = 0;
3914
3915 /* Handle case where we are called by netpoll with a budget of 0 */
3916 if (unlikely(!budget)) {
3917 idpf_tx_splitq_clean_all(q_vector, budget, &work_done);
3918
3919 return 0;
3920 }
3921
3922 clean_complete = idpf_rx_splitq_clean_all(q_vector, budget, &work_done);
3923 clean_complete &= idpf_tx_splitq_clean_all(q_vector, budget, &work_done);
3924
3925 /* If work not completed, return budget and polling will return */
3926 if (!clean_complete)
3927 return budget;
3928
3929 work_done = min_t(int, work_done, budget - 1);
3930
3931 /* Exit the polling mode, but don't re-enable interrupts if stack might
3932 * poll us due to busy-polling
3933 */
3934 if (likely(napi_complete_done(napi, work_done)))
3935 idpf_vport_intr_update_itr_ena_irq(q_vector);
3936
3937 /* Switch to poll mode in the tear-down path after sending disable
3938 * queues virtchnl message, as the interrupts will be disabled after
3939 * that
3940 */
3941 if (unlikely(q_vector->num_txq && test_bit(__IDPF_Q_POLL_MODE,
3942 q_vector->tx[0]->flags)))
3943 return budget;
3944 else
3945 return work_done;
3946}
3947
3948/**
3949 * idpf_vport_intr_map_vector_to_qs - Map vectors to queues
3950 * @vport: virtual port
3951 *
3952 * Mapping for vectors to queues
3953 */
3954static void idpf_vport_intr_map_vector_to_qs(struct idpf_vport *vport)
3955{
3956 u16 num_txq_grp = vport->num_txq_grp;
3957 int i, j, qv_idx, bufq_vidx = 0;
3958 struct idpf_rxq_group *rx_qgrp;
3959 struct idpf_txq_group *tx_qgrp;
3960 struct idpf_queue *q, *bufq;
3961 u16 q_index;
3962
3963 for (i = 0, qv_idx = 0; i < vport->num_rxq_grp; i++) {
3964 u16 num_rxq;
3965
3966 rx_qgrp = &vport->rxq_grps[i];
3967 if (idpf_is_queue_model_split(vport->rxq_model))
3968 num_rxq = rx_qgrp->splitq.num_rxq_sets;
3969 else
3970 num_rxq = rx_qgrp->singleq.num_rxq;
3971
3972 for (j = 0; j < num_rxq; j++) {
3973 if (qv_idx >= vport->num_q_vectors)
3974 qv_idx = 0;
3975
3976 if (idpf_is_queue_model_split(vport->rxq_model))
3977 q = &rx_qgrp->splitq.rxq_sets[j]->rxq;
3978 else
3979 q = rx_qgrp->singleq.rxqs[j];
3980 q->q_vector = &vport->q_vectors[qv_idx];
3981 q_index = q->q_vector->num_rxq;
3982 q->q_vector->rx[q_index] = q;
3983 q->q_vector->num_rxq++;
3984 qv_idx++;
3985 }
3986
3987 if (idpf_is_queue_model_split(vport->rxq_model)) {
3988 for (j = 0; j < vport->num_bufqs_per_qgrp; j++) {
3989 bufq = &rx_qgrp->splitq.bufq_sets[j].bufq;
3990 bufq->q_vector = &vport->q_vectors[bufq_vidx];
3991 q_index = bufq->q_vector->num_bufq;
3992 bufq->q_vector->bufq[q_index] = bufq;
3993 bufq->q_vector->num_bufq++;
3994 }
3995 if (++bufq_vidx >= vport->num_q_vectors)
3996 bufq_vidx = 0;
3997 }
3998 }
3999
4000 for (i = 0, qv_idx = 0; i < num_txq_grp; i++) {
4001 u16 num_txq;
4002
4003 tx_qgrp = &vport->txq_grps[i];
4004 num_txq = tx_qgrp->num_txq;
4005
4006 if (idpf_is_queue_model_split(vport->txq_model)) {
4007 if (qv_idx >= vport->num_q_vectors)
4008 qv_idx = 0;
4009
4010 q = tx_qgrp->complq;
4011 q->q_vector = &vport->q_vectors[qv_idx];
4012 q_index = q->q_vector->num_txq;
4013 q->q_vector->tx[q_index] = q;
4014 q->q_vector->num_txq++;
4015 qv_idx++;
4016 } else {
4017 for (j = 0; j < num_txq; j++) {
4018 if (qv_idx >= vport->num_q_vectors)
4019 qv_idx = 0;
4020
4021 q = tx_qgrp->txqs[j];
4022 q->q_vector = &vport->q_vectors[qv_idx];
4023 q_index = q->q_vector->num_txq;
4024 q->q_vector->tx[q_index] = q;
4025 q->q_vector->num_txq++;
4026
4027 qv_idx++;
4028 }
4029 }
4030 }
4031}
4032
4033/**
4034 * idpf_vport_intr_init_vec_idx - Initialize the vector indexes
4035 * @vport: virtual port
4036 *
4037 * Initialize vector indexes with values returened over mailbox
4038 */
4039static int idpf_vport_intr_init_vec_idx(struct idpf_vport *vport)
4040{
4041 struct idpf_adapter *adapter = vport->adapter;
4042 struct virtchnl2_alloc_vectors *ac;
4043 u16 *vecids, total_vecs;
4044 int i;
4045
4046 ac = adapter->req_vec_chunks;
4047 if (!ac) {
4048 for (i = 0; i < vport->num_q_vectors; i++)
4049 vport->q_vectors[i].v_idx = vport->q_vector_idxs[i];
4050
4051 return 0;
4052 }
4053
4054 total_vecs = idpf_get_reserved_vecs(adapter);
4055 vecids = kcalloc(total_vecs, sizeof(u16), GFP_KERNEL);
4056 if (!vecids)
4057 return -ENOMEM;
4058
4059 idpf_get_vec_ids(adapter, vecids, total_vecs, &ac->vchunks);
4060
4061 for (i = 0; i < vport->num_q_vectors; i++)
4062 vport->q_vectors[i].v_idx = vecids[vport->q_vector_idxs[i]];
4063
4064 kfree(vecids);
4065
4066 return 0;
4067}
4068
4069/**
4070 * idpf_vport_intr_napi_add_all- Register napi handler for all qvectors
4071 * @vport: virtual port structure
4072 */
4073static void idpf_vport_intr_napi_add_all(struct idpf_vport *vport)
4074{
4075 int (*napi_poll)(struct napi_struct *napi, int budget);
4076 u16 v_idx;
4077
4078 if (idpf_is_queue_model_split(vport->txq_model))
4079 napi_poll = idpf_vport_splitq_napi_poll;
4080 else
4081 napi_poll = idpf_vport_singleq_napi_poll;
4082
4083 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
4084 struct idpf_q_vector *q_vector = &vport->q_vectors[v_idx];
4085
4086 netif_napi_add(vport->netdev, &q_vector->napi, napi_poll);
4087
4088 /* only set affinity_mask if the CPU is online */
4089 if (cpu_online(v_idx))
4090 cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
4091 }
4092}
4093
4094/**
4095 * idpf_vport_intr_alloc - Allocate memory for interrupt vectors
4096 * @vport: virtual port
4097 *
4098 * We allocate one q_vector per queue interrupt. If allocation fails we
4099 * return -ENOMEM.
4100 */
4101int idpf_vport_intr_alloc(struct idpf_vport *vport)
4102{
4103 u16 txqs_per_vector, rxqs_per_vector, bufqs_per_vector;
4104 struct idpf_q_vector *q_vector;
4105 int v_idx, err;
4106
4107 vport->q_vectors = kcalloc(vport->num_q_vectors,
4108 sizeof(struct idpf_q_vector), GFP_KERNEL);
4109 if (!vport->q_vectors)
4110 return -ENOMEM;
4111
4112 txqs_per_vector = DIV_ROUND_UP(vport->num_txq, vport->num_q_vectors);
4113 rxqs_per_vector = DIV_ROUND_UP(vport->num_rxq, vport->num_q_vectors);
4114 bufqs_per_vector = vport->num_bufqs_per_qgrp *
4115 DIV_ROUND_UP(vport->num_rxq_grp,
4116 vport->num_q_vectors);
4117
4118 for (v_idx = 0; v_idx < vport->num_q_vectors; v_idx++) {
4119 q_vector = &vport->q_vectors[v_idx];
4120 q_vector->vport = vport;
4121
4122 q_vector->tx_itr_value = IDPF_ITR_TX_DEF;
4123 q_vector->tx_intr_mode = IDPF_ITR_DYNAMIC;
4124 q_vector->tx_itr_idx = VIRTCHNL2_ITR_IDX_1;
4125
4126 q_vector->rx_itr_value = IDPF_ITR_RX_DEF;
4127 q_vector->rx_intr_mode = IDPF_ITR_DYNAMIC;
4128 q_vector->rx_itr_idx = VIRTCHNL2_ITR_IDX_0;
4129
4130 q_vector->tx = kcalloc(txqs_per_vector,
4131 sizeof(struct idpf_queue *),
4132 GFP_KERNEL);
4133 if (!q_vector->tx) {
4134 err = -ENOMEM;
4135 goto error;
4136 }
4137
4138 q_vector->rx = kcalloc(rxqs_per_vector,
4139 sizeof(struct idpf_queue *),
4140 GFP_KERNEL);
4141 if (!q_vector->rx) {
4142 err = -ENOMEM;
4143 goto error;
4144 }
4145
4146 if (!idpf_is_queue_model_split(vport->rxq_model))
4147 continue;
4148
4149 q_vector->bufq = kcalloc(bufqs_per_vector,
4150 sizeof(struct idpf_queue *),
4151 GFP_KERNEL);
4152 if (!q_vector->bufq) {
4153 err = -ENOMEM;
4154 goto error;
4155 }
4156 }
4157
4158 return 0;
4159
4160error:
4161 idpf_vport_intr_rel(vport);
4162
4163 return err;
4164}
4165
4166/**
4167 * idpf_vport_intr_init - Setup all vectors for the given vport
4168 * @vport: virtual port
4169 *
4170 * Returns 0 on success or negative on failure
4171 */
4172int idpf_vport_intr_init(struct idpf_vport *vport)
4173{
4174 char *int_name;
4175 int err;
4176
4177 err = idpf_vport_intr_init_vec_idx(vport);
4178 if (err)
4179 return err;
4180
4181 idpf_vport_intr_map_vector_to_qs(vport);
4182 idpf_vport_intr_napi_add_all(vport);
4183
4184 err = vport->adapter->dev_ops.reg_ops.intr_reg_init(vport);
4185 if (err)
4186 goto unroll_vectors_alloc;
4187
4188 int_name = kasprintf(GFP_KERNEL, "%s-%s",
4189 dev_driver_string(&vport->adapter->pdev->dev),
4190 vport->netdev->name);
4191
4192 err = idpf_vport_intr_req_irq(vport, int_name);
4193 if (err)
4194 goto unroll_vectors_alloc;
4195
4196 return 0;
4197
4198unroll_vectors_alloc:
4199 idpf_vport_intr_napi_del_all(vport);
4200
4201 return err;
4202}
4203
4204void idpf_vport_intr_ena(struct idpf_vport *vport)
4205{
4206 idpf_vport_intr_napi_ena_all(vport);
4207 idpf_vport_intr_ena_irq_all(vport);
4208}
4209
4210/**
4211 * idpf_config_rss - Send virtchnl messages to configure RSS
4212 * @vport: virtual port
4213 *
4214 * Return 0 on success, negative on failure
4215 */
4216int idpf_config_rss(struct idpf_vport *vport)
4217{
4218 int err;
4219
4220 err = idpf_send_get_set_rss_key_msg(vport, false);
4221 if (err)
4222 return err;
4223
4224 return idpf_send_get_set_rss_lut_msg(vport, false);
4225}
4226
4227/**
4228 * idpf_fill_dflt_rss_lut - Fill the indirection table with the default values
4229 * @vport: virtual port structure
4230 */
4231static void idpf_fill_dflt_rss_lut(struct idpf_vport *vport)
4232{
4233 struct idpf_adapter *adapter = vport->adapter;
4234 u16 num_active_rxq = vport->num_rxq;
4235 struct idpf_rss_data *rss_data;
4236 int i;
4237
4238 rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4239
4240 for (i = 0; i < rss_data->rss_lut_size; i++) {
4241 rss_data->rss_lut[i] = i % num_active_rxq;
4242 rss_data->cached_lut[i] = rss_data->rss_lut[i];
4243 }
4244}
4245
4246/**
4247 * idpf_init_rss - Allocate and initialize RSS resources
4248 * @vport: virtual port
4249 *
4250 * Return 0 on success, negative on failure
4251 */
4252int idpf_init_rss(struct idpf_vport *vport)
4253{
4254 struct idpf_adapter *adapter = vport->adapter;
4255 struct idpf_rss_data *rss_data;
4256 u32 lut_size;
4257
4258 rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4259
4260 lut_size = rss_data->rss_lut_size * sizeof(u32);
4261 rss_data->rss_lut = kzalloc(lut_size, GFP_KERNEL);
4262 if (!rss_data->rss_lut)
4263 return -ENOMEM;
4264
4265 rss_data->cached_lut = kzalloc(lut_size, GFP_KERNEL);
4266 if (!rss_data->cached_lut) {
4267 kfree(rss_data->rss_lut);
4268 rss_data->rss_lut = NULL;
4269
4270 return -ENOMEM;
4271 }
4272
4273 /* Fill the default RSS lut values */
4274 idpf_fill_dflt_rss_lut(vport);
4275
4276 return idpf_config_rss(vport);
4277}
4278
4279/**
4280 * idpf_deinit_rss - Release RSS resources
4281 * @vport: virtual port
4282 */
4283void idpf_deinit_rss(struct idpf_vport *vport)
4284{
4285 struct idpf_adapter *adapter = vport->adapter;
4286 struct idpf_rss_data *rss_data;
4287
4288 rss_data = &adapter->vport_config[vport->idx]->user_config.rss_data;
4289 kfree(rss_data->cached_lut);
4290 rss_data->cached_lut = NULL;
4291 kfree(rss_data->rss_lut);
4292 rss_data->rss_lut = NULL;
4293}