Loading...
Note: File does not exist in v6.13.7.
1// SPDX-License-Identifier: GPL-2.0
2/*******************************************************************************
3 *
4 * Intel Ethernet Controller XL710 Family Linux Virtual Function Driver
5 * Copyright(c) 2013 - 2016 Intel Corporation.
6 *
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms and conditions of the GNU General Public License,
9 * version 2, as published by the Free Software Foundation.
10 *
11 * This program is distributed in the hope it will be useful, but WITHOUT
12 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * more details.
15 *
16 * You should have received a copy of the GNU General Public License along
17 * with this program. If not, see <http://www.gnu.org/licenses/>.
18 *
19 * The full GNU General Public License is included in this distribution in
20 * the file called "COPYING".
21 *
22 * Contact Information:
23 * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 * Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25 *
26 ******************************************************************************/
27
28#include <linux/prefetch.h>
29#include <net/busy_poll.h>
30
31#include "i40evf.h"
32#include "i40e_trace.h"
33#include "i40e_prototype.h"
34
35static inline __le64 build_ctob(u32 td_cmd, u32 td_offset, unsigned int size,
36 u32 td_tag)
37{
38 return cpu_to_le64(I40E_TX_DESC_DTYPE_DATA |
39 ((u64)td_cmd << I40E_TXD_QW1_CMD_SHIFT) |
40 ((u64)td_offset << I40E_TXD_QW1_OFFSET_SHIFT) |
41 ((u64)size << I40E_TXD_QW1_TX_BUF_SZ_SHIFT) |
42 ((u64)td_tag << I40E_TXD_QW1_L2TAG1_SHIFT));
43}
44
45#define I40E_TXD_CMD (I40E_TX_DESC_CMD_EOP | I40E_TX_DESC_CMD_RS)
46
47/**
48 * i40e_unmap_and_free_tx_resource - Release a Tx buffer
49 * @ring: the ring that owns the buffer
50 * @tx_buffer: the buffer to free
51 **/
52static void i40e_unmap_and_free_tx_resource(struct i40e_ring *ring,
53 struct i40e_tx_buffer *tx_buffer)
54{
55 if (tx_buffer->skb) {
56 if (tx_buffer->tx_flags & I40E_TX_FLAGS_FD_SB)
57 kfree(tx_buffer->raw_buf);
58 else
59 dev_kfree_skb_any(tx_buffer->skb);
60 if (dma_unmap_len(tx_buffer, len))
61 dma_unmap_single(ring->dev,
62 dma_unmap_addr(tx_buffer, dma),
63 dma_unmap_len(tx_buffer, len),
64 DMA_TO_DEVICE);
65 } else if (dma_unmap_len(tx_buffer, len)) {
66 dma_unmap_page(ring->dev,
67 dma_unmap_addr(tx_buffer, dma),
68 dma_unmap_len(tx_buffer, len),
69 DMA_TO_DEVICE);
70 }
71
72 tx_buffer->next_to_watch = NULL;
73 tx_buffer->skb = NULL;
74 dma_unmap_len_set(tx_buffer, len, 0);
75 /* tx_buffer must be completely set up in the transmit path */
76}
77
78/**
79 * i40evf_clean_tx_ring - Free any empty Tx buffers
80 * @tx_ring: ring to be cleaned
81 **/
82void i40evf_clean_tx_ring(struct i40e_ring *tx_ring)
83{
84 unsigned long bi_size;
85 u16 i;
86
87 /* ring already cleared, nothing to do */
88 if (!tx_ring->tx_bi)
89 return;
90
91 /* Free all the Tx ring sk_buffs */
92 for (i = 0; i < tx_ring->count; i++)
93 i40e_unmap_and_free_tx_resource(tx_ring, &tx_ring->tx_bi[i]);
94
95 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
96 memset(tx_ring->tx_bi, 0, bi_size);
97
98 /* Zero out the descriptor ring */
99 memset(tx_ring->desc, 0, tx_ring->size);
100
101 tx_ring->next_to_use = 0;
102 tx_ring->next_to_clean = 0;
103
104 if (!tx_ring->netdev)
105 return;
106
107 /* cleanup Tx queue statistics */
108 netdev_tx_reset_queue(txring_txq(tx_ring));
109}
110
111/**
112 * i40evf_free_tx_resources - Free Tx resources per queue
113 * @tx_ring: Tx descriptor ring for a specific queue
114 *
115 * Free all transmit software resources
116 **/
117void i40evf_free_tx_resources(struct i40e_ring *tx_ring)
118{
119 i40evf_clean_tx_ring(tx_ring);
120 kfree(tx_ring->tx_bi);
121 tx_ring->tx_bi = NULL;
122
123 if (tx_ring->desc) {
124 dma_free_coherent(tx_ring->dev, tx_ring->size,
125 tx_ring->desc, tx_ring->dma);
126 tx_ring->desc = NULL;
127 }
128}
129
130/**
131 * i40evf_get_tx_pending - how many Tx descriptors not processed
132 * @tx_ring: the ring of descriptors
133 * @in_sw: is tx_pending being checked in SW or HW
134 *
135 * Since there is no access to the ring head register
136 * in XL710, we need to use our local copies
137 **/
138u32 i40evf_get_tx_pending(struct i40e_ring *ring, bool in_sw)
139{
140 u32 head, tail;
141
142 head = ring->next_to_clean;
143 tail = readl(ring->tail);
144
145 if (head != tail)
146 return (head < tail) ?
147 tail - head : (tail + ring->count - head);
148
149 return 0;
150}
151
152/**
153 * i40evf_detect_recover_hung - Function to detect and recover hung_queues
154 * @vsi: pointer to vsi struct with tx queues
155 *
156 * VSI has netdev and netdev has TX queues. This function is to check each of
157 * those TX queues if they are hung, trigger recovery by issuing SW interrupt.
158 **/
159void i40evf_detect_recover_hung(struct i40e_vsi *vsi)
160{
161 struct i40e_ring *tx_ring = NULL;
162 struct net_device *netdev;
163 unsigned int i;
164 int packets;
165
166 if (!vsi)
167 return;
168
169 if (test_bit(__I40E_VSI_DOWN, vsi->state))
170 return;
171
172 netdev = vsi->netdev;
173 if (!netdev)
174 return;
175
176 if (!netif_carrier_ok(netdev))
177 return;
178
179 for (i = 0; i < vsi->back->num_active_queues; i++) {
180 tx_ring = &vsi->back->tx_rings[i];
181 if (tx_ring && tx_ring->desc) {
182 /* If packet counter has not changed the queue is
183 * likely stalled, so force an interrupt for this
184 * queue.
185 *
186 * prev_pkt_ctr would be negative if there was no
187 * pending work.
188 */
189 packets = tx_ring->stats.packets & INT_MAX;
190 if (tx_ring->tx_stats.prev_pkt_ctr == packets) {
191 i40evf_force_wb(vsi, tx_ring->q_vector);
192 continue;
193 }
194
195 /* Memory barrier between read of packet count and call
196 * to i40evf_get_tx_pending()
197 */
198 smp_rmb();
199 tx_ring->tx_stats.prev_pkt_ctr =
200 i40evf_get_tx_pending(tx_ring, true) ? packets : -1;
201 }
202 }
203}
204
205#define WB_STRIDE 4
206
207/**
208 * i40e_clean_tx_irq - Reclaim resources after transmit completes
209 * @vsi: the VSI we care about
210 * @tx_ring: Tx ring to clean
211 * @napi_budget: Used to determine if we are in netpoll
212 *
213 * Returns true if there's any budget left (e.g. the clean is finished)
214 **/
215static bool i40e_clean_tx_irq(struct i40e_vsi *vsi,
216 struct i40e_ring *tx_ring, int napi_budget)
217{
218 u16 i = tx_ring->next_to_clean;
219 struct i40e_tx_buffer *tx_buf;
220 struct i40e_tx_desc *tx_desc;
221 unsigned int total_bytes = 0, total_packets = 0;
222 unsigned int budget = vsi->work_limit;
223
224 tx_buf = &tx_ring->tx_bi[i];
225 tx_desc = I40E_TX_DESC(tx_ring, i);
226 i -= tx_ring->count;
227
228 do {
229 struct i40e_tx_desc *eop_desc = tx_buf->next_to_watch;
230
231 /* if next_to_watch is not set then there is no work pending */
232 if (!eop_desc)
233 break;
234
235 /* prevent any other reads prior to eop_desc */
236 smp_rmb();
237
238 i40e_trace(clean_tx_irq, tx_ring, tx_desc, tx_buf);
239 /* if the descriptor isn't done, no work yet to do */
240 if (!(eop_desc->cmd_type_offset_bsz &
241 cpu_to_le64(I40E_TX_DESC_DTYPE_DESC_DONE)))
242 break;
243
244 /* clear next_to_watch to prevent false hangs */
245 tx_buf->next_to_watch = NULL;
246
247 /* update the statistics for this packet */
248 total_bytes += tx_buf->bytecount;
249 total_packets += tx_buf->gso_segs;
250
251 /* free the skb */
252 napi_consume_skb(tx_buf->skb, napi_budget);
253
254 /* unmap skb header data */
255 dma_unmap_single(tx_ring->dev,
256 dma_unmap_addr(tx_buf, dma),
257 dma_unmap_len(tx_buf, len),
258 DMA_TO_DEVICE);
259
260 /* clear tx_buffer data */
261 tx_buf->skb = NULL;
262 dma_unmap_len_set(tx_buf, len, 0);
263
264 /* unmap remaining buffers */
265 while (tx_desc != eop_desc) {
266 i40e_trace(clean_tx_irq_unmap,
267 tx_ring, tx_desc, tx_buf);
268
269 tx_buf++;
270 tx_desc++;
271 i++;
272 if (unlikely(!i)) {
273 i -= tx_ring->count;
274 tx_buf = tx_ring->tx_bi;
275 tx_desc = I40E_TX_DESC(tx_ring, 0);
276 }
277
278 /* unmap any remaining paged data */
279 if (dma_unmap_len(tx_buf, len)) {
280 dma_unmap_page(tx_ring->dev,
281 dma_unmap_addr(tx_buf, dma),
282 dma_unmap_len(tx_buf, len),
283 DMA_TO_DEVICE);
284 dma_unmap_len_set(tx_buf, len, 0);
285 }
286 }
287
288 /* move us one more past the eop_desc for start of next pkt */
289 tx_buf++;
290 tx_desc++;
291 i++;
292 if (unlikely(!i)) {
293 i -= tx_ring->count;
294 tx_buf = tx_ring->tx_bi;
295 tx_desc = I40E_TX_DESC(tx_ring, 0);
296 }
297
298 prefetch(tx_desc);
299
300 /* update budget accounting */
301 budget--;
302 } while (likely(budget));
303
304 i += tx_ring->count;
305 tx_ring->next_to_clean = i;
306 u64_stats_update_begin(&tx_ring->syncp);
307 tx_ring->stats.bytes += total_bytes;
308 tx_ring->stats.packets += total_packets;
309 u64_stats_update_end(&tx_ring->syncp);
310 tx_ring->q_vector->tx.total_bytes += total_bytes;
311 tx_ring->q_vector->tx.total_packets += total_packets;
312
313 if (tx_ring->flags & I40E_TXR_FLAGS_WB_ON_ITR) {
314 /* check to see if there are < 4 descriptors
315 * waiting to be written back, then kick the hardware to force
316 * them to be written back in case we stay in NAPI.
317 * In this mode on X722 we do not enable Interrupt.
318 */
319 unsigned int j = i40evf_get_tx_pending(tx_ring, false);
320
321 if (budget &&
322 ((j / WB_STRIDE) == 0) && (j > 0) &&
323 !test_bit(__I40E_VSI_DOWN, vsi->state) &&
324 (I40E_DESC_UNUSED(tx_ring) != tx_ring->count))
325 tx_ring->arm_wb = true;
326 }
327
328 /* notify netdev of completed buffers */
329 netdev_tx_completed_queue(txring_txq(tx_ring),
330 total_packets, total_bytes);
331
332#define TX_WAKE_THRESHOLD ((s16)(DESC_NEEDED * 2))
333 if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
334 (I40E_DESC_UNUSED(tx_ring) >= TX_WAKE_THRESHOLD))) {
335 /* Make sure that anybody stopping the queue after this
336 * sees the new next_to_clean.
337 */
338 smp_mb();
339 if (__netif_subqueue_stopped(tx_ring->netdev,
340 tx_ring->queue_index) &&
341 !test_bit(__I40E_VSI_DOWN, vsi->state)) {
342 netif_wake_subqueue(tx_ring->netdev,
343 tx_ring->queue_index);
344 ++tx_ring->tx_stats.restart_queue;
345 }
346 }
347
348 return !!budget;
349}
350
351/**
352 * i40evf_enable_wb_on_itr - Arm hardware to do a wb, interrupts are not enabled
353 * @vsi: the VSI we care about
354 * @q_vector: the vector on which to enable writeback
355 *
356 **/
357static void i40e_enable_wb_on_itr(struct i40e_vsi *vsi,
358 struct i40e_q_vector *q_vector)
359{
360 u16 flags = q_vector->tx.ring[0].flags;
361 u32 val;
362
363 if (!(flags & I40E_TXR_FLAGS_WB_ON_ITR))
364 return;
365
366 if (q_vector->arm_wb_state)
367 return;
368
369 val = I40E_VFINT_DYN_CTLN1_WB_ON_ITR_MASK |
370 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK; /* set noitr */
371
372 wr32(&vsi->back->hw,
373 I40E_VFINT_DYN_CTLN1(q_vector->reg_idx), val);
374 q_vector->arm_wb_state = true;
375}
376
377/**
378 * i40evf_force_wb - Issue SW Interrupt so HW does a wb
379 * @vsi: the VSI we care about
380 * @q_vector: the vector on which to force writeback
381 *
382 **/
383void i40evf_force_wb(struct i40e_vsi *vsi, struct i40e_q_vector *q_vector)
384{
385 u32 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
386 I40E_VFINT_DYN_CTLN1_ITR_INDX_MASK | /* set noitr */
387 I40E_VFINT_DYN_CTLN1_SWINT_TRIG_MASK |
388 I40E_VFINT_DYN_CTLN1_SW_ITR_INDX_ENA_MASK
389 /* allow 00 to be written to the index */;
390
391 wr32(&vsi->back->hw,
392 I40E_VFINT_DYN_CTLN1(q_vector->reg_idx),
393 val);
394}
395
396static inline bool i40e_container_is_rx(struct i40e_q_vector *q_vector,
397 struct i40e_ring_container *rc)
398{
399 return &q_vector->rx == rc;
400}
401
402static inline unsigned int i40e_itr_divisor(struct i40e_q_vector *q_vector)
403{
404 unsigned int divisor;
405
406 switch (q_vector->adapter->link_speed) {
407 case I40E_LINK_SPEED_40GB:
408 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 1024;
409 break;
410 case I40E_LINK_SPEED_25GB:
411 case I40E_LINK_SPEED_20GB:
412 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 512;
413 break;
414 default:
415 case I40E_LINK_SPEED_10GB:
416 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 256;
417 break;
418 case I40E_LINK_SPEED_1GB:
419 case I40E_LINK_SPEED_100MB:
420 divisor = I40E_ITR_ADAPTIVE_MIN_INC * 32;
421 break;
422 }
423
424 return divisor;
425}
426
427/**
428 * i40e_update_itr - update the dynamic ITR value based on statistics
429 * @q_vector: structure containing interrupt and ring information
430 * @rc: structure containing ring performance data
431 *
432 * Stores a new ITR value based on packets and byte
433 * counts during the last interrupt. The advantage of per interrupt
434 * computation is faster updates and more accurate ITR for the current
435 * traffic pattern. Constants in this function were computed
436 * based on theoretical maximum wire speed and thresholds were set based
437 * on testing data as well as attempting to minimize response time
438 * while increasing bulk throughput.
439 **/
440static void i40e_update_itr(struct i40e_q_vector *q_vector,
441 struct i40e_ring_container *rc)
442{
443 unsigned int avg_wire_size, packets, bytes, itr;
444 unsigned long next_update = jiffies;
445
446 /* If we don't have any rings just leave ourselves set for maximum
447 * possible latency so we take ourselves out of the equation.
448 */
449 if (!rc->ring || !ITR_IS_DYNAMIC(rc->ring->itr_setting))
450 return;
451
452 /* For Rx we want to push the delay up and default to low latency.
453 * for Tx we want to pull the delay down and default to high latency.
454 */
455 itr = i40e_container_is_rx(q_vector, rc) ?
456 I40E_ITR_ADAPTIVE_MIN_USECS | I40E_ITR_ADAPTIVE_LATENCY :
457 I40E_ITR_ADAPTIVE_MAX_USECS | I40E_ITR_ADAPTIVE_LATENCY;
458
459 /* If we didn't update within up to 1 - 2 jiffies we can assume
460 * that either packets are coming in so slow there hasn't been
461 * any work, or that there is so much work that NAPI is dealing
462 * with interrupt moderation and we don't need to do anything.
463 */
464 if (time_after(next_update, rc->next_update))
465 goto clear_counts;
466
467 /* If itr_countdown is set it means we programmed an ITR within
468 * the last 4 interrupt cycles. This has a side effect of us
469 * potentially firing an early interrupt. In order to work around
470 * this we need to throw out any data received for a few
471 * interrupts following the update.
472 */
473 if (q_vector->itr_countdown) {
474 itr = rc->target_itr;
475 goto clear_counts;
476 }
477
478 packets = rc->total_packets;
479 bytes = rc->total_bytes;
480
481 if (i40e_container_is_rx(q_vector, rc)) {
482 /* If Rx there are 1 to 4 packets and bytes are less than
483 * 9000 assume insufficient data to use bulk rate limiting
484 * approach unless Tx is already in bulk rate limiting. We
485 * are likely latency driven.
486 */
487 if (packets && packets < 4 && bytes < 9000 &&
488 (q_vector->tx.target_itr & I40E_ITR_ADAPTIVE_LATENCY)) {
489 itr = I40E_ITR_ADAPTIVE_LATENCY;
490 goto adjust_by_size;
491 }
492 } else if (packets < 4) {
493 /* If we have Tx and Rx ITR maxed and Tx ITR is running in
494 * bulk mode and we are receiving 4 or fewer packets just
495 * reset the ITR_ADAPTIVE_LATENCY bit for latency mode so
496 * that the Rx can relax.
497 */
498 if (rc->target_itr == I40E_ITR_ADAPTIVE_MAX_USECS &&
499 (q_vector->rx.target_itr & I40E_ITR_MASK) ==
500 I40E_ITR_ADAPTIVE_MAX_USECS)
501 goto clear_counts;
502 } else if (packets > 32) {
503 /* If we have processed over 32 packets in a single interrupt
504 * for Tx assume we need to switch over to "bulk" mode.
505 */
506 rc->target_itr &= ~I40E_ITR_ADAPTIVE_LATENCY;
507 }
508
509 /* We have no packets to actually measure against. This means
510 * either one of the other queues on this vector is active or
511 * we are a Tx queue doing TSO with too high of an interrupt rate.
512 *
513 * Between 4 and 56 we can assume that our current interrupt delay
514 * is only slightly too low. As such we should increase it by a small
515 * fixed amount.
516 */
517 if (packets < 56) {
518 itr = rc->target_itr + I40E_ITR_ADAPTIVE_MIN_INC;
519 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
520 itr &= I40E_ITR_ADAPTIVE_LATENCY;
521 itr += I40E_ITR_ADAPTIVE_MAX_USECS;
522 }
523 goto clear_counts;
524 }
525
526 if (packets <= 256) {
527 itr = min(q_vector->tx.current_itr, q_vector->rx.current_itr);
528 itr &= I40E_ITR_MASK;
529
530 /* Between 56 and 112 is our "goldilocks" zone where we are
531 * working out "just right". Just report that our current
532 * ITR is good for us.
533 */
534 if (packets <= 112)
535 goto clear_counts;
536
537 /* If packet count is 128 or greater we are likely looking
538 * at a slight overrun of the delay we want. Try halving
539 * our delay to see if that will cut the number of packets
540 * in half per interrupt.
541 */
542 itr /= 2;
543 itr &= I40E_ITR_MASK;
544 if (itr < I40E_ITR_ADAPTIVE_MIN_USECS)
545 itr = I40E_ITR_ADAPTIVE_MIN_USECS;
546
547 goto clear_counts;
548 }
549
550 /* The paths below assume we are dealing with a bulk ITR since
551 * number of packets is greater than 256. We are just going to have
552 * to compute a value and try to bring the count under control,
553 * though for smaller packet sizes there isn't much we can do as
554 * NAPI polling will likely be kicking in sooner rather than later.
555 */
556 itr = I40E_ITR_ADAPTIVE_BULK;
557
558adjust_by_size:
559 /* If packet counts are 256 or greater we can assume we have a gross
560 * overestimation of what the rate should be. Instead of trying to fine
561 * tune it just use the formula below to try and dial in an exact value
562 * give the current packet size of the frame.
563 */
564 avg_wire_size = bytes / packets;
565
566 /* The following is a crude approximation of:
567 * wmem_default / (size + overhead) = desired_pkts_per_int
568 * rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
569 * (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
570 *
571 * Assuming wmem_default is 212992 and overhead is 640 bytes per
572 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
573 * formula down to
574 *
575 * (170 * (size + 24)) / (size + 640) = ITR
576 *
577 * We first do some math on the packet size and then finally bitshift
578 * by 8 after rounding up. We also have to account for PCIe link speed
579 * difference as ITR scales based on this.
580 */
581 if (avg_wire_size <= 60) {
582 /* Start at 250k ints/sec */
583 avg_wire_size = 4096;
584 } else if (avg_wire_size <= 380) {
585 /* 250K ints/sec to 60K ints/sec */
586 avg_wire_size *= 40;
587 avg_wire_size += 1696;
588 } else if (avg_wire_size <= 1084) {
589 /* 60K ints/sec to 36K ints/sec */
590 avg_wire_size *= 15;
591 avg_wire_size += 11452;
592 } else if (avg_wire_size <= 1980) {
593 /* 36K ints/sec to 30K ints/sec */
594 avg_wire_size *= 5;
595 avg_wire_size += 22420;
596 } else {
597 /* plateau at a limit of 30K ints/sec */
598 avg_wire_size = 32256;
599 }
600
601 /* If we are in low latency mode halve our delay which doubles the
602 * rate to somewhere between 100K to 16K ints/sec
603 */
604 if (itr & I40E_ITR_ADAPTIVE_LATENCY)
605 avg_wire_size /= 2;
606
607 /* Resultant value is 256 times larger than it needs to be. This
608 * gives us room to adjust the value as needed to either increase
609 * or decrease the value based on link speeds of 10G, 2.5G, 1G, etc.
610 *
611 * Use addition as we have already recorded the new latency flag
612 * for the ITR value.
613 */
614 itr += DIV_ROUND_UP(avg_wire_size, i40e_itr_divisor(q_vector)) *
615 I40E_ITR_ADAPTIVE_MIN_INC;
616
617 if ((itr & I40E_ITR_MASK) > I40E_ITR_ADAPTIVE_MAX_USECS) {
618 itr &= I40E_ITR_ADAPTIVE_LATENCY;
619 itr += I40E_ITR_ADAPTIVE_MAX_USECS;
620 }
621
622clear_counts:
623 /* write back value */
624 rc->target_itr = itr;
625
626 /* next update should occur within next jiffy */
627 rc->next_update = next_update + 1;
628
629 rc->total_bytes = 0;
630 rc->total_packets = 0;
631}
632
633/**
634 * i40evf_setup_tx_descriptors - Allocate the Tx descriptors
635 * @tx_ring: the tx ring to set up
636 *
637 * Return 0 on success, negative on error
638 **/
639int i40evf_setup_tx_descriptors(struct i40e_ring *tx_ring)
640{
641 struct device *dev = tx_ring->dev;
642 int bi_size;
643
644 if (!dev)
645 return -ENOMEM;
646
647 /* warn if we are about to overwrite the pointer */
648 WARN_ON(tx_ring->tx_bi);
649 bi_size = sizeof(struct i40e_tx_buffer) * tx_ring->count;
650 tx_ring->tx_bi = kzalloc(bi_size, GFP_KERNEL);
651 if (!tx_ring->tx_bi)
652 goto err;
653
654 /* round up to nearest 4K */
655 tx_ring->size = tx_ring->count * sizeof(struct i40e_tx_desc);
656 tx_ring->size = ALIGN(tx_ring->size, 4096);
657 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
658 &tx_ring->dma, GFP_KERNEL);
659 if (!tx_ring->desc) {
660 dev_info(dev, "Unable to allocate memory for the Tx descriptor ring, size=%d\n",
661 tx_ring->size);
662 goto err;
663 }
664
665 tx_ring->next_to_use = 0;
666 tx_ring->next_to_clean = 0;
667 tx_ring->tx_stats.prev_pkt_ctr = -1;
668 return 0;
669
670err:
671 kfree(tx_ring->tx_bi);
672 tx_ring->tx_bi = NULL;
673 return -ENOMEM;
674}
675
676/**
677 * i40evf_clean_rx_ring - Free Rx buffers
678 * @rx_ring: ring to be cleaned
679 **/
680void i40evf_clean_rx_ring(struct i40e_ring *rx_ring)
681{
682 unsigned long bi_size;
683 u16 i;
684
685 /* ring already cleared, nothing to do */
686 if (!rx_ring->rx_bi)
687 return;
688
689 if (rx_ring->skb) {
690 dev_kfree_skb(rx_ring->skb);
691 rx_ring->skb = NULL;
692 }
693
694 /* Free all the Rx ring sk_buffs */
695 for (i = 0; i < rx_ring->count; i++) {
696 struct i40e_rx_buffer *rx_bi = &rx_ring->rx_bi[i];
697
698 if (!rx_bi->page)
699 continue;
700
701 /* Invalidate cache lines that may have been written to by
702 * device so that we avoid corrupting memory.
703 */
704 dma_sync_single_range_for_cpu(rx_ring->dev,
705 rx_bi->dma,
706 rx_bi->page_offset,
707 rx_ring->rx_buf_len,
708 DMA_FROM_DEVICE);
709
710 /* free resources associated with mapping */
711 dma_unmap_page_attrs(rx_ring->dev, rx_bi->dma,
712 i40e_rx_pg_size(rx_ring),
713 DMA_FROM_DEVICE,
714 I40E_RX_DMA_ATTR);
715
716 __page_frag_cache_drain(rx_bi->page, rx_bi->pagecnt_bias);
717
718 rx_bi->page = NULL;
719 rx_bi->page_offset = 0;
720 }
721
722 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
723 memset(rx_ring->rx_bi, 0, bi_size);
724
725 /* Zero out the descriptor ring */
726 memset(rx_ring->desc, 0, rx_ring->size);
727
728 rx_ring->next_to_alloc = 0;
729 rx_ring->next_to_clean = 0;
730 rx_ring->next_to_use = 0;
731}
732
733/**
734 * i40evf_free_rx_resources - Free Rx resources
735 * @rx_ring: ring to clean the resources from
736 *
737 * Free all receive software resources
738 **/
739void i40evf_free_rx_resources(struct i40e_ring *rx_ring)
740{
741 i40evf_clean_rx_ring(rx_ring);
742 kfree(rx_ring->rx_bi);
743 rx_ring->rx_bi = NULL;
744
745 if (rx_ring->desc) {
746 dma_free_coherent(rx_ring->dev, rx_ring->size,
747 rx_ring->desc, rx_ring->dma);
748 rx_ring->desc = NULL;
749 }
750}
751
752/**
753 * i40evf_setup_rx_descriptors - Allocate Rx descriptors
754 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
755 *
756 * Returns 0 on success, negative on failure
757 **/
758int i40evf_setup_rx_descriptors(struct i40e_ring *rx_ring)
759{
760 struct device *dev = rx_ring->dev;
761 int bi_size;
762
763 /* warn if we are about to overwrite the pointer */
764 WARN_ON(rx_ring->rx_bi);
765 bi_size = sizeof(struct i40e_rx_buffer) * rx_ring->count;
766 rx_ring->rx_bi = kzalloc(bi_size, GFP_KERNEL);
767 if (!rx_ring->rx_bi)
768 goto err;
769
770 u64_stats_init(&rx_ring->syncp);
771
772 /* Round up to nearest 4K */
773 rx_ring->size = rx_ring->count * sizeof(union i40e_32byte_rx_desc);
774 rx_ring->size = ALIGN(rx_ring->size, 4096);
775 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
776 &rx_ring->dma, GFP_KERNEL);
777
778 if (!rx_ring->desc) {
779 dev_info(dev, "Unable to allocate memory for the Rx descriptor ring, size=%d\n",
780 rx_ring->size);
781 goto err;
782 }
783
784 rx_ring->next_to_alloc = 0;
785 rx_ring->next_to_clean = 0;
786 rx_ring->next_to_use = 0;
787
788 return 0;
789err:
790 kfree(rx_ring->rx_bi);
791 rx_ring->rx_bi = NULL;
792 return -ENOMEM;
793}
794
795/**
796 * i40e_release_rx_desc - Store the new tail and head values
797 * @rx_ring: ring to bump
798 * @val: new head index
799 **/
800static inline void i40e_release_rx_desc(struct i40e_ring *rx_ring, u32 val)
801{
802 rx_ring->next_to_use = val;
803
804 /* update next to alloc since we have filled the ring */
805 rx_ring->next_to_alloc = val;
806
807 /* Force memory writes to complete before letting h/w
808 * know there are new descriptors to fetch. (Only
809 * applicable for weak-ordered memory model archs,
810 * such as IA-64).
811 */
812 wmb();
813 writel(val, rx_ring->tail);
814}
815
816/**
817 * i40e_rx_offset - Return expected offset into page to access data
818 * @rx_ring: Ring we are requesting offset of
819 *
820 * Returns the offset value for ring into the data buffer.
821 */
822static inline unsigned int i40e_rx_offset(struct i40e_ring *rx_ring)
823{
824 return ring_uses_build_skb(rx_ring) ? I40E_SKB_PAD : 0;
825}
826
827/**
828 * i40e_alloc_mapped_page - recycle or make a new page
829 * @rx_ring: ring to use
830 * @bi: rx_buffer struct to modify
831 *
832 * Returns true if the page was successfully allocated or
833 * reused.
834 **/
835static bool i40e_alloc_mapped_page(struct i40e_ring *rx_ring,
836 struct i40e_rx_buffer *bi)
837{
838 struct page *page = bi->page;
839 dma_addr_t dma;
840
841 /* since we are recycling buffers we should seldom need to alloc */
842 if (likely(page)) {
843 rx_ring->rx_stats.page_reuse_count++;
844 return true;
845 }
846
847 /* alloc new page for storage */
848 page = dev_alloc_pages(i40e_rx_pg_order(rx_ring));
849 if (unlikely(!page)) {
850 rx_ring->rx_stats.alloc_page_failed++;
851 return false;
852 }
853
854 /* map page for use */
855 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
856 i40e_rx_pg_size(rx_ring),
857 DMA_FROM_DEVICE,
858 I40E_RX_DMA_ATTR);
859
860 /* if mapping failed free memory back to system since
861 * there isn't much point in holding memory we can't use
862 */
863 if (dma_mapping_error(rx_ring->dev, dma)) {
864 __free_pages(page, i40e_rx_pg_order(rx_ring));
865 rx_ring->rx_stats.alloc_page_failed++;
866 return false;
867 }
868
869 bi->dma = dma;
870 bi->page = page;
871 bi->page_offset = i40e_rx_offset(rx_ring);
872
873 /* initialize pagecnt_bias to 1 representing we fully own page */
874 bi->pagecnt_bias = 1;
875
876 return true;
877}
878
879/**
880 * i40e_receive_skb - Send a completed packet up the stack
881 * @rx_ring: rx ring in play
882 * @skb: packet to send up
883 * @vlan_tag: vlan tag for packet
884 **/
885static void i40e_receive_skb(struct i40e_ring *rx_ring,
886 struct sk_buff *skb, u16 vlan_tag)
887{
888 struct i40e_q_vector *q_vector = rx_ring->q_vector;
889
890 if ((rx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
891 (vlan_tag & VLAN_VID_MASK))
892 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vlan_tag);
893
894 napi_gro_receive(&q_vector->napi, skb);
895}
896
897/**
898 * i40evf_alloc_rx_buffers - Replace used receive buffers
899 * @rx_ring: ring to place buffers on
900 * @cleaned_count: number of buffers to replace
901 *
902 * Returns false if all allocations were successful, true if any fail
903 **/
904bool i40evf_alloc_rx_buffers(struct i40e_ring *rx_ring, u16 cleaned_count)
905{
906 u16 ntu = rx_ring->next_to_use;
907 union i40e_rx_desc *rx_desc;
908 struct i40e_rx_buffer *bi;
909
910 /* do nothing if no valid netdev defined */
911 if (!rx_ring->netdev || !cleaned_count)
912 return false;
913
914 rx_desc = I40E_RX_DESC(rx_ring, ntu);
915 bi = &rx_ring->rx_bi[ntu];
916
917 do {
918 if (!i40e_alloc_mapped_page(rx_ring, bi))
919 goto no_buffers;
920
921 /* sync the buffer for use by the device */
922 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
923 bi->page_offset,
924 rx_ring->rx_buf_len,
925 DMA_FROM_DEVICE);
926
927 /* Refresh the desc even if buffer_addrs didn't change
928 * because each write-back erases this info.
929 */
930 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
931
932 rx_desc++;
933 bi++;
934 ntu++;
935 if (unlikely(ntu == rx_ring->count)) {
936 rx_desc = I40E_RX_DESC(rx_ring, 0);
937 bi = rx_ring->rx_bi;
938 ntu = 0;
939 }
940
941 /* clear the status bits for the next_to_use descriptor */
942 rx_desc->wb.qword1.status_error_len = 0;
943
944 cleaned_count--;
945 } while (cleaned_count);
946
947 if (rx_ring->next_to_use != ntu)
948 i40e_release_rx_desc(rx_ring, ntu);
949
950 return false;
951
952no_buffers:
953 if (rx_ring->next_to_use != ntu)
954 i40e_release_rx_desc(rx_ring, ntu);
955
956 /* make sure to come back via polling to try again after
957 * allocation failure
958 */
959 return true;
960}
961
962/**
963 * i40e_rx_checksum - Indicate in skb if hw indicated a good cksum
964 * @vsi: the VSI we care about
965 * @skb: skb currently being received and modified
966 * @rx_desc: the receive descriptor
967 **/
968static inline void i40e_rx_checksum(struct i40e_vsi *vsi,
969 struct sk_buff *skb,
970 union i40e_rx_desc *rx_desc)
971{
972 struct i40e_rx_ptype_decoded decoded;
973 u32 rx_error, rx_status;
974 bool ipv4, ipv6;
975 u8 ptype;
976 u64 qword;
977
978 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
979 ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >> I40E_RXD_QW1_PTYPE_SHIFT;
980 rx_error = (qword & I40E_RXD_QW1_ERROR_MASK) >>
981 I40E_RXD_QW1_ERROR_SHIFT;
982 rx_status = (qword & I40E_RXD_QW1_STATUS_MASK) >>
983 I40E_RXD_QW1_STATUS_SHIFT;
984 decoded = decode_rx_desc_ptype(ptype);
985
986 skb->ip_summed = CHECKSUM_NONE;
987
988 skb_checksum_none_assert(skb);
989
990 /* Rx csum enabled and ip headers found? */
991 if (!(vsi->netdev->features & NETIF_F_RXCSUM))
992 return;
993
994 /* did the hardware decode the packet and checksum? */
995 if (!(rx_status & BIT(I40E_RX_DESC_STATUS_L3L4P_SHIFT)))
996 return;
997
998 /* both known and outer_ip must be set for the below code to work */
999 if (!(decoded.known && decoded.outer_ip))
1000 return;
1001
1002 ipv4 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
1003 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV4);
1004 ipv6 = (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP) &&
1005 (decoded.outer_ip_ver == I40E_RX_PTYPE_OUTER_IPV6);
1006
1007 if (ipv4 &&
1008 (rx_error & (BIT(I40E_RX_DESC_ERROR_IPE_SHIFT) |
1009 BIT(I40E_RX_DESC_ERROR_EIPE_SHIFT))))
1010 goto checksum_fail;
1011
1012 /* likely incorrect csum if alternate IP extension headers found */
1013 if (ipv6 &&
1014 rx_status & BIT(I40E_RX_DESC_STATUS_IPV6EXADD_SHIFT))
1015 /* don't increment checksum err here, non-fatal err */
1016 return;
1017
1018 /* there was some L4 error, count error and punt packet to the stack */
1019 if (rx_error & BIT(I40E_RX_DESC_ERROR_L4E_SHIFT))
1020 goto checksum_fail;
1021
1022 /* handle packets that were not able to be checksummed due
1023 * to arrival speed, in this case the stack can compute
1024 * the csum.
1025 */
1026 if (rx_error & BIT(I40E_RX_DESC_ERROR_PPRS_SHIFT))
1027 return;
1028
1029 /* Only report checksum unnecessary for TCP, UDP, or SCTP */
1030 switch (decoded.inner_prot) {
1031 case I40E_RX_PTYPE_INNER_PROT_TCP:
1032 case I40E_RX_PTYPE_INNER_PROT_UDP:
1033 case I40E_RX_PTYPE_INNER_PROT_SCTP:
1034 skb->ip_summed = CHECKSUM_UNNECESSARY;
1035 /* fall though */
1036 default:
1037 break;
1038 }
1039
1040 return;
1041
1042checksum_fail:
1043 vsi->back->hw_csum_rx_error++;
1044}
1045
1046/**
1047 * i40e_ptype_to_htype - get a hash type
1048 * @ptype: the ptype value from the descriptor
1049 *
1050 * Returns a hash type to be used by skb_set_hash
1051 **/
1052static inline int i40e_ptype_to_htype(u8 ptype)
1053{
1054 struct i40e_rx_ptype_decoded decoded = decode_rx_desc_ptype(ptype);
1055
1056 if (!decoded.known)
1057 return PKT_HASH_TYPE_NONE;
1058
1059 if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1060 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY4)
1061 return PKT_HASH_TYPE_L4;
1062 else if (decoded.outer_ip == I40E_RX_PTYPE_OUTER_IP &&
1063 decoded.payload_layer == I40E_RX_PTYPE_PAYLOAD_LAYER_PAY3)
1064 return PKT_HASH_TYPE_L3;
1065 else
1066 return PKT_HASH_TYPE_L2;
1067}
1068
1069/**
1070 * i40e_rx_hash - set the hash value in the skb
1071 * @ring: descriptor ring
1072 * @rx_desc: specific descriptor
1073 **/
1074static inline void i40e_rx_hash(struct i40e_ring *ring,
1075 union i40e_rx_desc *rx_desc,
1076 struct sk_buff *skb,
1077 u8 rx_ptype)
1078{
1079 u32 hash;
1080 const __le64 rss_mask =
1081 cpu_to_le64((u64)I40E_RX_DESC_FLTSTAT_RSS_HASH <<
1082 I40E_RX_DESC_STATUS_FLTSTAT_SHIFT);
1083
1084 if (ring->netdev->features & NETIF_F_RXHASH)
1085 return;
1086
1087 if ((rx_desc->wb.qword1.status_error_len & rss_mask) == rss_mask) {
1088 hash = le32_to_cpu(rx_desc->wb.qword0.hi_dword.rss);
1089 skb_set_hash(skb, hash, i40e_ptype_to_htype(rx_ptype));
1090 }
1091}
1092
1093/**
1094 * i40evf_process_skb_fields - Populate skb header fields from Rx descriptor
1095 * @rx_ring: rx descriptor ring packet is being transacted on
1096 * @rx_desc: pointer to the EOP Rx descriptor
1097 * @skb: pointer to current skb being populated
1098 * @rx_ptype: the packet type decoded by hardware
1099 *
1100 * This function checks the ring, descriptor, and packet information in
1101 * order to populate the hash, checksum, VLAN, protocol, and
1102 * other fields within the skb.
1103 **/
1104static inline
1105void i40evf_process_skb_fields(struct i40e_ring *rx_ring,
1106 union i40e_rx_desc *rx_desc, struct sk_buff *skb,
1107 u8 rx_ptype)
1108{
1109 i40e_rx_hash(rx_ring, rx_desc, skb, rx_ptype);
1110
1111 i40e_rx_checksum(rx_ring->vsi, skb, rx_desc);
1112
1113 skb_record_rx_queue(skb, rx_ring->queue_index);
1114
1115 /* modifies the skb - consumes the enet header */
1116 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1117}
1118
1119/**
1120 * i40e_cleanup_headers - Correct empty headers
1121 * @rx_ring: rx descriptor ring packet is being transacted on
1122 * @skb: pointer to current skb being fixed
1123 *
1124 * Also address the case where we are pulling data in on pages only
1125 * and as such no data is present in the skb header.
1126 *
1127 * In addition if skb is not at least 60 bytes we need to pad it so that
1128 * it is large enough to qualify as a valid Ethernet frame.
1129 *
1130 * Returns true if an error was encountered and skb was freed.
1131 **/
1132static bool i40e_cleanup_headers(struct i40e_ring *rx_ring, struct sk_buff *skb)
1133{
1134 /* if eth_skb_pad returns an error the skb was freed */
1135 if (eth_skb_pad(skb))
1136 return true;
1137
1138 return false;
1139}
1140
1141/**
1142 * i40e_reuse_rx_page - page flip buffer and store it back on the ring
1143 * @rx_ring: rx descriptor ring to store buffers on
1144 * @old_buff: donor buffer to have page reused
1145 *
1146 * Synchronizes page for reuse by the adapter
1147 **/
1148static void i40e_reuse_rx_page(struct i40e_ring *rx_ring,
1149 struct i40e_rx_buffer *old_buff)
1150{
1151 struct i40e_rx_buffer *new_buff;
1152 u16 nta = rx_ring->next_to_alloc;
1153
1154 new_buff = &rx_ring->rx_bi[nta];
1155
1156 /* update, and store next to alloc */
1157 nta++;
1158 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
1159
1160 /* transfer page from old buffer to new buffer */
1161 new_buff->dma = old_buff->dma;
1162 new_buff->page = old_buff->page;
1163 new_buff->page_offset = old_buff->page_offset;
1164 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
1165}
1166
1167/**
1168 * i40e_page_is_reusable - check if any reuse is possible
1169 * @page: page struct to check
1170 *
1171 * A page is not reusable if it was allocated under low memory
1172 * conditions, or it's not in the same NUMA node as this CPU.
1173 */
1174static inline bool i40e_page_is_reusable(struct page *page)
1175{
1176 return (page_to_nid(page) == numa_mem_id()) &&
1177 !page_is_pfmemalloc(page);
1178}
1179
1180/**
1181 * i40e_can_reuse_rx_page - Determine if this page can be reused by
1182 * the adapter for another receive
1183 *
1184 * @rx_buffer: buffer containing the page
1185 *
1186 * If page is reusable, rx_buffer->page_offset is adjusted to point to
1187 * an unused region in the page.
1188 *
1189 * For small pages, @truesize will be a constant value, half the size
1190 * of the memory at page. We'll attempt to alternate between high and
1191 * low halves of the page, with one half ready for use by the hardware
1192 * and the other half being consumed by the stack. We use the page
1193 * ref count to determine whether the stack has finished consuming the
1194 * portion of this page that was passed up with a previous packet. If
1195 * the page ref count is >1, we'll assume the "other" half page is
1196 * still busy, and this page cannot be reused.
1197 *
1198 * For larger pages, @truesize will be the actual space used by the
1199 * received packet (adjusted upward to an even multiple of the cache
1200 * line size). This will advance through the page by the amount
1201 * actually consumed by the received packets while there is still
1202 * space for a buffer. Each region of larger pages will be used at
1203 * most once, after which the page will not be reused.
1204 *
1205 * In either case, if the page is reusable its refcount is increased.
1206 **/
1207static bool i40e_can_reuse_rx_page(struct i40e_rx_buffer *rx_buffer)
1208{
1209 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
1210 struct page *page = rx_buffer->page;
1211
1212 /* Is any reuse possible? */
1213 if (unlikely(!i40e_page_is_reusable(page)))
1214 return false;
1215
1216#if (PAGE_SIZE < 8192)
1217 /* if we are only owner of page we can reuse it */
1218 if (unlikely((page_count(page) - pagecnt_bias) > 1))
1219 return false;
1220#else
1221#define I40E_LAST_OFFSET \
1222 (SKB_WITH_OVERHEAD(PAGE_SIZE) - I40E_RXBUFFER_2048)
1223 if (rx_buffer->page_offset > I40E_LAST_OFFSET)
1224 return false;
1225#endif
1226
1227 /* If we have drained the page fragment pool we need to update
1228 * the pagecnt_bias and page count so that we fully restock the
1229 * number of references the driver holds.
1230 */
1231 if (unlikely(!pagecnt_bias)) {
1232 page_ref_add(page, USHRT_MAX);
1233 rx_buffer->pagecnt_bias = USHRT_MAX;
1234 }
1235
1236 return true;
1237}
1238
1239/**
1240 * i40e_add_rx_frag - Add contents of Rx buffer to sk_buff
1241 * @rx_ring: rx descriptor ring to transact packets on
1242 * @rx_buffer: buffer containing page to add
1243 * @skb: sk_buff to place the data into
1244 * @size: packet length from rx_desc
1245 *
1246 * This function will add the data contained in rx_buffer->page to the skb.
1247 * It will just attach the page as a frag to the skb.
1248 *
1249 * The function will then update the page offset.
1250 **/
1251static void i40e_add_rx_frag(struct i40e_ring *rx_ring,
1252 struct i40e_rx_buffer *rx_buffer,
1253 struct sk_buff *skb,
1254 unsigned int size)
1255{
1256#if (PAGE_SIZE < 8192)
1257 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1258#else
1259 unsigned int truesize = SKB_DATA_ALIGN(size + i40e_rx_offset(rx_ring));
1260#endif
1261
1262 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1263 rx_buffer->page_offset, size, truesize);
1264
1265 /* page is being used so we must update the page offset */
1266#if (PAGE_SIZE < 8192)
1267 rx_buffer->page_offset ^= truesize;
1268#else
1269 rx_buffer->page_offset += truesize;
1270#endif
1271}
1272
1273/**
1274 * i40e_get_rx_buffer - Fetch Rx buffer and synchronize data for use
1275 * @rx_ring: rx descriptor ring to transact packets on
1276 * @size: size of buffer to add to skb
1277 *
1278 * This function will pull an Rx buffer from the ring and synchronize it
1279 * for use by the CPU.
1280 */
1281static struct i40e_rx_buffer *i40e_get_rx_buffer(struct i40e_ring *rx_ring,
1282 const unsigned int size)
1283{
1284 struct i40e_rx_buffer *rx_buffer;
1285
1286 rx_buffer = &rx_ring->rx_bi[rx_ring->next_to_clean];
1287 prefetchw(rx_buffer->page);
1288
1289 /* we are reusing so sync this buffer for CPU use */
1290 dma_sync_single_range_for_cpu(rx_ring->dev,
1291 rx_buffer->dma,
1292 rx_buffer->page_offset,
1293 size,
1294 DMA_FROM_DEVICE);
1295
1296 /* We have pulled a buffer for use, so decrement pagecnt_bias */
1297 rx_buffer->pagecnt_bias--;
1298
1299 return rx_buffer;
1300}
1301
1302/**
1303 * i40e_construct_skb - Allocate skb and populate it
1304 * @rx_ring: rx descriptor ring to transact packets on
1305 * @rx_buffer: rx buffer to pull data from
1306 * @size: size of buffer to add to skb
1307 *
1308 * This function allocates an skb. It then populates it with the page
1309 * data from the current receive descriptor, taking care to set up the
1310 * skb correctly.
1311 */
1312static struct sk_buff *i40e_construct_skb(struct i40e_ring *rx_ring,
1313 struct i40e_rx_buffer *rx_buffer,
1314 unsigned int size)
1315{
1316 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1317#if (PAGE_SIZE < 8192)
1318 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1319#else
1320 unsigned int truesize = SKB_DATA_ALIGN(size);
1321#endif
1322 unsigned int headlen;
1323 struct sk_buff *skb;
1324
1325 /* prefetch first cache line of first page */
1326 prefetch(va);
1327#if L1_CACHE_BYTES < 128
1328 prefetch(va + L1_CACHE_BYTES);
1329#endif
1330
1331 /* allocate a skb to store the frags */
1332 skb = __napi_alloc_skb(&rx_ring->q_vector->napi,
1333 I40E_RX_HDR_SIZE,
1334 GFP_ATOMIC | __GFP_NOWARN);
1335 if (unlikely(!skb))
1336 return NULL;
1337
1338 /* Determine available headroom for copy */
1339 headlen = size;
1340 if (headlen > I40E_RX_HDR_SIZE)
1341 headlen = eth_get_headlen(va, I40E_RX_HDR_SIZE);
1342
1343 /* align pull length to size of long to optimize memcpy performance */
1344 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
1345
1346 /* update all of the pointers */
1347 size -= headlen;
1348 if (size) {
1349 skb_add_rx_frag(skb, 0, rx_buffer->page,
1350 rx_buffer->page_offset + headlen,
1351 size, truesize);
1352
1353 /* buffer is used by skb, update page_offset */
1354#if (PAGE_SIZE < 8192)
1355 rx_buffer->page_offset ^= truesize;
1356#else
1357 rx_buffer->page_offset += truesize;
1358#endif
1359 } else {
1360 /* buffer is unused, reset bias back to rx_buffer */
1361 rx_buffer->pagecnt_bias++;
1362 }
1363
1364 return skb;
1365}
1366
1367/**
1368 * i40e_build_skb - Build skb around an existing buffer
1369 * @rx_ring: Rx descriptor ring to transact packets on
1370 * @rx_buffer: Rx buffer to pull data from
1371 * @size: size of buffer to add to skb
1372 *
1373 * This function builds an skb around an existing Rx buffer, taking care
1374 * to set up the skb correctly and avoid any memcpy overhead.
1375 */
1376static struct sk_buff *i40e_build_skb(struct i40e_ring *rx_ring,
1377 struct i40e_rx_buffer *rx_buffer,
1378 unsigned int size)
1379{
1380 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
1381#if (PAGE_SIZE < 8192)
1382 unsigned int truesize = i40e_rx_pg_size(rx_ring) / 2;
1383#else
1384 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1385 SKB_DATA_ALIGN(I40E_SKB_PAD + size);
1386#endif
1387 struct sk_buff *skb;
1388
1389 /* prefetch first cache line of first page */
1390 prefetch(va);
1391#if L1_CACHE_BYTES < 128
1392 prefetch(va + L1_CACHE_BYTES);
1393#endif
1394 /* build an skb around the page buffer */
1395 skb = build_skb(va - I40E_SKB_PAD, truesize);
1396 if (unlikely(!skb))
1397 return NULL;
1398
1399 /* update pointers within the skb to store the data */
1400 skb_reserve(skb, I40E_SKB_PAD);
1401 __skb_put(skb, size);
1402
1403 /* buffer is used by skb, update page_offset */
1404#if (PAGE_SIZE < 8192)
1405 rx_buffer->page_offset ^= truesize;
1406#else
1407 rx_buffer->page_offset += truesize;
1408#endif
1409
1410 return skb;
1411}
1412
1413/**
1414 * i40e_put_rx_buffer - Clean up used buffer and either recycle or free
1415 * @rx_ring: rx descriptor ring to transact packets on
1416 * @rx_buffer: rx buffer to pull data from
1417 *
1418 * This function will clean up the contents of the rx_buffer. It will
1419 * either recycle the buffer or unmap it and free the associated resources.
1420 */
1421static void i40e_put_rx_buffer(struct i40e_ring *rx_ring,
1422 struct i40e_rx_buffer *rx_buffer)
1423{
1424 if (i40e_can_reuse_rx_page(rx_buffer)) {
1425 /* hand second half of page back to the ring */
1426 i40e_reuse_rx_page(rx_ring, rx_buffer);
1427 rx_ring->rx_stats.page_reuse_count++;
1428 } else {
1429 /* we are not reusing the buffer so unmap it */
1430 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
1431 i40e_rx_pg_size(rx_ring),
1432 DMA_FROM_DEVICE, I40E_RX_DMA_ATTR);
1433 __page_frag_cache_drain(rx_buffer->page,
1434 rx_buffer->pagecnt_bias);
1435 }
1436
1437 /* clear contents of buffer_info */
1438 rx_buffer->page = NULL;
1439}
1440
1441/**
1442 * i40e_is_non_eop - process handling of non-EOP buffers
1443 * @rx_ring: Rx ring being processed
1444 * @rx_desc: Rx descriptor for current buffer
1445 * @skb: Current socket buffer containing buffer in progress
1446 *
1447 * This function updates next to clean. If the buffer is an EOP buffer
1448 * this function exits returning false, otherwise it will place the
1449 * sk_buff in the next buffer to be chained and return true indicating
1450 * that this is in fact a non-EOP buffer.
1451 **/
1452static bool i40e_is_non_eop(struct i40e_ring *rx_ring,
1453 union i40e_rx_desc *rx_desc,
1454 struct sk_buff *skb)
1455{
1456 u32 ntc = rx_ring->next_to_clean + 1;
1457
1458 /* fetch, update, and store next to clean */
1459 ntc = (ntc < rx_ring->count) ? ntc : 0;
1460 rx_ring->next_to_clean = ntc;
1461
1462 prefetch(I40E_RX_DESC(rx_ring, ntc));
1463
1464 /* if we are the last buffer then there is nothing else to do */
1465#define I40E_RXD_EOF BIT(I40E_RX_DESC_STATUS_EOF_SHIFT)
1466 if (likely(i40e_test_staterr(rx_desc, I40E_RXD_EOF)))
1467 return false;
1468
1469 rx_ring->rx_stats.non_eop_descs++;
1470
1471 return true;
1472}
1473
1474/**
1475 * i40e_clean_rx_irq - Clean completed descriptors from Rx ring - bounce buf
1476 * @rx_ring: rx descriptor ring to transact packets on
1477 * @budget: Total limit on number of packets to process
1478 *
1479 * This function provides a "bounce buffer" approach to Rx interrupt
1480 * processing. The advantage to this is that on systems that have
1481 * expensive overhead for IOMMU access this provides a means of avoiding
1482 * it by maintaining the mapping of the page to the system.
1483 *
1484 * Returns amount of work completed
1485 **/
1486static int i40e_clean_rx_irq(struct i40e_ring *rx_ring, int budget)
1487{
1488 unsigned int total_rx_bytes = 0, total_rx_packets = 0;
1489 struct sk_buff *skb = rx_ring->skb;
1490 u16 cleaned_count = I40E_DESC_UNUSED(rx_ring);
1491 bool failure = false;
1492
1493 while (likely(total_rx_packets < (unsigned int)budget)) {
1494 struct i40e_rx_buffer *rx_buffer;
1495 union i40e_rx_desc *rx_desc;
1496 unsigned int size;
1497 u16 vlan_tag;
1498 u8 rx_ptype;
1499 u64 qword;
1500
1501 /* return some buffers to hardware, one at a time is too slow */
1502 if (cleaned_count >= I40E_RX_BUFFER_WRITE) {
1503 failure = failure ||
1504 i40evf_alloc_rx_buffers(rx_ring, cleaned_count);
1505 cleaned_count = 0;
1506 }
1507
1508 rx_desc = I40E_RX_DESC(rx_ring, rx_ring->next_to_clean);
1509
1510 /* status_error_len will always be zero for unused descriptors
1511 * because it's cleared in cleanup, and overlaps with hdr_addr
1512 * which is always zero because packet split isn't used, if the
1513 * hardware wrote DD then the length will be non-zero
1514 */
1515 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1516
1517 /* This memory barrier is needed to keep us from reading
1518 * any other fields out of the rx_desc until we have
1519 * verified the descriptor has been written back.
1520 */
1521 dma_rmb();
1522
1523 size = (qword & I40E_RXD_QW1_LENGTH_PBUF_MASK) >>
1524 I40E_RXD_QW1_LENGTH_PBUF_SHIFT;
1525 if (!size)
1526 break;
1527
1528 i40e_trace(clean_rx_irq, rx_ring, rx_desc, skb);
1529 rx_buffer = i40e_get_rx_buffer(rx_ring, size);
1530
1531 /* retrieve a buffer from the ring */
1532 if (skb)
1533 i40e_add_rx_frag(rx_ring, rx_buffer, skb, size);
1534 else if (ring_uses_build_skb(rx_ring))
1535 skb = i40e_build_skb(rx_ring, rx_buffer, size);
1536 else
1537 skb = i40e_construct_skb(rx_ring, rx_buffer, size);
1538
1539 /* exit if we failed to retrieve a buffer */
1540 if (!skb) {
1541 rx_ring->rx_stats.alloc_buff_failed++;
1542 rx_buffer->pagecnt_bias++;
1543 break;
1544 }
1545
1546 i40e_put_rx_buffer(rx_ring, rx_buffer);
1547 cleaned_count++;
1548
1549 if (i40e_is_non_eop(rx_ring, rx_desc, skb))
1550 continue;
1551
1552 /* ERR_MASK will only have valid bits if EOP set, and
1553 * what we are doing here is actually checking
1554 * I40E_RX_DESC_ERROR_RXE_SHIFT, since it is the zeroth bit in
1555 * the error field
1556 */
1557 if (unlikely(i40e_test_staterr(rx_desc, BIT(I40E_RXD_QW1_ERROR_SHIFT)))) {
1558 dev_kfree_skb_any(skb);
1559 skb = NULL;
1560 continue;
1561 }
1562
1563 if (i40e_cleanup_headers(rx_ring, skb)) {
1564 skb = NULL;
1565 continue;
1566 }
1567
1568 /* probably a little skewed due to removing CRC */
1569 total_rx_bytes += skb->len;
1570
1571 qword = le64_to_cpu(rx_desc->wb.qword1.status_error_len);
1572 rx_ptype = (qword & I40E_RXD_QW1_PTYPE_MASK) >>
1573 I40E_RXD_QW1_PTYPE_SHIFT;
1574
1575 /* populate checksum, VLAN, and protocol */
1576 i40evf_process_skb_fields(rx_ring, rx_desc, skb, rx_ptype);
1577
1578
1579 vlan_tag = (qword & BIT(I40E_RX_DESC_STATUS_L2TAG1P_SHIFT)) ?
1580 le16_to_cpu(rx_desc->wb.qword0.lo_dword.l2tag1) : 0;
1581
1582 i40e_trace(clean_rx_irq_rx, rx_ring, rx_desc, skb);
1583 i40e_receive_skb(rx_ring, skb, vlan_tag);
1584 skb = NULL;
1585
1586 /* update budget accounting */
1587 total_rx_packets++;
1588 }
1589
1590 rx_ring->skb = skb;
1591
1592 u64_stats_update_begin(&rx_ring->syncp);
1593 rx_ring->stats.packets += total_rx_packets;
1594 rx_ring->stats.bytes += total_rx_bytes;
1595 u64_stats_update_end(&rx_ring->syncp);
1596 rx_ring->q_vector->rx.total_packets += total_rx_packets;
1597 rx_ring->q_vector->rx.total_bytes += total_rx_bytes;
1598
1599 /* guarantee a trip back through this routine if there was a failure */
1600 return failure ? budget : (int)total_rx_packets;
1601}
1602
1603static inline u32 i40e_buildreg_itr(const int type, u16 itr)
1604{
1605 u32 val;
1606
1607 /* We don't bother with setting the CLEARPBA bit as the data sheet
1608 * points out doing so is "meaningless since it was already
1609 * auto-cleared". The auto-clearing happens when the interrupt is
1610 * asserted.
1611 *
1612 * Hardware errata 28 for also indicates that writing to a
1613 * xxINT_DYN_CTLx CSR with INTENA_MSK (bit 31) set to 0 will clear
1614 * an event in the PBA anyway so we need to rely on the automask
1615 * to hold pending events for us until the interrupt is re-enabled
1616 *
1617 * The itr value is reported in microseconds, and the register
1618 * value is recorded in 2 microsecond units. For this reason we
1619 * only need to shift by the interval shift - 1 instead of the
1620 * full value.
1621 */
1622 itr &= I40E_ITR_MASK;
1623
1624 val = I40E_VFINT_DYN_CTLN1_INTENA_MASK |
1625 (type << I40E_VFINT_DYN_CTLN1_ITR_INDX_SHIFT) |
1626 (itr << (I40E_VFINT_DYN_CTLN1_INTERVAL_SHIFT - 1));
1627
1628 return val;
1629}
1630
1631/* a small macro to shorten up some long lines */
1632#define INTREG I40E_VFINT_DYN_CTLN1
1633
1634/* The act of updating the ITR will cause it to immediately trigger. In order
1635 * to prevent this from throwing off adaptive update statistics we defer the
1636 * update so that it can only happen so often. So after either Tx or Rx are
1637 * updated we make the adaptive scheme wait until either the ITR completely
1638 * expires via the next_update expiration or we have been through at least
1639 * 3 interrupts.
1640 */
1641#define ITR_COUNTDOWN_START 3
1642
1643/**
1644 * i40e_update_enable_itr - Update itr and re-enable MSIX interrupt
1645 * @vsi: the VSI we care about
1646 * @q_vector: q_vector for which itr is being updated and interrupt enabled
1647 *
1648 **/
1649static inline void i40e_update_enable_itr(struct i40e_vsi *vsi,
1650 struct i40e_q_vector *q_vector)
1651{
1652 struct i40e_hw *hw = &vsi->back->hw;
1653 u32 intval;
1654
1655 /* These will do nothing if dynamic updates are not enabled */
1656 i40e_update_itr(q_vector, &q_vector->tx);
1657 i40e_update_itr(q_vector, &q_vector->rx);
1658
1659 /* This block of logic allows us to get away with only updating
1660 * one ITR value with each interrupt. The idea is to perform a
1661 * pseudo-lazy update with the following criteria.
1662 *
1663 * 1. Rx is given higher priority than Tx if both are in same state
1664 * 2. If we must reduce an ITR that is given highest priority.
1665 * 3. We then give priority to increasing ITR based on amount.
1666 */
1667 if (q_vector->rx.target_itr < q_vector->rx.current_itr) {
1668 /* Rx ITR needs to be reduced, this is highest priority */
1669 intval = i40e_buildreg_itr(I40E_RX_ITR,
1670 q_vector->rx.target_itr);
1671 q_vector->rx.current_itr = q_vector->rx.target_itr;
1672 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1673 } else if ((q_vector->tx.target_itr < q_vector->tx.current_itr) ||
1674 ((q_vector->rx.target_itr - q_vector->rx.current_itr) <
1675 (q_vector->tx.target_itr - q_vector->tx.current_itr))) {
1676 /* Tx ITR needs to be reduced, this is second priority
1677 * Tx ITR needs to be increased more than Rx, fourth priority
1678 */
1679 intval = i40e_buildreg_itr(I40E_TX_ITR,
1680 q_vector->tx.target_itr);
1681 q_vector->tx.current_itr = q_vector->tx.target_itr;
1682 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1683 } else if (q_vector->rx.current_itr != q_vector->rx.target_itr) {
1684 /* Rx ITR needs to be increased, third priority */
1685 intval = i40e_buildreg_itr(I40E_RX_ITR,
1686 q_vector->rx.target_itr);
1687 q_vector->rx.current_itr = q_vector->rx.target_itr;
1688 q_vector->itr_countdown = ITR_COUNTDOWN_START;
1689 } else {
1690 /* No ITR update, lowest priority */
1691 intval = i40e_buildreg_itr(I40E_ITR_NONE, 0);
1692 if (q_vector->itr_countdown)
1693 q_vector->itr_countdown--;
1694 }
1695
1696 if (!test_bit(__I40E_VSI_DOWN, vsi->state))
1697 wr32(hw, INTREG(q_vector->reg_idx), intval);
1698}
1699
1700/**
1701 * i40evf_napi_poll - NAPI polling Rx/Tx cleanup routine
1702 * @napi: napi struct with our devices info in it
1703 * @budget: amount of work driver is allowed to do this pass, in packets
1704 *
1705 * This function will clean all queues associated with a q_vector.
1706 *
1707 * Returns the amount of work done
1708 **/
1709int i40evf_napi_poll(struct napi_struct *napi, int budget)
1710{
1711 struct i40e_q_vector *q_vector =
1712 container_of(napi, struct i40e_q_vector, napi);
1713 struct i40e_vsi *vsi = q_vector->vsi;
1714 struct i40e_ring *ring;
1715 bool clean_complete = true;
1716 bool arm_wb = false;
1717 int budget_per_ring;
1718 int work_done = 0;
1719
1720 if (test_bit(__I40E_VSI_DOWN, vsi->state)) {
1721 napi_complete(napi);
1722 return 0;
1723 }
1724
1725 /* Since the actual Tx work is minimal, we can give the Tx a larger
1726 * budget and be more aggressive about cleaning up the Tx descriptors.
1727 */
1728 i40e_for_each_ring(ring, q_vector->tx) {
1729 if (!i40e_clean_tx_irq(vsi, ring, budget)) {
1730 clean_complete = false;
1731 continue;
1732 }
1733 arm_wb |= ring->arm_wb;
1734 ring->arm_wb = false;
1735 }
1736
1737 /* Handle case where we are called by netpoll with a budget of 0 */
1738 if (budget <= 0)
1739 goto tx_only;
1740
1741 /* We attempt to distribute budget to each Rx queue fairly, but don't
1742 * allow the budget to go below 1 because that would exit polling early.
1743 */
1744 budget_per_ring = max(budget/q_vector->num_ringpairs, 1);
1745
1746 i40e_for_each_ring(ring, q_vector->rx) {
1747 int cleaned = i40e_clean_rx_irq(ring, budget_per_ring);
1748
1749 work_done += cleaned;
1750 /* if we clean as many as budgeted, we must not be done */
1751 if (cleaned >= budget_per_ring)
1752 clean_complete = false;
1753 }
1754
1755 /* If work not completed, return budget and polling will return */
1756 if (!clean_complete) {
1757 int cpu_id = smp_processor_id();
1758
1759 /* It is possible that the interrupt affinity has changed but,
1760 * if the cpu is pegged at 100%, polling will never exit while
1761 * traffic continues and the interrupt will be stuck on this
1762 * cpu. We check to make sure affinity is correct before we
1763 * continue to poll, otherwise we must stop polling so the
1764 * interrupt can move to the correct cpu.
1765 */
1766 if (!cpumask_test_cpu(cpu_id, &q_vector->affinity_mask)) {
1767 /* Tell napi that we are done polling */
1768 napi_complete_done(napi, work_done);
1769
1770 /* Force an interrupt */
1771 i40evf_force_wb(vsi, q_vector);
1772
1773 /* Return budget-1 so that polling stops */
1774 return budget - 1;
1775 }
1776tx_only:
1777 if (arm_wb) {
1778 q_vector->tx.ring[0].tx_stats.tx_force_wb++;
1779 i40e_enable_wb_on_itr(vsi, q_vector);
1780 }
1781 return budget;
1782 }
1783
1784 if (vsi->back->flags & I40E_TXR_FLAGS_WB_ON_ITR)
1785 q_vector->arm_wb_state = false;
1786
1787 /* Work is done so exit the polling mode and re-enable the interrupt */
1788 napi_complete_done(napi, work_done);
1789
1790 i40e_update_enable_itr(vsi, q_vector);
1791
1792 return min(work_done, budget - 1);
1793}
1794
1795/**
1796 * i40evf_tx_prepare_vlan_flags - prepare generic TX VLAN tagging flags for HW
1797 * @skb: send buffer
1798 * @tx_ring: ring to send buffer on
1799 * @flags: the tx flags to be set
1800 *
1801 * Checks the skb and set up correspondingly several generic transmit flags
1802 * related to VLAN tagging for the HW, such as VLAN, DCB, etc.
1803 *
1804 * Returns error code indicate the frame should be dropped upon error and the
1805 * otherwise returns 0 to indicate the flags has been set properly.
1806 **/
1807static inline int i40evf_tx_prepare_vlan_flags(struct sk_buff *skb,
1808 struct i40e_ring *tx_ring,
1809 u32 *flags)
1810{
1811 __be16 protocol = skb->protocol;
1812 u32 tx_flags = 0;
1813
1814 if (protocol == htons(ETH_P_8021Q) &&
1815 !(tx_ring->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)) {
1816 /* When HW VLAN acceleration is turned off by the user the
1817 * stack sets the protocol to 8021q so that the driver
1818 * can take any steps required to support the SW only
1819 * VLAN handling. In our case the driver doesn't need
1820 * to take any further steps so just set the protocol
1821 * to the encapsulated ethertype.
1822 */
1823 skb->protocol = vlan_get_protocol(skb);
1824 goto out;
1825 }
1826
1827 /* if we have a HW VLAN tag being added, default to the HW one */
1828 if (skb_vlan_tag_present(skb)) {
1829 tx_flags |= skb_vlan_tag_get(skb) << I40E_TX_FLAGS_VLAN_SHIFT;
1830 tx_flags |= I40E_TX_FLAGS_HW_VLAN;
1831 /* else if it is a SW VLAN, check the next protocol and store the tag */
1832 } else if (protocol == htons(ETH_P_8021Q)) {
1833 struct vlan_hdr *vhdr, _vhdr;
1834
1835 vhdr = skb_header_pointer(skb, ETH_HLEN, sizeof(_vhdr), &_vhdr);
1836 if (!vhdr)
1837 return -EINVAL;
1838
1839 protocol = vhdr->h_vlan_encapsulated_proto;
1840 tx_flags |= ntohs(vhdr->h_vlan_TCI) << I40E_TX_FLAGS_VLAN_SHIFT;
1841 tx_flags |= I40E_TX_FLAGS_SW_VLAN;
1842 }
1843
1844out:
1845 *flags = tx_flags;
1846 return 0;
1847}
1848
1849/**
1850 * i40e_tso - set up the tso context descriptor
1851 * @first: pointer to first Tx buffer for xmit
1852 * @hdr_len: ptr to the size of the packet header
1853 * @cd_type_cmd_tso_mss: Quad Word 1
1854 *
1855 * Returns 0 if no TSO can happen, 1 if tso is going, or error
1856 **/
1857static int i40e_tso(struct i40e_tx_buffer *first, u8 *hdr_len,
1858 u64 *cd_type_cmd_tso_mss)
1859{
1860 struct sk_buff *skb = first->skb;
1861 u64 cd_cmd, cd_tso_len, cd_mss;
1862 union {
1863 struct iphdr *v4;
1864 struct ipv6hdr *v6;
1865 unsigned char *hdr;
1866 } ip;
1867 union {
1868 struct tcphdr *tcp;
1869 struct udphdr *udp;
1870 unsigned char *hdr;
1871 } l4;
1872 u32 paylen, l4_offset;
1873 u16 gso_segs, gso_size;
1874 int err;
1875
1876 if (skb->ip_summed != CHECKSUM_PARTIAL)
1877 return 0;
1878
1879 if (!skb_is_gso(skb))
1880 return 0;
1881
1882 err = skb_cow_head(skb, 0);
1883 if (err < 0)
1884 return err;
1885
1886 ip.hdr = skb_network_header(skb);
1887 l4.hdr = skb_transport_header(skb);
1888
1889 /* initialize outer IP header fields */
1890 if (ip.v4->version == 4) {
1891 ip.v4->tot_len = 0;
1892 ip.v4->check = 0;
1893 } else {
1894 ip.v6->payload_len = 0;
1895 }
1896
1897 if (skb_shinfo(skb)->gso_type & (SKB_GSO_GRE |
1898 SKB_GSO_GRE_CSUM |
1899 SKB_GSO_IPXIP4 |
1900 SKB_GSO_IPXIP6 |
1901 SKB_GSO_UDP_TUNNEL |
1902 SKB_GSO_UDP_TUNNEL_CSUM)) {
1903 if (!(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
1904 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM)) {
1905 l4.udp->len = 0;
1906
1907 /* determine offset of outer transport header */
1908 l4_offset = l4.hdr - skb->data;
1909
1910 /* remove payload length from outer checksum */
1911 paylen = skb->len - l4_offset;
1912 csum_replace_by_diff(&l4.udp->check,
1913 (__force __wsum)htonl(paylen));
1914 }
1915
1916 /* reset pointers to inner headers */
1917 ip.hdr = skb_inner_network_header(skb);
1918 l4.hdr = skb_inner_transport_header(skb);
1919
1920 /* initialize inner IP header fields */
1921 if (ip.v4->version == 4) {
1922 ip.v4->tot_len = 0;
1923 ip.v4->check = 0;
1924 } else {
1925 ip.v6->payload_len = 0;
1926 }
1927 }
1928
1929 /* determine offset of inner transport header */
1930 l4_offset = l4.hdr - skb->data;
1931
1932 /* remove payload length from inner checksum */
1933 paylen = skb->len - l4_offset;
1934 csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
1935
1936 /* compute length of segmentation header */
1937 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
1938
1939 /* pull values out of skb_shinfo */
1940 gso_size = skb_shinfo(skb)->gso_size;
1941 gso_segs = skb_shinfo(skb)->gso_segs;
1942
1943 /* update GSO size and bytecount with header size */
1944 first->gso_segs = gso_segs;
1945 first->bytecount += (first->gso_segs - 1) * *hdr_len;
1946
1947 /* find the field values */
1948 cd_cmd = I40E_TX_CTX_DESC_TSO;
1949 cd_tso_len = skb->len - *hdr_len;
1950 cd_mss = gso_size;
1951 *cd_type_cmd_tso_mss |= (cd_cmd << I40E_TXD_CTX_QW1_CMD_SHIFT) |
1952 (cd_tso_len << I40E_TXD_CTX_QW1_TSO_LEN_SHIFT) |
1953 (cd_mss << I40E_TXD_CTX_QW1_MSS_SHIFT);
1954 return 1;
1955}
1956
1957/**
1958 * i40e_tx_enable_csum - Enable Tx checksum offloads
1959 * @skb: send buffer
1960 * @tx_flags: pointer to Tx flags currently set
1961 * @td_cmd: Tx descriptor command bits to set
1962 * @td_offset: Tx descriptor header offsets to set
1963 * @tx_ring: Tx descriptor ring
1964 * @cd_tunneling: ptr to context desc bits
1965 **/
1966static int i40e_tx_enable_csum(struct sk_buff *skb, u32 *tx_flags,
1967 u32 *td_cmd, u32 *td_offset,
1968 struct i40e_ring *tx_ring,
1969 u32 *cd_tunneling)
1970{
1971 union {
1972 struct iphdr *v4;
1973 struct ipv6hdr *v6;
1974 unsigned char *hdr;
1975 } ip;
1976 union {
1977 struct tcphdr *tcp;
1978 struct udphdr *udp;
1979 unsigned char *hdr;
1980 } l4;
1981 unsigned char *exthdr;
1982 u32 offset, cmd = 0;
1983 __be16 frag_off;
1984 u8 l4_proto = 0;
1985
1986 if (skb->ip_summed != CHECKSUM_PARTIAL)
1987 return 0;
1988
1989 ip.hdr = skb_network_header(skb);
1990 l4.hdr = skb_transport_header(skb);
1991
1992 /* compute outer L2 header size */
1993 offset = ((ip.hdr - skb->data) / 2) << I40E_TX_DESC_LENGTH_MACLEN_SHIFT;
1994
1995 if (skb->encapsulation) {
1996 u32 tunnel = 0;
1997 /* define outer network header type */
1998 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
1999 tunnel |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
2000 I40E_TX_CTX_EXT_IP_IPV4 :
2001 I40E_TX_CTX_EXT_IP_IPV4_NO_CSUM;
2002
2003 l4_proto = ip.v4->protocol;
2004 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
2005 tunnel |= I40E_TX_CTX_EXT_IP_IPV6;
2006
2007 exthdr = ip.hdr + sizeof(*ip.v6);
2008 l4_proto = ip.v6->nexthdr;
2009 if (l4.hdr != exthdr)
2010 ipv6_skip_exthdr(skb, exthdr - skb->data,
2011 &l4_proto, &frag_off);
2012 }
2013
2014 /* define outer transport */
2015 switch (l4_proto) {
2016 case IPPROTO_UDP:
2017 tunnel |= I40E_TXD_CTX_UDP_TUNNELING;
2018 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
2019 break;
2020 case IPPROTO_GRE:
2021 tunnel |= I40E_TXD_CTX_GRE_TUNNELING;
2022 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
2023 break;
2024 case IPPROTO_IPIP:
2025 case IPPROTO_IPV6:
2026 *tx_flags |= I40E_TX_FLAGS_VXLAN_TUNNEL;
2027 l4.hdr = skb_inner_network_header(skb);
2028 break;
2029 default:
2030 if (*tx_flags & I40E_TX_FLAGS_TSO)
2031 return -1;
2032
2033 skb_checksum_help(skb);
2034 return 0;
2035 }
2036
2037 /* compute outer L3 header size */
2038 tunnel |= ((l4.hdr - ip.hdr) / 4) <<
2039 I40E_TXD_CTX_QW0_EXT_IPLEN_SHIFT;
2040
2041 /* switch IP header pointer from outer to inner header */
2042 ip.hdr = skb_inner_network_header(skb);
2043
2044 /* compute tunnel header size */
2045 tunnel |= ((ip.hdr - l4.hdr) / 2) <<
2046 I40E_TXD_CTX_QW0_NATLEN_SHIFT;
2047
2048 /* indicate if we need to offload outer UDP header */
2049 if ((*tx_flags & I40E_TX_FLAGS_TSO) &&
2050 !(skb_shinfo(skb)->gso_type & SKB_GSO_PARTIAL) &&
2051 (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_TUNNEL_CSUM))
2052 tunnel |= I40E_TXD_CTX_QW0_L4T_CS_MASK;
2053
2054 /* record tunnel offload values */
2055 *cd_tunneling |= tunnel;
2056
2057 /* switch L4 header pointer from outer to inner */
2058 l4.hdr = skb_inner_transport_header(skb);
2059 l4_proto = 0;
2060
2061 /* reset type as we transition from outer to inner headers */
2062 *tx_flags &= ~(I40E_TX_FLAGS_IPV4 | I40E_TX_FLAGS_IPV6);
2063 if (ip.v4->version == 4)
2064 *tx_flags |= I40E_TX_FLAGS_IPV4;
2065 if (ip.v6->version == 6)
2066 *tx_flags |= I40E_TX_FLAGS_IPV6;
2067 }
2068
2069 /* Enable IP checksum offloads */
2070 if (*tx_flags & I40E_TX_FLAGS_IPV4) {
2071 l4_proto = ip.v4->protocol;
2072 /* the stack computes the IP header already, the only time we
2073 * need the hardware to recompute it is in the case of TSO.
2074 */
2075 cmd |= (*tx_flags & I40E_TX_FLAGS_TSO) ?
2076 I40E_TX_DESC_CMD_IIPT_IPV4_CSUM :
2077 I40E_TX_DESC_CMD_IIPT_IPV4;
2078 } else if (*tx_flags & I40E_TX_FLAGS_IPV6) {
2079 cmd |= I40E_TX_DESC_CMD_IIPT_IPV6;
2080
2081 exthdr = ip.hdr + sizeof(*ip.v6);
2082 l4_proto = ip.v6->nexthdr;
2083 if (l4.hdr != exthdr)
2084 ipv6_skip_exthdr(skb, exthdr - skb->data,
2085 &l4_proto, &frag_off);
2086 }
2087
2088 /* compute inner L3 header size */
2089 offset |= ((l4.hdr - ip.hdr) / 4) << I40E_TX_DESC_LENGTH_IPLEN_SHIFT;
2090
2091 /* Enable L4 checksum offloads */
2092 switch (l4_proto) {
2093 case IPPROTO_TCP:
2094 /* enable checksum offloads */
2095 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_TCP;
2096 offset |= l4.tcp->doff << I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2097 break;
2098 case IPPROTO_SCTP:
2099 /* enable SCTP checksum offload */
2100 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_SCTP;
2101 offset |= (sizeof(struct sctphdr) >> 2) <<
2102 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2103 break;
2104 case IPPROTO_UDP:
2105 /* enable UDP checksum offload */
2106 cmd |= I40E_TX_DESC_CMD_L4T_EOFT_UDP;
2107 offset |= (sizeof(struct udphdr) >> 2) <<
2108 I40E_TX_DESC_LENGTH_L4_FC_LEN_SHIFT;
2109 break;
2110 default:
2111 if (*tx_flags & I40E_TX_FLAGS_TSO)
2112 return -1;
2113 skb_checksum_help(skb);
2114 return 0;
2115 }
2116
2117 *td_cmd |= cmd;
2118 *td_offset |= offset;
2119
2120 return 1;
2121}
2122
2123/**
2124 * i40e_create_tx_ctx Build the Tx context descriptor
2125 * @tx_ring: ring to create the descriptor on
2126 * @cd_type_cmd_tso_mss: Quad Word 1
2127 * @cd_tunneling: Quad Word 0 - bits 0-31
2128 * @cd_l2tag2: Quad Word 0 - bits 32-63
2129 **/
2130static void i40e_create_tx_ctx(struct i40e_ring *tx_ring,
2131 const u64 cd_type_cmd_tso_mss,
2132 const u32 cd_tunneling, const u32 cd_l2tag2)
2133{
2134 struct i40e_tx_context_desc *context_desc;
2135 int i = tx_ring->next_to_use;
2136
2137 if ((cd_type_cmd_tso_mss == I40E_TX_DESC_DTYPE_CONTEXT) &&
2138 !cd_tunneling && !cd_l2tag2)
2139 return;
2140
2141 /* grab the next descriptor */
2142 context_desc = I40E_TX_CTXTDESC(tx_ring, i);
2143
2144 i++;
2145 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
2146
2147 /* cpu_to_le32 and assign to struct fields */
2148 context_desc->tunneling_params = cpu_to_le32(cd_tunneling);
2149 context_desc->l2tag2 = cpu_to_le16(cd_l2tag2);
2150 context_desc->rsvd = cpu_to_le16(0);
2151 context_desc->type_cmd_tso_mss = cpu_to_le64(cd_type_cmd_tso_mss);
2152}
2153
2154/**
2155 * __i40evf_chk_linearize - Check if there are more than 8 buffers per packet
2156 * @skb: send buffer
2157 *
2158 * Note: Our HW can't DMA more than 8 buffers to build a packet on the wire
2159 * and so we need to figure out the cases where we need to linearize the skb.
2160 *
2161 * For TSO we need to count the TSO header and segment payload separately.
2162 * As such we need to check cases where we have 7 fragments or more as we
2163 * can potentially require 9 DMA transactions, 1 for the TSO header, 1 for
2164 * the segment payload in the first descriptor, and another 7 for the
2165 * fragments.
2166 **/
2167bool __i40evf_chk_linearize(struct sk_buff *skb)
2168{
2169 const struct skb_frag_struct *frag, *stale;
2170 int nr_frags, sum;
2171
2172 /* no need to check if number of frags is less than 7 */
2173 nr_frags = skb_shinfo(skb)->nr_frags;
2174 if (nr_frags < (I40E_MAX_BUFFER_TXD - 1))
2175 return false;
2176
2177 /* We need to walk through the list and validate that each group
2178 * of 6 fragments totals at least gso_size.
2179 */
2180 nr_frags -= I40E_MAX_BUFFER_TXD - 2;
2181 frag = &skb_shinfo(skb)->frags[0];
2182
2183 /* Initialize size to the negative value of gso_size minus 1. We
2184 * use this as the worst case scenerio in which the frag ahead
2185 * of us only provides one byte which is why we are limited to 6
2186 * descriptors for a single transmit as the header and previous
2187 * fragment are already consuming 2 descriptors.
2188 */
2189 sum = 1 - skb_shinfo(skb)->gso_size;
2190
2191 /* Add size of frags 0 through 4 to create our initial sum */
2192 sum += skb_frag_size(frag++);
2193 sum += skb_frag_size(frag++);
2194 sum += skb_frag_size(frag++);
2195 sum += skb_frag_size(frag++);
2196 sum += skb_frag_size(frag++);
2197
2198 /* Walk through fragments adding latest fragment, testing it, and
2199 * then removing stale fragments from the sum.
2200 */
2201 for (stale = &skb_shinfo(skb)->frags[0];; stale++) {
2202 int stale_size = skb_frag_size(stale);
2203
2204 sum += skb_frag_size(frag++);
2205
2206 /* The stale fragment may present us with a smaller
2207 * descriptor than the actual fragment size. To account
2208 * for that we need to remove all the data on the front and
2209 * figure out what the remainder would be in the last
2210 * descriptor associated with the fragment.
2211 */
2212 if (stale_size > I40E_MAX_DATA_PER_TXD) {
2213 int align_pad = -(stale->page_offset) &
2214 (I40E_MAX_READ_REQ_SIZE - 1);
2215
2216 sum -= align_pad;
2217 stale_size -= align_pad;
2218
2219 do {
2220 sum -= I40E_MAX_DATA_PER_TXD_ALIGNED;
2221 stale_size -= I40E_MAX_DATA_PER_TXD_ALIGNED;
2222 } while (stale_size > I40E_MAX_DATA_PER_TXD);
2223 }
2224
2225 /* if sum is negative we failed to make sufficient progress */
2226 if (sum < 0)
2227 return true;
2228
2229 if (!nr_frags--)
2230 break;
2231
2232 sum -= stale_size;
2233 }
2234
2235 return false;
2236}
2237
2238/**
2239 * __i40evf_maybe_stop_tx - 2nd level check for tx stop conditions
2240 * @tx_ring: the ring to be checked
2241 * @size: the size buffer we want to assure is available
2242 *
2243 * Returns -EBUSY if a stop is needed, else 0
2244 **/
2245int __i40evf_maybe_stop_tx(struct i40e_ring *tx_ring, int size)
2246{
2247 netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
2248 /* Memory barrier before checking head and tail */
2249 smp_mb();
2250
2251 /* Check again in a case another CPU has just made room available. */
2252 if (likely(I40E_DESC_UNUSED(tx_ring) < size))
2253 return -EBUSY;
2254
2255 /* A reprieve! - use start_queue because it doesn't call schedule */
2256 netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
2257 ++tx_ring->tx_stats.restart_queue;
2258 return 0;
2259}
2260
2261/**
2262 * i40evf_tx_map - Build the Tx descriptor
2263 * @tx_ring: ring to send buffer on
2264 * @skb: send buffer
2265 * @first: first buffer info buffer to use
2266 * @tx_flags: collected send information
2267 * @hdr_len: size of the packet header
2268 * @td_cmd: the command field in the descriptor
2269 * @td_offset: offset for checksum or crc
2270 **/
2271static inline void i40evf_tx_map(struct i40e_ring *tx_ring, struct sk_buff *skb,
2272 struct i40e_tx_buffer *first, u32 tx_flags,
2273 const u8 hdr_len, u32 td_cmd, u32 td_offset)
2274{
2275 unsigned int data_len = skb->data_len;
2276 unsigned int size = skb_headlen(skb);
2277 struct skb_frag_struct *frag;
2278 struct i40e_tx_buffer *tx_bi;
2279 struct i40e_tx_desc *tx_desc;
2280 u16 i = tx_ring->next_to_use;
2281 u32 td_tag = 0;
2282 dma_addr_t dma;
2283
2284 if (tx_flags & I40E_TX_FLAGS_HW_VLAN) {
2285 td_cmd |= I40E_TX_DESC_CMD_IL2TAG1;
2286 td_tag = (tx_flags & I40E_TX_FLAGS_VLAN_MASK) >>
2287 I40E_TX_FLAGS_VLAN_SHIFT;
2288 }
2289
2290 first->tx_flags = tx_flags;
2291
2292 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
2293
2294 tx_desc = I40E_TX_DESC(tx_ring, i);
2295 tx_bi = first;
2296
2297 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
2298 unsigned int max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2299
2300 if (dma_mapping_error(tx_ring->dev, dma))
2301 goto dma_error;
2302
2303 /* record length, and DMA address */
2304 dma_unmap_len_set(tx_bi, len, size);
2305 dma_unmap_addr_set(tx_bi, dma, dma);
2306
2307 /* align size to end of page */
2308 max_data += -dma & (I40E_MAX_READ_REQ_SIZE - 1);
2309 tx_desc->buffer_addr = cpu_to_le64(dma);
2310
2311 while (unlikely(size > I40E_MAX_DATA_PER_TXD)) {
2312 tx_desc->cmd_type_offset_bsz =
2313 build_ctob(td_cmd, td_offset,
2314 max_data, td_tag);
2315
2316 tx_desc++;
2317 i++;
2318
2319 if (i == tx_ring->count) {
2320 tx_desc = I40E_TX_DESC(tx_ring, 0);
2321 i = 0;
2322 }
2323
2324 dma += max_data;
2325 size -= max_data;
2326
2327 max_data = I40E_MAX_DATA_PER_TXD_ALIGNED;
2328 tx_desc->buffer_addr = cpu_to_le64(dma);
2329 }
2330
2331 if (likely(!data_len))
2332 break;
2333
2334 tx_desc->cmd_type_offset_bsz = build_ctob(td_cmd, td_offset,
2335 size, td_tag);
2336
2337 tx_desc++;
2338 i++;
2339
2340 if (i == tx_ring->count) {
2341 tx_desc = I40E_TX_DESC(tx_ring, 0);
2342 i = 0;
2343 }
2344
2345 size = skb_frag_size(frag);
2346 data_len -= size;
2347
2348 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
2349 DMA_TO_DEVICE);
2350
2351 tx_bi = &tx_ring->tx_bi[i];
2352 }
2353
2354 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
2355
2356 i++;
2357 if (i == tx_ring->count)
2358 i = 0;
2359
2360 tx_ring->next_to_use = i;
2361
2362 i40e_maybe_stop_tx(tx_ring, DESC_NEEDED);
2363
2364 /* write last descriptor with RS and EOP bits */
2365 td_cmd |= I40E_TXD_CMD;
2366 tx_desc->cmd_type_offset_bsz =
2367 build_ctob(td_cmd, td_offset, size, td_tag);
2368
2369 /* Force memory writes to complete before letting h/w know there
2370 * are new descriptors to fetch.
2371 *
2372 * We also use this memory barrier to make certain all of the
2373 * status bits have been updated before next_to_watch is written.
2374 */
2375 wmb();
2376
2377 /* set next_to_watch value indicating a packet is present */
2378 first->next_to_watch = tx_desc;
2379
2380 /* notify HW of packet */
2381 if (netif_xmit_stopped(txring_txq(tx_ring)) || !skb->xmit_more) {
2382 writel(i, tx_ring->tail);
2383
2384 /* we need this if more than one processor can write to our tail
2385 * at a time, it synchronizes IO on IA64/Altix systems
2386 */
2387 mmiowb();
2388 }
2389
2390 return;
2391
2392dma_error:
2393 dev_info(tx_ring->dev, "TX DMA map failed\n");
2394
2395 /* clear dma mappings for failed tx_bi map */
2396 for (;;) {
2397 tx_bi = &tx_ring->tx_bi[i];
2398 i40e_unmap_and_free_tx_resource(tx_ring, tx_bi);
2399 if (tx_bi == first)
2400 break;
2401 if (i == 0)
2402 i = tx_ring->count;
2403 i--;
2404 }
2405
2406 tx_ring->next_to_use = i;
2407}
2408
2409/**
2410 * i40e_xmit_frame_ring - Sends buffer on Tx ring
2411 * @skb: send buffer
2412 * @tx_ring: ring to send buffer on
2413 *
2414 * Returns NETDEV_TX_OK if sent, else an error code
2415 **/
2416static netdev_tx_t i40e_xmit_frame_ring(struct sk_buff *skb,
2417 struct i40e_ring *tx_ring)
2418{
2419 u64 cd_type_cmd_tso_mss = I40E_TX_DESC_DTYPE_CONTEXT;
2420 u32 cd_tunneling = 0, cd_l2tag2 = 0;
2421 struct i40e_tx_buffer *first;
2422 u32 td_offset = 0;
2423 u32 tx_flags = 0;
2424 __be16 protocol;
2425 u32 td_cmd = 0;
2426 u8 hdr_len = 0;
2427 int tso, count;
2428
2429 /* prefetch the data, we'll need it later */
2430 prefetch(skb->data);
2431
2432 i40e_trace(xmit_frame_ring, skb, tx_ring);
2433
2434 count = i40e_xmit_descriptor_count(skb);
2435 if (i40e_chk_linearize(skb, count)) {
2436 if (__skb_linearize(skb)) {
2437 dev_kfree_skb_any(skb);
2438 return NETDEV_TX_OK;
2439 }
2440 count = i40e_txd_use_count(skb->len);
2441 tx_ring->tx_stats.tx_linearize++;
2442 }
2443
2444 /* need: 1 descriptor per page * PAGE_SIZE/I40E_MAX_DATA_PER_TXD,
2445 * + 1 desc for skb_head_len/I40E_MAX_DATA_PER_TXD,
2446 * + 4 desc gap to avoid the cache line where head is,
2447 * + 1 desc for context descriptor,
2448 * otherwise try next time
2449 */
2450 if (i40e_maybe_stop_tx(tx_ring, count + 4 + 1)) {
2451 tx_ring->tx_stats.tx_busy++;
2452 return NETDEV_TX_BUSY;
2453 }
2454
2455 /* record the location of the first descriptor for this packet */
2456 first = &tx_ring->tx_bi[tx_ring->next_to_use];
2457 first->skb = skb;
2458 first->bytecount = skb->len;
2459 first->gso_segs = 1;
2460
2461 /* prepare the xmit flags */
2462 if (i40evf_tx_prepare_vlan_flags(skb, tx_ring, &tx_flags))
2463 goto out_drop;
2464
2465 /* obtain protocol of skb */
2466 protocol = vlan_get_protocol(skb);
2467
2468 /* setup IPv4/IPv6 offloads */
2469 if (protocol == htons(ETH_P_IP))
2470 tx_flags |= I40E_TX_FLAGS_IPV4;
2471 else if (protocol == htons(ETH_P_IPV6))
2472 tx_flags |= I40E_TX_FLAGS_IPV6;
2473
2474 tso = i40e_tso(first, &hdr_len, &cd_type_cmd_tso_mss);
2475
2476 if (tso < 0)
2477 goto out_drop;
2478 else if (tso)
2479 tx_flags |= I40E_TX_FLAGS_TSO;
2480
2481 /* Always offload the checksum, since it's in the data descriptor */
2482 tso = i40e_tx_enable_csum(skb, &tx_flags, &td_cmd, &td_offset,
2483 tx_ring, &cd_tunneling);
2484 if (tso < 0)
2485 goto out_drop;
2486
2487 skb_tx_timestamp(skb);
2488
2489 /* always enable CRC insertion offload */
2490 td_cmd |= I40E_TX_DESC_CMD_ICRC;
2491
2492 i40e_create_tx_ctx(tx_ring, cd_type_cmd_tso_mss,
2493 cd_tunneling, cd_l2tag2);
2494
2495 i40evf_tx_map(tx_ring, skb, first, tx_flags, hdr_len,
2496 td_cmd, td_offset);
2497
2498 return NETDEV_TX_OK;
2499
2500out_drop:
2501 i40e_trace(xmit_frame_ring_drop, first->skb, tx_ring);
2502 dev_kfree_skb_any(first->skb);
2503 first->skb = NULL;
2504 return NETDEV_TX_OK;
2505}
2506
2507/**
2508 * i40evf_xmit_frame - Selects the correct VSI and Tx queue to send buffer
2509 * @skb: send buffer
2510 * @netdev: network interface device structure
2511 *
2512 * Returns NETDEV_TX_OK if sent, else an error code
2513 **/
2514netdev_tx_t i40evf_xmit_frame(struct sk_buff *skb, struct net_device *netdev)
2515{
2516 struct i40evf_adapter *adapter = netdev_priv(netdev);
2517 struct i40e_ring *tx_ring = &adapter->tx_rings[skb->queue_mapping];
2518
2519 /* hardware can't handle really short frames, hardware padding works
2520 * beyond this point
2521 */
2522 if (unlikely(skb->len < I40E_MIN_TX_LEN)) {
2523 if (skb_pad(skb, I40E_MIN_TX_LEN - skb->len))
2524 return NETDEV_TX_OK;
2525 skb->len = I40E_MIN_TX_LEN;
2526 skb_set_tail_pointer(skb, I40E_MIN_TX_LEN);
2527 }
2528
2529 return i40e_xmit_frame_ring(skb, tx_ring);
2530}