Loading...
Note: File does not exist in v3.5.6.
1/****************************************************************************
2 * Driver for Solarflare Solarstorm network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2010 Solarflare Communications Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/pci.h>
12#include <linux/tcp.h>
13#include <linux/ip.h>
14#include <linux/in.h>
15#include <linux/ipv6.h>
16#include <linux/slab.h>
17#include <net/ipv6.h>
18#include <linux/if_ether.h>
19#include <linux/highmem.h>
20#include "net_driver.h"
21#include "efx.h"
22#include "nic.h"
23#include "workarounds.h"
24
25/*
26 * TX descriptor ring full threshold
27 *
28 * The tx_queue descriptor ring fill-level must fall below this value
29 * before we restart the netif queue
30 */
31#define EFX_TXQ_THRESHOLD(_efx) ((_efx)->txq_entries / 2u)
32
33static void efx_dequeue_buffer(struct efx_tx_queue *tx_queue,
34 struct efx_tx_buffer *buffer)
35{
36 if (buffer->unmap_len) {
37 struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
38 dma_addr_t unmap_addr = (buffer->dma_addr + buffer->len -
39 buffer->unmap_len);
40 if (buffer->unmap_single)
41 pci_unmap_single(pci_dev, unmap_addr, buffer->unmap_len,
42 PCI_DMA_TODEVICE);
43 else
44 pci_unmap_page(pci_dev, unmap_addr, buffer->unmap_len,
45 PCI_DMA_TODEVICE);
46 buffer->unmap_len = 0;
47 buffer->unmap_single = false;
48 }
49
50 if (buffer->skb) {
51 dev_kfree_skb_any((struct sk_buff *) buffer->skb);
52 buffer->skb = NULL;
53 netif_vdbg(tx_queue->efx, tx_done, tx_queue->efx->net_dev,
54 "TX queue %d transmission id %x complete\n",
55 tx_queue->queue, tx_queue->read_count);
56 }
57}
58
59/**
60 * struct efx_tso_header - a DMA mapped buffer for packet headers
61 * @next: Linked list of free ones.
62 * The list is protected by the TX queue lock.
63 * @dma_unmap_len: Length to unmap for an oversize buffer, or 0.
64 * @dma_addr: The DMA address of the header below.
65 *
66 * This controls the memory used for a TSO header. Use TSOH_DATA()
67 * to find the packet header data. Use TSOH_SIZE() to calculate the
68 * total size required for a given packet header length. TSO headers
69 * in the free list are exactly %TSOH_STD_SIZE bytes in size.
70 */
71struct efx_tso_header {
72 union {
73 struct efx_tso_header *next;
74 size_t unmap_len;
75 };
76 dma_addr_t dma_addr;
77};
78
79static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
80 struct sk_buff *skb);
81static void efx_fini_tso(struct efx_tx_queue *tx_queue);
82static void efx_tsoh_heap_free(struct efx_tx_queue *tx_queue,
83 struct efx_tso_header *tsoh);
84
85static void efx_tsoh_free(struct efx_tx_queue *tx_queue,
86 struct efx_tx_buffer *buffer)
87{
88 if (buffer->tsoh) {
89 if (likely(!buffer->tsoh->unmap_len)) {
90 buffer->tsoh->next = tx_queue->tso_headers_free;
91 tx_queue->tso_headers_free = buffer->tsoh;
92 } else {
93 efx_tsoh_heap_free(tx_queue, buffer->tsoh);
94 }
95 buffer->tsoh = NULL;
96 }
97}
98
99
100static inline unsigned
101efx_max_tx_len(struct efx_nic *efx, dma_addr_t dma_addr)
102{
103 /* Depending on the NIC revision, we can use descriptor
104 * lengths up to 8K or 8K-1. However, since PCI Express
105 * devices must split read requests at 4K boundaries, there is
106 * little benefit from using descriptors that cross those
107 * boundaries and we keep things simple by not doing so.
108 */
109 unsigned len = (~dma_addr & 0xfff) + 1;
110
111 /* Work around hardware bug for unaligned buffers. */
112 if (EFX_WORKAROUND_5391(efx) && (dma_addr & 0xf))
113 len = min_t(unsigned, len, 512 - (dma_addr & 0xf));
114
115 return len;
116}
117
118/*
119 * Add a socket buffer to a TX queue
120 *
121 * This maps all fragments of a socket buffer for DMA and adds them to
122 * the TX queue. The queue's insert pointer will be incremented by
123 * the number of fragments in the socket buffer.
124 *
125 * If any DMA mapping fails, any mapped fragments will be unmapped,
126 * the queue's insert pointer will be restored to its original value.
127 *
128 * This function is split out from efx_hard_start_xmit to allow the
129 * loopback test to direct packets via specific TX queues.
130 *
131 * Returns NETDEV_TX_OK or NETDEV_TX_BUSY
132 * You must hold netif_tx_lock() to call this function.
133 */
134netdev_tx_t efx_enqueue_skb(struct efx_tx_queue *tx_queue, struct sk_buff *skb)
135{
136 struct efx_nic *efx = tx_queue->efx;
137 struct pci_dev *pci_dev = efx->pci_dev;
138 struct efx_tx_buffer *buffer;
139 skb_frag_t *fragment;
140 struct page *page;
141 int page_offset;
142 unsigned int len, unmap_len = 0, fill_level, insert_ptr;
143 dma_addr_t dma_addr, unmap_addr = 0;
144 unsigned int dma_len;
145 bool unmap_single;
146 int q_space, i = 0;
147 netdev_tx_t rc = NETDEV_TX_OK;
148
149 EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
150
151 if (skb_shinfo(skb)->gso_size)
152 return efx_enqueue_skb_tso(tx_queue, skb);
153
154 /* Get size of the initial fragment */
155 len = skb_headlen(skb);
156
157 /* Pad if necessary */
158 if (EFX_WORKAROUND_15592(efx) && skb->len <= 32) {
159 EFX_BUG_ON_PARANOID(skb->data_len);
160 len = 32 + 1;
161 if (skb_pad(skb, len - skb->len))
162 return NETDEV_TX_OK;
163 }
164
165 fill_level = tx_queue->insert_count - tx_queue->old_read_count;
166 q_space = efx->txq_entries - 1 - fill_level;
167
168 /* Map for DMA. Use pci_map_single rather than pci_map_page
169 * since this is more efficient on machines with sparse
170 * memory.
171 */
172 unmap_single = true;
173 dma_addr = pci_map_single(pci_dev, skb->data, len, PCI_DMA_TODEVICE);
174
175 /* Process all fragments */
176 while (1) {
177 if (unlikely(pci_dma_mapping_error(pci_dev, dma_addr)))
178 goto pci_err;
179
180 /* Store fields for marking in the per-fragment final
181 * descriptor */
182 unmap_len = len;
183 unmap_addr = dma_addr;
184
185 /* Add to TX queue, splitting across DMA boundaries */
186 do {
187 if (unlikely(q_space-- <= 0)) {
188 /* It might be that completions have
189 * happened since the xmit path last
190 * checked. Update the xmit path's
191 * copy of read_count.
192 */
193 netif_tx_stop_queue(tx_queue->core_txq);
194 /* This memory barrier protects the
195 * change of queue state from the access
196 * of read_count. */
197 smp_mb();
198 tx_queue->old_read_count =
199 ACCESS_ONCE(tx_queue->read_count);
200 fill_level = (tx_queue->insert_count
201 - tx_queue->old_read_count);
202 q_space = efx->txq_entries - 1 - fill_level;
203 if (unlikely(q_space-- <= 0)) {
204 rc = NETDEV_TX_BUSY;
205 goto unwind;
206 }
207 smp_mb();
208 if (likely(!efx->loopback_selftest))
209 netif_tx_start_queue(
210 tx_queue->core_txq);
211 }
212
213 insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
214 buffer = &tx_queue->buffer[insert_ptr];
215 efx_tsoh_free(tx_queue, buffer);
216 EFX_BUG_ON_PARANOID(buffer->tsoh);
217 EFX_BUG_ON_PARANOID(buffer->skb);
218 EFX_BUG_ON_PARANOID(buffer->len);
219 EFX_BUG_ON_PARANOID(!buffer->continuation);
220 EFX_BUG_ON_PARANOID(buffer->unmap_len);
221
222 dma_len = efx_max_tx_len(efx, dma_addr);
223 if (likely(dma_len >= len))
224 dma_len = len;
225
226 /* Fill out per descriptor fields */
227 buffer->len = dma_len;
228 buffer->dma_addr = dma_addr;
229 len -= dma_len;
230 dma_addr += dma_len;
231 ++tx_queue->insert_count;
232 } while (len);
233
234 /* Transfer ownership of the unmapping to the final buffer */
235 buffer->unmap_single = unmap_single;
236 buffer->unmap_len = unmap_len;
237 unmap_len = 0;
238
239 /* Get address and size of next fragment */
240 if (i >= skb_shinfo(skb)->nr_frags)
241 break;
242 fragment = &skb_shinfo(skb)->frags[i];
243 len = fragment->size;
244 page = fragment->page;
245 page_offset = fragment->page_offset;
246 i++;
247 /* Map for DMA */
248 unmap_single = false;
249 dma_addr = pci_map_page(pci_dev, page, page_offset, len,
250 PCI_DMA_TODEVICE);
251 }
252
253 /* Transfer ownership of the skb to the final buffer */
254 buffer->skb = skb;
255 buffer->continuation = false;
256
257 /* Pass off to hardware */
258 efx_nic_push_buffers(tx_queue);
259
260 return NETDEV_TX_OK;
261
262 pci_err:
263 netif_err(efx, tx_err, efx->net_dev,
264 " TX queue %d could not map skb with %d bytes %d "
265 "fragments for DMA\n", tx_queue->queue, skb->len,
266 skb_shinfo(skb)->nr_frags + 1);
267
268 /* Mark the packet as transmitted, and free the SKB ourselves */
269 dev_kfree_skb_any(skb);
270
271 unwind:
272 /* Work backwards until we hit the original insert pointer value */
273 while (tx_queue->insert_count != tx_queue->write_count) {
274 --tx_queue->insert_count;
275 insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
276 buffer = &tx_queue->buffer[insert_ptr];
277 efx_dequeue_buffer(tx_queue, buffer);
278 buffer->len = 0;
279 }
280
281 /* Free the fragment we were mid-way through pushing */
282 if (unmap_len) {
283 if (unmap_single)
284 pci_unmap_single(pci_dev, unmap_addr, unmap_len,
285 PCI_DMA_TODEVICE);
286 else
287 pci_unmap_page(pci_dev, unmap_addr, unmap_len,
288 PCI_DMA_TODEVICE);
289 }
290
291 return rc;
292}
293
294/* Remove packets from the TX queue
295 *
296 * This removes packets from the TX queue, up to and including the
297 * specified index.
298 */
299static void efx_dequeue_buffers(struct efx_tx_queue *tx_queue,
300 unsigned int index)
301{
302 struct efx_nic *efx = tx_queue->efx;
303 unsigned int stop_index, read_ptr;
304
305 stop_index = (index + 1) & tx_queue->ptr_mask;
306 read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
307
308 while (read_ptr != stop_index) {
309 struct efx_tx_buffer *buffer = &tx_queue->buffer[read_ptr];
310 if (unlikely(buffer->len == 0)) {
311 netif_err(efx, tx_err, efx->net_dev,
312 "TX queue %d spurious TX completion id %x\n",
313 tx_queue->queue, read_ptr);
314 efx_schedule_reset(efx, RESET_TYPE_TX_SKIP);
315 return;
316 }
317
318 efx_dequeue_buffer(tx_queue, buffer);
319 buffer->continuation = true;
320 buffer->len = 0;
321
322 ++tx_queue->read_count;
323 read_ptr = tx_queue->read_count & tx_queue->ptr_mask;
324 }
325}
326
327/* Initiate a packet transmission. We use one channel per CPU
328 * (sharing when we have more CPUs than channels). On Falcon, the TX
329 * completion events will be directed back to the CPU that transmitted
330 * the packet, which should be cache-efficient.
331 *
332 * Context: non-blocking.
333 * Note that returning anything other than NETDEV_TX_OK will cause the
334 * OS to free the skb.
335 */
336netdev_tx_t efx_hard_start_xmit(struct sk_buff *skb,
337 struct net_device *net_dev)
338{
339 struct efx_nic *efx = netdev_priv(net_dev);
340 struct efx_tx_queue *tx_queue;
341 unsigned index, type;
342
343 EFX_WARN_ON_PARANOID(!netif_device_present(net_dev));
344
345 index = skb_get_queue_mapping(skb);
346 type = skb->ip_summed == CHECKSUM_PARTIAL ? EFX_TXQ_TYPE_OFFLOAD : 0;
347 if (index >= efx->n_tx_channels) {
348 index -= efx->n_tx_channels;
349 type |= EFX_TXQ_TYPE_HIGHPRI;
350 }
351 tx_queue = efx_get_tx_queue(efx, index, type);
352
353 return efx_enqueue_skb(tx_queue, skb);
354}
355
356void efx_init_tx_queue_core_txq(struct efx_tx_queue *tx_queue)
357{
358 struct efx_nic *efx = tx_queue->efx;
359
360 /* Must be inverse of queue lookup in efx_hard_start_xmit() */
361 tx_queue->core_txq =
362 netdev_get_tx_queue(efx->net_dev,
363 tx_queue->queue / EFX_TXQ_TYPES +
364 ((tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI) ?
365 efx->n_tx_channels : 0));
366}
367
368int efx_setup_tc(struct net_device *net_dev, u8 num_tc)
369{
370 struct efx_nic *efx = netdev_priv(net_dev);
371 struct efx_channel *channel;
372 struct efx_tx_queue *tx_queue;
373 unsigned tc;
374 int rc;
375
376 if (efx_nic_rev(efx) < EFX_REV_FALCON_B0 || num_tc > EFX_MAX_TX_TC)
377 return -EINVAL;
378
379 if (num_tc == net_dev->num_tc)
380 return 0;
381
382 for (tc = 0; tc < num_tc; tc++) {
383 net_dev->tc_to_txq[tc].offset = tc * efx->n_tx_channels;
384 net_dev->tc_to_txq[tc].count = efx->n_tx_channels;
385 }
386
387 if (num_tc > net_dev->num_tc) {
388 /* Initialise high-priority queues as necessary */
389 efx_for_each_channel(channel, efx) {
390 efx_for_each_possible_channel_tx_queue(tx_queue,
391 channel) {
392 if (!(tx_queue->queue & EFX_TXQ_TYPE_HIGHPRI))
393 continue;
394 if (!tx_queue->buffer) {
395 rc = efx_probe_tx_queue(tx_queue);
396 if (rc)
397 return rc;
398 }
399 if (!tx_queue->initialised)
400 efx_init_tx_queue(tx_queue);
401 efx_init_tx_queue_core_txq(tx_queue);
402 }
403 }
404 } else {
405 /* Reduce number of classes before number of queues */
406 net_dev->num_tc = num_tc;
407 }
408
409 rc = netif_set_real_num_tx_queues(net_dev,
410 max_t(int, num_tc, 1) *
411 efx->n_tx_channels);
412 if (rc)
413 return rc;
414
415 /* Do not destroy high-priority queues when they become
416 * unused. We would have to flush them first, and it is
417 * fairly difficult to flush a subset of TX queues. Leave
418 * it to efx_fini_channels().
419 */
420
421 net_dev->num_tc = num_tc;
422 return 0;
423}
424
425void efx_xmit_done(struct efx_tx_queue *tx_queue, unsigned int index)
426{
427 unsigned fill_level;
428 struct efx_nic *efx = tx_queue->efx;
429
430 EFX_BUG_ON_PARANOID(index > tx_queue->ptr_mask);
431
432 efx_dequeue_buffers(tx_queue, index);
433
434 /* See if we need to restart the netif queue. This barrier
435 * separates the update of read_count from the test of the
436 * queue state. */
437 smp_mb();
438 if (unlikely(netif_tx_queue_stopped(tx_queue->core_txq)) &&
439 likely(efx->port_enabled) &&
440 likely(netif_device_present(efx->net_dev))) {
441 fill_level = tx_queue->insert_count - tx_queue->read_count;
442 if (fill_level < EFX_TXQ_THRESHOLD(efx)) {
443 EFX_BUG_ON_PARANOID(!efx_dev_registered(efx));
444 netif_tx_wake_queue(tx_queue->core_txq);
445 }
446 }
447
448 /* Check whether the hardware queue is now empty */
449 if ((int)(tx_queue->read_count - tx_queue->old_write_count) >= 0) {
450 tx_queue->old_write_count = ACCESS_ONCE(tx_queue->write_count);
451 if (tx_queue->read_count == tx_queue->old_write_count) {
452 smp_mb();
453 tx_queue->empty_read_count =
454 tx_queue->read_count | EFX_EMPTY_COUNT_VALID;
455 }
456 }
457}
458
459int efx_probe_tx_queue(struct efx_tx_queue *tx_queue)
460{
461 struct efx_nic *efx = tx_queue->efx;
462 unsigned int entries;
463 int i, rc;
464
465 /* Create the smallest power-of-two aligned ring */
466 entries = max(roundup_pow_of_two(efx->txq_entries), EFX_MIN_DMAQ_SIZE);
467 EFX_BUG_ON_PARANOID(entries > EFX_MAX_DMAQ_SIZE);
468 tx_queue->ptr_mask = entries - 1;
469
470 netif_dbg(efx, probe, efx->net_dev,
471 "creating TX queue %d size %#x mask %#x\n",
472 tx_queue->queue, efx->txq_entries, tx_queue->ptr_mask);
473
474 /* Allocate software ring */
475 tx_queue->buffer = kzalloc(entries * sizeof(*tx_queue->buffer),
476 GFP_KERNEL);
477 if (!tx_queue->buffer)
478 return -ENOMEM;
479 for (i = 0; i <= tx_queue->ptr_mask; ++i)
480 tx_queue->buffer[i].continuation = true;
481
482 /* Allocate hardware ring */
483 rc = efx_nic_probe_tx(tx_queue);
484 if (rc)
485 goto fail;
486
487 return 0;
488
489 fail:
490 kfree(tx_queue->buffer);
491 tx_queue->buffer = NULL;
492 return rc;
493}
494
495void efx_init_tx_queue(struct efx_tx_queue *tx_queue)
496{
497 netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
498 "initialising TX queue %d\n", tx_queue->queue);
499
500 tx_queue->insert_count = 0;
501 tx_queue->write_count = 0;
502 tx_queue->old_write_count = 0;
503 tx_queue->read_count = 0;
504 tx_queue->old_read_count = 0;
505 tx_queue->empty_read_count = 0 | EFX_EMPTY_COUNT_VALID;
506
507 /* Set up TX descriptor ring */
508 efx_nic_init_tx(tx_queue);
509
510 tx_queue->initialised = true;
511}
512
513void efx_release_tx_buffers(struct efx_tx_queue *tx_queue)
514{
515 struct efx_tx_buffer *buffer;
516
517 if (!tx_queue->buffer)
518 return;
519
520 /* Free any buffers left in the ring */
521 while (tx_queue->read_count != tx_queue->write_count) {
522 buffer = &tx_queue->buffer[tx_queue->read_count & tx_queue->ptr_mask];
523 efx_dequeue_buffer(tx_queue, buffer);
524 buffer->continuation = true;
525 buffer->len = 0;
526
527 ++tx_queue->read_count;
528 }
529}
530
531void efx_fini_tx_queue(struct efx_tx_queue *tx_queue)
532{
533 if (!tx_queue->initialised)
534 return;
535
536 netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
537 "shutting down TX queue %d\n", tx_queue->queue);
538
539 tx_queue->initialised = false;
540
541 /* Flush TX queue, remove descriptor ring */
542 efx_nic_fini_tx(tx_queue);
543
544 efx_release_tx_buffers(tx_queue);
545
546 /* Free up TSO header cache */
547 efx_fini_tso(tx_queue);
548}
549
550void efx_remove_tx_queue(struct efx_tx_queue *tx_queue)
551{
552 if (!tx_queue->buffer)
553 return;
554
555 netif_dbg(tx_queue->efx, drv, tx_queue->efx->net_dev,
556 "destroying TX queue %d\n", tx_queue->queue);
557 efx_nic_remove_tx(tx_queue);
558
559 kfree(tx_queue->buffer);
560 tx_queue->buffer = NULL;
561}
562
563
564/* Efx TCP segmentation acceleration.
565 *
566 * Why? Because by doing it here in the driver we can go significantly
567 * faster than the GSO.
568 *
569 * Requires TX checksum offload support.
570 */
571
572/* Number of bytes inserted at the start of a TSO header buffer,
573 * similar to NET_IP_ALIGN.
574 */
575#ifdef CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS
576#define TSOH_OFFSET 0
577#else
578#define TSOH_OFFSET NET_IP_ALIGN
579#endif
580
581#define TSOH_BUFFER(tsoh) ((u8 *)(tsoh + 1) + TSOH_OFFSET)
582
583/* Total size of struct efx_tso_header, buffer and padding */
584#define TSOH_SIZE(hdr_len) \
585 (sizeof(struct efx_tso_header) + TSOH_OFFSET + hdr_len)
586
587/* Size of blocks on free list. Larger blocks must be allocated from
588 * the heap.
589 */
590#define TSOH_STD_SIZE 128
591
592#define PTR_DIFF(p1, p2) ((u8 *)(p1) - (u8 *)(p2))
593#define ETH_HDR_LEN(skb) (skb_network_header(skb) - (skb)->data)
594#define SKB_TCP_OFF(skb) PTR_DIFF(tcp_hdr(skb), (skb)->data)
595#define SKB_IPV4_OFF(skb) PTR_DIFF(ip_hdr(skb), (skb)->data)
596#define SKB_IPV6_OFF(skb) PTR_DIFF(ipv6_hdr(skb), (skb)->data)
597
598/**
599 * struct tso_state - TSO state for an SKB
600 * @out_len: Remaining length in current segment
601 * @seqnum: Current sequence number
602 * @ipv4_id: Current IPv4 ID, host endian
603 * @packet_space: Remaining space in current packet
604 * @dma_addr: DMA address of current position
605 * @in_len: Remaining length in current SKB fragment
606 * @unmap_len: Length of SKB fragment
607 * @unmap_addr: DMA address of SKB fragment
608 * @unmap_single: DMA single vs page mapping flag
609 * @protocol: Network protocol (after any VLAN header)
610 * @header_len: Number of bytes of header
611 * @full_packet_size: Number of bytes to put in each outgoing segment
612 *
613 * The state used during segmentation. It is put into this data structure
614 * just to make it easy to pass into inline functions.
615 */
616struct tso_state {
617 /* Output position */
618 unsigned out_len;
619 unsigned seqnum;
620 unsigned ipv4_id;
621 unsigned packet_space;
622
623 /* Input position */
624 dma_addr_t dma_addr;
625 unsigned in_len;
626 unsigned unmap_len;
627 dma_addr_t unmap_addr;
628 bool unmap_single;
629
630 __be16 protocol;
631 unsigned header_len;
632 int full_packet_size;
633};
634
635
636/*
637 * Verify that our various assumptions about sk_buffs and the conditions
638 * under which TSO will be attempted hold true. Return the protocol number.
639 */
640static __be16 efx_tso_check_protocol(struct sk_buff *skb)
641{
642 __be16 protocol = skb->protocol;
643
644 EFX_BUG_ON_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
645 protocol);
646 if (protocol == htons(ETH_P_8021Q)) {
647 /* Find the encapsulated protocol; reset network header
648 * and transport header based on that. */
649 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
650 protocol = veh->h_vlan_encapsulated_proto;
651 skb_set_network_header(skb, sizeof(*veh));
652 if (protocol == htons(ETH_P_IP))
653 skb_set_transport_header(skb, sizeof(*veh) +
654 4 * ip_hdr(skb)->ihl);
655 else if (protocol == htons(ETH_P_IPV6))
656 skb_set_transport_header(skb, sizeof(*veh) +
657 sizeof(struct ipv6hdr));
658 }
659
660 if (protocol == htons(ETH_P_IP)) {
661 EFX_BUG_ON_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP);
662 } else {
663 EFX_BUG_ON_PARANOID(protocol != htons(ETH_P_IPV6));
664 EFX_BUG_ON_PARANOID(ipv6_hdr(skb)->nexthdr != NEXTHDR_TCP);
665 }
666 EFX_BUG_ON_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data)
667 + (tcp_hdr(skb)->doff << 2u)) >
668 skb_headlen(skb));
669
670 return protocol;
671}
672
673
674/*
675 * Allocate a page worth of efx_tso_header structures, and string them
676 * into the tx_queue->tso_headers_free linked list. Return 0 or -ENOMEM.
677 */
678static int efx_tsoh_block_alloc(struct efx_tx_queue *tx_queue)
679{
680
681 struct pci_dev *pci_dev = tx_queue->efx->pci_dev;
682 struct efx_tso_header *tsoh;
683 dma_addr_t dma_addr;
684 u8 *base_kva, *kva;
685
686 base_kva = pci_alloc_consistent(pci_dev, PAGE_SIZE, &dma_addr);
687 if (base_kva == NULL) {
688 netif_err(tx_queue->efx, tx_err, tx_queue->efx->net_dev,
689 "Unable to allocate page for TSO headers\n");
690 return -ENOMEM;
691 }
692
693 /* pci_alloc_consistent() allocates pages. */
694 EFX_BUG_ON_PARANOID(dma_addr & (PAGE_SIZE - 1u));
695
696 for (kva = base_kva; kva < base_kva + PAGE_SIZE; kva += TSOH_STD_SIZE) {
697 tsoh = (struct efx_tso_header *)kva;
698 tsoh->dma_addr = dma_addr + (TSOH_BUFFER(tsoh) - base_kva);
699 tsoh->next = tx_queue->tso_headers_free;
700 tx_queue->tso_headers_free = tsoh;
701 }
702
703 return 0;
704}
705
706
707/* Free up a TSO header, and all others in the same page. */
708static void efx_tsoh_block_free(struct efx_tx_queue *tx_queue,
709 struct efx_tso_header *tsoh,
710 struct pci_dev *pci_dev)
711{
712 struct efx_tso_header **p;
713 unsigned long base_kva;
714 dma_addr_t base_dma;
715
716 base_kva = (unsigned long)tsoh & PAGE_MASK;
717 base_dma = tsoh->dma_addr & PAGE_MASK;
718
719 p = &tx_queue->tso_headers_free;
720 while (*p != NULL) {
721 if (((unsigned long)*p & PAGE_MASK) == base_kva)
722 *p = (*p)->next;
723 else
724 p = &(*p)->next;
725 }
726
727 pci_free_consistent(pci_dev, PAGE_SIZE, (void *)base_kva, base_dma);
728}
729
730static struct efx_tso_header *
731efx_tsoh_heap_alloc(struct efx_tx_queue *tx_queue, size_t header_len)
732{
733 struct efx_tso_header *tsoh;
734
735 tsoh = kmalloc(TSOH_SIZE(header_len), GFP_ATOMIC | GFP_DMA);
736 if (unlikely(!tsoh))
737 return NULL;
738
739 tsoh->dma_addr = pci_map_single(tx_queue->efx->pci_dev,
740 TSOH_BUFFER(tsoh), header_len,
741 PCI_DMA_TODEVICE);
742 if (unlikely(pci_dma_mapping_error(tx_queue->efx->pci_dev,
743 tsoh->dma_addr))) {
744 kfree(tsoh);
745 return NULL;
746 }
747
748 tsoh->unmap_len = header_len;
749 return tsoh;
750}
751
752static void
753efx_tsoh_heap_free(struct efx_tx_queue *tx_queue, struct efx_tso_header *tsoh)
754{
755 pci_unmap_single(tx_queue->efx->pci_dev,
756 tsoh->dma_addr, tsoh->unmap_len,
757 PCI_DMA_TODEVICE);
758 kfree(tsoh);
759}
760
761/**
762 * efx_tx_queue_insert - push descriptors onto the TX queue
763 * @tx_queue: Efx TX queue
764 * @dma_addr: DMA address of fragment
765 * @len: Length of fragment
766 * @final_buffer: The final buffer inserted into the queue
767 *
768 * Push descriptors onto the TX queue. Return 0 on success or 1 if
769 * @tx_queue full.
770 */
771static int efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
772 dma_addr_t dma_addr, unsigned len,
773 struct efx_tx_buffer **final_buffer)
774{
775 struct efx_tx_buffer *buffer;
776 struct efx_nic *efx = tx_queue->efx;
777 unsigned dma_len, fill_level, insert_ptr;
778 int q_space;
779
780 EFX_BUG_ON_PARANOID(len <= 0);
781
782 fill_level = tx_queue->insert_count - tx_queue->old_read_count;
783 /* -1 as there is no way to represent all descriptors used */
784 q_space = efx->txq_entries - 1 - fill_level;
785
786 while (1) {
787 if (unlikely(q_space-- <= 0)) {
788 /* It might be that completions have happened
789 * since the xmit path last checked. Update
790 * the xmit path's copy of read_count.
791 */
792 netif_tx_stop_queue(tx_queue->core_txq);
793 /* This memory barrier protects the change of
794 * queue state from the access of read_count. */
795 smp_mb();
796 tx_queue->old_read_count =
797 ACCESS_ONCE(tx_queue->read_count);
798 fill_level = (tx_queue->insert_count
799 - tx_queue->old_read_count);
800 q_space = efx->txq_entries - 1 - fill_level;
801 if (unlikely(q_space-- <= 0)) {
802 *final_buffer = NULL;
803 return 1;
804 }
805 smp_mb();
806 netif_tx_start_queue(tx_queue->core_txq);
807 }
808
809 insert_ptr = tx_queue->insert_count & tx_queue->ptr_mask;
810 buffer = &tx_queue->buffer[insert_ptr];
811 ++tx_queue->insert_count;
812
813 EFX_BUG_ON_PARANOID(tx_queue->insert_count -
814 tx_queue->read_count >=
815 efx->txq_entries);
816
817 efx_tsoh_free(tx_queue, buffer);
818 EFX_BUG_ON_PARANOID(buffer->len);
819 EFX_BUG_ON_PARANOID(buffer->unmap_len);
820 EFX_BUG_ON_PARANOID(buffer->skb);
821 EFX_BUG_ON_PARANOID(!buffer->continuation);
822 EFX_BUG_ON_PARANOID(buffer->tsoh);
823
824 buffer->dma_addr = dma_addr;
825
826 dma_len = efx_max_tx_len(efx, dma_addr);
827
828 /* If there is enough space to send then do so */
829 if (dma_len >= len)
830 break;
831
832 buffer->len = dma_len; /* Don't set the other members */
833 dma_addr += dma_len;
834 len -= dma_len;
835 }
836
837 EFX_BUG_ON_PARANOID(!len);
838 buffer->len = len;
839 *final_buffer = buffer;
840 return 0;
841}
842
843
844/*
845 * Put a TSO header into the TX queue.
846 *
847 * This is special-cased because we know that it is small enough to fit in
848 * a single fragment, and we know it doesn't cross a page boundary. It
849 * also allows us to not worry about end-of-packet etc.
850 */
851static void efx_tso_put_header(struct efx_tx_queue *tx_queue,
852 struct efx_tso_header *tsoh, unsigned len)
853{
854 struct efx_tx_buffer *buffer;
855
856 buffer = &tx_queue->buffer[tx_queue->insert_count & tx_queue->ptr_mask];
857 efx_tsoh_free(tx_queue, buffer);
858 EFX_BUG_ON_PARANOID(buffer->len);
859 EFX_BUG_ON_PARANOID(buffer->unmap_len);
860 EFX_BUG_ON_PARANOID(buffer->skb);
861 EFX_BUG_ON_PARANOID(!buffer->continuation);
862 EFX_BUG_ON_PARANOID(buffer->tsoh);
863 buffer->len = len;
864 buffer->dma_addr = tsoh->dma_addr;
865 buffer->tsoh = tsoh;
866
867 ++tx_queue->insert_count;
868}
869
870
871/* Remove descriptors put into a tx_queue. */
872static void efx_enqueue_unwind(struct efx_tx_queue *tx_queue)
873{
874 struct efx_tx_buffer *buffer;
875 dma_addr_t unmap_addr;
876
877 /* Work backwards until we hit the original insert pointer value */
878 while (tx_queue->insert_count != tx_queue->write_count) {
879 --tx_queue->insert_count;
880 buffer = &tx_queue->buffer[tx_queue->insert_count &
881 tx_queue->ptr_mask];
882 efx_tsoh_free(tx_queue, buffer);
883 EFX_BUG_ON_PARANOID(buffer->skb);
884 if (buffer->unmap_len) {
885 unmap_addr = (buffer->dma_addr + buffer->len -
886 buffer->unmap_len);
887 if (buffer->unmap_single)
888 pci_unmap_single(tx_queue->efx->pci_dev,
889 unmap_addr, buffer->unmap_len,
890 PCI_DMA_TODEVICE);
891 else
892 pci_unmap_page(tx_queue->efx->pci_dev,
893 unmap_addr, buffer->unmap_len,
894 PCI_DMA_TODEVICE);
895 buffer->unmap_len = 0;
896 }
897 buffer->len = 0;
898 buffer->continuation = true;
899 }
900}
901
902
903/* Parse the SKB header and initialise state. */
904static void tso_start(struct tso_state *st, const struct sk_buff *skb)
905{
906 /* All ethernet/IP/TCP headers combined size is TCP header size
907 * plus offset of TCP header relative to start of packet.
908 */
909 st->header_len = ((tcp_hdr(skb)->doff << 2u)
910 + PTR_DIFF(tcp_hdr(skb), skb->data));
911 st->full_packet_size = st->header_len + skb_shinfo(skb)->gso_size;
912
913 if (st->protocol == htons(ETH_P_IP))
914 st->ipv4_id = ntohs(ip_hdr(skb)->id);
915 else
916 st->ipv4_id = 0;
917 st->seqnum = ntohl(tcp_hdr(skb)->seq);
918
919 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->urg);
920 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->syn);
921 EFX_BUG_ON_PARANOID(tcp_hdr(skb)->rst);
922
923 st->packet_space = st->full_packet_size;
924 st->out_len = skb->len - st->header_len;
925 st->unmap_len = 0;
926 st->unmap_single = false;
927}
928
929static int tso_get_fragment(struct tso_state *st, struct efx_nic *efx,
930 skb_frag_t *frag)
931{
932 st->unmap_addr = pci_map_page(efx->pci_dev, frag->page,
933 frag->page_offset, frag->size,
934 PCI_DMA_TODEVICE);
935 if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) {
936 st->unmap_single = false;
937 st->unmap_len = frag->size;
938 st->in_len = frag->size;
939 st->dma_addr = st->unmap_addr;
940 return 0;
941 }
942 return -ENOMEM;
943}
944
945static int tso_get_head_fragment(struct tso_state *st, struct efx_nic *efx,
946 const struct sk_buff *skb)
947{
948 int hl = st->header_len;
949 int len = skb_headlen(skb) - hl;
950
951 st->unmap_addr = pci_map_single(efx->pci_dev, skb->data + hl,
952 len, PCI_DMA_TODEVICE);
953 if (likely(!pci_dma_mapping_error(efx->pci_dev, st->unmap_addr))) {
954 st->unmap_single = true;
955 st->unmap_len = len;
956 st->in_len = len;
957 st->dma_addr = st->unmap_addr;
958 return 0;
959 }
960 return -ENOMEM;
961}
962
963
964/**
965 * tso_fill_packet_with_fragment - form descriptors for the current fragment
966 * @tx_queue: Efx TX queue
967 * @skb: Socket buffer
968 * @st: TSO state
969 *
970 * Form descriptors for the current fragment, until we reach the end
971 * of fragment or end-of-packet. Return 0 on success, 1 if not enough
972 * space in @tx_queue.
973 */
974static int tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue,
975 const struct sk_buff *skb,
976 struct tso_state *st)
977{
978 struct efx_tx_buffer *buffer;
979 int n, end_of_packet, rc;
980
981 if (st->in_len == 0)
982 return 0;
983 if (st->packet_space == 0)
984 return 0;
985
986 EFX_BUG_ON_PARANOID(st->in_len <= 0);
987 EFX_BUG_ON_PARANOID(st->packet_space <= 0);
988
989 n = min(st->in_len, st->packet_space);
990
991 st->packet_space -= n;
992 st->out_len -= n;
993 st->in_len -= n;
994
995 rc = efx_tx_queue_insert(tx_queue, st->dma_addr, n, &buffer);
996 if (likely(rc == 0)) {
997 if (st->out_len == 0)
998 /* Transfer ownership of the skb */
999 buffer->skb = skb;
1000
1001 end_of_packet = st->out_len == 0 || st->packet_space == 0;
1002 buffer->continuation = !end_of_packet;
1003
1004 if (st->in_len == 0) {
1005 /* Transfer ownership of the pci mapping */
1006 buffer->unmap_len = st->unmap_len;
1007 buffer->unmap_single = st->unmap_single;
1008 st->unmap_len = 0;
1009 }
1010 }
1011
1012 st->dma_addr += n;
1013 return rc;
1014}
1015
1016
1017/**
1018 * tso_start_new_packet - generate a new header and prepare for the new packet
1019 * @tx_queue: Efx TX queue
1020 * @skb: Socket buffer
1021 * @st: TSO state
1022 *
1023 * Generate a new header and prepare for the new packet. Return 0 on
1024 * success, or -1 if failed to alloc header.
1025 */
1026static int tso_start_new_packet(struct efx_tx_queue *tx_queue,
1027 const struct sk_buff *skb,
1028 struct tso_state *st)
1029{
1030 struct efx_tso_header *tsoh;
1031 struct tcphdr *tsoh_th;
1032 unsigned ip_length;
1033 u8 *header;
1034
1035 /* Allocate a DMA-mapped header buffer. */
1036 if (likely(TSOH_SIZE(st->header_len) <= TSOH_STD_SIZE)) {
1037 if (tx_queue->tso_headers_free == NULL) {
1038 if (efx_tsoh_block_alloc(tx_queue))
1039 return -1;
1040 }
1041 EFX_BUG_ON_PARANOID(!tx_queue->tso_headers_free);
1042 tsoh = tx_queue->tso_headers_free;
1043 tx_queue->tso_headers_free = tsoh->next;
1044 tsoh->unmap_len = 0;
1045 } else {
1046 tx_queue->tso_long_headers++;
1047 tsoh = efx_tsoh_heap_alloc(tx_queue, st->header_len);
1048 if (unlikely(!tsoh))
1049 return -1;
1050 }
1051
1052 header = TSOH_BUFFER(tsoh);
1053 tsoh_th = (struct tcphdr *)(header + SKB_TCP_OFF(skb));
1054
1055 /* Copy and update the headers. */
1056 memcpy(header, skb->data, st->header_len);
1057
1058 tsoh_th->seq = htonl(st->seqnum);
1059 st->seqnum += skb_shinfo(skb)->gso_size;
1060 if (st->out_len > skb_shinfo(skb)->gso_size) {
1061 /* This packet will not finish the TSO burst. */
1062 ip_length = st->full_packet_size - ETH_HDR_LEN(skb);
1063 tsoh_th->fin = 0;
1064 tsoh_th->psh = 0;
1065 } else {
1066 /* This packet will be the last in the TSO burst. */
1067 ip_length = st->header_len - ETH_HDR_LEN(skb) + st->out_len;
1068 tsoh_th->fin = tcp_hdr(skb)->fin;
1069 tsoh_th->psh = tcp_hdr(skb)->psh;
1070 }
1071
1072 if (st->protocol == htons(ETH_P_IP)) {
1073 struct iphdr *tsoh_iph =
1074 (struct iphdr *)(header + SKB_IPV4_OFF(skb));
1075
1076 tsoh_iph->tot_len = htons(ip_length);
1077
1078 /* Linux leaves suitable gaps in the IP ID space for us to fill. */
1079 tsoh_iph->id = htons(st->ipv4_id);
1080 st->ipv4_id++;
1081 } else {
1082 struct ipv6hdr *tsoh_iph =
1083 (struct ipv6hdr *)(header + SKB_IPV6_OFF(skb));
1084
1085 tsoh_iph->payload_len = htons(ip_length - sizeof(*tsoh_iph));
1086 }
1087
1088 st->packet_space = skb_shinfo(skb)->gso_size;
1089 ++tx_queue->tso_packets;
1090
1091 /* Form a descriptor for this header. */
1092 efx_tso_put_header(tx_queue, tsoh, st->header_len);
1093
1094 return 0;
1095}
1096
1097
1098/**
1099 * efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
1100 * @tx_queue: Efx TX queue
1101 * @skb: Socket buffer
1102 *
1103 * Context: You must hold netif_tx_lock() to call this function.
1104 *
1105 * Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
1106 * @skb was not enqueued. In all cases @skb is consumed. Return
1107 * %NETDEV_TX_OK or %NETDEV_TX_BUSY.
1108 */
1109static int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
1110 struct sk_buff *skb)
1111{
1112 struct efx_nic *efx = tx_queue->efx;
1113 int frag_i, rc, rc2 = NETDEV_TX_OK;
1114 struct tso_state state;
1115
1116 /* Find the packet protocol and sanity-check it */
1117 state.protocol = efx_tso_check_protocol(skb);
1118
1119 EFX_BUG_ON_PARANOID(tx_queue->write_count != tx_queue->insert_count);
1120
1121 tso_start(&state, skb);
1122
1123 /* Assume that skb header area contains exactly the headers, and
1124 * all payload is in the frag list.
1125 */
1126 if (skb_headlen(skb) == state.header_len) {
1127 /* Grab the first payload fragment. */
1128 EFX_BUG_ON_PARANOID(skb_shinfo(skb)->nr_frags < 1);
1129 frag_i = 0;
1130 rc = tso_get_fragment(&state, efx,
1131 skb_shinfo(skb)->frags + frag_i);
1132 if (rc)
1133 goto mem_err;
1134 } else {
1135 rc = tso_get_head_fragment(&state, efx, skb);
1136 if (rc)
1137 goto mem_err;
1138 frag_i = -1;
1139 }
1140
1141 if (tso_start_new_packet(tx_queue, skb, &state) < 0)
1142 goto mem_err;
1143
1144 while (1) {
1145 rc = tso_fill_packet_with_fragment(tx_queue, skb, &state);
1146 if (unlikely(rc)) {
1147 rc2 = NETDEV_TX_BUSY;
1148 goto unwind;
1149 }
1150
1151 /* Move onto the next fragment? */
1152 if (state.in_len == 0) {
1153 if (++frag_i >= skb_shinfo(skb)->nr_frags)
1154 /* End of payload reached. */
1155 break;
1156 rc = tso_get_fragment(&state, efx,
1157 skb_shinfo(skb)->frags + frag_i);
1158 if (rc)
1159 goto mem_err;
1160 }
1161
1162 /* Start at new packet? */
1163 if (state.packet_space == 0 &&
1164 tso_start_new_packet(tx_queue, skb, &state) < 0)
1165 goto mem_err;
1166 }
1167
1168 /* Pass off to hardware */
1169 efx_nic_push_buffers(tx_queue);
1170
1171 tx_queue->tso_bursts++;
1172 return NETDEV_TX_OK;
1173
1174 mem_err:
1175 netif_err(efx, tx_err, efx->net_dev,
1176 "Out of memory for TSO headers, or PCI mapping error\n");
1177 dev_kfree_skb_any(skb);
1178
1179 unwind:
1180 /* Free the DMA mapping we were in the process of writing out */
1181 if (state.unmap_len) {
1182 if (state.unmap_single)
1183 pci_unmap_single(efx->pci_dev, state.unmap_addr,
1184 state.unmap_len, PCI_DMA_TODEVICE);
1185 else
1186 pci_unmap_page(efx->pci_dev, state.unmap_addr,
1187 state.unmap_len, PCI_DMA_TODEVICE);
1188 }
1189
1190 efx_enqueue_unwind(tx_queue);
1191 return rc2;
1192}
1193
1194
1195/*
1196 * Free up all TSO datastructures associated with tx_queue. This
1197 * routine should be called only once the tx_queue is both empty and
1198 * will no longer be used.
1199 */
1200static void efx_fini_tso(struct efx_tx_queue *tx_queue)
1201{
1202 unsigned i;
1203
1204 if (tx_queue->buffer) {
1205 for (i = 0; i <= tx_queue->ptr_mask; ++i)
1206 efx_tsoh_free(tx_queue, &tx_queue->buffer[i]);
1207 }
1208
1209 while (tx_queue->tso_headers_free != NULL)
1210 efx_tsoh_block_free(tx_queue, tx_queue->tso_headers_free,
1211 tx_queue->efx->pci_dev);
1212}