Loading...
Note: File does not exist in v3.1.
1/****************************************************************************
2 * Driver for Solarflare network controllers and boards
3 * Copyright 2005-2006 Fen Systems Ltd.
4 * Copyright 2005-2015 Solarflare Communications Inc.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms of the GNU General Public License version 2 as published
8 * by the Free Software Foundation, incorporated herein by reference.
9 */
10
11#include <linux/pci.h>
12#include <linux/tcp.h>
13#include <linux/ip.h>
14#include <linux/in.h>
15#include <linux/ipv6.h>
16#include <linux/slab.h>
17#include <net/ipv6.h>
18#include <linux/if_ether.h>
19#include <linux/highmem.h>
20#include <linux/moduleparam.h>
21#include <linux/cache.h>
22#include "net_driver.h"
23#include "efx.h"
24#include "io.h"
25#include "nic.h"
26#include "tx.h"
27#include "workarounds.h"
28#include "ef10_regs.h"
29
30/* Efx legacy TCP segmentation acceleration.
31 *
32 * Utilises firmware support to go faster than GSO (but not as fast as TSOv2).
33 *
34 * Requires TX checksum offload support.
35 */
36
37#define PTR_DIFF(p1, p2) ((u8 *)(p1) - (u8 *)(p2))
38
39/**
40 * struct tso_state - TSO state for an SKB
41 * @out_len: Remaining length in current segment
42 * @seqnum: Current sequence number
43 * @ipv4_id: Current IPv4 ID, host endian
44 * @packet_space: Remaining space in current packet
45 * @dma_addr: DMA address of current position
46 * @in_len: Remaining length in current SKB fragment
47 * @unmap_len: Length of SKB fragment
48 * @unmap_addr: DMA address of SKB fragment
49 * @protocol: Network protocol (after any VLAN header)
50 * @ip_off: Offset of IP header
51 * @tcp_off: Offset of TCP header
52 * @header_len: Number of bytes of header
53 * @ip_base_len: IPv4 tot_len or IPv6 payload_len, before TCP payload
54 * @header_dma_addr: Header DMA address
55 * @header_unmap_len: Header DMA mapped length
56 *
57 * The state used during segmentation. It is put into this data structure
58 * just to make it easy to pass into inline functions.
59 */
60struct tso_state {
61 /* Output position */
62 unsigned int out_len;
63 unsigned int seqnum;
64 u16 ipv4_id;
65 unsigned int packet_space;
66
67 /* Input position */
68 dma_addr_t dma_addr;
69 unsigned int in_len;
70 unsigned int unmap_len;
71 dma_addr_t unmap_addr;
72
73 __be16 protocol;
74 unsigned int ip_off;
75 unsigned int tcp_off;
76 unsigned int header_len;
77 unsigned int ip_base_len;
78 dma_addr_t header_dma_addr;
79 unsigned int header_unmap_len;
80};
81
82static inline void prefetch_ptr(struct efx_tx_queue *tx_queue)
83{
84 unsigned int insert_ptr = efx_tx_queue_get_insert_index(tx_queue);
85 char *ptr;
86
87 ptr = (char *) (tx_queue->buffer + insert_ptr);
88 prefetch(ptr);
89 prefetch(ptr + 0x80);
90
91 ptr = (char *) (((efx_qword_t *)tx_queue->txd.buf.addr) + insert_ptr);
92 prefetch(ptr);
93 prefetch(ptr + 0x80);
94}
95
96/**
97 * efx_tx_queue_insert - push descriptors onto the TX queue
98 * @tx_queue: Efx TX queue
99 * @dma_addr: DMA address of fragment
100 * @len: Length of fragment
101 * @final_buffer: The final buffer inserted into the queue
102 *
103 * Push descriptors onto the TX queue.
104 */
105static void efx_tx_queue_insert(struct efx_tx_queue *tx_queue,
106 dma_addr_t dma_addr, unsigned int len,
107 struct efx_tx_buffer **final_buffer)
108{
109 struct efx_tx_buffer *buffer;
110 unsigned int dma_len;
111
112 EFX_WARN_ON_ONCE_PARANOID(len <= 0);
113
114 while (1) {
115 buffer = efx_tx_queue_get_insert_buffer(tx_queue);
116 ++tx_queue->insert_count;
117
118 EFX_WARN_ON_ONCE_PARANOID(tx_queue->insert_count -
119 tx_queue->read_count >=
120 tx_queue->efx->txq_entries);
121
122 buffer->dma_addr = dma_addr;
123
124 dma_len = tx_queue->efx->type->tx_limit_len(tx_queue,
125 dma_addr, len);
126
127 /* If there's space for everything this is our last buffer. */
128 if (dma_len >= len)
129 break;
130
131 buffer->len = dma_len;
132 buffer->flags = EFX_TX_BUF_CONT;
133 dma_addr += dma_len;
134 len -= dma_len;
135 }
136
137 EFX_WARN_ON_ONCE_PARANOID(!len);
138 buffer->len = len;
139 *final_buffer = buffer;
140}
141
142/*
143 * Verify that our various assumptions about sk_buffs and the conditions
144 * under which TSO will be attempted hold true. Return the protocol number.
145 */
146static __be16 efx_tso_check_protocol(struct sk_buff *skb)
147{
148 __be16 protocol = skb->protocol;
149
150 EFX_WARN_ON_ONCE_PARANOID(((struct ethhdr *)skb->data)->h_proto !=
151 protocol);
152 if (protocol == htons(ETH_P_8021Q)) {
153 struct vlan_ethhdr *veh = (struct vlan_ethhdr *)skb->data;
154
155 protocol = veh->h_vlan_encapsulated_proto;
156 }
157
158 if (protocol == htons(ETH_P_IP)) {
159 EFX_WARN_ON_ONCE_PARANOID(ip_hdr(skb)->protocol != IPPROTO_TCP);
160 } else {
161 EFX_WARN_ON_ONCE_PARANOID(protocol != htons(ETH_P_IPV6));
162 EFX_WARN_ON_ONCE_PARANOID(ipv6_hdr(skb)->nexthdr != NEXTHDR_TCP);
163 }
164 EFX_WARN_ON_ONCE_PARANOID((PTR_DIFF(tcp_hdr(skb), skb->data) +
165 (tcp_hdr(skb)->doff << 2u)) >
166 skb_headlen(skb));
167
168 return protocol;
169}
170
171/* Parse the SKB header and initialise state. */
172static int tso_start(struct tso_state *st, struct efx_nic *efx,
173 struct efx_tx_queue *tx_queue,
174 const struct sk_buff *skb)
175{
176 struct device *dma_dev = &efx->pci_dev->dev;
177 unsigned int header_len, in_len;
178 dma_addr_t dma_addr;
179
180 st->ip_off = skb_network_header(skb) - skb->data;
181 st->tcp_off = skb_transport_header(skb) - skb->data;
182 header_len = st->tcp_off + (tcp_hdr(skb)->doff << 2u);
183 in_len = skb_headlen(skb) - header_len;
184 st->header_len = header_len;
185 st->in_len = in_len;
186 if (st->protocol == htons(ETH_P_IP)) {
187 st->ip_base_len = st->header_len - st->ip_off;
188 st->ipv4_id = ntohs(ip_hdr(skb)->id);
189 } else {
190 st->ip_base_len = st->header_len - st->tcp_off;
191 st->ipv4_id = 0;
192 }
193 st->seqnum = ntohl(tcp_hdr(skb)->seq);
194
195 EFX_WARN_ON_ONCE_PARANOID(tcp_hdr(skb)->urg);
196 EFX_WARN_ON_ONCE_PARANOID(tcp_hdr(skb)->syn);
197 EFX_WARN_ON_ONCE_PARANOID(tcp_hdr(skb)->rst);
198
199 st->out_len = skb->len - header_len;
200
201 dma_addr = dma_map_single(dma_dev, skb->data,
202 skb_headlen(skb), DMA_TO_DEVICE);
203 st->header_dma_addr = dma_addr;
204 st->header_unmap_len = skb_headlen(skb);
205 st->dma_addr = dma_addr + header_len;
206 st->unmap_len = 0;
207
208 return unlikely(dma_mapping_error(dma_dev, dma_addr)) ? -ENOMEM : 0;
209}
210
211static int tso_get_fragment(struct tso_state *st, struct efx_nic *efx,
212 skb_frag_t *frag)
213{
214 st->unmap_addr = skb_frag_dma_map(&efx->pci_dev->dev, frag, 0,
215 skb_frag_size(frag), DMA_TO_DEVICE);
216 if (likely(!dma_mapping_error(&efx->pci_dev->dev, st->unmap_addr))) {
217 st->unmap_len = skb_frag_size(frag);
218 st->in_len = skb_frag_size(frag);
219 st->dma_addr = st->unmap_addr;
220 return 0;
221 }
222 return -ENOMEM;
223}
224
225
226/**
227 * tso_fill_packet_with_fragment - form descriptors for the current fragment
228 * @tx_queue: Efx TX queue
229 * @skb: Socket buffer
230 * @st: TSO state
231 *
232 * Form descriptors for the current fragment, until we reach the end
233 * of fragment or end-of-packet.
234 */
235static void tso_fill_packet_with_fragment(struct efx_tx_queue *tx_queue,
236 const struct sk_buff *skb,
237 struct tso_state *st)
238{
239 struct efx_tx_buffer *buffer;
240 int n;
241
242 if (st->in_len == 0)
243 return;
244 if (st->packet_space == 0)
245 return;
246
247 EFX_WARN_ON_ONCE_PARANOID(st->in_len <= 0);
248 EFX_WARN_ON_ONCE_PARANOID(st->packet_space <= 0);
249
250 n = min(st->in_len, st->packet_space);
251
252 st->packet_space -= n;
253 st->out_len -= n;
254 st->in_len -= n;
255
256 efx_tx_queue_insert(tx_queue, st->dma_addr, n, &buffer);
257
258 if (st->out_len == 0) {
259 /* Transfer ownership of the skb */
260 buffer->skb = skb;
261 buffer->flags = EFX_TX_BUF_SKB;
262 } else if (st->packet_space != 0) {
263 buffer->flags = EFX_TX_BUF_CONT;
264 }
265
266 if (st->in_len == 0) {
267 /* Transfer ownership of the DMA mapping */
268 buffer->unmap_len = st->unmap_len;
269 buffer->dma_offset = buffer->unmap_len - buffer->len;
270 st->unmap_len = 0;
271 }
272
273 st->dma_addr += n;
274}
275
276
277#define TCP_FLAGS_OFFSET 13
278
279/**
280 * tso_start_new_packet - generate a new header and prepare for the new packet
281 * @tx_queue: Efx TX queue
282 * @skb: Socket buffer
283 * @st: TSO state
284 *
285 * Generate a new header and prepare for the new packet. Return 0 on
286 * success, or -%ENOMEM if failed to alloc header, or other negative error.
287 */
288static int tso_start_new_packet(struct efx_tx_queue *tx_queue,
289 const struct sk_buff *skb,
290 struct tso_state *st)
291{
292 struct efx_tx_buffer *buffer =
293 efx_tx_queue_get_insert_buffer(tx_queue);
294 bool is_last = st->out_len <= skb_shinfo(skb)->gso_size;
295 u8 tcp_flags_mask, tcp_flags;
296
297 if (!is_last) {
298 st->packet_space = skb_shinfo(skb)->gso_size;
299 tcp_flags_mask = 0x09; /* mask out FIN and PSH */
300 } else {
301 st->packet_space = st->out_len;
302 tcp_flags_mask = 0x00;
303 }
304
305 if (WARN_ON(!st->header_unmap_len))
306 return -EINVAL;
307 /* Send the original headers with a TSO option descriptor
308 * in front
309 */
310 tcp_flags = ((u8 *)tcp_hdr(skb))[TCP_FLAGS_OFFSET] & ~tcp_flags_mask;
311
312 buffer->flags = EFX_TX_BUF_OPTION;
313 buffer->len = 0;
314 buffer->unmap_len = 0;
315 EFX_POPULATE_QWORD_5(buffer->option,
316 ESF_DZ_TX_DESC_IS_OPT, 1,
317 ESF_DZ_TX_OPTION_TYPE,
318 ESE_DZ_TX_OPTION_DESC_TSO,
319 ESF_DZ_TX_TSO_TCP_FLAGS, tcp_flags,
320 ESF_DZ_TX_TSO_IP_ID, st->ipv4_id,
321 ESF_DZ_TX_TSO_TCP_SEQNO, st->seqnum);
322 ++tx_queue->insert_count;
323
324 /* We mapped the headers in tso_start(). Unmap them
325 * when the last segment is completed.
326 */
327 buffer = efx_tx_queue_get_insert_buffer(tx_queue);
328 buffer->dma_addr = st->header_dma_addr;
329 buffer->len = st->header_len;
330 if (is_last) {
331 buffer->flags = EFX_TX_BUF_CONT | EFX_TX_BUF_MAP_SINGLE;
332 buffer->unmap_len = st->header_unmap_len;
333 buffer->dma_offset = 0;
334 /* Ensure we only unmap them once in case of a
335 * later DMA mapping error and rollback
336 */
337 st->header_unmap_len = 0;
338 } else {
339 buffer->flags = EFX_TX_BUF_CONT;
340 buffer->unmap_len = 0;
341 }
342 ++tx_queue->insert_count;
343
344 st->seqnum += skb_shinfo(skb)->gso_size;
345
346 /* Linux leaves suitable gaps in the IP ID space for us to fill. */
347 ++st->ipv4_id;
348
349 return 0;
350}
351
352/**
353 * efx_enqueue_skb_tso - segment and transmit a TSO socket buffer
354 * @tx_queue: Efx TX queue
355 * @skb: Socket buffer
356 * @data_mapped: Did we map the data? Always set to true
357 * by this on success.
358 *
359 * Context: You must hold netif_tx_lock() to call this function.
360 *
361 * Add socket buffer @skb to @tx_queue, doing TSO or return != 0 if
362 * @skb was not enqueued. @skb is consumed unless return value is
363 * %EINVAL.
364 */
365int efx_enqueue_skb_tso(struct efx_tx_queue *tx_queue,
366 struct sk_buff *skb,
367 bool *data_mapped)
368{
369 struct efx_nic *efx = tx_queue->efx;
370 int frag_i, rc;
371 struct tso_state state;
372
373 if (tx_queue->tso_version != 1)
374 return -EINVAL;
375
376 prefetch(skb->data);
377
378 /* Find the packet protocol and sanity-check it */
379 state.protocol = efx_tso_check_protocol(skb);
380
381 EFX_WARN_ON_ONCE_PARANOID(tx_queue->write_count != tx_queue->insert_count);
382
383 rc = tso_start(&state, efx, tx_queue, skb);
384 if (rc)
385 goto fail;
386
387 if (likely(state.in_len == 0)) {
388 /* Grab the first payload fragment. */
389 EFX_WARN_ON_ONCE_PARANOID(skb_shinfo(skb)->nr_frags < 1);
390 frag_i = 0;
391 rc = tso_get_fragment(&state, efx,
392 skb_shinfo(skb)->frags + frag_i);
393 if (rc)
394 goto fail;
395 } else {
396 /* Payload starts in the header area. */
397 frag_i = -1;
398 }
399
400 rc = tso_start_new_packet(tx_queue, skb, &state);
401 if (rc)
402 goto fail;
403
404 prefetch_ptr(tx_queue);
405
406 while (1) {
407 tso_fill_packet_with_fragment(tx_queue, skb, &state);
408
409 /* Move onto the next fragment? */
410 if (state.in_len == 0) {
411 if (++frag_i >= skb_shinfo(skb)->nr_frags)
412 /* End of payload reached. */
413 break;
414 rc = tso_get_fragment(&state, efx,
415 skb_shinfo(skb)->frags + frag_i);
416 if (rc)
417 goto fail;
418 }
419
420 /* Start at new packet? */
421 if (state.packet_space == 0) {
422 rc = tso_start_new_packet(tx_queue, skb, &state);
423 if (rc)
424 goto fail;
425 }
426 }
427
428 *data_mapped = true;
429
430 return 0;
431
432fail:
433 if (rc == -ENOMEM)
434 netif_err(efx, tx_err, efx->net_dev,
435 "Out of memory for TSO headers, or DMA mapping error\n");
436 else
437 netif_err(efx, tx_err, efx->net_dev, "TSO failed, rc = %d\n", rc);
438
439 /* Free the DMA mapping we were in the process of writing out */
440 if (state.unmap_len) {
441 dma_unmap_page(&efx->pci_dev->dev, state.unmap_addr,
442 state.unmap_len, DMA_TO_DEVICE);
443 }
444
445 /* Free the header DMA mapping */
446 if (state.header_unmap_len)
447 dma_unmap_single(&efx->pci_dev->dev, state.header_dma_addr,
448 state.header_unmap_len, DMA_TO_DEVICE);
449
450 return rc;
451}