Loading...
Note: File does not exist in v3.5.6.
1/*
2 * Copyright (c) 2005-2008 Chelsio, Inc. All rights reserved.
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and/or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/skbuff.h>
33#include <linux/netdevice.h>
34#include <linux/etherdevice.h>
35#include <linux/if_vlan.h>
36#include <linux/ip.h>
37#include <linux/tcp.h>
38#include <linux/dma-mapping.h>
39#include <linux/slab.h>
40#include <linux/prefetch.h>
41#include <net/arp.h>
42#include "common.h"
43#include "regs.h"
44#include "sge_defs.h"
45#include "t3_cpl.h"
46#include "firmware_exports.h"
47#include "cxgb3_offload.h"
48
49#define USE_GTS 0
50
51#define SGE_RX_SM_BUF_SIZE 1536
52
53#define SGE_RX_COPY_THRES 256
54#define SGE_RX_PULL_LEN 128
55
56#define SGE_PG_RSVD SMP_CACHE_BYTES
57/*
58 * Page chunk size for FL0 buffers if FL0 is to be populated with page chunks.
59 * It must be a divisor of PAGE_SIZE. If set to 0 FL0 will use sk_buffs
60 * directly.
61 */
62#define FL0_PG_CHUNK_SIZE 2048
63#define FL0_PG_ORDER 0
64#define FL0_PG_ALLOC_SIZE (PAGE_SIZE << FL0_PG_ORDER)
65#define FL1_PG_CHUNK_SIZE (PAGE_SIZE > 8192 ? 16384 : 8192)
66#define FL1_PG_ORDER (PAGE_SIZE > 8192 ? 0 : 1)
67#define FL1_PG_ALLOC_SIZE (PAGE_SIZE << FL1_PG_ORDER)
68
69#define SGE_RX_DROP_THRES 16
70#define RX_RECLAIM_PERIOD (HZ/4)
71
72/*
73 * Max number of Rx buffers we replenish at a time.
74 */
75#define MAX_RX_REFILL 16U
76/*
77 * Period of the Tx buffer reclaim timer. This timer does not need to run
78 * frequently as Tx buffers are usually reclaimed by new Tx packets.
79 */
80#define TX_RECLAIM_PERIOD (HZ / 4)
81#define TX_RECLAIM_TIMER_CHUNK 64U
82#define TX_RECLAIM_CHUNK 16U
83
84/* WR size in bytes */
85#define WR_LEN (WR_FLITS * 8)
86
87/*
88 * Types of Tx queues in each queue set. Order here matters, do not change.
89 */
90enum { TXQ_ETH, TXQ_OFLD, TXQ_CTRL };
91
92/* Values for sge_txq.flags */
93enum {
94 TXQ_RUNNING = 1 << 0, /* fetch engine is running */
95 TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */
96};
97
98struct tx_desc {
99 __be64 flit[TX_DESC_FLITS];
100};
101
102struct rx_desc {
103 __be32 addr_lo;
104 __be32 len_gen;
105 __be32 gen2;
106 __be32 addr_hi;
107};
108
109struct tx_sw_desc { /* SW state per Tx descriptor */
110 struct sk_buff *skb;
111 u8 eop; /* set if last descriptor for packet */
112 u8 addr_idx; /* buffer index of first SGL entry in descriptor */
113 u8 fragidx; /* first page fragment associated with descriptor */
114 s8 sflit; /* start flit of first SGL entry in descriptor */
115};
116
117struct rx_sw_desc { /* SW state per Rx descriptor */
118 union {
119 struct sk_buff *skb;
120 struct fl_pg_chunk pg_chunk;
121 };
122 DEFINE_DMA_UNMAP_ADDR(dma_addr);
123};
124
125struct rsp_desc { /* response queue descriptor */
126 struct rss_header rss_hdr;
127 __be32 flags;
128 __be32 len_cq;
129 u8 imm_data[47];
130 u8 intr_gen;
131};
132
133/*
134 * Holds unmapping information for Tx packets that need deferred unmapping.
135 * This structure lives at skb->head and must be allocated by callers.
136 */
137struct deferred_unmap_info {
138 struct pci_dev *pdev;
139 dma_addr_t addr[MAX_SKB_FRAGS + 1];
140};
141
142/*
143 * Maps a number of flits to the number of Tx descriptors that can hold them.
144 * The formula is
145 *
146 * desc = 1 + (flits - 2) / (WR_FLITS - 1).
147 *
148 * HW allows up to 4 descriptors to be combined into a WR.
149 */
150static u8 flit_desc_map[] = {
151 0,
152#if SGE_NUM_GENBITS == 1
153 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
154 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
155 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
156 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4
157#elif SGE_NUM_GENBITS == 2
158 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
159 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
160 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3,
161 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
162#else
163# error "SGE_NUM_GENBITS must be 1 or 2"
164#endif
165};
166
167static inline struct sge_qset *fl_to_qset(const struct sge_fl *q, int qidx)
168{
169 return container_of(q, struct sge_qset, fl[qidx]);
170}
171
172static inline struct sge_qset *rspq_to_qset(const struct sge_rspq *q)
173{
174 return container_of(q, struct sge_qset, rspq);
175}
176
177static inline struct sge_qset *txq_to_qset(const struct sge_txq *q, int qidx)
178{
179 return container_of(q, struct sge_qset, txq[qidx]);
180}
181
182/**
183 * refill_rspq - replenish an SGE response queue
184 * @adapter: the adapter
185 * @q: the response queue to replenish
186 * @credits: how many new responses to make available
187 *
188 * Replenishes a response queue by making the supplied number of responses
189 * available to HW.
190 */
191static inline void refill_rspq(struct adapter *adapter,
192 const struct sge_rspq *q, unsigned int credits)
193{
194 rmb();
195 t3_write_reg(adapter, A_SG_RSPQ_CREDIT_RETURN,
196 V_RSPQ(q->cntxt_id) | V_CREDITS(credits));
197}
198
199/**
200 * need_skb_unmap - does the platform need unmapping of sk_buffs?
201 *
202 * Returns true if the platform needs sk_buff unmapping. The compiler
203 * optimizes away unnecessary code if this returns true.
204 */
205static inline int need_skb_unmap(void)
206{
207#ifdef CONFIG_NEED_DMA_MAP_STATE
208 return 1;
209#else
210 return 0;
211#endif
212}
213
214/**
215 * unmap_skb - unmap a packet main body and its page fragments
216 * @skb: the packet
217 * @q: the Tx queue containing Tx descriptors for the packet
218 * @cidx: index of Tx descriptor
219 * @pdev: the PCI device
220 *
221 * Unmap the main body of an sk_buff and its page fragments, if any.
222 * Because of the fairly complicated structure of our SGLs and the desire
223 * to conserve space for metadata, the information necessary to unmap an
224 * sk_buff is spread across the sk_buff itself (buffer lengths), the HW Tx
225 * descriptors (the physical addresses of the various data buffers), and
226 * the SW descriptor state (assorted indices). The send functions
227 * initialize the indices for the first packet descriptor so we can unmap
228 * the buffers held in the first Tx descriptor here, and we have enough
229 * information at this point to set the state for the next Tx descriptor.
230 *
231 * Note that it is possible to clean up the first descriptor of a packet
232 * before the send routines have written the next descriptors, but this
233 * race does not cause any problem. We just end up writing the unmapping
234 * info for the descriptor first.
235 */
236static inline void unmap_skb(struct sk_buff *skb, struct sge_txq *q,
237 unsigned int cidx, struct pci_dev *pdev)
238{
239 const struct sg_ent *sgp;
240 struct tx_sw_desc *d = &q->sdesc[cidx];
241 int nfrags, frag_idx, curflit, j = d->addr_idx;
242
243 sgp = (struct sg_ent *)&q->desc[cidx].flit[d->sflit];
244 frag_idx = d->fragidx;
245
246 if (frag_idx == 0 && skb_headlen(skb)) {
247 pci_unmap_single(pdev, be64_to_cpu(sgp->addr[0]),
248 skb_headlen(skb), PCI_DMA_TODEVICE);
249 j = 1;
250 }
251
252 curflit = d->sflit + 1 + j;
253 nfrags = skb_shinfo(skb)->nr_frags;
254
255 while (frag_idx < nfrags && curflit < WR_FLITS) {
256 pci_unmap_page(pdev, be64_to_cpu(sgp->addr[j]),
257 skb_shinfo(skb)->frags[frag_idx].size,
258 PCI_DMA_TODEVICE);
259 j ^= 1;
260 if (j == 0) {
261 sgp++;
262 curflit++;
263 }
264 curflit++;
265 frag_idx++;
266 }
267
268 if (frag_idx < nfrags) { /* SGL continues into next Tx descriptor */
269 d = cidx + 1 == q->size ? q->sdesc : d + 1;
270 d->fragidx = frag_idx;
271 d->addr_idx = j;
272 d->sflit = curflit - WR_FLITS - j; /* sflit can be -1 */
273 }
274}
275
276/**
277 * free_tx_desc - reclaims Tx descriptors and their buffers
278 * @adapter: the adapter
279 * @q: the Tx queue to reclaim descriptors from
280 * @n: the number of descriptors to reclaim
281 *
282 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
283 * Tx buffers. Called with the Tx queue lock held.
284 */
285static void free_tx_desc(struct adapter *adapter, struct sge_txq *q,
286 unsigned int n)
287{
288 struct tx_sw_desc *d;
289 struct pci_dev *pdev = adapter->pdev;
290 unsigned int cidx = q->cidx;
291
292 const int need_unmap = need_skb_unmap() &&
293 q->cntxt_id >= FW_TUNNEL_SGEEC_START;
294
295 d = &q->sdesc[cidx];
296 while (n--) {
297 if (d->skb) { /* an SGL is present */
298 if (need_unmap)
299 unmap_skb(d->skb, q, cidx, pdev);
300 if (d->eop) {
301 kfree_skb(d->skb);
302 d->skb = NULL;
303 }
304 }
305 ++d;
306 if (++cidx == q->size) {
307 cidx = 0;
308 d = q->sdesc;
309 }
310 }
311 q->cidx = cidx;
312}
313
314/**
315 * reclaim_completed_tx - reclaims completed Tx descriptors
316 * @adapter: the adapter
317 * @q: the Tx queue to reclaim completed descriptors from
318 * @chunk: maximum number of descriptors to reclaim
319 *
320 * Reclaims Tx descriptors that the SGE has indicated it has processed,
321 * and frees the associated buffers if possible. Called with the Tx
322 * queue's lock held.
323 */
324static inline unsigned int reclaim_completed_tx(struct adapter *adapter,
325 struct sge_txq *q,
326 unsigned int chunk)
327{
328 unsigned int reclaim = q->processed - q->cleaned;
329
330 reclaim = min(chunk, reclaim);
331 if (reclaim) {
332 free_tx_desc(adapter, q, reclaim);
333 q->cleaned += reclaim;
334 q->in_use -= reclaim;
335 }
336 return q->processed - q->cleaned;
337}
338
339/**
340 * should_restart_tx - are there enough resources to restart a Tx queue?
341 * @q: the Tx queue
342 *
343 * Checks if there are enough descriptors to restart a suspended Tx queue.
344 */
345static inline int should_restart_tx(const struct sge_txq *q)
346{
347 unsigned int r = q->processed - q->cleaned;
348
349 return q->in_use - r < (q->size >> 1);
350}
351
352static void clear_rx_desc(struct pci_dev *pdev, const struct sge_fl *q,
353 struct rx_sw_desc *d)
354{
355 if (q->use_pages && d->pg_chunk.page) {
356 (*d->pg_chunk.p_cnt)--;
357 if (!*d->pg_chunk.p_cnt)
358 pci_unmap_page(pdev,
359 d->pg_chunk.mapping,
360 q->alloc_size, PCI_DMA_FROMDEVICE);
361
362 put_page(d->pg_chunk.page);
363 d->pg_chunk.page = NULL;
364 } else {
365 pci_unmap_single(pdev, dma_unmap_addr(d, dma_addr),
366 q->buf_size, PCI_DMA_FROMDEVICE);
367 kfree_skb(d->skb);
368 d->skb = NULL;
369 }
370}
371
372/**
373 * free_rx_bufs - free the Rx buffers on an SGE free list
374 * @pdev: the PCI device associated with the adapter
375 * @rxq: the SGE free list to clean up
376 *
377 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from
378 * this queue should be stopped before calling this function.
379 */
380static void free_rx_bufs(struct pci_dev *pdev, struct sge_fl *q)
381{
382 unsigned int cidx = q->cidx;
383
384 while (q->credits--) {
385 struct rx_sw_desc *d = &q->sdesc[cidx];
386
387
388 clear_rx_desc(pdev, q, d);
389 if (++cidx == q->size)
390 cidx = 0;
391 }
392
393 if (q->pg_chunk.page) {
394 __free_pages(q->pg_chunk.page, q->order);
395 q->pg_chunk.page = NULL;
396 }
397}
398
399/**
400 * add_one_rx_buf - add a packet buffer to a free-buffer list
401 * @va: buffer start VA
402 * @len: the buffer length
403 * @d: the HW Rx descriptor to write
404 * @sd: the SW Rx descriptor to write
405 * @gen: the generation bit value
406 * @pdev: the PCI device associated with the adapter
407 *
408 * Add a buffer of the given length to the supplied HW and SW Rx
409 * descriptors.
410 */
411static inline int add_one_rx_buf(void *va, unsigned int len,
412 struct rx_desc *d, struct rx_sw_desc *sd,
413 unsigned int gen, struct pci_dev *pdev)
414{
415 dma_addr_t mapping;
416
417 mapping = pci_map_single(pdev, va, len, PCI_DMA_FROMDEVICE);
418 if (unlikely(pci_dma_mapping_error(pdev, mapping)))
419 return -ENOMEM;
420
421 dma_unmap_addr_set(sd, dma_addr, mapping);
422
423 d->addr_lo = cpu_to_be32(mapping);
424 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
425 wmb();
426 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
427 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
428 return 0;
429}
430
431static inline int add_one_rx_chunk(dma_addr_t mapping, struct rx_desc *d,
432 unsigned int gen)
433{
434 d->addr_lo = cpu_to_be32(mapping);
435 d->addr_hi = cpu_to_be32((u64) mapping >> 32);
436 wmb();
437 d->len_gen = cpu_to_be32(V_FLD_GEN1(gen));
438 d->gen2 = cpu_to_be32(V_FLD_GEN2(gen));
439 return 0;
440}
441
442static int alloc_pg_chunk(struct adapter *adapter, struct sge_fl *q,
443 struct rx_sw_desc *sd, gfp_t gfp,
444 unsigned int order)
445{
446 if (!q->pg_chunk.page) {
447 dma_addr_t mapping;
448
449 q->pg_chunk.page = alloc_pages(gfp, order);
450 if (unlikely(!q->pg_chunk.page))
451 return -ENOMEM;
452 q->pg_chunk.va = page_address(q->pg_chunk.page);
453 q->pg_chunk.p_cnt = q->pg_chunk.va + (PAGE_SIZE << order) -
454 SGE_PG_RSVD;
455 q->pg_chunk.offset = 0;
456 mapping = pci_map_page(adapter->pdev, q->pg_chunk.page,
457 0, q->alloc_size, PCI_DMA_FROMDEVICE);
458 q->pg_chunk.mapping = mapping;
459 }
460 sd->pg_chunk = q->pg_chunk;
461
462 prefetch(sd->pg_chunk.p_cnt);
463
464 q->pg_chunk.offset += q->buf_size;
465 if (q->pg_chunk.offset == (PAGE_SIZE << order))
466 q->pg_chunk.page = NULL;
467 else {
468 q->pg_chunk.va += q->buf_size;
469 get_page(q->pg_chunk.page);
470 }
471
472 if (sd->pg_chunk.offset == 0)
473 *sd->pg_chunk.p_cnt = 1;
474 else
475 *sd->pg_chunk.p_cnt += 1;
476
477 return 0;
478}
479
480static inline void ring_fl_db(struct adapter *adap, struct sge_fl *q)
481{
482 if (q->pend_cred >= q->credits / 4) {
483 q->pend_cred = 0;
484 wmb();
485 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id));
486 }
487}
488
489/**
490 * refill_fl - refill an SGE free-buffer list
491 * @adapter: the adapter
492 * @q: the free-list to refill
493 * @n: the number of new buffers to allocate
494 * @gfp: the gfp flags for allocating new buffers
495 *
496 * (Re)populate an SGE free-buffer list with up to @n new packet buffers,
497 * allocated with the supplied gfp flags. The caller must assure that
498 * @n does not exceed the queue's capacity.
499 */
500static int refill_fl(struct adapter *adap, struct sge_fl *q, int n, gfp_t gfp)
501{
502 struct rx_sw_desc *sd = &q->sdesc[q->pidx];
503 struct rx_desc *d = &q->desc[q->pidx];
504 unsigned int count = 0;
505
506 while (n--) {
507 dma_addr_t mapping;
508 int err;
509
510 if (q->use_pages) {
511 if (unlikely(alloc_pg_chunk(adap, q, sd, gfp,
512 q->order))) {
513nomem: q->alloc_failed++;
514 break;
515 }
516 mapping = sd->pg_chunk.mapping + sd->pg_chunk.offset;
517 dma_unmap_addr_set(sd, dma_addr, mapping);
518
519 add_one_rx_chunk(mapping, d, q->gen);
520 pci_dma_sync_single_for_device(adap->pdev, mapping,
521 q->buf_size - SGE_PG_RSVD,
522 PCI_DMA_FROMDEVICE);
523 } else {
524 void *buf_start;
525
526 struct sk_buff *skb = alloc_skb(q->buf_size, gfp);
527 if (!skb)
528 goto nomem;
529
530 sd->skb = skb;
531 buf_start = skb->data;
532 err = add_one_rx_buf(buf_start, q->buf_size, d, sd,
533 q->gen, adap->pdev);
534 if (unlikely(err)) {
535 clear_rx_desc(adap->pdev, q, sd);
536 break;
537 }
538 }
539
540 d++;
541 sd++;
542 if (++q->pidx == q->size) {
543 q->pidx = 0;
544 q->gen ^= 1;
545 sd = q->sdesc;
546 d = q->desc;
547 }
548 count++;
549 }
550
551 q->credits += count;
552 q->pend_cred += count;
553 ring_fl_db(adap, q);
554
555 return count;
556}
557
558static inline void __refill_fl(struct adapter *adap, struct sge_fl *fl)
559{
560 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits),
561 GFP_ATOMIC | __GFP_COMP);
562}
563
564/**
565 * recycle_rx_buf - recycle a receive buffer
566 * @adapter: the adapter
567 * @q: the SGE free list
568 * @idx: index of buffer to recycle
569 *
570 * Recycles the specified buffer on the given free list by adding it at
571 * the next available slot on the list.
572 */
573static void recycle_rx_buf(struct adapter *adap, struct sge_fl *q,
574 unsigned int idx)
575{
576 struct rx_desc *from = &q->desc[idx];
577 struct rx_desc *to = &q->desc[q->pidx];
578
579 q->sdesc[q->pidx] = q->sdesc[idx];
580 to->addr_lo = from->addr_lo; /* already big endian */
581 to->addr_hi = from->addr_hi; /* likewise */
582 wmb();
583 to->len_gen = cpu_to_be32(V_FLD_GEN1(q->gen));
584 to->gen2 = cpu_to_be32(V_FLD_GEN2(q->gen));
585
586 if (++q->pidx == q->size) {
587 q->pidx = 0;
588 q->gen ^= 1;
589 }
590
591 q->credits++;
592 q->pend_cred++;
593 ring_fl_db(adap, q);
594}
595
596/**
597 * alloc_ring - allocate resources for an SGE descriptor ring
598 * @pdev: the PCI device
599 * @nelem: the number of descriptors
600 * @elem_size: the size of each descriptor
601 * @sw_size: the size of the SW state associated with each ring element
602 * @phys: the physical address of the allocated ring
603 * @metadata: address of the array holding the SW state for the ring
604 *
605 * Allocates resources for an SGE descriptor ring, such as Tx queues,
606 * free buffer lists, or response queues. Each SGE ring requires
607 * space for its HW descriptors plus, optionally, space for the SW state
608 * associated with each HW entry (the metadata). The function returns
609 * three values: the virtual address for the HW ring (the return value
610 * of the function), the physical address of the HW ring, and the address
611 * of the SW ring.
612 */
613static void *alloc_ring(struct pci_dev *pdev, size_t nelem, size_t elem_size,
614 size_t sw_size, dma_addr_t * phys, void *metadata)
615{
616 size_t len = nelem * elem_size;
617 void *s = NULL;
618 void *p = dma_alloc_coherent(&pdev->dev, len, phys, GFP_KERNEL);
619
620 if (!p)
621 return NULL;
622 if (sw_size && metadata) {
623 s = kcalloc(nelem, sw_size, GFP_KERNEL);
624
625 if (!s) {
626 dma_free_coherent(&pdev->dev, len, p, *phys);
627 return NULL;
628 }
629 *(void **)metadata = s;
630 }
631 memset(p, 0, len);
632 return p;
633}
634
635/**
636 * t3_reset_qset - reset a sge qset
637 * @q: the queue set
638 *
639 * Reset the qset structure.
640 * the NAPI structure is preserved in the event of
641 * the qset's reincarnation, for example during EEH recovery.
642 */
643static void t3_reset_qset(struct sge_qset *q)
644{
645 if (q->adap &&
646 !(q->adap->flags & NAPI_INIT)) {
647 memset(q, 0, sizeof(*q));
648 return;
649 }
650
651 q->adap = NULL;
652 memset(&q->rspq, 0, sizeof(q->rspq));
653 memset(q->fl, 0, sizeof(struct sge_fl) * SGE_RXQ_PER_SET);
654 memset(q->txq, 0, sizeof(struct sge_txq) * SGE_TXQ_PER_SET);
655 q->txq_stopped = 0;
656 q->tx_reclaim_timer.function = NULL; /* for t3_stop_sge_timers() */
657 q->rx_reclaim_timer.function = NULL;
658 q->nomem = 0;
659 napi_free_frags(&q->napi);
660}
661
662
663/**
664 * free_qset - free the resources of an SGE queue set
665 * @adapter: the adapter owning the queue set
666 * @q: the queue set
667 *
668 * Release the HW and SW resources associated with an SGE queue set, such
669 * as HW contexts, packet buffers, and descriptor rings. Traffic to the
670 * queue set must be quiesced prior to calling this.
671 */
672static void t3_free_qset(struct adapter *adapter, struct sge_qset *q)
673{
674 int i;
675 struct pci_dev *pdev = adapter->pdev;
676
677 for (i = 0; i < SGE_RXQ_PER_SET; ++i)
678 if (q->fl[i].desc) {
679 spin_lock_irq(&adapter->sge.reg_lock);
680 t3_sge_disable_fl(adapter, q->fl[i].cntxt_id);
681 spin_unlock_irq(&adapter->sge.reg_lock);
682 free_rx_bufs(pdev, &q->fl[i]);
683 kfree(q->fl[i].sdesc);
684 dma_free_coherent(&pdev->dev,
685 q->fl[i].size *
686 sizeof(struct rx_desc), q->fl[i].desc,
687 q->fl[i].phys_addr);
688 }
689
690 for (i = 0; i < SGE_TXQ_PER_SET; ++i)
691 if (q->txq[i].desc) {
692 spin_lock_irq(&adapter->sge.reg_lock);
693 t3_sge_enable_ecntxt(adapter, q->txq[i].cntxt_id, 0);
694 spin_unlock_irq(&adapter->sge.reg_lock);
695 if (q->txq[i].sdesc) {
696 free_tx_desc(adapter, &q->txq[i],
697 q->txq[i].in_use);
698 kfree(q->txq[i].sdesc);
699 }
700 dma_free_coherent(&pdev->dev,
701 q->txq[i].size *
702 sizeof(struct tx_desc),
703 q->txq[i].desc, q->txq[i].phys_addr);
704 __skb_queue_purge(&q->txq[i].sendq);
705 }
706
707 if (q->rspq.desc) {
708 spin_lock_irq(&adapter->sge.reg_lock);
709 t3_sge_disable_rspcntxt(adapter, q->rspq.cntxt_id);
710 spin_unlock_irq(&adapter->sge.reg_lock);
711 dma_free_coherent(&pdev->dev,
712 q->rspq.size * sizeof(struct rsp_desc),
713 q->rspq.desc, q->rspq.phys_addr);
714 }
715
716 t3_reset_qset(q);
717}
718
719/**
720 * init_qset_cntxt - initialize an SGE queue set context info
721 * @qs: the queue set
722 * @id: the queue set id
723 *
724 * Initializes the TIDs and context ids for the queues of a queue set.
725 */
726static void init_qset_cntxt(struct sge_qset *qs, unsigned int id)
727{
728 qs->rspq.cntxt_id = id;
729 qs->fl[0].cntxt_id = 2 * id;
730 qs->fl[1].cntxt_id = 2 * id + 1;
731 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id;
732 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id;
733 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id;
734 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id;
735 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id;
736}
737
738/**
739 * sgl_len - calculates the size of an SGL of the given capacity
740 * @n: the number of SGL entries
741 *
742 * Calculates the number of flits needed for a scatter/gather list that
743 * can hold the given number of entries.
744 */
745static inline unsigned int sgl_len(unsigned int n)
746{
747 /* alternatively: 3 * (n / 2) + 2 * (n & 1) */
748 return (3 * n) / 2 + (n & 1);
749}
750
751/**
752 * flits_to_desc - returns the num of Tx descriptors for the given flits
753 * @n: the number of flits
754 *
755 * Calculates the number of Tx descriptors needed for the supplied number
756 * of flits.
757 */
758static inline unsigned int flits_to_desc(unsigned int n)
759{
760 BUG_ON(n >= ARRAY_SIZE(flit_desc_map));
761 return flit_desc_map[n];
762}
763
764/**
765 * get_packet - return the next ingress packet buffer from a free list
766 * @adap: the adapter that received the packet
767 * @fl: the SGE free list holding the packet
768 * @len: the packet length including any SGE padding
769 * @drop_thres: # of remaining buffers before we start dropping packets
770 *
771 * Get the next packet from a free list and complete setup of the
772 * sk_buff. If the packet is small we make a copy and recycle the
773 * original buffer, otherwise we use the original buffer itself. If a
774 * positive drop threshold is supplied packets are dropped and their
775 * buffers recycled if (a) the number of remaining buffers is under the
776 * threshold and the packet is too big to copy, or (b) the packet should
777 * be copied but there is no memory for the copy.
778 */
779static struct sk_buff *get_packet(struct adapter *adap, struct sge_fl *fl,
780 unsigned int len, unsigned int drop_thres)
781{
782 struct sk_buff *skb = NULL;
783 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
784
785 prefetch(sd->skb->data);
786 fl->credits--;
787
788 if (len <= SGE_RX_COPY_THRES) {
789 skb = alloc_skb(len, GFP_ATOMIC);
790 if (likely(skb != NULL)) {
791 __skb_put(skb, len);
792 pci_dma_sync_single_for_cpu(adap->pdev,
793 dma_unmap_addr(sd, dma_addr), len,
794 PCI_DMA_FROMDEVICE);
795 memcpy(skb->data, sd->skb->data, len);
796 pci_dma_sync_single_for_device(adap->pdev,
797 dma_unmap_addr(sd, dma_addr), len,
798 PCI_DMA_FROMDEVICE);
799 } else if (!drop_thres)
800 goto use_orig_buf;
801recycle:
802 recycle_rx_buf(adap, fl, fl->cidx);
803 return skb;
804 }
805
806 if (unlikely(fl->credits < drop_thres) &&
807 refill_fl(adap, fl, min(MAX_RX_REFILL, fl->size - fl->credits - 1),
808 GFP_ATOMIC | __GFP_COMP) == 0)
809 goto recycle;
810
811use_orig_buf:
812 pci_unmap_single(adap->pdev, dma_unmap_addr(sd, dma_addr),
813 fl->buf_size, PCI_DMA_FROMDEVICE);
814 skb = sd->skb;
815 skb_put(skb, len);
816 __refill_fl(adap, fl);
817 return skb;
818}
819
820/**
821 * get_packet_pg - return the next ingress packet buffer from a free list
822 * @adap: the adapter that received the packet
823 * @fl: the SGE free list holding the packet
824 * @len: the packet length including any SGE padding
825 * @drop_thres: # of remaining buffers before we start dropping packets
826 *
827 * Get the next packet from a free list populated with page chunks.
828 * If the packet is small we make a copy and recycle the original buffer,
829 * otherwise we attach the original buffer as a page fragment to a fresh
830 * sk_buff. If a positive drop threshold is supplied packets are dropped
831 * and their buffers recycled if (a) the number of remaining buffers is
832 * under the threshold and the packet is too big to copy, or (b) there's
833 * no system memory.
834 *
835 * Note: this function is similar to @get_packet but deals with Rx buffers
836 * that are page chunks rather than sk_buffs.
837 */
838static struct sk_buff *get_packet_pg(struct adapter *adap, struct sge_fl *fl,
839 struct sge_rspq *q, unsigned int len,
840 unsigned int drop_thres)
841{
842 struct sk_buff *newskb, *skb;
843 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
844
845 dma_addr_t dma_addr = dma_unmap_addr(sd, dma_addr);
846
847 newskb = skb = q->pg_skb;
848 if (!skb && (len <= SGE_RX_COPY_THRES)) {
849 newskb = alloc_skb(len, GFP_ATOMIC);
850 if (likely(newskb != NULL)) {
851 __skb_put(newskb, len);
852 pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
853 PCI_DMA_FROMDEVICE);
854 memcpy(newskb->data, sd->pg_chunk.va, len);
855 pci_dma_sync_single_for_device(adap->pdev, dma_addr,
856 len,
857 PCI_DMA_FROMDEVICE);
858 } else if (!drop_thres)
859 return NULL;
860recycle:
861 fl->credits--;
862 recycle_rx_buf(adap, fl, fl->cidx);
863 q->rx_recycle_buf++;
864 return newskb;
865 }
866
867 if (unlikely(q->rx_recycle_buf || (!skb && fl->credits <= drop_thres)))
868 goto recycle;
869
870 prefetch(sd->pg_chunk.p_cnt);
871
872 if (!skb)
873 newskb = alloc_skb(SGE_RX_PULL_LEN, GFP_ATOMIC);
874
875 if (unlikely(!newskb)) {
876 if (!drop_thres)
877 return NULL;
878 goto recycle;
879 }
880
881 pci_dma_sync_single_for_cpu(adap->pdev, dma_addr, len,
882 PCI_DMA_FROMDEVICE);
883 (*sd->pg_chunk.p_cnt)--;
884 if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
885 pci_unmap_page(adap->pdev,
886 sd->pg_chunk.mapping,
887 fl->alloc_size,
888 PCI_DMA_FROMDEVICE);
889 if (!skb) {
890 __skb_put(newskb, SGE_RX_PULL_LEN);
891 memcpy(newskb->data, sd->pg_chunk.va, SGE_RX_PULL_LEN);
892 skb_fill_page_desc(newskb, 0, sd->pg_chunk.page,
893 sd->pg_chunk.offset + SGE_RX_PULL_LEN,
894 len - SGE_RX_PULL_LEN);
895 newskb->len = len;
896 newskb->data_len = len - SGE_RX_PULL_LEN;
897 newskb->truesize += newskb->data_len;
898 } else {
899 skb_fill_page_desc(newskb, skb_shinfo(newskb)->nr_frags,
900 sd->pg_chunk.page,
901 sd->pg_chunk.offset, len);
902 newskb->len += len;
903 newskb->data_len += len;
904 newskb->truesize += len;
905 }
906
907 fl->credits--;
908 /*
909 * We do not refill FLs here, we let the caller do it to overlap a
910 * prefetch.
911 */
912 return newskb;
913}
914
915/**
916 * get_imm_packet - return the next ingress packet buffer from a response
917 * @resp: the response descriptor containing the packet data
918 *
919 * Return a packet containing the immediate data of the given response.
920 */
921static inline struct sk_buff *get_imm_packet(const struct rsp_desc *resp)
922{
923 struct sk_buff *skb = alloc_skb(IMMED_PKT_SIZE, GFP_ATOMIC);
924
925 if (skb) {
926 __skb_put(skb, IMMED_PKT_SIZE);
927 skb_copy_to_linear_data(skb, resp->imm_data, IMMED_PKT_SIZE);
928 }
929 return skb;
930}
931
932/**
933 * calc_tx_descs - calculate the number of Tx descriptors for a packet
934 * @skb: the packet
935 *
936 * Returns the number of Tx descriptors needed for the given Ethernet
937 * packet. Ethernet packets require addition of WR and CPL headers.
938 */
939static inline unsigned int calc_tx_descs(const struct sk_buff *skb)
940{
941 unsigned int flits;
942
943 if (skb->len <= WR_LEN - sizeof(struct cpl_tx_pkt))
944 return 1;
945
946 flits = sgl_len(skb_shinfo(skb)->nr_frags + 1) + 2;
947 if (skb_shinfo(skb)->gso_size)
948 flits++;
949 return flits_to_desc(flits);
950}
951
952/**
953 * make_sgl - populate a scatter/gather list for a packet
954 * @skb: the packet
955 * @sgp: the SGL to populate
956 * @start: start address of skb main body data to include in the SGL
957 * @len: length of skb main body data to include in the SGL
958 * @pdev: the PCI device
959 *
960 * Generates a scatter/gather list for the buffers that make up a packet
961 * and returns the SGL size in 8-byte words. The caller must size the SGL
962 * appropriately.
963 */
964static inline unsigned int make_sgl(const struct sk_buff *skb,
965 struct sg_ent *sgp, unsigned char *start,
966 unsigned int len, struct pci_dev *pdev)
967{
968 dma_addr_t mapping;
969 unsigned int i, j = 0, nfrags;
970
971 if (len) {
972 mapping = pci_map_single(pdev, start, len, PCI_DMA_TODEVICE);
973 sgp->len[0] = cpu_to_be32(len);
974 sgp->addr[0] = cpu_to_be64(mapping);
975 j = 1;
976 }
977
978 nfrags = skb_shinfo(skb)->nr_frags;
979 for (i = 0; i < nfrags; i++) {
980 skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
981
982 mapping = pci_map_page(pdev, frag->page, frag->page_offset,
983 frag->size, PCI_DMA_TODEVICE);
984 sgp->len[j] = cpu_to_be32(frag->size);
985 sgp->addr[j] = cpu_to_be64(mapping);
986 j ^= 1;
987 if (j == 0)
988 ++sgp;
989 }
990 if (j)
991 sgp->len[j] = 0;
992 return ((nfrags + (len != 0)) * 3) / 2 + j;
993}
994
995/**
996 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell
997 * @adap: the adapter
998 * @q: the Tx queue
999 *
1000 * Ring the doorbel if a Tx queue is asleep. There is a natural race,
1001 * where the HW is going to sleep just after we checked, however,
1002 * then the interrupt handler will detect the outstanding TX packet
1003 * and ring the doorbell for us.
1004 *
1005 * When GTS is disabled we unconditionally ring the doorbell.
1006 */
1007static inline void check_ring_tx_db(struct adapter *adap, struct sge_txq *q)
1008{
1009#if USE_GTS
1010 clear_bit(TXQ_LAST_PKT_DB, &q->flags);
1011 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) {
1012 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1013 t3_write_reg(adap, A_SG_KDOORBELL,
1014 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1015 }
1016#else
1017 wmb(); /* write descriptors before telling HW */
1018 t3_write_reg(adap, A_SG_KDOORBELL,
1019 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1020#endif
1021}
1022
1023static inline void wr_gen2(struct tx_desc *d, unsigned int gen)
1024{
1025#if SGE_NUM_GENBITS == 2
1026 d->flit[TX_DESC_FLITS - 1] = cpu_to_be64(gen);
1027#endif
1028}
1029
1030/**
1031 * write_wr_hdr_sgl - write a WR header and, optionally, SGL
1032 * @ndesc: number of Tx descriptors spanned by the SGL
1033 * @skb: the packet corresponding to the WR
1034 * @d: first Tx descriptor to be written
1035 * @pidx: index of above descriptors
1036 * @q: the SGE Tx queue
1037 * @sgl: the SGL
1038 * @flits: number of flits to the start of the SGL in the first descriptor
1039 * @sgl_flits: the SGL size in flits
1040 * @gen: the Tx descriptor generation
1041 * @wr_hi: top 32 bits of WR header based on WR type (big endian)
1042 * @wr_lo: low 32 bits of WR header based on WR type (big endian)
1043 *
1044 * Write a work request header and an associated SGL. If the SGL is
1045 * small enough to fit into one Tx descriptor it has already been written
1046 * and we just need to write the WR header. Otherwise we distribute the
1047 * SGL across the number of descriptors it spans.
1048 */
1049static void write_wr_hdr_sgl(unsigned int ndesc, struct sk_buff *skb,
1050 struct tx_desc *d, unsigned int pidx,
1051 const struct sge_txq *q,
1052 const struct sg_ent *sgl,
1053 unsigned int flits, unsigned int sgl_flits,
1054 unsigned int gen, __be32 wr_hi,
1055 __be32 wr_lo)
1056{
1057 struct work_request_hdr *wrp = (struct work_request_hdr *)d;
1058 struct tx_sw_desc *sd = &q->sdesc[pidx];
1059
1060 sd->skb = skb;
1061 if (need_skb_unmap()) {
1062 sd->fragidx = 0;
1063 sd->addr_idx = 0;
1064 sd->sflit = flits;
1065 }
1066
1067 if (likely(ndesc == 1)) {
1068 sd->eop = 1;
1069 wrp->wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) |
1070 V_WR_SGLSFLT(flits)) | wr_hi;
1071 wmb();
1072 wrp->wr_lo = htonl(V_WR_LEN(flits + sgl_flits) |
1073 V_WR_GEN(gen)) | wr_lo;
1074 wr_gen2(d, gen);
1075 } else {
1076 unsigned int ogen = gen;
1077 const u64 *fp = (const u64 *)sgl;
1078 struct work_request_hdr *wp = wrp;
1079
1080 wrp->wr_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) |
1081 V_WR_SGLSFLT(flits)) | wr_hi;
1082
1083 while (sgl_flits) {
1084 unsigned int avail = WR_FLITS - flits;
1085
1086 if (avail > sgl_flits)
1087 avail = sgl_flits;
1088 memcpy(&d->flit[flits], fp, avail * sizeof(*fp));
1089 sgl_flits -= avail;
1090 ndesc--;
1091 if (!sgl_flits)
1092 break;
1093
1094 fp += avail;
1095 d++;
1096 sd->eop = 0;
1097 sd++;
1098 if (++pidx == q->size) {
1099 pidx = 0;
1100 gen ^= 1;
1101 d = q->desc;
1102 sd = q->sdesc;
1103 }
1104
1105 sd->skb = skb;
1106 wrp = (struct work_request_hdr *)d;
1107 wrp->wr_hi = htonl(V_WR_DATATYPE(1) |
1108 V_WR_SGLSFLT(1)) | wr_hi;
1109 wrp->wr_lo = htonl(V_WR_LEN(min(WR_FLITS,
1110 sgl_flits + 1)) |
1111 V_WR_GEN(gen)) | wr_lo;
1112 wr_gen2(d, gen);
1113 flits = 1;
1114 }
1115 sd->eop = 1;
1116 wrp->wr_hi |= htonl(F_WR_EOP);
1117 wmb();
1118 wp->wr_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo;
1119 wr_gen2((struct tx_desc *)wp, ogen);
1120 WARN_ON(ndesc != 0);
1121 }
1122}
1123
1124/**
1125 * write_tx_pkt_wr - write a TX_PKT work request
1126 * @adap: the adapter
1127 * @skb: the packet to send
1128 * @pi: the egress interface
1129 * @pidx: index of the first Tx descriptor to write
1130 * @gen: the generation value to use
1131 * @q: the Tx queue
1132 * @ndesc: number of descriptors the packet will occupy
1133 * @compl: the value of the COMPL bit to use
1134 *
1135 * Generate a TX_PKT work request to send the supplied packet.
1136 */
1137static void write_tx_pkt_wr(struct adapter *adap, struct sk_buff *skb,
1138 const struct port_info *pi,
1139 unsigned int pidx, unsigned int gen,
1140 struct sge_txq *q, unsigned int ndesc,
1141 unsigned int compl)
1142{
1143 unsigned int flits, sgl_flits, cntrl, tso_info;
1144 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1145 struct tx_desc *d = &q->desc[pidx];
1146 struct cpl_tx_pkt *cpl = (struct cpl_tx_pkt *)d;
1147
1148 cpl->len = htonl(skb->len);
1149 cntrl = V_TXPKT_INTF(pi->port_id);
1150
1151 if (vlan_tx_tag_present(skb))
1152 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN(vlan_tx_tag_get(skb));
1153
1154 tso_info = V_LSO_MSS(skb_shinfo(skb)->gso_size);
1155 if (tso_info) {
1156 int eth_type;
1157 struct cpl_tx_pkt_lso *hdr = (struct cpl_tx_pkt_lso *)cpl;
1158
1159 d->flit[2] = 0;
1160 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT_LSO);
1161 hdr->cntrl = htonl(cntrl);
1162 eth_type = skb_network_offset(skb) == ETH_HLEN ?
1163 CPL_ETH_II : CPL_ETH_II_VLAN;
1164 tso_info |= V_LSO_ETH_TYPE(eth_type) |
1165 V_LSO_IPHDR_WORDS(ip_hdr(skb)->ihl) |
1166 V_LSO_TCPHDR_WORDS(tcp_hdr(skb)->doff);
1167 hdr->lso_info = htonl(tso_info);
1168 flits = 3;
1169 } else {
1170 cntrl |= V_TXPKT_OPCODE(CPL_TX_PKT);
1171 cntrl |= F_TXPKT_IPCSUM_DIS; /* SW calculates IP csum */
1172 cntrl |= V_TXPKT_L4CSUM_DIS(skb->ip_summed != CHECKSUM_PARTIAL);
1173 cpl->cntrl = htonl(cntrl);
1174
1175 if (skb->len <= WR_LEN - sizeof(*cpl)) {
1176 q->sdesc[pidx].skb = NULL;
1177 if (!skb->data_len)
1178 skb_copy_from_linear_data(skb, &d->flit[2],
1179 skb->len);
1180 else
1181 skb_copy_bits(skb, 0, &d->flit[2], skb->len);
1182
1183 flits = (skb->len + 7) / 8 + 2;
1184 cpl->wr.wr_hi = htonl(V_WR_BCNTLFLT(skb->len & 7) |
1185 V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT)
1186 | F_WR_SOP | F_WR_EOP | compl);
1187 wmb();
1188 cpl->wr.wr_lo = htonl(V_WR_LEN(flits) | V_WR_GEN(gen) |
1189 V_WR_TID(q->token));
1190 wr_gen2(d, gen);
1191 kfree_skb(skb);
1192 return;
1193 }
1194
1195 flits = 2;
1196 }
1197
1198 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1199 sgl_flits = make_sgl(skb, sgp, skb->data, skb_headlen(skb), adap->pdev);
1200
1201 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits, gen,
1202 htonl(V_WR_OP(FW_WROPCODE_TUNNEL_TX_PKT) | compl),
1203 htonl(V_WR_TID(q->token)));
1204}
1205
1206static inline void t3_stop_tx_queue(struct netdev_queue *txq,
1207 struct sge_qset *qs, struct sge_txq *q)
1208{
1209 netif_tx_stop_queue(txq);
1210 set_bit(TXQ_ETH, &qs->txq_stopped);
1211 q->stops++;
1212}
1213
1214/**
1215 * eth_xmit - add a packet to the Ethernet Tx queue
1216 * @skb: the packet
1217 * @dev: the egress net device
1218 *
1219 * Add a packet to an SGE Tx queue. Runs with softirqs disabled.
1220 */
1221netdev_tx_t t3_eth_xmit(struct sk_buff *skb, struct net_device *dev)
1222{
1223 int qidx;
1224 unsigned int ndesc, pidx, credits, gen, compl;
1225 const struct port_info *pi = netdev_priv(dev);
1226 struct adapter *adap = pi->adapter;
1227 struct netdev_queue *txq;
1228 struct sge_qset *qs;
1229 struct sge_txq *q;
1230
1231 /*
1232 * The chip min packet length is 9 octets but play safe and reject
1233 * anything shorter than an Ethernet header.
1234 */
1235 if (unlikely(skb->len < ETH_HLEN)) {
1236 dev_kfree_skb(skb);
1237 return NETDEV_TX_OK;
1238 }
1239
1240 qidx = skb_get_queue_mapping(skb);
1241 qs = &pi->qs[qidx];
1242 q = &qs->txq[TXQ_ETH];
1243 txq = netdev_get_tx_queue(dev, qidx);
1244
1245 reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1246
1247 credits = q->size - q->in_use;
1248 ndesc = calc_tx_descs(skb);
1249
1250 if (unlikely(credits < ndesc)) {
1251 t3_stop_tx_queue(txq, qs, q);
1252 dev_err(&adap->pdev->dev,
1253 "%s: Tx ring %u full while queue awake!\n",
1254 dev->name, q->cntxt_id & 7);
1255 return NETDEV_TX_BUSY;
1256 }
1257
1258 q->in_use += ndesc;
1259 if (unlikely(credits - ndesc < q->stop_thres)) {
1260 t3_stop_tx_queue(txq, qs, q);
1261
1262 if (should_restart_tx(q) &&
1263 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1264 q->restarts++;
1265 netif_tx_start_queue(txq);
1266 }
1267 }
1268
1269 gen = q->gen;
1270 q->unacked += ndesc;
1271 compl = (q->unacked & 8) << (S_WR_COMPL - 3);
1272 q->unacked &= 7;
1273 pidx = q->pidx;
1274 q->pidx += ndesc;
1275 if (q->pidx >= q->size) {
1276 q->pidx -= q->size;
1277 q->gen ^= 1;
1278 }
1279
1280 /* update port statistics */
1281 if (skb->ip_summed == CHECKSUM_COMPLETE)
1282 qs->port_stats[SGE_PSTAT_TX_CSUM]++;
1283 if (skb_shinfo(skb)->gso_size)
1284 qs->port_stats[SGE_PSTAT_TSO]++;
1285 if (vlan_tx_tag_present(skb))
1286 qs->port_stats[SGE_PSTAT_VLANINS]++;
1287
1288 /*
1289 * We do not use Tx completion interrupts to free DMAd Tx packets.
1290 * This is good for performance but means that we rely on new Tx
1291 * packets arriving to run the destructors of completed packets,
1292 * which open up space in their sockets' send queues. Sometimes
1293 * we do not get such new packets causing Tx to stall. A single
1294 * UDP transmitter is a good example of this situation. We have
1295 * a clean up timer that periodically reclaims completed packets
1296 * but it doesn't run often enough (nor do we want it to) to prevent
1297 * lengthy stalls. A solution to this problem is to run the
1298 * destructor early, after the packet is queued but before it's DMAd.
1299 * A cons is that we lie to socket memory accounting, but the amount
1300 * of extra memory is reasonable (limited by the number of Tx
1301 * descriptors), the packets do actually get freed quickly by new
1302 * packets almost always, and for protocols like TCP that wait for
1303 * acks to really free up the data the extra memory is even less.
1304 * On the positive side we run the destructors on the sending CPU
1305 * rather than on a potentially different completing CPU, usually a
1306 * good thing. We also run them without holding our Tx queue lock,
1307 * unlike what reclaim_completed_tx() would otherwise do.
1308 *
1309 * Run the destructor before telling the DMA engine about the packet
1310 * to make sure it doesn't complete and get freed prematurely.
1311 */
1312 if (likely(!skb_shared(skb)))
1313 skb_orphan(skb);
1314
1315 write_tx_pkt_wr(adap, skb, pi, pidx, gen, q, ndesc, compl);
1316 check_ring_tx_db(adap, q);
1317 return NETDEV_TX_OK;
1318}
1319
1320/**
1321 * write_imm - write a packet into a Tx descriptor as immediate data
1322 * @d: the Tx descriptor to write
1323 * @skb: the packet
1324 * @len: the length of packet data to write as immediate data
1325 * @gen: the generation bit value to write
1326 *
1327 * Writes a packet as immediate data into a Tx descriptor. The packet
1328 * contains a work request at its beginning. We must write the packet
1329 * carefully so the SGE doesn't read it accidentally before it's written
1330 * in its entirety.
1331 */
1332static inline void write_imm(struct tx_desc *d, struct sk_buff *skb,
1333 unsigned int len, unsigned int gen)
1334{
1335 struct work_request_hdr *from = (struct work_request_hdr *)skb->data;
1336 struct work_request_hdr *to = (struct work_request_hdr *)d;
1337
1338 if (likely(!skb->data_len))
1339 memcpy(&to[1], &from[1], len - sizeof(*from));
1340 else
1341 skb_copy_bits(skb, sizeof(*from), &to[1], len - sizeof(*from));
1342
1343 to->wr_hi = from->wr_hi | htonl(F_WR_SOP | F_WR_EOP |
1344 V_WR_BCNTLFLT(len & 7));
1345 wmb();
1346 to->wr_lo = from->wr_lo | htonl(V_WR_GEN(gen) |
1347 V_WR_LEN((len + 7) / 8));
1348 wr_gen2(d, gen);
1349 kfree_skb(skb);
1350}
1351
1352/**
1353 * check_desc_avail - check descriptor availability on a send queue
1354 * @adap: the adapter
1355 * @q: the send queue
1356 * @skb: the packet needing the descriptors
1357 * @ndesc: the number of Tx descriptors needed
1358 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1359 *
1360 * Checks if the requested number of Tx descriptors is available on an
1361 * SGE send queue. If the queue is already suspended or not enough
1362 * descriptors are available the packet is queued for later transmission.
1363 * Must be called with the Tx queue locked.
1364 *
1365 * Returns 0 if enough descriptors are available, 1 if there aren't
1366 * enough descriptors and the packet has been queued, and 2 if the caller
1367 * needs to retry because there weren't enough descriptors at the
1368 * beginning of the call but some freed up in the mean time.
1369 */
1370static inline int check_desc_avail(struct adapter *adap, struct sge_txq *q,
1371 struct sk_buff *skb, unsigned int ndesc,
1372 unsigned int qid)
1373{
1374 if (unlikely(!skb_queue_empty(&q->sendq))) {
1375 addq_exit:__skb_queue_tail(&q->sendq, skb);
1376 return 1;
1377 }
1378 if (unlikely(q->size - q->in_use < ndesc)) {
1379 struct sge_qset *qs = txq_to_qset(q, qid);
1380
1381 set_bit(qid, &qs->txq_stopped);
1382 smp_mb__after_clear_bit();
1383
1384 if (should_restart_tx(q) &&
1385 test_and_clear_bit(qid, &qs->txq_stopped))
1386 return 2;
1387
1388 q->stops++;
1389 goto addq_exit;
1390 }
1391 return 0;
1392}
1393
1394/**
1395 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1396 * @q: the SGE control Tx queue
1397 *
1398 * This is a variant of reclaim_completed_tx() that is used for Tx queues
1399 * that send only immediate data (presently just the control queues) and
1400 * thus do not have any sk_buffs to release.
1401 */
1402static inline void reclaim_completed_tx_imm(struct sge_txq *q)
1403{
1404 unsigned int reclaim = q->processed - q->cleaned;
1405
1406 q->in_use -= reclaim;
1407 q->cleaned += reclaim;
1408}
1409
1410static inline int immediate(const struct sk_buff *skb)
1411{
1412 return skb->len <= WR_LEN;
1413}
1414
1415/**
1416 * ctrl_xmit - send a packet through an SGE control Tx queue
1417 * @adap: the adapter
1418 * @q: the control queue
1419 * @skb: the packet
1420 *
1421 * Send a packet through an SGE control Tx queue. Packets sent through
1422 * a control queue must fit entirely as immediate data in a single Tx
1423 * descriptor and have no page fragments.
1424 */
1425static int ctrl_xmit(struct adapter *adap, struct sge_txq *q,
1426 struct sk_buff *skb)
1427{
1428 int ret;
1429 struct work_request_hdr *wrp = (struct work_request_hdr *)skb->data;
1430
1431 if (unlikely(!immediate(skb))) {
1432 WARN_ON(1);
1433 dev_kfree_skb(skb);
1434 return NET_XMIT_SUCCESS;
1435 }
1436
1437 wrp->wr_hi |= htonl(F_WR_SOP | F_WR_EOP);
1438 wrp->wr_lo = htonl(V_WR_TID(q->token));
1439
1440 spin_lock(&q->lock);
1441 again:reclaim_completed_tx_imm(q);
1442
1443 ret = check_desc_avail(adap, q, skb, 1, TXQ_CTRL);
1444 if (unlikely(ret)) {
1445 if (ret == 1) {
1446 spin_unlock(&q->lock);
1447 return NET_XMIT_CN;
1448 }
1449 goto again;
1450 }
1451
1452 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1453
1454 q->in_use++;
1455 if (++q->pidx >= q->size) {
1456 q->pidx = 0;
1457 q->gen ^= 1;
1458 }
1459 spin_unlock(&q->lock);
1460 wmb();
1461 t3_write_reg(adap, A_SG_KDOORBELL,
1462 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1463 return NET_XMIT_SUCCESS;
1464}
1465
1466/**
1467 * restart_ctrlq - restart a suspended control queue
1468 * @qs: the queue set cotaining the control queue
1469 *
1470 * Resumes transmission on a suspended Tx control queue.
1471 */
1472static void restart_ctrlq(unsigned long data)
1473{
1474 struct sk_buff *skb;
1475 struct sge_qset *qs = (struct sge_qset *)data;
1476 struct sge_txq *q = &qs->txq[TXQ_CTRL];
1477
1478 spin_lock(&q->lock);
1479 again:reclaim_completed_tx_imm(q);
1480
1481 while (q->in_use < q->size &&
1482 (skb = __skb_dequeue(&q->sendq)) != NULL) {
1483
1484 write_imm(&q->desc[q->pidx], skb, skb->len, q->gen);
1485
1486 if (++q->pidx >= q->size) {
1487 q->pidx = 0;
1488 q->gen ^= 1;
1489 }
1490 q->in_use++;
1491 }
1492
1493 if (!skb_queue_empty(&q->sendq)) {
1494 set_bit(TXQ_CTRL, &qs->txq_stopped);
1495 smp_mb__after_clear_bit();
1496
1497 if (should_restart_tx(q) &&
1498 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped))
1499 goto again;
1500 q->stops++;
1501 }
1502
1503 spin_unlock(&q->lock);
1504 wmb();
1505 t3_write_reg(qs->adap, A_SG_KDOORBELL,
1506 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1507}
1508
1509/*
1510 * Send a management message through control queue 0
1511 */
1512int t3_mgmt_tx(struct adapter *adap, struct sk_buff *skb)
1513{
1514 int ret;
1515 local_bh_disable();
1516 ret = ctrl_xmit(adap, &adap->sge.qs[0].txq[TXQ_CTRL], skb);
1517 local_bh_enable();
1518
1519 return ret;
1520}
1521
1522/**
1523 * deferred_unmap_destructor - unmap a packet when it is freed
1524 * @skb: the packet
1525 *
1526 * This is the packet destructor used for Tx packets that need to remain
1527 * mapped until they are freed rather than until their Tx descriptors are
1528 * freed.
1529 */
1530static void deferred_unmap_destructor(struct sk_buff *skb)
1531{
1532 int i;
1533 const dma_addr_t *p;
1534 const struct skb_shared_info *si;
1535 const struct deferred_unmap_info *dui;
1536
1537 dui = (struct deferred_unmap_info *)skb->head;
1538 p = dui->addr;
1539
1540 if (skb->tail - skb->transport_header)
1541 pci_unmap_single(dui->pdev, *p++,
1542 skb->tail - skb->transport_header,
1543 PCI_DMA_TODEVICE);
1544
1545 si = skb_shinfo(skb);
1546 for (i = 0; i < si->nr_frags; i++)
1547 pci_unmap_page(dui->pdev, *p++, si->frags[i].size,
1548 PCI_DMA_TODEVICE);
1549}
1550
1551static void setup_deferred_unmapping(struct sk_buff *skb, struct pci_dev *pdev,
1552 const struct sg_ent *sgl, int sgl_flits)
1553{
1554 dma_addr_t *p;
1555 struct deferred_unmap_info *dui;
1556
1557 dui = (struct deferred_unmap_info *)skb->head;
1558 dui->pdev = pdev;
1559 for (p = dui->addr; sgl_flits >= 3; sgl++, sgl_flits -= 3) {
1560 *p++ = be64_to_cpu(sgl->addr[0]);
1561 *p++ = be64_to_cpu(sgl->addr[1]);
1562 }
1563 if (sgl_flits)
1564 *p = be64_to_cpu(sgl->addr[0]);
1565}
1566
1567/**
1568 * write_ofld_wr - write an offload work request
1569 * @adap: the adapter
1570 * @skb: the packet to send
1571 * @q: the Tx queue
1572 * @pidx: index of the first Tx descriptor to write
1573 * @gen: the generation value to use
1574 * @ndesc: number of descriptors the packet will occupy
1575 *
1576 * Write an offload work request to send the supplied packet. The packet
1577 * data already carry the work request with most fields populated.
1578 */
1579static void write_ofld_wr(struct adapter *adap, struct sk_buff *skb,
1580 struct sge_txq *q, unsigned int pidx,
1581 unsigned int gen, unsigned int ndesc)
1582{
1583 unsigned int sgl_flits, flits;
1584 struct work_request_hdr *from;
1585 struct sg_ent *sgp, sgl[MAX_SKB_FRAGS / 2 + 1];
1586 struct tx_desc *d = &q->desc[pidx];
1587
1588 if (immediate(skb)) {
1589 q->sdesc[pidx].skb = NULL;
1590 write_imm(d, skb, skb->len, gen);
1591 return;
1592 }
1593
1594 /* Only TX_DATA builds SGLs */
1595
1596 from = (struct work_request_hdr *)skb->data;
1597 memcpy(&d->flit[1], &from[1],
1598 skb_transport_offset(skb) - sizeof(*from));
1599
1600 flits = skb_transport_offset(skb) / 8;
1601 sgp = ndesc == 1 ? (struct sg_ent *)&d->flit[flits] : sgl;
1602 sgl_flits = make_sgl(skb, sgp, skb_transport_header(skb),
1603 skb->tail - skb->transport_header,
1604 adap->pdev);
1605 if (need_skb_unmap()) {
1606 setup_deferred_unmapping(skb, adap->pdev, sgp, sgl_flits);
1607 skb->destructor = deferred_unmap_destructor;
1608 }
1609
1610 write_wr_hdr_sgl(ndesc, skb, d, pidx, q, sgl, flits, sgl_flits,
1611 gen, from->wr_hi, from->wr_lo);
1612}
1613
1614/**
1615 * calc_tx_descs_ofld - calculate # of Tx descriptors for an offload packet
1616 * @skb: the packet
1617 *
1618 * Returns the number of Tx descriptors needed for the given offload
1619 * packet. These packets are already fully constructed.
1620 */
1621static inline unsigned int calc_tx_descs_ofld(const struct sk_buff *skb)
1622{
1623 unsigned int flits, cnt;
1624
1625 if (skb->len <= WR_LEN)
1626 return 1; /* packet fits as immediate data */
1627
1628 flits = skb_transport_offset(skb) / 8; /* headers */
1629 cnt = skb_shinfo(skb)->nr_frags;
1630 if (skb->tail != skb->transport_header)
1631 cnt++;
1632 return flits_to_desc(flits + sgl_len(cnt));
1633}
1634
1635/**
1636 * ofld_xmit - send a packet through an offload queue
1637 * @adap: the adapter
1638 * @q: the Tx offload queue
1639 * @skb: the packet
1640 *
1641 * Send an offload packet through an SGE offload queue.
1642 */
1643static int ofld_xmit(struct adapter *adap, struct sge_txq *q,
1644 struct sk_buff *skb)
1645{
1646 int ret;
1647 unsigned int ndesc = calc_tx_descs_ofld(skb), pidx, gen;
1648
1649 spin_lock(&q->lock);
1650again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1651
1652 ret = check_desc_avail(adap, q, skb, ndesc, TXQ_OFLD);
1653 if (unlikely(ret)) {
1654 if (ret == 1) {
1655 skb->priority = ndesc; /* save for restart */
1656 spin_unlock(&q->lock);
1657 return NET_XMIT_CN;
1658 }
1659 goto again;
1660 }
1661
1662 gen = q->gen;
1663 q->in_use += ndesc;
1664 pidx = q->pidx;
1665 q->pidx += ndesc;
1666 if (q->pidx >= q->size) {
1667 q->pidx -= q->size;
1668 q->gen ^= 1;
1669 }
1670 spin_unlock(&q->lock);
1671
1672 write_ofld_wr(adap, skb, q, pidx, gen, ndesc);
1673 check_ring_tx_db(adap, q);
1674 return NET_XMIT_SUCCESS;
1675}
1676
1677/**
1678 * restart_offloadq - restart a suspended offload queue
1679 * @qs: the queue set cotaining the offload queue
1680 *
1681 * Resumes transmission on a suspended Tx offload queue.
1682 */
1683static void restart_offloadq(unsigned long data)
1684{
1685 struct sk_buff *skb;
1686 struct sge_qset *qs = (struct sge_qset *)data;
1687 struct sge_txq *q = &qs->txq[TXQ_OFLD];
1688 const struct port_info *pi = netdev_priv(qs->netdev);
1689 struct adapter *adap = pi->adapter;
1690
1691 spin_lock(&q->lock);
1692again: reclaim_completed_tx(adap, q, TX_RECLAIM_CHUNK);
1693
1694 while ((skb = skb_peek(&q->sendq)) != NULL) {
1695 unsigned int gen, pidx;
1696 unsigned int ndesc = skb->priority;
1697
1698 if (unlikely(q->size - q->in_use < ndesc)) {
1699 set_bit(TXQ_OFLD, &qs->txq_stopped);
1700 smp_mb__after_clear_bit();
1701
1702 if (should_restart_tx(q) &&
1703 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped))
1704 goto again;
1705 q->stops++;
1706 break;
1707 }
1708
1709 gen = q->gen;
1710 q->in_use += ndesc;
1711 pidx = q->pidx;
1712 q->pidx += ndesc;
1713 if (q->pidx >= q->size) {
1714 q->pidx -= q->size;
1715 q->gen ^= 1;
1716 }
1717 __skb_unlink(skb, &q->sendq);
1718 spin_unlock(&q->lock);
1719
1720 write_ofld_wr(adap, skb, q, pidx, gen, ndesc);
1721 spin_lock(&q->lock);
1722 }
1723 spin_unlock(&q->lock);
1724
1725#if USE_GTS
1726 set_bit(TXQ_RUNNING, &q->flags);
1727 set_bit(TXQ_LAST_PKT_DB, &q->flags);
1728#endif
1729 wmb();
1730 t3_write_reg(adap, A_SG_KDOORBELL,
1731 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id));
1732}
1733
1734/**
1735 * queue_set - return the queue set a packet should use
1736 * @skb: the packet
1737 *
1738 * Maps a packet to the SGE queue set it should use. The desired queue
1739 * set is carried in bits 1-3 in the packet's priority.
1740 */
1741static inline int queue_set(const struct sk_buff *skb)
1742{
1743 return skb->priority >> 1;
1744}
1745
1746/**
1747 * is_ctrl_pkt - return whether an offload packet is a control packet
1748 * @skb: the packet
1749 *
1750 * Determines whether an offload packet should use an OFLD or a CTRL
1751 * Tx queue. This is indicated by bit 0 in the packet's priority.
1752 */
1753static inline int is_ctrl_pkt(const struct sk_buff *skb)
1754{
1755 return skb->priority & 1;
1756}
1757
1758/**
1759 * t3_offload_tx - send an offload packet
1760 * @tdev: the offload device to send to
1761 * @skb: the packet
1762 *
1763 * Sends an offload packet. We use the packet priority to select the
1764 * appropriate Tx queue as follows: bit 0 indicates whether the packet
1765 * should be sent as regular or control, bits 1-3 select the queue set.
1766 */
1767int t3_offload_tx(struct t3cdev *tdev, struct sk_buff *skb)
1768{
1769 struct adapter *adap = tdev2adap(tdev);
1770 struct sge_qset *qs = &adap->sge.qs[queue_set(skb)];
1771
1772 if (unlikely(is_ctrl_pkt(skb)))
1773 return ctrl_xmit(adap, &qs->txq[TXQ_CTRL], skb);
1774
1775 return ofld_xmit(adap, &qs->txq[TXQ_OFLD], skb);
1776}
1777
1778/**
1779 * offload_enqueue - add an offload packet to an SGE offload receive queue
1780 * @q: the SGE response queue
1781 * @skb: the packet
1782 *
1783 * Add a new offload packet to an SGE response queue's offload packet
1784 * queue. If the packet is the first on the queue it schedules the RX
1785 * softirq to process the queue.
1786 */
1787static inline void offload_enqueue(struct sge_rspq *q, struct sk_buff *skb)
1788{
1789 int was_empty = skb_queue_empty(&q->rx_queue);
1790
1791 __skb_queue_tail(&q->rx_queue, skb);
1792
1793 if (was_empty) {
1794 struct sge_qset *qs = rspq_to_qset(q);
1795
1796 napi_schedule(&qs->napi);
1797 }
1798}
1799
1800/**
1801 * deliver_partial_bundle - deliver a (partial) bundle of Rx offload pkts
1802 * @tdev: the offload device that will be receiving the packets
1803 * @q: the SGE response queue that assembled the bundle
1804 * @skbs: the partial bundle
1805 * @n: the number of packets in the bundle
1806 *
1807 * Delivers a (partial) bundle of Rx offload packets to an offload device.
1808 */
1809static inline void deliver_partial_bundle(struct t3cdev *tdev,
1810 struct sge_rspq *q,
1811 struct sk_buff *skbs[], int n)
1812{
1813 if (n) {
1814 q->offload_bundles++;
1815 tdev->recv(tdev, skbs, n);
1816 }
1817}
1818
1819/**
1820 * ofld_poll - NAPI handler for offload packets in interrupt mode
1821 * @dev: the network device doing the polling
1822 * @budget: polling budget
1823 *
1824 * The NAPI handler for offload packets when a response queue is serviced
1825 * by the hard interrupt handler, i.e., when it's operating in non-polling
1826 * mode. Creates small packet batches and sends them through the offload
1827 * receive handler. Batches need to be of modest size as we do prefetches
1828 * on the packets in each.
1829 */
1830static int ofld_poll(struct napi_struct *napi, int budget)
1831{
1832 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
1833 struct sge_rspq *q = &qs->rspq;
1834 struct adapter *adapter = qs->adap;
1835 int work_done = 0;
1836
1837 while (work_done < budget) {
1838 struct sk_buff *skb, *tmp, *skbs[RX_BUNDLE_SIZE];
1839 struct sk_buff_head queue;
1840 int ngathered;
1841
1842 spin_lock_irq(&q->lock);
1843 __skb_queue_head_init(&queue);
1844 skb_queue_splice_init(&q->rx_queue, &queue);
1845 if (skb_queue_empty(&queue)) {
1846 napi_complete(napi);
1847 spin_unlock_irq(&q->lock);
1848 return work_done;
1849 }
1850 spin_unlock_irq(&q->lock);
1851
1852 ngathered = 0;
1853 skb_queue_walk_safe(&queue, skb, tmp) {
1854 if (work_done >= budget)
1855 break;
1856 work_done++;
1857
1858 __skb_unlink(skb, &queue);
1859 prefetch(skb->data);
1860 skbs[ngathered] = skb;
1861 if (++ngathered == RX_BUNDLE_SIZE) {
1862 q->offload_bundles++;
1863 adapter->tdev.recv(&adapter->tdev, skbs,
1864 ngathered);
1865 ngathered = 0;
1866 }
1867 }
1868 if (!skb_queue_empty(&queue)) {
1869 /* splice remaining packets back onto Rx queue */
1870 spin_lock_irq(&q->lock);
1871 skb_queue_splice(&queue, &q->rx_queue);
1872 spin_unlock_irq(&q->lock);
1873 }
1874 deliver_partial_bundle(&adapter->tdev, q, skbs, ngathered);
1875 }
1876
1877 return work_done;
1878}
1879
1880/**
1881 * rx_offload - process a received offload packet
1882 * @tdev: the offload device receiving the packet
1883 * @rq: the response queue that received the packet
1884 * @skb: the packet
1885 * @rx_gather: a gather list of packets if we are building a bundle
1886 * @gather_idx: index of the next available slot in the bundle
1887 *
1888 * Process an ingress offload pakcet and add it to the offload ingress
1889 * queue. Returns the index of the next available slot in the bundle.
1890 */
1891static inline int rx_offload(struct t3cdev *tdev, struct sge_rspq *rq,
1892 struct sk_buff *skb, struct sk_buff *rx_gather[],
1893 unsigned int gather_idx)
1894{
1895 skb_reset_mac_header(skb);
1896 skb_reset_network_header(skb);
1897 skb_reset_transport_header(skb);
1898
1899 if (rq->polling) {
1900 rx_gather[gather_idx++] = skb;
1901 if (gather_idx == RX_BUNDLE_SIZE) {
1902 tdev->recv(tdev, rx_gather, RX_BUNDLE_SIZE);
1903 gather_idx = 0;
1904 rq->offload_bundles++;
1905 }
1906 } else
1907 offload_enqueue(rq, skb);
1908
1909 return gather_idx;
1910}
1911
1912/**
1913 * restart_tx - check whether to restart suspended Tx queues
1914 * @qs: the queue set to resume
1915 *
1916 * Restarts suspended Tx queues of an SGE queue set if they have enough
1917 * free resources to resume operation.
1918 */
1919static void restart_tx(struct sge_qset *qs)
1920{
1921 if (test_bit(TXQ_ETH, &qs->txq_stopped) &&
1922 should_restart_tx(&qs->txq[TXQ_ETH]) &&
1923 test_and_clear_bit(TXQ_ETH, &qs->txq_stopped)) {
1924 qs->txq[TXQ_ETH].restarts++;
1925 if (netif_running(qs->netdev))
1926 netif_tx_wake_queue(qs->tx_q);
1927 }
1928
1929 if (test_bit(TXQ_OFLD, &qs->txq_stopped) &&
1930 should_restart_tx(&qs->txq[TXQ_OFLD]) &&
1931 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) {
1932 qs->txq[TXQ_OFLD].restarts++;
1933 tasklet_schedule(&qs->txq[TXQ_OFLD].qresume_tsk);
1934 }
1935 if (test_bit(TXQ_CTRL, &qs->txq_stopped) &&
1936 should_restart_tx(&qs->txq[TXQ_CTRL]) &&
1937 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) {
1938 qs->txq[TXQ_CTRL].restarts++;
1939 tasklet_schedule(&qs->txq[TXQ_CTRL].qresume_tsk);
1940 }
1941}
1942
1943/**
1944 * cxgb3_arp_process - process an ARP request probing a private IP address
1945 * @adapter: the adapter
1946 * @skb: the skbuff containing the ARP request
1947 *
1948 * Check if the ARP request is probing the private IP address
1949 * dedicated to iSCSI, generate an ARP reply if so.
1950 */
1951static void cxgb3_arp_process(struct port_info *pi, struct sk_buff *skb)
1952{
1953 struct net_device *dev = skb->dev;
1954 struct arphdr *arp;
1955 unsigned char *arp_ptr;
1956 unsigned char *sha;
1957 __be32 sip, tip;
1958
1959 if (!dev)
1960 return;
1961
1962 skb_reset_network_header(skb);
1963 arp = arp_hdr(skb);
1964
1965 if (arp->ar_op != htons(ARPOP_REQUEST))
1966 return;
1967
1968 arp_ptr = (unsigned char *)(arp + 1);
1969 sha = arp_ptr;
1970 arp_ptr += dev->addr_len;
1971 memcpy(&sip, arp_ptr, sizeof(sip));
1972 arp_ptr += sizeof(sip);
1973 arp_ptr += dev->addr_len;
1974 memcpy(&tip, arp_ptr, sizeof(tip));
1975
1976 if (tip != pi->iscsi_ipv4addr)
1977 return;
1978
1979 arp_send(ARPOP_REPLY, ETH_P_ARP, sip, dev, tip, sha,
1980 pi->iscsic.mac_addr, sha);
1981
1982}
1983
1984static inline int is_arp(struct sk_buff *skb)
1985{
1986 return skb->protocol == htons(ETH_P_ARP);
1987}
1988
1989static void cxgb3_process_iscsi_prov_pack(struct port_info *pi,
1990 struct sk_buff *skb)
1991{
1992 if (is_arp(skb)) {
1993 cxgb3_arp_process(pi, skb);
1994 return;
1995 }
1996
1997 if (pi->iscsic.recv)
1998 pi->iscsic.recv(pi, skb);
1999
2000}
2001
2002/**
2003 * rx_eth - process an ingress ethernet packet
2004 * @adap: the adapter
2005 * @rq: the response queue that received the packet
2006 * @skb: the packet
2007 * @pad: amount of padding at the start of the buffer
2008 *
2009 * Process an ingress ethernet pakcet and deliver it to the stack.
2010 * The padding is 2 if the packet was delivered in an Rx buffer and 0
2011 * if it was immediate data in a response.
2012 */
2013static void rx_eth(struct adapter *adap, struct sge_rspq *rq,
2014 struct sk_buff *skb, int pad, int lro)
2015{
2016 struct cpl_rx_pkt *p = (struct cpl_rx_pkt *)(skb->data + pad);
2017 struct sge_qset *qs = rspq_to_qset(rq);
2018 struct port_info *pi;
2019
2020 skb_pull(skb, sizeof(*p) + pad);
2021 skb->protocol = eth_type_trans(skb, adap->port[p->iff]);
2022 pi = netdev_priv(skb->dev);
2023 if ((skb->dev->features & NETIF_F_RXCSUM) && p->csum_valid &&
2024 p->csum == htons(0xffff) && !p->fragment) {
2025 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2026 skb->ip_summed = CHECKSUM_UNNECESSARY;
2027 } else
2028 skb_checksum_none_assert(skb);
2029 skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);
2030
2031 if (p->vlan_valid) {
2032 qs->port_stats[SGE_PSTAT_VLANEX]++;
2033 __vlan_hwaccel_put_tag(skb, ntohs(p->vlan));
2034 }
2035 if (rq->polling) {
2036 if (lro)
2037 napi_gro_receive(&qs->napi, skb);
2038 else {
2039 if (unlikely(pi->iscsic.flags))
2040 cxgb3_process_iscsi_prov_pack(pi, skb);
2041 netif_receive_skb(skb);
2042 }
2043 } else
2044 netif_rx(skb);
2045}
2046
2047static inline int is_eth_tcp(u32 rss)
2048{
2049 return G_HASHTYPE(ntohl(rss)) == RSS_HASH_4_TUPLE;
2050}
2051
2052/**
2053 * lro_add_page - add a page chunk to an LRO session
2054 * @adap: the adapter
2055 * @qs: the associated queue set
2056 * @fl: the free list containing the page chunk to add
2057 * @len: packet length
2058 * @complete: Indicates the last fragment of a frame
2059 *
2060 * Add a received packet contained in a page chunk to an existing LRO
2061 * session.
2062 */
2063static void lro_add_page(struct adapter *adap, struct sge_qset *qs,
2064 struct sge_fl *fl, int len, int complete)
2065{
2066 struct rx_sw_desc *sd = &fl->sdesc[fl->cidx];
2067 struct port_info *pi = netdev_priv(qs->netdev);
2068 struct sk_buff *skb = NULL;
2069 struct cpl_rx_pkt *cpl;
2070 struct skb_frag_struct *rx_frag;
2071 int nr_frags;
2072 int offset = 0;
2073
2074 if (!qs->nomem) {
2075 skb = napi_get_frags(&qs->napi);
2076 qs->nomem = !skb;
2077 }
2078
2079 fl->credits--;
2080
2081 pci_dma_sync_single_for_cpu(adap->pdev,
2082 dma_unmap_addr(sd, dma_addr),
2083 fl->buf_size - SGE_PG_RSVD,
2084 PCI_DMA_FROMDEVICE);
2085
2086 (*sd->pg_chunk.p_cnt)--;
2087 if (!*sd->pg_chunk.p_cnt && sd->pg_chunk.page != fl->pg_chunk.page)
2088 pci_unmap_page(adap->pdev,
2089 sd->pg_chunk.mapping,
2090 fl->alloc_size,
2091 PCI_DMA_FROMDEVICE);
2092
2093 if (!skb) {
2094 put_page(sd->pg_chunk.page);
2095 if (complete)
2096 qs->nomem = 0;
2097 return;
2098 }
2099
2100 rx_frag = skb_shinfo(skb)->frags;
2101 nr_frags = skb_shinfo(skb)->nr_frags;
2102
2103 if (!nr_frags) {
2104 offset = 2 + sizeof(struct cpl_rx_pkt);
2105 cpl = qs->lro_va = sd->pg_chunk.va + 2;
2106
2107 if ((qs->netdev->features & NETIF_F_RXCSUM) &&
2108 cpl->csum_valid && cpl->csum == htons(0xffff)) {
2109 skb->ip_summed = CHECKSUM_UNNECESSARY;
2110 qs->port_stats[SGE_PSTAT_RX_CSUM_GOOD]++;
2111 } else
2112 skb->ip_summed = CHECKSUM_NONE;
2113 } else
2114 cpl = qs->lro_va;
2115
2116 len -= offset;
2117
2118 rx_frag += nr_frags;
2119 rx_frag->page = sd->pg_chunk.page;
2120 rx_frag->page_offset = sd->pg_chunk.offset + offset;
2121 rx_frag->size = len;
2122
2123 skb->len += len;
2124 skb->data_len += len;
2125 skb->truesize += len;
2126 skb_shinfo(skb)->nr_frags++;
2127
2128 if (!complete)
2129 return;
2130
2131 skb_record_rx_queue(skb, qs - &adap->sge.qs[pi->first_qset]);
2132
2133 if (cpl->vlan_valid)
2134 __vlan_hwaccel_put_tag(skb, ntohs(cpl->vlan));
2135 napi_gro_frags(&qs->napi);
2136}
2137
2138/**
2139 * handle_rsp_cntrl_info - handles control information in a response
2140 * @qs: the queue set corresponding to the response
2141 * @flags: the response control flags
2142 *
2143 * Handles the control information of an SGE response, such as GTS
2144 * indications and completion credits for the queue set's Tx queues.
2145 * HW coalesces credits, we don't do any extra SW coalescing.
2146 */
2147static inline void handle_rsp_cntrl_info(struct sge_qset *qs, u32 flags)
2148{
2149 unsigned int credits;
2150
2151#if USE_GTS
2152 if (flags & F_RSPD_TXQ0_GTS)
2153 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags);
2154#endif
2155
2156 credits = G_RSPD_TXQ0_CR(flags);
2157 if (credits)
2158 qs->txq[TXQ_ETH].processed += credits;
2159
2160 credits = G_RSPD_TXQ2_CR(flags);
2161 if (credits)
2162 qs->txq[TXQ_CTRL].processed += credits;
2163
2164# if USE_GTS
2165 if (flags & F_RSPD_TXQ1_GTS)
2166 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags);
2167# endif
2168 credits = G_RSPD_TXQ1_CR(flags);
2169 if (credits)
2170 qs->txq[TXQ_OFLD].processed += credits;
2171}
2172
2173/**
2174 * check_ring_db - check if we need to ring any doorbells
2175 * @adapter: the adapter
2176 * @qs: the queue set whose Tx queues are to be examined
2177 * @sleeping: indicates which Tx queue sent GTS
2178 *
2179 * Checks if some of a queue set's Tx queues need to ring their doorbells
2180 * to resume transmission after idling while they still have unprocessed
2181 * descriptors.
2182 */
2183static void check_ring_db(struct adapter *adap, struct sge_qset *qs,
2184 unsigned int sleeping)
2185{
2186 if (sleeping & F_RSPD_TXQ0_GTS) {
2187 struct sge_txq *txq = &qs->txq[TXQ_ETH];
2188
2189 if (txq->cleaned + txq->in_use != txq->processed &&
2190 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2191 set_bit(TXQ_RUNNING, &txq->flags);
2192 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2193 V_EGRCNTX(txq->cntxt_id));
2194 }
2195 }
2196
2197 if (sleeping & F_RSPD_TXQ1_GTS) {
2198 struct sge_txq *txq = &qs->txq[TXQ_OFLD];
2199
2200 if (txq->cleaned + txq->in_use != txq->processed &&
2201 !test_and_set_bit(TXQ_LAST_PKT_DB, &txq->flags)) {
2202 set_bit(TXQ_RUNNING, &txq->flags);
2203 t3_write_reg(adap, A_SG_KDOORBELL, F_SELEGRCNTX |
2204 V_EGRCNTX(txq->cntxt_id));
2205 }
2206 }
2207}
2208
2209/**
2210 * is_new_response - check if a response is newly written
2211 * @r: the response descriptor
2212 * @q: the response queue
2213 *
2214 * Returns true if a response descriptor contains a yet unprocessed
2215 * response.
2216 */
2217static inline int is_new_response(const struct rsp_desc *r,
2218 const struct sge_rspq *q)
2219{
2220 return (r->intr_gen & F_RSPD_GEN2) == q->gen;
2221}
2222
2223static inline void clear_rspq_bufstate(struct sge_rspq * const q)
2224{
2225 q->pg_skb = NULL;
2226 q->rx_recycle_buf = 0;
2227}
2228
2229#define RSPD_GTS_MASK (F_RSPD_TXQ0_GTS | F_RSPD_TXQ1_GTS)
2230#define RSPD_CTRL_MASK (RSPD_GTS_MASK | \
2231 V_RSPD_TXQ0_CR(M_RSPD_TXQ0_CR) | \
2232 V_RSPD_TXQ1_CR(M_RSPD_TXQ1_CR) | \
2233 V_RSPD_TXQ2_CR(M_RSPD_TXQ2_CR))
2234
2235/* How long to delay the next interrupt in case of memory shortage, in 0.1us. */
2236#define NOMEM_INTR_DELAY 2500
2237
2238/**
2239 * process_responses - process responses from an SGE response queue
2240 * @adap: the adapter
2241 * @qs: the queue set to which the response queue belongs
2242 * @budget: how many responses can be processed in this round
2243 *
2244 * Process responses from an SGE response queue up to the supplied budget.
2245 * Responses include received packets as well as credits and other events
2246 * for the queues that belong to the response queue's queue set.
2247 * A negative budget is effectively unlimited.
2248 *
2249 * Additionally choose the interrupt holdoff time for the next interrupt
2250 * on this queue. If the system is under memory shortage use a fairly
2251 * long delay to help recovery.
2252 */
2253static int process_responses(struct adapter *adap, struct sge_qset *qs,
2254 int budget)
2255{
2256 struct sge_rspq *q = &qs->rspq;
2257 struct rsp_desc *r = &q->desc[q->cidx];
2258 int budget_left = budget;
2259 unsigned int sleeping = 0;
2260 struct sk_buff *offload_skbs[RX_BUNDLE_SIZE];
2261 int ngathered = 0;
2262
2263 q->next_holdoff = q->holdoff_tmr;
2264
2265 while (likely(budget_left && is_new_response(r, q))) {
2266 int packet_complete, eth, ethpad = 2;
2267 int lro = !!(qs->netdev->features & NETIF_F_GRO);
2268 struct sk_buff *skb = NULL;
2269 u32 len, flags;
2270 __be32 rss_hi, rss_lo;
2271
2272 rmb();
2273 eth = r->rss_hdr.opcode == CPL_RX_PKT;
2274 rss_hi = *(const __be32 *)r;
2275 rss_lo = r->rss_hdr.rss_hash_val;
2276 flags = ntohl(r->flags);
2277
2278 if (unlikely(flags & F_RSPD_ASYNC_NOTIF)) {
2279 skb = alloc_skb(AN_PKT_SIZE, GFP_ATOMIC);
2280 if (!skb)
2281 goto no_mem;
2282
2283 memcpy(__skb_put(skb, AN_PKT_SIZE), r, AN_PKT_SIZE);
2284 skb->data[0] = CPL_ASYNC_NOTIF;
2285 rss_hi = htonl(CPL_ASYNC_NOTIF << 24);
2286 q->async_notif++;
2287 } else if (flags & F_RSPD_IMM_DATA_VALID) {
2288 skb = get_imm_packet(r);
2289 if (unlikely(!skb)) {
2290no_mem:
2291 q->next_holdoff = NOMEM_INTR_DELAY;
2292 q->nomem++;
2293 /* consume one credit since we tried */
2294 budget_left--;
2295 break;
2296 }
2297 q->imm_data++;
2298 ethpad = 0;
2299 } else if ((len = ntohl(r->len_cq)) != 0) {
2300 struct sge_fl *fl;
2301
2302 lro &= eth && is_eth_tcp(rss_hi);
2303
2304 fl = (len & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0];
2305 if (fl->use_pages) {
2306 void *addr = fl->sdesc[fl->cidx].pg_chunk.va;
2307
2308 prefetch(addr);
2309#if L1_CACHE_BYTES < 128
2310 prefetch(addr + L1_CACHE_BYTES);
2311#endif
2312 __refill_fl(adap, fl);
2313 if (lro > 0) {
2314 lro_add_page(adap, qs, fl,
2315 G_RSPD_LEN(len),
2316 flags & F_RSPD_EOP);
2317 goto next_fl;
2318 }
2319
2320 skb = get_packet_pg(adap, fl, q,
2321 G_RSPD_LEN(len),
2322 eth ?
2323 SGE_RX_DROP_THRES : 0);
2324 q->pg_skb = skb;
2325 } else
2326 skb = get_packet(adap, fl, G_RSPD_LEN(len),
2327 eth ? SGE_RX_DROP_THRES : 0);
2328 if (unlikely(!skb)) {
2329 if (!eth)
2330 goto no_mem;
2331 q->rx_drops++;
2332 } else if (unlikely(r->rss_hdr.opcode == CPL_TRACE_PKT))
2333 __skb_pull(skb, 2);
2334next_fl:
2335 if (++fl->cidx == fl->size)
2336 fl->cidx = 0;
2337 } else
2338 q->pure_rsps++;
2339
2340 if (flags & RSPD_CTRL_MASK) {
2341 sleeping |= flags & RSPD_GTS_MASK;
2342 handle_rsp_cntrl_info(qs, flags);
2343 }
2344
2345 r++;
2346 if (unlikely(++q->cidx == q->size)) {
2347 q->cidx = 0;
2348 q->gen ^= 1;
2349 r = q->desc;
2350 }
2351 prefetch(r);
2352
2353 if (++q->credits >= (q->size / 4)) {
2354 refill_rspq(adap, q, q->credits);
2355 q->credits = 0;
2356 }
2357
2358 packet_complete = flags &
2359 (F_RSPD_EOP | F_RSPD_IMM_DATA_VALID |
2360 F_RSPD_ASYNC_NOTIF);
2361
2362 if (skb != NULL && packet_complete) {
2363 if (eth)
2364 rx_eth(adap, q, skb, ethpad, lro);
2365 else {
2366 q->offload_pkts++;
2367 /* Preserve the RSS info in csum & priority */
2368 skb->csum = rss_hi;
2369 skb->priority = rss_lo;
2370 ngathered = rx_offload(&adap->tdev, q, skb,
2371 offload_skbs,
2372 ngathered);
2373 }
2374
2375 if (flags & F_RSPD_EOP)
2376 clear_rspq_bufstate(q);
2377 }
2378 --budget_left;
2379 }
2380
2381 deliver_partial_bundle(&adap->tdev, q, offload_skbs, ngathered);
2382
2383 if (sleeping)
2384 check_ring_db(adap, qs, sleeping);
2385
2386 smp_mb(); /* commit Tx queue .processed updates */
2387 if (unlikely(qs->txq_stopped != 0))
2388 restart_tx(qs);
2389
2390 budget -= budget_left;
2391 return budget;
2392}
2393
2394static inline int is_pure_response(const struct rsp_desc *r)
2395{
2396 __be32 n = r->flags & htonl(F_RSPD_ASYNC_NOTIF | F_RSPD_IMM_DATA_VALID);
2397
2398 return (n | r->len_cq) == 0;
2399}
2400
2401/**
2402 * napi_rx_handler - the NAPI handler for Rx processing
2403 * @napi: the napi instance
2404 * @budget: how many packets we can process in this round
2405 *
2406 * Handler for new data events when using NAPI.
2407 */
2408static int napi_rx_handler(struct napi_struct *napi, int budget)
2409{
2410 struct sge_qset *qs = container_of(napi, struct sge_qset, napi);
2411 struct adapter *adap = qs->adap;
2412 int work_done = process_responses(adap, qs, budget);
2413
2414 if (likely(work_done < budget)) {
2415 napi_complete(napi);
2416
2417 /*
2418 * Because we don't atomically flush the following
2419 * write it is possible that in very rare cases it can
2420 * reach the device in a way that races with a new
2421 * response being written plus an error interrupt
2422 * causing the NAPI interrupt handler below to return
2423 * unhandled status to the OS. To protect against
2424 * this would require flushing the write and doing
2425 * both the write and the flush with interrupts off.
2426 * Way too expensive and unjustifiable given the
2427 * rarity of the race.
2428 *
2429 * The race cannot happen at all with MSI-X.
2430 */
2431 t3_write_reg(adap, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) |
2432 V_NEWTIMER(qs->rspq.next_holdoff) |
2433 V_NEWINDEX(qs->rspq.cidx));
2434 }
2435 return work_done;
2436}
2437
2438/*
2439 * Returns true if the device is already scheduled for polling.
2440 */
2441static inline int napi_is_scheduled(struct napi_struct *napi)
2442{
2443 return test_bit(NAPI_STATE_SCHED, &napi->state);
2444}
2445
2446/**
2447 * process_pure_responses - process pure responses from a response queue
2448 * @adap: the adapter
2449 * @qs: the queue set owning the response queue
2450 * @r: the first pure response to process
2451 *
2452 * A simpler version of process_responses() that handles only pure (i.e.,
2453 * non data-carrying) responses. Such respones are too light-weight to
2454 * justify calling a softirq under NAPI, so we handle them specially in
2455 * the interrupt handler. The function is called with a pointer to a
2456 * response, which the caller must ensure is a valid pure response.
2457 *
2458 * Returns 1 if it encounters a valid data-carrying response, 0 otherwise.
2459 */
2460static int process_pure_responses(struct adapter *adap, struct sge_qset *qs,
2461 struct rsp_desc *r)
2462{
2463 struct sge_rspq *q = &qs->rspq;
2464 unsigned int sleeping = 0;
2465
2466 do {
2467 u32 flags = ntohl(r->flags);
2468
2469 r++;
2470 if (unlikely(++q->cidx == q->size)) {
2471 q->cidx = 0;
2472 q->gen ^= 1;
2473 r = q->desc;
2474 }
2475 prefetch(r);
2476
2477 if (flags & RSPD_CTRL_MASK) {
2478 sleeping |= flags & RSPD_GTS_MASK;
2479 handle_rsp_cntrl_info(qs, flags);
2480 }
2481
2482 q->pure_rsps++;
2483 if (++q->credits >= (q->size / 4)) {
2484 refill_rspq(adap, q, q->credits);
2485 q->credits = 0;
2486 }
2487 if (!is_new_response(r, q))
2488 break;
2489 rmb();
2490 } while (is_pure_response(r));
2491
2492 if (sleeping)
2493 check_ring_db(adap, qs, sleeping);
2494
2495 smp_mb(); /* commit Tx queue .processed updates */
2496 if (unlikely(qs->txq_stopped != 0))
2497 restart_tx(qs);
2498
2499 return is_new_response(r, q);
2500}
2501
2502/**
2503 * handle_responses - decide what to do with new responses in NAPI mode
2504 * @adap: the adapter
2505 * @q: the response queue
2506 *
2507 * This is used by the NAPI interrupt handlers to decide what to do with
2508 * new SGE responses. If there are no new responses it returns -1. If
2509 * there are new responses and they are pure (i.e., non-data carrying)
2510 * it handles them straight in hard interrupt context as they are very
2511 * cheap and don't deliver any packets. Finally, if there are any data
2512 * signaling responses it schedules the NAPI handler. Returns 1 if it
2513 * schedules NAPI, 0 if all new responses were pure.
2514 *
2515 * The caller must ascertain NAPI is not already running.
2516 */
2517static inline int handle_responses(struct adapter *adap, struct sge_rspq *q)
2518{
2519 struct sge_qset *qs = rspq_to_qset(q);
2520 struct rsp_desc *r = &q->desc[q->cidx];
2521
2522 if (!is_new_response(r, q))
2523 return -1;
2524 rmb();
2525 if (is_pure_response(r) && process_pure_responses(adap, qs, r) == 0) {
2526 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2527 V_NEWTIMER(q->holdoff_tmr) | V_NEWINDEX(q->cidx));
2528 return 0;
2529 }
2530 napi_schedule(&qs->napi);
2531 return 1;
2532}
2533
2534/*
2535 * The MSI-X interrupt handler for an SGE response queue for the non-NAPI case
2536 * (i.e., response queue serviced in hard interrupt).
2537 */
2538static irqreturn_t t3_sge_intr_msix(int irq, void *cookie)
2539{
2540 struct sge_qset *qs = cookie;
2541 struct adapter *adap = qs->adap;
2542 struct sge_rspq *q = &qs->rspq;
2543
2544 spin_lock(&q->lock);
2545 if (process_responses(adap, qs, -1) == 0)
2546 q->unhandled_irqs++;
2547 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2548 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2549 spin_unlock(&q->lock);
2550 return IRQ_HANDLED;
2551}
2552
2553/*
2554 * The MSI-X interrupt handler for an SGE response queue for the NAPI case
2555 * (i.e., response queue serviced by NAPI polling).
2556 */
2557static irqreturn_t t3_sge_intr_msix_napi(int irq, void *cookie)
2558{
2559 struct sge_qset *qs = cookie;
2560 struct sge_rspq *q = &qs->rspq;
2561
2562 spin_lock(&q->lock);
2563
2564 if (handle_responses(qs->adap, q) < 0)
2565 q->unhandled_irqs++;
2566 spin_unlock(&q->lock);
2567 return IRQ_HANDLED;
2568}
2569
2570/*
2571 * The non-NAPI MSI interrupt handler. This needs to handle data events from
2572 * SGE response queues as well as error and other async events as they all use
2573 * the same MSI vector. We use one SGE response queue per port in this mode
2574 * and protect all response queues with queue 0's lock.
2575 */
2576static irqreturn_t t3_intr_msi(int irq, void *cookie)
2577{
2578 int new_packets = 0;
2579 struct adapter *adap = cookie;
2580 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2581
2582 spin_lock(&q->lock);
2583
2584 if (process_responses(adap, &adap->sge.qs[0], -1)) {
2585 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q->cntxt_id) |
2586 V_NEWTIMER(q->next_holdoff) | V_NEWINDEX(q->cidx));
2587 new_packets = 1;
2588 }
2589
2590 if (adap->params.nports == 2 &&
2591 process_responses(adap, &adap->sge.qs[1], -1)) {
2592 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2593
2594 t3_write_reg(adap, A_SG_GTS, V_RSPQ(q1->cntxt_id) |
2595 V_NEWTIMER(q1->next_holdoff) |
2596 V_NEWINDEX(q1->cidx));
2597 new_packets = 1;
2598 }
2599
2600 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2601 q->unhandled_irqs++;
2602
2603 spin_unlock(&q->lock);
2604 return IRQ_HANDLED;
2605}
2606
2607static int rspq_check_napi(struct sge_qset *qs)
2608{
2609 struct sge_rspq *q = &qs->rspq;
2610
2611 if (!napi_is_scheduled(&qs->napi) &&
2612 is_new_response(&q->desc[q->cidx], q)) {
2613 napi_schedule(&qs->napi);
2614 return 1;
2615 }
2616 return 0;
2617}
2618
2619/*
2620 * The MSI interrupt handler for the NAPI case (i.e., response queues serviced
2621 * by NAPI polling). Handles data events from SGE response queues as well as
2622 * error and other async events as they all use the same MSI vector. We use
2623 * one SGE response queue per port in this mode and protect all response
2624 * queues with queue 0's lock.
2625 */
2626static irqreturn_t t3_intr_msi_napi(int irq, void *cookie)
2627{
2628 int new_packets;
2629 struct adapter *adap = cookie;
2630 struct sge_rspq *q = &adap->sge.qs[0].rspq;
2631
2632 spin_lock(&q->lock);
2633
2634 new_packets = rspq_check_napi(&adap->sge.qs[0]);
2635 if (adap->params.nports == 2)
2636 new_packets += rspq_check_napi(&adap->sge.qs[1]);
2637 if (!new_packets && t3_slow_intr_handler(adap) == 0)
2638 q->unhandled_irqs++;
2639
2640 spin_unlock(&q->lock);
2641 return IRQ_HANDLED;
2642}
2643
2644/*
2645 * A helper function that processes responses and issues GTS.
2646 */
2647static inline int process_responses_gts(struct adapter *adap,
2648 struct sge_rspq *rq)
2649{
2650 int work;
2651
2652 work = process_responses(adap, rspq_to_qset(rq), -1);
2653 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) |
2654 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx));
2655 return work;
2656}
2657
2658/*
2659 * The legacy INTx interrupt handler. This needs to handle data events from
2660 * SGE response queues as well as error and other async events as they all use
2661 * the same interrupt pin. We use one SGE response queue per port in this mode
2662 * and protect all response queues with queue 0's lock.
2663 */
2664static irqreturn_t t3_intr(int irq, void *cookie)
2665{
2666 int work_done, w0, w1;
2667 struct adapter *adap = cookie;
2668 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2669 struct sge_rspq *q1 = &adap->sge.qs[1].rspq;
2670
2671 spin_lock(&q0->lock);
2672
2673 w0 = is_new_response(&q0->desc[q0->cidx], q0);
2674 w1 = adap->params.nports == 2 &&
2675 is_new_response(&q1->desc[q1->cidx], q1);
2676
2677 if (likely(w0 | w1)) {
2678 t3_write_reg(adap, A_PL_CLI, 0);
2679 t3_read_reg(adap, A_PL_CLI); /* flush */
2680
2681 if (likely(w0))
2682 process_responses_gts(adap, q0);
2683
2684 if (w1)
2685 process_responses_gts(adap, q1);
2686
2687 work_done = w0 | w1;
2688 } else
2689 work_done = t3_slow_intr_handler(adap);
2690
2691 spin_unlock(&q0->lock);
2692 return IRQ_RETVAL(work_done != 0);
2693}
2694
2695/*
2696 * Interrupt handler for legacy INTx interrupts for T3B-based cards.
2697 * Handles data events from SGE response queues as well as error and other
2698 * async events as they all use the same interrupt pin. We use one SGE
2699 * response queue per port in this mode and protect all response queues with
2700 * queue 0's lock.
2701 */
2702static irqreturn_t t3b_intr(int irq, void *cookie)
2703{
2704 u32 map;
2705 struct adapter *adap = cookie;
2706 struct sge_rspq *q0 = &adap->sge.qs[0].rspq;
2707
2708 t3_write_reg(adap, A_PL_CLI, 0);
2709 map = t3_read_reg(adap, A_SG_DATA_INTR);
2710
2711 if (unlikely(!map)) /* shared interrupt, most likely */
2712 return IRQ_NONE;
2713
2714 spin_lock(&q0->lock);
2715
2716 if (unlikely(map & F_ERRINTR))
2717 t3_slow_intr_handler(adap);
2718
2719 if (likely(map & 1))
2720 process_responses_gts(adap, q0);
2721
2722 if (map & 2)
2723 process_responses_gts(adap, &adap->sge.qs[1].rspq);
2724
2725 spin_unlock(&q0->lock);
2726 return IRQ_HANDLED;
2727}
2728
2729/*
2730 * NAPI interrupt handler for legacy INTx interrupts for T3B-based cards.
2731 * Handles data events from SGE response queues as well as error and other
2732 * async events as they all use the same interrupt pin. We use one SGE
2733 * response queue per port in this mode and protect all response queues with
2734 * queue 0's lock.
2735 */
2736static irqreturn_t t3b_intr_napi(int irq, void *cookie)
2737{
2738 u32 map;
2739 struct adapter *adap = cookie;
2740 struct sge_qset *qs0 = &adap->sge.qs[0];
2741 struct sge_rspq *q0 = &qs0->rspq;
2742
2743 t3_write_reg(adap, A_PL_CLI, 0);
2744 map = t3_read_reg(adap, A_SG_DATA_INTR);
2745
2746 if (unlikely(!map)) /* shared interrupt, most likely */
2747 return IRQ_NONE;
2748
2749 spin_lock(&q0->lock);
2750
2751 if (unlikely(map & F_ERRINTR))
2752 t3_slow_intr_handler(adap);
2753
2754 if (likely(map & 1))
2755 napi_schedule(&qs0->napi);
2756
2757 if (map & 2)
2758 napi_schedule(&adap->sge.qs[1].napi);
2759
2760 spin_unlock(&q0->lock);
2761 return IRQ_HANDLED;
2762}
2763
2764/**
2765 * t3_intr_handler - select the top-level interrupt handler
2766 * @adap: the adapter
2767 * @polling: whether using NAPI to service response queues
2768 *
2769 * Selects the top-level interrupt handler based on the type of interrupts
2770 * (MSI-X, MSI, or legacy) and whether NAPI will be used to service the
2771 * response queues.
2772 */
2773irq_handler_t t3_intr_handler(struct adapter *adap, int polling)
2774{
2775 if (adap->flags & USING_MSIX)
2776 return polling ? t3_sge_intr_msix_napi : t3_sge_intr_msix;
2777 if (adap->flags & USING_MSI)
2778 return polling ? t3_intr_msi_napi : t3_intr_msi;
2779 if (adap->params.rev > 0)
2780 return polling ? t3b_intr_napi : t3b_intr;
2781 return t3_intr;
2782}
2783
2784#define SGE_PARERR (F_CPPARITYERROR | F_OCPARITYERROR | F_RCPARITYERROR | \
2785 F_IRPARITYERROR | V_ITPARITYERROR(M_ITPARITYERROR) | \
2786 V_FLPARITYERROR(M_FLPARITYERROR) | F_LODRBPARITYERROR | \
2787 F_HIDRBPARITYERROR | F_LORCQPARITYERROR | \
2788 F_HIRCQPARITYERROR)
2789#define SGE_FRAMINGERR (F_UC_REQ_FRAMINGERROR | F_R_REQ_FRAMINGERROR)
2790#define SGE_FATALERR (SGE_PARERR | SGE_FRAMINGERR | F_RSPQCREDITOVERFOW | \
2791 F_RSPQDISABLED)
2792
2793/**
2794 * t3_sge_err_intr_handler - SGE async event interrupt handler
2795 * @adapter: the adapter
2796 *
2797 * Interrupt handler for SGE asynchronous (non-data) events.
2798 */
2799void t3_sge_err_intr_handler(struct adapter *adapter)
2800{
2801 unsigned int v, status = t3_read_reg(adapter, A_SG_INT_CAUSE) &
2802 ~F_FLEMPTY;
2803
2804 if (status & SGE_PARERR)
2805 CH_ALERT(adapter, "SGE parity error (0x%x)\n",
2806 status & SGE_PARERR);
2807 if (status & SGE_FRAMINGERR)
2808 CH_ALERT(adapter, "SGE framing error (0x%x)\n",
2809 status & SGE_FRAMINGERR);
2810
2811 if (status & F_RSPQCREDITOVERFOW)
2812 CH_ALERT(adapter, "SGE response queue credit overflow\n");
2813
2814 if (status & F_RSPQDISABLED) {
2815 v = t3_read_reg(adapter, A_SG_RSPQ_FL_STATUS);
2816
2817 CH_ALERT(adapter,
2818 "packet delivered to disabled response queue "
2819 "(0x%x)\n", (v >> S_RSPQ0DISABLED) & 0xff);
2820 }
2821
2822 if (status & (F_HIPIODRBDROPERR | F_LOPIODRBDROPERR))
2823 queue_work(cxgb3_wq, &adapter->db_drop_task);
2824
2825 if (status & (F_HIPRIORITYDBFULL | F_LOPRIORITYDBFULL))
2826 queue_work(cxgb3_wq, &adapter->db_full_task);
2827
2828 if (status & (F_HIPRIORITYDBEMPTY | F_LOPRIORITYDBEMPTY))
2829 queue_work(cxgb3_wq, &adapter->db_empty_task);
2830
2831 t3_write_reg(adapter, A_SG_INT_CAUSE, status);
2832 if (status & SGE_FATALERR)
2833 t3_fatal_err(adapter);
2834}
2835
2836/**
2837 * sge_timer_tx - perform periodic maintenance of an SGE qset
2838 * @data: the SGE queue set to maintain
2839 *
2840 * Runs periodically from a timer to perform maintenance of an SGE queue
2841 * set. It performs two tasks:
2842 *
2843 * Cleans up any completed Tx descriptors that may still be pending.
2844 * Normal descriptor cleanup happens when new packets are added to a Tx
2845 * queue so this timer is relatively infrequent and does any cleanup only
2846 * if the Tx queue has not seen any new packets in a while. We make a
2847 * best effort attempt to reclaim descriptors, in that we don't wait
2848 * around if we cannot get a queue's lock (which most likely is because
2849 * someone else is queueing new packets and so will also handle the clean
2850 * up). Since control queues use immediate data exclusively we don't
2851 * bother cleaning them up here.
2852 *
2853 */
2854static void sge_timer_tx(unsigned long data)
2855{
2856 struct sge_qset *qs = (struct sge_qset *)data;
2857 struct port_info *pi = netdev_priv(qs->netdev);
2858 struct adapter *adap = pi->adapter;
2859 unsigned int tbd[SGE_TXQ_PER_SET] = {0, 0};
2860 unsigned long next_period;
2861
2862 if (__netif_tx_trylock(qs->tx_q)) {
2863 tbd[TXQ_ETH] = reclaim_completed_tx(adap, &qs->txq[TXQ_ETH],
2864 TX_RECLAIM_TIMER_CHUNK);
2865 __netif_tx_unlock(qs->tx_q);
2866 }
2867
2868 if (spin_trylock(&qs->txq[TXQ_OFLD].lock)) {
2869 tbd[TXQ_OFLD] = reclaim_completed_tx(adap, &qs->txq[TXQ_OFLD],
2870 TX_RECLAIM_TIMER_CHUNK);
2871 spin_unlock(&qs->txq[TXQ_OFLD].lock);
2872 }
2873
2874 next_period = TX_RECLAIM_PERIOD >>
2875 (max(tbd[TXQ_ETH], tbd[TXQ_OFLD]) /
2876 TX_RECLAIM_TIMER_CHUNK);
2877 mod_timer(&qs->tx_reclaim_timer, jiffies + next_period);
2878}
2879
2880/*
2881 * sge_timer_rx - perform periodic maintenance of an SGE qset
2882 * @data: the SGE queue set to maintain
2883 *
2884 * a) Replenishes Rx queues that have run out due to memory shortage.
2885 * Normally new Rx buffers are added when existing ones are consumed but
2886 * when out of memory a queue can become empty. We try to add only a few
2887 * buffers here, the queue will be replenished fully as these new buffers
2888 * are used up if memory shortage has subsided.
2889 *
2890 * b) Return coalesced response queue credits in case a response queue is
2891 * starved.
2892 *
2893 */
2894static void sge_timer_rx(unsigned long data)
2895{
2896 spinlock_t *lock;
2897 struct sge_qset *qs = (struct sge_qset *)data;
2898 struct port_info *pi = netdev_priv(qs->netdev);
2899 struct adapter *adap = pi->adapter;
2900 u32 status;
2901
2902 lock = adap->params.rev > 0 ?
2903 &qs->rspq.lock : &adap->sge.qs[0].rspq.lock;
2904
2905 if (!spin_trylock_irq(lock))
2906 goto out;
2907
2908 if (napi_is_scheduled(&qs->napi))
2909 goto unlock;
2910
2911 if (adap->params.rev < 4) {
2912 status = t3_read_reg(adap, A_SG_RSPQ_FL_STATUS);
2913
2914 if (status & (1 << qs->rspq.cntxt_id)) {
2915 qs->rspq.starved++;
2916 if (qs->rspq.credits) {
2917 qs->rspq.credits--;
2918 refill_rspq(adap, &qs->rspq, 1);
2919 qs->rspq.restarted++;
2920 t3_write_reg(adap, A_SG_RSPQ_FL_STATUS,
2921 1 << qs->rspq.cntxt_id);
2922 }
2923 }
2924 }
2925
2926 if (qs->fl[0].credits < qs->fl[0].size)
2927 __refill_fl(adap, &qs->fl[0]);
2928 if (qs->fl[1].credits < qs->fl[1].size)
2929 __refill_fl(adap, &qs->fl[1]);
2930
2931unlock:
2932 spin_unlock_irq(lock);
2933out:
2934 mod_timer(&qs->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
2935}
2936
2937/**
2938 * t3_update_qset_coalesce - update coalescing settings for a queue set
2939 * @qs: the SGE queue set
2940 * @p: new queue set parameters
2941 *
2942 * Update the coalescing settings for an SGE queue set. Nothing is done
2943 * if the queue set is not initialized yet.
2944 */
2945void t3_update_qset_coalesce(struct sge_qset *qs, const struct qset_params *p)
2946{
2947 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U);/* can't be 0 */
2948 qs->rspq.polling = p->polling;
2949 qs->napi.poll = p->polling ? napi_rx_handler : ofld_poll;
2950}
2951
2952/**
2953 * t3_sge_alloc_qset - initialize an SGE queue set
2954 * @adapter: the adapter
2955 * @id: the queue set id
2956 * @nports: how many Ethernet ports will be using this queue set
2957 * @irq_vec_idx: the IRQ vector index for response queue interrupts
2958 * @p: configuration parameters for this queue set
2959 * @ntxq: number of Tx queues for the queue set
2960 * @netdev: net device associated with this queue set
2961 * @netdevq: net device TX queue associated with this queue set
2962 *
2963 * Allocate resources and initialize an SGE queue set. A queue set
2964 * comprises a response queue, two Rx free-buffer queues, and up to 3
2965 * Tx queues. The Tx queues are assigned roles in the order Ethernet
2966 * queue, offload queue, and control queue.
2967 */
2968int t3_sge_alloc_qset(struct adapter *adapter, unsigned int id, int nports,
2969 int irq_vec_idx, const struct qset_params *p,
2970 int ntxq, struct net_device *dev,
2971 struct netdev_queue *netdevq)
2972{
2973 int i, avail, ret = -ENOMEM;
2974 struct sge_qset *q = &adapter->sge.qs[id];
2975
2976 init_qset_cntxt(q, id);
2977 setup_timer(&q->tx_reclaim_timer, sge_timer_tx, (unsigned long)q);
2978 setup_timer(&q->rx_reclaim_timer, sge_timer_rx, (unsigned long)q);
2979
2980 q->fl[0].desc = alloc_ring(adapter->pdev, p->fl_size,
2981 sizeof(struct rx_desc),
2982 sizeof(struct rx_sw_desc),
2983 &q->fl[0].phys_addr, &q->fl[0].sdesc);
2984 if (!q->fl[0].desc)
2985 goto err;
2986
2987 q->fl[1].desc = alloc_ring(adapter->pdev, p->jumbo_size,
2988 sizeof(struct rx_desc),
2989 sizeof(struct rx_sw_desc),
2990 &q->fl[1].phys_addr, &q->fl[1].sdesc);
2991 if (!q->fl[1].desc)
2992 goto err;
2993
2994 q->rspq.desc = alloc_ring(adapter->pdev, p->rspq_size,
2995 sizeof(struct rsp_desc), 0,
2996 &q->rspq.phys_addr, NULL);
2997 if (!q->rspq.desc)
2998 goto err;
2999
3000 for (i = 0; i < ntxq; ++i) {
3001 /*
3002 * The control queue always uses immediate data so does not
3003 * need to keep track of any sk_buffs.
3004 */
3005 size_t sz = i == TXQ_CTRL ? 0 : sizeof(struct tx_sw_desc);
3006
3007 q->txq[i].desc = alloc_ring(adapter->pdev, p->txq_size[i],
3008 sizeof(struct tx_desc), sz,
3009 &q->txq[i].phys_addr,
3010 &q->txq[i].sdesc);
3011 if (!q->txq[i].desc)
3012 goto err;
3013
3014 q->txq[i].gen = 1;
3015 q->txq[i].size = p->txq_size[i];
3016 spin_lock_init(&q->txq[i].lock);
3017 skb_queue_head_init(&q->txq[i].sendq);
3018 }
3019
3020 tasklet_init(&q->txq[TXQ_OFLD].qresume_tsk, restart_offloadq,
3021 (unsigned long)q);
3022 tasklet_init(&q->txq[TXQ_CTRL].qresume_tsk, restart_ctrlq,
3023 (unsigned long)q);
3024
3025 q->fl[0].gen = q->fl[1].gen = 1;
3026 q->fl[0].size = p->fl_size;
3027 q->fl[1].size = p->jumbo_size;
3028
3029 q->rspq.gen = 1;
3030 q->rspq.size = p->rspq_size;
3031 spin_lock_init(&q->rspq.lock);
3032 skb_queue_head_init(&q->rspq.rx_queue);
3033
3034 q->txq[TXQ_ETH].stop_thres = nports *
3035 flits_to_desc(sgl_len(MAX_SKB_FRAGS + 1) + 3);
3036
3037#if FL0_PG_CHUNK_SIZE > 0
3038 q->fl[0].buf_size = FL0_PG_CHUNK_SIZE;
3039#else
3040 q->fl[0].buf_size = SGE_RX_SM_BUF_SIZE + sizeof(struct cpl_rx_data);
3041#endif
3042#if FL1_PG_CHUNK_SIZE > 0
3043 q->fl[1].buf_size = FL1_PG_CHUNK_SIZE;
3044#else
3045 q->fl[1].buf_size = is_offload(adapter) ?
3046 (16 * 1024) - SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
3047 MAX_FRAME_SIZE + 2 + sizeof(struct cpl_rx_pkt);
3048#endif
3049
3050 q->fl[0].use_pages = FL0_PG_CHUNK_SIZE > 0;
3051 q->fl[1].use_pages = FL1_PG_CHUNK_SIZE > 0;
3052 q->fl[0].order = FL0_PG_ORDER;
3053 q->fl[1].order = FL1_PG_ORDER;
3054 q->fl[0].alloc_size = FL0_PG_ALLOC_SIZE;
3055 q->fl[1].alloc_size = FL1_PG_ALLOC_SIZE;
3056
3057 spin_lock_irq(&adapter->sge.reg_lock);
3058
3059 /* FL threshold comparison uses < */
3060 ret = t3_sge_init_rspcntxt(adapter, q->rspq.cntxt_id, irq_vec_idx,
3061 q->rspq.phys_addr, q->rspq.size,
3062 q->fl[0].buf_size - SGE_PG_RSVD, 1, 0);
3063 if (ret)
3064 goto err_unlock;
3065
3066 for (i = 0; i < SGE_RXQ_PER_SET; ++i) {
3067 ret = t3_sge_init_flcntxt(adapter, q->fl[i].cntxt_id, 0,
3068 q->fl[i].phys_addr, q->fl[i].size,
3069 q->fl[i].buf_size - SGE_PG_RSVD,
3070 p->cong_thres, 1, 0);
3071 if (ret)
3072 goto err_unlock;
3073 }
3074
3075 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_ETH].cntxt_id, USE_GTS,
3076 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr,
3077 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token,
3078 1, 0);
3079 if (ret)
3080 goto err_unlock;
3081
3082 if (ntxq > 1) {
3083 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_OFLD].cntxt_id,
3084 USE_GTS, SGE_CNTXT_OFLD, id,
3085 q->txq[TXQ_OFLD].phys_addr,
3086 q->txq[TXQ_OFLD].size, 0, 1, 0);
3087 if (ret)
3088 goto err_unlock;
3089 }
3090
3091 if (ntxq > 2) {
3092 ret = t3_sge_init_ecntxt(adapter, q->txq[TXQ_CTRL].cntxt_id, 0,
3093 SGE_CNTXT_CTRL, id,
3094 q->txq[TXQ_CTRL].phys_addr,
3095 q->txq[TXQ_CTRL].size,
3096 q->txq[TXQ_CTRL].token, 1, 0);
3097 if (ret)
3098 goto err_unlock;
3099 }
3100
3101 spin_unlock_irq(&adapter->sge.reg_lock);
3102
3103 q->adap = adapter;
3104 q->netdev = dev;
3105 q->tx_q = netdevq;
3106 t3_update_qset_coalesce(q, p);
3107
3108 avail = refill_fl(adapter, &q->fl[0], q->fl[0].size,
3109 GFP_KERNEL | __GFP_COMP);
3110 if (!avail) {
3111 CH_ALERT(adapter, "free list queue 0 initialization failed\n");
3112 goto err;
3113 }
3114 if (avail < q->fl[0].size)
3115 CH_WARN(adapter, "free list queue 0 enabled with %d credits\n",
3116 avail);
3117
3118 avail = refill_fl(adapter, &q->fl[1], q->fl[1].size,
3119 GFP_KERNEL | __GFP_COMP);
3120 if (avail < q->fl[1].size)
3121 CH_WARN(adapter, "free list queue 1 enabled with %d credits\n",
3122 avail);
3123 refill_rspq(adapter, &q->rspq, q->rspq.size - 1);
3124
3125 t3_write_reg(adapter, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) |
3126 V_NEWTIMER(q->rspq.holdoff_tmr));
3127
3128 return 0;
3129
3130err_unlock:
3131 spin_unlock_irq(&adapter->sge.reg_lock);
3132err:
3133 t3_free_qset(adapter, q);
3134 return ret;
3135}
3136
3137/**
3138 * t3_start_sge_timers - start SGE timer call backs
3139 * @adap: the adapter
3140 *
3141 * Starts each SGE queue set's timer call back
3142 */
3143void t3_start_sge_timers(struct adapter *adap)
3144{
3145 int i;
3146
3147 for (i = 0; i < SGE_QSETS; ++i) {
3148 struct sge_qset *q = &adap->sge.qs[i];
3149
3150 if (q->tx_reclaim_timer.function)
3151 mod_timer(&q->tx_reclaim_timer, jiffies + TX_RECLAIM_PERIOD);
3152
3153 if (q->rx_reclaim_timer.function)
3154 mod_timer(&q->rx_reclaim_timer, jiffies + RX_RECLAIM_PERIOD);
3155 }
3156}
3157
3158/**
3159 * t3_stop_sge_timers - stop SGE timer call backs
3160 * @adap: the adapter
3161 *
3162 * Stops each SGE queue set's timer call back
3163 */
3164void t3_stop_sge_timers(struct adapter *adap)
3165{
3166 int i;
3167
3168 for (i = 0; i < SGE_QSETS; ++i) {
3169 struct sge_qset *q = &adap->sge.qs[i];
3170
3171 if (q->tx_reclaim_timer.function)
3172 del_timer_sync(&q->tx_reclaim_timer);
3173 if (q->rx_reclaim_timer.function)
3174 del_timer_sync(&q->rx_reclaim_timer);
3175 }
3176}
3177
3178/**
3179 * t3_free_sge_resources - free SGE resources
3180 * @adap: the adapter
3181 *
3182 * Frees resources used by the SGE queue sets.
3183 */
3184void t3_free_sge_resources(struct adapter *adap)
3185{
3186 int i;
3187
3188 for (i = 0; i < SGE_QSETS; ++i)
3189 t3_free_qset(adap, &adap->sge.qs[i]);
3190}
3191
3192/**
3193 * t3_sge_start - enable SGE
3194 * @adap: the adapter
3195 *
3196 * Enables the SGE for DMAs. This is the last step in starting packet
3197 * transfers.
3198 */
3199void t3_sge_start(struct adapter *adap)
3200{
3201 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, F_GLOBALENABLE);
3202}
3203
3204/**
3205 * t3_sge_stop - disable SGE operation
3206 * @adap: the adapter
3207 *
3208 * Disables the DMA engine. This can be called in emeregencies (e.g.,
3209 * from error interrupts) or from normal process context. In the latter
3210 * case it also disables any pending queue restart tasklets. Note that
3211 * if it is called in interrupt context it cannot disable the restart
3212 * tasklets as it cannot wait, however the tasklets will have no effect
3213 * since the doorbells are disabled and the driver will call this again
3214 * later from process context, at which time the tasklets will be stopped
3215 * if they are still running.
3216 */
3217void t3_sge_stop(struct adapter *adap)
3218{
3219 t3_set_reg_field(adap, A_SG_CONTROL, F_GLOBALENABLE, 0);
3220 if (!in_interrupt()) {
3221 int i;
3222
3223 for (i = 0; i < SGE_QSETS; ++i) {
3224 struct sge_qset *qs = &adap->sge.qs[i];
3225
3226 tasklet_kill(&qs->txq[TXQ_OFLD].qresume_tsk);
3227 tasklet_kill(&qs->txq[TXQ_CTRL].qresume_tsk);
3228 }
3229 }
3230}
3231
3232/**
3233 * t3_sge_init - initialize SGE
3234 * @adap: the adapter
3235 * @p: the SGE parameters
3236 *
3237 * Performs SGE initialization needed every time after a chip reset.
3238 * We do not initialize any of the queue sets here, instead the driver
3239 * top-level must request those individually. We also do not enable DMA
3240 * here, that should be done after the queues have been set up.
3241 */
3242void t3_sge_init(struct adapter *adap, struct sge_params *p)
3243{
3244 unsigned int ctrl, ups = ffs(pci_resource_len(adap->pdev, 2) >> 12);
3245
3246 ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL |
3247 F_CQCRDTCTRL | F_CONGMODE | F_TNLFLMODE | F_FATLPERREN |
3248 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS |
3249 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING;
3250#if SGE_NUM_GENBITS == 1
3251 ctrl |= F_EGRGENCTRL;
3252#endif
3253 if (adap->params.rev > 0) {
3254 if (!(adap->flags & (USING_MSIX | USING_MSI)))
3255 ctrl |= F_ONEINTMULTQ | F_OPTONEINTMULTQ;
3256 }
3257 t3_write_reg(adap, A_SG_CONTROL, ctrl);
3258 t3_write_reg(adap, A_SG_EGR_RCQ_DRB_THRSH, V_HIRCQDRBTHRSH(512) |
3259 V_LORCQDRBTHRSH(512));
3260 t3_write_reg(adap, A_SG_TIMER_TICK, core_ticks_per_usec(adap) / 10);
3261 t3_write_reg(adap, A_SG_CMDQ_CREDIT_TH, V_THRESHOLD(32) |
3262 V_TIMEOUT(200 * core_ticks_per_usec(adap)));
3263 t3_write_reg(adap, A_SG_HI_DRB_HI_THRSH,
3264 adap->params.rev < T3_REV_C ? 1000 : 500);
3265 t3_write_reg(adap, A_SG_HI_DRB_LO_THRSH, 256);
3266 t3_write_reg(adap, A_SG_LO_DRB_HI_THRSH, 1000);
3267 t3_write_reg(adap, A_SG_LO_DRB_LO_THRSH, 256);
3268 t3_write_reg(adap, A_SG_OCO_BASE, V_BASE1(0xfff));
3269 t3_write_reg(adap, A_SG_DRB_PRI_THRESH, 63 * 1024);
3270}
3271
3272/**
3273 * t3_sge_prep - one-time SGE initialization
3274 * @adap: the associated adapter
3275 * @p: SGE parameters
3276 *
3277 * Performs one-time initialization of SGE SW state. Includes determining
3278 * defaults for the assorted SGE parameters, which admins can change until
3279 * they are used to initialize the SGE.
3280 */
3281void t3_sge_prep(struct adapter *adap, struct sge_params *p)
3282{
3283 int i;
3284
3285 p->max_pkt_size = (16 * 1024) - sizeof(struct cpl_rx_data) -
3286 SKB_DATA_ALIGN(sizeof(struct skb_shared_info));
3287
3288 for (i = 0; i < SGE_QSETS; ++i) {
3289 struct qset_params *q = p->qset + i;
3290
3291 q->polling = adap->params.rev > 0;
3292 q->coalesce_usecs = 5;
3293 q->rspq_size = 1024;
3294 q->fl_size = 1024;
3295 q->jumbo_size = 512;
3296 q->txq_size[TXQ_ETH] = 1024;
3297 q->txq_size[TXQ_OFLD] = 1024;
3298 q->txq_size[TXQ_CTRL] = 256;
3299 q->cong_thres = 0;
3300 }
3301
3302 spin_lock_init(&adap->sge.reg_lock);
3303}