Loading...
Note: File does not exist in v6.13.7.
1/*
2 * This file is part of the Chelsio T4 Ethernet driver for Linux.
3 *
4 * Copyright (c) 2003-2010 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
36
37#include <linux/bitmap.h>
38#include <linux/crc32.h>
39#include <linux/ctype.h>
40#include <linux/debugfs.h>
41#include <linux/err.h>
42#include <linux/etherdevice.h>
43#include <linux/firmware.h>
44#include <linux/if_vlan.h>
45#include <linux/init.h>
46#include <linux/log2.h>
47#include <linux/mdio.h>
48#include <linux/module.h>
49#include <linux/moduleparam.h>
50#include <linux/mutex.h>
51#include <linux/netdevice.h>
52#include <linux/pci.h>
53#include <linux/aer.h>
54#include <linux/rtnetlink.h>
55#include <linux/sched.h>
56#include <linux/seq_file.h>
57#include <linux/sockios.h>
58#include <linux/vmalloc.h>
59#include <linux/workqueue.h>
60#include <net/neighbour.h>
61#include <net/netevent.h>
62#include <asm/uaccess.h>
63
64#include "cxgb4.h"
65#include "t4_regs.h"
66#include "t4_msg.h"
67#include "t4fw_api.h"
68#include "l2t.h"
69
70#define DRV_VERSION "1.3.0-ko"
71#define DRV_DESC "Chelsio T4 Network Driver"
72
73/*
74 * Max interrupt hold-off timer value in us. Queues fall back to this value
75 * under extreme memory pressure so it's largish to give the system time to
76 * recover.
77 */
78#define MAX_SGE_TIMERVAL 200U
79
80#ifdef CONFIG_PCI_IOV
81/*
82 * Virtual Function provisioning constants. We need two extra Ingress Queues
83 * with Interrupt capability to serve as the VF's Firmware Event Queue and
84 * Forwarded Interrupt Queue (when using MSI mode) -- neither will have Free
85 * Lists associated with them). For each Ethernet/Control Egress Queue and
86 * for each Free List, we need an Egress Context.
87 */
88enum {
89 VFRES_NPORTS = 1, /* # of "ports" per VF */
90 VFRES_NQSETS = 2, /* # of "Queue Sets" per VF */
91
92 VFRES_NVI = VFRES_NPORTS, /* # of Virtual Interfaces */
93 VFRES_NETHCTRL = VFRES_NQSETS, /* # of EQs used for ETH or CTRL Qs */
94 VFRES_NIQFLINT = VFRES_NQSETS+2,/* # of ingress Qs/w Free List(s)/intr */
95 VFRES_NIQ = 0, /* # of non-fl/int ingress queues */
96 VFRES_NEQ = VFRES_NQSETS*2, /* # of egress queues */
97 VFRES_TC = 0, /* PCI-E traffic class */
98 VFRES_NEXACTF = 16, /* # of exact MPS filters */
99
100 VFRES_R_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF|FW_CMD_CAP_PORT,
101 VFRES_WX_CAPS = FW_CMD_CAP_DMAQ|FW_CMD_CAP_VF,
102};
103
104/*
105 * Provide a Port Access Rights Mask for the specified PF/VF. This is very
106 * static and likely not to be useful in the long run. We really need to
107 * implement some form of persistent configuration which the firmware
108 * controls.
109 */
110static unsigned int pfvfres_pmask(struct adapter *adapter,
111 unsigned int pf, unsigned int vf)
112{
113 unsigned int portn, portvec;
114
115 /*
116 * Give PF's access to all of the ports.
117 */
118 if (vf == 0)
119 return FW_PFVF_CMD_PMASK_MASK;
120
121 /*
122 * For VFs, we'll assign them access to the ports based purely on the
123 * PF. We assign active ports in order, wrapping around if there are
124 * fewer active ports than PFs: e.g. active port[pf % nports].
125 * Unfortunately the adapter's port_info structs haven't been
126 * initialized yet so we have to compute this.
127 */
128 if (adapter->params.nports == 0)
129 return 0;
130
131 portn = pf % adapter->params.nports;
132 portvec = adapter->params.portvec;
133 for (;;) {
134 /*
135 * Isolate the lowest set bit in the port vector. If we're at
136 * the port number that we want, return that as the pmask.
137 * otherwise mask that bit out of the port vector and
138 * decrement our port number ...
139 */
140 unsigned int pmask = portvec ^ (portvec & (portvec-1));
141 if (portn == 0)
142 return pmask;
143 portn--;
144 portvec &= ~pmask;
145 }
146 /*NOTREACHED*/
147}
148#endif
149
150enum {
151 MEMWIN0_APERTURE = 65536,
152 MEMWIN0_BASE = 0x30000,
153 MEMWIN1_APERTURE = 32768,
154 MEMWIN1_BASE = 0x28000,
155 MEMWIN2_APERTURE = 2048,
156 MEMWIN2_BASE = 0x1b800,
157};
158
159enum {
160 MAX_TXQ_ENTRIES = 16384,
161 MAX_CTRL_TXQ_ENTRIES = 1024,
162 MAX_RSPQ_ENTRIES = 16384,
163 MAX_RX_BUFFERS = 16384,
164 MIN_TXQ_ENTRIES = 32,
165 MIN_CTRL_TXQ_ENTRIES = 32,
166 MIN_RSPQ_ENTRIES = 128,
167 MIN_FL_ENTRIES = 16
168};
169
170#define DFLT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK | \
171 NETIF_MSG_TIMER | NETIF_MSG_IFDOWN | NETIF_MSG_IFUP |\
172 NETIF_MSG_RX_ERR | NETIF_MSG_TX_ERR)
173
174#define CH_DEVICE(devid, data) { PCI_VDEVICE(CHELSIO, devid), (data) }
175
176static DEFINE_PCI_DEVICE_TABLE(cxgb4_pci_tbl) = {
177 CH_DEVICE(0xa000, 0), /* PE10K */
178 CH_DEVICE(0x4001, -1),
179 CH_DEVICE(0x4002, -1),
180 CH_DEVICE(0x4003, -1),
181 CH_DEVICE(0x4004, -1),
182 CH_DEVICE(0x4005, -1),
183 CH_DEVICE(0x4006, -1),
184 CH_DEVICE(0x4007, -1),
185 CH_DEVICE(0x4008, -1),
186 CH_DEVICE(0x4009, -1),
187 CH_DEVICE(0x400a, -1),
188 CH_DEVICE(0x4401, 4),
189 CH_DEVICE(0x4402, 4),
190 CH_DEVICE(0x4403, 4),
191 CH_DEVICE(0x4404, 4),
192 CH_DEVICE(0x4405, 4),
193 CH_DEVICE(0x4406, 4),
194 CH_DEVICE(0x4407, 4),
195 CH_DEVICE(0x4408, 4),
196 CH_DEVICE(0x4409, 4),
197 CH_DEVICE(0x440a, 4),
198 { 0, }
199};
200
201#define FW_FNAME "cxgb4/t4fw.bin"
202
203MODULE_DESCRIPTION(DRV_DESC);
204MODULE_AUTHOR("Chelsio Communications");
205MODULE_LICENSE("Dual BSD/GPL");
206MODULE_VERSION(DRV_VERSION);
207MODULE_DEVICE_TABLE(pci, cxgb4_pci_tbl);
208MODULE_FIRMWARE(FW_FNAME);
209
210static int dflt_msg_enable = DFLT_MSG_ENABLE;
211
212module_param(dflt_msg_enable, int, 0644);
213MODULE_PARM_DESC(dflt_msg_enable, "Chelsio T4 default message enable bitmap");
214
215/*
216 * The driver uses the best interrupt scheme available on a platform in the
217 * order MSI-X, MSI, legacy INTx interrupts. This parameter determines which
218 * of these schemes the driver may consider as follows:
219 *
220 * msi = 2: choose from among all three options
221 * msi = 1: only consider MSI and INTx interrupts
222 * msi = 0: force INTx interrupts
223 */
224static int msi = 2;
225
226module_param(msi, int, 0644);
227MODULE_PARM_DESC(msi, "whether to use INTx (0), MSI (1) or MSI-X (2)");
228
229/*
230 * Queue interrupt hold-off timer values. Queues default to the first of these
231 * upon creation.
232 */
233static unsigned int intr_holdoff[SGE_NTIMERS - 1] = { 5, 10, 20, 50, 100 };
234
235module_param_array(intr_holdoff, uint, NULL, 0644);
236MODULE_PARM_DESC(intr_holdoff, "values for queue interrupt hold-off timers "
237 "0..4 in microseconds");
238
239static unsigned int intr_cnt[SGE_NCOUNTERS - 1] = { 4, 8, 16 };
240
241module_param_array(intr_cnt, uint, NULL, 0644);
242MODULE_PARM_DESC(intr_cnt,
243 "thresholds 1..3 for queue interrupt packet counters");
244
245static int vf_acls;
246
247#ifdef CONFIG_PCI_IOV
248module_param(vf_acls, bool, 0644);
249MODULE_PARM_DESC(vf_acls, "if set enable virtualization L2 ACL enforcement");
250
251static unsigned int num_vf[4];
252
253module_param_array(num_vf, uint, NULL, 0644);
254MODULE_PARM_DESC(num_vf, "number of VFs for each of PFs 0-3");
255#endif
256
257static struct dentry *cxgb4_debugfs_root;
258
259static LIST_HEAD(adapter_list);
260static DEFINE_MUTEX(uld_mutex);
261static struct cxgb4_uld_info ulds[CXGB4_ULD_MAX];
262static const char *uld_str[] = { "RDMA", "iSCSI" };
263
264static void link_report(struct net_device *dev)
265{
266 if (!netif_carrier_ok(dev))
267 netdev_info(dev, "link down\n");
268 else {
269 static const char *fc[] = { "no", "Rx", "Tx", "Tx/Rx" };
270
271 const char *s = "10Mbps";
272 const struct port_info *p = netdev_priv(dev);
273
274 switch (p->link_cfg.speed) {
275 case SPEED_10000:
276 s = "10Gbps";
277 break;
278 case SPEED_1000:
279 s = "1000Mbps";
280 break;
281 case SPEED_100:
282 s = "100Mbps";
283 break;
284 }
285
286 netdev_info(dev, "link up, %s, full-duplex, %s PAUSE\n", s,
287 fc[p->link_cfg.fc]);
288 }
289}
290
291void t4_os_link_changed(struct adapter *adapter, int port_id, int link_stat)
292{
293 struct net_device *dev = adapter->port[port_id];
294
295 /* Skip changes from disabled ports. */
296 if (netif_running(dev) && link_stat != netif_carrier_ok(dev)) {
297 if (link_stat)
298 netif_carrier_on(dev);
299 else
300 netif_carrier_off(dev);
301
302 link_report(dev);
303 }
304}
305
306void t4_os_portmod_changed(const struct adapter *adap, int port_id)
307{
308 static const char *mod_str[] = {
309 NULL, "LR", "SR", "ER", "passive DA", "active DA", "LRM"
310 };
311
312 const struct net_device *dev = adap->port[port_id];
313 const struct port_info *pi = netdev_priv(dev);
314
315 if (pi->mod_type == FW_PORT_MOD_TYPE_NONE)
316 netdev_info(dev, "port module unplugged\n");
317 else if (pi->mod_type < ARRAY_SIZE(mod_str))
318 netdev_info(dev, "%s module inserted\n", mod_str[pi->mod_type]);
319}
320
321/*
322 * Configure the exact and hash address filters to handle a port's multicast
323 * and secondary unicast MAC addresses.
324 */
325static int set_addr_filters(const struct net_device *dev, bool sleep)
326{
327 u64 mhash = 0;
328 u64 uhash = 0;
329 bool free = true;
330 u16 filt_idx[7];
331 const u8 *addr[7];
332 int ret, naddr = 0;
333 const struct netdev_hw_addr *ha;
334 int uc_cnt = netdev_uc_count(dev);
335 int mc_cnt = netdev_mc_count(dev);
336 const struct port_info *pi = netdev_priv(dev);
337 unsigned int mb = pi->adapter->fn;
338
339 /* first do the secondary unicast addresses */
340 netdev_for_each_uc_addr(ha, dev) {
341 addr[naddr++] = ha->addr;
342 if (--uc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
343 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
344 naddr, addr, filt_idx, &uhash, sleep);
345 if (ret < 0)
346 return ret;
347
348 free = false;
349 naddr = 0;
350 }
351 }
352
353 /* next set up the multicast addresses */
354 netdev_for_each_mc_addr(ha, dev) {
355 addr[naddr++] = ha->addr;
356 if (--mc_cnt == 0 || naddr >= ARRAY_SIZE(addr)) {
357 ret = t4_alloc_mac_filt(pi->adapter, mb, pi->viid, free,
358 naddr, addr, filt_idx, &mhash, sleep);
359 if (ret < 0)
360 return ret;
361
362 free = false;
363 naddr = 0;
364 }
365 }
366
367 return t4_set_addr_hash(pi->adapter, mb, pi->viid, uhash != 0,
368 uhash | mhash, sleep);
369}
370
371/*
372 * Set Rx properties of a port, such as promiscruity, address filters, and MTU.
373 * If @mtu is -1 it is left unchanged.
374 */
375static int set_rxmode(struct net_device *dev, int mtu, bool sleep_ok)
376{
377 int ret;
378 struct port_info *pi = netdev_priv(dev);
379
380 ret = set_addr_filters(dev, sleep_ok);
381 if (ret == 0)
382 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, mtu,
383 (dev->flags & IFF_PROMISC) ? 1 : 0,
384 (dev->flags & IFF_ALLMULTI) ? 1 : 0, 1, -1,
385 sleep_ok);
386 return ret;
387}
388
389/**
390 * link_start - enable a port
391 * @dev: the port to enable
392 *
393 * Performs the MAC and PHY actions needed to enable a port.
394 */
395static int link_start(struct net_device *dev)
396{
397 int ret;
398 struct port_info *pi = netdev_priv(dev);
399 unsigned int mb = pi->adapter->fn;
400
401 /*
402 * We do not set address filters and promiscuity here, the stack does
403 * that step explicitly.
404 */
405 ret = t4_set_rxmode(pi->adapter, mb, pi->viid, dev->mtu, -1, -1, -1,
406 !!(dev->features & NETIF_F_HW_VLAN_RX), true);
407 if (ret == 0) {
408 ret = t4_change_mac(pi->adapter, mb, pi->viid,
409 pi->xact_addr_filt, dev->dev_addr, true,
410 true);
411 if (ret >= 0) {
412 pi->xact_addr_filt = ret;
413 ret = 0;
414 }
415 }
416 if (ret == 0)
417 ret = t4_link_start(pi->adapter, mb, pi->tx_chan,
418 &pi->link_cfg);
419 if (ret == 0)
420 ret = t4_enable_vi(pi->adapter, mb, pi->viid, true, true);
421 return ret;
422}
423
424/*
425 * Response queue handler for the FW event queue.
426 */
427static int fwevtq_handler(struct sge_rspq *q, const __be64 *rsp,
428 const struct pkt_gl *gl)
429{
430 u8 opcode = ((const struct rss_header *)rsp)->opcode;
431
432 rsp++; /* skip RSS header */
433 if (likely(opcode == CPL_SGE_EGR_UPDATE)) {
434 const struct cpl_sge_egr_update *p = (void *)rsp;
435 unsigned int qid = EGR_QID(ntohl(p->opcode_qid));
436 struct sge_txq *txq;
437
438 txq = q->adap->sge.egr_map[qid - q->adap->sge.egr_start];
439 txq->restarts++;
440 if ((u8 *)txq < (u8 *)q->adap->sge.ofldtxq) {
441 struct sge_eth_txq *eq;
442
443 eq = container_of(txq, struct sge_eth_txq, q);
444 netif_tx_wake_queue(eq->txq);
445 } else {
446 struct sge_ofld_txq *oq;
447
448 oq = container_of(txq, struct sge_ofld_txq, q);
449 tasklet_schedule(&oq->qresume_tsk);
450 }
451 } else if (opcode == CPL_FW6_MSG || opcode == CPL_FW4_MSG) {
452 const struct cpl_fw6_msg *p = (void *)rsp;
453
454 if (p->type == 0)
455 t4_handle_fw_rpl(q->adap, p->data);
456 } else if (opcode == CPL_L2T_WRITE_RPL) {
457 const struct cpl_l2t_write_rpl *p = (void *)rsp;
458
459 do_l2t_write_rpl(q->adap, p);
460 } else
461 dev_err(q->adap->pdev_dev,
462 "unexpected CPL %#x on FW event queue\n", opcode);
463 return 0;
464}
465
466/**
467 * uldrx_handler - response queue handler for ULD queues
468 * @q: the response queue that received the packet
469 * @rsp: the response queue descriptor holding the offload message
470 * @gl: the gather list of packet fragments
471 *
472 * Deliver an ingress offload packet to a ULD. All processing is done by
473 * the ULD, we just maintain statistics.
474 */
475static int uldrx_handler(struct sge_rspq *q, const __be64 *rsp,
476 const struct pkt_gl *gl)
477{
478 struct sge_ofld_rxq *rxq = container_of(q, struct sge_ofld_rxq, rspq);
479
480 if (ulds[q->uld].rx_handler(q->adap->uld_handle[q->uld], rsp, gl)) {
481 rxq->stats.nomem++;
482 return -1;
483 }
484 if (gl == NULL)
485 rxq->stats.imm++;
486 else if (gl == CXGB4_MSG_AN)
487 rxq->stats.an++;
488 else
489 rxq->stats.pkts++;
490 return 0;
491}
492
493static void disable_msi(struct adapter *adapter)
494{
495 if (adapter->flags & USING_MSIX) {
496 pci_disable_msix(adapter->pdev);
497 adapter->flags &= ~USING_MSIX;
498 } else if (adapter->flags & USING_MSI) {
499 pci_disable_msi(adapter->pdev);
500 adapter->flags &= ~USING_MSI;
501 }
502}
503
504/*
505 * Interrupt handler for non-data events used with MSI-X.
506 */
507static irqreturn_t t4_nondata_intr(int irq, void *cookie)
508{
509 struct adapter *adap = cookie;
510
511 u32 v = t4_read_reg(adap, MYPF_REG(PL_PF_INT_CAUSE));
512 if (v & PFSW) {
513 adap->swintr = 1;
514 t4_write_reg(adap, MYPF_REG(PL_PF_INT_CAUSE), v);
515 }
516 t4_slow_intr_handler(adap);
517 return IRQ_HANDLED;
518}
519
520/*
521 * Name the MSI-X interrupts.
522 */
523static void name_msix_vecs(struct adapter *adap)
524{
525 int i, j, msi_idx = 2, n = sizeof(adap->msix_info[0].desc);
526
527 /* non-data interrupts */
528 snprintf(adap->msix_info[0].desc, n, "%s", adap->port[0]->name);
529
530 /* FW events */
531 snprintf(adap->msix_info[1].desc, n, "%s-FWeventq",
532 adap->port[0]->name);
533
534 /* Ethernet queues */
535 for_each_port(adap, j) {
536 struct net_device *d = adap->port[j];
537 const struct port_info *pi = netdev_priv(d);
538
539 for (i = 0; i < pi->nqsets; i++, msi_idx++)
540 snprintf(adap->msix_info[msi_idx].desc, n, "%s-Rx%d",
541 d->name, i);
542 }
543
544 /* offload queues */
545 for_each_ofldrxq(&adap->sge, i)
546 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-ofld%d",
547 adap->port[0]->name, i);
548
549 for_each_rdmarxq(&adap->sge, i)
550 snprintf(adap->msix_info[msi_idx++].desc, n, "%s-rdma%d",
551 adap->port[0]->name, i);
552}
553
554static int request_msix_queue_irqs(struct adapter *adap)
555{
556 struct sge *s = &adap->sge;
557 int err, ethqidx, ofldqidx = 0, rdmaqidx = 0, msi = 2;
558
559 err = request_irq(adap->msix_info[1].vec, t4_sge_intr_msix, 0,
560 adap->msix_info[1].desc, &s->fw_evtq);
561 if (err)
562 return err;
563
564 for_each_ethrxq(s, ethqidx) {
565 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
566 adap->msix_info[msi].desc,
567 &s->ethrxq[ethqidx].rspq);
568 if (err)
569 goto unwind;
570 msi++;
571 }
572 for_each_ofldrxq(s, ofldqidx) {
573 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
574 adap->msix_info[msi].desc,
575 &s->ofldrxq[ofldqidx].rspq);
576 if (err)
577 goto unwind;
578 msi++;
579 }
580 for_each_rdmarxq(s, rdmaqidx) {
581 err = request_irq(adap->msix_info[msi].vec, t4_sge_intr_msix, 0,
582 adap->msix_info[msi].desc,
583 &s->rdmarxq[rdmaqidx].rspq);
584 if (err)
585 goto unwind;
586 msi++;
587 }
588 return 0;
589
590unwind:
591 while (--rdmaqidx >= 0)
592 free_irq(adap->msix_info[--msi].vec,
593 &s->rdmarxq[rdmaqidx].rspq);
594 while (--ofldqidx >= 0)
595 free_irq(adap->msix_info[--msi].vec,
596 &s->ofldrxq[ofldqidx].rspq);
597 while (--ethqidx >= 0)
598 free_irq(adap->msix_info[--msi].vec, &s->ethrxq[ethqidx].rspq);
599 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
600 return err;
601}
602
603static void free_msix_queue_irqs(struct adapter *adap)
604{
605 int i, msi = 2;
606 struct sge *s = &adap->sge;
607
608 free_irq(adap->msix_info[1].vec, &s->fw_evtq);
609 for_each_ethrxq(s, i)
610 free_irq(adap->msix_info[msi++].vec, &s->ethrxq[i].rspq);
611 for_each_ofldrxq(s, i)
612 free_irq(adap->msix_info[msi++].vec, &s->ofldrxq[i].rspq);
613 for_each_rdmarxq(s, i)
614 free_irq(adap->msix_info[msi++].vec, &s->rdmarxq[i].rspq);
615}
616
617/**
618 * write_rss - write the RSS table for a given port
619 * @pi: the port
620 * @queues: array of queue indices for RSS
621 *
622 * Sets up the portion of the HW RSS table for the port's VI to distribute
623 * packets to the Rx queues in @queues.
624 */
625static int write_rss(const struct port_info *pi, const u16 *queues)
626{
627 u16 *rss;
628 int i, err;
629 const struct sge_eth_rxq *q = &pi->adapter->sge.ethrxq[pi->first_qset];
630
631 rss = kmalloc(pi->rss_size * sizeof(u16), GFP_KERNEL);
632 if (!rss)
633 return -ENOMEM;
634
635 /* map the queue indices to queue ids */
636 for (i = 0; i < pi->rss_size; i++, queues++)
637 rss[i] = q[*queues].rspq.abs_id;
638
639 err = t4_config_rss_range(pi->adapter, pi->adapter->fn, pi->viid, 0,
640 pi->rss_size, rss, pi->rss_size);
641 kfree(rss);
642 return err;
643}
644
645/**
646 * setup_rss - configure RSS
647 * @adap: the adapter
648 *
649 * Sets up RSS for each port.
650 */
651static int setup_rss(struct adapter *adap)
652{
653 int i, err;
654
655 for_each_port(adap, i) {
656 const struct port_info *pi = adap2pinfo(adap, i);
657
658 err = write_rss(pi, pi->rss);
659 if (err)
660 return err;
661 }
662 return 0;
663}
664
665/*
666 * Return the channel of the ingress queue with the given qid.
667 */
668static unsigned int rxq_to_chan(const struct sge *p, unsigned int qid)
669{
670 qid -= p->ingr_start;
671 return netdev2pinfo(p->ingr_map[qid]->netdev)->tx_chan;
672}
673
674/*
675 * Wait until all NAPI handlers are descheduled.
676 */
677static void quiesce_rx(struct adapter *adap)
678{
679 int i;
680
681 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
682 struct sge_rspq *q = adap->sge.ingr_map[i];
683
684 if (q && q->handler)
685 napi_disable(&q->napi);
686 }
687}
688
689/*
690 * Enable NAPI scheduling and interrupt generation for all Rx queues.
691 */
692static void enable_rx(struct adapter *adap)
693{
694 int i;
695
696 for (i = 0; i < ARRAY_SIZE(adap->sge.ingr_map); i++) {
697 struct sge_rspq *q = adap->sge.ingr_map[i];
698
699 if (!q)
700 continue;
701 if (q->handler)
702 napi_enable(&q->napi);
703 /* 0-increment GTS to start the timer and enable interrupts */
704 t4_write_reg(adap, MYPF_REG(SGE_PF_GTS),
705 SEINTARM(q->intr_params) |
706 INGRESSQID(q->cntxt_id));
707 }
708}
709
710/**
711 * setup_sge_queues - configure SGE Tx/Rx/response queues
712 * @adap: the adapter
713 *
714 * Determines how many sets of SGE queues to use and initializes them.
715 * We support multiple queue sets per port if we have MSI-X, otherwise
716 * just one queue set per port.
717 */
718static int setup_sge_queues(struct adapter *adap)
719{
720 int err, msi_idx, i, j;
721 struct sge *s = &adap->sge;
722
723 bitmap_zero(s->starving_fl, MAX_EGRQ);
724 bitmap_zero(s->txq_maperr, MAX_EGRQ);
725
726 if (adap->flags & USING_MSIX)
727 msi_idx = 1; /* vector 0 is for non-queue interrupts */
728 else {
729 err = t4_sge_alloc_rxq(adap, &s->intrq, false, adap->port[0], 0,
730 NULL, NULL);
731 if (err)
732 return err;
733 msi_idx = -((int)s->intrq.abs_id + 1);
734 }
735
736 err = t4_sge_alloc_rxq(adap, &s->fw_evtq, true, adap->port[0],
737 msi_idx, NULL, fwevtq_handler);
738 if (err) {
739freeout: t4_free_sge_resources(adap);
740 return err;
741 }
742
743 for_each_port(adap, i) {
744 struct net_device *dev = adap->port[i];
745 struct port_info *pi = netdev_priv(dev);
746 struct sge_eth_rxq *q = &s->ethrxq[pi->first_qset];
747 struct sge_eth_txq *t = &s->ethtxq[pi->first_qset];
748
749 for (j = 0; j < pi->nqsets; j++, q++) {
750 if (msi_idx > 0)
751 msi_idx++;
752 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev,
753 msi_idx, &q->fl,
754 t4_ethrx_handler);
755 if (err)
756 goto freeout;
757 q->rspq.idx = j;
758 memset(&q->stats, 0, sizeof(q->stats));
759 }
760 for (j = 0; j < pi->nqsets; j++, t++) {
761 err = t4_sge_alloc_eth_txq(adap, t, dev,
762 netdev_get_tx_queue(dev, j),
763 s->fw_evtq.cntxt_id);
764 if (err)
765 goto freeout;
766 }
767 }
768
769 j = s->ofldqsets / adap->params.nports; /* ofld queues per channel */
770 for_each_ofldrxq(s, i) {
771 struct sge_ofld_rxq *q = &s->ofldrxq[i];
772 struct net_device *dev = adap->port[i / j];
773
774 if (msi_idx > 0)
775 msi_idx++;
776 err = t4_sge_alloc_rxq(adap, &q->rspq, false, dev, msi_idx,
777 &q->fl, uldrx_handler);
778 if (err)
779 goto freeout;
780 memset(&q->stats, 0, sizeof(q->stats));
781 s->ofld_rxq[i] = q->rspq.abs_id;
782 err = t4_sge_alloc_ofld_txq(adap, &s->ofldtxq[i], dev,
783 s->fw_evtq.cntxt_id);
784 if (err)
785 goto freeout;
786 }
787
788 for_each_rdmarxq(s, i) {
789 struct sge_ofld_rxq *q = &s->rdmarxq[i];
790
791 if (msi_idx > 0)
792 msi_idx++;
793 err = t4_sge_alloc_rxq(adap, &q->rspq, false, adap->port[i],
794 msi_idx, &q->fl, uldrx_handler);
795 if (err)
796 goto freeout;
797 memset(&q->stats, 0, sizeof(q->stats));
798 s->rdma_rxq[i] = q->rspq.abs_id;
799 }
800
801 for_each_port(adap, i) {
802 /*
803 * Note that ->rdmarxq[i].rspq.cntxt_id below is 0 if we don't
804 * have RDMA queues, and that's the right value.
805 */
806 err = t4_sge_alloc_ctrl_txq(adap, &s->ctrlq[i], adap->port[i],
807 s->fw_evtq.cntxt_id,
808 s->rdmarxq[i].rspq.cntxt_id);
809 if (err)
810 goto freeout;
811 }
812
813 t4_write_reg(adap, MPS_TRC_RSS_CONTROL,
814 RSSCONTROL(netdev2pinfo(adap->port[0])->tx_chan) |
815 QUEUENUMBER(s->ethrxq[0].rspq.abs_id));
816 return 0;
817}
818
819/*
820 * Returns 0 if new FW was successfully loaded, a positive errno if a load was
821 * started but failed, and a negative errno if flash load couldn't start.
822 */
823static int upgrade_fw(struct adapter *adap)
824{
825 int ret;
826 u32 vers;
827 const struct fw_hdr *hdr;
828 const struct firmware *fw;
829 struct device *dev = adap->pdev_dev;
830
831 ret = request_firmware(&fw, FW_FNAME, dev);
832 if (ret < 0) {
833 dev_err(dev, "unable to load firmware image " FW_FNAME
834 ", error %d\n", ret);
835 return ret;
836 }
837
838 hdr = (const struct fw_hdr *)fw->data;
839 vers = ntohl(hdr->fw_ver);
840 if (FW_HDR_FW_VER_MAJOR_GET(vers) != FW_VERSION_MAJOR) {
841 ret = -EINVAL; /* wrong major version, won't do */
842 goto out;
843 }
844
845 /*
846 * If the flash FW is unusable or we found something newer, load it.
847 */
848 if (FW_HDR_FW_VER_MAJOR_GET(adap->params.fw_vers) != FW_VERSION_MAJOR ||
849 vers > adap->params.fw_vers) {
850 ret = -t4_load_fw(adap, fw->data, fw->size);
851 if (!ret)
852 dev_info(dev, "firmware upgraded to version %pI4 from "
853 FW_FNAME "\n", &hdr->fw_ver);
854 }
855out: release_firmware(fw);
856 return ret;
857}
858
859/*
860 * Allocate a chunk of memory using kmalloc or, if that fails, vmalloc.
861 * The allocated memory is cleared.
862 */
863void *t4_alloc_mem(size_t size)
864{
865 void *p = kzalloc(size, GFP_KERNEL);
866
867 if (!p)
868 p = vzalloc(size);
869 return p;
870}
871
872/*
873 * Free memory allocated through alloc_mem().
874 */
875static void t4_free_mem(void *addr)
876{
877 if (is_vmalloc_addr(addr))
878 vfree(addr);
879 else
880 kfree(addr);
881}
882
883static inline int is_offload(const struct adapter *adap)
884{
885 return adap->params.offload;
886}
887
888/*
889 * Implementation of ethtool operations.
890 */
891
892static u32 get_msglevel(struct net_device *dev)
893{
894 return netdev2adap(dev)->msg_enable;
895}
896
897static void set_msglevel(struct net_device *dev, u32 val)
898{
899 netdev2adap(dev)->msg_enable = val;
900}
901
902static char stats_strings[][ETH_GSTRING_LEN] = {
903 "TxOctetsOK ",
904 "TxFramesOK ",
905 "TxBroadcastFrames ",
906 "TxMulticastFrames ",
907 "TxUnicastFrames ",
908 "TxErrorFrames ",
909
910 "TxFrames64 ",
911 "TxFrames65To127 ",
912 "TxFrames128To255 ",
913 "TxFrames256To511 ",
914 "TxFrames512To1023 ",
915 "TxFrames1024To1518 ",
916 "TxFrames1519ToMax ",
917
918 "TxFramesDropped ",
919 "TxPauseFrames ",
920 "TxPPP0Frames ",
921 "TxPPP1Frames ",
922 "TxPPP2Frames ",
923 "TxPPP3Frames ",
924 "TxPPP4Frames ",
925 "TxPPP5Frames ",
926 "TxPPP6Frames ",
927 "TxPPP7Frames ",
928
929 "RxOctetsOK ",
930 "RxFramesOK ",
931 "RxBroadcastFrames ",
932 "RxMulticastFrames ",
933 "RxUnicastFrames ",
934
935 "RxFramesTooLong ",
936 "RxJabberErrors ",
937 "RxFCSErrors ",
938 "RxLengthErrors ",
939 "RxSymbolErrors ",
940 "RxRuntFrames ",
941
942 "RxFrames64 ",
943 "RxFrames65To127 ",
944 "RxFrames128To255 ",
945 "RxFrames256To511 ",
946 "RxFrames512To1023 ",
947 "RxFrames1024To1518 ",
948 "RxFrames1519ToMax ",
949
950 "RxPauseFrames ",
951 "RxPPP0Frames ",
952 "RxPPP1Frames ",
953 "RxPPP2Frames ",
954 "RxPPP3Frames ",
955 "RxPPP4Frames ",
956 "RxPPP5Frames ",
957 "RxPPP6Frames ",
958 "RxPPP7Frames ",
959
960 "RxBG0FramesDropped ",
961 "RxBG1FramesDropped ",
962 "RxBG2FramesDropped ",
963 "RxBG3FramesDropped ",
964 "RxBG0FramesTrunc ",
965 "RxBG1FramesTrunc ",
966 "RxBG2FramesTrunc ",
967 "RxBG3FramesTrunc ",
968
969 "TSO ",
970 "TxCsumOffload ",
971 "RxCsumGood ",
972 "VLANextractions ",
973 "VLANinsertions ",
974 "GROpackets ",
975 "GROmerged ",
976};
977
978static int get_sset_count(struct net_device *dev, int sset)
979{
980 switch (sset) {
981 case ETH_SS_STATS:
982 return ARRAY_SIZE(stats_strings);
983 default:
984 return -EOPNOTSUPP;
985 }
986}
987
988#define T4_REGMAP_SIZE (160 * 1024)
989
990static int get_regs_len(struct net_device *dev)
991{
992 return T4_REGMAP_SIZE;
993}
994
995static int get_eeprom_len(struct net_device *dev)
996{
997 return EEPROMSIZE;
998}
999
1000static void get_drvinfo(struct net_device *dev, struct ethtool_drvinfo *info)
1001{
1002 struct adapter *adapter = netdev2adap(dev);
1003
1004 strcpy(info->driver, KBUILD_MODNAME);
1005 strcpy(info->version, DRV_VERSION);
1006 strcpy(info->bus_info, pci_name(adapter->pdev));
1007
1008 if (!adapter->params.fw_vers)
1009 strcpy(info->fw_version, "N/A");
1010 else
1011 snprintf(info->fw_version, sizeof(info->fw_version),
1012 "%u.%u.%u.%u, TP %u.%u.%u.%u",
1013 FW_HDR_FW_VER_MAJOR_GET(adapter->params.fw_vers),
1014 FW_HDR_FW_VER_MINOR_GET(adapter->params.fw_vers),
1015 FW_HDR_FW_VER_MICRO_GET(adapter->params.fw_vers),
1016 FW_HDR_FW_VER_BUILD_GET(adapter->params.fw_vers),
1017 FW_HDR_FW_VER_MAJOR_GET(adapter->params.tp_vers),
1018 FW_HDR_FW_VER_MINOR_GET(adapter->params.tp_vers),
1019 FW_HDR_FW_VER_MICRO_GET(adapter->params.tp_vers),
1020 FW_HDR_FW_VER_BUILD_GET(adapter->params.tp_vers));
1021}
1022
1023static void get_strings(struct net_device *dev, u32 stringset, u8 *data)
1024{
1025 if (stringset == ETH_SS_STATS)
1026 memcpy(data, stats_strings, sizeof(stats_strings));
1027}
1028
1029/*
1030 * port stats maintained per queue of the port. They should be in the same
1031 * order as in stats_strings above.
1032 */
1033struct queue_port_stats {
1034 u64 tso;
1035 u64 tx_csum;
1036 u64 rx_csum;
1037 u64 vlan_ex;
1038 u64 vlan_ins;
1039 u64 gro_pkts;
1040 u64 gro_merged;
1041};
1042
1043static void collect_sge_port_stats(const struct adapter *adap,
1044 const struct port_info *p, struct queue_port_stats *s)
1045{
1046 int i;
1047 const struct sge_eth_txq *tx = &adap->sge.ethtxq[p->first_qset];
1048 const struct sge_eth_rxq *rx = &adap->sge.ethrxq[p->first_qset];
1049
1050 memset(s, 0, sizeof(*s));
1051 for (i = 0; i < p->nqsets; i++, rx++, tx++) {
1052 s->tso += tx->tso;
1053 s->tx_csum += tx->tx_cso;
1054 s->rx_csum += rx->stats.rx_cso;
1055 s->vlan_ex += rx->stats.vlan_ex;
1056 s->vlan_ins += tx->vlan_ins;
1057 s->gro_pkts += rx->stats.lro_pkts;
1058 s->gro_merged += rx->stats.lro_merged;
1059 }
1060}
1061
1062static void get_stats(struct net_device *dev, struct ethtool_stats *stats,
1063 u64 *data)
1064{
1065 struct port_info *pi = netdev_priv(dev);
1066 struct adapter *adapter = pi->adapter;
1067
1068 t4_get_port_stats(adapter, pi->tx_chan, (struct port_stats *)data);
1069
1070 data += sizeof(struct port_stats) / sizeof(u64);
1071 collect_sge_port_stats(adapter, pi, (struct queue_port_stats *)data);
1072}
1073
1074/*
1075 * Return a version number to identify the type of adapter. The scheme is:
1076 * - bits 0..9: chip version
1077 * - bits 10..15: chip revision
1078 * - bits 16..23: register dump version
1079 */
1080static inline unsigned int mk_adap_vers(const struct adapter *ap)
1081{
1082 return 4 | (ap->params.rev << 10) | (1 << 16);
1083}
1084
1085static void reg_block_dump(struct adapter *ap, void *buf, unsigned int start,
1086 unsigned int end)
1087{
1088 u32 *p = buf + start;
1089
1090 for ( ; start <= end; start += sizeof(u32))
1091 *p++ = t4_read_reg(ap, start);
1092}
1093
1094static void get_regs(struct net_device *dev, struct ethtool_regs *regs,
1095 void *buf)
1096{
1097 static const unsigned int reg_ranges[] = {
1098 0x1008, 0x1108,
1099 0x1180, 0x11b4,
1100 0x11fc, 0x123c,
1101 0x1300, 0x173c,
1102 0x1800, 0x18fc,
1103 0x3000, 0x30d8,
1104 0x30e0, 0x5924,
1105 0x5960, 0x59d4,
1106 0x5a00, 0x5af8,
1107 0x6000, 0x6098,
1108 0x6100, 0x6150,
1109 0x6200, 0x6208,
1110 0x6240, 0x6248,
1111 0x6280, 0x6338,
1112 0x6370, 0x638c,
1113 0x6400, 0x643c,
1114 0x6500, 0x6524,
1115 0x6a00, 0x6a38,
1116 0x6a60, 0x6a78,
1117 0x6b00, 0x6b84,
1118 0x6bf0, 0x6c84,
1119 0x6cf0, 0x6d84,
1120 0x6df0, 0x6e84,
1121 0x6ef0, 0x6f84,
1122 0x6ff0, 0x7084,
1123 0x70f0, 0x7184,
1124 0x71f0, 0x7284,
1125 0x72f0, 0x7384,
1126 0x73f0, 0x7450,
1127 0x7500, 0x7530,
1128 0x7600, 0x761c,
1129 0x7680, 0x76cc,
1130 0x7700, 0x7798,
1131 0x77c0, 0x77fc,
1132 0x7900, 0x79fc,
1133 0x7b00, 0x7c38,
1134 0x7d00, 0x7efc,
1135 0x8dc0, 0x8e1c,
1136 0x8e30, 0x8e78,
1137 0x8ea0, 0x8f6c,
1138 0x8fc0, 0x9074,
1139 0x90fc, 0x90fc,
1140 0x9400, 0x9458,
1141 0x9600, 0x96bc,
1142 0x9800, 0x9808,
1143 0x9820, 0x983c,
1144 0x9850, 0x9864,
1145 0x9c00, 0x9c6c,
1146 0x9c80, 0x9cec,
1147 0x9d00, 0x9d6c,
1148 0x9d80, 0x9dec,
1149 0x9e00, 0x9e6c,
1150 0x9e80, 0x9eec,
1151 0x9f00, 0x9f6c,
1152 0x9f80, 0x9fec,
1153 0xd004, 0xd03c,
1154 0xdfc0, 0xdfe0,
1155 0xe000, 0xea7c,
1156 0xf000, 0x11190,
1157 0x19040, 0x1906c,
1158 0x19078, 0x19080,
1159 0x1908c, 0x19124,
1160 0x19150, 0x191b0,
1161 0x191d0, 0x191e8,
1162 0x19238, 0x1924c,
1163 0x193f8, 0x19474,
1164 0x19490, 0x194f8,
1165 0x19800, 0x19f30,
1166 0x1a000, 0x1a06c,
1167 0x1a0b0, 0x1a120,
1168 0x1a128, 0x1a138,
1169 0x1a190, 0x1a1c4,
1170 0x1a1fc, 0x1a1fc,
1171 0x1e040, 0x1e04c,
1172 0x1e284, 0x1e28c,
1173 0x1e2c0, 0x1e2c0,
1174 0x1e2e0, 0x1e2e0,
1175 0x1e300, 0x1e384,
1176 0x1e3c0, 0x1e3c8,
1177 0x1e440, 0x1e44c,
1178 0x1e684, 0x1e68c,
1179 0x1e6c0, 0x1e6c0,
1180 0x1e6e0, 0x1e6e0,
1181 0x1e700, 0x1e784,
1182 0x1e7c0, 0x1e7c8,
1183 0x1e840, 0x1e84c,
1184 0x1ea84, 0x1ea8c,
1185 0x1eac0, 0x1eac0,
1186 0x1eae0, 0x1eae0,
1187 0x1eb00, 0x1eb84,
1188 0x1ebc0, 0x1ebc8,
1189 0x1ec40, 0x1ec4c,
1190 0x1ee84, 0x1ee8c,
1191 0x1eec0, 0x1eec0,
1192 0x1eee0, 0x1eee0,
1193 0x1ef00, 0x1ef84,
1194 0x1efc0, 0x1efc8,
1195 0x1f040, 0x1f04c,
1196 0x1f284, 0x1f28c,
1197 0x1f2c0, 0x1f2c0,
1198 0x1f2e0, 0x1f2e0,
1199 0x1f300, 0x1f384,
1200 0x1f3c0, 0x1f3c8,
1201 0x1f440, 0x1f44c,
1202 0x1f684, 0x1f68c,
1203 0x1f6c0, 0x1f6c0,
1204 0x1f6e0, 0x1f6e0,
1205 0x1f700, 0x1f784,
1206 0x1f7c0, 0x1f7c8,
1207 0x1f840, 0x1f84c,
1208 0x1fa84, 0x1fa8c,
1209 0x1fac0, 0x1fac0,
1210 0x1fae0, 0x1fae0,
1211 0x1fb00, 0x1fb84,
1212 0x1fbc0, 0x1fbc8,
1213 0x1fc40, 0x1fc4c,
1214 0x1fe84, 0x1fe8c,
1215 0x1fec0, 0x1fec0,
1216 0x1fee0, 0x1fee0,
1217 0x1ff00, 0x1ff84,
1218 0x1ffc0, 0x1ffc8,
1219 0x20000, 0x2002c,
1220 0x20100, 0x2013c,
1221 0x20190, 0x201c8,
1222 0x20200, 0x20318,
1223 0x20400, 0x20528,
1224 0x20540, 0x20614,
1225 0x21000, 0x21040,
1226 0x2104c, 0x21060,
1227 0x210c0, 0x210ec,
1228 0x21200, 0x21268,
1229 0x21270, 0x21284,
1230 0x212fc, 0x21388,
1231 0x21400, 0x21404,
1232 0x21500, 0x21518,
1233 0x2152c, 0x2153c,
1234 0x21550, 0x21554,
1235 0x21600, 0x21600,
1236 0x21608, 0x21628,
1237 0x21630, 0x2163c,
1238 0x21700, 0x2171c,
1239 0x21780, 0x2178c,
1240 0x21800, 0x21c38,
1241 0x21c80, 0x21d7c,
1242 0x21e00, 0x21e04,
1243 0x22000, 0x2202c,
1244 0x22100, 0x2213c,
1245 0x22190, 0x221c8,
1246 0x22200, 0x22318,
1247 0x22400, 0x22528,
1248 0x22540, 0x22614,
1249 0x23000, 0x23040,
1250 0x2304c, 0x23060,
1251 0x230c0, 0x230ec,
1252 0x23200, 0x23268,
1253 0x23270, 0x23284,
1254 0x232fc, 0x23388,
1255 0x23400, 0x23404,
1256 0x23500, 0x23518,
1257 0x2352c, 0x2353c,
1258 0x23550, 0x23554,
1259 0x23600, 0x23600,
1260 0x23608, 0x23628,
1261 0x23630, 0x2363c,
1262 0x23700, 0x2371c,
1263 0x23780, 0x2378c,
1264 0x23800, 0x23c38,
1265 0x23c80, 0x23d7c,
1266 0x23e00, 0x23e04,
1267 0x24000, 0x2402c,
1268 0x24100, 0x2413c,
1269 0x24190, 0x241c8,
1270 0x24200, 0x24318,
1271 0x24400, 0x24528,
1272 0x24540, 0x24614,
1273 0x25000, 0x25040,
1274 0x2504c, 0x25060,
1275 0x250c0, 0x250ec,
1276 0x25200, 0x25268,
1277 0x25270, 0x25284,
1278 0x252fc, 0x25388,
1279 0x25400, 0x25404,
1280 0x25500, 0x25518,
1281 0x2552c, 0x2553c,
1282 0x25550, 0x25554,
1283 0x25600, 0x25600,
1284 0x25608, 0x25628,
1285 0x25630, 0x2563c,
1286 0x25700, 0x2571c,
1287 0x25780, 0x2578c,
1288 0x25800, 0x25c38,
1289 0x25c80, 0x25d7c,
1290 0x25e00, 0x25e04,
1291 0x26000, 0x2602c,
1292 0x26100, 0x2613c,
1293 0x26190, 0x261c8,
1294 0x26200, 0x26318,
1295 0x26400, 0x26528,
1296 0x26540, 0x26614,
1297 0x27000, 0x27040,
1298 0x2704c, 0x27060,
1299 0x270c0, 0x270ec,
1300 0x27200, 0x27268,
1301 0x27270, 0x27284,
1302 0x272fc, 0x27388,
1303 0x27400, 0x27404,
1304 0x27500, 0x27518,
1305 0x2752c, 0x2753c,
1306 0x27550, 0x27554,
1307 0x27600, 0x27600,
1308 0x27608, 0x27628,
1309 0x27630, 0x2763c,
1310 0x27700, 0x2771c,
1311 0x27780, 0x2778c,
1312 0x27800, 0x27c38,
1313 0x27c80, 0x27d7c,
1314 0x27e00, 0x27e04
1315 };
1316
1317 int i;
1318 struct adapter *ap = netdev2adap(dev);
1319
1320 regs->version = mk_adap_vers(ap);
1321
1322 memset(buf, 0, T4_REGMAP_SIZE);
1323 for (i = 0; i < ARRAY_SIZE(reg_ranges); i += 2)
1324 reg_block_dump(ap, buf, reg_ranges[i], reg_ranges[i + 1]);
1325}
1326
1327static int restart_autoneg(struct net_device *dev)
1328{
1329 struct port_info *p = netdev_priv(dev);
1330
1331 if (!netif_running(dev))
1332 return -EAGAIN;
1333 if (p->link_cfg.autoneg != AUTONEG_ENABLE)
1334 return -EINVAL;
1335 t4_restart_aneg(p->adapter, p->adapter->fn, p->tx_chan);
1336 return 0;
1337}
1338
1339static int identify_port(struct net_device *dev,
1340 enum ethtool_phys_id_state state)
1341{
1342 unsigned int val;
1343 struct adapter *adap = netdev2adap(dev);
1344
1345 if (state == ETHTOOL_ID_ACTIVE)
1346 val = 0xffff;
1347 else if (state == ETHTOOL_ID_INACTIVE)
1348 val = 0;
1349 else
1350 return -EINVAL;
1351
1352 return t4_identify_port(adap, adap->fn, netdev2pinfo(dev)->viid, val);
1353}
1354
1355static unsigned int from_fw_linkcaps(unsigned int type, unsigned int caps)
1356{
1357 unsigned int v = 0;
1358
1359 if (type == FW_PORT_TYPE_BT_SGMII || type == FW_PORT_TYPE_BT_XFI ||
1360 type == FW_PORT_TYPE_BT_XAUI) {
1361 v |= SUPPORTED_TP;
1362 if (caps & FW_PORT_CAP_SPEED_100M)
1363 v |= SUPPORTED_100baseT_Full;
1364 if (caps & FW_PORT_CAP_SPEED_1G)
1365 v |= SUPPORTED_1000baseT_Full;
1366 if (caps & FW_PORT_CAP_SPEED_10G)
1367 v |= SUPPORTED_10000baseT_Full;
1368 } else if (type == FW_PORT_TYPE_KX4 || type == FW_PORT_TYPE_KX) {
1369 v |= SUPPORTED_Backplane;
1370 if (caps & FW_PORT_CAP_SPEED_1G)
1371 v |= SUPPORTED_1000baseKX_Full;
1372 if (caps & FW_PORT_CAP_SPEED_10G)
1373 v |= SUPPORTED_10000baseKX4_Full;
1374 } else if (type == FW_PORT_TYPE_KR)
1375 v |= SUPPORTED_Backplane | SUPPORTED_10000baseKR_Full;
1376 else if (type == FW_PORT_TYPE_BP_AP)
1377 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1378 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full;
1379 else if (type == FW_PORT_TYPE_BP4_AP)
1380 v |= SUPPORTED_Backplane | SUPPORTED_10000baseR_FEC |
1381 SUPPORTED_10000baseKR_Full | SUPPORTED_1000baseKX_Full |
1382 SUPPORTED_10000baseKX4_Full;
1383 else if (type == FW_PORT_TYPE_FIBER_XFI ||
1384 type == FW_PORT_TYPE_FIBER_XAUI || type == FW_PORT_TYPE_SFP)
1385 v |= SUPPORTED_FIBRE;
1386
1387 if (caps & FW_PORT_CAP_ANEG)
1388 v |= SUPPORTED_Autoneg;
1389 return v;
1390}
1391
1392static unsigned int to_fw_linkcaps(unsigned int caps)
1393{
1394 unsigned int v = 0;
1395
1396 if (caps & ADVERTISED_100baseT_Full)
1397 v |= FW_PORT_CAP_SPEED_100M;
1398 if (caps & ADVERTISED_1000baseT_Full)
1399 v |= FW_PORT_CAP_SPEED_1G;
1400 if (caps & ADVERTISED_10000baseT_Full)
1401 v |= FW_PORT_CAP_SPEED_10G;
1402 return v;
1403}
1404
1405static int get_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1406{
1407 const struct port_info *p = netdev_priv(dev);
1408
1409 if (p->port_type == FW_PORT_TYPE_BT_SGMII ||
1410 p->port_type == FW_PORT_TYPE_BT_XFI ||
1411 p->port_type == FW_PORT_TYPE_BT_XAUI)
1412 cmd->port = PORT_TP;
1413 else if (p->port_type == FW_PORT_TYPE_FIBER_XFI ||
1414 p->port_type == FW_PORT_TYPE_FIBER_XAUI)
1415 cmd->port = PORT_FIBRE;
1416 else if (p->port_type == FW_PORT_TYPE_SFP) {
1417 if (p->mod_type == FW_PORT_MOD_TYPE_TWINAX_PASSIVE ||
1418 p->mod_type == FW_PORT_MOD_TYPE_TWINAX_ACTIVE)
1419 cmd->port = PORT_DA;
1420 else
1421 cmd->port = PORT_FIBRE;
1422 } else
1423 cmd->port = PORT_OTHER;
1424
1425 if (p->mdio_addr >= 0) {
1426 cmd->phy_address = p->mdio_addr;
1427 cmd->transceiver = XCVR_EXTERNAL;
1428 cmd->mdio_support = p->port_type == FW_PORT_TYPE_BT_SGMII ?
1429 MDIO_SUPPORTS_C22 : MDIO_SUPPORTS_C45;
1430 } else {
1431 cmd->phy_address = 0; /* not really, but no better option */
1432 cmd->transceiver = XCVR_INTERNAL;
1433 cmd->mdio_support = 0;
1434 }
1435
1436 cmd->supported = from_fw_linkcaps(p->port_type, p->link_cfg.supported);
1437 cmd->advertising = from_fw_linkcaps(p->port_type,
1438 p->link_cfg.advertising);
1439 ethtool_cmd_speed_set(cmd,
1440 netif_carrier_ok(dev) ? p->link_cfg.speed : 0);
1441 cmd->duplex = DUPLEX_FULL;
1442 cmd->autoneg = p->link_cfg.autoneg;
1443 cmd->maxtxpkt = 0;
1444 cmd->maxrxpkt = 0;
1445 return 0;
1446}
1447
1448static unsigned int speed_to_caps(int speed)
1449{
1450 if (speed == SPEED_100)
1451 return FW_PORT_CAP_SPEED_100M;
1452 if (speed == SPEED_1000)
1453 return FW_PORT_CAP_SPEED_1G;
1454 if (speed == SPEED_10000)
1455 return FW_PORT_CAP_SPEED_10G;
1456 return 0;
1457}
1458
1459static int set_settings(struct net_device *dev, struct ethtool_cmd *cmd)
1460{
1461 unsigned int cap;
1462 struct port_info *p = netdev_priv(dev);
1463 struct link_config *lc = &p->link_cfg;
1464 u32 speed = ethtool_cmd_speed(cmd);
1465
1466 if (cmd->duplex != DUPLEX_FULL) /* only full-duplex supported */
1467 return -EINVAL;
1468
1469 if (!(lc->supported & FW_PORT_CAP_ANEG)) {
1470 /*
1471 * PHY offers a single speed. See if that's what's
1472 * being requested.
1473 */
1474 if (cmd->autoneg == AUTONEG_DISABLE &&
1475 (lc->supported & speed_to_caps(speed)))
1476 return 0;
1477 return -EINVAL;
1478 }
1479
1480 if (cmd->autoneg == AUTONEG_DISABLE) {
1481 cap = speed_to_caps(speed);
1482
1483 if (!(lc->supported & cap) || (speed == SPEED_1000) ||
1484 (speed == SPEED_10000))
1485 return -EINVAL;
1486 lc->requested_speed = cap;
1487 lc->advertising = 0;
1488 } else {
1489 cap = to_fw_linkcaps(cmd->advertising);
1490 if (!(lc->supported & cap))
1491 return -EINVAL;
1492 lc->requested_speed = 0;
1493 lc->advertising = cap | FW_PORT_CAP_ANEG;
1494 }
1495 lc->autoneg = cmd->autoneg;
1496
1497 if (netif_running(dev))
1498 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1499 lc);
1500 return 0;
1501}
1502
1503static void get_pauseparam(struct net_device *dev,
1504 struct ethtool_pauseparam *epause)
1505{
1506 struct port_info *p = netdev_priv(dev);
1507
1508 epause->autoneg = (p->link_cfg.requested_fc & PAUSE_AUTONEG) != 0;
1509 epause->rx_pause = (p->link_cfg.fc & PAUSE_RX) != 0;
1510 epause->tx_pause = (p->link_cfg.fc & PAUSE_TX) != 0;
1511}
1512
1513static int set_pauseparam(struct net_device *dev,
1514 struct ethtool_pauseparam *epause)
1515{
1516 struct port_info *p = netdev_priv(dev);
1517 struct link_config *lc = &p->link_cfg;
1518
1519 if (epause->autoneg == AUTONEG_DISABLE)
1520 lc->requested_fc = 0;
1521 else if (lc->supported & FW_PORT_CAP_ANEG)
1522 lc->requested_fc = PAUSE_AUTONEG;
1523 else
1524 return -EINVAL;
1525
1526 if (epause->rx_pause)
1527 lc->requested_fc |= PAUSE_RX;
1528 if (epause->tx_pause)
1529 lc->requested_fc |= PAUSE_TX;
1530 if (netif_running(dev))
1531 return t4_link_start(p->adapter, p->adapter->fn, p->tx_chan,
1532 lc);
1533 return 0;
1534}
1535
1536static void get_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1537{
1538 const struct port_info *pi = netdev_priv(dev);
1539 const struct sge *s = &pi->adapter->sge;
1540
1541 e->rx_max_pending = MAX_RX_BUFFERS;
1542 e->rx_mini_max_pending = MAX_RSPQ_ENTRIES;
1543 e->rx_jumbo_max_pending = 0;
1544 e->tx_max_pending = MAX_TXQ_ENTRIES;
1545
1546 e->rx_pending = s->ethrxq[pi->first_qset].fl.size - 8;
1547 e->rx_mini_pending = s->ethrxq[pi->first_qset].rspq.size;
1548 e->rx_jumbo_pending = 0;
1549 e->tx_pending = s->ethtxq[pi->first_qset].q.size;
1550}
1551
1552static int set_sge_param(struct net_device *dev, struct ethtool_ringparam *e)
1553{
1554 int i;
1555 const struct port_info *pi = netdev_priv(dev);
1556 struct adapter *adapter = pi->adapter;
1557 struct sge *s = &adapter->sge;
1558
1559 if (e->rx_pending > MAX_RX_BUFFERS || e->rx_jumbo_pending ||
1560 e->tx_pending > MAX_TXQ_ENTRIES ||
1561 e->rx_mini_pending > MAX_RSPQ_ENTRIES ||
1562 e->rx_mini_pending < MIN_RSPQ_ENTRIES ||
1563 e->rx_pending < MIN_FL_ENTRIES || e->tx_pending < MIN_TXQ_ENTRIES)
1564 return -EINVAL;
1565
1566 if (adapter->flags & FULL_INIT_DONE)
1567 return -EBUSY;
1568
1569 for (i = 0; i < pi->nqsets; ++i) {
1570 s->ethtxq[pi->first_qset + i].q.size = e->tx_pending;
1571 s->ethrxq[pi->first_qset + i].fl.size = e->rx_pending + 8;
1572 s->ethrxq[pi->first_qset + i].rspq.size = e->rx_mini_pending;
1573 }
1574 return 0;
1575}
1576
1577static int closest_timer(const struct sge *s, int time)
1578{
1579 int i, delta, match = 0, min_delta = INT_MAX;
1580
1581 for (i = 0; i < ARRAY_SIZE(s->timer_val); i++) {
1582 delta = time - s->timer_val[i];
1583 if (delta < 0)
1584 delta = -delta;
1585 if (delta < min_delta) {
1586 min_delta = delta;
1587 match = i;
1588 }
1589 }
1590 return match;
1591}
1592
1593static int closest_thres(const struct sge *s, int thres)
1594{
1595 int i, delta, match = 0, min_delta = INT_MAX;
1596
1597 for (i = 0; i < ARRAY_SIZE(s->counter_val); i++) {
1598 delta = thres - s->counter_val[i];
1599 if (delta < 0)
1600 delta = -delta;
1601 if (delta < min_delta) {
1602 min_delta = delta;
1603 match = i;
1604 }
1605 }
1606 return match;
1607}
1608
1609/*
1610 * Return a queue's interrupt hold-off time in us. 0 means no timer.
1611 */
1612static unsigned int qtimer_val(const struct adapter *adap,
1613 const struct sge_rspq *q)
1614{
1615 unsigned int idx = q->intr_params >> 1;
1616
1617 return idx < SGE_NTIMERS ? adap->sge.timer_val[idx] : 0;
1618}
1619
1620/**
1621 * set_rxq_intr_params - set a queue's interrupt holdoff parameters
1622 * @adap: the adapter
1623 * @q: the Rx queue
1624 * @us: the hold-off time in us, or 0 to disable timer
1625 * @cnt: the hold-off packet count, or 0 to disable counter
1626 *
1627 * Sets an Rx queue's interrupt hold-off time and packet count. At least
1628 * one of the two needs to be enabled for the queue to generate interrupts.
1629 */
1630static int set_rxq_intr_params(struct adapter *adap, struct sge_rspq *q,
1631 unsigned int us, unsigned int cnt)
1632{
1633 if ((us | cnt) == 0)
1634 cnt = 1;
1635
1636 if (cnt) {
1637 int err;
1638 u32 v, new_idx;
1639
1640 new_idx = closest_thres(&adap->sge, cnt);
1641 if (q->desc && q->pktcnt_idx != new_idx) {
1642 /* the queue has already been created, update it */
1643 v = FW_PARAMS_MNEM(FW_PARAMS_MNEM_DMAQ) |
1644 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DMAQ_IQ_INTCNTTHRESH) |
1645 FW_PARAMS_PARAM_YZ(q->cntxt_id);
1646 err = t4_set_params(adap, adap->fn, adap->fn, 0, 1, &v,
1647 &new_idx);
1648 if (err)
1649 return err;
1650 }
1651 q->pktcnt_idx = new_idx;
1652 }
1653
1654 us = us == 0 ? 6 : closest_timer(&adap->sge, us);
1655 q->intr_params = QINTR_TIMER_IDX(us) | (cnt > 0 ? QINTR_CNT_EN : 0);
1656 return 0;
1657}
1658
1659static int set_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1660{
1661 const struct port_info *pi = netdev_priv(dev);
1662 struct adapter *adap = pi->adapter;
1663
1664 return set_rxq_intr_params(adap, &adap->sge.ethrxq[pi->first_qset].rspq,
1665 c->rx_coalesce_usecs, c->rx_max_coalesced_frames);
1666}
1667
1668static int get_coalesce(struct net_device *dev, struct ethtool_coalesce *c)
1669{
1670 const struct port_info *pi = netdev_priv(dev);
1671 const struct adapter *adap = pi->adapter;
1672 const struct sge_rspq *rq = &adap->sge.ethrxq[pi->first_qset].rspq;
1673
1674 c->rx_coalesce_usecs = qtimer_val(adap, rq);
1675 c->rx_max_coalesced_frames = (rq->intr_params & QINTR_CNT_EN) ?
1676 adap->sge.counter_val[rq->pktcnt_idx] : 0;
1677 return 0;
1678}
1679
1680/**
1681 * eeprom_ptov - translate a physical EEPROM address to virtual
1682 * @phys_addr: the physical EEPROM address
1683 * @fn: the PCI function number
1684 * @sz: size of function-specific area
1685 *
1686 * Translate a physical EEPROM address to virtual. The first 1K is
1687 * accessed through virtual addresses starting at 31K, the rest is
1688 * accessed through virtual addresses starting at 0.
1689 *
1690 * The mapping is as follows:
1691 * [0..1K) -> [31K..32K)
1692 * [1K..1K+A) -> [31K-A..31K)
1693 * [1K+A..ES) -> [0..ES-A-1K)
1694 *
1695 * where A = @fn * @sz, and ES = EEPROM size.
1696 */
1697static int eeprom_ptov(unsigned int phys_addr, unsigned int fn, unsigned int sz)
1698{
1699 fn *= sz;
1700 if (phys_addr < 1024)
1701 return phys_addr + (31 << 10);
1702 if (phys_addr < 1024 + fn)
1703 return 31744 - fn + phys_addr - 1024;
1704 if (phys_addr < EEPROMSIZE)
1705 return phys_addr - 1024 - fn;
1706 return -EINVAL;
1707}
1708
1709/*
1710 * The next two routines implement eeprom read/write from physical addresses.
1711 */
1712static int eeprom_rd_phys(struct adapter *adap, unsigned int phys_addr, u32 *v)
1713{
1714 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1715
1716 if (vaddr >= 0)
1717 vaddr = pci_read_vpd(adap->pdev, vaddr, sizeof(u32), v);
1718 return vaddr < 0 ? vaddr : 0;
1719}
1720
1721static int eeprom_wr_phys(struct adapter *adap, unsigned int phys_addr, u32 v)
1722{
1723 int vaddr = eeprom_ptov(phys_addr, adap->fn, EEPROMPFSIZE);
1724
1725 if (vaddr >= 0)
1726 vaddr = pci_write_vpd(adap->pdev, vaddr, sizeof(u32), &v);
1727 return vaddr < 0 ? vaddr : 0;
1728}
1729
1730#define EEPROM_MAGIC 0x38E2F10C
1731
1732static int get_eeprom(struct net_device *dev, struct ethtool_eeprom *e,
1733 u8 *data)
1734{
1735 int i, err = 0;
1736 struct adapter *adapter = netdev2adap(dev);
1737
1738 u8 *buf = kmalloc(EEPROMSIZE, GFP_KERNEL);
1739 if (!buf)
1740 return -ENOMEM;
1741
1742 e->magic = EEPROM_MAGIC;
1743 for (i = e->offset & ~3; !err && i < e->offset + e->len; i += 4)
1744 err = eeprom_rd_phys(adapter, i, (u32 *)&buf[i]);
1745
1746 if (!err)
1747 memcpy(data, buf + e->offset, e->len);
1748 kfree(buf);
1749 return err;
1750}
1751
1752static int set_eeprom(struct net_device *dev, struct ethtool_eeprom *eeprom,
1753 u8 *data)
1754{
1755 u8 *buf;
1756 int err = 0;
1757 u32 aligned_offset, aligned_len, *p;
1758 struct adapter *adapter = netdev2adap(dev);
1759
1760 if (eeprom->magic != EEPROM_MAGIC)
1761 return -EINVAL;
1762
1763 aligned_offset = eeprom->offset & ~3;
1764 aligned_len = (eeprom->len + (eeprom->offset & 3) + 3) & ~3;
1765
1766 if (adapter->fn > 0) {
1767 u32 start = 1024 + adapter->fn * EEPROMPFSIZE;
1768
1769 if (aligned_offset < start ||
1770 aligned_offset + aligned_len > start + EEPROMPFSIZE)
1771 return -EPERM;
1772 }
1773
1774 if (aligned_offset != eeprom->offset || aligned_len != eeprom->len) {
1775 /*
1776 * RMW possibly needed for first or last words.
1777 */
1778 buf = kmalloc(aligned_len, GFP_KERNEL);
1779 if (!buf)
1780 return -ENOMEM;
1781 err = eeprom_rd_phys(adapter, aligned_offset, (u32 *)buf);
1782 if (!err && aligned_len > 4)
1783 err = eeprom_rd_phys(adapter,
1784 aligned_offset + aligned_len - 4,
1785 (u32 *)&buf[aligned_len - 4]);
1786 if (err)
1787 goto out;
1788 memcpy(buf + (eeprom->offset & 3), data, eeprom->len);
1789 } else
1790 buf = data;
1791
1792 err = t4_seeprom_wp(adapter, false);
1793 if (err)
1794 goto out;
1795
1796 for (p = (u32 *)buf; !err && aligned_len; aligned_len -= 4, p++) {
1797 err = eeprom_wr_phys(adapter, aligned_offset, *p);
1798 aligned_offset += 4;
1799 }
1800
1801 if (!err)
1802 err = t4_seeprom_wp(adapter, true);
1803out:
1804 if (buf != data)
1805 kfree(buf);
1806 return err;
1807}
1808
1809static int set_flash(struct net_device *netdev, struct ethtool_flash *ef)
1810{
1811 int ret;
1812 const struct firmware *fw;
1813 struct adapter *adap = netdev2adap(netdev);
1814
1815 ef->data[sizeof(ef->data) - 1] = '\0';
1816 ret = request_firmware(&fw, ef->data, adap->pdev_dev);
1817 if (ret < 0)
1818 return ret;
1819
1820 ret = t4_load_fw(adap, fw->data, fw->size);
1821 release_firmware(fw);
1822 if (!ret)
1823 dev_info(adap->pdev_dev, "loaded firmware %s\n", ef->data);
1824 return ret;
1825}
1826
1827#define WOL_SUPPORTED (WAKE_BCAST | WAKE_MAGIC)
1828#define BCAST_CRC 0xa0ccc1a6
1829
1830static void get_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1831{
1832 wol->supported = WAKE_BCAST | WAKE_MAGIC;
1833 wol->wolopts = netdev2adap(dev)->wol;
1834 memset(&wol->sopass, 0, sizeof(wol->sopass));
1835}
1836
1837static int set_wol(struct net_device *dev, struct ethtool_wolinfo *wol)
1838{
1839 int err = 0;
1840 struct port_info *pi = netdev_priv(dev);
1841
1842 if (wol->wolopts & ~WOL_SUPPORTED)
1843 return -EINVAL;
1844 t4_wol_magic_enable(pi->adapter, pi->tx_chan,
1845 (wol->wolopts & WAKE_MAGIC) ? dev->dev_addr : NULL);
1846 if (wol->wolopts & WAKE_BCAST) {
1847 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0xfe, ~0ULL,
1848 ~0ULL, 0, false);
1849 if (!err)
1850 err = t4_wol_pat_enable(pi->adapter, pi->tx_chan, 1,
1851 ~6ULL, ~0ULL, BCAST_CRC, true);
1852 } else
1853 t4_wol_pat_enable(pi->adapter, pi->tx_chan, 0, 0, 0, 0, false);
1854 return err;
1855}
1856
1857static int cxgb_set_features(struct net_device *dev, u32 features)
1858{
1859 const struct port_info *pi = netdev_priv(dev);
1860 u32 changed = dev->features ^ features;
1861 int err;
1862
1863 if (!(changed & NETIF_F_HW_VLAN_RX))
1864 return 0;
1865
1866 err = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, -1,
1867 -1, -1, -1,
1868 !!(features & NETIF_F_HW_VLAN_RX), true);
1869 if (unlikely(err))
1870 dev->features = features ^ NETIF_F_HW_VLAN_RX;
1871 return err;
1872}
1873
1874static int get_rss_table(struct net_device *dev, struct ethtool_rxfh_indir *p)
1875{
1876 const struct port_info *pi = netdev_priv(dev);
1877 unsigned int n = min_t(unsigned int, p->size, pi->rss_size);
1878
1879 p->size = pi->rss_size;
1880 while (n--)
1881 p->ring_index[n] = pi->rss[n];
1882 return 0;
1883}
1884
1885static int set_rss_table(struct net_device *dev,
1886 const struct ethtool_rxfh_indir *p)
1887{
1888 unsigned int i;
1889 struct port_info *pi = netdev_priv(dev);
1890
1891 if (p->size != pi->rss_size)
1892 return -EINVAL;
1893 for (i = 0; i < p->size; i++)
1894 if (p->ring_index[i] >= pi->nqsets)
1895 return -EINVAL;
1896 for (i = 0; i < p->size; i++)
1897 pi->rss[i] = p->ring_index[i];
1898 if (pi->adapter->flags & FULL_INIT_DONE)
1899 return write_rss(pi, pi->rss);
1900 return 0;
1901}
1902
1903static int get_rxnfc(struct net_device *dev, struct ethtool_rxnfc *info,
1904 void *rules)
1905{
1906 const struct port_info *pi = netdev_priv(dev);
1907
1908 switch (info->cmd) {
1909 case ETHTOOL_GRXFH: {
1910 unsigned int v = pi->rss_mode;
1911
1912 info->data = 0;
1913 switch (info->flow_type) {
1914 case TCP_V4_FLOW:
1915 if (v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN)
1916 info->data = RXH_IP_SRC | RXH_IP_DST |
1917 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1918 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1919 info->data = RXH_IP_SRC | RXH_IP_DST;
1920 break;
1921 case UDP_V4_FLOW:
1922 if ((v & FW_RSS_VI_CONFIG_CMD_IP4FOURTUPEN) &&
1923 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1924 info->data = RXH_IP_SRC | RXH_IP_DST |
1925 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1926 else if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1927 info->data = RXH_IP_SRC | RXH_IP_DST;
1928 break;
1929 case SCTP_V4_FLOW:
1930 case AH_ESP_V4_FLOW:
1931 case IPV4_FLOW:
1932 if (v & FW_RSS_VI_CONFIG_CMD_IP4TWOTUPEN)
1933 info->data = RXH_IP_SRC | RXH_IP_DST;
1934 break;
1935 case TCP_V6_FLOW:
1936 if (v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN)
1937 info->data = RXH_IP_SRC | RXH_IP_DST |
1938 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1939 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1940 info->data = RXH_IP_SRC | RXH_IP_DST;
1941 break;
1942 case UDP_V6_FLOW:
1943 if ((v & FW_RSS_VI_CONFIG_CMD_IP6FOURTUPEN) &&
1944 (v & FW_RSS_VI_CONFIG_CMD_UDPEN))
1945 info->data = RXH_IP_SRC | RXH_IP_DST |
1946 RXH_L4_B_0_1 | RXH_L4_B_2_3;
1947 else if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1948 info->data = RXH_IP_SRC | RXH_IP_DST;
1949 break;
1950 case SCTP_V6_FLOW:
1951 case AH_ESP_V6_FLOW:
1952 case IPV6_FLOW:
1953 if (v & FW_RSS_VI_CONFIG_CMD_IP6TWOTUPEN)
1954 info->data = RXH_IP_SRC | RXH_IP_DST;
1955 break;
1956 }
1957 return 0;
1958 }
1959 case ETHTOOL_GRXRINGS:
1960 info->data = pi->nqsets;
1961 return 0;
1962 }
1963 return -EOPNOTSUPP;
1964}
1965
1966static struct ethtool_ops cxgb_ethtool_ops = {
1967 .get_settings = get_settings,
1968 .set_settings = set_settings,
1969 .get_drvinfo = get_drvinfo,
1970 .get_msglevel = get_msglevel,
1971 .set_msglevel = set_msglevel,
1972 .get_ringparam = get_sge_param,
1973 .set_ringparam = set_sge_param,
1974 .get_coalesce = get_coalesce,
1975 .set_coalesce = set_coalesce,
1976 .get_eeprom_len = get_eeprom_len,
1977 .get_eeprom = get_eeprom,
1978 .set_eeprom = set_eeprom,
1979 .get_pauseparam = get_pauseparam,
1980 .set_pauseparam = set_pauseparam,
1981 .get_link = ethtool_op_get_link,
1982 .get_strings = get_strings,
1983 .set_phys_id = identify_port,
1984 .nway_reset = restart_autoneg,
1985 .get_sset_count = get_sset_count,
1986 .get_ethtool_stats = get_stats,
1987 .get_regs_len = get_regs_len,
1988 .get_regs = get_regs,
1989 .get_wol = get_wol,
1990 .set_wol = set_wol,
1991 .get_rxnfc = get_rxnfc,
1992 .get_rxfh_indir = get_rss_table,
1993 .set_rxfh_indir = set_rss_table,
1994 .flash_device = set_flash,
1995};
1996
1997/*
1998 * debugfs support
1999 */
2000
2001static int mem_open(struct inode *inode, struct file *file)
2002{
2003 file->private_data = inode->i_private;
2004 return 0;
2005}
2006
2007static ssize_t mem_read(struct file *file, char __user *buf, size_t count,
2008 loff_t *ppos)
2009{
2010 loff_t pos = *ppos;
2011 loff_t avail = file->f_path.dentry->d_inode->i_size;
2012 unsigned int mem = (uintptr_t)file->private_data & 3;
2013 struct adapter *adap = file->private_data - mem;
2014
2015 if (pos < 0)
2016 return -EINVAL;
2017 if (pos >= avail)
2018 return 0;
2019 if (count > avail - pos)
2020 count = avail - pos;
2021
2022 while (count) {
2023 size_t len;
2024 int ret, ofst;
2025 __be32 data[16];
2026
2027 if (mem == MEM_MC)
2028 ret = t4_mc_read(adap, pos, data, NULL);
2029 else
2030 ret = t4_edc_read(adap, mem, pos, data, NULL);
2031 if (ret)
2032 return ret;
2033
2034 ofst = pos % sizeof(data);
2035 len = min(count, sizeof(data) - ofst);
2036 if (copy_to_user(buf, (u8 *)data + ofst, len))
2037 return -EFAULT;
2038
2039 buf += len;
2040 pos += len;
2041 count -= len;
2042 }
2043 count = pos - *ppos;
2044 *ppos = pos;
2045 return count;
2046}
2047
2048static const struct file_operations mem_debugfs_fops = {
2049 .owner = THIS_MODULE,
2050 .open = mem_open,
2051 .read = mem_read,
2052 .llseek = default_llseek,
2053};
2054
2055static void __devinit add_debugfs_mem(struct adapter *adap, const char *name,
2056 unsigned int idx, unsigned int size_mb)
2057{
2058 struct dentry *de;
2059
2060 de = debugfs_create_file(name, S_IRUSR, adap->debugfs_root,
2061 (void *)adap + idx, &mem_debugfs_fops);
2062 if (de && de->d_inode)
2063 de->d_inode->i_size = size_mb << 20;
2064}
2065
2066static int __devinit setup_debugfs(struct adapter *adap)
2067{
2068 int i;
2069
2070 if (IS_ERR_OR_NULL(adap->debugfs_root))
2071 return -1;
2072
2073 i = t4_read_reg(adap, MA_TARGET_MEM_ENABLE);
2074 if (i & EDRAM0_ENABLE)
2075 add_debugfs_mem(adap, "edc0", MEM_EDC0, 5);
2076 if (i & EDRAM1_ENABLE)
2077 add_debugfs_mem(adap, "edc1", MEM_EDC1, 5);
2078 if (i & EXT_MEM_ENABLE)
2079 add_debugfs_mem(adap, "mc", MEM_MC,
2080 EXT_MEM_SIZE_GET(t4_read_reg(adap, MA_EXT_MEMORY_BAR)));
2081 if (adap->l2t)
2082 debugfs_create_file("l2t", S_IRUSR, adap->debugfs_root, adap,
2083 &t4_l2t_fops);
2084 return 0;
2085}
2086
2087/*
2088 * upper-layer driver support
2089 */
2090
2091/*
2092 * Allocate an active-open TID and set it to the supplied value.
2093 */
2094int cxgb4_alloc_atid(struct tid_info *t, void *data)
2095{
2096 int atid = -1;
2097
2098 spin_lock_bh(&t->atid_lock);
2099 if (t->afree) {
2100 union aopen_entry *p = t->afree;
2101
2102 atid = p - t->atid_tab;
2103 t->afree = p->next;
2104 p->data = data;
2105 t->atids_in_use++;
2106 }
2107 spin_unlock_bh(&t->atid_lock);
2108 return atid;
2109}
2110EXPORT_SYMBOL(cxgb4_alloc_atid);
2111
2112/*
2113 * Release an active-open TID.
2114 */
2115void cxgb4_free_atid(struct tid_info *t, unsigned int atid)
2116{
2117 union aopen_entry *p = &t->atid_tab[atid];
2118
2119 spin_lock_bh(&t->atid_lock);
2120 p->next = t->afree;
2121 t->afree = p;
2122 t->atids_in_use--;
2123 spin_unlock_bh(&t->atid_lock);
2124}
2125EXPORT_SYMBOL(cxgb4_free_atid);
2126
2127/*
2128 * Allocate a server TID and set it to the supplied value.
2129 */
2130int cxgb4_alloc_stid(struct tid_info *t, int family, void *data)
2131{
2132 int stid;
2133
2134 spin_lock_bh(&t->stid_lock);
2135 if (family == PF_INET) {
2136 stid = find_first_zero_bit(t->stid_bmap, t->nstids);
2137 if (stid < t->nstids)
2138 __set_bit(stid, t->stid_bmap);
2139 else
2140 stid = -1;
2141 } else {
2142 stid = bitmap_find_free_region(t->stid_bmap, t->nstids, 2);
2143 if (stid < 0)
2144 stid = -1;
2145 }
2146 if (stid >= 0) {
2147 t->stid_tab[stid].data = data;
2148 stid += t->stid_base;
2149 t->stids_in_use++;
2150 }
2151 spin_unlock_bh(&t->stid_lock);
2152 return stid;
2153}
2154EXPORT_SYMBOL(cxgb4_alloc_stid);
2155
2156/*
2157 * Release a server TID.
2158 */
2159void cxgb4_free_stid(struct tid_info *t, unsigned int stid, int family)
2160{
2161 stid -= t->stid_base;
2162 spin_lock_bh(&t->stid_lock);
2163 if (family == PF_INET)
2164 __clear_bit(stid, t->stid_bmap);
2165 else
2166 bitmap_release_region(t->stid_bmap, stid, 2);
2167 t->stid_tab[stid].data = NULL;
2168 t->stids_in_use--;
2169 spin_unlock_bh(&t->stid_lock);
2170}
2171EXPORT_SYMBOL(cxgb4_free_stid);
2172
2173/*
2174 * Populate a TID_RELEASE WR. Caller must properly size the skb.
2175 */
2176static void mk_tid_release(struct sk_buff *skb, unsigned int chan,
2177 unsigned int tid)
2178{
2179 struct cpl_tid_release *req;
2180
2181 set_wr_txq(skb, CPL_PRIORITY_SETUP, chan);
2182 req = (struct cpl_tid_release *)__skb_put(skb, sizeof(*req));
2183 INIT_TP_WR(req, tid);
2184 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_TID_RELEASE, tid));
2185}
2186
2187/*
2188 * Queue a TID release request and if necessary schedule a work queue to
2189 * process it.
2190 */
2191static void cxgb4_queue_tid_release(struct tid_info *t, unsigned int chan,
2192 unsigned int tid)
2193{
2194 void **p = &t->tid_tab[tid];
2195 struct adapter *adap = container_of(t, struct adapter, tids);
2196
2197 spin_lock_bh(&adap->tid_release_lock);
2198 *p = adap->tid_release_head;
2199 /* Low 2 bits encode the Tx channel number */
2200 adap->tid_release_head = (void **)((uintptr_t)p | chan);
2201 if (!adap->tid_release_task_busy) {
2202 adap->tid_release_task_busy = true;
2203 schedule_work(&adap->tid_release_task);
2204 }
2205 spin_unlock_bh(&adap->tid_release_lock);
2206}
2207
2208/*
2209 * Process the list of pending TID release requests.
2210 */
2211static void process_tid_release_list(struct work_struct *work)
2212{
2213 struct sk_buff *skb;
2214 struct adapter *adap;
2215
2216 adap = container_of(work, struct adapter, tid_release_task);
2217
2218 spin_lock_bh(&adap->tid_release_lock);
2219 while (adap->tid_release_head) {
2220 void **p = adap->tid_release_head;
2221 unsigned int chan = (uintptr_t)p & 3;
2222 p = (void *)p - chan;
2223
2224 adap->tid_release_head = *p;
2225 *p = NULL;
2226 spin_unlock_bh(&adap->tid_release_lock);
2227
2228 while (!(skb = alloc_skb(sizeof(struct cpl_tid_release),
2229 GFP_KERNEL)))
2230 schedule_timeout_uninterruptible(1);
2231
2232 mk_tid_release(skb, chan, p - adap->tids.tid_tab);
2233 t4_ofld_send(adap, skb);
2234 spin_lock_bh(&adap->tid_release_lock);
2235 }
2236 adap->tid_release_task_busy = false;
2237 spin_unlock_bh(&adap->tid_release_lock);
2238}
2239
2240/*
2241 * Release a TID and inform HW. If we are unable to allocate the release
2242 * message we defer to a work queue.
2243 */
2244void cxgb4_remove_tid(struct tid_info *t, unsigned int chan, unsigned int tid)
2245{
2246 void *old;
2247 struct sk_buff *skb;
2248 struct adapter *adap = container_of(t, struct adapter, tids);
2249
2250 old = t->tid_tab[tid];
2251 skb = alloc_skb(sizeof(struct cpl_tid_release), GFP_ATOMIC);
2252 if (likely(skb)) {
2253 t->tid_tab[tid] = NULL;
2254 mk_tid_release(skb, chan, tid);
2255 t4_ofld_send(adap, skb);
2256 } else
2257 cxgb4_queue_tid_release(t, chan, tid);
2258 if (old)
2259 atomic_dec(&t->tids_in_use);
2260}
2261EXPORT_SYMBOL(cxgb4_remove_tid);
2262
2263/*
2264 * Allocate and initialize the TID tables. Returns 0 on success.
2265 */
2266static int tid_init(struct tid_info *t)
2267{
2268 size_t size;
2269 unsigned int natids = t->natids;
2270
2271 size = t->ntids * sizeof(*t->tid_tab) + natids * sizeof(*t->atid_tab) +
2272 t->nstids * sizeof(*t->stid_tab) +
2273 BITS_TO_LONGS(t->nstids) * sizeof(long);
2274 t->tid_tab = t4_alloc_mem(size);
2275 if (!t->tid_tab)
2276 return -ENOMEM;
2277
2278 t->atid_tab = (union aopen_entry *)&t->tid_tab[t->ntids];
2279 t->stid_tab = (struct serv_entry *)&t->atid_tab[natids];
2280 t->stid_bmap = (unsigned long *)&t->stid_tab[t->nstids];
2281 spin_lock_init(&t->stid_lock);
2282 spin_lock_init(&t->atid_lock);
2283
2284 t->stids_in_use = 0;
2285 t->afree = NULL;
2286 t->atids_in_use = 0;
2287 atomic_set(&t->tids_in_use, 0);
2288
2289 /* Setup the free list for atid_tab and clear the stid bitmap. */
2290 if (natids) {
2291 while (--natids)
2292 t->atid_tab[natids - 1].next = &t->atid_tab[natids];
2293 t->afree = t->atid_tab;
2294 }
2295 bitmap_zero(t->stid_bmap, t->nstids);
2296 return 0;
2297}
2298
2299/**
2300 * cxgb4_create_server - create an IP server
2301 * @dev: the device
2302 * @stid: the server TID
2303 * @sip: local IP address to bind server to
2304 * @sport: the server's TCP port
2305 * @queue: queue to direct messages from this server to
2306 *
2307 * Create an IP server for the given port and address.
2308 * Returns <0 on error and one of the %NET_XMIT_* values on success.
2309 */
2310int cxgb4_create_server(const struct net_device *dev, unsigned int stid,
2311 __be32 sip, __be16 sport, unsigned int queue)
2312{
2313 unsigned int chan;
2314 struct sk_buff *skb;
2315 struct adapter *adap;
2316 struct cpl_pass_open_req *req;
2317
2318 skb = alloc_skb(sizeof(*req), GFP_KERNEL);
2319 if (!skb)
2320 return -ENOMEM;
2321
2322 adap = netdev2adap(dev);
2323 req = (struct cpl_pass_open_req *)__skb_put(skb, sizeof(*req));
2324 INIT_TP_WR(req, 0);
2325 OPCODE_TID(req) = htonl(MK_OPCODE_TID(CPL_PASS_OPEN_REQ, stid));
2326 req->local_port = sport;
2327 req->peer_port = htons(0);
2328 req->local_ip = sip;
2329 req->peer_ip = htonl(0);
2330 chan = rxq_to_chan(&adap->sge, queue);
2331 req->opt0 = cpu_to_be64(TX_CHAN(chan));
2332 req->opt1 = cpu_to_be64(CONN_POLICY_ASK |
2333 SYN_RSS_ENABLE | SYN_RSS_QUEUE(queue));
2334 return t4_mgmt_tx(adap, skb);
2335}
2336EXPORT_SYMBOL(cxgb4_create_server);
2337
2338/**
2339 * cxgb4_best_mtu - find the entry in the MTU table closest to an MTU
2340 * @mtus: the HW MTU table
2341 * @mtu: the target MTU
2342 * @idx: index of selected entry in the MTU table
2343 *
2344 * Returns the index and the value in the HW MTU table that is closest to
2345 * but does not exceed @mtu, unless @mtu is smaller than any value in the
2346 * table, in which case that smallest available value is selected.
2347 */
2348unsigned int cxgb4_best_mtu(const unsigned short *mtus, unsigned short mtu,
2349 unsigned int *idx)
2350{
2351 unsigned int i = 0;
2352
2353 while (i < NMTUS - 1 && mtus[i + 1] <= mtu)
2354 ++i;
2355 if (idx)
2356 *idx = i;
2357 return mtus[i];
2358}
2359EXPORT_SYMBOL(cxgb4_best_mtu);
2360
2361/**
2362 * cxgb4_port_chan - get the HW channel of a port
2363 * @dev: the net device for the port
2364 *
2365 * Return the HW Tx channel of the given port.
2366 */
2367unsigned int cxgb4_port_chan(const struct net_device *dev)
2368{
2369 return netdev2pinfo(dev)->tx_chan;
2370}
2371EXPORT_SYMBOL(cxgb4_port_chan);
2372
2373/**
2374 * cxgb4_port_viid - get the VI id of a port
2375 * @dev: the net device for the port
2376 *
2377 * Return the VI id of the given port.
2378 */
2379unsigned int cxgb4_port_viid(const struct net_device *dev)
2380{
2381 return netdev2pinfo(dev)->viid;
2382}
2383EXPORT_SYMBOL(cxgb4_port_viid);
2384
2385/**
2386 * cxgb4_port_idx - get the index of a port
2387 * @dev: the net device for the port
2388 *
2389 * Return the index of the given port.
2390 */
2391unsigned int cxgb4_port_idx(const struct net_device *dev)
2392{
2393 return netdev2pinfo(dev)->port_id;
2394}
2395EXPORT_SYMBOL(cxgb4_port_idx);
2396
2397void cxgb4_get_tcp_stats(struct pci_dev *pdev, struct tp_tcp_stats *v4,
2398 struct tp_tcp_stats *v6)
2399{
2400 struct adapter *adap = pci_get_drvdata(pdev);
2401
2402 spin_lock(&adap->stats_lock);
2403 t4_tp_get_tcp_stats(adap, v4, v6);
2404 spin_unlock(&adap->stats_lock);
2405}
2406EXPORT_SYMBOL(cxgb4_get_tcp_stats);
2407
2408void cxgb4_iscsi_init(struct net_device *dev, unsigned int tag_mask,
2409 const unsigned int *pgsz_order)
2410{
2411 struct adapter *adap = netdev2adap(dev);
2412
2413 t4_write_reg(adap, ULP_RX_ISCSI_TAGMASK, tag_mask);
2414 t4_write_reg(adap, ULP_RX_ISCSI_PSZ, HPZ0(pgsz_order[0]) |
2415 HPZ1(pgsz_order[1]) | HPZ2(pgsz_order[2]) |
2416 HPZ3(pgsz_order[3]));
2417}
2418EXPORT_SYMBOL(cxgb4_iscsi_init);
2419
2420static struct pci_driver cxgb4_driver;
2421
2422static void check_neigh_update(struct neighbour *neigh)
2423{
2424 const struct device *parent;
2425 const struct net_device *netdev = neigh->dev;
2426
2427 if (netdev->priv_flags & IFF_802_1Q_VLAN)
2428 netdev = vlan_dev_real_dev(netdev);
2429 parent = netdev->dev.parent;
2430 if (parent && parent->driver == &cxgb4_driver.driver)
2431 t4_l2t_update(dev_get_drvdata(parent), neigh);
2432}
2433
2434static int netevent_cb(struct notifier_block *nb, unsigned long event,
2435 void *data)
2436{
2437 switch (event) {
2438 case NETEVENT_NEIGH_UPDATE:
2439 check_neigh_update(data);
2440 break;
2441 case NETEVENT_REDIRECT:
2442 default:
2443 break;
2444 }
2445 return 0;
2446}
2447
2448static bool netevent_registered;
2449static struct notifier_block cxgb4_netevent_nb = {
2450 .notifier_call = netevent_cb
2451};
2452
2453static void uld_attach(struct adapter *adap, unsigned int uld)
2454{
2455 void *handle;
2456 struct cxgb4_lld_info lli;
2457
2458 lli.pdev = adap->pdev;
2459 lli.l2t = adap->l2t;
2460 lli.tids = &adap->tids;
2461 lli.ports = adap->port;
2462 lli.vr = &adap->vres;
2463 lli.mtus = adap->params.mtus;
2464 if (uld == CXGB4_ULD_RDMA) {
2465 lli.rxq_ids = adap->sge.rdma_rxq;
2466 lli.nrxq = adap->sge.rdmaqs;
2467 } else if (uld == CXGB4_ULD_ISCSI) {
2468 lli.rxq_ids = adap->sge.ofld_rxq;
2469 lli.nrxq = adap->sge.ofldqsets;
2470 }
2471 lli.ntxq = adap->sge.ofldqsets;
2472 lli.nchan = adap->params.nports;
2473 lli.nports = adap->params.nports;
2474 lli.wr_cred = adap->params.ofldq_wr_cred;
2475 lli.adapter_type = adap->params.rev;
2476 lli.iscsi_iolen = MAXRXDATA_GET(t4_read_reg(adap, TP_PARA_REG2));
2477 lli.udb_density = 1 << QUEUESPERPAGEPF0_GET(
2478 t4_read_reg(adap, SGE_EGRESS_QUEUES_PER_PAGE_PF) >>
2479 (adap->fn * 4));
2480 lli.ucq_density = 1 << QUEUESPERPAGEPF0_GET(
2481 t4_read_reg(adap, SGE_INGRESS_QUEUES_PER_PAGE_PF) >>
2482 (adap->fn * 4));
2483 lli.gts_reg = adap->regs + MYPF_REG(SGE_PF_GTS);
2484 lli.db_reg = adap->regs + MYPF_REG(SGE_PF_KDOORBELL);
2485 lli.fw_vers = adap->params.fw_vers;
2486
2487 handle = ulds[uld].add(&lli);
2488 if (IS_ERR(handle)) {
2489 dev_warn(adap->pdev_dev,
2490 "could not attach to the %s driver, error %ld\n",
2491 uld_str[uld], PTR_ERR(handle));
2492 return;
2493 }
2494
2495 adap->uld_handle[uld] = handle;
2496
2497 if (!netevent_registered) {
2498 register_netevent_notifier(&cxgb4_netevent_nb);
2499 netevent_registered = true;
2500 }
2501
2502 if (adap->flags & FULL_INIT_DONE)
2503 ulds[uld].state_change(handle, CXGB4_STATE_UP);
2504}
2505
2506static void attach_ulds(struct adapter *adap)
2507{
2508 unsigned int i;
2509
2510 mutex_lock(&uld_mutex);
2511 list_add_tail(&adap->list_node, &adapter_list);
2512 for (i = 0; i < CXGB4_ULD_MAX; i++)
2513 if (ulds[i].add)
2514 uld_attach(adap, i);
2515 mutex_unlock(&uld_mutex);
2516}
2517
2518static void detach_ulds(struct adapter *adap)
2519{
2520 unsigned int i;
2521
2522 mutex_lock(&uld_mutex);
2523 list_del(&adap->list_node);
2524 for (i = 0; i < CXGB4_ULD_MAX; i++)
2525 if (adap->uld_handle[i]) {
2526 ulds[i].state_change(adap->uld_handle[i],
2527 CXGB4_STATE_DETACH);
2528 adap->uld_handle[i] = NULL;
2529 }
2530 if (netevent_registered && list_empty(&adapter_list)) {
2531 unregister_netevent_notifier(&cxgb4_netevent_nb);
2532 netevent_registered = false;
2533 }
2534 mutex_unlock(&uld_mutex);
2535}
2536
2537static void notify_ulds(struct adapter *adap, enum cxgb4_state new_state)
2538{
2539 unsigned int i;
2540
2541 mutex_lock(&uld_mutex);
2542 for (i = 0; i < CXGB4_ULD_MAX; i++)
2543 if (adap->uld_handle[i])
2544 ulds[i].state_change(adap->uld_handle[i], new_state);
2545 mutex_unlock(&uld_mutex);
2546}
2547
2548/**
2549 * cxgb4_register_uld - register an upper-layer driver
2550 * @type: the ULD type
2551 * @p: the ULD methods
2552 *
2553 * Registers an upper-layer driver with this driver and notifies the ULD
2554 * about any presently available devices that support its type. Returns
2555 * %-EBUSY if a ULD of the same type is already registered.
2556 */
2557int cxgb4_register_uld(enum cxgb4_uld type, const struct cxgb4_uld_info *p)
2558{
2559 int ret = 0;
2560 struct adapter *adap;
2561
2562 if (type >= CXGB4_ULD_MAX)
2563 return -EINVAL;
2564 mutex_lock(&uld_mutex);
2565 if (ulds[type].add) {
2566 ret = -EBUSY;
2567 goto out;
2568 }
2569 ulds[type] = *p;
2570 list_for_each_entry(adap, &adapter_list, list_node)
2571 uld_attach(adap, type);
2572out: mutex_unlock(&uld_mutex);
2573 return ret;
2574}
2575EXPORT_SYMBOL(cxgb4_register_uld);
2576
2577/**
2578 * cxgb4_unregister_uld - unregister an upper-layer driver
2579 * @type: the ULD type
2580 *
2581 * Unregisters an existing upper-layer driver.
2582 */
2583int cxgb4_unregister_uld(enum cxgb4_uld type)
2584{
2585 struct adapter *adap;
2586
2587 if (type >= CXGB4_ULD_MAX)
2588 return -EINVAL;
2589 mutex_lock(&uld_mutex);
2590 list_for_each_entry(adap, &adapter_list, list_node)
2591 adap->uld_handle[type] = NULL;
2592 ulds[type].add = NULL;
2593 mutex_unlock(&uld_mutex);
2594 return 0;
2595}
2596EXPORT_SYMBOL(cxgb4_unregister_uld);
2597
2598/**
2599 * cxgb_up - enable the adapter
2600 * @adap: adapter being enabled
2601 *
2602 * Called when the first port is enabled, this function performs the
2603 * actions necessary to make an adapter operational, such as completing
2604 * the initialization of HW modules, and enabling interrupts.
2605 *
2606 * Must be called with the rtnl lock held.
2607 */
2608static int cxgb_up(struct adapter *adap)
2609{
2610 int err;
2611
2612 err = setup_sge_queues(adap);
2613 if (err)
2614 goto out;
2615 err = setup_rss(adap);
2616 if (err)
2617 goto freeq;
2618
2619 if (adap->flags & USING_MSIX) {
2620 name_msix_vecs(adap);
2621 err = request_irq(adap->msix_info[0].vec, t4_nondata_intr, 0,
2622 adap->msix_info[0].desc, adap);
2623 if (err)
2624 goto irq_err;
2625
2626 err = request_msix_queue_irqs(adap);
2627 if (err) {
2628 free_irq(adap->msix_info[0].vec, adap);
2629 goto irq_err;
2630 }
2631 } else {
2632 err = request_irq(adap->pdev->irq, t4_intr_handler(adap),
2633 (adap->flags & USING_MSI) ? 0 : IRQF_SHARED,
2634 adap->port[0]->name, adap);
2635 if (err)
2636 goto irq_err;
2637 }
2638 enable_rx(adap);
2639 t4_sge_start(adap);
2640 t4_intr_enable(adap);
2641 adap->flags |= FULL_INIT_DONE;
2642 notify_ulds(adap, CXGB4_STATE_UP);
2643 out:
2644 return err;
2645 irq_err:
2646 dev_err(adap->pdev_dev, "request_irq failed, err %d\n", err);
2647 freeq:
2648 t4_free_sge_resources(adap);
2649 goto out;
2650}
2651
2652static void cxgb_down(struct adapter *adapter)
2653{
2654 t4_intr_disable(adapter);
2655 cancel_work_sync(&adapter->tid_release_task);
2656 adapter->tid_release_task_busy = false;
2657 adapter->tid_release_head = NULL;
2658
2659 if (adapter->flags & USING_MSIX) {
2660 free_msix_queue_irqs(adapter);
2661 free_irq(adapter->msix_info[0].vec, adapter);
2662 } else
2663 free_irq(adapter->pdev->irq, adapter);
2664 quiesce_rx(adapter);
2665 t4_sge_stop(adapter);
2666 t4_free_sge_resources(adapter);
2667 adapter->flags &= ~FULL_INIT_DONE;
2668}
2669
2670/*
2671 * net_device operations
2672 */
2673static int cxgb_open(struct net_device *dev)
2674{
2675 int err;
2676 struct port_info *pi = netdev_priv(dev);
2677 struct adapter *adapter = pi->adapter;
2678
2679 netif_carrier_off(dev);
2680
2681 if (!(adapter->flags & FULL_INIT_DONE)) {
2682 err = cxgb_up(adapter);
2683 if (err < 0)
2684 return err;
2685 }
2686
2687 err = link_start(dev);
2688 if (!err)
2689 netif_tx_start_all_queues(dev);
2690 return err;
2691}
2692
2693static int cxgb_close(struct net_device *dev)
2694{
2695 struct port_info *pi = netdev_priv(dev);
2696 struct adapter *adapter = pi->adapter;
2697
2698 netif_tx_stop_all_queues(dev);
2699 netif_carrier_off(dev);
2700 return t4_enable_vi(adapter, adapter->fn, pi->viid, false, false);
2701}
2702
2703static struct rtnl_link_stats64 *cxgb_get_stats(struct net_device *dev,
2704 struct rtnl_link_stats64 *ns)
2705{
2706 struct port_stats stats;
2707 struct port_info *p = netdev_priv(dev);
2708 struct adapter *adapter = p->adapter;
2709
2710 spin_lock(&adapter->stats_lock);
2711 t4_get_port_stats(adapter, p->tx_chan, &stats);
2712 spin_unlock(&adapter->stats_lock);
2713
2714 ns->tx_bytes = stats.tx_octets;
2715 ns->tx_packets = stats.tx_frames;
2716 ns->rx_bytes = stats.rx_octets;
2717 ns->rx_packets = stats.rx_frames;
2718 ns->multicast = stats.rx_mcast_frames;
2719
2720 /* detailed rx_errors */
2721 ns->rx_length_errors = stats.rx_jabber + stats.rx_too_long +
2722 stats.rx_runt;
2723 ns->rx_over_errors = 0;
2724 ns->rx_crc_errors = stats.rx_fcs_err;
2725 ns->rx_frame_errors = stats.rx_symbol_err;
2726 ns->rx_fifo_errors = stats.rx_ovflow0 + stats.rx_ovflow1 +
2727 stats.rx_ovflow2 + stats.rx_ovflow3 +
2728 stats.rx_trunc0 + stats.rx_trunc1 +
2729 stats.rx_trunc2 + stats.rx_trunc3;
2730 ns->rx_missed_errors = 0;
2731
2732 /* detailed tx_errors */
2733 ns->tx_aborted_errors = 0;
2734 ns->tx_carrier_errors = 0;
2735 ns->tx_fifo_errors = 0;
2736 ns->tx_heartbeat_errors = 0;
2737 ns->tx_window_errors = 0;
2738
2739 ns->tx_errors = stats.tx_error_frames;
2740 ns->rx_errors = stats.rx_symbol_err + stats.rx_fcs_err +
2741 ns->rx_length_errors + stats.rx_len_err + ns->rx_fifo_errors;
2742 return ns;
2743}
2744
2745static int cxgb_ioctl(struct net_device *dev, struct ifreq *req, int cmd)
2746{
2747 unsigned int mbox;
2748 int ret = 0, prtad, devad;
2749 struct port_info *pi = netdev_priv(dev);
2750 struct mii_ioctl_data *data = (struct mii_ioctl_data *)&req->ifr_data;
2751
2752 switch (cmd) {
2753 case SIOCGMIIPHY:
2754 if (pi->mdio_addr < 0)
2755 return -EOPNOTSUPP;
2756 data->phy_id = pi->mdio_addr;
2757 break;
2758 case SIOCGMIIREG:
2759 case SIOCSMIIREG:
2760 if (mdio_phy_id_is_c45(data->phy_id)) {
2761 prtad = mdio_phy_id_prtad(data->phy_id);
2762 devad = mdio_phy_id_devad(data->phy_id);
2763 } else if (data->phy_id < 32) {
2764 prtad = data->phy_id;
2765 devad = 0;
2766 data->reg_num &= 0x1f;
2767 } else
2768 return -EINVAL;
2769
2770 mbox = pi->adapter->fn;
2771 if (cmd == SIOCGMIIREG)
2772 ret = t4_mdio_rd(pi->adapter, mbox, prtad, devad,
2773 data->reg_num, &data->val_out);
2774 else
2775 ret = t4_mdio_wr(pi->adapter, mbox, prtad, devad,
2776 data->reg_num, data->val_in);
2777 break;
2778 default:
2779 return -EOPNOTSUPP;
2780 }
2781 return ret;
2782}
2783
2784static void cxgb_set_rxmode(struct net_device *dev)
2785{
2786 /* unfortunately we can't return errors to the stack */
2787 set_rxmode(dev, -1, false);
2788}
2789
2790static int cxgb_change_mtu(struct net_device *dev, int new_mtu)
2791{
2792 int ret;
2793 struct port_info *pi = netdev_priv(dev);
2794
2795 if (new_mtu < 81 || new_mtu > MAX_MTU) /* accommodate SACK */
2796 return -EINVAL;
2797 ret = t4_set_rxmode(pi->adapter, pi->adapter->fn, pi->viid, new_mtu, -1,
2798 -1, -1, -1, true);
2799 if (!ret)
2800 dev->mtu = new_mtu;
2801 return ret;
2802}
2803
2804static int cxgb_set_mac_addr(struct net_device *dev, void *p)
2805{
2806 int ret;
2807 struct sockaddr *addr = p;
2808 struct port_info *pi = netdev_priv(dev);
2809
2810 if (!is_valid_ether_addr(addr->sa_data))
2811 return -EINVAL;
2812
2813 ret = t4_change_mac(pi->adapter, pi->adapter->fn, pi->viid,
2814 pi->xact_addr_filt, addr->sa_data, true, true);
2815 if (ret < 0)
2816 return ret;
2817
2818 memcpy(dev->dev_addr, addr->sa_data, dev->addr_len);
2819 pi->xact_addr_filt = ret;
2820 return 0;
2821}
2822
2823#ifdef CONFIG_NET_POLL_CONTROLLER
2824static void cxgb_netpoll(struct net_device *dev)
2825{
2826 struct port_info *pi = netdev_priv(dev);
2827 struct adapter *adap = pi->adapter;
2828
2829 if (adap->flags & USING_MSIX) {
2830 int i;
2831 struct sge_eth_rxq *rx = &adap->sge.ethrxq[pi->first_qset];
2832
2833 for (i = pi->nqsets; i; i--, rx++)
2834 t4_sge_intr_msix(0, &rx->rspq);
2835 } else
2836 t4_intr_handler(adap)(0, adap);
2837}
2838#endif
2839
2840static const struct net_device_ops cxgb4_netdev_ops = {
2841 .ndo_open = cxgb_open,
2842 .ndo_stop = cxgb_close,
2843 .ndo_start_xmit = t4_eth_xmit,
2844 .ndo_get_stats64 = cxgb_get_stats,
2845 .ndo_set_rx_mode = cxgb_set_rxmode,
2846 .ndo_set_mac_address = cxgb_set_mac_addr,
2847 .ndo_set_features = cxgb_set_features,
2848 .ndo_validate_addr = eth_validate_addr,
2849 .ndo_do_ioctl = cxgb_ioctl,
2850 .ndo_change_mtu = cxgb_change_mtu,
2851#ifdef CONFIG_NET_POLL_CONTROLLER
2852 .ndo_poll_controller = cxgb_netpoll,
2853#endif
2854};
2855
2856void t4_fatal_err(struct adapter *adap)
2857{
2858 t4_set_reg_field(adap, SGE_CONTROL, GLOBALENABLE, 0);
2859 t4_intr_disable(adap);
2860 dev_alert(adap->pdev_dev, "encountered fatal error, adapter stopped\n");
2861}
2862
2863static void setup_memwin(struct adapter *adap)
2864{
2865 u32 bar0;
2866
2867 bar0 = pci_resource_start(adap->pdev, 0); /* truncation intentional */
2868 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 0),
2869 (bar0 + MEMWIN0_BASE) | BIR(0) |
2870 WINDOW(ilog2(MEMWIN0_APERTURE) - 10));
2871 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 1),
2872 (bar0 + MEMWIN1_BASE) | BIR(0) |
2873 WINDOW(ilog2(MEMWIN1_APERTURE) - 10));
2874 t4_write_reg(adap, PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 2),
2875 (bar0 + MEMWIN2_BASE) | BIR(0) |
2876 WINDOW(ilog2(MEMWIN2_APERTURE) - 10));
2877 if (adap->vres.ocq.size) {
2878 unsigned int start, sz_kb;
2879
2880 start = pci_resource_start(adap->pdev, 2) +
2881 OCQ_WIN_OFFSET(adap->pdev, &adap->vres);
2882 sz_kb = roundup_pow_of_two(adap->vres.ocq.size) >> 10;
2883 t4_write_reg(adap,
2884 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_BASE_WIN, 3),
2885 start | BIR(1) | WINDOW(ilog2(sz_kb)));
2886 t4_write_reg(adap,
2887 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3),
2888 adap->vres.ocq.start);
2889 t4_read_reg(adap,
2890 PCIE_MEM_ACCESS_REG(PCIE_MEM_ACCESS_OFFSET, 3));
2891 }
2892}
2893
2894static int adap_init1(struct adapter *adap, struct fw_caps_config_cmd *c)
2895{
2896 u32 v;
2897 int ret;
2898
2899 /* get device capabilities */
2900 memset(c, 0, sizeof(*c));
2901 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2902 FW_CMD_REQUEST | FW_CMD_READ);
2903 c->retval_len16 = htonl(FW_LEN16(*c));
2904 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), c);
2905 if (ret < 0)
2906 return ret;
2907
2908 /* select capabilities we'll be using */
2909 if (c->niccaps & htons(FW_CAPS_CONFIG_NIC_VM)) {
2910 if (!vf_acls)
2911 c->niccaps ^= htons(FW_CAPS_CONFIG_NIC_VM);
2912 else
2913 c->niccaps = htons(FW_CAPS_CONFIG_NIC_VM);
2914 } else if (vf_acls) {
2915 dev_err(adap->pdev_dev, "virtualization ACLs not supported");
2916 return ret;
2917 }
2918 c->op_to_write = htonl(FW_CMD_OP(FW_CAPS_CONFIG_CMD) |
2919 FW_CMD_REQUEST | FW_CMD_WRITE);
2920 ret = t4_wr_mbox(adap, adap->fn, c, sizeof(*c), NULL);
2921 if (ret < 0)
2922 return ret;
2923
2924 ret = t4_config_glbl_rss(adap, adap->fn,
2925 FW_RSS_GLB_CONFIG_CMD_MODE_BASICVIRTUAL,
2926 FW_RSS_GLB_CONFIG_CMD_TNLMAPEN |
2927 FW_RSS_GLB_CONFIG_CMD_TNLALLLKP);
2928 if (ret < 0)
2929 return ret;
2930
2931 ret = t4_cfg_pfvf(adap, adap->fn, adap->fn, 0, MAX_EGRQ, 64, MAX_INGQ,
2932 0, 0, 4, 0xf, 0xf, 16, FW_CMD_CAP_PF, FW_CMD_CAP_PF);
2933 if (ret < 0)
2934 return ret;
2935
2936 t4_sge_init(adap);
2937
2938 /* tweak some settings */
2939 t4_write_reg(adap, TP_SHIFT_CNT, 0x64f8849);
2940 t4_write_reg(adap, ULP_RX_TDDP_PSZ, HPZ0(PAGE_SHIFT - 12));
2941 t4_write_reg(adap, TP_PIO_ADDR, TP_INGRESS_CONFIG);
2942 v = t4_read_reg(adap, TP_PIO_DATA);
2943 t4_write_reg(adap, TP_PIO_DATA, v & ~CSUM_HAS_PSEUDO_HDR);
2944
2945 /* get basic stuff going */
2946 return t4_early_init(adap, adap->fn);
2947}
2948
2949/*
2950 * Max # of ATIDs. The absolute HW max is 16K but we keep it lower.
2951 */
2952#define MAX_ATIDS 8192U
2953
2954/*
2955 * Phase 0 of initialization: contact FW, obtain config, perform basic init.
2956 */
2957static int adap_init0(struct adapter *adap)
2958{
2959 int ret;
2960 u32 v, port_vec;
2961 enum dev_state state;
2962 u32 params[7], val[7];
2963 struct fw_caps_config_cmd c;
2964
2965 ret = t4_check_fw_version(adap);
2966 if (ret == -EINVAL || ret > 0) {
2967 if (upgrade_fw(adap) >= 0) /* recache FW version */
2968 ret = t4_check_fw_version(adap);
2969 }
2970 if (ret < 0)
2971 return ret;
2972
2973 /* contact FW, request master */
2974 ret = t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, &state);
2975 if (ret < 0) {
2976 dev_err(adap->pdev_dev, "could not connect to FW, error %d\n",
2977 ret);
2978 return ret;
2979 }
2980
2981 /* reset device */
2982 ret = t4_fw_reset(adap, adap->fn, PIORSTMODE | PIORST);
2983 if (ret < 0)
2984 goto bye;
2985
2986 for (v = 0; v < SGE_NTIMERS - 1; v++)
2987 adap->sge.timer_val[v] = min(intr_holdoff[v], MAX_SGE_TIMERVAL);
2988 adap->sge.timer_val[SGE_NTIMERS - 1] = MAX_SGE_TIMERVAL;
2989 adap->sge.counter_val[0] = 1;
2990 for (v = 1; v < SGE_NCOUNTERS; v++)
2991 adap->sge.counter_val[v] = min(intr_cnt[v - 1],
2992 THRESHOLD_3_MASK);
2993#define FW_PARAM_DEV(param) \
2994 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_DEV) | \
2995 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_DEV_##param))
2996
2997 params[0] = FW_PARAM_DEV(CCLK);
2998 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 1, params, val);
2999 if (ret < 0)
3000 goto bye;
3001 adap->params.vpd.cclk = val[0];
3002
3003 ret = adap_init1(adap, &c);
3004 if (ret < 0)
3005 goto bye;
3006
3007#define FW_PARAM_PFVF(param) \
3008 (FW_PARAMS_MNEM(FW_PARAMS_MNEM_PFVF) | \
3009 FW_PARAMS_PARAM_X(FW_PARAMS_PARAM_PFVF_##param) | \
3010 FW_PARAMS_PARAM_Y(adap->fn))
3011
3012 params[0] = FW_PARAM_DEV(PORTVEC);
3013 params[1] = FW_PARAM_PFVF(L2T_START);
3014 params[2] = FW_PARAM_PFVF(L2T_END);
3015 params[3] = FW_PARAM_PFVF(FILTER_START);
3016 params[4] = FW_PARAM_PFVF(FILTER_END);
3017 params[5] = FW_PARAM_PFVF(IQFLINT_START);
3018 params[6] = FW_PARAM_PFVF(EQ_START);
3019 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 7, params, val);
3020 if (ret < 0)
3021 goto bye;
3022 port_vec = val[0];
3023 adap->tids.ftid_base = val[3];
3024 adap->tids.nftids = val[4] - val[3] + 1;
3025 adap->sge.ingr_start = val[5];
3026 adap->sge.egr_start = val[6];
3027
3028 if (c.ofldcaps) {
3029 /* query offload-related parameters */
3030 params[0] = FW_PARAM_DEV(NTID);
3031 params[1] = FW_PARAM_PFVF(SERVER_START);
3032 params[2] = FW_PARAM_PFVF(SERVER_END);
3033 params[3] = FW_PARAM_PFVF(TDDP_START);
3034 params[4] = FW_PARAM_PFVF(TDDP_END);
3035 params[5] = FW_PARAM_DEV(FLOWC_BUFFIFO_SZ);
3036 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3037 val);
3038 if (ret < 0)
3039 goto bye;
3040 adap->tids.ntids = val[0];
3041 adap->tids.natids = min(adap->tids.ntids / 2, MAX_ATIDS);
3042 adap->tids.stid_base = val[1];
3043 adap->tids.nstids = val[2] - val[1] + 1;
3044 adap->vres.ddp.start = val[3];
3045 adap->vres.ddp.size = val[4] - val[3] + 1;
3046 adap->params.ofldq_wr_cred = val[5];
3047 adap->params.offload = 1;
3048 }
3049 if (c.rdmacaps) {
3050 params[0] = FW_PARAM_PFVF(STAG_START);
3051 params[1] = FW_PARAM_PFVF(STAG_END);
3052 params[2] = FW_PARAM_PFVF(RQ_START);
3053 params[3] = FW_PARAM_PFVF(RQ_END);
3054 params[4] = FW_PARAM_PFVF(PBL_START);
3055 params[5] = FW_PARAM_PFVF(PBL_END);
3056 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3057 val);
3058 if (ret < 0)
3059 goto bye;
3060 adap->vres.stag.start = val[0];
3061 adap->vres.stag.size = val[1] - val[0] + 1;
3062 adap->vres.rq.start = val[2];
3063 adap->vres.rq.size = val[3] - val[2] + 1;
3064 adap->vres.pbl.start = val[4];
3065 adap->vres.pbl.size = val[5] - val[4] + 1;
3066
3067 params[0] = FW_PARAM_PFVF(SQRQ_START);
3068 params[1] = FW_PARAM_PFVF(SQRQ_END);
3069 params[2] = FW_PARAM_PFVF(CQ_START);
3070 params[3] = FW_PARAM_PFVF(CQ_END);
3071 params[4] = FW_PARAM_PFVF(OCQ_START);
3072 params[5] = FW_PARAM_PFVF(OCQ_END);
3073 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 6, params,
3074 val);
3075 if (ret < 0)
3076 goto bye;
3077 adap->vres.qp.start = val[0];
3078 adap->vres.qp.size = val[1] - val[0] + 1;
3079 adap->vres.cq.start = val[2];
3080 adap->vres.cq.size = val[3] - val[2] + 1;
3081 adap->vres.ocq.start = val[4];
3082 adap->vres.ocq.size = val[5] - val[4] + 1;
3083 }
3084 if (c.iscsicaps) {
3085 params[0] = FW_PARAM_PFVF(ISCSI_START);
3086 params[1] = FW_PARAM_PFVF(ISCSI_END);
3087 ret = t4_query_params(adap, adap->fn, adap->fn, 0, 2, params,
3088 val);
3089 if (ret < 0)
3090 goto bye;
3091 adap->vres.iscsi.start = val[0];
3092 adap->vres.iscsi.size = val[1] - val[0] + 1;
3093 }
3094#undef FW_PARAM_PFVF
3095#undef FW_PARAM_DEV
3096
3097 adap->params.nports = hweight32(port_vec);
3098 adap->params.portvec = port_vec;
3099 adap->flags |= FW_OK;
3100
3101 /* These are finalized by FW initialization, load their values now */
3102 v = t4_read_reg(adap, TP_TIMER_RESOLUTION);
3103 adap->params.tp.tre = TIMERRESOLUTION_GET(v);
3104 t4_read_mtu_tbl(adap, adap->params.mtus, NULL);
3105 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3106 adap->params.b_wnd);
3107
3108#ifdef CONFIG_PCI_IOV
3109 /*
3110 * Provision resource limits for Virtual Functions. We currently
3111 * grant them all the same static resource limits except for the Port
3112 * Access Rights Mask which we're assigning based on the PF. All of
3113 * the static provisioning stuff for both the PF and VF really needs
3114 * to be managed in a persistent manner for each device which the
3115 * firmware controls.
3116 */
3117 {
3118 int pf, vf;
3119
3120 for (pf = 0; pf < ARRAY_SIZE(num_vf); pf++) {
3121 if (num_vf[pf] <= 0)
3122 continue;
3123
3124 /* VF numbering starts at 1! */
3125 for (vf = 1; vf <= num_vf[pf]; vf++) {
3126 ret = t4_cfg_pfvf(adap, adap->fn, pf, vf,
3127 VFRES_NEQ, VFRES_NETHCTRL,
3128 VFRES_NIQFLINT, VFRES_NIQ,
3129 VFRES_TC, VFRES_NVI,
3130 FW_PFVF_CMD_CMASK_MASK,
3131 pfvfres_pmask(adap, pf, vf),
3132 VFRES_NEXACTF,
3133 VFRES_R_CAPS, VFRES_WX_CAPS);
3134 if (ret < 0)
3135 dev_warn(adap->pdev_dev, "failed to "
3136 "provision pf/vf=%d/%d; "
3137 "err=%d\n", pf, vf, ret);
3138 }
3139 }
3140 }
3141#endif
3142
3143 setup_memwin(adap);
3144 return 0;
3145
3146 /*
3147 * If a command timed out or failed with EIO FW does not operate within
3148 * its spec or something catastrophic happened to HW/FW, stop issuing
3149 * commands.
3150 */
3151bye: if (ret != -ETIMEDOUT && ret != -EIO)
3152 t4_fw_bye(adap, adap->fn);
3153 return ret;
3154}
3155
3156/* EEH callbacks */
3157
3158static pci_ers_result_t eeh_err_detected(struct pci_dev *pdev,
3159 pci_channel_state_t state)
3160{
3161 int i;
3162 struct adapter *adap = pci_get_drvdata(pdev);
3163
3164 if (!adap)
3165 goto out;
3166
3167 rtnl_lock();
3168 adap->flags &= ~FW_OK;
3169 notify_ulds(adap, CXGB4_STATE_START_RECOVERY);
3170 for_each_port(adap, i) {
3171 struct net_device *dev = adap->port[i];
3172
3173 netif_device_detach(dev);
3174 netif_carrier_off(dev);
3175 }
3176 if (adap->flags & FULL_INIT_DONE)
3177 cxgb_down(adap);
3178 rtnl_unlock();
3179 pci_disable_device(pdev);
3180out: return state == pci_channel_io_perm_failure ?
3181 PCI_ERS_RESULT_DISCONNECT : PCI_ERS_RESULT_NEED_RESET;
3182}
3183
3184static pci_ers_result_t eeh_slot_reset(struct pci_dev *pdev)
3185{
3186 int i, ret;
3187 struct fw_caps_config_cmd c;
3188 struct adapter *adap = pci_get_drvdata(pdev);
3189
3190 if (!adap) {
3191 pci_restore_state(pdev);
3192 pci_save_state(pdev);
3193 return PCI_ERS_RESULT_RECOVERED;
3194 }
3195
3196 if (pci_enable_device(pdev)) {
3197 dev_err(&pdev->dev, "cannot reenable PCI device after reset\n");
3198 return PCI_ERS_RESULT_DISCONNECT;
3199 }
3200
3201 pci_set_master(pdev);
3202 pci_restore_state(pdev);
3203 pci_save_state(pdev);
3204 pci_cleanup_aer_uncorrect_error_status(pdev);
3205
3206 if (t4_wait_dev_ready(adap) < 0)
3207 return PCI_ERS_RESULT_DISCONNECT;
3208 if (t4_fw_hello(adap, adap->fn, adap->fn, MASTER_MUST, NULL))
3209 return PCI_ERS_RESULT_DISCONNECT;
3210 adap->flags |= FW_OK;
3211 if (adap_init1(adap, &c))
3212 return PCI_ERS_RESULT_DISCONNECT;
3213
3214 for_each_port(adap, i) {
3215 struct port_info *p = adap2pinfo(adap, i);
3216
3217 ret = t4_alloc_vi(adap, adap->fn, p->tx_chan, adap->fn, 0, 1,
3218 NULL, NULL);
3219 if (ret < 0)
3220 return PCI_ERS_RESULT_DISCONNECT;
3221 p->viid = ret;
3222 p->xact_addr_filt = -1;
3223 }
3224
3225 t4_load_mtus(adap, adap->params.mtus, adap->params.a_wnd,
3226 adap->params.b_wnd);
3227 setup_memwin(adap);
3228 if (cxgb_up(adap))
3229 return PCI_ERS_RESULT_DISCONNECT;
3230 return PCI_ERS_RESULT_RECOVERED;
3231}
3232
3233static void eeh_resume(struct pci_dev *pdev)
3234{
3235 int i;
3236 struct adapter *adap = pci_get_drvdata(pdev);
3237
3238 if (!adap)
3239 return;
3240
3241 rtnl_lock();
3242 for_each_port(adap, i) {
3243 struct net_device *dev = adap->port[i];
3244
3245 if (netif_running(dev)) {
3246 link_start(dev);
3247 cxgb_set_rxmode(dev);
3248 }
3249 netif_device_attach(dev);
3250 }
3251 rtnl_unlock();
3252}
3253
3254static struct pci_error_handlers cxgb4_eeh = {
3255 .error_detected = eeh_err_detected,
3256 .slot_reset = eeh_slot_reset,
3257 .resume = eeh_resume,
3258};
3259
3260static inline bool is_10g_port(const struct link_config *lc)
3261{
3262 return (lc->supported & FW_PORT_CAP_SPEED_10G) != 0;
3263}
3264
3265static inline void init_rspq(struct sge_rspq *q, u8 timer_idx, u8 pkt_cnt_idx,
3266 unsigned int size, unsigned int iqe_size)
3267{
3268 q->intr_params = QINTR_TIMER_IDX(timer_idx) |
3269 (pkt_cnt_idx < SGE_NCOUNTERS ? QINTR_CNT_EN : 0);
3270 q->pktcnt_idx = pkt_cnt_idx < SGE_NCOUNTERS ? pkt_cnt_idx : 0;
3271 q->iqe_len = iqe_size;
3272 q->size = size;
3273}
3274
3275/*
3276 * Perform default configuration of DMA queues depending on the number and type
3277 * of ports we found and the number of available CPUs. Most settings can be
3278 * modified by the admin prior to actual use.
3279 */
3280static void __devinit cfg_queues(struct adapter *adap)
3281{
3282 struct sge *s = &adap->sge;
3283 int i, q10g = 0, n10g = 0, qidx = 0;
3284
3285 for_each_port(adap, i)
3286 n10g += is_10g_port(&adap2pinfo(adap, i)->link_cfg);
3287
3288 /*
3289 * We default to 1 queue per non-10G port and up to # of cores queues
3290 * per 10G port.
3291 */
3292 if (n10g)
3293 q10g = (MAX_ETH_QSETS - (adap->params.nports - n10g)) / n10g;
3294 if (q10g > num_online_cpus())
3295 q10g = num_online_cpus();
3296
3297 for_each_port(adap, i) {
3298 struct port_info *pi = adap2pinfo(adap, i);
3299
3300 pi->first_qset = qidx;
3301 pi->nqsets = is_10g_port(&pi->link_cfg) ? q10g : 1;
3302 qidx += pi->nqsets;
3303 }
3304
3305 s->ethqsets = qidx;
3306 s->max_ethqsets = qidx; /* MSI-X may lower it later */
3307
3308 if (is_offload(adap)) {
3309 /*
3310 * For offload we use 1 queue/channel if all ports are up to 1G,
3311 * otherwise we divide all available queues amongst the channels
3312 * capped by the number of available cores.
3313 */
3314 if (n10g) {
3315 i = min_t(int, ARRAY_SIZE(s->ofldrxq),
3316 num_online_cpus());
3317 s->ofldqsets = roundup(i, adap->params.nports);
3318 } else
3319 s->ofldqsets = adap->params.nports;
3320 /* For RDMA one Rx queue per channel suffices */
3321 s->rdmaqs = adap->params.nports;
3322 }
3323
3324 for (i = 0; i < ARRAY_SIZE(s->ethrxq); i++) {
3325 struct sge_eth_rxq *r = &s->ethrxq[i];
3326
3327 init_rspq(&r->rspq, 0, 0, 1024, 64);
3328 r->fl.size = 72;
3329 }
3330
3331 for (i = 0; i < ARRAY_SIZE(s->ethtxq); i++)
3332 s->ethtxq[i].q.size = 1024;
3333
3334 for (i = 0; i < ARRAY_SIZE(s->ctrlq); i++)
3335 s->ctrlq[i].q.size = 512;
3336
3337 for (i = 0; i < ARRAY_SIZE(s->ofldtxq); i++)
3338 s->ofldtxq[i].q.size = 1024;
3339
3340 for (i = 0; i < ARRAY_SIZE(s->ofldrxq); i++) {
3341 struct sge_ofld_rxq *r = &s->ofldrxq[i];
3342
3343 init_rspq(&r->rspq, 0, 0, 1024, 64);
3344 r->rspq.uld = CXGB4_ULD_ISCSI;
3345 r->fl.size = 72;
3346 }
3347
3348 for (i = 0; i < ARRAY_SIZE(s->rdmarxq); i++) {
3349 struct sge_ofld_rxq *r = &s->rdmarxq[i];
3350
3351 init_rspq(&r->rspq, 0, 0, 511, 64);
3352 r->rspq.uld = CXGB4_ULD_RDMA;
3353 r->fl.size = 72;
3354 }
3355
3356 init_rspq(&s->fw_evtq, 6, 0, 512, 64);
3357 init_rspq(&s->intrq, 6, 0, 2 * MAX_INGQ, 64);
3358}
3359
3360/*
3361 * Reduce the number of Ethernet queues across all ports to at most n.
3362 * n provides at least one queue per port.
3363 */
3364static void __devinit reduce_ethqs(struct adapter *adap, int n)
3365{
3366 int i;
3367 struct port_info *pi;
3368
3369 while (n < adap->sge.ethqsets)
3370 for_each_port(adap, i) {
3371 pi = adap2pinfo(adap, i);
3372 if (pi->nqsets > 1) {
3373 pi->nqsets--;
3374 adap->sge.ethqsets--;
3375 if (adap->sge.ethqsets <= n)
3376 break;
3377 }
3378 }
3379
3380 n = 0;
3381 for_each_port(adap, i) {
3382 pi = adap2pinfo(adap, i);
3383 pi->first_qset = n;
3384 n += pi->nqsets;
3385 }
3386}
3387
3388/* 2 MSI-X vectors needed for the FW queue and non-data interrupts */
3389#define EXTRA_VECS 2
3390
3391static int __devinit enable_msix(struct adapter *adap)
3392{
3393 int ofld_need = 0;
3394 int i, err, want, need;
3395 struct sge *s = &adap->sge;
3396 unsigned int nchan = adap->params.nports;
3397 struct msix_entry entries[MAX_INGQ + 1];
3398
3399 for (i = 0; i < ARRAY_SIZE(entries); ++i)
3400 entries[i].entry = i;
3401
3402 want = s->max_ethqsets + EXTRA_VECS;
3403 if (is_offload(adap)) {
3404 want += s->rdmaqs + s->ofldqsets;
3405 /* need nchan for each possible ULD */
3406 ofld_need = 2 * nchan;
3407 }
3408 need = adap->params.nports + EXTRA_VECS + ofld_need;
3409
3410 while ((err = pci_enable_msix(adap->pdev, entries, want)) >= need)
3411 want = err;
3412
3413 if (!err) {
3414 /*
3415 * Distribute available vectors to the various queue groups.
3416 * Every group gets its minimum requirement and NIC gets top
3417 * priority for leftovers.
3418 */
3419 i = want - EXTRA_VECS - ofld_need;
3420 if (i < s->max_ethqsets) {
3421 s->max_ethqsets = i;
3422 if (i < s->ethqsets)
3423 reduce_ethqs(adap, i);
3424 }
3425 if (is_offload(adap)) {
3426 i = want - EXTRA_VECS - s->max_ethqsets;
3427 i -= ofld_need - nchan;
3428 s->ofldqsets = (i / nchan) * nchan; /* round down */
3429 }
3430 for (i = 0; i < want; ++i)
3431 adap->msix_info[i].vec = entries[i].vector;
3432 } else if (err > 0)
3433 dev_info(adap->pdev_dev,
3434 "only %d MSI-X vectors left, not using MSI-X\n", err);
3435 return err;
3436}
3437
3438#undef EXTRA_VECS
3439
3440static int __devinit init_rss(struct adapter *adap)
3441{
3442 unsigned int i, j;
3443
3444 for_each_port(adap, i) {
3445 struct port_info *pi = adap2pinfo(adap, i);
3446
3447 pi->rss = kcalloc(pi->rss_size, sizeof(u16), GFP_KERNEL);
3448 if (!pi->rss)
3449 return -ENOMEM;
3450 for (j = 0; j < pi->rss_size; j++)
3451 pi->rss[j] = j % pi->nqsets;
3452 }
3453 return 0;
3454}
3455
3456static void __devinit print_port_info(const struct net_device *dev)
3457{
3458 static const char *base[] = {
3459 "R XFI", "R XAUI", "T SGMII", "T XFI", "T XAUI", "KX4", "CX4",
3460 "KX", "KR", "R SFP+", "KR/KX", "KR/KX/KX4"
3461 };
3462
3463 char buf[80];
3464 char *bufp = buf;
3465 const char *spd = "";
3466 const struct port_info *pi = netdev_priv(dev);
3467 const struct adapter *adap = pi->adapter;
3468
3469 if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_2_5GB)
3470 spd = " 2.5 GT/s";
3471 else if (adap->params.pci.speed == PCI_EXP_LNKSTA_CLS_5_0GB)
3472 spd = " 5 GT/s";
3473
3474 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_100M)
3475 bufp += sprintf(bufp, "100/");
3476 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_1G)
3477 bufp += sprintf(bufp, "1000/");
3478 if (pi->link_cfg.supported & FW_PORT_CAP_SPEED_10G)
3479 bufp += sprintf(bufp, "10G/");
3480 if (bufp != buf)
3481 --bufp;
3482 sprintf(bufp, "BASE-%s", base[pi->port_type]);
3483
3484 netdev_info(dev, "Chelsio %s rev %d %s %sNIC PCIe x%d%s%s\n",
3485 adap->params.vpd.id, adap->params.rev, buf,
3486 is_offload(adap) ? "R" : "", adap->params.pci.width, spd,
3487 (adap->flags & USING_MSIX) ? " MSI-X" :
3488 (adap->flags & USING_MSI) ? " MSI" : "");
3489 netdev_info(dev, "S/N: %s, E/C: %s\n",
3490 adap->params.vpd.sn, adap->params.vpd.ec);
3491}
3492
3493static void __devinit enable_pcie_relaxed_ordering(struct pci_dev *dev)
3494{
3495 u16 v;
3496 int pos;
3497
3498 pos = pci_pcie_cap(dev);
3499 if (pos > 0) {
3500 pci_read_config_word(dev, pos + PCI_EXP_DEVCTL, &v);
3501 v |= PCI_EXP_DEVCTL_RELAX_EN;
3502 pci_write_config_word(dev, pos + PCI_EXP_DEVCTL, v);
3503 }
3504}
3505
3506/*
3507 * Free the following resources:
3508 * - memory used for tables
3509 * - MSI/MSI-X
3510 * - net devices
3511 * - resources FW is holding for us
3512 */
3513static void free_some_resources(struct adapter *adapter)
3514{
3515 unsigned int i;
3516
3517 t4_free_mem(adapter->l2t);
3518 t4_free_mem(adapter->tids.tid_tab);
3519 disable_msi(adapter);
3520
3521 for_each_port(adapter, i)
3522 if (adapter->port[i]) {
3523 kfree(adap2pinfo(adapter, i)->rss);
3524 free_netdev(adapter->port[i]);
3525 }
3526 if (adapter->flags & FW_OK)
3527 t4_fw_bye(adapter, adapter->fn);
3528}
3529
3530#define TSO_FLAGS (NETIF_F_TSO | NETIF_F_TSO6 | NETIF_F_TSO_ECN)
3531#define VLAN_FEAT (NETIF_F_SG | NETIF_F_IP_CSUM | TSO_FLAGS | \
3532 NETIF_F_IPV6_CSUM | NETIF_F_HIGHDMA)
3533
3534static int __devinit init_one(struct pci_dev *pdev,
3535 const struct pci_device_id *ent)
3536{
3537 int func, i, err;
3538 struct port_info *pi;
3539 unsigned int highdma = 0;
3540 struct adapter *adapter = NULL;
3541
3542 printk_once(KERN_INFO "%s - version %s\n", DRV_DESC, DRV_VERSION);
3543
3544 err = pci_request_regions(pdev, KBUILD_MODNAME);
3545 if (err) {
3546 /* Just info, some other driver may have claimed the device. */
3547 dev_info(&pdev->dev, "cannot obtain PCI resources\n");
3548 return err;
3549 }
3550
3551 /* We control everything through one PF */
3552 func = PCI_FUNC(pdev->devfn);
3553 if (func != ent->driver_data) {
3554 pci_save_state(pdev); /* to restore SR-IOV later */
3555 goto sriov;
3556 }
3557
3558 err = pci_enable_device(pdev);
3559 if (err) {
3560 dev_err(&pdev->dev, "cannot enable PCI device\n");
3561 goto out_release_regions;
3562 }
3563
3564 if (!pci_set_dma_mask(pdev, DMA_BIT_MASK(64))) {
3565 highdma = NETIF_F_HIGHDMA;
3566 err = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
3567 if (err) {
3568 dev_err(&pdev->dev, "unable to obtain 64-bit DMA for "
3569 "coherent allocations\n");
3570 goto out_disable_device;
3571 }
3572 } else {
3573 err = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
3574 if (err) {
3575 dev_err(&pdev->dev, "no usable DMA configuration\n");
3576 goto out_disable_device;
3577 }
3578 }
3579
3580 pci_enable_pcie_error_reporting(pdev);
3581 enable_pcie_relaxed_ordering(pdev);
3582 pci_set_master(pdev);
3583 pci_save_state(pdev);
3584
3585 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL);
3586 if (!adapter) {
3587 err = -ENOMEM;
3588 goto out_disable_device;
3589 }
3590
3591 adapter->regs = pci_ioremap_bar(pdev, 0);
3592 if (!adapter->regs) {
3593 dev_err(&pdev->dev, "cannot map device registers\n");
3594 err = -ENOMEM;
3595 goto out_free_adapter;
3596 }
3597
3598 adapter->pdev = pdev;
3599 adapter->pdev_dev = &pdev->dev;
3600 adapter->fn = func;
3601 adapter->msg_enable = dflt_msg_enable;
3602 memset(adapter->chan_map, 0xff, sizeof(adapter->chan_map));
3603
3604 spin_lock_init(&adapter->stats_lock);
3605 spin_lock_init(&adapter->tid_release_lock);
3606
3607 INIT_WORK(&adapter->tid_release_task, process_tid_release_list);
3608
3609 err = t4_prep_adapter(adapter);
3610 if (err)
3611 goto out_unmap_bar;
3612 err = adap_init0(adapter);
3613 if (err)
3614 goto out_unmap_bar;
3615
3616 for_each_port(adapter, i) {
3617 struct net_device *netdev;
3618
3619 netdev = alloc_etherdev_mq(sizeof(struct port_info),
3620 MAX_ETH_QSETS);
3621 if (!netdev) {
3622 err = -ENOMEM;
3623 goto out_free_dev;
3624 }
3625
3626 SET_NETDEV_DEV(netdev, &pdev->dev);
3627
3628 adapter->port[i] = netdev;
3629 pi = netdev_priv(netdev);
3630 pi->adapter = adapter;
3631 pi->xact_addr_filt = -1;
3632 pi->port_id = i;
3633 netdev->irq = pdev->irq;
3634
3635 netdev->hw_features = NETIF_F_SG | TSO_FLAGS |
3636 NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
3637 NETIF_F_RXCSUM | NETIF_F_RXHASH |
3638 NETIF_F_HW_VLAN_TX | NETIF_F_HW_VLAN_RX;
3639 netdev->features |= netdev->hw_features | highdma;
3640 netdev->vlan_features = netdev->features & VLAN_FEAT;
3641
3642 netdev->netdev_ops = &cxgb4_netdev_ops;
3643 SET_ETHTOOL_OPS(netdev, &cxgb_ethtool_ops);
3644 }
3645
3646 pci_set_drvdata(pdev, adapter);
3647
3648 if (adapter->flags & FW_OK) {
3649 err = t4_port_init(adapter, func, func, 0);
3650 if (err)
3651 goto out_free_dev;
3652 }
3653
3654 /*
3655 * Configure queues and allocate tables now, they can be needed as
3656 * soon as the first register_netdev completes.
3657 */
3658 cfg_queues(adapter);
3659
3660 adapter->l2t = t4_init_l2t();
3661 if (!adapter->l2t) {
3662 /* We tolerate a lack of L2T, giving up some functionality */
3663 dev_warn(&pdev->dev, "could not allocate L2T, continuing\n");
3664 adapter->params.offload = 0;
3665 }
3666
3667 if (is_offload(adapter) && tid_init(&adapter->tids) < 0) {
3668 dev_warn(&pdev->dev, "could not allocate TID table, "
3669 "continuing\n");
3670 adapter->params.offload = 0;
3671 }
3672
3673 /* See what interrupts we'll be using */
3674 if (msi > 1 && enable_msix(adapter) == 0)
3675 adapter->flags |= USING_MSIX;
3676 else if (msi > 0 && pci_enable_msi(pdev) == 0)
3677 adapter->flags |= USING_MSI;
3678
3679 err = init_rss(adapter);
3680 if (err)
3681 goto out_free_dev;
3682
3683 /*
3684 * The card is now ready to go. If any errors occur during device
3685 * registration we do not fail the whole card but rather proceed only
3686 * with the ports we manage to register successfully. However we must
3687 * register at least one net device.
3688 */
3689 for_each_port(adapter, i) {
3690 pi = adap2pinfo(adapter, i);
3691 netif_set_real_num_tx_queues(adapter->port[i], pi->nqsets);
3692 netif_set_real_num_rx_queues(adapter->port[i], pi->nqsets);
3693
3694 err = register_netdev(adapter->port[i]);
3695 if (err)
3696 break;
3697 adapter->chan_map[pi->tx_chan] = i;
3698 print_port_info(adapter->port[i]);
3699 }
3700 if (i == 0) {
3701 dev_err(&pdev->dev, "could not register any net devices\n");
3702 goto out_free_dev;
3703 }
3704 if (err) {
3705 dev_warn(&pdev->dev, "only %d net devices registered\n", i);
3706 err = 0;
3707 }
3708
3709 if (cxgb4_debugfs_root) {
3710 adapter->debugfs_root = debugfs_create_dir(pci_name(pdev),
3711 cxgb4_debugfs_root);
3712 setup_debugfs(adapter);
3713 }
3714
3715 /* PCIe EEH recovery on powerpc platforms needs fundamental reset */
3716 pdev->needs_freset = 1;
3717
3718 if (is_offload(adapter))
3719 attach_ulds(adapter);
3720
3721sriov:
3722#ifdef CONFIG_PCI_IOV
3723 if (func < ARRAY_SIZE(num_vf) && num_vf[func] > 0)
3724 if (pci_enable_sriov(pdev, num_vf[func]) == 0)
3725 dev_info(&pdev->dev,
3726 "instantiated %u virtual functions\n",
3727 num_vf[func]);
3728#endif
3729 return 0;
3730
3731 out_free_dev:
3732 free_some_resources(adapter);
3733 out_unmap_bar:
3734 iounmap(adapter->regs);
3735 out_free_adapter:
3736 kfree(adapter);
3737 out_disable_device:
3738 pci_disable_pcie_error_reporting(pdev);
3739 pci_disable_device(pdev);
3740 out_release_regions:
3741 pci_release_regions(pdev);
3742 pci_set_drvdata(pdev, NULL);
3743 return err;
3744}
3745
3746static void __devexit remove_one(struct pci_dev *pdev)
3747{
3748 struct adapter *adapter = pci_get_drvdata(pdev);
3749
3750 pci_disable_sriov(pdev);
3751
3752 if (adapter) {
3753 int i;
3754
3755 if (is_offload(adapter))
3756 detach_ulds(adapter);
3757
3758 for_each_port(adapter, i)
3759 if (adapter->port[i]->reg_state == NETREG_REGISTERED)
3760 unregister_netdev(adapter->port[i]);
3761
3762 if (adapter->debugfs_root)
3763 debugfs_remove_recursive(adapter->debugfs_root);
3764
3765 if (adapter->flags & FULL_INIT_DONE)
3766 cxgb_down(adapter);
3767
3768 free_some_resources(adapter);
3769 iounmap(adapter->regs);
3770 kfree(adapter);
3771 pci_disable_pcie_error_reporting(pdev);
3772 pci_disable_device(pdev);
3773 pci_release_regions(pdev);
3774 pci_set_drvdata(pdev, NULL);
3775 } else
3776 pci_release_regions(pdev);
3777}
3778
3779static struct pci_driver cxgb4_driver = {
3780 .name = KBUILD_MODNAME,
3781 .id_table = cxgb4_pci_tbl,
3782 .probe = init_one,
3783 .remove = __devexit_p(remove_one),
3784 .err_handler = &cxgb4_eeh,
3785};
3786
3787static int __init cxgb4_init_module(void)
3788{
3789 int ret;
3790
3791 /* Debugfs support is optional, just warn if this fails */
3792 cxgb4_debugfs_root = debugfs_create_dir(KBUILD_MODNAME, NULL);
3793 if (!cxgb4_debugfs_root)
3794 pr_warning("could not create debugfs entry, continuing\n");
3795
3796 ret = pci_register_driver(&cxgb4_driver);
3797 if (ret < 0)
3798 debugfs_remove(cxgb4_debugfs_root);
3799 return ret;
3800}
3801
3802static void __exit cxgb4_cleanup_module(void)
3803{
3804 pci_unregister_driver(&cxgb4_driver);
3805 debugfs_remove(cxgb4_debugfs_root); /* NULL ok */
3806}
3807
3808module_init(cxgb4_init_module);
3809module_exit(cxgb4_cleanup_module);