Loading...
1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2012 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
30#include <linux/module.h>
31#include <linux/types.h>
32#include <linux/init.h>
33#include <linux/bitops.h>
34#include <linux/vmalloc.h>
35#include <linux/pagemap.h>
36#include <linux/netdevice.h>
37#include <linux/ipv6.h>
38#include <linux/slab.h>
39#include <net/checksum.h>
40#include <net/ip6_checksum.h>
41#include <linux/net_tstamp.h>
42#include <linux/mii.h>
43#include <linux/ethtool.h>
44#include <linux/if.h>
45#include <linux/if_vlan.h>
46#include <linux/pci.h>
47#include <linux/pci-aspm.h>
48#include <linux/delay.h>
49#include <linux/interrupt.h>
50#include <linux/ip.h>
51#include <linux/tcp.h>
52#include <linux/sctp.h>
53#include <linux/if_ether.h>
54#include <linux/aer.h>
55#include <linux/prefetch.h>
56#include <linux/pm_runtime.h>
57#ifdef CONFIG_IGB_DCA
58#include <linux/dca.h>
59#endif
60#include "igb.h"
61
62#define MAJ 3
63#define MIN 4
64#define BUILD 7
65#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
66__stringify(BUILD) "-k"
67char igb_driver_name[] = "igb";
68char igb_driver_version[] = DRV_VERSION;
69static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
71static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
72
73static const struct e1000_info *igb_info_tbl[] = {
74 [board_82575] = &e1000_82575_info,
75};
76
77static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
104 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
105 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
106 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
107 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
108 /* required last entry */
109 {0, }
110};
111
112MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
113
114void igb_reset(struct igb_adapter *);
115static int igb_setup_all_tx_resources(struct igb_adapter *);
116static int igb_setup_all_rx_resources(struct igb_adapter *);
117static void igb_free_all_tx_resources(struct igb_adapter *);
118static void igb_free_all_rx_resources(struct igb_adapter *);
119static void igb_setup_mrqc(struct igb_adapter *);
120static int igb_probe(struct pci_dev *, const struct pci_device_id *);
121static void __devexit igb_remove(struct pci_dev *pdev);
122static int igb_sw_init(struct igb_adapter *);
123static int igb_open(struct net_device *);
124static int igb_close(struct net_device *);
125static void igb_configure_tx(struct igb_adapter *);
126static void igb_configure_rx(struct igb_adapter *);
127static void igb_clean_all_tx_rings(struct igb_adapter *);
128static void igb_clean_all_rx_rings(struct igb_adapter *);
129static void igb_clean_tx_ring(struct igb_ring *);
130static void igb_clean_rx_ring(struct igb_ring *);
131static void igb_set_rx_mode(struct net_device *);
132static void igb_update_phy_info(unsigned long);
133static void igb_watchdog(unsigned long);
134static void igb_watchdog_task(struct work_struct *);
135static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
136static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
137 struct rtnl_link_stats64 *stats);
138static int igb_change_mtu(struct net_device *, int);
139static int igb_set_mac(struct net_device *, void *);
140static void igb_set_uta(struct igb_adapter *adapter);
141static irqreturn_t igb_intr(int irq, void *);
142static irqreturn_t igb_intr_msi(int irq, void *);
143static irqreturn_t igb_msix_other(int irq, void *);
144static irqreturn_t igb_msix_ring(int irq, void *);
145#ifdef CONFIG_IGB_DCA
146static void igb_update_dca(struct igb_q_vector *);
147static void igb_setup_dca(struct igb_adapter *);
148#endif /* CONFIG_IGB_DCA */
149static int igb_poll(struct napi_struct *, int);
150static bool igb_clean_tx_irq(struct igb_q_vector *);
151static bool igb_clean_rx_irq(struct igb_q_vector *, int);
152static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
153static void igb_tx_timeout(struct net_device *);
154static void igb_reset_task(struct work_struct *);
155static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
156static int igb_vlan_rx_add_vid(struct net_device *, u16);
157static int igb_vlan_rx_kill_vid(struct net_device *, u16);
158static void igb_restore_vlan(struct igb_adapter *);
159static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
160static void igb_ping_all_vfs(struct igb_adapter *);
161static void igb_msg_task(struct igb_adapter *);
162static void igb_vmm_control(struct igb_adapter *);
163static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
164static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
165static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
166static int igb_ndo_set_vf_vlan(struct net_device *netdev,
167 int vf, u16 vlan, u8 qos);
168static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
169static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
170 struct ifla_vf_info *ivi);
171static void igb_check_vf_rate_limit(struct igb_adapter *);
172
173#ifdef CONFIG_PCI_IOV
174static int igb_vf_configure(struct igb_adapter *adapter, int vf);
175static int igb_find_enabled_vfs(struct igb_adapter *adapter);
176static int igb_check_vf_assignment(struct igb_adapter *adapter);
177#endif
178
179#ifdef CONFIG_PM
180#ifdef CONFIG_PM_SLEEP
181static int igb_suspend(struct device *);
182#endif
183static int igb_resume(struct device *);
184#ifdef CONFIG_PM_RUNTIME
185static int igb_runtime_suspend(struct device *dev);
186static int igb_runtime_resume(struct device *dev);
187static int igb_runtime_idle(struct device *dev);
188#endif
189static const struct dev_pm_ops igb_pm_ops = {
190 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
191 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
192 igb_runtime_idle)
193};
194#endif
195static void igb_shutdown(struct pci_dev *);
196#ifdef CONFIG_IGB_DCA
197static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
198static struct notifier_block dca_notifier = {
199 .notifier_call = igb_notify_dca,
200 .next = NULL,
201 .priority = 0
202};
203#endif
204#ifdef CONFIG_NET_POLL_CONTROLLER
205/* for netdump / net console */
206static void igb_netpoll(struct net_device *);
207#endif
208#ifdef CONFIG_PCI_IOV
209static unsigned int max_vfs = 0;
210module_param(max_vfs, uint, 0);
211MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
212 "per physical function");
213#endif /* CONFIG_PCI_IOV */
214
215static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
216 pci_channel_state_t);
217static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
218static void igb_io_resume(struct pci_dev *);
219
220static struct pci_error_handlers igb_err_handler = {
221 .error_detected = igb_io_error_detected,
222 .slot_reset = igb_io_slot_reset,
223 .resume = igb_io_resume,
224};
225
226static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
227
228static struct pci_driver igb_driver = {
229 .name = igb_driver_name,
230 .id_table = igb_pci_tbl,
231 .probe = igb_probe,
232 .remove = __devexit_p(igb_remove),
233#ifdef CONFIG_PM
234 .driver.pm = &igb_pm_ops,
235#endif
236 .shutdown = igb_shutdown,
237 .err_handler = &igb_err_handler
238};
239
240MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
241MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
242MODULE_LICENSE("GPL");
243MODULE_VERSION(DRV_VERSION);
244
245#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
246static int debug = -1;
247module_param(debug, int, 0);
248MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
249
250struct igb_reg_info {
251 u32 ofs;
252 char *name;
253};
254
255static const struct igb_reg_info igb_reg_info_tbl[] = {
256
257 /* General Registers */
258 {E1000_CTRL, "CTRL"},
259 {E1000_STATUS, "STATUS"},
260 {E1000_CTRL_EXT, "CTRL_EXT"},
261
262 /* Interrupt Registers */
263 {E1000_ICR, "ICR"},
264
265 /* RX Registers */
266 {E1000_RCTL, "RCTL"},
267 {E1000_RDLEN(0), "RDLEN"},
268 {E1000_RDH(0), "RDH"},
269 {E1000_RDT(0), "RDT"},
270 {E1000_RXDCTL(0), "RXDCTL"},
271 {E1000_RDBAL(0), "RDBAL"},
272 {E1000_RDBAH(0), "RDBAH"},
273
274 /* TX Registers */
275 {E1000_TCTL, "TCTL"},
276 {E1000_TDBAL(0), "TDBAL"},
277 {E1000_TDBAH(0), "TDBAH"},
278 {E1000_TDLEN(0), "TDLEN"},
279 {E1000_TDH(0), "TDH"},
280 {E1000_TDT(0), "TDT"},
281 {E1000_TXDCTL(0), "TXDCTL"},
282 {E1000_TDFH, "TDFH"},
283 {E1000_TDFT, "TDFT"},
284 {E1000_TDFHS, "TDFHS"},
285 {E1000_TDFPC, "TDFPC"},
286
287 /* List Terminator */
288 {}
289};
290
291/*
292 * igb_regdump - register printout routine
293 */
294static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
295{
296 int n = 0;
297 char rname[16];
298 u32 regs[8];
299
300 switch (reginfo->ofs) {
301 case E1000_RDLEN(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_RDLEN(n));
304 break;
305 case E1000_RDH(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_RDH(n));
308 break;
309 case E1000_RDT(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_RDT(n));
312 break;
313 case E1000_RXDCTL(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_RXDCTL(n));
316 break;
317 case E1000_RDBAL(0):
318 for (n = 0; n < 4; n++)
319 regs[n] = rd32(E1000_RDBAL(n));
320 break;
321 case E1000_RDBAH(0):
322 for (n = 0; n < 4; n++)
323 regs[n] = rd32(E1000_RDBAH(n));
324 break;
325 case E1000_TDBAL(0):
326 for (n = 0; n < 4; n++)
327 regs[n] = rd32(E1000_RDBAL(n));
328 break;
329 case E1000_TDBAH(0):
330 for (n = 0; n < 4; n++)
331 regs[n] = rd32(E1000_TDBAH(n));
332 break;
333 case E1000_TDLEN(0):
334 for (n = 0; n < 4; n++)
335 regs[n] = rd32(E1000_TDLEN(n));
336 break;
337 case E1000_TDH(0):
338 for (n = 0; n < 4; n++)
339 regs[n] = rd32(E1000_TDH(n));
340 break;
341 case E1000_TDT(0):
342 for (n = 0; n < 4; n++)
343 regs[n] = rd32(E1000_TDT(n));
344 break;
345 case E1000_TXDCTL(0):
346 for (n = 0; n < 4; n++)
347 regs[n] = rd32(E1000_TXDCTL(n));
348 break;
349 default:
350 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
351 return;
352 }
353
354 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
355 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
356 regs[2], regs[3]);
357}
358
359/*
360 * igb_dump - Print registers, tx-rings and rx-rings
361 */
362static void igb_dump(struct igb_adapter *adapter)
363{
364 struct net_device *netdev = adapter->netdev;
365 struct e1000_hw *hw = &adapter->hw;
366 struct igb_reg_info *reginfo;
367 struct igb_ring *tx_ring;
368 union e1000_adv_tx_desc *tx_desc;
369 struct my_u0 { u64 a; u64 b; } *u0;
370 struct igb_ring *rx_ring;
371 union e1000_adv_rx_desc *rx_desc;
372 u32 staterr;
373 u16 i, n;
374
375 if (!netif_msg_hw(adapter))
376 return;
377
378 /* Print netdevice Info */
379 if (netdev) {
380 dev_info(&adapter->pdev->dev, "Net device Info\n");
381 pr_info("Device Name state trans_start "
382 "last_rx\n");
383 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
384 netdev->state, netdev->trans_start, netdev->last_rx);
385 }
386
387 /* Print Registers */
388 dev_info(&adapter->pdev->dev, "Register Dump\n");
389 pr_info(" Register Name Value\n");
390 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
391 reginfo->name; reginfo++) {
392 igb_regdump(hw, reginfo);
393 }
394
395 /* Print TX Ring Summary */
396 if (!netdev || !netif_running(netdev))
397 goto exit;
398
399 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
400 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
401 for (n = 0; n < adapter->num_tx_queues; n++) {
402 struct igb_tx_buffer *buffer_info;
403 tx_ring = adapter->tx_ring[n];
404 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
405 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
406 n, tx_ring->next_to_use, tx_ring->next_to_clean,
407 (u64)buffer_info->dma,
408 buffer_info->length,
409 buffer_info->next_to_watch,
410 (u64)buffer_info->time_stamp);
411 }
412
413 /* Print TX Rings */
414 if (!netif_msg_tx_done(adapter))
415 goto rx_ring_summary;
416
417 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
418
419 /* Transmit Descriptor Formats
420 *
421 * Advanced Transmit Descriptor
422 * +--------------------------------------------------------------+
423 * 0 | Buffer Address [63:0] |
424 * +--------------------------------------------------------------+
425 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
426 * +--------------------------------------------------------------+
427 * 63 46 45 40 39 38 36 35 32 31 24 15 0
428 */
429
430 for (n = 0; n < adapter->num_tx_queues; n++) {
431 tx_ring = adapter->tx_ring[n];
432 pr_info("------------------------------------\n");
433 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
434 pr_info("------------------------------------\n");
435 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
436 "[bi->dma ] leng ntw timestamp "
437 "bi->skb\n");
438
439 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
440 const char *next_desc;
441 struct igb_tx_buffer *buffer_info;
442 tx_desc = IGB_TX_DESC(tx_ring, i);
443 buffer_info = &tx_ring->tx_buffer_info[i];
444 u0 = (struct my_u0 *)tx_desc;
445 if (i == tx_ring->next_to_use &&
446 i == tx_ring->next_to_clean)
447 next_desc = " NTC/U";
448 else if (i == tx_ring->next_to_use)
449 next_desc = " NTU";
450 else if (i == tx_ring->next_to_clean)
451 next_desc = " NTC";
452 else
453 next_desc = "";
454
455 pr_info("T [0x%03X] %016llX %016llX %016llX"
456 " %04X %p %016llX %p%s\n", i,
457 le64_to_cpu(u0->a),
458 le64_to_cpu(u0->b),
459 (u64)buffer_info->dma,
460 buffer_info->length,
461 buffer_info->next_to_watch,
462 (u64)buffer_info->time_stamp,
463 buffer_info->skb, next_desc);
464
465 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
466 print_hex_dump(KERN_INFO, "",
467 DUMP_PREFIX_ADDRESS,
468 16, 1, phys_to_virt(buffer_info->dma),
469 buffer_info->length, true);
470 }
471 }
472
473 /* Print RX Rings Summary */
474rx_ring_summary:
475 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
476 pr_info("Queue [NTU] [NTC]\n");
477 for (n = 0; n < adapter->num_rx_queues; n++) {
478 rx_ring = adapter->rx_ring[n];
479 pr_info(" %5d %5X %5X\n",
480 n, rx_ring->next_to_use, rx_ring->next_to_clean);
481 }
482
483 /* Print RX Rings */
484 if (!netif_msg_rx_status(adapter))
485 goto exit;
486
487 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
488
489 /* Advanced Receive Descriptor (Read) Format
490 * 63 1 0
491 * +-----------------------------------------------------+
492 * 0 | Packet Buffer Address [63:1] |A0/NSE|
493 * +----------------------------------------------+------+
494 * 8 | Header Buffer Address [63:1] | DD |
495 * +-----------------------------------------------------+
496 *
497 *
498 * Advanced Receive Descriptor (Write-Back) Format
499 *
500 * 63 48 47 32 31 30 21 20 17 16 4 3 0
501 * +------------------------------------------------------+
502 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
503 * | Checksum Ident | | | | Type | Type |
504 * +------------------------------------------------------+
505 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
506 * +------------------------------------------------------+
507 * 63 48 47 32 31 20 19 0
508 */
509
510 for (n = 0; n < adapter->num_rx_queues; n++) {
511 rx_ring = adapter->rx_ring[n];
512 pr_info("------------------------------------\n");
513 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
514 pr_info("------------------------------------\n");
515 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
516 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
517 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
518 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
519
520 for (i = 0; i < rx_ring->count; i++) {
521 const char *next_desc;
522 struct igb_rx_buffer *buffer_info;
523 buffer_info = &rx_ring->rx_buffer_info[i];
524 rx_desc = IGB_RX_DESC(rx_ring, i);
525 u0 = (struct my_u0 *)rx_desc;
526 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
527
528 if (i == rx_ring->next_to_use)
529 next_desc = " NTU";
530 else if (i == rx_ring->next_to_clean)
531 next_desc = " NTC";
532 else
533 next_desc = "";
534
535 if (staterr & E1000_RXD_STAT_DD) {
536 /* Descriptor Done */
537 pr_info("%s[0x%03X] %016llX %016llX -------"
538 "--------- %p%s\n", "RWB", i,
539 le64_to_cpu(u0->a),
540 le64_to_cpu(u0->b),
541 buffer_info->skb, next_desc);
542 } else {
543 pr_info("%s[0x%03X] %016llX %016llX %016llX"
544 " %p%s\n", "R ", i,
545 le64_to_cpu(u0->a),
546 le64_to_cpu(u0->b),
547 (u64)buffer_info->dma,
548 buffer_info->skb, next_desc);
549
550 if (netif_msg_pktdata(adapter)) {
551 print_hex_dump(KERN_INFO, "",
552 DUMP_PREFIX_ADDRESS,
553 16, 1,
554 phys_to_virt(buffer_info->dma),
555 IGB_RX_HDR_LEN, true);
556 print_hex_dump(KERN_INFO, "",
557 DUMP_PREFIX_ADDRESS,
558 16, 1,
559 phys_to_virt(
560 buffer_info->page_dma +
561 buffer_info->page_offset),
562 PAGE_SIZE/2, true);
563 }
564 }
565 }
566 }
567
568exit:
569 return;
570}
571
572/**
573 * igb_get_hw_dev - return device
574 * used by hardware layer to print debugging information
575 **/
576struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
577{
578 struct igb_adapter *adapter = hw->back;
579 return adapter->netdev;
580}
581
582/**
583 * igb_init_module - Driver Registration Routine
584 *
585 * igb_init_module is the first routine called when the driver is
586 * loaded. All it does is register with the PCI subsystem.
587 **/
588static int __init igb_init_module(void)
589{
590 int ret;
591 pr_info("%s - version %s\n",
592 igb_driver_string, igb_driver_version);
593
594 pr_info("%s\n", igb_copyright);
595
596#ifdef CONFIG_IGB_DCA
597 dca_register_notify(&dca_notifier);
598#endif
599 ret = pci_register_driver(&igb_driver);
600 return ret;
601}
602
603module_init(igb_init_module);
604
605/**
606 * igb_exit_module - Driver Exit Cleanup Routine
607 *
608 * igb_exit_module is called just before the driver is removed
609 * from memory.
610 **/
611static void __exit igb_exit_module(void)
612{
613#ifdef CONFIG_IGB_DCA
614 dca_unregister_notify(&dca_notifier);
615#endif
616 pci_unregister_driver(&igb_driver);
617}
618
619module_exit(igb_exit_module);
620
621#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
622/**
623 * igb_cache_ring_register - Descriptor ring to register mapping
624 * @adapter: board private structure to initialize
625 *
626 * Once we know the feature-set enabled for the device, we'll cache
627 * the register offset the descriptor ring is assigned to.
628 **/
629static void igb_cache_ring_register(struct igb_adapter *adapter)
630{
631 int i = 0, j = 0;
632 u32 rbase_offset = adapter->vfs_allocated_count;
633
634 switch (adapter->hw.mac.type) {
635 case e1000_82576:
636 /* The queues are allocated for virtualization such that VF 0
637 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
638 * In order to avoid collision we start at the first free queue
639 * and continue consuming queues in the same sequence
640 */
641 if (adapter->vfs_allocated_count) {
642 for (; i < adapter->rss_queues; i++)
643 adapter->rx_ring[i]->reg_idx = rbase_offset +
644 Q_IDX_82576(i);
645 }
646 case e1000_82575:
647 case e1000_82580:
648 case e1000_i350:
649 case e1000_i210:
650 case e1000_i211:
651 default:
652 for (; i < adapter->num_rx_queues; i++)
653 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
654 for (; j < adapter->num_tx_queues; j++)
655 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
656 break;
657 }
658}
659
660static void igb_free_queues(struct igb_adapter *adapter)
661{
662 int i;
663
664 for (i = 0; i < adapter->num_tx_queues; i++) {
665 kfree(adapter->tx_ring[i]);
666 adapter->tx_ring[i] = NULL;
667 }
668 for (i = 0; i < adapter->num_rx_queues; i++) {
669 kfree(adapter->rx_ring[i]);
670 adapter->rx_ring[i] = NULL;
671 }
672 adapter->num_rx_queues = 0;
673 adapter->num_tx_queues = 0;
674}
675
676/**
677 * igb_alloc_queues - Allocate memory for all rings
678 * @adapter: board private structure to initialize
679 *
680 * We allocate one ring per queue at run-time since we don't know the
681 * number of queues at compile-time.
682 **/
683static int igb_alloc_queues(struct igb_adapter *adapter)
684{
685 struct igb_ring *ring;
686 int i;
687 int orig_node = adapter->node;
688
689 for (i = 0; i < adapter->num_tx_queues; i++) {
690 if (orig_node == -1) {
691 int cur_node = next_online_node(adapter->node);
692 if (cur_node == MAX_NUMNODES)
693 cur_node = first_online_node;
694 adapter->node = cur_node;
695 }
696 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
697 adapter->node);
698 if (!ring)
699 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
700 if (!ring)
701 goto err;
702 ring->count = adapter->tx_ring_count;
703 ring->queue_index = i;
704 ring->dev = &adapter->pdev->dev;
705 ring->netdev = adapter->netdev;
706 ring->numa_node = adapter->node;
707 /* For 82575, context index must be unique per ring. */
708 if (adapter->hw.mac.type == e1000_82575)
709 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
710 adapter->tx_ring[i] = ring;
711 }
712 /* Restore the adapter's original node */
713 adapter->node = orig_node;
714
715 for (i = 0; i < adapter->num_rx_queues; i++) {
716 if (orig_node == -1) {
717 int cur_node = next_online_node(adapter->node);
718 if (cur_node == MAX_NUMNODES)
719 cur_node = first_online_node;
720 adapter->node = cur_node;
721 }
722 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
723 adapter->node);
724 if (!ring)
725 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
726 if (!ring)
727 goto err;
728 ring->count = adapter->rx_ring_count;
729 ring->queue_index = i;
730 ring->dev = &adapter->pdev->dev;
731 ring->netdev = adapter->netdev;
732 ring->numa_node = adapter->node;
733 /* set flag indicating ring supports SCTP checksum offload */
734 if (adapter->hw.mac.type >= e1000_82576)
735 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
736
737 /*
738 * On i350, i210, and i211, loopback VLAN packets
739 * have the tag byte-swapped.
740 * */
741 if (adapter->hw.mac.type >= e1000_i350)
742 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
743
744 adapter->rx_ring[i] = ring;
745 }
746 /* Restore the adapter's original node */
747 adapter->node = orig_node;
748
749 igb_cache_ring_register(adapter);
750
751 return 0;
752
753err:
754 /* Restore the adapter's original node */
755 adapter->node = orig_node;
756 igb_free_queues(adapter);
757
758 return -ENOMEM;
759}
760
761/**
762 * igb_write_ivar - configure ivar for given MSI-X vector
763 * @hw: pointer to the HW structure
764 * @msix_vector: vector number we are allocating to a given ring
765 * @index: row index of IVAR register to write within IVAR table
766 * @offset: column offset of in IVAR, should be multiple of 8
767 *
768 * This function is intended to handle the writing of the IVAR register
769 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
770 * each containing an cause allocation for an Rx and Tx ring, and a
771 * variable number of rows depending on the number of queues supported.
772 **/
773static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
774 int index, int offset)
775{
776 u32 ivar = array_rd32(E1000_IVAR0, index);
777
778 /* clear any bits that are currently set */
779 ivar &= ~((u32)0xFF << offset);
780
781 /* write vector and valid bit */
782 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
783
784 array_wr32(E1000_IVAR0, index, ivar);
785}
786
787#define IGB_N0_QUEUE -1
788static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
789{
790 struct igb_adapter *adapter = q_vector->adapter;
791 struct e1000_hw *hw = &adapter->hw;
792 int rx_queue = IGB_N0_QUEUE;
793 int tx_queue = IGB_N0_QUEUE;
794 u32 msixbm = 0;
795
796 if (q_vector->rx.ring)
797 rx_queue = q_vector->rx.ring->reg_idx;
798 if (q_vector->tx.ring)
799 tx_queue = q_vector->tx.ring->reg_idx;
800
801 switch (hw->mac.type) {
802 case e1000_82575:
803 /* The 82575 assigns vectors using a bitmask, which matches the
804 bitmask for the EICR/EIMS/EIMC registers. To assign one
805 or more queues to a vector, we write the appropriate bits
806 into the MSIXBM register for that vector. */
807 if (rx_queue > IGB_N0_QUEUE)
808 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
809 if (tx_queue > IGB_N0_QUEUE)
810 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
811 if (!adapter->msix_entries && msix_vector == 0)
812 msixbm |= E1000_EIMS_OTHER;
813 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
814 q_vector->eims_value = msixbm;
815 break;
816 case e1000_82576:
817 /*
818 * 82576 uses a table that essentially consists of 2 columns
819 * with 8 rows. The ordering is column-major so we use the
820 * lower 3 bits as the row index, and the 4th bit as the
821 * column offset.
822 */
823 if (rx_queue > IGB_N0_QUEUE)
824 igb_write_ivar(hw, msix_vector,
825 rx_queue & 0x7,
826 (rx_queue & 0x8) << 1);
827 if (tx_queue > IGB_N0_QUEUE)
828 igb_write_ivar(hw, msix_vector,
829 tx_queue & 0x7,
830 ((tx_queue & 0x8) << 1) + 8);
831 q_vector->eims_value = 1 << msix_vector;
832 break;
833 case e1000_82580:
834 case e1000_i350:
835 case e1000_i210:
836 case e1000_i211:
837 /*
838 * On 82580 and newer adapters the scheme is similar to 82576
839 * however instead of ordering column-major we have things
840 * ordered row-major. So we traverse the table by using
841 * bit 0 as the column offset, and the remaining bits as the
842 * row index.
843 */
844 if (rx_queue > IGB_N0_QUEUE)
845 igb_write_ivar(hw, msix_vector,
846 rx_queue >> 1,
847 (rx_queue & 0x1) << 4);
848 if (tx_queue > IGB_N0_QUEUE)
849 igb_write_ivar(hw, msix_vector,
850 tx_queue >> 1,
851 ((tx_queue & 0x1) << 4) + 8);
852 q_vector->eims_value = 1 << msix_vector;
853 break;
854 default:
855 BUG();
856 break;
857 }
858
859 /* add q_vector eims value to global eims_enable_mask */
860 adapter->eims_enable_mask |= q_vector->eims_value;
861
862 /* configure q_vector to set itr on first interrupt */
863 q_vector->set_itr = 1;
864}
865
866/**
867 * igb_configure_msix - Configure MSI-X hardware
868 *
869 * igb_configure_msix sets up the hardware to properly
870 * generate MSI-X interrupts.
871 **/
872static void igb_configure_msix(struct igb_adapter *adapter)
873{
874 u32 tmp;
875 int i, vector = 0;
876 struct e1000_hw *hw = &adapter->hw;
877
878 adapter->eims_enable_mask = 0;
879
880 /* set vector for other causes, i.e. link changes */
881 switch (hw->mac.type) {
882 case e1000_82575:
883 tmp = rd32(E1000_CTRL_EXT);
884 /* enable MSI-X PBA support*/
885 tmp |= E1000_CTRL_EXT_PBA_CLR;
886
887 /* Auto-Mask interrupts upon ICR read. */
888 tmp |= E1000_CTRL_EXT_EIAME;
889 tmp |= E1000_CTRL_EXT_IRCA;
890
891 wr32(E1000_CTRL_EXT, tmp);
892
893 /* enable msix_other interrupt */
894 array_wr32(E1000_MSIXBM(0), vector++,
895 E1000_EIMS_OTHER);
896 adapter->eims_other = E1000_EIMS_OTHER;
897
898 break;
899
900 case e1000_82576:
901 case e1000_82580:
902 case e1000_i350:
903 case e1000_i210:
904 case e1000_i211:
905 /* Turn on MSI-X capability first, or our settings
906 * won't stick. And it will take days to debug. */
907 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
908 E1000_GPIE_PBA | E1000_GPIE_EIAME |
909 E1000_GPIE_NSICR);
910
911 /* enable msix_other interrupt */
912 adapter->eims_other = 1 << vector;
913 tmp = (vector++ | E1000_IVAR_VALID) << 8;
914
915 wr32(E1000_IVAR_MISC, tmp);
916 break;
917 default:
918 /* do nothing, since nothing else supports MSI-X */
919 break;
920 } /* switch (hw->mac.type) */
921
922 adapter->eims_enable_mask |= adapter->eims_other;
923
924 for (i = 0; i < adapter->num_q_vectors; i++)
925 igb_assign_vector(adapter->q_vector[i], vector++);
926
927 wrfl();
928}
929
930/**
931 * igb_request_msix - Initialize MSI-X interrupts
932 *
933 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
934 * kernel.
935 **/
936static int igb_request_msix(struct igb_adapter *adapter)
937{
938 struct net_device *netdev = adapter->netdev;
939 struct e1000_hw *hw = &adapter->hw;
940 int i, err = 0, vector = 0;
941
942 err = request_irq(adapter->msix_entries[vector].vector,
943 igb_msix_other, 0, netdev->name, adapter);
944 if (err)
945 goto out;
946 vector++;
947
948 for (i = 0; i < adapter->num_q_vectors; i++) {
949 struct igb_q_vector *q_vector = adapter->q_vector[i];
950
951 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
952
953 if (q_vector->rx.ring && q_vector->tx.ring)
954 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
955 q_vector->rx.ring->queue_index);
956 else if (q_vector->tx.ring)
957 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
958 q_vector->tx.ring->queue_index);
959 else if (q_vector->rx.ring)
960 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
961 q_vector->rx.ring->queue_index);
962 else
963 sprintf(q_vector->name, "%s-unused", netdev->name);
964
965 err = request_irq(adapter->msix_entries[vector].vector,
966 igb_msix_ring, 0, q_vector->name,
967 q_vector);
968 if (err)
969 goto out;
970 vector++;
971 }
972
973 igb_configure_msix(adapter);
974 return 0;
975out:
976 return err;
977}
978
979static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
980{
981 if (adapter->msix_entries) {
982 pci_disable_msix(adapter->pdev);
983 kfree(adapter->msix_entries);
984 adapter->msix_entries = NULL;
985 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
986 pci_disable_msi(adapter->pdev);
987 }
988}
989
990/**
991 * igb_free_q_vectors - Free memory allocated for interrupt vectors
992 * @adapter: board private structure to initialize
993 *
994 * This function frees the memory allocated to the q_vectors. In addition if
995 * NAPI is enabled it will delete any references to the NAPI struct prior
996 * to freeing the q_vector.
997 **/
998static void igb_free_q_vectors(struct igb_adapter *adapter)
999{
1000 int v_idx;
1001
1002 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1003 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1004 adapter->q_vector[v_idx] = NULL;
1005 if (!q_vector)
1006 continue;
1007 netif_napi_del(&q_vector->napi);
1008 kfree(q_vector);
1009 }
1010 adapter->num_q_vectors = 0;
1011}
1012
1013/**
1014 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1015 *
1016 * This function resets the device so that it has 0 rx queues, tx queues, and
1017 * MSI-X interrupts allocated.
1018 */
1019static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1020{
1021 igb_free_queues(adapter);
1022 igb_free_q_vectors(adapter);
1023 igb_reset_interrupt_capability(adapter);
1024}
1025
1026/**
1027 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1028 *
1029 * Attempt to configure interrupts using the best available
1030 * capabilities of the hardware and kernel.
1031 **/
1032static int igb_set_interrupt_capability(struct igb_adapter *adapter)
1033{
1034 int err;
1035 int numvecs, i;
1036
1037 /* Number of supported queues. */
1038 adapter->num_rx_queues = adapter->rss_queues;
1039 if (adapter->vfs_allocated_count)
1040 adapter->num_tx_queues = 1;
1041 else
1042 adapter->num_tx_queues = adapter->rss_queues;
1043
1044 /* start with one vector for every rx queue */
1045 numvecs = adapter->num_rx_queues;
1046
1047 /* if tx handler is separate add 1 for every tx queue */
1048 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1049 numvecs += adapter->num_tx_queues;
1050
1051 /* i210 and i211 can only have 4 MSIX vectors for rx/tx queues. */
1052 if ((adapter->hw.mac.type == e1000_i210)
1053 || (adapter->hw.mac.type == e1000_i211))
1054 numvecs = 4;
1055
1056 /* store the number of vectors reserved for queues */
1057 adapter->num_q_vectors = numvecs;
1058
1059 /* add 1 vector for link status interrupts */
1060 numvecs++;
1061 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1062 GFP_KERNEL);
1063
1064 if (!adapter->msix_entries)
1065 goto msi_only;
1066
1067 for (i = 0; i < numvecs; i++)
1068 adapter->msix_entries[i].entry = i;
1069
1070 err = pci_enable_msix(adapter->pdev,
1071 adapter->msix_entries,
1072 numvecs);
1073 if (err == 0)
1074 goto out;
1075
1076 igb_reset_interrupt_capability(adapter);
1077
1078 /* If we can't do MSI-X, try MSI */
1079msi_only:
1080#ifdef CONFIG_PCI_IOV
1081 /* disable SR-IOV for non MSI-X configurations */
1082 if (adapter->vf_data) {
1083 struct e1000_hw *hw = &adapter->hw;
1084 /* disable iov and allow time for transactions to clear */
1085 pci_disable_sriov(adapter->pdev);
1086 msleep(500);
1087
1088 kfree(adapter->vf_data);
1089 adapter->vf_data = NULL;
1090 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1091 wrfl();
1092 msleep(100);
1093 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1094 }
1095#endif
1096 adapter->vfs_allocated_count = 0;
1097 adapter->rss_queues = 1;
1098 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1099 adapter->num_rx_queues = 1;
1100 adapter->num_tx_queues = 1;
1101 adapter->num_q_vectors = 1;
1102 if (!pci_enable_msi(adapter->pdev))
1103 adapter->flags |= IGB_FLAG_HAS_MSI;
1104out:
1105 /* Notify the stack of the (possibly) reduced queue counts. */
1106 rtnl_lock();
1107 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1108 err = netif_set_real_num_rx_queues(adapter->netdev,
1109 adapter->num_rx_queues);
1110 rtnl_unlock();
1111 return err;
1112}
1113
1114/**
1115 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1116 * @adapter: board private structure to initialize
1117 *
1118 * We allocate one q_vector per queue interrupt. If allocation fails we
1119 * return -ENOMEM.
1120 **/
1121static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1122{
1123 struct igb_q_vector *q_vector;
1124 struct e1000_hw *hw = &adapter->hw;
1125 int v_idx;
1126 int orig_node = adapter->node;
1127
1128 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1129 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1130 adapter->num_tx_queues)) &&
1131 (adapter->num_rx_queues == v_idx))
1132 adapter->node = orig_node;
1133 if (orig_node == -1) {
1134 int cur_node = next_online_node(adapter->node);
1135 if (cur_node == MAX_NUMNODES)
1136 cur_node = first_online_node;
1137 adapter->node = cur_node;
1138 }
1139 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1140 adapter->node);
1141 if (!q_vector)
1142 q_vector = kzalloc(sizeof(struct igb_q_vector),
1143 GFP_KERNEL);
1144 if (!q_vector)
1145 goto err_out;
1146 q_vector->adapter = adapter;
1147 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1148 q_vector->itr_val = IGB_START_ITR;
1149 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1150 adapter->q_vector[v_idx] = q_vector;
1151 }
1152 /* Restore the adapter's original node */
1153 adapter->node = orig_node;
1154
1155 return 0;
1156
1157err_out:
1158 /* Restore the adapter's original node */
1159 adapter->node = orig_node;
1160 igb_free_q_vectors(adapter);
1161 return -ENOMEM;
1162}
1163
1164static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1165 int ring_idx, int v_idx)
1166{
1167 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1168
1169 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1170 q_vector->rx.ring->q_vector = q_vector;
1171 q_vector->rx.count++;
1172 q_vector->itr_val = adapter->rx_itr_setting;
1173 if (q_vector->itr_val && q_vector->itr_val <= 3)
1174 q_vector->itr_val = IGB_START_ITR;
1175}
1176
1177static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1178 int ring_idx, int v_idx)
1179{
1180 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1181
1182 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1183 q_vector->tx.ring->q_vector = q_vector;
1184 q_vector->tx.count++;
1185 q_vector->itr_val = adapter->tx_itr_setting;
1186 q_vector->tx.work_limit = adapter->tx_work_limit;
1187 if (q_vector->itr_val && q_vector->itr_val <= 3)
1188 q_vector->itr_val = IGB_START_ITR;
1189}
1190
1191/**
1192 * igb_map_ring_to_vector - maps allocated queues to vectors
1193 *
1194 * This function maps the recently allocated queues to vectors.
1195 **/
1196static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1197{
1198 int i;
1199 int v_idx = 0;
1200
1201 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1202 (adapter->num_q_vectors < adapter->num_tx_queues))
1203 return -ENOMEM;
1204
1205 if (adapter->num_q_vectors >=
1206 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1207 for (i = 0; i < adapter->num_rx_queues; i++)
1208 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1209 for (i = 0; i < adapter->num_tx_queues; i++)
1210 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1211 } else {
1212 for (i = 0; i < adapter->num_rx_queues; i++) {
1213 if (i < adapter->num_tx_queues)
1214 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1215 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1216 }
1217 for (; i < adapter->num_tx_queues; i++)
1218 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1219 }
1220 return 0;
1221}
1222
1223/**
1224 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1225 *
1226 * This function initializes the interrupts and allocates all of the queues.
1227 **/
1228static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1229{
1230 struct pci_dev *pdev = adapter->pdev;
1231 int err;
1232
1233 err = igb_set_interrupt_capability(adapter);
1234 if (err)
1235 return err;
1236
1237 err = igb_alloc_q_vectors(adapter);
1238 if (err) {
1239 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1240 goto err_alloc_q_vectors;
1241 }
1242
1243 err = igb_alloc_queues(adapter);
1244 if (err) {
1245 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1246 goto err_alloc_queues;
1247 }
1248
1249 err = igb_map_ring_to_vector(adapter);
1250 if (err) {
1251 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1252 goto err_map_queues;
1253 }
1254
1255
1256 return 0;
1257err_map_queues:
1258 igb_free_queues(adapter);
1259err_alloc_queues:
1260 igb_free_q_vectors(adapter);
1261err_alloc_q_vectors:
1262 igb_reset_interrupt_capability(adapter);
1263 return err;
1264}
1265
1266/**
1267 * igb_request_irq - initialize interrupts
1268 *
1269 * Attempts to configure interrupts using the best available
1270 * capabilities of the hardware and kernel.
1271 **/
1272static int igb_request_irq(struct igb_adapter *adapter)
1273{
1274 struct net_device *netdev = adapter->netdev;
1275 struct pci_dev *pdev = adapter->pdev;
1276 int err = 0;
1277
1278 if (adapter->msix_entries) {
1279 err = igb_request_msix(adapter);
1280 if (!err)
1281 goto request_done;
1282 /* fall back to MSI */
1283 igb_clear_interrupt_scheme(adapter);
1284 if (!pci_enable_msi(pdev))
1285 adapter->flags |= IGB_FLAG_HAS_MSI;
1286 igb_free_all_tx_resources(adapter);
1287 igb_free_all_rx_resources(adapter);
1288 adapter->num_tx_queues = 1;
1289 adapter->num_rx_queues = 1;
1290 adapter->num_q_vectors = 1;
1291 err = igb_alloc_q_vectors(adapter);
1292 if (err) {
1293 dev_err(&pdev->dev,
1294 "Unable to allocate memory for vectors\n");
1295 goto request_done;
1296 }
1297 err = igb_alloc_queues(adapter);
1298 if (err) {
1299 dev_err(&pdev->dev,
1300 "Unable to allocate memory for queues\n");
1301 igb_free_q_vectors(adapter);
1302 goto request_done;
1303 }
1304 igb_setup_all_tx_resources(adapter);
1305 igb_setup_all_rx_resources(adapter);
1306 }
1307
1308 igb_assign_vector(adapter->q_vector[0], 0);
1309
1310 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1311 err = request_irq(pdev->irq, igb_intr_msi, 0,
1312 netdev->name, adapter);
1313 if (!err)
1314 goto request_done;
1315
1316 /* fall back to legacy interrupts */
1317 igb_reset_interrupt_capability(adapter);
1318 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1319 }
1320
1321 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1322 netdev->name, adapter);
1323
1324 if (err)
1325 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1326 err);
1327
1328request_done:
1329 return err;
1330}
1331
1332static void igb_free_irq(struct igb_adapter *adapter)
1333{
1334 if (adapter->msix_entries) {
1335 int vector = 0, i;
1336
1337 free_irq(adapter->msix_entries[vector++].vector, adapter);
1338
1339 for (i = 0; i < adapter->num_q_vectors; i++)
1340 free_irq(adapter->msix_entries[vector++].vector,
1341 adapter->q_vector[i]);
1342 } else {
1343 free_irq(adapter->pdev->irq, adapter);
1344 }
1345}
1346
1347/**
1348 * igb_irq_disable - Mask off interrupt generation on the NIC
1349 * @adapter: board private structure
1350 **/
1351static void igb_irq_disable(struct igb_adapter *adapter)
1352{
1353 struct e1000_hw *hw = &adapter->hw;
1354
1355 /*
1356 * we need to be careful when disabling interrupts. The VFs are also
1357 * mapped into these registers and so clearing the bits can cause
1358 * issues on the VF drivers so we only need to clear what we set
1359 */
1360 if (adapter->msix_entries) {
1361 u32 regval = rd32(E1000_EIAM);
1362 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1363 wr32(E1000_EIMC, adapter->eims_enable_mask);
1364 regval = rd32(E1000_EIAC);
1365 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1366 }
1367
1368 wr32(E1000_IAM, 0);
1369 wr32(E1000_IMC, ~0);
1370 wrfl();
1371 if (adapter->msix_entries) {
1372 int i;
1373 for (i = 0; i < adapter->num_q_vectors; i++)
1374 synchronize_irq(adapter->msix_entries[i].vector);
1375 } else {
1376 synchronize_irq(adapter->pdev->irq);
1377 }
1378}
1379
1380/**
1381 * igb_irq_enable - Enable default interrupt generation settings
1382 * @adapter: board private structure
1383 **/
1384static void igb_irq_enable(struct igb_adapter *adapter)
1385{
1386 struct e1000_hw *hw = &adapter->hw;
1387
1388 if (adapter->msix_entries) {
1389 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1390 u32 regval = rd32(E1000_EIAC);
1391 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1392 regval = rd32(E1000_EIAM);
1393 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1394 wr32(E1000_EIMS, adapter->eims_enable_mask);
1395 if (adapter->vfs_allocated_count) {
1396 wr32(E1000_MBVFIMR, 0xFF);
1397 ims |= E1000_IMS_VMMB;
1398 }
1399 wr32(E1000_IMS, ims);
1400 } else {
1401 wr32(E1000_IMS, IMS_ENABLE_MASK |
1402 E1000_IMS_DRSTA);
1403 wr32(E1000_IAM, IMS_ENABLE_MASK |
1404 E1000_IMS_DRSTA);
1405 }
1406}
1407
1408static void igb_update_mng_vlan(struct igb_adapter *adapter)
1409{
1410 struct e1000_hw *hw = &adapter->hw;
1411 u16 vid = adapter->hw.mng_cookie.vlan_id;
1412 u16 old_vid = adapter->mng_vlan_id;
1413
1414 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1415 /* add VID to filter table */
1416 igb_vfta_set(hw, vid, true);
1417 adapter->mng_vlan_id = vid;
1418 } else {
1419 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1420 }
1421
1422 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1423 (vid != old_vid) &&
1424 !test_bit(old_vid, adapter->active_vlans)) {
1425 /* remove VID from filter table */
1426 igb_vfta_set(hw, old_vid, false);
1427 }
1428}
1429
1430/**
1431 * igb_release_hw_control - release control of the h/w to f/w
1432 * @adapter: address of board private structure
1433 *
1434 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1435 * For ASF and Pass Through versions of f/w this means that the
1436 * driver is no longer loaded.
1437 *
1438 **/
1439static void igb_release_hw_control(struct igb_adapter *adapter)
1440{
1441 struct e1000_hw *hw = &adapter->hw;
1442 u32 ctrl_ext;
1443
1444 /* Let firmware take over control of h/w */
1445 ctrl_ext = rd32(E1000_CTRL_EXT);
1446 wr32(E1000_CTRL_EXT,
1447 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1448}
1449
1450/**
1451 * igb_get_hw_control - get control of the h/w from f/w
1452 * @adapter: address of board private structure
1453 *
1454 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1455 * For ASF and Pass Through versions of f/w this means that
1456 * the driver is loaded.
1457 *
1458 **/
1459static void igb_get_hw_control(struct igb_adapter *adapter)
1460{
1461 struct e1000_hw *hw = &adapter->hw;
1462 u32 ctrl_ext;
1463
1464 /* Let firmware know the driver has taken over */
1465 ctrl_ext = rd32(E1000_CTRL_EXT);
1466 wr32(E1000_CTRL_EXT,
1467 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1468}
1469
1470/**
1471 * igb_configure - configure the hardware for RX and TX
1472 * @adapter: private board structure
1473 **/
1474static void igb_configure(struct igb_adapter *adapter)
1475{
1476 struct net_device *netdev = adapter->netdev;
1477 int i;
1478
1479 igb_get_hw_control(adapter);
1480 igb_set_rx_mode(netdev);
1481
1482 igb_restore_vlan(adapter);
1483
1484 igb_setup_tctl(adapter);
1485 igb_setup_mrqc(adapter);
1486 igb_setup_rctl(adapter);
1487
1488 igb_configure_tx(adapter);
1489 igb_configure_rx(adapter);
1490
1491 igb_rx_fifo_flush_82575(&adapter->hw);
1492
1493 /* call igb_desc_unused which always leaves
1494 * at least 1 descriptor unused to make sure
1495 * next_to_use != next_to_clean */
1496 for (i = 0; i < adapter->num_rx_queues; i++) {
1497 struct igb_ring *ring = adapter->rx_ring[i];
1498 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1499 }
1500}
1501
1502/**
1503 * igb_power_up_link - Power up the phy/serdes link
1504 * @adapter: address of board private structure
1505 **/
1506void igb_power_up_link(struct igb_adapter *adapter)
1507{
1508 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1509 igb_power_up_phy_copper(&adapter->hw);
1510 else
1511 igb_power_up_serdes_link_82575(&adapter->hw);
1512 igb_reset_phy(&adapter->hw);
1513}
1514
1515/**
1516 * igb_power_down_link - Power down the phy/serdes link
1517 * @adapter: address of board private structure
1518 */
1519static void igb_power_down_link(struct igb_adapter *adapter)
1520{
1521 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1522 igb_power_down_phy_copper_82575(&adapter->hw);
1523 else
1524 igb_shutdown_serdes_link_82575(&adapter->hw);
1525}
1526
1527/**
1528 * igb_up - Open the interface and prepare it to handle traffic
1529 * @adapter: board private structure
1530 **/
1531int igb_up(struct igb_adapter *adapter)
1532{
1533 struct e1000_hw *hw = &adapter->hw;
1534 int i;
1535
1536 /* hardware has been reset, we need to reload some things */
1537 igb_configure(adapter);
1538
1539 clear_bit(__IGB_DOWN, &adapter->state);
1540
1541 for (i = 0; i < adapter->num_q_vectors; i++)
1542 napi_enable(&(adapter->q_vector[i]->napi));
1543
1544 if (adapter->msix_entries)
1545 igb_configure_msix(adapter);
1546 else
1547 igb_assign_vector(adapter->q_vector[0], 0);
1548
1549 /* Clear any pending interrupts. */
1550 rd32(E1000_ICR);
1551 igb_irq_enable(adapter);
1552
1553 /* notify VFs that reset has been completed */
1554 if (adapter->vfs_allocated_count) {
1555 u32 reg_data = rd32(E1000_CTRL_EXT);
1556 reg_data |= E1000_CTRL_EXT_PFRSTD;
1557 wr32(E1000_CTRL_EXT, reg_data);
1558 }
1559
1560 netif_tx_start_all_queues(adapter->netdev);
1561
1562 /* start the watchdog. */
1563 hw->mac.get_link_status = 1;
1564 schedule_work(&adapter->watchdog_task);
1565
1566 return 0;
1567}
1568
1569void igb_down(struct igb_adapter *adapter)
1570{
1571 struct net_device *netdev = adapter->netdev;
1572 struct e1000_hw *hw = &adapter->hw;
1573 u32 tctl, rctl;
1574 int i;
1575
1576 /* signal that we're down so the interrupt handler does not
1577 * reschedule our watchdog timer */
1578 set_bit(__IGB_DOWN, &adapter->state);
1579
1580 /* disable receives in the hardware */
1581 rctl = rd32(E1000_RCTL);
1582 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1583 /* flush and sleep below */
1584
1585 netif_tx_stop_all_queues(netdev);
1586
1587 /* disable transmits in the hardware */
1588 tctl = rd32(E1000_TCTL);
1589 tctl &= ~E1000_TCTL_EN;
1590 wr32(E1000_TCTL, tctl);
1591 /* flush both disables and wait for them to finish */
1592 wrfl();
1593 msleep(10);
1594
1595 for (i = 0; i < adapter->num_q_vectors; i++)
1596 napi_disable(&(adapter->q_vector[i]->napi));
1597
1598 igb_irq_disable(adapter);
1599
1600 del_timer_sync(&adapter->watchdog_timer);
1601 del_timer_sync(&adapter->phy_info_timer);
1602
1603 netif_carrier_off(netdev);
1604
1605 /* record the stats before reset*/
1606 spin_lock(&adapter->stats64_lock);
1607 igb_update_stats(adapter, &adapter->stats64);
1608 spin_unlock(&adapter->stats64_lock);
1609
1610 adapter->link_speed = 0;
1611 adapter->link_duplex = 0;
1612
1613 if (!pci_channel_offline(adapter->pdev))
1614 igb_reset(adapter);
1615 igb_clean_all_tx_rings(adapter);
1616 igb_clean_all_rx_rings(adapter);
1617#ifdef CONFIG_IGB_DCA
1618
1619 /* since we reset the hardware DCA settings were cleared */
1620 igb_setup_dca(adapter);
1621#endif
1622}
1623
1624void igb_reinit_locked(struct igb_adapter *adapter)
1625{
1626 WARN_ON(in_interrupt());
1627 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1628 msleep(1);
1629 igb_down(adapter);
1630 igb_up(adapter);
1631 clear_bit(__IGB_RESETTING, &adapter->state);
1632}
1633
1634void igb_reset(struct igb_adapter *adapter)
1635{
1636 struct pci_dev *pdev = adapter->pdev;
1637 struct e1000_hw *hw = &adapter->hw;
1638 struct e1000_mac_info *mac = &hw->mac;
1639 struct e1000_fc_info *fc = &hw->fc;
1640 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1641 u16 hwm;
1642
1643 /* Repartition Pba for greater than 9k mtu
1644 * To take effect CTRL.RST is required.
1645 */
1646 switch (mac->type) {
1647 case e1000_i350:
1648 case e1000_82580:
1649 pba = rd32(E1000_RXPBS);
1650 pba = igb_rxpbs_adjust_82580(pba);
1651 break;
1652 case e1000_82576:
1653 pba = rd32(E1000_RXPBS);
1654 pba &= E1000_RXPBS_SIZE_MASK_82576;
1655 break;
1656 case e1000_82575:
1657 case e1000_i210:
1658 case e1000_i211:
1659 default:
1660 pba = E1000_PBA_34K;
1661 break;
1662 }
1663
1664 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1665 (mac->type < e1000_82576)) {
1666 /* adjust PBA for jumbo frames */
1667 wr32(E1000_PBA, pba);
1668
1669 /* To maintain wire speed transmits, the Tx FIFO should be
1670 * large enough to accommodate two full transmit packets,
1671 * rounded up to the next 1KB and expressed in KB. Likewise,
1672 * the Rx FIFO should be large enough to accommodate at least
1673 * one full receive packet and is similarly rounded up and
1674 * expressed in KB. */
1675 pba = rd32(E1000_PBA);
1676 /* upper 16 bits has Tx packet buffer allocation size in KB */
1677 tx_space = pba >> 16;
1678 /* lower 16 bits has Rx packet buffer allocation size in KB */
1679 pba &= 0xffff;
1680 /* the tx fifo also stores 16 bytes of information about the tx
1681 * but don't include ethernet FCS because hardware appends it */
1682 min_tx_space = (adapter->max_frame_size +
1683 sizeof(union e1000_adv_tx_desc) -
1684 ETH_FCS_LEN) * 2;
1685 min_tx_space = ALIGN(min_tx_space, 1024);
1686 min_tx_space >>= 10;
1687 /* software strips receive CRC, so leave room for it */
1688 min_rx_space = adapter->max_frame_size;
1689 min_rx_space = ALIGN(min_rx_space, 1024);
1690 min_rx_space >>= 10;
1691
1692 /* If current Tx allocation is less than the min Tx FIFO size,
1693 * and the min Tx FIFO size is less than the current Rx FIFO
1694 * allocation, take space away from current Rx allocation */
1695 if (tx_space < min_tx_space &&
1696 ((min_tx_space - tx_space) < pba)) {
1697 pba = pba - (min_tx_space - tx_space);
1698
1699 /* if short on rx space, rx wins and must trump tx
1700 * adjustment */
1701 if (pba < min_rx_space)
1702 pba = min_rx_space;
1703 }
1704 wr32(E1000_PBA, pba);
1705 }
1706
1707 /* flow control settings */
1708 /* The high water mark must be low enough to fit one full frame
1709 * (or the size used for early receive) above it in the Rx FIFO.
1710 * Set it to the lower of:
1711 * - 90% of the Rx FIFO size, or
1712 * - the full Rx FIFO size minus one full frame */
1713 hwm = min(((pba << 10) * 9 / 10),
1714 ((pba << 10) - 2 * adapter->max_frame_size));
1715
1716 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1717 fc->low_water = fc->high_water - 16;
1718 fc->pause_time = 0xFFFF;
1719 fc->send_xon = 1;
1720 fc->current_mode = fc->requested_mode;
1721
1722 /* disable receive for all VFs and wait one second */
1723 if (adapter->vfs_allocated_count) {
1724 int i;
1725 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1726 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1727
1728 /* ping all the active vfs to let them know we are going down */
1729 igb_ping_all_vfs(adapter);
1730
1731 /* disable transmits and receives */
1732 wr32(E1000_VFRE, 0);
1733 wr32(E1000_VFTE, 0);
1734 }
1735
1736 /* Allow time for pending master requests to run */
1737 hw->mac.ops.reset_hw(hw);
1738 wr32(E1000_WUC, 0);
1739
1740 if (hw->mac.ops.init_hw(hw))
1741 dev_err(&pdev->dev, "Hardware Error\n");
1742
1743 /*
1744 * Flow control settings reset on hardware reset, so guarantee flow
1745 * control is off when forcing speed.
1746 */
1747 if (!hw->mac.autoneg)
1748 igb_force_mac_fc(hw);
1749
1750 igb_init_dmac(adapter, pba);
1751 if (!netif_running(adapter->netdev))
1752 igb_power_down_link(adapter);
1753
1754 igb_update_mng_vlan(adapter);
1755
1756 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1757 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1758
1759 igb_get_phy_info(hw);
1760}
1761
1762static netdev_features_t igb_fix_features(struct net_device *netdev,
1763 netdev_features_t features)
1764{
1765 /*
1766 * Since there is no support for separate rx/tx vlan accel
1767 * enable/disable make sure tx flag is always in same state as rx.
1768 */
1769 if (features & NETIF_F_HW_VLAN_RX)
1770 features |= NETIF_F_HW_VLAN_TX;
1771 else
1772 features &= ~NETIF_F_HW_VLAN_TX;
1773
1774 return features;
1775}
1776
1777static int igb_set_features(struct net_device *netdev,
1778 netdev_features_t features)
1779{
1780 netdev_features_t changed = netdev->features ^ features;
1781 struct igb_adapter *adapter = netdev_priv(netdev);
1782
1783 if (changed & NETIF_F_HW_VLAN_RX)
1784 igb_vlan_mode(netdev, features);
1785
1786 if (!(changed & NETIF_F_RXALL))
1787 return 0;
1788
1789 netdev->features = features;
1790
1791 if (netif_running(netdev))
1792 igb_reinit_locked(adapter);
1793 else
1794 igb_reset(adapter);
1795
1796 return 0;
1797}
1798
1799static const struct net_device_ops igb_netdev_ops = {
1800 .ndo_open = igb_open,
1801 .ndo_stop = igb_close,
1802 .ndo_start_xmit = igb_xmit_frame,
1803 .ndo_get_stats64 = igb_get_stats64,
1804 .ndo_set_rx_mode = igb_set_rx_mode,
1805 .ndo_set_mac_address = igb_set_mac,
1806 .ndo_change_mtu = igb_change_mtu,
1807 .ndo_do_ioctl = igb_ioctl,
1808 .ndo_tx_timeout = igb_tx_timeout,
1809 .ndo_validate_addr = eth_validate_addr,
1810 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1811 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1812 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1813 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1814 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1815 .ndo_get_vf_config = igb_ndo_get_vf_config,
1816#ifdef CONFIG_NET_POLL_CONTROLLER
1817 .ndo_poll_controller = igb_netpoll,
1818#endif
1819 .ndo_fix_features = igb_fix_features,
1820 .ndo_set_features = igb_set_features,
1821};
1822
1823/**
1824 * igb_probe - Device Initialization Routine
1825 * @pdev: PCI device information struct
1826 * @ent: entry in igb_pci_tbl
1827 *
1828 * Returns 0 on success, negative on failure
1829 *
1830 * igb_probe initializes an adapter identified by a pci_dev structure.
1831 * The OS initialization, configuring of the adapter private structure,
1832 * and a hardware reset occur.
1833 **/
1834static int __devinit igb_probe(struct pci_dev *pdev,
1835 const struct pci_device_id *ent)
1836{
1837 struct net_device *netdev;
1838 struct igb_adapter *adapter;
1839 struct e1000_hw *hw;
1840 u16 eeprom_data = 0;
1841 s32 ret_val;
1842 static int global_quad_port_a; /* global quad port a indication */
1843 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1844 unsigned long mmio_start, mmio_len;
1845 int err, pci_using_dac;
1846 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1847 u8 part_str[E1000_PBANUM_LENGTH];
1848
1849 /* Catch broken hardware that put the wrong VF device ID in
1850 * the PCIe SR-IOV capability.
1851 */
1852 if (pdev->is_virtfn) {
1853 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1854 pci_name(pdev), pdev->vendor, pdev->device);
1855 return -EINVAL;
1856 }
1857
1858 err = pci_enable_device_mem(pdev);
1859 if (err)
1860 return err;
1861
1862 pci_using_dac = 0;
1863 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1864 if (!err) {
1865 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1866 if (!err)
1867 pci_using_dac = 1;
1868 } else {
1869 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1870 if (err) {
1871 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1872 if (err) {
1873 dev_err(&pdev->dev, "No usable DMA "
1874 "configuration, aborting\n");
1875 goto err_dma;
1876 }
1877 }
1878 }
1879
1880 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1881 IORESOURCE_MEM),
1882 igb_driver_name);
1883 if (err)
1884 goto err_pci_reg;
1885
1886 pci_enable_pcie_error_reporting(pdev);
1887
1888 pci_set_master(pdev);
1889 pci_save_state(pdev);
1890
1891 err = -ENOMEM;
1892 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1893 IGB_MAX_TX_QUEUES);
1894 if (!netdev)
1895 goto err_alloc_etherdev;
1896
1897 SET_NETDEV_DEV(netdev, &pdev->dev);
1898
1899 pci_set_drvdata(pdev, netdev);
1900 adapter = netdev_priv(netdev);
1901 adapter->netdev = netdev;
1902 adapter->pdev = pdev;
1903 hw = &adapter->hw;
1904 hw->back = adapter;
1905 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
1906
1907 mmio_start = pci_resource_start(pdev, 0);
1908 mmio_len = pci_resource_len(pdev, 0);
1909
1910 err = -EIO;
1911 hw->hw_addr = ioremap(mmio_start, mmio_len);
1912 if (!hw->hw_addr)
1913 goto err_ioremap;
1914
1915 netdev->netdev_ops = &igb_netdev_ops;
1916 igb_set_ethtool_ops(netdev);
1917 netdev->watchdog_timeo = 5 * HZ;
1918
1919 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1920
1921 netdev->mem_start = mmio_start;
1922 netdev->mem_end = mmio_start + mmio_len;
1923
1924 /* PCI config space info */
1925 hw->vendor_id = pdev->vendor;
1926 hw->device_id = pdev->device;
1927 hw->revision_id = pdev->revision;
1928 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1929 hw->subsystem_device_id = pdev->subsystem_device;
1930
1931 /* Copy the default MAC, PHY and NVM function pointers */
1932 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1933 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1934 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1935 /* Initialize skew-specific constants */
1936 err = ei->get_invariants(hw);
1937 if (err)
1938 goto err_sw_init;
1939
1940 /* setup the private structure */
1941 err = igb_sw_init(adapter);
1942 if (err)
1943 goto err_sw_init;
1944
1945 igb_get_bus_info_pcie(hw);
1946
1947 hw->phy.autoneg_wait_to_complete = false;
1948
1949 /* Copper options */
1950 if (hw->phy.media_type == e1000_media_type_copper) {
1951 hw->phy.mdix = AUTO_ALL_MODES;
1952 hw->phy.disable_polarity_correction = false;
1953 hw->phy.ms_type = e1000_ms_hw_default;
1954 }
1955
1956 if (igb_check_reset_block(hw))
1957 dev_info(&pdev->dev,
1958 "PHY reset is blocked due to SOL/IDER session.\n");
1959
1960 /*
1961 * features is initialized to 0 in allocation, it might have bits
1962 * set by igb_sw_init so we should use an or instead of an
1963 * assignment.
1964 */
1965 netdev->features |= NETIF_F_SG |
1966 NETIF_F_IP_CSUM |
1967 NETIF_F_IPV6_CSUM |
1968 NETIF_F_TSO |
1969 NETIF_F_TSO6 |
1970 NETIF_F_RXHASH |
1971 NETIF_F_RXCSUM |
1972 NETIF_F_HW_VLAN_RX |
1973 NETIF_F_HW_VLAN_TX;
1974
1975 /* copy netdev features into list of user selectable features */
1976 netdev->hw_features |= netdev->features;
1977 netdev->hw_features |= NETIF_F_RXALL;
1978
1979 /* set this bit last since it cannot be part of hw_features */
1980 netdev->features |= NETIF_F_HW_VLAN_FILTER;
1981
1982 netdev->vlan_features |= NETIF_F_TSO |
1983 NETIF_F_TSO6 |
1984 NETIF_F_IP_CSUM |
1985 NETIF_F_IPV6_CSUM |
1986 NETIF_F_SG;
1987
1988 netdev->priv_flags |= IFF_SUPP_NOFCS;
1989
1990 if (pci_using_dac) {
1991 netdev->features |= NETIF_F_HIGHDMA;
1992 netdev->vlan_features |= NETIF_F_HIGHDMA;
1993 }
1994
1995 if (hw->mac.type >= e1000_82576) {
1996 netdev->hw_features |= NETIF_F_SCTP_CSUM;
1997 netdev->features |= NETIF_F_SCTP_CSUM;
1998 }
1999
2000 netdev->priv_flags |= IFF_UNICAST_FLT;
2001
2002 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
2003
2004 /* before reading the NVM, reset the controller to put the device in a
2005 * known good starting state */
2006 hw->mac.ops.reset_hw(hw);
2007
2008 /*
2009 * make sure the NVM is good , i211 parts have special NVM that
2010 * doesn't contain a checksum
2011 */
2012 if (hw->mac.type != e1000_i211) {
2013 if (hw->nvm.ops.validate(hw) < 0) {
2014 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2015 err = -EIO;
2016 goto err_eeprom;
2017 }
2018 }
2019
2020 /* copy the MAC address out of the NVM */
2021 if (hw->mac.ops.read_mac_addr(hw))
2022 dev_err(&pdev->dev, "NVM Read Error\n");
2023
2024 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2025 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2026
2027 if (!is_valid_ether_addr(netdev->perm_addr)) {
2028 dev_err(&pdev->dev, "Invalid MAC Address\n");
2029 err = -EIO;
2030 goto err_eeprom;
2031 }
2032
2033 setup_timer(&adapter->watchdog_timer, igb_watchdog,
2034 (unsigned long) adapter);
2035 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2036 (unsigned long) adapter);
2037
2038 INIT_WORK(&adapter->reset_task, igb_reset_task);
2039 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2040
2041 /* Initialize link properties that are user-changeable */
2042 adapter->fc_autoneg = true;
2043 hw->mac.autoneg = true;
2044 hw->phy.autoneg_advertised = 0x2f;
2045
2046 hw->fc.requested_mode = e1000_fc_default;
2047 hw->fc.current_mode = e1000_fc_default;
2048
2049 igb_validate_mdi_setting(hw);
2050
2051 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2052 * enable the ACPI Magic Packet filter
2053 */
2054
2055 if (hw->bus.func == 0)
2056 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2057 else if (hw->mac.type >= e1000_82580)
2058 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2059 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2060 &eeprom_data);
2061 else if (hw->bus.func == 1)
2062 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2063
2064 if (eeprom_data & eeprom_apme_mask)
2065 adapter->eeprom_wol |= E1000_WUFC_MAG;
2066
2067 /* now that we have the eeprom settings, apply the special cases where
2068 * the eeprom may be wrong or the board simply won't support wake on
2069 * lan on a particular port */
2070 switch (pdev->device) {
2071 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2072 adapter->eeprom_wol = 0;
2073 break;
2074 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2075 case E1000_DEV_ID_82576_FIBER:
2076 case E1000_DEV_ID_82576_SERDES:
2077 /* Wake events only supported on port A for dual fiber
2078 * regardless of eeprom setting */
2079 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2080 adapter->eeprom_wol = 0;
2081 break;
2082 case E1000_DEV_ID_82576_QUAD_COPPER:
2083 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2084 /* if quad port adapter, disable WoL on all but port A */
2085 if (global_quad_port_a != 0)
2086 adapter->eeprom_wol = 0;
2087 else
2088 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2089 /* Reset for multiple quad port adapters */
2090 if (++global_quad_port_a == 4)
2091 global_quad_port_a = 0;
2092 break;
2093 }
2094
2095 /* initialize the wol settings based on the eeprom settings */
2096 adapter->wol = adapter->eeprom_wol;
2097 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
2098
2099 /* reset the hardware with the new settings */
2100 igb_reset(adapter);
2101
2102 /* let the f/w know that the h/w is now under the control of the
2103 * driver. */
2104 igb_get_hw_control(adapter);
2105
2106 strcpy(netdev->name, "eth%d");
2107 err = register_netdev(netdev);
2108 if (err)
2109 goto err_register;
2110
2111 /* carrier off reporting is important to ethtool even BEFORE open */
2112 netif_carrier_off(netdev);
2113
2114#ifdef CONFIG_IGB_DCA
2115 if (dca_add_requester(&pdev->dev) == 0) {
2116 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2117 dev_info(&pdev->dev, "DCA enabled\n");
2118 igb_setup_dca(adapter);
2119 }
2120
2121#endif
2122#ifdef CONFIG_IGB_PTP
2123 /* do hw tstamp init after resetting */
2124 igb_ptp_init(adapter);
2125
2126#endif
2127 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2128 /* print bus type/speed/width info */
2129 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
2130 netdev->name,
2131 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
2132 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
2133 "unknown"),
2134 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2135 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2136 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2137 "unknown"),
2138 netdev->dev_addr);
2139
2140 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2141 if (ret_val)
2142 strcpy(part_str, "Unknown");
2143 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2144 dev_info(&pdev->dev,
2145 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2146 adapter->msix_entries ? "MSI-X" :
2147 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2148 adapter->num_rx_queues, adapter->num_tx_queues);
2149 switch (hw->mac.type) {
2150 case e1000_i350:
2151 case e1000_i210:
2152 case e1000_i211:
2153 igb_set_eee_i350(hw);
2154 break;
2155 default:
2156 break;
2157 }
2158
2159 pm_runtime_put_noidle(&pdev->dev);
2160 return 0;
2161
2162err_register:
2163 igb_release_hw_control(adapter);
2164err_eeprom:
2165 if (!igb_check_reset_block(hw))
2166 igb_reset_phy(hw);
2167
2168 if (hw->flash_address)
2169 iounmap(hw->flash_address);
2170err_sw_init:
2171 igb_clear_interrupt_scheme(adapter);
2172 iounmap(hw->hw_addr);
2173err_ioremap:
2174 free_netdev(netdev);
2175err_alloc_etherdev:
2176 pci_release_selected_regions(pdev,
2177 pci_select_bars(pdev, IORESOURCE_MEM));
2178err_pci_reg:
2179err_dma:
2180 pci_disable_device(pdev);
2181 return err;
2182}
2183
2184/**
2185 * igb_remove - Device Removal Routine
2186 * @pdev: PCI device information struct
2187 *
2188 * igb_remove is called by the PCI subsystem to alert the driver
2189 * that it should release a PCI device. The could be caused by a
2190 * Hot-Plug event, or because the driver is going to be removed from
2191 * memory.
2192 **/
2193static void __devexit igb_remove(struct pci_dev *pdev)
2194{
2195 struct net_device *netdev = pci_get_drvdata(pdev);
2196 struct igb_adapter *adapter = netdev_priv(netdev);
2197 struct e1000_hw *hw = &adapter->hw;
2198
2199 pm_runtime_get_noresume(&pdev->dev);
2200#ifdef CONFIG_IGB_PTP
2201 igb_ptp_remove(adapter);
2202
2203#endif
2204 /*
2205 * The watchdog timer may be rescheduled, so explicitly
2206 * disable watchdog from being rescheduled.
2207 */
2208 set_bit(__IGB_DOWN, &adapter->state);
2209 del_timer_sync(&adapter->watchdog_timer);
2210 del_timer_sync(&adapter->phy_info_timer);
2211
2212 cancel_work_sync(&adapter->reset_task);
2213 cancel_work_sync(&adapter->watchdog_task);
2214
2215#ifdef CONFIG_IGB_DCA
2216 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2217 dev_info(&pdev->dev, "DCA disabled\n");
2218 dca_remove_requester(&pdev->dev);
2219 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2220 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2221 }
2222#endif
2223
2224 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2225 * would have already happened in close and is redundant. */
2226 igb_release_hw_control(adapter);
2227
2228 unregister_netdev(netdev);
2229
2230 igb_clear_interrupt_scheme(adapter);
2231
2232#ifdef CONFIG_PCI_IOV
2233 /* reclaim resources allocated to VFs */
2234 if (adapter->vf_data) {
2235 /* disable iov and allow time for transactions to clear */
2236 if (!igb_check_vf_assignment(adapter)) {
2237 pci_disable_sriov(pdev);
2238 msleep(500);
2239 } else {
2240 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2241 }
2242
2243 kfree(adapter->vf_data);
2244 adapter->vf_data = NULL;
2245 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2246 wrfl();
2247 msleep(100);
2248 dev_info(&pdev->dev, "IOV Disabled\n");
2249 }
2250#endif
2251
2252 iounmap(hw->hw_addr);
2253 if (hw->flash_address)
2254 iounmap(hw->flash_address);
2255 pci_release_selected_regions(pdev,
2256 pci_select_bars(pdev, IORESOURCE_MEM));
2257
2258 kfree(adapter->shadow_vfta);
2259 free_netdev(netdev);
2260
2261 pci_disable_pcie_error_reporting(pdev);
2262
2263 pci_disable_device(pdev);
2264}
2265
2266/**
2267 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2268 * @adapter: board private structure to initialize
2269 *
2270 * This function initializes the vf specific data storage and then attempts to
2271 * allocate the VFs. The reason for ordering it this way is because it is much
2272 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2273 * the memory for the VFs.
2274 **/
2275static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2276{
2277#ifdef CONFIG_PCI_IOV
2278 struct pci_dev *pdev = adapter->pdev;
2279 struct e1000_hw *hw = &adapter->hw;
2280 int old_vfs = igb_find_enabled_vfs(adapter);
2281 int i;
2282
2283 /* Virtualization features not supported on i210 family. */
2284 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2285 return;
2286
2287 if (old_vfs) {
2288 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2289 "max_vfs setting of %d\n", old_vfs, max_vfs);
2290 adapter->vfs_allocated_count = old_vfs;
2291 }
2292
2293 if (!adapter->vfs_allocated_count)
2294 return;
2295
2296 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2297 sizeof(struct vf_data_storage), GFP_KERNEL);
2298
2299 /* if allocation failed then we do not support SR-IOV */
2300 if (!adapter->vf_data) {
2301 adapter->vfs_allocated_count = 0;
2302 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2303 "Data Storage\n");
2304 goto out;
2305 }
2306
2307 if (!old_vfs) {
2308 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2309 goto err_out;
2310 }
2311 dev_info(&pdev->dev, "%d VFs allocated\n",
2312 adapter->vfs_allocated_count);
2313 for (i = 0; i < adapter->vfs_allocated_count; i++)
2314 igb_vf_configure(adapter, i);
2315
2316 /* DMA Coalescing is not supported in IOV mode. */
2317 adapter->flags &= ~IGB_FLAG_DMAC;
2318 goto out;
2319err_out:
2320 kfree(adapter->vf_data);
2321 adapter->vf_data = NULL;
2322 adapter->vfs_allocated_count = 0;
2323out:
2324 return;
2325#endif /* CONFIG_PCI_IOV */
2326}
2327
2328/**
2329 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2330 * @adapter: board private structure to initialize
2331 *
2332 * igb_sw_init initializes the Adapter private data structure.
2333 * Fields are initialized based on PCI device information and
2334 * OS network device settings (MTU size).
2335 **/
2336static int __devinit igb_sw_init(struct igb_adapter *adapter)
2337{
2338 struct e1000_hw *hw = &adapter->hw;
2339 struct net_device *netdev = adapter->netdev;
2340 struct pci_dev *pdev = adapter->pdev;
2341
2342 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2343
2344 /* set default ring sizes */
2345 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2346 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2347
2348 /* set default ITR values */
2349 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2350 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2351
2352 /* set default work limits */
2353 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2354
2355 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2356 VLAN_HLEN;
2357 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2358
2359 adapter->node = -1;
2360
2361 spin_lock_init(&adapter->stats64_lock);
2362#ifdef CONFIG_PCI_IOV
2363 switch (hw->mac.type) {
2364 case e1000_82576:
2365 case e1000_i350:
2366 if (max_vfs > 7) {
2367 dev_warn(&pdev->dev,
2368 "Maximum of 7 VFs per PF, using max\n");
2369 adapter->vfs_allocated_count = 7;
2370 } else
2371 adapter->vfs_allocated_count = max_vfs;
2372 break;
2373 case e1000_i210:
2374 case e1000_i211:
2375 adapter->vfs_allocated_count = 0;
2376 break;
2377 default:
2378 break;
2379 }
2380#endif /* CONFIG_PCI_IOV */
2381 switch (hw->mac.type) {
2382 case e1000_i210:
2383 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES_I210,
2384 num_online_cpus());
2385 break;
2386 case e1000_i211:
2387 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES_I211,
2388 num_online_cpus());
2389 break;
2390 default:
2391 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES,
2392 num_online_cpus());
2393 break;
2394 }
2395 /* i350 cannot do RSS and SR-IOV at the same time */
2396 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2397 adapter->rss_queues = 1;
2398
2399 /*
2400 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2401 * then we should combine the queues into a queue pair in order to
2402 * conserve interrupts due to limited supply
2403 */
2404 if ((adapter->rss_queues > 4) ||
2405 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2406 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2407
2408 /* Setup and initialize a copy of the hw vlan table array */
2409 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2410 E1000_VLAN_FILTER_TBL_SIZE,
2411 GFP_ATOMIC);
2412
2413 /* This call may decrease the number of queues */
2414 if (igb_init_interrupt_scheme(adapter)) {
2415 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2416 return -ENOMEM;
2417 }
2418
2419 igb_probe_vfs(adapter);
2420
2421 /* Explicitly disable IRQ since the NIC can be in any state. */
2422 igb_irq_disable(adapter);
2423
2424 if (hw->mac.type >= e1000_i350)
2425 adapter->flags &= ~IGB_FLAG_DMAC;
2426
2427 set_bit(__IGB_DOWN, &adapter->state);
2428 return 0;
2429}
2430
2431/**
2432 * igb_open - Called when a network interface is made active
2433 * @netdev: network interface device structure
2434 *
2435 * Returns 0 on success, negative value on failure
2436 *
2437 * The open entry point is called when a network interface is made
2438 * active by the system (IFF_UP). At this point all resources needed
2439 * for transmit and receive operations are allocated, the interrupt
2440 * handler is registered with the OS, the watchdog timer is started,
2441 * and the stack is notified that the interface is ready.
2442 **/
2443static int __igb_open(struct net_device *netdev, bool resuming)
2444{
2445 struct igb_adapter *adapter = netdev_priv(netdev);
2446 struct e1000_hw *hw = &adapter->hw;
2447 struct pci_dev *pdev = adapter->pdev;
2448 int err;
2449 int i;
2450
2451 /* disallow open during test */
2452 if (test_bit(__IGB_TESTING, &adapter->state)) {
2453 WARN_ON(resuming);
2454 return -EBUSY;
2455 }
2456
2457 if (!resuming)
2458 pm_runtime_get_sync(&pdev->dev);
2459
2460 netif_carrier_off(netdev);
2461
2462 /* allocate transmit descriptors */
2463 err = igb_setup_all_tx_resources(adapter);
2464 if (err)
2465 goto err_setup_tx;
2466
2467 /* allocate receive descriptors */
2468 err = igb_setup_all_rx_resources(adapter);
2469 if (err)
2470 goto err_setup_rx;
2471
2472 igb_power_up_link(adapter);
2473
2474 /* before we allocate an interrupt, we must be ready to handle it.
2475 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2476 * as soon as we call pci_request_irq, so we have to setup our
2477 * clean_rx handler before we do so. */
2478 igb_configure(adapter);
2479
2480 err = igb_request_irq(adapter);
2481 if (err)
2482 goto err_req_irq;
2483
2484 /* From here on the code is the same as igb_up() */
2485 clear_bit(__IGB_DOWN, &adapter->state);
2486
2487 for (i = 0; i < adapter->num_q_vectors; i++)
2488 napi_enable(&(adapter->q_vector[i]->napi));
2489
2490 /* Clear any pending interrupts. */
2491 rd32(E1000_ICR);
2492
2493 igb_irq_enable(adapter);
2494
2495 /* notify VFs that reset has been completed */
2496 if (adapter->vfs_allocated_count) {
2497 u32 reg_data = rd32(E1000_CTRL_EXT);
2498 reg_data |= E1000_CTRL_EXT_PFRSTD;
2499 wr32(E1000_CTRL_EXT, reg_data);
2500 }
2501
2502 netif_tx_start_all_queues(netdev);
2503
2504 if (!resuming)
2505 pm_runtime_put(&pdev->dev);
2506
2507 /* start the watchdog. */
2508 hw->mac.get_link_status = 1;
2509 schedule_work(&adapter->watchdog_task);
2510
2511 return 0;
2512
2513err_req_irq:
2514 igb_release_hw_control(adapter);
2515 igb_power_down_link(adapter);
2516 igb_free_all_rx_resources(adapter);
2517err_setup_rx:
2518 igb_free_all_tx_resources(adapter);
2519err_setup_tx:
2520 igb_reset(adapter);
2521 if (!resuming)
2522 pm_runtime_put(&pdev->dev);
2523
2524 return err;
2525}
2526
2527static int igb_open(struct net_device *netdev)
2528{
2529 return __igb_open(netdev, false);
2530}
2531
2532/**
2533 * igb_close - Disables a network interface
2534 * @netdev: network interface device structure
2535 *
2536 * Returns 0, this is not allowed to fail
2537 *
2538 * The close entry point is called when an interface is de-activated
2539 * by the OS. The hardware is still under the driver's control, but
2540 * needs to be disabled. A global MAC reset is issued to stop the
2541 * hardware, and all transmit and receive resources are freed.
2542 **/
2543static int __igb_close(struct net_device *netdev, bool suspending)
2544{
2545 struct igb_adapter *adapter = netdev_priv(netdev);
2546 struct pci_dev *pdev = adapter->pdev;
2547
2548 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2549
2550 if (!suspending)
2551 pm_runtime_get_sync(&pdev->dev);
2552
2553 igb_down(adapter);
2554 igb_free_irq(adapter);
2555
2556 igb_free_all_tx_resources(adapter);
2557 igb_free_all_rx_resources(adapter);
2558
2559 if (!suspending)
2560 pm_runtime_put_sync(&pdev->dev);
2561 return 0;
2562}
2563
2564static int igb_close(struct net_device *netdev)
2565{
2566 return __igb_close(netdev, false);
2567}
2568
2569/**
2570 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2571 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2572 *
2573 * Return 0 on success, negative on failure
2574 **/
2575int igb_setup_tx_resources(struct igb_ring *tx_ring)
2576{
2577 struct device *dev = tx_ring->dev;
2578 int orig_node = dev_to_node(dev);
2579 int size;
2580
2581 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2582 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2583 if (!tx_ring->tx_buffer_info)
2584 tx_ring->tx_buffer_info = vzalloc(size);
2585 if (!tx_ring->tx_buffer_info)
2586 goto err;
2587
2588 /* round up to nearest 4K */
2589 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2590 tx_ring->size = ALIGN(tx_ring->size, 4096);
2591
2592 set_dev_node(dev, tx_ring->numa_node);
2593 tx_ring->desc = dma_alloc_coherent(dev,
2594 tx_ring->size,
2595 &tx_ring->dma,
2596 GFP_KERNEL);
2597 set_dev_node(dev, orig_node);
2598 if (!tx_ring->desc)
2599 tx_ring->desc = dma_alloc_coherent(dev,
2600 tx_ring->size,
2601 &tx_ring->dma,
2602 GFP_KERNEL);
2603
2604 if (!tx_ring->desc)
2605 goto err;
2606
2607 tx_ring->next_to_use = 0;
2608 tx_ring->next_to_clean = 0;
2609
2610 return 0;
2611
2612err:
2613 vfree(tx_ring->tx_buffer_info);
2614 dev_err(dev,
2615 "Unable to allocate memory for the transmit descriptor ring\n");
2616 return -ENOMEM;
2617}
2618
2619/**
2620 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2621 * (Descriptors) for all queues
2622 * @adapter: board private structure
2623 *
2624 * Return 0 on success, negative on failure
2625 **/
2626static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2627{
2628 struct pci_dev *pdev = adapter->pdev;
2629 int i, err = 0;
2630
2631 for (i = 0; i < adapter->num_tx_queues; i++) {
2632 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2633 if (err) {
2634 dev_err(&pdev->dev,
2635 "Allocation for Tx Queue %u failed\n", i);
2636 for (i--; i >= 0; i--)
2637 igb_free_tx_resources(adapter->tx_ring[i]);
2638 break;
2639 }
2640 }
2641
2642 return err;
2643}
2644
2645/**
2646 * igb_setup_tctl - configure the transmit control registers
2647 * @adapter: Board private structure
2648 **/
2649void igb_setup_tctl(struct igb_adapter *adapter)
2650{
2651 struct e1000_hw *hw = &adapter->hw;
2652 u32 tctl;
2653
2654 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2655 wr32(E1000_TXDCTL(0), 0);
2656
2657 /* Program the Transmit Control Register */
2658 tctl = rd32(E1000_TCTL);
2659 tctl &= ~E1000_TCTL_CT;
2660 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2661 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2662
2663 igb_config_collision_dist(hw);
2664
2665 /* Enable transmits */
2666 tctl |= E1000_TCTL_EN;
2667
2668 wr32(E1000_TCTL, tctl);
2669}
2670
2671/**
2672 * igb_configure_tx_ring - Configure transmit ring after Reset
2673 * @adapter: board private structure
2674 * @ring: tx ring to configure
2675 *
2676 * Configure a transmit ring after a reset.
2677 **/
2678void igb_configure_tx_ring(struct igb_adapter *adapter,
2679 struct igb_ring *ring)
2680{
2681 struct e1000_hw *hw = &adapter->hw;
2682 u32 txdctl = 0;
2683 u64 tdba = ring->dma;
2684 int reg_idx = ring->reg_idx;
2685
2686 /* disable the queue */
2687 wr32(E1000_TXDCTL(reg_idx), 0);
2688 wrfl();
2689 mdelay(10);
2690
2691 wr32(E1000_TDLEN(reg_idx),
2692 ring->count * sizeof(union e1000_adv_tx_desc));
2693 wr32(E1000_TDBAL(reg_idx),
2694 tdba & 0x00000000ffffffffULL);
2695 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2696
2697 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2698 wr32(E1000_TDH(reg_idx), 0);
2699 writel(0, ring->tail);
2700
2701 txdctl |= IGB_TX_PTHRESH;
2702 txdctl |= IGB_TX_HTHRESH << 8;
2703 txdctl |= IGB_TX_WTHRESH << 16;
2704
2705 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2706 wr32(E1000_TXDCTL(reg_idx), txdctl);
2707}
2708
2709/**
2710 * igb_configure_tx - Configure transmit Unit after Reset
2711 * @adapter: board private structure
2712 *
2713 * Configure the Tx unit of the MAC after a reset.
2714 **/
2715static void igb_configure_tx(struct igb_adapter *adapter)
2716{
2717 int i;
2718
2719 for (i = 0; i < adapter->num_tx_queues; i++)
2720 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2721}
2722
2723/**
2724 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2725 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2726 *
2727 * Returns 0 on success, negative on failure
2728 **/
2729int igb_setup_rx_resources(struct igb_ring *rx_ring)
2730{
2731 struct device *dev = rx_ring->dev;
2732 int orig_node = dev_to_node(dev);
2733 int size, desc_len;
2734
2735 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
2736 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2737 if (!rx_ring->rx_buffer_info)
2738 rx_ring->rx_buffer_info = vzalloc(size);
2739 if (!rx_ring->rx_buffer_info)
2740 goto err;
2741
2742 desc_len = sizeof(union e1000_adv_rx_desc);
2743
2744 /* Round up to nearest 4K */
2745 rx_ring->size = rx_ring->count * desc_len;
2746 rx_ring->size = ALIGN(rx_ring->size, 4096);
2747
2748 set_dev_node(dev, rx_ring->numa_node);
2749 rx_ring->desc = dma_alloc_coherent(dev,
2750 rx_ring->size,
2751 &rx_ring->dma,
2752 GFP_KERNEL);
2753 set_dev_node(dev, orig_node);
2754 if (!rx_ring->desc)
2755 rx_ring->desc = dma_alloc_coherent(dev,
2756 rx_ring->size,
2757 &rx_ring->dma,
2758 GFP_KERNEL);
2759
2760 if (!rx_ring->desc)
2761 goto err;
2762
2763 rx_ring->next_to_clean = 0;
2764 rx_ring->next_to_use = 0;
2765
2766 return 0;
2767
2768err:
2769 vfree(rx_ring->rx_buffer_info);
2770 rx_ring->rx_buffer_info = NULL;
2771 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2772 " ring\n");
2773 return -ENOMEM;
2774}
2775
2776/**
2777 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2778 * (Descriptors) for all queues
2779 * @adapter: board private structure
2780 *
2781 * Return 0 on success, negative on failure
2782 **/
2783static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2784{
2785 struct pci_dev *pdev = adapter->pdev;
2786 int i, err = 0;
2787
2788 for (i = 0; i < adapter->num_rx_queues; i++) {
2789 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2790 if (err) {
2791 dev_err(&pdev->dev,
2792 "Allocation for Rx Queue %u failed\n", i);
2793 for (i--; i >= 0; i--)
2794 igb_free_rx_resources(adapter->rx_ring[i]);
2795 break;
2796 }
2797 }
2798
2799 return err;
2800}
2801
2802/**
2803 * igb_setup_mrqc - configure the multiple receive queue control registers
2804 * @adapter: Board private structure
2805 **/
2806static void igb_setup_mrqc(struct igb_adapter *adapter)
2807{
2808 struct e1000_hw *hw = &adapter->hw;
2809 u32 mrqc, rxcsum;
2810 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2811 union e1000_reta {
2812 u32 dword;
2813 u8 bytes[4];
2814 } reta;
2815 static const u8 rsshash[40] = {
2816 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2817 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2818 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2819 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2820
2821 /* Fill out hash function seeds */
2822 for (j = 0; j < 10; j++) {
2823 u32 rsskey = rsshash[(j * 4)];
2824 rsskey |= rsshash[(j * 4) + 1] << 8;
2825 rsskey |= rsshash[(j * 4) + 2] << 16;
2826 rsskey |= rsshash[(j * 4) + 3] << 24;
2827 array_wr32(E1000_RSSRK(0), j, rsskey);
2828 }
2829
2830 num_rx_queues = adapter->rss_queues;
2831
2832 if (adapter->vfs_allocated_count) {
2833 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2834 switch (hw->mac.type) {
2835 case e1000_i350:
2836 case e1000_82580:
2837 num_rx_queues = 1;
2838 shift = 0;
2839 break;
2840 case e1000_82576:
2841 shift = 3;
2842 num_rx_queues = 2;
2843 break;
2844 case e1000_82575:
2845 shift = 2;
2846 shift2 = 6;
2847 default:
2848 break;
2849 }
2850 } else {
2851 if (hw->mac.type == e1000_82575)
2852 shift = 6;
2853 }
2854
2855 for (j = 0; j < (32 * 4); j++) {
2856 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2857 if (shift2)
2858 reta.bytes[j & 3] |= num_rx_queues << shift2;
2859 if ((j & 3) == 3)
2860 wr32(E1000_RETA(j >> 2), reta.dword);
2861 }
2862
2863 /*
2864 * Disable raw packet checksumming so that RSS hash is placed in
2865 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2866 * offloads as they are enabled by default
2867 */
2868 rxcsum = rd32(E1000_RXCSUM);
2869 rxcsum |= E1000_RXCSUM_PCSD;
2870
2871 if (adapter->hw.mac.type >= e1000_82576)
2872 /* Enable Receive Checksum Offload for SCTP */
2873 rxcsum |= E1000_RXCSUM_CRCOFL;
2874
2875 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2876 wr32(E1000_RXCSUM, rxcsum);
2877 /*
2878 * Generate RSS hash based on TCP port numbers and/or
2879 * IPv4/v6 src and dst addresses since UDP cannot be
2880 * hashed reliably due to IP fragmentation
2881 */
2882
2883 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2884 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2885 E1000_MRQC_RSS_FIELD_IPV6 |
2886 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2887 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2888
2889 /* If VMDq is enabled then we set the appropriate mode for that, else
2890 * we default to RSS so that an RSS hash is calculated per packet even
2891 * if we are only using one queue */
2892 if (adapter->vfs_allocated_count) {
2893 if (hw->mac.type > e1000_82575) {
2894 /* Set the default pool for the PF's first queue */
2895 u32 vtctl = rd32(E1000_VT_CTL);
2896 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2897 E1000_VT_CTL_DISABLE_DEF_POOL);
2898 vtctl |= adapter->vfs_allocated_count <<
2899 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2900 wr32(E1000_VT_CTL, vtctl);
2901 }
2902 if (adapter->rss_queues > 1)
2903 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2904 else
2905 mrqc |= E1000_MRQC_ENABLE_VMDQ;
2906 } else {
2907 if (hw->mac.type != e1000_i211)
2908 mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
2909 }
2910 igb_vmm_control(adapter);
2911
2912 wr32(E1000_MRQC, mrqc);
2913}
2914
2915/**
2916 * igb_setup_rctl - configure the receive control registers
2917 * @adapter: Board private structure
2918 **/
2919void igb_setup_rctl(struct igb_adapter *adapter)
2920{
2921 struct e1000_hw *hw = &adapter->hw;
2922 u32 rctl;
2923
2924 rctl = rd32(E1000_RCTL);
2925
2926 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2927 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2928
2929 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2930 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2931
2932 /*
2933 * enable stripping of CRC. It's unlikely this will break BMC
2934 * redirection as it did with e1000. Newer features require
2935 * that the HW strips the CRC.
2936 */
2937 rctl |= E1000_RCTL_SECRC;
2938
2939 /* disable store bad packets and clear size bits. */
2940 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2941
2942 /* enable LPE to prevent packets larger than max_frame_size */
2943 rctl |= E1000_RCTL_LPE;
2944
2945 /* disable queue 0 to prevent tail write w/o re-config */
2946 wr32(E1000_RXDCTL(0), 0);
2947
2948 /* Attention!!! For SR-IOV PF driver operations you must enable
2949 * queue drop for all VF and PF queues to prevent head of line blocking
2950 * if an un-trusted VF does not provide descriptors to hardware.
2951 */
2952 if (adapter->vfs_allocated_count) {
2953 /* set all queue drop enable bits */
2954 wr32(E1000_QDE, ALL_QUEUES);
2955 }
2956
2957 /* This is useful for sniffing bad packets. */
2958 if (adapter->netdev->features & NETIF_F_RXALL) {
2959 /* UPE and MPE will be handled by normal PROMISC logic
2960 * in e1000e_set_rx_mode */
2961 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
2962 E1000_RCTL_BAM | /* RX All Bcast Pkts */
2963 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
2964
2965 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
2966 E1000_RCTL_DPF | /* Allow filtered pause */
2967 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
2968 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
2969 * and that breaks VLANs.
2970 */
2971 }
2972
2973 wr32(E1000_RCTL, rctl);
2974}
2975
2976static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2977 int vfn)
2978{
2979 struct e1000_hw *hw = &adapter->hw;
2980 u32 vmolr;
2981
2982 /* if it isn't the PF check to see if VFs are enabled and
2983 * increase the size to support vlan tags */
2984 if (vfn < adapter->vfs_allocated_count &&
2985 adapter->vf_data[vfn].vlans_enabled)
2986 size += VLAN_TAG_SIZE;
2987
2988 vmolr = rd32(E1000_VMOLR(vfn));
2989 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2990 vmolr |= size | E1000_VMOLR_LPE;
2991 wr32(E1000_VMOLR(vfn), vmolr);
2992
2993 return 0;
2994}
2995
2996/**
2997 * igb_rlpml_set - set maximum receive packet size
2998 * @adapter: board private structure
2999 *
3000 * Configure maximum receivable packet size.
3001 **/
3002static void igb_rlpml_set(struct igb_adapter *adapter)
3003{
3004 u32 max_frame_size = adapter->max_frame_size;
3005 struct e1000_hw *hw = &adapter->hw;
3006 u16 pf_id = adapter->vfs_allocated_count;
3007
3008 if (pf_id) {
3009 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3010 /*
3011 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3012 * to our max jumbo frame size, in case we need to enable
3013 * jumbo frames on one of the rings later.
3014 * This will not pass over-length frames into the default
3015 * queue because it's gated by the VMOLR.RLPML.
3016 */
3017 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3018 }
3019
3020 wr32(E1000_RLPML, max_frame_size);
3021}
3022
3023static inline void igb_set_vmolr(struct igb_adapter *adapter,
3024 int vfn, bool aupe)
3025{
3026 struct e1000_hw *hw = &adapter->hw;
3027 u32 vmolr;
3028
3029 /*
3030 * This register exists only on 82576 and newer so if we are older then
3031 * we should exit and do nothing
3032 */
3033 if (hw->mac.type < e1000_82576)
3034 return;
3035
3036 vmolr = rd32(E1000_VMOLR(vfn));
3037 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3038 if (aupe)
3039 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3040 else
3041 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3042
3043 /* clear all bits that might not be set */
3044 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3045
3046 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3047 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3048 /*
3049 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3050 * multicast packets
3051 */
3052 if (vfn <= adapter->vfs_allocated_count)
3053 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3054
3055 wr32(E1000_VMOLR(vfn), vmolr);
3056}
3057
3058/**
3059 * igb_configure_rx_ring - Configure a receive ring after Reset
3060 * @adapter: board private structure
3061 * @ring: receive ring to be configured
3062 *
3063 * Configure the Rx unit of the MAC after a reset.
3064 **/
3065void igb_configure_rx_ring(struct igb_adapter *adapter,
3066 struct igb_ring *ring)
3067{
3068 struct e1000_hw *hw = &adapter->hw;
3069 u64 rdba = ring->dma;
3070 int reg_idx = ring->reg_idx;
3071 u32 srrctl = 0, rxdctl = 0;
3072
3073 /* disable the queue */
3074 wr32(E1000_RXDCTL(reg_idx), 0);
3075
3076 /* Set DMA base address registers */
3077 wr32(E1000_RDBAL(reg_idx),
3078 rdba & 0x00000000ffffffffULL);
3079 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3080 wr32(E1000_RDLEN(reg_idx),
3081 ring->count * sizeof(union e1000_adv_rx_desc));
3082
3083 /* initialize head and tail */
3084 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3085 wr32(E1000_RDH(reg_idx), 0);
3086 writel(0, ring->tail);
3087
3088 /* set descriptor configuration */
3089 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3090#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3091 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3092#else
3093 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3094#endif
3095 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3096 if (hw->mac.type >= e1000_82580)
3097 srrctl |= E1000_SRRCTL_TIMESTAMP;
3098 /* Only set Drop Enable if we are supporting multiple queues */
3099 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3100 srrctl |= E1000_SRRCTL_DROP_EN;
3101
3102 wr32(E1000_SRRCTL(reg_idx), srrctl);
3103
3104 /* set filtering for VMDQ pools */
3105 igb_set_vmolr(adapter, reg_idx & 0x7, true);
3106
3107 rxdctl |= IGB_RX_PTHRESH;
3108 rxdctl |= IGB_RX_HTHRESH << 8;
3109 rxdctl |= IGB_RX_WTHRESH << 16;
3110
3111 /* enable receive descriptor fetching */
3112 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3113 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3114}
3115
3116/**
3117 * igb_configure_rx - Configure receive Unit after Reset
3118 * @adapter: board private structure
3119 *
3120 * Configure the Rx unit of the MAC after a reset.
3121 **/
3122static void igb_configure_rx(struct igb_adapter *adapter)
3123{
3124 int i;
3125
3126 /* set UTA to appropriate mode */
3127 igb_set_uta(adapter);
3128
3129 /* set the correct pool for the PF default MAC address in entry 0 */
3130 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3131 adapter->vfs_allocated_count);
3132
3133 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3134 * the Base and Length of the Rx Descriptor Ring */
3135 for (i = 0; i < adapter->num_rx_queues; i++)
3136 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3137}
3138
3139/**
3140 * igb_free_tx_resources - Free Tx Resources per Queue
3141 * @tx_ring: Tx descriptor ring for a specific queue
3142 *
3143 * Free all transmit software resources
3144 **/
3145void igb_free_tx_resources(struct igb_ring *tx_ring)
3146{
3147 igb_clean_tx_ring(tx_ring);
3148
3149 vfree(tx_ring->tx_buffer_info);
3150 tx_ring->tx_buffer_info = NULL;
3151
3152 /* if not set, then don't free */
3153 if (!tx_ring->desc)
3154 return;
3155
3156 dma_free_coherent(tx_ring->dev, tx_ring->size,
3157 tx_ring->desc, tx_ring->dma);
3158
3159 tx_ring->desc = NULL;
3160}
3161
3162/**
3163 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3164 * @adapter: board private structure
3165 *
3166 * Free all transmit software resources
3167 **/
3168static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3169{
3170 int i;
3171
3172 for (i = 0; i < adapter->num_tx_queues; i++)
3173 igb_free_tx_resources(adapter->tx_ring[i]);
3174}
3175
3176void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3177 struct igb_tx_buffer *tx_buffer)
3178{
3179 if (tx_buffer->skb) {
3180 dev_kfree_skb_any(tx_buffer->skb);
3181 if (tx_buffer->dma)
3182 dma_unmap_single(ring->dev,
3183 tx_buffer->dma,
3184 tx_buffer->length,
3185 DMA_TO_DEVICE);
3186 } else if (tx_buffer->dma) {
3187 dma_unmap_page(ring->dev,
3188 tx_buffer->dma,
3189 tx_buffer->length,
3190 DMA_TO_DEVICE);
3191 }
3192 tx_buffer->next_to_watch = NULL;
3193 tx_buffer->skb = NULL;
3194 tx_buffer->dma = 0;
3195 /* buffer_info must be completely set up in the transmit path */
3196}
3197
3198/**
3199 * igb_clean_tx_ring - Free Tx Buffers
3200 * @tx_ring: ring to be cleaned
3201 **/
3202static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3203{
3204 struct igb_tx_buffer *buffer_info;
3205 unsigned long size;
3206 u16 i;
3207
3208 if (!tx_ring->tx_buffer_info)
3209 return;
3210 /* Free all the Tx ring sk_buffs */
3211
3212 for (i = 0; i < tx_ring->count; i++) {
3213 buffer_info = &tx_ring->tx_buffer_info[i];
3214 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3215 }
3216
3217 netdev_tx_reset_queue(txring_txq(tx_ring));
3218
3219 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3220 memset(tx_ring->tx_buffer_info, 0, size);
3221
3222 /* Zero out the descriptor ring */
3223 memset(tx_ring->desc, 0, tx_ring->size);
3224
3225 tx_ring->next_to_use = 0;
3226 tx_ring->next_to_clean = 0;
3227}
3228
3229/**
3230 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3231 * @adapter: board private structure
3232 **/
3233static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3234{
3235 int i;
3236
3237 for (i = 0; i < adapter->num_tx_queues; i++)
3238 igb_clean_tx_ring(adapter->tx_ring[i]);
3239}
3240
3241/**
3242 * igb_free_rx_resources - Free Rx Resources
3243 * @rx_ring: ring to clean the resources from
3244 *
3245 * Free all receive software resources
3246 **/
3247void igb_free_rx_resources(struct igb_ring *rx_ring)
3248{
3249 igb_clean_rx_ring(rx_ring);
3250
3251 vfree(rx_ring->rx_buffer_info);
3252 rx_ring->rx_buffer_info = NULL;
3253
3254 /* if not set, then don't free */
3255 if (!rx_ring->desc)
3256 return;
3257
3258 dma_free_coherent(rx_ring->dev, rx_ring->size,
3259 rx_ring->desc, rx_ring->dma);
3260
3261 rx_ring->desc = NULL;
3262}
3263
3264/**
3265 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3266 * @adapter: board private structure
3267 *
3268 * Free all receive software resources
3269 **/
3270static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3271{
3272 int i;
3273
3274 for (i = 0; i < adapter->num_rx_queues; i++)
3275 igb_free_rx_resources(adapter->rx_ring[i]);
3276}
3277
3278/**
3279 * igb_clean_rx_ring - Free Rx Buffers per Queue
3280 * @rx_ring: ring to free buffers from
3281 **/
3282static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3283{
3284 unsigned long size;
3285 u16 i;
3286
3287 if (!rx_ring->rx_buffer_info)
3288 return;
3289
3290 /* Free all the Rx ring sk_buffs */
3291 for (i = 0; i < rx_ring->count; i++) {
3292 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3293 if (buffer_info->dma) {
3294 dma_unmap_single(rx_ring->dev,
3295 buffer_info->dma,
3296 IGB_RX_HDR_LEN,
3297 DMA_FROM_DEVICE);
3298 buffer_info->dma = 0;
3299 }
3300
3301 if (buffer_info->skb) {
3302 dev_kfree_skb(buffer_info->skb);
3303 buffer_info->skb = NULL;
3304 }
3305 if (buffer_info->page_dma) {
3306 dma_unmap_page(rx_ring->dev,
3307 buffer_info->page_dma,
3308 PAGE_SIZE / 2,
3309 DMA_FROM_DEVICE);
3310 buffer_info->page_dma = 0;
3311 }
3312 if (buffer_info->page) {
3313 put_page(buffer_info->page);
3314 buffer_info->page = NULL;
3315 buffer_info->page_offset = 0;
3316 }
3317 }
3318
3319 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3320 memset(rx_ring->rx_buffer_info, 0, size);
3321
3322 /* Zero out the descriptor ring */
3323 memset(rx_ring->desc, 0, rx_ring->size);
3324
3325 rx_ring->next_to_clean = 0;
3326 rx_ring->next_to_use = 0;
3327}
3328
3329/**
3330 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3331 * @adapter: board private structure
3332 **/
3333static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3334{
3335 int i;
3336
3337 for (i = 0; i < adapter->num_rx_queues; i++)
3338 igb_clean_rx_ring(adapter->rx_ring[i]);
3339}
3340
3341/**
3342 * igb_set_mac - Change the Ethernet Address of the NIC
3343 * @netdev: network interface device structure
3344 * @p: pointer to an address structure
3345 *
3346 * Returns 0 on success, negative on failure
3347 **/
3348static int igb_set_mac(struct net_device *netdev, void *p)
3349{
3350 struct igb_adapter *adapter = netdev_priv(netdev);
3351 struct e1000_hw *hw = &adapter->hw;
3352 struct sockaddr *addr = p;
3353
3354 if (!is_valid_ether_addr(addr->sa_data))
3355 return -EADDRNOTAVAIL;
3356
3357 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3358 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3359
3360 /* set the correct pool for the new PF MAC address in entry 0 */
3361 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3362 adapter->vfs_allocated_count);
3363
3364 return 0;
3365}
3366
3367/**
3368 * igb_write_mc_addr_list - write multicast addresses to MTA
3369 * @netdev: network interface device structure
3370 *
3371 * Writes multicast address list to the MTA hash table.
3372 * Returns: -ENOMEM on failure
3373 * 0 on no addresses written
3374 * X on writing X addresses to MTA
3375 **/
3376static int igb_write_mc_addr_list(struct net_device *netdev)
3377{
3378 struct igb_adapter *adapter = netdev_priv(netdev);
3379 struct e1000_hw *hw = &adapter->hw;
3380 struct netdev_hw_addr *ha;
3381 u8 *mta_list;
3382 int i;
3383
3384 if (netdev_mc_empty(netdev)) {
3385 /* nothing to program, so clear mc list */
3386 igb_update_mc_addr_list(hw, NULL, 0);
3387 igb_restore_vf_multicasts(adapter);
3388 return 0;
3389 }
3390
3391 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3392 if (!mta_list)
3393 return -ENOMEM;
3394
3395 /* The shared function expects a packed array of only addresses. */
3396 i = 0;
3397 netdev_for_each_mc_addr(ha, netdev)
3398 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3399
3400 igb_update_mc_addr_list(hw, mta_list, i);
3401 kfree(mta_list);
3402
3403 return netdev_mc_count(netdev);
3404}
3405
3406/**
3407 * igb_write_uc_addr_list - write unicast addresses to RAR table
3408 * @netdev: network interface device structure
3409 *
3410 * Writes unicast address list to the RAR table.
3411 * Returns: -ENOMEM on failure/insufficient address space
3412 * 0 on no addresses written
3413 * X on writing X addresses to the RAR table
3414 **/
3415static int igb_write_uc_addr_list(struct net_device *netdev)
3416{
3417 struct igb_adapter *adapter = netdev_priv(netdev);
3418 struct e1000_hw *hw = &adapter->hw;
3419 unsigned int vfn = adapter->vfs_allocated_count;
3420 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3421 int count = 0;
3422
3423 /* return ENOMEM indicating insufficient memory for addresses */
3424 if (netdev_uc_count(netdev) > rar_entries)
3425 return -ENOMEM;
3426
3427 if (!netdev_uc_empty(netdev) && rar_entries) {
3428 struct netdev_hw_addr *ha;
3429
3430 netdev_for_each_uc_addr(ha, netdev) {
3431 if (!rar_entries)
3432 break;
3433 igb_rar_set_qsel(adapter, ha->addr,
3434 rar_entries--,
3435 vfn);
3436 count++;
3437 }
3438 }
3439 /* write the addresses in reverse order to avoid write combining */
3440 for (; rar_entries > 0 ; rar_entries--) {
3441 wr32(E1000_RAH(rar_entries), 0);
3442 wr32(E1000_RAL(rar_entries), 0);
3443 }
3444 wrfl();
3445
3446 return count;
3447}
3448
3449/**
3450 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3451 * @netdev: network interface device structure
3452 *
3453 * The set_rx_mode entry point is called whenever the unicast or multicast
3454 * address lists or the network interface flags are updated. This routine is
3455 * responsible for configuring the hardware for proper unicast, multicast,
3456 * promiscuous mode, and all-multi behavior.
3457 **/
3458static void igb_set_rx_mode(struct net_device *netdev)
3459{
3460 struct igb_adapter *adapter = netdev_priv(netdev);
3461 struct e1000_hw *hw = &adapter->hw;
3462 unsigned int vfn = adapter->vfs_allocated_count;
3463 u32 rctl, vmolr = 0;
3464 int count;
3465
3466 /* Check for Promiscuous and All Multicast modes */
3467 rctl = rd32(E1000_RCTL);
3468
3469 /* clear the effected bits */
3470 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3471
3472 if (netdev->flags & IFF_PROMISC) {
3473 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3474 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3475 } else {
3476 if (netdev->flags & IFF_ALLMULTI) {
3477 rctl |= E1000_RCTL_MPE;
3478 vmolr |= E1000_VMOLR_MPME;
3479 } else {
3480 /*
3481 * Write addresses to the MTA, if the attempt fails
3482 * then we should just turn on promiscuous mode so
3483 * that we can at least receive multicast traffic
3484 */
3485 count = igb_write_mc_addr_list(netdev);
3486 if (count < 0) {
3487 rctl |= E1000_RCTL_MPE;
3488 vmolr |= E1000_VMOLR_MPME;
3489 } else if (count) {
3490 vmolr |= E1000_VMOLR_ROMPE;
3491 }
3492 }
3493 /*
3494 * Write addresses to available RAR registers, if there is not
3495 * sufficient space to store all the addresses then enable
3496 * unicast promiscuous mode
3497 */
3498 count = igb_write_uc_addr_list(netdev);
3499 if (count < 0) {
3500 rctl |= E1000_RCTL_UPE;
3501 vmolr |= E1000_VMOLR_ROPE;
3502 }
3503 rctl |= E1000_RCTL_VFE;
3504 }
3505 wr32(E1000_RCTL, rctl);
3506
3507 /*
3508 * In order to support SR-IOV and eventually VMDq it is necessary to set
3509 * the VMOLR to enable the appropriate modes. Without this workaround
3510 * we will have issues with VLAN tag stripping not being done for frames
3511 * that are only arriving because we are the default pool
3512 */
3513 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
3514 return;
3515
3516 vmolr |= rd32(E1000_VMOLR(vfn)) &
3517 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3518 wr32(E1000_VMOLR(vfn), vmolr);
3519 igb_restore_vf_multicasts(adapter);
3520}
3521
3522static void igb_check_wvbr(struct igb_adapter *adapter)
3523{
3524 struct e1000_hw *hw = &adapter->hw;
3525 u32 wvbr = 0;
3526
3527 switch (hw->mac.type) {
3528 case e1000_82576:
3529 case e1000_i350:
3530 if (!(wvbr = rd32(E1000_WVBR)))
3531 return;
3532 break;
3533 default:
3534 break;
3535 }
3536
3537 adapter->wvbr |= wvbr;
3538}
3539
3540#define IGB_STAGGERED_QUEUE_OFFSET 8
3541
3542static void igb_spoof_check(struct igb_adapter *adapter)
3543{
3544 int j;
3545
3546 if (!adapter->wvbr)
3547 return;
3548
3549 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3550 if (adapter->wvbr & (1 << j) ||
3551 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3552 dev_warn(&adapter->pdev->dev,
3553 "Spoof event(s) detected on VF %d\n", j);
3554 adapter->wvbr &=
3555 ~((1 << j) |
3556 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3557 }
3558 }
3559}
3560
3561/* Need to wait a few seconds after link up to get diagnostic information from
3562 * the phy */
3563static void igb_update_phy_info(unsigned long data)
3564{
3565 struct igb_adapter *adapter = (struct igb_adapter *) data;
3566 igb_get_phy_info(&adapter->hw);
3567}
3568
3569/**
3570 * igb_has_link - check shared code for link and determine up/down
3571 * @adapter: pointer to driver private info
3572 **/
3573bool igb_has_link(struct igb_adapter *adapter)
3574{
3575 struct e1000_hw *hw = &adapter->hw;
3576 bool link_active = false;
3577 s32 ret_val = 0;
3578
3579 /* get_link_status is set on LSC (link status) interrupt or
3580 * rx sequence error interrupt. get_link_status will stay
3581 * false until the e1000_check_for_link establishes link
3582 * for copper adapters ONLY
3583 */
3584 switch (hw->phy.media_type) {
3585 case e1000_media_type_copper:
3586 if (hw->mac.get_link_status) {
3587 ret_val = hw->mac.ops.check_for_link(hw);
3588 link_active = !hw->mac.get_link_status;
3589 } else {
3590 link_active = true;
3591 }
3592 break;
3593 case e1000_media_type_internal_serdes:
3594 ret_val = hw->mac.ops.check_for_link(hw);
3595 link_active = hw->mac.serdes_has_link;
3596 break;
3597 default:
3598 case e1000_media_type_unknown:
3599 break;
3600 }
3601
3602 return link_active;
3603}
3604
3605static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3606{
3607 bool ret = false;
3608 u32 ctrl_ext, thstat;
3609
3610 /* check for thermal sensor event on i350 copper only */
3611 if (hw->mac.type == e1000_i350) {
3612 thstat = rd32(E1000_THSTAT);
3613 ctrl_ext = rd32(E1000_CTRL_EXT);
3614
3615 if ((hw->phy.media_type == e1000_media_type_copper) &&
3616 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3617 ret = !!(thstat & event);
3618 }
3619 }
3620
3621 return ret;
3622}
3623
3624/**
3625 * igb_watchdog - Timer Call-back
3626 * @data: pointer to adapter cast into an unsigned long
3627 **/
3628static void igb_watchdog(unsigned long data)
3629{
3630 struct igb_adapter *adapter = (struct igb_adapter *)data;
3631 /* Do the rest outside of interrupt context */
3632 schedule_work(&adapter->watchdog_task);
3633}
3634
3635static void igb_watchdog_task(struct work_struct *work)
3636{
3637 struct igb_adapter *adapter = container_of(work,
3638 struct igb_adapter,
3639 watchdog_task);
3640 struct e1000_hw *hw = &adapter->hw;
3641 struct net_device *netdev = adapter->netdev;
3642 u32 link;
3643 int i;
3644
3645 link = igb_has_link(adapter);
3646 if (link) {
3647 /* Cancel scheduled suspend requests. */
3648 pm_runtime_resume(netdev->dev.parent);
3649
3650 if (!netif_carrier_ok(netdev)) {
3651 u32 ctrl;
3652 hw->mac.ops.get_speed_and_duplex(hw,
3653 &adapter->link_speed,
3654 &adapter->link_duplex);
3655
3656 ctrl = rd32(E1000_CTRL);
3657 /* Links status message must follow this format */
3658 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3659 "Duplex, Flow Control: %s\n",
3660 netdev->name,
3661 adapter->link_speed,
3662 adapter->link_duplex == FULL_DUPLEX ?
3663 "Full" : "Half",
3664 (ctrl & E1000_CTRL_TFCE) &&
3665 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3666 (ctrl & E1000_CTRL_RFCE) ? "RX" :
3667 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
3668
3669 /* check for thermal sensor event */
3670 if (igb_thermal_sensor_event(hw,
3671 E1000_THSTAT_LINK_THROTTLE)) {
3672 netdev_info(netdev, "The network adapter link "
3673 "speed was downshifted because it "
3674 "overheated\n");
3675 }
3676
3677 /* adjust timeout factor according to speed/duplex */
3678 adapter->tx_timeout_factor = 1;
3679 switch (adapter->link_speed) {
3680 case SPEED_10:
3681 adapter->tx_timeout_factor = 14;
3682 break;
3683 case SPEED_100:
3684 /* maybe add some timeout factor ? */
3685 break;
3686 }
3687
3688 netif_carrier_on(netdev);
3689
3690 igb_ping_all_vfs(adapter);
3691 igb_check_vf_rate_limit(adapter);
3692
3693 /* link state has changed, schedule phy info update */
3694 if (!test_bit(__IGB_DOWN, &adapter->state))
3695 mod_timer(&adapter->phy_info_timer,
3696 round_jiffies(jiffies + 2 * HZ));
3697 }
3698 } else {
3699 if (netif_carrier_ok(netdev)) {
3700 adapter->link_speed = 0;
3701 adapter->link_duplex = 0;
3702
3703 /* check for thermal sensor event */
3704 if (igb_thermal_sensor_event(hw,
3705 E1000_THSTAT_PWR_DOWN)) {
3706 netdev_err(netdev, "The network adapter was "
3707 "stopped because it overheated\n");
3708 }
3709
3710 /* Links status message must follow this format */
3711 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3712 netdev->name);
3713 netif_carrier_off(netdev);
3714
3715 igb_ping_all_vfs(adapter);
3716
3717 /* link state has changed, schedule phy info update */
3718 if (!test_bit(__IGB_DOWN, &adapter->state))
3719 mod_timer(&adapter->phy_info_timer,
3720 round_jiffies(jiffies + 2 * HZ));
3721
3722 pm_schedule_suspend(netdev->dev.parent,
3723 MSEC_PER_SEC * 5);
3724 }
3725 }
3726
3727 spin_lock(&adapter->stats64_lock);
3728 igb_update_stats(adapter, &adapter->stats64);
3729 spin_unlock(&adapter->stats64_lock);
3730
3731 for (i = 0; i < adapter->num_tx_queues; i++) {
3732 struct igb_ring *tx_ring = adapter->tx_ring[i];
3733 if (!netif_carrier_ok(netdev)) {
3734 /* We've lost link, so the controller stops DMA,
3735 * but we've got queued Tx work that's never going
3736 * to get done, so reset controller to flush Tx.
3737 * (Do the reset outside of interrupt context). */
3738 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3739 adapter->tx_timeout_count++;
3740 schedule_work(&adapter->reset_task);
3741 /* return immediately since reset is imminent */
3742 return;
3743 }
3744 }
3745
3746 /* Force detection of hung controller every watchdog period */
3747 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
3748 }
3749
3750 /* Cause software interrupt to ensure rx ring is cleaned */
3751 if (adapter->msix_entries) {
3752 u32 eics = 0;
3753 for (i = 0; i < adapter->num_q_vectors; i++)
3754 eics |= adapter->q_vector[i]->eims_value;
3755 wr32(E1000_EICS, eics);
3756 } else {
3757 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3758 }
3759
3760 igb_spoof_check(adapter);
3761
3762 /* Reset the timer */
3763 if (!test_bit(__IGB_DOWN, &adapter->state))
3764 mod_timer(&adapter->watchdog_timer,
3765 round_jiffies(jiffies + 2 * HZ));
3766}
3767
3768enum latency_range {
3769 lowest_latency = 0,
3770 low_latency = 1,
3771 bulk_latency = 2,
3772 latency_invalid = 255
3773};
3774
3775/**
3776 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3777 *
3778 * Stores a new ITR value based on strictly on packet size. This
3779 * algorithm is less sophisticated than that used in igb_update_itr,
3780 * due to the difficulty of synchronizing statistics across multiple
3781 * receive rings. The divisors and thresholds used by this function
3782 * were determined based on theoretical maximum wire speed and testing
3783 * data, in order to minimize response time while increasing bulk
3784 * throughput.
3785 * This functionality is controlled by the InterruptThrottleRate module
3786 * parameter (see igb_param.c)
3787 * NOTE: This function is called only when operating in a multiqueue
3788 * receive environment.
3789 * @q_vector: pointer to q_vector
3790 **/
3791static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3792{
3793 int new_val = q_vector->itr_val;
3794 int avg_wire_size = 0;
3795 struct igb_adapter *adapter = q_vector->adapter;
3796 unsigned int packets;
3797
3798 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3799 * ints/sec - ITR timer value of 120 ticks.
3800 */
3801 if (adapter->link_speed != SPEED_1000) {
3802 new_val = IGB_4K_ITR;
3803 goto set_itr_val;
3804 }
3805
3806 packets = q_vector->rx.total_packets;
3807 if (packets)
3808 avg_wire_size = q_vector->rx.total_bytes / packets;
3809
3810 packets = q_vector->tx.total_packets;
3811 if (packets)
3812 avg_wire_size = max_t(u32, avg_wire_size,
3813 q_vector->tx.total_bytes / packets);
3814
3815 /* if avg_wire_size isn't set no work was done */
3816 if (!avg_wire_size)
3817 goto clear_counts;
3818
3819 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3820 avg_wire_size += 24;
3821
3822 /* Don't starve jumbo frames */
3823 avg_wire_size = min(avg_wire_size, 3000);
3824
3825 /* Give a little boost to mid-size frames */
3826 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3827 new_val = avg_wire_size / 3;
3828 else
3829 new_val = avg_wire_size / 2;
3830
3831 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3832 if (new_val < IGB_20K_ITR &&
3833 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3834 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3835 new_val = IGB_20K_ITR;
3836
3837set_itr_val:
3838 if (new_val != q_vector->itr_val) {
3839 q_vector->itr_val = new_val;
3840 q_vector->set_itr = 1;
3841 }
3842clear_counts:
3843 q_vector->rx.total_bytes = 0;
3844 q_vector->rx.total_packets = 0;
3845 q_vector->tx.total_bytes = 0;
3846 q_vector->tx.total_packets = 0;
3847}
3848
3849/**
3850 * igb_update_itr - update the dynamic ITR value based on statistics
3851 * Stores a new ITR value based on packets and byte
3852 * counts during the last interrupt. The advantage of per interrupt
3853 * computation is faster updates and more accurate ITR for the current
3854 * traffic pattern. Constants in this function were computed
3855 * based on theoretical maximum wire speed and thresholds were set based
3856 * on testing data as well as attempting to minimize response time
3857 * while increasing bulk throughput.
3858 * this functionality is controlled by the InterruptThrottleRate module
3859 * parameter (see igb_param.c)
3860 * NOTE: These calculations are only valid when operating in a single-
3861 * queue environment.
3862 * @q_vector: pointer to q_vector
3863 * @ring_container: ring info to update the itr for
3864 **/
3865static void igb_update_itr(struct igb_q_vector *q_vector,
3866 struct igb_ring_container *ring_container)
3867{
3868 unsigned int packets = ring_container->total_packets;
3869 unsigned int bytes = ring_container->total_bytes;
3870 u8 itrval = ring_container->itr;
3871
3872 /* no packets, exit with status unchanged */
3873 if (packets == 0)
3874 return;
3875
3876 switch (itrval) {
3877 case lowest_latency:
3878 /* handle TSO and jumbo frames */
3879 if (bytes/packets > 8000)
3880 itrval = bulk_latency;
3881 else if ((packets < 5) && (bytes > 512))
3882 itrval = low_latency;
3883 break;
3884 case low_latency: /* 50 usec aka 20000 ints/s */
3885 if (bytes > 10000) {
3886 /* this if handles the TSO accounting */
3887 if (bytes/packets > 8000) {
3888 itrval = bulk_latency;
3889 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3890 itrval = bulk_latency;
3891 } else if ((packets > 35)) {
3892 itrval = lowest_latency;
3893 }
3894 } else if (bytes/packets > 2000) {
3895 itrval = bulk_latency;
3896 } else if (packets <= 2 && bytes < 512) {
3897 itrval = lowest_latency;
3898 }
3899 break;
3900 case bulk_latency: /* 250 usec aka 4000 ints/s */
3901 if (bytes > 25000) {
3902 if (packets > 35)
3903 itrval = low_latency;
3904 } else if (bytes < 1500) {
3905 itrval = low_latency;
3906 }
3907 break;
3908 }
3909
3910 /* clear work counters since we have the values we need */
3911 ring_container->total_bytes = 0;
3912 ring_container->total_packets = 0;
3913
3914 /* write updated itr to ring container */
3915 ring_container->itr = itrval;
3916}
3917
3918static void igb_set_itr(struct igb_q_vector *q_vector)
3919{
3920 struct igb_adapter *adapter = q_vector->adapter;
3921 u32 new_itr = q_vector->itr_val;
3922 u8 current_itr = 0;
3923
3924 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3925 if (adapter->link_speed != SPEED_1000) {
3926 current_itr = 0;
3927 new_itr = IGB_4K_ITR;
3928 goto set_itr_now;
3929 }
3930
3931 igb_update_itr(q_vector, &q_vector->tx);
3932 igb_update_itr(q_vector, &q_vector->rx);
3933
3934 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
3935
3936 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3937 if (current_itr == lowest_latency &&
3938 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3939 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3940 current_itr = low_latency;
3941
3942 switch (current_itr) {
3943 /* counts and packets in update_itr are dependent on these numbers */
3944 case lowest_latency:
3945 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
3946 break;
3947 case low_latency:
3948 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
3949 break;
3950 case bulk_latency:
3951 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
3952 break;
3953 default:
3954 break;
3955 }
3956
3957set_itr_now:
3958 if (new_itr != q_vector->itr_val) {
3959 /* this attempts to bias the interrupt rate towards Bulk
3960 * by adding intermediate steps when interrupt rate is
3961 * increasing */
3962 new_itr = new_itr > q_vector->itr_val ?
3963 max((new_itr * q_vector->itr_val) /
3964 (new_itr + (q_vector->itr_val >> 2)),
3965 new_itr) :
3966 new_itr;
3967 /* Don't write the value here; it resets the adapter's
3968 * internal timer, and causes us to delay far longer than
3969 * we should between interrupts. Instead, we write the ITR
3970 * value at the beginning of the next interrupt so the timing
3971 * ends up being correct.
3972 */
3973 q_vector->itr_val = new_itr;
3974 q_vector->set_itr = 1;
3975 }
3976}
3977
3978static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
3979 u32 type_tucmd, u32 mss_l4len_idx)
3980{
3981 struct e1000_adv_tx_context_desc *context_desc;
3982 u16 i = tx_ring->next_to_use;
3983
3984 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
3985
3986 i++;
3987 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
3988
3989 /* set bits to identify this as an advanced context descriptor */
3990 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3991
3992 /* For 82575, context index must be unique per ring. */
3993 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
3994 mss_l4len_idx |= tx_ring->reg_idx << 4;
3995
3996 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
3997 context_desc->seqnum_seed = 0;
3998 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
3999 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4000}
4001
4002static int igb_tso(struct igb_ring *tx_ring,
4003 struct igb_tx_buffer *first,
4004 u8 *hdr_len)
4005{
4006 struct sk_buff *skb = first->skb;
4007 u32 vlan_macip_lens, type_tucmd;
4008 u32 mss_l4len_idx, l4len;
4009
4010 if (!skb_is_gso(skb))
4011 return 0;
4012
4013 if (skb_header_cloned(skb)) {
4014 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4015 if (err)
4016 return err;
4017 }
4018
4019 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4020 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4021
4022 if (first->protocol == __constant_htons(ETH_P_IP)) {
4023 struct iphdr *iph = ip_hdr(skb);
4024 iph->tot_len = 0;
4025 iph->check = 0;
4026 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4027 iph->daddr, 0,
4028 IPPROTO_TCP,
4029 0);
4030 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4031 first->tx_flags |= IGB_TX_FLAGS_TSO |
4032 IGB_TX_FLAGS_CSUM |
4033 IGB_TX_FLAGS_IPV4;
4034 } else if (skb_is_gso_v6(skb)) {
4035 ipv6_hdr(skb)->payload_len = 0;
4036 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4037 &ipv6_hdr(skb)->daddr,
4038 0, IPPROTO_TCP, 0);
4039 first->tx_flags |= IGB_TX_FLAGS_TSO |
4040 IGB_TX_FLAGS_CSUM;
4041 }
4042
4043 /* compute header lengths */
4044 l4len = tcp_hdrlen(skb);
4045 *hdr_len = skb_transport_offset(skb) + l4len;
4046
4047 /* update gso size and bytecount with header size */
4048 first->gso_segs = skb_shinfo(skb)->gso_segs;
4049 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4050
4051 /* MSS L4LEN IDX */
4052 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4053 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4054
4055 /* VLAN MACLEN IPLEN */
4056 vlan_macip_lens = skb_network_header_len(skb);
4057 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4058 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4059
4060 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4061
4062 return 1;
4063}
4064
4065static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4066{
4067 struct sk_buff *skb = first->skb;
4068 u32 vlan_macip_lens = 0;
4069 u32 mss_l4len_idx = 0;
4070 u32 type_tucmd = 0;
4071
4072 if (skb->ip_summed != CHECKSUM_PARTIAL) {
4073 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4074 return;
4075 } else {
4076 u8 l4_hdr = 0;
4077 switch (first->protocol) {
4078 case __constant_htons(ETH_P_IP):
4079 vlan_macip_lens |= skb_network_header_len(skb);
4080 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4081 l4_hdr = ip_hdr(skb)->protocol;
4082 break;
4083 case __constant_htons(ETH_P_IPV6):
4084 vlan_macip_lens |= skb_network_header_len(skb);
4085 l4_hdr = ipv6_hdr(skb)->nexthdr;
4086 break;
4087 default:
4088 if (unlikely(net_ratelimit())) {
4089 dev_warn(tx_ring->dev,
4090 "partial checksum but proto=%x!\n",
4091 first->protocol);
4092 }
4093 break;
4094 }
4095
4096 switch (l4_hdr) {
4097 case IPPROTO_TCP:
4098 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4099 mss_l4len_idx = tcp_hdrlen(skb) <<
4100 E1000_ADVTXD_L4LEN_SHIFT;
4101 break;
4102 case IPPROTO_SCTP:
4103 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4104 mss_l4len_idx = sizeof(struct sctphdr) <<
4105 E1000_ADVTXD_L4LEN_SHIFT;
4106 break;
4107 case IPPROTO_UDP:
4108 mss_l4len_idx = sizeof(struct udphdr) <<
4109 E1000_ADVTXD_L4LEN_SHIFT;
4110 break;
4111 default:
4112 if (unlikely(net_ratelimit())) {
4113 dev_warn(tx_ring->dev,
4114 "partial checksum but l4 proto=%x!\n",
4115 l4_hdr);
4116 }
4117 break;
4118 }
4119
4120 /* update TX checksum flag */
4121 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4122 }
4123
4124 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4125 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4126
4127 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4128}
4129
4130static __le32 igb_tx_cmd_type(u32 tx_flags)
4131{
4132 /* set type for advanced descriptor with frame checksum insertion */
4133 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4134 E1000_ADVTXD_DCMD_IFCS |
4135 E1000_ADVTXD_DCMD_DEXT);
4136
4137 /* set HW vlan bit if vlan is present */
4138 if (tx_flags & IGB_TX_FLAGS_VLAN)
4139 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4140
4141 /* set timestamp bit if present */
4142 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4143 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4144
4145 /* set segmentation bits for TSO */
4146 if (tx_flags & IGB_TX_FLAGS_TSO)
4147 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4148
4149 return cmd_type;
4150}
4151
4152static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4153 union e1000_adv_tx_desc *tx_desc,
4154 u32 tx_flags, unsigned int paylen)
4155{
4156 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4157
4158 /* 82575 requires a unique index per ring if any offload is enabled */
4159 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4160 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4161 olinfo_status |= tx_ring->reg_idx << 4;
4162
4163 /* insert L4 checksum */
4164 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4165 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4166
4167 /* insert IPv4 checksum */
4168 if (tx_flags & IGB_TX_FLAGS_IPV4)
4169 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4170 }
4171
4172 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4173}
4174
4175/*
4176 * The largest size we can write to the descriptor is 65535. In order to
4177 * maintain a power of two alignment we have to limit ourselves to 32K.
4178 */
4179#define IGB_MAX_TXD_PWR 15
4180#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4181
4182static void igb_tx_map(struct igb_ring *tx_ring,
4183 struct igb_tx_buffer *first,
4184 const u8 hdr_len)
4185{
4186 struct sk_buff *skb = first->skb;
4187 struct igb_tx_buffer *tx_buffer_info;
4188 union e1000_adv_tx_desc *tx_desc;
4189 dma_addr_t dma;
4190 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4191 unsigned int data_len = skb->data_len;
4192 unsigned int size = skb_headlen(skb);
4193 unsigned int paylen = skb->len - hdr_len;
4194 __le32 cmd_type;
4195 u32 tx_flags = first->tx_flags;
4196 u16 i = tx_ring->next_to_use;
4197
4198 tx_desc = IGB_TX_DESC(tx_ring, i);
4199
4200 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4201 cmd_type = igb_tx_cmd_type(tx_flags);
4202
4203 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4204 if (dma_mapping_error(tx_ring->dev, dma))
4205 goto dma_error;
4206
4207 /* record length, and DMA address */
4208 first->length = size;
4209 first->dma = dma;
4210 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4211
4212 for (;;) {
4213 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4214 tx_desc->read.cmd_type_len =
4215 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4216
4217 i++;
4218 tx_desc++;
4219 if (i == tx_ring->count) {
4220 tx_desc = IGB_TX_DESC(tx_ring, 0);
4221 i = 0;
4222 }
4223
4224 dma += IGB_MAX_DATA_PER_TXD;
4225 size -= IGB_MAX_DATA_PER_TXD;
4226
4227 tx_desc->read.olinfo_status = 0;
4228 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4229 }
4230
4231 if (likely(!data_len))
4232 break;
4233
4234 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4235
4236 i++;
4237 tx_desc++;
4238 if (i == tx_ring->count) {
4239 tx_desc = IGB_TX_DESC(tx_ring, 0);
4240 i = 0;
4241 }
4242
4243 size = skb_frag_size(frag);
4244 data_len -= size;
4245
4246 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4247 size, DMA_TO_DEVICE);
4248 if (dma_mapping_error(tx_ring->dev, dma))
4249 goto dma_error;
4250
4251 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4252 tx_buffer_info->length = size;
4253 tx_buffer_info->dma = dma;
4254
4255 tx_desc->read.olinfo_status = 0;
4256 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4257
4258 frag++;
4259 }
4260
4261 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4262
4263 /* write last descriptor with RS and EOP bits */
4264 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4265 if (unlikely(skb->no_fcs))
4266 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
4267 tx_desc->read.cmd_type_len = cmd_type;
4268
4269 /* set the timestamp */
4270 first->time_stamp = jiffies;
4271
4272 /*
4273 * Force memory writes to complete before letting h/w know there
4274 * are new descriptors to fetch. (Only applicable for weak-ordered
4275 * memory model archs, such as IA-64).
4276 *
4277 * We also need this memory barrier to make certain all of the
4278 * status bits have been updated before next_to_watch is written.
4279 */
4280 wmb();
4281
4282 /* set next_to_watch value indicating a packet is present */
4283 first->next_to_watch = tx_desc;
4284
4285 i++;
4286 if (i == tx_ring->count)
4287 i = 0;
4288
4289 tx_ring->next_to_use = i;
4290
4291 writel(i, tx_ring->tail);
4292
4293 /* we need this if more than one processor can write to our tail
4294 * at a time, it syncronizes IO on IA64/Altix systems */
4295 mmiowb();
4296
4297 return;
4298
4299dma_error:
4300 dev_err(tx_ring->dev, "TX DMA map failed\n");
4301
4302 /* clear dma mappings for failed tx_buffer_info map */
4303 for (;;) {
4304 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4305 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4306 if (tx_buffer_info == first)
4307 break;
4308 if (i == 0)
4309 i = tx_ring->count;
4310 i--;
4311 }
4312
4313 tx_ring->next_to_use = i;
4314}
4315
4316static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4317{
4318 struct net_device *netdev = tx_ring->netdev;
4319
4320 netif_stop_subqueue(netdev, tx_ring->queue_index);
4321
4322 /* Herbert's original patch had:
4323 * smp_mb__after_netif_stop_queue();
4324 * but since that doesn't exist yet, just open code it. */
4325 smp_mb();
4326
4327 /* We need to check again in a case another CPU has just
4328 * made room available. */
4329 if (igb_desc_unused(tx_ring) < size)
4330 return -EBUSY;
4331
4332 /* A reprieve! */
4333 netif_wake_subqueue(netdev, tx_ring->queue_index);
4334
4335 u64_stats_update_begin(&tx_ring->tx_syncp2);
4336 tx_ring->tx_stats.restart_queue2++;
4337 u64_stats_update_end(&tx_ring->tx_syncp2);
4338
4339 return 0;
4340}
4341
4342static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4343{
4344 if (igb_desc_unused(tx_ring) >= size)
4345 return 0;
4346 return __igb_maybe_stop_tx(tx_ring, size);
4347}
4348
4349netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4350 struct igb_ring *tx_ring)
4351{
4352 struct igb_tx_buffer *first;
4353 int tso;
4354 u32 tx_flags = 0;
4355 __be16 protocol = vlan_get_protocol(skb);
4356 u8 hdr_len = 0;
4357
4358 /* need: 1 descriptor per page,
4359 * + 2 desc gap to keep tail from touching head,
4360 * + 1 desc for skb->data,
4361 * + 1 desc for context descriptor,
4362 * otherwise try next time */
4363 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4364 /* this is a hard error */
4365 return NETDEV_TX_BUSY;
4366 }
4367
4368 /* record the location of the first descriptor for this packet */
4369 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4370 first->skb = skb;
4371 first->bytecount = skb->len;
4372 first->gso_segs = 1;
4373
4374 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4375 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4376 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4377 }
4378
4379 if (vlan_tx_tag_present(skb)) {
4380 tx_flags |= IGB_TX_FLAGS_VLAN;
4381 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4382 }
4383
4384 /* record initial flags and protocol */
4385 first->tx_flags = tx_flags;
4386 first->protocol = protocol;
4387
4388 tso = igb_tso(tx_ring, first, &hdr_len);
4389 if (tso < 0)
4390 goto out_drop;
4391 else if (!tso)
4392 igb_tx_csum(tx_ring, first);
4393
4394 igb_tx_map(tx_ring, first, hdr_len);
4395
4396 /* Make sure there is space in the ring for the next send. */
4397 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4398
4399 return NETDEV_TX_OK;
4400
4401out_drop:
4402 igb_unmap_and_free_tx_resource(tx_ring, first);
4403
4404 return NETDEV_TX_OK;
4405}
4406
4407static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4408 struct sk_buff *skb)
4409{
4410 unsigned int r_idx = skb->queue_mapping;
4411
4412 if (r_idx >= adapter->num_tx_queues)
4413 r_idx = r_idx % adapter->num_tx_queues;
4414
4415 return adapter->tx_ring[r_idx];
4416}
4417
4418static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4419 struct net_device *netdev)
4420{
4421 struct igb_adapter *adapter = netdev_priv(netdev);
4422
4423 if (test_bit(__IGB_DOWN, &adapter->state)) {
4424 dev_kfree_skb_any(skb);
4425 return NETDEV_TX_OK;
4426 }
4427
4428 if (skb->len <= 0) {
4429 dev_kfree_skb_any(skb);
4430 return NETDEV_TX_OK;
4431 }
4432
4433 /*
4434 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4435 * in order to meet this minimum size requirement.
4436 */
4437 if (skb->len < 17) {
4438 if (skb_padto(skb, 17))
4439 return NETDEV_TX_OK;
4440 skb->len = 17;
4441 }
4442
4443 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4444}
4445
4446/**
4447 * igb_tx_timeout - Respond to a Tx Hang
4448 * @netdev: network interface device structure
4449 **/
4450static void igb_tx_timeout(struct net_device *netdev)
4451{
4452 struct igb_adapter *adapter = netdev_priv(netdev);
4453 struct e1000_hw *hw = &adapter->hw;
4454
4455 /* Do the reset outside of interrupt context */
4456 adapter->tx_timeout_count++;
4457
4458 if (hw->mac.type >= e1000_82580)
4459 hw->dev_spec._82575.global_device_reset = true;
4460
4461 schedule_work(&adapter->reset_task);
4462 wr32(E1000_EICS,
4463 (adapter->eims_enable_mask & ~adapter->eims_other));
4464}
4465
4466static void igb_reset_task(struct work_struct *work)
4467{
4468 struct igb_adapter *adapter;
4469 adapter = container_of(work, struct igb_adapter, reset_task);
4470
4471 igb_dump(adapter);
4472 netdev_err(adapter->netdev, "Reset adapter\n");
4473 igb_reinit_locked(adapter);
4474}
4475
4476/**
4477 * igb_get_stats64 - Get System Network Statistics
4478 * @netdev: network interface device structure
4479 * @stats: rtnl_link_stats64 pointer
4480 *
4481 **/
4482static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4483 struct rtnl_link_stats64 *stats)
4484{
4485 struct igb_adapter *adapter = netdev_priv(netdev);
4486
4487 spin_lock(&adapter->stats64_lock);
4488 igb_update_stats(adapter, &adapter->stats64);
4489 memcpy(stats, &adapter->stats64, sizeof(*stats));
4490 spin_unlock(&adapter->stats64_lock);
4491
4492 return stats;
4493}
4494
4495/**
4496 * igb_change_mtu - Change the Maximum Transfer Unit
4497 * @netdev: network interface device structure
4498 * @new_mtu: new value for maximum frame size
4499 *
4500 * Returns 0 on success, negative on failure
4501 **/
4502static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4503{
4504 struct igb_adapter *adapter = netdev_priv(netdev);
4505 struct pci_dev *pdev = adapter->pdev;
4506 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4507
4508 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4509 dev_err(&pdev->dev, "Invalid MTU setting\n");
4510 return -EINVAL;
4511 }
4512
4513#define MAX_STD_JUMBO_FRAME_SIZE 9238
4514 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4515 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4516 return -EINVAL;
4517 }
4518
4519 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4520 msleep(1);
4521
4522 /* igb_down has a dependency on max_frame_size */
4523 adapter->max_frame_size = max_frame;
4524
4525 if (netif_running(netdev))
4526 igb_down(adapter);
4527
4528 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4529 netdev->mtu, new_mtu);
4530 netdev->mtu = new_mtu;
4531
4532 if (netif_running(netdev))
4533 igb_up(adapter);
4534 else
4535 igb_reset(adapter);
4536
4537 clear_bit(__IGB_RESETTING, &adapter->state);
4538
4539 return 0;
4540}
4541
4542/**
4543 * igb_update_stats - Update the board statistics counters
4544 * @adapter: board private structure
4545 **/
4546
4547void igb_update_stats(struct igb_adapter *adapter,
4548 struct rtnl_link_stats64 *net_stats)
4549{
4550 struct e1000_hw *hw = &adapter->hw;
4551 struct pci_dev *pdev = adapter->pdev;
4552 u32 reg, mpc;
4553 u16 phy_tmp;
4554 int i;
4555 u64 bytes, packets;
4556 unsigned int start;
4557 u64 _bytes, _packets;
4558
4559#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4560
4561 /*
4562 * Prevent stats update while adapter is being reset, or if the pci
4563 * connection is down.
4564 */
4565 if (adapter->link_speed == 0)
4566 return;
4567 if (pci_channel_offline(pdev))
4568 return;
4569
4570 bytes = 0;
4571 packets = 0;
4572 for (i = 0; i < adapter->num_rx_queues; i++) {
4573 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4574 struct igb_ring *ring = adapter->rx_ring[i];
4575
4576 ring->rx_stats.drops += rqdpc_tmp;
4577 net_stats->rx_fifo_errors += rqdpc_tmp;
4578
4579 do {
4580 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4581 _bytes = ring->rx_stats.bytes;
4582 _packets = ring->rx_stats.packets;
4583 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4584 bytes += _bytes;
4585 packets += _packets;
4586 }
4587
4588 net_stats->rx_bytes = bytes;
4589 net_stats->rx_packets = packets;
4590
4591 bytes = 0;
4592 packets = 0;
4593 for (i = 0; i < adapter->num_tx_queues; i++) {
4594 struct igb_ring *ring = adapter->tx_ring[i];
4595 do {
4596 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4597 _bytes = ring->tx_stats.bytes;
4598 _packets = ring->tx_stats.packets;
4599 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4600 bytes += _bytes;
4601 packets += _packets;
4602 }
4603 net_stats->tx_bytes = bytes;
4604 net_stats->tx_packets = packets;
4605
4606 /* read stats registers */
4607 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4608 adapter->stats.gprc += rd32(E1000_GPRC);
4609 adapter->stats.gorc += rd32(E1000_GORCL);
4610 rd32(E1000_GORCH); /* clear GORCL */
4611 adapter->stats.bprc += rd32(E1000_BPRC);
4612 adapter->stats.mprc += rd32(E1000_MPRC);
4613 adapter->stats.roc += rd32(E1000_ROC);
4614
4615 adapter->stats.prc64 += rd32(E1000_PRC64);
4616 adapter->stats.prc127 += rd32(E1000_PRC127);
4617 adapter->stats.prc255 += rd32(E1000_PRC255);
4618 adapter->stats.prc511 += rd32(E1000_PRC511);
4619 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4620 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4621 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4622 adapter->stats.sec += rd32(E1000_SEC);
4623
4624 mpc = rd32(E1000_MPC);
4625 adapter->stats.mpc += mpc;
4626 net_stats->rx_fifo_errors += mpc;
4627 adapter->stats.scc += rd32(E1000_SCC);
4628 adapter->stats.ecol += rd32(E1000_ECOL);
4629 adapter->stats.mcc += rd32(E1000_MCC);
4630 adapter->stats.latecol += rd32(E1000_LATECOL);
4631 adapter->stats.dc += rd32(E1000_DC);
4632 adapter->stats.rlec += rd32(E1000_RLEC);
4633 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4634 adapter->stats.xontxc += rd32(E1000_XONTXC);
4635 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4636 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4637 adapter->stats.fcruc += rd32(E1000_FCRUC);
4638 adapter->stats.gptc += rd32(E1000_GPTC);
4639 adapter->stats.gotc += rd32(E1000_GOTCL);
4640 rd32(E1000_GOTCH); /* clear GOTCL */
4641 adapter->stats.rnbc += rd32(E1000_RNBC);
4642 adapter->stats.ruc += rd32(E1000_RUC);
4643 adapter->stats.rfc += rd32(E1000_RFC);
4644 adapter->stats.rjc += rd32(E1000_RJC);
4645 adapter->stats.tor += rd32(E1000_TORH);
4646 adapter->stats.tot += rd32(E1000_TOTH);
4647 adapter->stats.tpr += rd32(E1000_TPR);
4648
4649 adapter->stats.ptc64 += rd32(E1000_PTC64);
4650 adapter->stats.ptc127 += rd32(E1000_PTC127);
4651 adapter->stats.ptc255 += rd32(E1000_PTC255);
4652 adapter->stats.ptc511 += rd32(E1000_PTC511);
4653 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4654 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4655
4656 adapter->stats.mptc += rd32(E1000_MPTC);
4657 adapter->stats.bptc += rd32(E1000_BPTC);
4658
4659 adapter->stats.tpt += rd32(E1000_TPT);
4660 adapter->stats.colc += rd32(E1000_COLC);
4661
4662 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4663 /* read internal phy specific stats */
4664 reg = rd32(E1000_CTRL_EXT);
4665 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4666 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4667 adapter->stats.tncrs += rd32(E1000_TNCRS);
4668 }
4669
4670 adapter->stats.tsctc += rd32(E1000_TSCTC);
4671 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4672
4673 adapter->stats.iac += rd32(E1000_IAC);
4674 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4675 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4676 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4677 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4678 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4679 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4680 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4681 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4682
4683 /* Fill out the OS statistics structure */
4684 net_stats->multicast = adapter->stats.mprc;
4685 net_stats->collisions = adapter->stats.colc;
4686
4687 /* Rx Errors */
4688
4689 /* RLEC on some newer hardware can be incorrect so build
4690 * our own version based on RUC and ROC */
4691 net_stats->rx_errors = adapter->stats.rxerrc +
4692 adapter->stats.crcerrs + adapter->stats.algnerrc +
4693 adapter->stats.ruc + adapter->stats.roc +
4694 adapter->stats.cexterr;
4695 net_stats->rx_length_errors = adapter->stats.ruc +
4696 adapter->stats.roc;
4697 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4698 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4699 net_stats->rx_missed_errors = adapter->stats.mpc;
4700
4701 /* Tx Errors */
4702 net_stats->tx_errors = adapter->stats.ecol +
4703 adapter->stats.latecol;
4704 net_stats->tx_aborted_errors = adapter->stats.ecol;
4705 net_stats->tx_window_errors = adapter->stats.latecol;
4706 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4707
4708 /* Tx Dropped needs to be maintained elsewhere */
4709
4710 /* Phy Stats */
4711 if (hw->phy.media_type == e1000_media_type_copper) {
4712 if ((adapter->link_speed == SPEED_1000) &&
4713 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4714 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4715 adapter->phy_stats.idle_errors += phy_tmp;
4716 }
4717 }
4718
4719 /* Management Stats */
4720 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4721 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4722 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4723
4724 /* OS2BMC Stats */
4725 reg = rd32(E1000_MANC);
4726 if (reg & E1000_MANC_EN_BMC2OS) {
4727 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4728 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4729 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4730 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4731 }
4732}
4733
4734static irqreturn_t igb_msix_other(int irq, void *data)
4735{
4736 struct igb_adapter *adapter = data;
4737 struct e1000_hw *hw = &adapter->hw;
4738 u32 icr = rd32(E1000_ICR);
4739 /* reading ICR causes bit 31 of EICR to be cleared */
4740
4741 if (icr & E1000_ICR_DRSTA)
4742 schedule_work(&adapter->reset_task);
4743
4744 if (icr & E1000_ICR_DOUTSYNC) {
4745 /* HW is reporting DMA is out of sync */
4746 adapter->stats.doosync++;
4747 /* The DMA Out of Sync is also indication of a spoof event
4748 * in IOV mode. Check the Wrong VM Behavior register to
4749 * see if it is really a spoof event. */
4750 igb_check_wvbr(adapter);
4751 }
4752
4753 /* Check for a mailbox event */
4754 if (icr & E1000_ICR_VMMB)
4755 igb_msg_task(adapter);
4756
4757 if (icr & E1000_ICR_LSC) {
4758 hw->mac.get_link_status = 1;
4759 /* guard against interrupt when we're going down */
4760 if (!test_bit(__IGB_DOWN, &adapter->state))
4761 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4762 }
4763
4764 wr32(E1000_EIMS, adapter->eims_other);
4765
4766 return IRQ_HANDLED;
4767}
4768
4769static void igb_write_itr(struct igb_q_vector *q_vector)
4770{
4771 struct igb_adapter *adapter = q_vector->adapter;
4772 u32 itr_val = q_vector->itr_val & 0x7FFC;
4773
4774 if (!q_vector->set_itr)
4775 return;
4776
4777 if (!itr_val)
4778 itr_val = 0x4;
4779
4780 if (adapter->hw.mac.type == e1000_82575)
4781 itr_val |= itr_val << 16;
4782 else
4783 itr_val |= E1000_EITR_CNT_IGNR;
4784
4785 writel(itr_val, q_vector->itr_register);
4786 q_vector->set_itr = 0;
4787}
4788
4789static irqreturn_t igb_msix_ring(int irq, void *data)
4790{
4791 struct igb_q_vector *q_vector = data;
4792
4793 /* Write the ITR value calculated from the previous interrupt. */
4794 igb_write_itr(q_vector);
4795
4796 napi_schedule(&q_vector->napi);
4797
4798 return IRQ_HANDLED;
4799}
4800
4801#ifdef CONFIG_IGB_DCA
4802static void igb_update_dca(struct igb_q_vector *q_vector)
4803{
4804 struct igb_adapter *adapter = q_vector->adapter;
4805 struct e1000_hw *hw = &adapter->hw;
4806 int cpu = get_cpu();
4807
4808 if (q_vector->cpu == cpu)
4809 goto out_no_update;
4810
4811 if (q_vector->tx.ring) {
4812 int q = q_vector->tx.ring->reg_idx;
4813 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4814 if (hw->mac.type == e1000_82575) {
4815 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4816 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4817 } else {
4818 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4819 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4820 E1000_DCA_TXCTRL_CPUID_SHIFT;
4821 }
4822 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4823 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4824 }
4825 if (q_vector->rx.ring) {
4826 int q = q_vector->rx.ring->reg_idx;
4827 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4828 if (hw->mac.type == e1000_82575) {
4829 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4830 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4831 } else {
4832 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4833 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4834 E1000_DCA_RXCTRL_CPUID_SHIFT;
4835 }
4836 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4837 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4838 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4839 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4840 }
4841 q_vector->cpu = cpu;
4842out_no_update:
4843 put_cpu();
4844}
4845
4846static void igb_setup_dca(struct igb_adapter *adapter)
4847{
4848 struct e1000_hw *hw = &adapter->hw;
4849 int i;
4850
4851 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4852 return;
4853
4854 /* Always use CB2 mode, difference is masked in the CB driver. */
4855 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4856
4857 for (i = 0; i < adapter->num_q_vectors; i++) {
4858 adapter->q_vector[i]->cpu = -1;
4859 igb_update_dca(adapter->q_vector[i]);
4860 }
4861}
4862
4863static int __igb_notify_dca(struct device *dev, void *data)
4864{
4865 struct net_device *netdev = dev_get_drvdata(dev);
4866 struct igb_adapter *adapter = netdev_priv(netdev);
4867 struct pci_dev *pdev = adapter->pdev;
4868 struct e1000_hw *hw = &adapter->hw;
4869 unsigned long event = *(unsigned long *)data;
4870
4871 switch (event) {
4872 case DCA_PROVIDER_ADD:
4873 /* if already enabled, don't do it again */
4874 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4875 break;
4876 if (dca_add_requester(dev) == 0) {
4877 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4878 dev_info(&pdev->dev, "DCA enabled\n");
4879 igb_setup_dca(adapter);
4880 break;
4881 }
4882 /* Fall Through since DCA is disabled. */
4883 case DCA_PROVIDER_REMOVE:
4884 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4885 /* without this a class_device is left
4886 * hanging around in the sysfs model */
4887 dca_remove_requester(dev);
4888 dev_info(&pdev->dev, "DCA disabled\n");
4889 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4890 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4891 }
4892 break;
4893 }
4894
4895 return 0;
4896}
4897
4898static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4899 void *p)
4900{
4901 int ret_val;
4902
4903 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4904 __igb_notify_dca);
4905
4906 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4907}
4908#endif /* CONFIG_IGB_DCA */
4909
4910#ifdef CONFIG_PCI_IOV
4911static int igb_vf_configure(struct igb_adapter *adapter, int vf)
4912{
4913 unsigned char mac_addr[ETH_ALEN];
4914 struct pci_dev *pdev = adapter->pdev;
4915 struct e1000_hw *hw = &adapter->hw;
4916 struct pci_dev *pvfdev;
4917 unsigned int device_id;
4918 u16 thisvf_devfn;
4919
4920 random_ether_addr(mac_addr);
4921 igb_set_vf_mac(adapter, vf, mac_addr);
4922
4923 switch (adapter->hw.mac.type) {
4924 case e1000_82576:
4925 device_id = IGB_82576_VF_DEV_ID;
4926 /* VF Stride for 82576 is 2 */
4927 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
4928 (pdev->devfn & 1);
4929 break;
4930 case e1000_i350:
4931 device_id = IGB_I350_VF_DEV_ID;
4932 /* VF Stride for I350 is 4 */
4933 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
4934 (pdev->devfn & 3);
4935 break;
4936 default:
4937 device_id = 0;
4938 thisvf_devfn = 0;
4939 break;
4940 }
4941
4942 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
4943 while (pvfdev) {
4944 if (pvfdev->devfn == thisvf_devfn)
4945 break;
4946 pvfdev = pci_get_device(hw->vendor_id,
4947 device_id, pvfdev);
4948 }
4949
4950 if (pvfdev)
4951 adapter->vf_data[vf].vfdev = pvfdev;
4952 else
4953 dev_err(&pdev->dev,
4954 "Couldn't find pci dev ptr for VF %4.4x\n",
4955 thisvf_devfn);
4956 return pvfdev != NULL;
4957}
4958
4959static int igb_find_enabled_vfs(struct igb_adapter *adapter)
4960{
4961 struct e1000_hw *hw = &adapter->hw;
4962 struct pci_dev *pdev = adapter->pdev;
4963 struct pci_dev *pvfdev;
4964 u16 vf_devfn = 0;
4965 u16 vf_stride;
4966 unsigned int device_id;
4967 int vfs_found = 0;
4968
4969 switch (adapter->hw.mac.type) {
4970 case e1000_82576:
4971 device_id = IGB_82576_VF_DEV_ID;
4972 /* VF Stride for 82576 is 2 */
4973 vf_stride = 2;
4974 break;
4975 case e1000_i350:
4976 device_id = IGB_I350_VF_DEV_ID;
4977 /* VF Stride for I350 is 4 */
4978 vf_stride = 4;
4979 break;
4980 default:
4981 device_id = 0;
4982 vf_stride = 0;
4983 break;
4984 }
4985
4986 vf_devfn = pdev->devfn + 0x80;
4987 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
4988 while (pvfdev) {
4989 if (pvfdev->devfn == vf_devfn &&
4990 (pvfdev->bus->number >= pdev->bus->number))
4991 vfs_found++;
4992 vf_devfn += vf_stride;
4993 pvfdev = pci_get_device(hw->vendor_id,
4994 device_id, pvfdev);
4995 }
4996
4997 return vfs_found;
4998}
4999
5000static int igb_check_vf_assignment(struct igb_adapter *adapter)
5001{
5002 int i;
5003 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5004 if (adapter->vf_data[i].vfdev) {
5005 if (adapter->vf_data[i].vfdev->dev_flags &
5006 PCI_DEV_FLAGS_ASSIGNED)
5007 return true;
5008 }
5009 }
5010 return false;
5011}
5012
5013#endif
5014static void igb_ping_all_vfs(struct igb_adapter *adapter)
5015{
5016 struct e1000_hw *hw = &adapter->hw;
5017 u32 ping;
5018 int i;
5019
5020 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5021 ping = E1000_PF_CONTROL_MSG;
5022 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5023 ping |= E1000_VT_MSGTYPE_CTS;
5024 igb_write_mbx(hw, &ping, 1, i);
5025 }
5026}
5027
5028static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5029{
5030 struct e1000_hw *hw = &adapter->hw;
5031 u32 vmolr = rd32(E1000_VMOLR(vf));
5032 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5033
5034 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5035 IGB_VF_FLAG_MULTI_PROMISC);
5036 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5037
5038 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5039 vmolr |= E1000_VMOLR_MPME;
5040 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5041 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5042 } else {
5043 /*
5044 * if we have hashes and we are clearing a multicast promisc
5045 * flag we need to write the hashes to the MTA as this step
5046 * was previously skipped
5047 */
5048 if (vf_data->num_vf_mc_hashes > 30) {
5049 vmolr |= E1000_VMOLR_MPME;
5050 } else if (vf_data->num_vf_mc_hashes) {
5051 int j;
5052 vmolr |= E1000_VMOLR_ROMPE;
5053 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5054 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5055 }
5056 }
5057
5058 wr32(E1000_VMOLR(vf), vmolr);
5059
5060 /* there are flags left unprocessed, likely not supported */
5061 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5062 return -EINVAL;
5063
5064 return 0;
5065
5066}
5067
5068static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5069 u32 *msgbuf, u32 vf)
5070{
5071 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5072 u16 *hash_list = (u16 *)&msgbuf[1];
5073 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5074 int i;
5075
5076 /* salt away the number of multicast addresses assigned
5077 * to this VF for later use to restore when the PF multi cast
5078 * list changes
5079 */
5080 vf_data->num_vf_mc_hashes = n;
5081
5082 /* only up to 30 hash values supported */
5083 if (n > 30)
5084 n = 30;
5085
5086 /* store the hashes for later use */
5087 for (i = 0; i < n; i++)
5088 vf_data->vf_mc_hashes[i] = hash_list[i];
5089
5090 /* Flush and reset the mta with the new values */
5091 igb_set_rx_mode(adapter->netdev);
5092
5093 return 0;
5094}
5095
5096static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5097{
5098 struct e1000_hw *hw = &adapter->hw;
5099 struct vf_data_storage *vf_data;
5100 int i, j;
5101
5102 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5103 u32 vmolr = rd32(E1000_VMOLR(i));
5104 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5105
5106 vf_data = &adapter->vf_data[i];
5107
5108 if ((vf_data->num_vf_mc_hashes > 30) ||
5109 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5110 vmolr |= E1000_VMOLR_MPME;
5111 } else if (vf_data->num_vf_mc_hashes) {
5112 vmolr |= E1000_VMOLR_ROMPE;
5113 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5114 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5115 }
5116 wr32(E1000_VMOLR(i), vmolr);
5117 }
5118}
5119
5120static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5121{
5122 struct e1000_hw *hw = &adapter->hw;
5123 u32 pool_mask, reg, vid;
5124 int i;
5125
5126 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5127
5128 /* Find the vlan filter for this id */
5129 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5130 reg = rd32(E1000_VLVF(i));
5131
5132 /* remove the vf from the pool */
5133 reg &= ~pool_mask;
5134
5135 /* if pool is empty then remove entry from vfta */
5136 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5137 (reg & E1000_VLVF_VLANID_ENABLE)) {
5138 reg = 0;
5139 vid = reg & E1000_VLVF_VLANID_MASK;
5140 igb_vfta_set(hw, vid, false);
5141 }
5142
5143 wr32(E1000_VLVF(i), reg);
5144 }
5145
5146 adapter->vf_data[vf].vlans_enabled = 0;
5147}
5148
5149static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5150{
5151 struct e1000_hw *hw = &adapter->hw;
5152 u32 reg, i;
5153
5154 /* The vlvf table only exists on 82576 hardware and newer */
5155 if (hw->mac.type < e1000_82576)
5156 return -1;
5157
5158 /* we only need to do this if VMDq is enabled */
5159 if (!adapter->vfs_allocated_count)
5160 return -1;
5161
5162 /* Find the vlan filter for this id */
5163 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5164 reg = rd32(E1000_VLVF(i));
5165 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5166 vid == (reg & E1000_VLVF_VLANID_MASK))
5167 break;
5168 }
5169
5170 if (add) {
5171 if (i == E1000_VLVF_ARRAY_SIZE) {
5172 /* Did not find a matching VLAN ID entry that was
5173 * enabled. Search for a free filter entry, i.e.
5174 * one without the enable bit set
5175 */
5176 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5177 reg = rd32(E1000_VLVF(i));
5178 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5179 break;
5180 }
5181 }
5182 if (i < E1000_VLVF_ARRAY_SIZE) {
5183 /* Found an enabled/available entry */
5184 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5185
5186 /* if !enabled we need to set this up in vfta */
5187 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5188 /* add VID to filter table */
5189 igb_vfta_set(hw, vid, true);
5190 reg |= E1000_VLVF_VLANID_ENABLE;
5191 }
5192 reg &= ~E1000_VLVF_VLANID_MASK;
5193 reg |= vid;
5194 wr32(E1000_VLVF(i), reg);
5195
5196 /* do not modify RLPML for PF devices */
5197 if (vf >= adapter->vfs_allocated_count)
5198 return 0;
5199
5200 if (!adapter->vf_data[vf].vlans_enabled) {
5201 u32 size;
5202 reg = rd32(E1000_VMOLR(vf));
5203 size = reg & E1000_VMOLR_RLPML_MASK;
5204 size += 4;
5205 reg &= ~E1000_VMOLR_RLPML_MASK;
5206 reg |= size;
5207 wr32(E1000_VMOLR(vf), reg);
5208 }
5209
5210 adapter->vf_data[vf].vlans_enabled++;
5211 }
5212 } else {
5213 if (i < E1000_VLVF_ARRAY_SIZE) {
5214 /* remove vf from the pool */
5215 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5216 /* if pool is empty then remove entry from vfta */
5217 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5218 reg = 0;
5219 igb_vfta_set(hw, vid, false);
5220 }
5221 wr32(E1000_VLVF(i), reg);
5222
5223 /* do not modify RLPML for PF devices */
5224 if (vf >= adapter->vfs_allocated_count)
5225 return 0;
5226
5227 adapter->vf_data[vf].vlans_enabled--;
5228 if (!adapter->vf_data[vf].vlans_enabled) {
5229 u32 size;
5230 reg = rd32(E1000_VMOLR(vf));
5231 size = reg & E1000_VMOLR_RLPML_MASK;
5232 size -= 4;
5233 reg &= ~E1000_VMOLR_RLPML_MASK;
5234 reg |= size;
5235 wr32(E1000_VMOLR(vf), reg);
5236 }
5237 }
5238 }
5239 return 0;
5240}
5241
5242static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5243{
5244 struct e1000_hw *hw = &adapter->hw;
5245
5246 if (vid)
5247 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5248 else
5249 wr32(E1000_VMVIR(vf), 0);
5250}
5251
5252static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5253 int vf, u16 vlan, u8 qos)
5254{
5255 int err = 0;
5256 struct igb_adapter *adapter = netdev_priv(netdev);
5257
5258 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5259 return -EINVAL;
5260 if (vlan || qos) {
5261 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5262 if (err)
5263 goto out;
5264 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5265 igb_set_vmolr(adapter, vf, !vlan);
5266 adapter->vf_data[vf].pf_vlan = vlan;
5267 adapter->vf_data[vf].pf_qos = qos;
5268 dev_info(&adapter->pdev->dev,
5269 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5270 if (test_bit(__IGB_DOWN, &adapter->state)) {
5271 dev_warn(&adapter->pdev->dev,
5272 "The VF VLAN has been set,"
5273 " but the PF device is not up.\n");
5274 dev_warn(&adapter->pdev->dev,
5275 "Bring the PF device up before"
5276 " attempting to use the VF device.\n");
5277 }
5278 } else {
5279 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5280 false, vf);
5281 igb_set_vmvir(adapter, vlan, vf);
5282 igb_set_vmolr(adapter, vf, true);
5283 adapter->vf_data[vf].pf_vlan = 0;
5284 adapter->vf_data[vf].pf_qos = 0;
5285 }
5286out:
5287 return err;
5288}
5289
5290static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5291{
5292 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5293 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5294
5295 return igb_vlvf_set(adapter, vid, add, vf);
5296}
5297
5298static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5299{
5300 /* clear flags - except flag that indicates PF has set the MAC */
5301 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5302 adapter->vf_data[vf].last_nack = jiffies;
5303
5304 /* reset offloads to defaults */
5305 igb_set_vmolr(adapter, vf, true);
5306
5307 /* reset vlans for device */
5308 igb_clear_vf_vfta(adapter, vf);
5309 if (adapter->vf_data[vf].pf_vlan)
5310 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5311 adapter->vf_data[vf].pf_vlan,
5312 adapter->vf_data[vf].pf_qos);
5313 else
5314 igb_clear_vf_vfta(adapter, vf);
5315
5316 /* reset multicast table array for vf */
5317 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5318
5319 /* Flush and reset the mta with the new values */
5320 igb_set_rx_mode(adapter->netdev);
5321}
5322
5323static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5324{
5325 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5326
5327 /* generate a new mac address as we were hotplug removed/added */
5328 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5329 random_ether_addr(vf_mac);
5330
5331 /* process remaining reset events */
5332 igb_vf_reset(adapter, vf);
5333}
5334
5335static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5336{
5337 struct e1000_hw *hw = &adapter->hw;
5338 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5339 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5340 u32 reg, msgbuf[3];
5341 u8 *addr = (u8 *)(&msgbuf[1]);
5342
5343 /* process all the same items cleared in a function level reset */
5344 igb_vf_reset(adapter, vf);
5345
5346 /* set vf mac address */
5347 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5348
5349 /* enable transmit and receive for vf */
5350 reg = rd32(E1000_VFTE);
5351 wr32(E1000_VFTE, reg | (1 << vf));
5352 reg = rd32(E1000_VFRE);
5353 wr32(E1000_VFRE, reg | (1 << vf));
5354
5355 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5356
5357 /* reply to reset with ack and vf mac address */
5358 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5359 memcpy(addr, vf_mac, 6);
5360 igb_write_mbx(hw, msgbuf, 3, vf);
5361}
5362
5363static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5364{
5365 /*
5366 * The VF MAC Address is stored in a packed array of bytes
5367 * starting at the second 32 bit word of the msg array
5368 */
5369 unsigned char *addr = (char *)&msg[1];
5370 int err = -1;
5371
5372 if (is_valid_ether_addr(addr))
5373 err = igb_set_vf_mac(adapter, vf, addr);
5374
5375 return err;
5376}
5377
5378static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5379{
5380 struct e1000_hw *hw = &adapter->hw;
5381 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5382 u32 msg = E1000_VT_MSGTYPE_NACK;
5383
5384 /* if device isn't clear to send it shouldn't be reading either */
5385 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5386 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5387 igb_write_mbx(hw, &msg, 1, vf);
5388 vf_data->last_nack = jiffies;
5389 }
5390}
5391
5392static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5393{
5394 struct pci_dev *pdev = adapter->pdev;
5395 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5396 struct e1000_hw *hw = &adapter->hw;
5397 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5398 s32 retval;
5399
5400 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5401
5402 if (retval) {
5403 /* if receive failed revoke VF CTS stats and restart init */
5404 dev_err(&pdev->dev, "Error receiving message from VF\n");
5405 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5406 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5407 return;
5408 goto out;
5409 }
5410
5411 /* this is a message we already processed, do nothing */
5412 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5413 return;
5414
5415 /*
5416 * until the vf completes a reset it should not be
5417 * allowed to start any configuration.
5418 */
5419
5420 if (msgbuf[0] == E1000_VF_RESET) {
5421 igb_vf_reset_msg(adapter, vf);
5422 return;
5423 }
5424
5425 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5426 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5427 return;
5428 retval = -1;
5429 goto out;
5430 }
5431
5432 switch ((msgbuf[0] & 0xFFFF)) {
5433 case E1000_VF_SET_MAC_ADDR:
5434 retval = -EINVAL;
5435 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5436 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5437 else
5438 dev_warn(&pdev->dev,
5439 "VF %d attempted to override administratively "
5440 "set MAC address\nReload the VF driver to "
5441 "resume operations\n", vf);
5442 break;
5443 case E1000_VF_SET_PROMISC:
5444 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5445 break;
5446 case E1000_VF_SET_MULTICAST:
5447 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5448 break;
5449 case E1000_VF_SET_LPE:
5450 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5451 break;
5452 case E1000_VF_SET_VLAN:
5453 retval = -1;
5454 if (vf_data->pf_vlan)
5455 dev_warn(&pdev->dev,
5456 "VF %d attempted to override administratively "
5457 "set VLAN tag\nReload the VF driver to "
5458 "resume operations\n", vf);
5459 else
5460 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5461 break;
5462 default:
5463 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5464 retval = -1;
5465 break;
5466 }
5467
5468 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5469out:
5470 /* notify the VF of the results of what it sent us */
5471 if (retval)
5472 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5473 else
5474 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5475
5476 igb_write_mbx(hw, msgbuf, 1, vf);
5477}
5478
5479static void igb_msg_task(struct igb_adapter *adapter)
5480{
5481 struct e1000_hw *hw = &adapter->hw;
5482 u32 vf;
5483
5484 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5485 /* process any reset requests */
5486 if (!igb_check_for_rst(hw, vf))
5487 igb_vf_reset_event(adapter, vf);
5488
5489 /* process any messages pending */
5490 if (!igb_check_for_msg(hw, vf))
5491 igb_rcv_msg_from_vf(adapter, vf);
5492
5493 /* process any acks */
5494 if (!igb_check_for_ack(hw, vf))
5495 igb_rcv_ack_from_vf(adapter, vf);
5496 }
5497}
5498
5499/**
5500 * igb_set_uta - Set unicast filter table address
5501 * @adapter: board private structure
5502 *
5503 * The unicast table address is a register array of 32-bit registers.
5504 * The table is meant to be used in a way similar to how the MTA is used
5505 * however due to certain limitations in the hardware it is necessary to
5506 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5507 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
5508 **/
5509static void igb_set_uta(struct igb_adapter *adapter)
5510{
5511 struct e1000_hw *hw = &adapter->hw;
5512 int i;
5513
5514 /* The UTA table only exists on 82576 hardware and newer */
5515 if (hw->mac.type < e1000_82576)
5516 return;
5517
5518 /* we only need to do this if VMDq is enabled */
5519 if (!adapter->vfs_allocated_count)
5520 return;
5521
5522 for (i = 0; i < hw->mac.uta_reg_count; i++)
5523 array_wr32(E1000_UTA, i, ~0);
5524}
5525
5526/**
5527 * igb_intr_msi - Interrupt Handler
5528 * @irq: interrupt number
5529 * @data: pointer to a network interface device structure
5530 **/
5531static irqreturn_t igb_intr_msi(int irq, void *data)
5532{
5533 struct igb_adapter *adapter = data;
5534 struct igb_q_vector *q_vector = adapter->q_vector[0];
5535 struct e1000_hw *hw = &adapter->hw;
5536 /* read ICR disables interrupts using IAM */
5537 u32 icr = rd32(E1000_ICR);
5538
5539 igb_write_itr(q_vector);
5540
5541 if (icr & E1000_ICR_DRSTA)
5542 schedule_work(&adapter->reset_task);
5543
5544 if (icr & E1000_ICR_DOUTSYNC) {
5545 /* HW is reporting DMA is out of sync */
5546 adapter->stats.doosync++;
5547 }
5548
5549 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5550 hw->mac.get_link_status = 1;
5551 if (!test_bit(__IGB_DOWN, &adapter->state))
5552 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5553 }
5554
5555 napi_schedule(&q_vector->napi);
5556
5557 return IRQ_HANDLED;
5558}
5559
5560/**
5561 * igb_intr - Legacy Interrupt Handler
5562 * @irq: interrupt number
5563 * @data: pointer to a network interface device structure
5564 **/
5565static irqreturn_t igb_intr(int irq, void *data)
5566{
5567 struct igb_adapter *adapter = data;
5568 struct igb_q_vector *q_vector = adapter->q_vector[0];
5569 struct e1000_hw *hw = &adapter->hw;
5570 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5571 * need for the IMC write */
5572 u32 icr = rd32(E1000_ICR);
5573
5574 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5575 * not set, then the adapter didn't send an interrupt */
5576 if (!(icr & E1000_ICR_INT_ASSERTED))
5577 return IRQ_NONE;
5578
5579 igb_write_itr(q_vector);
5580
5581 if (icr & E1000_ICR_DRSTA)
5582 schedule_work(&adapter->reset_task);
5583
5584 if (icr & E1000_ICR_DOUTSYNC) {
5585 /* HW is reporting DMA is out of sync */
5586 adapter->stats.doosync++;
5587 }
5588
5589 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5590 hw->mac.get_link_status = 1;
5591 /* guard against interrupt when we're going down */
5592 if (!test_bit(__IGB_DOWN, &adapter->state))
5593 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5594 }
5595
5596 napi_schedule(&q_vector->napi);
5597
5598 return IRQ_HANDLED;
5599}
5600
5601static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5602{
5603 struct igb_adapter *adapter = q_vector->adapter;
5604 struct e1000_hw *hw = &adapter->hw;
5605
5606 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5607 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5608 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5609 igb_set_itr(q_vector);
5610 else
5611 igb_update_ring_itr(q_vector);
5612 }
5613
5614 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5615 if (adapter->msix_entries)
5616 wr32(E1000_EIMS, q_vector->eims_value);
5617 else
5618 igb_irq_enable(adapter);
5619 }
5620}
5621
5622/**
5623 * igb_poll - NAPI Rx polling callback
5624 * @napi: napi polling structure
5625 * @budget: count of how many packets we should handle
5626 **/
5627static int igb_poll(struct napi_struct *napi, int budget)
5628{
5629 struct igb_q_vector *q_vector = container_of(napi,
5630 struct igb_q_vector,
5631 napi);
5632 bool clean_complete = true;
5633
5634#ifdef CONFIG_IGB_DCA
5635 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5636 igb_update_dca(q_vector);
5637#endif
5638 if (q_vector->tx.ring)
5639 clean_complete = igb_clean_tx_irq(q_vector);
5640
5641 if (q_vector->rx.ring)
5642 clean_complete &= igb_clean_rx_irq(q_vector, budget);
5643
5644 /* If all work not completed, return budget and keep polling */
5645 if (!clean_complete)
5646 return budget;
5647
5648 /* If not enough Rx work done, exit the polling mode */
5649 napi_complete(napi);
5650 igb_ring_irq_enable(q_vector);
5651
5652 return 0;
5653}
5654
5655#ifdef CONFIG_IGB_PTP
5656/**
5657 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5658 * @q_vector: pointer to q_vector containing needed info
5659 * @buffer: pointer to igb_tx_buffer structure
5660 *
5661 * If we were asked to do hardware stamping and such a time stamp is
5662 * available, then it must have been for this skb here because we only
5663 * allow only one such packet into the queue.
5664 */
5665static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5666 struct igb_tx_buffer *buffer_info)
5667{
5668 struct igb_adapter *adapter = q_vector->adapter;
5669 struct e1000_hw *hw = &adapter->hw;
5670 struct skb_shared_hwtstamps shhwtstamps;
5671 u64 regval;
5672
5673 /* if skb does not support hw timestamp or TX stamp not valid exit */
5674 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
5675 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5676 return;
5677
5678 regval = rd32(E1000_TXSTMPL);
5679 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5680
5681 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5682 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5683}
5684
5685#endif
5686/**
5687 * igb_clean_tx_irq - Reclaim resources after transmit completes
5688 * @q_vector: pointer to q_vector containing needed info
5689 * returns true if ring is completely cleaned
5690 **/
5691static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5692{
5693 struct igb_adapter *adapter = q_vector->adapter;
5694 struct igb_ring *tx_ring = q_vector->tx.ring;
5695 struct igb_tx_buffer *tx_buffer;
5696 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5697 unsigned int total_bytes = 0, total_packets = 0;
5698 unsigned int budget = q_vector->tx.work_limit;
5699 unsigned int i = tx_ring->next_to_clean;
5700
5701 if (test_bit(__IGB_DOWN, &adapter->state))
5702 return true;
5703
5704 tx_buffer = &tx_ring->tx_buffer_info[i];
5705 tx_desc = IGB_TX_DESC(tx_ring, i);
5706 i -= tx_ring->count;
5707
5708 for (; budget; budget--) {
5709 eop_desc = tx_buffer->next_to_watch;
5710
5711 /* prevent any other reads prior to eop_desc */
5712 rmb();
5713
5714 /* if next_to_watch is not set then there is no work pending */
5715 if (!eop_desc)
5716 break;
5717
5718 /* if DD is not set pending work has not been completed */
5719 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5720 break;
5721
5722 /* clear next_to_watch to prevent false hangs */
5723 tx_buffer->next_to_watch = NULL;
5724
5725 /* update the statistics for this packet */
5726 total_bytes += tx_buffer->bytecount;
5727 total_packets += tx_buffer->gso_segs;
5728
5729#ifdef CONFIG_IGB_PTP
5730 /* retrieve hardware timestamp */
5731 igb_tx_hwtstamp(q_vector, tx_buffer);
5732
5733#endif
5734 /* free the skb */
5735 dev_kfree_skb_any(tx_buffer->skb);
5736 tx_buffer->skb = NULL;
5737
5738 /* unmap skb header data */
5739 dma_unmap_single(tx_ring->dev,
5740 tx_buffer->dma,
5741 tx_buffer->length,
5742 DMA_TO_DEVICE);
5743
5744 /* clear last DMA location and unmap remaining buffers */
5745 while (tx_desc != eop_desc) {
5746 tx_buffer->dma = 0;
5747
5748 tx_buffer++;
5749 tx_desc++;
5750 i++;
5751 if (unlikely(!i)) {
5752 i -= tx_ring->count;
5753 tx_buffer = tx_ring->tx_buffer_info;
5754 tx_desc = IGB_TX_DESC(tx_ring, 0);
5755 }
5756
5757 /* unmap any remaining paged data */
5758 if (tx_buffer->dma) {
5759 dma_unmap_page(tx_ring->dev,
5760 tx_buffer->dma,
5761 tx_buffer->length,
5762 DMA_TO_DEVICE);
5763 }
5764 }
5765
5766 /* clear last DMA location */
5767 tx_buffer->dma = 0;
5768
5769 /* move us one more past the eop_desc for start of next pkt */
5770 tx_buffer++;
5771 tx_desc++;
5772 i++;
5773 if (unlikely(!i)) {
5774 i -= tx_ring->count;
5775 tx_buffer = tx_ring->tx_buffer_info;
5776 tx_desc = IGB_TX_DESC(tx_ring, 0);
5777 }
5778 }
5779
5780 netdev_tx_completed_queue(txring_txq(tx_ring),
5781 total_packets, total_bytes);
5782 i += tx_ring->count;
5783 tx_ring->next_to_clean = i;
5784 u64_stats_update_begin(&tx_ring->tx_syncp);
5785 tx_ring->tx_stats.bytes += total_bytes;
5786 tx_ring->tx_stats.packets += total_packets;
5787 u64_stats_update_end(&tx_ring->tx_syncp);
5788 q_vector->tx.total_bytes += total_bytes;
5789 q_vector->tx.total_packets += total_packets;
5790
5791 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
5792 struct e1000_hw *hw = &adapter->hw;
5793
5794 eop_desc = tx_buffer->next_to_watch;
5795
5796 /* Detect a transmit hang in hardware, this serializes the
5797 * check with the clearing of time_stamp and movement of i */
5798 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5799 if (eop_desc &&
5800 time_after(jiffies, tx_buffer->time_stamp +
5801 (adapter->tx_timeout_factor * HZ)) &&
5802 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5803
5804 /* detected Tx unit hang */
5805 dev_err(tx_ring->dev,
5806 "Detected Tx Unit Hang\n"
5807 " Tx Queue <%d>\n"
5808 " TDH <%x>\n"
5809 " TDT <%x>\n"
5810 " next_to_use <%x>\n"
5811 " next_to_clean <%x>\n"
5812 "buffer_info[next_to_clean]\n"
5813 " time_stamp <%lx>\n"
5814 " next_to_watch <%p>\n"
5815 " jiffies <%lx>\n"
5816 " desc.status <%x>\n",
5817 tx_ring->queue_index,
5818 rd32(E1000_TDH(tx_ring->reg_idx)),
5819 readl(tx_ring->tail),
5820 tx_ring->next_to_use,
5821 tx_ring->next_to_clean,
5822 tx_buffer->time_stamp,
5823 eop_desc,
5824 jiffies,
5825 eop_desc->wb.status);
5826 netif_stop_subqueue(tx_ring->netdev,
5827 tx_ring->queue_index);
5828
5829 /* we are about to reset, no point in enabling stuff */
5830 return true;
5831 }
5832 }
5833
5834 if (unlikely(total_packets &&
5835 netif_carrier_ok(tx_ring->netdev) &&
5836 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5837 /* Make sure that anybody stopping the queue after this
5838 * sees the new next_to_clean.
5839 */
5840 smp_mb();
5841 if (__netif_subqueue_stopped(tx_ring->netdev,
5842 tx_ring->queue_index) &&
5843 !(test_bit(__IGB_DOWN, &adapter->state))) {
5844 netif_wake_subqueue(tx_ring->netdev,
5845 tx_ring->queue_index);
5846
5847 u64_stats_update_begin(&tx_ring->tx_syncp);
5848 tx_ring->tx_stats.restart_queue++;
5849 u64_stats_update_end(&tx_ring->tx_syncp);
5850 }
5851 }
5852
5853 return !!budget;
5854}
5855
5856static inline void igb_rx_checksum(struct igb_ring *ring,
5857 union e1000_adv_rx_desc *rx_desc,
5858 struct sk_buff *skb)
5859{
5860 skb_checksum_none_assert(skb);
5861
5862 /* Ignore Checksum bit is set */
5863 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
5864 return;
5865
5866 /* Rx checksum disabled via ethtool */
5867 if (!(ring->netdev->features & NETIF_F_RXCSUM))
5868 return;
5869
5870 /* TCP/UDP checksum error bit is set */
5871 if (igb_test_staterr(rx_desc,
5872 E1000_RXDEXT_STATERR_TCPE |
5873 E1000_RXDEXT_STATERR_IPE)) {
5874 /*
5875 * work around errata with sctp packets where the TCPE aka
5876 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5877 * packets, (aka let the stack check the crc32c)
5878 */
5879 if (!((skb->len == 60) &&
5880 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
5881 u64_stats_update_begin(&ring->rx_syncp);
5882 ring->rx_stats.csum_err++;
5883 u64_stats_update_end(&ring->rx_syncp);
5884 }
5885 /* let the stack verify checksum errors */
5886 return;
5887 }
5888 /* It must be a TCP or UDP packet with a valid checksum */
5889 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5890 E1000_RXD_STAT_UDPCS))
5891 skb->ip_summed = CHECKSUM_UNNECESSARY;
5892
5893 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5894 le32_to_cpu(rx_desc->wb.upper.status_error));
5895}
5896
5897static inline void igb_rx_hash(struct igb_ring *ring,
5898 union e1000_adv_rx_desc *rx_desc,
5899 struct sk_buff *skb)
5900{
5901 if (ring->netdev->features & NETIF_F_RXHASH)
5902 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5903}
5904
5905#ifdef CONFIG_IGB_PTP
5906static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
5907 union e1000_adv_rx_desc *rx_desc,
5908 struct sk_buff *skb)
5909{
5910 struct igb_adapter *adapter = q_vector->adapter;
5911 struct e1000_hw *hw = &adapter->hw;
5912 u64 regval;
5913
5914 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
5915 E1000_RXDADV_STAT_TS))
5916 return;
5917
5918 /*
5919 * If this bit is set, then the RX registers contain the time stamp. No
5920 * other packet will be time stamped until we read these registers, so
5921 * read the registers to make them available again. Because only one
5922 * packet can be time stamped at a time, we know that the register
5923 * values must belong to this one here and therefore we don't need to
5924 * compare any of the additional attributes stored for it.
5925 *
5926 * If nothing went wrong, then it should have a shared tx_flags that we
5927 * can turn into a skb_shared_hwtstamps.
5928 */
5929 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
5930 u32 *stamp = (u32 *)skb->data;
5931 regval = le32_to_cpu(*(stamp + 2));
5932 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5933 skb_pull(skb, IGB_TS_HDR_LEN);
5934 } else {
5935 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5936 return;
5937
5938 regval = rd32(E1000_RXSTMPL);
5939 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5940 }
5941
5942 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5943}
5944
5945#endif
5946static void igb_rx_vlan(struct igb_ring *ring,
5947 union e1000_adv_rx_desc *rx_desc,
5948 struct sk_buff *skb)
5949{
5950 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
5951 u16 vid;
5952 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
5953 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
5954 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
5955 else
5956 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
5957
5958 __vlan_hwaccel_put_tag(skb, vid);
5959 }
5960}
5961
5962static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
5963{
5964 /* HW will not DMA in data larger than the given buffer, even if it
5965 * parses the (NFS, of course) header to be larger. In that case, it
5966 * fills the header buffer and spills the rest into the page.
5967 */
5968 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5969 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5970 if (hlen > IGB_RX_HDR_LEN)
5971 hlen = IGB_RX_HDR_LEN;
5972 return hlen;
5973}
5974
5975static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
5976{
5977 struct igb_ring *rx_ring = q_vector->rx.ring;
5978 union e1000_adv_rx_desc *rx_desc;
5979 const int current_node = numa_node_id();
5980 unsigned int total_bytes = 0, total_packets = 0;
5981 u16 cleaned_count = igb_desc_unused(rx_ring);
5982 u16 i = rx_ring->next_to_clean;
5983
5984 rx_desc = IGB_RX_DESC(rx_ring, i);
5985
5986 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
5987 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
5988 struct sk_buff *skb = buffer_info->skb;
5989 union e1000_adv_rx_desc *next_rxd;
5990
5991 buffer_info->skb = NULL;
5992 prefetch(skb->data);
5993
5994 i++;
5995 if (i == rx_ring->count)
5996 i = 0;
5997
5998 next_rxd = IGB_RX_DESC(rx_ring, i);
5999 prefetch(next_rxd);
6000
6001 /*
6002 * This memory barrier is needed to keep us from reading
6003 * any other fields out of the rx_desc until we know the
6004 * RXD_STAT_DD bit is set
6005 */
6006 rmb();
6007
6008 if (!skb_is_nonlinear(skb)) {
6009 __skb_put(skb, igb_get_hlen(rx_desc));
6010 dma_unmap_single(rx_ring->dev, buffer_info->dma,
6011 IGB_RX_HDR_LEN,
6012 DMA_FROM_DEVICE);
6013 buffer_info->dma = 0;
6014 }
6015
6016 if (rx_desc->wb.upper.length) {
6017 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
6018
6019 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
6020 buffer_info->page,
6021 buffer_info->page_offset,
6022 length);
6023
6024 skb->len += length;
6025 skb->data_len += length;
6026 skb->truesize += PAGE_SIZE / 2;
6027
6028 if ((page_count(buffer_info->page) != 1) ||
6029 (page_to_nid(buffer_info->page) != current_node))
6030 buffer_info->page = NULL;
6031 else
6032 get_page(buffer_info->page);
6033
6034 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6035 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6036 buffer_info->page_dma = 0;
6037 }
6038
6039 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
6040 struct igb_rx_buffer *next_buffer;
6041 next_buffer = &rx_ring->rx_buffer_info[i];
6042 buffer_info->skb = next_buffer->skb;
6043 buffer_info->dma = next_buffer->dma;
6044 next_buffer->skb = skb;
6045 next_buffer->dma = 0;
6046 goto next_desc;
6047 }
6048
6049 if (unlikely((igb_test_staterr(rx_desc,
6050 E1000_RXDEXT_ERR_FRAME_ERR_MASK))
6051 && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
6052 dev_kfree_skb_any(skb);
6053 goto next_desc;
6054 }
6055
6056#ifdef CONFIG_IGB_PTP
6057 igb_rx_hwtstamp(q_vector, rx_desc, skb);
6058#endif
6059 igb_rx_hash(rx_ring, rx_desc, skb);
6060 igb_rx_checksum(rx_ring, rx_desc, skb);
6061 igb_rx_vlan(rx_ring, rx_desc, skb);
6062
6063 total_bytes += skb->len;
6064 total_packets++;
6065
6066 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6067
6068 napi_gro_receive(&q_vector->napi, skb);
6069
6070 budget--;
6071next_desc:
6072 if (!budget)
6073 break;
6074
6075 cleaned_count++;
6076 /* return some buffers to hardware, one at a time is too slow */
6077 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
6078 igb_alloc_rx_buffers(rx_ring, cleaned_count);
6079 cleaned_count = 0;
6080 }
6081
6082 /* use prefetched values */
6083 rx_desc = next_rxd;
6084 }
6085
6086 rx_ring->next_to_clean = i;
6087 u64_stats_update_begin(&rx_ring->rx_syncp);
6088 rx_ring->rx_stats.packets += total_packets;
6089 rx_ring->rx_stats.bytes += total_bytes;
6090 u64_stats_update_end(&rx_ring->rx_syncp);
6091 q_vector->rx.total_packets += total_packets;
6092 q_vector->rx.total_bytes += total_bytes;
6093
6094 if (cleaned_count)
6095 igb_alloc_rx_buffers(rx_ring, cleaned_count);
6096
6097 return !!budget;
6098}
6099
6100static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
6101 struct igb_rx_buffer *bi)
6102{
6103 struct sk_buff *skb = bi->skb;
6104 dma_addr_t dma = bi->dma;
6105
6106 if (dma)
6107 return true;
6108
6109 if (likely(!skb)) {
6110 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6111 IGB_RX_HDR_LEN);
6112 bi->skb = skb;
6113 if (!skb) {
6114 rx_ring->rx_stats.alloc_failed++;
6115 return false;
6116 }
6117
6118 /* initialize skb for ring */
6119 skb_record_rx_queue(skb, rx_ring->queue_index);
6120 }
6121
6122 dma = dma_map_single(rx_ring->dev, skb->data,
6123 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6124
6125 if (dma_mapping_error(rx_ring->dev, dma)) {
6126 rx_ring->rx_stats.alloc_failed++;
6127 return false;
6128 }
6129
6130 bi->dma = dma;
6131 return true;
6132}
6133
6134static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
6135 struct igb_rx_buffer *bi)
6136{
6137 struct page *page = bi->page;
6138 dma_addr_t page_dma = bi->page_dma;
6139 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6140
6141 if (page_dma)
6142 return true;
6143
6144 if (!page) {
6145 page = alloc_page(GFP_ATOMIC | __GFP_COLD);
6146 bi->page = page;
6147 if (unlikely(!page)) {
6148 rx_ring->rx_stats.alloc_failed++;
6149 return false;
6150 }
6151 }
6152
6153 page_dma = dma_map_page(rx_ring->dev, page,
6154 page_offset, PAGE_SIZE / 2,
6155 DMA_FROM_DEVICE);
6156
6157 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6158 rx_ring->rx_stats.alloc_failed++;
6159 return false;
6160 }
6161
6162 bi->page_dma = page_dma;
6163 bi->page_offset = page_offset;
6164 return true;
6165}
6166
6167/**
6168 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
6169 * @adapter: address of board private structure
6170 **/
6171void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
6172{
6173 union e1000_adv_rx_desc *rx_desc;
6174 struct igb_rx_buffer *bi;
6175 u16 i = rx_ring->next_to_use;
6176
6177 rx_desc = IGB_RX_DESC(rx_ring, i);
6178 bi = &rx_ring->rx_buffer_info[i];
6179 i -= rx_ring->count;
6180
6181 while (cleaned_count--) {
6182 if (!igb_alloc_mapped_skb(rx_ring, bi))
6183 break;
6184
6185 /* Refresh the desc even if buffer_addrs didn't change
6186 * because each write-back erases this info. */
6187 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
6188
6189 if (!igb_alloc_mapped_page(rx_ring, bi))
6190 break;
6191
6192 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
6193
6194 rx_desc++;
6195 bi++;
6196 i++;
6197 if (unlikely(!i)) {
6198 rx_desc = IGB_RX_DESC(rx_ring, 0);
6199 bi = rx_ring->rx_buffer_info;
6200 i -= rx_ring->count;
6201 }
6202
6203 /* clear the hdr_addr for the next_to_use descriptor */
6204 rx_desc->read.hdr_addr = 0;
6205 }
6206
6207 i += rx_ring->count;
6208
6209 if (rx_ring->next_to_use != i) {
6210 rx_ring->next_to_use = i;
6211
6212 /* Force memory writes to complete before letting h/w
6213 * know there are new descriptors to fetch. (Only
6214 * applicable for weak-ordered memory model archs,
6215 * such as IA-64). */
6216 wmb();
6217 writel(i, rx_ring->tail);
6218 }
6219}
6220
6221/**
6222 * igb_mii_ioctl -
6223 * @netdev:
6224 * @ifreq:
6225 * @cmd:
6226 **/
6227static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6228{
6229 struct igb_adapter *adapter = netdev_priv(netdev);
6230 struct mii_ioctl_data *data = if_mii(ifr);
6231
6232 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6233 return -EOPNOTSUPP;
6234
6235 switch (cmd) {
6236 case SIOCGMIIPHY:
6237 data->phy_id = adapter->hw.phy.addr;
6238 break;
6239 case SIOCGMIIREG:
6240 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6241 &data->val_out))
6242 return -EIO;
6243 break;
6244 case SIOCSMIIREG:
6245 default:
6246 return -EOPNOTSUPP;
6247 }
6248 return 0;
6249}
6250
6251/**
6252 * igb_hwtstamp_ioctl - control hardware time stamping
6253 * @netdev:
6254 * @ifreq:
6255 * @cmd:
6256 *
6257 * Outgoing time stamping can be enabled and disabled. Play nice and
6258 * disable it when requested, although it shouldn't case any overhead
6259 * when no packet needs it. At most one packet in the queue may be
6260 * marked for time stamping, otherwise it would be impossible to tell
6261 * for sure to which packet the hardware time stamp belongs.
6262 *
6263 * Incoming time stamping has to be configured via the hardware
6264 * filters. Not all combinations are supported, in particular event
6265 * type has to be specified. Matching the kind of event packet is
6266 * not supported, with the exception of "all V2 events regardless of
6267 * level 2 or 4".
6268 *
6269 **/
6270static int igb_hwtstamp_ioctl(struct net_device *netdev,
6271 struct ifreq *ifr, int cmd)
6272{
6273 struct igb_adapter *adapter = netdev_priv(netdev);
6274 struct e1000_hw *hw = &adapter->hw;
6275 struct hwtstamp_config config;
6276 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6277 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6278 u32 tsync_rx_cfg = 0;
6279 bool is_l4 = false;
6280 bool is_l2 = false;
6281 u32 regval;
6282
6283 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6284 return -EFAULT;
6285
6286 /* reserved for future extensions */
6287 if (config.flags)
6288 return -EINVAL;
6289
6290 switch (config.tx_type) {
6291 case HWTSTAMP_TX_OFF:
6292 tsync_tx_ctl = 0;
6293 case HWTSTAMP_TX_ON:
6294 break;
6295 default:
6296 return -ERANGE;
6297 }
6298
6299 switch (config.rx_filter) {
6300 case HWTSTAMP_FILTER_NONE:
6301 tsync_rx_ctl = 0;
6302 break;
6303 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6304 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6305 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6306 case HWTSTAMP_FILTER_ALL:
6307 /*
6308 * register TSYNCRXCFG must be set, therefore it is not
6309 * possible to time stamp both Sync and Delay_Req messages
6310 * => fall back to time stamping all packets
6311 */
6312 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6313 config.rx_filter = HWTSTAMP_FILTER_ALL;
6314 break;
6315 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
6316 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6317 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
6318 is_l4 = true;
6319 break;
6320 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
6321 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6322 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
6323 is_l4 = true;
6324 break;
6325 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6326 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
6327 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6328 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
6329 is_l2 = true;
6330 is_l4 = true;
6331 config.rx_filter = HWTSTAMP_FILTER_SOME;
6332 break;
6333 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6334 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
6335 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6336 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
6337 is_l2 = true;
6338 is_l4 = true;
6339 config.rx_filter = HWTSTAMP_FILTER_SOME;
6340 break;
6341 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6342 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6343 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
6344 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
6345 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
6346 is_l2 = true;
6347 is_l4 = true;
6348 break;
6349 default:
6350 return -ERANGE;
6351 }
6352
6353 if (hw->mac.type == e1000_82575) {
6354 if (tsync_rx_ctl | tsync_tx_ctl)
6355 return -EINVAL;
6356 return 0;
6357 }
6358
6359 /*
6360 * Per-packet timestamping only works if all packets are
6361 * timestamped, so enable timestamping in all packets as
6362 * long as one rx filter was configured.
6363 */
6364 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
6365 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6366 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6367 }
6368
6369 /* enable/disable TX */
6370 regval = rd32(E1000_TSYNCTXCTL);
6371 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6372 regval |= tsync_tx_ctl;
6373 wr32(E1000_TSYNCTXCTL, regval);
6374
6375 /* enable/disable RX */
6376 regval = rd32(E1000_TSYNCRXCTL);
6377 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6378 regval |= tsync_rx_ctl;
6379 wr32(E1000_TSYNCRXCTL, regval);
6380
6381 /* define which PTP packets are time stamped */
6382 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
6383
6384 /* define ethertype filter for timestamped packets */
6385 if (is_l2)
6386 wr32(E1000_ETQF(3),
6387 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6388 E1000_ETQF_1588 | /* enable timestamping */
6389 ETH_P_1588)); /* 1588 eth protocol type */
6390 else
6391 wr32(E1000_ETQF(3), 0);
6392
6393#define PTP_PORT 319
6394 /* L4 Queue Filter[3]: filter by destination port and protocol */
6395 if (is_l4) {
6396 u32 ftqf = (IPPROTO_UDP /* UDP */
6397 | E1000_FTQF_VF_BP /* VF not compared */
6398 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6399 | E1000_FTQF_MASK); /* mask all inputs */
6400 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6401
6402 wr32(E1000_IMIR(3), htons(PTP_PORT));
6403 wr32(E1000_IMIREXT(3),
6404 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6405 if (hw->mac.type == e1000_82576) {
6406 /* enable source port check */
6407 wr32(E1000_SPQF(3), htons(PTP_PORT));
6408 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6409 }
6410 wr32(E1000_FTQF(3), ftqf);
6411 } else {
6412 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6413 }
6414 wrfl();
6415
6416 adapter->hwtstamp_config = config;
6417
6418 /* clear TX/RX time stamp registers, just to be sure */
6419 regval = rd32(E1000_TXSTMPH);
6420 regval = rd32(E1000_RXSTMPH);
6421
6422 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6423 -EFAULT : 0;
6424}
6425
6426/**
6427 * igb_ioctl -
6428 * @netdev:
6429 * @ifreq:
6430 * @cmd:
6431 **/
6432static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6433{
6434 switch (cmd) {
6435 case SIOCGMIIPHY:
6436 case SIOCGMIIREG:
6437 case SIOCSMIIREG:
6438 return igb_mii_ioctl(netdev, ifr, cmd);
6439 case SIOCSHWTSTAMP:
6440 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
6441 default:
6442 return -EOPNOTSUPP;
6443 }
6444}
6445
6446s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6447{
6448 struct igb_adapter *adapter = hw->back;
6449 u16 cap_offset;
6450
6451 cap_offset = adapter->pdev->pcie_cap;
6452 if (!cap_offset)
6453 return -E1000_ERR_CONFIG;
6454
6455 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6456
6457 return 0;
6458}
6459
6460s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6461{
6462 struct igb_adapter *adapter = hw->back;
6463 u16 cap_offset;
6464
6465 cap_offset = adapter->pdev->pcie_cap;
6466 if (!cap_offset)
6467 return -E1000_ERR_CONFIG;
6468
6469 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6470
6471 return 0;
6472}
6473
6474static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
6475{
6476 struct igb_adapter *adapter = netdev_priv(netdev);
6477 struct e1000_hw *hw = &adapter->hw;
6478 u32 ctrl, rctl;
6479 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
6480
6481 if (enable) {
6482 /* enable VLAN tag insert/strip */
6483 ctrl = rd32(E1000_CTRL);
6484 ctrl |= E1000_CTRL_VME;
6485 wr32(E1000_CTRL, ctrl);
6486
6487 /* Disable CFI check */
6488 rctl = rd32(E1000_RCTL);
6489 rctl &= ~E1000_RCTL_CFIEN;
6490 wr32(E1000_RCTL, rctl);
6491 } else {
6492 /* disable VLAN tag insert/strip */
6493 ctrl = rd32(E1000_CTRL);
6494 ctrl &= ~E1000_CTRL_VME;
6495 wr32(E1000_CTRL, ctrl);
6496 }
6497
6498 igb_rlpml_set(adapter);
6499}
6500
6501static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6502{
6503 struct igb_adapter *adapter = netdev_priv(netdev);
6504 struct e1000_hw *hw = &adapter->hw;
6505 int pf_id = adapter->vfs_allocated_count;
6506
6507 /* attempt to add filter to vlvf array */
6508 igb_vlvf_set(adapter, vid, true, pf_id);
6509
6510 /* add the filter since PF can receive vlans w/o entry in vlvf */
6511 igb_vfta_set(hw, vid, true);
6512
6513 set_bit(vid, adapter->active_vlans);
6514
6515 return 0;
6516}
6517
6518static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6519{
6520 struct igb_adapter *adapter = netdev_priv(netdev);
6521 struct e1000_hw *hw = &adapter->hw;
6522 int pf_id = adapter->vfs_allocated_count;
6523 s32 err;
6524
6525 /* remove vlan from VLVF table array */
6526 err = igb_vlvf_set(adapter, vid, false, pf_id);
6527
6528 /* if vid was not present in VLVF just remove it from table */
6529 if (err)
6530 igb_vfta_set(hw, vid, false);
6531
6532 clear_bit(vid, adapter->active_vlans);
6533
6534 return 0;
6535}
6536
6537static void igb_restore_vlan(struct igb_adapter *adapter)
6538{
6539 u16 vid;
6540
6541 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6542
6543 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6544 igb_vlan_rx_add_vid(adapter->netdev, vid);
6545}
6546
6547int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
6548{
6549 struct pci_dev *pdev = adapter->pdev;
6550 struct e1000_mac_info *mac = &adapter->hw.mac;
6551
6552 mac->autoneg = 0;
6553
6554 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6555 * for the switch() below to work */
6556 if ((spd & 1) || (dplx & ~1))
6557 goto err_inval;
6558
6559 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6560 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6561 spd != SPEED_1000 &&
6562 dplx != DUPLEX_FULL)
6563 goto err_inval;
6564
6565 switch (spd + dplx) {
6566 case SPEED_10 + DUPLEX_HALF:
6567 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6568 break;
6569 case SPEED_10 + DUPLEX_FULL:
6570 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6571 break;
6572 case SPEED_100 + DUPLEX_HALF:
6573 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6574 break;
6575 case SPEED_100 + DUPLEX_FULL:
6576 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6577 break;
6578 case SPEED_1000 + DUPLEX_FULL:
6579 mac->autoneg = 1;
6580 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6581 break;
6582 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6583 default:
6584 goto err_inval;
6585 }
6586 return 0;
6587
6588err_inval:
6589 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6590 return -EINVAL;
6591}
6592
6593static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6594 bool runtime)
6595{
6596 struct net_device *netdev = pci_get_drvdata(pdev);
6597 struct igb_adapter *adapter = netdev_priv(netdev);
6598 struct e1000_hw *hw = &adapter->hw;
6599 u32 ctrl, rctl, status;
6600 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
6601#ifdef CONFIG_PM
6602 int retval = 0;
6603#endif
6604
6605 netif_device_detach(netdev);
6606
6607 if (netif_running(netdev))
6608 __igb_close(netdev, true);
6609
6610 igb_clear_interrupt_scheme(adapter);
6611
6612#ifdef CONFIG_PM
6613 retval = pci_save_state(pdev);
6614 if (retval)
6615 return retval;
6616#endif
6617
6618 status = rd32(E1000_STATUS);
6619 if (status & E1000_STATUS_LU)
6620 wufc &= ~E1000_WUFC_LNKC;
6621
6622 if (wufc) {
6623 igb_setup_rctl(adapter);
6624 igb_set_rx_mode(netdev);
6625
6626 /* turn on all-multi mode if wake on multicast is enabled */
6627 if (wufc & E1000_WUFC_MC) {
6628 rctl = rd32(E1000_RCTL);
6629 rctl |= E1000_RCTL_MPE;
6630 wr32(E1000_RCTL, rctl);
6631 }
6632
6633 ctrl = rd32(E1000_CTRL);
6634 /* advertise wake from D3Cold */
6635 #define E1000_CTRL_ADVD3WUC 0x00100000
6636 /* phy power management enable */
6637 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6638 ctrl |= E1000_CTRL_ADVD3WUC;
6639 wr32(E1000_CTRL, ctrl);
6640
6641 /* Allow time for pending master requests to run */
6642 igb_disable_pcie_master(hw);
6643
6644 wr32(E1000_WUC, E1000_WUC_PME_EN);
6645 wr32(E1000_WUFC, wufc);
6646 } else {
6647 wr32(E1000_WUC, 0);
6648 wr32(E1000_WUFC, 0);
6649 }
6650
6651 *enable_wake = wufc || adapter->en_mng_pt;
6652 if (!*enable_wake)
6653 igb_power_down_link(adapter);
6654 else
6655 igb_power_up_link(adapter);
6656
6657 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6658 * would have already happened in close and is redundant. */
6659 igb_release_hw_control(adapter);
6660
6661 pci_disable_device(pdev);
6662
6663 return 0;
6664}
6665
6666#ifdef CONFIG_PM
6667#ifdef CONFIG_PM_SLEEP
6668static int igb_suspend(struct device *dev)
6669{
6670 int retval;
6671 bool wake;
6672 struct pci_dev *pdev = to_pci_dev(dev);
6673
6674 retval = __igb_shutdown(pdev, &wake, 0);
6675 if (retval)
6676 return retval;
6677
6678 if (wake) {
6679 pci_prepare_to_sleep(pdev);
6680 } else {
6681 pci_wake_from_d3(pdev, false);
6682 pci_set_power_state(pdev, PCI_D3hot);
6683 }
6684
6685 return 0;
6686}
6687#endif /* CONFIG_PM_SLEEP */
6688
6689static int igb_resume(struct device *dev)
6690{
6691 struct pci_dev *pdev = to_pci_dev(dev);
6692 struct net_device *netdev = pci_get_drvdata(pdev);
6693 struct igb_adapter *adapter = netdev_priv(netdev);
6694 struct e1000_hw *hw = &adapter->hw;
6695 u32 err;
6696
6697 pci_set_power_state(pdev, PCI_D0);
6698 pci_restore_state(pdev);
6699 pci_save_state(pdev);
6700
6701 err = pci_enable_device_mem(pdev);
6702 if (err) {
6703 dev_err(&pdev->dev,
6704 "igb: Cannot enable PCI device from suspend\n");
6705 return err;
6706 }
6707 pci_set_master(pdev);
6708
6709 pci_enable_wake(pdev, PCI_D3hot, 0);
6710 pci_enable_wake(pdev, PCI_D3cold, 0);
6711
6712 if (igb_init_interrupt_scheme(adapter)) {
6713 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6714 return -ENOMEM;
6715 }
6716
6717 igb_reset(adapter);
6718
6719 /* let the f/w know that the h/w is now under the control of the
6720 * driver. */
6721 igb_get_hw_control(adapter);
6722
6723 wr32(E1000_WUS, ~0);
6724
6725 if (netdev->flags & IFF_UP) {
6726 err = __igb_open(netdev, true);
6727 if (err)
6728 return err;
6729 }
6730
6731 netif_device_attach(netdev);
6732 return 0;
6733}
6734
6735#ifdef CONFIG_PM_RUNTIME
6736static int igb_runtime_idle(struct device *dev)
6737{
6738 struct pci_dev *pdev = to_pci_dev(dev);
6739 struct net_device *netdev = pci_get_drvdata(pdev);
6740 struct igb_adapter *adapter = netdev_priv(netdev);
6741
6742 if (!igb_has_link(adapter))
6743 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6744
6745 return -EBUSY;
6746}
6747
6748static int igb_runtime_suspend(struct device *dev)
6749{
6750 struct pci_dev *pdev = to_pci_dev(dev);
6751 int retval;
6752 bool wake;
6753
6754 retval = __igb_shutdown(pdev, &wake, 1);
6755 if (retval)
6756 return retval;
6757
6758 if (wake) {
6759 pci_prepare_to_sleep(pdev);
6760 } else {
6761 pci_wake_from_d3(pdev, false);
6762 pci_set_power_state(pdev, PCI_D3hot);
6763 }
6764
6765 return 0;
6766}
6767
6768static int igb_runtime_resume(struct device *dev)
6769{
6770 return igb_resume(dev);
6771}
6772#endif /* CONFIG_PM_RUNTIME */
6773#endif
6774
6775static void igb_shutdown(struct pci_dev *pdev)
6776{
6777 bool wake;
6778
6779 __igb_shutdown(pdev, &wake, 0);
6780
6781 if (system_state == SYSTEM_POWER_OFF) {
6782 pci_wake_from_d3(pdev, wake);
6783 pci_set_power_state(pdev, PCI_D3hot);
6784 }
6785}
6786
6787#ifdef CONFIG_NET_POLL_CONTROLLER
6788/*
6789 * Polling 'interrupt' - used by things like netconsole to send skbs
6790 * without having to re-enable interrupts. It's not called while
6791 * the interrupt routine is executing.
6792 */
6793static void igb_netpoll(struct net_device *netdev)
6794{
6795 struct igb_adapter *adapter = netdev_priv(netdev);
6796 struct e1000_hw *hw = &adapter->hw;
6797 struct igb_q_vector *q_vector;
6798 int i;
6799
6800 for (i = 0; i < adapter->num_q_vectors; i++) {
6801 q_vector = adapter->q_vector[i];
6802 if (adapter->msix_entries)
6803 wr32(E1000_EIMC, q_vector->eims_value);
6804 else
6805 igb_irq_disable(adapter);
6806 napi_schedule(&q_vector->napi);
6807 }
6808}
6809#endif /* CONFIG_NET_POLL_CONTROLLER */
6810
6811/**
6812 * igb_io_error_detected - called when PCI error is detected
6813 * @pdev: Pointer to PCI device
6814 * @state: The current pci connection state
6815 *
6816 * This function is called after a PCI bus error affecting
6817 * this device has been detected.
6818 */
6819static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6820 pci_channel_state_t state)
6821{
6822 struct net_device *netdev = pci_get_drvdata(pdev);
6823 struct igb_adapter *adapter = netdev_priv(netdev);
6824
6825 netif_device_detach(netdev);
6826
6827 if (state == pci_channel_io_perm_failure)
6828 return PCI_ERS_RESULT_DISCONNECT;
6829
6830 if (netif_running(netdev))
6831 igb_down(adapter);
6832 pci_disable_device(pdev);
6833
6834 /* Request a slot slot reset. */
6835 return PCI_ERS_RESULT_NEED_RESET;
6836}
6837
6838/**
6839 * igb_io_slot_reset - called after the pci bus has been reset.
6840 * @pdev: Pointer to PCI device
6841 *
6842 * Restart the card from scratch, as if from a cold-boot. Implementation
6843 * resembles the first-half of the igb_resume routine.
6844 */
6845static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6846{
6847 struct net_device *netdev = pci_get_drvdata(pdev);
6848 struct igb_adapter *adapter = netdev_priv(netdev);
6849 struct e1000_hw *hw = &adapter->hw;
6850 pci_ers_result_t result;
6851 int err;
6852
6853 if (pci_enable_device_mem(pdev)) {
6854 dev_err(&pdev->dev,
6855 "Cannot re-enable PCI device after reset.\n");
6856 result = PCI_ERS_RESULT_DISCONNECT;
6857 } else {
6858 pci_set_master(pdev);
6859 pci_restore_state(pdev);
6860 pci_save_state(pdev);
6861
6862 pci_enable_wake(pdev, PCI_D3hot, 0);
6863 pci_enable_wake(pdev, PCI_D3cold, 0);
6864
6865 igb_reset(adapter);
6866 wr32(E1000_WUS, ~0);
6867 result = PCI_ERS_RESULT_RECOVERED;
6868 }
6869
6870 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6871 if (err) {
6872 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6873 "failed 0x%0x\n", err);
6874 /* non-fatal, continue */
6875 }
6876
6877 return result;
6878}
6879
6880/**
6881 * igb_io_resume - called when traffic can start flowing again.
6882 * @pdev: Pointer to PCI device
6883 *
6884 * This callback is called when the error recovery driver tells us that
6885 * its OK to resume normal operation. Implementation resembles the
6886 * second-half of the igb_resume routine.
6887 */
6888static void igb_io_resume(struct pci_dev *pdev)
6889{
6890 struct net_device *netdev = pci_get_drvdata(pdev);
6891 struct igb_adapter *adapter = netdev_priv(netdev);
6892
6893 if (netif_running(netdev)) {
6894 if (igb_up(adapter)) {
6895 dev_err(&pdev->dev, "igb_up failed after reset\n");
6896 return;
6897 }
6898 }
6899
6900 netif_device_attach(netdev);
6901
6902 /* let the f/w know that the h/w is now under the control of the
6903 * driver. */
6904 igb_get_hw_control(adapter);
6905}
6906
6907static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6908 u8 qsel)
6909{
6910 u32 rar_low, rar_high;
6911 struct e1000_hw *hw = &adapter->hw;
6912
6913 /* HW expects these in little endian so we reverse the byte order
6914 * from network order (big endian) to little endian
6915 */
6916 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6917 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6918 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6919
6920 /* Indicate to hardware the Address is Valid. */
6921 rar_high |= E1000_RAH_AV;
6922
6923 if (hw->mac.type == e1000_82575)
6924 rar_high |= E1000_RAH_POOL_1 * qsel;
6925 else
6926 rar_high |= E1000_RAH_POOL_1 << qsel;
6927
6928 wr32(E1000_RAL(index), rar_low);
6929 wrfl();
6930 wr32(E1000_RAH(index), rar_high);
6931 wrfl();
6932}
6933
6934static int igb_set_vf_mac(struct igb_adapter *adapter,
6935 int vf, unsigned char *mac_addr)
6936{
6937 struct e1000_hw *hw = &adapter->hw;
6938 /* VF MAC addresses start at end of receive addresses and moves
6939 * torwards the first, as a result a collision should not be possible */
6940 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6941
6942 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6943
6944 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6945
6946 return 0;
6947}
6948
6949static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6950{
6951 struct igb_adapter *adapter = netdev_priv(netdev);
6952 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6953 return -EINVAL;
6954 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6955 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6956 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6957 " change effective.");
6958 if (test_bit(__IGB_DOWN, &adapter->state)) {
6959 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6960 " but the PF device is not up.\n");
6961 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6962 " attempting to use the VF device.\n");
6963 }
6964 return igb_set_vf_mac(adapter, vf, mac);
6965}
6966
6967static int igb_link_mbps(int internal_link_speed)
6968{
6969 switch (internal_link_speed) {
6970 case SPEED_100:
6971 return 100;
6972 case SPEED_1000:
6973 return 1000;
6974 default:
6975 return 0;
6976 }
6977}
6978
6979static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6980 int link_speed)
6981{
6982 int rf_dec, rf_int;
6983 u32 bcnrc_val;
6984
6985 if (tx_rate != 0) {
6986 /* Calculate the rate factor values to set */
6987 rf_int = link_speed / tx_rate;
6988 rf_dec = (link_speed - (rf_int * tx_rate));
6989 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6990
6991 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6992 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6993 E1000_RTTBCNRC_RF_INT_MASK);
6994 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6995 } else {
6996 bcnrc_val = 0;
6997 }
6998
6999 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
7000 wr32(E1000_RTTBCNRC, bcnrc_val);
7001}
7002
7003static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
7004{
7005 int actual_link_speed, i;
7006 bool reset_rate = false;
7007
7008 /* VF TX rate limit was not set or not supported */
7009 if ((adapter->vf_rate_link_speed == 0) ||
7010 (adapter->hw.mac.type != e1000_82576))
7011 return;
7012
7013 actual_link_speed = igb_link_mbps(adapter->link_speed);
7014 if (actual_link_speed != adapter->vf_rate_link_speed) {
7015 reset_rate = true;
7016 adapter->vf_rate_link_speed = 0;
7017 dev_info(&adapter->pdev->dev,
7018 "Link speed has been changed. VF Transmit "
7019 "rate is disabled\n");
7020 }
7021
7022 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7023 if (reset_rate)
7024 adapter->vf_data[i].tx_rate = 0;
7025
7026 igb_set_vf_rate_limit(&adapter->hw, i,
7027 adapter->vf_data[i].tx_rate,
7028 actual_link_speed);
7029 }
7030}
7031
7032static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
7033{
7034 struct igb_adapter *adapter = netdev_priv(netdev);
7035 struct e1000_hw *hw = &adapter->hw;
7036 int actual_link_speed;
7037
7038 if (hw->mac.type != e1000_82576)
7039 return -EOPNOTSUPP;
7040
7041 actual_link_speed = igb_link_mbps(adapter->link_speed);
7042 if ((vf >= adapter->vfs_allocated_count) ||
7043 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7044 (tx_rate < 0) || (tx_rate > actual_link_speed))
7045 return -EINVAL;
7046
7047 adapter->vf_rate_link_speed = actual_link_speed;
7048 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7049 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7050
7051 return 0;
7052}
7053
7054static int igb_ndo_get_vf_config(struct net_device *netdev,
7055 int vf, struct ifla_vf_info *ivi)
7056{
7057 struct igb_adapter *adapter = netdev_priv(netdev);
7058 if (vf >= adapter->vfs_allocated_count)
7059 return -EINVAL;
7060 ivi->vf = vf;
7061 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
7062 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
7063 ivi->vlan = adapter->vf_data[vf].pf_vlan;
7064 ivi->qos = adapter->vf_data[vf].pf_qos;
7065 return 0;
7066}
7067
7068static void igb_vmm_control(struct igb_adapter *adapter)
7069{
7070 struct e1000_hw *hw = &adapter->hw;
7071 u32 reg;
7072
7073 switch (hw->mac.type) {
7074 case e1000_82575:
7075 case e1000_i210:
7076 case e1000_i211:
7077 default:
7078 /* replication is not supported for 82575 */
7079 return;
7080 case e1000_82576:
7081 /* notify HW that the MAC is adding vlan tags */
7082 reg = rd32(E1000_DTXCTL);
7083 reg |= E1000_DTXCTL_VLAN_ADDED;
7084 wr32(E1000_DTXCTL, reg);
7085 case e1000_82580:
7086 /* enable replication vlan tag stripping */
7087 reg = rd32(E1000_RPLOLR);
7088 reg |= E1000_RPLOLR_STRVLAN;
7089 wr32(E1000_RPLOLR, reg);
7090 case e1000_i350:
7091 /* none of the above registers are supported by i350 */
7092 break;
7093 }
7094
7095 if (adapter->vfs_allocated_count) {
7096 igb_vmdq_set_loopback_pf(hw, true);
7097 igb_vmdq_set_replication_pf(hw, true);
7098 igb_vmdq_set_anti_spoofing_pf(hw, true,
7099 adapter->vfs_allocated_count);
7100 } else {
7101 igb_vmdq_set_loopback_pf(hw, false);
7102 igb_vmdq_set_replication_pf(hw, false);
7103 }
7104}
7105
7106static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
7107{
7108 struct e1000_hw *hw = &adapter->hw;
7109 u32 dmac_thr;
7110 u16 hwm;
7111
7112 if (hw->mac.type > e1000_82580) {
7113 if (adapter->flags & IGB_FLAG_DMAC) {
7114 u32 reg;
7115
7116 /* force threshold to 0. */
7117 wr32(E1000_DMCTXTH, 0);
7118
7119 /*
7120 * DMA Coalescing high water mark needs to be greater
7121 * than the Rx threshold. Set hwm to PBA - max frame
7122 * size in 16B units, capping it at PBA - 6KB.
7123 */
7124 hwm = 64 * pba - adapter->max_frame_size / 16;
7125 if (hwm < 64 * (pba - 6))
7126 hwm = 64 * (pba - 6);
7127 reg = rd32(E1000_FCRTC);
7128 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
7129 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
7130 & E1000_FCRTC_RTH_COAL_MASK);
7131 wr32(E1000_FCRTC, reg);
7132
7133 /*
7134 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
7135 * frame size, capping it at PBA - 10KB.
7136 */
7137 dmac_thr = pba - adapter->max_frame_size / 512;
7138 if (dmac_thr < pba - 10)
7139 dmac_thr = pba - 10;
7140 reg = rd32(E1000_DMACR);
7141 reg &= ~E1000_DMACR_DMACTHR_MASK;
7142 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7143 & E1000_DMACR_DMACTHR_MASK);
7144
7145 /* transition to L0x or L1 if available..*/
7146 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7147
7148 /* watchdog timer= +-1000 usec in 32usec intervals */
7149 reg |= (1000 >> 5);
7150
7151 /* Disable BMC-to-OS Watchdog Enable */
7152 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
7153 wr32(E1000_DMACR, reg);
7154
7155 /*
7156 * no lower threshold to disable
7157 * coalescing(smart fifb)-UTRESH=0
7158 */
7159 wr32(E1000_DMCRTRH, 0);
7160
7161 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7162
7163 wr32(E1000_DMCTLX, reg);
7164
7165 /*
7166 * free space in tx packet buffer to wake from
7167 * DMA coal
7168 */
7169 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7170 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7171
7172 /*
7173 * make low power state decision controlled
7174 * by DMA coal
7175 */
7176 reg = rd32(E1000_PCIEMISC);
7177 reg &= ~E1000_PCIEMISC_LX_DECISION;
7178 wr32(E1000_PCIEMISC, reg);
7179 } /* endif adapter->dmac is not disabled */
7180 } else if (hw->mac.type == e1000_82580) {
7181 u32 reg = rd32(E1000_PCIEMISC);
7182 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7183 wr32(E1000_DMACR, 0);
7184 }
7185}
7186
7187/* igb_main.c */
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2007 - 2018 Intel Corporation. */
3
4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6#include <linux/module.h>
7#include <linux/types.h>
8#include <linux/init.h>
9#include <linux/bitops.h>
10#include <linux/vmalloc.h>
11#include <linux/pagemap.h>
12#include <linux/netdevice.h>
13#include <linux/ipv6.h>
14#include <linux/slab.h>
15#include <net/checksum.h>
16#include <net/ip6_checksum.h>
17#include <net/pkt_sched.h>
18#include <net/pkt_cls.h>
19#include <linux/net_tstamp.h>
20#include <linux/mii.h>
21#include <linux/ethtool.h>
22#include <linux/if.h>
23#include <linux/if_vlan.h>
24#include <linux/pci.h>
25#include <linux/delay.h>
26#include <linux/interrupt.h>
27#include <linux/ip.h>
28#include <linux/tcp.h>
29#include <linux/sctp.h>
30#include <linux/if_ether.h>
31#include <linux/aer.h>
32#include <linux/prefetch.h>
33#include <linux/pm_runtime.h>
34#include <linux/etherdevice.h>
35#ifdef CONFIG_IGB_DCA
36#include <linux/dca.h>
37#endif
38#include <linux/i2c.h>
39#include "igb.h"
40
41enum queue_mode {
42 QUEUE_MODE_STRICT_PRIORITY,
43 QUEUE_MODE_STREAM_RESERVATION,
44};
45
46enum tx_queue_prio {
47 TX_QUEUE_PRIO_HIGH,
48 TX_QUEUE_PRIO_LOW,
49};
50
51char igb_driver_name[] = "igb";
52static const char igb_driver_string[] =
53 "Intel(R) Gigabit Ethernet Network Driver";
54static const char igb_copyright[] =
55 "Copyright (c) 2007-2014 Intel Corporation.";
56
57static const struct e1000_info *igb_info_tbl[] = {
58 [board_82575] = &e1000_82575_info,
59};
60
61static const struct pci_device_id igb_pci_tbl[] = {
62 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
63 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
97 /* required last entry */
98 {0, }
99};
100
101MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
102
103static int igb_setup_all_tx_resources(struct igb_adapter *);
104static int igb_setup_all_rx_resources(struct igb_adapter *);
105static void igb_free_all_tx_resources(struct igb_adapter *);
106static void igb_free_all_rx_resources(struct igb_adapter *);
107static void igb_setup_mrqc(struct igb_adapter *);
108static int igb_probe(struct pci_dev *, const struct pci_device_id *);
109static void igb_remove(struct pci_dev *pdev);
110static int igb_sw_init(struct igb_adapter *);
111int igb_open(struct net_device *);
112int igb_close(struct net_device *);
113static void igb_configure(struct igb_adapter *);
114static void igb_configure_tx(struct igb_adapter *);
115static void igb_configure_rx(struct igb_adapter *);
116static void igb_clean_all_tx_rings(struct igb_adapter *);
117static void igb_clean_all_rx_rings(struct igb_adapter *);
118static void igb_clean_tx_ring(struct igb_ring *);
119static void igb_clean_rx_ring(struct igb_ring *);
120static void igb_set_rx_mode(struct net_device *);
121static void igb_update_phy_info(struct timer_list *);
122static void igb_watchdog(struct timer_list *);
123static void igb_watchdog_task(struct work_struct *);
124static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
125static void igb_get_stats64(struct net_device *dev,
126 struct rtnl_link_stats64 *stats);
127static int igb_change_mtu(struct net_device *, int);
128static int igb_set_mac(struct net_device *, void *);
129static void igb_set_uta(struct igb_adapter *adapter, bool set);
130static irqreturn_t igb_intr(int irq, void *);
131static irqreturn_t igb_intr_msi(int irq, void *);
132static irqreturn_t igb_msix_other(int irq, void *);
133static irqreturn_t igb_msix_ring(int irq, void *);
134#ifdef CONFIG_IGB_DCA
135static void igb_update_dca(struct igb_q_vector *);
136static void igb_setup_dca(struct igb_adapter *);
137#endif /* CONFIG_IGB_DCA */
138static int igb_poll(struct napi_struct *, int);
139static bool igb_clean_tx_irq(struct igb_q_vector *, int);
140static int igb_clean_rx_irq(struct igb_q_vector *, int);
141static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
142static void igb_tx_timeout(struct net_device *, unsigned int txqueue);
143static void igb_reset_task(struct work_struct *);
144static void igb_vlan_mode(struct net_device *netdev,
145 netdev_features_t features);
146static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
147static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
148static void igb_restore_vlan(struct igb_adapter *);
149static void igb_rar_set_index(struct igb_adapter *, u32);
150static void igb_ping_all_vfs(struct igb_adapter *);
151static void igb_msg_task(struct igb_adapter *);
152static void igb_vmm_control(struct igb_adapter *);
153static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
154static void igb_flush_mac_table(struct igb_adapter *);
155static int igb_available_rars(struct igb_adapter *, u8);
156static void igb_set_default_mac_filter(struct igb_adapter *);
157static int igb_uc_sync(struct net_device *, const unsigned char *);
158static int igb_uc_unsync(struct net_device *, const unsigned char *);
159static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
160static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
161static int igb_ndo_set_vf_vlan(struct net_device *netdev,
162 int vf, u16 vlan, u8 qos, __be16 vlan_proto);
163static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
164static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
165 bool setting);
166static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf,
167 bool setting);
168static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
169 struct ifla_vf_info *ivi);
170static void igb_check_vf_rate_limit(struct igb_adapter *);
171static void igb_nfc_filter_exit(struct igb_adapter *adapter);
172static void igb_nfc_filter_restore(struct igb_adapter *adapter);
173
174#ifdef CONFIG_PCI_IOV
175static int igb_vf_configure(struct igb_adapter *adapter, int vf);
176static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs);
177static int igb_disable_sriov(struct pci_dev *dev);
178static int igb_pci_disable_sriov(struct pci_dev *dev);
179#endif
180
181static int igb_suspend(struct device *);
182static int igb_resume(struct device *);
183static int igb_runtime_suspend(struct device *dev);
184static int igb_runtime_resume(struct device *dev);
185static int igb_runtime_idle(struct device *dev);
186static const struct dev_pm_ops igb_pm_ops = {
187 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
188 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
189 igb_runtime_idle)
190};
191static void igb_shutdown(struct pci_dev *);
192static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
193#ifdef CONFIG_IGB_DCA
194static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
195static struct notifier_block dca_notifier = {
196 .notifier_call = igb_notify_dca,
197 .next = NULL,
198 .priority = 0
199};
200#endif
201#ifdef CONFIG_PCI_IOV
202static unsigned int max_vfs;
203module_param(max_vfs, uint, 0);
204MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
205#endif /* CONFIG_PCI_IOV */
206
207static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
208 pci_channel_state_t);
209static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
210static void igb_io_resume(struct pci_dev *);
211
212static const struct pci_error_handlers igb_err_handler = {
213 .error_detected = igb_io_error_detected,
214 .slot_reset = igb_io_slot_reset,
215 .resume = igb_io_resume,
216};
217
218static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
219
220static struct pci_driver igb_driver = {
221 .name = igb_driver_name,
222 .id_table = igb_pci_tbl,
223 .probe = igb_probe,
224 .remove = igb_remove,
225#ifdef CONFIG_PM
226 .driver.pm = &igb_pm_ops,
227#endif
228 .shutdown = igb_shutdown,
229 .sriov_configure = igb_pci_sriov_configure,
230 .err_handler = &igb_err_handler
231};
232
233MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
234MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
235MODULE_LICENSE("GPL v2");
236
237#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
238static int debug = -1;
239module_param(debug, int, 0);
240MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
241
242struct igb_reg_info {
243 u32 ofs;
244 char *name;
245};
246
247static const struct igb_reg_info igb_reg_info_tbl[] = {
248
249 /* General Registers */
250 {E1000_CTRL, "CTRL"},
251 {E1000_STATUS, "STATUS"},
252 {E1000_CTRL_EXT, "CTRL_EXT"},
253
254 /* Interrupt Registers */
255 {E1000_ICR, "ICR"},
256
257 /* RX Registers */
258 {E1000_RCTL, "RCTL"},
259 {E1000_RDLEN(0), "RDLEN"},
260 {E1000_RDH(0), "RDH"},
261 {E1000_RDT(0), "RDT"},
262 {E1000_RXDCTL(0), "RXDCTL"},
263 {E1000_RDBAL(0), "RDBAL"},
264 {E1000_RDBAH(0), "RDBAH"},
265
266 /* TX Registers */
267 {E1000_TCTL, "TCTL"},
268 {E1000_TDBAL(0), "TDBAL"},
269 {E1000_TDBAH(0), "TDBAH"},
270 {E1000_TDLEN(0), "TDLEN"},
271 {E1000_TDH(0), "TDH"},
272 {E1000_TDT(0), "TDT"},
273 {E1000_TXDCTL(0), "TXDCTL"},
274 {E1000_TDFH, "TDFH"},
275 {E1000_TDFT, "TDFT"},
276 {E1000_TDFHS, "TDFHS"},
277 {E1000_TDFPC, "TDFPC"},
278
279 /* List Terminator */
280 {}
281};
282
283/* igb_regdump - register printout routine */
284static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
285{
286 int n = 0;
287 char rname[16];
288 u32 regs[8];
289
290 switch (reginfo->ofs) {
291 case E1000_RDLEN(0):
292 for (n = 0; n < 4; n++)
293 regs[n] = rd32(E1000_RDLEN(n));
294 break;
295 case E1000_RDH(0):
296 for (n = 0; n < 4; n++)
297 regs[n] = rd32(E1000_RDH(n));
298 break;
299 case E1000_RDT(0):
300 for (n = 0; n < 4; n++)
301 regs[n] = rd32(E1000_RDT(n));
302 break;
303 case E1000_RXDCTL(0):
304 for (n = 0; n < 4; n++)
305 regs[n] = rd32(E1000_RXDCTL(n));
306 break;
307 case E1000_RDBAL(0):
308 for (n = 0; n < 4; n++)
309 regs[n] = rd32(E1000_RDBAL(n));
310 break;
311 case E1000_RDBAH(0):
312 for (n = 0; n < 4; n++)
313 regs[n] = rd32(E1000_RDBAH(n));
314 break;
315 case E1000_TDBAL(0):
316 for (n = 0; n < 4; n++)
317 regs[n] = rd32(E1000_RDBAL(n));
318 break;
319 case E1000_TDBAH(0):
320 for (n = 0; n < 4; n++)
321 regs[n] = rd32(E1000_TDBAH(n));
322 break;
323 case E1000_TDLEN(0):
324 for (n = 0; n < 4; n++)
325 regs[n] = rd32(E1000_TDLEN(n));
326 break;
327 case E1000_TDH(0):
328 for (n = 0; n < 4; n++)
329 regs[n] = rd32(E1000_TDH(n));
330 break;
331 case E1000_TDT(0):
332 for (n = 0; n < 4; n++)
333 regs[n] = rd32(E1000_TDT(n));
334 break;
335 case E1000_TXDCTL(0):
336 for (n = 0; n < 4; n++)
337 regs[n] = rd32(E1000_TXDCTL(n));
338 break;
339 default:
340 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
341 return;
342 }
343
344 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
345 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
346 regs[2], regs[3]);
347}
348
349/* igb_dump - Print registers, Tx-rings and Rx-rings */
350static void igb_dump(struct igb_adapter *adapter)
351{
352 struct net_device *netdev = adapter->netdev;
353 struct e1000_hw *hw = &adapter->hw;
354 struct igb_reg_info *reginfo;
355 struct igb_ring *tx_ring;
356 union e1000_adv_tx_desc *tx_desc;
357 struct my_u0 { u64 a; u64 b; } *u0;
358 struct igb_ring *rx_ring;
359 union e1000_adv_rx_desc *rx_desc;
360 u32 staterr;
361 u16 i, n;
362
363 if (!netif_msg_hw(adapter))
364 return;
365
366 /* Print netdevice Info */
367 if (netdev) {
368 dev_info(&adapter->pdev->dev, "Net device Info\n");
369 pr_info("Device Name state trans_start\n");
370 pr_info("%-15s %016lX %016lX\n", netdev->name,
371 netdev->state, dev_trans_start(netdev));
372 }
373
374 /* Print Registers */
375 dev_info(&adapter->pdev->dev, "Register Dump\n");
376 pr_info(" Register Name Value\n");
377 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
378 reginfo->name; reginfo++) {
379 igb_regdump(hw, reginfo);
380 }
381
382 /* Print TX Ring Summary */
383 if (!netdev || !netif_running(netdev))
384 goto exit;
385
386 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
387 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
388 for (n = 0; n < adapter->num_tx_queues; n++) {
389 struct igb_tx_buffer *buffer_info;
390 tx_ring = adapter->tx_ring[n];
391 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
392 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
393 n, tx_ring->next_to_use, tx_ring->next_to_clean,
394 (u64)dma_unmap_addr(buffer_info, dma),
395 dma_unmap_len(buffer_info, len),
396 buffer_info->next_to_watch,
397 (u64)buffer_info->time_stamp);
398 }
399
400 /* Print TX Rings */
401 if (!netif_msg_tx_done(adapter))
402 goto rx_ring_summary;
403
404 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
405
406 /* Transmit Descriptor Formats
407 *
408 * Advanced Transmit Descriptor
409 * +--------------------------------------------------------------+
410 * 0 | Buffer Address [63:0] |
411 * +--------------------------------------------------------------+
412 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
413 * +--------------------------------------------------------------+
414 * 63 46 45 40 39 38 36 35 32 31 24 15 0
415 */
416
417 for (n = 0; n < adapter->num_tx_queues; n++) {
418 tx_ring = adapter->tx_ring[n];
419 pr_info("------------------------------------\n");
420 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
421 pr_info("------------------------------------\n");
422 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n");
423
424 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
425 const char *next_desc;
426 struct igb_tx_buffer *buffer_info;
427 tx_desc = IGB_TX_DESC(tx_ring, i);
428 buffer_info = &tx_ring->tx_buffer_info[i];
429 u0 = (struct my_u0 *)tx_desc;
430 if (i == tx_ring->next_to_use &&
431 i == tx_ring->next_to_clean)
432 next_desc = " NTC/U";
433 else if (i == tx_ring->next_to_use)
434 next_desc = " NTU";
435 else if (i == tx_ring->next_to_clean)
436 next_desc = " NTC";
437 else
438 next_desc = "";
439
440 pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n",
441 i, le64_to_cpu(u0->a),
442 le64_to_cpu(u0->b),
443 (u64)dma_unmap_addr(buffer_info, dma),
444 dma_unmap_len(buffer_info, len),
445 buffer_info->next_to_watch,
446 (u64)buffer_info->time_stamp,
447 buffer_info->skb, next_desc);
448
449 if (netif_msg_pktdata(adapter) && buffer_info->skb)
450 print_hex_dump(KERN_INFO, "",
451 DUMP_PREFIX_ADDRESS,
452 16, 1, buffer_info->skb->data,
453 dma_unmap_len(buffer_info, len),
454 true);
455 }
456 }
457
458 /* Print RX Rings Summary */
459rx_ring_summary:
460 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
461 pr_info("Queue [NTU] [NTC]\n");
462 for (n = 0; n < adapter->num_rx_queues; n++) {
463 rx_ring = adapter->rx_ring[n];
464 pr_info(" %5d %5X %5X\n",
465 n, rx_ring->next_to_use, rx_ring->next_to_clean);
466 }
467
468 /* Print RX Rings */
469 if (!netif_msg_rx_status(adapter))
470 goto exit;
471
472 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
473
474 /* Advanced Receive Descriptor (Read) Format
475 * 63 1 0
476 * +-----------------------------------------------------+
477 * 0 | Packet Buffer Address [63:1] |A0/NSE|
478 * +----------------------------------------------+------+
479 * 8 | Header Buffer Address [63:1] | DD |
480 * +-----------------------------------------------------+
481 *
482 *
483 * Advanced Receive Descriptor (Write-Back) Format
484 *
485 * 63 48 47 32 31 30 21 20 17 16 4 3 0
486 * +------------------------------------------------------+
487 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
488 * | Checksum Ident | | | | Type | Type |
489 * +------------------------------------------------------+
490 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
491 * +------------------------------------------------------+
492 * 63 48 47 32 31 20 19 0
493 */
494
495 for (n = 0; n < adapter->num_rx_queues; n++) {
496 rx_ring = adapter->rx_ring[n];
497 pr_info("------------------------------------\n");
498 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
499 pr_info("------------------------------------\n");
500 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
501 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
502
503 for (i = 0; i < rx_ring->count; i++) {
504 const char *next_desc;
505 struct igb_rx_buffer *buffer_info;
506 buffer_info = &rx_ring->rx_buffer_info[i];
507 rx_desc = IGB_RX_DESC(rx_ring, i);
508 u0 = (struct my_u0 *)rx_desc;
509 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
510
511 if (i == rx_ring->next_to_use)
512 next_desc = " NTU";
513 else if (i == rx_ring->next_to_clean)
514 next_desc = " NTC";
515 else
516 next_desc = "";
517
518 if (staterr & E1000_RXD_STAT_DD) {
519 /* Descriptor Done */
520 pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n",
521 "RWB", i,
522 le64_to_cpu(u0->a),
523 le64_to_cpu(u0->b),
524 next_desc);
525 } else {
526 pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n",
527 "R ", i,
528 le64_to_cpu(u0->a),
529 le64_to_cpu(u0->b),
530 (u64)buffer_info->dma,
531 next_desc);
532
533 if (netif_msg_pktdata(adapter) &&
534 buffer_info->dma && buffer_info->page) {
535 print_hex_dump(KERN_INFO, "",
536 DUMP_PREFIX_ADDRESS,
537 16, 1,
538 page_address(buffer_info->page) +
539 buffer_info->page_offset,
540 igb_rx_bufsz(rx_ring), true);
541 }
542 }
543 }
544 }
545
546exit:
547 return;
548}
549
550/**
551 * igb_get_i2c_data - Reads the I2C SDA data bit
552 * @hw: pointer to hardware structure
553 * @i2cctl: Current value of I2CCTL register
554 *
555 * Returns the I2C data bit value
556 **/
557static int igb_get_i2c_data(void *data)
558{
559 struct igb_adapter *adapter = (struct igb_adapter *)data;
560 struct e1000_hw *hw = &adapter->hw;
561 s32 i2cctl = rd32(E1000_I2CPARAMS);
562
563 return !!(i2cctl & E1000_I2C_DATA_IN);
564}
565
566/**
567 * igb_set_i2c_data - Sets the I2C data bit
568 * @data: pointer to hardware structure
569 * @state: I2C data value (0 or 1) to set
570 *
571 * Sets the I2C data bit
572 **/
573static void igb_set_i2c_data(void *data, int state)
574{
575 struct igb_adapter *adapter = (struct igb_adapter *)data;
576 struct e1000_hw *hw = &adapter->hw;
577 s32 i2cctl = rd32(E1000_I2CPARAMS);
578
579 if (state)
580 i2cctl |= E1000_I2C_DATA_OUT;
581 else
582 i2cctl &= ~E1000_I2C_DATA_OUT;
583
584 i2cctl &= ~E1000_I2C_DATA_OE_N;
585 i2cctl |= E1000_I2C_CLK_OE_N;
586 wr32(E1000_I2CPARAMS, i2cctl);
587 wrfl();
588
589}
590
591/**
592 * igb_set_i2c_clk - Sets the I2C SCL clock
593 * @data: pointer to hardware structure
594 * @state: state to set clock
595 *
596 * Sets the I2C clock line to state
597 **/
598static void igb_set_i2c_clk(void *data, int state)
599{
600 struct igb_adapter *adapter = (struct igb_adapter *)data;
601 struct e1000_hw *hw = &adapter->hw;
602 s32 i2cctl = rd32(E1000_I2CPARAMS);
603
604 if (state) {
605 i2cctl |= E1000_I2C_CLK_OUT;
606 i2cctl &= ~E1000_I2C_CLK_OE_N;
607 } else {
608 i2cctl &= ~E1000_I2C_CLK_OUT;
609 i2cctl &= ~E1000_I2C_CLK_OE_N;
610 }
611 wr32(E1000_I2CPARAMS, i2cctl);
612 wrfl();
613}
614
615/**
616 * igb_get_i2c_clk - Gets the I2C SCL clock state
617 * @data: pointer to hardware structure
618 *
619 * Gets the I2C clock state
620 **/
621static int igb_get_i2c_clk(void *data)
622{
623 struct igb_adapter *adapter = (struct igb_adapter *)data;
624 struct e1000_hw *hw = &adapter->hw;
625 s32 i2cctl = rd32(E1000_I2CPARAMS);
626
627 return !!(i2cctl & E1000_I2C_CLK_IN);
628}
629
630static const struct i2c_algo_bit_data igb_i2c_algo = {
631 .setsda = igb_set_i2c_data,
632 .setscl = igb_set_i2c_clk,
633 .getsda = igb_get_i2c_data,
634 .getscl = igb_get_i2c_clk,
635 .udelay = 5,
636 .timeout = 20,
637};
638
639/**
640 * igb_get_hw_dev - return device
641 * @hw: pointer to hardware structure
642 *
643 * used by hardware layer to print debugging information
644 **/
645struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
646{
647 struct igb_adapter *adapter = hw->back;
648 return adapter->netdev;
649}
650
651/**
652 * igb_init_module - Driver Registration Routine
653 *
654 * igb_init_module is the first routine called when the driver is
655 * loaded. All it does is register with the PCI subsystem.
656 **/
657static int __init igb_init_module(void)
658{
659 int ret;
660
661 pr_info("%s\n", igb_driver_string);
662 pr_info("%s\n", igb_copyright);
663
664#ifdef CONFIG_IGB_DCA
665 dca_register_notify(&dca_notifier);
666#endif
667 ret = pci_register_driver(&igb_driver);
668 return ret;
669}
670
671module_init(igb_init_module);
672
673/**
674 * igb_exit_module - Driver Exit Cleanup Routine
675 *
676 * igb_exit_module is called just before the driver is removed
677 * from memory.
678 **/
679static void __exit igb_exit_module(void)
680{
681#ifdef CONFIG_IGB_DCA
682 dca_unregister_notify(&dca_notifier);
683#endif
684 pci_unregister_driver(&igb_driver);
685}
686
687module_exit(igb_exit_module);
688
689#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
690/**
691 * igb_cache_ring_register - Descriptor ring to register mapping
692 * @adapter: board private structure to initialize
693 *
694 * Once we know the feature-set enabled for the device, we'll cache
695 * the register offset the descriptor ring is assigned to.
696 **/
697static void igb_cache_ring_register(struct igb_adapter *adapter)
698{
699 int i = 0, j = 0;
700 u32 rbase_offset = adapter->vfs_allocated_count;
701
702 switch (adapter->hw.mac.type) {
703 case e1000_82576:
704 /* The queues are allocated for virtualization such that VF 0
705 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
706 * In order to avoid collision we start at the first free queue
707 * and continue consuming queues in the same sequence
708 */
709 if (adapter->vfs_allocated_count) {
710 for (; i < adapter->rss_queues; i++)
711 adapter->rx_ring[i]->reg_idx = rbase_offset +
712 Q_IDX_82576(i);
713 }
714 fallthrough;
715 case e1000_82575:
716 case e1000_82580:
717 case e1000_i350:
718 case e1000_i354:
719 case e1000_i210:
720 case e1000_i211:
721 default:
722 for (; i < adapter->num_rx_queues; i++)
723 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
724 for (; j < adapter->num_tx_queues; j++)
725 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
726 break;
727 }
728}
729
730u32 igb_rd32(struct e1000_hw *hw, u32 reg)
731{
732 struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
733 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
734 u32 value = 0;
735
736 if (E1000_REMOVED(hw_addr))
737 return ~value;
738
739 value = readl(&hw_addr[reg]);
740
741 /* reads should not return all F's */
742 if (!(~value) && (!reg || !(~readl(hw_addr)))) {
743 struct net_device *netdev = igb->netdev;
744 hw->hw_addr = NULL;
745 netdev_err(netdev, "PCIe link lost\n");
746 WARN(pci_device_is_present(igb->pdev),
747 "igb: Failed to read reg 0x%x!\n", reg);
748 }
749
750 return value;
751}
752
753/**
754 * igb_write_ivar - configure ivar for given MSI-X vector
755 * @hw: pointer to the HW structure
756 * @msix_vector: vector number we are allocating to a given ring
757 * @index: row index of IVAR register to write within IVAR table
758 * @offset: column offset of in IVAR, should be multiple of 8
759 *
760 * This function is intended to handle the writing of the IVAR register
761 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
762 * each containing an cause allocation for an Rx and Tx ring, and a
763 * variable number of rows depending on the number of queues supported.
764 **/
765static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
766 int index, int offset)
767{
768 u32 ivar = array_rd32(E1000_IVAR0, index);
769
770 /* clear any bits that are currently set */
771 ivar &= ~((u32)0xFF << offset);
772
773 /* write vector and valid bit */
774 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
775
776 array_wr32(E1000_IVAR0, index, ivar);
777}
778
779#define IGB_N0_QUEUE -1
780static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
781{
782 struct igb_adapter *adapter = q_vector->adapter;
783 struct e1000_hw *hw = &adapter->hw;
784 int rx_queue = IGB_N0_QUEUE;
785 int tx_queue = IGB_N0_QUEUE;
786 u32 msixbm = 0;
787
788 if (q_vector->rx.ring)
789 rx_queue = q_vector->rx.ring->reg_idx;
790 if (q_vector->tx.ring)
791 tx_queue = q_vector->tx.ring->reg_idx;
792
793 switch (hw->mac.type) {
794 case e1000_82575:
795 /* The 82575 assigns vectors using a bitmask, which matches the
796 * bitmask for the EICR/EIMS/EIMC registers. To assign one
797 * or more queues to a vector, we write the appropriate bits
798 * into the MSIXBM register for that vector.
799 */
800 if (rx_queue > IGB_N0_QUEUE)
801 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
802 if (tx_queue > IGB_N0_QUEUE)
803 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
804 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
805 msixbm |= E1000_EIMS_OTHER;
806 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
807 q_vector->eims_value = msixbm;
808 break;
809 case e1000_82576:
810 /* 82576 uses a table that essentially consists of 2 columns
811 * with 8 rows. The ordering is column-major so we use the
812 * lower 3 bits as the row index, and the 4th bit as the
813 * column offset.
814 */
815 if (rx_queue > IGB_N0_QUEUE)
816 igb_write_ivar(hw, msix_vector,
817 rx_queue & 0x7,
818 (rx_queue & 0x8) << 1);
819 if (tx_queue > IGB_N0_QUEUE)
820 igb_write_ivar(hw, msix_vector,
821 tx_queue & 0x7,
822 ((tx_queue & 0x8) << 1) + 8);
823 q_vector->eims_value = BIT(msix_vector);
824 break;
825 case e1000_82580:
826 case e1000_i350:
827 case e1000_i354:
828 case e1000_i210:
829 case e1000_i211:
830 /* On 82580 and newer adapters the scheme is similar to 82576
831 * however instead of ordering column-major we have things
832 * ordered row-major. So we traverse the table by using
833 * bit 0 as the column offset, and the remaining bits as the
834 * row index.
835 */
836 if (rx_queue > IGB_N0_QUEUE)
837 igb_write_ivar(hw, msix_vector,
838 rx_queue >> 1,
839 (rx_queue & 0x1) << 4);
840 if (tx_queue > IGB_N0_QUEUE)
841 igb_write_ivar(hw, msix_vector,
842 tx_queue >> 1,
843 ((tx_queue & 0x1) << 4) + 8);
844 q_vector->eims_value = BIT(msix_vector);
845 break;
846 default:
847 BUG();
848 break;
849 }
850
851 /* add q_vector eims value to global eims_enable_mask */
852 adapter->eims_enable_mask |= q_vector->eims_value;
853
854 /* configure q_vector to set itr on first interrupt */
855 q_vector->set_itr = 1;
856}
857
858/**
859 * igb_configure_msix - Configure MSI-X hardware
860 * @adapter: board private structure to initialize
861 *
862 * igb_configure_msix sets up the hardware to properly
863 * generate MSI-X interrupts.
864 **/
865static void igb_configure_msix(struct igb_adapter *adapter)
866{
867 u32 tmp;
868 int i, vector = 0;
869 struct e1000_hw *hw = &adapter->hw;
870
871 adapter->eims_enable_mask = 0;
872
873 /* set vector for other causes, i.e. link changes */
874 switch (hw->mac.type) {
875 case e1000_82575:
876 tmp = rd32(E1000_CTRL_EXT);
877 /* enable MSI-X PBA support*/
878 tmp |= E1000_CTRL_EXT_PBA_CLR;
879
880 /* Auto-Mask interrupts upon ICR read. */
881 tmp |= E1000_CTRL_EXT_EIAME;
882 tmp |= E1000_CTRL_EXT_IRCA;
883
884 wr32(E1000_CTRL_EXT, tmp);
885
886 /* enable msix_other interrupt */
887 array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
888 adapter->eims_other = E1000_EIMS_OTHER;
889
890 break;
891
892 case e1000_82576:
893 case e1000_82580:
894 case e1000_i350:
895 case e1000_i354:
896 case e1000_i210:
897 case e1000_i211:
898 /* Turn on MSI-X capability first, or our settings
899 * won't stick. And it will take days to debug.
900 */
901 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
902 E1000_GPIE_PBA | E1000_GPIE_EIAME |
903 E1000_GPIE_NSICR);
904
905 /* enable msix_other interrupt */
906 adapter->eims_other = BIT(vector);
907 tmp = (vector++ | E1000_IVAR_VALID) << 8;
908
909 wr32(E1000_IVAR_MISC, tmp);
910 break;
911 default:
912 /* do nothing, since nothing else supports MSI-X */
913 break;
914 } /* switch (hw->mac.type) */
915
916 adapter->eims_enable_mask |= adapter->eims_other;
917
918 for (i = 0; i < adapter->num_q_vectors; i++)
919 igb_assign_vector(adapter->q_vector[i], vector++);
920
921 wrfl();
922}
923
924/**
925 * igb_request_msix - Initialize MSI-X interrupts
926 * @adapter: board private structure to initialize
927 *
928 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
929 * kernel.
930 **/
931static int igb_request_msix(struct igb_adapter *adapter)
932{
933 struct net_device *netdev = adapter->netdev;
934 int i, err = 0, vector = 0, free_vector = 0;
935
936 err = request_irq(adapter->msix_entries[vector].vector,
937 igb_msix_other, 0, netdev->name, adapter);
938 if (err)
939 goto err_out;
940
941 for (i = 0; i < adapter->num_q_vectors; i++) {
942 struct igb_q_vector *q_vector = adapter->q_vector[i];
943
944 vector++;
945
946 q_vector->itr_register = adapter->io_addr + E1000_EITR(vector);
947
948 if (q_vector->rx.ring && q_vector->tx.ring)
949 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
950 q_vector->rx.ring->queue_index);
951 else if (q_vector->tx.ring)
952 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
953 q_vector->tx.ring->queue_index);
954 else if (q_vector->rx.ring)
955 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
956 q_vector->rx.ring->queue_index);
957 else
958 sprintf(q_vector->name, "%s-unused", netdev->name);
959
960 err = request_irq(adapter->msix_entries[vector].vector,
961 igb_msix_ring, 0, q_vector->name,
962 q_vector);
963 if (err)
964 goto err_free;
965 }
966
967 igb_configure_msix(adapter);
968 return 0;
969
970err_free:
971 /* free already assigned IRQs */
972 free_irq(adapter->msix_entries[free_vector++].vector, adapter);
973
974 vector--;
975 for (i = 0; i < vector; i++) {
976 free_irq(adapter->msix_entries[free_vector++].vector,
977 adapter->q_vector[i]);
978 }
979err_out:
980 return err;
981}
982
983/**
984 * igb_free_q_vector - Free memory allocated for specific interrupt vector
985 * @adapter: board private structure to initialize
986 * @v_idx: Index of vector to be freed
987 *
988 * This function frees the memory allocated to the q_vector.
989 **/
990static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
991{
992 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
993
994 adapter->q_vector[v_idx] = NULL;
995
996 /* igb_get_stats64() might access the rings on this vector,
997 * we must wait a grace period before freeing it.
998 */
999 if (q_vector)
1000 kfree_rcu(q_vector, rcu);
1001}
1002
1003/**
1004 * igb_reset_q_vector - Reset config for interrupt vector
1005 * @adapter: board private structure to initialize
1006 * @v_idx: Index of vector to be reset
1007 *
1008 * If NAPI is enabled it will delete any references to the
1009 * NAPI struct. This is preparation for igb_free_q_vector.
1010 **/
1011static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
1012{
1013 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1014
1015 /* Coming from igb_set_interrupt_capability, the vectors are not yet
1016 * allocated. So, q_vector is NULL so we should stop here.
1017 */
1018 if (!q_vector)
1019 return;
1020
1021 if (q_vector->tx.ring)
1022 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
1023
1024 if (q_vector->rx.ring)
1025 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
1026
1027 netif_napi_del(&q_vector->napi);
1028
1029}
1030
1031static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
1032{
1033 int v_idx = adapter->num_q_vectors;
1034
1035 if (adapter->flags & IGB_FLAG_HAS_MSIX)
1036 pci_disable_msix(adapter->pdev);
1037 else if (adapter->flags & IGB_FLAG_HAS_MSI)
1038 pci_disable_msi(adapter->pdev);
1039
1040 while (v_idx--)
1041 igb_reset_q_vector(adapter, v_idx);
1042}
1043
1044/**
1045 * igb_free_q_vectors - Free memory allocated for interrupt vectors
1046 * @adapter: board private structure to initialize
1047 *
1048 * This function frees the memory allocated to the q_vectors. In addition if
1049 * NAPI is enabled it will delete any references to the NAPI struct prior
1050 * to freeing the q_vector.
1051 **/
1052static void igb_free_q_vectors(struct igb_adapter *adapter)
1053{
1054 int v_idx = adapter->num_q_vectors;
1055
1056 adapter->num_tx_queues = 0;
1057 adapter->num_rx_queues = 0;
1058 adapter->num_q_vectors = 0;
1059
1060 while (v_idx--) {
1061 igb_reset_q_vector(adapter, v_idx);
1062 igb_free_q_vector(adapter, v_idx);
1063 }
1064}
1065
1066/**
1067 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1068 * @adapter: board private structure to initialize
1069 *
1070 * This function resets the device so that it has 0 Rx queues, Tx queues, and
1071 * MSI-X interrupts allocated.
1072 */
1073static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1074{
1075 igb_free_q_vectors(adapter);
1076 igb_reset_interrupt_capability(adapter);
1077}
1078
1079/**
1080 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1081 * @adapter: board private structure to initialize
1082 * @msix: boolean value of MSIX capability
1083 *
1084 * Attempt to configure interrupts using the best available
1085 * capabilities of the hardware and kernel.
1086 **/
1087static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
1088{
1089 int err;
1090 int numvecs, i;
1091
1092 if (!msix)
1093 goto msi_only;
1094 adapter->flags |= IGB_FLAG_HAS_MSIX;
1095
1096 /* Number of supported queues. */
1097 adapter->num_rx_queues = adapter->rss_queues;
1098 if (adapter->vfs_allocated_count)
1099 adapter->num_tx_queues = 1;
1100 else
1101 adapter->num_tx_queues = adapter->rss_queues;
1102
1103 /* start with one vector for every Rx queue */
1104 numvecs = adapter->num_rx_queues;
1105
1106 /* if Tx handler is separate add 1 for every Tx queue */
1107 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1108 numvecs += adapter->num_tx_queues;
1109
1110 /* store the number of vectors reserved for queues */
1111 adapter->num_q_vectors = numvecs;
1112
1113 /* add 1 vector for link status interrupts */
1114 numvecs++;
1115 for (i = 0; i < numvecs; i++)
1116 adapter->msix_entries[i].entry = i;
1117
1118 err = pci_enable_msix_range(adapter->pdev,
1119 adapter->msix_entries,
1120 numvecs,
1121 numvecs);
1122 if (err > 0)
1123 return;
1124
1125 igb_reset_interrupt_capability(adapter);
1126
1127 /* If we can't do MSI-X, try MSI */
1128msi_only:
1129 adapter->flags &= ~IGB_FLAG_HAS_MSIX;
1130#ifdef CONFIG_PCI_IOV
1131 /* disable SR-IOV for non MSI-X configurations */
1132 if (adapter->vf_data) {
1133 struct e1000_hw *hw = &adapter->hw;
1134 /* disable iov and allow time for transactions to clear */
1135 pci_disable_sriov(adapter->pdev);
1136 msleep(500);
1137
1138 kfree(adapter->vf_mac_list);
1139 adapter->vf_mac_list = NULL;
1140 kfree(adapter->vf_data);
1141 adapter->vf_data = NULL;
1142 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1143 wrfl();
1144 msleep(100);
1145 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1146 }
1147#endif
1148 adapter->vfs_allocated_count = 0;
1149 adapter->rss_queues = 1;
1150 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1151 adapter->num_rx_queues = 1;
1152 adapter->num_tx_queues = 1;
1153 adapter->num_q_vectors = 1;
1154 if (!pci_enable_msi(adapter->pdev))
1155 adapter->flags |= IGB_FLAG_HAS_MSI;
1156}
1157
1158static void igb_add_ring(struct igb_ring *ring,
1159 struct igb_ring_container *head)
1160{
1161 head->ring = ring;
1162 head->count++;
1163}
1164
1165/**
1166 * igb_alloc_q_vector - Allocate memory for a single interrupt vector
1167 * @adapter: board private structure to initialize
1168 * @v_count: q_vectors allocated on adapter, used for ring interleaving
1169 * @v_idx: index of vector in adapter struct
1170 * @txr_count: total number of Tx rings to allocate
1171 * @txr_idx: index of first Tx ring to allocate
1172 * @rxr_count: total number of Rx rings to allocate
1173 * @rxr_idx: index of first Rx ring to allocate
1174 *
1175 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1176 **/
1177static int igb_alloc_q_vector(struct igb_adapter *adapter,
1178 int v_count, int v_idx,
1179 int txr_count, int txr_idx,
1180 int rxr_count, int rxr_idx)
1181{
1182 struct igb_q_vector *q_vector;
1183 struct igb_ring *ring;
1184 int ring_count;
1185 size_t size;
1186
1187 /* igb only supports 1 Tx and/or 1 Rx queue per vector */
1188 if (txr_count > 1 || rxr_count > 1)
1189 return -ENOMEM;
1190
1191 ring_count = txr_count + rxr_count;
1192 size = struct_size(q_vector, ring, ring_count);
1193
1194 /* allocate q_vector and rings */
1195 q_vector = adapter->q_vector[v_idx];
1196 if (!q_vector) {
1197 q_vector = kzalloc(size, GFP_KERNEL);
1198 } else if (size > ksize(q_vector)) {
1199 kfree_rcu(q_vector, rcu);
1200 q_vector = kzalloc(size, GFP_KERNEL);
1201 } else {
1202 memset(q_vector, 0, size);
1203 }
1204 if (!q_vector)
1205 return -ENOMEM;
1206
1207 /* initialize NAPI */
1208 netif_napi_add(adapter->netdev, &q_vector->napi,
1209 igb_poll, 64);
1210
1211 /* tie q_vector and adapter together */
1212 adapter->q_vector[v_idx] = q_vector;
1213 q_vector->adapter = adapter;
1214
1215 /* initialize work limits */
1216 q_vector->tx.work_limit = adapter->tx_work_limit;
1217
1218 /* initialize ITR configuration */
1219 q_vector->itr_register = adapter->io_addr + E1000_EITR(0);
1220 q_vector->itr_val = IGB_START_ITR;
1221
1222 /* initialize pointer to rings */
1223 ring = q_vector->ring;
1224
1225 /* intialize ITR */
1226 if (rxr_count) {
1227 /* rx or rx/tx vector */
1228 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
1229 q_vector->itr_val = adapter->rx_itr_setting;
1230 } else {
1231 /* tx only vector */
1232 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
1233 q_vector->itr_val = adapter->tx_itr_setting;
1234 }
1235
1236 if (txr_count) {
1237 /* assign generic ring traits */
1238 ring->dev = &adapter->pdev->dev;
1239 ring->netdev = adapter->netdev;
1240
1241 /* configure backlink on ring */
1242 ring->q_vector = q_vector;
1243
1244 /* update q_vector Tx values */
1245 igb_add_ring(ring, &q_vector->tx);
1246
1247 /* For 82575, context index must be unique per ring. */
1248 if (adapter->hw.mac.type == e1000_82575)
1249 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
1250
1251 /* apply Tx specific ring traits */
1252 ring->count = adapter->tx_ring_count;
1253 ring->queue_index = txr_idx;
1254
1255 ring->cbs_enable = false;
1256 ring->idleslope = 0;
1257 ring->sendslope = 0;
1258 ring->hicredit = 0;
1259 ring->locredit = 0;
1260
1261 u64_stats_init(&ring->tx_syncp);
1262 u64_stats_init(&ring->tx_syncp2);
1263
1264 /* assign ring to adapter */
1265 adapter->tx_ring[txr_idx] = ring;
1266
1267 /* push pointer to next ring */
1268 ring++;
1269 }
1270
1271 if (rxr_count) {
1272 /* assign generic ring traits */
1273 ring->dev = &adapter->pdev->dev;
1274 ring->netdev = adapter->netdev;
1275
1276 /* configure backlink on ring */
1277 ring->q_vector = q_vector;
1278
1279 /* update q_vector Rx values */
1280 igb_add_ring(ring, &q_vector->rx);
1281
1282 /* set flag indicating ring supports SCTP checksum offload */
1283 if (adapter->hw.mac.type >= e1000_82576)
1284 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1285
1286 /* On i350, i354, i210, and i211, loopback VLAN packets
1287 * have the tag byte-swapped.
1288 */
1289 if (adapter->hw.mac.type >= e1000_i350)
1290 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1291
1292 /* apply Rx specific ring traits */
1293 ring->count = adapter->rx_ring_count;
1294 ring->queue_index = rxr_idx;
1295
1296 u64_stats_init(&ring->rx_syncp);
1297
1298 /* assign ring to adapter */
1299 adapter->rx_ring[rxr_idx] = ring;
1300 }
1301
1302 return 0;
1303}
1304
1305
1306/**
1307 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1308 * @adapter: board private structure to initialize
1309 *
1310 * We allocate one q_vector per queue interrupt. If allocation fails we
1311 * return -ENOMEM.
1312 **/
1313static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1314{
1315 int q_vectors = adapter->num_q_vectors;
1316 int rxr_remaining = adapter->num_rx_queues;
1317 int txr_remaining = adapter->num_tx_queues;
1318 int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1319 int err;
1320
1321 if (q_vectors >= (rxr_remaining + txr_remaining)) {
1322 for (; rxr_remaining; v_idx++) {
1323 err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1324 0, 0, 1, rxr_idx);
1325
1326 if (err)
1327 goto err_out;
1328
1329 /* update counts and index */
1330 rxr_remaining--;
1331 rxr_idx++;
1332 }
1333 }
1334
1335 for (; v_idx < q_vectors; v_idx++) {
1336 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1337 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1338
1339 err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1340 tqpv, txr_idx, rqpv, rxr_idx);
1341
1342 if (err)
1343 goto err_out;
1344
1345 /* update counts and index */
1346 rxr_remaining -= rqpv;
1347 txr_remaining -= tqpv;
1348 rxr_idx++;
1349 txr_idx++;
1350 }
1351
1352 return 0;
1353
1354err_out:
1355 adapter->num_tx_queues = 0;
1356 adapter->num_rx_queues = 0;
1357 adapter->num_q_vectors = 0;
1358
1359 while (v_idx--)
1360 igb_free_q_vector(adapter, v_idx);
1361
1362 return -ENOMEM;
1363}
1364
1365/**
1366 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1367 * @adapter: board private structure to initialize
1368 * @msix: boolean value of MSIX capability
1369 *
1370 * This function initializes the interrupts and allocates all of the queues.
1371 **/
1372static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1373{
1374 struct pci_dev *pdev = adapter->pdev;
1375 int err;
1376
1377 igb_set_interrupt_capability(adapter, msix);
1378
1379 err = igb_alloc_q_vectors(adapter);
1380 if (err) {
1381 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1382 goto err_alloc_q_vectors;
1383 }
1384
1385 igb_cache_ring_register(adapter);
1386
1387 return 0;
1388
1389err_alloc_q_vectors:
1390 igb_reset_interrupt_capability(adapter);
1391 return err;
1392}
1393
1394/**
1395 * igb_request_irq - initialize interrupts
1396 * @adapter: board private structure to initialize
1397 *
1398 * Attempts to configure interrupts using the best available
1399 * capabilities of the hardware and kernel.
1400 **/
1401static int igb_request_irq(struct igb_adapter *adapter)
1402{
1403 struct net_device *netdev = adapter->netdev;
1404 struct pci_dev *pdev = adapter->pdev;
1405 int err = 0;
1406
1407 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1408 err = igb_request_msix(adapter);
1409 if (!err)
1410 goto request_done;
1411 /* fall back to MSI */
1412 igb_free_all_tx_resources(adapter);
1413 igb_free_all_rx_resources(adapter);
1414
1415 igb_clear_interrupt_scheme(adapter);
1416 err = igb_init_interrupt_scheme(adapter, false);
1417 if (err)
1418 goto request_done;
1419
1420 igb_setup_all_tx_resources(adapter);
1421 igb_setup_all_rx_resources(adapter);
1422 igb_configure(adapter);
1423 }
1424
1425 igb_assign_vector(adapter->q_vector[0], 0);
1426
1427 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1428 err = request_irq(pdev->irq, igb_intr_msi, 0,
1429 netdev->name, adapter);
1430 if (!err)
1431 goto request_done;
1432
1433 /* fall back to legacy interrupts */
1434 igb_reset_interrupt_capability(adapter);
1435 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1436 }
1437
1438 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1439 netdev->name, adapter);
1440
1441 if (err)
1442 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1443 err);
1444
1445request_done:
1446 return err;
1447}
1448
1449static void igb_free_irq(struct igb_adapter *adapter)
1450{
1451 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1452 int vector = 0, i;
1453
1454 free_irq(adapter->msix_entries[vector++].vector, adapter);
1455
1456 for (i = 0; i < adapter->num_q_vectors; i++)
1457 free_irq(adapter->msix_entries[vector++].vector,
1458 adapter->q_vector[i]);
1459 } else {
1460 free_irq(adapter->pdev->irq, adapter);
1461 }
1462}
1463
1464/**
1465 * igb_irq_disable - Mask off interrupt generation on the NIC
1466 * @adapter: board private structure
1467 **/
1468static void igb_irq_disable(struct igb_adapter *adapter)
1469{
1470 struct e1000_hw *hw = &adapter->hw;
1471
1472 /* we need to be careful when disabling interrupts. The VFs are also
1473 * mapped into these registers and so clearing the bits can cause
1474 * issues on the VF drivers so we only need to clear what we set
1475 */
1476 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1477 u32 regval = rd32(E1000_EIAM);
1478
1479 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1480 wr32(E1000_EIMC, adapter->eims_enable_mask);
1481 regval = rd32(E1000_EIAC);
1482 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1483 }
1484
1485 wr32(E1000_IAM, 0);
1486 wr32(E1000_IMC, ~0);
1487 wrfl();
1488 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1489 int i;
1490
1491 for (i = 0; i < adapter->num_q_vectors; i++)
1492 synchronize_irq(adapter->msix_entries[i].vector);
1493 } else {
1494 synchronize_irq(adapter->pdev->irq);
1495 }
1496}
1497
1498/**
1499 * igb_irq_enable - Enable default interrupt generation settings
1500 * @adapter: board private structure
1501 **/
1502static void igb_irq_enable(struct igb_adapter *adapter)
1503{
1504 struct e1000_hw *hw = &adapter->hw;
1505
1506 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1507 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1508 u32 regval = rd32(E1000_EIAC);
1509
1510 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1511 regval = rd32(E1000_EIAM);
1512 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1513 wr32(E1000_EIMS, adapter->eims_enable_mask);
1514 if (adapter->vfs_allocated_count) {
1515 wr32(E1000_MBVFIMR, 0xFF);
1516 ims |= E1000_IMS_VMMB;
1517 }
1518 wr32(E1000_IMS, ims);
1519 } else {
1520 wr32(E1000_IMS, IMS_ENABLE_MASK |
1521 E1000_IMS_DRSTA);
1522 wr32(E1000_IAM, IMS_ENABLE_MASK |
1523 E1000_IMS_DRSTA);
1524 }
1525}
1526
1527static void igb_update_mng_vlan(struct igb_adapter *adapter)
1528{
1529 struct e1000_hw *hw = &adapter->hw;
1530 u16 pf_id = adapter->vfs_allocated_count;
1531 u16 vid = adapter->hw.mng_cookie.vlan_id;
1532 u16 old_vid = adapter->mng_vlan_id;
1533
1534 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1535 /* add VID to filter table */
1536 igb_vfta_set(hw, vid, pf_id, true, true);
1537 adapter->mng_vlan_id = vid;
1538 } else {
1539 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1540 }
1541
1542 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1543 (vid != old_vid) &&
1544 !test_bit(old_vid, adapter->active_vlans)) {
1545 /* remove VID from filter table */
1546 igb_vfta_set(hw, vid, pf_id, false, true);
1547 }
1548}
1549
1550/**
1551 * igb_release_hw_control - release control of the h/w to f/w
1552 * @adapter: address of board private structure
1553 *
1554 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1555 * For ASF and Pass Through versions of f/w this means that the
1556 * driver is no longer loaded.
1557 **/
1558static void igb_release_hw_control(struct igb_adapter *adapter)
1559{
1560 struct e1000_hw *hw = &adapter->hw;
1561 u32 ctrl_ext;
1562
1563 /* Let firmware take over control of h/w */
1564 ctrl_ext = rd32(E1000_CTRL_EXT);
1565 wr32(E1000_CTRL_EXT,
1566 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1567}
1568
1569/**
1570 * igb_get_hw_control - get control of the h/w from f/w
1571 * @adapter: address of board private structure
1572 *
1573 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1574 * For ASF and Pass Through versions of f/w this means that
1575 * the driver is loaded.
1576 **/
1577static void igb_get_hw_control(struct igb_adapter *adapter)
1578{
1579 struct e1000_hw *hw = &adapter->hw;
1580 u32 ctrl_ext;
1581
1582 /* Let firmware know the driver has taken over */
1583 ctrl_ext = rd32(E1000_CTRL_EXT);
1584 wr32(E1000_CTRL_EXT,
1585 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1586}
1587
1588static void enable_fqtss(struct igb_adapter *adapter, bool enable)
1589{
1590 struct net_device *netdev = adapter->netdev;
1591 struct e1000_hw *hw = &adapter->hw;
1592
1593 WARN_ON(hw->mac.type != e1000_i210);
1594
1595 if (enable)
1596 adapter->flags |= IGB_FLAG_FQTSS;
1597 else
1598 adapter->flags &= ~IGB_FLAG_FQTSS;
1599
1600 if (netif_running(netdev))
1601 schedule_work(&adapter->reset_task);
1602}
1603
1604static bool is_fqtss_enabled(struct igb_adapter *adapter)
1605{
1606 return (adapter->flags & IGB_FLAG_FQTSS) ? true : false;
1607}
1608
1609static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue,
1610 enum tx_queue_prio prio)
1611{
1612 u32 val;
1613
1614 WARN_ON(hw->mac.type != e1000_i210);
1615 WARN_ON(queue < 0 || queue > 4);
1616
1617 val = rd32(E1000_I210_TXDCTL(queue));
1618
1619 if (prio == TX_QUEUE_PRIO_HIGH)
1620 val |= E1000_TXDCTL_PRIORITY;
1621 else
1622 val &= ~E1000_TXDCTL_PRIORITY;
1623
1624 wr32(E1000_I210_TXDCTL(queue), val);
1625}
1626
1627static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode)
1628{
1629 u32 val;
1630
1631 WARN_ON(hw->mac.type != e1000_i210);
1632 WARN_ON(queue < 0 || queue > 1);
1633
1634 val = rd32(E1000_I210_TQAVCC(queue));
1635
1636 if (mode == QUEUE_MODE_STREAM_RESERVATION)
1637 val |= E1000_TQAVCC_QUEUEMODE;
1638 else
1639 val &= ~E1000_TQAVCC_QUEUEMODE;
1640
1641 wr32(E1000_I210_TQAVCC(queue), val);
1642}
1643
1644static bool is_any_cbs_enabled(struct igb_adapter *adapter)
1645{
1646 int i;
1647
1648 for (i = 0; i < adapter->num_tx_queues; i++) {
1649 if (adapter->tx_ring[i]->cbs_enable)
1650 return true;
1651 }
1652
1653 return false;
1654}
1655
1656static bool is_any_txtime_enabled(struct igb_adapter *adapter)
1657{
1658 int i;
1659
1660 for (i = 0; i < adapter->num_tx_queues; i++) {
1661 if (adapter->tx_ring[i]->launchtime_enable)
1662 return true;
1663 }
1664
1665 return false;
1666}
1667
1668/**
1669 * igb_config_tx_modes - Configure "Qav Tx mode" features on igb
1670 * @adapter: pointer to adapter struct
1671 * @queue: queue number
1672 *
1673 * Configure CBS and Launchtime for a given hardware queue.
1674 * Parameters are retrieved from the correct Tx ring, so
1675 * igb_save_cbs_params() and igb_save_txtime_params() should be used
1676 * for setting those correctly prior to this function being called.
1677 **/
1678static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
1679{
1680 struct igb_ring *ring = adapter->tx_ring[queue];
1681 struct net_device *netdev = adapter->netdev;
1682 struct e1000_hw *hw = &adapter->hw;
1683 u32 tqavcc, tqavctrl;
1684 u16 value;
1685
1686 WARN_ON(hw->mac.type != e1000_i210);
1687 WARN_ON(queue < 0 || queue > 1);
1688
1689 /* If any of the Qav features is enabled, configure queues as SR and
1690 * with HIGH PRIO. If none is, then configure them with LOW PRIO and
1691 * as SP.
1692 */
1693 if (ring->cbs_enable || ring->launchtime_enable) {
1694 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
1695 set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
1696 } else {
1697 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
1698 set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
1699 }
1700
1701 /* If CBS is enabled, set DataTranARB and config its parameters. */
1702 if (ring->cbs_enable || queue == 0) {
1703 /* i210 does not allow the queue 0 to be in the Strict
1704 * Priority mode while the Qav mode is enabled, so,
1705 * instead of disabling strict priority mode, we give
1706 * queue 0 the maximum of credits possible.
1707 *
1708 * See section 8.12.19 of the i210 datasheet, "Note:
1709 * Queue0 QueueMode must be set to 1b when
1710 * TransmitMode is set to Qav."
1711 */
1712 if (queue == 0 && !ring->cbs_enable) {
1713 /* max "linkspeed" idleslope in kbps */
1714 ring->idleslope = 1000000;
1715 ring->hicredit = ETH_FRAME_LEN;
1716 }
1717
1718 /* Always set data transfer arbitration to credit-based
1719 * shaper algorithm on TQAVCTRL if CBS is enabled for any of
1720 * the queues.
1721 */
1722 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1723 tqavctrl |= E1000_TQAVCTRL_DATATRANARB;
1724 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1725
1726 /* According to i210 datasheet section 7.2.7.7, we should set
1727 * the 'idleSlope' field from TQAVCC register following the
1728 * equation:
1729 *
1730 * For 100 Mbps link speed:
1731 *
1732 * value = BW * 0x7735 * 0.2 (E1)
1733 *
1734 * For 1000Mbps link speed:
1735 *
1736 * value = BW * 0x7735 * 2 (E2)
1737 *
1738 * E1 and E2 can be merged into one equation as shown below.
1739 * Note that 'link-speed' is in Mbps.
1740 *
1741 * value = BW * 0x7735 * 2 * link-speed
1742 * -------------- (E3)
1743 * 1000
1744 *
1745 * 'BW' is the percentage bandwidth out of full link speed
1746 * which can be found with the following equation. Note that
1747 * idleSlope here is the parameter from this function which
1748 * is in kbps.
1749 *
1750 * BW = idleSlope
1751 * ----------------- (E4)
1752 * link-speed * 1000
1753 *
1754 * That said, we can come up with a generic equation to
1755 * calculate the value we should set it TQAVCC register by
1756 * replacing 'BW' in E3 by E4. The resulting equation is:
1757 *
1758 * value = idleSlope * 0x7735 * 2 * link-speed
1759 * ----------------- -------------- (E5)
1760 * link-speed * 1000 1000
1761 *
1762 * 'link-speed' is present in both sides of the fraction so
1763 * it is canceled out. The final equation is the following:
1764 *
1765 * value = idleSlope * 61034
1766 * ----------------- (E6)
1767 * 1000000
1768 *
1769 * NOTE: For i210, given the above, we can see that idleslope
1770 * is represented in 16.38431 kbps units by the value at
1771 * the TQAVCC register (1Gbps / 61034), which reduces
1772 * the granularity for idleslope increments.
1773 * For instance, if you want to configure a 2576kbps
1774 * idleslope, the value to be written on the register
1775 * would have to be 157.23. If rounded down, you end
1776 * up with less bandwidth available than originally
1777 * required (~2572 kbps). If rounded up, you end up
1778 * with a higher bandwidth (~2589 kbps). Below the
1779 * approach we take is to always round up the
1780 * calculated value, so the resulting bandwidth might
1781 * be slightly higher for some configurations.
1782 */
1783 value = DIV_ROUND_UP_ULL(ring->idleslope * 61034ULL, 1000000);
1784
1785 tqavcc = rd32(E1000_I210_TQAVCC(queue));
1786 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1787 tqavcc |= value;
1788 wr32(E1000_I210_TQAVCC(queue), tqavcc);
1789
1790 wr32(E1000_I210_TQAVHC(queue),
1791 0x80000000 + ring->hicredit * 0x7735);
1792 } else {
1793
1794 /* Set idleSlope to zero. */
1795 tqavcc = rd32(E1000_I210_TQAVCC(queue));
1796 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1797 wr32(E1000_I210_TQAVCC(queue), tqavcc);
1798
1799 /* Set hiCredit to zero. */
1800 wr32(E1000_I210_TQAVHC(queue), 0);
1801
1802 /* If CBS is not enabled for any queues anymore, then return to
1803 * the default state of Data Transmission Arbitration on
1804 * TQAVCTRL.
1805 */
1806 if (!is_any_cbs_enabled(adapter)) {
1807 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1808 tqavctrl &= ~E1000_TQAVCTRL_DATATRANARB;
1809 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1810 }
1811 }
1812
1813 /* If LaunchTime is enabled, set DataTranTIM. */
1814 if (ring->launchtime_enable) {
1815 /* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled
1816 * for any of the SR queues, and configure fetchtime delta.
1817 * XXX NOTE:
1818 * - LaunchTime will be enabled for all SR queues.
1819 * - A fixed offset can be added relative to the launch
1820 * time of all packets if configured at reg LAUNCH_OS0.
1821 * We are keeping it as 0 for now (default value).
1822 */
1823 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1824 tqavctrl |= E1000_TQAVCTRL_DATATRANTIM |
1825 E1000_TQAVCTRL_FETCHTIME_DELTA;
1826 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1827 } else {
1828 /* If Launchtime is not enabled for any SR queues anymore,
1829 * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta,
1830 * effectively disabling Launchtime.
1831 */
1832 if (!is_any_txtime_enabled(adapter)) {
1833 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1834 tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM;
1835 tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA;
1836 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1837 }
1838 }
1839
1840 /* XXX: In i210 controller the sendSlope and loCredit parameters from
1841 * CBS are not configurable by software so we don't do any 'controller
1842 * configuration' in respect to these parameters.
1843 */
1844
1845 netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n",
1846 ring->cbs_enable ? "enabled" : "disabled",
1847 ring->launchtime_enable ? "enabled" : "disabled",
1848 queue,
1849 ring->idleslope, ring->sendslope,
1850 ring->hicredit, ring->locredit);
1851}
1852
1853static int igb_save_txtime_params(struct igb_adapter *adapter, int queue,
1854 bool enable)
1855{
1856 struct igb_ring *ring;
1857
1858 if (queue < 0 || queue > adapter->num_tx_queues)
1859 return -EINVAL;
1860
1861 ring = adapter->tx_ring[queue];
1862 ring->launchtime_enable = enable;
1863
1864 return 0;
1865}
1866
1867static int igb_save_cbs_params(struct igb_adapter *adapter, int queue,
1868 bool enable, int idleslope, int sendslope,
1869 int hicredit, int locredit)
1870{
1871 struct igb_ring *ring;
1872
1873 if (queue < 0 || queue > adapter->num_tx_queues)
1874 return -EINVAL;
1875
1876 ring = adapter->tx_ring[queue];
1877
1878 ring->cbs_enable = enable;
1879 ring->idleslope = idleslope;
1880 ring->sendslope = sendslope;
1881 ring->hicredit = hicredit;
1882 ring->locredit = locredit;
1883
1884 return 0;
1885}
1886
1887/**
1888 * igb_setup_tx_mode - Switch to/from Qav Tx mode when applicable
1889 * @adapter: pointer to adapter struct
1890 *
1891 * Configure TQAVCTRL register switching the controller's Tx mode
1892 * if FQTSS mode is enabled or disabled. Additionally, will issue
1893 * a call to igb_config_tx_modes() per queue so any previously saved
1894 * Tx parameters are applied.
1895 **/
1896static void igb_setup_tx_mode(struct igb_adapter *adapter)
1897{
1898 struct net_device *netdev = adapter->netdev;
1899 struct e1000_hw *hw = &adapter->hw;
1900 u32 val;
1901
1902 /* Only i210 controller supports changing the transmission mode. */
1903 if (hw->mac.type != e1000_i210)
1904 return;
1905
1906 if (is_fqtss_enabled(adapter)) {
1907 int i, max_queue;
1908
1909 /* Configure TQAVCTRL register: set transmit mode to 'Qav',
1910 * set data fetch arbitration to 'round robin', set SP_WAIT_SR
1911 * so SP queues wait for SR ones.
1912 */
1913 val = rd32(E1000_I210_TQAVCTRL);
1914 val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR;
1915 val &= ~E1000_TQAVCTRL_DATAFETCHARB;
1916 wr32(E1000_I210_TQAVCTRL, val);
1917
1918 /* Configure Tx and Rx packet buffers sizes as described in
1919 * i210 datasheet section 7.2.7.7.
1920 */
1921 val = rd32(E1000_TXPBS);
1922 val &= ~I210_TXPBSIZE_MASK;
1923 val |= I210_TXPBSIZE_PB0_8KB | I210_TXPBSIZE_PB1_8KB |
1924 I210_TXPBSIZE_PB2_4KB | I210_TXPBSIZE_PB3_4KB;
1925 wr32(E1000_TXPBS, val);
1926
1927 val = rd32(E1000_RXPBS);
1928 val &= ~I210_RXPBSIZE_MASK;
1929 val |= I210_RXPBSIZE_PB_30KB;
1930 wr32(E1000_RXPBS, val);
1931
1932 /* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ
1933 * register should not exceed the buffer size programmed in
1934 * TXPBS. The smallest buffer size programmed in TXPBS is 4kB
1935 * so according to the datasheet we should set MAX_TPKT_SIZE to
1936 * 4kB / 64.
1937 *
1938 * However, when we do so, no frame from queue 2 and 3 are
1939 * transmitted. It seems the MAX_TPKT_SIZE should not be great
1940 * or _equal_ to the buffer size programmed in TXPBS. For this
1941 * reason, we set set MAX_ TPKT_SIZE to (4kB - 1) / 64.
1942 */
1943 val = (4096 - 1) / 64;
1944 wr32(E1000_I210_DTXMXPKTSZ, val);
1945
1946 /* Since FQTSS mode is enabled, apply any CBS configuration
1947 * previously set. If no previous CBS configuration has been
1948 * done, then the initial configuration is applied, which means
1949 * CBS is disabled.
1950 */
1951 max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ?
1952 adapter->num_tx_queues : I210_SR_QUEUES_NUM;
1953
1954 for (i = 0; i < max_queue; i++) {
1955 igb_config_tx_modes(adapter, i);
1956 }
1957 } else {
1958 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
1959 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
1960 wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT);
1961
1962 val = rd32(E1000_I210_TQAVCTRL);
1963 /* According to Section 8.12.21, the other flags we've set when
1964 * enabling FQTSS are not relevant when disabling FQTSS so we
1965 * don't set they here.
1966 */
1967 val &= ~E1000_TQAVCTRL_XMIT_MODE;
1968 wr32(E1000_I210_TQAVCTRL, val);
1969 }
1970
1971 netdev_dbg(netdev, "FQTSS %s\n", (is_fqtss_enabled(adapter)) ?
1972 "enabled" : "disabled");
1973}
1974
1975/**
1976 * igb_configure - configure the hardware for RX and TX
1977 * @adapter: private board structure
1978 **/
1979static void igb_configure(struct igb_adapter *adapter)
1980{
1981 struct net_device *netdev = adapter->netdev;
1982 int i;
1983
1984 igb_get_hw_control(adapter);
1985 igb_set_rx_mode(netdev);
1986 igb_setup_tx_mode(adapter);
1987
1988 igb_restore_vlan(adapter);
1989
1990 igb_setup_tctl(adapter);
1991 igb_setup_mrqc(adapter);
1992 igb_setup_rctl(adapter);
1993
1994 igb_nfc_filter_restore(adapter);
1995 igb_configure_tx(adapter);
1996 igb_configure_rx(adapter);
1997
1998 igb_rx_fifo_flush_82575(&adapter->hw);
1999
2000 /* call igb_desc_unused which always leaves
2001 * at least 1 descriptor unused to make sure
2002 * next_to_use != next_to_clean
2003 */
2004 for (i = 0; i < adapter->num_rx_queues; i++) {
2005 struct igb_ring *ring = adapter->rx_ring[i];
2006 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
2007 }
2008}
2009
2010/**
2011 * igb_power_up_link - Power up the phy/serdes link
2012 * @adapter: address of board private structure
2013 **/
2014void igb_power_up_link(struct igb_adapter *adapter)
2015{
2016 igb_reset_phy(&adapter->hw);
2017
2018 if (adapter->hw.phy.media_type == e1000_media_type_copper)
2019 igb_power_up_phy_copper(&adapter->hw);
2020 else
2021 igb_power_up_serdes_link_82575(&adapter->hw);
2022
2023 igb_setup_link(&adapter->hw);
2024}
2025
2026/**
2027 * igb_power_down_link - Power down the phy/serdes link
2028 * @adapter: address of board private structure
2029 */
2030static void igb_power_down_link(struct igb_adapter *adapter)
2031{
2032 if (adapter->hw.phy.media_type == e1000_media_type_copper)
2033 igb_power_down_phy_copper_82575(&adapter->hw);
2034 else
2035 igb_shutdown_serdes_link_82575(&adapter->hw);
2036}
2037
2038/**
2039 * Detect and switch function for Media Auto Sense
2040 * @adapter: address of the board private structure
2041 **/
2042static void igb_check_swap_media(struct igb_adapter *adapter)
2043{
2044 struct e1000_hw *hw = &adapter->hw;
2045 u32 ctrl_ext, connsw;
2046 bool swap_now = false;
2047
2048 ctrl_ext = rd32(E1000_CTRL_EXT);
2049 connsw = rd32(E1000_CONNSW);
2050
2051 /* need to live swap if current media is copper and we have fiber/serdes
2052 * to go to.
2053 */
2054
2055 if ((hw->phy.media_type == e1000_media_type_copper) &&
2056 (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
2057 swap_now = true;
2058 } else if ((hw->phy.media_type != e1000_media_type_copper) &&
2059 !(connsw & E1000_CONNSW_SERDESD)) {
2060 /* copper signal takes time to appear */
2061 if (adapter->copper_tries < 4) {
2062 adapter->copper_tries++;
2063 connsw |= E1000_CONNSW_AUTOSENSE_CONF;
2064 wr32(E1000_CONNSW, connsw);
2065 return;
2066 } else {
2067 adapter->copper_tries = 0;
2068 if ((connsw & E1000_CONNSW_PHYSD) &&
2069 (!(connsw & E1000_CONNSW_PHY_PDN))) {
2070 swap_now = true;
2071 connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
2072 wr32(E1000_CONNSW, connsw);
2073 }
2074 }
2075 }
2076
2077 if (!swap_now)
2078 return;
2079
2080 switch (hw->phy.media_type) {
2081 case e1000_media_type_copper:
2082 netdev_info(adapter->netdev,
2083 "MAS: changing media to fiber/serdes\n");
2084 ctrl_ext |=
2085 E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2086 adapter->flags |= IGB_FLAG_MEDIA_RESET;
2087 adapter->copper_tries = 0;
2088 break;
2089 case e1000_media_type_internal_serdes:
2090 case e1000_media_type_fiber:
2091 netdev_info(adapter->netdev,
2092 "MAS: changing media to copper\n");
2093 ctrl_ext &=
2094 ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2095 adapter->flags |= IGB_FLAG_MEDIA_RESET;
2096 break;
2097 default:
2098 /* shouldn't get here during regular operation */
2099 netdev_err(adapter->netdev,
2100 "AMS: Invalid media type found, returning\n");
2101 break;
2102 }
2103 wr32(E1000_CTRL_EXT, ctrl_ext);
2104}
2105
2106/**
2107 * igb_up - Open the interface and prepare it to handle traffic
2108 * @adapter: board private structure
2109 **/
2110int igb_up(struct igb_adapter *adapter)
2111{
2112 struct e1000_hw *hw = &adapter->hw;
2113 int i;
2114
2115 /* hardware has been reset, we need to reload some things */
2116 igb_configure(adapter);
2117
2118 clear_bit(__IGB_DOWN, &adapter->state);
2119
2120 for (i = 0; i < adapter->num_q_vectors; i++)
2121 napi_enable(&(adapter->q_vector[i]->napi));
2122
2123 if (adapter->flags & IGB_FLAG_HAS_MSIX)
2124 igb_configure_msix(adapter);
2125 else
2126 igb_assign_vector(adapter->q_vector[0], 0);
2127
2128 /* Clear any pending interrupts. */
2129 rd32(E1000_TSICR);
2130 rd32(E1000_ICR);
2131 igb_irq_enable(adapter);
2132
2133 /* notify VFs that reset has been completed */
2134 if (adapter->vfs_allocated_count) {
2135 u32 reg_data = rd32(E1000_CTRL_EXT);
2136
2137 reg_data |= E1000_CTRL_EXT_PFRSTD;
2138 wr32(E1000_CTRL_EXT, reg_data);
2139 }
2140
2141 netif_tx_start_all_queues(adapter->netdev);
2142
2143 /* start the watchdog. */
2144 hw->mac.get_link_status = 1;
2145 schedule_work(&adapter->watchdog_task);
2146
2147 if ((adapter->flags & IGB_FLAG_EEE) &&
2148 (!hw->dev_spec._82575.eee_disable))
2149 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;
2150
2151 return 0;
2152}
2153
2154void igb_down(struct igb_adapter *adapter)
2155{
2156 struct net_device *netdev = adapter->netdev;
2157 struct e1000_hw *hw = &adapter->hw;
2158 u32 tctl, rctl;
2159 int i;
2160
2161 /* signal that we're down so the interrupt handler does not
2162 * reschedule our watchdog timer
2163 */
2164 set_bit(__IGB_DOWN, &adapter->state);
2165
2166 /* disable receives in the hardware */
2167 rctl = rd32(E1000_RCTL);
2168 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
2169 /* flush and sleep below */
2170
2171 igb_nfc_filter_exit(adapter);
2172
2173 netif_carrier_off(netdev);
2174 netif_tx_stop_all_queues(netdev);
2175
2176 /* disable transmits in the hardware */
2177 tctl = rd32(E1000_TCTL);
2178 tctl &= ~E1000_TCTL_EN;
2179 wr32(E1000_TCTL, tctl);
2180 /* flush both disables and wait for them to finish */
2181 wrfl();
2182 usleep_range(10000, 11000);
2183
2184 igb_irq_disable(adapter);
2185
2186 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
2187
2188 for (i = 0; i < adapter->num_q_vectors; i++) {
2189 if (adapter->q_vector[i]) {
2190 napi_synchronize(&adapter->q_vector[i]->napi);
2191 napi_disable(&adapter->q_vector[i]->napi);
2192 }
2193 }
2194
2195 del_timer_sync(&adapter->watchdog_timer);
2196 del_timer_sync(&adapter->phy_info_timer);
2197
2198 /* record the stats before reset*/
2199 spin_lock(&adapter->stats64_lock);
2200 igb_update_stats(adapter);
2201 spin_unlock(&adapter->stats64_lock);
2202
2203 adapter->link_speed = 0;
2204 adapter->link_duplex = 0;
2205
2206 if (!pci_channel_offline(adapter->pdev))
2207 igb_reset(adapter);
2208
2209 /* clear VLAN promisc flag so VFTA will be updated if necessary */
2210 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
2211
2212 igb_clean_all_tx_rings(adapter);
2213 igb_clean_all_rx_rings(adapter);
2214#ifdef CONFIG_IGB_DCA
2215
2216 /* since we reset the hardware DCA settings were cleared */
2217 igb_setup_dca(adapter);
2218#endif
2219}
2220
2221void igb_reinit_locked(struct igb_adapter *adapter)
2222{
2223 WARN_ON(in_interrupt());
2224 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
2225 usleep_range(1000, 2000);
2226 igb_down(adapter);
2227 igb_up(adapter);
2228 clear_bit(__IGB_RESETTING, &adapter->state);
2229}
2230
2231/** igb_enable_mas - Media Autosense re-enable after swap
2232 *
2233 * @adapter: adapter struct
2234 **/
2235static void igb_enable_mas(struct igb_adapter *adapter)
2236{
2237 struct e1000_hw *hw = &adapter->hw;
2238 u32 connsw = rd32(E1000_CONNSW);
2239
2240 /* configure for SerDes media detect */
2241 if ((hw->phy.media_type == e1000_media_type_copper) &&
2242 (!(connsw & E1000_CONNSW_SERDESD))) {
2243 connsw |= E1000_CONNSW_ENRGSRC;
2244 connsw |= E1000_CONNSW_AUTOSENSE_EN;
2245 wr32(E1000_CONNSW, connsw);
2246 wrfl();
2247 }
2248}
2249
2250void igb_reset(struct igb_adapter *adapter)
2251{
2252 struct pci_dev *pdev = adapter->pdev;
2253 struct e1000_hw *hw = &adapter->hw;
2254 struct e1000_mac_info *mac = &hw->mac;
2255 struct e1000_fc_info *fc = &hw->fc;
2256 u32 pba, hwm;
2257
2258 /* Repartition Pba for greater than 9k mtu
2259 * To take effect CTRL.RST is required.
2260 */
2261 switch (mac->type) {
2262 case e1000_i350:
2263 case e1000_i354:
2264 case e1000_82580:
2265 pba = rd32(E1000_RXPBS);
2266 pba = igb_rxpbs_adjust_82580(pba);
2267 break;
2268 case e1000_82576:
2269 pba = rd32(E1000_RXPBS);
2270 pba &= E1000_RXPBS_SIZE_MASK_82576;
2271 break;
2272 case e1000_82575:
2273 case e1000_i210:
2274 case e1000_i211:
2275 default:
2276 pba = E1000_PBA_34K;
2277 break;
2278 }
2279
2280 if (mac->type == e1000_82575) {
2281 u32 min_rx_space, min_tx_space, needed_tx_space;
2282
2283 /* write Rx PBA so that hardware can report correct Tx PBA */
2284 wr32(E1000_PBA, pba);
2285
2286 /* To maintain wire speed transmits, the Tx FIFO should be
2287 * large enough to accommodate two full transmit packets,
2288 * rounded up to the next 1KB and expressed in KB. Likewise,
2289 * the Rx FIFO should be large enough to accommodate at least
2290 * one full receive packet and is similarly rounded up and
2291 * expressed in KB.
2292 */
2293 min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024);
2294
2295 /* The Tx FIFO also stores 16 bytes of information about the Tx
2296 * but don't include Ethernet FCS because hardware appends it.
2297 * We only need to round down to the nearest 512 byte block
2298 * count since the value we care about is 2 frames, not 1.
2299 */
2300 min_tx_space = adapter->max_frame_size;
2301 min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN;
2302 min_tx_space = DIV_ROUND_UP(min_tx_space, 512);
2303
2304 /* upper 16 bits has Tx packet buffer allocation size in KB */
2305 needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16);
2306
2307 /* If current Tx allocation is less than the min Tx FIFO size,
2308 * and the min Tx FIFO size is less than the current Rx FIFO
2309 * allocation, take space away from current Rx allocation.
2310 */
2311 if (needed_tx_space < pba) {
2312 pba -= needed_tx_space;
2313
2314 /* if short on Rx space, Rx wins and must trump Tx
2315 * adjustment
2316 */
2317 if (pba < min_rx_space)
2318 pba = min_rx_space;
2319 }
2320
2321 /* adjust PBA for jumbo frames */
2322 wr32(E1000_PBA, pba);
2323 }
2324
2325 /* flow control settings
2326 * The high water mark must be low enough to fit one full frame
2327 * after transmitting the pause frame. As such we must have enough
2328 * space to allow for us to complete our current transmit and then
2329 * receive the frame that is in progress from the link partner.
2330 * Set it to:
2331 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
2332 */
2333 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
2334
2335 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */
2336 fc->low_water = fc->high_water - 16;
2337 fc->pause_time = 0xFFFF;
2338 fc->send_xon = 1;
2339 fc->current_mode = fc->requested_mode;
2340
2341 /* disable receive for all VFs and wait one second */
2342 if (adapter->vfs_allocated_count) {
2343 int i;
2344
2345 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
2346 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
2347
2348 /* ping all the active vfs to let them know we are going down */
2349 igb_ping_all_vfs(adapter);
2350
2351 /* disable transmits and receives */
2352 wr32(E1000_VFRE, 0);
2353 wr32(E1000_VFTE, 0);
2354 }
2355
2356 /* Allow time for pending master requests to run */
2357 hw->mac.ops.reset_hw(hw);
2358 wr32(E1000_WUC, 0);
2359
2360 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
2361 /* need to resetup here after media swap */
2362 adapter->ei.get_invariants(hw);
2363 adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
2364 }
2365 if ((mac->type == e1000_82575 || mac->type == e1000_i350) &&
2366 (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
2367 igb_enable_mas(adapter);
2368 }
2369 if (hw->mac.ops.init_hw(hw))
2370 dev_err(&pdev->dev, "Hardware Error\n");
2371
2372 /* RAR registers were cleared during init_hw, clear mac table */
2373 igb_flush_mac_table(adapter);
2374 __dev_uc_unsync(adapter->netdev, NULL);
2375
2376 /* Recover default RAR entry */
2377 igb_set_default_mac_filter(adapter);
2378
2379 /* Flow control settings reset on hardware reset, so guarantee flow
2380 * control is off when forcing speed.
2381 */
2382 if (!hw->mac.autoneg)
2383 igb_force_mac_fc(hw);
2384
2385 igb_init_dmac(adapter, pba);
2386#ifdef CONFIG_IGB_HWMON
2387 /* Re-initialize the thermal sensor on i350 devices. */
2388 if (!test_bit(__IGB_DOWN, &adapter->state)) {
2389 if (mac->type == e1000_i350 && hw->bus.func == 0) {
2390 /* If present, re-initialize the external thermal sensor
2391 * interface.
2392 */
2393 if (adapter->ets)
2394 mac->ops.init_thermal_sensor_thresh(hw);
2395 }
2396 }
2397#endif
2398 /* Re-establish EEE setting */
2399 if (hw->phy.media_type == e1000_media_type_copper) {
2400 switch (mac->type) {
2401 case e1000_i350:
2402 case e1000_i210:
2403 case e1000_i211:
2404 igb_set_eee_i350(hw, true, true);
2405 break;
2406 case e1000_i354:
2407 igb_set_eee_i354(hw, true, true);
2408 break;
2409 default:
2410 break;
2411 }
2412 }
2413 if (!netif_running(adapter->netdev))
2414 igb_power_down_link(adapter);
2415
2416 igb_update_mng_vlan(adapter);
2417
2418 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
2419 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
2420
2421 /* Re-enable PTP, where applicable. */
2422 if (adapter->ptp_flags & IGB_PTP_ENABLED)
2423 igb_ptp_reset(adapter);
2424
2425 igb_get_phy_info(hw);
2426}
2427
2428static netdev_features_t igb_fix_features(struct net_device *netdev,
2429 netdev_features_t features)
2430{
2431 /* Since there is no support for separate Rx/Tx vlan accel
2432 * enable/disable make sure Tx flag is always in same state as Rx.
2433 */
2434 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2435 features |= NETIF_F_HW_VLAN_CTAG_TX;
2436 else
2437 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2438
2439 return features;
2440}
2441
2442static int igb_set_features(struct net_device *netdev,
2443 netdev_features_t features)
2444{
2445 netdev_features_t changed = netdev->features ^ features;
2446 struct igb_adapter *adapter = netdev_priv(netdev);
2447
2448 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2449 igb_vlan_mode(netdev, features);
2450
2451 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2452 return 0;
2453
2454 if (!(features & NETIF_F_NTUPLE)) {
2455 struct hlist_node *node2;
2456 struct igb_nfc_filter *rule;
2457
2458 spin_lock(&adapter->nfc_lock);
2459 hlist_for_each_entry_safe(rule, node2,
2460 &adapter->nfc_filter_list, nfc_node) {
2461 igb_erase_filter(adapter, rule);
2462 hlist_del(&rule->nfc_node);
2463 kfree(rule);
2464 }
2465 spin_unlock(&adapter->nfc_lock);
2466 adapter->nfc_filter_count = 0;
2467 }
2468
2469 netdev->features = features;
2470
2471 if (netif_running(netdev))
2472 igb_reinit_locked(adapter);
2473 else
2474 igb_reset(adapter);
2475
2476 return 1;
2477}
2478
2479static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
2480 struct net_device *dev,
2481 const unsigned char *addr, u16 vid,
2482 u16 flags,
2483 struct netlink_ext_ack *extack)
2484{
2485 /* guarantee we can provide a unique filter for the unicast address */
2486 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
2487 struct igb_adapter *adapter = netdev_priv(dev);
2488 int vfn = adapter->vfs_allocated_count;
2489
2490 if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn))
2491 return -ENOMEM;
2492 }
2493
2494 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
2495}
2496
2497#define IGB_MAX_MAC_HDR_LEN 127
2498#define IGB_MAX_NETWORK_HDR_LEN 511
2499
2500static netdev_features_t
2501igb_features_check(struct sk_buff *skb, struct net_device *dev,
2502 netdev_features_t features)
2503{
2504 unsigned int network_hdr_len, mac_hdr_len;
2505
2506 /* Make certain the headers can be described by a context descriptor */
2507 mac_hdr_len = skb_network_header(skb) - skb->data;
2508 if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN))
2509 return features & ~(NETIF_F_HW_CSUM |
2510 NETIF_F_SCTP_CRC |
2511 NETIF_F_GSO_UDP_L4 |
2512 NETIF_F_HW_VLAN_CTAG_TX |
2513 NETIF_F_TSO |
2514 NETIF_F_TSO6);
2515
2516 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2517 if (unlikely(network_hdr_len > IGB_MAX_NETWORK_HDR_LEN))
2518 return features & ~(NETIF_F_HW_CSUM |
2519 NETIF_F_SCTP_CRC |
2520 NETIF_F_GSO_UDP_L4 |
2521 NETIF_F_TSO |
2522 NETIF_F_TSO6);
2523
2524 /* We can only support IPV4 TSO in tunnels if we can mangle the
2525 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2526 */
2527 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2528 features &= ~NETIF_F_TSO;
2529
2530 return features;
2531}
2532
2533static void igb_offload_apply(struct igb_adapter *adapter, s32 queue)
2534{
2535 if (!is_fqtss_enabled(adapter)) {
2536 enable_fqtss(adapter, true);
2537 return;
2538 }
2539
2540 igb_config_tx_modes(adapter, queue);
2541
2542 if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter))
2543 enable_fqtss(adapter, false);
2544}
2545
2546static int igb_offload_cbs(struct igb_adapter *adapter,
2547 struct tc_cbs_qopt_offload *qopt)
2548{
2549 struct e1000_hw *hw = &adapter->hw;
2550 int err;
2551
2552 /* CBS offloading is only supported by i210 controller. */
2553 if (hw->mac.type != e1000_i210)
2554 return -EOPNOTSUPP;
2555
2556 /* CBS offloading is only supported by queue 0 and queue 1. */
2557 if (qopt->queue < 0 || qopt->queue > 1)
2558 return -EINVAL;
2559
2560 err = igb_save_cbs_params(adapter, qopt->queue, qopt->enable,
2561 qopt->idleslope, qopt->sendslope,
2562 qopt->hicredit, qopt->locredit);
2563 if (err)
2564 return err;
2565
2566 igb_offload_apply(adapter, qopt->queue);
2567
2568 return 0;
2569}
2570
2571#define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
2572#define VLAN_PRIO_FULL_MASK (0x07)
2573
2574static int igb_parse_cls_flower(struct igb_adapter *adapter,
2575 struct flow_cls_offload *f,
2576 int traffic_class,
2577 struct igb_nfc_filter *input)
2578{
2579 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
2580 struct flow_dissector *dissector = rule->match.dissector;
2581 struct netlink_ext_ack *extack = f->common.extack;
2582
2583 if (dissector->used_keys &
2584 ~(BIT(FLOW_DISSECTOR_KEY_BASIC) |
2585 BIT(FLOW_DISSECTOR_KEY_CONTROL) |
2586 BIT(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2587 BIT(FLOW_DISSECTOR_KEY_VLAN))) {
2588 NL_SET_ERR_MSG_MOD(extack,
2589 "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported");
2590 return -EOPNOTSUPP;
2591 }
2592
2593 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2594 struct flow_match_eth_addrs match;
2595
2596 flow_rule_match_eth_addrs(rule, &match);
2597 if (!is_zero_ether_addr(match.mask->dst)) {
2598 if (!is_broadcast_ether_addr(match.mask->dst)) {
2599 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address");
2600 return -EINVAL;
2601 }
2602
2603 input->filter.match_flags |=
2604 IGB_FILTER_FLAG_DST_MAC_ADDR;
2605 ether_addr_copy(input->filter.dst_addr, match.key->dst);
2606 }
2607
2608 if (!is_zero_ether_addr(match.mask->src)) {
2609 if (!is_broadcast_ether_addr(match.mask->src)) {
2610 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address");
2611 return -EINVAL;
2612 }
2613
2614 input->filter.match_flags |=
2615 IGB_FILTER_FLAG_SRC_MAC_ADDR;
2616 ether_addr_copy(input->filter.src_addr, match.key->src);
2617 }
2618 }
2619
2620 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
2621 struct flow_match_basic match;
2622
2623 flow_rule_match_basic(rule, &match);
2624 if (match.mask->n_proto) {
2625 if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) {
2626 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter");
2627 return -EINVAL;
2628 }
2629
2630 input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE;
2631 input->filter.etype = match.key->n_proto;
2632 }
2633 }
2634
2635 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
2636 struct flow_match_vlan match;
2637
2638 flow_rule_match_vlan(rule, &match);
2639 if (match.mask->vlan_priority) {
2640 if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) {
2641 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority");
2642 return -EINVAL;
2643 }
2644
2645 input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI;
2646 input->filter.vlan_tci = match.key->vlan_priority;
2647 }
2648 }
2649
2650 input->action = traffic_class;
2651 input->cookie = f->cookie;
2652
2653 return 0;
2654}
2655
2656static int igb_configure_clsflower(struct igb_adapter *adapter,
2657 struct flow_cls_offload *cls_flower)
2658{
2659 struct netlink_ext_ack *extack = cls_flower->common.extack;
2660 struct igb_nfc_filter *filter, *f;
2661 int err, tc;
2662
2663 tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
2664 if (tc < 0) {
2665 NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class");
2666 return -EINVAL;
2667 }
2668
2669 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
2670 if (!filter)
2671 return -ENOMEM;
2672
2673 err = igb_parse_cls_flower(adapter, cls_flower, tc, filter);
2674 if (err < 0)
2675 goto err_parse;
2676
2677 spin_lock(&adapter->nfc_lock);
2678
2679 hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) {
2680 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2681 err = -EEXIST;
2682 NL_SET_ERR_MSG_MOD(extack,
2683 "This filter is already set in ethtool");
2684 goto err_locked;
2685 }
2686 }
2687
2688 hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) {
2689 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2690 err = -EEXIST;
2691 NL_SET_ERR_MSG_MOD(extack,
2692 "This filter is already set in cls_flower");
2693 goto err_locked;
2694 }
2695 }
2696
2697 err = igb_add_filter(adapter, filter);
2698 if (err < 0) {
2699 NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter");
2700 goto err_locked;
2701 }
2702
2703 hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list);
2704
2705 spin_unlock(&adapter->nfc_lock);
2706
2707 return 0;
2708
2709err_locked:
2710 spin_unlock(&adapter->nfc_lock);
2711
2712err_parse:
2713 kfree(filter);
2714
2715 return err;
2716}
2717
2718static int igb_delete_clsflower(struct igb_adapter *adapter,
2719 struct flow_cls_offload *cls_flower)
2720{
2721 struct igb_nfc_filter *filter;
2722 int err;
2723
2724 spin_lock(&adapter->nfc_lock);
2725
2726 hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node)
2727 if (filter->cookie == cls_flower->cookie)
2728 break;
2729
2730 if (!filter) {
2731 err = -ENOENT;
2732 goto out;
2733 }
2734
2735 err = igb_erase_filter(adapter, filter);
2736 if (err < 0)
2737 goto out;
2738
2739 hlist_del(&filter->nfc_node);
2740 kfree(filter);
2741
2742out:
2743 spin_unlock(&adapter->nfc_lock);
2744
2745 return err;
2746}
2747
2748static int igb_setup_tc_cls_flower(struct igb_adapter *adapter,
2749 struct flow_cls_offload *cls_flower)
2750{
2751 switch (cls_flower->command) {
2752 case FLOW_CLS_REPLACE:
2753 return igb_configure_clsflower(adapter, cls_flower);
2754 case FLOW_CLS_DESTROY:
2755 return igb_delete_clsflower(adapter, cls_flower);
2756 case FLOW_CLS_STATS:
2757 return -EOPNOTSUPP;
2758 default:
2759 return -EOPNOTSUPP;
2760 }
2761}
2762
2763static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2764 void *cb_priv)
2765{
2766 struct igb_adapter *adapter = cb_priv;
2767
2768 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
2769 return -EOPNOTSUPP;
2770
2771 switch (type) {
2772 case TC_SETUP_CLSFLOWER:
2773 return igb_setup_tc_cls_flower(adapter, type_data);
2774
2775 default:
2776 return -EOPNOTSUPP;
2777 }
2778}
2779
2780static int igb_offload_txtime(struct igb_adapter *adapter,
2781 struct tc_etf_qopt_offload *qopt)
2782{
2783 struct e1000_hw *hw = &adapter->hw;
2784 int err;
2785
2786 /* Launchtime offloading is only supported by i210 controller. */
2787 if (hw->mac.type != e1000_i210)
2788 return -EOPNOTSUPP;
2789
2790 /* Launchtime offloading is only supported by queues 0 and 1. */
2791 if (qopt->queue < 0 || qopt->queue > 1)
2792 return -EINVAL;
2793
2794 err = igb_save_txtime_params(adapter, qopt->queue, qopt->enable);
2795 if (err)
2796 return err;
2797
2798 igb_offload_apply(adapter, qopt->queue);
2799
2800 return 0;
2801}
2802
2803static LIST_HEAD(igb_block_cb_list);
2804
2805static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
2806 void *type_data)
2807{
2808 struct igb_adapter *adapter = netdev_priv(dev);
2809
2810 switch (type) {
2811 case TC_SETUP_QDISC_CBS:
2812 return igb_offload_cbs(adapter, type_data);
2813 case TC_SETUP_BLOCK:
2814 return flow_block_cb_setup_simple(type_data,
2815 &igb_block_cb_list,
2816 igb_setup_tc_block_cb,
2817 adapter, adapter, true);
2818
2819 case TC_SETUP_QDISC_ETF:
2820 return igb_offload_txtime(adapter, type_data);
2821
2822 default:
2823 return -EOPNOTSUPP;
2824 }
2825}
2826
2827static const struct net_device_ops igb_netdev_ops = {
2828 .ndo_open = igb_open,
2829 .ndo_stop = igb_close,
2830 .ndo_start_xmit = igb_xmit_frame,
2831 .ndo_get_stats64 = igb_get_stats64,
2832 .ndo_set_rx_mode = igb_set_rx_mode,
2833 .ndo_set_mac_address = igb_set_mac,
2834 .ndo_change_mtu = igb_change_mtu,
2835 .ndo_do_ioctl = igb_ioctl,
2836 .ndo_tx_timeout = igb_tx_timeout,
2837 .ndo_validate_addr = eth_validate_addr,
2838 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
2839 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
2840 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
2841 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
2842 .ndo_set_vf_rate = igb_ndo_set_vf_bw,
2843 .ndo_set_vf_spoofchk = igb_ndo_set_vf_spoofchk,
2844 .ndo_set_vf_trust = igb_ndo_set_vf_trust,
2845 .ndo_get_vf_config = igb_ndo_get_vf_config,
2846 .ndo_fix_features = igb_fix_features,
2847 .ndo_set_features = igb_set_features,
2848 .ndo_fdb_add = igb_ndo_fdb_add,
2849 .ndo_features_check = igb_features_check,
2850 .ndo_setup_tc = igb_setup_tc,
2851};
2852
2853/**
2854 * igb_set_fw_version - Configure version string for ethtool
2855 * @adapter: adapter struct
2856 **/
2857void igb_set_fw_version(struct igb_adapter *adapter)
2858{
2859 struct e1000_hw *hw = &adapter->hw;
2860 struct e1000_fw_version fw;
2861
2862 igb_get_fw_version(hw, &fw);
2863
2864 switch (hw->mac.type) {
2865 case e1000_i210:
2866 case e1000_i211:
2867 if (!(igb_get_flash_presence_i210(hw))) {
2868 snprintf(adapter->fw_version,
2869 sizeof(adapter->fw_version),
2870 "%2d.%2d-%d",
2871 fw.invm_major, fw.invm_minor,
2872 fw.invm_img_type);
2873 break;
2874 }
2875 fallthrough;
2876 default:
2877 /* if option is rom valid, display its version too */
2878 if (fw.or_valid) {
2879 snprintf(adapter->fw_version,
2880 sizeof(adapter->fw_version),
2881 "%d.%d, 0x%08x, %d.%d.%d",
2882 fw.eep_major, fw.eep_minor, fw.etrack_id,
2883 fw.or_major, fw.or_build, fw.or_patch);
2884 /* no option rom */
2885 } else if (fw.etrack_id != 0X0000) {
2886 snprintf(adapter->fw_version,
2887 sizeof(adapter->fw_version),
2888 "%d.%d, 0x%08x",
2889 fw.eep_major, fw.eep_minor, fw.etrack_id);
2890 } else {
2891 snprintf(adapter->fw_version,
2892 sizeof(adapter->fw_version),
2893 "%d.%d.%d",
2894 fw.eep_major, fw.eep_minor, fw.eep_build);
2895 }
2896 break;
2897 }
2898}
2899
2900/**
2901 * igb_init_mas - init Media Autosense feature if enabled in the NVM
2902 *
2903 * @adapter: adapter struct
2904 **/
2905static void igb_init_mas(struct igb_adapter *adapter)
2906{
2907 struct e1000_hw *hw = &adapter->hw;
2908 u16 eeprom_data;
2909
2910 hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data);
2911 switch (hw->bus.func) {
2912 case E1000_FUNC_0:
2913 if (eeprom_data & IGB_MAS_ENABLE_0) {
2914 adapter->flags |= IGB_FLAG_MAS_ENABLE;
2915 netdev_info(adapter->netdev,
2916 "MAS: Enabling Media Autosense for port %d\n",
2917 hw->bus.func);
2918 }
2919 break;
2920 case E1000_FUNC_1:
2921 if (eeprom_data & IGB_MAS_ENABLE_1) {
2922 adapter->flags |= IGB_FLAG_MAS_ENABLE;
2923 netdev_info(adapter->netdev,
2924 "MAS: Enabling Media Autosense for port %d\n",
2925 hw->bus.func);
2926 }
2927 break;
2928 case E1000_FUNC_2:
2929 if (eeprom_data & IGB_MAS_ENABLE_2) {
2930 adapter->flags |= IGB_FLAG_MAS_ENABLE;
2931 netdev_info(adapter->netdev,
2932 "MAS: Enabling Media Autosense for port %d\n",
2933 hw->bus.func);
2934 }
2935 break;
2936 case E1000_FUNC_3:
2937 if (eeprom_data & IGB_MAS_ENABLE_3) {
2938 adapter->flags |= IGB_FLAG_MAS_ENABLE;
2939 netdev_info(adapter->netdev,
2940 "MAS: Enabling Media Autosense for port %d\n",
2941 hw->bus.func);
2942 }
2943 break;
2944 default:
2945 /* Shouldn't get here */
2946 netdev_err(adapter->netdev,
2947 "MAS: Invalid port configuration, returning\n");
2948 break;
2949 }
2950}
2951
2952/**
2953 * igb_init_i2c - Init I2C interface
2954 * @adapter: pointer to adapter structure
2955 **/
2956static s32 igb_init_i2c(struct igb_adapter *adapter)
2957{
2958 s32 status = 0;
2959
2960 /* I2C interface supported on i350 devices */
2961 if (adapter->hw.mac.type != e1000_i350)
2962 return 0;
2963
2964 /* Initialize the i2c bus which is controlled by the registers.
2965 * This bus will use the i2c_algo_bit structue that implements
2966 * the protocol through toggling of the 4 bits in the register.
2967 */
2968 adapter->i2c_adap.owner = THIS_MODULE;
2969 adapter->i2c_algo = igb_i2c_algo;
2970 adapter->i2c_algo.data = adapter;
2971 adapter->i2c_adap.algo_data = &adapter->i2c_algo;
2972 adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
2973 strlcpy(adapter->i2c_adap.name, "igb BB",
2974 sizeof(adapter->i2c_adap.name));
2975 status = i2c_bit_add_bus(&adapter->i2c_adap);
2976 return status;
2977}
2978
2979/**
2980 * igb_probe - Device Initialization Routine
2981 * @pdev: PCI device information struct
2982 * @ent: entry in igb_pci_tbl
2983 *
2984 * Returns 0 on success, negative on failure
2985 *
2986 * igb_probe initializes an adapter identified by a pci_dev structure.
2987 * The OS initialization, configuring of the adapter private structure,
2988 * and a hardware reset occur.
2989 **/
2990static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2991{
2992 struct net_device *netdev;
2993 struct igb_adapter *adapter;
2994 struct e1000_hw *hw;
2995 u16 eeprom_data = 0;
2996 s32 ret_val;
2997 static int global_quad_port_a; /* global quad port a indication */
2998 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
2999 int err, pci_using_dac;
3000 u8 part_str[E1000_PBANUM_LENGTH];
3001
3002 /* Catch broken hardware that put the wrong VF device ID in
3003 * the PCIe SR-IOV capability.
3004 */
3005 if (pdev->is_virtfn) {
3006 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
3007 pci_name(pdev), pdev->vendor, pdev->device);
3008 return -EINVAL;
3009 }
3010
3011 err = pci_enable_device_mem(pdev);
3012 if (err)
3013 return err;
3014
3015 pci_using_dac = 0;
3016 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3017 if (!err) {
3018 pci_using_dac = 1;
3019 } else {
3020 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3021 if (err) {
3022 dev_err(&pdev->dev,
3023 "No usable DMA configuration, aborting\n");
3024 goto err_dma;
3025 }
3026 }
3027
3028 err = pci_request_mem_regions(pdev, igb_driver_name);
3029 if (err)
3030 goto err_pci_reg;
3031
3032 pci_enable_pcie_error_reporting(pdev);
3033
3034 pci_set_master(pdev);
3035 pci_save_state(pdev);
3036
3037 err = -ENOMEM;
3038 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
3039 IGB_MAX_TX_QUEUES);
3040 if (!netdev)
3041 goto err_alloc_etherdev;
3042
3043 SET_NETDEV_DEV(netdev, &pdev->dev);
3044
3045 pci_set_drvdata(pdev, netdev);
3046 adapter = netdev_priv(netdev);
3047 adapter->netdev = netdev;
3048 adapter->pdev = pdev;
3049 hw = &adapter->hw;
3050 hw->back = adapter;
3051 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3052
3053 err = -EIO;
3054 adapter->io_addr = pci_iomap(pdev, 0, 0);
3055 if (!adapter->io_addr)
3056 goto err_ioremap;
3057 /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
3058 hw->hw_addr = adapter->io_addr;
3059
3060 netdev->netdev_ops = &igb_netdev_ops;
3061 igb_set_ethtool_ops(netdev);
3062 netdev->watchdog_timeo = 5 * HZ;
3063
3064 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
3065
3066 netdev->mem_start = pci_resource_start(pdev, 0);
3067 netdev->mem_end = pci_resource_end(pdev, 0);
3068
3069 /* PCI config space info */
3070 hw->vendor_id = pdev->vendor;
3071 hw->device_id = pdev->device;
3072 hw->revision_id = pdev->revision;
3073 hw->subsystem_vendor_id = pdev->subsystem_vendor;
3074 hw->subsystem_device_id = pdev->subsystem_device;
3075
3076 /* Copy the default MAC, PHY and NVM function pointers */
3077 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
3078 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
3079 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
3080 /* Initialize skew-specific constants */
3081 err = ei->get_invariants(hw);
3082 if (err)
3083 goto err_sw_init;
3084
3085 /* setup the private structure */
3086 err = igb_sw_init(adapter);
3087 if (err)
3088 goto err_sw_init;
3089
3090 igb_get_bus_info_pcie(hw);
3091
3092 hw->phy.autoneg_wait_to_complete = false;
3093
3094 /* Copper options */
3095 if (hw->phy.media_type == e1000_media_type_copper) {
3096 hw->phy.mdix = AUTO_ALL_MODES;
3097 hw->phy.disable_polarity_correction = false;
3098 hw->phy.ms_type = e1000_ms_hw_default;
3099 }
3100
3101 if (igb_check_reset_block(hw))
3102 dev_info(&pdev->dev,
3103 "PHY reset is blocked due to SOL/IDER session.\n");
3104
3105 /* features is initialized to 0 in allocation, it might have bits
3106 * set by igb_sw_init so we should use an or instead of an
3107 * assignment.
3108 */
3109 netdev->features |= NETIF_F_SG |
3110 NETIF_F_TSO |
3111 NETIF_F_TSO6 |
3112 NETIF_F_RXHASH |
3113 NETIF_F_RXCSUM |
3114 NETIF_F_HW_CSUM;
3115
3116 if (hw->mac.type >= e1000_82576)
3117 netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;
3118
3119 if (hw->mac.type >= e1000_i350)
3120 netdev->features |= NETIF_F_HW_TC;
3121
3122#define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
3123 NETIF_F_GSO_GRE_CSUM | \
3124 NETIF_F_GSO_IPXIP4 | \
3125 NETIF_F_GSO_IPXIP6 | \
3126 NETIF_F_GSO_UDP_TUNNEL | \
3127 NETIF_F_GSO_UDP_TUNNEL_CSUM)
3128
3129 netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES;
3130 netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES;
3131
3132 /* copy netdev features into list of user selectable features */
3133 netdev->hw_features |= netdev->features |
3134 NETIF_F_HW_VLAN_CTAG_RX |
3135 NETIF_F_HW_VLAN_CTAG_TX |
3136 NETIF_F_RXALL;
3137
3138 if (hw->mac.type >= e1000_i350)
3139 netdev->hw_features |= NETIF_F_NTUPLE;
3140
3141 if (pci_using_dac)
3142 netdev->features |= NETIF_F_HIGHDMA;
3143
3144 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
3145 netdev->mpls_features |= NETIF_F_HW_CSUM;
3146 netdev->hw_enc_features |= netdev->vlan_features;
3147
3148 /* set this bit last since it cannot be part of vlan_features */
3149 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
3150 NETIF_F_HW_VLAN_CTAG_RX |
3151 NETIF_F_HW_VLAN_CTAG_TX;
3152
3153 netdev->priv_flags |= IFF_SUPP_NOFCS;
3154
3155 netdev->priv_flags |= IFF_UNICAST_FLT;
3156
3157 /* MTU range: 68 - 9216 */
3158 netdev->min_mtu = ETH_MIN_MTU;
3159 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
3160
3161 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
3162
3163 /* before reading the NVM, reset the controller to put the device in a
3164 * known good starting state
3165 */
3166 hw->mac.ops.reset_hw(hw);
3167
3168 /* make sure the NVM is good , i211/i210 parts can have special NVM
3169 * that doesn't contain a checksum
3170 */
3171 switch (hw->mac.type) {
3172 case e1000_i210:
3173 case e1000_i211:
3174 if (igb_get_flash_presence_i210(hw)) {
3175 if (hw->nvm.ops.validate(hw) < 0) {
3176 dev_err(&pdev->dev,
3177 "The NVM Checksum Is Not Valid\n");
3178 err = -EIO;
3179 goto err_eeprom;
3180 }
3181 }
3182 break;
3183 default:
3184 if (hw->nvm.ops.validate(hw) < 0) {
3185 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
3186 err = -EIO;
3187 goto err_eeprom;
3188 }
3189 break;
3190 }
3191
3192 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
3193 /* copy the MAC address out of the NVM */
3194 if (hw->mac.ops.read_mac_addr(hw))
3195 dev_err(&pdev->dev, "NVM Read Error\n");
3196 }
3197
3198 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
3199
3200 if (!is_valid_ether_addr(netdev->dev_addr)) {
3201 dev_err(&pdev->dev, "Invalid MAC Address\n");
3202 err = -EIO;
3203 goto err_eeprom;
3204 }
3205
3206 igb_set_default_mac_filter(adapter);
3207
3208 /* get firmware version for ethtool -i */
3209 igb_set_fw_version(adapter);
3210
3211 /* configure RXPBSIZE and TXPBSIZE */
3212 if (hw->mac.type == e1000_i210) {
3213 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
3214 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
3215 }
3216
3217 timer_setup(&adapter->watchdog_timer, igb_watchdog, 0);
3218 timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0);
3219
3220 INIT_WORK(&adapter->reset_task, igb_reset_task);
3221 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
3222
3223 /* Initialize link properties that are user-changeable */
3224 adapter->fc_autoneg = true;
3225 hw->mac.autoneg = true;
3226 hw->phy.autoneg_advertised = 0x2f;
3227
3228 hw->fc.requested_mode = e1000_fc_default;
3229 hw->fc.current_mode = e1000_fc_default;
3230
3231 igb_validate_mdi_setting(hw);
3232
3233 /* By default, support wake on port A */
3234 if (hw->bus.func == 0)
3235 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3236
3237 /* Check the NVM for wake support on non-port A ports */
3238 if (hw->mac.type >= e1000_82580)
3239 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
3240 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
3241 &eeprom_data);
3242 else if (hw->bus.func == 1)
3243 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
3244
3245 if (eeprom_data & IGB_EEPROM_APME)
3246 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3247
3248 /* now that we have the eeprom settings, apply the special cases where
3249 * the eeprom may be wrong or the board simply won't support wake on
3250 * lan on a particular port
3251 */
3252 switch (pdev->device) {
3253 case E1000_DEV_ID_82575GB_QUAD_COPPER:
3254 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3255 break;
3256 case E1000_DEV_ID_82575EB_FIBER_SERDES:
3257 case E1000_DEV_ID_82576_FIBER:
3258 case E1000_DEV_ID_82576_SERDES:
3259 /* Wake events only supported on port A for dual fiber
3260 * regardless of eeprom setting
3261 */
3262 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
3263 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3264 break;
3265 case E1000_DEV_ID_82576_QUAD_COPPER:
3266 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
3267 /* if quad port adapter, disable WoL on all but port A */
3268 if (global_quad_port_a != 0)
3269 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3270 else
3271 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
3272 /* Reset for multiple quad port adapters */
3273 if (++global_quad_port_a == 4)
3274 global_quad_port_a = 0;
3275 break;
3276 default:
3277 /* If the device can't wake, don't set software support */
3278 if (!device_can_wakeup(&adapter->pdev->dev))
3279 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3280 }
3281
3282 /* initialize the wol settings based on the eeprom settings */
3283 if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
3284 adapter->wol |= E1000_WUFC_MAG;
3285
3286 /* Some vendors want WoL disabled by default, but still supported */
3287 if ((hw->mac.type == e1000_i350) &&
3288 (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
3289 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3290 adapter->wol = 0;
3291 }
3292
3293 /* Some vendors want the ability to Use the EEPROM setting as
3294 * enable/disable only, and not for capability
3295 */
3296 if (((hw->mac.type == e1000_i350) ||
3297 (hw->mac.type == e1000_i354)) &&
3298 (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) {
3299 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3300 adapter->wol = 0;
3301 }
3302 if (hw->mac.type == e1000_i350) {
3303 if (((pdev->subsystem_device == 0x5001) ||
3304 (pdev->subsystem_device == 0x5002)) &&
3305 (hw->bus.func == 0)) {
3306 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3307 adapter->wol = 0;
3308 }
3309 if (pdev->subsystem_device == 0x1F52)
3310 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3311 }
3312
3313 device_set_wakeup_enable(&adapter->pdev->dev,
3314 adapter->flags & IGB_FLAG_WOL_SUPPORTED);
3315
3316 /* reset the hardware with the new settings */
3317 igb_reset(adapter);
3318
3319 /* Init the I2C interface */
3320 err = igb_init_i2c(adapter);
3321 if (err) {
3322 dev_err(&pdev->dev, "failed to init i2c interface\n");
3323 goto err_eeprom;
3324 }
3325
3326 /* let the f/w know that the h/w is now under the control of the
3327 * driver.
3328 */
3329 igb_get_hw_control(adapter);
3330
3331 strcpy(netdev->name, "eth%d");
3332 err = register_netdev(netdev);
3333 if (err)
3334 goto err_register;
3335
3336 /* carrier off reporting is important to ethtool even BEFORE open */
3337 netif_carrier_off(netdev);
3338
3339#ifdef CONFIG_IGB_DCA
3340 if (dca_add_requester(&pdev->dev) == 0) {
3341 adapter->flags |= IGB_FLAG_DCA_ENABLED;
3342 dev_info(&pdev->dev, "DCA enabled\n");
3343 igb_setup_dca(adapter);
3344 }
3345
3346#endif
3347#ifdef CONFIG_IGB_HWMON
3348 /* Initialize the thermal sensor on i350 devices. */
3349 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
3350 u16 ets_word;
3351
3352 /* Read the NVM to determine if this i350 device supports an
3353 * external thermal sensor.
3354 */
3355 hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
3356 if (ets_word != 0x0000 && ets_word != 0xFFFF)
3357 adapter->ets = true;
3358 else
3359 adapter->ets = false;
3360 if (igb_sysfs_init(adapter))
3361 dev_err(&pdev->dev,
3362 "failed to allocate sysfs resources\n");
3363 } else {
3364 adapter->ets = false;
3365 }
3366#endif
3367 /* Check if Media Autosense is enabled */
3368 adapter->ei = *ei;
3369 if (hw->dev_spec._82575.mas_capable)
3370 igb_init_mas(adapter);
3371
3372 /* do hw tstamp init after resetting */
3373 igb_ptp_init(adapter);
3374
3375 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
3376 /* print bus type/speed/width info, not applicable to i354 */
3377 if (hw->mac.type != e1000_i354) {
3378 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
3379 netdev->name,
3380 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
3381 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
3382 "unknown"),
3383 ((hw->bus.width == e1000_bus_width_pcie_x4) ?
3384 "Width x4" :
3385 (hw->bus.width == e1000_bus_width_pcie_x2) ?
3386 "Width x2" :
3387 (hw->bus.width == e1000_bus_width_pcie_x1) ?
3388 "Width x1" : "unknown"), netdev->dev_addr);
3389 }
3390
3391 if ((hw->mac.type >= e1000_i210 ||
3392 igb_get_flash_presence_i210(hw))) {
3393 ret_val = igb_read_part_string(hw, part_str,
3394 E1000_PBANUM_LENGTH);
3395 } else {
3396 ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
3397 }
3398
3399 if (ret_val)
3400 strcpy(part_str, "Unknown");
3401 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
3402 dev_info(&pdev->dev,
3403 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
3404 (adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" :
3405 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
3406 adapter->num_rx_queues, adapter->num_tx_queues);
3407 if (hw->phy.media_type == e1000_media_type_copper) {
3408 switch (hw->mac.type) {
3409 case e1000_i350:
3410 case e1000_i210:
3411 case e1000_i211:
3412 /* Enable EEE for internal copper PHY devices */
3413 err = igb_set_eee_i350(hw, true, true);
3414 if ((!err) &&
3415 (!hw->dev_spec._82575.eee_disable)) {
3416 adapter->eee_advert =
3417 MDIO_EEE_100TX | MDIO_EEE_1000T;
3418 adapter->flags |= IGB_FLAG_EEE;
3419 }
3420 break;
3421 case e1000_i354:
3422 if ((rd32(E1000_CTRL_EXT) &
3423 E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3424 err = igb_set_eee_i354(hw, true, true);
3425 if ((!err) &&
3426 (!hw->dev_spec._82575.eee_disable)) {
3427 adapter->eee_advert =
3428 MDIO_EEE_100TX | MDIO_EEE_1000T;
3429 adapter->flags |= IGB_FLAG_EEE;
3430 }
3431 }
3432 break;
3433 default:
3434 break;
3435 }
3436 }
3437
3438 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
3439
3440 pm_runtime_put_noidle(&pdev->dev);
3441 return 0;
3442
3443err_register:
3444 igb_release_hw_control(adapter);
3445 memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
3446err_eeprom:
3447 if (!igb_check_reset_block(hw))
3448 igb_reset_phy(hw);
3449
3450 if (hw->flash_address)
3451 iounmap(hw->flash_address);
3452err_sw_init:
3453 kfree(adapter->mac_table);
3454 kfree(adapter->shadow_vfta);
3455 igb_clear_interrupt_scheme(adapter);
3456#ifdef CONFIG_PCI_IOV
3457 igb_disable_sriov(pdev);
3458#endif
3459 pci_iounmap(pdev, adapter->io_addr);
3460err_ioremap:
3461 free_netdev(netdev);
3462err_alloc_etherdev:
3463 pci_release_mem_regions(pdev);
3464err_pci_reg:
3465err_dma:
3466 pci_disable_device(pdev);
3467 return err;
3468}
3469
3470#ifdef CONFIG_PCI_IOV
3471static int igb_disable_sriov(struct pci_dev *pdev)
3472{
3473 struct net_device *netdev = pci_get_drvdata(pdev);
3474 struct igb_adapter *adapter = netdev_priv(netdev);
3475 struct e1000_hw *hw = &adapter->hw;
3476
3477 /* reclaim resources allocated to VFs */
3478 if (adapter->vf_data) {
3479 /* disable iov and allow time for transactions to clear */
3480 if (pci_vfs_assigned(pdev)) {
3481 dev_warn(&pdev->dev,
3482 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
3483 return -EPERM;
3484 } else {
3485 pci_disable_sriov(pdev);
3486 msleep(500);
3487 }
3488
3489 kfree(adapter->vf_mac_list);
3490 adapter->vf_mac_list = NULL;
3491 kfree(adapter->vf_data);
3492 adapter->vf_data = NULL;
3493 adapter->vfs_allocated_count = 0;
3494 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
3495 wrfl();
3496 msleep(100);
3497 dev_info(&pdev->dev, "IOV Disabled\n");
3498
3499 /* Re-enable DMA Coalescing flag since IOV is turned off */
3500 adapter->flags |= IGB_FLAG_DMAC;
3501 }
3502
3503 return 0;
3504}
3505
3506static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs)
3507{
3508 struct net_device *netdev = pci_get_drvdata(pdev);
3509 struct igb_adapter *adapter = netdev_priv(netdev);
3510 int old_vfs = pci_num_vf(pdev);
3511 struct vf_mac_filter *mac_list;
3512 int err = 0;
3513 int num_vf_mac_filters, i;
3514
3515 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) {
3516 err = -EPERM;
3517 goto out;
3518 }
3519 if (!num_vfs)
3520 goto out;
3521
3522 if (old_vfs) {
3523 dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
3524 old_vfs, max_vfs);
3525 adapter->vfs_allocated_count = old_vfs;
3526 } else
3527 adapter->vfs_allocated_count = num_vfs;
3528
3529 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
3530 sizeof(struct vf_data_storage), GFP_KERNEL);
3531
3532 /* if allocation failed then we do not support SR-IOV */
3533 if (!adapter->vf_data) {
3534 adapter->vfs_allocated_count = 0;
3535 err = -ENOMEM;
3536 goto out;
3537 }
3538
3539 /* Due to the limited number of RAR entries calculate potential
3540 * number of MAC filters available for the VFs. Reserve entries
3541 * for PF default MAC, PF MAC filters and at least one RAR entry
3542 * for each VF for VF MAC.
3543 */
3544 num_vf_mac_filters = adapter->hw.mac.rar_entry_count -
3545 (1 + IGB_PF_MAC_FILTERS_RESERVED +
3546 adapter->vfs_allocated_count);
3547
3548 adapter->vf_mac_list = kcalloc(num_vf_mac_filters,
3549 sizeof(struct vf_mac_filter),
3550 GFP_KERNEL);
3551
3552 mac_list = adapter->vf_mac_list;
3553 INIT_LIST_HEAD(&adapter->vf_macs.l);
3554
3555 if (adapter->vf_mac_list) {
3556 /* Initialize list of VF MAC filters */
3557 for (i = 0; i < num_vf_mac_filters; i++) {
3558 mac_list->vf = -1;
3559 mac_list->free = true;
3560 list_add(&mac_list->l, &adapter->vf_macs.l);
3561 mac_list++;
3562 }
3563 } else {
3564 /* If we could not allocate memory for the VF MAC filters
3565 * we can continue without this feature but warn user.
3566 */
3567 dev_err(&pdev->dev,
3568 "Unable to allocate memory for VF MAC filter list\n");
3569 }
3570
3571 /* only call pci_enable_sriov() if no VFs are allocated already */
3572 if (!old_vfs) {
3573 err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
3574 if (err)
3575 goto err_out;
3576 }
3577 dev_info(&pdev->dev, "%d VFs allocated\n",
3578 adapter->vfs_allocated_count);
3579 for (i = 0; i < adapter->vfs_allocated_count; i++)
3580 igb_vf_configure(adapter, i);
3581
3582 /* DMA Coalescing is not supported in IOV mode. */
3583 adapter->flags &= ~IGB_FLAG_DMAC;
3584 goto out;
3585
3586err_out:
3587 kfree(adapter->vf_mac_list);
3588 adapter->vf_mac_list = NULL;
3589 kfree(adapter->vf_data);
3590 adapter->vf_data = NULL;
3591 adapter->vfs_allocated_count = 0;
3592out:
3593 return err;
3594}
3595
3596#endif
3597/**
3598 * igb_remove_i2c - Cleanup I2C interface
3599 * @adapter: pointer to adapter structure
3600 **/
3601static void igb_remove_i2c(struct igb_adapter *adapter)
3602{
3603 /* free the adapter bus structure */
3604 i2c_del_adapter(&adapter->i2c_adap);
3605}
3606
3607/**
3608 * igb_remove - Device Removal Routine
3609 * @pdev: PCI device information struct
3610 *
3611 * igb_remove is called by the PCI subsystem to alert the driver
3612 * that it should release a PCI device. The could be caused by a
3613 * Hot-Plug event, or because the driver is going to be removed from
3614 * memory.
3615 **/
3616static void igb_remove(struct pci_dev *pdev)
3617{
3618 struct net_device *netdev = pci_get_drvdata(pdev);
3619 struct igb_adapter *adapter = netdev_priv(netdev);
3620 struct e1000_hw *hw = &adapter->hw;
3621
3622 pm_runtime_get_noresume(&pdev->dev);
3623#ifdef CONFIG_IGB_HWMON
3624 igb_sysfs_exit(adapter);
3625#endif
3626 igb_remove_i2c(adapter);
3627 igb_ptp_stop(adapter);
3628 /* The watchdog timer may be rescheduled, so explicitly
3629 * disable watchdog from being rescheduled.
3630 */
3631 set_bit(__IGB_DOWN, &adapter->state);
3632 del_timer_sync(&adapter->watchdog_timer);
3633 del_timer_sync(&adapter->phy_info_timer);
3634
3635 cancel_work_sync(&adapter->reset_task);
3636 cancel_work_sync(&adapter->watchdog_task);
3637
3638#ifdef CONFIG_IGB_DCA
3639 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
3640 dev_info(&pdev->dev, "DCA disabled\n");
3641 dca_remove_requester(&pdev->dev);
3642 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
3643 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
3644 }
3645#endif
3646
3647 /* Release control of h/w to f/w. If f/w is AMT enabled, this
3648 * would have already happened in close and is redundant.
3649 */
3650 igb_release_hw_control(adapter);
3651
3652#ifdef CONFIG_PCI_IOV
3653 igb_disable_sriov(pdev);
3654#endif
3655
3656 unregister_netdev(netdev);
3657
3658 igb_clear_interrupt_scheme(adapter);
3659
3660 pci_iounmap(pdev, adapter->io_addr);
3661 if (hw->flash_address)
3662 iounmap(hw->flash_address);
3663 pci_release_mem_regions(pdev);
3664
3665 kfree(adapter->mac_table);
3666 kfree(adapter->shadow_vfta);
3667 free_netdev(netdev);
3668
3669 pci_disable_pcie_error_reporting(pdev);
3670
3671 pci_disable_device(pdev);
3672}
3673
3674/**
3675 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
3676 * @adapter: board private structure to initialize
3677 *
3678 * This function initializes the vf specific data storage and then attempts to
3679 * allocate the VFs. The reason for ordering it this way is because it is much
3680 * mor expensive time wise to disable SR-IOV than it is to allocate and free
3681 * the memory for the VFs.
3682 **/
3683static void igb_probe_vfs(struct igb_adapter *adapter)
3684{
3685#ifdef CONFIG_PCI_IOV
3686 struct pci_dev *pdev = adapter->pdev;
3687 struct e1000_hw *hw = &adapter->hw;
3688
3689 /* Virtualization features not supported on i210 family. */
3690 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
3691 return;
3692
3693 /* Of the below we really only want the effect of getting
3694 * IGB_FLAG_HAS_MSIX set (if available), without which
3695 * igb_enable_sriov() has no effect.
3696 */
3697 igb_set_interrupt_capability(adapter, true);
3698 igb_reset_interrupt_capability(adapter);
3699
3700 pci_sriov_set_totalvfs(pdev, 7);
3701 igb_enable_sriov(pdev, max_vfs);
3702
3703#endif /* CONFIG_PCI_IOV */
3704}
3705
3706unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter)
3707{
3708 struct e1000_hw *hw = &adapter->hw;
3709 unsigned int max_rss_queues;
3710
3711 /* Determine the maximum number of RSS queues supported. */
3712 switch (hw->mac.type) {
3713 case e1000_i211:
3714 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
3715 break;
3716 case e1000_82575:
3717 case e1000_i210:
3718 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
3719 break;
3720 case e1000_i350:
3721 /* I350 cannot do RSS and SR-IOV at the same time */
3722 if (!!adapter->vfs_allocated_count) {
3723 max_rss_queues = 1;
3724 break;
3725 }
3726 fallthrough;
3727 case e1000_82576:
3728 if (!!adapter->vfs_allocated_count) {
3729 max_rss_queues = 2;
3730 break;
3731 }
3732 fallthrough;
3733 case e1000_82580:
3734 case e1000_i354:
3735 default:
3736 max_rss_queues = IGB_MAX_RX_QUEUES;
3737 break;
3738 }
3739
3740 return max_rss_queues;
3741}
3742
3743static void igb_init_queue_configuration(struct igb_adapter *adapter)
3744{
3745 u32 max_rss_queues;
3746
3747 max_rss_queues = igb_get_max_rss_queues(adapter);
3748 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
3749
3750 igb_set_flag_queue_pairs(adapter, max_rss_queues);
3751}
3752
3753void igb_set_flag_queue_pairs(struct igb_adapter *adapter,
3754 const u32 max_rss_queues)
3755{
3756 struct e1000_hw *hw = &adapter->hw;
3757
3758 /* Determine if we need to pair queues. */
3759 switch (hw->mac.type) {
3760 case e1000_82575:
3761 case e1000_i211:
3762 /* Device supports enough interrupts without queue pairing. */
3763 break;
3764 case e1000_82576:
3765 case e1000_82580:
3766 case e1000_i350:
3767 case e1000_i354:
3768 case e1000_i210:
3769 default:
3770 /* If rss_queues > half of max_rss_queues, pair the queues in
3771 * order to conserve interrupts due to limited supply.
3772 */
3773 if (adapter->rss_queues > (max_rss_queues / 2))
3774 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
3775 else
3776 adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS;
3777 break;
3778 }
3779}
3780
3781/**
3782 * igb_sw_init - Initialize general software structures (struct igb_adapter)
3783 * @adapter: board private structure to initialize
3784 *
3785 * igb_sw_init initializes the Adapter private data structure.
3786 * Fields are initialized based on PCI device information and
3787 * OS network device settings (MTU size).
3788 **/
3789static int igb_sw_init(struct igb_adapter *adapter)
3790{
3791 struct e1000_hw *hw = &adapter->hw;
3792 struct net_device *netdev = adapter->netdev;
3793 struct pci_dev *pdev = adapter->pdev;
3794
3795 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
3796
3797 /* set default ring sizes */
3798 adapter->tx_ring_count = IGB_DEFAULT_TXD;
3799 adapter->rx_ring_count = IGB_DEFAULT_RXD;
3800
3801 /* set default ITR values */
3802 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
3803 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
3804
3805 /* set default work limits */
3806 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
3807
3808 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
3809 VLAN_HLEN;
3810 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
3811
3812 spin_lock_init(&adapter->nfc_lock);
3813 spin_lock_init(&adapter->stats64_lock);
3814#ifdef CONFIG_PCI_IOV
3815 switch (hw->mac.type) {
3816 case e1000_82576:
3817 case e1000_i350:
3818 if (max_vfs > 7) {
3819 dev_warn(&pdev->dev,
3820 "Maximum of 7 VFs per PF, using max\n");
3821 max_vfs = adapter->vfs_allocated_count = 7;
3822 } else
3823 adapter->vfs_allocated_count = max_vfs;
3824 if (adapter->vfs_allocated_count)
3825 dev_warn(&pdev->dev,
3826 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
3827 break;
3828 default:
3829 break;
3830 }
3831#endif /* CONFIG_PCI_IOV */
3832
3833 /* Assume MSI-X interrupts, will be checked during IRQ allocation */
3834 adapter->flags |= IGB_FLAG_HAS_MSIX;
3835
3836 adapter->mac_table = kcalloc(hw->mac.rar_entry_count,
3837 sizeof(struct igb_mac_addr),
3838 GFP_KERNEL);
3839 if (!adapter->mac_table)
3840 return -ENOMEM;
3841
3842 igb_probe_vfs(adapter);
3843
3844 igb_init_queue_configuration(adapter);
3845
3846 /* Setup and initialize a copy of the hw vlan table array */
3847 adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
3848 GFP_KERNEL);
3849 if (!adapter->shadow_vfta)
3850 return -ENOMEM;
3851
3852 /* This call may decrease the number of queues */
3853 if (igb_init_interrupt_scheme(adapter, true)) {
3854 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
3855 return -ENOMEM;
3856 }
3857
3858 /* Explicitly disable IRQ since the NIC can be in any state. */
3859 igb_irq_disable(adapter);
3860
3861 if (hw->mac.type >= e1000_i350)
3862 adapter->flags &= ~IGB_FLAG_DMAC;
3863
3864 set_bit(__IGB_DOWN, &adapter->state);
3865 return 0;
3866}
3867
3868/**
3869 * igb_open - Called when a network interface is made active
3870 * @netdev: network interface device structure
3871 *
3872 * Returns 0 on success, negative value on failure
3873 *
3874 * The open entry point is called when a network interface is made
3875 * active by the system (IFF_UP). At this point all resources needed
3876 * for transmit and receive operations are allocated, the interrupt
3877 * handler is registered with the OS, the watchdog timer is started,
3878 * and the stack is notified that the interface is ready.
3879 **/
3880static int __igb_open(struct net_device *netdev, bool resuming)
3881{
3882 struct igb_adapter *adapter = netdev_priv(netdev);
3883 struct e1000_hw *hw = &adapter->hw;
3884 struct pci_dev *pdev = adapter->pdev;
3885 int err;
3886 int i;
3887
3888 /* disallow open during test */
3889 if (test_bit(__IGB_TESTING, &adapter->state)) {
3890 WARN_ON(resuming);
3891 return -EBUSY;
3892 }
3893
3894 if (!resuming)
3895 pm_runtime_get_sync(&pdev->dev);
3896
3897 netif_carrier_off(netdev);
3898
3899 /* allocate transmit descriptors */
3900 err = igb_setup_all_tx_resources(adapter);
3901 if (err)
3902 goto err_setup_tx;
3903
3904 /* allocate receive descriptors */
3905 err = igb_setup_all_rx_resources(adapter);
3906 if (err)
3907 goto err_setup_rx;
3908
3909 igb_power_up_link(adapter);
3910
3911 /* before we allocate an interrupt, we must be ready to handle it.
3912 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
3913 * as soon as we call pci_request_irq, so we have to setup our
3914 * clean_rx handler before we do so.
3915 */
3916 igb_configure(adapter);
3917
3918 err = igb_request_irq(adapter);
3919 if (err)
3920 goto err_req_irq;
3921
3922 /* Notify the stack of the actual queue counts. */
3923 err = netif_set_real_num_tx_queues(adapter->netdev,
3924 adapter->num_tx_queues);
3925 if (err)
3926 goto err_set_queues;
3927
3928 err = netif_set_real_num_rx_queues(adapter->netdev,
3929 adapter->num_rx_queues);
3930 if (err)
3931 goto err_set_queues;
3932
3933 /* From here on the code is the same as igb_up() */
3934 clear_bit(__IGB_DOWN, &adapter->state);
3935
3936 for (i = 0; i < adapter->num_q_vectors; i++)
3937 napi_enable(&(adapter->q_vector[i]->napi));
3938
3939 /* Clear any pending interrupts. */
3940 rd32(E1000_TSICR);
3941 rd32(E1000_ICR);
3942
3943 igb_irq_enable(adapter);
3944
3945 /* notify VFs that reset has been completed */
3946 if (adapter->vfs_allocated_count) {
3947 u32 reg_data = rd32(E1000_CTRL_EXT);
3948
3949 reg_data |= E1000_CTRL_EXT_PFRSTD;
3950 wr32(E1000_CTRL_EXT, reg_data);
3951 }
3952
3953 netif_tx_start_all_queues(netdev);
3954
3955 if (!resuming)
3956 pm_runtime_put(&pdev->dev);
3957
3958 /* start the watchdog. */
3959 hw->mac.get_link_status = 1;
3960 schedule_work(&adapter->watchdog_task);
3961
3962 return 0;
3963
3964err_set_queues:
3965 igb_free_irq(adapter);
3966err_req_irq:
3967 igb_release_hw_control(adapter);
3968 igb_power_down_link(adapter);
3969 igb_free_all_rx_resources(adapter);
3970err_setup_rx:
3971 igb_free_all_tx_resources(adapter);
3972err_setup_tx:
3973 igb_reset(adapter);
3974 if (!resuming)
3975 pm_runtime_put(&pdev->dev);
3976
3977 return err;
3978}
3979
3980int igb_open(struct net_device *netdev)
3981{
3982 return __igb_open(netdev, false);
3983}
3984
3985/**
3986 * igb_close - Disables a network interface
3987 * @netdev: network interface device structure
3988 *
3989 * Returns 0, this is not allowed to fail
3990 *
3991 * The close entry point is called when an interface is de-activated
3992 * by the OS. The hardware is still under the driver's control, but
3993 * needs to be disabled. A global MAC reset is issued to stop the
3994 * hardware, and all transmit and receive resources are freed.
3995 **/
3996static int __igb_close(struct net_device *netdev, bool suspending)
3997{
3998 struct igb_adapter *adapter = netdev_priv(netdev);
3999 struct pci_dev *pdev = adapter->pdev;
4000
4001 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
4002
4003 if (!suspending)
4004 pm_runtime_get_sync(&pdev->dev);
4005
4006 igb_down(adapter);
4007 igb_free_irq(adapter);
4008
4009 igb_free_all_tx_resources(adapter);
4010 igb_free_all_rx_resources(adapter);
4011
4012 if (!suspending)
4013 pm_runtime_put_sync(&pdev->dev);
4014 return 0;
4015}
4016
4017int igb_close(struct net_device *netdev)
4018{
4019 if (netif_device_present(netdev) || netdev->dismantle)
4020 return __igb_close(netdev, false);
4021 return 0;
4022}
4023
4024/**
4025 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
4026 * @tx_ring: tx descriptor ring (for a specific queue) to setup
4027 *
4028 * Return 0 on success, negative on failure
4029 **/
4030int igb_setup_tx_resources(struct igb_ring *tx_ring)
4031{
4032 struct device *dev = tx_ring->dev;
4033 int size;
4034
4035 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
4036
4037 tx_ring->tx_buffer_info = vmalloc(size);
4038 if (!tx_ring->tx_buffer_info)
4039 goto err;
4040
4041 /* round up to nearest 4K */
4042 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
4043 tx_ring->size = ALIGN(tx_ring->size, 4096);
4044
4045 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
4046 &tx_ring->dma, GFP_KERNEL);
4047 if (!tx_ring->desc)
4048 goto err;
4049
4050 tx_ring->next_to_use = 0;
4051 tx_ring->next_to_clean = 0;
4052
4053 return 0;
4054
4055err:
4056 vfree(tx_ring->tx_buffer_info);
4057 tx_ring->tx_buffer_info = NULL;
4058 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
4059 return -ENOMEM;
4060}
4061
4062/**
4063 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
4064 * (Descriptors) for all queues
4065 * @adapter: board private structure
4066 *
4067 * Return 0 on success, negative on failure
4068 **/
4069static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
4070{
4071 struct pci_dev *pdev = adapter->pdev;
4072 int i, err = 0;
4073
4074 for (i = 0; i < adapter->num_tx_queues; i++) {
4075 err = igb_setup_tx_resources(adapter->tx_ring[i]);
4076 if (err) {
4077 dev_err(&pdev->dev,
4078 "Allocation for Tx Queue %u failed\n", i);
4079 for (i--; i >= 0; i--)
4080 igb_free_tx_resources(adapter->tx_ring[i]);
4081 break;
4082 }
4083 }
4084
4085 return err;
4086}
4087
4088/**
4089 * igb_setup_tctl - configure the transmit control registers
4090 * @adapter: Board private structure
4091 **/
4092void igb_setup_tctl(struct igb_adapter *adapter)
4093{
4094 struct e1000_hw *hw = &adapter->hw;
4095 u32 tctl;
4096
4097 /* disable queue 0 which is enabled by default on 82575 and 82576 */
4098 wr32(E1000_TXDCTL(0), 0);
4099
4100 /* Program the Transmit Control Register */
4101 tctl = rd32(E1000_TCTL);
4102 tctl &= ~E1000_TCTL_CT;
4103 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
4104 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
4105
4106 igb_config_collision_dist(hw);
4107
4108 /* Enable transmits */
4109 tctl |= E1000_TCTL_EN;
4110
4111 wr32(E1000_TCTL, tctl);
4112}
4113
4114/**
4115 * igb_configure_tx_ring - Configure transmit ring after Reset
4116 * @adapter: board private structure
4117 * @ring: tx ring to configure
4118 *
4119 * Configure a transmit ring after a reset.
4120 **/
4121void igb_configure_tx_ring(struct igb_adapter *adapter,
4122 struct igb_ring *ring)
4123{
4124 struct e1000_hw *hw = &adapter->hw;
4125 u32 txdctl = 0;
4126 u64 tdba = ring->dma;
4127 int reg_idx = ring->reg_idx;
4128
4129 wr32(E1000_TDLEN(reg_idx),
4130 ring->count * sizeof(union e1000_adv_tx_desc));
4131 wr32(E1000_TDBAL(reg_idx),
4132 tdba & 0x00000000ffffffffULL);
4133 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
4134
4135 ring->tail = adapter->io_addr + E1000_TDT(reg_idx);
4136 wr32(E1000_TDH(reg_idx), 0);
4137 writel(0, ring->tail);
4138
4139 txdctl |= IGB_TX_PTHRESH;
4140 txdctl |= IGB_TX_HTHRESH << 8;
4141 txdctl |= IGB_TX_WTHRESH << 16;
4142
4143 /* reinitialize tx_buffer_info */
4144 memset(ring->tx_buffer_info, 0,
4145 sizeof(struct igb_tx_buffer) * ring->count);
4146
4147 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
4148 wr32(E1000_TXDCTL(reg_idx), txdctl);
4149}
4150
4151/**
4152 * igb_configure_tx - Configure transmit Unit after Reset
4153 * @adapter: board private structure
4154 *
4155 * Configure the Tx unit of the MAC after a reset.
4156 **/
4157static void igb_configure_tx(struct igb_adapter *adapter)
4158{
4159 struct e1000_hw *hw = &adapter->hw;
4160 int i;
4161
4162 /* disable the queues */
4163 for (i = 0; i < adapter->num_tx_queues; i++)
4164 wr32(E1000_TXDCTL(adapter->tx_ring[i]->reg_idx), 0);
4165
4166 wrfl();
4167 usleep_range(10000, 20000);
4168
4169 for (i = 0; i < adapter->num_tx_queues; i++)
4170 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
4171}
4172
4173/**
4174 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
4175 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
4176 *
4177 * Returns 0 on success, negative on failure
4178 **/
4179int igb_setup_rx_resources(struct igb_ring *rx_ring)
4180{
4181 struct device *dev = rx_ring->dev;
4182 int size;
4183
4184 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
4185
4186 rx_ring->rx_buffer_info = vmalloc(size);
4187 if (!rx_ring->rx_buffer_info)
4188 goto err;
4189
4190 /* Round up to nearest 4K */
4191 rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
4192 rx_ring->size = ALIGN(rx_ring->size, 4096);
4193
4194 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
4195 &rx_ring->dma, GFP_KERNEL);
4196 if (!rx_ring->desc)
4197 goto err;
4198
4199 rx_ring->next_to_alloc = 0;
4200 rx_ring->next_to_clean = 0;
4201 rx_ring->next_to_use = 0;
4202
4203 return 0;
4204
4205err:
4206 vfree(rx_ring->rx_buffer_info);
4207 rx_ring->rx_buffer_info = NULL;
4208 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
4209 return -ENOMEM;
4210}
4211
4212/**
4213 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
4214 * (Descriptors) for all queues
4215 * @adapter: board private structure
4216 *
4217 * Return 0 on success, negative on failure
4218 **/
4219static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
4220{
4221 struct pci_dev *pdev = adapter->pdev;
4222 int i, err = 0;
4223
4224 for (i = 0; i < adapter->num_rx_queues; i++) {
4225 err = igb_setup_rx_resources(adapter->rx_ring[i]);
4226 if (err) {
4227 dev_err(&pdev->dev,
4228 "Allocation for Rx Queue %u failed\n", i);
4229 for (i--; i >= 0; i--)
4230 igb_free_rx_resources(adapter->rx_ring[i]);
4231 break;
4232 }
4233 }
4234
4235 return err;
4236}
4237
4238/**
4239 * igb_setup_mrqc - configure the multiple receive queue control registers
4240 * @adapter: Board private structure
4241 **/
4242static void igb_setup_mrqc(struct igb_adapter *adapter)
4243{
4244 struct e1000_hw *hw = &adapter->hw;
4245 u32 mrqc, rxcsum;
4246 u32 j, num_rx_queues;
4247 u32 rss_key[10];
4248
4249 netdev_rss_key_fill(rss_key, sizeof(rss_key));
4250 for (j = 0; j < 10; j++)
4251 wr32(E1000_RSSRK(j), rss_key[j]);
4252
4253 num_rx_queues = adapter->rss_queues;
4254
4255 switch (hw->mac.type) {
4256 case e1000_82576:
4257 /* 82576 supports 2 RSS queues for SR-IOV */
4258 if (adapter->vfs_allocated_count)
4259 num_rx_queues = 2;
4260 break;
4261 default:
4262 break;
4263 }
4264
4265 if (adapter->rss_indir_tbl_init != num_rx_queues) {
4266 for (j = 0; j < IGB_RETA_SIZE; j++)
4267 adapter->rss_indir_tbl[j] =
4268 (j * num_rx_queues) / IGB_RETA_SIZE;
4269 adapter->rss_indir_tbl_init = num_rx_queues;
4270 }
4271 igb_write_rss_indir_tbl(adapter);
4272
4273 /* Disable raw packet checksumming so that RSS hash is placed in
4274 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
4275 * offloads as they are enabled by default
4276 */
4277 rxcsum = rd32(E1000_RXCSUM);
4278 rxcsum |= E1000_RXCSUM_PCSD;
4279
4280 if (adapter->hw.mac.type >= e1000_82576)
4281 /* Enable Receive Checksum Offload for SCTP */
4282 rxcsum |= E1000_RXCSUM_CRCOFL;
4283
4284 /* Don't need to set TUOFL or IPOFL, they default to 1 */
4285 wr32(E1000_RXCSUM, rxcsum);
4286
4287 /* Generate RSS hash based on packet types, TCP/UDP
4288 * port numbers and/or IPv4/v6 src and dst addresses
4289 */
4290 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
4291 E1000_MRQC_RSS_FIELD_IPV4_TCP |
4292 E1000_MRQC_RSS_FIELD_IPV6 |
4293 E1000_MRQC_RSS_FIELD_IPV6_TCP |
4294 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
4295
4296 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
4297 mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
4298 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
4299 mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;
4300
4301 /* If VMDq is enabled then we set the appropriate mode for that, else
4302 * we default to RSS so that an RSS hash is calculated per packet even
4303 * if we are only using one queue
4304 */
4305 if (adapter->vfs_allocated_count) {
4306 if (hw->mac.type > e1000_82575) {
4307 /* Set the default pool for the PF's first queue */
4308 u32 vtctl = rd32(E1000_VT_CTL);
4309
4310 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
4311 E1000_VT_CTL_DISABLE_DEF_POOL);
4312 vtctl |= adapter->vfs_allocated_count <<
4313 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
4314 wr32(E1000_VT_CTL, vtctl);
4315 }
4316 if (adapter->rss_queues > 1)
4317 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ;
4318 else
4319 mrqc |= E1000_MRQC_ENABLE_VMDQ;
4320 } else {
4321 if (hw->mac.type != e1000_i211)
4322 mrqc |= E1000_MRQC_ENABLE_RSS_MQ;
4323 }
4324 igb_vmm_control(adapter);
4325
4326 wr32(E1000_MRQC, mrqc);
4327}
4328
4329/**
4330 * igb_setup_rctl - configure the receive control registers
4331 * @adapter: Board private structure
4332 **/
4333void igb_setup_rctl(struct igb_adapter *adapter)
4334{
4335 struct e1000_hw *hw = &adapter->hw;
4336 u32 rctl;
4337
4338 rctl = rd32(E1000_RCTL);
4339
4340 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4341 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
4342
4343 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
4344 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4345
4346 /* enable stripping of CRC. It's unlikely this will break BMC
4347 * redirection as it did with e1000. Newer features require
4348 * that the HW strips the CRC.
4349 */
4350 rctl |= E1000_RCTL_SECRC;
4351
4352 /* disable store bad packets and clear size bits. */
4353 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
4354
4355 /* enable LPE to allow for reception of jumbo frames */
4356 rctl |= E1000_RCTL_LPE;
4357
4358 /* disable queue 0 to prevent tail write w/o re-config */
4359 wr32(E1000_RXDCTL(0), 0);
4360
4361 /* Attention!!! For SR-IOV PF driver operations you must enable
4362 * queue drop for all VF and PF queues to prevent head of line blocking
4363 * if an un-trusted VF does not provide descriptors to hardware.
4364 */
4365 if (adapter->vfs_allocated_count) {
4366 /* set all queue drop enable bits */
4367 wr32(E1000_QDE, ALL_QUEUES);
4368 }
4369
4370 /* This is useful for sniffing bad packets. */
4371 if (adapter->netdev->features & NETIF_F_RXALL) {
4372 /* UPE and MPE will be handled by normal PROMISC logic
4373 * in e1000e_set_rx_mode
4374 */
4375 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
4376 E1000_RCTL_BAM | /* RX All Bcast Pkts */
4377 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
4378
4379 rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */
4380 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
4381 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
4382 * and that breaks VLANs.
4383 */
4384 }
4385
4386 wr32(E1000_RCTL, rctl);
4387}
4388
4389static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
4390 int vfn)
4391{
4392 struct e1000_hw *hw = &adapter->hw;
4393 u32 vmolr;
4394
4395 if (size > MAX_JUMBO_FRAME_SIZE)
4396 size = MAX_JUMBO_FRAME_SIZE;
4397
4398 vmolr = rd32(E1000_VMOLR(vfn));
4399 vmolr &= ~E1000_VMOLR_RLPML_MASK;
4400 vmolr |= size | E1000_VMOLR_LPE;
4401 wr32(E1000_VMOLR(vfn), vmolr);
4402
4403 return 0;
4404}
4405
4406static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
4407 int vfn, bool enable)
4408{
4409 struct e1000_hw *hw = &adapter->hw;
4410 u32 val, reg;
4411
4412 if (hw->mac.type < e1000_82576)
4413 return;
4414
4415 if (hw->mac.type == e1000_i350)
4416 reg = E1000_DVMOLR(vfn);
4417 else
4418 reg = E1000_VMOLR(vfn);
4419
4420 val = rd32(reg);
4421 if (enable)
4422 val |= E1000_VMOLR_STRVLAN;
4423 else
4424 val &= ~(E1000_VMOLR_STRVLAN);
4425 wr32(reg, val);
4426}
4427
4428static inline void igb_set_vmolr(struct igb_adapter *adapter,
4429 int vfn, bool aupe)
4430{
4431 struct e1000_hw *hw = &adapter->hw;
4432 u32 vmolr;
4433
4434 /* This register exists only on 82576 and newer so if we are older then
4435 * we should exit and do nothing
4436 */
4437 if (hw->mac.type < e1000_82576)
4438 return;
4439
4440 vmolr = rd32(E1000_VMOLR(vfn));
4441 if (aupe)
4442 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
4443 else
4444 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
4445
4446 /* clear all bits that might not be set */
4447 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
4448
4449 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
4450 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
4451 /* for VMDq only allow the VFs and pool 0 to accept broadcast and
4452 * multicast packets
4453 */
4454 if (vfn <= adapter->vfs_allocated_count)
4455 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
4456
4457 wr32(E1000_VMOLR(vfn), vmolr);
4458}
4459
4460/**
4461 * igb_setup_srrctl - configure the split and replication receive control
4462 * registers
4463 * @adapter: Board private structure
4464 * @ring: receive ring to be configured
4465 **/
4466void igb_setup_srrctl(struct igb_adapter *adapter, struct igb_ring *ring)
4467{
4468 struct e1000_hw *hw = &adapter->hw;
4469 int reg_idx = ring->reg_idx;
4470 u32 srrctl = 0;
4471
4472 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4473 if (ring_uses_large_buffer(ring))
4474 srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4475 else
4476 srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4477 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4478 if (hw->mac.type >= e1000_82580)
4479 srrctl |= E1000_SRRCTL_TIMESTAMP;
4480 /* Only set Drop Enable if VFs allocated, or we are supporting multiple
4481 * queues and rx flow control is disabled
4482 */
4483 if (adapter->vfs_allocated_count ||
4484 (!(hw->fc.current_mode & e1000_fc_rx_pause) &&
4485 adapter->num_rx_queues > 1))
4486 srrctl |= E1000_SRRCTL_DROP_EN;
4487
4488 wr32(E1000_SRRCTL(reg_idx), srrctl);
4489}
4490
4491/**
4492 * igb_configure_rx_ring - Configure a receive ring after Reset
4493 * @adapter: board private structure
4494 * @ring: receive ring to be configured
4495 *
4496 * Configure the Rx unit of the MAC after a reset.
4497 **/
4498void igb_configure_rx_ring(struct igb_adapter *adapter,
4499 struct igb_ring *ring)
4500{
4501 struct e1000_hw *hw = &adapter->hw;
4502 union e1000_adv_rx_desc *rx_desc;
4503 u64 rdba = ring->dma;
4504 int reg_idx = ring->reg_idx;
4505 u32 rxdctl = 0;
4506
4507 /* disable the queue */
4508 wr32(E1000_RXDCTL(reg_idx), 0);
4509
4510 /* Set DMA base address registers */
4511 wr32(E1000_RDBAL(reg_idx),
4512 rdba & 0x00000000ffffffffULL);
4513 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
4514 wr32(E1000_RDLEN(reg_idx),
4515 ring->count * sizeof(union e1000_adv_rx_desc));
4516
4517 /* initialize head and tail */
4518 ring->tail = adapter->io_addr + E1000_RDT(reg_idx);
4519 wr32(E1000_RDH(reg_idx), 0);
4520 writel(0, ring->tail);
4521
4522 /* set descriptor configuration */
4523 igb_setup_srrctl(adapter, ring);
4524
4525 /* set filtering for VMDQ pools */
4526 igb_set_vmolr(adapter, reg_idx & 0x7, true);
4527
4528 rxdctl |= IGB_RX_PTHRESH;
4529 rxdctl |= IGB_RX_HTHRESH << 8;
4530 rxdctl |= IGB_RX_WTHRESH << 16;
4531
4532 /* initialize rx_buffer_info */
4533 memset(ring->rx_buffer_info, 0,
4534 sizeof(struct igb_rx_buffer) * ring->count);
4535
4536 /* initialize Rx descriptor 0 */
4537 rx_desc = IGB_RX_DESC(ring, 0);
4538 rx_desc->wb.upper.length = 0;
4539
4540 /* enable receive descriptor fetching */
4541 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4542 wr32(E1000_RXDCTL(reg_idx), rxdctl);
4543}
4544
4545static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
4546 struct igb_ring *rx_ring)
4547{
4548 /* set build_skb and buffer size flags */
4549 clear_ring_build_skb_enabled(rx_ring);
4550 clear_ring_uses_large_buffer(rx_ring);
4551
4552 if (adapter->flags & IGB_FLAG_RX_LEGACY)
4553 return;
4554
4555 set_ring_build_skb_enabled(rx_ring);
4556
4557#if (PAGE_SIZE < 8192)
4558 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
4559 return;
4560
4561 set_ring_uses_large_buffer(rx_ring);
4562#endif
4563}
4564
4565/**
4566 * igb_configure_rx - Configure receive Unit after Reset
4567 * @adapter: board private structure
4568 *
4569 * Configure the Rx unit of the MAC after a reset.
4570 **/
4571static void igb_configure_rx(struct igb_adapter *adapter)
4572{
4573 int i;
4574
4575 /* set the correct pool for the PF default MAC address in entry 0 */
4576 igb_set_default_mac_filter(adapter);
4577
4578 /* Setup the HW Rx Head and Tail Descriptor Pointers and
4579 * the Base and Length of the Rx Descriptor Ring
4580 */
4581 for (i = 0; i < adapter->num_rx_queues; i++) {
4582 struct igb_ring *rx_ring = adapter->rx_ring[i];
4583
4584 igb_set_rx_buffer_len(adapter, rx_ring);
4585 igb_configure_rx_ring(adapter, rx_ring);
4586 }
4587}
4588
4589/**
4590 * igb_free_tx_resources - Free Tx Resources per Queue
4591 * @tx_ring: Tx descriptor ring for a specific queue
4592 *
4593 * Free all transmit software resources
4594 **/
4595void igb_free_tx_resources(struct igb_ring *tx_ring)
4596{
4597 igb_clean_tx_ring(tx_ring);
4598
4599 vfree(tx_ring->tx_buffer_info);
4600 tx_ring->tx_buffer_info = NULL;
4601
4602 /* if not set, then don't free */
4603 if (!tx_ring->desc)
4604 return;
4605
4606 dma_free_coherent(tx_ring->dev, tx_ring->size,
4607 tx_ring->desc, tx_ring->dma);
4608
4609 tx_ring->desc = NULL;
4610}
4611
4612/**
4613 * igb_free_all_tx_resources - Free Tx Resources for All Queues
4614 * @adapter: board private structure
4615 *
4616 * Free all transmit software resources
4617 **/
4618static void igb_free_all_tx_resources(struct igb_adapter *adapter)
4619{
4620 int i;
4621
4622 for (i = 0; i < adapter->num_tx_queues; i++)
4623 if (adapter->tx_ring[i])
4624 igb_free_tx_resources(adapter->tx_ring[i]);
4625}
4626
4627/**
4628 * igb_clean_tx_ring - Free Tx Buffers
4629 * @tx_ring: ring to be cleaned
4630 **/
4631static void igb_clean_tx_ring(struct igb_ring *tx_ring)
4632{
4633 u16 i = tx_ring->next_to_clean;
4634 struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
4635
4636 while (i != tx_ring->next_to_use) {
4637 union e1000_adv_tx_desc *eop_desc, *tx_desc;
4638
4639 /* Free all the Tx ring sk_buffs */
4640 dev_kfree_skb_any(tx_buffer->skb);
4641
4642 /* unmap skb header data */
4643 dma_unmap_single(tx_ring->dev,
4644 dma_unmap_addr(tx_buffer, dma),
4645 dma_unmap_len(tx_buffer, len),
4646 DMA_TO_DEVICE);
4647
4648 /* check for eop_desc to determine the end of the packet */
4649 eop_desc = tx_buffer->next_to_watch;
4650 tx_desc = IGB_TX_DESC(tx_ring, i);
4651
4652 /* unmap remaining buffers */
4653 while (tx_desc != eop_desc) {
4654 tx_buffer++;
4655 tx_desc++;
4656 i++;
4657 if (unlikely(i == tx_ring->count)) {
4658 i = 0;
4659 tx_buffer = tx_ring->tx_buffer_info;
4660 tx_desc = IGB_TX_DESC(tx_ring, 0);
4661 }
4662
4663 /* unmap any remaining paged data */
4664 if (dma_unmap_len(tx_buffer, len))
4665 dma_unmap_page(tx_ring->dev,
4666 dma_unmap_addr(tx_buffer, dma),
4667 dma_unmap_len(tx_buffer, len),
4668 DMA_TO_DEVICE);
4669 }
4670
4671 /* move us one more past the eop_desc for start of next pkt */
4672 tx_buffer++;
4673 i++;
4674 if (unlikely(i == tx_ring->count)) {
4675 i = 0;
4676 tx_buffer = tx_ring->tx_buffer_info;
4677 }
4678 }
4679
4680 /* reset BQL for queue */
4681 netdev_tx_reset_queue(txring_txq(tx_ring));
4682
4683 /* reset next_to_use and next_to_clean */
4684 tx_ring->next_to_use = 0;
4685 tx_ring->next_to_clean = 0;
4686}
4687
4688/**
4689 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
4690 * @adapter: board private structure
4691 **/
4692static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
4693{
4694 int i;
4695
4696 for (i = 0; i < adapter->num_tx_queues; i++)
4697 if (adapter->tx_ring[i])
4698 igb_clean_tx_ring(adapter->tx_ring[i]);
4699}
4700
4701/**
4702 * igb_free_rx_resources - Free Rx Resources
4703 * @rx_ring: ring to clean the resources from
4704 *
4705 * Free all receive software resources
4706 **/
4707void igb_free_rx_resources(struct igb_ring *rx_ring)
4708{
4709 igb_clean_rx_ring(rx_ring);
4710
4711 vfree(rx_ring->rx_buffer_info);
4712 rx_ring->rx_buffer_info = NULL;
4713
4714 /* if not set, then don't free */
4715 if (!rx_ring->desc)
4716 return;
4717
4718 dma_free_coherent(rx_ring->dev, rx_ring->size,
4719 rx_ring->desc, rx_ring->dma);
4720
4721 rx_ring->desc = NULL;
4722}
4723
4724/**
4725 * igb_free_all_rx_resources - Free Rx Resources for All Queues
4726 * @adapter: board private structure
4727 *
4728 * Free all receive software resources
4729 **/
4730static void igb_free_all_rx_resources(struct igb_adapter *adapter)
4731{
4732 int i;
4733
4734 for (i = 0; i < adapter->num_rx_queues; i++)
4735 if (adapter->rx_ring[i])
4736 igb_free_rx_resources(adapter->rx_ring[i]);
4737}
4738
4739/**
4740 * igb_clean_rx_ring - Free Rx Buffers per Queue
4741 * @rx_ring: ring to free buffers from
4742 **/
4743static void igb_clean_rx_ring(struct igb_ring *rx_ring)
4744{
4745 u16 i = rx_ring->next_to_clean;
4746
4747 dev_kfree_skb(rx_ring->skb);
4748 rx_ring->skb = NULL;
4749
4750 /* Free all the Rx ring sk_buffs */
4751 while (i != rx_ring->next_to_alloc) {
4752 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
4753
4754 /* Invalidate cache lines that may have been written to by
4755 * device so that we avoid corrupting memory.
4756 */
4757 dma_sync_single_range_for_cpu(rx_ring->dev,
4758 buffer_info->dma,
4759 buffer_info->page_offset,
4760 igb_rx_bufsz(rx_ring),
4761 DMA_FROM_DEVICE);
4762
4763 /* free resources associated with mapping */
4764 dma_unmap_page_attrs(rx_ring->dev,
4765 buffer_info->dma,
4766 igb_rx_pg_size(rx_ring),
4767 DMA_FROM_DEVICE,
4768 IGB_RX_DMA_ATTR);
4769 __page_frag_cache_drain(buffer_info->page,
4770 buffer_info->pagecnt_bias);
4771
4772 i++;
4773 if (i == rx_ring->count)
4774 i = 0;
4775 }
4776
4777 rx_ring->next_to_alloc = 0;
4778 rx_ring->next_to_clean = 0;
4779 rx_ring->next_to_use = 0;
4780}
4781
4782/**
4783 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
4784 * @adapter: board private structure
4785 **/
4786static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
4787{
4788 int i;
4789
4790 for (i = 0; i < adapter->num_rx_queues; i++)
4791 if (adapter->rx_ring[i])
4792 igb_clean_rx_ring(adapter->rx_ring[i]);
4793}
4794
4795/**
4796 * igb_set_mac - Change the Ethernet Address of the NIC
4797 * @netdev: network interface device structure
4798 * @p: pointer to an address structure
4799 *
4800 * Returns 0 on success, negative on failure
4801 **/
4802static int igb_set_mac(struct net_device *netdev, void *p)
4803{
4804 struct igb_adapter *adapter = netdev_priv(netdev);
4805 struct e1000_hw *hw = &adapter->hw;
4806 struct sockaddr *addr = p;
4807
4808 if (!is_valid_ether_addr(addr->sa_data))
4809 return -EADDRNOTAVAIL;
4810
4811 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
4812 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
4813
4814 /* set the correct pool for the new PF MAC address in entry 0 */
4815 igb_set_default_mac_filter(adapter);
4816
4817 return 0;
4818}
4819
4820/**
4821 * igb_write_mc_addr_list - write multicast addresses to MTA
4822 * @netdev: network interface device structure
4823 *
4824 * Writes multicast address list to the MTA hash table.
4825 * Returns: -ENOMEM on failure
4826 * 0 on no addresses written
4827 * X on writing X addresses to MTA
4828 **/
4829static int igb_write_mc_addr_list(struct net_device *netdev)
4830{
4831 struct igb_adapter *adapter = netdev_priv(netdev);
4832 struct e1000_hw *hw = &adapter->hw;
4833 struct netdev_hw_addr *ha;
4834 u8 *mta_list;
4835 int i;
4836
4837 if (netdev_mc_empty(netdev)) {
4838 /* nothing to program, so clear mc list */
4839 igb_update_mc_addr_list(hw, NULL, 0);
4840 igb_restore_vf_multicasts(adapter);
4841 return 0;
4842 }
4843
4844 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
4845 if (!mta_list)
4846 return -ENOMEM;
4847
4848 /* The shared function expects a packed array of only addresses. */
4849 i = 0;
4850 netdev_for_each_mc_addr(ha, netdev)
4851 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
4852
4853 igb_update_mc_addr_list(hw, mta_list, i);
4854 kfree(mta_list);
4855
4856 return netdev_mc_count(netdev);
4857}
4858
4859static int igb_vlan_promisc_enable(struct igb_adapter *adapter)
4860{
4861 struct e1000_hw *hw = &adapter->hw;
4862 u32 i, pf_id;
4863
4864 switch (hw->mac.type) {
4865 case e1000_i210:
4866 case e1000_i211:
4867 case e1000_i350:
4868 /* VLAN filtering needed for VLAN prio filter */
4869 if (adapter->netdev->features & NETIF_F_NTUPLE)
4870 break;
4871 fallthrough;
4872 case e1000_82576:
4873 case e1000_82580:
4874 case e1000_i354:
4875 /* VLAN filtering needed for pool filtering */
4876 if (adapter->vfs_allocated_count)
4877 break;
4878 fallthrough;
4879 default:
4880 return 1;
4881 }
4882
4883 /* We are already in VLAN promisc, nothing to do */
4884 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
4885 return 0;
4886
4887 if (!adapter->vfs_allocated_count)
4888 goto set_vfta;
4889
4890 /* Add PF to all active pools */
4891 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
4892
4893 for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
4894 u32 vlvf = rd32(E1000_VLVF(i));
4895
4896 vlvf |= BIT(pf_id);
4897 wr32(E1000_VLVF(i), vlvf);
4898 }
4899
4900set_vfta:
4901 /* Set all bits in the VLAN filter table array */
4902 for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;)
4903 hw->mac.ops.write_vfta(hw, i, ~0U);
4904
4905 /* Set flag so we don't redo unnecessary work */
4906 adapter->flags |= IGB_FLAG_VLAN_PROMISC;
4907
4908 return 0;
4909}
4910
4911#define VFTA_BLOCK_SIZE 8
4912static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset)
4913{
4914 struct e1000_hw *hw = &adapter->hw;
4915 u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
4916 u32 vid_start = vfta_offset * 32;
4917 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
4918 u32 i, vid, word, bits, pf_id;
4919
4920 /* guarantee that we don't scrub out management VLAN */
4921 vid = adapter->mng_vlan_id;
4922 if (vid >= vid_start && vid < vid_end)
4923 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
4924
4925 if (!adapter->vfs_allocated_count)
4926 goto set_vfta;
4927
4928 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
4929
4930 for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
4931 u32 vlvf = rd32(E1000_VLVF(i));
4932
4933 /* pull VLAN ID from VLVF */
4934 vid = vlvf & VLAN_VID_MASK;
4935
4936 /* only concern ourselves with a certain range */
4937 if (vid < vid_start || vid >= vid_end)
4938 continue;
4939
4940 if (vlvf & E1000_VLVF_VLANID_ENABLE) {
4941 /* record VLAN ID in VFTA */
4942 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
4943
4944 /* if PF is part of this then continue */
4945 if (test_bit(vid, adapter->active_vlans))
4946 continue;
4947 }
4948
4949 /* remove PF from the pool */
4950 bits = ~BIT(pf_id);
4951 bits &= rd32(E1000_VLVF(i));
4952 wr32(E1000_VLVF(i), bits);
4953 }
4954
4955set_vfta:
4956 /* extract values from active_vlans and write back to VFTA */
4957 for (i = VFTA_BLOCK_SIZE; i--;) {
4958 vid = (vfta_offset + i) * 32;
4959 word = vid / BITS_PER_LONG;
4960 bits = vid % BITS_PER_LONG;
4961
4962 vfta[i] |= adapter->active_vlans[word] >> bits;
4963
4964 hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]);
4965 }
4966}
4967
4968static void igb_vlan_promisc_disable(struct igb_adapter *adapter)
4969{
4970 u32 i;
4971
4972 /* We are not in VLAN promisc, nothing to do */
4973 if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC))
4974 return;
4975
4976 /* Set flag so we don't redo unnecessary work */
4977 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
4978
4979 for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE)
4980 igb_scrub_vfta(adapter, i);
4981}
4982
4983/**
4984 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
4985 * @netdev: network interface device structure
4986 *
4987 * The set_rx_mode entry point is called whenever the unicast or multicast
4988 * address lists or the network interface flags are updated. This routine is
4989 * responsible for configuring the hardware for proper unicast, multicast,
4990 * promiscuous mode, and all-multi behavior.
4991 **/
4992static void igb_set_rx_mode(struct net_device *netdev)
4993{
4994 struct igb_adapter *adapter = netdev_priv(netdev);
4995 struct e1000_hw *hw = &adapter->hw;
4996 unsigned int vfn = adapter->vfs_allocated_count;
4997 u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
4998 int count;
4999
5000 /* Check for Promiscuous and All Multicast modes */
5001 if (netdev->flags & IFF_PROMISC) {
5002 rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE;
5003 vmolr |= E1000_VMOLR_MPME;
5004
5005 /* enable use of UTA filter to force packets to default pool */
5006 if (hw->mac.type == e1000_82576)
5007 vmolr |= E1000_VMOLR_ROPE;
5008 } else {
5009 if (netdev->flags & IFF_ALLMULTI) {
5010 rctl |= E1000_RCTL_MPE;
5011 vmolr |= E1000_VMOLR_MPME;
5012 } else {
5013 /* Write addresses to the MTA, if the attempt fails
5014 * then we should just turn on promiscuous mode so
5015 * that we can at least receive multicast traffic
5016 */
5017 count = igb_write_mc_addr_list(netdev);
5018 if (count < 0) {
5019 rctl |= E1000_RCTL_MPE;
5020 vmolr |= E1000_VMOLR_MPME;
5021 } else if (count) {
5022 vmolr |= E1000_VMOLR_ROMPE;
5023 }
5024 }
5025 }
5026
5027 /* Write addresses to available RAR registers, if there is not
5028 * sufficient space to store all the addresses then enable
5029 * unicast promiscuous mode
5030 */
5031 if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) {
5032 rctl |= E1000_RCTL_UPE;
5033 vmolr |= E1000_VMOLR_ROPE;
5034 }
5035
5036 /* enable VLAN filtering by default */
5037 rctl |= E1000_RCTL_VFE;
5038
5039 /* disable VLAN filtering for modes that require it */
5040 if ((netdev->flags & IFF_PROMISC) ||
5041 (netdev->features & NETIF_F_RXALL)) {
5042 /* if we fail to set all rules then just clear VFE */
5043 if (igb_vlan_promisc_enable(adapter))
5044 rctl &= ~E1000_RCTL_VFE;
5045 } else {
5046 igb_vlan_promisc_disable(adapter);
5047 }
5048
5049 /* update state of unicast, multicast, and VLAN filtering modes */
5050 rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE |
5051 E1000_RCTL_VFE);
5052 wr32(E1000_RCTL, rctl);
5053
5054#if (PAGE_SIZE < 8192)
5055 if (!adapter->vfs_allocated_count) {
5056 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5057 rlpml = IGB_MAX_FRAME_BUILD_SKB;
5058 }
5059#endif
5060 wr32(E1000_RLPML, rlpml);
5061
5062 /* In order to support SR-IOV and eventually VMDq it is necessary to set
5063 * the VMOLR to enable the appropriate modes. Without this workaround
5064 * we will have issues with VLAN tag stripping not being done for frames
5065 * that are only arriving because we are the default pool
5066 */
5067 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
5068 return;
5069
5070 /* set UTA to appropriate mode */
5071 igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE));
5072
5073 vmolr |= rd32(E1000_VMOLR(vfn)) &
5074 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
5075
5076 /* enable Rx jumbo frames, restrict as needed to support build_skb */
5077 vmolr &= ~E1000_VMOLR_RLPML_MASK;
5078#if (PAGE_SIZE < 8192)
5079 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5080 vmolr |= IGB_MAX_FRAME_BUILD_SKB;
5081 else
5082#endif
5083 vmolr |= MAX_JUMBO_FRAME_SIZE;
5084 vmolr |= E1000_VMOLR_LPE;
5085
5086 wr32(E1000_VMOLR(vfn), vmolr);
5087
5088 igb_restore_vf_multicasts(adapter);
5089}
5090
5091static void igb_check_wvbr(struct igb_adapter *adapter)
5092{
5093 struct e1000_hw *hw = &adapter->hw;
5094 u32 wvbr = 0;
5095
5096 switch (hw->mac.type) {
5097 case e1000_82576:
5098 case e1000_i350:
5099 wvbr = rd32(E1000_WVBR);
5100 if (!wvbr)
5101 return;
5102 break;
5103 default:
5104 break;
5105 }
5106
5107 adapter->wvbr |= wvbr;
5108}
5109
5110#define IGB_STAGGERED_QUEUE_OFFSET 8
5111
5112static void igb_spoof_check(struct igb_adapter *adapter)
5113{
5114 int j;
5115
5116 if (!adapter->wvbr)
5117 return;
5118
5119 for (j = 0; j < adapter->vfs_allocated_count; j++) {
5120 if (adapter->wvbr & BIT(j) ||
5121 adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) {
5122 dev_warn(&adapter->pdev->dev,
5123 "Spoof event(s) detected on VF %d\n", j);
5124 adapter->wvbr &=
5125 ~(BIT(j) |
5126 BIT(j + IGB_STAGGERED_QUEUE_OFFSET));
5127 }
5128 }
5129}
5130
5131/* Need to wait a few seconds after link up to get diagnostic information from
5132 * the phy
5133 */
5134static void igb_update_phy_info(struct timer_list *t)
5135{
5136 struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5137 igb_get_phy_info(&adapter->hw);
5138}
5139
5140/**
5141 * igb_has_link - check shared code for link and determine up/down
5142 * @adapter: pointer to driver private info
5143 **/
5144bool igb_has_link(struct igb_adapter *adapter)
5145{
5146 struct e1000_hw *hw = &adapter->hw;
5147 bool link_active = false;
5148
5149 /* get_link_status is set on LSC (link status) interrupt or
5150 * rx sequence error interrupt. get_link_status will stay
5151 * false until the e1000_check_for_link establishes link
5152 * for copper adapters ONLY
5153 */
5154 switch (hw->phy.media_type) {
5155 case e1000_media_type_copper:
5156 if (!hw->mac.get_link_status)
5157 return true;
5158 fallthrough;
5159 case e1000_media_type_internal_serdes:
5160 hw->mac.ops.check_for_link(hw);
5161 link_active = !hw->mac.get_link_status;
5162 break;
5163 default:
5164 case e1000_media_type_unknown:
5165 break;
5166 }
5167
5168 if (((hw->mac.type == e1000_i210) ||
5169 (hw->mac.type == e1000_i211)) &&
5170 (hw->phy.id == I210_I_PHY_ID)) {
5171 if (!netif_carrier_ok(adapter->netdev)) {
5172 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5173 } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
5174 adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
5175 adapter->link_check_timeout = jiffies;
5176 }
5177 }
5178
5179 return link_active;
5180}
5181
5182static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
5183{
5184 bool ret = false;
5185 u32 ctrl_ext, thstat;
5186
5187 /* check for thermal sensor event on i350 copper only */
5188 if (hw->mac.type == e1000_i350) {
5189 thstat = rd32(E1000_THSTAT);
5190 ctrl_ext = rd32(E1000_CTRL_EXT);
5191
5192 if ((hw->phy.media_type == e1000_media_type_copper) &&
5193 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
5194 ret = !!(thstat & event);
5195 }
5196
5197 return ret;
5198}
5199
5200/**
5201 * igb_check_lvmmc - check for malformed packets received
5202 * and indicated in LVMMC register
5203 * @adapter: pointer to adapter
5204 **/
5205static void igb_check_lvmmc(struct igb_adapter *adapter)
5206{
5207 struct e1000_hw *hw = &adapter->hw;
5208 u32 lvmmc;
5209
5210 lvmmc = rd32(E1000_LVMMC);
5211 if (lvmmc) {
5212 if (unlikely(net_ratelimit())) {
5213 netdev_warn(adapter->netdev,
5214 "malformed Tx packet detected and dropped, LVMMC:0x%08x\n",
5215 lvmmc);
5216 }
5217 }
5218}
5219
5220/**
5221 * igb_watchdog - Timer Call-back
5222 * @data: pointer to adapter cast into an unsigned long
5223 **/
5224static void igb_watchdog(struct timer_list *t)
5225{
5226 struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5227 /* Do the rest outside of interrupt context */
5228 schedule_work(&adapter->watchdog_task);
5229}
5230
5231static void igb_watchdog_task(struct work_struct *work)
5232{
5233 struct igb_adapter *adapter = container_of(work,
5234 struct igb_adapter,
5235 watchdog_task);
5236 struct e1000_hw *hw = &adapter->hw;
5237 struct e1000_phy_info *phy = &hw->phy;
5238 struct net_device *netdev = adapter->netdev;
5239 u32 link;
5240 int i;
5241 u32 connsw;
5242 u16 phy_data, retry_count = 20;
5243
5244 link = igb_has_link(adapter);
5245
5246 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
5247 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5248 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5249 else
5250 link = false;
5251 }
5252
5253 /* Force link down if we have fiber to swap to */
5254 if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5255 if (hw->phy.media_type == e1000_media_type_copper) {
5256 connsw = rd32(E1000_CONNSW);
5257 if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
5258 link = 0;
5259 }
5260 }
5261 if (link) {
5262 /* Perform a reset if the media type changed. */
5263 if (hw->dev_spec._82575.media_changed) {
5264 hw->dev_spec._82575.media_changed = false;
5265 adapter->flags |= IGB_FLAG_MEDIA_RESET;
5266 igb_reset(adapter);
5267 }
5268 /* Cancel scheduled suspend requests. */
5269 pm_runtime_resume(netdev->dev.parent);
5270
5271 if (!netif_carrier_ok(netdev)) {
5272 u32 ctrl;
5273
5274 hw->mac.ops.get_speed_and_duplex(hw,
5275 &adapter->link_speed,
5276 &adapter->link_duplex);
5277
5278 ctrl = rd32(E1000_CTRL);
5279 /* Links status message must follow this format */
5280 netdev_info(netdev,
5281 "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5282 netdev->name,
5283 adapter->link_speed,
5284 adapter->link_duplex == FULL_DUPLEX ?
5285 "Full" : "Half",
5286 (ctrl & E1000_CTRL_TFCE) &&
5287 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
5288 (ctrl & E1000_CTRL_RFCE) ? "RX" :
5289 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
5290
5291 /* disable EEE if enabled */
5292 if ((adapter->flags & IGB_FLAG_EEE) &&
5293 (adapter->link_duplex == HALF_DUPLEX)) {
5294 dev_info(&adapter->pdev->dev,
5295 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
5296 adapter->hw.dev_spec._82575.eee_disable = true;
5297 adapter->flags &= ~IGB_FLAG_EEE;
5298 }
5299
5300 /* check if SmartSpeed worked */
5301 igb_check_downshift(hw);
5302 if (phy->speed_downgraded)
5303 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5304
5305 /* check for thermal sensor event */
5306 if (igb_thermal_sensor_event(hw,
5307 E1000_THSTAT_LINK_THROTTLE))
5308 netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n");
5309
5310 /* adjust timeout factor according to speed/duplex */
5311 adapter->tx_timeout_factor = 1;
5312 switch (adapter->link_speed) {
5313 case SPEED_10:
5314 adapter->tx_timeout_factor = 14;
5315 break;
5316 case SPEED_100:
5317 /* maybe add some timeout factor ? */
5318 break;
5319 }
5320
5321 if (adapter->link_speed != SPEED_1000)
5322 goto no_wait;
5323
5324 /* wait for Remote receiver status OK */
5325retry_read_status:
5326 if (!igb_read_phy_reg(hw, PHY_1000T_STATUS,
5327 &phy_data)) {
5328 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5329 retry_count) {
5330 msleep(100);
5331 retry_count--;
5332 goto retry_read_status;
5333 } else if (!retry_count) {
5334 dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
5335 }
5336 } else {
5337 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
5338 }
5339no_wait:
5340 netif_carrier_on(netdev);
5341
5342 igb_ping_all_vfs(adapter);
5343 igb_check_vf_rate_limit(adapter);
5344
5345 /* link state has changed, schedule phy info update */
5346 if (!test_bit(__IGB_DOWN, &adapter->state))
5347 mod_timer(&adapter->phy_info_timer,
5348 round_jiffies(jiffies + 2 * HZ));
5349 }
5350 } else {
5351 if (netif_carrier_ok(netdev)) {
5352 adapter->link_speed = 0;
5353 adapter->link_duplex = 0;
5354
5355 /* check for thermal sensor event */
5356 if (igb_thermal_sensor_event(hw,
5357 E1000_THSTAT_PWR_DOWN)) {
5358 netdev_err(netdev, "The network adapter was stopped because it overheated\n");
5359 }
5360
5361 /* Links status message must follow this format */
5362 netdev_info(netdev, "igb: %s NIC Link is Down\n",
5363 netdev->name);
5364 netif_carrier_off(netdev);
5365
5366 igb_ping_all_vfs(adapter);
5367
5368 /* link state has changed, schedule phy info update */
5369 if (!test_bit(__IGB_DOWN, &adapter->state))
5370 mod_timer(&adapter->phy_info_timer,
5371 round_jiffies(jiffies + 2 * HZ));
5372
5373 /* link is down, time to check for alternate media */
5374 if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5375 igb_check_swap_media(adapter);
5376 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5377 schedule_work(&adapter->reset_task);
5378 /* return immediately */
5379 return;
5380 }
5381 }
5382 pm_schedule_suspend(netdev->dev.parent,
5383 MSEC_PER_SEC * 5);
5384
5385 /* also check for alternate media here */
5386 } else if (!netif_carrier_ok(netdev) &&
5387 (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
5388 igb_check_swap_media(adapter);
5389 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5390 schedule_work(&adapter->reset_task);
5391 /* return immediately */
5392 return;
5393 }
5394 }
5395 }
5396
5397 spin_lock(&adapter->stats64_lock);
5398 igb_update_stats(adapter);
5399 spin_unlock(&adapter->stats64_lock);
5400
5401 for (i = 0; i < adapter->num_tx_queues; i++) {
5402 struct igb_ring *tx_ring = adapter->tx_ring[i];
5403 if (!netif_carrier_ok(netdev)) {
5404 /* We've lost link, so the controller stops DMA,
5405 * but we've got queued Tx work that's never going
5406 * to get done, so reset controller to flush Tx.
5407 * (Do the reset outside of interrupt context).
5408 */
5409 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
5410 adapter->tx_timeout_count++;
5411 schedule_work(&adapter->reset_task);
5412 /* return immediately since reset is imminent */
5413 return;
5414 }
5415 }
5416
5417 /* Force detection of hung controller every watchdog period */
5418 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5419 }
5420
5421 /* Cause software interrupt to ensure Rx ring is cleaned */
5422 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
5423 u32 eics = 0;
5424
5425 for (i = 0; i < adapter->num_q_vectors; i++)
5426 eics |= adapter->q_vector[i]->eims_value;
5427 wr32(E1000_EICS, eics);
5428 } else {
5429 wr32(E1000_ICS, E1000_ICS_RXDMT0);
5430 }
5431
5432 igb_spoof_check(adapter);
5433 igb_ptp_rx_hang(adapter);
5434 igb_ptp_tx_hang(adapter);
5435
5436 /* Check LVMMC register on i350/i354 only */
5437 if ((adapter->hw.mac.type == e1000_i350) ||
5438 (adapter->hw.mac.type == e1000_i354))
5439 igb_check_lvmmc(adapter);
5440
5441 /* Reset the timer */
5442 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5443 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
5444 mod_timer(&adapter->watchdog_timer,
5445 round_jiffies(jiffies + HZ));
5446 else
5447 mod_timer(&adapter->watchdog_timer,
5448 round_jiffies(jiffies + 2 * HZ));
5449 }
5450}
5451
5452enum latency_range {
5453 lowest_latency = 0,
5454 low_latency = 1,
5455 bulk_latency = 2,
5456 latency_invalid = 255
5457};
5458
5459/**
5460 * igb_update_ring_itr - update the dynamic ITR value based on packet size
5461 * @q_vector: pointer to q_vector
5462 *
5463 * Stores a new ITR value based on strictly on packet size. This
5464 * algorithm is less sophisticated than that used in igb_update_itr,
5465 * due to the difficulty of synchronizing statistics across multiple
5466 * receive rings. The divisors and thresholds used by this function
5467 * were determined based on theoretical maximum wire speed and testing
5468 * data, in order to minimize response time while increasing bulk
5469 * throughput.
5470 * This functionality is controlled by ethtool's coalescing settings.
5471 * NOTE: This function is called only when operating in a multiqueue
5472 * receive environment.
5473 **/
5474static void igb_update_ring_itr(struct igb_q_vector *q_vector)
5475{
5476 int new_val = q_vector->itr_val;
5477 int avg_wire_size = 0;
5478 struct igb_adapter *adapter = q_vector->adapter;
5479 unsigned int packets;
5480
5481 /* For non-gigabit speeds, just fix the interrupt rate at 4000
5482 * ints/sec - ITR timer value of 120 ticks.
5483 */
5484 if (adapter->link_speed != SPEED_1000) {
5485 new_val = IGB_4K_ITR;
5486 goto set_itr_val;
5487 }
5488
5489 packets = q_vector->rx.total_packets;
5490 if (packets)
5491 avg_wire_size = q_vector->rx.total_bytes / packets;
5492
5493 packets = q_vector->tx.total_packets;
5494 if (packets)
5495 avg_wire_size = max_t(u32, avg_wire_size,
5496 q_vector->tx.total_bytes / packets);
5497
5498 /* if avg_wire_size isn't set no work was done */
5499 if (!avg_wire_size)
5500 goto clear_counts;
5501
5502 /* Add 24 bytes to size to account for CRC, preamble, and gap */
5503 avg_wire_size += 24;
5504
5505 /* Don't starve jumbo frames */
5506 avg_wire_size = min(avg_wire_size, 3000);
5507
5508 /* Give a little boost to mid-size frames */
5509 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
5510 new_val = avg_wire_size / 3;
5511 else
5512 new_val = avg_wire_size / 2;
5513
5514 /* conservative mode (itr 3) eliminates the lowest_latency setting */
5515 if (new_val < IGB_20K_ITR &&
5516 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5517 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5518 new_val = IGB_20K_ITR;
5519
5520set_itr_val:
5521 if (new_val != q_vector->itr_val) {
5522 q_vector->itr_val = new_val;
5523 q_vector->set_itr = 1;
5524 }
5525clear_counts:
5526 q_vector->rx.total_bytes = 0;
5527 q_vector->rx.total_packets = 0;
5528 q_vector->tx.total_bytes = 0;
5529 q_vector->tx.total_packets = 0;
5530}
5531
5532/**
5533 * igb_update_itr - update the dynamic ITR value based on statistics
5534 * @q_vector: pointer to q_vector
5535 * @ring_container: ring info to update the itr for
5536 *
5537 * Stores a new ITR value based on packets and byte
5538 * counts during the last interrupt. The advantage of per interrupt
5539 * computation is faster updates and more accurate ITR for the current
5540 * traffic pattern. Constants in this function were computed
5541 * based on theoretical maximum wire speed and thresholds were set based
5542 * on testing data as well as attempting to minimize response time
5543 * while increasing bulk throughput.
5544 * This functionality is controlled by ethtool's coalescing settings.
5545 * NOTE: These calculations are only valid when operating in a single-
5546 * queue environment.
5547 **/
5548static void igb_update_itr(struct igb_q_vector *q_vector,
5549 struct igb_ring_container *ring_container)
5550{
5551 unsigned int packets = ring_container->total_packets;
5552 unsigned int bytes = ring_container->total_bytes;
5553 u8 itrval = ring_container->itr;
5554
5555 /* no packets, exit with status unchanged */
5556 if (packets == 0)
5557 return;
5558
5559 switch (itrval) {
5560 case lowest_latency:
5561 /* handle TSO and jumbo frames */
5562 if (bytes/packets > 8000)
5563 itrval = bulk_latency;
5564 else if ((packets < 5) && (bytes > 512))
5565 itrval = low_latency;
5566 break;
5567 case low_latency: /* 50 usec aka 20000 ints/s */
5568 if (bytes > 10000) {
5569 /* this if handles the TSO accounting */
5570 if (bytes/packets > 8000)
5571 itrval = bulk_latency;
5572 else if ((packets < 10) || ((bytes/packets) > 1200))
5573 itrval = bulk_latency;
5574 else if ((packets > 35))
5575 itrval = lowest_latency;
5576 } else if (bytes/packets > 2000) {
5577 itrval = bulk_latency;
5578 } else if (packets <= 2 && bytes < 512) {
5579 itrval = lowest_latency;
5580 }
5581 break;
5582 case bulk_latency: /* 250 usec aka 4000 ints/s */
5583 if (bytes > 25000) {
5584 if (packets > 35)
5585 itrval = low_latency;
5586 } else if (bytes < 1500) {
5587 itrval = low_latency;
5588 }
5589 break;
5590 }
5591
5592 /* clear work counters since we have the values we need */
5593 ring_container->total_bytes = 0;
5594 ring_container->total_packets = 0;
5595
5596 /* write updated itr to ring container */
5597 ring_container->itr = itrval;
5598}
5599
5600static void igb_set_itr(struct igb_q_vector *q_vector)
5601{
5602 struct igb_adapter *adapter = q_vector->adapter;
5603 u32 new_itr = q_vector->itr_val;
5604 u8 current_itr = 0;
5605
5606 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
5607 if (adapter->link_speed != SPEED_1000) {
5608 current_itr = 0;
5609 new_itr = IGB_4K_ITR;
5610 goto set_itr_now;
5611 }
5612
5613 igb_update_itr(q_vector, &q_vector->tx);
5614 igb_update_itr(q_vector, &q_vector->rx);
5615
5616 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
5617
5618 /* conservative mode (itr 3) eliminates the lowest_latency setting */
5619 if (current_itr == lowest_latency &&
5620 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5621 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5622 current_itr = low_latency;
5623
5624 switch (current_itr) {
5625 /* counts and packets in update_itr are dependent on these numbers */
5626 case lowest_latency:
5627 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
5628 break;
5629 case low_latency:
5630 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
5631 break;
5632 case bulk_latency:
5633 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
5634 break;
5635 default:
5636 break;
5637 }
5638
5639set_itr_now:
5640 if (new_itr != q_vector->itr_val) {
5641 /* this attempts to bias the interrupt rate towards Bulk
5642 * by adding intermediate steps when interrupt rate is
5643 * increasing
5644 */
5645 new_itr = new_itr > q_vector->itr_val ?
5646 max((new_itr * q_vector->itr_val) /
5647 (new_itr + (q_vector->itr_val >> 2)),
5648 new_itr) : new_itr;
5649 /* Don't write the value here; it resets the adapter's
5650 * internal timer, and causes us to delay far longer than
5651 * we should between interrupts. Instead, we write the ITR
5652 * value at the beginning of the next interrupt so the timing
5653 * ends up being correct.
5654 */
5655 q_vector->itr_val = new_itr;
5656 q_vector->set_itr = 1;
5657 }
5658}
5659
5660static void igb_tx_ctxtdesc(struct igb_ring *tx_ring,
5661 struct igb_tx_buffer *first,
5662 u32 vlan_macip_lens, u32 type_tucmd,
5663 u32 mss_l4len_idx)
5664{
5665 struct e1000_adv_tx_context_desc *context_desc;
5666 u16 i = tx_ring->next_to_use;
5667 struct timespec64 ts;
5668
5669 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
5670
5671 i++;
5672 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
5673
5674 /* set bits to identify this as an advanced context descriptor */
5675 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
5676
5677 /* For 82575, context index must be unique per ring. */
5678 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5679 mss_l4len_idx |= tx_ring->reg_idx << 4;
5680
5681 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
5682 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
5683 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
5684
5685 /* We assume there is always a valid tx time available. Invalid times
5686 * should have been handled by the upper layers.
5687 */
5688 if (tx_ring->launchtime_enable) {
5689 ts = ktime_to_timespec64(first->skb->tstamp);
5690 first->skb->tstamp = ktime_set(0, 0);
5691 context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32);
5692 } else {
5693 context_desc->seqnum_seed = 0;
5694 }
5695}
5696
5697static int igb_tso(struct igb_ring *tx_ring,
5698 struct igb_tx_buffer *first,
5699 u8 *hdr_len)
5700{
5701 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
5702 struct sk_buff *skb = first->skb;
5703 union {
5704 struct iphdr *v4;
5705 struct ipv6hdr *v6;
5706 unsigned char *hdr;
5707 } ip;
5708 union {
5709 struct tcphdr *tcp;
5710 struct udphdr *udp;
5711 unsigned char *hdr;
5712 } l4;
5713 u32 paylen, l4_offset;
5714 int err;
5715
5716 if (skb->ip_summed != CHECKSUM_PARTIAL)
5717 return 0;
5718
5719 if (!skb_is_gso(skb))
5720 return 0;
5721
5722 err = skb_cow_head(skb, 0);
5723 if (err < 0)
5724 return err;
5725
5726 ip.hdr = skb_network_header(skb);
5727 l4.hdr = skb_checksum_start(skb);
5728
5729 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
5730 type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?
5731 E1000_ADVTXD_TUCMD_L4T_UDP : E1000_ADVTXD_TUCMD_L4T_TCP;
5732
5733 /* initialize outer IP header fields */
5734 if (ip.v4->version == 4) {
5735 unsigned char *csum_start = skb_checksum_start(skb);
5736 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
5737
5738 /* IP header will have to cancel out any data that
5739 * is not a part of the outer IP header
5740 */
5741 ip.v4->check = csum_fold(csum_partial(trans_start,
5742 csum_start - trans_start,
5743 0));
5744 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
5745
5746 ip.v4->tot_len = 0;
5747 first->tx_flags |= IGB_TX_FLAGS_TSO |
5748 IGB_TX_FLAGS_CSUM |
5749 IGB_TX_FLAGS_IPV4;
5750 } else {
5751 ip.v6->payload_len = 0;
5752 first->tx_flags |= IGB_TX_FLAGS_TSO |
5753 IGB_TX_FLAGS_CSUM;
5754 }
5755
5756 /* determine offset of inner transport header */
5757 l4_offset = l4.hdr - skb->data;
5758
5759 /* remove payload length from inner checksum */
5760 paylen = skb->len - l4_offset;
5761 if (type_tucmd & E1000_ADVTXD_TUCMD_L4T_TCP) {
5762 /* compute length of segmentation header */
5763 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
5764 csum_replace_by_diff(&l4.tcp->check,
5765 (__force __wsum)htonl(paylen));
5766 } else {
5767 /* compute length of segmentation header */
5768 *hdr_len = sizeof(*l4.udp) + l4_offset;
5769 csum_replace_by_diff(&l4.udp->check,
5770 (__force __wsum)htonl(paylen));
5771 }
5772
5773 /* update gso size and bytecount with header size */
5774 first->gso_segs = skb_shinfo(skb)->gso_segs;
5775 first->bytecount += (first->gso_segs - 1) * *hdr_len;
5776
5777 /* MSS L4LEN IDX */
5778 mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
5779 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
5780
5781 /* VLAN MACLEN IPLEN */
5782 vlan_macip_lens = l4.hdr - ip.hdr;
5783 vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
5784 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
5785
5786 igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens,
5787 type_tucmd, mss_l4len_idx);
5788
5789 return 1;
5790}
5791
5792static inline bool igb_ipv6_csum_is_sctp(struct sk_buff *skb)
5793{
5794 unsigned int offset = 0;
5795
5796 ipv6_find_hdr(skb, &offset, IPPROTO_SCTP, NULL, NULL);
5797
5798 return offset == skb_checksum_start_offset(skb);
5799}
5800
5801static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
5802{
5803 struct sk_buff *skb = first->skb;
5804 u32 vlan_macip_lens = 0;
5805 u32 type_tucmd = 0;
5806
5807 if (skb->ip_summed != CHECKSUM_PARTIAL) {
5808csum_failed:
5809 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN) &&
5810 !tx_ring->launchtime_enable)
5811 return;
5812 goto no_csum;
5813 }
5814
5815 switch (skb->csum_offset) {
5816 case offsetof(struct tcphdr, check):
5817 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
5818 fallthrough;
5819 case offsetof(struct udphdr, check):
5820 break;
5821 case offsetof(struct sctphdr, checksum):
5822 /* validate that this is actually an SCTP request */
5823 if (((first->protocol == htons(ETH_P_IP)) &&
5824 (ip_hdr(skb)->protocol == IPPROTO_SCTP)) ||
5825 ((first->protocol == htons(ETH_P_IPV6)) &&
5826 igb_ipv6_csum_is_sctp(skb))) {
5827 type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
5828 break;
5829 }
5830 fallthrough;
5831 default:
5832 skb_checksum_help(skb);
5833 goto csum_failed;
5834 }
5835
5836 /* update TX checksum flag */
5837 first->tx_flags |= IGB_TX_FLAGS_CSUM;
5838 vlan_macip_lens = skb_checksum_start_offset(skb) -
5839 skb_network_offset(skb);
5840no_csum:
5841 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
5842 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
5843
5844 igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
5845}
5846
5847#define IGB_SET_FLAG(_input, _flag, _result) \
5848 ((_flag <= _result) ? \
5849 ((u32)(_input & _flag) * (_result / _flag)) : \
5850 ((u32)(_input & _flag) / (_flag / _result)))
5851
5852static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
5853{
5854 /* set type for advanced descriptor with frame checksum insertion */
5855 u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
5856 E1000_ADVTXD_DCMD_DEXT |
5857 E1000_ADVTXD_DCMD_IFCS;
5858
5859 /* set HW vlan bit if vlan is present */
5860 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
5861 (E1000_ADVTXD_DCMD_VLE));
5862
5863 /* set segmentation bits for TSO */
5864 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
5865 (E1000_ADVTXD_DCMD_TSE));
5866
5867 /* set timestamp bit if present */
5868 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
5869 (E1000_ADVTXD_MAC_TSTAMP));
5870
5871 /* insert frame checksum */
5872 cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
5873
5874 return cmd_type;
5875}
5876
5877static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
5878 union e1000_adv_tx_desc *tx_desc,
5879 u32 tx_flags, unsigned int paylen)
5880{
5881 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
5882
5883 /* 82575 requires a unique index per ring */
5884 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5885 olinfo_status |= tx_ring->reg_idx << 4;
5886
5887 /* insert L4 checksum */
5888 olinfo_status |= IGB_SET_FLAG(tx_flags,
5889 IGB_TX_FLAGS_CSUM,
5890 (E1000_TXD_POPTS_TXSM << 8));
5891
5892 /* insert IPv4 checksum */
5893 olinfo_status |= IGB_SET_FLAG(tx_flags,
5894 IGB_TX_FLAGS_IPV4,
5895 (E1000_TXD_POPTS_IXSM << 8));
5896
5897 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
5898}
5899
5900static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
5901{
5902 struct net_device *netdev = tx_ring->netdev;
5903
5904 netif_stop_subqueue(netdev, tx_ring->queue_index);
5905
5906 /* Herbert's original patch had:
5907 * smp_mb__after_netif_stop_queue();
5908 * but since that doesn't exist yet, just open code it.
5909 */
5910 smp_mb();
5911
5912 /* We need to check again in a case another CPU has just
5913 * made room available.
5914 */
5915 if (igb_desc_unused(tx_ring) < size)
5916 return -EBUSY;
5917
5918 /* A reprieve! */
5919 netif_wake_subqueue(netdev, tx_ring->queue_index);
5920
5921 u64_stats_update_begin(&tx_ring->tx_syncp2);
5922 tx_ring->tx_stats.restart_queue2++;
5923 u64_stats_update_end(&tx_ring->tx_syncp2);
5924
5925 return 0;
5926}
5927
5928static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
5929{
5930 if (igb_desc_unused(tx_ring) >= size)
5931 return 0;
5932 return __igb_maybe_stop_tx(tx_ring, size);
5933}
5934
5935static int igb_tx_map(struct igb_ring *tx_ring,
5936 struct igb_tx_buffer *first,
5937 const u8 hdr_len)
5938{
5939 struct sk_buff *skb = first->skb;
5940 struct igb_tx_buffer *tx_buffer;
5941 union e1000_adv_tx_desc *tx_desc;
5942 skb_frag_t *frag;
5943 dma_addr_t dma;
5944 unsigned int data_len, size;
5945 u32 tx_flags = first->tx_flags;
5946 u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
5947 u16 i = tx_ring->next_to_use;
5948
5949 tx_desc = IGB_TX_DESC(tx_ring, i);
5950
5951 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
5952
5953 size = skb_headlen(skb);
5954 data_len = skb->data_len;
5955
5956 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
5957
5958 tx_buffer = first;
5959
5960 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
5961 if (dma_mapping_error(tx_ring->dev, dma))
5962 goto dma_error;
5963
5964 /* record length, and DMA address */
5965 dma_unmap_len_set(tx_buffer, len, size);
5966 dma_unmap_addr_set(tx_buffer, dma, dma);
5967
5968 tx_desc->read.buffer_addr = cpu_to_le64(dma);
5969
5970 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
5971 tx_desc->read.cmd_type_len =
5972 cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
5973
5974 i++;
5975 tx_desc++;
5976 if (i == tx_ring->count) {
5977 tx_desc = IGB_TX_DESC(tx_ring, 0);
5978 i = 0;
5979 }
5980 tx_desc->read.olinfo_status = 0;
5981
5982 dma += IGB_MAX_DATA_PER_TXD;
5983 size -= IGB_MAX_DATA_PER_TXD;
5984
5985 tx_desc->read.buffer_addr = cpu_to_le64(dma);
5986 }
5987
5988 if (likely(!data_len))
5989 break;
5990
5991 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
5992
5993 i++;
5994 tx_desc++;
5995 if (i == tx_ring->count) {
5996 tx_desc = IGB_TX_DESC(tx_ring, 0);
5997 i = 0;
5998 }
5999 tx_desc->read.olinfo_status = 0;
6000
6001 size = skb_frag_size(frag);
6002 data_len -= size;
6003
6004 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
6005 size, DMA_TO_DEVICE);
6006
6007 tx_buffer = &tx_ring->tx_buffer_info[i];
6008 }
6009
6010 /* write last descriptor with RS and EOP bits */
6011 cmd_type |= size | IGB_TXD_DCMD;
6012 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6013
6014 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
6015
6016 /* set the timestamp */
6017 first->time_stamp = jiffies;
6018
6019 skb_tx_timestamp(skb);
6020
6021 /* Force memory writes to complete before letting h/w know there
6022 * are new descriptors to fetch. (Only applicable for weak-ordered
6023 * memory model archs, such as IA-64).
6024 *
6025 * We also need this memory barrier to make certain all of the
6026 * status bits have been updated before next_to_watch is written.
6027 */
6028 dma_wmb();
6029
6030 /* set next_to_watch value indicating a packet is present */
6031 first->next_to_watch = tx_desc;
6032
6033 i++;
6034 if (i == tx_ring->count)
6035 i = 0;
6036
6037 tx_ring->next_to_use = i;
6038
6039 /* Make sure there is space in the ring for the next send. */
6040 igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6041
6042 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
6043 writel(i, tx_ring->tail);
6044 }
6045 return 0;
6046
6047dma_error:
6048 dev_err(tx_ring->dev, "TX DMA map failed\n");
6049 tx_buffer = &tx_ring->tx_buffer_info[i];
6050
6051 /* clear dma mappings for failed tx_buffer_info map */
6052 while (tx_buffer != first) {
6053 if (dma_unmap_len(tx_buffer, len))
6054 dma_unmap_page(tx_ring->dev,
6055 dma_unmap_addr(tx_buffer, dma),
6056 dma_unmap_len(tx_buffer, len),
6057 DMA_TO_DEVICE);
6058 dma_unmap_len_set(tx_buffer, len, 0);
6059
6060 if (i-- == 0)
6061 i += tx_ring->count;
6062 tx_buffer = &tx_ring->tx_buffer_info[i];
6063 }
6064
6065 if (dma_unmap_len(tx_buffer, len))
6066 dma_unmap_single(tx_ring->dev,
6067 dma_unmap_addr(tx_buffer, dma),
6068 dma_unmap_len(tx_buffer, len),
6069 DMA_TO_DEVICE);
6070 dma_unmap_len_set(tx_buffer, len, 0);
6071
6072 dev_kfree_skb_any(tx_buffer->skb);
6073 tx_buffer->skb = NULL;
6074
6075 tx_ring->next_to_use = i;
6076
6077 return -1;
6078}
6079
6080netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
6081 struct igb_ring *tx_ring)
6082{
6083 struct igb_tx_buffer *first;
6084 int tso;
6085 u32 tx_flags = 0;
6086 unsigned short f;
6087 u16 count = TXD_USE_COUNT(skb_headlen(skb));
6088 __be16 protocol = vlan_get_protocol(skb);
6089 u8 hdr_len = 0;
6090
6091 /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
6092 * + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
6093 * + 2 desc gap to keep tail from touching head,
6094 * + 1 desc for context descriptor,
6095 * otherwise try next time
6096 */
6097 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
6098 count += TXD_USE_COUNT(skb_frag_size(
6099 &skb_shinfo(skb)->frags[f]));
6100
6101 if (igb_maybe_stop_tx(tx_ring, count + 3)) {
6102 /* this is a hard error */
6103 return NETDEV_TX_BUSY;
6104 }
6105
6106 /* record the location of the first descriptor for this packet */
6107 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
6108 first->skb = skb;
6109 first->bytecount = skb->len;
6110 first->gso_segs = 1;
6111
6112 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
6113 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6114
6115 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
6116 !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
6117 &adapter->state)) {
6118 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
6119 tx_flags |= IGB_TX_FLAGS_TSTAMP;
6120
6121 adapter->ptp_tx_skb = skb_get(skb);
6122 adapter->ptp_tx_start = jiffies;
6123 if (adapter->hw.mac.type == e1000_82576)
6124 schedule_work(&adapter->ptp_tx_work);
6125 } else {
6126 adapter->tx_hwtstamp_skipped++;
6127 }
6128 }
6129
6130 if (skb_vlan_tag_present(skb)) {
6131 tx_flags |= IGB_TX_FLAGS_VLAN;
6132 tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
6133 }
6134
6135 /* record initial flags and protocol */
6136 first->tx_flags = tx_flags;
6137 first->protocol = protocol;
6138
6139 tso = igb_tso(tx_ring, first, &hdr_len);
6140 if (tso < 0)
6141 goto out_drop;
6142 else if (!tso)
6143 igb_tx_csum(tx_ring, first);
6144
6145 if (igb_tx_map(tx_ring, first, hdr_len))
6146 goto cleanup_tx_tstamp;
6147
6148 return NETDEV_TX_OK;
6149
6150out_drop:
6151 dev_kfree_skb_any(first->skb);
6152 first->skb = NULL;
6153cleanup_tx_tstamp:
6154 if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) {
6155 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6156
6157 dev_kfree_skb_any(adapter->ptp_tx_skb);
6158 adapter->ptp_tx_skb = NULL;
6159 if (adapter->hw.mac.type == e1000_82576)
6160 cancel_work_sync(&adapter->ptp_tx_work);
6161 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
6162 }
6163
6164 return NETDEV_TX_OK;
6165}
6166
6167static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
6168 struct sk_buff *skb)
6169{
6170 unsigned int r_idx = skb->queue_mapping;
6171
6172 if (r_idx >= adapter->num_tx_queues)
6173 r_idx = r_idx % adapter->num_tx_queues;
6174
6175 return adapter->tx_ring[r_idx];
6176}
6177
6178static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
6179 struct net_device *netdev)
6180{
6181 struct igb_adapter *adapter = netdev_priv(netdev);
6182
6183 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
6184 * in order to meet this minimum size requirement.
6185 */
6186 if (skb_put_padto(skb, 17))
6187 return NETDEV_TX_OK;
6188
6189 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
6190}
6191
6192/**
6193 * igb_tx_timeout - Respond to a Tx Hang
6194 * @netdev: network interface device structure
6195 **/
6196static void igb_tx_timeout(struct net_device *netdev, unsigned int txqueue)
6197{
6198 struct igb_adapter *adapter = netdev_priv(netdev);
6199 struct e1000_hw *hw = &adapter->hw;
6200
6201 /* Do the reset outside of interrupt context */
6202 adapter->tx_timeout_count++;
6203
6204 if (hw->mac.type >= e1000_82580)
6205 hw->dev_spec._82575.global_device_reset = true;
6206
6207 schedule_work(&adapter->reset_task);
6208 wr32(E1000_EICS,
6209 (adapter->eims_enable_mask & ~adapter->eims_other));
6210}
6211
6212static void igb_reset_task(struct work_struct *work)
6213{
6214 struct igb_adapter *adapter;
6215 adapter = container_of(work, struct igb_adapter, reset_task);
6216
6217 rtnl_lock();
6218 /* If we're already down or resetting, just bail */
6219 if (test_bit(__IGB_DOWN, &adapter->state) ||
6220 test_bit(__IGB_RESETTING, &adapter->state)) {
6221 rtnl_unlock();
6222 return;
6223 }
6224
6225 igb_dump(adapter);
6226 netdev_err(adapter->netdev, "Reset adapter\n");
6227 igb_reinit_locked(adapter);
6228 rtnl_unlock();
6229}
6230
6231/**
6232 * igb_get_stats64 - Get System Network Statistics
6233 * @netdev: network interface device structure
6234 * @stats: rtnl_link_stats64 pointer
6235 **/
6236static void igb_get_stats64(struct net_device *netdev,
6237 struct rtnl_link_stats64 *stats)
6238{
6239 struct igb_adapter *adapter = netdev_priv(netdev);
6240
6241 spin_lock(&adapter->stats64_lock);
6242 igb_update_stats(adapter);
6243 memcpy(stats, &adapter->stats64, sizeof(*stats));
6244 spin_unlock(&adapter->stats64_lock);
6245}
6246
6247/**
6248 * igb_change_mtu - Change the Maximum Transfer Unit
6249 * @netdev: network interface device structure
6250 * @new_mtu: new value for maximum frame size
6251 *
6252 * Returns 0 on success, negative on failure
6253 **/
6254static int igb_change_mtu(struct net_device *netdev, int new_mtu)
6255{
6256 struct igb_adapter *adapter = netdev_priv(netdev);
6257 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
6258
6259 /* adjust max frame to be at least the size of a standard frame */
6260 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
6261 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
6262
6263 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
6264 usleep_range(1000, 2000);
6265
6266 /* igb_down has a dependency on max_frame_size */
6267 adapter->max_frame_size = max_frame;
6268
6269 if (netif_running(netdev))
6270 igb_down(adapter);
6271
6272 netdev_dbg(netdev, "changing MTU from %d to %d\n",
6273 netdev->mtu, new_mtu);
6274 netdev->mtu = new_mtu;
6275
6276 if (netif_running(netdev))
6277 igb_up(adapter);
6278 else
6279 igb_reset(adapter);
6280
6281 clear_bit(__IGB_RESETTING, &adapter->state);
6282
6283 return 0;
6284}
6285
6286/**
6287 * igb_update_stats - Update the board statistics counters
6288 * @adapter: board private structure
6289 **/
6290void igb_update_stats(struct igb_adapter *adapter)
6291{
6292 struct rtnl_link_stats64 *net_stats = &adapter->stats64;
6293 struct e1000_hw *hw = &adapter->hw;
6294 struct pci_dev *pdev = adapter->pdev;
6295 u32 reg, mpc;
6296 int i;
6297 u64 bytes, packets;
6298 unsigned int start;
6299 u64 _bytes, _packets;
6300
6301 /* Prevent stats update while adapter is being reset, or if the pci
6302 * connection is down.
6303 */
6304 if (adapter->link_speed == 0)
6305 return;
6306 if (pci_channel_offline(pdev))
6307 return;
6308
6309 bytes = 0;
6310 packets = 0;
6311
6312 rcu_read_lock();
6313 for (i = 0; i < adapter->num_rx_queues; i++) {
6314 struct igb_ring *ring = adapter->rx_ring[i];
6315 u32 rqdpc = rd32(E1000_RQDPC(i));
6316 if (hw->mac.type >= e1000_i210)
6317 wr32(E1000_RQDPC(i), 0);
6318
6319 if (rqdpc) {
6320 ring->rx_stats.drops += rqdpc;
6321 net_stats->rx_fifo_errors += rqdpc;
6322 }
6323
6324 do {
6325 start = u64_stats_fetch_begin_irq(&ring->rx_syncp);
6326 _bytes = ring->rx_stats.bytes;
6327 _packets = ring->rx_stats.packets;
6328 } while (u64_stats_fetch_retry_irq(&ring->rx_syncp, start));
6329 bytes += _bytes;
6330 packets += _packets;
6331 }
6332
6333 net_stats->rx_bytes = bytes;
6334 net_stats->rx_packets = packets;
6335
6336 bytes = 0;
6337 packets = 0;
6338 for (i = 0; i < adapter->num_tx_queues; i++) {
6339 struct igb_ring *ring = adapter->tx_ring[i];
6340 do {
6341 start = u64_stats_fetch_begin_irq(&ring->tx_syncp);
6342 _bytes = ring->tx_stats.bytes;
6343 _packets = ring->tx_stats.packets;
6344 } while (u64_stats_fetch_retry_irq(&ring->tx_syncp, start));
6345 bytes += _bytes;
6346 packets += _packets;
6347 }
6348 net_stats->tx_bytes = bytes;
6349 net_stats->tx_packets = packets;
6350 rcu_read_unlock();
6351
6352 /* read stats registers */
6353 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
6354 adapter->stats.gprc += rd32(E1000_GPRC);
6355 adapter->stats.gorc += rd32(E1000_GORCL);
6356 rd32(E1000_GORCH); /* clear GORCL */
6357 adapter->stats.bprc += rd32(E1000_BPRC);
6358 adapter->stats.mprc += rd32(E1000_MPRC);
6359 adapter->stats.roc += rd32(E1000_ROC);
6360
6361 adapter->stats.prc64 += rd32(E1000_PRC64);
6362 adapter->stats.prc127 += rd32(E1000_PRC127);
6363 adapter->stats.prc255 += rd32(E1000_PRC255);
6364 adapter->stats.prc511 += rd32(E1000_PRC511);
6365 adapter->stats.prc1023 += rd32(E1000_PRC1023);
6366 adapter->stats.prc1522 += rd32(E1000_PRC1522);
6367 adapter->stats.symerrs += rd32(E1000_SYMERRS);
6368 adapter->stats.sec += rd32(E1000_SEC);
6369
6370 mpc = rd32(E1000_MPC);
6371 adapter->stats.mpc += mpc;
6372 net_stats->rx_fifo_errors += mpc;
6373 adapter->stats.scc += rd32(E1000_SCC);
6374 adapter->stats.ecol += rd32(E1000_ECOL);
6375 adapter->stats.mcc += rd32(E1000_MCC);
6376 adapter->stats.latecol += rd32(E1000_LATECOL);
6377 adapter->stats.dc += rd32(E1000_DC);
6378 adapter->stats.rlec += rd32(E1000_RLEC);
6379 adapter->stats.xonrxc += rd32(E1000_XONRXC);
6380 adapter->stats.xontxc += rd32(E1000_XONTXC);
6381 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
6382 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
6383 adapter->stats.fcruc += rd32(E1000_FCRUC);
6384 adapter->stats.gptc += rd32(E1000_GPTC);
6385 adapter->stats.gotc += rd32(E1000_GOTCL);
6386 rd32(E1000_GOTCH); /* clear GOTCL */
6387 adapter->stats.rnbc += rd32(E1000_RNBC);
6388 adapter->stats.ruc += rd32(E1000_RUC);
6389 adapter->stats.rfc += rd32(E1000_RFC);
6390 adapter->stats.rjc += rd32(E1000_RJC);
6391 adapter->stats.tor += rd32(E1000_TORH);
6392 adapter->stats.tot += rd32(E1000_TOTH);
6393 adapter->stats.tpr += rd32(E1000_TPR);
6394
6395 adapter->stats.ptc64 += rd32(E1000_PTC64);
6396 adapter->stats.ptc127 += rd32(E1000_PTC127);
6397 adapter->stats.ptc255 += rd32(E1000_PTC255);
6398 adapter->stats.ptc511 += rd32(E1000_PTC511);
6399 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
6400 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
6401
6402 adapter->stats.mptc += rd32(E1000_MPTC);
6403 adapter->stats.bptc += rd32(E1000_BPTC);
6404
6405 adapter->stats.tpt += rd32(E1000_TPT);
6406 adapter->stats.colc += rd32(E1000_COLC);
6407
6408 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
6409 /* read internal phy specific stats */
6410 reg = rd32(E1000_CTRL_EXT);
6411 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
6412 adapter->stats.rxerrc += rd32(E1000_RXERRC);
6413
6414 /* this stat has invalid values on i210/i211 */
6415 if ((hw->mac.type != e1000_i210) &&
6416 (hw->mac.type != e1000_i211))
6417 adapter->stats.tncrs += rd32(E1000_TNCRS);
6418 }
6419
6420 adapter->stats.tsctc += rd32(E1000_TSCTC);
6421 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
6422
6423 adapter->stats.iac += rd32(E1000_IAC);
6424 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
6425 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
6426 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
6427 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
6428 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
6429 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
6430 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
6431 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
6432
6433 /* Fill out the OS statistics structure */
6434 net_stats->multicast = adapter->stats.mprc;
6435 net_stats->collisions = adapter->stats.colc;
6436
6437 /* Rx Errors */
6438
6439 /* RLEC on some newer hardware can be incorrect so build
6440 * our own version based on RUC and ROC
6441 */
6442 net_stats->rx_errors = adapter->stats.rxerrc +
6443 adapter->stats.crcerrs + adapter->stats.algnerrc +
6444 adapter->stats.ruc + adapter->stats.roc +
6445 adapter->stats.cexterr;
6446 net_stats->rx_length_errors = adapter->stats.ruc +
6447 adapter->stats.roc;
6448 net_stats->rx_crc_errors = adapter->stats.crcerrs;
6449 net_stats->rx_frame_errors = adapter->stats.algnerrc;
6450 net_stats->rx_missed_errors = adapter->stats.mpc;
6451
6452 /* Tx Errors */
6453 net_stats->tx_errors = adapter->stats.ecol +
6454 adapter->stats.latecol;
6455 net_stats->tx_aborted_errors = adapter->stats.ecol;
6456 net_stats->tx_window_errors = adapter->stats.latecol;
6457 net_stats->tx_carrier_errors = adapter->stats.tncrs;
6458
6459 /* Tx Dropped needs to be maintained elsewhere */
6460
6461 /* Management Stats */
6462 adapter->stats.mgptc += rd32(E1000_MGTPTC);
6463 adapter->stats.mgprc += rd32(E1000_MGTPRC);
6464 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
6465
6466 /* OS2BMC Stats */
6467 reg = rd32(E1000_MANC);
6468 if (reg & E1000_MANC_EN_BMC2OS) {
6469 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
6470 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
6471 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
6472 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
6473 }
6474}
6475
6476static void igb_tsync_interrupt(struct igb_adapter *adapter)
6477{
6478 struct e1000_hw *hw = &adapter->hw;
6479 struct ptp_clock_event event;
6480 struct timespec64 ts;
6481 u32 ack = 0, tsauxc, sec, nsec, tsicr = rd32(E1000_TSICR);
6482
6483 if (tsicr & TSINTR_SYS_WRAP) {
6484 event.type = PTP_CLOCK_PPS;
6485 if (adapter->ptp_caps.pps)
6486 ptp_clock_event(adapter->ptp_clock, &event);
6487 ack |= TSINTR_SYS_WRAP;
6488 }
6489
6490 if (tsicr & E1000_TSICR_TXTS) {
6491 /* retrieve hardware timestamp */
6492 schedule_work(&adapter->ptp_tx_work);
6493 ack |= E1000_TSICR_TXTS;
6494 }
6495
6496 if (tsicr & TSINTR_TT0) {
6497 spin_lock(&adapter->tmreg_lock);
6498 ts = timespec64_add(adapter->perout[0].start,
6499 adapter->perout[0].period);
6500 /* u32 conversion of tv_sec is safe until y2106 */
6501 wr32(E1000_TRGTTIML0, ts.tv_nsec);
6502 wr32(E1000_TRGTTIMH0, (u32)ts.tv_sec);
6503 tsauxc = rd32(E1000_TSAUXC);
6504 tsauxc |= TSAUXC_EN_TT0;
6505 wr32(E1000_TSAUXC, tsauxc);
6506 adapter->perout[0].start = ts;
6507 spin_unlock(&adapter->tmreg_lock);
6508 ack |= TSINTR_TT0;
6509 }
6510
6511 if (tsicr & TSINTR_TT1) {
6512 spin_lock(&adapter->tmreg_lock);
6513 ts = timespec64_add(adapter->perout[1].start,
6514 adapter->perout[1].period);
6515 wr32(E1000_TRGTTIML1, ts.tv_nsec);
6516 wr32(E1000_TRGTTIMH1, (u32)ts.tv_sec);
6517 tsauxc = rd32(E1000_TSAUXC);
6518 tsauxc |= TSAUXC_EN_TT1;
6519 wr32(E1000_TSAUXC, tsauxc);
6520 adapter->perout[1].start = ts;
6521 spin_unlock(&adapter->tmreg_lock);
6522 ack |= TSINTR_TT1;
6523 }
6524
6525 if (tsicr & TSINTR_AUTT0) {
6526 nsec = rd32(E1000_AUXSTMPL0);
6527 sec = rd32(E1000_AUXSTMPH0);
6528 event.type = PTP_CLOCK_EXTTS;
6529 event.index = 0;
6530 event.timestamp = sec * 1000000000ULL + nsec;
6531 ptp_clock_event(adapter->ptp_clock, &event);
6532 ack |= TSINTR_AUTT0;
6533 }
6534
6535 if (tsicr & TSINTR_AUTT1) {
6536 nsec = rd32(E1000_AUXSTMPL1);
6537 sec = rd32(E1000_AUXSTMPH1);
6538 event.type = PTP_CLOCK_EXTTS;
6539 event.index = 1;
6540 event.timestamp = sec * 1000000000ULL + nsec;
6541 ptp_clock_event(adapter->ptp_clock, &event);
6542 ack |= TSINTR_AUTT1;
6543 }
6544
6545 /* acknowledge the interrupts */
6546 wr32(E1000_TSICR, ack);
6547}
6548
6549static irqreturn_t igb_msix_other(int irq, void *data)
6550{
6551 struct igb_adapter *adapter = data;
6552 struct e1000_hw *hw = &adapter->hw;
6553 u32 icr = rd32(E1000_ICR);
6554 /* reading ICR causes bit 31 of EICR to be cleared */
6555
6556 if (icr & E1000_ICR_DRSTA)
6557 schedule_work(&adapter->reset_task);
6558
6559 if (icr & E1000_ICR_DOUTSYNC) {
6560 /* HW is reporting DMA is out of sync */
6561 adapter->stats.doosync++;
6562 /* The DMA Out of Sync is also indication of a spoof event
6563 * in IOV mode. Check the Wrong VM Behavior register to
6564 * see if it is really a spoof event.
6565 */
6566 igb_check_wvbr(adapter);
6567 }
6568
6569 /* Check for a mailbox event */
6570 if (icr & E1000_ICR_VMMB)
6571 igb_msg_task(adapter);
6572
6573 if (icr & E1000_ICR_LSC) {
6574 hw->mac.get_link_status = 1;
6575 /* guard against interrupt when we're going down */
6576 if (!test_bit(__IGB_DOWN, &adapter->state))
6577 mod_timer(&adapter->watchdog_timer, jiffies + 1);
6578 }
6579
6580 if (icr & E1000_ICR_TS)
6581 igb_tsync_interrupt(adapter);
6582
6583 wr32(E1000_EIMS, adapter->eims_other);
6584
6585 return IRQ_HANDLED;
6586}
6587
6588static void igb_write_itr(struct igb_q_vector *q_vector)
6589{
6590 struct igb_adapter *adapter = q_vector->adapter;
6591 u32 itr_val = q_vector->itr_val & 0x7FFC;
6592
6593 if (!q_vector->set_itr)
6594 return;
6595
6596 if (!itr_val)
6597 itr_val = 0x4;
6598
6599 if (adapter->hw.mac.type == e1000_82575)
6600 itr_val |= itr_val << 16;
6601 else
6602 itr_val |= E1000_EITR_CNT_IGNR;
6603
6604 writel(itr_val, q_vector->itr_register);
6605 q_vector->set_itr = 0;
6606}
6607
6608static irqreturn_t igb_msix_ring(int irq, void *data)
6609{
6610 struct igb_q_vector *q_vector = data;
6611
6612 /* Write the ITR value calculated from the previous interrupt. */
6613 igb_write_itr(q_vector);
6614
6615 napi_schedule(&q_vector->napi);
6616
6617 return IRQ_HANDLED;
6618}
6619
6620#ifdef CONFIG_IGB_DCA
6621static void igb_update_tx_dca(struct igb_adapter *adapter,
6622 struct igb_ring *tx_ring,
6623 int cpu)
6624{
6625 struct e1000_hw *hw = &adapter->hw;
6626 u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
6627
6628 if (hw->mac.type != e1000_82575)
6629 txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;
6630
6631 /* We can enable relaxed ordering for reads, but not writes when
6632 * DCA is enabled. This is due to a known issue in some chipsets
6633 * which will cause the DCA tag to be cleared.
6634 */
6635 txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
6636 E1000_DCA_TXCTRL_DATA_RRO_EN |
6637 E1000_DCA_TXCTRL_DESC_DCA_EN;
6638
6639 wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
6640}
6641
6642static void igb_update_rx_dca(struct igb_adapter *adapter,
6643 struct igb_ring *rx_ring,
6644 int cpu)
6645{
6646 struct e1000_hw *hw = &adapter->hw;
6647 u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);
6648
6649 if (hw->mac.type != e1000_82575)
6650 rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;
6651
6652 /* We can enable relaxed ordering for reads, but not writes when
6653 * DCA is enabled. This is due to a known issue in some chipsets
6654 * which will cause the DCA tag to be cleared.
6655 */
6656 rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
6657 E1000_DCA_RXCTRL_DESC_DCA_EN;
6658
6659 wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
6660}
6661
6662static void igb_update_dca(struct igb_q_vector *q_vector)
6663{
6664 struct igb_adapter *adapter = q_vector->adapter;
6665 int cpu = get_cpu();
6666
6667 if (q_vector->cpu == cpu)
6668 goto out_no_update;
6669
6670 if (q_vector->tx.ring)
6671 igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);
6672
6673 if (q_vector->rx.ring)
6674 igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);
6675
6676 q_vector->cpu = cpu;
6677out_no_update:
6678 put_cpu();
6679}
6680
6681static void igb_setup_dca(struct igb_adapter *adapter)
6682{
6683 struct e1000_hw *hw = &adapter->hw;
6684 int i;
6685
6686 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
6687 return;
6688
6689 /* Always use CB2 mode, difference is masked in the CB driver. */
6690 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
6691
6692 for (i = 0; i < adapter->num_q_vectors; i++) {
6693 adapter->q_vector[i]->cpu = -1;
6694 igb_update_dca(adapter->q_vector[i]);
6695 }
6696}
6697
6698static int __igb_notify_dca(struct device *dev, void *data)
6699{
6700 struct net_device *netdev = dev_get_drvdata(dev);
6701 struct igb_adapter *adapter = netdev_priv(netdev);
6702 struct pci_dev *pdev = adapter->pdev;
6703 struct e1000_hw *hw = &adapter->hw;
6704 unsigned long event = *(unsigned long *)data;
6705
6706 switch (event) {
6707 case DCA_PROVIDER_ADD:
6708 /* if already enabled, don't do it again */
6709 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
6710 break;
6711 if (dca_add_requester(dev) == 0) {
6712 adapter->flags |= IGB_FLAG_DCA_ENABLED;
6713 dev_info(&pdev->dev, "DCA enabled\n");
6714 igb_setup_dca(adapter);
6715 break;
6716 }
6717 fallthrough; /* since DCA is disabled. */
6718 case DCA_PROVIDER_REMOVE:
6719 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
6720 /* without this a class_device is left
6721 * hanging around in the sysfs model
6722 */
6723 dca_remove_requester(dev);
6724 dev_info(&pdev->dev, "DCA disabled\n");
6725 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
6726 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
6727 }
6728 break;
6729 }
6730
6731 return 0;
6732}
6733
6734static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
6735 void *p)
6736{
6737 int ret_val;
6738
6739 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
6740 __igb_notify_dca);
6741
6742 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
6743}
6744#endif /* CONFIG_IGB_DCA */
6745
6746#ifdef CONFIG_PCI_IOV
6747static int igb_vf_configure(struct igb_adapter *adapter, int vf)
6748{
6749 unsigned char mac_addr[ETH_ALEN];
6750
6751 eth_zero_addr(mac_addr);
6752 igb_set_vf_mac(adapter, vf, mac_addr);
6753
6754 /* By default spoof check is enabled for all VFs */
6755 adapter->vf_data[vf].spoofchk_enabled = true;
6756
6757 /* By default VFs are not trusted */
6758 adapter->vf_data[vf].trusted = false;
6759
6760 return 0;
6761}
6762
6763#endif
6764static void igb_ping_all_vfs(struct igb_adapter *adapter)
6765{
6766 struct e1000_hw *hw = &adapter->hw;
6767 u32 ping;
6768 int i;
6769
6770 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
6771 ping = E1000_PF_CONTROL_MSG;
6772 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
6773 ping |= E1000_VT_MSGTYPE_CTS;
6774 igb_write_mbx(hw, &ping, 1, i);
6775 }
6776}
6777
6778static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
6779{
6780 struct e1000_hw *hw = &adapter->hw;
6781 u32 vmolr = rd32(E1000_VMOLR(vf));
6782 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6783
6784 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
6785 IGB_VF_FLAG_MULTI_PROMISC);
6786 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
6787
6788 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
6789 vmolr |= E1000_VMOLR_MPME;
6790 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
6791 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
6792 } else {
6793 /* if we have hashes and we are clearing a multicast promisc
6794 * flag we need to write the hashes to the MTA as this step
6795 * was previously skipped
6796 */
6797 if (vf_data->num_vf_mc_hashes > 30) {
6798 vmolr |= E1000_VMOLR_MPME;
6799 } else if (vf_data->num_vf_mc_hashes) {
6800 int j;
6801
6802 vmolr |= E1000_VMOLR_ROMPE;
6803 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
6804 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
6805 }
6806 }
6807
6808 wr32(E1000_VMOLR(vf), vmolr);
6809
6810 /* there are flags left unprocessed, likely not supported */
6811 if (*msgbuf & E1000_VT_MSGINFO_MASK)
6812 return -EINVAL;
6813
6814 return 0;
6815}
6816
6817static int igb_set_vf_multicasts(struct igb_adapter *adapter,
6818 u32 *msgbuf, u32 vf)
6819{
6820 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
6821 u16 *hash_list = (u16 *)&msgbuf[1];
6822 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
6823 int i;
6824
6825 /* salt away the number of multicast addresses assigned
6826 * to this VF for later use to restore when the PF multi cast
6827 * list changes
6828 */
6829 vf_data->num_vf_mc_hashes = n;
6830
6831 /* only up to 30 hash values supported */
6832 if (n > 30)
6833 n = 30;
6834
6835 /* store the hashes for later use */
6836 for (i = 0; i < n; i++)
6837 vf_data->vf_mc_hashes[i] = hash_list[i];
6838
6839 /* Flush and reset the mta with the new values */
6840 igb_set_rx_mode(adapter->netdev);
6841
6842 return 0;
6843}
6844
6845static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
6846{
6847 struct e1000_hw *hw = &adapter->hw;
6848 struct vf_data_storage *vf_data;
6849 int i, j;
6850
6851 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6852 u32 vmolr = rd32(E1000_VMOLR(i));
6853
6854 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
6855
6856 vf_data = &adapter->vf_data[i];
6857
6858 if ((vf_data->num_vf_mc_hashes > 30) ||
6859 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
6860 vmolr |= E1000_VMOLR_MPME;
6861 } else if (vf_data->num_vf_mc_hashes) {
6862 vmolr |= E1000_VMOLR_ROMPE;
6863 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
6864 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
6865 }
6866 wr32(E1000_VMOLR(i), vmolr);
6867 }
6868}
6869
6870static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
6871{
6872 struct e1000_hw *hw = &adapter->hw;
6873 u32 pool_mask, vlvf_mask, i;
6874
6875 /* create mask for VF and other pools */
6876 pool_mask = E1000_VLVF_POOLSEL_MASK;
6877 vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf);
6878
6879 /* drop PF from pool bits */
6880 pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT +
6881 adapter->vfs_allocated_count);
6882
6883 /* Find the vlan filter for this id */
6884 for (i = E1000_VLVF_ARRAY_SIZE; i--;) {
6885 u32 vlvf = rd32(E1000_VLVF(i));
6886 u32 vfta_mask, vid, vfta;
6887
6888 /* remove the vf from the pool */
6889 if (!(vlvf & vlvf_mask))
6890 continue;
6891
6892 /* clear out bit from VLVF */
6893 vlvf ^= vlvf_mask;
6894
6895 /* if other pools are present, just remove ourselves */
6896 if (vlvf & pool_mask)
6897 goto update_vlvfb;
6898
6899 /* if PF is present, leave VFTA */
6900 if (vlvf & E1000_VLVF_POOLSEL_MASK)
6901 goto update_vlvf;
6902
6903 vid = vlvf & E1000_VLVF_VLANID_MASK;
6904 vfta_mask = BIT(vid % 32);
6905
6906 /* clear bit from VFTA */
6907 vfta = adapter->shadow_vfta[vid / 32];
6908 if (vfta & vfta_mask)
6909 hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask);
6910update_vlvf:
6911 /* clear pool selection enable */
6912 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
6913 vlvf &= E1000_VLVF_POOLSEL_MASK;
6914 else
6915 vlvf = 0;
6916update_vlvfb:
6917 /* clear pool bits */
6918 wr32(E1000_VLVF(i), vlvf);
6919 }
6920}
6921
6922static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan)
6923{
6924 u32 vlvf;
6925 int idx;
6926
6927 /* short cut the special case */
6928 if (vlan == 0)
6929 return 0;
6930
6931 /* Search for the VLAN id in the VLVF entries */
6932 for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) {
6933 vlvf = rd32(E1000_VLVF(idx));
6934 if ((vlvf & VLAN_VID_MASK) == vlan)
6935 break;
6936 }
6937
6938 return idx;
6939}
6940
6941static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid)
6942{
6943 struct e1000_hw *hw = &adapter->hw;
6944 u32 bits, pf_id;
6945 int idx;
6946
6947 idx = igb_find_vlvf_entry(hw, vid);
6948 if (!idx)
6949 return;
6950
6951 /* See if any other pools are set for this VLAN filter
6952 * entry other than the PF.
6953 */
6954 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
6955 bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK;
6956 bits &= rd32(E1000_VLVF(idx));
6957
6958 /* Disable the filter so this falls into the default pool. */
6959 if (!bits) {
6960 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
6961 wr32(E1000_VLVF(idx), BIT(pf_id));
6962 else
6963 wr32(E1000_VLVF(idx), 0);
6964 }
6965}
6966
6967static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid,
6968 bool add, u32 vf)
6969{
6970 int pf_id = adapter->vfs_allocated_count;
6971 struct e1000_hw *hw = &adapter->hw;
6972 int err;
6973
6974 /* If VLAN overlaps with one the PF is currently monitoring make
6975 * sure that we are able to allocate a VLVF entry. This may be
6976 * redundant but it guarantees PF will maintain visibility to
6977 * the VLAN.
6978 */
6979 if (add && test_bit(vid, adapter->active_vlans)) {
6980 err = igb_vfta_set(hw, vid, pf_id, true, false);
6981 if (err)
6982 return err;
6983 }
6984
6985 err = igb_vfta_set(hw, vid, vf, add, false);
6986
6987 if (add && !err)
6988 return err;
6989
6990 /* If we failed to add the VF VLAN or we are removing the VF VLAN
6991 * we may need to drop the PF pool bit in order to allow us to free
6992 * up the VLVF resources.
6993 */
6994 if (test_bit(vid, adapter->active_vlans) ||
6995 (adapter->flags & IGB_FLAG_VLAN_PROMISC))
6996 igb_update_pf_vlvf(adapter, vid);
6997
6998 return err;
6999}
7000
7001static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
7002{
7003 struct e1000_hw *hw = &adapter->hw;
7004
7005 if (vid)
7006 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
7007 else
7008 wr32(E1000_VMVIR(vf), 0);
7009}
7010
7011static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf,
7012 u16 vlan, u8 qos)
7013{
7014 int err;
7015
7016 err = igb_set_vf_vlan(adapter, vlan, true, vf);
7017 if (err)
7018 return err;
7019
7020 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
7021 igb_set_vmolr(adapter, vf, !vlan);
7022
7023 /* revoke access to previous VLAN */
7024 if (vlan != adapter->vf_data[vf].pf_vlan)
7025 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7026 false, vf);
7027
7028 adapter->vf_data[vf].pf_vlan = vlan;
7029 adapter->vf_data[vf].pf_qos = qos;
7030 igb_set_vf_vlan_strip(adapter, vf, true);
7031 dev_info(&adapter->pdev->dev,
7032 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
7033 if (test_bit(__IGB_DOWN, &adapter->state)) {
7034 dev_warn(&adapter->pdev->dev,
7035 "The VF VLAN has been set, but the PF device is not up.\n");
7036 dev_warn(&adapter->pdev->dev,
7037 "Bring the PF device up before attempting to use the VF device.\n");
7038 }
7039
7040 return err;
7041}
7042
7043static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf)
7044{
7045 /* Restore tagless access via VLAN 0 */
7046 igb_set_vf_vlan(adapter, 0, true, vf);
7047
7048 igb_set_vmvir(adapter, 0, vf);
7049 igb_set_vmolr(adapter, vf, true);
7050
7051 /* Remove any PF assigned VLAN */
7052 if (adapter->vf_data[vf].pf_vlan)
7053 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7054 false, vf);
7055
7056 adapter->vf_data[vf].pf_vlan = 0;
7057 adapter->vf_data[vf].pf_qos = 0;
7058 igb_set_vf_vlan_strip(adapter, vf, false);
7059
7060 return 0;
7061}
7062
7063static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf,
7064 u16 vlan, u8 qos, __be16 vlan_proto)
7065{
7066 struct igb_adapter *adapter = netdev_priv(netdev);
7067
7068 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
7069 return -EINVAL;
7070
7071 if (vlan_proto != htons(ETH_P_8021Q))
7072 return -EPROTONOSUPPORT;
7073
7074 return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) :
7075 igb_disable_port_vlan(adapter, vf);
7076}
7077
7078static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7079{
7080 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
7081 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
7082 int ret;
7083
7084 if (adapter->vf_data[vf].pf_vlan)
7085 return -1;
7086
7087 /* VLAN 0 is a special case, don't allow it to be removed */
7088 if (!vid && !add)
7089 return 0;
7090
7091 ret = igb_set_vf_vlan(adapter, vid, !!add, vf);
7092 if (!ret)
7093 igb_set_vf_vlan_strip(adapter, vf, !!vid);
7094 return ret;
7095}
7096
7097static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
7098{
7099 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7100
7101 /* clear flags - except flag that indicates PF has set the MAC */
7102 vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC;
7103 vf_data->last_nack = jiffies;
7104
7105 /* reset vlans for device */
7106 igb_clear_vf_vfta(adapter, vf);
7107 igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf);
7108 igb_set_vmvir(adapter, vf_data->pf_vlan |
7109 (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf);
7110 igb_set_vmolr(adapter, vf, !vf_data->pf_vlan);
7111 igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan));
7112
7113 /* reset multicast table array for vf */
7114 adapter->vf_data[vf].num_vf_mc_hashes = 0;
7115
7116 /* Flush and reset the mta with the new values */
7117 igb_set_rx_mode(adapter->netdev);
7118}
7119
7120static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
7121{
7122 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7123
7124 /* clear mac address as we were hotplug removed/added */
7125 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
7126 eth_zero_addr(vf_mac);
7127
7128 /* process remaining reset events */
7129 igb_vf_reset(adapter, vf);
7130}
7131
7132static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
7133{
7134 struct e1000_hw *hw = &adapter->hw;
7135 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7136 u32 reg, msgbuf[3];
7137 u8 *addr = (u8 *)(&msgbuf[1]);
7138
7139 /* process all the same items cleared in a function level reset */
7140 igb_vf_reset(adapter, vf);
7141
7142 /* set vf mac address */
7143 igb_set_vf_mac(adapter, vf, vf_mac);
7144
7145 /* enable transmit and receive for vf */
7146 reg = rd32(E1000_VFTE);
7147 wr32(E1000_VFTE, reg | BIT(vf));
7148 reg = rd32(E1000_VFRE);
7149 wr32(E1000_VFRE, reg | BIT(vf));
7150
7151 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
7152
7153 /* reply to reset with ack and vf mac address */
7154 if (!is_zero_ether_addr(vf_mac)) {
7155 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
7156 memcpy(addr, vf_mac, ETH_ALEN);
7157 } else {
7158 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
7159 }
7160 igb_write_mbx(hw, msgbuf, 3, vf);
7161}
7162
7163static void igb_flush_mac_table(struct igb_adapter *adapter)
7164{
7165 struct e1000_hw *hw = &adapter->hw;
7166 int i;
7167
7168 for (i = 0; i < hw->mac.rar_entry_count; i++) {
7169 adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
7170 eth_zero_addr(adapter->mac_table[i].addr);
7171 adapter->mac_table[i].queue = 0;
7172 igb_rar_set_index(adapter, i);
7173 }
7174}
7175
7176static int igb_available_rars(struct igb_adapter *adapter, u8 queue)
7177{
7178 struct e1000_hw *hw = &adapter->hw;
7179 /* do not count rar entries reserved for VFs MAC addresses */
7180 int rar_entries = hw->mac.rar_entry_count -
7181 adapter->vfs_allocated_count;
7182 int i, count = 0;
7183
7184 for (i = 0; i < rar_entries; i++) {
7185 /* do not count default entries */
7186 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT)
7187 continue;
7188
7189 /* do not count "in use" entries for different queues */
7190 if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) &&
7191 (adapter->mac_table[i].queue != queue))
7192 continue;
7193
7194 count++;
7195 }
7196
7197 return count;
7198}
7199
7200/* Set default MAC address for the PF in the first RAR entry */
7201static void igb_set_default_mac_filter(struct igb_adapter *adapter)
7202{
7203 struct igb_mac_addr *mac_table = &adapter->mac_table[0];
7204
7205 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
7206 mac_table->queue = adapter->vfs_allocated_count;
7207 mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7208
7209 igb_rar_set_index(adapter, 0);
7210}
7211
7212/* If the filter to be added and an already existing filter express
7213 * the same address and address type, it should be possible to only
7214 * override the other configurations, for example the queue to steer
7215 * traffic.
7216 */
7217static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry,
7218 const u8 *addr, const u8 flags)
7219{
7220 if (!(entry->state & IGB_MAC_STATE_IN_USE))
7221 return true;
7222
7223 if ((entry->state & IGB_MAC_STATE_SRC_ADDR) !=
7224 (flags & IGB_MAC_STATE_SRC_ADDR))
7225 return false;
7226
7227 if (!ether_addr_equal(addr, entry->addr))
7228 return false;
7229
7230 return true;
7231}
7232
7233/* Add a MAC filter for 'addr' directing matching traffic to 'queue',
7234 * 'flags' is used to indicate what kind of match is made, match is by
7235 * default for the destination address, if matching by source address
7236 * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used.
7237 */
7238static int igb_add_mac_filter_flags(struct igb_adapter *adapter,
7239 const u8 *addr, const u8 queue,
7240 const u8 flags)
7241{
7242 struct e1000_hw *hw = &adapter->hw;
7243 int rar_entries = hw->mac.rar_entry_count -
7244 adapter->vfs_allocated_count;
7245 int i;
7246
7247 if (is_zero_ether_addr(addr))
7248 return -EINVAL;
7249
7250 /* Search for the first empty entry in the MAC table.
7251 * Do not touch entries at the end of the table reserved for the VF MAC
7252 * addresses.
7253 */
7254 for (i = 0; i < rar_entries; i++) {
7255 if (!igb_mac_entry_can_be_used(&adapter->mac_table[i],
7256 addr, flags))
7257 continue;
7258
7259 ether_addr_copy(adapter->mac_table[i].addr, addr);
7260 adapter->mac_table[i].queue = queue;
7261 adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags;
7262
7263 igb_rar_set_index(adapter, i);
7264 return i;
7265 }
7266
7267 return -ENOSPC;
7268}
7269
7270static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7271 const u8 queue)
7272{
7273 return igb_add_mac_filter_flags(adapter, addr, queue, 0);
7274}
7275
7276/* Remove a MAC filter for 'addr' directing matching traffic to
7277 * 'queue', 'flags' is used to indicate what kind of match need to be
7278 * removed, match is by default for the destination address, if
7279 * matching by source address is to be removed the flag
7280 * IGB_MAC_STATE_SRC_ADDR can be used.
7281 */
7282static int igb_del_mac_filter_flags(struct igb_adapter *adapter,
7283 const u8 *addr, const u8 queue,
7284 const u8 flags)
7285{
7286 struct e1000_hw *hw = &adapter->hw;
7287 int rar_entries = hw->mac.rar_entry_count -
7288 adapter->vfs_allocated_count;
7289 int i;
7290
7291 if (is_zero_ether_addr(addr))
7292 return -EINVAL;
7293
7294 /* Search for matching entry in the MAC table based on given address
7295 * and queue. Do not touch entries at the end of the table reserved
7296 * for the VF MAC addresses.
7297 */
7298 for (i = 0; i < rar_entries; i++) {
7299 if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE))
7300 continue;
7301 if ((adapter->mac_table[i].state & flags) != flags)
7302 continue;
7303 if (adapter->mac_table[i].queue != queue)
7304 continue;
7305 if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
7306 continue;
7307
7308 /* When a filter for the default address is "deleted",
7309 * we return it to its initial configuration
7310 */
7311 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) {
7312 adapter->mac_table[i].state =
7313 IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7314 adapter->mac_table[i].queue =
7315 adapter->vfs_allocated_count;
7316 } else {
7317 adapter->mac_table[i].state = 0;
7318 adapter->mac_table[i].queue = 0;
7319 eth_zero_addr(adapter->mac_table[i].addr);
7320 }
7321
7322 igb_rar_set_index(adapter, i);
7323 return 0;
7324 }
7325
7326 return -ENOENT;
7327}
7328
7329static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7330 const u8 queue)
7331{
7332 return igb_del_mac_filter_flags(adapter, addr, queue, 0);
7333}
7334
7335int igb_add_mac_steering_filter(struct igb_adapter *adapter,
7336 const u8 *addr, u8 queue, u8 flags)
7337{
7338 struct e1000_hw *hw = &adapter->hw;
7339
7340 /* In theory, this should be supported on 82575 as well, but
7341 * that part wasn't easily accessible during development.
7342 */
7343 if (hw->mac.type != e1000_i210)
7344 return -EOPNOTSUPP;
7345
7346 return igb_add_mac_filter_flags(adapter, addr, queue,
7347 IGB_MAC_STATE_QUEUE_STEERING | flags);
7348}
7349
7350int igb_del_mac_steering_filter(struct igb_adapter *adapter,
7351 const u8 *addr, u8 queue, u8 flags)
7352{
7353 return igb_del_mac_filter_flags(adapter, addr, queue,
7354 IGB_MAC_STATE_QUEUE_STEERING | flags);
7355}
7356
7357static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr)
7358{
7359 struct igb_adapter *adapter = netdev_priv(netdev);
7360 int ret;
7361
7362 ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7363
7364 return min_t(int, ret, 0);
7365}
7366
7367static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr)
7368{
7369 struct igb_adapter *adapter = netdev_priv(netdev);
7370
7371 igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7372
7373 return 0;
7374}
7375
7376static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
7377 const u32 info, const u8 *addr)
7378{
7379 struct pci_dev *pdev = adapter->pdev;
7380 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7381 struct list_head *pos;
7382 struct vf_mac_filter *entry = NULL;
7383 int ret = 0;
7384
7385 switch (info) {
7386 case E1000_VF_MAC_FILTER_CLR:
7387 /* remove all unicast MAC filters related to the current VF */
7388 list_for_each(pos, &adapter->vf_macs.l) {
7389 entry = list_entry(pos, struct vf_mac_filter, l);
7390 if (entry->vf == vf) {
7391 entry->vf = -1;
7392 entry->free = true;
7393 igb_del_mac_filter(adapter, entry->vf_mac, vf);
7394 }
7395 }
7396 break;
7397 case E1000_VF_MAC_FILTER_ADD:
7398 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7399 !vf_data->trusted) {
7400 dev_warn(&pdev->dev,
7401 "VF %d requested MAC filter but is administratively denied\n",
7402 vf);
7403 return -EINVAL;
7404 }
7405 if (!is_valid_ether_addr(addr)) {
7406 dev_warn(&pdev->dev,
7407 "VF %d attempted to set invalid MAC filter\n",
7408 vf);
7409 return -EINVAL;
7410 }
7411
7412 /* try to find empty slot in the list */
7413 list_for_each(pos, &adapter->vf_macs.l) {
7414 entry = list_entry(pos, struct vf_mac_filter, l);
7415 if (entry->free)
7416 break;
7417 }
7418
7419 if (entry && entry->free) {
7420 entry->free = false;
7421 entry->vf = vf;
7422 ether_addr_copy(entry->vf_mac, addr);
7423
7424 ret = igb_add_mac_filter(adapter, addr, vf);
7425 ret = min_t(int, ret, 0);
7426 } else {
7427 ret = -ENOSPC;
7428 }
7429
7430 if (ret == -ENOSPC)
7431 dev_warn(&pdev->dev,
7432 "VF %d has requested MAC filter but there is no space for it\n",
7433 vf);
7434 break;
7435 default:
7436 ret = -EINVAL;
7437 break;
7438 }
7439
7440 return ret;
7441}
7442
7443static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
7444{
7445 struct pci_dev *pdev = adapter->pdev;
7446 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7447 u32 info = msg[0] & E1000_VT_MSGINFO_MASK;
7448
7449 /* The VF MAC Address is stored in a packed array of bytes
7450 * starting at the second 32 bit word of the msg array
7451 */
7452 unsigned char *addr = (unsigned char *)&msg[1];
7453 int ret = 0;
7454
7455 if (!info) {
7456 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7457 !vf_data->trusted) {
7458 dev_warn(&pdev->dev,
7459 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
7460 vf);
7461 return -EINVAL;
7462 }
7463
7464 if (!is_valid_ether_addr(addr)) {
7465 dev_warn(&pdev->dev,
7466 "VF %d attempted to set invalid MAC\n",
7467 vf);
7468 return -EINVAL;
7469 }
7470
7471 ret = igb_set_vf_mac(adapter, vf, addr);
7472 } else {
7473 ret = igb_set_vf_mac_filter(adapter, vf, info, addr);
7474 }
7475
7476 return ret;
7477}
7478
7479static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
7480{
7481 struct e1000_hw *hw = &adapter->hw;
7482 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7483 u32 msg = E1000_VT_MSGTYPE_NACK;
7484
7485 /* if device isn't clear to send it shouldn't be reading either */
7486 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
7487 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
7488 igb_write_mbx(hw, &msg, 1, vf);
7489 vf_data->last_nack = jiffies;
7490 }
7491}
7492
7493static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
7494{
7495 struct pci_dev *pdev = adapter->pdev;
7496 u32 msgbuf[E1000_VFMAILBOX_SIZE];
7497 struct e1000_hw *hw = &adapter->hw;
7498 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7499 s32 retval;
7500
7501 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false);
7502
7503 if (retval) {
7504 /* if receive failed revoke VF CTS stats and restart init */
7505 dev_err(&pdev->dev, "Error receiving message from VF\n");
7506 vf_data->flags &= ~IGB_VF_FLAG_CTS;
7507 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
7508 goto unlock;
7509 goto out;
7510 }
7511
7512 /* this is a message we already processed, do nothing */
7513 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
7514 goto unlock;
7515
7516 /* until the vf completes a reset it should not be
7517 * allowed to start any configuration.
7518 */
7519 if (msgbuf[0] == E1000_VF_RESET) {
7520 /* unlocks mailbox */
7521 igb_vf_reset_msg(adapter, vf);
7522 return;
7523 }
7524
7525 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
7526 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
7527 goto unlock;
7528 retval = -1;
7529 goto out;
7530 }
7531
7532 switch ((msgbuf[0] & 0xFFFF)) {
7533 case E1000_VF_SET_MAC_ADDR:
7534 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
7535 break;
7536 case E1000_VF_SET_PROMISC:
7537 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
7538 break;
7539 case E1000_VF_SET_MULTICAST:
7540 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
7541 break;
7542 case E1000_VF_SET_LPE:
7543 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
7544 break;
7545 case E1000_VF_SET_VLAN:
7546 retval = -1;
7547 if (vf_data->pf_vlan)
7548 dev_warn(&pdev->dev,
7549 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
7550 vf);
7551 else
7552 retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf);
7553 break;
7554 default:
7555 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
7556 retval = -1;
7557 break;
7558 }
7559
7560 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
7561out:
7562 /* notify the VF of the results of what it sent us */
7563 if (retval)
7564 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
7565 else
7566 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
7567
7568 /* unlocks mailbox */
7569 igb_write_mbx(hw, msgbuf, 1, vf);
7570 return;
7571
7572unlock:
7573 igb_unlock_mbx(hw, vf);
7574}
7575
7576static void igb_msg_task(struct igb_adapter *adapter)
7577{
7578 struct e1000_hw *hw = &adapter->hw;
7579 u32 vf;
7580
7581 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
7582 /* process any reset requests */
7583 if (!igb_check_for_rst(hw, vf))
7584 igb_vf_reset_event(adapter, vf);
7585
7586 /* process any messages pending */
7587 if (!igb_check_for_msg(hw, vf))
7588 igb_rcv_msg_from_vf(adapter, vf);
7589
7590 /* process any acks */
7591 if (!igb_check_for_ack(hw, vf))
7592 igb_rcv_ack_from_vf(adapter, vf);
7593 }
7594}
7595
7596/**
7597 * igb_set_uta - Set unicast filter table address
7598 * @adapter: board private structure
7599 * @set: boolean indicating if we are setting or clearing bits
7600 *
7601 * The unicast table address is a register array of 32-bit registers.
7602 * The table is meant to be used in a way similar to how the MTA is used
7603 * however due to certain limitations in the hardware it is necessary to
7604 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
7605 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
7606 **/
7607static void igb_set_uta(struct igb_adapter *adapter, bool set)
7608{
7609 struct e1000_hw *hw = &adapter->hw;
7610 u32 uta = set ? ~0 : 0;
7611 int i;
7612
7613 /* we only need to do this if VMDq is enabled */
7614 if (!adapter->vfs_allocated_count)
7615 return;
7616
7617 for (i = hw->mac.uta_reg_count; i--;)
7618 array_wr32(E1000_UTA, i, uta);
7619}
7620
7621/**
7622 * igb_intr_msi - Interrupt Handler
7623 * @irq: interrupt number
7624 * @data: pointer to a network interface device structure
7625 **/
7626static irqreturn_t igb_intr_msi(int irq, void *data)
7627{
7628 struct igb_adapter *adapter = data;
7629 struct igb_q_vector *q_vector = adapter->q_vector[0];
7630 struct e1000_hw *hw = &adapter->hw;
7631 /* read ICR disables interrupts using IAM */
7632 u32 icr = rd32(E1000_ICR);
7633
7634 igb_write_itr(q_vector);
7635
7636 if (icr & E1000_ICR_DRSTA)
7637 schedule_work(&adapter->reset_task);
7638
7639 if (icr & E1000_ICR_DOUTSYNC) {
7640 /* HW is reporting DMA is out of sync */
7641 adapter->stats.doosync++;
7642 }
7643
7644 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
7645 hw->mac.get_link_status = 1;
7646 if (!test_bit(__IGB_DOWN, &adapter->state))
7647 mod_timer(&adapter->watchdog_timer, jiffies + 1);
7648 }
7649
7650 if (icr & E1000_ICR_TS)
7651 igb_tsync_interrupt(adapter);
7652
7653 napi_schedule(&q_vector->napi);
7654
7655 return IRQ_HANDLED;
7656}
7657
7658/**
7659 * igb_intr - Legacy Interrupt Handler
7660 * @irq: interrupt number
7661 * @data: pointer to a network interface device structure
7662 **/
7663static irqreturn_t igb_intr(int irq, void *data)
7664{
7665 struct igb_adapter *adapter = data;
7666 struct igb_q_vector *q_vector = adapter->q_vector[0];
7667 struct e1000_hw *hw = &adapter->hw;
7668 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
7669 * need for the IMC write
7670 */
7671 u32 icr = rd32(E1000_ICR);
7672
7673 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
7674 * not set, then the adapter didn't send an interrupt
7675 */
7676 if (!(icr & E1000_ICR_INT_ASSERTED))
7677 return IRQ_NONE;
7678
7679 igb_write_itr(q_vector);
7680
7681 if (icr & E1000_ICR_DRSTA)
7682 schedule_work(&adapter->reset_task);
7683
7684 if (icr & E1000_ICR_DOUTSYNC) {
7685 /* HW is reporting DMA is out of sync */
7686 adapter->stats.doosync++;
7687 }
7688
7689 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
7690 hw->mac.get_link_status = 1;
7691 /* guard against interrupt when we're going down */
7692 if (!test_bit(__IGB_DOWN, &adapter->state))
7693 mod_timer(&adapter->watchdog_timer, jiffies + 1);
7694 }
7695
7696 if (icr & E1000_ICR_TS)
7697 igb_tsync_interrupt(adapter);
7698
7699 napi_schedule(&q_vector->napi);
7700
7701 return IRQ_HANDLED;
7702}
7703
7704static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
7705{
7706 struct igb_adapter *adapter = q_vector->adapter;
7707 struct e1000_hw *hw = &adapter->hw;
7708
7709 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
7710 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
7711 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
7712 igb_set_itr(q_vector);
7713 else
7714 igb_update_ring_itr(q_vector);
7715 }
7716
7717 if (!test_bit(__IGB_DOWN, &adapter->state)) {
7718 if (adapter->flags & IGB_FLAG_HAS_MSIX)
7719 wr32(E1000_EIMS, q_vector->eims_value);
7720 else
7721 igb_irq_enable(adapter);
7722 }
7723}
7724
7725/**
7726 * igb_poll - NAPI Rx polling callback
7727 * @napi: napi polling structure
7728 * @budget: count of how many packets we should handle
7729 **/
7730static int igb_poll(struct napi_struct *napi, int budget)
7731{
7732 struct igb_q_vector *q_vector = container_of(napi,
7733 struct igb_q_vector,
7734 napi);
7735 bool clean_complete = true;
7736 int work_done = 0;
7737
7738#ifdef CONFIG_IGB_DCA
7739 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
7740 igb_update_dca(q_vector);
7741#endif
7742 if (q_vector->tx.ring)
7743 clean_complete = igb_clean_tx_irq(q_vector, budget);
7744
7745 if (q_vector->rx.ring) {
7746 int cleaned = igb_clean_rx_irq(q_vector, budget);
7747
7748 work_done += cleaned;
7749 if (cleaned >= budget)
7750 clean_complete = false;
7751 }
7752
7753 /* If all work not completed, return budget and keep polling */
7754 if (!clean_complete)
7755 return budget;
7756
7757 /* Exit the polling mode, but don't re-enable interrupts if stack might
7758 * poll us due to busy-polling
7759 */
7760 if (likely(napi_complete_done(napi, work_done)))
7761 igb_ring_irq_enable(q_vector);
7762
7763 return min(work_done, budget - 1);
7764}
7765
7766/**
7767 * igb_clean_tx_irq - Reclaim resources after transmit completes
7768 * @q_vector: pointer to q_vector containing needed info
7769 * @napi_budget: Used to determine if we are in netpoll
7770 *
7771 * returns true if ring is completely cleaned
7772 **/
7773static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget)
7774{
7775 struct igb_adapter *adapter = q_vector->adapter;
7776 struct igb_ring *tx_ring = q_vector->tx.ring;
7777 struct igb_tx_buffer *tx_buffer;
7778 union e1000_adv_tx_desc *tx_desc;
7779 unsigned int total_bytes = 0, total_packets = 0;
7780 unsigned int budget = q_vector->tx.work_limit;
7781 unsigned int i = tx_ring->next_to_clean;
7782
7783 if (test_bit(__IGB_DOWN, &adapter->state))
7784 return true;
7785
7786 tx_buffer = &tx_ring->tx_buffer_info[i];
7787 tx_desc = IGB_TX_DESC(tx_ring, i);
7788 i -= tx_ring->count;
7789
7790 do {
7791 union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
7792
7793 /* if next_to_watch is not set then there is no work pending */
7794 if (!eop_desc)
7795 break;
7796
7797 /* prevent any other reads prior to eop_desc */
7798 smp_rmb();
7799
7800 /* if DD is not set pending work has not been completed */
7801 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
7802 break;
7803
7804 /* clear next_to_watch to prevent false hangs */
7805 tx_buffer->next_to_watch = NULL;
7806
7807 /* update the statistics for this packet */
7808 total_bytes += tx_buffer->bytecount;
7809 total_packets += tx_buffer->gso_segs;
7810
7811 /* free the skb */
7812 napi_consume_skb(tx_buffer->skb, napi_budget);
7813
7814 /* unmap skb header data */
7815 dma_unmap_single(tx_ring->dev,
7816 dma_unmap_addr(tx_buffer, dma),
7817 dma_unmap_len(tx_buffer, len),
7818 DMA_TO_DEVICE);
7819
7820 /* clear tx_buffer data */
7821 dma_unmap_len_set(tx_buffer, len, 0);
7822
7823 /* clear last DMA location and unmap remaining buffers */
7824 while (tx_desc != eop_desc) {
7825 tx_buffer++;
7826 tx_desc++;
7827 i++;
7828 if (unlikely(!i)) {
7829 i -= tx_ring->count;
7830 tx_buffer = tx_ring->tx_buffer_info;
7831 tx_desc = IGB_TX_DESC(tx_ring, 0);
7832 }
7833
7834 /* unmap any remaining paged data */
7835 if (dma_unmap_len(tx_buffer, len)) {
7836 dma_unmap_page(tx_ring->dev,
7837 dma_unmap_addr(tx_buffer, dma),
7838 dma_unmap_len(tx_buffer, len),
7839 DMA_TO_DEVICE);
7840 dma_unmap_len_set(tx_buffer, len, 0);
7841 }
7842 }
7843
7844 /* move us one more past the eop_desc for start of next pkt */
7845 tx_buffer++;
7846 tx_desc++;
7847 i++;
7848 if (unlikely(!i)) {
7849 i -= tx_ring->count;
7850 tx_buffer = tx_ring->tx_buffer_info;
7851 tx_desc = IGB_TX_DESC(tx_ring, 0);
7852 }
7853
7854 /* issue prefetch for next Tx descriptor */
7855 prefetch(tx_desc);
7856
7857 /* update budget accounting */
7858 budget--;
7859 } while (likely(budget));
7860
7861 netdev_tx_completed_queue(txring_txq(tx_ring),
7862 total_packets, total_bytes);
7863 i += tx_ring->count;
7864 tx_ring->next_to_clean = i;
7865 u64_stats_update_begin(&tx_ring->tx_syncp);
7866 tx_ring->tx_stats.bytes += total_bytes;
7867 tx_ring->tx_stats.packets += total_packets;
7868 u64_stats_update_end(&tx_ring->tx_syncp);
7869 q_vector->tx.total_bytes += total_bytes;
7870 q_vector->tx.total_packets += total_packets;
7871
7872 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
7873 struct e1000_hw *hw = &adapter->hw;
7874
7875 /* Detect a transmit hang in hardware, this serializes the
7876 * check with the clearing of time_stamp and movement of i
7877 */
7878 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
7879 if (tx_buffer->next_to_watch &&
7880 time_after(jiffies, tx_buffer->time_stamp +
7881 (adapter->tx_timeout_factor * HZ)) &&
7882 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
7883
7884 /* detected Tx unit hang */
7885 dev_err(tx_ring->dev,
7886 "Detected Tx Unit Hang\n"
7887 " Tx Queue <%d>\n"
7888 " TDH <%x>\n"
7889 " TDT <%x>\n"
7890 " next_to_use <%x>\n"
7891 " next_to_clean <%x>\n"
7892 "buffer_info[next_to_clean]\n"
7893 " time_stamp <%lx>\n"
7894 " next_to_watch <%p>\n"
7895 " jiffies <%lx>\n"
7896 " desc.status <%x>\n",
7897 tx_ring->queue_index,
7898 rd32(E1000_TDH(tx_ring->reg_idx)),
7899 readl(tx_ring->tail),
7900 tx_ring->next_to_use,
7901 tx_ring->next_to_clean,
7902 tx_buffer->time_stamp,
7903 tx_buffer->next_to_watch,
7904 jiffies,
7905 tx_buffer->next_to_watch->wb.status);
7906 netif_stop_subqueue(tx_ring->netdev,
7907 tx_ring->queue_index);
7908
7909 /* we are about to reset, no point in enabling stuff */
7910 return true;
7911 }
7912 }
7913
7914#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
7915 if (unlikely(total_packets &&
7916 netif_carrier_ok(tx_ring->netdev) &&
7917 igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
7918 /* Make sure that anybody stopping the queue after this
7919 * sees the new next_to_clean.
7920 */
7921 smp_mb();
7922 if (__netif_subqueue_stopped(tx_ring->netdev,
7923 tx_ring->queue_index) &&
7924 !(test_bit(__IGB_DOWN, &adapter->state))) {
7925 netif_wake_subqueue(tx_ring->netdev,
7926 tx_ring->queue_index);
7927
7928 u64_stats_update_begin(&tx_ring->tx_syncp);
7929 tx_ring->tx_stats.restart_queue++;
7930 u64_stats_update_end(&tx_ring->tx_syncp);
7931 }
7932 }
7933
7934 return !!budget;
7935}
7936
7937/**
7938 * igb_reuse_rx_page - page flip buffer and store it back on the ring
7939 * @rx_ring: rx descriptor ring to store buffers on
7940 * @old_buff: donor buffer to have page reused
7941 *
7942 * Synchronizes page for reuse by the adapter
7943 **/
7944static void igb_reuse_rx_page(struct igb_ring *rx_ring,
7945 struct igb_rx_buffer *old_buff)
7946{
7947 struct igb_rx_buffer *new_buff;
7948 u16 nta = rx_ring->next_to_alloc;
7949
7950 new_buff = &rx_ring->rx_buffer_info[nta];
7951
7952 /* update, and store next to alloc */
7953 nta++;
7954 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
7955
7956 /* Transfer page from old buffer to new buffer.
7957 * Move each member individually to avoid possible store
7958 * forwarding stalls.
7959 */
7960 new_buff->dma = old_buff->dma;
7961 new_buff->page = old_buff->page;
7962 new_buff->page_offset = old_buff->page_offset;
7963 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
7964}
7965
7966static inline bool igb_page_is_reserved(struct page *page)
7967{
7968 return (page_to_nid(page) != numa_mem_id()) || page_is_pfmemalloc(page);
7969}
7970
7971static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer)
7972{
7973 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
7974 struct page *page = rx_buffer->page;
7975
7976 /* avoid re-using remote pages */
7977 if (unlikely(igb_page_is_reserved(page)))
7978 return false;
7979
7980#if (PAGE_SIZE < 8192)
7981 /* if we are only owner of page we can reuse it */
7982 if (unlikely((page_ref_count(page) - pagecnt_bias) > 1))
7983 return false;
7984#else
7985#define IGB_LAST_OFFSET \
7986 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)
7987
7988 if (rx_buffer->page_offset > IGB_LAST_OFFSET)
7989 return false;
7990#endif
7991
7992 /* If we have drained the page fragment pool we need to update
7993 * the pagecnt_bias and page count so that we fully restock the
7994 * number of references the driver holds.
7995 */
7996 if (unlikely(!pagecnt_bias)) {
7997 page_ref_add(page, USHRT_MAX);
7998 rx_buffer->pagecnt_bias = USHRT_MAX;
7999 }
8000
8001 return true;
8002}
8003
8004/**
8005 * igb_add_rx_frag - Add contents of Rx buffer to sk_buff
8006 * @rx_ring: rx descriptor ring to transact packets on
8007 * @rx_buffer: buffer containing page to add
8008 * @skb: sk_buff to place the data into
8009 * @size: size of buffer to be added
8010 *
8011 * This function will add the data contained in rx_buffer->page to the skb.
8012 **/
8013static void igb_add_rx_frag(struct igb_ring *rx_ring,
8014 struct igb_rx_buffer *rx_buffer,
8015 struct sk_buff *skb,
8016 unsigned int size)
8017{
8018#if (PAGE_SIZE < 8192)
8019 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8020#else
8021 unsigned int truesize = ring_uses_build_skb(rx_ring) ?
8022 SKB_DATA_ALIGN(IGB_SKB_PAD + size) :
8023 SKB_DATA_ALIGN(size);
8024#endif
8025 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
8026 rx_buffer->page_offset, size, truesize);
8027#if (PAGE_SIZE < 8192)
8028 rx_buffer->page_offset ^= truesize;
8029#else
8030 rx_buffer->page_offset += truesize;
8031#endif
8032}
8033
8034static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,
8035 struct igb_rx_buffer *rx_buffer,
8036 union e1000_adv_rx_desc *rx_desc,
8037 unsigned int size)
8038{
8039 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
8040#if (PAGE_SIZE < 8192)
8041 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8042#else
8043 unsigned int truesize = SKB_DATA_ALIGN(size);
8044#endif
8045 unsigned int headlen;
8046 struct sk_buff *skb;
8047
8048 /* prefetch first cache line of first page */
8049 prefetch(va);
8050#if L1_CACHE_BYTES < 128
8051 prefetch(va + L1_CACHE_BYTES);
8052#endif
8053
8054 /* allocate a skb to store the frags */
8055 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
8056 if (unlikely(!skb))
8057 return NULL;
8058
8059 if (unlikely(igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))) {
8060 igb_ptp_rx_pktstamp(rx_ring->q_vector, va, skb);
8061 va += IGB_TS_HDR_LEN;
8062 size -= IGB_TS_HDR_LEN;
8063 }
8064
8065 /* Determine available headroom for copy */
8066 headlen = size;
8067 if (headlen > IGB_RX_HDR_LEN)
8068 headlen = eth_get_headlen(skb->dev, va, IGB_RX_HDR_LEN);
8069
8070 /* align pull length to size of long to optimize memcpy performance */
8071 memcpy(__skb_put(skb, headlen), va, ALIGN(headlen, sizeof(long)));
8072
8073 /* update all of the pointers */
8074 size -= headlen;
8075 if (size) {
8076 skb_add_rx_frag(skb, 0, rx_buffer->page,
8077 (va + headlen) - page_address(rx_buffer->page),
8078 size, truesize);
8079#if (PAGE_SIZE < 8192)
8080 rx_buffer->page_offset ^= truesize;
8081#else
8082 rx_buffer->page_offset += truesize;
8083#endif
8084 } else {
8085 rx_buffer->pagecnt_bias++;
8086 }
8087
8088 return skb;
8089}
8090
8091static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring,
8092 struct igb_rx_buffer *rx_buffer,
8093 union e1000_adv_rx_desc *rx_desc,
8094 unsigned int size)
8095{
8096 void *va = page_address(rx_buffer->page) + rx_buffer->page_offset;
8097#if (PAGE_SIZE < 8192)
8098 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8099#else
8100 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
8101 SKB_DATA_ALIGN(IGB_SKB_PAD + size);
8102#endif
8103 struct sk_buff *skb;
8104
8105 /* prefetch first cache line of first page */
8106 prefetch(va);
8107#if L1_CACHE_BYTES < 128
8108 prefetch(va + L1_CACHE_BYTES);
8109#endif
8110
8111 /* build an skb around the page buffer */
8112 skb = build_skb(va - IGB_SKB_PAD, truesize);
8113 if (unlikely(!skb))
8114 return NULL;
8115
8116 /* update pointers within the skb to store the data */
8117 skb_reserve(skb, IGB_SKB_PAD);
8118 __skb_put(skb, size);
8119
8120 /* pull timestamp out of packet data */
8121 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
8122 igb_ptp_rx_pktstamp(rx_ring->q_vector, skb->data, skb);
8123 __skb_pull(skb, IGB_TS_HDR_LEN);
8124 }
8125
8126 /* update buffer offset */
8127#if (PAGE_SIZE < 8192)
8128 rx_buffer->page_offset ^= truesize;
8129#else
8130 rx_buffer->page_offset += truesize;
8131#endif
8132
8133 return skb;
8134}
8135
8136static inline void igb_rx_checksum(struct igb_ring *ring,
8137 union e1000_adv_rx_desc *rx_desc,
8138 struct sk_buff *skb)
8139{
8140 skb_checksum_none_assert(skb);
8141
8142 /* Ignore Checksum bit is set */
8143 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
8144 return;
8145
8146 /* Rx checksum disabled via ethtool */
8147 if (!(ring->netdev->features & NETIF_F_RXCSUM))
8148 return;
8149
8150 /* TCP/UDP checksum error bit is set */
8151 if (igb_test_staterr(rx_desc,
8152 E1000_RXDEXT_STATERR_TCPE |
8153 E1000_RXDEXT_STATERR_IPE)) {
8154 /* work around errata with sctp packets where the TCPE aka
8155 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
8156 * packets, (aka let the stack check the crc32c)
8157 */
8158 if (!((skb->len == 60) &&
8159 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
8160 u64_stats_update_begin(&ring->rx_syncp);
8161 ring->rx_stats.csum_err++;
8162 u64_stats_update_end(&ring->rx_syncp);
8163 }
8164 /* let the stack verify checksum errors */
8165 return;
8166 }
8167 /* It must be a TCP or UDP packet with a valid checksum */
8168 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
8169 E1000_RXD_STAT_UDPCS))
8170 skb->ip_summed = CHECKSUM_UNNECESSARY;
8171
8172 dev_dbg(ring->dev, "cksum success: bits %08X\n",
8173 le32_to_cpu(rx_desc->wb.upper.status_error));
8174}
8175
8176static inline void igb_rx_hash(struct igb_ring *ring,
8177 union e1000_adv_rx_desc *rx_desc,
8178 struct sk_buff *skb)
8179{
8180 if (ring->netdev->features & NETIF_F_RXHASH)
8181 skb_set_hash(skb,
8182 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
8183 PKT_HASH_TYPE_L3);
8184}
8185
8186/**
8187 * igb_is_non_eop - process handling of non-EOP buffers
8188 * @rx_ring: Rx ring being processed
8189 * @rx_desc: Rx descriptor for current buffer
8190 * @skb: current socket buffer containing buffer in progress
8191 *
8192 * This function updates next to clean. If the buffer is an EOP buffer
8193 * this function exits returning false, otherwise it will place the
8194 * sk_buff in the next buffer to be chained and return true indicating
8195 * that this is in fact a non-EOP buffer.
8196 **/
8197static bool igb_is_non_eop(struct igb_ring *rx_ring,
8198 union e1000_adv_rx_desc *rx_desc)
8199{
8200 u32 ntc = rx_ring->next_to_clean + 1;
8201
8202 /* fetch, update, and store next to clean */
8203 ntc = (ntc < rx_ring->count) ? ntc : 0;
8204 rx_ring->next_to_clean = ntc;
8205
8206 prefetch(IGB_RX_DESC(rx_ring, ntc));
8207
8208 if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
8209 return false;
8210
8211 return true;
8212}
8213
8214/**
8215 * igb_cleanup_headers - Correct corrupted or empty headers
8216 * @rx_ring: rx descriptor ring packet is being transacted on
8217 * @rx_desc: pointer to the EOP Rx descriptor
8218 * @skb: pointer to current skb being fixed
8219 *
8220 * Address the case where we are pulling data in on pages only
8221 * and as such no data is present in the skb header.
8222 *
8223 * In addition if skb is not at least 60 bytes we need to pad it so that
8224 * it is large enough to qualify as a valid Ethernet frame.
8225 *
8226 * Returns true if an error was encountered and skb was freed.
8227 **/
8228static bool igb_cleanup_headers(struct igb_ring *rx_ring,
8229 union e1000_adv_rx_desc *rx_desc,
8230 struct sk_buff *skb)
8231{
8232 if (unlikely((igb_test_staterr(rx_desc,
8233 E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
8234 struct net_device *netdev = rx_ring->netdev;
8235 if (!(netdev->features & NETIF_F_RXALL)) {
8236 dev_kfree_skb_any(skb);
8237 return true;
8238 }
8239 }
8240
8241 /* if eth_skb_pad returns an error the skb was freed */
8242 if (eth_skb_pad(skb))
8243 return true;
8244
8245 return false;
8246}
8247
8248/**
8249 * igb_process_skb_fields - Populate skb header fields from Rx descriptor
8250 * @rx_ring: rx descriptor ring packet is being transacted on
8251 * @rx_desc: pointer to the EOP Rx descriptor
8252 * @skb: pointer to current skb being populated
8253 *
8254 * This function checks the ring, descriptor, and packet information in
8255 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
8256 * other fields within the skb.
8257 **/
8258static void igb_process_skb_fields(struct igb_ring *rx_ring,
8259 union e1000_adv_rx_desc *rx_desc,
8260 struct sk_buff *skb)
8261{
8262 struct net_device *dev = rx_ring->netdev;
8263
8264 igb_rx_hash(rx_ring, rx_desc, skb);
8265
8266 igb_rx_checksum(rx_ring, rx_desc, skb);
8267
8268 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) &&
8269 !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))
8270 igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);
8271
8272 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
8273 igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
8274 u16 vid;
8275
8276 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
8277 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
8278 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
8279 else
8280 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
8281
8282 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
8283 }
8284
8285 skb_record_rx_queue(skb, rx_ring->queue_index);
8286
8287 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8288}
8289
8290static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring,
8291 const unsigned int size)
8292{
8293 struct igb_rx_buffer *rx_buffer;
8294
8295 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
8296 prefetchw(rx_buffer->page);
8297
8298 /* we are reusing so sync this buffer for CPU use */
8299 dma_sync_single_range_for_cpu(rx_ring->dev,
8300 rx_buffer->dma,
8301 rx_buffer->page_offset,
8302 size,
8303 DMA_FROM_DEVICE);
8304
8305 rx_buffer->pagecnt_bias--;
8306
8307 return rx_buffer;
8308}
8309
8310static void igb_put_rx_buffer(struct igb_ring *rx_ring,
8311 struct igb_rx_buffer *rx_buffer)
8312{
8313 if (igb_can_reuse_rx_page(rx_buffer)) {
8314 /* hand second half of page back to the ring */
8315 igb_reuse_rx_page(rx_ring, rx_buffer);
8316 } else {
8317 /* We are not reusing the buffer so unmap it and free
8318 * any references we are holding to it
8319 */
8320 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
8321 igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
8322 IGB_RX_DMA_ATTR);
8323 __page_frag_cache_drain(rx_buffer->page,
8324 rx_buffer->pagecnt_bias);
8325 }
8326
8327 /* clear contents of rx_buffer */
8328 rx_buffer->page = NULL;
8329}
8330
8331static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
8332{
8333 struct igb_ring *rx_ring = q_vector->rx.ring;
8334 struct sk_buff *skb = rx_ring->skb;
8335 unsigned int total_bytes = 0, total_packets = 0;
8336 u16 cleaned_count = igb_desc_unused(rx_ring);
8337
8338 while (likely(total_packets < budget)) {
8339 union e1000_adv_rx_desc *rx_desc;
8340 struct igb_rx_buffer *rx_buffer;
8341 unsigned int size;
8342
8343 /* return some buffers to hardware, one at a time is too slow */
8344 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
8345 igb_alloc_rx_buffers(rx_ring, cleaned_count);
8346 cleaned_count = 0;
8347 }
8348
8349 rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
8350 size = le16_to_cpu(rx_desc->wb.upper.length);
8351 if (!size)
8352 break;
8353
8354 /* This memory barrier is needed to keep us from reading
8355 * any other fields out of the rx_desc until we know the
8356 * descriptor has been written back
8357 */
8358 dma_rmb();
8359
8360 rx_buffer = igb_get_rx_buffer(rx_ring, size);
8361
8362 /* retrieve a buffer from the ring */
8363 if (skb)
8364 igb_add_rx_frag(rx_ring, rx_buffer, skb, size);
8365 else if (ring_uses_build_skb(rx_ring))
8366 skb = igb_build_skb(rx_ring, rx_buffer, rx_desc, size);
8367 else
8368 skb = igb_construct_skb(rx_ring, rx_buffer,
8369 rx_desc, size);
8370
8371 /* exit if we failed to retrieve a buffer */
8372 if (!skb) {
8373 rx_ring->rx_stats.alloc_failed++;
8374 rx_buffer->pagecnt_bias++;
8375 break;
8376 }
8377
8378 igb_put_rx_buffer(rx_ring, rx_buffer);
8379 cleaned_count++;
8380
8381 /* fetch next buffer in frame if non-eop */
8382 if (igb_is_non_eop(rx_ring, rx_desc))
8383 continue;
8384
8385 /* verify the packet layout is correct */
8386 if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
8387 skb = NULL;
8388 continue;
8389 }
8390
8391 /* probably a little skewed due to removing CRC */
8392 total_bytes += skb->len;
8393
8394 /* populate checksum, timestamp, VLAN, and protocol */
8395 igb_process_skb_fields(rx_ring, rx_desc, skb);
8396
8397 napi_gro_receive(&q_vector->napi, skb);
8398
8399 /* reset skb pointer */
8400 skb = NULL;
8401
8402 /* update budget accounting */
8403 total_packets++;
8404 }
8405
8406 /* place incomplete frames back on ring for completion */
8407 rx_ring->skb = skb;
8408
8409 u64_stats_update_begin(&rx_ring->rx_syncp);
8410 rx_ring->rx_stats.packets += total_packets;
8411 rx_ring->rx_stats.bytes += total_bytes;
8412 u64_stats_update_end(&rx_ring->rx_syncp);
8413 q_vector->rx.total_packets += total_packets;
8414 q_vector->rx.total_bytes += total_bytes;
8415
8416 if (cleaned_count)
8417 igb_alloc_rx_buffers(rx_ring, cleaned_count);
8418
8419 return total_packets;
8420}
8421
8422static inline unsigned int igb_rx_offset(struct igb_ring *rx_ring)
8423{
8424 return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0;
8425}
8426
8427static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
8428 struct igb_rx_buffer *bi)
8429{
8430 struct page *page = bi->page;
8431 dma_addr_t dma;
8432
8433 /* since we are recycling buffers we should seldom need to alloc */
8434 if (likely(page))
8435 return true;
8436
8437 /* alloc new page for storage */
8438 page = dev_alloc_pages(igb_rx_pg_order(rx_ring));
8439 if (unlikely(!page)) {
8440 rx_ring->rx_stats.alloc_failed++;
8441 return false;
8442 }
8443
8444 /* map page for use */
8445 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
8446 igb_rx_pg_size(rx_ring),
8447 DMA_FROM_DEVICE,
8448 IGB_RX_DMA_ATTR);
8449
8450 /* if mapping failed free memory back to system since
8451 * there isn't much point in holding memory we can't use
8452 */
8453 if (dma_mapping_error(rx_ring->dev, dma)) {
8454 __free_pages(page, igb_rx_pg_order(rx_ring));
8455
8456 rx_ring->rx_stats.alloc_failed++;
8457 return false;
8458 }
8459
8460 bi->dma = dma;
8461 bi->page = page;
8462 bi->page_offset = igb_rx_offset(rx_ring);
8463 bi->pagecnt_bias = 1;
8464
8465 return true;
8466}
8467
8468/**
8469 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
8470 * @adapter: address of board private structure
8471 **/
8472void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
8473{
8474 union e1000_adv_rx_desc *rx_desc;
8475 struct igb_rx_buffer *bi;
8476 u16 i = rx_ring->next_to_use;
8477 u16 bufsz;
8478
8479 /* nothing to do */
8480 if (!cleaned_count)
8481 return;
8482
8483 rx_desc = IGB_RX_DESC(rx_ring, i);
8484 bi = &rx_ring->rx_buffer_info[i];
8485 i -= rx_ring->count;
8486
8487 bufsz = igb_rx_bufsz(rx_ring);
8488
8489 do {
8490 if (!igb_alloc_mapped_page(rx_ring, bi))
8491 break;
8492
8493 /* sync the buffer for use by the device */
8494 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
8495 bi->page_offset, bufsz,
8496 DMA_FROM_DEVICE);
8497
8498 /* Refresh the desc even if buffer_addrs didn't change
8499 * because each write-back erases this info.
8500 */
8501 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
8502
8503 rx_desc++;
8504 bi++;
8505 i++;
8506 if (unlikely(!i)) {
8507 rx_desc = IGB_RX_DESC(rx_ring, 0);
8508 bi = rx_ring->rx_buffer_info;
8509 i -= rx_ring->count;
8510 }
8511
8512 /* clear the length for the next_to_use descriptor */
8513 rx_desc->wb.upper.length = 0;
8514
8515 cleaned_count--;
8516 } while (cleaned_count);
8517
8518 i += rx_ring->count;
8519
8520 if (rx_ring->next_to_use != i) {
8521 /* record the next descriptor to use */
8522 rx_ring->next_to_use = i;
8523
8524 /* update next to alloc since we have filled the ring */
8525 rx_ring->next_to_alloc = i;
8526
8527 /* Force memory writes to complete before letting h/w
8528 * know there are new descriptors to fetch. (Only
8529 * applicable for weak-ordered memory model archs,
8530 * such as IA-64).
8531 */
8532 dma_wmb();
8533 writel(i, rx_ring->tail);
8534 }
8535}
8536
8537/**
8538 * igb_mii_ioctl -
8539 * @netdev:
8540 * @ifreq:
8541 * @cmd:
8542 **/
8543static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
8544{
8545 struct igb_adapter *adapter = netdev_priv(netdev);
8546 struct mii_ioctl_data *data = if_mii(ifr);
8547
8548 if (adapter->hw.phy.media_type != e1000_media_type_copper)
8549 return -EOPNOTSUPP;
8550
8551 switch (cmd) {
8552 case SIOCGMIIPHY:
8553 data->phy_id = adapter->hw.phy.addr;
8554 break;
8555 case SIOCGMIIREG:
8556 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
8557 &data->val_out))
8558 return -EIO;
8559 break;
8560 case SIOCSMIIREG:
8561 default:
8562 return -EOPNOTSUPP;
8563 }
8564 return 0;
8565}
8566
8567/**
8568 * igb_ioctl -
8569 * @netdev:
8570 * @ifreq:
8571 * @cmd:
8572 **/
8573static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
8574{
8575 switch (cmd) {
8576 case SIOCGMIIPHY:
8577 case SIOCGMIIREG:
8578 case SIOCSMIIREG:
8579 return igb_mii_ioctl(netdev, ifr, cmd);
8580 case SIOCGHWTSTAMP:
8581 return igb_ptp_get_ts_config(netdev, ifr);
8582 case SIOCSHWTSTAMP:
8583 return igb_ptp_set_ts_config(netdev, ifr);
8584 default:
8585 return -EOPNOTSUPP;
8586 }
8587}
8588
8589void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
8590{
8591 struct igb_adapter *adapter = hw->back;
8592
8593 pci_read_config_word(adapter->pdev, reg, value);
8594}
8595
8596void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
8597{
8598 struct igb_adapter *adapter = hw->back;
8599
8600 pci_write_config_word(adapter->pdev, reg, *value);
8601}
8602
8603s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
8604{
8605 struct igb_adapter *adapter = hw->back;
8606
8607 if (pcie_capability_read_word(adapter->pdev, reg, value))
8608 return -E1000_ERR_CONFIG;
8609
8610 return 0;
8611}
8612
8613s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
8614{
8615 struct igb_adapter *adapter = hw->back;
8616
8617 if (pcie_capability_write_word(adapter->pdev, reg, *value))
8618 return -E1000_ERR_CONFIG;
8619
8620 return 0;
8621}
8622
8623static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
8624{
8625 struct igb_adapter *adapter = netdev_priv(netdev);
8626 struct e1000_hw *hw = &adapter->hw;
8627 u32 ctrl, rctl;
8628 bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
8629
8630 if (enable) {
8631 /* enable VLAN tag insert/strip */
8632 ctrl = rd32(E1000_CTRL);
8633 ctrl |= E1000_CTRL_VME;
8634 wr32(E1000_CTRL, ctrl);
8635
8636 /* Disable CFI check */
8637 rctl = rd32(E1000_RCTL);
8638 rctl &= ~E1000_RCTL_CFIEN;
8639 wr32(E1000_RCTL, rctl);
8640 } else {
8641 /* disable VLAN tag insert/strip */
8642 ctrl = rd32(E1000_CTRL);
8643 ctrl &= ~E1000_CTRL_VME;
8644 wr32(E1000_CTRL, ctrl);
8645 }
8646
8647 igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable);
8648}
8649
8650static int igb_vlan_rx_add_vid(struct net_device *netdev,
8651 __be16 proto, u16 vid)
8652{
8653 struct igb_adapter *adapter = netdev_priv(netdev);
8654 struct e1000_hw *hw = &adapter->hw;
8655 int pf_id = adapter->vfs_allocated_count;
8656
8657 /* add the filter since PF can receive vlans w/o entry in vlvf */
8658 if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
8659 igb_vfta_set(hw, vid, pf_id, true, !!vid);
8660
8661 set_bit(vid, adapter->active_vlans);
8662
8663 return 0;
8664}
8665
8666static int igb_vlan_rx_kill_vid(struct net_device *netdev,
8667 __be16 proto, u16 vid)
8668{
8669 struct igb_adapter *adapter = netdev_priv(netdev);
8670 int pf_id = adapter->vfs_allocated_count;
8671 struct e1000_hw *hw = &adapter->hw;
8672
8673 /* remove VID from filter table */
8674 if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
8675 igb_vfta_set(hw, vid, pf_id, false, true);
8676
8677 clear_bit(vid, adapter->active_vlans);
8678
8679 return 0;
8680}
8681
8682static void igb_restore_vlan(struct igb_adapter *adapter)
8683{
8684 u16 vid = 1;
8685
8686 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
8687 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
8688
8689 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
8690 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
8691}
8692
8693int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
8694{
8695 struct pci_dev *pdev = adapter->pdev;
8696 struct e1000_mac_info *mac = &adapter->hw.mac;
8697
8698 mac->autoneg = 0;
8699
8700 /* Make sure dplx is at most 1 bit and lsb of speed is not set
8701 * for the switch() below to work
8702 */
8703 if ((spd & 1) || (dplx & ~1))
8704 goto err_inval;
8705
8706 /* Fiber NIC's only allow 1000 gbps Full duplex
8707 * and 100Mbps Full duplex for 100baseFx sfp
8708 */
8709 if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
8710 switch (spd + dplx) {
8711 case SPEED_10 + DUPLEX_HALF:
8712 case SPEED_10 + DUPLEX_FULL:
8713 case SPEED_100 + DUPLEX_HALF:
8714 goto err_inval;
8715 default:
8716 break;
8717 }
8718 }
8719
8720 switch (spd + dplx) {
8721 case SPEED_10 + DUPLEX_HALF:
8722 mac->forced_speed_duplex = ADVERTISE_10_HALF;
8723 break;
8724 case SPEED_10 + DUPLEX_FULL:
8725 mac->forced_speed_duplex = ADVERTISE_10_FULL;
8726 break;
8727 case SPEED_100 + DUPLEX_HALF:
8728 mac->forced_speed_duplex = ADVERTISE_100_HALF;
8729 break;
8730 case SPEED_100 + DUPLEX_FULL:
8731 mac->forced_speed_duplex = ADVERTISE_100_FULL;
8732 break;
8733 case SPEED_1000 + DUPLEX_FULL:
8734 mac->autoneg = 1;
8735 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
8736 break;
8737 case SPEED_1000 + DUPLEX_HALF: /* not supported */
8738 default:
8739 goto err_inval;
8740 }
8741
8742 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
8743 adapter->hw.phy.mdix = AUTO_ALL_MODES;
8744
8745 return 0;
8746
8747err_inval:
8748 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
8749 return -EINVAL;
8750}
8751
8752static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
8753 bool runtime)
8754{
8755 struct net_device *netdev = pci_get_drvdata(pdev);
8756 struct igb_adapter *adapter = netdev_priv(netdev);
8757 struct e1000_hw *hw = &adapter->hw;
8758 u32 ctrl, rctl, status;
8759 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
8760 bool wake;
8761
8762 rtnl_lock();
8763 netif_device_detach(netdev);
8764
8765 if (netif_running(netdev))
8766 __igb_close(netdev, true);
8767
8768 igb_ptp_suspend(adapter);
8769
8770 igb_clear_interrupt_scheme(adapter);
8771 rtnl_unlock();
8772
8773 status = rd32(E1000_STATUS);
8774 if (status & E1000_STATUS_LU)
8775 wufc &= ~E1000_WUFC_LNKC;
8776
8777 if (wufc) {
8778 igb_setup_rctl(adapter);
8779 igb_set_rx_mode(netdev);
8780
8781 /* turn on all-multi mode if wake on multicast is enabled */
8782 if (wufc & E1000_WUFC_MC) {
8783 rctl = rd32(E1000_RCTL);
8784 rctl |= E1000_RCTL_MPE;
8785 wr32(E1000_RCTL, rctl);
8786 }
8787
8788 ctrl = rd32(E1000_CTRL);
8789 ctrl |= E1000_CTRL_ADVD3WUC;
8790 wr32(E1000_CTRL, ctrl);
8791
8792 /* Allow time for pending master requests to run */
8793 igb_disable_pcie_master(hw);
8794
8795 wr32(E1000_WUC, E1000_WUC_PME_EN);
8796 wr32(E1000_WUFC, wufc);
8797 } else {
8798 wr32(E1000_WUC, 0);
8799 wr32(E1000_WUFC, 0);
8800 }
8801
8802 wake = wufc || adapter->en_mng_pt;
8803 if (!wake)
8804 igb_power_down_link(adapter);
8805 else
8806 igb_power_up_link(adapter);
8807
8808 if (enable_wake)
8809 *enable_wake = wake;
8810
8811 /* Release control of h/w to f/w. If f/w is AMT enabled, this
8812 * would have already happened in close and is redundant.
8813 */
8814 igb_release_hw_control(adapter);
8815
8816 pci_disable_device(pdev);
8817
8818 return 0;
8819}
8820
8821static void igb_deliver_wake_packet(struct net_device *netdev)
8822{
8823 struct igb_adapter *adapter = netdev_priv(netdev);
8824 struct e1000_hw *hw = &adapter->hw;
8825 struct sk_buff *skb;
8826 u32 wupl;
8827
8828 wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK;
8829
8830 /* WUPM stores only the first 128 bytes of the wake packet.
8831 * Read the packet only if we have the whole thing.
8832 */
8833 if ((wupl == 0) || (wupl > E1000_WUPM_BYTES))
8834 return;
8835
8836 skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES);
8837 if (!skb)
8838 return;
8839
8840 skb_put(skb, wupl);
8841
8842 /* Ensure reads are 32-bit aligned */
8843 wupl = roundup(wupl, 4);
8844
8845 memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl);
8846
8847 skb->protocol = eth_type_trans(skb, netdev);
8848 netif_rx(skb);
8849}
8850
8851static int __maybe_unused igb_suspend(struct device *dev)
8852{
8853 return __igb_shutdown(to_pci_dev(dev), NULL, 0);
8854}
8855
8856static int __maybe_unused igb_resume(struct device *dev)
8857{
8858 struct pci_dev *pdev = to_pci_dev(dev);
8859 struct net_device *netdev = pci_get_drvdata(pdev);
8860 struct igb_adapter *adapter = netdev_priv(netdev);
8861 struct e1000_hw *hw = &adapter->hw;
8862 u32 err, val;
8863
8864 pci_set_power_state(pdev, PCI_D0);
8865 pci_restore_state(pdev);
8866 pci_save_state(pdev);
8867
8868 if (!pci_device_is_present(pdev))
8869 return -ENODEV;
8870 err = pci_enable_device_mem(pdev);
8871 if (err) {
8872 dev_err(&pdev->dev,
8873 "igb: Cannot enable PCI device from suspend\n");
8874 return err;
8875 }
8876 pci_set_master(pdev);
8877
8878 pci_enable_wake(pdev, PCI_D3hot, 0);
8879 pci_enable_wake(pdev, PCI_D3cold, 0);
8880
8881 if (igb_init_interrupt_scheme(adapter, true)) {
8882 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
8883 return -ENOMEM;
8884 }
8885
8886 igb_reset(adapter);
8887
8888 /* let the f/w know that the h/w is now under the control of the
8889 * driver.
8890 */
8891 igb_get_hw_control(adapter);
8892
8893 val = rd32(E1000_WUS);
8894 if (val & WAKE_PKT_WUS)
8895 igb_deliver_wake_packet(netdev);
8896
8897 wr32(E1000_WUS, ~0);
8898
8899 rtnl_lock();
8900 if (!err && netif_running(netdev))
8901 err = __igb_open(netdev, true);
8902
8903 if (!err)
8904 netif_device_attach(netdev);
8905 rtnl_unlock();
8906
8907 return err;
8908}
8909
8910static int __maybe_unused igb_runtime_idle(struct device *dev)
8911{
8912 struct net_device *netdev = dev_get_drvdata(dev);
8913 struct igb_adapter *adapter = netdev_priv(netdev);
8914
8915 if (!igb_has_link(adapter))
8916 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
8917
8918 return -EBUSY;
8919}
8920
8921static int __maybe_unused igb_runtime_suspend(struct device *dev)
8922{
8923 return __igb_shutdown(to_pci_dev(dev), NULL, 1);
8924}
8925
8926static int __maybe_unused igb_runtime_resume(struct device *dev)
8927{
8928 return igb_resume(dev);
8929}
8930
8931static void igb_shutdown(struct pci_dev *pdev)
8932{
8933 bool wake;
8934
8935 __igb_shutdown(pdev, &wake, 0);
8936
8937 if (system_state == SYSTEM_POWER_OFF) {
8938 pci_wake_from_d3(pdev, wake);
8939 pci_set_power_state(pdev, PCI_D3hot);
8940 }
8941}
8942
8943#ifdef CONFIG_PCI_IOV
8944static int igb_sriov_reinit(struct pci_dev *dev)
8945{
8946 struct net_device *netdev = pci_get_drvdata(dev);
8947 struct igb_adapter *adapter = netdev_priv(netdev);
8948 struct pci_dev *pdev = adapter->pdev;
8949
8950 rtnl_lock();
8951
8952 if (netif_running(netdev))
8953 igb_close(netdev);
8954 else
8955 igb_reset(adapter);
8956
8957 igb_clear_interrupt_scheme(adapter);
8958
8959 igb_init_queue_configuration(adapter);
8960
8961 if (igb_init_interrupt_scheme(adapter, true)) {
8962 rtnl_unlock();
8963 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
8964 return -ENOMEM;
8965 }
8966
8967 if (netif_running(netdev))
8968 igb_open(netdev);
8969
8970 rtnl_unlock();
8971
8972 return 0;
8973}
8974
8975static int igb_pci_disable_sriov(struct pci_dev *dev)
8976{
8977 int err = igb_disable_sriov(dev);
8978
8979 if (!err)
8980 err = igb_sriov_reinit(dev);
8981
8982 return err;
8983}
8984
8985static int igb_pci_enable_sriov(struct pci_dev *dev, int num_vfs)
8986{
8987 int err = igb_enable_sriov(dev, num_vfs);
8988
8989 if (err)
8990 goto out;
8991
8992 err = igb_sriov_reinit(dev);
8993 if (!err)
8994 return num_vfs;
8995
8996out:
8997 return err;
8998}
8999
9000#endif
9001static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
9002{
9003#ifdef CONFIG_PCI_IOV
9004 if (num_vfs == 0)
9005 return igb_pci_disable_sriov(dev);
9006 else
9007 return igb_pci_enable_sriov(dev, num_vfs);
9008#endif
9009 return 0;
9010}
9011
9012/**
9013 * igb_io_error_detected - called when PCI error is detected
9014 * @pdev: Pointer to PCI device
9015 * @state: The current pci connection state
9016 *
9017 * This function is called after a PCI bus error affecting
9018 * this device has been detected.
9019 **/
9020static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
9021 pci_channel_state_t state)
9022{
9023 struct net_device *netdev = pci_get_drvdata(pdev);
9024 struct igb_adapter *adapter = netdev_priv(netdev);
9025
9026 netif_device_detach(netdev);
9027
9028 if (state == pci_channel_io_perm_failure)
9029 return PCI_ERS_RESULT_DISCONNECT;
9030
9031 if (netif_running(netdev))
9032 igb_down(adapter);
9033 pci_disable_device(pdev);
9034
9035 /* Request a slot slot reset. */
9036 return PCI_ERS_RESULT_NEED_RESET;
9037}
9038
9039/**
9040 * igb_io_slot_reset - called after the pci bus has been reset.
9041 * @pdev: Pointer to PCI device
9042 *
9043 * Restart the card from scratch, as if from a cold-boot. Implementation
9044 * resembles the first-half of the igb_resume routine.
9045 **/
9046static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
9047{
9048 struct net_device *netdev = pci_get_drvdata(pdev);
9049 struct igb_adapter *adapter = netdev_priv(netdev);
9050 struct e1000_hw *hw = &adapter->hw;
9051 pci_ers_result_t result;
9052
9053 if (pci_enable_device_mem(pdev)) {
9054 dev_err(&pdev->dev,
9055 "Cannot re-enable PCI device after reset.\n");
9056 result = PCI_ERS_RESULT_DISCONNECT;
9057 } else {
9058 pci_set_master(pdev);
9059 pci_restore_state(pdev);
9060 pci_save_state(pdev);
9061
9062 pci_enable_wake(pdev, PCI_D3hot, 0);
9063 pci_enable_wake(pdev, PCI_D3cold, 0);
9064
9065 /* In case of PCI error, adapter lose its HW address
9066 * so we should re-assign it here.
9067 */
9068 hw->hw_addr = adapter->io_addr;
9069
9070 igb_reset(adapter);
9071 wr32(E1000_WUS, ~0);
9072 result = PCI_ERS_RESULT_RECOVERED;
9073 }
9074
9075 return result;
9076}
9077
9078/**
9079 * igb_io_resume - called when traffic can start flowing again.
9080 * @pdev: Pointer to PCI device
9081 *
9082 * This callback is called when the error recovery driver tells us that
9083 * its OK to resume normal operation. Implementation resembles the
9084 * second-half of the igb_resume routine.
9085 */
9086static void igb_io_resume(struct pci_dev *pdev)
9087{
9088 struct net_device *netdev = pci_get_drvdata(pdev);
9089 struct igb_adapter *adapter = netdev_priv(netdev);
9090
9091 if (netif_running(netdev)) {
9092 if (igb_up(adapter)) {
9093 dev_err(&pdev->dev, "igb_up failed after reset\n");
9094 return;
9095 }
9096 }
9097
9098 netif_device_attach(netdev);
9099
9100 /* let the f/w know that the h/w is now under the control of the
9101 * driver.
9102 */
9103 igb_get_hw_control(adapter);
9104}
9105
9106/**
9107 * igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
9108 * @adapter: Pointer to adapter structure
9109 * @index: Index of the RAR entry which need to be synced with MAC table
9110 **/
9111static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
9112{
9113 struct e1000_hw *hw = &adapter->hw;
9114 u32 rar_low, rar_high;
9115 u8 *addr = adapter->mac_table[index].addr;
9116
9117 /* HW expects these to be in network order when they are plugged
9118 * into the registers which are little endian. In order to guarantee
9119 * that ordering we need to do an leXX_to_cpup here in order to be
9120 * ready for the byteswap that occurs with writel
9121 */
9122 rar_low = le32_to_cpup((__le32 *)(addr));
9123 rar_high = le16_to_cpup((__le16 *)(addr + 4));
9124
9125 /* Indicate to hardware the Address is Valid. */
9126 if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) {
9127 if (is_valid_ether_addr(addr))
9128 rar_high |= E1000_RAH_AV;
9129
9130 if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR)
9131 rar_high |= E1000_RAH_ASEL_SRC_ADDR;
9132
9133 switch (hw->mac.type) {
9134 case e1000_82575:
9135 case e1000_i210:
9136 if (adapter->mac_table[index].state &
9137 IGB_MAC_STATE_QUEUE_STEERING)
9138 rar_high |= E1000_RAH_QSEL_ENABLE;
9139
9140 rar_high |= E1000_RAH_POOL_1 *
9141 adapter->mac_table[index].queue;
9142 break;
9143 default:
9144 rar_high |= E1000_RAH_POOL_1 <<
9145 adapter->mac_table[index].queue;
9146 break;
9147 }
9148 }
9149
9150 wr32(E1000_RAL(index), rar_low);
9151 wrfl();
9152 wr32(E1000_RAH(index), rar_high);
9153 wrfl();
9154}
9155
9156static int igb_set_vf_mac(struct igb_adapter *adapter,
9157 int vf, unsigned char *mac_addr)
9158{
9159 struct e1000_hw *hw = &adapter->hw;
9160 /* VF MAC addresses start at end of receive addresses and moves
9161 * towards the first, as a result a collision should not be possible
9162 */
9163 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
9164 unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses;
9165
9166 ether_addr_copy(vf_mac_addr, mac_addr);
9167 ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr);
9168 adapter->mac_table[rar_entry].queue = vf;
9169 adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE;
9170 igb_rar_set_index(adapter, rar_entry);
9171
9172 return 0;
9173}
9174
9175static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
9176{
9177 struct igb_adapter *adapter = netdev_priv(netdev);
9178
9179 if (vf >= adapter->vfs_allocated_count)
9180 return -EINVAL;
9181
9182 /* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC
9183 * flag and allows to overwrite the MAC via VF netdev. This
9184 * is necessary to allow libvirt a way to restore the original
9185 * MAC after unbinding vfio-pci and reloading igbvf after shutting
9186 * down a VM.
9187 */
9188 if (is_zero_ether_addr(mac)) {
9189 adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC;
9190 dev_info(&adapter->pdev->dev,
9191 "remove administratively set MAC on VF %d\n",
9192 vf);
9193 } else if (is_valid_ether_addr(mac)) {
9194 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
9195 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n",
9196 mac, vf);
9197 dev_info(&adapter->pdev->dev,
9198 "Reload the VF driver to make this change effective.");
9199 /* Generate additional warning if PF is down */
9200 if (test_bit(__IGB_DOWN, &adapter->state)) {
9201 dev_warn(&adapter->pdev->dev,
9202 "The VF MAC address has been set, but the PF device is not up.\n");
9203 dev_warn(&adapter->pdev->dev,
9204 "Bring the PF device up before attempting to use the VF device.\n");
9205 }
9206 } else {
9207 return -EINVAL;
9208 }
9209 return igb_set_vf_mac(adapter, vf, mac);
9210}
9211
9212static int igb_link_mbps(int internal_link_speed)
9213{
9214 switch (internal_link_speed) {
9215 case SPEED_100:
9216 return 100;
9217 case SPEED_1000:
9218 return 1000;
9219 default:
9220 return 0;
9221 }
9222}
9223
9224static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
9225 int link_speed)
9226{
9227 int rf_dec, rf_int;
9228 u32 bcnrc_val;
9229
9230 if (tx_rate != 0) {
9231 /* Calculate the rate factor values to set */
9232 rf_int = link_speed / tx_rate;
9233 rf_dec = (link_speed - (rf_int * tx_rate));
9234 rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) /
9235 tx_rate;
9236
9237 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
9238 bcnrc_val |= ((rf_int << E1000_RTTBCNRC_RF_INT_SHIFT) &
9239 E1000_RTTBCNRC_RF_INT_MASK);
9240 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
9241 } else {
9242 bcnrc_val = 0;
9243 }
9244
9245 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
9246 /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
9247 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
9248 */
9249 wr32(E1000_RTTBCNRM, 0x14);
9250 wr32(E1000_RTTBCNRC, bcnrc_val);
9251}
9252
9253static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
9254{
9255 int actual_link_speed, i;
9256 bool reset_rate = false;
9257
9258 /* VF TX rate limit was not set or not supported */
9259 if ((adapter->vf_rate_link_speed == 0) ||
9260 (adapter->hw.mac.type != e1000_82576))
9261 return;
9262
9263 actual_link_speed = igb_link_mbps(adapter->link_speed);
9264 if (actual_link_speed != adapter->vf_rate_link_speed) {
9265 reset_rate = true;
9266 adapter->vf_rate_link_speed = 0;
9267 dev_info(&adapter->pdev->dev,
9268 "Link speed has been changed. VF Transmit rate is disabled\n");
9269 }
9270
9271 for (i = 0; i < adapter->vfs_allocated_count; i++) {
9272 if (reset_rate)
9273 adapter->vf_data[i].tx_rate = 0;
9274
9275 igb_set_vf_rate_limit(&adapter->hw, i,
9276 adapter->vf_data[i].tx_rate,
9277 actual_link_speed);
9278 }
9279}
9280
9281static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf,
9282 int min_tx_rate, int max_tx_rate)
9283{
9284 struct igb_adapter *adapter = netdev_priv(netdev);
9285 struct e1000_hw *hw = &adapter->hw;
9286 int actual_link_speed;
9287
9288 if (hw->mac.type != e1000_82576)
9289 return -EOPNOTSUPP;
9290
9291 if (min_tx_rate)
9292 return -EINVAL;
9293
9294 actual_link_speed = igb_link_mbps(adapter->link_speed);
9295 if ((vf >= adapter->vfs_allocated_count) ||
9296 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
9297 (max_tx_rate < 0) ||
9298 (max_tx_rate > actual_link_speed))
9299 return -EINVAL;
9300
9301 adapter->vf_rate_link_speed = actual_link_speed;
9302 adapter->vf_data[vf].tx_rate = (u16)max_tx_rate;
9303 igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed);
9304
9305 return 0;
9306}
9307
9308static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
9309 bool setting)
9310{
9311 struct igb_adapter *adapter = netdev_priv(netdev);
9312 struct e1000_hw *hw = &adapter->hw;
9313 u32 reg_val, reg_offset;
9314
9315 if (!adapter->vfs_allocated_count)
9316 return -EOPNOTSUPP;
9317
9318 if (vf >= adapter->vfs_allocated_count)
9319 return -EINVAL;
9320
9321 reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
9322 reg_val = rd32(reg_offset);
9323 if (setting)
9324 reg_val |= (BIT(vf) |
9325 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9326 else
9327 reg_val &= ~(BIT(vf) |
9328 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9329 wr32(reg_offset, reg_val);
9330
9331 adapter->vf_data[vf].spoofchk_enabled = setting;
9332 return 0;
9333}
9334
9335static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting)
9336{
9337 struct igb_adapter *adapter = netdev_priv(netdev);
9338
9339 if (vf >= adapter->vfs_allocated_count)
9340 return -EINVAL;
9341 if (adapter->vf_data[vf].trusted == setting)
9342 return 0;
9343
9344 adapter->vf_data[vf].trusted = setting;
9345
9346 dev_info(&adapter->pdev->dev, "VF %u is %strusted\n",
9347 vf, setting ? "" : "not ");
9348 return 0;
9349}
9350
9351static int igb_ndo_get_vf_config(struct net_device *netdev,
9352 int vf, struct ifla_vf_info *ivi)
9353{
9354 struct igb_adapter *adapter = netdev_priv(netdev);
9355 if (vf >= adapter->vfs_allocated_count)
9356 return -EINVAL;
9357 ivi->vf = vf;
9358 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
9359 ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
9360 ivi->min_tx_rate = 0;
9361 ivi->vlan = adapter->vf_data[vf].pf_vlan;
9362 ivi->qos = adapter->vf_data[vf].pf_qos;
9363 ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
9364 ivi->trusted = adapter->vf_data[vf].trusted;
9365 return 0;
9366}
9367
9368static void igb_vmm_control(struct igb_adapter *adapter)
9369{
9370 struct e1000_hw *hw = &adapter->hw;
9371 u32 reg;
9372
9373 switch (hw->mac.type) {
9374 case e1000_82575:
9375 case e1000_i210:
9376 case e1000_i211:
9377 case e1000_i354:
9378 default:
9379 /* replication is not supported for 82575 */
9380 return;
9381 case e1000_82576:
9382 /* notify HW that the MAC is adding vlan tags */
9383 reg = rd32(E1000_DTXCTL);
9384 reg |= E1000_DTXCTL_VLAN_ADDED;
9385 wr32(E1000_DTXCTL, reg);
9386 fallthrough;
9387 case e1000_82580:
9388 /* enable replication vlan tag stripping */
9389 reg = rd32(E1000_RPLOLR);
9390 reg |= E1000_RPLOLR_STRVLAN;
9391 wr32(E1000_RPLOLR, reg);
9392 fallthrough;
9393 case e1000_i350:
9394 /* none of the above registers are supported by i350 */
9395 break;
9396 }
9397
9398 if (adapter->vfs_allocated_count) {
9399 igb_vmdq_set_loopback_pf(hw, true);
9400 igb_vmdq_set_replication_pf(hw, true);
9401 igb_vmdq_set_anti_spoofing_pf(hw, true,
9402 adapter->vfs_allocated_count);
9403 } else {
9404 igb_vmdq_set_loopback_pf(hw, false);
9405 igb_vmdq_set_replication_pf(hw, false);
9406 }
9407}
9408
9409static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
9410{
9411 struct e1000_hw *hw = &adapter->hw;
9412 u32 dmac_thr;
9413 u16 hwm;
9414
9415 if (hw->mac.type > e1000_82580) {
9416 if (adapter->flags & IGB_FLAG_DMAC) {
9417 u32 reg;
9418
9419 /* force threshold to 0. */
9420 wr32(E1000_DMCTXTH, 0);
9421
9422 /* DMA Coalescing high water mark needs to be greater
9423 * than the Rx threshold. Set hwm to PBA - max frame
9424 * size in 16B units, capping it at PBA - 6KB.
9425 */
9426 hwm = 64 * (pba - 6);
9427 reg = rd32(E1000_FCRTC);
9428 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
9429 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
9430 & E1000_FCRTC_RTH_COAL_MASK);
9431 wr32(E1000_FCRTC, reg);
9432
9433 /* Set the DMA Coalescing Rx threshold to PBA - 2 * max
9434 * frame size, capping it at PBA - 10KB.
9435 */
9436 dmac_thr = pba - 10;
9437 reg = rd32(E1000_DMACR);
9438 reg &= ~E1000_DMACR_DMACTHR_MASK;
9439 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
9440 & E1000_DMACR_DMACTHR_MASK);
9441
9442 /* transition to L0x or L1 if available..*/
9443 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
9444
9445 /* watchdog timer= +-1000 usec in 32usec intervals */
9446 reg |= (1000 >> 5);
9447
9448 /* Disable BMC-to-OS Watchdog Enable */
9449 if (hw->mac.type != e1000_i354)
9450 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
9451
9452 wr32(E1000_DMACR, reg);
9453
9454 /* no lower threshold to disable
9455 * coalescing(smart fifb)-UTRESH=0
9456 */
9457 wr32(E1000_DMCRTRH, 0);
9458
9459 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
9460
9461 wr32(E1000_DMCTLX, reg);
9462
9463 /* free space in tx packet buffer to wake from
9464 * DMA coal
9465 */
9466 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
9467 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
9468
9469 /* make low power state decision controlled
9470 * by DMA coal
9471 */
9472 reg = rd32(E1000_PCIEMISC);
9473 reg &= ~E1000_PCIEMISC_LX_DECISION;
9474 wr32(E1000_PCIEMISC, reg);
9475 } /* endif adapter->dmac is not disabled */
9476 } else if (hw->mac.type == e1000_82580) {
9477 u32 reg = rd32(E1000_PCIEMISC);
9478
9479 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
9480 wr32(E1000_DMACR, 0);
9481 }
9482}
9483
9484/**
9485 * igb_read_i2c_byte - Reads 8 bit word over I2C
9486 * @hw: pointer to hardware structure
9487 * @byte_offset: byte offset to read
9488 * @dev_addr: device address
9489 * @data: value read
9490 *
9491 * Performs byte read operation over I2C interface at
9492 * a specified device address.
9493 **/
9494s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
9495 u8 dev_addr, u8 *data)
9496{
9497 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
9498 struct i2c_client *this_client = adapter->i2c_client;
9499 s32 status;
9500 u16 swfw_mask = 0;
9501
9502 if (!this_client)
9503 return E1000_ERR_I2C;
9504
9505 swfw_mask = E1000_SWFW_PHY0_SM;
9506
9507 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
9508 return E1000_ERR_SWFW_SYNC;
9509
9510 status = i2c_smbus_read_byte_data(this_client, byte_offset);
9511 hw->mac.ops.release_swfw_sync(hw, swfw_mask);
9512
9513 if (status < 0)
9514 return E1000_ERR_I2C;
9515 else {
9516 *data = status;
9517 return 0;
9518 }
9519}
9520
9521/**
9522 * igb_write_i2c_byte - Writes 8 bit word over I2C
9523 * @hw: pointer to hardware structure
9524 * @byte_offset: byte offset to write
9525 * @dev_addr: device address
9526 * @data: value to write
9527 *
9528 * Performs byte write operation over I2C interface at
9529 * a specified device address.
9530 **/
9531s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
9532 u8 dev_addr, u8 data)
9533{
9534 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
9535 struct i2c_client *this_client = adapter->i2c_client;
9536 s32 status;
9537 u16 swfw_mask = E1000_SWFW_PHY0_SM;
9538
9539 if (!this_client)
9540 return E1000_ERR_I2C;
9541
9542 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
9543 return E1000_ERR_SWFW_SYNC;
9544 status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
9545 hw->mac.ops.release_swfw_sync(hw, swfw_mask);
9546
9547 if (status)
9548 return E1000_ERR_I2C;
9549 else
9550 return 0;
9551
9552}
9553
9554int igb_reinit_queues(struct igb_adapter *adapter)
9555{
9556 struct net_device *netdev = adapter->netdev;
9557 struct pci_dev *pdev = adapter->pdev;
9558 int err = 0;
9559
9560 if (netif_running(netdev))
9561 igb_close(netdev);
9562
9563 igb_reset_interrupt_capability(adapter);
9564
9565 if (igb_init_interrupt_scheme(adapter, true)) {
9566 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
9567 return -ENOMEM;
9568 }
9569
9570 if (netif_running(netdev))
9571 err = igb_open(netdev);
9572
9573 return err;
9574}
9575
9576static void igb_nfc_filter_exit(struct igb_adapter *adapter)
9577{
9578 struct igb_nfc_filter *rule;
9579
9580 spin_lock(&adapter->nfc_lock);
9581
9582 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
9583 igb_erase_filter(adapter, rule);
9584
9585 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
9586 igb_erase_filter(adapter, rule);
9587
9588 spin_unlock(&adapter->nfc_lock);
9589}
9590
9591static void igb_nfc_filter_restore(struct igb_adapter *adapter)
9592{
9593 struct igb_nfc_filter *rule;
9594
9595 spin_lock(&adapter->nfc_lock);
9596
9597 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
9598 igb_add_filter(adapter, rule);
9599
9600 spin_unlock(&adapter->nfc_lock);
9601}
9602/* igb_main.c */