Loading...
Note: File does not exist in v3.5.6.
1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2011 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#include <linux/module.h>
29#include <linux/types.h>
30#include <linux/init.h>
31#include <linux/bitops.h>
32#include <linux/vmalloc.h>
33#include <linux/pagemap.h>
34#include <linux/netdevice.h>
35#include <linux/ipv6.h>
36#include <linux/slab.h>
37#include <net/checksum.h>
38#include <net/ip6_checksum.h>
39#include <linux/net_tstamp.h>
40#include <linux/mii.h>
41#include <linux/ethtool.h>
42#include <linux/if_vlan.h>
43#include <linux/pci.h>
44#include <linux/pci-aspm.h>
45#include <linux/delay.h>
46#include <linux/interrupt.h>
47#include <linux/if_ether.h>
48#include <linux/aer.h>
49#include <linux/prefetch.h>
50#ifdef CONFIG_IGB_DCA
51#include <linux/dca.h>
52#endif
53#include "igb.h"
54
55#define MAJ 3
56#define MIN 0
57#define BUILD 6
58#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
59__stringify(BUILD) "-k"
60char igb_driver_name[] = "igb";
61char igb_driver_version[] = DRV_VERSION;
62static const char igb_driver_string[] =
63 "Intel(R) Gigabit Ethernet Network Driver";
64static const char igb_copyright[] = "Copyright (c) 2007-2011 Intel Corporation.";
65
66static const struct e1000_info *igb_info_tbl[] = {
67 [board_82575] = &e1000_82575_info,
68};
69
70static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
96 /* required last entry */
97 {0, }
98};
99
100MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
101
102void igb_reset(struct igb_adapter *);
103static int igb_setup_all_tx_resources(struct igb_adapter *);
104static int igb_setup_all_rx_resources(struct igb_adapter *);
105static void igb_free_all_tx_resources(struct igb_adapter *);
106static void igb_free_all_rx_resources(struct igb_adapter *);
107static void igb_setup_mrqc(struct igb_adapter *);
108static int igb_probe(struct pci_dev *, const struct pci_device_id *);
109static void __devexit igb_remove(struct pci_dev *pdev);
110static void igb_init_hw_timer(struct igb_adapter *adapter);
111static int igb_sw_init(struct igb_adapter *);
112static int igb_open(struct net_device *);
113static int igb_close(struct net_device *);
114static void igb_configure_tx(struct igb_adapter *);
115static void igb_configure_rx(struct igb_adapter *);
116static void igb_clean_all_tx_rings(struct igb_adapter *);
117static void igb_clean_all_rx_rings(struct igb_adapter *);
118static void igb_clean_tx_ring(struct igb_ring *);
119static void igb_clean_rx_ring(struct igb_ring *);
120static void igb_set_rx_mode(struct net_device *);
121static void igb_update_phy_info(unsigned long);
122static void igb_watchdog(unsigned long);
123static void igb_watchdog_task(struct work_struct *);
124static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb, struct net_device *);
125static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
126 struct rtnl_link_stats64 *stats);
127static int igb_change_mtu(struct net_device *, int);
128static int igb_set_mac(struct net_device *, void *);
129static void igb_set_uta(struct igb_adapter *adapter);
130static irqreturn_t igb_intr(int irq, void *);
131static irqreturn_t igb_intr_msi(int irq, void *);
132static irqreturn_t igb_msix_other(int irq, void *);
133static irqreturn_t igb_msix_ring(int irq, void *);
134#ifdef CONFIG_IGB_DCA
135static void igb_update_dca(struct igb_q_vector *);
136static void igb_setup_dca(struct igb_adapter *);
137#endif /* CONFIG_IGB_DCA */
138static bool igb_clean_tx_irq(struct igb_q_vector *);
139static int igb_poll(struct napi_struct *, int);
140static bool igb_clean_rx_irq_adv(struct igb_q_vector *, int *, int);
141static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
142static void igb_tx_timeout(struct net_device *);
143static void igb_reset_task(struct work_struct *);
144static void igb_vlan_mode(struct net_device *netdev, u32 features);
145static void igb_vlan_rx_add_vid(struct net_device *, u16);
146static void igb_vlan_rx_kill_vid(struct net_device *, u16);
147static void igb_restore_vlan(struct igb_adapter *);
148static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
149static void igb_ping_all_vfs(struct igb_adapter *);
150static void igb_msg_task(struct igb_adapter *);
151static void igb_vmm_control(struct igb_adapter *);
152static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
153static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
154static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
155static int igb_ndo_set_vf_vlan(struct net_device *netdev,
156 int vf, u16 vlan, u8 qos);
157static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
158static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
159 struct ifla_vf_info *ivi);
160static void igb_check_vf_rate_limit(struct igb_adapter *);
161
162#ifdef CONFIG_PM
163static int igb_suspend(struct pci_dev *, pm_message_t);
164static int igb_resume(struct pci_dev *);
165#endif
166static void igb_shutdown(struct pci_dev *);
167#ifdef CONFIG_IGB_DCA
168static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
169static struct notifier_block dca_notifier = {
170 .notifier_call = igb_notify_dca,
171 .next = NULL,
172 .priority = 0
173};
174#endif
175#ifdef CONFIG_NET_POLL_CONTROLLER
176/* for netdump / net console */
177static void igb_netpoll(struct net_device *);
178#endif
179#ifdef CONFIG_PCI_IOV
180static unsigned int max_vfs = 0;
181module_param(max_vfs, uint, 0);
182MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
183 "per physical function");
184#endif /* CONFIG_PCI_IOV */
185
186static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
187 pci_channel_state_t);
188static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
189static void igb_io_resume(struct pci_dev *);
190
191static struct pci_error_handlers igb_err_handler = {
192 .error_detected = igb_io_error_detected,
193 .slot_reset = igb_io_slot_reset,
194 .resume = igb_io_resume,
195};
196
197
198static struct pci_driver igb_driver = {
199 .name = igb_driver_name,
200 .id_table = igb_pci_tbl,
201 .probe = igb_probe,
202 .remove = __devexit_p(igb_remove),
203#ifdef CONFIG_PM
204 /* Power Management Hooks */
205 .suspend = igb_suspend,
206 .resume = igb_resume,
207#endif
208 .shutdown = igb_shutdown,
209 .err_handler = &igb_err_handler
210};
211
212MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
213MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
214MODULE_LICENSE("GPL");
215MODULE_VERSION(DRV_VERSION);
216
217struct igb_reg_info {
218 u32 ofs;
219 char *name;
220};
221
222static const struct igb_reg_info igb_reg_info_tbl[] = {
223
224 /* General Registers */
225 {E1000_CTRL, "CTRL"},
226 {E1000_STATUS, "STATUS"},
227 {E1000_CTRL_EXT, "CTRL_EXT"},
228
229 /* Interrupt Registers */
230 {E1000_ICR, "ICR"},
231
232 /* RX Registers */
233 {E1000_RCTL, "RCTL"},
234 {E1000_RDLEN(0), "RDLEN"},
235 {E1000_RDH(0), "RDH"},
236 {E1000_RDT(0), "RDT"},
237 {E1000_RXDCTL(0), "RXDCTL"},
238 {E1000_RDBAL(0), "RDBAL"},
239 {E1000_RDBAH(0), "RDBAH"},
240
241 /* TX Registers */
242 {E1000_TCTL, "TCTL"},
243 {E1000_TDBAL(0), "TDBAL"},
244 {E1000_TDBAH(0), "TDBAH"},
245 {E1000_TDLEN(0), "TDLEN"},
246 {E1000_TDH(0), "TDH"},
247 {E1000_TDT(0), "TDT"},
248 {E1000_TXDCTL(0), "TXDCTL"},
249 {E1000_TDFH, "TDFH"},
250 {E1000_TDFT, "TDFT"},
251 {E1000_TDFHS, "TDFHS"},
252 {E1000_TDFPC, "TDFPC"},
253
254 /* List Terminator */
255 {}
256};
257
258/*
259 * igb_regdump - register printout routine
260 */
261static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
262{
263 int n = 0;
264 char rname[16];
265 u32 regs[8];
266
267 switch (reginfo->ofs) {
268 case E1000_RDLEN(0):
269 for (n = 0; n < 4; n++)
270 regs[n] = rd32(E1000_RDLEN(n));
271 break;
272 case E1000_RDH(0):
273 for (n = 0; n < 4; n++)
274 regs[n] = rd32(E1000_RDH(n));
275 break;
276 case E1000_RDT(0):
277 for (n = 0; n < 4; n++)
278 regs[n] = rd32(E1000_RDT(n));
279 break;
280 case E1000_RXDCTL(0):
281 for (n = 0; n < 4; n++)
282 regs[n] = rd32(E1000_RXDCTL(n));
283 break;
284 case E1000_RDBAL(0):
285 for (n = 0; n < 4; n++)
286 regs[n] = rd32(E1000_RDBAL(n));
287 break;
288 case E1000_RDBAH(0):
289 for (n = 0; n < 4; n++)
290 regs[n] = rd32(E1000_RDBAH(n));
291 break;
292 case E1000_TDBAL(0):
293 for (n = 0; n < 4; n++)
294 regs[n] = rd32(E1000_RDBAL(n));
295 break;
296 case E1000_TDBAH(0):
297 for (n = 0; n < 4; n++)
298 regs[n] = rd32(E1000_TDBAH(n));
299 break;
300 case E1000_TDLEN(0):
301 for (n = 0; n < 4; n++)
302 regs[n] = rd32(E1000_TDLEN(n));
303 break;
304 case E1000_TDH(0):
305 for (n = 0; n < 4; n++)
306 regs[n] = rd32(E1000_TDH(n));
307 break;
308 case E1000_TDT(0):
309 for (n = 0; n < 4; n++)
310 regs[n] = rd32(E1000_TDT(n));
311 break;
312 case E1000_TXDCTL(0):
313 for (n = 0; n < 4; n++)
314 regs[n] = rd32(E1000_TXDCTL(n));
315 break;
316 default:
317 printk(KERN_INFO "%-15s %08x\n",
318 reginfo->name, rd32(reginfo->ofs));
319 return;
320 }
321
322 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
323 printk(KERN_INFO "%-15s ", rname);
324 for (n = 0; n < 4; n++)
325 printk(KERN_CONT "%08x ", regs[n]);
326 printk(KERN_CONT "\n");
327}
328
329/*
330 * igb_dump - Print registers, tx-rings and rx-rings
331 */
332static void igb_dump(struct igb_adapter *adapter)
333{
334 struct net_device *netdev = adapter->netdev;
335 struct e1000_hw *hw = &adapter->hw;
336 struct igb_reg_info *reginfo;
337 int n = 0;
338 struct igb_ring *tx_ring;
339 union e1000_adv_tx_desc *tx_desc;
340 struct my_u0 { u64 a; u64 b; } *u0;
341 struct igb_buffer *buffer_info;
342 struct igb_ring *rx_ring;
343 union e1000_adv_rx_desc *rx_desc;
344 u32 staterr;
345 int i = 0;
346
347 if (!netif_msg_hw(adapter))
348 return;
349
350 /* Print netdevice Info */
351 if (netdev) {
352 dev_info(&adapter->pdev->dev, "Net device Info\n");
353 printk(KERN_INFO "Device Name state "
354 "trans_start last_rx\n");
355 printk(KERN_INFO "%-15s %016lX %016lX %016lX\n",
356 netdev->name,
357 netdev->state,
358 netdev->trans_start,
359 netdev->last_rx);
360 }
361
362 /* Print Registers */
363 dev_info(&adapter->pdev->dev, "Register Dump\n");
364 printk(KERN_INFO " Register Name Value\n");
365 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
366 reginfo->name; reginfo++) {
367 igb_regdump(hw, reginfo);
368 }
369
370 /* Print TX Ring Summary */
371 if (!netdev || !netif_running(netdev))
372 goto exit;
373
374 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
375 printk(KERN_INFO "Queue [NTU] [NTC] [bi(ntc)->dma ]"
376 " leng ntw timestamp\n");
377 for (n = 0; n < adapter->num_tx_queues; n++) {
378 tx_ring = adapter->tx_ring[n];
379 buffer_info = &tx_ring->buffer_info[tx_ring->next_to_clean];
380 printk(KERN_INFO " %5d %5X %5X %016llX %04X %3X %016llX\n",
381 n, tx_ring->next_to_use, tx_ring->next_to_clean,
382 (u64)buffer_info->dma,
383 buffer_info->length,
384 buffer_info->next_to_watch,
385 (u64)buffer_info->time_stamp);
386 }
387
388 /* Print TX Rings */
389 if (!netif_msg_tx_done(adapter))
390 goto rx_ring_summary;
391
392 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
393
394 /* Transmit Descriptor Formats
395 *
396 * Advanced Transmit Descriptor
397 * +--------------------------------------------------------------+
398 * 0 | Buffer Address [63:0] |
399 * +--------------------------------------------------------------+
400 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
401 * +--------------------------------------------------------------+
402 * 63 46 45 40 39 38 36 35 32 31 24 15 0
403 */
404
405 for (n = 0; n < adapter->num_tx_queues; n++) {
406 tx_ring = adapter->tx_ring[n];
407 printk(KERN_INFO "------------------------------------\n");
408 printk(KERN_INFO "TX QUEUE INDEX = %d\n", tx_ring->queue_index);
409 printk(KERN_INFO "------------------------------------\n");
410 printk(KERN_INFO "T [desc] [address 63:0 ] "
411 "[PlPOCIStDDM Ln] [bi->dma ] "
412 "leng ntw timestamp bi->skb\n");
413
414 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
415 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
416 buffer_info = &tx_ring->buffer_info[i];
417 u0 = (struct my_u0 *)tx_desc;
418 printk(KERN_INFO "T [0x%03X] %016llX %016llX %016llX"
419 " %04X %3X %016llX %p", i,
420 le64_to_cpu(u0->a),
421 le64_to_cpu(u0->b),
422 (u64)buffer_info->dma,
423 buffer_info->length,
424 buffer_info->next_to_watch,
425 (u64)buffer_info->time_stamp,
426 buffer_info->skb);
427 if (i == tx_ring->next_to_use &&
428 i == tx_ring->next_to_clean)
429 printk(KERN_CONT " NTC/U\n");
430 else if (i == tx_ring->next_to_use)
431 printk(KERN_CONT " NTU\n");
432 else if (i == tx_ring->next_to_clean)
433 printk(KERN_CONT " NTC\n");
434 else
435 printk(KERN_CONT "\n");
436
437 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
438 print_hex_dump(KERN_INFO, "",
439 DUMP_PREFIX_ADDRESS,
440 16, 1, phys_to_virt(buffer_info->dma),
441 buffer_info->length, true);
442 }
443 }
444
445 /* Print RX Rings Summary */
446rx_ring_summary:
447 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
448 printk(KERN_INFO "Queue [NTU] [NTC]\n");
449 for (n = 0; n < adapter->num_rx_queues; n++) {
450 rx_ring = adapter->rx_ring[n];
451 printk(KERN_INFO " %5d %5X %5X\n", n,
452 rx_ring->next_to_use, rx_ring->next_to_clean);
453 }
454
455 /* Print RX Rings */
456 if (!netif_msg_rx_status(adapter))
457 goto exit;
458
459 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
460
461 /* Advanced Receive Descriptor (Read) Format
462 * 63 1 0
463 * +-----------------------------------------------------+
464 * 0 | Packet Buffer Address [63:1] |A0/NSE|
465 * +----------------------------------------------+------+
466 * 8 | Header Buffer Address [63:1] | DD |
467 * +-----------------------------------------------------+
468 *
469 *
470 * Advanced Receive Descriptor (Write-Back) Format
471 *
472 * 63 48 47 32 31 30 21 20 17 16 4 3 0
473 * +------------------------------------------------------+
474 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
475 * | Checksum Ident | | | | Type | Type |
476 * +------------------------------------------------------+
477 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
478 * +------------------------------------------------------+
479 * 63 48 47 32 31 20 19 0
480 */
481
482 for (n = 0; n < adapter->num_rx_queues; n++) {
483 rx_ring = adapter->rx_ring[n];
484 printk(KERN_INFO "------------------------------------\n");
485 printk(KERN_INFO "RX QUEUE INDEX = %d\n", rx_ring->queue_index);
486 printk(KERN_INFO "------------------------------------\n");
487 printk(KERN_INFO "R [desc] [ PktBuf A0] "
488 "[ HeadBuf DD] [bi->dma ] [bi->skb] "
489 "<-- Adv Rx Read format\n");
490 printk(KERN_INFO "RWB[desc] [PcsmIpSHl PtRs] "
491 "[vl er S cks ln] ---------------- [bi->skb] "
492 "<-- Adv Rx Write-Back format\n");
493
494 for (i = 0; i < rx_ring->count; i++) {
495 buffer_info = &rx_ring->buffer_info[i];
496 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
497 u0 = (struct my_u0 *)rx_desc;
498 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
499 if (staterr & E1000_RXD_STAT_DD) {
500 /* Descriptor Done */
501 printk(KERN_INFO "RWB[0x%03X] %016llX "
502 "%016llX ---------------- %p", i,
503 le64_to_cpu(u0->a),
504 le64_to_cpu(u0->b),
505 buffer_info->skb);
506 } else {
507 printk(KERN_INFO "R [0x%03X] %016llX "
508 "%016llX %016llX %p", i,
509 le64_to_cpu(u0->a),
510 le64_to_cpu(u0->b),
511 (u64)buffer_info->dma,
512 buffer_info->skb);
513
514 if (netif_msg_pktdata(adapter)) {
515 print_hex_dump(KERN_INFO, "",
516 DUMP_PREFIX_ADDRESS,
517 16, 1,
518 phys_to_virt(buffer_info->dma),
519 rx_ring->rx_buffer_len, true);
520 if (rx_ring->rx_buffer_len
521 < IGB_RXBUFFER_1024)
522 print_hex_dump(KERN_INFO, "",
523 DUMP_PREFIX_ADDRESS,
524 16, 1,
525 phys_to_virt(
526 buffer_info->page_dma +
527 buffer_info->page_offset),
528 PAGE_SIZE/2, true);
529 }
530 }
531
532 if (i == rx_ring->next_to_use)
533 printk(KERN_CONT " NTU\n");
534 else if (i == rx_ring->next_to_clean)
535 printk(KERN_CONT " NTC\n");
536 else
537 printk(KERN_CONT "\n");
538
539 }
540 }
541
542exit:
543 return;
544}
545
546
547/**
548 * igb_read_clock - read raw cycle counter (to be used by time counter)
549 */
550static cycle_t igb_read_clock(const struct cyclecounter *tc)
551{
552 struct igb_adapter *adapter =
553 container_of(tc, struct igb_adapter, cycles);
554 struct e1000_hw *hw = &adapter->hw;
555 u64 stamp = 0;
556 int shift = 0;
557
558 /*
559 * The timestamp latches on lowest register read. For the 82580
560 * the lowest register is SYSTIMR instead of SYSTIML. However we never
561 * adjusted TIMINCA so SYSTIMR will just read as all 0s so ignore it.
562 */
563 if (hw->mac.type == e1000_82580) {
564 stamp = rd32(E1000_SYSTIMR) >> 8;
565 shift = IGB_82580_TSYNC_SHIFT;
566 }
567
568 stamp |= (u64)rd32(E1000_SYSTIML) << shift;
569 stamp |= (u64)rd32(E1000_SYSTIMH) << (shift + 32);
570 return stamp;
571}
572
573/**
574 * igb_get_hw_dev - return device
575 * used by hardware layer to print debugging information
576 **/
577struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
578{
579 struct igb_adapter *adapter = hw->back;
580 return adapter->netdev;
581}
582
583/**
584 * igb_init_module - Driver Registration Routine
585 *
586 * igb_init_module is the first routine called when the driver is
587 * loaded. All it does is register with the PCI subsystem.
588 **/
589static int __init igb_init_module(void)
590{
591 int ret;
592 printk(KERN_INFO "%s - version %s\n",
593 igb_driver_string, igb_driver_version);
594
595 printk(KERN_INFO "%s\n", igb_copyright);
596
597#ifdef CONFIG_IGB_DCA
598 dca_register_notify(&dca_notifier);
599#endif
600 ret = pci_register_driver(&igb_driver);
601 return ret;
602}
603
604module_init(igb_init_module);
605
606/**
607 * igb_exit_module - Driver Exit Cleanup Routine
608 *
609 * igb_exit_module is called just before the driver is removed
610 * from memory.
611 **/
612static void __exit igb_exit_module(void)
613{
614#ifdef CONFIG_IGB_DCA
615 dca_unregister_notify(&dca_notifier);
616#endif
617 pci_unregister_driver(&igb_driver);
618}
619
620module_exit(igb_exit_module);
621
622#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
623/**
624 * igb_cache_ring_register - Descriptor ring to register mapping
625 * @adapter: board private structure to initialize
626 *
627 * Once we know the feature-set enabled for the device, we'll cache
628 * the register offset the descriptor ring is assigned to.
629 **/
630static void igb_cache_ring_register(struct igb_adapter *adapter)
631{
632 int i = 0, j = 0;
633 u32 rbase_offset = adapter->vfs_allocated_count;
634
635 switch (adapter->hw.mac.type) {
636 case e1000_82576:
637 /* The queues are allocated for virtualization such that VF 0
638 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
639 * In order to avoid collision we start at the first free queue
640 * and continue consuming queues in the same sequence
641 */
642 if (adapter->vfs_allocated_count) {
643 for (; i < adapter->rss_queues; i++)
644 adapter->rx_ring[i]->reg_idx = rbase_offset +
645 Q_IDX_82576(i);
646 }
647 case e1000_82575:
648 case e1000_82580:
649 case e1000_i350:
650 default:
651 for (; i < adapter->num_rx_queues; i++)
652 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
653 for (; j < adapter->num_tx_queues; j++)
654 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
655 break;
656 }
657}
658
659static void igb_free_queues(struct igb_adapter *adapter)
660{
661 int i;
662
663 for (i = 0; i < adapter->num_tx_queues; i++) {
664 kfree(adapter->tx_ring[i]);
665 adapter->tx_ring[i] = NULL;
666 }
667 for (i = 0; i < adapter->num_rx_queues; i++) {
668 kfree(adapter->rx_ring[i]);
669 adapter->rx_ring[i] = NULL;
670 }
671 adapter->num_rx_queues = 0;
672 adapter->num_tx_queues = 0;
673}
674
675/**
676 * igb_alloc_queues - Allocate memory for all rings
677 * @adapter: board private structure to initialize
678 *
679 * We allocate one ring per queue at run-time since we don't know the
680 * number of queues at compile-time.
681 **/
682static int igb_alloc_queues(struct igb_adapter *adapter)
683{
684 struct igb_ring *ring;
685 int i;
686
687 for (i = 0; i < adapter->num_tx_queues; i++) {
688 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
689 if (!ring)
690 goto err;
691 ring->count = adapter->tx_ring_count;
692 ring->queue_index = i;
693 ring->dev = &adapter->pdev->dev;
694 ring->netdev = adapter->netdev;
695 /* For 82575, context index must be unique per ring. */
696 if (adapter->hw.mac.type == e1000_82575)
697 ring->flags = IGB_RING_FLAG_TX_CTX_IDX;
698 adapter->tx_ring[i] = ring;
699 }
700
701 for (i = 0; i < adapter->num_rx_queues; i++) {
702 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
703 if (!ring)
704 goto err;
705 ring->count = adapter->rx_ring_count;
706 ring->queue_index = i;
707 ring->dev = &adapter->pdev->dev;
708 ring->netdev = adapter->netdev;
709 ring->rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
710 ring->flags = IGB_RING_FLAG_RX_CSUM; /* enable rx checksum */
711 /* set flag indicating ring supports SCTP checksum offload */
712 if (adapter->hw.mac.type >= e1000_82576)
713 ring->flags |= IGB_RING_FLAG_RX_SCTP_CSUM;
714 adapter->rx_ring[i] = ring;
715 }
716
717 igb_cache_ring_register(adapter);
718
719 return 0;
720
721err:
722 igb_free_queues(adapter);
723
724 return -ENOMEM;
725}
726
727#define IGB_N0_QUEUE -1
728static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
729{
730 u32 msixbm = 0;
731 struct igb_adapter *adapter = q_vector->adapter;
732 struct e1000_hw *hw = &adapter->hw;
733 u32 ivar, index;
734 int rx_queue = IGB_N0_QUEUE;
735 int tx_queue = IGB_N0_QUEUE;
736
737 if (q_vector->rx_ring)
738 rx_queue = q_vector->rx_ring->reg_idx;
739 if (q_vector->tx_ring)
740 tx_queue = q_vector->tx_ring->reg_idx;
741
742 switch (hw->mac.type) {
743 case e1000_82575:
744 /* The 82575 assigns vectors using a bitmask, which matches the
745 bitmask for the EICR/EIMS/EIMC registers. To assign one
746 or more queues to a vector, we write the appropriate bits
747 into the MSIXBM register for that vector. */
748 if (rx_queue > IGB_N0_QUEUE)
749 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
750 if (tx_queue > IGB_N0_QUEUE)
751 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
752 if (!adapter->msix_entries && msix_vector == 0)
753 msixbm |= E1000_EIMS_OTHER;
754 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
755 q_vector->eims_value = msixbm;
756 break;
757 case e1000_82576:
758 /* 82576 uses a table-based method for assigning vectors.
759 Each queue has a single entry in the table to which we write
760 a vector number along with a "valid" bit. Sadly, the layout
761 of the table is somewhat counterintuitive. */
762 if (rx_queue > IGB_N0_QUEUE) {
763 index = (rx_queue & 0x7);
764 ivar = array_rd32(E1000_IVAR0, index);
765 if (rx_queue < 8) {
766 /* vector goes into low byte of register */
767 ivar = ivar & 0xFFFFFF00;
768 ivar |= msix_vector | E1000_IVAR_VALID;
769 } else {
770 /* vector goes into third byte of register */
771 ivar = ivar & 0xFF00FFFF;
772 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
773 }
774 array_wr32(E1000_IVAR0, index, ivar);
775 }
776 if (tx_queue > IGB_N0_QUEUE) {
777 index = (tx_queue & 0x7);
778 ivar = array_rd32(E1000_IVAR0, index);
779 if (tx_queue < 8) {
780 /* vector goes into second byte of register */
781 ivar = ivar & 0xFFFF00FF;
782 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
783 } else {
784 /* vector goes into high byte of register */
785 ivar = ivar & 0x00FFFFFF;
786 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
787 }
788 array_wr32(E1000_IVAR0, index, ivar);
789 }
790 q_vector->eims_value = 1 << msix_vector;
791 break;
792 case e1000_82580:
793 case e1000_i350:
794 /* 82580 uses the same table-based approach as 82576 but has fewer
795 entries as a result we carry over for queues greater than 4. */
796 if (rx_queue > IGB_N0_QUEUE) {
797 index = (rx_queue >> 1);
798 ivar = array_rd32(E1000_IVAR0, index);
799 if (rx_queue & 0x1) {
800 /* vector goes into third byte of register */
801 ivar = ivar & 0xFF00FFFF;
802 ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
803 } else {
804 /* vector goes into low byte of register */
805 ivar = ivar & 0xFFFFFF00;
806 ivar |= msix_vector | E1000_IVAR_VALID;
807 }
808 array_wr32(E1000_IVAR0, index, ivar);
809 }
810 if (tx_queue > IGB_N0_QUEUE) {
811 index = (tx_queue >> 1);
812 ivar = array_rd32(E1000_IVAR0, index);
813 if (tx_queue & 0x1) {
814 /* vector goes into high byte of register */
815 ivar = ivar & 0x00FFFFFF;
816 ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
817 } else {
818 /* vector goes into second byte of register */
819 ivar = ivar & 0xFFFF00FF;
820 ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
821 }
822 array_wr32(E1000_IVAR0, index, ivar);
823 }
824 q_vector->eims_value = 1 << msix_vector;
825 break;
826 default:
827 BUG();
828 break;
829 }
830
831 /* add q_vector eims value to global eims_enable_mask */
832 adapter->eims_enable_mask |= q_vector->eims_value;
833
834 /* configure q_vector to set itr on first interrupt */
835 q_vector->set_itr = 1;
836}
837
838/**
839 * igb_configure_msix - Configure MSI-X hardware
840 *
841 * igb_configure_msix sets up the hardware to properly
842 * generate MSI-X interrupts.
843 **/
844static void igb_configure_msix(struct igb_adapter *adapter)
845{
846 u32 tmp;
847 int i, vector = 0;
848 struct e1000_hw *hw = &adapter->hw;
849
850 adapter->eims_enable_mask = 0;
851
852 /* set vector for other causes, i.e. link changes */
853 switch (hw->mac.type) {
854 case e1000_82575:
855 tmp = rd32(E1000_CTRL_EXT);
856 /* enable MSI-X PBA support*/
857 tmp |= E1000_CTRL_EXT_PBA_CLR;
858
859 /* Auto-Mask interrupts upon ICR read. */
860 tmp |= E1000_CTRL_EXT_EIAME;
861 tmp |= E1000_CTRL_EXT_IRCA;
862
863 wr32(E1000_CTRL_EXT, tmp);
864
865 /* enable msix_other interrupt */
866 array_wr32(E1000_MSIXBM(0), vector++,
867 E1000_EIMS_OTHER);
868 adapter->eims_other = E1000_EIMS_OTHER;
869
870 break;
871
872 case e1000_82576:
873 case e1000_82580:
874 case e1000_i350:
875 /* Turn on MSI-X capability first, or our settings
876 * won't stick. And it will take days to debug. */
877 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
878 E1000_GPIE_PBA | E1000_GPIE_EIAME |
879 E1000_GPIE_NSICR);
880
881 /* enable msix_other interrupt */
882 adapter->eims_other = 1 << vector;
883 tmp = (vector++ | E1000_IVAR_VALID) << 8;
884
885 wr32(E1000_IVAR_MISC, tmp);
886 break;
887 default:
888 /* do nothing, since nothing else supports MSI-X */
889 break;
890 } /* switch (hw->mac.type) */
891
892 adapter->eims_enable_mask |= adapter->eims_other;
893
894 for (i = 0; i < adapter->num_q_vectors; i++)
895 igb_assign_vector(adapter->q_vector[i], vector++);
896
897 wrfl();
898}
899
900/**
901 * igb_request_msix - Initialize MSI-X interrupts
902 *
903 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
904 * kernel.
905 **/
906static int igb_request_msix(struct igb_adapter *adapter)
907{
908 struct net_device *netdev = adapter->netdev;
909 struct e1000_hw *hw = &adapter->hw;
910 int i, err = 0, vector = 0;
911
912 err = request_irq(adapter->msix_entries[vector].vector,
913 igb_msix_other, 0, netdev->name, adapter);
914 if (err)
915 goto out;
916 vector++;
917
918 for (i = 0; i < adapter->num_q_vectors; i++) {
919 struct igb_q_vector *q_vector = adapter->q_vector[i];
920
921 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
922
923 if (q_vector->rx_ring && q_vector->tx_ring)
924 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
925 q_vector->rx_ring->queue_index);
926 else if (q_vector->tx_ring)
927 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
928 q_vector->tx_ring->queue_index);
929 else if (q_vector->rx_ring)
930 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
931 q_vector->rx_ring->queue_index);
932 else
933 sprintf(q_vector->name, "%s-unused", netdev->name);
934
935 err = request_irq(adapter->msix_entries[vector].vector,
936 igb_msix_ring, 0, q_vector->name,
937 q_vector);
938 if (err)
939 goto out;
940 vector++;
941 }
942
943 igb_configure_msix(adapter);
944 return 0;
945out:
946 return err;
947}
948
949static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
950{
951 if (adapter->msix_entries) {
952 pci_disable_msix(adapter->pdev);
953 kfree(adapter->msix_entries);
954 adapter->msix_entries = NULL;
955 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
956 pci_disable_msi(adapter->pdev);
957 }
958}
959
960/**
961 * igb_free_q_vectors - Free memory allocated for interrupt vectors
962 * @adapter: board private structure to initialize
963 *
964 * This function frees the memory allocated to the q_vectors. In addition if
965 * NAPI is enabled it will delete any references to the NAPI struct prior
966 * to freeing the q_vector.
967 **/
968static void igb_free_q_vectors(struct igb_adapter *adapter)
969{
970 int v_idx;
971
972 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
973 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
974 adapter->q_vector[v_idx] = NULL;
975 if (!q_vector)
976 continue;
977 netif_napi_del(&q_vector->napi);
978 kfree(q_vector);
979 }
980 adapter->num_q_vectors = 0;
981}
982
983/**
984 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
985 *
986 * This function resets the device so that it has 0 rx queues, tx queues, and
987 * MSI-X interrupts allocated.
988 */
989static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
990{
991 igb_free_queues(adapter);
992 igb_free_q_vectors(adapter);
993 igb_reset_interrupt_capability(adapter);
994}
995
996/**
997 * igb_set_interrupt_capability - set MSI or MSI-X if supported
998 *
999 * Attempt to configure interrupts using the best available
1000 * capabilities of the hardware and kernel.
1001 **/
1002static int igb_set_interrupt_capability(struct igb_adapter *adapter)
1003{
1004 int err;
1005 int numvecs, i;
1006
1007 /* Number of supported queues. */
1008 adapter->num_rx_queues = adapter->rss_queues;
1009 if (adapter->vfs_allocated_count)
1010 adapter->num_tx_queues = 1;
1011 else
1012 adapter->num_tx_queues = adapter->rss_queues;
1013
1014 /* start with one vector for every rx queue */
1015 numvecs = adapter->num_rx_queues;
1016
1017 /* if tx handler is separate add 1 for every tx queue */
1018 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1019 numvecs += adapter->num_tx_queues;
1020
1021 /* store the number of vectors reserved for queues */
1022 adapter->num_q_vectors = numvecs;
1023
1024 /* add 1 vector for link status interrupts */
1025 numvecs++;
1026 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1027 GFP_KERNEL);
1028 if (!adapter->msix_entries)
1029 goto msi_only;
1030
1031 for (i = 0; i < numvecs; i++)
1032 adapter->msix_entries[i].entry = i;
1033
1034 err = pci_enable_msix(adapter->pdev,
1035 adapter->msix_entries,
1036 numvecs);
1037 if (err == 0)
1038 goto out;
1039
1040 igb_reset_interrupt_capability(adapter);
1041
1042 /* If we can't do MSI-X, try MSI */
1043msi_only:
1044#ifdef CONFIG_PCI_IOV
1045 /* disable SR-IOV for non MSI-X configurations */
1046 if (adapter->vf_data) {
1047 struct e1000_hw *hw = &adapter->hw;
1048 /* disable iov and allow time for transactions to clear */
1049 pci_disable_sriov(adapter->pdev);
1050 msleep(500);
1051
1052 kfree(adapter->vf_data);
1053 adapter->vf_data = NULL;
1054 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1055 wrfl();
1056 msleep(100);
1057 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1058 }
1059#endif
1060 adapter->vfs_allocated_count = 0;
1061 adapter->rss_queues = 1;
1062 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1063 adapter->num_rx_queues = 1;
1064 adapter->num_tx_queues = 1;
1065 adapter->num_q_vectors = 1;
1066 if (!pci_enable_msi(adapter->pdev))
1067 adapter->flags |= IGB_FLAG_HAS_MSI;
1068out:
1069 /* Notify the stack of the (possibly) reduced queue counts. */
1070 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1071 return netif_set_real_num_rx_queues(adapter->netdev,
1072 adapter->num_rx_queues);
1073}
1074
1075/**
1076 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1077 * @adapter: board private structure to initialize
1078 *
1079 * We allocate one q_vector per queue interrupt. If allocation fails we
1080 * return -ENOMEM.
1081 **/
1082static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1083{
1084 struct igb_q_vector *q_vector;
1085 struct e1000_hw *hw = &adapter->hw;
1086 int v_idx;
1087
1088 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1089 q_vector = kzalloc(sizeof(struct igb_q_vector), GFP_KERNEL);
1090 if (!q_vector)
1091 goto err_out;
1092 q_vector->adapter = adapter;
1093 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1094 q_vector->itr_val = IGB_START_ITR;
1095 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1096 adapter->q_vector[v_idx] = q_vector;
1097 }
1098 return 0;
1099
1100err_out:
1101 igb_free_q_vectors(adapter);
1102 return -ENOMEM;
1103}
1104
1105static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1106 int ring_idx, int v_idx)
1107{
1108 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1109
1110 q_vector->rx_ring = adapter->rx_ring[ring_idx];
1111 q_vector->rx_ring->q_vector = q_vector;
1112 q_vector->itr_val = adapter->rx_itr_setting;
1113 if (q_vector->itr_val && q_vector->itr_val <= 3)
1114 q_vector->itr_val = IGB_START_ITR;
1115}
1116
1117static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1118 int ring_idx, int v_idx)
1119{
1120 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1121
1122 q_vector->tx_ring = adapter->tx_ring[ring_idx];
1123 q_vector->tx_ring->q_vector = q_vector;
1124 q_vector->itr_val = adapter->tx_itr_setting;
1125 if (q_vector->itr_val && q_vector->itr_val <= 3)
1126 q_vector->itr_val = IGB_START_ITR;
1127}
1128
1129/**
1130 * igb_map_ring_to_vector - maps allocated queues to vectors
1131 *
1132 * This function maps the recently allocated queues to vectors.
1133 **/
1134static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1135{
1136 int i;
1137 int v_idx = 0;
1138
1139 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1140 (adapter->num_q_vectors < adapter->num_tx_queues))
1141 return -ENOMEM;
1142
1143 if (adapter->num_q_vectors >=
1144 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1145 for (i = 0; i < adapter->num_rx_queues; i++)
1146 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1147 for (i = 0; i < adapter->num_tx_queues; i++)
1148 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1149 } else {
1150 for (i = 0; i < adapter->num_rx_queues; i++) {
1151 if (i < adapter->num_tx_queues)
1152 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1153 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1154 }
1155 for (; i < adapter->num_tx_queues; i++)
1156 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1157 }
1158 return 0;
1159}
1160
1161/**
1162 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1163 *
1164 * This function initializes the interrupts and allocates all of the queues.
1165 **/
1166static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1167{
1168 struct pci_dev *pdev = adapter->pdev;
1169 int err;
1170
1171 err = igb_set_interrupt_capability(adapter);
1172 if (err)
1173 return err;
1174
1175 err = igb_alloc_q_vectors(adapter);
1176 if (err) {
1177 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1178 goto err_alloc_q_vectors;
1179 }
1180
1181 err = igb_alloc_queues(adapter);
1182 if (err) {
1183 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1184 goto err_alloc_queues;
1185 }
1186
1187 err = igb_map_ring_to_vector(adapter);
1188 if (err) {
1189 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1190 goto err_map_queues;
1191 }
1192
1193
1194 return 0;
1195err_map_queues:
1196 igb_free_queues(adapter);
1197err_alloc_queues:
1198 igb_free_q_vectors(adapter);
1199err_alloc_q_vectors:
1200 igb_reset_interrupt_capability(adapter);
1201 return err;
1202}
1203
1204/**
1205 * igb_request_irq - initialize interrupts
1206 *
1207 * Attempts to configure interrupts using the best available
1208 * capabilities of the hardware and kernel.
1209 **/
1210static int igb_request_irq(struct igb_adapter *adapter)
1211{
1212 struct net_device *netdev = adapter->netdev;
1213 struct pci_dev *pdev = adapter->pdev;
1214 int err = 0;
1215
1216 if (adapter->msix_entries) {
1217 err = igb_request_msix(adapter);
1218 if (!err)
1219 goto request_done;
1220 /* fall back to MSI */
1221 igb_clear_interrupt_scheme(adapter);
1222 if (!pci_enable_msi(adapter->pdev))
1223 adapter->flags |= IGB_FLAG_HAS_MSI;
1224 igb_free_all_tx_resources(adapter);
1225 igb_free_all_rx_resources(adapter);
1226 adapter->num_tx_queues = 1;
1227 adapter->num_rx_queues = 1;
1228 adapter->num_q_vectors = 1;
1229 err = igb_alloc_q_vectors(adapter);
1230 if (err) {
1231 dev_err(&pdev->dev,
1232 "Unable to allocate memory for vectors\n");
1233 goto request_done;
1234 }
1235 err = igb_alloc_queues(adapter);
1236 if (err) {
1237 dev_err(&pdev->dev,
1238 "Unable to allocate memory for queues\n");
1239 igb_free_q_vectors(adapter);
1240 goto request_done;
1241 }
1242 igb_setup_all_tx_resources(adapter);
1243 igb_setup_all_rx_resources(adapter);
1244 } else {
1245 igb_assign_vector(adapter->q_vector[0], 0);
1246 }
1247
1248 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1249 err = request_irq(adapter->pdev->irq, igb_intr_msi, 0,
1250 netdev->name, adapter);
1251 if (!err)
1252 goto request_done;
1253
1254 /* fall back to legacy interrupts */
1255 igb_reset_interrupt_capability(adapter);
1256 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1257 }
1258
1259 err = request_irq(adapter->pdev->irq, igb_intr, IRQF_SHARED,
1260 netdev->name, adapter);
1261
1262 if (err)
1263 dev_err(&adapter->pdev->dev, "Error %d getting interrupt\n",
1264 err);
1265
1266request_done:
1267 return err;
1268}
1269
1270static void igb_free_irq(struct igb_adapter *adapter)
1271{
1272 if (adapter->msix_entries) {
1273 int vector = 0, i;
1274
1275 free_irq(adapter->msix_entries[vector++].vector, adapter);
1276
1277 for (i = 0; i < adapter->num_q_vectors; i++) {
1278 struct igb_q_vector *q_vector = adapter->q_vector[i];
1279 free_irq(adapter->msix_entries[vector++].vector,
1280 q_vector);
1281 }
1282 } else {
1283 free_irq(adapter->pdev->irq, adapter);
1284 }
1285}
1286
1287/**
1288 * igb_irq_disable - Mask off interrupt generation on the NIC
1289 * @adapter: board private structure
1290 **/
1291static void igb_irq_disable(struct igb_adapter *adapter)
1292{
1293 struct e1000_hw *hw = &adapter->hw;
1294
1295 /*
1296 * we need to be careful when disabling interrupts. The VFs are also
1297 * mapped into these registers and so clearing the bits can cause
1298 * issues on the VF drivers so we only need to clear what we set
1299 */
1300 if (adapter->msix_entries) {
1301 u32 regval = rd32(E1000_EIAM);
1302 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1303 wr32(E1000_EIMC, adapter->eims_enable_mask);
1304 regval = rd32(E1000_EIAC);
1305 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1306 }
1307
1308 wr32(E1000_IAM, 0);
1309 wr32(E1000_IMC, ~0);
1310 wrfl();
1311 if (adapter->msix_entries) {
1312 int i;
1313 for (i = 0; i < adapter->num_q_vectors; i++)
1314 synchronize_irq(adapter->msix_entries[i].vector);
1315 } else {
1316 synchronize_irq(adapter->pdev->irq);
1317 }
1318}
1319
1320/**
1321 * igb_irq_enable - Enable default interrupt generation settings
1322 * @adapter: board private structure
1323 **/
1324static void igb_irq_enable(struct igb_adapter *adapter)
1325{
1326 struct e1000_hw *hw = &adapter->hw;
1327
1328 if (adapter->msix_entries) {
1329 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC;
1330 u32 regval = rd32(E1000_EIAC);
1331 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1332 regval = rd32(E1000_EIAM);
1333 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1334 wr32(E1000_EIMS, adapter->eims_enable_mask);
1335 if (adapter->vfs_allocated_count) {
1336 wr32(E1000_MBVFIMR, 0xFF);
1337 ims |= E1000_IMS_VMMB;
1338 }
1339 if (adapter->hw.mac.type == e1000_82580)
1340 ims |= E1000_IMS_DRSTA;
1341
1342 wr32(E1000_IMS, ims);
1343 } else {
1344 wr32(E1000_IMS, IMS_ENABLE_MASK |
1345 E1000_IMS_DRSTA);
1346 wr32(E1000_IAM, IMS_ENABLE_MASK |
1347 E1000_IMS_DRSTA);
1348 }
1349}
1350
1351static void igb_update_mng_vlan(struct igb_adapter *adapter)
1352{
1353 struct e1000_hw *hw = &adapter->hw;
1354 u16 vid = adapter->hw.mng_cookie.vlan_id;
1355 u16 old_vid = adapter->mng_vlan_id;
1356
1357 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1358 /* add VID to filter table */
1359 igb_vfta_set(hw, vid, true);
1360 adapter->mng_vlan_id = vid;
1361 } else {
1362 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1363 }
1364
1365 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1366 (vid != old_vid) &&
1367 !test_bit(old_vid, adapter->active_vlans)) {
1368 /* remove VID from filter table */
1369 igb_vfta_set(hw, old_vid, false);
1370 }
1371}
1372
1373/**
1374 * igb_release_hw_control - release control of the h/w to f/w
1375 * @adapter: address of board private structure
1376 *
1377 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1378 * For ASF and Pass Through versions of f/w this means that the
1379 * driver is no longer loaded.
1380 *
1381 **/
1382static void igb_release_hw_control(struct igb_adapter *adapter)
1383{
1384 struct e1000_hw *hw = &adapter->hw;
1385 u32 ctrl_ext;
1386
1387 /* Let firmware take over control of h/w */
1388 ctrl_ext = rd32(E1000_CTRL_EXT);
1389 wr32(E1000_CTRL_EXT,
1390 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1391}
1392
1393/**
1394 * igb_get_hw_control - get control of the h/w from f/w
1395 * @adapter: address of board private structure
1396 *
1397 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1398 * For ASF and Pass Through versions of f/w this means that
1399 * the driver is loaded.
1400 *
1401 **/
1402static void igb_get_hw_control(struct igb_adapter *adapter)
1403{
1404 struct e1000_hw *hw = &adapter->hw;
1405 u32 ctrl_ext;
1406
1407 /* Let firmware know the driver has taken over */
1408 ctrl_ext = rd32(E1000_CTRL_EXT);
1409 wr32(E1000_CTRL_EXT,
1410 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1411}
1412
1413/**
1414 * igb_configure - configure the hardware for RX and TX
1415 * @adapter: private board structure
1416 **/
1417static void igb_configure(struct igb_adapter *adapter)
1418{
1419 struct net_device *netdev = adapter->netdev;
1420 int i;
1421
1422 igb_get_hw_control(adapter);
1423 igb_set_rx_mode(netdev);
1424
1425 igb_restore_vlan(adapter);
1426
1427 igb_setup_tctl(adapter);
1428 igb_setup_mrqc(adapter);
1429 igb_setup_rctl(adapter);
1430
1431 igb_configure_tx(adapter);
1432 igb_configure_rx(adapter);
1433
1434 igb_rx_fifo_flush_82575(&adapter->hw);
1435
1436 /* call igb_desc_unused which always leaves
1437 * at least 1 descriptor unused to make sure
1438 * next_to_use != next_to_clean */
1439 for (i = 0; i < adapter->num_rx_queues; i++) {
1440 struct igb_ring *ring = adapter->rx_ring[i];
1441 igb_alloc_rx_buffers_adv(ring, igb_desc_unused(ring));
1442 }
1443}
1444
1445/**
1446 * igb_power_up_link - Power up the phy/serdes link
1447 * @adapter: address of board private structure
1448 **/
1449void igb_power_up_link(struct igb_adapter *adapter)
1450{
1451 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1452 igb_power_up_phy_copper(&adapter->hw);
1453 else
1454 igb_power_up_serdes_link_82575(&adapter->hw);
1455}
1456
1457/**
1458 * igb_power_down_link - Power down the phy/serdes link
1459 * @adapter: address of board private structure
1460 */
1461static void igb_power_down_link(struct igb_adapter *adapter)
1462{
1463 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1464 igb_power_down_phy_copper_82575(&adapter->hw);
1465 else
1466 igb_shutdown_serdes_link_82575(&adapter->hw);
1467}
1468
1469/**
1470 * igb_up - Open the interface and prepare it to handle traffic
1471 * @adapter: board private structure
1472 **/
1473int igb_up(struct igb_adapter *adapter)
1474{
1475 struct e1000_hw *hw = &adapter->hw;
1476 int i;
1477
1478 /* hardware has been reset, we need to reload some things */
1479 igb_configure(adapter);
1480
1481 clear_bit(__IGB_DOWN, &adapter->state);
1482
1483 for (i = 0; i < adapter->num_q_vectors; i++) {
1484 struct igb_q_vector *q_vector = adapter->q_vector[i];
1485 napi_enable(&q_vector->napi);
1486 }
1487 if (adapter->msix_entries)
1488 igb_configure_msix(adapter);
1489 else
1490 igb_assign_vector(adapter->q_vector[0], 0);
1491
1492 /* Clear any pending interrupts. */
1493 rd32(E1000_ICR);
1494 igb_irq_enable(adapter);
1495
1496 /* notify VFs that reset has been completed */
1497 if (adapter->vfs_allocated_count) {
1498 u32 reg_data = rd32(E1000_CTRL_EXT);
1499 reg_data |= E1000_CTRL_EXT_PFRSTD;
1500 wr32(E1000_CTRL_EXT, reg_data);
1501 }
1502
1503 netif_tx_start_all_queues(adapter->netdev);
1504
1505 /* start the watchdog. */
1506 hw->mac.get_link_status = 1;
1507 schedule_work(&adapter->watchdog_task);
1508
1509 return 0;
1510}
1511
1512void igb_down(struct igb_adapter *adapter)
1513{
1514 struct net_device *netdev = adapter->netdev;
1515 struct e1000_hw *hw = &adapter->hw;
1516 u32 tctl, rctl;
1517 int i;
1518
1519 /* signal that we're down so the interrupt handler does not
1520 * reschedule our watchdog timer */
1521 set_bit(__IGB_DOWN, &adapter->state);
1522
1523 /* disable receives in the hardware */
1524 rctl = rd32(E1000_RCTL);
1525 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1526 /* flush and sleep below */
1527
1528 netif_tx_stop_all_queues(netdev);
1529
1530 /* disable transmits in the hardware */
1531 tctl = rd32(E1000_TCTL);
1532 tctl &= ~E1000_TCTL_EN;
1533 wr32(E1000_TCTL, tctl);
1534 /* flush both disables and wait for them to finish */
1535 wrfl();
1536 msleep(10);
1537
1538 for (i = 0; i < adapter->num_q_vectors; i++) {
1539 struct igb_q_vector *q_vector = adapter->q_vector[i];
1540 napi_disable(&q_vector->napi);
1541 }
1542
1543 igb_irq_disable(adapter);
1544
1545 del_timer_sync(&adapter->watchdog_timer);
1546 del_timer_sync(&adapter->phy_info_timer);
1547
1548 netif_carrier_off(netdev);
1549
1550 /* record the stats before reset*/
1551 spin_lock(&adapter->stats64_lock);
1552 igb_update_stats(adapter, &adapter->stats64);
1553 spin_unlock(&adapter->stats64_lock);
1554
1555 adapter->link_speed = 0;
1556 adapter->link_duplex = 0;
1557
1558 if (!pci_channel_offline(adapter->pdev))
1559 igb_reset(adapter);
1560 igb_clean_all_tx_rings(adapter);
1561 igb_clean_all_rx_rings(adapter);
1562#ifdef CONFIG_IGB_DCA
1563
1564 /* since we reset the hardware DCA settings were cleared */
1565 igb_setup_dca(adapter);
1566#endif
1567}
1568
1569void igb_reinit_locked(struct igb_adapter *adapter)
1570{
1571 WARN_ON(in_interrupt());
1572 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1573 msleep(1);
1574 igb_down(adapter);
1575 igb_up(adapter);
1576 clear_bit(__IGB_RESETTING, &adapter->state);
1577}
1578
1579void igb_reset(struct igb_adapter *adapter)
1580{
1581 struct pci_dev *pdev = adapter->pdev;
1582 struct e1000_hw *hw = &adapter->hw;
1583 struct e1000_mac_info *mac = &hw->mac;
1584 struct e1000_fc_info *fc = &hw->fc;
1585 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1586 u16 hwm;
1587
1588 /* Repartition Pba for greater than 9k mtu
1589 * To take effect CTRL.RST is required.
1590 */
1591 switch (mac->type) {
1592 case e1000_i350:
1593 case e1000_82580:
1594 pba = rd32(E1000_RXPBS);
1595 pba = igb_rxpbs_adjust_82580(pba);
1596 break;
1597 case e1000_82576:
1598 pba = rd32(E1000_RXPBS);
1599 pba &= E1000_RXPBS_SIZE_MASK_82576;
1600 break;
1601 case e1000_82575:
1602 default:
1603 pba = E1000_PBA_34K;
1604 break;
1605 }
1606
1607 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1608 (mac->type < e1000_82576)) {
1609 /* adjust PBA for jumbo frames */
1610 wr32(E1000_PBA, pba);
1611
1612 /* To maintain wire speed transmits, the Tx FIFO should be
1613 * large enough to accommodate two full transmit packets,
1614 * rounded up to the next 1KB and expressed in KB. Likewise,
1615 * the Rx FIFO should be large enough to accommodate at least
1616 * one full receive packet and is similarly rounded up and
1617 * expressed in KB. */
1618 pba = rd32(E1000_PBA);
1619 /* upper 16 bits has Tx packet buffer allocation size in KB */
1620 tx_space = pba >> 16;
1621 /* lower 16 bits has Rx packet buffer allocation size in KB */
1622 pba &= 0xffff;
1623 /* the tx fifo also stores 16 bytes of information about the tx
1624 * but don't include ethernet FCS because hardware appends it */
1625 min_tx_space = (adapter->max_frame_size +
1626 sizeof(union e1000_adv_tx_desc) -
1627 ETH_FCS_LEN) * 2;
1628 min_tx_space = ALIGN(min_tx_space, 1024);
1629 min_tx_space >>= 10;
1630 /* software strips receive CRC, so leave room for it */
1631 min_rx_space = adapter->max_frame_size;
1632 min_rx_space = ALIGN(min_rx_space, 1024);
1633 min_rx_space >>= 10;
1634
1635 /* If current Tx allocation is less than the min Tx FIFO size,
1636 * and the min Tx FIFO size is less than the current Rx FIFO
1637 * allocation, take space away from current Rx allocation */
1638 if (tx_space < min_tx_space &&
1639 ((min_tx_space - tx_space) < pba)) {
1640 pba = pba - (min_tx_space - tx_space);
1641
1642 /* if short on rx space, rx wins and must trump tx
1643 * adjustment */
1644 if (pba < min_rx_space)
1645 pba = min_rx_space;
1646 }
1647 wr32(E1000_PBA, pba);
1648 }
1649
1650 /* flow control settings */
1651 /* The high water mark must be low enough to fit one full frame
1652 * (or the size used for early receive) above it in the Rx FIFO.
1653 * Set it to the lower of:
1654 * - 90% of the Rx FIFO size, or
1655 * - the full Rx FIFO size minus one full frame */
1656 hwm = min(((pba << 10) * 9 / 10),
1657 ((pba << 10) - 2 * adapter->max_frame_size));
1658
1659 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1660 fc->low_water = fc->high_water - 16;
1661 fc->pause_time = 0xFFFF;
1662 fc->send_xon = 1;
1663 fc->current_mode = fc->requested_mode;
1664
1665 /* disable receive for all VFs and wait one second */
1666 if (adapter->vfs_allocated_count) {
1667 int i;
1668 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1669 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1670
1671 /* ping all the active vfs to let them know we are going down */
1672 igb_ping_all_vfs(adapter);
1673
1674 /* disable transmits and receives */
1675 wr32(E1000_VFRE, 0);
1676 wr32(E1000_VFTE, 0);
1677 }
1678
1679 /* Allow time for pending master requests to run */
1680 hw->mac.ops.reset_hw(hw);
1681 wr32(E1000_WUC, 0);
1682
1683 if (hw->mac.ops.init_hw(hw))
1684 dev_err(&pdev->dev, "Hardware Error\n");
1685 if (hw->mac.type > e1000_82580) {
1686 if (adapter->flags & IGB_FLAG_DMAC) {
1687 u32 reg;
1688
1689 /*
1690 * DMA Coalescing high water mark needs to be higher
1691 * than * the * Rx threshold. The Rx threshold is
1692 * currently * pba - 6, so we * should use a high water
1693 * mark of pba * - 4. */
1694 hwm = (pba - 4) << 10;
1695
1696 reg = (((pba-6) << E1000_DMACR_DMACTHR_SHIFT)
1697 & E1000_DMACR_DMACTHR_MASK);
1698
1699 /* transition to L0x or L1 if available..*/
1700 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
1701
1702 /* watchdog timer= +-1000 usec in 32usec intervals */
1703 reg |= (1000 >> 5);
1704 wr32(E1000_DMACR, reg);
1705
1706 /* no lower threshold to disable coalescing(smart fifb)
1707 * -UTRESH=0*/
1708 wr32(E1000_DMCRTRH, 0);
1709
1710 /* set hwm to PBA - 2 * max frame size */
1711 wr32(E1000_FCRTC, hwm);
1712
1713 /*
1714 * This sets the time to wait before requesting tran-
1715 * sition to * low power state to number of usecs needed
1716 * to receive 1 512 * byte frame at gigabit line rate
1717 */
1718 reg = rd32(E1000_DMCTLX);
1719 reg |= IGB_DMCTLX_DCFLUSH_DIS;
1720
1721 /* Delay 255 usec before entering Lx state. */
1722 reg |= 0xFF;
1723 wr32(E1000_DMCTLX, reg);
1724
1725 /* free space in Tx packet buffer to wake from DMAC */
1726 wr32(E1000_DMCTXTH,
1727 (IGB_MIN_TXPBSIZE -
1728 (IGB_TX_BUF_4096 + adapter->max_frame_size))
1729 >> 6);
1730
1731 /* make low power state decision controlled by DMAC */
1732 reg = rd32(E1000_PCIEMISC);
1733 reg |= E1000_PCIEMISC_LX_DECISION;
1734 wr32(E1000_PCIEMISC, reg);
1735 } /* end if IGB_FLAG_DMAC set */
1736 }
1737 if (hw->mac.type == e1000_82580) {
1738 u32 reg = rd32(E1000_PCIEMISC);
1739 wr32(E1000_PCIEMISC,
1740 reg & ~E1000_PCIEMISC_LX_DECISION);
1741 }
1742 if (!netif_running(adapter->netdev))
1743 igb_power_down_link(adapter);
1744
1745 igb_update_mng_vlan(adapter);
1746
1747 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1748 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1749
1750 igb_get_phy_info(hw);
1751}
1752
1753static u32 igb_fix_features(struct net_device *netdev, u32 features)
1754{
1755 /*
1756 * Since there is no support for separate rx/tx vlan accel
1757 * enable/disable make sure tx flag is always in same state as rx.
1758 */
1759 if (features & NETIF_F_HW_VLAN_RX)
1760 features |= NETIF_F_HW_VLAN_TX;
1761 else
1762 features &= ~NETIF_F_HW_VLAN_TX;
1763
1764 return features;
1765}
1766
1767static int igb_set_features(struct net_device *netdev, u32 features)
1768{
1769 struct igb_adapter *adapter = netdev_priv(netdev);
1770 int i;
1771 u32 changed = netdev->features ^ features;
1772
1773 for (i = 0; i < adapter->num_rx_queues; i++) {
1774 if (features & NETIF_F_RXCSUM)
1775 adapter->rx_ring[i]->flags |= IGB_RING_FLAG_RX_CSUM;
1776 else
1777 adapter->rx_ring[i]->flags &= ~IGB_RING_FLAG_RX_CSUM;
1778 }
1779
1780 if (changed & NETIF_F_HW_VLAN_RX)
1781 igb_vlan_mode(netdev, features);
1782
1783 return 0;
1784}
1785
1786static const struct net_device_ops igb_netdev_ops = {
1787 .ndo_open = igb_open,
1788 .ndo_stop = igb_close,
1789 .ndo_start_xmit = igb_xmit_frame_adv,
1790 .ndo_get_stats64 = igb_get_stats64,
1791 .ndo_set_rx_mode = igb_set_rx_mode,
1792 .ndo_set_multicast_list = igb_set_rx_mode,
1793 .ndo_set_mac_address = igb_set_mac,
1794 .ndo_change_mtu = igb_change_mtu,
1795 .ndo_do_ioctl = igb_ioctl,
1796 .ndo_tx_timeout = igb_tx_timeout,
1797 .ndo_validate_addr = eth_validate_addr,
1798 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1799 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1800 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1801 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1802 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1803 .ndo_get_vf_config = igb_ndo_get_vf_config,
1804#ifdef CONFIG_NET_POLL_CONTROLLER
1805 .ndo_poll_controller = igb_netpoll,
1806#endif
1807 .ndo_fix_features = igb_fix_features,
1808 .ndo_set_features = igb_set_features,
1809};
1810
1811/**
1812 * igb_probe - Device Initialization Routine
1813 * @pdev: PCI device information struct
1814 * @ent: entry in igb_pci_tbl
1815 *
1816 * Returns 0 on success, negative on failure
1817 *
1818 * igb_probe initializes an adapter identified by a pci_dev structure.
1819 * The OS initialization, configuring of the adapter private structure,
1820 * and a hardware reset occur.
1821 **/
1822static int __devinit igb_probe(struct pci_dev *pdev,
1823 const struct pci_device_id *ent)
1824{
1825 struct net_device *netdev;
1826 struct igb_adapter *adapter;
1827 struct e1000_hw *hw;
1828 u16 eeprom_data = 0;
1829 s32 ret_val;
1830 static int global_quad_port_a; /* global quad port a indication */
1831 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1832 unsigned long mmio_start, mmio_len;
1833 int err, pci_using_dac;
1834 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1835 u8 part_str[E1000_PBANUM_LENGTH];
1836
1837 /* Catch broken hardware that put the wrong VF device ID in
1838 * the PCIe SR-IOV capability.
1839 */
1840 if (pdev->is_virtfn) {
1841 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1842 pci_name(pdev), pdev->vendor, pdev->device);
1843 return -EINVAL;
1844 }
1845
1846 err = pci_enable_device_mem(pdev);
1847 if (err)
1848 return err;
1849
1850 pci_using_dac = 0;
1851 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1852 if (!err) {
1853 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1854 if (!err)
1855 pci_using_dac = 1;
1856 } else {
1857 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1858 if (err) {
1859 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1860 if (err) {
1861 dev_err(&pdev->dev, "No usable DMA "
1862 "configuration, aborting\n");
1863 goto err_dma;
1864 }
1865 }
1866 }
1867
1868 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1869 IORESOURCE_MEM),
1870 igb_driver_name);
1871 if (err)
1872 goto err_pci_reg;
1873
1874 pci_enable_pcie_error_reporting(pdev);
1875
1876 pci_set_master(pdev);
1877 pci_save_state(pdev);
1878
1879 err = -ENOMEM;
1880 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1881 IGB_ABS_MAX_TX_QUEUES);
1882 if (!netdev)
1883 goto err_alloc_etherdev;
1884
1885 SET_NETDEV_DEV(netdev, &pdev->dev);
1886
1887 pci_set_drvdata(pdev, netdev);
1888 adapter = netdev_priv(netdev);
1889 adapter->netdev = netdev;
1890 adapter->pdev = pdev;
1891 hw = &adapter->hw;
1892 hw->back = adapter;
1893 adapter->msg_enable = NETIF_MSG_DRV | NETIF_MSG_PROBE;
1894
1895 mmio_start = pci_resource_start(pdev, 0);
1896 mmio_len = pci_resource_len(pdev, 0);
1897
1898 err = -EIO;
1899 hw->hw_addr = ioremap(mmio_start, mmio_len);
1900 if (!hw->hw_addr)
1901 goto err_ioremap;
1902
1903 netdev->netdev_ops = &igb_netdev_ops;
1904 igb_set_ethtool_ops(netdev);
1905 netdev->watchdog_timeo = 5 * HZ;
1906
1907 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1908
1909 netdev->mem_start = mmio_start;
1910 netdev->mem_end = mmio_start + mmio_len;
1911
1912 /* PCI config space info */
1913 hw->vendor_id = pdev->vendor;
1914 hw->device_id = pdev->device;
1915 hw->revision_id = pdev->revision;
1916 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1917 hw->subsystem_device_id = pdev->subsystem_device;
1918
1919 /* Copy the default MAC, PHY and NVM function pointers */
1920 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1921 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1922 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1923 /* Initialize skew-specific constants */
1924 err = ei->get_invariants(hw);
1925 if (err)
1926 goto err_sw_init;
1927
1928 /* setup the private structure */
1929 err = igb_sw_init(adapter);
1930 if (err)
1931 goto err_sw_init;
1932
1933 igb_get_bus_info_pcie(hw);
1934
1935 hw->phy.autoneg_wait_to_complete = false;
1936
1937 /* Copper options */
1938 if (hw->phy.media_type == e1000_media_type_copper) {
1939 hw->phy.mdix = AUTO_ALL_MODES;
1940 hw->phy.disable_polarity_correction = false;
1941 hw->phy.ms_type = e1000_ms_hw_default;
1942 }
1943
1944 if (igb_check_reset_block(hw))
1945 dev_info(&pdev->dev,
1946 "PHY reset is blocked due to SOL/IDER session.\n");
1947
1948 netdev->hw_features = NETIF_F_SG |
1949 NETIF_F_IP_CSUM |
1950 NETIF_F_IPV6_CSUM |
1951 NETIF_F_TSO |
1952 NETIF_F_TSO6 |
1953 NETIF_F_RXCSUM |
1954 NETIF_F_HW_VLAN_RX;
1955
1956 netdev->features = netdev->hw_features |
1957 NETIF_F_HW_VLAN_TX |
1958 NETIF_F_HW_VLAN_FILTER;
1959
1960 netdev->vlan_features |= NETIF_F_TSO;
1961 netdev->vlan_features |= NETIF_F_TSO6;
1962 netdev->vlan_features |= NETIF_F_IP_CSUM;
1963 netdev->vlan_features |= NETIF_F_IPV6_CSUM;
1964 netdev->vlan_features |= NETIF_F_SG;
1965
1966 if (pci_using_dac) {
1967 netdev->features |= NETIF_F_HIGHDMA;
1968 netdev->vlan_features |= NETIF_F_HIGHDMA;
1969 }
1970
1971 if (hw->mac.type >= e1000_82576) {
1972 netdev->hw_features |= NETIF_F_SCTP_CSUM;
1973 netdev->features |= NETIF_F_SCTP_CSUM;
1974 }
1975
1976 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
1977
1978 /* before reading the NVM, reset the controller to put the device in a
1979 * known good starting state */
1980 hw->mac.ops.reset_hw(hw);
1981
1982 /* make sure the NVM is good */
1983 if (hw->nvm.ops.validate(hw) < 0) {
1984 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
1985 err = -EIO;
1986 goto err_eeprom;
1987 }
1988
1989 /* copy the MAC address out of the NVM */
1990 if (hw->mac.ops.read_mac_addr(hw))
1991 dev_err(&pdev->dev, "NVM Read Error\n");
1992
1993 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
1994 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
1995
1996 if (!is_valid_ether_addr(netdev->perm_addr)) {
1997 dev_err(&pdev->dev, "Invalid MAC Address\n");
1998 err = -EIO;
1999 goto err_eeprom;
2000 }
2001
2002 setup_timer(&adapter->watchdog_timer, igb_watchdog,
2003 (unsigned long) adapter);
2004 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2005 (unsigned long) adapter);
2006
2007 INIT_WORK(&adapter->reset_task, igb_reset_task);
2008 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2009
2010 /* Initialize link properties that are user-changeable */
2011 adapter->fc_autoneg = true;
2012 hw->mac.autoneg = true;
2013 hw->phy.autoneg_advertised = 0x2f;
2014
2015 hw->fc.requested_mode = e1000_fc_default;
2016 hw->fc.current_mode = e1000_fc_default;
2017
2018 igb_validate_mdi_setting(hw);
2019
2020 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2021 * enable the ACPI Magic Packet filter
2022 */
2023
2024 if (hw->bus.func == 0)
2025 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2026 else if (hw->mac.type >= e1000_82580)
2027 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2028 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2029 &eeprom_data);
2030 else if (hw->bus.func == 1)
2031 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2032
2033 if (eeprom_data & eeprom_apme_mask)
2034 adapter->eeprom_wol |= E1000_WUFC_MAG;
2035
2036 /* now that we have the eeprom settings, apply the special cases where
2037 * the eeprom may be wrong or the board simply won't support wake on
2038 * lan on a particular port */
2039 switch (pdev->device) {
2040 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2041 adapter->eeprom_wol = 0;
2042 break;
2043 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2044 case E1000_DEV_ID_82576_FIBER:
2045 case E1000_DEV_ID_82576_SERDES:
2046 /* Wake events only supported on port A for dual fiber
2047 * regardless of eeprom setting */
2048 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2049 adapter->eeprom_wol = 0;
2050 break;
2051 case E1000_DEV_ID_82576_QUAD_COPPER:
2052 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2053 /* if quad port adapter, disable WoL on all but port A */
2054 if (global_quad_port_a != 0)
2055 adapter->eeprom_wol = 0;
2056 else
2057 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2058 /* Reset for multiple quad port adapters */
2059 if (++global_quad_port_a == 4)
2060 global_quad_port_a = 0;
2061 break;
2062 }
2063
2064 /* initialize the wol settings based on the eeprom settings */
2065 adapter->wol = adapter->eeprom_wol;
2066 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
2067
2068 /* reset the hardware with the new settings */
2069 igb_reset(adapter);
2070
2071 /* let the f/w know that the h/w is now under the control of the
2072 * driver. */
2073 igb_get_hw_control(adapter);
2074
2075 strcpy(netdev->name, "eth%d");
2076 err = register_netdev(netdev);
2077 if (err)
2078 goto err_register;
2079
2080 igb_vlan_mode(netdev, netdev->features);
2081
2082 /* carrier off reporting is important to ethtool even BEFORE open */
2083 netif_carrier_off(netdev);
2084
2085#ifdef CONFIG_IGB_DCA
2086 if (dca_add_requester(&pdev->dev) == 0) {
2087 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2088 dev_info(&pdev->dev, "DCA enabled\n");
2089 igb_setup_dca(adapter);
2090 }
2091
2092#endif
2093 /* do hw tstamp init after resetting */
2094 igb_init_hw_timer(adapter);
2095
2096 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2097 /* print bus type/speed/width info */
2098 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
2099 netdev->name,
2100 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
2101 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
2102 "unknown"),
2103 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2104 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2105 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2106 "unknown"),
2107 netdev->dev_addr);
2108
2109 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2110 if (ret_val)
2111 strcpy(part_str, "Unknown");
2112 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2113 dev_info(&pdev->dev,
2114 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2115 adapter->msix_entries ? "MSI-X" :
2116 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2117 adapter->num_rx_queues, adapter->num_tx_queues);
2118 switch (hw->mac.type) {
2119 case e1000_i350:
2120 igb_set_eee_i350(hw);
2121 break;
2122 default:
2123 break;
2124 }
2125 return 0;
2126
2127err_register:
2128 igb_release_hw_control(adapter);
2129err_eeprom:
2130 if (!igb_check_reset_block(hw))
2131 igb_reset_phy(hw);
2132
2133 if (hw->flash_address)
2134 iounmap(hw->flash_address);
2135err_sw_init:
2136 igb_clear_interrupt_scheme(adapter);
2137 iounmap(hw->hw_addr);
2138err_ioremap:
2139 free_netdev(netdev);
2140err_alloc_etherdev:
2141 pci_release_selected_regions(pdev,
2142 pci_select_bars(pdev, IORESOURCE_MEM));
2143err_pci_reg:
2144err_dma:
2145 pci_disable_device(pdev);
2146 return err;
2147}
2148
2149/**
2150 * igb_remove - Device Removal Routine
2151 * @pdev: PCI device information struct
2152 *
2153 * igb_remove is called by the PCI subsystem to alert the driver
2154 * that it should release a PCI device. The could be caused by a
2155 * Hot-Plug event, or because the driver is going to be removed from
2156 * memory.
2157 **/
2158static void __devexit igb_remove(struct pci_dev *pdev)
2159{
2160 struct net_device *netdev = pci_get_drvdata(pdev);
2161 struct igb_adapter *adapter = netdev_priv(netdev);
2162 struct e1000_hw *hw = &adapter->hw;
2163
2164 /*
2165 * The watchdog timer may be rescheduled, so explicitly
2166 * disable watchdog from being rescheduled.
2167 */
2168 set_bit(__IGB_DOWN, &adapter->state);
2169 del_timer_sync(&adapter->watchdog_timer);
2170 del_timer_sync(&adapter->phy_info_timer);
2171
2172 cancel_work_sync(&adapter->reset_task);
2173 cancel_work_sync(&adapter->watchdog_task);
2174
2175#ifdef CONFIG_IGB_DCA
2176 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2177 dev_info(&pdev->dev, "DCA disabled\n");
2178 dca_remove_requester(&pdev->dev);
2179 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2180 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2181 }
2182#endif
2183
2184 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2185 * would have already happened in close and is redundant. */
2186 igb_release_hw_control(adapter);
2187
2188 unregister_netdev(netdev);
2189
2190 igb_clear_interrupt_scheme(adapter);
2191
2192#ifdef CONFIG_PCI_IOV
2193 /* reclaim resources allocated to VFs */
2194 if (adapter->vf_data) {
2195 /* disable iov and allow time for transactions to clear */
2196 pci_disable_sriov(pdev);
2197 msleep(500);
2198
2199 kfree(adapter->vf_data);
2200 adapter->vf_data = NULL;
2201 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2202 wrfl();
2203 msleep(100);
2204 dev_info(&pdev->dev, "IOV Disabled\n");
2205 }
2206#endif
2207
2208 iounmap(hw->hw_addr);
2209 if (hw->flash_address)
2210 iounmap(hw->flash_address);
2211 pci_release_selected_regions(pdev,
2212 pci_select_bars(pdev, IORESOURCE_MEM));
2213
2214 free_netdev(netdev);
2215
2216 pci_disable_pcie_error_reporting(pdev);
2217
2218 pci_disable_device(pdev);
2219}
2220
2221/**
2222 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2223 * @adapter: board private structure to initialize
2224 *
2225 * This function initializes the vf specific data storage and then attempts to
2226 * allocate the VFs. The reason for ordering it this way is because it is much
2227 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2228 * the memory for the VFs.
2229 **/
2230static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2231{
2232#ifdef CONFIG_PCI_IOV
2233 struct pci_dev *pdev = adapter->pdev;
2234
2235 if (adapter->vfs_allocated_count) {
2236 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2237 sizeof(struct vf_data_storage),
2238 GFP_KERNEL);
2239 /* if allocation failed then we do not support SR-IOV */
2240 if (!adapter->vf_data) {
2241 adapter->vfs_allocated_count = 0;
2242 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2243 "Data Storage\n");
2244 }
2245 }
2246
2247 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count)) {
2248 kfree(adapter->vf_data);
2249 adapter->vf_data = NULL;
2250#endif /* CONFIG_PCI_IOV */
2251 adapter->vfs_allocated_count = 0;
2252#ifdef CONFIG_PCI_IOV
2253 } else {
2254 unsigned char mac_addr[ETH_ALEN];
2255 int i;
2256 dev_info(&pdev->dev, "%d vfs allocated\n",
2257 adapter->vfs_allocated_count);
2258 for (i = 0; i < adapter->vfs_allocated_count; i++) {
2259 random_ether_addr(mac_addr);
2260 igb_set_vf_mac(adapter, i, mac_addr);
2261 }
2262 /* DMA Coalescing is not supported in IOV mode. */
2263 if (adapter->flags & IGB_FLAG_DMAC)
2264 adapter->flags &= ~IGB_FLAG_DMAC;
2265 }
2266#endif /* CONFIG_PCI_IOV */
2267}
2268
2269
2270/**
2271 * igb_init_hw_timer - Initialize hardware timer used with IEEE 1588 timestamp
2272 * @adapter: board private structure to initialize
2273 *
2274 * igb_init_hw_timer initializes the function pointer and values for the hw
2275 * timer found in hardware.
2276 **/
2277static void igb_init_hw_timer(struct igb_adapter *adapter)
2278{
2279 struct e1000_hw *hw = &adapter->hw;
2280
2281 switch (hw->mac.type) {
2282 case e1000_i350:
2283 case e1000_82580:
2284 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2285 adapter->cycles.read = igb_read_clock;
2286 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2287 adapter->cycles.mult = 1;
2288 /*
2289 * The 82580 timesync updates the system timer every 8ns by 8ns
2290 * and the value cannot be shifted. Instead we need to shift
2291 * the registers to generate a 64bit timer value. As a result
2292 * SYSTIMR/L/H, TXSTMPL/H, RXSTMPL/H all have to be shifted by
2293 * 24 in order to generate a larger value for synchronization.
2294 */
2295 adapter->cycles.shift = IGB_82580_TSYNC_SHIFT;
2296 /* disable system timer temporarily by setting bit 31 */
2297 wr32(E1000_TSAUXC, 0x80000000);
2298 wrfl();
2299
2300 /* Set registers so that rollover occurs soon to test this. */
2301 wr32(E1000_SYSTIMR, 0x00000000);
2302 wr32(E1000_SYSTIML, 0x80000000);
2303 wr32(E1000_SYSTIMH, 0x000000FF);
2304 wrfl();
2305
2306 /* enable system timer by clearing bit 31 */
2307 wr32(E1000_TSAUXC, 0x0);
2308 wrfl();
2309
2310 timecounter_init(&adapter->clock,
2311 &adapter->cycles,
2312 ktime_to_ns(ktime_get_real()));
2313 /*
2314 * Synchronize our NIC clock against system wall clock. NIC
2315 * time stamp reading requires ~3us per sample, each sample
2316 * was pretty stable even under load => only require 10
2317 * samples for each offset comparison.
2318 */
2319 memset(&adapter->compare, 0, sizeof(adapter->compare));
2320 adapter->compare.source = &adapter->clock;
2321 adapter->compare.target = ktime_get_real;
2322 adapter->compare.num_samples = 10;
2323 timecompare_update(&adapter->compare, 0);
2324 break;
2325 case e1000_82576:
2326 /*
2327 * Initialize hardware timer: we keep it running just in case
2328 * that some program needs it later on.
2329 */
2330 memset(&adapter->cycles, 0, sizeof(adapter->cycles));
2331 adapter->cycles.read = igb_read_clock;
2332 adapter->cycles.mask = CLOCKSOURCE_MASK(64);
2333 adapter->cycles.mult = 1;
2334 /**
2335 * Scale the NIC clock cycle by a large factor so that
2336 * relatively small clock corrections can be added or
2337 * subtracted at each clock tick. The drawbacks of a large
2338 * factor are a) that the clock register overflows more quickly
2339 * (not such a big deal) and b) that the increment per tick has
2340 * to fit into 24 bits. As a result we need to use a shift of
2341 * 19 so we can fit a value of 16 into the TIMINCA register.
2342 */
2343 adapter->cycles.shift = IGB_82576_TSYNC_SHIFT;
2344 wr32(E1000_TIMINCA,
2345 (1 << E1000_TIMINCA_16NS_SHIFT) |
2346 (16 << IGB_82576_TSYNC_SHIFT));
2347
2348 /* Set registers so that rollover occurs soon to test this. */
2349 wr32(E1000_SYSTIML, 0x00000000);
2350 wr32(E1000_SYSTIMH, 0xFF800000);
2351 wrfl();
2352
2353 timecounter_init(&adapter->clock,
2354 &adapter->cycles,
2355 ktime_to_ns(ktime_get_real()));
2356 /*
2357 * Synchronize our NIC clock against system wall clock. NIC
2358 * time stamp reading requires ~3us per sample, each sample
2359 * was pretty stable even under load => only require 10
2360 * samples for each offset comparison.
2361 */
2362 memset(&adapter->compare, 0, sizeof(adapter->compare));
2363 adapter->compare.source = &adapter->clock;
2364 adapter->compare.target = ktime_get_real;
2365 adapter->compare.num_samples = 10;
2366 timecompare_update(&adapter->compare, 0);
2367 break;
2368 case e1000_82575:
2369 /* 82575 does not support timesync */
2370 default:
2371 break;
2372 }
2373
2374}
2375
2376/**
2377 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2378 * @adapter: board private structure to initialize
2379 *
2380 * igb_sw_init initializes the Adapter private data structure.
2381 * Fields are initialized based on PCI device information and
2382 * OS network device settings (MTU size).
2383 **/
2384static int __devinit igb_sw_init(struct igb_adapter *adapter)
2385{
2386 struct e1000_hw *hw = &adapter->hw;
2387 struct net_device *netdev = adapter->netdev;
2388 struct pci_dev *pdev = adapter->pdev;
2389
2390 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2391
2392 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2393 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2394 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2395 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2396
2397 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
2398 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2399
2400 spin_lock_init(&adapter->stats64_lock);
2401#ifdef CONFIG_PCI_IOV
2402 switch (hw->mac.type) {
2403 case e1000_82576:
2404 case e1000_i350:
2405 if (max_vfs > 7) {
2406 dev_warn(&pdev->dev,
2407 "Maximum of 7 VFs per PF, using max\n");
2408 adapter->vfs_allocated_count = 7;
2409 } else
2410 adapter->vfs_allocated_count = max_vfs;
2411 break;
2412 default:
2413 break;
2414 }
2415#endif /* CONFIG_PCI_IOV */
2416 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES, num_online_cpus());
2417 /* i350 cannot do RSS and SR-IOV at the same time */
2418 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2419 adapter->rss_queues = 1;
2420
2421 /*
2422 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2423 * then we should combine the queues into a queue pair in order to
2424 * conserve interrupts due to limited supply
2425 */
2426 if ((adapter->rss_queues > 4) ||
2427 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2428 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2429
2430 /* This call may decrease the number of queues */
2431 if (igb_init_interrupt_scheme(adapter)) {
2432 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2433 return -ENOMEM;
2434 }
2435
2436 igb_probe_vfs(adapter);
2437
2438 /* Explicitly disable IRQ since the NIC can be in any state. */
2439 igb_irq_disable(adapter);
2440
2441 if (hw->mac.type == e1000_i350)
2442 adapter->flags &= ~IGB_FLAG_DMAC;
2443
2444 set_bit(__IGB_DOWN, &adapter->state);
2445 return 0;
2446}
2447
2448/**
2449 * igb_open - Called when a network interface is made active
2450 * @netdev: network interface device structure
2451 *
2452 * Returns 0 on success, negative value on failure
2453 *
2454 * The open entry point is called when a network interface is made
2455 * active by the system (IFF_UP). At this point all resources needed
2456 * for transmit and receive operations are allocated, the interrupt
2457 * handler is registered with the OS, the watchdog timer is started,
2458 * and the stack is notified that the interface is ready.
2459 **/
2460static int igb_open(struct net_device *netdev)
2461{
2462 struct igb_adapter *adapter = netdev_priv(netdev);
2463 struct e1000_hw *hw = &adapter->hw;
2464 int err;
2465 int i;
2466
2467 /* disallow open during test */
2468 if (test_bit(__IGB_TESTING, &adapter->state))
2469 return -EBUSY;
2470
2471 netif_carrier_off(netdev);
2472
2473 /* allocate transmit descriptors */
2474 err = igb_setup_all_tx_resources(adapter);
2475 if (err)
2476 goto err_setup_tx;
2477
2478 /* allocate receive descriptors */
2479 err = igb_setup_all_rx_resources(adapter);
2480 if (err)
2481 goto err_setup_rx;
2482
2483 igb_power_up_link(adapter);
2484
2485 /* before we allocate an interrupt, we must be ready to handle it.
2486 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2487 * as soon as we call pci_request_irq, so we have to setup our
2488 * clean_rx handler before we do so. */
2489 igb_configure(adapter);
2490
2491 err = igb_request_irq(adapter);
2492 if (err)
2493 goto err_req_irq;
2494
2495 /* From here on the code is the same as igb_up() */
2496 clear_bit(__IGB_DOWN, &adapter->state);
2497
2498 for (i = 0; i < adapter->num_q_vectors; i++) {
2499 struct igb_q_vector *q_vector = adapter->q_vector[i];
2500 napi_enable(&q_vector->napi);
2501 }
2502
2503 /* Clear any pending interrupts. */
2504 rd32(E1000_ICR);
2505
2506 igb_irq_enable(adapter);
2507
2508 /* notify VFs that reset has been completed */
2509 if (adapter->vfs_allocated_count) {
2510 u32 reg_data = rd32(E1000_CTRL_EXT);
2511 reg_data |= E1000_CTRL_EXT_PFRSTD;
2512 wr32(E1000_CTRL_EXT, reg_data);
2513 }
2514
2515 netif_tx_start_all_queues(netdev);
2516
2517 /* start the watchdog. */
2518 hw->mac.get_link_status = 1;
2519 schedule_work(&adapter->watchdog_task);
2520
2521 return 0;
2522
2523err_req_irq:
2524 igb_release_hw_control(adapter);
2525 igb_power_down_link(adapter);
2526 igb_free_all_rx_resources(adapter);
2527err_setup_rx:
2528 igb_free_all_tx_resources(adapter);
2529err_setup_tx:
2530 igb_reset(adapter);
2531
2532 return err;
2533}
2534
2535/**
2536 * igb_close - Disables a network interface
2537 * @netdev: network interface device structure
2538 *
2539 * Returns 0, this is not allowed to fail
2540 *
2541 * The close entry point is called when an interface is de-activated
2542 * by the OS. The hardware is still under the driver's control, but
2543 * needs to be disabled. A global MAC reset is issued to stop the
2544 * hardware, and all transmit and receive resources are freed.
2545 **/
2546static int igb_close(struct net_device *netdev)
2547{
2548 struct igb_adapter *adapter = netdev_priv(netdev);
2549
2550 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2551 igb_down(adapter);
2552
2553 igb_free_irq(adapter);
2554
2555 igb_free_all_tx_resources(adapter);
2556 igb_free_all_rx_resources(adapter);
2557
2558 return 0;
2559}
2560
2561/**
2562 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2563 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2564 *
2565 * Return 0 on success, negative on failure
2566 **/
2567int igb_setup_tx_resources(struct igb_ring *tx_ring)
2568{
2569 struct device *dev = tx_ring->dev;
2570 int size;
2571
2572 size = sizeof(struct igb_buffer) * tx_ring->count;
2573 tx_ring->buffer_info = vzalloc(size);
2574 if (!tx_ring->buffer_info)
2575 goto err;
2576
2577 /* round up to nearest 4K */
2578 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2579 tx_ring->size = ALIGN(tx_ring->size, 4096);
2580
2581 tx_ring->desc = dma_alloc_coherent(dev,
2582 tx_ring->size,
2583 &tx_ring->dma,
2584 GFP_KERNEL);
2585
2586 if (!tx_ring->desc)
2587 goto err;
2588
2589 tx_ring->next_to_use = 0;
2590 tx_ring->next_to_clean = 0;
2591 return 0;
2592
2593err:
2594 vfree(tx_ring->buffer_info);
2595 dev_err(dev,
2596 "Unable to allocate memory for the transmit descriptor ring\n");
2597 return -ENOMEM;
2598}
2599
2600/**
2601 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2602 * (Descriptors) for all queues
2603 * @adapter: board private structure
2604 *
2605 * Return 0 on success, negative on failure
2606 **/
2607static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2608{
2609 struct pci_dev *pdev = adapter->pdev;
2610 int i, err = 0;
2611
2612 for (i = 0; i < adapter->num_tx_queues; i++) {
2613 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2614 if (err) {
2615 dev_err(&pdev->dev,
2616 "Allocation for Tx Queue %u failed\n", i);
2617 for (i--; i >= 0; i--)
2618 igb_free_tx_resources(adapter->tx_ring[i]);
2619 break;
2620 }
2621 }
2622
2623 for (i = 0; i < IGB_ABS_MAX_TX_QUEUES; i++) {
2624 int r_idx = i % adapter->num_tx_queues;
2625 adapter->multi_tx_table[i] = adapter->tx_ring[r_idx];
2626 }
2627 return err;
2628}
2629
2630/**
2631 * igb_setup_tctl - configure the transmit control registers
2632 * @adapter: Board private structure
2633 **/
2634void igb_setup_tctl(struct igb_adapter *adapter)
2635{
2636 struct e1000_hw *hw = &adapter->hw;
2637 u32 tctl;
2638
2639 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2640 wr32(E1000_TXDCTL(0), 0);
2641
2642 /* Program the Transmit Control Register */
2643 tctl = rd32(E1000_TCTL);
2644 tctl &= ~E1000_TCTL_CT;
2645 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2646 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2647
2648 igb_config_collision_dist(hw);
2649
2650 /* Enable transmits */
2651 tctl |= E1000_TCTL_EN;
2652
2653 wr32(E1000_TCTL, tctl);
2654}
2655
2656/**
2657 * igb_configure_tx_ring - Configure transmit ring after Reset
2658 * @adapter: board private structure
2659 * @ring: tx ring to configure
2660 *
2661 * Configure a transmit ring after a reset.
2662 **/
2663void igb_configure_tx_ring(struct igb_adapter *adapter,
2664 struct igb_ring *ring)
2665{
2666 struct e1000_hw *hw = &adapter->hw;
2667 u32 txdctl;
2668 u64 tdba = ring->dma;
2669 int reg_idx = ring->reg_idx;
2670
2671 /* disable the queue */
2672 txdctl = rd32(E1000_TXDCTL(reg_idx));
2673 wr32(E1000_TXDCTL(reg_idx),
2674 txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
2675 wrfl();
2676 mdelay(10);
2677
2678 wr32(E1000_TDLEN(reg_idx),
2679 ring->count * sizeof(union e1000_adv_tx_desc));
2680 wr32(E1000_TDBAL(reg_idx),
2681 tdba & 0x00000000ffffffffULL);
2682 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2683
2684 ring->head = hw->hw_addr + E1000_TDH(reg_idx);
2685 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2686 writel(0, ring->head);
2687 writel(0, ring->tail);
2688
2689 txdctl |= IGB_TX_PTHRESH;
2690 txdctl |= IGB_TX_HTHRESH << 8;
2691 txdctl |= IGB_TX_WTHRESH << 16;
2692
2693 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2694 wr32(E1000_TXDCTL(reg_idx), txdctl);
2695}
2696
2697/**
2698 * igb_configure_tx - Configure transmit Unit after Reset
2699 * @adapter: board private structure
2700 *
2701 * Configure the Tx unit of the MAC after a reset.
2702 **/
2703static void igb_configure_tx(struct igb_adapter *adapter)
2704{
2705 int i;
2706
2707 for (i = 0; i < adapter->num_tx_queues; i++)
2708 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2709}
2710
2711/**
2712 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2713 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2714 *
2715 * Returns 0 on success, negative on failure
2716 **/
2717int igb_setup_rx_resources(struct igb_ring *rx_ring)
2718{
2719 struct device *dev = rx_ring->dev;
2720 int size, desc_len;
2721
2722 size = sizeof(struct igb_buffer) * rx_ring->count;
2723 rx_ring->buffer_info = vzalloc(size);
2724 if (!rx_ring->buffer_info)
2725 goto err;
2726
2727 desc_len = sizeof(union e1000_adv_rx_desc);
2728
2729 /* Round up to nearest 4K */
2730 rx_ring->size = rx_ring->count * desc_len;
2731 rx_ring->size = ALIGN(rx_ring->size, 4096);
2732
2733 rx_ring->desc = dma_alloc_coherent(dev,
2734 rx_ring->size,
2735 &rx_ring->dma,
2736 GFP_KERNEL);
2737
2738 if (!rx_ring->desc)
2739 goto err;
2740
2741 rx_ring->next_to_clean = 0;
2742 rx_ring->next_to_use = 0;
2743
2744 return 0;
2745
2746err:
2747 vfree(rx_ring->buffer_info);
2748 rx_ring->buffer_info = NULL;
2749 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2750 " ring\n");
2751 return -ENOMEM;
2752}
2753
2754/**
2755 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2756 * (Descriptors) for all queues
2757 * @adapter: board private structure
2758 *
2759 * Return 0 on success, negative on failure
2760 **/
2761static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2762{
2763 struct pci_dev *pdev = adapter->pdev;
2764 int i, err = 0;
2765
2766 for (i = 0; i < adapter->num_rx_queues; i++) {
2767 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2768 if (err) {
2769 dev_err(&pdev->dev,
2770 "Allocation for Rx Queue %u failed\n", i);
2771 for (i--; i >= 0; i--)
2772 igb_free_rx_resources(adapter->rx_ring[i]);
2773 break;
2774 }
2775 }
2776
2777 return err;
2778}
2779
2780/**
2781 * igb_setup_mrqc - configure the multiple receive queue control registers
2782 * @adapter: Board private structure
2783 **/
2784static void igb_setup_mrqc(struct igb_adapter *adapter)
2785{
2786 struct e1000_hw *hw = &adapter->hw;
2787 u32 mrqc, rxcsum;
2788 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2789 union e1000_reta {
2790 u32 dword;
2791 u8 bytes[4];
2792 } reta;
2793 static const u8 rsshash[40] = {
2794 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2795 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2796 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2797 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2798
2799 /* Fill out hash function seeds */
2800 for (j = 0; j < 10; j++) {
2801 u32 rsskey = rsshash[(j * 4)];
2802 rsskey |= rsshash[(j * 4) + 1] << 8;
2803 rsskey |= rsshash[(j * 4) + 2] << 16;
2804 rsskey |= rsshash[(j * 4) + 3] << 24;
2805 array_wr32(E1000_RSSRK(0), j, rsskey);
2806 }
2807
2808 num_rx_queues = adapter->rss_queues;
2809
2810 if (adapter->vfs_allocated_count) {
2811 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2812 switch (hw->mac.type) {
2813 case e1000_i350:
2814 case e1000_82580:
2815 num_rx_queues = 1;
2816 shift = 0;
2817 break;
2818 case e1000_82576:
2819 shift = 3;
2820 num_rx_queues = 2;
2821 break;
2822 case e1000_82575:
2823 shift = 2;
2824 shift2 = 6;
2825 default:
2826 break;
2827 }
2828 } else {
2829 if (hw->mac.type == e1000_82575)
2830 shift = 6;
2831 }
2832
2833 for (j = 0; j < (32 * 4); j++) {
2834 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2835 if (shift2)
2836 reta.bytes[j & 3] |= num_rx_queues << shift2;
2837 if ((j & 3) == 3)
2838 wr32(E1000_RETA(j >> 2), reta.dword);
2839 }
2840
2841 /*
2842 * Disable raw packet checksumming so that RSS hash is placed in
2843 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2844 * offloads as they are enabled by default
2845 */
2846 rxcsum = rd32(E1000_RXCSUM);
2847 rxcsum |= E1000_RXCSUM_PCSD;
2848
2849 if (adapter->hw.mac.type >= e1000_82576)
2850 /* Enable Receive Checksum Offload for SCTP */
2851 rxcsum |= E1000_RXCSUM_CRCOFL;
2852
2853 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2854 wr32(E1000_RXCSUM, rxcsum);
2855
2856 /* If VMDq is enabled then we set the appropriate mode for that, else
2857 * we default to RSS so that an RSS hash is calculated per packet even
2858 * if we are only using one queue */
2859 if (adapter->vfs_allocated_count) {
2860 if (hw->mac.type > e1000_82575) {
2861 /* Set the default pool for the PF's first queue */
2862 u32 vtctl = rd32(E1000_VT_CTL);
2863 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2864 E1000_VT_CTL_DISABLE_DEF_POOL);
2865 vtctl |= adapter->vfs_allocated_count <<
2866 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2867 wr32(E1000_VT_CTL, vtctl);
2868 }
2869 if (adapter->rss_queues > 1)
2870 mrqc = E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2871 else
2872 mrqc = E1000_MRQC_ENABLE_VMDQ;
2873 } else {
2874 mrqc = E1000_MRQC_ENABLE_RSS_4Q;
2875 }
2876 igb_vmm_control(adapter);
2877
2878 /*
2879 * Generate RSS hash based on TCP port numbers and/or
2880 * IPv4/v6 src and dst addresses since UDP cannot be
2881 * hashed reliably due to IP fragmentation
2882 */
2883 mrqc |= E1000_MRQC_RSS_FIELD_IPV4 |
2884 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2885 E1000_MRQC_RSS_FIELD_IPV6 |
2886 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2887 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2888
2889 wr32(E1000_MRQC, mrqc);
2890}
2891
2892/**
2893 * igb_setup_rctl - configure the receive control registers
2894 * @adapter: Board private structure
2895 **/
2896void igb_setup_rctl(struct igb_adapter *adapter)
2897{
2898 struct e1000_hw *hw = &adapter->hw;
2899 u32 rctl;
2900
2901 rctl = rd32(E1000_RCTL);
2902
2903 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2904 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2905
2906 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2907 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2908
2909 /*
2910 * enable stripping of CRC. It's unlikely this will break BMC
2911 * redirection as it did with e1000. Newer features require
2912 * that the HW strips the CRC.
2913 */
2914 rctl |= E1000_RCTL_SECRC;
2915
2916 /* disable store bad packets and clear size bits. */
2917 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2918
2919 /* enable LPE to prevent packets larger than max_frame_size */
2920 rctl |= E1000_RCTL_LPE;
2921
2922 /* disable queue 0 to prevent tail write w/o re-config */
2923 wr32(E1000_RXDCTL(0), 0);
2924
2925 /* Attention!!! For SR-IOV PF driver operations you must enable
2926 * queue drop for all VF and PF queues to prevent head of line blocking
2927 * if an un-trusted VF does not provide descriptors to hardware.
2928 */
2929 if (adapter->vfs_allocated_count) {
2930 /* set all queue drop enable bits */
2931 wr32(E1000_QDE, ALL_QUEUES);
2932 }
2933
2934 wr32(E1000_RCTL, rctl);
2935}
2936
2937static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2938 int vfn)
2939{
2940 struct e1000_hw *hw = &adapter->hw;
2941 u32 vmolr;
2942
2943 /* if it isn't the PF check to see if VFs are enabled and
2944 * increase the size to support vlan tags */
2945 if (vfn < adapter->vfs_allocated_count &&
2946 adapter->vf_data[vfn].vlans_enabled)
2947 size += VLAN_TAG_SIZE;
2948
2949 vmolr = rd32(E1000_VMOLR(vfn));
2950 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2951 vmolr |= size | E1000_VMOLR_LPE;
2952 wr32(E1000_VMOLR(vfn), vmolr);
2953
2954 return 0;
2955}
2956
2957/**
2958 * igb_rlpml_set - set maximum receive packet size
2959 * @adapter: board private structure
2960 *
2961 * Configure maximum receivable packet size.
2962 **/
2963static void igb_rlpml_set(struct igb_adapter *adapter)
2964{
2965 u32 max_frame_size;
2966 struct e1000_hw *hw = &adapter->hw;
2967 u16 pf_id = adapter->vfs_allocated_count;
2968
2969 max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
2970
2971 /* if vfs are enabled we set RLPML to the largest possible request
2972 * size and set the VMOLR RLPML to the size we need */
2973 if (pf_id) {
2974 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
2975 max_frame_size = MAX_JUMBO_FRAME_SIZE;
2976 }
2977
2978 wr32(E1000_RLPML, max_frame_size);
2979}
2980
2981static inline void igb_set_vmolr(struct igb_adapter *adapter,
2982 int vfn, bool aupe)
2983{
2984 struct e1000_hw *hw = &adapter->hw;
2985 u32 vmolr;
2986
2987 /*
2988 * This register exists only on 82576 and newer so if we are older then
2989 * we should exit and do nothing
2990 */
2991 if (hw->mac.type < e1000_82576)
2992 return;
2993
2994 vmolr = rd32(E1000_VMOLR(vfn));
2995 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
2996 if (aupe)
2997 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
2998 else
2999 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3000
3001 /* clear all bits that might not be set */
3002 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3003
3004 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3005 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3006 /*
3007 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3008 * multicast packets
3009 */
3010 if (vfn <= adapter->vfs_allocated_count)
3011 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3012
3013 wr32(E1000_VMOLR(vfn), vmolr);
3014}
3015
3016/**
3017 * igb_configure_rx_ring - Configure a receive ring after Reset
3018 * @adapter: board private structure
3019 * @ring: receive ring to be configured
3020 *
3021 * Configure the Rx unit of the MAC after a reset.
3022 **/
3023void igb_configure_rx_ring(struct igb_adapter *adapter,
3024 struct igb_ring *ring)
3025{
3026 struct e1000_hw *hw = &adapter->hw;
3027 u64 rdba = ring->dma;
3028 int reg_idx = ring->reg_idx;
3029 u32 srrctl, rxdctl;
3030
3031 /* disable the queue */
3032 rxdctl = rd32(E1000_RXDCTL(reg_idx));
3033 wr32(E1000_RXDCTL(reg_idx),
3034 rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
3035
3036 /* Set DMA base address registers */
3037 wr32(E1000_RDBAL(reg_idx),
3038 rdba & 0x00000000ffffffffULL);
3039 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3040 wr32(E1000_RDLEN(reg_idx),
3041 ring->count * sizeof(union e1000_adv_rx_desc));
3042
3043 /* initialize head and tail */
3044 ring->head = hw->hw_addr + E1000_RDH(reg_idx);
3045 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3046 writel(0, ring->head);
3047 writel(0, ring->tail);
3048
3049 /* set descriptor configuration */
3050 if (ring->rx_buffer_len < IGB_RXBUFFER_1024) {
3051 srrctl = ALIGN(ring->rx_buffer_len, 64) <<
3052 E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3053#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3054 srrctl |= IGB_RXBUFFER_16384 >>
3055 E1000_SRRCTL_BSIZEPKT_SHIFT;
3056#else
3057 srrctl |= (PAGE_SIZE / 2) >>
3058 E1000_SRRCTL_BSIZEPKT_SHIFT;
3059#endif
3060 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3061 } else {
3062 srrctl = ALIGN(ring->rx_buffer_len, 1024) >>
3063 E1000_SRRCTL_BSIZEPKT_SHIFT;
3064 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
3065 }
3066 if (hw->mac.type == e1000_82580)
3067 srrctl |= E1000_SRRCTL_TIMESTAMP;
3068 /* Only set Drop Enable if we are supporting multiple queues */
3069 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3070 srrctl |= E1000_SRRCTL_DROP_EN;
3071
3072 wr32(E1000_SRRCTL(reg_idx), srrctl);
3073
3074 /* set filtering for VMDQ pools */
3075 igb_set_vmolr(adapter, reg_idx & 0x7, true);
3076
3077 /* enable receive descriptor fetching */
3078 rxdctl = rd32(E1000_RXDCTL(reg_idx));
3079 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3080 rxdctl &= 0xFFF00000;
3081 rxdctl |= IGB_RX_PTHRESH;
3082 rxdctl |= IGB_RX_HTHRESH << 8;
3083 rxdctl |= IGB_RX_WTHRESH << 16;
3084 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3085}
3086
3087/**
3088 * igb_configure_rx - Configure receive Unit after Reset
3089 * @adapter: board private structure
3090 *
3091 * Configure the Rx unit of the MAC after a reset.
3092 **/
3093static void igb_configure_rx(struct igb_adapter *adapter)
3094{
3095 int i;
3096
3097 /* set UTA to appropriate mode */
3098 igb_set_uta(adapter);
3099
3100 /* set the correct pool for the PF default MAC address in entry 0 */
3101 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3102 adapter->vfs_allocated_count);
3103
3104 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3105 * the Base and Length of the Rx Descriptor Ring */
3106 for (i = 0; i < adapter->num_rx_queues; i++)
3107 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3108}
3109
3110/**
3111 * igb_free_tx_resources - Free Tx Resources per Queue
3112 * @tx_ring: Tx descriptor ring for a specific queue
3113 *
3114 * Free all transmit software resources
3115 **/
3116void igb_free_tx_resources(struct igb_ring *tx_ring)
3117{
3118 igb_clean_tx_ring(tx_ring);
3119
3120 vfree(tx_ring->buffer_info);
3121 tx_ring->buffer_info = NULL;
3122
3123 /* if not set, then don't free */
3124 if (!tx_ring->desc)
3125 return;
3126
3127 dma_free_coherent(tx_ring->dev, tx_ring->size,
3128 tx_ring->desc, tx_ring->dma);
3129
3130 tx_ring->desc = NULL;
3131}
3132
3133/**
3134 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3135 * @adapter: board private structure
3136 *
3137 * Free all transmit software resources
3138 **/
3139static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3140{
3141 int i;
3142
3143 for (i = 0; i < adapter->num_tx_queues; i++)
3144 igb_free_tx_resources(adapter->tx_ring[i]);
3145}
3146
3147void igb_unmap_and_free_tx_resource(struct igb_ring *tx_ring,
3148 struct igb_buffer *buffer_info)
3149{
3150 if (buffer_info->dma) {
3151 if (buffer_info->mapped_as_page)
3152 dma_unmap_page(tx_ring->dev,
3153 buffer_info->dma,
3154 buffer_info->length,
3155 DMA_TO_DEVICE);
3156 else
3157 dma_unmap_single(tx_ring->dev,
3158 buffer_info->dma,
3159 buffer_info->length,
3160 DMA_TO_DEVICE);
3161 buffer_info->dma = 0;
3162 }
3163 if (buffer_info->skb) {
3164 dev_kfree_skb_any(buffer_info->skb);
3165 buffer_info->skb = NULL;
3166 }
3167 buffer_info->time_stamp = 0;
3168 buffer_info->length = 0;
3169 buffer_info->next_to_watch = 0;
3170 buffer_info->mapped_as_page = false;
3171}
3172
3173/**
3174 * igb_clean_tx_ring - Free Tx Buffers
3175 * @tx_ring: ring to be cleaned
3176 **/
3177static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3178{
3179 struct igb_buffer *buffer_info;
3180 unsigned long size;
3181 unsigned int i;
3182
3183 if (!tx_ring->buffer_info)
3184 return;
3185 /* Free all the Tx ring sk_buffs */
3186
3187 for (i = 0; i < tx_ring->count; i++) {
3188 buffer_info = &tx_ring->buffer_info[i];
3189 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3190 }
3191
3192 size = sizeof(struct igb_buffer) * tx_ring->count;
3193 memset(tx_ring->buffer_info, 0, size);
3194
3195 /* Zero out the descriptor ring */
3196 memset(tx_ring->desc, 0, tx_ring->size);
3197
3198 tx_ring->next_to_use = 0;
3199 tx_ring->next_to_clean = 0;
3200}
3201
3202/**
3203 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3204 * @adapter: board private structure
3205 **/
3206static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3207{
3208 int i;
3209
3210 for (i = 0; i < adapter->num_tx_queues; i++)
3211 igb_clean_tx_ring(adapter->tx_ring[i]);
3212}
3213
3214/**
3215 * igb_free_rx_resources - Free Rx Resources
3216 * @rx_ring: ring to clean the resources from
3217 *
3218 * Free all receive software resources
3219 **/
3220void igb_free_rx_resources(struct igb_ring *rx_ring)
3221{
3222 igb_clean_rx_ring(rx_ring);
3223
3224 vfree(rx_ring->buffer_info);
3225 rx_ring->buffer_info = NULL;
3226
3227 /* if not set, then don't free */
3228 if (!rx_ring->desc)
3229 return;
3230
3231 dma_free_coherent(rx_ring->dev, rx_ring->size,
3232 rx_ring->desc, rx_ring->dma);
3233
3234 rx_ring->desc = NULL;
3235}
3236
3237/**
3238 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3239 * @adapter: board private structure
3240 *
3241 * Free all receive software resources
3242 **/
3243static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3244{
3245 int i;
3246
3247 for (i = 0; i < adapter->num_rx_queues; i++)
3248 igb_free_rx_resources(adapter->rx_ring[i]);
3249}
3250
3251/**
3252 * igb_clean_rx_ring - Free Rx Buffers per Queue
3253 * @rx_ring: ring to free buffers from
3254 **/
3255static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3256{
3257 struct igb_buffer *buffer_info;
3258 unsigned long size;
3259 unsigned int i;
3260
3261 if (!rx_ring->buffer_info)
3262 return;
3263
3264 /* Free all the Rx ring sk_buffs */
3265 for (i = 0; i < rx_ring->count; i++) {
3266 buffer_info = &rx_ring->buffer_info[i];
3267 if (buffer_info->dma) {
3268 dma_unmap_single(rx_ring->dev,
3269 buffer_info->dma,
3270 rx_ring->rx_buffer_len,
3271 DMA_FROM_DEVICE);
3272 buffer_info->dma = 0;
3273 }
3274
3275 if (buffer_info->skb) {
3276 dev_kfree_skb(buffer_info->skb);
3277 buffer_info->skb = NULL;
3278 }
3279 if (buffer_info->page_dma) {
3280 dma_unmap_page(rx_ring->dev,
3281 buffer_info->page_dma,
3282 PAGE_SIZE / 2,
3283 DMA_FROM_DEVICE);
3284 buffer_info->page_dma = 0;
3285 }
3286 if (buffer_info->page) {
3287 put_page(buffer_info->page);
3288 buffer_info->page = NULL;
3289 buffer_info->page_offset = 0;
3290 }
3291 }
3292
3293 size = sizeof(struct igb_buffer) * rx_ring->count;
3294 memset(rx_ring->buffer_info, 0, size);
3295
3296 /* Zero out the descriptor ring */
3297 memset(rx_ring->desc, 0, rx_ring->size);
3298
3299 rx_ring->next_to_clean = 0;
3300 rx_ring->next_to_use = 0;
3301}
3302
3303/**
3304 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3305 * @adapter: board private structure
3306 **/
3307static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3308{
3309 int i;
3310
3311 for (i = 0; i < adapter->num_rx_queues; i++)
3312 igb_clean_rx_ring(adapter->rx_ring[i]);
3313}
3314
3315/**
3316 * igb_set_mac - Change the Ethernet Address of the NIC
3317 * @netdev: network interface device structure
3318 * @p: pointer to an address structure
3319 *
3320 * Returns 0 on success, negative on failure
3321 **/
3322static int igb_set_mac(struct net_device *netdev, void *p)
3323{
3324 struct igb_adapter *adapter = netdev_priv(netdev);
3325 struct e1000_hw *hw = &adapter->hw;
3326 struct sockaddr *addr = p;
3327
3328 if (!is_valid_ether_addr(addr->sa_data))
3329 return -EADDRNOTAVAIL;
3330
3331 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3332 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3333
3334 /* set the correct pool for the new PF MAC address in entry 0 */
3335 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3336 adapter->vfs_allocated_count);
3337
3338 return 0;
3339}
3340
3341/**
3342 * igb_write_mc_addr_list - write multicast addresses to MTA
3343 * @netdev: network interface device structure
3344 *
3345 * Writes multicast address list to the MTA hash table.
3346 * Returns: -ENOMEM on failure
3347 * 0 on no addresses written
3348 * X on writing X addresses to MTA
3349 **/
3350static int igb_write_mc_addr_list(struct net_device *netdev)
3351{
3352 struct igb_adapter *adapter = netdev_priv(netdev);
3353 struct e1000_hw *hw = &adapter->hw;
3354 struct netdev_hw_addr *ha;
3355 u8 *mta_list;
3356 int i;
3357
3358 if (netdev_mc_empty(netdev)) {
3359 /* nothing to program, so clear mc list */
3360 igb_update_mc_addr_list(hw, NULL, 0);
3361 igb_restore_vf_multicasts(adapter);
3362 return 0;
3363 }
3364
3365 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3366 if (!mta_list)
3367 return -ENOMEM;
3368
3369 /* The shared function expects a packed array of only addresses. */
3370 i = 0;
3371 netdev_for_each_mc_addr(ha, netdev)
3372 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3373
3374 igb_update_mc_addr_list(hw, mta_list, i);
3375 kfree(mta_list);
3376
3377 return netdev_mc_count(netdev);
3378}
3379
3380/**
3381 * igb_write_uc_addr_list - write unicast addresses to RAR table
3382 * @netdev: network interface device structure
3383 *
3384 * Writes unicast address list to the RAR table.
3385 * Returns: -ENOMEM on failure/insufficient address space
3386 * 0 on no addresses written
3387 * X on writing X addresses to the RAR table
3388 **/
3389static int igb_write_uc_addr_list(struct net_device *netdev)
3390{
3391 struct igb_adapter *adapter = netdev_priv(netdev);
3392 struct e1000_hw *hw = &adapter->hw;
3393 unsigned int vfn = adapter->vfs_allocated_count;
3394 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3395 int count = 0;
3396
3397 /* return ENOMEM indicating insufficient memory for addresses */
3398 if (netdev_uc_count(netdev) > rar_entries)
3399 return -ENOMEM;
3400
3401 if (!netdev_uc_empty(netdev) && rar_entries) {
3402 struct netdev_hw_addr *ha;
3403
3404 netdev_for_each_uc_addr(ha, netdev) {
3405 if (!rar_entries)
3406 break;
3407 igb_rar_set_qsel(adapter, ha->addr,
3408 rar_entries--,
3409 vfn);
3410 count++;
3411 }
3412 }
3413 /* write the addresses in reverse order to avoid write combining */
3414 for (; rar_entries > 0 ; rar_entries--) {
3415 wr32(E1000_RAH(rar_entries), 0);
3416 wr32(E1000_RAL(rar_entries), 0);
3417 }
3418 wrfl();
3419
3420 return count;
3421}
3422
3423/**
3424 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3425 * @netdev: network interface device structure
3426 *
3427 * The set_rx_mode entry point is called whenever the unicast or multicast
3428 * address lists or the network interface flags are updated. This routine is
3429 * responsible for configuring the hardware for proper unicast, multicast,
3430 * promiscuous mode, and all-multi behavior.
3431 **/
3432static void igb_set_rx_mode(struct net_device *netdev)
3433{
3434 struct igb_adapter *adapter = netdev_priv(netdev);
3435 struct e1000_hw *hw = &adapter->hw;
3436 unsigned int vfn = adapter->vfs_allocated_count;
3437 u32 rctl, vmolr = 0;
3438 int count;
3439
3440 /* Check for Promiscuous and All Multicast modes */
3441 rctl = rd32(E1000_RCTL);
3442
3443 /* clear the effected bits */
3444 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3445
3446 if (netdev->flags & IFF_PROMISC) {
3447 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3448 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3449 } else {
3450 if (netdev->flags & IFF_ALLMULTI) {
3451 rctl |= E1000_RCTL_MPE;
3452 vmolr |= E1000_VMOLR_MPME;
3453 } else {
3454 /*
3455 * Write addresses to the MTA, if the attempt fails
3456 * then we should just turn on promiscuous mode so
3457 * that we can at least receive multicast traffic
3458 */
3459 count = igb_write_mc_addr_list(netdev);
3460 if (count < 0) {
3461 rctl |= E1000_RCTL_MPE;
3462 vmolr |= E1000_VMOLR_MPME;
3463 } else if (count) {
3464 vmolr |= E1000_VMOLR_ROMPE;
3465 }
3466 }
3467 /*
3468 * Write addresses to available RAR registers, if there is not
3469 * sufficient space to store all the addresses then enable
3470 * unicast promiscuous mode
3471 */
3472 count = igb_write_uc_addr_list(netdev);
3473 if (count < 0) {
3474 rctl |= E1000_RCTL_UPE;
3475 vmolr |= E1000_VMOLR_ROPE;
3476 }
3477 rctl |= E1000_RCTL_VFE;
3478 }
3479 wr32(E1000_RCTL, rctl);
3480
3481 /*
3482 * In order to support SR-IOV and eventually VMDq it is necessary to set
3483 * the VMOLR to enable the appropriate modes. Without this workaround
3484 * we will have issues with VLAN tag stripping not being done for frames
3485 * that are only arriving because we are the default pool
3486 */
3487 if (hw->mac.type < e1000_82576)
3488 return;
3489
3490 vmolr |= rd32(E1000_VMOLR(vfn)) &
3491 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3492 wr32(E1000_VMOLR(vfn), vmolr);
3493 igb_restore_vf_multicasts(adapter);
3494}
3495
3496static void igb_check_wvbr(struct igb_adapter *adapter)
3497{
3498 struct e1000_hw *hw = &adapter->hw;
3499 u32 wvbr = 0;
3500
3501 switch (hw->mac.type) {
3502 case e1000_82576:
3503 case e1000_i350:
3504 if (!(wvbr = rd32(E1000_WVBR)))
3505 return;
3506 break;
3507 default:
3508 break;
3509 }
3510
3511 adapter->wvbr |= wvbr;
3512}
3513
3514#define IGB_STAGGERED_QUEUE_OFFSET 8
3515
3516static void igb_spoof_check(struct igb_adapter *adapter)
3517{
3518 int j;
3519
3520 if (!adapter->wvbr)
3521 return;
3522
3523 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3524 if (adapter->wvbr & (1 << j) ||
3525 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3526 dev_warn(&adapter->pdev->dev,
3527 "Spoof event(s) detected on VF %d\n", j);
3528 adapter->wvbr &=
3529 ~((1 << j) |
3530 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3531 }
3532 }
3533}
3534
3535/* Need to wait a few seconds after link up to get diagnostic information from
3536 * the phy */
3537static void igb_update_phy_info(unsigned long data)
3538{
3539 struct igb_adapter *adapter = (struct igb_adapter *) data;
3540 igb_get_phy_info(&adapter->hw);
3541}
3542
3543/**
3544 * igb_has_link - check shared code for link and determine up/down
3545 * @adapter: pointer to driver private info
3546 **/
3547bool igb_has_link(struct igb_adapter *adapter)
3548{
3549 struct e1000_hw *hw = &adapter->hw;
3550 bool link_active = false;
3551 s32 ret_val = 0;
3552
3553 /* get_link_status is set on LSC (link status) interrupt or
3554 * rx sequence error interrupt. get_link_status will stay
3555 * false until the e1000_check_for_link establishes link
3556 * for copper adapters ONLY
3557 */
3558 switch (hw->phy.media_type) {
3559 case e1000_media_type_copper:
3560 if (hw->mac.get_link_status) {
3561 ret_val = hw->mac.ops.check_for_link(hw);
3562 link_active = !hw->mac.get_link_status;
3563 } else {
3564 link_active = true;
3565 }
3566 break;
3567 case e1000_media_type_internal_serdes:
3568 ret_val = hw->mac.ops.check_for_link(hw);
3569 link_active = hw->mac.serdes_has_link;
3570 break;
3571 default:
3572 case e1000_media_type_unknown:
3573 break;
3574 }
3575
3576 return link_active;
3577}
3578
3579static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3580{
3581 bool ret = false;
3582 u32 ctrl_ext, thstat;
3583
3584 /* check for thermal sensor event on i350, copper only */
3585 if (hw->mac.type == e1000_i350) {
3586 thstat = rd32(E1000_THSTAT);
3587 ctrl_ext = rd32(E1000_CTRL_EXT);
3588
3589 if ((hw->phy.media_type == e1000_media_type_copper) &&
3590 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3591 ret = !!(thstat & event);
3592 }
3593 }
3594
3595 return ret;
3596}
3597
3598/**
3599 * igb_watchdog - Timer Call-back
3600 * @data: pointer to adapter cast into an unsigned long
3601 **/
3602static void igb_watchdog(unsigned long data)
3603{
3604 struct igb_adapter *adapter = (struct igb_adapter *)data;
3605 /* Do the rest outside of interrupt context */
3606 schedule_work(&adapter->watchdog_task);
3607}
3608
3609static void igb_watchdog_task(struct work_struct *work)
3610{
3611 struct igb_adapter *adapter = container_of(work,
3612 struct igb_adapter,
3613 watchdog_task);
3614 struct e1000_hw *hw = &adapter->hw;
3615 struct net_device *netdev = adapter->netdev;
3616 u32 link;
3617 int i;
3618
3619 link = igb_has_link(adapter);
3620 if (link) {
3621 if (!netif_carrier_ok(netdev)) {
3622 u32 ctrl;
3623 hw->mac.ops.get_speed_and_duplex(hw,
3624 &adapter->link_speed,
3625 &adapter->link_duplex);
3626
3627 ctrl = rd32(E1000_CTRL);
3628 /* Links status message must follow this format */
3629 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s, "
3630 "Flow Control: %s\n",
3631 netdev->name,
3632 adapter->link_speed,
3633 adapter->link_duplex == FULL_DUPLEX ?
3634 "Full Duplex" : "Half Duplex",
3635 ((ctrl & E1000_CTRL_TFCE) &&
3636 (ctrl & E1000_CTRL_RFCE)) ? "RX/TX" :
3637 ((ctrl & E1000_CTRL_RFCE) ? "RX" :
3638 ((ctrl & E1000_CTRL_TFCE) ? "TX" : "None")));
3639
3640 /* check for thermal sensor event */
3641 if (igb_thermal_sensor_event(hw, E1000_THSTAT_LINK_THROTTLE)) {
3642 printk(KERN_INFO "igb: %s The network adapter "
3643 "link speed was downshifted "
3644 "because it overheated.\n",
3645 netdev->name);
3646 }
3647
3648 /* adjust timeout factor according to speed/duplex */
3649 adapter->tx_timeout_factor = 1;
3650 switch (adapter->link_speed) {
3651 case SPEED_10:
3652 adapter->tx_timeout_factor = 14;
3653 break;
3654 case SPEED_100:
3655 /* maybe add some timeout factor ? */
3656 break;
3657 }
3658
3659 netif_carrier_on(netdev);
3660
3661 igb_ping_all_vfs(adapter);
3662 igb_check_vf_rate_limit(adapter);
3663
3664 /* link state has changed, schedule phy info update */
3665 if (!test_bit(__IGB_DOWN, &adapter->state))
3666 mod_timer(&adapter->phy_info_timer,
3667 round_jiffies(jiffies + 2 * HZ));
3668 }
3669 } else {
3670 if (netif_carrier_ok(netdev)) {
3671 adapter->link_speed = 0;
3672 adapter->link_duplex = 0;
3673
3674 /* check for thermal sensor event */
3675 if (igb_thermal_sensor_event(hw, E1000_THSTAT_PWR_DOWN)) {
3676 printk(KERN_ERR "igb: %s The network adapter "
3677 "was stopped because it "
3678 "overheated.\n",
3679 netdev->name);
3680 }
3681
3682 /* Links status message must follow this format */
3683 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3684 netdev->name);
3685 netif_carrier_off(netdev);
3686
3687 igb_ping_all_vfs(adapter);
3688
3689 /* link state has changed, schedule phy info update */
3690 if (!test_bit(__IGB_DOWN, &adapter->state))
3691 mod_timer(&adapter->phy_info_timer,
3692 round_jiffies(jiffies + 2 * HZ));
3693 }
3694 }
3695
3696 spin_lock(&adapter->stats64_lock);
3697 igb_update_stats(adapter, &adapter->stats64);
3698 spin_unlock(&adapter->stats64_lock);
3699
3700 for (i = 0; i < adapter->num_tx_queues; i++) {
3701 struct igb_ring *tx_ring = adapter->tx_ring[i];
3702 if (!netif_carrier_ok(netdev)) {
3703 /* We've lost link, so the controller stops DMA,
3704 * but we've got queued Tx work that's never going
3705 * to get done, so reset controller to flush Tx.
3706 * (Do the reset outside of interrupt context). */
3707 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3708 adapter->tx_timeout_count++;
3709 schedule_work(&adapter->reset_task);
3710 /* return immediately since reset is imminent */
3711 return;
3712 }
3713 }
3714
3715 /* Force detection of hung controller every watchdog period */
3716 tx_ring->detect_tx_hung = true;
3717 }
3718
3719 /* Cause software interrupt to ensure rx ring is cleaned */
3720 if (adapter->msix_entries) {
3721 u32 eics = 0;
3722 for (i = 0; i < adapter->num_q_vectors; i++) {
3723 struct igb_q_vector *q_vector = adapter->q_vector[i];
3724 eics |= q_vector->eims_value;
3725 }
3726 wr32(E1000_EICS, eics);
3727 } else {
3728 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3729 }
3730
3731 igb_spoof_check(adapter);
3732
3733 /* Reset the timer */
3734 if (!test_bit(__IGB_DOWN, &adapter->state))
3735 mod_timer(&adapter->watchdog_timer,
3736 round_jiffies(jiffies + 2 * HZ));
3737}
3738
3739enum latency_range {
3740 lowest_latency = 0,
3741 low_latency = 1,
3742 bulk_latency = 2,
3743 latency_invalid = 255
3744};
3745
3746/**
3747 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3748 *
3749 * Stores a new ITR value based on strictly on packet size. This
3750 * algorithm is less sophisticated than that used in igb_update_itr,
3751 * due to the difficulty of synchronizing statistics across multiple
3752 * receive rings. The divisors and thresholds used by this function
3753 * were determined based on theoretical maximum wire speed and testing
3754 * data, in order to minimize response time while increasing bulk
3755 * throughput.
3756 * This functionality is controlled by the InterruptThrottleRate module
3757 * parameter (see igb_param.c)
3758 * NOTE: This function is called only when operating in a multiqueue
3759 * receive environment.
3760 * @q_vector: pointer to q_vector
3761 **/
3762static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3763{
3764 int new_val = q_vector->itr_val;
3765 int avg_wire_size = 0;
3766 struct igb_adapter *adapter = q_vector->adapter;
3767 struct igb_ring *ring;
3768 unsigned int packets;
3769
3770 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3771 * ints/sec - ITR timer value of 120 ticks.
3772 */
3773 if (adapter->link_speed != SPEED_1000) {
3774 new_val = 976;
3775 goto set_itr_val;
3776 }
3777
3778 ring = q_vector->rx_ring;
3779 if (ring) {
3780 packets = ACCESS_ONCE(ring->total_packets);
3781
3782 if (packets)
3783 avg_wire_size = ring->total_bytes / packets;
3784 }
3785
3786 ring = q_vector->tx_ring;
3787 if (ring) {
3788 packets = ACCESS_ONCE(ring->total_packets);
3789
3790 if (packets)
3791 avg_wire_size = max_t(u32, avg_wire_size,
3792 ring->total_bytes / packets);
3793 }
3794
3795 /* if avg_wire_size isn't set no work was done */
3796 if (!avg_wire_size)
3797 goto clear_counts;
3798
3799 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3800 avg_wire_size += 24;
3801
3802 /* Don't starve jumbo frames */
3803 avg_wire_size = min(avg_wire_size, 3000);
3804
3805 /* Give a little boost to mid-size frames */
3806 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3807 new_val = avg_wire_size / 3;
3808 else
3809 new_val = avg_wire_size / 2;
3810
3811 /* when in itr mode 3 do not exceed 20K ints/sec */
3812 if (adapter->rx_itr_setting == 3 && new_val < 196)
3813 new_val = 196;
3814
3815set_itr_val:
3816 if (new_val != q_vector->itr_val) {
3817 q_vector->itr_val = new_val;
3818 q_vector->set_itr = 1;
3819 }
3820clear_counts:
3821 if (q_vector->rx_ring) {
3822 q_vector->rx_ring->total_bytes = 0;
3823 q_vector->rx_ring->total_packets = 0;
3824 }
3825 if (q_vector->tx_ring) {
3826 q_vector->tx_ring->total_bytes = 0;
3827 q_vector->tx_ring->total_packets = 0;
3828 }
3829}
3830
3831/**
3832 * igb_update_itr - update the dynamic ITR value based on statistics
3833 * Stores a new ITR value based on packets and byte
3834 * counts during the last interrupt. The advantage of per interrupt
3835 * computation is faster updates and more accurate ITR for the current
3836 * traffic pattern. Constants in this function were computed
3837 * based on theoretical maximum wire speed and thresholds were set based
3838 * on testing data as well as attempting to minimize response time
3839 * while increasing bulk throughput.
3840 * this functionality is controlled by the InterruptThrottleRate module
3841 * parameter (see igb_param.c)
3842 * NOTE: These calculations are only valid when operating in a single-
3843 * queue environment.
3844 * @adapter: pointer to adapter
3845 * @itr_setting: current q_vector->itr_val
3846 * @packets: the number of packets during this measurement interval
3847 * @bytes: the number of bytes during this measurement interval
3848 **/
3849static unsigned int igb_update_itr(struct igb_adapter *adapter, u16 itr_setting,
3850 int packets, int bytes)
3851{
3852 unsigned int retval = itr_setting;
3853
3854 if (packets == 0)
3855 goto update_itr_done;
3856
3857 switch (itr_setting) {
3858 case lowest_latency:
3859 /* handle TSO and jumbo frames */
3860 if (bytes/packets > 8000)
3861 retval = bulk_latency;
3862 else if ((packets < 5) && (bytes > 512))
3863 retval = low_latency;
3864 break;
3865 case low_latency: /* 50 usec aka 20000 ints/s */
3866 if (bytes > 10000) {
3867 /* this if handles the TSO accounting */
3868 if (bytes/packets > 8000) {
3869 retval = bulk_latency;
3870 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3871 retval = bulk_latency;
3872 } else if ((packets > 35)) {
3873 retval = lowest_latency;
3874 }
3875 } else if (bytes/packets > 2000) {
3876 retval = bulk_latency;
3877 } else if (packets <= 2 && bytes < 512) {
3878 retval = lowest_latency;
3879 }
3880 break;
3881 case bulk_latency: /* 250 usec aka 4000 ints/s */
3882 if (bytes > 25000) {
3883 if (packets > 35)
3884 retval = low_latency;
3885 } else if (bytes < 1500) {
3886 retval = low_latency;
3887 }
3888 break;
3889 }
3890
3891update_itr_done:
3892 return retval;
3893}
3894
3895static void igb_set_itr(struct igb_adapter *adapter)
3896{
3897 struct igb_q_vector *q_vector = adapter->q_vector[0];
3898 u16 current_itr;
3899 u32 new_itr = q_vector->itr_val;
3900
3901 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3902 if (adapter->link_speed != SPEED_1000) {
3903 current_itr = 0;
3904 new_itr = 4000;
3905 goto set_itr_now;
3906 }
3907
3908 adapter->rx_itr = igb_update_itr(adapter,
3909 adapter->rx_itr,
3910 q_vector->rx_ring->total_packets,
3911 q_vector->rx_ring->total_bytes);
3912
3913 adapter->tx_itr = igb_update_itr(adapter,
3914 adapter->tx_itr,
3915 q_vector->tx_ring->total_packets,
3916 q_vector->tx_ring->total_bytes);
3917 current_itr = max(adapter->rx_itr, adapter->tx_itr);
3918
3919 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3920 if (adapter->rx_itr_setting == 3 && current_itr == lowest_latency)
3921 current_itr = low_latency;
3922
3923 switch (current_itr) {
3924 /* counts and packets in update_itr are dependent on these numbers */
3925 case lowest_latency:
3926 new_itr = 56; /* aka 70,000 ints/sec */
3927 break;
3928 case low_latency:
3929 new_itr = 196; /* aka 20,000 ints/sec */
3930 break;
3931 case bulk_latency:
3932 new_itr = 980; /* aka 4,000 ints/sec */
3933 break;
3934 default:
3935 break;
3936 }
3937
3938set_itr_now:
3939 q_vector->rx_ring->total_bytes = 0;
3940 q_vector->rx_ring->total_packets = 0;
3941 q_vector->tx_ring->total_bytes = 0;
3942 q_vector->tx_ring->total_packets = 0;
3943
3944 if (new_itr != q_vector->itr_val) {
3945 /* this attempts to bias the interrupt rate towards Bulk
3946 * by adding intermediate steps when interrupt rate is
3947 * increasing */
3948 new_itr = new_itr > q_vector->itr_val ?
3949 max((new_itr * q_vector->itr_val) /
3950 (new_itr + (q_vector->itr_val >> 2)),
3951 new_itr) :
3952 new_itr;
3953 /* Don't write the value here; it resets the adapter's
3954 * internal timer, and causes us to delay far longer than
3955 * we should between interrupts. Instead, we write the ITR
3956 * value at the beginning of the next interrupt so the timing
3957 * ends up being correct.
3958 */
3959 q_vector->itr_val = new_itr;
3960 q_vector->set_itr = 1;
3961 }
3962}
3963
3964#define IGB_TX_FLAGS_CSUM 0x00000001
3965#define IGB_TX_FLAGS_VLAN 0x00000002
3966#define IGB_TX_FLAGS_TSO 0x00000004
3967#define IGB_TX_FLAGS_IPV4 0x00000008
3968#define IGB_TX_FLAGS_TSTAMP 0x00000010
3969#define IGB_TX_FLAGS_VLAN_MASK 0xffff0000
3970#define IGB_TX_FLAGS_VLAN_SHIFT 16
3971
3972static inline int igb_tso_adv(struct igb_ring *tx_ring,
3973 struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
3974{
3975 struct e1000_adv_tx_context_desc *context_desc;
3976 unsigned int i;
3977 int err;
3978 struct igb_buffer *buffer_info;
3979 u32 info = 0, tu_cmd = 0;
3980 u32 mss_l4len_idx;
3981 u8 l4len;
3982
3983 if (skb_header_cloned(skb)) {
3984 err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
3985 if (err)
3986 return err;
3987 }
3988
3989 l4len = tcp_hdrlen(skb);
3990 *hdr_len += l4len;
3991
3992 if (skb->protocol == htons(ETH_P_IP)) {
3993 struct iphdr *iph = ip_hdr(skb);
3994 iph->tot_len = 0;
3995 iph->check = 0;
3996 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
3997 iph->daddr, 0,
3998 IPPROTO_TCP,
3999 0);
4000 } else if (skb_is_gso_v6(skb)) {
4001 ipv6_hdr(skb)->payload_len = 0;
4002 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4003 &ipv6_hdr(skb)->daddr,
4004 0, IPPROTO_TCP, 0);
4005 }
4006
4007 i = tx_ring->next_to_use;
4008
4009 buffer_info = &tx_ring->buffer_info[i];
4010 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
4011 /* VLAN MACLEN IPLEN */
4012 if (tx_flags & IGB_TX_FLAGS_VLAN)
4013 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
4014 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
4015 *hdr_len += skb_network_offset(skb);
4016 info |= skb_network_header_len(skb);
4017 *hdr_len += skb_network_header_len(skb);
4018 context_desc->vlan_macip_lens = cpu_to_le32(info);
4019
4020 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4021 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
4022
4023 if (skb->protocol == htons(ETH_P_IP))
4024 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
4025 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4026
4027 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
4028
4029 /* MSS L4LEN IDX */
4030 mss_l4len_idx = (skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT);
4031 mss_l4len_idx |= (l4len << E1000_ADVTXD_L4LEN_SHIFT);
4032
4033 /* For 82575, context index must be unique per ring. */
4034 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
4035 mss_l4len_idx |= tx_ring->reg_idx << 4;
4036
4037 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4038 context_desc->seqnum_seed = 0;
4039
4040 buffer_info->time_stamp = jiffies;
4041 buffer_info->next_to_watch = i;
4042 buffer_info->dma = 0;
4043 i++;
4044 if (i == tx_ring->count)
4045 i = 0;
4046
4047 tx_ring->next_to_use = i;
4048
4049 return true;
4050}
4051
4052static inline bool igb_tx_csum_adv(struct igb_ring *tx_ring,
4053 struct sk_buff *skb, u32 tx_flags)
4054{
4055 struct e1000_adv_tx_context_desc *context_desc;
4056 struct device *dev = tx_ring->dev;
4057 struct igb_buffer *buffer_info;
4058 u32 info = 0, tu_cmd = 0;
4059 unsigned int i;
4060
4061 if ((skb->ip_summed == CHECKSUM_PARTIAL) ||
4062 (tx_flags & IGB_TX_FLAGS_VLAN)) {
4063 i = tx_ring->next_to_use;
4064 buffer_info = &tx_ring->buffer_info[i];
4065 context_desc = E1000_TX_CTXTDESC_ADV(*tx_ring, i);
4066
4067 if (tx_flags & IGB_TX_FLAGS_VLAN)
4068 info |= (tx_flags & IGB_TX_FLAGS_VLAN_MASK);
4069
4070 info |= (skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT);
4071 if (skb->ip_summed == CHECKSUM_PARTIAL)
4072 info |= skb_network_header_len(skb);
4073
4074 context_desc->vlan_macip_lens = cpu_to_le32(info);
4075
4076 tu_cmd |= (E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT);
4077
4078 if (skb->ip_summed == CHECKSUM_PARTIAL) {
4079 __be16 protocol;
4080
4081 if (skb->protocol == cpu_to_be16(ETH_P_8021Q)) {
4082 const struct vlan_ethhdr *vhdr =
4083 (const struct vlan_ethhdr*)skb->data;
4084
4085 protocol = vhdr->h_vlan_encapsulated_proto;
4086 } else {
4087 protocol = skb->protocol;
4088 }
4089
4090 switch (protocol) {
4091 case cpu_to_be16(ETH_P_IP):
4092 tu_cmd |= E1000_ADVTXD_TUCMD_IPV4;
4093 if (ip_hdr(skb)->protocol == IPPROTO_TCP)
4094 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4095 else if (ip_hdr(skb)->protocol == IPPROTO_SCTP)
4096 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4097 break;
4098 case cpu_to_be16(ETH_P_IPV6):
4099 /* XXX what about other V6 headers?? */
4100 if (ipv6_hdr(skb)->nexthdr == IPPROTO_TCP)
4101 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4102 else if (ipv6_hdr(skb)->nexthdr == IPPROTO_SCTP)
4103 tu_cmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4104 break;
4105 default:
4106 if (unlikely(net_ratelimit()))
4107 dev_warn(dev,
4108 "partial checksum but proto=%x!\n",
4109 skb->protocol);
4110 break;
4111 }
4112 }
4113
4114 context_desc->type_tucmd_mlhl = cpu_to_le32(tu_cmd);
4115 context_desc->seqnum_seed = 0;
4116 if (tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX)
4117 context_desc->mss_l4len_idx =
4118 cpu_to_le32(tx_ring->reg_idx << 4);
4119
4120 buffer_info->time_stamp = jiffies;
4121 buffer_info->next_to_watch = i;
4122 buffer_info->dma = 0;
4123
4124 i++;
4125 if (i == tx_ring->count)
4126 i = 0;
4127 tx_ring->next_to_use = i;
4128
4129 return true;
4130 }
4131 return false;
4132}
4133
4134#define IGB_MAX_TXD_PWR 16
4135#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4136
4137static inline int igb_tx_map_adv(struct igb_ring *tx_ring, struct sk_buff *skb,
4138 unsigned int first)
4139{
4140 struct igb_buffer *buffer_info;
4141 struct device *dev = tx_ring->dev;
4142 unsigned int hlen = skb_headlen(skb);
4143 unsigned int count = 0, i;
4144 unsigned int f;
4145 u16 gso_segs = skb_shinfo(skb)->gso_segs ?: 1;
4146
4147 i = tx_ring->next_to_use;
4148
4149 buffer_info = &tx_ring->buffer_info[i];
4150 BUG_ON(hlen >= IGB_MAX_DATA_PER_TXD);
4151 buffer_info->length = hlen;
4152 /* set time_stamp *before* dma to help avoid a possible race */
4153 buffer_info->time_stamp = jiffies;
4154 buffer_info->next_to_watch = i;
4155 buffer_info->dma = dma_map_single(dev, skb->data, hlen,
4156 DMA_TO_DEVICE);
4157 if (dma_mapping_error(dev, buffer_info->dma))
4158 goto dma_error;
4159
4160 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
4161 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[f];
4162 unsigned int len = frag->size;
4163
4164 count++;
4165 i++;
4166 if (i == tx_ring->count)
4167 i = 0;
4168
4169 buffer_info = &tx_ring->buffer_info[i];
4170 BUG_ON(len >= IGB_MAX_DATA_PER_TXD);
4171 buffer_info->length = len;
4172 buffer_info->time_stamp = jiffies;
4173 buffer_info->next_to_watch = i;
4174 buffer_info->mapped_as_page = true;
4175 buffer_info->dma = dma_map_page(dev,
4176 frag->page,
4177 frag->page_offset,
4178 len,
4179 DMA_TO_DEVICE);
4180 if (dma_mapping_error(dev, buffer_info->dma))
4181 goto dma_error;
4182
4183 }
4184
4185 tx_ring->buffer_info[i].skb = skb;
4186 tx_ring->buffer_info[i].tx_flags = skb_shinfo(skb)->tx_flags;
4187 /* multiply data chunks by size of headers */
4188 tx_ring->buffer_info[i].bytecount = ((gso_segs - 1) * hlen) + skb->len;
4189 tx_ring->buffer_info[i].gso_segs = gso_segs;
4190 tx_ring->buffer_info[first].next_to_watch = i;
4191
4192 return ++count;
4193
4194dma_error:
4195 dev_err(dev, "TX DMA map failed\n");
4196
4197 /* clear timestamp and dma mappings for failed buffer_info mapping */
4198 buffer_info->dma = 0;
4199 buffer_info->time_stamp = 0;
4200 buffer_info->length = 0;
4201 buffer_info->next_to_watch = 0;
4202 buffer_info->mapped_as_page = false;
4203
4204 /* clear timestamp and dma mappings for remaining portion of packet */
4205 while (count--) {
4206 if (i == 0)
4207 i = tx_ring->count;
4208 i--;
4209 buffer_info = &tx_ring->buffer_info[i];
4210 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
4211 }
4212
4213 return 0;
4214}
4215
4216static inline void igb_tx_queue_adv(struct igb_ring *tx_ring,
4217 u32 tx_flags, int count, u32 paylen,
4218 u8 hdr_len)
4219{
4220 union e1000_adv_tx_desc *tx_desc;
4221 struct igb_buffer *buffer_info;
4222 u32 olinfo_status = 0, cmd_type_len;
4223 unsigned int i = tx_ring->next_to_use;
4224
4225 cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
4226 E1000_ADVTXD_DCMD_DEXT);
4227
4228 if (tx_flags & IGB_TX_FLAGS_VLAN)
4229 cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
4230
4231 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4232 cmd_type_len |= E1000_ADVTXD_MAC_TSTAMP;
4233
4234 if (tx_flags & IGB_TX_FLAGS_TSO) {
4235 cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
4236
4237 /* insert tcp checksum */
4238 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4239
4240 /* insert ip checksum */
4241 if (tx_flags & IGB_TX_FLAGS_IPV4)
4242 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4243
4244 } else if (tx_flags & IGB_TX_FLAGS_CSUM) {
4245 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4246 }
4247
4248 if ((tx_ring->flags & IGB_RING_FLAG_TX_CTX_IDX) &&
4249 (tx_flags & (IGB_TX_FLAGS_CSUM |
4250 IGB_TX_FLAGS_TSO |
4251 IGB_TX_FLAGS_VLAN)))
4252 olinfo_status |= tx_ring->reg_idx << 4;
4253
4254 olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
4255
4256 do {
4257 buffer_info = &tx_ring->buffer_info[i];
4258 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
4259 tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
4260 tx_desc->read.cmd_type_len =
4261 cpu_to_le32(cmd_type_len | buffer_info->length);
4262 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4263 count--;
4264 i++;
4265 if (i == tx_ring->count)
4266 i = 0;
4267 } while (count > 0);
4268
4269 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_ADVTXD_DCMD);
4270 /* Force memory writes to complete before letting h/w
4271 * know there are new descriptors to fetch. (Only
4272 * applicable for weak-ordered memory model archs,
4273 * such as IA-64). */
4274 wmb();
4275
4276 tx_ring->next_to_use = i;
4277 writel(i, tx_ring->tail);
4278 /* we need this if more than one processor can write to our tail
4279 * at a time, it syncronizes IO on IA64/Altix systems */
4280 mmiowb();
4281}
4282
4283static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4284{
4285 struct net_device *netdev = tx_ring->netdev;
4286
4287 netif_stop_subqueue(netdev, tx_ring->queue_index);
4288
4289 /* Herbert's original patch had:
4290 * smp_mb__after_netif_stop_queue();
4291 * but since that doesn't exist yet, just open code it. */
4292 smp_mb();
4293
4294 /* We need to check again in a case another CPU has just
4295 * made room available. */
4296 if (igb_desc_unused(tx_ring) < size)
4297 return -EBUSY;
4298
4299 /* A reprieve! */
4300 netif_wake_subqueue(netdev, tx_ring->queue_index);
4301
4302 u64_stats_update_begin(&tx_ring->tx_syncp2);
4303 tx_ring->tx_stats.restart_queue2++;
4304 u64_stats_update_end(&tx_ring->tx_syncp2);
4305
4306 return 0;
4307}
4308
4309static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, int size)
4310{
4311 if (igb_desc_unused(tx_ring) >= size)
4312 return 0;
4313 return __igb_maybe_stop_tx(tx_ring, size);
4314}
4315
4316netdev_tx_t igb_xmit_frame_ring_adv(struct sk_buff *skb,
4317 struct igb_ring *tx_ring)
4318{
4319 int tso = 0, count;
4320 u32 tx_flags = 0;
4321 u16 first;
4322 u8 hdr_len = 0;
4323
4324 /* need: 1 descriptor per page,
4325 * + 2 desc gap to keep tail from touching head,
4326 * + 1 desc for skb->data,
4327 * + 1 desc for context descriptor,
4328 * otherwise try next time */
4329 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4330 /* this is a hard error */
4331 return NETDEV_TX_BUSY;
4332 }
4333
4334 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4335 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4336 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4337 }
4338
4339 if (vlan_tx_tag_present(skb)) {
4340 tx_flags |= IGB_TX_FLAGS_VLAN;
4341 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4342 }
4343
4344 if (skb->protocol == htons(ETH_P_IP))
4345 tx_flags |= IGB_TX_FLAGS_IPV4;
4346
4347 first = tx_ring->next_to_use;
4348 if (skb_is_gso(skb)) {
4349 tso = igb_tso_adv(tx_ring, skb, tx_flags, &hdr_len);
4350
4351 if (tso < 0) {
4352 dev_kfree_skb_any(skb);
4353 return NETDEV_TX_OK;
4354 }
4355 }
4356
4357 if (tso)
4358 tx_flags |= IGB_TX_FLAGS_TSO;
4359 else if (igb_tx_csum_adv(tx_ring, skb, tx_flags) &&
4360 (skb->ip_summed == CHECKSUM_PARTIAL))
4361 tx_flags |= IGB_TX_FLAGS_CSUM;
4362
4363 /*
4364 * count reflects descriptors mapped, if 0 or less then mapping error
4365 * has occurred and we need to rewind the descriptor queue
4366 */
4367 count = igb_tx_map_adv(tx_ring, skb, first);
4368 if (!count) {
4369 dev_kfree_skb_any(skb);
4370 tx_ring->buffer_info[first].time_stamp = 0;
4371 tx_ring->next_to_use = first;
4372 return NETDEV_TX_OK;
4373 }
4374
4375 igb_tx_queue_adv(tx_ring, tx_flags, count, skb->len, hdr_len);
4376
4377 /* Make sure there is space in the ring for the next send. */
4378 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4379
4380 return NETDEV_TX_OK;
4381}
4382
4383static netdev_tx_t igb_xmit_frame_adv(struct sk_buff *skb,
4384 struct net_device *netdev)
4385{
4386 struct igb_adapter *adapter = netdev_priv(netdev);
4387 struct igb_ring *tx_ring;
4388 int r_idx = 0;
4389
4390 if (test_bit(__IGB_DOWN, &adapter->state)) {
4391 dev_kfree_skb_any(skb);
4392 return NETDEV_TX_OK;
4393 }
4394
4395 if (skb->len <= 0) {
4396 dev_kfree_skb_any(skb);
4397 return NETDEV_TX_OK;
4398 }
4399
4400 r_idx = skb->queue_mapping & (IGB_ABS_MAX_TX_QUEUES - 1);
4401 tx_ring = adapter->multi_tx_table[r_idx];
4402
4403 /* This goes back to the question of how to logically map a tx queue
4404 * to a flow. Right now, performance is impacted slightly negatively
4405 * if using multiple tx queues. If the stack breaks away from a
4406 * single qdisc implementation, we can look at this again. */
4407 return igb_xmit_frame_ring_adv(skb, tx_ring);
4408}
4409
4410/**
4411 * igb_tx_timeout - Respond to a Tx Hang
4412 * @netdev: network interface device structure
4413 **/
4414static void igb_tx_timeout(struct net_device *netdev)
4415{
4416 struct igb_adapter *adapter = netdev_priv(netdev);
4417 struct e1000_hw *hw = &adapter->hw;
4418
4419 /* Do the reset outside of interrupt context */
4420 adapter->tx_timeout_count++;
4421
4422 if (hw->mac.type == e1000_82580)
4423 hw->dev_spec._82575.global_device_reset = true;
4424
4425 schedule_work(&adapter->reset_task);
4426 wr32(E1000_EICS,
4427 (adapter->eims_enable_mask & ~adapter->eims_other));
4428}
4429
4430static void igb_reset_task(struct work_struct *work)
4431{
4432 struct igb_adapter *adapter;
4433 adapter = container_of(work, struct igb_adapter, reset_task);
4434
4435 igb_dump(adapter);
4436 netdev_err(adapter->netdev, "Reset adapter\n");
4437 igb_reinit_locked(adapter);
4438}
4439
4440/**
4441 * igb_get_stats64 - Get System Network Statistics
4442 * @netdev: network interface device structure
4443 * @stats: rtnl_link_stats64 pointer
4444 *
4445 **/
4446static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4447 struct rtnl_link_stats64 *stats)
4448{
4449 struct igb_adapter *adapter = netdev_priv(netdev);
4450
4451 spin_lock(&adapter->stats64_lock);
4452 igb_update_stats(adapter, &adapter->stats64);
4453 memcpy(stats, &adapter->stats64, sizeof(*stats));
4454 spin_unlock(&adapter->stats64_lock);
4455
4456 return stats;
4457}
4458
4459/**
4460 * igb_change_mtu - Change the Maximum Transfer Unit
4461 * @netdev: network interface device structure
4462 * @new_mtu: new value for maximum frame size
4463 *
4464 * Returns 0 on success, negative on failure
4465 **/
4466static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4467{
4468 struct igb_adapter *adapter = netdev_priv(netdev);
4469 struct pci_dev *pdev = adapter->pdev;
4470 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
4471 u32 rx_buffer_len, i;
4472
4473 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4474 dev_err(&pdev->dev, "Invalid MTU setting\n");
4475 return -EINVAL;
4476 }
4477
4478 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4479 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4480 return -EINVAL;
4481 }
4482
4483 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4484 msleep(1);
4485
4486 /* igb_down has a dependency on max_frame_size */
4487 adapter->max_frame_size = max_frame;
4488
4489 /* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
4490 * means we reserve 2 more, this pushes us to allocate from the next
4491 * larger slab size.
4492 * i.e. RXBUFFER_2048 --> size-4096 slab
4493 */
4494
4495 if (adapter->hw.mac.type == e1000_82580)
4496 max_frame += IGB_TS_HDR_LEN;
4497
4498 if (max_frame <= IGB_RXBUFFER_1024)
4499 rx_buffer_len = IGB_RXBUFFER_1024;
4500 else if (max_frame <= MAXIMUM_ETHERNET_VLAN_SIZE)
4501 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE;
4502 else
4503 rx_buffer_len = IGB_RXBUFFER_128;
4504
4505 if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN + IGB_TS_HDR_LEN) ||
4506 (max_frame == MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN))
4507 rx_buffer_len = MAXIMUM_ETHERNET_VLAN_SIZE + IGB_TS_HDR_LEN;
4508
4509 if ((adapter->hw.mac.type == e1000_82580) &&
4510 (rx_buffer_len == IGB_RXBUFFER_128))
4511 rx_buffer_len += IGB_RXBUFFER_64;
4512
4513 if (netif_running(netdev))
4514 igb_down(adapter);
4515
4516 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4517 netdev->mtu, new_mtu);
4518 netdev->mtu = new_mtu;
4519
4520 for (i = 0; i < adapter->num_rx_queues; i++)
4521 adapter->rx_ring[i]->rx_buffer_len = rx_buffer_len;
4522
4523 if (netif_running(netdev))
4524 igb_up(adapter);
4525 else
4526 igb_reset(adapter);
4527
4528 clear_bit(__IGB_RESETTING, &adapter->state);
4529
4530 return 0;
4531}
4532
4533/**
4534 * igb_update_stats - Update the board statistics counters
4535 * @adapter: board private structure
4536 **/
4537
4538void igb_update_stats(struct igb_adapter *adapter,
4539 struct rtnl_link_stats64 *net_stats)
4540{
4541 struct e1000_hw *hw = &adapter->hw;
4542 struct pci_dev *pdev = adapter->pdev;
4543 u32 reg, mpc;
4544 u16 phy_tmp;
4545 int i;
4546 u64 bytes, packets;
4547 unsigned int start;
4548 u64 _bytes, _packets;
4549
4550#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4551
4552 /*
4553 * Prevent stats update while adapter is being reset, or if the pci
4554 * connection is down.
4555 */
4556 if (adapter->link_speed == 0)
4557 return;
4558 if (pci_channel_offline(pdev))
4559 return;
4560
4561 bytes = 0;
4562 packets = 0;
4563 for (i = 0; i < adapter->num_rx_queues; i++) {
4564 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4565 struct igb_ring *ring = adapter->rx_ring[i];
4566
4567 ring->rx_stats.drops += rqdpc_tmp;
4568 net_stats->rx_fifo_errors += rqdpc_tmp;
4569
4570 do {
4571 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4572 _bytes = ring->rx_stats.bytes;
4573 _packets = ring->rx_stats.packets;
4574 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4575 bytes += _bytes;
4576 packets += _packets;
4577 }
4578
4579 net_stats->rx_bytes = bytes;
4580 net_stats->rx_packets = packets;
4581
4582 bytes = 0;
4583 packets = 0;
4584 for (i = 0; i < adapter->num_tx_queues; i++) {
4585 struct igb_ring *ring = adapter->tx_ring[i];
4586 do {
4587 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4588 _bytes = ring->tx_stats.bytes;
4589 _packets = ring->tx_stats.packets;
4590 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4591 bytes += _bytes;
4592 packets += _packets;
4593 }
4594 net_stats->tx_bytes = bytes;
4595 net_stats->tx_packets = packets;
4596
4597 /* read stats registers */
4598 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4599 adapter->stats.gprc += rd32(E1000_GPRC);
4600 adapter->stats.gorc += rd32(E1000_GORCL);
4601 rd32(E1000_GORCH); /* clear GORCL */
4602 adapter->stats.bprc += rd32(E1000_BPRC);
4603 adapter->stats.mprc += rd32(E1000_MPRC);
4604 adapter->stats.roc += rd32(E1000_ROC);
4605
4606 adapter->stats.prc64 += rd32(E1000_PRC64);
4607 adapter->stats.prc127 += rd32(E1000_PRC127);
4608 adapter->stats.prc255 += rd32(E1000_PRC255);
4609 adapter->stats.prc511 += rd32(E1000_PRC511);
4610 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4611 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4612 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4613 adapter->stats.sec += rd32(E1000_SEC);
4614
4615 mpc = rd32(E1000_MPC);
4616 adapter->stats.mpc += mpc;
4617 net_stats->rx_fifo_errors += mpc;
4618 adapter->stats.scc += rd32(E1000_SCC);
4619 adapter->stats.ecol += rd32(E1000_ECOL);
4620 adapter->stats.mcc += rd32(E1000_MCC);
4621 adapter->stats.latecol += rd32(E1000_LATECOL);
4622 adapter->stats.dc += rd32(E1000_DC);
4623 adapter->stats.rlec += rd32(E1000_RLEC);
4624 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4625 adapter->stats.xontxc += rd32(E1000_XONTXC);
4626 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4627 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4628 adapter->stats.fcruc += rd32(E1000_FCRUC);
4629 adapter->stats.gptc += rd32(E1000_GPTC);
4630 adapter->stats.gotc += rd32(E1000_GOTCL);
4631 rd32(E1000_GOTCH); /* clear GOTCL */
4632 adapter->stats.rnbc += rd32(E1000_RNBC);
4633 adapter->stats.ruc += rd32(E1000_RUC);
4634 adapter->stats.rfc += rd32(E1000_RFC);
4635 adapter->stats.rjc += rd32(E1000_RJC);
4636 adapter->stats.tor += rd32(E1000_TORH);
4637 adapter->stats.tot += rd32(E1000_TOTH);
4638 adapter->stats.tpr += rd32(E1000_TPR);
4639
4640 adapter->stats.ptc64 += rd32(E1000_PTC64);
4641 adapter->stats.ptc127 += rd32(E1000_PTC127);
4642 adapter->stats.ptc255 += rd32(E1000_PTC255);
4643 adapter->stats.ptc511 += rd32(E1000_PTC511);
4644 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4645 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4646
4647 adapter->stats.mptc += rd32(E1000_MPTC);
4648 adapter->stats.bptc += rd32(E1000_BPTC);
4649
4650 adapter->stats.tpt += rd32(E1000_TPT);
4651 adapter->stats.colc += rd32(E1000_COLC);
4652
4653 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4654 /* read internal phy specific stats */
4655 reg = rd32(E1000_CTRL_EXT);
4656 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4657 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4658 adapter->stats.tncrs += rd32(E1000_TNCRS);
4659 }
4660
4661 adapter->stats.tsctc += rd32(E1000_TSCTC);
4662 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4663
4664 adapter->stats.iac += rd32(E1000_IAC);
4665 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4666 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4667 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4668 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4669 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4670 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4671 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4672 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4673
4674 /* Fill out the OS statistics structure */
4675 net_stats->multicast = adapter->stats.mprc;
4676 net_stats->collisions = adapter->stats.colc;
4677
4678 /* Rx Errors */
4679
4680 /* RLEC on some newer hardware can be incorrect so build
4681 * our own version based on RUC and ROC */
4682 net_stats->rx_errors = adapter->stats.rxerrc +
4683 adapter->stats.crcerrs + adapter->stats.algnerrc +
4684 adapter->stats.ruc + adapter->stats.roc +
4685 adapter->stats.cexterr;
4686 net_stats->rx_length_errors = adapter->stats.ruc +
4687 adapter->stats.roc;
4688 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4689 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4690 net_stats->rx_missed_errors = adapter->stats.mpc;
4691
4692 /* Tx Errors */
4693 net_stats->tx_errors = adapter->stats.ecol +
4694 adapter->stats.latecol;
4695 net_stats->tx_aborted_errors = adapter->stats.ecol;
4696 net_stats->tx_window_errors = adapter->stats.latecol;
4697 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4698
4699 /* Tx Dropped needs to be maintained elsewhere */
4700
4701 /* Phy Stats */
4702 if (hw->phy.media_type == e1000_media_type_copper) {
4703 if ((adapter->link_speed == SPEED_1000) &&
4704 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4705 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4706 adapter->phy_stats.idle_errors += phy_tmp;
4707 }
4708 }
4709
4710 /* Management Stats */
4711 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4712 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4713 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4714
4715 /* OS2BMC Stats */
4716 reg = rd32(E1000_MANC);
4717 if (reg & E1000_MANC_EN_BMC2OS) {
4718 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4719 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4720 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4721 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4722 }
4723}
4724
4725static irqreturn_t igb_msix_other(int irq, void *data)
4726{
4727 struct igb_adapter *adapter = data;
4728 struct e1000_hw *hw = &adapter->hw;
4729 u32 icr = rd32(E1000_ICR);
4730 /* reading ICR causes bit 31 of EICR to be cleared */
4731
4732 if (icr & E1000_ICR_DRSTA)
4733 schedule_work(&adapter->reset_task);
4734
4735 if (icr & E1000_ICR_DOUTSYNC) {
4736 /* HW is reporting DMA is out of sync */
4737 adapter->stats.doosync++;
4738 /* The DMA Out of Sync is also indication of a spoof event
4739 * in IOV mode. Check the Wrong VM Behavior register to
4740 * see if it is really a spoof event. */
4741 igb_check_wvbr(adapter);
4742 }
4743
4744 /* Check for a mailbox event */
4745 if (icr & E1000_ICR_VMMB)
4746 igb_msg_task(adapter);
4747
4748 if (icr & E1000_ICR_LSC) {
4749 hw->mac.get_link_status = 1;
4750 /* guard against interrupt when we're going down */
4751 if (!test_bit(__IGB_DOWN, &adapter->state))
4752 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4753 }
4754
4755 if (adapter->vfs_allocated_count)
4756 wr32(E1000_IMS, E1000_IMS_LSC |
4757 E1000_IMS_VMMB |
4758 E1000_IMS_DOUTSYNC);
4759 else
4760 wr32(E1000_IMS, E1000_IMS_LSC | E1000_IMS_DOUTSYNC);
4761 wr32(E1000_EIMS, adapter->eims_other);
4762
4763 return IRQ_HANDLED;
4764}
4765
4766static void igb_write_itr(struct igb_q_vector *q_vector)
4767{
4768 struct igb_adapter *adapter = q_vector->adapter;
4769 u32 itr_val = q_vector->itr_val & 0x7FFC;
4770
4771 if (!q_vector->set_itr)
4772 return;
4773
4774 if (!itr_val)
4775 itr_val = 0x4;
4776
4777 if (adapter->hw.mac.type == e1000_82575)
4778 itr_val |= itr_val << 16;
4779 else
4780 itr_val |= 0x8000000;
4781
4782 writel(itr_val, q_vector->itr_register);
4783 q_vector->set_itr = 0;
4784}
4785
4786static irqreturn_t igb_msix_ring(int irq, void *data)
4787{
4788 struct igb_q_vector *q_vector = data;
4789
4790 /* Write the ITR value calculated from the previous interrupt. */
4791 igb_write_itr(q_vector);
4792
4793 napi_schedule(&q_vector->napi);
4794
4795 return IRQ_HANDLED;
4796}
4797
4798#ifdef CONFIG_IGB_DCA
4799static void igb_update_dca(struct igb_q_vector *q_vector)
4800{
4801 struct igb_adapter *adapter = q_vector->adapter;
4802 struct e1000_hw *hw = &adapter->hw;
4803 int cpu = get_cpu();
4804
4805 if (q_vector->cpu == cpu)
4806 goto out_no_update;
4807
4808 if (q_vector->tx_ring) {
4809 int q = q_vector->tx_ring->reg_idx;
4810 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4811 if (hw->mac.type == e1000_82575) {
4812 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4813 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4814 } else {
4815 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4816 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4817 E1000_DCA_TXCTRL_CPUID_SHIFT;
4818 }
4819 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4820 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4821 }
4822 if (q_vector->rx_ring) {
4823 int q = q_vector->rx_ring->reg_idx;
4824 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4825 if (hw->mac.type == e1000_82575) {
4826 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4827 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4828 } else {
4829 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4830 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4831 E1000_DCA_RXCTRL_CPUID_SHIFT;
4832 }
4833 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4834 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4835 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4836 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4837 }
4838 q_vector->cpu = cpu;
4839out_no_update:
4840 put_cpu();
4841}
4842
4843static void igb_setup_dca(struct igb_adapter *adapter)
4844{
4845 struct e1000_hw *hw = &adapter->hw;
4846 int i;
4847
4848 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4849 return;
4850
4851 /* Always use CB2 mode, difference is masked in the CB driver. */
4852 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4853
4854 for (i = 0; i < adapter->num_q_vectors; i++) {
4855 adapter->q_vector[i]->cpu = -1;
4856 igb_update_dca(adapter->q_vector[i]);
4857 }
4858}
4859
4860static int __igb_notify_dca(struct device *dev, void *data)
4861{
4862 struct net_device *netdev = dev_get_drvdata(dev);
4863 struct igb_adapter *adapter = netdev_priv(netdev);
4864 struct pci_dev *pdev = adapter->pdev;
4865 struct e1000_hw *hw = &adapter->hw;
4866 unsigned long event = *(unsigned long *)data;
4867
4868 switch (event) {
4869 case DCA_PROVIDER_ADD:
4870 /* if already enabled, don't do it again */
4871 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4872 break;
4873 if (dca_add_requester(dev) == 0) {
4874 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4875 dev_info(&pdev->dev, "DCA enabled\n");
4876 igb_setup_dca(adapter);
4877 break;
4878 }
4879 /* Fall Through since DCA is disabled. */
4880 case DCA_PROVIDER_REMOVE:
4881 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4882 /* without this a class_device is left
4883 * hanging around in the sysfs model */
4884 dca_remove_requester(dev);
4885 dev_info(&pdev->dev, "DCA disabled\n");
4886 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4887 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4888 }
4889 break;
4890 }
4891
4892 return 0;
4893}
4894
4895static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4896 void *p)
4897{
4898 int ret_val;
4899
4900 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4901 __igb_notify_dca);
4902
4903 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4904}
4905#endif /* CONFIG_IGB_DCA */
4906
4907static void igb_ping_all_vfs(struct igb_adapter *adapter)
4908{
4909 struct e1000_hw *hw = &adapter->hw;
4910 u32 ping;
4911 int i;
4912
4913 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
4914 ping = E1000_PF_CONTROL_MSG;
4915 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
4916 ping |= E1000_VT_MSGTYPE_CTS;
4917 igb_write_mbx(hw, &ping, 1, i);
4918 }
4919}
4920
4921static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
4922{
4923 struct e1000_hw *hw = &adapter->hw;
4924 u32 vmolr = rd32(E1000_VMOLR(vf));
4925 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4926
4927 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
4928 IGB_VF_FLAG_MULTI_PROMISC);
4929 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4930
4931 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
4932 vmolr |= E1000_VMOLR_MPME;
4933 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
4934 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
4935 } else {
4936 /*
4937 * if we have hashes and we are clearing a multicast promisc
4938 * flag we need to write the hashes to the MTA as this step
4939 * was previously skipped
4940 */
4941 if (vf_data->num_vf_mc_hashes > 30) {
4942 vmolr |= E1000_VMOLR_MPME;
4943 } else if (vf_data->num_vf_mc_hashes) {
4944 int j;
4945 vmolr |= E1000_VMOLR_ROMPE;
4946 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
4947 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
4948 }
4949 }
4950
4951 wr32(E1000_VMOLR(vf), vmolr);
4952
4953 /* there are flags left unprocessed, likely not supported */
4954 if (*msgbuf & E1000_VT_MSGINFO_MASK)
4955 return -EINVAL;
4956
4957 return 0;
4958
4959}
4960
4961static int igb_set_vf_multicasts(struct igb_adapter *adapter,
4962 u32 *msgbuf, u32 vf)
4963{
4964 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
4965 u16 *hash_list = (u16 *)&msgbuf[1];
4966 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
4967 int i;
4968
4969 /* salt away the number of multicast addresses assigned
4970 * to this VF for later use to restore when the PF multi cast
4971 * list changes
4972 */
4973 vf_data->num_vf_mc_hashes = n;
4974
4975 /* only up to 30 hash values supported */
4976 if (n > 30)
4977 n = 30;
4978
4979 /* store the hashes for later use */
4980 for (i = 0; i < n; i++)
4981 vf_data->vf_mc_hashes[i] = hash_list[i];
4982
4983 /* Flush and reset the mta with the new values */
4984 igb_set_rx_mode(adapter->netdev);
4985
4986 return 0;
4987}
4988
4989static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
4990{
4991 struct e1000_hw *hw = &adapter->hw;
4992 struct vf_data_storage *vf_data;
4993 int i, j;
4994
4995 for (i = 0; i < adapter->vfs_allocated_count; i++) {
4996 u32 vmolr = rd32(E1000_VMOLR(i));
4997 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
4998
4999 vf_data = &adapter->vf_data[i];
5000
5001 if ((vf_data->num_vf_mc_hashes > 30) ||
5002 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5003 vmolr |= E1000_VMOLR_MPME;
5004 } else if (vf_data->num_vf_mc_hashes) {
5005 vmolr |= E1000_VMOLR_ROMPE;
5006 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5007 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5008 }
5009 wr32(E1000_VMOLR(i), vmolr);
5010 }
5011}
5012
5013static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5014{
5015 struct e1000_hw *hw = &adapter->hw;
5016 u32 pool_mask, reg, vid;
5017 int i;
5018
5019 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5020
5021 /* Find the vlan filter for this id */
5022 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5023 reg = rd32(E1000_VLVF(i));
5024
5025 /* remove the vf from the pool */
5026 reg &= ~pool_mask;
5027
5028 /* if pool is empty then remove entry from vfta */
5029 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5030 (reg & E1000_VLVF_VLANID_ENABLE)) {
5031 reg = 0;
5032 vid = reg & E1000_VLVF_VLANID_MASK;
5033 igb_vfta_set(hw, vid, false);
5034 }
5035
5036 wr32(E1000_VLVF(i), reg);
5037 }
5038
5039 adapter->vf_data[vf].vlans_enabled = 0;
5040}
5041
5042static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5043{
5044 struct e1000_hw *hw = &adapter->hw;
5045 u32 reg, i;
5046
5047 /* The vlvf table only exists on 82576 hardware and newer */
5048 if (hw->mac.type < e1000_82576)
5049 return -1;
5050
5051 /* we only need to do this if VMDq is enabled */
5052 if (!adapter->vfs_allocated_count)
5053 return -1;
5054
5055 /* Find the vlan filter for this id */
5056 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5057 reg = rd32(E1000_VLVF(i));
5058 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5059 vid == (reg & E1000_VLVF_VLANID_MASK))
5060 break;
5061 }
5062
5063 if (add) {
5064 if (i == E1000_VLVF_ARRAY_SIZE) {
5065 /* Did not find a matching VLAN ID entry that was
5066 * enabled. Search for a free filter entry, i.e.
5067 * one without the enable bit set
5068 */
5069 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5070 reg = rd32(E1000_VLVF(i));
5071 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5072 break;
5073 }
5074 }
5075 if (i < E1000_VLVF_ARRAY_SIZE) {
5076 /* Found an enabled/available entry */
5077 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5078
5079 /* if !enabled we need to set this up in vfta */
5080 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5081 /* add VID to filter table */
5082 igb_vfta_set(hw, vid, true);
5083 reg |= E1000_VLVF_VLANID_ENABLE;
5084 }
5085 reg &= ~E1000_VLVF_VLANID_MASK;
5086 reg |= vid;
5087 wr32(E1000_VLVF(i), reg);
5088
5089 /* do not modify RLPML for PF devices */
5090 if (vf >= adapter->vfs_allocated_count)
5091 return 0;
5092
5093 if (!adapter->vf_data[vf].vlans_enabled) {
5094 u32 size;
5095 reg = rd32(E1000_VMOLR(vf));
5096 size = reg & E1000_VMOLR_RLPML_MASK;
5097 size += 4;
5098 reg &= ~E1000_VMOLR_RLPML_MASK;
5099 reg |= size;
5100 wr32(E1000_VMOLR(vf), reg);
5101 }
5102
5103 adapter->vf_data[vf].vlans_enabled++;
5104 return 0;
5105 }
5106 } else {
5107 if (i < E1000_VLVF_ARRAY_SIZE) {
5108 /* remove vf from the pool */
5109 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5110 /* if pool is empty then remove entry from vfta */
5111 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5112 reg = 0;
5113 igb_vfta_set(hw, vid, false);
5114 }
5115 wr32(E1000_VLVF(i), reg);
5116
5117 /* do not modify RLPML for PF devices */
5118 if (vf >= adapter->vfs_allocated_count)
5119 return 0;
5120
5121 adapter->vf_data[vf].vlans_enabled--;
5122 if (!adapter->vf_data[vf].vlans_enabled) {
5123 u32 size;
5124 reg = rd32(E1000_VMOLR(vf));
5125 size = reg & E1000_VMOLR_RLPML_MASK;
5126 size -= 4;
5127 reg &= ~E1000_VMOLR_RLPML_MASK;
5128 reg |= size;
5129 wr32(E1000_VMOLR(vf), reg);
5130 }
5131 }
5132 }
5133 return 0;
5134}
5135
5136static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5137{
5138 struct e1000_hw *hw = &adapter->hw;
5139
5140 if (vid)
5141 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5142 else
5143 wr32(E1000_VMVIR(vf), 0);
5144}
5145
5146static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5147 int vf, u16 vlan, u8 qos)
5148{
5149 int err = 0;
5150 struct igb_adapter *adapter = netdev_priv(netdev);
5151
5152 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5153 return -EINVAL;
5154 if (vlan || qos) {
5155 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5156 if (err)
5157 goto out;
5158 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5159 igb_set_vmolr(adapter, vf, !vlan);
5160 adapter->vf_data[vf].pf_vlan = vlan;
5161 adapter->vf_data[vf].pf_qos = qos;
5162 dev_info(&adapter->pdev->dev,
5163 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5164 if (test_bit(__IGB_DOWN, &adapter->state)) {
5165 dev_warn(&adapter->pdev->dev,
5166 "The VF VLAN has been set,"
5167 " but the PF device is not up.\n");
5168 dev_warn(&adapter->pdev->dev,
5169 "Bring the PF device up before"
5170 " attempting to use the VF device.\n");
5171 }
5172 } else {
5173 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5174 false, vf);
5175 igb_set_vmvir(adapter, vlan, vf);
5176 igb_set_vmolr(adapter, vf, true);
5177 adapter->vf_data[vf].pf_vlan = 0;
5178 adapter->vf_data[vf].pf_qos = 0;
5179 }
5180out:
5181 return err;
5182}
5183
5184static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5185{
5186 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5187 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5188
5189 return igb_vlvf_set(adapter, vid, add, vf);
5190}
5191
5192static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5193{
5194 /* clear flags - except flag that indicates PF has set the MAC */
5195 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5196 adapter->vf_data[vf].last_nack = jiffies;
5197
5198 /* reset offloads to defaults */
5199 igb_set_vmolr(adapter, vf, true);
5200
5201 /* reset vlans for device */
5202 igb_clear_vf_vfta(adapter, vf);
5203 if (adapter->vf_data[vf].pf_vlan)
5204 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5205 adapter->vf_data[vf].pf_vlan,
5206 adapter->vf_data[vf].pf_qos);
5207 else
5208 igb_clear_vf_vfta(adapter, vf);
5209
5210 /* reset multicast table array for vf */
5211 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5212
5213 /* Flush and reset the mta with the new values */
5214 igb_set_rx_mode(adapter->netdev);
5215}
5216
5217static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5218{
5219 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5220
5221 /* generate a new mac address as we were hotplug removed/added */
5222 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5223 random_ether_addr(vf_mac);
5224
5225 /* process remaining reset events */
5226 igb_vf_reset(adapter, vf);
5227}
5228
5229static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5230{
5231 struct e1000_hw *hw = &adapter->hw;
5232 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5233 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5234 u32 reg, msgbuf[3];
5235 u8 *addr = (u8 *)(&msgbuf[1]);
5236
5237 /* process all the same items cleared in a function level reset */
5238 igb_vf_reset(adapter, vf);
5239
5240 /* set vf mac address */
5241 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5242
5243 /* enable transmit and receive for vf */
5244 reg = rd32(E1000_VFTE);
5245 wr32(E1000_VFTE, reg | (1 << vf));
5246 reg = rd32(E1000_VFRE);
5247 wr32(E1000_VFRE, reg | (1 << vf));
5248
5249 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5250
5251 /* reply to reset with ack and vf mac address */
5252 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5253 memcpy(addr, vf_mac, 6);
5254 igb_write_mbx(hw, msgbuf, 3, vf);
5255}
5256
5257static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5258{
5259 /*
5260 * The VF MAC Address is stored in a packed array of bytes
5261 * starting at the second 32 bit word of the msg array
5262 */
5263 unsigned char *addr = (char *)&msg[1];
5264 int err = -1;
5265
5266 if (is_valid_ether_addr(addr))
5267 err = igb_set_vf_mac(adapter, vf, addr);
5268
5269 return err;
5270}
5271
5272static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5273{
5274 struct e1000_hw *hw = &adapter->hw;
5275 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5276 u32 msg = E1000_VT_MSGTYPE_NACK;
5277
5278 /* if device isn't clear to send it shouldn't be reading either */
5279 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5280 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5281 igb_write_mbx(hw, &msg, 1, vf);
5282 vf_data->last_nack = jiffies;
5283 }
5284}
5285
5286static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5287{
5288 struct pci_dev *pdev = adapter->pdev;
5289 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5290 struct e1000_hw *hw = &adapter->hw;
5291 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5292 s32 retval;
5293
5294 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5295
5296 if (retval) {
5297 /* if receive failed revoke VF CTS stats and restart init */
5298 dev_err(&pdev->dev, "Error receiving message from VF\n");
5299 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5300 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5301 return;
5302 goto out;
5303 }
5304
5305 /* this is a message we already processed, do nothing */
5306 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5307 return;
5308
5309 /*
5310 * until the vf completes a reset it should not be
5311 * allowed to start any configuration.
5312 */
5313
5314 if (msgbuf[0] == E1000_VF_RESET) {
5315 igb_vf_reset_msg(adapter, vf);
5316 return;
5317 }
5318
5319 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5320 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5321 return;
5322 retval = -1;
5323 goto out;
5324 }
5325
5326 switch ((msgbuf[0] & 0xFFFF)) {
5327 case E1000_VF_SET_MAC_ADDR:
5328 retval = -EINVAL;
5329 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5330 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5331 else
5332 dev_warn(&pdev->dev,
5333 "VF %d attempted to override administratively "
5334 "set MAC address\nReload the VF driver to "
5335 "resume operations\n", vf);
5336 break;
5337 case E1000_VF_SET_PROMISC:
5338 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5339 break;
5340 case E1000_VF_SET_MULTICAST:
5341 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5342 break;
5343 case E1000_VF_SET_LPE:
5344 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5345 break;
5346 case E1000_VF_SET_VLAN:
5347 retval = -1;
5348 if (vf_data->pf_vlan)
5349 dev_warn(&pdev->dev,
5350 "VF %d attempted to override administratively "
5351 "set VLAN tag\nReload the VF driver to "
5352 "resume operations\n", vf);
5353 else
5354 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5355 break;
5356 default:
5357 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5358 retval = -1;
5359 break;
5360 }
5361
5362 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5363out:
5364 /* notify the VF of the results of what it sent us */
5365 if (retval)
5366 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5367 else
5368 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5369
5370 igb_write_mbx(hw, msgbuf, 1, vf);
5371}
5372
5373static void igb_msg_task(struct igb_adapter *adapter)
5374{
5375 struct e1000_hw *hw = &adapter->hw;
5376 u32 vf;
5377
5378 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5379 /* process any reset requests */
5380 if (!igb_check_for_rst(hw, vf))
5381 igb_vf_reset_event(adapter, vf);
5382
5383 /* process any messages pending */
5384 if (!igb_check_for_msg(hw, vf))
5385 igb_rcv_msg_from_vf(adapter, vf);
5386
5387 /* process any acks */
5388 if (!igb_check_for_ack(hw, vf))
5389 igb_rcv_ack_from_vf(adapter, vf);
5390 }
5391}
5392
5393/**
5394 * igb_set_uta - Set unicast filter table address
5395 * @adapter: board private structure
5396 *
5397 * The unicast table address is a register array of 32-bit registers.
5398 * The table is meant to be used in a way similar to how the MTA is used
5399 * however due to certain limitations in the hardware it is necessary to
5400 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5401 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
5402 **/
5403static void igb_set_uta(struct igb_adapter *adapter)
5404{
5405 struct e1000_hw *hw = &adapter->hw;
5406 int i;
5407
5408 /* The UTA table only exists on 82576 hardware and newer */
5409 if (hw->mac.type < e1000_82576)
5410 return;
5411
5412 /* we only need to do this if VMDq is enabled */
5413 if (!adapter->vfs_allocated_count)
5414 return;
5415
5416 for (i = 0; i < hw->mac.uta_reg_count; i++)
5417 array_wr32(E1000_UTA, i, ~0);
5418}
5419
5420/**
5421 * igb_intr_msi - Interrupt Handler
5422 * @irq: interrupt number
5423 * @data: pointer to a network interface device structure
5424 **/
5425static irqreturn_t igb_intr_msi(int irq, void *data)
5426{
5427 struct igb_adapter *adapter = data;
5428 struct igb_q_vector *q_vector = adapter->q_vector[0];
5429 struct e1000_hw *hw = &adapter->hw;
5430 /* read ICR disables interrupts using IAM */
5431 u32 icr = rd32(E1000_ICR);
5432
5433 igb_write_itr(q_vector);
5434
5435 if (icr & E1000_ICR_DRSTA)
5436 schedule_work(&adapter->reset_task);
5437
5438 if (icr & E1000_ICR_DOUTSYNC) {
5439 /* HW is reporting DMA is out of sync */
5440 adapter->stats.doosync++;
5441 }
5442
5443 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5444 hw->mac.get_link_status = 1;
5445 if (!test_bit(__IGB_DOWN, &adapter->state))
5446 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5447 }
5448
5449 napi_schedule(&q_vector->napi);
5450
5451 return IRQ_HANDLED;
5452}
5453
5454/**
5455 * igb_intr - Legacy Interrupt Handler
5456 * @irq: interrupt number
5457 * @data: pointer to a network interface device structure
5458 **/
5459static irqreturn_t igb_intr(int irq, void *data)
5460{
5461 struct igb_adapter *adapter = data;
5462 struct igb_q_vector *q_vector = adapter->q_vector[0];
5463 struct e1000_hw *hw = &adapter->hw;
5464 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5465 * need for the IMC write */
5466 u32 icr = rd32(E1000_ICR);
5467 if (!icr)
5468 return IRQ_NONE; /* Not our interrupt */
5469
5470 igb_write_itr(q_vector);
5471
5472 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5473 * not set, then the adapter didn't send an interrupt */
5474 if (!(icr & E1000_ICR_INT_ASSERTED))
5475 return IRQ_NONE;
5476
5477 if (icr & E1000_ICR_DRSTA)
5478 schedule_work(&adapter->reset_task);
5479
5480 if (icr & E1000_ICR_DOUTSYNC) {
5481 /* HW is reporting DMA is out of sync */
5482 adapter->stats.doosync++;
5483 }
5484
5485 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5486 hw->mac.get_link_status = 1;
5487 /* guard against interrupt when we're going down */
5488 if (!test_bit(__IGB_DOWN, &adapter->state))
5489 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5490 }
5491
5492 napi_schedule(&q_vector->napi);
5493
5494 return IRQ_HANDLED;
5495}
5496
5497static inline void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5498{
5499 struct igb_adapter *adapter = q_vector->adapter;
5500 struct e1000_hw *hw = &adapter->hw;
5501
5502 if ((q_vector->rx_ring && (adapter->rx_itr_setting & 3)) ||
5503 (!q_vector->rx_ring && (adapter->tx_itr_setting & 3))) {
5504 if (!adapter->msix_entries)
5505 igb_set_itr(adapter);
5506 else
5507 igb_update_ring_itr(q_vector);
5508 }
5509
5510 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5511 if (adapter->msix_entries)
5512 wr32(E1000_EIMS, q_vector->eims_value);
5513 else
5514 igb_irq_enable(adapter);
5515 }
5516}
5517
5518/**
5519 * igb_poll - NAPI Rx polling callback
5520 * @napi: napi polling structure
5521 * @budget: count of how many packets we should handle
5522 **/
5523static int igb_poll(struct napi_struct *napi, int budget)
5524{
5525 struct igb_q_vector *q_vector = container_of(napi,
5526 struct igb_q_vector,
5527 napi);
5528 int tx_clean_complete = 1, work_done = 0;
5529
5530#ifdef CONFIG_IGB_DCA
5531 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5532 igb_update_dca(q_vector);
5533#endif
5534 if (q_vector->tx_ring)
5535 tx_clean_complete = igb_clean_tx_irq(q_vector);
5536
5537 if (q_vector->rx_ring)
5538 igb_clean_rx_irq_adv(q_vector, &work_done, budget);
5539
5540 if (!tx_clean_complete)
5541 work_done = budget;
5542
5543 /* If not enough Rx work done, exit the polling mode */
5544 if (work_done < budget) {
5545 napi_complete(napi);
5546 igb_ring_irq_enable(q_vector);
5547 }
5548
5549 return work_done;
5550}
5551
5552/**
5553 * igb_systim_to_hwtstamp - convert system time value to hw timestamp
5554 * @adapter: board private structure
5555 * @shhwtstamps: timestamp structure to update
5556 * @regval: unsigned 64bit system time value.
5557 *
5558 * We need to convert the system time value stored in the RX/TXSTMP registers
5559 * into a hwtstamp which can be used by the upper level timestamping functions
5560 */
5561static void igb_systim_to_hwtstamp(struct igb_adapter *adapter,
5562 struct skb_shared_hwtstamps *shhwtstamps,
5563 u64 regval)
5564{
5565 u64 ns;
5566
5567 /*
5568 * The 82580 starts with 1ns at bit 0 in RX/TXSTMPL, shift this up to
5569 * 24 to match clock shift we setup earlier.
5570 */
5571 if (adapter->hw.mac.type == e1000_82580)
5572 regval <<= IGB_82580_TSYNC_SHIFT;
5573
5574 ns = timecounter_cyc2time(&adapter->clock, regval);
5575 timecompare_update(&adapter->compare, ns);
5576 memset(shhwtstamps, 0, sizeof(struct skb_shared_hwtstamps));
5577 shhwtstamps->hwtstamp = ns_to_ktime(ns);
5578 shhwtstamps->syststamp = timecompare_transform(&adapter->compare, ns);
5579}
5580
5581/**
5582 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5583 * @q_vector: pointer to q_vector containing needed info
5584 * @buffer: pointer to igb_buffer structure
5585 *
5586 * If we were asked to do hardware stamping and such a time stamp is
5587 * available, then it must have been for this skb here because we only
5588 * allow only one such packet into the queue.
5589 */
5590static void igb_tx_hwtstamp(struct igb_q_vector *q_vector, struct igb_buffer *buffer_info)
5591{
5592 struct igb_adapter *adapter = q_vector->adapter;
5593 struct e1000_hw *hw = &adapter->hw;
5594 struct skb_shared_hwtstamps shhwtstamps;
5595 u64 regval;
5596
5597 /* if skb does not support hw timestamp or TX stamp not valid exit */
5598 if (likely(!(buffer_info->tx_flags & SKBTX_HW_TSTAMP)) ||
5599 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5600 return;
5601
5602 regval = rd32(E1000_TXSTMPL);
5603 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5604
5605 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5606 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5607}
5608
5609/**
5610 * igb_clean_tx_irq - Reclaim resources after transmit completes
5611 * @q_vector: pointer to q_vector containing needed info
5612 * returns true if ring is completely cleaned
5613 **/
5614static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5615{
5616 struct igb_adapter *adapter = q_vector->adapter;
5617 struct igb_ring *tx_ring = q_vector->tx_ring;
5618 struct net_device *netdev = tx_ring->netdev;
5619 struct e1000_hw *hw = &adapter->hw;
5620 struct igb_buffer *buffer_info;
5621 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5622 unsigned int total_bytes = 0, total_packets = 0;
5623 unsigned int i, eop, count = 0;
5624 bool cleaned = false;
5625
5626 i = tx_ring->next_to_clean;
5627 eop = tx_ring->buffer_info[i].next_to_watch;
5628 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5629
5630 while ((eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)) &&
5631 (count < tx_ring->count)) {
5632 rmb(); /* read buffer_info after eop_desc status */
5633 for (cleaned = false; !cleaned; count++) {
5634 tx_desc = E1000_TX_DESC_ADV(*tx_ring, i);
5635 buffer_info = &tx_ring->buffer_info[i];
5636 cleaned = (i == eop);
5637
5638 if (buffer_info->skb) {
5639 total_bytes += buffer_info->bytecount;
5640 /* gso_segs is currently only valid for tcp */
5641 total_packets += buffer_info->gso_segs;
5642 igb_tx_hwtstamp(q_vector, buffer_info);
5643 }
5644
5645 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
5646 tx_desc->wb.status = 0;
5647
5648 i++;
5649 if (i == tx_ring->count)
5650 i = 0;
5651 }
5652 eop = tx_ring->buffer_info[i].next_to_watch;
5653 eop_desc = E1000_TX_DESC_ADV(*tx_ring, eop);
5654 }
5655
5656 tx_ring->next_to_clean = i;
5657
5658 if (unlikely(count &&
5659 netif_carrier_ok(netdev) &&
5660 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5661 /* Make sure that anybody stopping the queue after this
5662 * sees the new next_to_clean.
5663 */
5664 smp_mb();
5665 if (__netif_subqueue_stopped(netdev, tx_ring->queue_index) &&
5666 !(test_bit(__IGB_DOWN, &adapter->state))) {
5667 netif_wake_subqueue(netdev, tx_ring->queue_index);
5668
5669 u64_stats_update_begin(&tx_ring->tx_syncp);
5670 tx_ring->tx_stats.restart_queue++;
5671 u64_stats_update_end(&tx_ring->tx_syncp);
5672 }
5673 }
5674
5675 if (tx_ring->detect_tx_hung) {
5676 /* Detect a transmit hang in hardware, this serializes the
5677 * check with the clearing of time_stamp and movement of i */
5678 tx_ring->detect_tx_hung = false;
5679 if (tx_ring->buffer_info[i].time_stamp &&
5680 time_after(jiffies, tx_ring->buffer_info[i].time_stamp +
5681 (adapter->tx_timeout_factor * HZ)) &&
5682 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5683
5684 /* detected Tx unit hang */
5685 dev_err(tx_ring->dev,
5686 "Detected Tx Unit Hang\n"
5687 " Tx Queue <%d>\n"
5688 " TDH <%x>\n"
5689 " TDT <%x>\n"
5690 " next_to_use <%x>\n"
5691 " next_to_clean <%x>\n"
5692 "buffer_info[next_to_clean]\n"
5693 " time_stamp <%lx>\n"
5694 " next_to_watch <%x>\n"
5695 " jiffies <%lx>\n"
5696 " desc.status <%x>\n",
5697 tx_ring->queue_index,
5698 readl(tx_ring->head),
5699 readl(tx_ring->tail),
5700 tx_ring->next_to_use,
5701 tx_ring->next_to_clean,
5702 tx_ring->buffer_info[eop].time_stamp,
5703 eop,
5704 jiffies,
5705 eop_desc->wb.status);
5706 netif_stop_subqueue(netdev, tx_ring->queue_index);
5707 }
5708 }
5709 tx_ring->total_bytes += total_bytes;
5710 tx_ring->total_packets += total_packets;
5711 u64_stats_update_begin(&tx_ring->tx_syncp);
5712 tx_ring->tx_stats.bytes += total_bytes;
5713 tx_ring->tx_stats.packets += total_packets;
5714 u64_stats_update_end(&tx_ring->tx_syncp);
5715 return count < tx_ring->count;
5716}
5717
5718static inline void igb_rx_checksum_adv(struct igb_ring *ring,
5719 u32 status_err, struct sk_buff *skb)
5720{
5721 skb_checksum_none_assert(skb);
5722
5723 /* Ignore Checksum bit is set or checksum is disabled through ethtool */
5724 if (!(ring->flags & IGB_RING_FLAG_RX_CSUM) ||
5725 (status_err & E1000_RXD_STAT_IXSM))
5726 return;
5727
5728 /* TCP/UDP checksum error bit is set */
5729 if (status_err &
5730 (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
5731 /*
5732 * work around errata with sctp packets where the TCPE aka
5733 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5734 * packets, (aka let the stack check the crc32c)
5735 */
5736 if ((skb->len == 60) &&
5737 (ring->flags & IGB_RING_FLAG_RX_SCTP_CSUM)) {
5738 u64_stats_update_begin(&ring->rx_syncp);
5739 ring->rx_stats.csum_err++;
5740 u64_stats_update_end(&ring->rx_syncp);
5741 }
5742 /* let the stack verify checksum errors */
5743 return;
5744 }
5745 /* It must be a TCP or UDP packet with a valid checksum */
5746 if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
5747 skb->ip_summed = CHECKSUM_UNNECESSARY;
5748
5749 dev_dbg(ring->dev, "cksum success: bits %08X\n", status_err);
5750}
5751
5752static void igb_rx_hwtstamp(struct igb_q_vector *q_vector, u32 staterr,
5753 struct sk_buff *skb)
5754{
5755 struct igb_adapter *adapter = q_vector->adapter;
5756 struct e1000_hw *hw = &adapter->hw;
5757 u64 regval;
5758
5759 /*
5760 * If this bit is set, then the RX registers contain the time stamp. No
5761 * other packet will be time stamped until we read these registers, so
5762 * read the registers to make them available again. Because only one
5763 * packet can be time stamped at a time, we know that the register
5764 * values must belong to this one here and therefore we don't need to
5765 * compare any of the additional attributes stored for it.
5766 *
5767 * If nothing went wrong, then it should have a shared tx_flags that we
5768 * can turn into a skb_shared_hwtstamps.
5769 */
5770 if (staterr & E1000_RXDADV_STAT_TSIP) {
5771 u32 *stamp = (u32 *)skb->data;
5772 regval = le32_to_cpu(*(stamp + 2));
5773 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5774 skb_pull(skb, IGB_TS_HDR_LEN);
5775 } else {
5776 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5777 return;
5778
5779 regval = rd32(E1000_RXSTMPL);
5780 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5781 }
5782
5783 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5784}
5785static inline u16 igb_get_hlen(struct igb_ring *rx_ring,
5786 union e1000_adv_rx_desc *rx_desc)
5787{
5788 /* HW will not DMA in data larger than the given buffer, even if it
5789 * parses the (NFS, of course) header to be larger. In that case, it
5790 * fills the header buffer and spills the rest into the page.
5791 */
5792 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5793 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5794 if (hlen > rx_ring->rx_buffer_len)
5795 hlen = rx_ring->rx_buffer_len;
5796 return hlen;
5797}
5798
5799static bool igb_clean_rx_irq_adv(struct igb_q_vector *q_vector,
5800 int *work_done, int budget)
5801{
5802 struct igb_ring *rx_ring = q_vector->rx_ring;
5803 struct net_device *netdev = rx_ring->netdev;
5804 struct device *dev = rx_ring->dev;
5805 union e1000_adv_rx_desc *rx_desc , *next_rxd;
5806 struct igb_buffer *buffer_info , *next_buffer;
5807 struct sk_buff *skb;
5808 bool cleaned = false;
5809 int cleaned_count = 0;
5810 int current_node = numa_node_id();
5811 unsigned int total_bytes = 0, total_packets = 0;
5812 unsigned int i;
5813 u32 staterr;
5814 u16 length;
5815
5816 i = rx_ring->next_to_clean;
5817 buffer_info = &rx_ring->buffer_info[i];
5818 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5819 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5820
5821 while (staterr & E1000_RXD_STAT_DD) {
5822 if (*work_done >= budget)
5823 break;
5824 (*work_done)++;
5825 rmb(); /* read descriptor and rx_buffer_info after status DD */
5826
5827 skb = buffer_info->skb;
5828 prefetch(skb->data - NET_IP_ALIGN);
5829 buffer_info->skb = NULL;
5830
5831 i++;
5832 if (i == rx_ring->count)
5833 i = 0;
5834
5835 next_rxd = E1000_RX_DESC_ADV(*rx_ring, i);
5836 prefetch(next_rxd);
5837 next_buffer = &rx_ring->buffer_info[i];
5838
5839 length = le16_to_cpu(rx_desc->wb.upper.length);
5840 cleaned = true;
5841 cleaned_count++;
5842
5843 if (buffer_info->dma) {
5844 dma_unmap_single(dev, buffer_info->dma,
5845 rx_ring->rx_buffer_len,
5846 DMA_FROM_DEVICE);
5847 buffer_info->dma = 0;
5848 if (rx_ring->rx_buffer_len >= IGB_RXBUFFER_1024) {
5849 skb_put(skb, length);
5850 goto send_up;
5851 }
5852 skb_put(skb, igb_get_hlen(rx_ring, rx_desc));
5853 }
5854
5855 if (length) {
5856 dma_unmap_page(dev, buffer_info->page_dma,
5857 PAGE_SIZE / 2, DMA_FROM_DEVICE);
5858 buffer_info->page_dma = 0;
5859
5860 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
5861 buffer_info->page,
5862 buffer_info->page_offset,
5863 length);
5864
5865 if ((page_count(buffer_info->page) != 1) ||
5866 (page_to_nid(buffer_info->page) != current_node))
5867 buffer_info->page = NULL;
5868 else
5869 get_page(buffer_info->page);
5870
5871 skb->len += length;
5872 skb->data_len += length;
5873 skb->truesize += length;
5874 }
5875
5876 if (!(staterr & E1000_RXD_STAT_EOP)) {
5877 buffer_info->skb = next_buffer->skb;
5878 buffer_info->dma = next_buffer->dma;
5879 next_buffer->skb = skb;
5880 next_buffer->dma = 0;
5881 goto next_desc;
5882 }
5883send_up:
5884 if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
5885 dev_kfree_skb_irq(skb);
5886 goto next_desc;
5887 }
5888
5889 if (staterr & (E1000_RXDADV_STAT_TSIP | E1000_RXDADV_STAT_TS))
5890 igb_rx_hwtstamp(q_vector, staterr, skb);
5891 total_bytes += skb->len;
5892 total_packets++;
5893
5894 igb_rx_checksum_adv(rx_ring, staterr, skb);
5895
5896 skb->protocol = eth_type_trans(skb, netdev);
5897 skb_record_rx_queue(skb, rx_ring->queue_index);
5898
5899 if (staterr & E1000_RXD_STAT_VP) {
5900 u16 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
5901
5902 __vlan_hwaccel_put_tag(skb, vid);
5903 }
5904 napi_gro_receive(&q_vector->napi, skb);
5905
5906next_desc:
5907 rx_desc->wb.upper.status_error = 0;
5908
5909 /* return some buffers to hardware, one at a time is too slow */
5910 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
5911 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5912 cleaned_count = 0;
5913 }
5914
5915 /* use prefetched values */
5916 rx_desc = next_rxd;
5917 buffer_info = next_buffer;
5918 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
5919 }
5920
5921 rx_ring->next_to_clean = i;
5922 cleaned_count = igb_desc_unused(rx_ring);
5923
5924 if (cleaned_count)
5925 igb_alloc_rx_buffers_adv(rx_ring, cleaned_count);
5926
5927 rx_ring->total_packets += total_packets;
5928 rx_ring->total_bytes += total_bytes;
5929 u64_stats_update_begin(&rx_ring->rx_syncp);
5930 rx_ring->rx_stats.packets += total_packets;
5931 rx_ring->rx_stats.bytes += total_bytes;
5932 u64_stats_update_end(&rx_ring->rx_syncp);
5933 return cleaned;
5934}
5935
5936/**
5937 * igb_alloc_rx_buffers_adv - Replace used receive buffers; packet split
5938 * @adapter: address of board private structure
5939 **/
5940void igb_alloc_rx_buffers_adv(struct igb_ring *rx_ring, int cleaned_count)
5941{
5942 struct net_device *netdev = rx_ring->netdev;
5943 union e1000_adv_rx_desc *rx_desc;
5944 struct igb_buffer *buffer_info;
5945 struct sk_buff *skb;
5946 unsigned int i;
5947 int bufsz;
5948
5949 i = rx_ring->next_to_use;
5950 buffer_info = &rx_ring->buffer_info[i];
5951
5952 bufsz = rx_ring->rx_buffer_len;
5953
5954 while (cleaned_count--) {
5955 rx_desc = E1000_RX_DESC_ADV(*rx_ring, i);
5956
5957 if ((bufsz < IGB_RXBUFFER_1024) && !buffer_info->page_dma) {
5958 if (!buffer_info->page) {
5959 buffer_info->page = netdev_alloc_page(netdev);
5960 if (unlikely(!buffer_info->page)) {
5961 u64_stats_update_begin(&rx_ring->rx_syncp);
5962 rx_ring->rx_stats.alloc_failed++;
5963 u64_stats_update_end(&rx_ring->rx_syncp);
5964 goto no_buffers;
5965 }
5966 buffer_info->page_offset = 0;
5967 } else {
5968 buffer_info->page_offset ^= PAGE_SIZE / 2;
5969 }
5970 buffer_info->page_dma =
5971 dma_map_page(rx_ring->dev, buffer_info->page,
5972 buffer_info->page_offset,
5973 PAGE_SIZE / 2,
5974 DMA_FROM_DEVICE);
5975 if (dma_mapping_error(rx_ring->dev,
5976 buffer_info->page_dma)) {
5977 buffer_info->page_dma = 0;
5978 u64_stats_update_begin(&rx_ring->rx_syncp);
5979 rx_ring->rx_stats.alloc_failed++;
5980 u64_stats_update_end(&rx_ring->rx_syncp);
5981 goto no_buffers;
5982 }
5983 }
5984
5985 skb = buffer_info->skb;
5986 if (!skb) {
5987 skb = netdev_alloc_skb_ip_align(netdev, bufsz);
5988 if (unlikely(!skb)) {
5989 u64_stats_update_begin(&rx_ring->rx_syncp);
5990 rx_ring->rx_stats.alloc_failed++;
5991 u64_stats_update_end(&rx_ring->rx_syncp);
5992 goto no_buffers;
5993 }
5994
5995 buffer_info->skb = skb;
5996 }
5997 if (!buffer_info->dma) {
5998 buffer_info->dma = dma_map_single(rx_ring->dev,
5999 skb->data,
6000 bufsz,
6001 DMA_FROM_DEVICE);
6002 if (dma_mapping_error(rx_ring->dev,
6003 buffer_info->dma)) {
6004 buffer_info->dma = 0;
6005 u64_stats_update_begin(&rx_ring->rx_syncp);
6006 rx_ring->rx_stats.alloc_failed++;
6007 u64_stats_update_end(&rx_ring->rx_syncp);
6008 goto no_buffers;
6009 }
6010 }
6011 /* Refresh the desc even if buffer_addrs didn't change because
6012 * each write-back erases this info. */
6013 if (bufsz < IGB_RXBUFFER_1024) {
6014 rx_desc->read.pkt_addr =
6015 cpu_to_le64(buffer_info->page_dma);
6016 rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
6017 } else {
6018 rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
6019 rx_desc->read.hdr_addr = 0;
6020 }
6021
6022 i++;
6023 if (i == rx_ring->count)
6024 i = 0;
6025 buffer_info = &rx_ring->buffer_info[i];
6026 }
6027
6028no_buffers:
6029 if (rx_ring->next_to_use != i) {
6030 rx_ring->next_to_use = i;
6031 if (i == 0)
6032 i = (rx_ring->count - 1);
6033 else
6034 i--;
6035
6036 /* Force memory writes to complete before letting h/w
6037 * know there are new descriptors to fetch. (Only
6038 * applicable for weak-ordered memory model archs,
6039 * such as IA-64). */
6040 wmb();
6041 writel(i, rx_ring->tail);
6042 }
6043}
6044
6045/**
6046 * igb_mii_ioctl -
6047 * @netdev:
6048 * @ifreq:
6049 * @cmd:
6050 **/
6051static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6052{
6053 struct igb_adapter *adapter = netdev_priv(netdev);
6054 struct mii_ioctl_data *data = if_mii(ifr);
6055
6056 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6057 return -EOPNOTSUPP;
6058
6059 switch (cmd) {
6060 case SIOCGMIIPHY:
6061 data->phy_id = adapter->hw.phy.addr;
6062 break;
6063 case SIOCGMIIREG:
6064 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6065 &data->val_out))
6066 return -EIO;
6067 break;
6068 case SIOCSMIIREG:
6069 default:
6070 return -EOPNOTSUPP;
6071 }
6072 return 0;
6073}
6074
6075/**
6076 * igb_hwtstamp_ioctl - control hardware time stamping
6077 * @netdev:
6078 * @ifreq:
6079 * @cmd:
6080 *
6081 * Outgoing time stamping can be enabled and disabled. Play nice and
6082 * disable it when requested, although it shouldn't case any overhead
6083 * when no packet needs it. At most one packet in the queue may be
6084 * marked for time stamping, otherwise it would be impossible to tell
6085 * for sure to which packet the hardware time stamp belongs.
6086 *
6087 * Incoming time stamping has to be configured via the hardware
6088 * filters. Not all combinations are supported, in particular event
6089 * type has to be specified. Matching the kind of event packet is
6090 * not supported, with the exception of "all V2 events regardless of
6091 * level 2 or 4".
6092 *
6093 **/
6094static int igb_hwtstamp_ioctl(struct net_device *netdev,
6095 struct ifreq *ifr, int cmd)
6096{
6097 struct igb_adapter *adapter = netdev_priv(netdev);
6098 struct e1000_hw *hw = &adapter->hw;
6099 struct hwtstamp_config config;
6100 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6101 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6102 u32 tsync_rx_cfg = 0;
6103 bool is_l4 = false;
6104 bool is_l2 = false;
6105 u32 regval;
6106
6107 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6108 return -EFAULT;
6109
6110 /* reserved for future extensions */
6111 if (config.flags)
6112 return -EINVAL;
6113
6114 switch (config.tx_type) {
6115 case HWTSTAMP_TX_OFF:
6116 tsync_tx_ctl = 0;
6117 case HWTSTAMP_TX_ON:
6118 break;
6119 default:
6120 return -ERANGE;
6121 }
6122
6123 switch (config.rx_filter) {
6124 case HWTSTAMP_FILTER_NONE:
6125 tsync_rx_ctl = 0;
6126 break;
6127 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6128 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6129 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6130 case HWTSTAMP_FILTER_ALL:
6131 /*
6132 * register TSYNCRXCFG must be set, therefore it is not
6133 * possible to time stamp both Sync and Delay_Req messages
6134 * => fall back to time stamping all packets
6135 */
6136 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6137 config.rx_filter = HWTSTAMP_FILTER_ALL;
6138 break;
6139 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
6140 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6141 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
6142 is_l4 = true;
6143 break;
6144 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
6145 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6146 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
6147 is_l4 = true;
6148 break;
6149 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6150 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
6151 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6152 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
6153 is_l2 = true;
6154 is_l4 = true;
6155 config.rx_filter = HWTSTAMP_FILTER_SOME;
6156 break;
6157 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6158 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
6159 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6160 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
6161 is_l2 = true;
6162 is_l4 = true;
6163 config.rx_filter = HWTSTAMP_FILTER_SOME;
6164 break;
6165 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6166 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6167 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
6168 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
6169 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
6170 is_l2 = true;
6171 break;
6172 default:
6173 return -ERANGE;
6174 }
6175
6176 if (hw->mac.type == e1000_82575) {
6177 if (tsync_rx_ctl | tsync_tx_ctl)
6178 return -EINVAL;
6179 return 0;
6180 }
6181
6182 /*
6183 * Per-packet timestamping only works if all packets are
6184 * timestamped, so enable timestamping in all packets as
6185 * long as one rx filter was configured.
6186 */
6187 if ((hw->mac.type == e1000_82580) && tsync_rx_ctl) {
6188 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6189 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6190 }
6191
6192 /* enable/disable TX */
6193 regval = rd32(E1000_TSYNCTXCTL);
6194 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6195 regval |= tsync_tx_ctl;
6196 wr32(E1000_TSYNCTXCTL, regval);
6197
6198 /* enable/disable RX */
6199 regval = rd32(E1000_TSYNCRXCTL);
6200 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6201 regval |= tsync_rx_ctl;
6202 wr32(E1000_TSYNCRXCTL, regval);
6203
6204 /* define which PTP packets are time stamped */
6205 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
6206
6207 /* define ethertype filter for timestamped packets */
6208 if (is_l2)
6209 wr32(E1000_ETQF(3),
6210 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6211 E1000_ETQF_1588 | /* enable timestamping */
6212 ETH_P_1588)); /* 1588 eth protocol type */
6213 else
6214 wr32(E1000_ETQF(3), 0);
6215
6216#define PTP_PORT 319
6217 /* L4 Queue Filter[3]: filter by destination port and protocol */
6218 if (is_l4) {
6219 u32 ftqf = (IPPROTO_UDP /* UDP */
6220 | E1000_FTQF_VF_BP /* VF not compared */
6221 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6222 | E1000_FTQF_MASK); /* mask all inputs */
6223 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6224
6225 wr32(E1000_IMIR(3), htons(PTP_PORT));
6226 wr32(E1000_IMIREXT(3),
6227 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6228 if (hw->mac.type == e1000_82576) {
6229 /* enable source port check */
6230 wr32(E1000_SPQF(3), htons(PTP_PORT));
6231 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6232 }
6233 wr32(E1000_FTQF(3), ftqf);
6234 } else {
6235 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6236 }
6237 wrfl();
6238
6239 adapter->hwtstamp_config = config;
6240
6241 /* clear TX/RX time stamp registers, just to be sure */
6242 regval = rd32(E1000_TXSTMPH);
6243 regval = rd32(E1000_RXSTMPH);
6244
6245 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6246 -EFAULT : 0;
6247}
6248
6249/**
6250 * igb_ioctl -
6251 * @netdev:
6252 * @ifreq:
6253 * @cmd:
6254 **/
6255static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6256{
6257 switch (cmd) {
6258 case SIOCGMIIPHY:
6259 case SIOCGMIIREG:
6260 case SIOCSMIIREG:
6261 return igb_mii_ioctl(netdev, ifr, cmd);
6262 case SIOCSHWTSTAMP:
6263 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
6264 default:
6265 return -EOPNOTSUPP;
6266 }
6267}
6268
6269s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6270{
6271 struct igb_adapter *adapter = hw->back;
6272 u16 cap_offset;
6273
6274 cap_offset = adapter->pdev->pcie_cap;
6275 if (!cap_offset)
6276 return -E1000_ERR_CONFIG;
6277
6278 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6279
6280 return 0;
6281}
6282
6283s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6284{
6285 struct igb_adapter *adapter = hw->back;
6286 u16 cap_offset;
6287
6288 cap_offset = adapter->pdev->pcie_cap;
6289 if (!cap_offset)
6290 return -E1000_ERR_CONFIG;
6291
6292 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6293
6294 return 0;
6295}
6296
6297static void igb_vlan_mode(struct net_device *netdev, u32 features)
6298{
6299 struct igb_adapter *adapter = netdev_priv(netdev);
6300 struct e1000_hw *hw = &adapter->hw;
6301 u32 ctrl, rctl;
6302
6303 igb_irq_disable(adapter);
6304
6305 if (features & NETIF_F_HW_VLAN_RX) {
6306 /* enable VLAN tag insert/strip */
6307 ctrl = rd32(E1000_CTRL);
6308 ctrl |= E1000_CTRL_VME;
6309 wr32(E1000_CTRL, ctrl);
6310
6311 /* Disable CFI check */
6312 rctl = rd32(E1000_RCTL);
6313 rctl &= ~E1000_RCTL_CFIEN;
6314 wr32(E1000_RCTL, rctl);
6315 } else {
6316 /* disable VLAN tag insert/strip */
6317 ctrl = rd32(E1000_CTRL);
6318 ctrl &= ~E1000_CTRL_VME;
6319 wr32(E1000_CTRL, ctrl);
6320 }
6321
6322 igb_rlpml_set(adapter);
6323
6324 if (!test_bit(__IGB_DOWN, &adapter->state))
6325 igb_irq_enable(adapter);
6326}
6327
6328static void igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6329{
6330 struct igb_adapter *adapter = netdev_priv(netdev);
6331 struct e1000_hw *hw = &adapter->hw;
6332 int pf_id = adapter->vfs_allocated_count;
6333
6334 /* attempt to add filter to vlvf array */
6335 igb_vlvf_set(adapter, vid, true, pf_id);
6336
6337 /* add the filter since PF can receive vlans w/o entry in vlvf */
6338 igb_vfta_set(hw, vid, true);
6339
6340 set_bit(vid, adapter->active_vlans);
6341}
6342
6343static void igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6344{
6345 struct igb_adapter *adapter = netdev_priv(netdev);
6346 struct e1000_hw *hw = &adapter->hw;
6347 int pf_id = adapter->vfs_allocated_count;
6348 s32 err;
6349
6350 igb_irq_disable(adapter);
6351
6352 if (!test_bit(__IGB_DOWN, &adapter->state))
6353 igb_irq_enable(adapter);
6354
6355 /* remove vlan from VLVF table array */
6356 err = igb_vlvf_set(adapter, vid, false, pf_id);
6357
6358 /* if vid was not present in VLVF just remove it from table */
6359 if (err)
6360 igb_vfta_set(hw, vid, false);
6361
6362 clear_bit(vid, adapter->active_vlans);
6363}
6364
6365static void igb_restore_vlan(struct igb_adapter *adapter)
6366{
6367 u16 vid;
6368
6369 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6370 igb_vlan_rx_add_vid(adapter->netdev, vid);
6371}
6372
6373int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
6374{
6375 struct pci_dev *pdev = adapter->pdev;
6376 struct e1000_mac_info *mac = &adapter->hw.mac;
6377
6378 mac->autoneg = 0;
6379
6380 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6381 * for the switch() below to work */
6382 if ((spd & 1) || (dplx & ~1))
6383 goto err_inval;
6384
6385 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6386 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6387 spd != SPEED_1000 &&
6388 dplx != DUPLEX_FULL)
6389 goto err_inval;
6390
6391 switch (spd + dplx) {
6392 case SPEED_10 + DUPLEX_HALF:
6393 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6394 break;
6395 case SPEED_10 + DUPLEX_FULL:
6396 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6397 break;
6398 case SPEED_100 + DUPLEX_HALF:
6399 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6400 break;
6401 case SPEED_100 + DUPLEX_FULL:
6402 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6403 break;
6404 case SPEED_1000 + DUPLEX_FULL:
6405 mac->autoneg = 1;
6406 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6407 break;
6408 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6409 default:
6410 goto err_inval;
6411 }
6412 return 0;
6413
6414err_inval:
6415 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6416 return -EINVAL;
6417}
6418
6419static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake)
6420{
6421 struct net_device *netdev = pci_get_drvdata(pdev);
6422 struct igb_adapter *adapter = netdev_priv(netdev);
6423 struct e1000_hw *hw = &adapter->hw;
6424 u32 ctrl, rctl, status;
6425 u32 wufc = adapter->wol;
6426#ifdef CONFIG_PM
6427 int retval = 0;
6428#endif
6429
6430 netif_device_detach(netdev);
6431
6432 if (netif_running(netdev))
6433 igb_close(netdev);
6434
6435 igb_clear_interrupt_scheme(adapter);
6436
6437#ifdef CONFIG_PM
6438 retval = pci_save_state(pdev);
6439 if (retval)
6440 return retval;
6441#endif
6442
6443 status = rd32(E1000_STATUS);
6444 if (status & E1000_STATUS_LU)
6445 wufc &= ~E1000_WUFC_LNKC;
6446
6447 if (wufc) {
6448 igb_setup_rctl(adapter);
6449 igb_set_rx_mode(netdev);
6450
6451 /* turn on all-multi mode if wake on multicast is enabled */
6452 if (wufc & E1000_WUFC_MC) {
6453 rctl = rd32(E1000_RCTL);
6454 rctl |= E1000_RCTL_MPE;
6455 wr32(E1000_RCTL, rctl);
6456 }
6457
6458 ctrl = rd32(E1000_CTRL);
6459 /* advertise wake from D3Cold */
6460 #define E1000_CTRL_ADVD3WUC 0x00100000
6461 /* phy power management enable */
6462 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6463 ctrl |= E1000_CTRL_ADVD3WUC;
6464 wr32(E1000_CTRL, ctrl);
6465
6466 /* Allow time for pending master requests to run */
6467 igb_disable_pcie_master(hw);
6468
6469 wr32(E1000_WUC, E1000_WUC_PME_EN);
6470 wr32(E1000_WUFC, wufc);
6471 } else {
6472 wr32(E1000_WUC, 0);
6473 wr32(E1000_WUFC, 0);
6474 }
6475
6476 *enable_wake = wufc || adapter->en_mng_pt;
6477 if (!*enable_wake)
6478 igb_power_down_link(adapter);
6479 else
6480 igb_power_up_link(adapter);
6481
6482 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6483 * would have already happened in close and is redundant. */
6484 igb_release_hw_control(adapter);
6485
6486 pci_disable_device(pdev);
6487
6488 return 0;
6489}
6490
6491#ifdef CONFIG_PM
6492static int igb_suspend(struct pci_dev *pdev, pm_message_t state)
6493{
6494 int retval;
6495 bool wake;
6496
6497 retval = __igb_shutdown(pdev, &wake);
6498 if (retval)
6499 return retval;
6500
6501 if (wake) {
6502 pci_prepare_to_sleep(pdev);
6503 } else {
6504 pci_wake_from_d3(pdev, false);
6505 pci_set_power_state(pdev, PCI_D3hot);
6506 }
6507
6508 return 0;
6509}
6510
6511static int igb_resume(struct pci_dev *pdev)
6512{
6513 struct net_device *netdev = pci_get_drvdata(pdev);
6514 struct igb_adapter *adapter = netdev_priv(netdev);
6515 struct e1000_hw *hw = &adapter->hw;
6516 u32 err;
6517
6518 pci_set_power_state(pdev, PCI_D0);
6519 pci_restore_state(pdev);
6520 pci_save_state(pdev);
6521
6522 err = pci_enable_device_mem(pdev);
6523 if (err) {
6524 dev_err(&pdev->dev,
6525 "igb: Cannot enable PCI device from suspend\n");
6526 return err;
6527 }
6528 pci_set_master(pdev);
6529
6530 pci_enable_wake(pdev, PCI_D3hot, 0);
6531 pci_enable_wake(pdev, PCI_D3cold, 0);
6532
6533 if (igb_init_interrupt_scheme(adapter)) {
6534 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6535 return -ENOMEM;
6536 }
6537
6538 igb_reset(adapter);
6539
6540 /* let the f/w know that the h/w is now under the control of the
6541 * driver. */
6542 igb_get_hw_control(adapter);
6543
6544 wr32(E1000_WUS, ~0);
6545
6546 if (netif_running(netdev)) {
6547 err = igb_open(netdev);
6548 if (err)
6549 return err;
6550 }
6551
6552 netif_device_attach(netdev);
6553
6554 return 0;
6555}
6556#endif
6557
6558static void igb_shutdown(struct pci_dev *pdev)
6559{
6560 bool wake;
6561
6562 __igb_shutdown(pdev, &wake);
6563
6564 if (system_state == SYSTEM_POWER_OFF) {
6565 pci_wake_from_d3(pdev, wake);
6566 pci_set_power_state(pdev, PCI_D3hot);
6567 }
6568}
6569
6570#ifdef CONFIG_NET_POLL_CONTROLLER
6571/*
6572 * Polling 'interrupt' - used by things like netconsole to send skbs
6573 * without having to re-enable interrupts. It's not called while
6574 * the interrupt routine is executing.
6575 */
6576static void igb_netpoll(struct net_device *netdev)
6577{
6578 struct igb_adapter *adapter = netdev_priv(netdev);
6579 struct e1000_hw *hw = &adapter->hw;
6580 int i;
6581
6582 if (!adapter->msix_entries) {
6583 struct igb_q_vector *q_vector = adapter->q_vector[0];
6584 igb_irq_disable(adapter);
6585 napi_schedule(&q_vector->napi);
6586 return;
6587 }
6588
6589 for (i = 0; i < adapter->num_q_vectors; i++) {
6590 struct igb_q_vector *q_vector = adapter->q_vector[i];
6591 wr32(E1000_EIMC, q_vector->eims_value);
6592 napi_schedule(&q_vector->napi);
6593 }
6594}
6595#endif /* CONFIG_NET_POLL_CONTROLLER */
6596
6597/**
6598 * igb_io_error_detected - called when PCI error is detected
6599 * @pdev: Pointer to PCI device
6600 * @state: The current pci connection state
6601 *
6602 * This function is called after a PCI bus error affecting
6603 * this device has been detected.
6604 */
6605static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6606 pci_channel_state_t state)
6607{
6608 struct net_device *netdev = pci_get_drvdata(pdev);
6609 struct igb_adapter *adapter = netdev_priv(netdev);
6610
6611 netif_device_detach(netdev);
6612
6613 if (state == pci_channel_io_perm_failure)
6614 return PCI_ERS_RESULT_DISCONNECT;
6615
6616 if (netif_running(netdev))
6617 igb_down(adapter);
6618 pci_disable_device(pdev);
6619
6620 /* Request a slot slot reset. */
6621 return PCI_ERS_RESULT_NEED_RESET;
6622}
6623
6624/**
6625 * igb_io_slot_reset - called after the pci bus has been reset.
6626 * @pdev: Pointer to PCI device
6627 *
6628 * Restart the card from scratch, as if from a cold-boot. Implementation
6629 * resembles the first-half of the igb_resume routine.
6630 */
6631static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6632{
6633 struct net_device *netdev = pci_get_drvdata(pdev);
6634 struct igb_adapter *adapter = netdev_priv(netdev);
6635 struct e1000_hw *hw = &adapter->hw;
6636 pci_ers_result_t result;
6637 int err;
6638
6639 if (pci_enable_device_mem(pdev)) {
6640 dev_err(&pdev->dev,
6641 "Cannot re-enable PCI device after reset.\n");
6642 result = PCI_ERS_RESULT_DISCONNECT;
6643 } else {
6644 pci_set_master(pdev);
6645 pci_restore_state(pdev);
6646 pci_save_state(pdev);
6647
6648 pci_enable_wake(pdev, PCI_D3hot, 0);
6649 pci_enable_wake(pdev, PCI_D3cold, 0);
6650
6651 igb_reset(adapter);
6652 wr32(E1000_WUS, ~0);
6653 result = PCI_ERS_RESULT_RECOVERED;
6654 }
6655
6656 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6657 if (err) {
6658 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6659 "failed 0x%0x\n", err);
6660 /* non-fatal, continue */
6661 }
6662
6663 return result;
6664}
6665
6666/**
6667 * igb_io_resume - called when traffic can start flowing again.
6668 * @pdev: Pointer to PCI device
6669 *
6670 * This callback is called when the error recovery driver tells us that
6671 * its OK to resume normal operation. Implementation resembles the
6672 * second-half of the igb_resume routine.
6673 */
6674static void igb_io_resume(struct pci_dev *pdev)
6675{
6676 struct net_device *netdev = pci_get_drvdata(pdev);
6677 struct igb_adapter *adapter = netdev_priv(netdev);
6678
6679 if (netif_running(netdev)) {
6680 if (igb_up(adapter)) {
6681 dev_err(&pdev->dev, "igb_up failed after reset\n");
6682 return;
6683 }
6684 }
6685
6686 netif_device_attach(netdev);
6687
6688 /* let the f/w know that the h/w is now under the control of the
6689 * driver. */
6690 igb_get_hw_control(adapter);
6691}
6692
6693static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6694 u8 qsel)
6695{
6696 u32 rar_low, rar_high;
6697 struct e1000_hw *hw = &adapter->hw;
6698
6699 /* HW expects these in little endian so we reverse the byte order
6700 * from network order (big endian) to little endian
6701 */
6702 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6703 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6704 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6705
6706 /* Indicate to hardware the Address is Valid. */
6707 rar_high |= E1000_RAH_AV;
6708
6709 if (hw->mac.type == e1000_82575)
6710 rar_high |= E1000_RAH_POOL_1 * qsel;
6711 else
6712 rar_high |= E1000_RAH_POOL_1 << qsel;
6713
6714 wr32(E1000_RAL(index), rar_low);
6715 wrfl();
6716 wr32(E1000_RAH(index), rar_high);
6717 wrfl();
6718}
6719
6720static int igb_set_vf_mac(struct igb_adapter *adapter,
6721 int vf, unsigned char *mac_addr)
6722{
6723 struct e1000_hw *hw = &adapter->hw;
6724 /* VF MAC addresses start at end of receive addresses and moves
6725 * torwards the first, as a result a collision should not be possible */
6726 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6727
6728 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6729
6730 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6731
6732 return 0;
6733}
6734
6735static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6736{
6737 struct igb_adapter *adapter = netdev_priv(netdev);
6738 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6739 return -EINVAL;
6740 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6741 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6742 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6743 " change effective.");
6744 if (test_bit(__IGB_DOWN, &adapter->state)) {
6745 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6746 " but the PF device is not up.\n");
6747 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6748 " attempting to use the VF device.\n");
6749 }
6750 return igb_set_vf_mac(adapter, vf, mac);
6751}
6752
6753static int igb_link_mbps(int internal_link_speed)
6754{
6755 switch (internal_link_speed) {
6756 case SPEED_100:
6757 return 100;
6758 case SPEED_1000:
6759 return 1000;
6760 default:
6761 return 0;
6762 }
6763}
6764
6765static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6766 int link_speed)
6767{
6768 int rf_dec, rf_int;
6769 u32 bcnrc_val;
6770
6771 if (tx_rate != 0) {
6772 /* Calculate the rate factor values to set */
6773 rf_int = link_speed / tx_rate;
6774 rf_dec = (link_speed - (rf_int * tx_rate));
6775 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6776
6777 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6778 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6779 E1000_RTTBCNRC_RF_INT_MASK);
6780 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6781 } else {
6782 bcnrc_val = 0;
6783 }
6784
6785 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
6786 wr32(E1000_RTTBCNRC, bcnrc_val);
6787}
6788
6789static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
6790{
6791 int actual_link_speed, i;
6792 bool reset_rate = false;
6793
6794 /* VF TX rate limit was not set or not supported */
6795 if ((adapter->vf_rate_link_speed == 0) ||
6796 (adapter->hw.mac.type != e1000_82576))
6797 return;
6798
6799 actual_link_speed = igb_link_mbps(adapter->link_speed);
6800 if (actual_link_speed != adapter->vf_rate_link_speed) {
6801 reset_rate = true;
6802 adapter->vf_rate_link_speed = 0;
6803 dev_info(&adapter->pdev->dev,
6804 "Link speed has been changed. VF Transmit "
6805 "rate is disabled\n");
6806 }
6807
6808 for (i = 0; i < adapter->vfs_allocated_count; i++) {
6809 if (reset_rate)
6810 adapter->vf_data[i].tx_rate = 0;
6811
6812 igb_set_vf_rate_limit(&adapter->hw, i,
6813 adapter->vf_data[i].tx_rate,
6814 actual_link_speed);
6815 }
6816}
6817
6818static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
6819{
6820 struct igb_adapter *adapter = netdev_priv(netdev);
6821 struct e1000_hw *hw = &adapter->hw;
6822 int actual_link_speed;
6823
6824 if (hw->mac.type != e1000_82576)
6825 return -EOPNOTSUPP;
6826
6827 actual_link_speed = igb_link_mbps(adapter->link_speed);
6828 if ((vf >= adapter->vfs_allocated_count) ||
6829 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
6830 (tx_rate < 0) || (tx_rate > actual_link_speed))
6831 return -EINVAL;
6832
6833 adapter->vf_rate_link_speed = actual_link_speed;
6834 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
6835 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
6836
6837 return 0;
6838}
6839
6840static int igb_ndo_get_vf_config(struct net_device *netdev,
6841 int vf, struct ifla_vf_info *ivi)
6842{
6843 struct igb_adapter *adapter = netdev_priv(netdev);
6844 if (vf >= adapter->vfs_allocated_count)
6845 return -EINVAL;
6846 ivi->vf = vf;
6847 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
6848 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
6849 ivi->vlan = adapter->vf_data[vf].pf_vlan;
6850 ivi->qos = adapter->vf_data[vf].pf_qos;
6851 return 0;
6852}
6853
6854static void igb_vmm_control(struct igb_adapter *adapter)
6855{
6856 struct e1000_hw *hw = &adapter->hw;
6857 u32 reg;
6858
6859 switch (hw->mac.type) {
6860 case e1000_82575:
6861 default:
6862 /* replication is not supported for 82575 */
6863 return;
6864 case e1000_82576:
6865 /* notify HW that the MAC is adding vlan tags */
6866 reg = rd32(E1000_DTXCTL);
6867 reg |= E1000_DTXCTL_VLAN_ADDED;
6868 wr32(E1000_DTXCTL, reg);
6869 case e1000_82580:
6870 /* enable replication vlan tag stripping */
6871 reg = rd32(E1000_RPLOLR);
6872 reg |= E1000_RPLOLR_STRVLAN;
6873 wr32(E1000_RPLOLR, reg);
6874 case e1000_i350:
6875 /* none of the above registers are supported by i350 */
6876 break;
6877 }
6878
6879 if (adapter->vfs_allocated_count) {
6880 igb_vmdq_set_loopback_pf(hw, true);
6881 igb_vmdq_set_replication_pf(hw, true);
6882 igb_vmdq_set_anti_spoofing_pf(hw, true,
6883 adapter->vfs_allocated_count);
6884 } else {
6885 igb_vmdq_set_loopback_pf(hw, false);
6886 igb_vmdq_set_replication_pf(hw, false);
6887 }
6888}
6889
6890/* igb_main.c */