Loading...
1// SPDX-License-Identifier: GPL-2.0
2/* Copyright(c) 2007 - 2018 Intel Corporation. */
3
4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6#include <linux/module.h>
7#include <linux/types.h>
8#include <linux/init.h>
9#include <linux/bitops.h>
10#include <linux/vmalloc.h>
11#include <linux/pagemap.h>
12#include <linux/netdevice.h>
13#include <linux/ipv6.h>
14#include <linux/slab.h>
15#include <net/checksum.h>
16#include <net/ip6_checksum.h>
17#include <net/pkt_sched.h>
18#include <net/pkt_cls.h>
19#include <linux/net_tstamp.h>
20#include <linux/mii.h>
21#include <linux/ethtool.h>
22#include <linux/if.h>
23#include <linux/if_vlan.h>
24#include <linux/pci.h>
25#include <linux/delay.h>
26#include <linux/interrupt.h>
27#include <linux/ip.h>
28#include <linux/tcp.h>
29#include <linux/sctp.h>
30#include <linux/if_ether.h>
31#include <linux/prefetch.h>
32#include <linux/bpf.h>
33#include <linux/bpf_trace.h>
34#include <linux/pm_runtime.h>
35#include <linux/etherdevice.h>
36#ifdef CONFIG_IGB_DCA
37#include <linux/dca.h>
38#endif
39#include <linux/i2c.h>
40#include "igb.h"
41
42enum queue_mode {
43 QUEUE_MODE_STRICT_PRIORITY,
44 QUEUE_MODE_STREAM_RESERVATION,
45};
46
47enum tx_queue_prio {
48 TX_QUEUE_PRIO_HIGH,
49 TX_QUEUE_PRIO_LOW,
50};
51
52char igb_driver_name[] = "igb";
53static const char igb_driver_string[] =
54 "Intel(R) Gigabit Ethernet Network Driver";
55static const char igb_copyright[] =
56 "Copyright (c) 2007-2014 Intel Corporation.";
57
58static const struct e1000_info *igb_info_tbl[] = {
59 [board_82575] = &e1000_82575_info,
60};
61
62static const struct pci_device_id igb_pci_tbl[] = {
63 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_1GBPS) },
64 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_SGMII) },
65 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I354_BACKPLANE_2_5GBPS) },
66 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
67 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
68 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
69 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
70 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
71 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER_FLASHLESS), board_82575 },
72 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES_FLASHLESS), board_82575 },
73 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
74 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
75 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
76 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
77 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
98 /* required last entry */
99 {0, }
100};
101
102MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
103
104static int igb_setup_all_tx_resources(struct igb_adapter *);
105static int igb_setup_all_rx_resources(struct igb_adapter *);
106static void igb_free_all_tx_resources(struct igb_adapter *);
107static void igb_free_all_rx_resources(struct igb_adapter *);
108static void igb_setup_mrqc(struct igb_adapter *);
109static int igb_probe(struct pci_dev *, const struct pci_device_id *);
110static void igb_remove(struct pci_dev *pdev);
111static void igb_init_queue_configuration(struct igb_adapter *adapter);
112static int igb_sw_init(struct igb_adapter *);
113int igb_open(struct net_device *);
114int igb_close(struct net_device *);
115static void igb_configure(struct igb_adapter *);
116static void igb_configure_tx(struct igb_adapter *);
117static void igb_configure_rx(struct igb_adapter *);
118static void igb_clean_all_tx_rings(struct igb_adapter *);
119static void igb_clean_all_rx_rings(struct igb_adapter *);
120static void igb_clean_tx_ring(struct igb_ring *);
121static void igb_clean_rx_ring(struct igb_ring *);
122static void igb_set_rx_mode(struct net_device *);
123static void igb_update_phy_info(struct timer_list *);
124static void igb_watchdog(struct timer_list *);
125static void igb_watchdog_task(struct work_struct *);
126static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
127static void igb_get_stats64(struct net_device *dev,
128 struct rtnl_link_stats64 *stats);
129static int igb_change_mtu(struct net_device *, int);
130static int igb_set_mac(struct net_device *, void *);
131static void igb_set_uta(struct igb_adapter *adapter, bool set);
132static irqreturn_t igb_intr(int irq, void *);
133static irqreturn_t igb_intr_msi(int irq, void *);
134static irqreturn_t igb_msix_other(int irq, void *);
135static irqreturn_t igb_msix_ring(int irq, void *);
136#ifdef CONFIG_IGB_DCA
137static void igb_update_dca(struct igb_q_vector *);
138static void igb_setup_dca(struct igb_adapter *);
139#endif /* CONFIG_IGB_DCA */
140static int igb_poll(struct napi_struct *, int);
141static bool igb_clean_tx_irq(struct igb_q_vector *, int);
142static int igb_clean_rx_irq(struct igb_q_vector *, int);
143static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
144static void igb_tx_timeout(struct net_device *, unsigned int txqueue);
145static void igb_reset_task(struct work_struct *);
146static void igb_vlan_mode(struct net_device *netdev,
147 netdev_features_t features);
148static int igb_vlan_rx_add_vid(struct net_device *, __be16, u16);
149static int igb_vlan_rx_kill_vid(struct net_device *, __be16, u16);
150static void igb_restore_vlan(struct igb_adapter *);
151static void igb_rar_set_index(struct igb_adapter *, u32);
152static void igb_ping_all_vfs(struct igb_adapter *);
153static void igb_msg_task(struct igb_adapter *);
154static void igb_vmm_control(struct igb_adapter *);
155static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
156static void igb_flush_mac_table(struct igb_adapter *);
157static int igb_available_rars(struct igb_adapter *, u8);
158static void igb_set_default_mac_filter(struct igb_adapter *);
159static int igb_uc_sync(struct net_device *, const unsigned char *);
160static int igb_uc_unsync(struct net_device *, const unsigned char *);
161static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
162static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
163static int igb_ndo_set_vf_vlan(struct net_device *netdev,
164 int vf, u16 vlan, u8 qos, __be16 vlan_proto);
165static int igb_ndo_set_vf_bw(struct net_device *, int, int, int);
166static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
167 bool setting);
168static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf,
169 bool setting);
170static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
171 struct ifla_vf_info *ivi);
172static void igb_check_vf_rate_limit(struct igb_adapter *);
173static void igb_nfc_filter_exit(struct igb_adapter *adapter);
174static void igb_nfc_filter_restore(struct igb_adapter *adapter);
175
176#ifdef CONFIG_PCI_IOV
177static int igb_vf_configure(struct igb_adapter *adapter, int vf);
178static int igb_disable_sriov(struct pci_dev *dev, bool reinit);
179#endif
180
181static int igb_suspend(struct device *);
182static int igb_resume(struct device *);
183static int igb_runtime_suspend(struct device *dev);
184static int igb_runtime_resume(struct device *dev);
185static int igb_runtime_idle(struct device *dev);
186#ifdef CONFIG_PM
187static const struct dev_pm_ops igb_pm_ops = {
188 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
189 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
190 igb_runtime_idle)
191};
192#endif
193static void igb_shutdown(struct pci_dev *);
194static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs);
195#ifdef CONFIG_IGB_DCA
196static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
197static struct notifier_block dca_notifier = {
198 .notifier_call = igb_notify_dca,
199 .next = NULL,
200 .priority = 0
201};
202#endif
203#ifdef CONFIG_PCI_IOV
204static unsigned int max_vfs;
205module_param(max_vfs, uint, 0);
206MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate per physical function");
207#endif /* CONFIG_PCI_IOV */
208
209static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
210 pci_channel_state_t);
211static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
212static void igb_io_resume(struct pci_dev *);
213
214static const struct pci_error_handlers igb_err_handler = {
215 .error_detected = igb_io_error_detected,
216 .slot_reset = igb_io_slot_reset,
217 .resume = igb_io_resume,
218};
219
220static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
221
222static struct pci_driver igb_driver = {
223 .name = igb_driver_name,
224 .id_table = igb_pci_tbl,
225 .probe = igb_probe,
226 .remove = igb_remove,
227#ifdef CONFIG_PM
228 .driver.pm = &igb_pm_ops,
229#endif
230 .shutdown = igb_shutdown,
231 .sriov_configure = igb_pci_sriov_configure,
232 .err_handler = &igb_err_handler
233};
234
235MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
236MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
237MODULE_LICENSE("GPL v2");
238
239#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
240static int debug = -1;
241module_param(debug, int, 0);
242MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
243
244struct igb_reg_info {
245 u32 ofs;
246 char *name;
247};
248
249static const struct igb_reg_info igb_reg_info_tbl[] = {
250
251 /* General Registers */
252 {E1000_CTRL, "CTRL"},
253 {E1000_STATUS, "STATUS"},
254 {E1000_CTRL_EXT, "CTRL_EXT"},
255
256 /* Interrupt Registers */
257 {E1000_ICR, "ICR"},
258
259 /* RX Registers */
260 {E1000_RCTL, "RCTL"},
261 {E1000_RDLEN(0), "RDLEN"},
262 {E1000_RDH(0), "RDH"},
263 {E1000_RDT(0), "RDT"},
264 {E1000_RXDCTL(0), "RXDCTL"},
265 {E1000_RDBAL(0), "RDBAL"},
266 {E1000_RDBAH(0), "RDBAH"},
267
268 /* TX Registers */
269 {E1000_TCTL, "TCTL"},
270 {E1000_TDBAL(0), "TDBAL"},
271 {E1000_TDBAH(0), "TDBAH"},
272 {E1000_TDLEN(0), "TDLEN"},
273 {E1000_TDH(0), "TDH"},
274 {E1000_TDT(0), "TDT"},
275 {E1000_TXDCTL(0), "TXDCTL"},
276 {E1000_TDFH, "TDFH"},
277 {E1000_TDFT, "TDFT"},
278 {E1000_TDFHS, "TDFHS"},
279 {E1000_TDFPC, "TDFPC"},
280
281 /* List Terminator */
282 {}
283};
284
285/* igb_regdump - register printout routine */
286static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
287{
288 int n = 0;
289 char rname[16];
290 u32 regs[8];
291
292 switch (reginfo->ofs) {
293 case E1000_RDLEN(0):
294 for (n = 0; n < 4; n++)
295 regs[n] = rd32(E1000_RDLEN(n));
296 break;
297 case E1000_RDH(0):
298 for (n = 0; n < 4; n++)
299 regs[n] = rd32(E1000_RDH(n));
300 break;
301 case E1000_RDT(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_RDT(n));
304 break;
305 case E1000_RXDCTL(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_RXDCTL(n));
308 break;
309 case E1000_RDBAL(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_RDBAL(n));
312 break;
313 case E1000_RDBAH(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_RDBAH(n));
316 break;
317 case E1000_TDBAL(0):
318 for (n = 0; n < 4; n++)
319 regs[n] = rd32(E1000_TDBAL(n));
320 break;
321 case E1000_TDBAH(0):
322 for (n = 0; n < 4; n++)
323 regs[n] = rd32(E1000_TDBAH(n));
324 break;
325 case E1000_TDLEN(0):
326 for (n = 0; n < 4; n++)
327 regs[n] = rd32(E1000_TDLEN(n));
328 break;
329 case E1000_TDH(0):
330 for (n = 0; n < 4; n++)
331 regs[n] = rd32(E1000_TDH(n));
332 break;
333 case E1000_TDT(0):
334 for (n = 0; n < 4; n++)
335 regs[n] = rd32(E1000_TDT(n));
336 break;
337 case E1000_TXDCTL(0):
338 for (n = 0; n < 4; n++)
339 regs[n] = rd32(E1000_TXDCTL(n));
340 break;
341 default:
342 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
343 return;
344 }
345
346 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
347 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
348 regs[2], regs[3]);
349}
350
351/* igb_dump - Print registers, Tx-rings and Rx-rings */
352static void igb_dump(struct igb_adapter *adapter)
353{
354 struct net_device *netdev = adapter->netdev;
355 struct e1000_hw *hw = &adapter->hw;
356 struct igb_reg_info *reginfo;
357 struct igb_ring *tx_ring;
358 union e1000_adv_tx_desc *tx_desc;
359 struct my_u0 { __le64 a; __le64 b; } *u0;
360 struct igb_ring *rx_ring;
361 union e1000_adv_rx_desc *rx_desc;
362 u32 staterr;
363 u16 i, n;
364
365 if (!netif_msg_hw(adapter))
366 return;
367
368 /* Print netdevice Info */
369 if (netdev) {
370 dev_info(&adapter->pdev->dev, "Net device Info\n");
371 pr_info("Device Name state trans_start\n");
372 pr_info("%-15s %016lX %016lX\n", netdev->name,
373 netdev->state, dev_trans_start(netdev));
374 }
375
376 /* Print Registers */
377 dev_info(&adapter->pdev->dev, "Register Dump\n");
378 pr_info(" Register Name Value\n");
379 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
380 reginfo->name; reginfo++) {
381 igb_regdump(hw, reginfo);
382 }
383
384 /* Print TX Ring Summary */
385 if (!netdev || !netif_running(netdev))
386 goto exit;
387
388 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
389 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
390 for (n = 0; n < adapter->num_tx_queues; n++) {
391 struct igb_tx_buffer *buffer_info;
392 tx_ring = adapter->tx_ring[n];
393 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
394 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
395 n, tx_ring->next_to_use, tx_ring->next_to_clean,
396 (u64)dma_unmap_addr(buffer_info, dma),
397 dma_unmap_len(buffer_info, len),
398 buffer_info->next_to_watch,
399 (u64)buffer_info->time_stamp);
400 }
401
402 /* Print TX Rings */
403 if (!netif_msg_tx_done(adapter))
404 goto rx_ring_summary;
405
406 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
407
408 /* Transmit Descriptor Formats
409 *
410 * Advanced Transmit Descriptor
411 * +--------------------------------------------------------------+
412 * 0 | Buffer Address [63:0] |
413 * +--------------------------------------------------------------+
414 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
415 * +--------------------------------------------------------------+
416 * 63 46 45 40 39 38 36 35 32 31 24 15 0
417 */
418
419 for (n = 0; n < adapter->num_tx_queues; n++) {
420 tx_ring = adapter->tx_ring[n];
421 pr_info("------------------------------------\n");
422 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
423 pr_info("------------------------------------\n");
424 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] [bi->dma ] leng ntw timestamp bi->skb\n");
425
426 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
427 const char *next_desc;
428 struct igb_tx_buffer *buffer_info;
429 tx_desc = IGB_TX_DESC(tx_ring, i);
430 buffer_info = &tx_ring->tx_buffer_info[i];
431 u0 = (struct my_u0 *)tx_desc;
432 if (i == tx_ring->next_to_use &&
433 i == tx_ring->next_to_clean)
434 next_desc = " NTC/U";
435 else if (i == tx_ring->next_to_use)
436 next_desc = " NTU";
437 else if (i == tx_ring->next_to_clean)
438 next_desc = " NTC";
439 else
440 next_desc = "";
441
442 pr_info("T [0x%03X] %016llX %016llX %016llX %04X %p %016llX %p%s\n",
443 i, le64_to_cpu(u0->a),
444 le64_to_cpu(u0->b),
445 (u64)dma_unmap_addr(buffer_info, dma),
446 dma_unmap_len(buffer_info, len),
447 buffer_info->next_to_watch,
448 (u64)buffer_info->time_stamp,
449 buffer_info->skb, next_desc);
450
451 if (netif_msg_pktdata(adapter) && buffer_info->skb)
452 print_hex_dump(KERN_INFO, "",
453 DUMP_PREFIX_ADDRESS,
454 16, 1, buffer_info->skb->data,
455 dma_unmap_len(buffer_info, len),
456 true);
457 }
458 }
459
460 /* Print RX Rings Summary */
461rx_ring_summary:
462 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
463 pr_info("Queue [NTU] [NTC]\n");
464 for (n = 0; n < adapter->num_rx_queues; n++) {
465 rx_ring = adapter->rx_ring[n];
466 pr_info(" %5d %5X %5X\n",
467 n, rx_ring->next_to_use, rx_ring->next_to_clean);
468 }
469
470 /* Print RX Rings */
471 if (!netif_msg_rx_status(adapter))
472 goto exit;
473
474 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
475
476 /* Advanced Receive Descriptor (Read) Format
477 * 63 1 0
478 * +-----------------------------------------------------+
479 * 0 | Packet Buffer Address [63:1] |A0/NSE|
480 * +----------------------------------------------+------+
481 * 8 | Header Buffer Address [63:1] | DD |
482 * +-----------------------------------------------------+
483 *
484 *
485 * Advanced Receive Descriptor (Write-Back) Format
486 *
487 * 63 48 47 32 31 30 21 20 17 16 4 3 0
488 * +------------------------------------------------------+
489 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
490 * | Checksum Ident | | | | Type | Type |
491 * +------------------------------------------------------+
492 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
493 * +------------------------------------------------------+
494 * 63 48 47 32 31 20 19 0
495 */
496
497 for (n = 0; n < adapter->num_rx_queues; n++) {
498 rx_ring = adapter->rx_ring[n];
499 pr_info("------------------------------------\n");
500 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
501 pr_info("------------------------------------\n");
502 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] [bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
503 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] ---------------- [bi->skb] <-- Adv Rx Write-Back format\n");
504
505 for (i = 0; i < rx_ring->count; i++) {
506 const char *next_desc;
507 struct igb_rx_buffer *buffer_info;
508 buffer_info = &rx_ring->rx_buffer_info[i];
509 rx_desc = IGB_RX_DESC(rx_ring, i);
510 u0 = (struct my_u0 *)rx_desc;
511 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
512
513 if (i == rx_ring->next_to_use)
514 next_desc = " NTU";
515 else if (i == rx_ring->next_to_clean)
516 next_desc = " NTC";
517 else
518 next_desc = "";
519
520 if (staterr & E1000_RXD_STAT_DD) {
521 /* Descriptor Done */
522 pr_info("%s[0x%03X] %016llX %016llX ---------------- %s\n",
523 "RWB", i,
524 le64_to_cpu(u0->a),
525 le64_to_cpu(u0->b),
526 next_desc);
527 } else {
528 pr_info("%s[0x%03X] %016llX %016llX %016llX %s\n",
529 "R ", i,
530 le64_to_cpu(u0->a),
531 le64_to_cpu(u0->b),
532 (u64)buffer_info->dma,
533 next_desc);
534
535 if (netif_msg_pktdata(adapter) &&
536 buffer_info->dma && buffer_info->page) {
537 print_hex_dump(KERN_INFO, "",
538 DUMP_PREFIX_ADDRESS,
539 16, 1,
540 page_address(buffer_info->page) +
541 buffer_info->page_offset,
542 igb_rx_bufsz(rx_ring), true);
543 }
544 }
545 }
546 }
547
548exit:
549 return;
550}
551
552/**
553 * igb_get_i2c_data - Reads the I2C SDA data bit
554 * @data: opaque pointer to adapter struct
555 *
556 * Returns the I2C data bit value
557 **/
558static int igb_get_i2c_data(void *data)
559{
560 struct igb_adapter *adapter = (struct igb_adapter *)data;
561 struct e1000_hw *hw = &adapter->hw;
562 s32 i2cctl = rd32(E1000_I2CPARAMS);
563
564 return !!(i2cctl & E1000_I2C_DATA_IN);
565}
566
567/**
568 * igb_set_i2c_data - Sets the I2C data bit
569 * @data: pointer to hardware structure
570 * @state: I2C data value (0 or 1) to set
571 *
572 * Sets the I2C data bit
573 **/
574static void igb_set_i2c_data(void *data, int state)
575{
576 struct igb_adapter *adapter = (struct igb_adapter *)data;
577 struct e1000_hw *hw = &adapter->hw;
578 s32 i2cctl = rd32(E1000_I2CPARAMS);
579
580 if (state) {
581 i2cctl |= E1000_I2C_DATA_OUT | E1000_I2C_DATA_OE_N;
582 } else {
583 i2cctl &= ~E1000_I2C_DATA_OE_N;
584 i2cctl &= ~E1000_I2C_DATA_OUT;
585 }
586
587 wr32(E1000_I2CPARAMS, i2cctl);
588 wrfl();
589}
590
591/**
592 * igb_set_i2c_clk - Sets the I2C SCL clock
593 * @data: pointer to hardware structure
594 * @state: state to set clock
595 *
596 * Sets the I2C clock line to state
597 **/
598static void igb_set_i2c_clk(void *data, int state)
599{
600 struct igb_adapter *adapter = (struct igb_adapter *)data;
601 struct e1000_hw *hw = &adapter->hw;
602 s32 i2cctl = rd32(E1000_I2CPARAMS);
603
604 if (state) {
605 i2cctl |= E1000_I2C_CLK_OUT | E1000_I2C_CLK_OE_N;
606 } else {
607 i2cctl &= ~E1000_I2C_CLK_OUT;
608 i2cctl &= ~E1000_I2C_CLK_OE_N;
609 }
610 wr32(E1000_I2CPARAMS, i2cctl);
611 wrfl();
612}
613
614/**
615 * igb_get_i2c_clk - Gets the I2C SCL clock state
616 * @data: pointer to hardware structure
617 *
618 * Gets the I2C clock state
619 **/
620static int igb_get_i2c_clk(void *data)
621{
622 struct igb_adapter *adapter = (struct igb_adapter *)data;
623 struct e1000_hw *hw = &adapter->hw;
624 s32 i2cctl = rd32(E1000_I2CPARAMS);
625
626 return !!(i2cctl & E1000_I2C_CLK_IN);
627}
628
629static const struct i2c_algo_bit_data igb_i2c_algo = {
630 .setsda = igb_set_i2c_data,
631 .setscl = igb_set_i2c_clk,
632 .getsda = igb_get_i2c_data,
633 .getscl = igb_get_i2c_clk,
634 .udelay = 5,
635 .timeout = 20,
636};
637
638/**
639 * igb_get_hw_dev - return device
640 * @hw: pointer to hardware structure
641 *
642 * used by hardware layer to print debugging information
643 **/
644struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
645{
646 struct igb_adapter *adapter = hw->back;
647 return adapter->netdev;
648}
649
650/**
651 * igb_init_module - Driver Registration Routine
652 *
653 * igb_init_module is the first routine called when the driver is
654 * loaded. All it does is register with the PCI subsystem.
655 **/
656static int __init igb_init_module(void)
657{
658 int ret;
659
660 pr_info("%s\n", igb_driver_string);
661 pr_info("%s\n", igb_copyright);
662
663#ifdef CONFIG_IGB_DCA
664 dca_register_notify(&dca_notifier);
665#endif
666 ret = pci_register_driver(&igb_driver);
667 return ret;
668}
669
670module_init(igb_init_module);
671
672/**
673 * igb_exit_module - Driver Exit Cleanup Routine
674 *
675 * igb_exit_module is called just before the driver is removed
676 * from memory.
677 **/
678static void __exit igb_exit_module(void)
679{
680#ifdef CONFIG_IGB_DCA
681 dca_unregister_notify(&dca_notifier);
682#endif
683 pci_unregister_driver(&igb_driver);
684}
685
686module_exit(igb_exit_module);
687
688#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
689/**
690 * igb_cache_ring_register - Descriptor ring to register mapping
691 * @adapter: board private structure to initialize
692 *
693 * Once we know the feature-set enabled for the device, we'll cache
694 * the register offset the descriptor ring is assigned to.
695 **/
696static void igb_cache_ring_register(struct igb_adapter *adapter)
697{
698 int i = 0, j = 0;
699 u32 rbase_offset = adapter->vfs_allocated_count;
700
701 switch (adapter->hw.mac.type) {
702 case e1000_82576:
703 /* The queues are allocated for virtualization such that VF 0
704 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
705 * In order to avoid collision we start at the first free queue
706 * and continue consuming queues in the same sequence
707 */
708 if (adapter->vfs_allocated_count) {
709 for (; i < adapter->rss_queues; i++)
710 adapter->rx_ring[i]->reg_idx = rbase_offset +
711 Q_IDX_82576(i);
712 }
713 fallthrough;
714 case e1000_82575:
715 case e1000_82580:
716 case e1000_i350:
717 case e1000_i354:
718 case e1000_i210:
719 case e1000_i211:
720 default:
721 for (; i < adapter->num_rx_queues; i++)
722 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
723 for (; j < adapter->num_tx_queues; j++)
724 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
725 break;
726 }
727}
728
729u32 igb_rd32(struct e1000_hw *hw, u32 reg)
730{
731 struct igb_adapter *igb = container_of(hw, struct igb_adapter, hw);
732 u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
733 u32 value = 0;
734
735 if (E1000_REMOVED(hw_addr))
736 return ~value;
737
738 value = readl(&hw_addr[reg]);
739
740 /* reads should not return all F's */
741 if (!(~value) && (!reg || !(~readl(hw_addr)))) {
742 struct net_device *netdev = igb->netdev;
743 hw->hw_addr = NULL;
744 netdev_err(netdev, "PCIe link lost\n");
745 WARN(pci_device_is_present(igb->pdev),
746 "igb: Failed to read reg 0x%x!\n", reg);
747 }
748
749 return value;
750}
751
752/**
753 * igb_write_ivar - configure ivar for given MSI-X vector
754 * @hw: pointer to the HW structure
755 * @msix_vector: vector number we are allocating to a given ring
756 * @index: row index of IVAR register to write within IVAR table
757 * @offset: column offset of in IVAR, should be multiple of 8
758 *
759 * This function is intended to handle the writing of the IVAR register
760 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
761 * each containing an cause allocation for an Rx and Tx ring, and a
762 * variable number of rows depending on the number of queues supported.
763 **/
764static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
765 int index, int offset)
766{
767 u32 ivar = array_rd32(E1000_IVAR0, index);
768
769 /* clear any bits that are currently set */
770 ivar &= ~((u32)0xFF << offset);
771
772 /* write vector and valid bit */
773 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
774
775 array_wr32(E1000_IVAR0, index, ivar);
776}
777
778#define IGB_N0_QUEUE -1
779static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
780{
781 struct igb_adapter *adapter = q_vector->adapter;
782 struct e1000_hw *hw = &adapter->hw;
783 int rx_queue = IGB_N0_QUEUE;
784 int tx_queue = IGB_N0_QUEUE;
785 u32 msixbm = 0;
786
787 if (q_vector->rx.ring)
788 rx_queue = q_vector->rx.ring->reg_idx;
789 if (q_vector->tx.ring)
790 tx_queue = q_vector->tx.ring->reg_idx;
791
792 switch (hw->mac.type) {
793 case e1000_82575:
794 /* The 82575 assigns vectors using a bitmask, which matches the
795 * bitmask for the EICR/EIMS/EIMC registers. To assign one
796 * or more queues to a vector, we write the appropriate bits
797 * into the MSIXBM register for that vector.
798 */
799 if (rx_queue > IGB_N0_QUEUE)
800 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
801 if (tx_queue > IGB_N0_QUEUE)
802 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
803 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) && msix_vector == 0)
804 msixbm |= E1000_EIMS_OTHER;
805 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
806 q_vector->eims_value = msixbm;
807 break;
808 case e1000_82576:
809 /* 82576 uses a table that essentially consists of 2 columns
810 * with 8 rows. The ordering is column-major so we use the
811 * lower 3 bits as the row index, and the 4th bit as the
812 * column offset.
813 */
814 if (rx_queue > IGB_N0_QUEUE)
815 igb_write_ivar(hw, msix_vector,
816 rx_queue & 0x7,
817 (rx_queue & 0x8) << 1);
818 if (tx_queue > IGB_N0_QUEUE)
819 igb_write_ivar(hw, msix_vector,
820 tx_queue & 0x7,
821 ((tx_queue & 0x8) << 1) + 8);
822 q_vector->eims_value = BIT(msix_vector);
823 break;
824 case e1000_82580:
825 case e1000_i350:
826 case e1000_i354:
827 case e1000_i210:
828 case e1000_i211:
829 /* On 82580 and newer adapters the scheme is similar to 82576
830 * however instead of ordering column-major we have things
831 * ordered row-major. So we traverse the table by using
832 * bit 0 as the column offset, and the remaining bits as the
833 * row index.
834 */
835 if (rx_queue > IGB_N0_QUEUE)
836 igb_write_ivar(hw, msix_vector,
837 rx_queue >> 1,
838 (rx_queue & 0x1) << 4);
839 if (tx_queue > IGB_N0_QUEUE)
840 igb_write_ivar(hw, msix_vector,
841 tx_queue >> 1,
842 ((tx_queue & 0x1) << 4) + 8);
843 q_vector->eims_value = BIT(msix_vector);
844 break;
845 default:
846 BUG();
847 break;
848 }
849
850 /* add q_vector eims value to global eims_enable_mask */
851 adapter->eims_enable_mask |= q_vector->eims_value;
852
853 /* configure q_vector to set itr on first interrupt */
854 q_vector->set_itr = 1;
855}
856
857/**
858 * igb_configure_msix - Configure MSI-X hardware
859 * @adapter: board private structure to initialize
860 *
861 * igb_configure_msix sets up the hardware to properly
862 * generate MSI-X interrupts.
863 **/
864static void igb_configure_msix(struct igb_adapter *adapter)
865{
866 u32 tmp;
867 int i, vector = 0;
868 struct e1000_hw *hw = &adapter->hw;
869
870 adapter->eims_enable_mask = 0;
871
872 /* set vector for other causes, i.e. link changes */
873 switch (hw->mac.type) {
874 case e1000_82575:
875 tmp = rd32(E1000_CTRL_EXT);
876 /* enable MSI-X PBA support*/
877 tmp |= E1000_CTRL_EXT_PBA_CLR;
878
879 /* Auto-Mask interrupts upon ICR read. */
880 tmp |= E1000_CTRL_EXT_EIAME;
881 tmp |= E1000_CTRL_EXT_IRCA;
882
883 wr32(E1000_CTRL_EXT, tmp);
884
885 /* enable msix_other interrupt */
886 array_wr32(E1000_MSIXBM(0), vector++, E1000_EIMS_OTHER);
887 adapter->eims_other = E1000_EIMS_OTHER;
888
889 break;
890
891 case e1000_82576:
892 case e1000_82580:
893 case e1000_i350:
894 case e1000_i354:
895 case e1000_i210:
896 case e1000_i211:
897 /* Turn on MSI-X capability first, or our settings
898 * won't stick. And it will take days to debug.
899 */
900 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
901 E1000_GPIE_PBA | E1000_GPIE_EIAME |
902 E1000_GPIE_NSICR);
903
904 /* enable msix_other interrupt */
905 adapter->eims_other = BIT(vector);
906 tmp = (vector++ | E1000_IVAR_VALID) << 8;
907
908 wr32(E1000_IVAR_MISC, tmp);
909 break;
910 default:
911 /* do nothing, since nothing else supports MSI-X */
912 break;
913 } /* switch (hw->mac.type) */
914
915 adapter->eims_enable_mask |= adapter->eims_other;
916
917 for (i = 0; i < adapter->num_q_vectors; i++)
918 igb_assign_vector(adapter->q_vector[i], vector++);
919
920 wrfl();
921}
922
923/**
924 * igb_request_msix - Initialize MSI-X interrupts
925 * @adapter: board private structure to initialize
926 *
927 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
928 * kernel.
929 **/
930static int igb_request_msix(struct igb_adapter *adapter)
931{
932 unsigned int num_q_vectors = adapter->num_q_vectors;
933 struct net_device *netdev = adapter->netdev;
934 int i, err = 0, vector = 0, free_vector = 0;
935
936 err = request_irq(adapter->msix_entries[vector].vector,
937 igb_msix_other, 0, netdev->name, adapter);
938 if (err)
939 goto err_out;
940
941 if (num_q_vectors > MAX_Q_VECTORS) {
942 num_q_vectors = MAX_Q_VECTORS;
943 dev_warn(&adapter->pdev->dev,
944 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
945 adapter->num_q_vectors, MAX_Q_VECTORS);
946 }
947 for (i = 0; i < num_q_vectors; i++) {
948 struct igb_q_vector *q_vector = adapter->q_vector[i];
949
950 vector++;
951
952 q_vector->itr_register = adapter->io_addr + E1000_EITR(vector);
953
954 if (q_vector->rx.ring && q_vector->tx.ring)
955 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
956 q_vector->rx.ring->queue_index);
957 else if (q_vector->tx.ring)
958 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
959 q_vector->tx.ring->queue_index);
960 else if (q_vector->rx.ring)
961 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
962 q_vector->rx.ring->queue_index);
963 else
964 sprintf(q_vector->name, "%s-unused", netdev->name);
965
966 err = request_irq(adapter->msix_entries[vector].vector,
967 igb_msix_ring, 0, q_vector->name,
968 q_vector);
969 if (err)
970 goto err_free;
971 }
972
973 igb_configure_msix(adapter);
974 return 0;
975
976err_free:
977 /* free already assigned IRQs */
978 free_irq(adapter->msix_entries[free_vector++].vector, adapter);
979
980 vector--;
981 for (i = 0; i < vector; i++) {
982 free_irq(adapter->msix_entries[free_vector++].vector,
983 adapter->q_vector[i]);
984 }
985err_out:
986 return err;
987}
988
989/**
990 * igb_free_q_vector - Free memory allocated for specific interrupt vector
991 * @adapter: board private structure to initialize
992 * @v_idx: Index of vector to be freed
993 *
994 * This function frees the memory allocated to the q_vector.
995 **/
996static void igb_free_q_vector(struct igb_adapter *adapter, int v_idx)
997{
998 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
999
1000 adapter->q_vector[v_idx] = NULL;
1001
1002 /* igb_get_stats64() might access the rings on this vector,
1003 * we must wait a grace period before freeing it.
1004 */
1005 if (q_vector)
1006 kfree_rcu(q_vector, rcu);
1007}
1008
1009/**
1010 * igb_reset_q_vector - Reset config for interrupt vector
1011 * @adapter: board private structure to initialize
1012 * @v_idx: Index of vector to be reset
1013 *
1014 * If NAPI is enabled it will delete any references to the
1015 * NAPI struct. This is preparation for igb_free_q_vector.
1016 **/
1017static void igb_reset_q_vector(struct igb_adapter *adapter, int v_idx)
1018{
1019 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1020
1021 /* Coming from igb_set_interrupt_capability, the vectors are not yet
1022 * allocated. So, q_vector is NULL so we should stop here.
1023 */
1024 if (!q_vector)
1025 return;
1026
1027 if (q_vector->tx.ring)
1028 adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
1029
1030 if (q_vector->rx.ring)
1031 adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
1032
1033 netif_napi_del(&q_vector->napi);
1034
1035}
1036
1037static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
1038{
1039 int v_idx = adapter->num_q_vectors;
1040
1041 if (adapter->flags & IGB_FLAG_HAS_MSIX)
1042 pci_disable_msix(adapter->pdev);
1043 else if (adapter->flags & IGB_FLAG_HAS_MSI)
1044 pci_disable_msi(adapter->pdev);
1045
1046 while (v_idx--)
1047 igb_reset_q_vector(adapter, v_idx);
1048}
1049
1050/**
1051 * igb_free_q_vectors - Free memory allocated for interrupt vectors
1052 * @adapter: board private structure to initialize
1053 *
1054 * This function frees the memory allocated to the q_vectors. In addition if
1055 * NAPI is enabled it will delete any references to the NAPI struct prior
1056 * to freeing the q_vector.
1057 **/
1058static void igb_free_q_vectors(struct igb_adapter *adapter)
1059{
1060 int v_idx = adapter->num_q_vectors;
1061
1062 adapter->num_tx_queues = 0;
1063 adapter->num_rx_queues = 0;
1064 adapter->num_q_vectors = 0;
1065
1066 while (v_idx--) {
1067 igb_reset_q_vector(adapter, v_idx);
1068 igb_free_q_vector(adapter, v_idx);
1069 }
1070}
1071
1072/**
1073 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1074 * @adapter: board private structure to initialize
1075 *
1076 * This function resets the device so that it has 0 Rx queues, Tx queues, and
1077 * MSI-X interrupts allocated.
1078 */
1079static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1080{
1081 igb_free_q_vectors(adapter);
1082 igb_reset_interrupt_capability(adapter);
1083}
1084
1085/**
1086 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1087 * @adapter: board private structure to initialize
1088 * @msix: boolean value of MSIX capability
1089 *
1090 * Attempt to configure interrupts using the best available
1091 * capabilities of the hardware and kernel.
1092 **/
1093static void igb_set_interrupt_capability(struct igb_adapter *adapter, bool msix)
1094{
1095 int err;
1096 int numvecs, i;
1097
1098 if (!msix)
1099 goto msi_only;
1100 adapter->flags |= IGB_FLAG_HAS_MSIX;
1101
1102 /* Number of supported queues. */
1103 adapter->num_rx_queues = adapter->rss_queues;
1104 if (adapter->vfs_allocated_count)
1105 adapter->num_tx_queues = 1;
1106 else
1107 adapter->num_tx_queues = adapter->rss_queues;
1108
1109 /* start with one vector for every Rx queue */
1110 numvecs = adapter->num_rx_queues;
1111
1112 /* if Tx handler is separate add 1 for every Tx queue */
1113 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1114 numvecs += adapter->num_tx_queues;
1115
1116 /* store the number of vectors reserved for queues */
1117 adapter->num_q_vectors = numvecs;
1118
1119 /* add 1 vector for link status interrupts */
1120 numvecs++;
1121 for (i = 0; i < numvecs; i++)
1122 adapter->msix_entries[i].entry = i;
1123
1124 err = pci_enable_msix_range(adapter->pdev,
1125 adapter->msix_entries,
1126 numvecs,
1127 numvecs);
1128 if (err > 0)
1129 return;
1130
1131 igb_reset_interrupt_capability(adapter);
1132
1133 /* If we can't do MSI-X, try MSI */
1134msi_only:
1135 adapter->flags &= ~IGB_FLAG_HAS_MSIX;
1136#ifdef CONFIG_PCI_IOV
1137 /* disable SR-IOV for non MSI-X configurations */
1138 if (adapter->vf_data) {
1139 struct e1000_hw *hw = &adapter->hw;
1140 /* disable iov and allow time for transactions to clear */
1141 pci_disable_sriov(adapter->pdev);
1142 msleep(500);
1143
1144 kfree(adapter->vf_mac_list);
1145 adapter->vf_mac_list = NULL;
1146 kfree(adapter->vf_data);
1147 adapter->vf_data = NULL;
1148 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1149 wrfl();
1150 msleep(100);
1151 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1152 }
1153#endif
1154 adapter->vfs_allocated_count = 0;
1155 adapter->rss_queues = 1;
1156 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1157 adapter->num_rx_queues = 1;
1158 adapter->num_tx_queues = 1;
1159 adapter->num_q_vectors = 1;
1160 if (!pci_enable_msi(adapter->pdev))
1161 adapter->flags |= IGB_FLAG_HAS_MSI;
1162}
1163
1164static void igb_add_ring(struct igb_ring *ring,
1165 struct igb_ring_container *head)
1166{
1167 head->ring = ring;
1168 head->count++;
1169}
1170
1171/**
1172 * igb_alloc_q_vector - Allocate memory for a single interrupt vector
1173 * @adapter: board private structure to initialize
1174 * @v_count: q_vectors allocated on adapter, used for ring interleaving
1175 * @v_idx: index of vector in adapter struct
1176 * @txr_count: total number of Tx rings to allocate
1177 * @txr_idx: index of first Tx ring to allocate
1178 * @rxr_count: total number of Rx rings to allocate
1179 * @rxr_idx: index of first Rx ring to allocate
1180 *
1181 * We allocate one q_vector. If allocation fails we return -ENOMEM.
1182 **/
1183static int igb_alloc_q_vector(struct igb_adapter *adapter,
1184 int v_count, int v_idx,
1185 int txr_count, int txr_idx,
1186 int rxr_count, int rxr_idx)
1187{
1188 struct igb_q_vector *q_vector;
1189 struct igb_ring *ring;
1190 int ring_count;
1191 size_t size;
1192
1193 /* igb only supports 1 Tx and/or 1 Rx queue per vector */
1194 if (txr_count > 1 || rxr_count > 1)
1195 return -ENOMEM;
1196
1197 ring_count = txr_count + rxr_count;
1198 size = kmalloc_size_roundup(struct_size(q_vector, ring, ring_count));
1199
1200 /* allocate q_vector and rings */
1201 q_vector = adapter->q_vector[v_idx];
1202 if (!q_vector) {
1203 q_vector = kzalloc(size, GFP_KERNEL);
1204 } else if (size > ksize(q_vector)) {
1205 struct igb_q_vector *new_q_vector;
1206
1207 new_q_vector = kzalloc(size, GFP_KERNEL);
1208 if (new_q_vector)
1209 kfree_rcu(q_vector, rcu);
1210 q_vector = new_q_vector;
1211 } else {
1212 memset(q_vector, 0, size);
1213 }
1214 if (!q_vector)
1215 return -ENOMEM;
1216
1217 /* initialize NAPI */
1218 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll);
1219
1220 /* tie q_vector and adapter together */
1221 adapter->q_vector[v_idx] = q_vector;
1222 q_vector->adapter = adapter;
1223
1224 /* initialize work limits */
1225 q_vector->tx.work_limit = adapter->tx_work_limit;
1226
1227 /* initialize ITR configuration */
1228 q_vector->itr_register = adapter->io_addr + E1000_EITR(0);
1229 q_vector->itr_val = IGB_START_ITR;
1230
1231 /* initialize pointer to rings */
1232 ring = q_vector->ring;
1233
1234 /* intialize ITR */
1235 if (rxr_count) {
1236 /* rx or rx/tx vector */
1237 if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
1238 q_vector->itr_val = adapter->rx_itr_setting;
1239 } else {
1240 /* tx only vector */
1241 if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
1242 q_vector->itr_val = adapter->tx_itr_setting;
1243 }
1244
1245 if (txr_count) {
1246 /* assign generic ring traits */
1247 ring->dev = &adapter->pdev->dev;
1248 ring->netdev = adapter->netdev;
1249
1250 /* configure backlink on ring */
1251 ring->q_vector = q_vector;
1252
1253 /* update q_vector Tx values */
1254 igb_add_ring(ring, &q_vector->tx);
1255
1256 /* For 82575, context index must be unique per ring. */
1257 if (adapter->hw.mac.type == e1000_82575)
1258 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
1259
1260 /* apply Tx specific ring traits */
1261 ring->count = adapter->tx_ring_count;
1262 ring->queue_index = txr_idx;
1263
1264 ring->cbs_enable = false;
1265 ring->idleslope = 0;
1266 ring->sendslope = 0;
1267 ring->hicredit = 0;
1268 ring->locredit = 0;
1269
1270 u64_stats_init(&ring->tx_syncp);
1271 u64_stats_init(&ring->tx_syncp2);
1272
1273 /* assign ring to adapter */
1274 adapter->tx_ring[txr_idx] = ring;
1275
1276 /* push pointer to next ring */
1277 ring++;
1278 }
1279
1280 if (rxr_count) {
1281 /* assign generic ring traits */
1282 ring->dev = &adapter->pdev->dev;
1283 ring->netdev = adapter->netdev;
1284
1285 /* configure backlink on ring */
1286 ring->q_vector = q_vector;
1287
1288 /* update q_vector Rx values */
1289 igb_add_ring(ring, &q_vector->rx);
1290
1291 /* set flag indicating ring supports SCTP checksum offload */
1292 if (adapter->hw.mac.type >= e1000_82576)
1293 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
1294
1295 /* On i350, i354, i210, and i211, loopback VLAN packets
1296 * have the tag byte-swapped.
1297 */
1298 if (adapter->hw.mac.type >= e1000_i350)
1299 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
1300
1301 /* apply Rx specific ring traits */
1302 ring->count = adapter->rx_ring_count;
1303 ring->queue_index = rxr_idx;
1304
1305 u64_stats_init(&ring->rx_syncp);
1306
1307 /* assign ring to adapter */
1308 adapter->rx_ring[rxr_idx] = ring;
1309 }
1310
1311 return 0;
1312}
1313
1314
1315/**
1316 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1317 * @adapter: board private structure to initialize
1318 *
1319 * We allocate one q_vector per queue interrupt. If allocation fails we
1320 * return -ENOMEM.
1321 **/
1322static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1323{
1324 int q_vectors = adapter->num_q_vectors;
1325 int rxr_remaining = adapter->num_rx_queues;
1326 int txr_remaining = adapter->num_tx_queues;
1327 int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1328 int err;
1329
1330 if (q_vectors >= (rxr_remaining + txr_remaining)) {
1331 for (; rxr_remaining; v_idx++) {
1332 err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1333 0, 0, 1, rxr_idx);
1334
1335 if (err)
1336 goto err_out;
1337
1338 /* update counts and index */
1339 rxr_remaining--;
1340 rxr_idx++;
1341 }
1342 }
1343
1344 for (; v_idx < q_vectors; v_idx++) {
1345 int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1346 int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1347
1348 err = igb_alloc_q_vector(adapter, q_vectors, v_idx,
1349 tqpv, txr_idx, rqpv, rxr_idx);
1350
1351 if (err)
1352 goto err_out;
1353
1354 /* update counts and index */
1355 rxr_remaining -= rqpv;
1356 txr_remaining -= tqpv;
1357 rxr_idx++;
1358 txr_idx++;
1359 }
1360
1361 return 0;
1362
1363err_out:
1364 adapter->num_tx_queues = 0;
1365 adapter->num_rx_queues = 0;
1366 adapter->num_q_vectors = 0;
1367
1368 while (v_idx--)
1369 igb_free_q_vector(adapter, v_idx);
1370
1371 return -ENOMEM;
1372}
1373
1374/**
1375 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1376 * @adapter: board private structure to initialize
1377 * @msix: boolean value of MSIX capability
1378 *
1379 * This function initializes the interrupts and allocates all of the queues.
1380 **/
1381static int igb_init_interrupt_scheme(struct igb_adapter *adapter, bool msix)
1382{
1383 struct pci_dev *pdev = adapter->pdev;
1384 int err;
1385
1386 igb_set_interrupt_capability(adapter, msix);
1387
1388 err = igb_alloc_q_vectors(adapter);
1389 if (err) {
1390 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1391 goto err_alloc_q_vectors;
1392 }
1393
1394 igb_cache_ring_register(adapter);
1395
1396 return 0;
1397
1398err_alloc_q_vectors:
1399 igb_reset_interrupt_capability(adapter);
1400 return err;
1401}
1402
1403/**
1404 * igb_request_irq - initialize interrupts
1405 * @adapter: board private structure to initialize
1406 *
1407 * Attempts to configure interrupts using the best available
1408 * capabilities of the hardware and kernel.
1409 **/
1410static int igb_request_irq(struct igb_adapter *adapter)
1411{
1412 struct net_device *netdev = adapter->netdev;
1413 struct pci_dev *pdev = adapter->pdev;
1414 int err = 0;
1415
1416 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1417 err = igb_request_msix(adapter);
1418 if (!err)
1419 goto request_done;
1420 /* fall back to MSI */
1421 igb_free_all_tx_resources(adapter);
1422 igb_free_all_rx_resources(adapter);
1423
1424 igb_clear_interrupt_scheme(adapter);
1425 err = igb_init_interrupt_scheme(adapter, false);
1426 if (err)
1427 goto request_done;
1428
1429 igb_setup_all_tx_resources(adapter);
1430 igb_setup_all_rx_resources(adapter);
1431 igb_configure(adapter);
1432 }
1433
1434 igb_assign_vector(adapter->q_vector[0], 0);
1435
1436 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1437 err = request_irq(pdev->irq, igb_intr_msi, 0,
1438 netdev->name, adapter);
1439 if (!err)
1440 goto request_done;
1441
1442 /* fall back to legacy interrupts */
1443 igb_reset_interrupt_capability(adapter);
1444 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1445 }
1446
1447 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1448 netdev->name, adapter);
1449
1450 if (err)
1451 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1452 err);
1453
1454request_done:
1455 return err;
1456}
1457
1458static void igb_free_irq(struct igb_adapter *adapter)
1459{
1460 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1461 int vector = 0, i;
1462
1463 free_irq(adapter->msix_entries[vector++].vector, adapter);
1464
1465 for (i = 0; i < adapter->num_q_vectors; i++)
1466 free_irq(adapter->msix_entries[vector++].vector,
1467 adapter->q_vector[i]);
1468 } else {
1469 free_irq(adapter->pdev->irq, adapter);
1470 }
1471}
1472
1473/**
1474 * igb_irq_disable - Mask off interrupt generation on the NIC
1475 * @adapter: board private structure
1476 **/
1477static void igb_irq_disable(struct igb_adapter *adapter)
1478{
1479 struct e1000_hw *hw = &adapter->hw;
1480
1481 /* we need to be careful when disabling interrupts. The VFs are also
1482 * mapped into these registers and so clearing the bits can cause
1483 * issues on the VF drivers so we only need to clear what we set
1484 */
1485 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1486 u32 regval = rd32(E1000_EIAM);
1487
1488 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1489 wr32(E1000_EIMC, adapter->eims_enable_mask);
1490 regval = rd32(E1000_EIAC);
1491 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1492 }
1493
1494 wr32(E1000_IAM, 0);
1495 wr32(E1000_IMC, ~0);
1496 wrfl();
1497 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1498 int i;
1499
1500 for (i = 0; i < adapter->num_q_vectors; i++)
1501 synchronize_irq(adapter->msix_entries[i].vector);
1502 } else {
1503 synchronize_irq(adapter->pdev->irq);
1504 }
1505}
1506
1507/**
1508 * igb_irq_enable - Enable default interrupt generation settings
1509 * @adapter: board private structure
1510 **/
1511static void igb_irq_enable(struct igb_adapter *adapter)
1512{
1513 struct e1000_hw *hw = &adapter->hw;
1514
1515 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
1516 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1517 u32 regval = rd32(E1000_EIAC);
1518
1519 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1520 regval = rd32(E1000_EIAM);
1521 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1522 wr32(E1000_EIMS, adapter->eims_enable_mask);
1523 if (adapter->vfs_allocated_count) {
1524 wr32(E1000_MBVFIMR, 0xFF);
1525 ims |= E1000_IMS_VMMB;
1526 }
1527 wr32(E1000_IMS, ims);
1528 } else {
1529 wr32(E1000_IMS, IMS_ENABLE_MASK |
1530 E1000_IMS_DRSTA);
1531 wr32(E1000_IAM, IMS_ENABLE_MASK |
1532 E1000_IMS_DRSTA);
1533 }
1534}
1535
1536static void igb_update_mng_vlan(struct igb_adapter *adapter)
1537{
1538 struct e1000_hw *hw = &adapter->hw;
1539 u16 pf_id = adapter->vfs_allocated_count;
1540 u16 vid = adapter->hw.mng_cookie.vlan_id;
1541 u16 old_vid = adapter->mng_vlan_id;
1542
1543 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1544 /* add VID to filter table */
1545 igb_vfta_set(hw, vid, pf_id, true, true);
1546 adapter->mng_vlan_id = vid;
1547 } else {
1548 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1549 }
1550
1551 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1552 (vid != old_vid) &&
1553 !test_bit(old_vid, adapter->active_vlans)) {
1554 /* remove VID from filter table */
1555 igb_vfta_set(hw, vid, pf_id, false, true);
1556 }
1557}
1558
1559/**
1560 * igb_release_hw_control - release control of the h/w to f/w
1561 * @adapter: address of board private structure
1562 *
1563 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1564 * For ASF and Pass Through versions of f/w this means that the
1565 * driver is no longer loaded.
1566 **/
1567static void igb_release_hw_control(struct igb_adapter *adapter)
1568{
1569 struct e1000_hw *hw = &adapter->hw;
1570 u32 ctrl_ext;
1571
1572 /* Let firmware take over control of h/w */
1573 ctrl_ext = rd32(E1000_CTRL_EXT);
1574 wr32(E1000_CTRL_EXT,
1575 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1576}
1577
1578/**
1579 * igb_get_hw_control - get control of the h/w from f/w
1580 * @adapter: address of board private structure
1581 *
1582 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1583 * For ASF and Pass Through versions of f/w this means that
1584 * the driver is loaded.
1585 **/
1586static void igb_get_hw_control(struct igb_adapter *adapter)
1587{
1588 struct e1000_hw *hw = &adapter->hw;
1589 u32 ctrl_ext;
1590
1591 /* Let firmware know the driver has taken over */
1592 ctrl_ext = rd32(E1000_CTRL_EXT);
1593 wr32(E1000_CTRL_EXT,
1594 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1595}
1596
1597static void enable_fqtss(struct igb_adapter *adapter, bool enable)
1598{
1599 struct net_device *netdev = adapter->netdev;
1600 struct e1000_hw *hw = &adapter->hw;
1601
1602 WARN_ON(hw->mac.type != e1000_i210);
1603
1604 if (enable)
1605 adapter->flags |= IGB_FLAG_FQTSS;
1606 else
1607 adapter->flags &= ~IGB_FLAG_FQTSS;
1608
1609 if (netif_running(netdev))
1610 schedule_work(&adapter->reset_task);
1611}
1612
1613static bool is_fqtss_enabled(struct igb_adapter *adapter)
1614{
1615 return (adapter->flags & IGB_FLAG_FQTSS) ? true : false;
1616}
1617
1618static void set_tx_desc_fetch_prio(struct e1000_hw *hw, int queue,
1619 enum tx_queue_prio prio)
1620{
1621 u32 val;
1622
1623 WARN_ON(hw->mac.type != e1000_i210);
1624 WARN_ON(queue < 0 || queue > 4);
1625
1626 val = rd32(E1000_I210_TXDCTL(queue));
1627
1628 if (prio == TX_QUEUE_PRIO_HIGH)
1629 val |= E1000_TXDCTL_PRIORITY;
1630 else
1631 val &= ~E1000_TXDCTL_PRIORITY;
1632
1633 wr32(E1000_I210_TXDCTL(queue), val);
1634}
1635
1636static void set_queue_mode(struct e1000_hw *hw, int queue, enum queue_mode mode)
1637{
1638 u32 val;
1639
1640 WARN_ON(hw->mac.type != e1000_i210);
1641 WARN_ON(queue < 0 || queue > 1);
1642
1643 val = rd32(E1000_I210_TQAVCC(queue));
1644
1645 if (mode == QUEUE_MODE_STREAM_RESERVATION)
1646 val |= E1000_TQAVCC_QUEUEMODE;
1647 else
1648 val &= ~E1000_TQAVCC_QUEUEMODE;
1649
1650 wr32(E1000_I210_TQAVCC(queue), val);
1651}
1652
1653static bool is_any_cbs_enabled(struct igb_adapter *adapter)
1654{
1655 int i;
1656
1657 for (i = 0; i < adapter->num_tx_queues; i++) {
1658 if (adapter->tx_ring[i]->cbs_enable)
1659 return true;
1660 }
1661
1662 return false;
1663}
1664
1665static bool is_any_txtime_enabled(struct igb_adapter *adapter)
1666{
1667 int i;
1668
1669 for (i = 0; i < adapter->num_tx_queues; i++) {
1670 if (adapter->tx_ring[i]->launchtime_enable)
1671 return true;
1672 }
1673
1674 return false;
1675}
1676
1677/**
1678 * igb_config_tx_modes - Configure "Qav Tx mode" features on igb
1679 * @adapter: pointer to adapter struct
1680 * @queue: queue number
1681 *
1682 * Configure CBS and Launchtime for a given hardware queue.
1683 * Parameters are retrieved from the correct Tx ring, so
1684 * igb_save_cbs_params() and igb_save_txtime_params() should be used
1685 * for setting those correctly prior to this function being called.
1686 **/
1687static void igb_config_tx_modes(struct igb_adapter *adapter, int queue)
1688{
1689 struct net_device *netdev = adapter->netdev;
1690 struct e1000_hw *hw = &adapter->hw;
1691 struct igb_ring *ring;
1692 u32 tqavcc, tqavctrl;
1693 u16 value;
1694
1695 WARN_ON(hw->mac.type != e1000_i210);
1696 WARN_ON(queue < 0 || queue > 1);
1697 ring = adapter->tx_ring[queue];
1698
1699 /* If any of the Qav features is enabled, configure queues as SR and
1700 * with HIGH PRIO. If none is, then configure them with LOW PRIO and
1701 * as SP.
1702 */
1703 if (ring->cbs_enable || ring->launchtime_enable) {
1704 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_HIGH);
1705 set_queue_mode(hw, queue, QUEUE_MODE_STREAM_RESERVATION);
1706 } else {
1707 set_tx_desc_fetch_prio(hw, queue, TX_QUEUE_PRIO_LOW);
1708 set_queue_mode(hw, queue, QUEUE_MODE_STRICT_PRIORITY);
1709 }
1710
1711 /* If CBS is enabled, set DataTranARB and config its parameters. */
1712 if (ring->cbs_enable || queue == 0) {
1713 /* i210 does not allow the queue 0 to be in the Strict
1714 * Priority mode while the Qav mode is enabled, so,
1715 * instead of disabling strict priority mode, we give
1716 * queue 0 the maximum of credits possible.
1717 *
1718 * See section 8.12.19 of the i210 datasheet, "Note:
1719 * Queue0 QueueMode must be set to 1b when
1720 * TransmitMode is set to Qav."
1721 */
1722 if (queue == 0 && !ring->cbs_enable) {
1723 /* max "linkspeed" idleslope in kbps */
1724 ring->idleslope = 1000000;
1725 ring->hicredit = ETH_FRAME_LEN;
1726 }
1727
1728 /* Always set data transfer arbitration to credit-based
1729 * shaper algorithm on TQAVCTRL if CBS is enabled for any of
1730 * the queues.
1731 */
1732 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1733 tqavctrl |= E1000_TQAVCTRL_DATATRANARB;
1734 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1735
1736 /* According to i210 datasheet section 7.2.7.7, we should set
1737 * the 'idleSlope' field from TQAVCC register following the
1738 * equation:
1739 *
1740 * For 100 Mbps link speed:
1741 *
1742 * value = BW * 0x7735 * 0.2 (E1)
1743 *
1744 * For 1000Mbps link speed:
1745 *
1746 * value = BW * 0x7735 * 2 (E2)
1747 *
1748 * E1 and E2 can be merged into one equation as shown below.
1749 * Note that 'link-speed' is in Mbps.
1750 *
1751 * value = BW * 0x7735 * 2 * link-speed
1752 * -------------- (E3)
1753 * 1000
1754 *
1755 * 'BW' is the percentage bandwidth out of full link speed
1756 * which can be found with the following equation. Note that
1757 * idleSlope here is the parameter from this function which
1758 * is in kbps.
1759 *
1760 * BW = idleSlope
1761 * ----------------- (E4)
1762 * link-speed * 1000
1763 *
1764 * That said, we can come up with a generic equation to
1765 * calculate the value we should set it TQAVCC register by
1766 * replacing 'BW' in E3 by E4. The resulting equation is:
1767 *
1768 * value = idleSlope * 0x7735 * 2 * link-speed
1769 * ----------------- -------------- (E5)
1770 * link-speed * 1000 1000
1771 *
1772 * 'link-speed' is present in both sides of the fraction so
1773 * it is canceled out. The final equation is the following:
1774 *
1775 * value = idleSlope * 61034
1776 * ----------------- (E6)
1777 * 1000000
1778 *
1779 * NOTE: For i210, given the above, we can see that idleslope
1780 * is represented in 16.38431 kbps units by the value at
1781 * the TQAVCC register (1Gbps / 61034), which reduces
1782 * the granularity for idleslope increments.
1783 * For instance, if you want to configure a 2576kbps
1784 * idleslope, the value to be written on the register
1785 * would have to be 157.23. If rounded down, you end
1786 * up with less bandwidth available than originally
1787 * required (~2572 kbps). If rounded up, you end up
1788 * with a higher bandwidth (~2589 kbps). Below the
1789 * approach we take is to always round up the
1790 * calculated value, so the resulting bandwidth might
1791 * be slightly higher for some configurations.
1792 */
1793 value = DIV_ROUND_UP_ULL(ring->idleslope * 61034ULL, 1000000);
1794
1795 tqavcc = rd32(E1000_I210_TQAVCC(queue));
1796 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1797 tqavcc |= value;
1798 wr32(E1000_I210_TQAVCC(queue), tqavcc);
1799
1800 wr32(E1000_I210_TQAVHC(queue),
1801 0x80000000 + ring->hicredit * 0x7735);
1802 } else {
1803
1804 /* Set idleSlope to zero. */
1805 tqavcc = rd32(E1000_I210_TQAVCC(queue));
1806 tqavcc &= ~E1000_TQAVCC_IDLESLOPE_MASK;
1807 wr32(E1000_I210_TQAVCC(queue), tqavcc);
1808
1809 /* Set hiCredit to zero. */
1810 wr32(E1000_I210_TQAVHC(queue), 0);
1811
1812 /* If CBS is not enabled for any queues anymore, then return to
1813 * the default state of Data Transmission Arbitration on
1814 * TQAVCTRL.
1815 */
1816 if (!is_any_cbs_enabled(adapter)) {
1817 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1818 tqavctrl &= ~E1000_TQAVCTRL_DATATRANARB;
1819 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1820 }
1821 }
1822
1823 /* If LaunchTime is enabled, set DataTranTIM. */
1824 if (ring->launchtime_enable) {
1825 /* Always set DataTranTIM on TQAVCTRL if LaunchTime is enabled
1826 * for any of the SR queues, and configure fetchtime delta.
1827 * XXX NOTE:
1828 * - LaunchTime will be enabled for all SR queues.
1829 * - A fixed offset can be added relative to the launch
1830 * time of all packets if configured at reg LAUNCH_OS0.
1831 * We are keeping it as 0 for now (default value).
1832 */
1833 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1834 tqavctrl |= E1000_TQAVCTRL_DATATRANTIM |
1835 E1000_TQAVCTRL_FETCHTIME_DELTA;
1836 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1837 } else {
1838 /* If Launchtime is not enabled for any SR queues anymore,
1839 * then clear DataTranTIM on TQAVCTRL and clear fetchtime delta,
1840 * effectively disabling Launchtime.
1841 */
1842 if (!is_any_txtime_enabled(adapter)) {
1843 tqavctrl = rd32(E1000_I210_TQAVCTRL);
1844 tqavctrl &= ~E1000_TQAVCTRL_DATATRANTIM;
1845 tqavctrl &= ~E1000_TQAVCTRL_FETCHTIME_DELTA;
1846 wr32(E1000_I210_TQAVCTRL, tqavctrl);
1847 }
1848 }
1849
1850 /* XXX: In i210 controller the sendSlope and loCredit parameters from
1851 * CBS are not configurable by software so we don't do any 'controller
1852 * configuration' in respect to these parameters.
1853 */
1854
1855 netdev_dbg(netdev, "Qav Tx mode: cbs %s, launchtime %s, queue %d idleslope %d sendslope %d hiCredit %d locredit %d\n",
1856 ring->cbs_enable ? "enabled" : "disabled",
1857 ring->launchtime_enable ? "enabled" : "disabled",
1858 queue,
1859 ring->idleslope, ring->sendslope,
1860 ring->hicredit, ring->locredit);
1861}
1862
1863static int igb_save_txtime_params(struct igb_adapter *adapter, int queue,
1864 bool enable)
1865{
1866 struct igb_ring *ring;
1867
1868 if (queue < 0 || queue > adapter->num_tx_queues)
1869 return -EINVAL;
1870
1871 ring = adapter->tx_ring[queue];
1872 ring->launchtime_enable = enable;
1873
1874 return 0;
1875}
1876
1877static int igb_save_cbs_params(struct igb_adapter *adapter, int queue,
1878 bool enable, int idleslope, int sendslope,
1879 int hicredit, int locredit)
1880{
1881 struct igb_ring *ring;
1882
1883 if (queue < 0 || queue > adapter->num_tx_queues)
1884 return -EINVAL;
1885
1886 ring = adapter->tx_ring[queue];
1887
1888 ring->cbs_enable = enable;
1889 ring->idleslope = idleslope;
1890 ring->sendslope = sendslope;
1891 ring->hicredit = hicredit;
1892 ring->locredit = locredit;
1893
1894 return 0;
1895}
1896
1897/**
1898 * igb_setup_tx_mode - Switch to/from Qav Tx mode when applicable
1899 * @adapter: pointer to adapter struct
1900 *
1901 * Configure TQAVCTRL register switching the controller's Tx mode
1902 * if FQTSS mode is enabled or disabled. Additionally, will issue
1903 * a call to igb_config_tx_modes() per queue so any previously saved
1904 * Tx parameters are applied.
1905 **/
1906static void igb_setup_tx_mode(struct igb_adapter *adapter)
1907{
1908 struct net_device *netdev = adapter->netdev;
1909 struct e1000_hw *hw = &adapter->hw;
1910 u32 val;
1911
1912 /* Only i210 controller supports changing the transmission mode. */
1913 if (hw->mac.type != e1000_i210)
1914 return;
1915
1916 if (is_fqtss_enabled(adapter)) {
1917 int i, max_queue;
1918
1919 /* Configure TQAVCTRL register: set transmit mode to 'Qav',
1920 * set data fetch arbitration to 'round robin', set SP_WAIT_SR
1921 * so SP queues wait for SR ones.
1922 */
1923 val = rd32(E1000_I210_TQAVCTRL);
1924 val |= E1000_TQAVCTRL_XMIT_MODE | E1000_TQAVCTRL_SP_WAIT_SR;
1925 val &= ~E1000_TQAVCTRL_DATAFETCHARB;
1926 wr32(E1000_I210_TQAVCTRL, val);
1927
1928 /* Configure Tx and Rx packet buffers sizes as described in
1929 * i210 datasheet section 7.2.7.7.
1930 */
1931 val = rd32(E1000_TXPBS);
1932 val &= ~I210_TXPBSIZE_MASK;
1933 val |= I210_TXPBSIZE_PB0_6KB | I210_TXPBSIZE_PB1_6KB |
1934 I210_TXPBSIZE_PB2_6KB | I210_TXPBSIZE_PB3_6KB;
1935 wr32(E1000_TXPBS, val);
1936
1937 val = rd32(E1000_RXPBS);
1938 val &= ~I210_RXPBSIZE_MASK;
1939 val |= I210_RXPBSIZE_PB_30KB;
1940 wr32(E1000_RXPBS, val);
1941
1942 /* Section 8.12.9 states that MAX_TPKT_SIZE from DTXMXPKTSZ
1943 * register should not exceed the buffer size programmed in
1944 * TXPBS. The smallest buffer size programmed in TXPBS is 4kB
1945 * so according to the datasheet we should set MAX_TPKT_SIZE to
1946 * 4kB / 64.
1947 *
1948 * However, when we do so, no frame from queue 2 and 3 are
1949 * transmitted. It seems the MAX_TPKT_SIZE should not be great
1950 * or _equal_ to the buffer size programmed in TXPBS. For this
1951 * reason, we set MAX_ TPKT_SIZE to (4kB - 1) / 64.
1952 */
1953 val = (4096 - 1) / 64;
1954 wr32(E1000_I210_DTXMXPKTSZ, val);
1955
1956 /* Since FQTSS mode is enabled, apply any CBS configuration
1957 * previously set. If no previous CBS configuration has been
1958 * done, then the initial configuration is applied, which means
1959 * CBS is disabled.
1960 */
1961 max_queue = (adapter->num_tx_queues < I210_SR_QUEUES_NUM) ?
1962 adapter->num_tx_queues : I210_SR_QUEUES_NUM;
1963
1964 for (i = 0; i < max_queue; i++) {
1965 igb_config_tx_modes(adapter, i);
1966 }
1967 } else {
1968 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
1969 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
1970 wr32(E1000_I210_DTXMXPKTSZ, I210_DTXMXPKTSZ_DEFAULT);
1971
1972 val = rd32(E1000_I210_TQAVCTRL);
1973 /* According to Section 8.12.21, the other flags we've set when
1974 * enabling FQTSS are not relevant when disabling FQTSS so we
1975 * don't set they here.
1976 */
1977 val &= ~E1000_TQAVCTRL_XMIT_MODE;
1978 wr32(E1000_I210_TQAVCTRL, val);
1979 }
1980
1981 netdev_dbg(netdev, "FQTSS %s\n", (is_fqtss_enabled(adapter)) ?
1982 "enabled" : "disabled");
1983}
1984
1985/**
1986 * igb_configure - configure the hardware for RX and TX
1987 * @adapter: private board structure
1988 **/
1989static void igb_configure(struct igb_adapter *adapter)
1990{
1991 struct net_device *netdev = adapter->netdev;
1992 int i;
1993
1994 igb_get_hw_control(adapter);
1995 igb_set_rx_mode(netdev);
1996 igb_setup_tx_mode(adapter);
1997
1998 igb_restore_vlan(adapter);
1999
2000 igb_setup_tctl(adapter);
2001 igb_setup_mrqc(adapter);
2002 igb_setup_rctl(adapter);
2003
2004 igb_nfc_filter_restore(adapter);
2005 igb_configure_tx(adapter);
2006 igb_configure_rx(adapter);
2007
2008 igb_rx_fifo_flush_82575(&adapter->hw);
2009
2010 /* call igb_desc_unused which always leaves
2011 * at least 1 descriptor unused to make sure
2012 * next_to_use != next_to_clean
2013 */
2014 for (i = 0; i < adapter->num_rx_queues; i++) {
2015 struct igb_ring *ring = adapter->rx_ring[i];
2016 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
2017 }
2018}
2019
2020/**
2021 * igb_power_up_link - Power up the phy/serdes link
2022 * @adapter: address of board private structure
2023 **/
2024void igb_power_up_link(struct igb_adapter *adapter)
2025{
2026 igb_reset_phy(&adapter->hw);
2027
2028 if (adapter->hw.phy.media_type == e1000_media_type_copper)
2029 igb_power_up_phy_copper(&adapter->hw);
2030 else
2031 igb_power_up_serdes_link_82575(&adapter->hw);
2032
2033 igb_setup_link(&adapter->hw);
2034}
2035
2036/**
2037 * igb_power_down_link - Power down the phy/serdes link
2038 * @adapter: address of board private structure
2039 */
2040static void igb_power_down_link(struct igb_adapter *adapter)
2041{
2042 if (adapter->hw.phy.media_type == e1000_media_type_copper)
2043 igb_power_down_phy_copper_82575(&adapter->hw);
2044 else
2045 igb_shutdown_serdes_link_82575(&adapter->hw);
2046}
2047
2048/**
2049 * igb_check_swap_media - Detect and switch function for Media Auto Sense
2050 * @adapter: address of the board private structure
2051 **/
2052static void igb_check_swap_media(struct igb_adapter *adapter)
2053{
2054 struct e1000_hw *hw = &adapter->hw;
2055 u32 ctrl_ext, connsw;
2056 bool swap_now = false;
2057
2058 ctrl_ext = rd32(E1000_CTRL_EXT);
2059 connsw = rd32(E1000_CONNSW);
2060
2061 /* need to live swap if current media is copper and we have fiber/serdes
2062 * to go to.
2063 */
2064
2065 if ((hw->phy.media_type == e1000_media_type_copper) &&
2066 (!(connsw & E1000_CONNSW_AUTOSENSE_EN))) {
2067 swap_now = true;
2068 } else if ((hw->phy.media_type != e1000_media_type_copper) &&
2069 !(connsw & E1000_CONNSW_SERDESD)) {
2070 /* copper signal takes time to appear */
2071 if (adapter->copper_tries < 4) {
2072 adapter->copper_tries++;
2073 connsw |= E1000_CONNSW_AUTOSENSE_CONF;
2074 wr32(E1000_CONNSW, connsw);
2075 return;
2076 } else {
2077 adapter->copper_tries = 0;
2078 if ((connsw & E1000_CONNSW_PHYSD) &&
2079 (!(connsw & E1000_CONNSW_PHY_PDN))) {
2080 swap_now = true;
2081 connsw &= ~E1000_CONNSW_AUTOSENSE_CONF;
2082 wr32(E1000_CONNSW, connsw);
2083 }
2084 }
2085 }
2086
2087 if (!swap_now)
2088 return;
2089
2090 switch (hw->phy.media_type) {
2091 case e1000_media_type_copper:
2092 netdev_info(adapter->netdev,
2093 "MAS: changing media to fiber/serdes\n");
2094 ctrl_ext |=
2095 E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2096 adapter->flags |= IGB_FLAG_MEDIA_RESET;
2097 adapter->copper_tries = 0;
2098 break;
2099 case e1000_media_type_internal_serdes:
2100 case e1000_media_type_fiber:
2101 netdev_info(adapter->netdev,
2102 "MAS: changing media to copper\n");
2103 ctrl_ext &=
2104 ~E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES;
2105 adapter->flags |= IGB_FLAG_MEDIA_RESET;
2106 break;
2107 default:
2108 /* shouldn't get here during regular operation */
2109 netdev_err(adapter->netdev,
2110 "AMS: Invalid media type found, returning\n");
2111 break;
2112 }
2113 wr32(E1000_CTRL_EXT, ctrl_ext);
2114}
2115
2116/**
2117 * igb_up - Open the interface and prepare it to handle traffic
2118 * @adapter: board private structure
2119 **/
2120int igb_up(struct igb_adapter *adapter)
2121{
2122 struct e1000_hw *hw = &adapter->hw;
2123 int i;
2124
2125 /* hardware has been reset, we need to reload some things */
2126 igb_configure(adapter);
2127
2128 clear_bit(__IGB_DOWN, &adapter->state);
2129
2130 for (i = 0; i < adapter->num_q_vectors; i++)
2131 napi_enable(&(adapter->q_vector[i]->napi));
2132
2133 if (adapter->flags & IGB_FLAG_HAS_MSIX)
2134 igb_configure_msix(adapter);
2135 else
2136 igb_assign_vector(adapter->q_vector[0], 0);
2137
2138 /* Clear any pending interrupts. */
2139 rd32(E1000_TSICR);
2140 rd32(E1000_ICR);
2141 igb_irq_enable(adapter);
2142
2143 /* notify VFs that reset has been completed */
2144 if (adapter->vfs_allocated_count) {
2145 u32 reg_data = rd32(E1000_CTRL_EXT);
2146
2147 reg_data |= E1000_CTRL_EXT_PFRSTD;
2148 wr32(E1000_CTRL_EXT, reg_data);
2149 }
2150
2151 netif_tx_start_all_queues(adapter->netdev);
2152
2153 /* start the watchdog. */
2154 hw->mac.get_link_status = 1;
2155 schedule_work(&adapter->watchdog_task);
2156
2157 if ((adapter->flags & IGB_FLAG_EEE) &&
2158 (!hw->dev_spec._82575.eee_disable))
2159 adapter->eee_advert = MDIO_EEE_100TX | MDIO_EEE_1000T;
2160
2161 return 0;
2162}
2163
2164void igb_down(struct igb_adapter *adapter)
2165{
2166 struct net_device *netdev = adapter->netdev;
2167 struct e1000_hw *hw = &adapter->hw;
2168 u32 tctl, rctl;
2169 int i;
2170
2171 /* signal that we're down so the interrupt handler does not
2172 * reschedule our watchdog timer
2173 */
2174 set_bit(__IGB_DOWN, &adapter->state);
2175
2176 /* disable receives in the hardware */
2177 rctl = rd32(E1000_RCTL);
2178 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
2179 /* flush and sleep below */
2180
2181 igb_nfc_filter_exit(adapter);
2182
2183 netif_carrier_off(netdev);
2184 netif_tx_stop_all_queues(netdev);
2185
2186 /* disable transmits in the hardware */
2187 tctl = rd32(E1000_TCTL);
2188 tctl &= ~E1000_TCTL_EN;
2189 wr32(E1000_TCTL, tctl);
2190 /* flush both disables and wait for them to finish */
2191 wrfl();
2192 usleep_range(10000, 11000);
2193
2194 igb_irq_disable(adapter);
2195
2196 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
2197
2198 for (i = 0; i < adapter->num_q_vectors; i++) {
2199 if (adapter->q_vector[i]) {
2200 napi_synchronize(&adapter->q_vector[i]->napi);
2201 napi_disable(&adapter->q_vector[i]->napi);
2202 }
2203 }
2204
2205 del_timer_sync(&adapter->watchdog_timer);
2206 del_timer_sync(&adapter->phy_info_timer);
2207
2208 /* record the stats before reset*/
2209 spin_lock(&adapter->stats64_lock);
2210 igb_update_stats(adapter);
2211 spin_unlock(&adapter->stats64_lock);
2212
2213 adapter->link_speed = 0;
2214 adapter->link_duplex = 0;
2215
2216 if (!pci_channel_offline(adapter->pdev))
2217 igb_reset(adapter);
2218
2219 /* clear VLAN promisc flag so VFTA will be updated if necessary */
2220 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
2221
2222 igb_clean_all_tx_rings(adapter);
2223 igb_clean_all_rx_rings(adapter);
2224#ifdef CONFIG_IGB_DCA
2225
2226 /* since we reset the hardware DCA settings were cleared */
2227 igb_setup_dca(adapter);
2228#endif
2229}
2230
2231void igb_reinit_locked(struct igb_adapter *adapter)
2232{
2233 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
2234 usleep_range(1000, 2000);
2235 igb_down(adapter);
2236 igb_up(adapter);
2237 clear_bit(__IGB_RESETTING, &adapter->state);
2238}
2239
2240/** igb_enable_mas - Media Autosense re-enable after swap
2241 *
2242 * @adapter: adapter struct
2243 **/
2244static void igb_enable_mas(struct igb_adapter *adapter)
2245{
2246 struct e1000_hw *hw = &adapter->hw;
2247 u32 connsw = rd32(E1000_CONNSW);
2248
2249 /* configure for SerDes media detect */
2250 if ((hw->phy.media_type == e1000_media_type_copper) &&
2251 (!(connsw & E1000_CONNSW_SERDESD))) {
2252 connsw |= E1000_CONNSW_ENRGSRC;
2253 connsw |= E1000_CONNSW_AUTOSENSE_EN;
2254 wr32(E1000_CONNSW, connsw);
2255 wrfl();
2256 }
2257}
2258
2259#ifdef CONFIG_IGB_HWMON
2260/**
2261 * igb_set_i2c_bb - Init I2C interface
2262 * @hw: pointer to hardware structure
2263 **/
2264static void igb_set_i2c_bb(struct e1000_hw *hw)
2265{
2266 u32 ctrl_ext;
2267 s32 i2cctl;
2268
2269 ctrl_ext = rd32(E1000_CTRL_EXT);
2270 ctrl_ext |= E1000_CTRL_I2C_ENA;
2271 wr32(E1000_CTRL_EXT, ctrl_ext);
2272 wrfl();
2273
2274 i2cctl = rd32(E1000_I2CPARAMS);
2275 i2cctl |= E1000_I2CBB_EN
2276 | E1000_I2C_CLK_OE_N
2277 | E1000_I2C_DATA_OE_N;
2278 wr32(E1000_I2CPARAMS, i2cctl);
2279 wrfl();
2280}
2281#endif
2282
2283void igb_reset(struct igb_adapter *adapter)
2284{
2285 struct pci_dev *pdev = adapter->pdev;
2286 struct e1000_hw *hw = &adapter->hw;
2287 struct e1000_mac_info *mac = &hw->mac;
2288 struct e1000_fc_info *fc = &hw->fc;
2289 u32 pba, hwm;
2290
2291 /* Repartition Pba for greater than 9k mtu
2292 * To take effect CTRL.RST is required.
2293 */
2294 switch (mac->type) {
2295 case e1000_i350:
2296 case e1000_i354:
2297 case e1000_82580:
2298 pba = rd32(E1000_RXPBS);
2299 pba = igb_rxpbs_adjust_82580(pba);
2300 break;
2301 case e1000_82576:
2302 pba = rd32(E1000_RXPBS);
2303 pba &= E1000_RXPBS_SIZE_MASK_82576;
2304 break;
2305 case e1000_82575:
2306 case e1000_i210:
2307 case e1000_i211:
2308 default:
2309 pba = E1000_PBA_34K;
2310 break;
2311 }
2312
2313 if (mac->type == e1000_82575) {
2314 u32 min_rx_space, min_tx_space, needed_tx_space;
2315
2316 /* write Rx PBA so that hardware can report correct Tx PBA */
2317 wr32(E1000_PBA, pba);
2318
2319 /* To maintain wire speed transmits, the Tx FIFO should be
2320 * large enough to accommodate two full transmit packets,
2321 * rounded up to the next 1KB and expressed in KB. Likewise,
2322 * the Rx FIFO should be large enough to accommodate at least
2323 * one full receive packet and is similarly rounded up and
2324 * expressed in KB.
2325 */
2326 min_rx_space = DIV_ROUND_UP(MAX_JUMBO_FRAME_SIZE, 1024);
2327
2328 /* The Tx FIFO also stores 16 bytes of information about the Tx
2329 * but don't include Ethernet FCS because hardware appends it.
2330 * We only need to round down to the nearest 512 byte block
2331 * count since the value we care about is 2 frames, not 1.
2332 */
2333 min_tx_space = adapter->max_frame_size;
2334 min_tx_space += sizeof(union e1000_adv_tx_desc) - ETH_FCS_LEN;
2335 min_tx_space = DIV_ROUND_UP(min_tx_space, 512);
2336
2337 /* upper 16 bits has Tx packet buffer allocation size in KB */
2338 needed_tx_space = min_tx_space - (rd32(E1000_PBA) >> 16);
2339
2340 /* If current Tx allocation is less than the min Tx FIFO size,
2341 * and the min Tx FIFO size is less than the current Rx FIFO
2342 * allocation, take space away from current Rx allocation.
2343 */
2344 if (needed_tx_space < pba) {
2345 pba -= needed_tx_space;
2346
2347 /* if short on Rx space, Rx wins and must trump Tx
2348 * adjustment
2349 */
2350 if (pba < min_rx_space)
2351 pba = min_rx_space;
2352 }
2353
2354 /* adjust PBA for jumbo frames */
2355 wr32(E1000_PBA, pba);
2356 }
2357
2358 /* flow control settings
2359 * The high water mark must be low enough to fit one full frame
2360 * after transmitting the pause frame. As such we must have enough
2361 * space to allow for us to complete our current transmit and then
2362 * receive the frame that is in progress from the link partner.
2363 * Set it to:
2364 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
2365 */
2366 hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
2367
2368 fc->high_water = hwm & 0xFFFFFFF0; /* 16-byte granularity */
2369 fc->low_water = fc->high_water - 16;
2370 fc->pause_time = 0xFFFF;
2371 fc->send_xon = 1;
2372 fc->current_mode = fc->requested_mode;
2373
2374 /* disable receive for all VFs and wait one second */
2375 if (adapter->vfs_allocated_count) {
2376 int i;
2377
2378 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
2379 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
2380
2381 /* ping all the active vfs to let them know we are going down */
2382 igb_ping_all_vfs(adapter);
2383
2384 /* disable transmits and receives */
2385 wr32(E1000_VFRE, 0);
2386 wr32(E1000_VFTE, 0);
2387 }
2388
2389 /* Allow time for pending master requests to run */
2390 hw->mac.ops.reset_hw(hw);
2391 wr32(E1000_WUC, 0);
2392
2393 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
2394 /* need to resetup here after media swap */
2395 adapter->ei.get_invariants(hw);
2396 adapter->flags &= ~IGB_FLAG_MEDIA_RESET;
2397 }
2398 if ((mac->type == e1000_82575 || mac->type == e1000_i350) &&
2399 (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
2400 igb_enable_mas(adapter);
2401 }
2402 if (hw->mac.ops.init_hw(hw))
2403 dev_err(&pdev->dev, "Hardware Error\n");
2404
2405 /* RAR registers were cleared during init_hw, clear mac table */
2406 igb_flush_mac_table(adapter);
2407 __dev_uc_unsync(adapter->netdev, NULL);
2408
2409 /* Recover default RAR entry */
2410 igb_set_default_mac_filter(adapter);
2411
2412 /* Flow control settings reset on hardware reset, so guarantee flow
2413 * control is off when forcing speed.
2414 */
2415 if (!hw->mac.autoneg)
2416 igb_force_mac_fc(hw);
2417
2418 igb_init_dmac(adapter, pba);
2419#ifdef CONFIG_IGB_HWMON
2420 /* Re-initialize the thermal sensor on i350 devices. */
2421 if (!test_bit(__IGB_DOWN, &adapter->state)) {
2422 if (mac->type == e1000_i350 && hw->bus.func == 0) {
2423 /* If present, re-initialize the external thermal sensor
2424 * interface.
2425 */
2426 if (adapter->ets)
2427 igb_set_i2c_bb(hw);
2428 mac->ops.init_thermal_sensor_thresh(hw);
2429 }
2430 }
2431#endif
2432 /* Re-establish EEE setting */
2433 if (hw->phy.media_type == e1000_media_type_copper) {
2434 switch (mac->type) {
2435 case e1000_i350:
2436 case e1000_i210:
2437 case e1000_i211:
2438 igb_set_eee_i350(hw, true, true);
2439 break;
2440 case e1000_i354:
2441 igb_set_eee_i354(hw, true, true);
2442 break;
2443 default:
2444 break;
2445 }
2446 }
2447 if (!netif_running(adapter->netdev))
2448 igb_power_down_link(adapter);
2449
2450 igb_update_mng_vlan(adapter);
2451
2452 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
2453 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
2454
2455 /* Re-enable PTP, where applicable. */
2456 if (adapter->ptp_flags & IGB_PTP_ENABLED)
2457 igb_ptp_reset(adapter);
2458
2459 igb_get_phy_info(hw);
2460}
2461
2462static netdev_features_t igb_fix_features(struct net_device *netdev,
2463 netdev_features_t features)
2464{
2465 /* Since there is no support for separate Rx/Tx vlan accel
2466 * enable/disable make sure Tx flag is always in same state as Rx.
2467 */
2468 if (features & NETIF_F_HW_VLAN_CTAG_RX)
2469 features |= NETIF_F_HW_VLAN_CTAG_TX;
2470 else
2471 features &= ~NETIF_F_HW_VLAN_CTAG_TX;
2472
2473 return features;
2474}
2475
2476static int igb_set_features(struct net_device *netdev,
2477 netdev_features_t features)
2478{
2479 netdev_features_t changed = netdev->features ^ features;
2480 struct igb_adapter *adapter = netdev_priv(netdev);
2481
2482 if (changed & NETIF_F_HW_VLAN_CTAG_RX)
2483 igb_vlan_mode(netdev, features);
2484
2485 if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
2486 return 0;
2487
2488 if (!(features & NETIF_F_NTUPLE)) {
2489 struct hlist_node *node2;
2490 struct igb_nfc_filter *rule;
2491
2492 spin_lock(&adapter->nfc_lock);
2493 hlist_for_each_entry_safe(rule, node2,
2494 &adapter->nfc_filter_list, nfc_node) {
2495 igb_erase_filter(adapter, rule);
2496 hlist_del(&rule->nfc_node);
2497 kfree(rule);
2498 }
2499 spin_unlock(&adapter->nfc_lock);
2500 adapter->nfc_filter_count = 0;
2501 }
2502
2503 netdev->features = features;
2504
2505 if (netif_running(netdev))
2506 igb_reinit_locked(adapter);
2507 else
2508 igb_reset(adapter);
2509
2510 return 1;
2511}
2512
2513static int igb_ndo_fdb_add(struct ndmsg *ndm, struct nlattr *tb[],
2514 struct net_device *dev,
2515 const unsigned char *addr, u16 vid,
2516 u16 flags,
2517 struct netlink_ext_ack *extack)
2518{
2519 /* guarantee we can provide a unique filter for the unicast address */
2520 if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr)) {
2521 struct igb_adapter *adapter = netdev_priv(dev);
2522 int vfn = adapter->vfs_allocated_count;
2523
2524 if (netdev_uc_count(dev) >= igb_available_rars(adapter, vfn))
2525 return -ENOMEM;
2526 }
2527
2528 return ndo_dflt_fdb_add(ndm, tb, dev, addr, vid, flags);
2529}
2530
2531#define IGB_MAX_MAC_HDR_LEN 127
2532#define IGB_MAX_NETWORK_HDR_LEN 511
2533
2534static netdev_features_t
2535igb_features_check(struct sk_buff *skb, struct net_device *dev,
2536 netdev_features_t features)
2537{
2538 unsigned int network_hdr_len, mac_hdr_len;
2539
2540 /* Make certain the headers can be described by a context descriptor */
2541 mac_hdr_len = skb_network_header(skb) - skb->data;
2542 if (unlikely(mac_hdr_len > IGB_MAX_MAC_HDR_LEN))
2543 return features & ~(NETIF_F_HW_CSUM |
2544 NETIF_F_SCTP_CRC |
2545 NETIF_F_GSO_UDP_L4 |
2546 NETIF_F_HW_VLAN_CTAG_TX |
2547 NETIF_F_TSO |
2548 NETIF_F_TSO6);
2549
2550 network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2551 if (unlikely(network_hdr_len > IGB_MAX_NETWORK_HDR_LEN))
2552 return features & ~(NETIF_F_HW_CSUM |
2553 NETIF_F_SCTP_CRC |
2554 NETIF_F_GSO_UDP_L4 |
2555 NETIF_F_TSO |
2556 NETIF_F_TSO6);
2557
2558 /* We can only support IPV4 TSO in tunnels if we can mangle the
2559 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2560 */
2561 if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2562 features &= ~NETIF_F_TSO;
2563
2564 return features;
2565}
2566
2567static void igb_offload_apply(struct igb_adapter *adapter, s32 queue)
2568{
2569 if (!is_fqtss_enabled(adapter)) {
2570 enable_fqtss(adapter, true);
2571 return;
2572 }
2573
2574 igb_config_tx_modes(adapter, queue);
2575
2576 if (!is_any_cbs_enabled(adapter) && !is_any_txtime_enabled(adapter))
2577 enable_fqtss(adapter, false);
2578}
2579
2580static int igb_offload_cbs(struct igb_adapter *adapter,
2581 struct tc_cbs_qopt_offload *qopt)
2582{
2583 struct e1000_hw *hw = &adapter->hw;
2584 int err;
2585
2586 /* CBS offloading is only supported by i210 controller. */
2587 if (hw->mac.type != e1000_i210)
2588 return -EOPNOTSUPP;
2589
2590 /* CBS offloading is only supported by queue 0 and queue 1. */
2591 if (qopt->queue < 0 || qopt->queue > 1)
2592 return -EINVAL;
2593
2594 err = igb_save_cbs_params(adapter, qopt->queue, qopt->enable,
2595 qopt->idleslope, qopt->sendslope,
2596 qopt->hicredit, qopt->locredit);
2597 if (err)
2598 return err;
2599
2600 igb_offload_apply(adapter, qopt->queue);
2601
2602 return 0;
2603}
2604
2605#define ETHER_TYPE_FULL_MASK ((__force __be16)~0)
2606#define VLAN_PRIO_FULL_MASK (0x07)
2607
2608static int igb_parse_cls_flower(struct igb_adapter *adapter,
2609 struct flow_cls_offload *f,
2610 int traffic_class,
2611 struct igb_nfc_filter *input)
2612{
2613 struct flow_rule *rule = flow_cls_offload_flow_rule(f);
2614 struct flow_dissector *dissector = rule->match.dissector;
2615 struct netlink_ext_ack *extack = f->common.extack;
2616
2617 if (dissector->used_keys &
2618 ~(BIT_ULL(FLOW_DISSECTOR_KEY_BASIC) |
2619 BIT_ULL(FLOW_DISSECTOR_KEY_CONTROL) |
2620 BIT_ULL(FLOW_DISSECTOR_KEY_ETH_ADDRS) |
2621 BIT_ULL(FLOW_DISSECTOR_KEY_VLAN))) {
2622 NL_SET_ERR_MSG_MOD(extack,
2623 "Unsupported key used, only BASIC, CONTROL, ETH_ADDRS and VLAN are supported");
2624 return -EOPNOTSUPP;
2625 }
2626
2627 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_ETH_ADDRS)) {
2628 struct flow_match_eth_addrs match;
2629
2630 flow_rule_match_eth_addrs(rule, &match);
2631 if (!is_zero_ether_addr(match.mask->dst)) {
2632 if (!is_broadcast_ether_addr(match.mask->dst)) {
2633 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for destination MAC address");
2634 return -EINVAL;
2635 }
2636
2637 input->filter.match_flags |=
2638 IGB_FILTER_FLAG_DST_MAC_ADDR;
2639 ether_addr_copy(input->filter.dst_addr, match.key->dst);
2640 }
2641
2642 if (!is_zero_ether_addr(match.mask->src)) {
2643 if (!is_broadcast_ether_addr(match.mask->src)) {
2644 NL_SET_ERR_MSG_MOD(extack, "Only full masks are supported for source MAC address");
2645 return -EINVAL;
2646 }
2647
2648 input->filter.match_flags |=
2649 IGB_FILTER_FLAG_SRC_MAC_ADDR;
2650 ether_addr_copy(input->filter.src_addr, match.key->src);
2651 }
2652 }
2653
2654 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_BASIC)) {
2655 struct flow_match_basic match;
2656
2657 flow_rule_match_basic(rule, &match);
2658 if (match.mask->n_proto) {
2659 if (match.mask->n_proto != ETHER_TYPE_FULL_MASK) {
2660 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for EtherType filter");
2661 return -EINVAL;
2662 }
2663
2664 input->filter.match_flags |= IGB_FILTER_FLAG_ETHER_TYPE;
2665 input->filter.etype = match.key->n_proto;
2666 }
2667 }
2668
2669 if (flow_rule_match_key(rule, FLOW_DISSECTOR_KEY_VLAN)) {
2670 struct flow_match_vlan match;
2671
2672 flow_rule_match_vlan(rule, &match);
2673 if (match.mask->vlan_priority) {
2674 if (match.mask->vlan_priority != VLAN_PRIO_FULL_MASK) {
2675 NL_SET_ERR_MSG_MOD(extack, "Only full mask is supported for VLAN priority");
2676 return -EINVAL;
2677 }
2678
2679 input->filter.match_flags |= IGB_FILTER_FLAG_VLAN_TCI;
2680 input->filter.vlan_tci =
2681 (__force __be16)match.key->vlan_priority;
2682 }
2683 }
2684
2685 input->action = traffic_class;
2686 input->cookie = f->cookie;
2687
2688 return 0;
2689}
2690
2691static int igb_configure_clsflower(struct igb_adapter *adapter,
2692 struct flow_cls_offload *cls_flower)
2693{
2694 struct netlink_ext_ack *extack = cls_flower->common.extack;
2695 struct igb_nfc_filter *filter, *f;
2696 int err, tc;
2697
2698 tc = tc_classid_to_hwtc(adapter->netdev, cls_flower->classid);
2699 if (tc < 0) {
2700 NL_SET_ERR_MSG_MOD(extack, "Invalid traffic class");
2701 return -EINVAL;
2702 }
2703
2704 filter = kzalloc(sizeof(*filter), GFP_KERNEL);
2705 if (!filter)
2706 return -ENOMEM;
2707
2708 err = igb_parse_cls_flower(adapter, cls_flower, tc, filter);
2709 if (err < 0)
2710 goto err_parse;
2711
2712 spin_lock(&adapter->nfc_lock);
2713
2714 hlist_for_each_entry(f, &adapter->nfc_filter_list, nfc_node) {
2715 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2716 err = -EEXIST;
2717 NL_SET_ERR_MSG_MOD(extack,
2718 "This filter is already set in ethtool");
2719 goto err_locked;
2720 }
2721 }
2722
2723 hlist_for_each_entry(f, &adapter->cls_flower_list, nfc_node) {
2724 if (!memcmp(&f->filter, &filter->filter, sizeof(f->filter))) {
2725 err = -EEXIST;
2726 NL_SET_ERR_MSG_MOD(extack,
2727 "This filter is already set in cls_flower");
2728 goto err_locked;
2729 }
2730 }
2731
2732 err = igb_add_filter(adapter, filter);
2733 if (err < 0) {
2734 NL_SET_ERR_MSG_MOD(extack, "Could not add filter to the adapter");
2735 goto err_locked;
2736 }
2737
2738 hlist_add_head(&filter->nfc_node, &adapter->cls_flower_list);
2739
2740 spin_unlock(&adapter->nfc_lock);
2741
2742 return 0;
2743
2744err_locked:
2745 spin_unlock(&adapter->nfc_lock);
2746
2747err_parse:
2748 kfree(filter);
2749
2750 return err;
2751}
2752
2753static int igb_delete_clsflower(struct igb_adapter *adapter,
2754 struct flow_cls_offload *cls_flower)
2755{
2756 struct igb_nfc_filter *filter;
2757 int err;
2758
2759 spin_lock(&adapter->nfc_lock);
2760
2761 hlist_for_each_entry(filter, &adapter->cls_flower_list, nfc_node)
2762 if (filter->cookie == cls_flower->cookie)
2763 break;
2764
2765 if (!filter) {
2766 err = -ENOENT;
2767 goto out;
2768 }
2769
2770 err = igb_erase_filter(adapter, filter);
2771 if (err < 0)
2772 goto out;
2773
2774 hlist_del(&filter->nfc_node);
2775 kfree(filter);
2776
2777out:
2778 spin_unlock(&adapter->nfc_lock);
2779
2780 return err;
2781}
2782
2783static int igb_setup_tc_cls_flower(struct igb_adapter *adapter,
2784 struct flow_cls_offload *cls_flower)
2785{
2786 switch (cls_flower->command) {
2787 case FLOW_CLS_REPLACE:
2788 return igb_configure_clsflower(adapter, cls_flower);
2789 case FLOW_CLS_DESTROY:
2790 return igb_delete_clsflower(adapter, cls_flower);
2791 case FLOW_CLS_STATS:
2792 return -EOPNOTSUPP;
2793 default:
2794 return -EOPNOTSUPP;
2795 }
2796}
2797
2798static int igb_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
2799 void *cb_priv)
2800{
2801 struct igb_adapter *adapter = cb_priv;
2802
2803 if (!tc_cls_can_offload_and_chain0(adapter->netdev, type_data))
2804 return -EOPNOTSUPP;
2805
2806 switch (type) {
2807 case TC_SETUP_CLSFLOWER:
2808 return igb_setup_tc_cls_flower(adapter, type_data);
2809
2810 default:
2811 return -EOPNOTSUPP;
2812 }
2813}
2814
2815static int igb_offload_txtime(struct igb_adapter *adapter,
2816 struct tc_etf_qopt_offload *qopt)
2817{
2818 struct e1000_hw *hw = &adapter->hw;
2819 int err;
2820
2821 /* Launchtime offloading is only supported by i210 controller. */
2822 if (hw->mac.type != e1000_i210)
2823 return -EOPNOTSUPP;
2824
2825 /* Launchtime offloading is only supported by queues 0 and 1. */
2826 if (qopt->queue < 0 || qopt->queue > 1)
2827 return -EINVAL;
2828
2829 err = igb_save_txtime_params(adapter, qopt->queue, qopt->enable);
2830 if (err)
2831 return err;
2832
2833 igb_offload_apply(adapter, qopt->queue);
2834
2835 return 0;
2836}
2837
2838static int igb_tc_query_caps(struct igb_adapter *adapter,
2839 struct tc_query_caps_base *base)
2840{
2841 switch (base->type) {
2842 case TC_SETUP_QDISC_TAPRIO: {
2843 struct tc_taprio_caps *caps = base->caps;
2844
2845 caps->broken_mqprio = true;
2846
2847 return 0;
2848 }
2849 default:
2850 return -EOPNOTSUPP;
2851 }
2852}
2853
2854static LIST_HEAD(igb_block_cb_list);
2855
2856static int igb_setup_tc(struct net_device *dev, enum tc_setup_type type,
2857 void *type_data)
2858{
2859 struct igb_adapter *adapter = netdev_priv(dev);
2860
2861 switch (type) {
2862 case TC_QUERY_CAPS:
2863 return igb_tc_query_caps(adapter, type_data);
2864 case TC_SETUP_QDISC_CBS:
2865 return igb_offload_cbs(adapter, type_data);
2866 case TC_SETUP_BLOCK:
2867 return flow_block_cb_setup_simple(type_data,
2868 &igb_block_cb_list,
2869 igb_setup_tc_block_cb,
2870 adapter, adapter, true);
2871
2872 case TC_SETUP_QDISC_ETF:
2873 return igb_offload_txtime(adapter, type_data);
2874
2875 default:
2876 return -EOPNOTSUPP;
2877 }
2878}
2879
2880static int igb_xdp_setup(struct net_device *dev, struct netdev_bpf *bpf)
2881{
2882 int i, frame_size = dev->mtu + IGB_ETH_PKT_HDR_PAD;
2883 struct igb_adapter *adapter = netdev_priv(dev);
2884 struct bpf_prog *prog = bpf->prog, *old_prog;
2885 bool running = netif_running(dev);
2886 bool need_reset;
2887
2888 /* verify igb ring attributes are sufficient for XDP */
2889 for (i = 0; i < adapter->num_rx_queues; i++) {
2890 struct igb_ring *ring = adapter->rx_ring[i];
2891
2892 if (frame_size > igb_rx_bufsz(ring)) {
2893 NL_SET_ERR_MSG_MOD(bpf->extack,
2894 "The RX buffer size is too small for the frame size");
2895 netdev_warn(dev, "XDP RX buffer size %d is too small for the frame size %d\n",
2896 igb_rx_bufsz(ring), frame_size);
2897 return -EINVAL;
2898 }
2899 }
2900
2901 old_prog = xchg(&adapter->xdp_prog, prog);
2902 need_reset = (!!prog != !!old_prog);
2903
2904 /* device is up and bpf is added/removed, must setup the RX queues */
2905 if (need_reset && running) {
2906 igb_close(dev);
2907 } else {
2908 for (i = 0; i < adapter->num_rx_queues; i++)
2909 (void)xchg(&adapter->rx_ring[i]->xdp_prog,
2910 adapter->xdp_prog);
2911 }
2912
2913 if (old_prog)
2914 bpf_prog_put(old_prog);
2915
2916 /* bpf is just replaced, RXQ and MTU are already setup */
2917 if (!need_reset) {
2918 return 0;
2919 } else {
2920 if (prog)
2921 xdp_features_set_redirect_target(dev, true);
2922 else
2923 xdp_features_clear_redirect_target(dev);
2924 }
2925
2926 if (running)
2927 igb_open(dev);
2928
2929 return 0;
2930}
2931
2932static int igb_xdp(struct net_device *dev, struct netdev_bpf *xdp)
2933{
2934 switch (xdp->command) {
2935 case XDP_SETUP_PROG:
2936 return igb_xdp_setup(dev, xdp);
2937 default:
2938 return -EINVAL;
2939 }
2940}
2941
2942static void igb_xdp_ring_update_tail(struct igb_ring *ring)
2943{
2944 /* Force memory writes to complete before letting h/w know there
2945 * are new descriptors to fetch.
2946 */
2947 wmb();
2948 writel(ring->next_to_use, ring->tail);
2949}
2950
2951static struct igb_ring *igb_xdp_tx_queue_mapping(struct igb_adapter *adapter)
2952{
2953 unsigned int r_idx = smp_processor_id();
2954
2955 if (r_idx >= adapter->num_tx_queues)
2956 r_idx = r_idx % adapter->num_tx_queues;
2957
2958 return adapter->tx_ring[r_idx];
2959}
2960
2961static int igb_xdp_xmit_back(struct igb_adapter *adapter, struct xdp_buff *xdp)
2962{
2963 struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2964 int cpu = smp_processor_id();
2965 struct igb_ring *tx_ring;
2966 struct netdev_queue *nq;
2967 u32 ret;
2968
2969 if (unlikely(!xdpf))
2970 return IGB_XDP_CONSUMED;
2971
2972 /* During program transitions its possible adapter->xdp_prog is assigned
2973 * but ring has not been configured yet. In this case simply abort xmit.
2974 */
2975 tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL;
2976 if (unlikely(!tx_ring))
2977 return IGB_XDP_CONSUMED;
2978
2979 nq = txring_txq(tx_ring);
2980 __netif_tx_lock(nq, cpu);
2981 /* Avoid transmit queue timeout since we share it with the slow path */
2982 txq_trans_cond_update(nq);
2983 ret = igb_xmit_xdp_ring(adapter, tx_ring, xdpf);
2984 __netif_tx_unlock(nq);
2985
2986 return ret;
2987}
2988
2989static int igb_xdp_xmit(struct net_device *dev, int n,
2990 struct xdp_frame **frames, u32 flags)
2991{
2992 struct igb_adapter *adapter = netdev_priv(dev);
2993 int cpu = smp_processor_id();
2994 struct igb_ring *tx_ring;
2995 struct netdev_queue *nq;
2996 int nxmit = 0;
2997 int i;
2998
2999 if (unlikely(test_bit(__IGB_DOWN, &adapter->state)))
3000 return -ENETDOWN;
3001
3002 if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
3003 return -EINVAL;
3004
3005 /* During program transitions its possible adapter->xdp_prog is assigned
3006 * but ring has not been configured yet. In this case simply abort xmit.
3007 */
3008 tx_ring = adapter->xdp_prog ? igb_xdp_tx_queue_mapping(adapter) : NULL;
3009 if (unlikely(!tx_ring))
3010 return -ENXIO;
3011
3012 nq = txring_txq(tx_ring);
3013 __netif_tx_lock(nq, cpu);
3014
3015 /* Avoid transmit queue timeout since we share it with the slow path */
3016 txq_trans_cond_update(nq);
3017
3018 for (i = 0; i < n; i++) {
3019 struct xdp_frame *xdpf = frames[i];
3020 int err;
3021
3022 err = igb_xmit_xdp_ring(adapter, tx_ring, xdpf);
3023 if (err != IGB_XDP_TX)
3024 break;
3025 nxmit++;
3026 }
3027
3028 __netif_tx_unlock(nq);
3029
3030 if (unlikely(flags & XDP_XMIT_FLUSH))
3031 igb_xdp_ring_update_tail(tx_ring);
3032
3033 return nxmit;
3034}
3035
3036static const struct net_device_ops igb_netdev_ops = {
3037 .ndo_open = igb_open,
3038 .ndo_stop = igb_close,
3039 .ndo_start_xmit = igb_xmit_frame,
3040 .ndo_get_stats64 = igb_get_stats64,
3041 .ndo_set_rx_mode = igb_set_rx_mode,
3042 .ndo_set_mac_address = igb_set_mac,
3043 .ndo_change_mtu = igb_change_mtu,
3044 .ndo_eth_ioctl = igb_ioctl,
3045 .ndo_tx_timeout = igb_tx_timeout,
3046 .ndo_validate_addr = eth_validate_addr,
3047 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
3048 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
3049 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
3050 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
3051 .ndo_set_vf_rate = igb_ndo_set_vf_bw,
3052 .ndo_set_vf_spoofchk = igb_ndo_set_vf_spoofchk,
3053 .ndo_set_vf_trust = igb_ndo_set_vf_trust,
3054 .ndo_get_vf_config = igb_ndo_get_vf_config,
3055 .ndo_fix_features = igb_fix_features,
3056 .ndo_set_features = igb_set_features,
3057 .ndo_fdb_add = igb_ndo_fdb_add,
3058 .ndo_features_check = igb_features_check,
3059 .ndo_setup_tc = igb_setup_tc,
3060 .ndo_bpf = igb_xdp,
3061 .ndo_xdp_xmit = igb_xdp_xmit,
3062};
3063
3064/**
3065 * igb_set_fw_version - Configure version string for ethtool
3066 * @adapter: adapter struct
3067 **/
3068void igb_set_fw_version(struct igb_adapter *adapter)
3069{
3070 struct e1000_hw *hw = &adapter->hw;
3071 struct e1000_fw_version fw;
3072
3073 igb_get_fw_version(hw, &fw);
3074
3075 switch (hw->mac.type) {
3076 case e1000_i210:
3077 case e1000_i211:
3078 if (!(igb_get_flash_presence_i210(hw))) {
3079 snprintf(adapter->fw_version,
3080 sizeof(adapter->fw_version),
3081 "%2d.%2d-%d",
3082 fw.invm_major, fw.invm_minor,
3083 fw.invm_img_type);
3084 break;
3085 }
3086 fallthrough;
3087 default:
3088 /* if option rom is valid, display its version too */
3089 if (fw.or_valid) {
3090 snprintf(adapter->fw_version,
3091 sizeof(adapter->fw_version),
3092 "%d.%d, 0x%08x, %d.%d.%d",
3093 fw.eep_major, fw.eep_minor, fw.etrack_id,
3094 fw.or_major, fw.or_build, fw.or_patch);
3095 /* no option rom */
3096 } else if (fw.etrack_id != 0X0000) {
3097 snprintf(adapter->fw_version,
3098 sizeof(adapter->fw_version),
3099 "%d.%d, 0x%08x",
3100 fw.eep_major, fw.eep_minor, fw.etrack_id);
3101 } else {
3102 snprintf(adapter->fw_version,
3103 sizeof(adapter->fw_version),
3104 "%d.%d.%d",
3105 fw.eep_major, fw.eep_minor, fw.eep_build);
3106 }
3107 break;
3108 }
3109}
3110
3111/**
3112 * igb_init_mas - init Media Autosense feature if enabled in the NVM
3113 *
3114 * @adapter: adapter struct
3115 **/
3116static void igb_init_mas(struct igb_adapter *adapter)
3117{
3118 struct e1000_hw *hw = &adapter->hw;
3119 u16 eeprom_data;
3120
3121 hw->nvm.ops.read(hw, NVM_COMPAT, 1, &eeprom_data);
3122 switch (hw->bus.func) {
3123 case E1000_FUNC_0:
3124 if (eeprom_data & IGB_MAS_ENABLE_0) {
3125 adapter->flags |= IGB_FLAG_MAS_ENABLE;
3126 netdev_info(adapter->netdev,
3127 "MAS: Enabling Media Autosense for port %d\n",
3128 hw->bus.func);
3129 }
3130 break;
3131 case E1000_FUNC_1:
3132 if (eeprom_data & IGB_MAS_ENABLE_1) {
3133 adapter->flags |= IGB_FLAG_MAS_ENABLE;
3134 netdev_info(adapter->netdev,
3135 "MAS: Enabling Media Autosense for port %d\n",
3136 hw->bus.func);
3137 }
3138 break;
3139 case E1000_FUNC_2:
3140 if (eeprom_data & IGB_MAS_ENABLE_2) {
3141 adapter->flags |= IGB_FLAG_MAS_ENABLE;
3142 netdev_info(adapter->netdev,
3143 "MAS: Enabling Media Autosense for port %d\n",
3144 hw->bus.func);
3145 }
3146 break;
3147 case E1000_FUNC_3:
3148 if (eeprom_data & IGB_MAS_ENABLE_3) {
3149 adapter->flags |= IGB_FLAG_MAS_ENABLE;
3150 netdev_info(adapter->netdev,
3151 "MAS: Enabling Media Autosense for port %d\n",
3152 hw->bus.func);
3153 }
3154 break;
3155 default:
3156 /* Shouldn't get here */
3157 netdev_err(adapter->netdev,
3158 "MAS: Invalid port configuration, returning\n");
3159 break;
3160 }
3161}
3162
3163/**
3164 * igb_init_i2c - Init I2C interface
3165 * @adapter: pointer to adapter structure
3166 **/
3167static s32 igb_init_i2c(struct igb_adapter *adapter)
3168{
3169 s32 status = 0;
3170
3171 /* I2C interface supported on i350 devices */
3172 if (adapter->hw.mac.type != e1000_i350)
3173 return 0;
3174
3175 /* Initialize the i2c bus which is controlled by the registers.
3176 * This bus will use the i2c_algo_bit structure that implements
3177 * the protocol through toggling of the 4 bits in the register.
3178 */
3179 adapter->i2c_adap.owner = THIS_MODULE;
3180 adapter->i2c_algo = igb_i2c_algo;
3181 adapter->i2c_algo.data = adapter;
3182 adapter->i2c_adap.algo_data = &adapter->i2c_algo;
3183 adapter->i2c_adap.dev.parent = &adapter->pdev->dev;
3184 strscpy(adapter->i2c_adap.name, "igb BB",
3185 sizeof(adapter->i2c_adap.name));
3186 status = i2c_bit_add_bus(&adapter->i2c_adap);
3187 return status;
3188}
3189
3190/**
3191 * igb_probe - Device Initialization Routine
3192 * @pdev: PCI device information struct
3193 * @ent: entry in igb_pci_tbl
3194 *
3195 * Returns 0 on success, negative on failure
3196 *
3197 * igb_probe initializes an adapter identified by a pci_dev structure.
3198 * The OS initialization, configuring of the adapter private structure,
3199 * and a hardware reset occur.
3200 **/
3201static int igb_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
3202{
3203 struct net_device *netdev;
3204 struct igb_adapter *adapter;
3205 struct e1000_hw *hw;
3206 u16 eeprom_data = 0;
3207 s32 ret_val;
3208 static int global_quad_port_a; /* global quad port a indication */
3209 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
3210 u8 part_str[E1000_PBANUM_LENGTH];
3211 int err;
3212
3213 /* Catch broken hardware that put the wrong VF device ID in
3214 * the PCIe SR-IOV capability.
3215 */
3216 if (pdev->is_virtfn) {
3217 WARN(1, KERN_ERR "%s (%x:%x) should not be a VF!\n",
3218 pci_name(pdev), pdev->vendor, pdev->device);
3219 return -EINVAL;
3220 }
3221
3222 err = pci_enable_device_mem(pdev);
3223 if (err)
3224 return err;
3225
3226 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3227 if (err) {
3228 dev_err(&pdev->dev,
3229 "No usable DMA configuration, aborting\n");
3230 goto err_dma;
3231 }
3232
3233 err = pci_request_mem_regions(pdev, igb_driver_name);
3234 if (err)
3235 goto err_pci_reg;
3236
3237 pci_set_master(pdev);
3238 pci_save_state(pdev);
3239
3240 err = -ENOMEM;
3241 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
3242 IGB_MAX_TX_QUEUES);
3243 if (!netdev)
3244 goto err_alloc_etherdev;
3245
3246 SET_NETDEV_DEV(netdev, &pdev->dev);
3247
3248 pci_set_drvdata(pdev, netdev);
3249 adapter = netdev_priv(netdev);
3250 adapter->netdev = netdev;
3251 adapter->pdev = pdev;
3252 hw = &adapter->hw;
3253 hw->back = adapter;
3254 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
3255
3256 err = -EIO;
3257 adapter->io_addr = pci_iomap(pdev, 0, 0);
3258 if (!adapter->io_addr)
3259 goto err_ioremap;
3260 /* hw->hw_addr can be altered, we'll use adapter->io_addr for unmap */
3261 hw->hw_addr = adapter->io_addr;
3262
3263 netdev->netdev_ops = &igb_netdev_ops;
3264 igb_set_ethtool_ops(netdev);
3265 netdev->watchdog_timeo = 5 * HZ;
3266
3267 strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
3268
3269 netdev->mem_start = pci_resource_start(pdev, 0);
3270 netdev->mem_end = pci_resource_end(pdev, 0);
3271
3272 /* PCI config space info */
3273 hw->vendor_id = pdev->vendor;
3274 hw->device_id = pdev->device;
3275 hw->revision_id = pdev->revision;
3276 hw->subsystem_vendor_id = pdev->subsystem_vendor;
3277 hw->subsystem_device_id = pdev->subsystem_device;
3278
3279 /* Copy the default MAC, PHY and NVM function pointers */
3280 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
3281 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
3282 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
3283 /* Initialize skew-specific constants */
3284 err = ei->get_invariants(hw);
3285 if (err)
3286 goto err_sw_init;
3287
3288 /* setup the private structure */
3289 err = igb_sw_init(adapter);
3290 if (err)
3291 goto err_sw_init;
3292
3293 igb_get_bus_info_pcie(hw);
3294
3295 hw->phy.autoneg_wait_to_complete = false;
3296
3297 /* Copper options */
3298 if (hw->phy.media_type == e1000_media_type_copper) {
3299 hw->phy.mdix = AUTO_ALL_MODES;
3300 hw->phy.disable_polarity_correction = false;
3301 hw->phy.ms_type = e1000_ms_hw_default;
3302 }
3303
3304 if (igb_check_reset_block(hw))
3305 dev_info(&pdev->dev,
3306 "PHY reset is blocked due to SOL/IDER session.\n");
3307
3308 /* features is initialized to 0 in allocation, it might have bits
3309 * set by igb_sw_init so we should use an or instead of an
3310 * assignment.
3311 */
3312 netdev->features |= NETIF_F_SG |
3313 NETIF_F_TSO |
3314 NETIF_F_TSO6 |
3315 NETIF_F_RXHASH |
3316 NETIF_F_RXCSUM |
3317 NETIF_F_HW_CSUM;
3318
3319 if (hw->mac.type >= e1000_82576)
3320 netdev->features |= NETIF_F_SCTP_CRC | NETIF_F_GSO_UDP_L4;
3321
3322 if (hw->mac.type >= e1000_i350)
3323 netdev->features |= NETIF_F_HW_TC;
3324
3325#define IGB_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
3326 NETIF_F_GSO_GRE_CSUM | \
3327 NETIF_F_GSO_IPXIP4 | \
3328 NETIF_F_GSO_IPXIP6 | \
3329 NETIF_F_GSO_UDP_TUNNEL | \
3330 NETIF_F_GSO_UDP_TUNNEL_CSUM)
3331
3332 netdev->gso_partial_features = IGB_GSO_PARTIAL_FEATURES;
3333 netdev->features |= NETIF_F_GSO_PARTIAL | IGB_GSO_PARTIAL_FEATURES;
3334
3335 /* copy netdev features into list of user selectable features */
3336 netdev->hw_features |= netdev->features |
3337 NETIF_F_HW_VLAN_CTAG_RX |
3338 NETIF_F_HW_VLAN_CTAG_TX |
3339 NETIF_F_RXALL;
3340
3341 if (hw->mac.type >= e1000_i350)
3342 netdev->hw_features |= NETIF_F_NTUPLE;
3343
3344 netdev->features |= NETIF_F_HIGHDMA;
3345
3346 netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
3347 netdev->mpls_features |= NETIF_F_HW_CSUM;
3348 netdev->hw_enc_features |= netdev->vlan_features;
3349
3350 /* set this bit last since it cannot be part of vlan_features */
3351 netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
3352 NETIF_F_HW_VLAN_CTAG_RX |
3353 NETIF_F_HW_VLAN_CTAG_TX;
3354
3355 netdev->priv_flags |= IFF_SUPP_NOFCS;
3356
3357 netdev->priv_flags |= IFF_UNICAST_FLT;
3358 netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT;
3359
3360 /* MTU range: 68 - 9216 */
3361 netdev->min_mtu = ETH_MIN_MTU;
3362 netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
3363
3364 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
3365
3366 /* before reading the NVM, reset the controller to put the device in a
3367 * known good starting state
3368 */
3369 hw->mac.ops.reset_hw(hw);
3370
3371 /* make sure the NVM is good , i211/i210 parts can have special NVM
3372 * that doesn't contain a checksum
3373 */
3374 switch (hw->mac.type) {
3375 case e1000_i210:
3376 case e1000_i211:
3377 if (igb_get_flash_presence_i210(hw)) {
3378 if (hw->nvm.ops.validate(hw) < 0) {
3379 dev_err(&pdev->dev,
3380 "The NVM Checksum Is Not Valid\n");
3381 err = -EIO;
3382 goto err_eeprom;
3383 }
3384 }
3385 break;
3386 default:
3387 if (hw->nvm.ops.validate(hw) < 0) {
3388 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
3389 err = -EIO;
3390 goto err_eeprom;
3391 }
3392 break;
3393 }
3394
3395 if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
3396 /* copy the MAC address out of the NVM */
3397 if (hw->mac.ops.read_mac_addr(hw))
3398 dev_err(&pdev->dev, "NVM Read Error\n");
3399 }
3400
3401 eth_hw_addr_set(netdev, hw->mac.addr);
3402
3403 if (!is_valid_ether_addr(netdev->dev_addr)) {
3404 dev_err(&pdev->dev, "Invalid MAC Address\n");
3405 err = -EIO;
3406 goto err_eeprom;
3407 }
3408
3409 igb_set_default_mac_filter(adapter);
3410
3411 /* get firmware version for ethtool -i */
3412 igb_set_fw_version(adapter);
3413
3414 /* configure RXPBSIZE and TXPBSIZE */
3415 if (hw->mac.type == e1000_i210) {
3416 wr32(E1000_RXPBS, I210_RXPBSIZE_DEFAULT);
3417 wr32(E1000_TXPBS, I210_TXPBSIZE_DEFAULT);
3418 }
3419
3420 timer_setup(&adapter->watchdog_timer, igb_watchdog, 0);
3421 timer_setup(&adapter->phy_info_timer, igb_update_phy_info, 0);
3422
3423 INIT_WORK(&adapter->reset_task, igb_reset_task);
3424 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
3425
3426 /* Initialize link properties that are user-changeable */
3427 adapter->fc_autoneg = true;
3428 hw->mac.autoneg = true;
3429 hw->phy.autoneg_advertised = 0x2f;
3430
3431 hw->fc.requested_mode = e1000_fc_default;
3432 hw->fc.current_mode = e1000_fc_default;
3433
3434 igb_validate_mdi_setting(hw);
3435
3436 /* By default, support wake on port A */
3437 if (hw->bus.func == 0)
3438 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3439
3440 /* Check the NVM for wake support on non-port A ports */
3441 if (hw->mac.type >= e1000_82580)
3442 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
3443 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
3444 &eeprom_data);
3445 else if (hw->bus.func == 1)
3446 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
3447
3448 if (eeprom_data & IGB_EEPROM_APME)
3449 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3450
3451 /* now that we have the eeprom settings, apply the special cases where
3452 * the eeprom may be wrong or the board simply won't support wake on
3453 * lan on a particular port
3454 */
3455 switch (pdev->device) {
3456 case E1000_DEV_ID_82575GB_QUAD_COPPER:
3457 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3458 break;
3459 case E1000_DEV_ID_82575EB_FIBER_SERDES:
3460 case E1000_DEV_ID_82576_FIBER:
3461 case E1000_DEV_ID_82576_SERDES:
3462 /* Wake events only supported on port A for dual fiber
3463 * regardless of eeprom setting
3464 */
3465 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
3466 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3467 break;
3468 case E1000_DEV_ID_82576_QUAD_COPPER:
3469 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
3470 /* if quad port adapter, disable WoL on all but port A */
3471 if (global_quad_port_a != 0)
3472 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3473 else
3474 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
3475 /* Reset for multiple quad port adapters */
3476 if (++global_quad_port_a == 4)
3477 global_quad_port_a = 0;
3478 break;
3479 default:
3480 /* If the device can't wake, don't set software support */
3481 if (!device_can_wakeup(&adapter->pdev->dev))
3482 adapter->flags &= ~IGB_FLAG_WOL_SUPPORTED;
3483 }
3484
3485 /* initialize the wol settings based on the eeprom settings */
3486 if (adapter->flags & IGB_FLAG_WOL_SUPPORTED)
3487 adapter->wol |= E1000_WUFC_MAG;
3488
3489 /* Some vendors want WoL disabled by default, but still supported */
3490 if ((hw->mac.type == e1000_i350) &&
3491 (pdev->subsystem_vendor == PCI_VENDOR_ID_HP)) {
3492 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3493 adapter->wol = 0;
3494 }
3495
3496 /* Some vendors want the ability to Use the EEPROM setting as
3497 * enable/disable only, and not for capability
3498 */
3499 if (((hw->mac.type == e1000_i350) ||
3500 (hw->mac.type == e1000_i354)) &&
3501 (pdev->subsystem_vendor == PCI_VENDOR_ID_DELL)) {
3502 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3503 adapter->wol = 0;
3504 }
3505 if (hw->mac.type == e1000_i350) {
3506 if (((pdev->subsystem_device == 0x5001) ||
3507 (pdev->subsystem_device == 0x5002)) &&
3508 (hw->bus.func == 0)) {
3509 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3510 adapter->wol = 0;
3511 }
3512 if (pdev->subsystem_device == 0x1F52)
3513 adapter->flags |= IGB_FLAG_WOL_SUPPORTED;
3514 }
3515
3516 device_set_wakeup_enable(&adapter->pdev->dev,
3517 adapter->flags & IGB_FLAG_WOL_SUPPORTED);
3518
3519 /* reset the hardware with the new settings */
3520 igb_reset(adapter);
3521
3522 /* Init the I2C interface */
3523 err = igb_init_i2c(adapter);
3524 if (err) {
3525 dev_err(&pdev->dev, "failed to init i2c interface\n");
3526 goto err_eeprom;
3527 }
3528
3529 /* let the f/w know that the h/w is now under the control of the
3530 * driver.
3531 */
3532 igb_get_hw_control(adapter);
3533
3534 strcpy(netdev->name, "eth%d");
3535 err = register_netdev(netdev);
3536 if (err)
3537 goto err_register;
3538
3539 /* carrier off reporting is important to ethtool even BEFORE open */
3540 netif_carrier_off(netdev);
3541
3542#ifdef CONFIG_IGB_DCA
3543 if (dca_add_requester(&pdev->dev) == 0) {
3544 adapter->flags |= IGB_FLAG_DCA_ENABLED;
3545 dev_info(&pdev->dev, "DCA enabled\n");
3546 igb_setup_dca(adapter);
3547 }
3548
3549#endif
3550#ifdef CONFIG_IGB_HWMON
3551 /* Initialize the thermal sensor on i350 devices. */
3552 if (hw->mac.type == e1000_i350 && hw->bus.func == 0) {
3553 u16 ets_word;
3554
3555 /* Read the NVM to determine if this i350 device supports an
3556 * external thermal sensor.
3557 */
3558 hw->nvm.ops.read(hw, NVM_ETS_CFG, 1, &ets_word);
3559 if (ets_word != 0x0000 && ets_word != 0xFFFF)
3560 adapter->ets = true;
3561 else
3562 adapter->ets = false;
3563 /* Only enable I2C bit banging if an external thermal
3564 * sensor is supported.
3565 */
3566 if (adapter->ets)
3567 igb_set_i2c_bb(hw);
3568 hw->mac.ops.init_thermal_sensor_thresh(hw);
3569 if (igb_sysfs_init(adapter))
3570 dev_err(&pdev->dev,
3571 "failed to allocate sysfs resources\n");
3572 } else {
3573 adapter->ets = false;
3574 }
3575#endif
3576 /* Check if Media Autosense is enabled */
3577 adapter->ei = *ei;
3578 if (hw->dev_spec._82575.mas_capable)
3579 igb_init_mas(adapter);
3580
3581 /* do hw tstamp init after resetting */
3582 igb_ptp_init(adapter);
3583
3584 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
3585 /* print bus type/speed/width info, not applicable to i354 */
3586 if (hw->mac.type != e1000_i354) {
3587 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
3588 netdev->name,
3589 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
3590 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
3591 "unknown"),
3592 ((hw->bus.width == e1000_bus_width_pcie_x4) ?
3593 "Width x4" :
3594 (hw->bus.width == e1000_bus_width_pcie_x2) ?
3595 "Width x2" :
3596 (hw->bus.width == e1000_bus_width_pcie_x1) ?
3597 "Width x1" : "unknown"), netdev->dev_addr);
3598 }
3599
3600 if ((hw->mac.type == e1000_82576 &&
3601 rd32(E1000_EECD) & E1000_EECD_PRES) ||
3602 (hw->mac.type >= e1000_i210 ||
3603 igb_get_flash_presence_i210(hw))) {
3604 ret_val = igb_read_part_string(hw, part_str,
3605 E1000_PBANUM_LENGTH);
3606 } else {
3607 ret_val = -E1000_ERR_INVM_VALUE_NOT_FOUND;
3608 }
3609
3610 if (ret_val)
3611 strcpy(part_str, "Unknown");
3612 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
3613 dev_info(&pdev->dev,
3614 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
3615 (adapter->flags & IGB_FLAG_HAS_MSIX) ? "MSI-X" :
3616 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
3617 adapter->num_rx_queues, adapter->num_tx_queues);
3618 if (hw->phy.media_type == e1000_media_type_copper) {
3619 switch (hw->mac.type) {
3620 case e1000_i350:
3621 case e1000_i210:
3622 case e1000_i211:
3623 /* Enable EEE for internal copper PHY devices */
3624 err = igb_set_eee_i350(hw, true, true);
3625 if ((!err) &&
3626 (!hw->dev_spec._82575.eee_disable)) {
3627 adapter->eee_advert =
3628 MDIO_EEE_100TX | MDIO_EEE_1000T;
3629 adapter->flags |= IGB_FLAG_EEE;
3630 }
3631 break;
3632 case e1000_i354:
3633 if ((rd32(E1000_CTRL_EXT) &
3634 E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3635 err = igb_set_eee_i354(hw, true, true);
3636 if ((!err) &&
3637 (!hw->dev_spec._82575.eee_disable)) {
3638 adapter->eee_advert =
3639 MDIO_EEE_100TX | MDIO_EEE_1000T;
3640 adapter->flags |= IGB_FLAG_EEE;
3641 }
3642 }
3643 break;
3644 default:
3645 break;
3646 }
3647 }
3648
3649 dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
3650
3651 pm_runtime_put_noidle(&pdev->dev);
3652 return 0;
3653
3654err_register:
3655 igb_release_hw_control(adapter);
3656 memset(&adapter->i2c_adap, 0, sizeof(adapter->i2c_adap));
3657err_eeprom:
3658 if (!igb_check_reset_block(hw))
3659 igb_reset_phy(hw);
3660
3661 if (hw->flash_address)
3662 iounmap(hw->flash_address);
3663err_sw_init:
3664 kfree(adapter->mac_table);
3665 kfree(adapter->shadow_vfta);
3666 igb_clear_interrupt_scheme(adapter);
3667#ifdef CONFIG_PCI_IOV
3668 igb_disable_sriov(pdev, false);
3669#endif
3670 pci_iounmap(pdev, adapter->io_addr);
3671err_ioremap:
3672 free_netdev(netdev);
3673err_alloc_etherdev:
3674 pci_release_mem_regions(pdev);
3675err_pci_reg:
3676err_dma:
3677 pci_disable_device(pdev);
3678 return err;
3679}
3680
3681#ifdef CONFIG_PCI_IOV
3682static int igb_sriov_reinit(struct pci_dev *dev)
3683{
3684 struct net_device *netdev = pci_get_drvdata(dev);
3685 struct igb_adapter *adapter = netdev_priv(netdev);
3686 struct pci_dev *pdev = adapter->pdev;
3687
3688 rtnl_lock();
3689
3690 if (netif_running(netdev))
3691 igb_close(netdev);
3692 else
3693 igb_reset(adapter);
3694
3695 igb_clear_interrupt_scheme(adapter);
3696
3697 igb_init_queue_configuration(adapter);
3698
3699 if (igb_init_interrupt_scheme(adapter, true)) {
3700 rtnl_unlock();
3701 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
3702 return -ENOMEM;
3703 }
3704
3705 if (netif_running(netdev))
3706 igb_open(netdev);
3707
3708 rtnl_unlock();
3709
3710 return 0;
3711}
3712
3713static int igb_disable_sriov(struct pci_dev *pdev, bool reinit)
3714{
3715 struct net_device *netdev = pci_get_drvdata(pdev);
3716 struct igb_adapter *adapter = netdev_priv(netdev);
3717 struct e1000_hw *hw = &adapter->hw;
3718 unsigned long flags;
3719
3720 /* reclaim resources allocated to VFs */
3721 if (adapter->vf_data) {
3722 /* disable iov and allow time for transactions to clear */
3723 if (pci_vfs_assigned(pdev)) {
3724 dev_warn(&pdev->dev,
3725 "Cannot deallocate SR-IOV virtual functions while they are assigned - VFs will not be deallocated\n");
3726 return -EPERM;
3727 } else {
3728 pci_disable_sriov(pdev);
3729 msleep(500);
3730 }
3731 spin_lock_irqsave(&adapter->vfs_lock, flags);
3732 kfree(adapter->vf_mac_list);
3733 adapter->vf_mac_list = NULL;
3734 kfree(adapter->vf_data);
3735 adapter->vf_data = NULL;
3736 adapter->vfs_allocated_count = 0;
3737 spin_unlock_irqrestore(&adapter->vfs_lock, flags);
3738 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
3739 wrfl();
3740 msleep(100);
3741 dev_info(&pdev->dev, "IOV Disabled\n");
3742
3743 /* Re-enable DMA Coalescing flag since IOV is turned off */
3744 adapter->flags |= IGB_FLAG_DMAC;
3745 }
3746
3747 return reinit ? igb_sriov_reinit(pdev) : 0;
3748}
3749
3750static int igb_enable_sriov(struct pci_dev *pdev, int num_vfs, bool reinit)
3751{
3752 struct net_device *netdev = pci_get_drvdata(pdev);
3753 struct igb_adapter *adapter = netdev_priv(netdev);
3754 int old_vfs = pci_num_vf(pdev);
3755 struct vf_mac_filter *mac_list;
3756 int err = 0;
3757 int num_vf_mac_filters, i;
3758
3759 if (!(adapter->flags & IGB_FLAG_HAS_MSIX) || num_vfs > 7) {
3760 err = -EPERM;
3761 goto out;
3762 }
3763 if (!num_vfs)
3764 goto out;
3765
3766 if (old_vfs) {
3767 dev_info(&pdev->dev, "%d pre-allocated VFs found - override max_vfs setting of %d\n",
3768 old_vfs, max_vfs);
3769 adapter->vfs_allocated_count = old_vfs;
3770 } else
3771 adapter->vfs_allocated_count = num_vfs;
3772
3773 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
3774 sizeof(struct vf_data_storage), GFP_KERNEL);
3775
3776 /* if allocation failed then we do not support SR-IOV */
3777 if (!adapter->vf_data) {
3778 adapter->vfs_allocated_count = 0;
3779 err = -ENOMEM;
3780 goto out;
3781 }
3782
3783 /* Due to the limited number of RAR entries calculate potential
3784 * number of MAC filters available for the VFs. Reserve entries
3785 * for PF default MAC, PF MAC filters and at least one RAR entry
3786 * for each VF for VF MAC.
3787 */
3788 num_vf_mac_filters = adapter->hw.mac.rar_entry_count -
3789 (1 + IGB_PF_MAC_FILTERS_RESERVED +
3790 adapter->vfs_allocated_count);
3791
3792 adapter->vf_mac_list = kcalloc(num_vf_mac_filters,
3793 sizeof(struct vf_mac_filter),
3794 GFP_KERNEL);
3795
3796 mac_list = adapter->vf_mac_list;
3797 INIT_LIST_HEAD(&adapter->vf_macs.l);
3798
3799 if (adapter->vf_mac_list) {
3800 /* Initialize list of VF MAC filters */
3801 for (i = 0; i < num_vf_mac_filters; i++) {
3802 mac_list->vf = -1;
3803 mac_list->free = true;
3804 list_add(&mac_list->l, &adapter->vf_macs.l);
3805 mac_list++;
3806 }
3807 } else {
3808 /* If we could not allocate memory for the VF MAC filters
3809 * we can continue without this feature but warn user.
3810 */
3811 dev_err(&pdev->dev,
3812 "Unable to allocate memory for VF MAC filter list\n");
3813 }
3814
3815 dev_info(&pdev->dev, "%d VFs allocated\n",
3816 adapter->vfs_allocated_count);
3817 for (i = 0; i < adapter->vfs_allocated_count; i++)
3818 igb_vf_configure(adapter, i);
3819
3820 /* DMA Coalescing is not supported in IOV mode. */
3821 adapter->flags &= ~IGB_FLAG_DMAC;
3822
3823 if (reinit) {
3824 err = igb_sriov_reinit(pdev);
3825 if (err)
3826 goto err_out;
3827 }
3828
3829 /* only call pci_enable_sriov() if no VFs are allocated already */
3830 if (!old_vfs) {
3831 err = pci_enable_sriov(pdev, adapter->vfs_allocated_count);
3832 if (err)
3833 goto err_out;
3834 }
3835
3836 goto out;
3837
3838err_out:
3839 kfree(adapter->vf_mac_list);
3840 adapter->vf_mac_list = NULL;
3841 kfree(adapter->vf_data);
3842 adapter->vf_data = NULL;
3843 adapter->vfs_allocated_count = 0;
3844out:
3845 return err;
3846}
3847
3848#endif
3849/**
3850 * igb_remove_i2c - Cleanup I2C interface
3851 * @adapter: pointer to adapter structure
3852 **/
3853static void igb_remove_i2c(struct igb_adapter *adapter)
3854{
3855 /* free the adapter bus structure */
3856 i2c_del_adapter(&adapter->i2c_adap);
3857}
3858
3859/**
3860 * igb_remove - Device Removal Routine
3861 * @pdev: PCI device information struct
3862 *
3863 * igb_remove is called by the PCI subsystem to alert the driver
3864 * that it should release a PCI device. The could be caused by a
3865 * Hot-Plug event, or because the driver is going to be removed from
3866 * memory.
3867 **/
3868static void igb_remove(struct pci_dev *pdev)
3869{
3870 struct net_device *netdev = pci_get_drvdata(pdev);
3871 struct igb_adapter *adapter = netdev_priv(netdev);
3872 struct e1000_hw *hw = &adapter->hw;
3873
3874 pm_runtime_get_noresume(&pdev->dev);
3875#ifdef CONFIG_IGB_HWMON
3876 igb_sysfs_exit(adapter);
3877#endif
3878 igb_remove_i2c(adapter);
3879 igb_ptp_stop(adapter);
3880 /* The watchdog timer may be rescheduled, so explicitly
3881 * disable watchdog from being rescheduled.
3882 */
3883 set_bit(__IGB_DOWN, &adapter->state);
3884 del_timer_sync(&adapter->watchdog_timer);
3885 del_timer_sync(&adapter->phy_info_timer);
3886
3887 cancel_work_sync(&adapter->reset_task);
3888 cancel_work_sync(&adapter->watchdog_task);
3889
3890#ifdef CONFIG_IGB_DCA
3891 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
3892 dev_info(&pdev->dev, "DCA disabled\n");
3893 dca_remove_requester(&pdev->dev);
3894 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
3895 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
3896 }
3897#endif
3898
3899 /* Release control of h/w to f/w. If f/w is AMT enabled, this
3900 * would have already happened in close and is redundant.
3901 */
3902 igb_release_hw_control(adapter);
3903
3904#ifdef CONFIG_PCI_IOV
3905 igb_disable_sriov(pdev, false);
3906#endif
3907
3908 unregister_netdev(netdev);
3909
3910 igb_clear_interrupt_scheme(adapter);
3911
3912 pci_iounmap(pdev, adapter->io_addr);
3913 if (hw->flash_address)
3914 iounmap(hw->flash_address);
3915 pci_release_mem_regions(pdev);
3916
3917 kfree(adapter->mac_table);
3918 kfree(adapter->shadow_vfta);
3919 free_netdev(netdev);
3920
3921 pci_disable_device(pdev);
3922}
3923
3924/**
3925 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
3926 * @adapter: board private structure to initialize
3927 *
3928 * This function initializes the vf specific data storage and then attempts to
3929 * allocate the VFs. The reason for ordering it this way is because it is much
3930 * mor expensive time wise to disable SR-IOV than it is to allocate and free
3931 * the memory for the VFs.
3932 **/
3933static void igb_probe_vfs(struct igb_adapter *adapter)
3934{
3935#ifdef CONFIG_PCI_IOV
3936 struct pci_dev *pdev = adapter->pdev;
3937 struct e1000_hw *hw = &adapter->hw;
3938
3939 /* Virtualization features not supported on i210 and 82580 family. */
3940 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211) ||
3941 (hw->mac.type == e1000_82580))
3942 return;
3943
3944 /* Of the below we really only want the effect of getting
3945 * IGB_FLAG_HAS_MSIX set (if available), without which
3946 * igb_enable_sriov() has no effect.
3947 */
3948 igb_set_interrupt_capability(adapter, true);
3949 igb_reset_interrupt_capability(adapter);
3950
3951 pci_sriov_set_totalvfs(pdev, 7);
3952 igb_enable_sriov(pdev, max_vfs, false);
3953
3954#endif /* CONFIG_PCI_IOV */
3955}
3956
3957unsigned int igb_get_max_rss_queues(struct igb_adapter *adapter)
3958{
3959 struct e1000_hw *hw = &adapter->hw;
3960 unsigned int max_rss_queues;
3961
3962 /* Determine the maximum number of RSS queues supported. */
3963 switch (hw->mac.type) {
3964 case e1000_i211:
3965 max_rss_queues = IGB_MAX_RX_QUEUES_I211;
3966 break;
3967 case e1000_82575:
3968 case e1000_i210:
3969 max_rss_queues = IGB_MAX_RX_QUEUES_82575;
3970 break;
3971 case e1000_i350:
3972 /* I350 cannot do RSS and SR-IOV at the same time */
3973 if (!!adapter->vfs_allocated_count) {
3974 max_rss_queues = 1;
3975 break;
3976 }
3977 fallthrough;
3978 case e1000_82576:
3979 if (!!adapter->vfs_allocated_count) {
3980 max_rss_queues = 2;
3981 break;
3982 }
3983 fallthrough;
3984 case e1000_82580:
3985 case e1000_i354:
3986 default:
3987 max_rss_queues = IGB_MAX_RX_QUEUES;
3988 break;
3989 }
3990
3991 return max_rss_queues;
3992}
3993
3994static void igb_init_queue_configuration(struct igb_adapter *adapter)
3995{
3996 u32 max_rss_queues;
3997
3998 max_rss_queues = igb_get_max_rss_queues(adapter);
3999 adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4000
4001 igb_set_flag_queue_pairs(adapter, max_rss_queues);
4002}
4003
4004void igb_set_flag_queue_pairs(struct igb_adapter *adapter,
4005 const u32 max_rss_queues)
4006{
4007 struct e1000_hw *hw = &adapter->hw;
4008
4009 /* Determine if we need to pair queues. */
4010 switch (hw->mac.type) {
4011 case e1000_82575:
4012 case e1000_i211:
4013 /* Device supports enough interrupts without queue pairing. */
4014 break;
4015 case e1000_82576:
4016 case e1000_82580:
4017 case e1000_i350:
4018 case e1000_i354:
4019 case e1000_i210:
4020 default:
4021 /* If rss_queues > half of max_rss_queues, pair the queues in
4022 * order to conserve interrupts due to limited supply.
4023 */
4024 if (adapter->rss_queues > (max_rss_queues / 2))
4025 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
4026 else
4027 adapter->flags &= ~IGB_FLAG_QUEUE_PAIRS;
4028 break;
4029 }
4030}
4031
4032/**
4033 * igb_sw_init - Initialize general software structures (struct igb_adapter)
4034 * @adapter: board private structure to initialize
4035 *
4036 * igb_sw_init initializes the Adapter private data structure.
4037 * Fields are initialized based on PCI device information and
4038 * OS network device settings (MTU size).
4039 **/
4040static int igb_sw_init(struct igb_adapter *adapter)
4041{
4042 struct e1000_hw *hw = &adapter->hw;
4043 struct net_device *netdev = adapter->netdev;
4044 struct pci_dev *pdev = adapter->pdev;
4045
4046 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4047
4048 /* set default ring sizes */
4049 adapter->tx_ring_count = IGB_DEFAULT_TXD;
4050 adapter->rx_ring_count = IGB_DEFAULT_RXD;
4051
4052 /* set default ITR values */
4053 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
4054 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
4055
4056 /* set default work limits */
4057 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
4058
4059 adapter->max_frame_size = netdev->mtu + IGB_ETH_PKT_HDR_PAD;
4060 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4061
4062 spin_lock_init(&adapter->nfc_lock);
4063 spin_lock_init(&adapter->stats64_lock);
4064
4065 /* init spinlock to avoid concurrency of VF resources */
4066 spin_lock_init(&adapter->vfs_lock);
4067#ifdef CONFIG_PCI_IOV
4068 switch (hw->mac.type) {
4069 case e1000_82576:
4070 case e1000_i350:
4071 if (max_vfs > 7) {
4072 dev_warn(&pdev->dev,
4073 "Maximum of 7 VFs per PF, using max\n");
4074 max_vfs = adapter->vfs_allocated_count = 7;
4075 } else
4076 adapter->vfs_allocated_count = max_vfs;
4077 if (adapter->vfs_allocated_count)
4078 dev_warn(&pdev->dev,
4079 "Enabling SR-IOV VFs using the module parameter is deprecated - please use the pci sysfs interface.\n");
4080 break;
4081 default:
4082 break;
4083 }
4084#endif /* CONFIG_PCI_IOV */
4085
4086 /* Assume MSI-X interrupts, will be checked during IRQ allocation */
4087 adapter->flags |= IGB_FLAG_HAS_MSIX;
4088
4089 adapter->mac_table = kcalloc(hw->mac.rar_entry_count,
4090 sizeof(struct igb_mac_addr),
4091 GFP_KERNEL);
4092 if (!adapter->mac_table)
4093 return -ENOMEM;
4094
4095 igb_probe_vfs(adapter);
4096
4097 igb_init_queue_configuration(adapter);
4098
4099 /* Setup and initialize a copy of the hw vlan table array */
4100 adapter->shadow_vfta = kcalloc(E1000_VLAN_FILTER_TBL_SIZE, sizeof(u32),
4101 GFP_KERNEL);
4102 if (!adapter->shadow_vfta)
4103 return -ENOMEM;
4104
4105 /* This call may decrease the number of queues */
4106 if (igb_init_interrupt_scheme(adapter, true)) {
4107 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
4108 return -ENOMEM;
4109 }
4110
4111 /* Explicitly disable IRQ since the NIC can be in any state. */
4112 igb_irq_disable(adapter);
4113
4114 if (hw->mac.type >= e1000_i350)
4115 adapter->flags &= ~IGB_FLAG_DMAC;
4116
4117 set_bit(__IGB_DOWN, &adapter->state);
4118 return 0;
4119}
4120
4121/**
4122 * __igb_open - Called when a network interface is made active
4123 * @netdev: network interface device structure
4124 * @resuming: indicates whether we are in a resume call
4125 *
4126 * Returns 0 on success, negative value on failure
4127 *
4128 * The open entry point is called when a network interface is made
4129 * active by the system (IFF_UP). At this point all resources needed
4130 * for transmit and receive operations are allocated, the interrupt
4131 * handler is registered with the OS, the watchdog timer is started,
4132 * and the stack is notified that the interface is ready.
4133 **/
4134static int __igb_open(struct net_device *netdev, bool resuming)
4135{
4136 struct igb_adapter *adapter = netdev_priv(netdev);
4137 struct e1000_hw *hw = &adapter->hw;
4138 struct pci_dev *pdev = adapter->pdev;
4139 int err;
4140 int i;
4141
4142 /* disallow open during test */
4143 if (test_bit(__IGB_TESTING, &adapter->state)) {
4144 WARN_ON(resuming);
4145 return -EBUSY;
4146 }
4147
4148 if (!resuming)
4149 pm_runtime_get_sync(&pdev->dev);
4150
4151 netif_carrier_off(netdev);
4152
4153 /* allocate transmit descriptors */
4154 err = igb_setup_all_tx_resources(adapter);
4155 if (err)
4156 goto err_setup_tx;
4157
4158 /* allocate receive descriptors */
4159 err = igb_setup_all_rx_resources(adapter);
4160 if (err)
4161 goto err_setup_rx;
4162
4163 igb_power_up_link(adapter);
4164
4165 /* before we allocate an interrupt, we must be ready to handle it.
4166 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
4167 * as soon as we call pci_request_irq, so we have to setup our
4168 * clean_rx handler before we do so.
4169 */
4170 igb_configure(adapter);
4171
4172 err = igb_request_irq(adapter);
4173 if (err)
4174 goto err_req_irq;
4175
4176 /* Notify the stack of the actual queue counts. */
4177 err = netif_set_real_num_tx_queues(adapter->netdev,
4178 adapter->num_tx_queues);
4179 if (err)
4180 goto err_set_queues;
4181
4182 err = netif_set_real_num_rx_queues(adapter->netdev,
4183 adapter->num_rx_queues);
4184 if (err)
4185 goto err_set_queues;
4186
4187 /* From here on the code is the same as igb_up() */
4188 clear_bit(__IGB_DOWN, &adapter->state);
4189
4190 for (i = 0; i < adapter->num_q_vectors; i++)
4191 napi_enable(&(adapter->q_vector[i]->napi));
4192
4193 /* Clear any pending interrupts. */
4194 rd32(E1000_TSICR);
4195 rd32(E1000_ICR);
4196
4197 igb_irq_enable(adapter);
4198
4199 /* notify VFs that reset has been completed */
4200 if (adapter->vfs_allocated_count) {
4201 u32 reg_data = rd32(E1000_CTRL_EXT);
4202
4203 reg_data |= E1000_CTRL_EXT_PFRSTD;
4204 wr32(E1000_CTRL_EXT, reg_data);
4205 }
4206
4207 netif_tx_start_all_queues(netdev);
4208
4209 if (!resuming)
4210 pm_runtime_put(&pdev->dev);
4211
4212 /* start the watchdog. */
4213 hw->mac.get_link_status = 1;
4214 schedule_work(&adapter->watchdog_task);
4215
4216 return 0;
4217
4218err_set_queues:
4219 igb_free_irq(adapter);
4220err_req_irq:
4221 igb_release_hw_control(adapter);
4222 igb_power_down_link(adapter);
4223 igb_free_all_rx_resources(adapter);
4224err_setup_rx:
4225 igb_free_all_tx_resources(adapter);
4226err_setup_tx:
4227 igb_reset(adapter);
4228 if (!resuming)
4229 pm_runtime_put(&pdev->dev);
4230
4231 return err;
4232}
4233
4234int igb_open(struct net_device *netdev)
4235{
4236 return __igb_open(netdev, false);
4237}
4238
4239/**
4240 * __igb_close - Disables a network interface
4241 * @netdev: network interface device structure
4242 * @suspending: indicates we are in a suspend call
4243 *
4244 * Returns 0, this is not allowed to fail
4245 *
4246 * The close entry point is called when an interface is de-activated
4247 * by the OS. The hardware is still under the driver's control, but
4248 * needs to be disabled. A global MAC reset is issued to stop the
4249 * hardware, and all transmit and receive resources are freed.
4250 **/
4251static int __igb_close(struct net_device *netdev, bool suspending)
4252{
4253 struct igb_adapter *adapter = netdev_priv(netdev);
4254 struct pci_dev *pdev = adapter->pdev;
4255
4256 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
4257
4258 if (!suspending)
4259 pm_runtime_get_sync(&pdev->dev);
4260
4261 igb_down(adapter);
4262 igb_free_irq(adapter);
4263
4264 igb_free_all_tx_resources(adapter);
4265 igb_free_all_rx_resources(adapter);
4266
4267 if (!suspending)
4268 pm_runtime_put_sync(&pdev->dev);
4269 return 0;
4270}
4271
4272int igb_close(struct net_device *netdev)
4273{
4274 if (netif_device_present(netdev) || netdev->dismantle)
4275 return __igb_close(netdev, false);
4276 return 0;
4277}
4278
4279/**
4280 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
4281 * @tx_ring: tx descriptor ring (for a specific queue) to setup
4282 *
4283 * Return 0 on success, negative on failure
4284 **/
4285int igb_setup_tx_resources(struct igb_ring *tx_ring)
4286{
4287 struct device *dev = tx_ring->dev;
4288 int size;
4289
4290 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
4291
4292 tx_ring->tx_buffer_info = vmalloc(size);
4293 if (!tx_ring->tx_buffer_info)
4294 goto err;
4295
4296 /* round up to nearest 4K */
4297 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
4298 tx_ring->size = ALIGN(tx_ring->size, 4096);
4299
4300 tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
4301 &tx_ring->dma, GFP_KERNEL);
4302 if (!tx_ring->desc)
4303 goto err;
4304
4305 tx_ring->next_to_use = 0;
4306 tx_ring->next_to_clean = 0;
4307
4308 return 0;
4309
4310err:
4311 vfree(tx_ring->tx_buffer_info);
4312 tx_ring->tx_buffer_info = NULL;
4313 dev_err(dev, "Unable to allocate memory for the Tx descriptor ring\n");
4314 return -ENOMEM;
4315}
4316
4317/**
4318 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
4319 * (Descriptors) for all queues
4320 * @adapter: board private structure
4321 *
4322 * Return 0 on success, negative on failure
4323 **/
4324static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
4325{
4326 struct pci_dev *pdev = adapter->pdev;
4327 int i, err = 0;
4328
4329 for (i = 0; i < adapter->num_tx_queues; i++) {
4330 err = igb_setup_tx_resources(adapter->tx_ring[i]);
4331 if (err) {
4332 dev_err(&pdev->dev,
4333 "Allocation for Tx Queue %u failed\n", i);
4334 for (i--; i >= 0; i--)
4335 igb_free_tx_resources(adapter->tx_ring[i]);
4336 break;
4337 }
4338 }
4339
4340 return err;
4341}
4342
4343/**
4344 * igb_setup_tctl - configure the transmit control registers
4345 * @adapter: Board private structure
4346 **/
4347void igb_setup_tctl(struct igb_adapter *adapter)
4348{
4349 struct e1000_hw *hw = &adapter->hw;
4350 u32 tctl;
4351
4352 /* disable queue 0 which is enabled by default on 82575 and 82576 */
4353 wr32(E1000_TXDCTL(0), 0);
4354
4355 /* Program the Transmit Control Register */
4356 tctl = rd32(E1000_TCTL);
4357 tctl &= ~E1000_TCTL_CT;
4358 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
4359 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
4360
4361 igb_config_collision_dist(hw);
4362
4363 /* Enable transmits */
4364 tctl |= E1000_TCTL_EN;
4365
4366 wr32(E1000_TCTL, tctl);
4367}
4368
4369/**
4370 * igb_configure_tx_ring - Configure transmit ring after Reset
4371 * @adapter: board private structure
4372 * @ring: tx ring to configure
4373 *
4374 * Configure a transmit ring after a reset.
4375 **/
4376void igb_configure_tx_ring(struct igb_adapter *adapter,
4377 struct igb_ring *ring)
4378{
4379 struct e1000_hw *hw = &adapter->hw;
4380 u32 txdctl = 0;
4381 u64 tdba = ring->dma;
4382 int reg_idx = ring->reg_idx;
4383
4384 wr32(E1000_TDLEN(reg_idx),
4385 ring->count * sizeof(union e1000_adv_tx_desc));
4386 wr32(E1000_TDBAL(reg_idx),
4387 tdba & 0x00000000ffffffffULL);
4388 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
4389
4390 ring->tail = adapter->io_addr + E1000_TDT(reg_idx);
4391 wr32(E1000_TDH(reg_idx), 0);
4392 writel(0, ring->tail);
4393
4394 txdctl |= IGB_TX_PTHRESH;
4395 txdctl |= IGB_TX_HTHRESH << 8;
4396 txdctl |= IGB_TX_WTHRESH << 16;
4397
4398 /* reinitialize tx_buffer_info */
4399 memset(ring->tx_buffer_info, 0,
4400 sizeof(struct igb_tx_buffer) * ring->count);
4401
4402 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
4403 wr32(E1000_TXDCTL(reg_idx), txdctl);
4404}
4405
4406/**
4407 * igb_configure_tx - Configure transmit Unit after Reset
4408 * @adapter: board private structure
4409 *
4410 * Configure the Tx unit of the MAC after a reset.
4411 **/
4412static void igb_configure_tx(struct igb_adapter *adapter)
4413{
4414 struct e1000_hw *hw = &adapter->hw;
4415 int i;
4416
4417 /* disable the queues */
4418 for (i = 0; i < adapter->num_tx_queues; i++)
4419 wr32(E1000_TXDCTL(adapter->tx_ring[i]->reg_idx), 0);
4420
4421 wrfl();
4422 usleep_range(10000, 20000);
4423
4424 for (i = 0; i < adapter->num_tx_queues; i++)
4425 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
4426}
4427
4428/**
4429 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
4430 * @rx_ring: Rx descriptor ring (for a specific queue) to setup
4431 *
4432 * Returns 0 on success, negative on failure
4433 **/
4434int igb_setup_rx_resources(struct igb_ring *rx_ring)
4435{
4436 struct igb_adapter *adapter = netdev_priv(rx_ring->netdev);
4437 struct device *dev = rx_ring->dev;
4438 int size, res;
4439
4440 /* XDP RX-queue info */
4441 if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
4442 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4443 res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, rx_ring->netdev,
4444 rx_ring->queue_index, 0);
4445 if (res < 0) {
4446 dev_err(dev, "Failed to register xdp_rxq index %u\n",
4447 rx_ring->queue_index);
4448 return res;
4449 }
4450
4451 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
4452
4453 rx_ring->rx_buffer_info = vmalloc(size);
4454 if (!rx_ring->rx_buffer_info)
4455 goto err;
4456
4457 /* Round up to nearest 4K */
4458 rx_ring->size = rx_ring->count * sizeof(union e1000_adv_rx_desc);
4459 rx_ring->size = ALIGN(rx_ring->size, 4096);
4460
4461 rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
4462 &rx_ring->dma, GFP_KERNEL);
4463 if (!rx_ring->desc)
4464 goto err;
4465
4466 rx_ring->next_to_alloc = 0;
4467 rx_ring->next_to_clean = 0;
4468 rx_ring->next_to_use = 0;
4469
4470 rx_ring->xdp_prog = adapter->xdp_prog;
4471
4472 return 0;
4473
4474err:
4475 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4476 vfree(rx_ring->rx_buffer_info);
4477 rx_ring->rx_buffer_info = NULL;
4478 dev_err(dev, "Unable to allocate memory for the Rx descriptor ring\n");
4479 return -ENOMEM;
4480}
4481
4482/**
4483 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
4484 * (Descriptors) for all queues
4485 * @adapter: board private structure
4486 *
4487 * Return 0 on success, negative on failure
4488 **/
4489static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
4490{
4491 struct pci_dev *pdev = adapter->pdev;
4492 int i, err = 0;
4493
4494 for (i = 0; i < adapter->num_rx_queues; i++) {
4495 err = igb_setup_rx_resources(adapter->rx_ring[i]);
4496 if (err) {
4497 dev_err(&pdev->dev,
4498 "Allocation for Rx Queue %u failed\n", i);
4499 for (i--; i >= 0; i--)
4500 igb_free_rx_resources(adapter->rx_ring[i]);
4501 break;
4502 }
4503 }
4504
4505 return err;
4506}
4507
4508/**
4509 * igb_setup_mrqc - configure the multiple receive queue control registers
4510 * @adapter: Board private structure
4511 **/
4512static void igb_setup_mrqc(struct igb_adapter *adapter)
4513{
4514 struct e1000_hw *hw = &adapter->hw;
4515 u32 mrqc, rxcsum;
4516 u32 j, num_rx_queues;
4517 u32 rss_key[10];
4518
4519 netdev_rss_key_fill(rss_key, sizeof(rss_key));
4520 for (j = 0; j < 10; j++)
4521 wr32(E1000_RSSRK(j), rss_key[j]);
4522
4523 num_rx_queues = adapter->rss_queues;
4524
4525 switch (hw->mac.type) {
4526 case e1000_82576:
4527 /* 82576 supports 2 RSS queues for SR-IOV */
4528 if (adapter->vfs_allocated_count)
4529 num_rx_queues = 2;
4530 break;
4531 default:
4532 break;
4533 }
4534
4535 if (adapter->rss_indir_tbl_init != num_rx_queues) {
4536 for (j = 0; j < IGB_RETA_SIZE; j++)
4537 adapter->rss_indir_tbl[j] =
4538 (j * num_rx_queues) / IGB_RETA_SIZE;
4539 adapter->rss_indir_tbl_init = num_rx_queues;
4540 }
4541 igb_write_rss_indir_tbl(adapter);
4542
4543 /* Disable raw packet checksumming so that RSS hash is placed in
4544 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
4545 * offloads as they are enabled by default
4546 */
4547 rxcsum = rd32(E1000_RXCSUM);
4548 rxcsum |= E1000_RXCSUM_PCSD;
4549
4550 if (adapter->hw.mac.type >= e1000_82576)
4551 /* Enable Receive Checksum Offload for SCTP */
4552 rxcsum |= E1000_RXCSUM_CRCOFL;
4553
4554 /* Don't need to set TUOFL or IPOFL, they default to 1 */
4555 wr32(E1000_RXCSUM, rxcsum);
4556
4557 /* Generate RSS hash based on packet types, TCP/UDP
4558 * port numbers and/or IPv4/v6 src and dst addresses
4559 */
4560 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
4561 E1000_MRQC_RSS_FIELD_IPV4_TCP |
4562 E1000_MRQC_RSS_FIELD_IPV6 |
4563 E1000_MRQC_RSS_FIELD_IPV6_TCP |
4564 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
4565
4566 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV4_UDP)
4567 mrqc |= E1000_MRQC_RSS_FIELD_IPV4_UDP;
4568 if (adapter->flags & IGB_FLAG_RSS_FIELD_IPV6_UDP)
4569 mrqc |= E1000_MRQC_RSS_FIELD_IPV6_UDP;
4570
4571 /* If VMDq is enabled then we set the appropriate mode for that, else
4572 * we default to RSS so that an RSS hash is calculated per packet even
4573 * if we are only using one queue
4574 */
4575 if (adapter->vfs_allocated_count) {
4576 if (hw->mac.type > e1000_82575) {
4577 /* Set the default pool for the PF's first queue */
4578 u32 vtctl = rd32(E1000_VT_CTL);
4579
4580 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
4581 E1000_VT_CTL_DISABLE_DEF_POOL);
4582 vtctl |= adapter->vfs_allocated_count <<
4583 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
4584 wr32(E1000_VT_CTL, vtctl);
4585 }
4586 if (adapter->rss_queues > 1)
4587 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_MQ;
4588 else
4589 mrqc |= E1000_MRQC_ENABLE_VMDQ;
4590 } else {
4591 mrqc |= E1000_MRQC_ENABLE_RSS_MQ;
4592 }
4593 igb_vmm_control(adapter);
4594
4595 wr32(E1000_MRQC, mrqc);
4596}
4597
4598/**
4599 * igb_setup_rctl - configure the receive control registers
4600 * @adapter: Board private structure
4601 **/
4602void igb_setup_rctl(struct igb_adapter *adapter)
4603{
4604 struct e1000_hw *hw = &adapter->hw;
4605 u32 rctl;
4606
4607 rctl = rd32(E1000_RCTL);
4608
4609 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
4610 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
4611
4612 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
4613 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
4614
4615 /* enable stripping of CRC. It's unlikely this will break BMC
4616 * redirection as it did with e1000. Newer features require
4617 * that the HW strips the CRC.
4618 */
4619 rctl |= E1000_RCTL_SECRC;
4620
4621 /* disable store bad packets and clear size bits. */
4622 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
4623
4624 /* enable LPE to allow for reception of jumbo frames */
4625 rctl |= E1000_RCTL_LPE;
4626
4627 /* disable queue 0 to prevent tail write w/o re-config */
4628 wr32(E1000_RXDCTL(0), 0);
4629
4630 /* Attention!!! For SR-IOV PF driver operations you must enable
4631 * queue drop for all VF and PF queues to prevent head of line blocking
4632 * if an un-trusted VF does not provide descriptors to hardware.
4633 */
4634 if (adapter->vfs_allocated_count) {
4635 /* set all queue drop enable bits */
4636 wr32(E1000_QDE, ALL_QUEUES);
4637 }
4638
4639 /* This is useful for sniffing bad packets. */
4640 if (adapter->netdev->features & NETIF_F_RXALL) {
4641 /* UPE and MPE will be handled by normal PROMISC logic
4642 * in e1000e_set_rx_mode
4643 */
4644 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
4645 E1000_RCTL_BAM | /* RX All Bcast Pkts */
4646 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
4647
4648 rctl &= ~(E1000_RCTL_DPF | /* Allow filtered pause */
4649 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
4650 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
4651 * and that breaks VLANs.
4652 */
4653 }
4654
4655 wr32(E1000_RCTL, rctl);
4656}
4657
4658static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
4659 int vfn)
4660{
4661 struct e1000_hw *hw = &adapter->hw;
4662 u32 vmolr;
4663
4664 if (size > MAX_JUMBO_FRAME_SIZE)
4665 size = MAX_JUMBO_FRAME_SIZE;
4666
4667 vmolr = rd32(E1000_VMOLR(vfn));
4668 vmolr &= ~E1000_VMOLR_RLPML_MASK;
4669 vmolr |= size | E1000_VMOLR_LPE;
4670 wr32(E1000_VMOLR(vfn), vmolr);
4671
4672 return 0;
4673}
4674
4675static inline void igb_set_vf_vlan_strip(struct igb_adapter *adapter,
4676 int vfn, bool enable)
4677{
4678 struct e1000_hw *hw = &adapter->hw;
4679 u32 val, reg;
4680
4681 if (hw->mac.type < e1000_82576)
4682 return;
4683
4684 if (hw->mac.type == e1000_i350)
4685 reg = E1000_DVMOLR(vfn);
4686 else
4687 reg = E1000_VMOLR(vfn);
4688
4689 val = rd32(reg);
4690 if (enable)
4691 val |= E1000_VMOLR_STRVLAN;
4692 else
4693 val &= ~(E1000_VMOLR_STRVLAN);
4694 wr32(reg, val);
4695}
4696
4697static inline void igb_set_vmolr(struct igb_adapter *adapter,
4698 int vfn, bool aupe)
4699{
4700 struct e1000_hw *hw = &adapter->hw;
4701 u32 vmolr;
4702
4703 /* This register exists only on 82576 and newer so if we are older then
4704 * we should exit and do nothing
4705 */
4706 if (hw->mac.type < e1000_82576)
4707 return;
4708
4709 vmolr = rd32(E1000_VMOLR(vfn));
4710 if (aupe)
4711 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
4712 else
4713 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
4714
4715 /* clear all bits that might not be set */
4716 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
4717
4718 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
4719 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
4720 /* for VMDq only allow the VFs and pool 0 to accept broadcast and
4721 * multicast packets
4722 */
4723 if (vfn <= adapter->vfs_allocated_count)
4724 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
4725
4726 wr32(E1000_VMOLR(vfn), vmolr);
4727}
4728
4729/**
4730 * igb_setup_srrctl - configure the split and replication receive control
4731 * registers
4732 * @adapter: Board private structure
4733 * @ring: receive ring to be configured
4734 **/
4735void igb_setup_srrctl(struct igb_adapter *adapter, struct igb_ring *ring)
4736{
4737 struct e1000_hw *hw = &adapter->hw;
4738 int reg_idx = ring->reg_idx;
4739 u32 srrctl = 0;
4740
4741 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
4742 if (ring_uses_large_buffer(ring))
4743 srrctl |= IGB_RXBUFFER_3072 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4744 else
4745 srrctl |= IGB_RXBUFFER_2048 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
4746 srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
4747 if (hw->mac.type >= e1000_82580)
4748 srrctl |= E1000_SRRCTL_TIMESTAMP;
4749 /* Only set Drop Enable if VFs allocated, or we are supporting multiple
4750 * queues and rx flow control is disabled
4751 */
4752 if (adapter->vfs_allocated_count ||
4753 (!(hw->fc.current_mode & e1000_fc_rx_pause) &&
4754 adapter->num_rx_queues > 1))
4755 srrctl |= E1000_SRRCTL_DROP_EN;
4756
4757 wr32(E1000_SRRCTL(reg_idx), srrctl);
4758}
4759
4760/**
4761 * igb_configure_rx_ring - Configure a receive ring after Reset
4762 * @adapter: board private structure
4763 * @ring: receive ring to be configured
4764 *
4765 * Configure the Rx unit of the MAC after a reset.
4766 **/
4767void igb_configure_rx_ring(struct igb_adapter *adapter,
4768 struct igb_ring *ring)
4769{
4770 struct e1000_hw *hw = &adapter->hw;
4771 union e1000_adv_rx_desc *rx_desc;
4772 u64 rdba = ring->dma;
4773 int reg_idx = ring->reg_idx;
4774 u32 rxdctl = 0;
4775
4776 xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
4777 WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
4778 MEM_TYPE_PAGE_SHARED, NULL));
4779
4780 /* disable the queue */
4781 wr32(E1000_RXDCTL(reg_idx), 0);
4782
4783 /* Set DMA base address registers */
4784 wr32(E1000_RDBAL(reg_idx),
4785 rdba & 0x00000000ffffffffULL);
4786 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
4787 wr32(E1000_RDLEN(reg_idx),
4788 ring->count * sizeof(union e1000_adv_rx_desc));
4789
4790 /* initialize head and tail */
4791 ring->tail = adapter->io_addr + E1000_RDT(reg_idx);
4792 wr32(E1000_RDH(reg_idx), 0);
4793 writel(0, ring->tail);
4794
4795 /* set descriptor configuration */
4796 igb_setup_srrctl(adapter, ring);
4797
4798 /* set filtering for VMDQ pools */
4799 igb_set_vmolr(adapter, reg_idx & 0x7, true);
4800
4801 rxdctl |= IGB_RX_PTHRESH;
4802 rxdctl |= IGB_RX_HTHRESH << 8;
4803 rxdctl |= IGB_RX_WTHRESH << 16;
4804
4805 /* initialize rx_buffer_info */
4806 memset(ring->rx_buffer_info, 0,
4807 sizeof(struct igb_rx_buffer) * ring->count);
4808
4809 /* initialize Rx descriptor 0 */
4810 rx_desc = IGB_RX_DESC(ring, 0);
4811 rx_desc->wb.upper.length = 0;
4812
4813 /* enable receive descriptor fetching */
4814 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
4815 wr32(E1000_RXDCTL(reg_idx), rxdctl);
4816}
4817
4818static void igb_set_rx_buffer_len(struct igb_adapter *adapter,
4819 struct igb_ring *rx_ring)
4820{
4821#if (PAGE_SIZE < 8192)
4822 struct e1000_hw *hw = &adapter->hw;
4823#endif
4824
4825 /* set build_skb and buffer size flags */
4826 clear_ring_build_skb_enabled(rx_ring);
4827 clear_ring_uses_large_buffer(rx_ring);
4828
4829 if (adapter->flags & IGB_FLAG_RX_LEGACY)
4830 return;
4831
4832 set_ring_build_skb_enabled(rx_ring);
4833
4834#if (PAGE_SIZE < 8192)
4835 if (adapter->max_frame_size > IGB_MAX_FRAME_BUILD_SKB ||
4836 rd32(E1000_RCTL) & E1000_RCTL_SBP)
4837 set_ring_uses_large_buffer(rx_ring);
4838#endif
4839}
4840
4841/**
4842 * igb_configure_rx - Configure receive Unit after Reset
4843 * @adapter: board private structure
4844 *
4845 * Configure the Rx unit of the MAC after a reset.
4846 **/
4847static void igb_configure_rx(struct igb_adapter *adapter)
4848{
4849 int i;
4850
4851 /* set the correct pool for the PF default MAC address in entry 0 */
4852 igb_set_default_mac_filter(adapter);
4853
4854 /* Setup the HW Rx Head and Tail Descriptor Pointers and
4855 * the Base and Length of the Rx Descriptor Ring
4856 */
4857 for (i = 0; i < adapter->num_rx_queues; i++) {
4858 struct igb_ring *rx_ring = adapter->rx_ring[i];
4859
4860 igb_set_rx_buffer_len(adapter, rx_ring);
4861 igb_configure_rx_ring(adapter, rx_ring);
4862 }
4863}
4864
4865/**
4866 * igb_free_tx_resources - Free Tx Resources per Queue
4867 * @tx_ring: Tx descriptor ring for a specific queue
4868 *
4869 * Free all transmit software resources
4870 **/
4871void igb_free_tx_resources(struct igb_ring *tx_ring)
4872{
4873 igb_clean_tx_ring(tx_ring);
4874
4875 vfree(tx_ring->tx_buffer_info);
4876 tx_ring->tx_buffer_info = NULL;
4877
4878 /* if not set, then don't free */
4879 if (!tx_ring->desc)
4880 return;
4881
4882 dma_free_coherent(tx_ring->dev, tx_ring->size,
4883 tx_ring->desc, tx_ring->dma);
4884
4885 tx_ring->desc = NULL;
4886}
4887
4888/**
4889 * igb_free_all_tx_resources - Free Tx Resources for All Queues
4890 * @adapter: board private structure
4891 *
4892 * Free all transmit software resources
4893 **/
4894static void igb_free_all_tx_resources(struct igb_adapter *adapter)
4895{
4896 int i;
4897
4898 for (i = 0; i < adapter->num_tx_queues; i++)
4899 if (adapter->tx_ring[i])
4900 igb_free_tx_resources(adapter->tx_ring[i]);
4901}
4902
4903/**
4904 * igb_clean_tx_ring - Free Tx Buffers
4905 * @tx_ring: ring to be cleaned
4906 **/
4907static void igb_clean_tx_ring(struct igb_ring *tx_ring)
4908{
4909 u16 i = tx_ring->next_to_clean;
4910 struct igb_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
4911
4912 while (i != tx_ring->next_to_use) {
4913 union e1000_adv_tx_desc *eop_desc, *tx_desc;
4914
4915 /* Free all the Tx ring sk_buffs or xdp frames */
4916 if (tx_buffer->type == IGB_TYPE_SKB)
4917 dev_kfree_skb_any(tx_buffer->skb);
4918 else
4919 xdp_return_frame(tx_buffer->xdpf);
4920
4921 /* unmap skb header data */
4922 dma_unmap_single(tx_ring->dev,
4923 dma_unmap_addr(tx_buffer, dma),
4924 dma_unmap_len(tx_buffer, len),
4925 DMA_TO_DEVICE);
4926
4927 /* check for eop_desc to determine the end of the packet */
4928 eop_desc = tx_buffer->next_to_watch;
4929 tx_desc = IGB_TX_DESC(tx_ring, i);
4930
4931 /* unmap remaining buffers */
4932 while (tx_desc != eop_desc) {
4933 tx_buffer++;
4934 tx_desc++;
4935 i++;
4936 if (unlikely(i == tx_ring->count)) {
4937 i = 0;
4938 tx_buffer = tx_ring->tx_buffer_info;
4939 tx_desc = IGB_TX_DESC(tx_ring, 0);
4940 }
4941
4942 /* unmap any remaining paged data */
4943 if (dma_unmap_len(tx_buffer, len))
4944 dma_unmap_page(tx_ring->dev,
4945 dma_unmap_addr(tx_buffer, dma),
4946 dma_unmap_len(tx_buffer, len),
4947 DMA_TO_DEVICE);
4948 }
4949
4950 tx_buffer->next_to_watch = NULL;
4951
4952 /* move us one more past the eop_desc for start of next pkt */
4953 tx_buffer++;
4954 i++;
4955 if (unlikely(i == tx_ring->count)) {
4956 i = 0;
4957 tx_buffer = tx_ring->tx_buffer_info;
4958 }
4959 }
4960
4961 /* reset BQL for queue */
4962 netdev_tx_reset_queue(txring_txq(tx_ring));
4963
4964 /* reset next_to_use and next_to_clean */
4965 tx_ring->next_to_use = 0;
4966 tx_ring->next_to_clean = 0;
4967}
4968
4969/**
4970 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
4971 * @adapter: board private structure
4972 **/
4973static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
4974{
4975 int i;
4976
4977 for (i = 0; i < adapter->num_tx_queues; i++)
4978 if (adapter->tx_ring[i])
4979 igb_clean_tx_ring(adapter->tx_ring[i]);
4980}
4981
4982/**
4983 * igb_free_rx_resources - Free Rx Resources
4984 * @rx_ring: ring to clean the resources from
4985 *
4986 * Free all receive software resources
4987 **/
4988void igb_free_rx_resources(struct igb_ring *rx_ring)
4989{
4990 igb_clean_rx_ring(rx_ring);
4991
4992 rx_ring->xdp_prog = NULL;
4993 xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
4994 vfree(rx_ring->rx_buffer_info);
4995 rx_ring->rx_buffer_info = NULL;
4996
4997 /* if not set, then don't free */
4998 if (!rx_ring->desc)
4999 return;
5000
5001 dma_free_coherent(rx_ring->dev, rx_ring->size,
5002 rx_ring->desc, rx_ring->dma);
5003
5004 rx_ring->desc = NULL;
5005}
5006
5007/**
5008 * igb_free_all_rx_resources - Free Rx Resources for All Queues
5009 * @adapter: board private structure
5010 *
5011 * Free all receive software resources
5012 **/
5013static void igb_free_all_rx_resources(struct igb_adapter *adapter)
5014{
5015 int i;
5016
5017 for (i = 0; i < adapter->num_rx_queues; i++)
5018 if (adapter->rx_ring[i])
5019 igb_free_rx_resources(adapter->rx_ring[i]);
5020}
5021
5022/**
5023 * igb_clean_rx_ring - Free Rx Buffers per Queue
5024 * @rx_ring: ring to free buffers from
5025 **/
5026static void igb_clean_rx_ring(struct igb_ring *rx_ring)
5027{
5028 u16 i = rx_ring->next_to_clean;
5029
5030 dev_kfree_skb(rx_ring->skb);
5031 rx_ring->skb = NULL;
5032
5033 /* Free all the Rx ring sk_buffs */
5034 while (i != rx_ring->next_to_alloc) {
5035 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
5036
5037 /* Invalidate cache lines that may have been written to by
5038 * device so that we avoid corrupting memory.
5039 */
5040 dma_sync_single_range_for_cpu(rx_ring->dev,
5041 buffer_info->dma,
5042 buffer_info->page_offset,
5043 igb_rx_bufsz(rx_ring),
5044 DMA_FROM_DEVICE);
5045
5046 /* free resources associated with mapping */
5047 dma_unmap_page_attrs(rx_ring->dev,
5048 buffer_info->dma,
5049 igb_rx_pg_size(rx_ring),
5050 DMA_FROM_DEVICE,
5051 IGB_RX_DMA_ATTR);
5052 __page_frag_cache_drain(buffer_info->page,
5053 buffer_info->pagecnt_bias);
5054
5055 i++;
5056 if (i == rx_ring->count)
5057 i = 0;
5058 }
5059
5060 rx_ring->next_to_alloc = 0;
5061 rx_ring->next_to_clean = 0;
5062 rx_ring->next_to_use = 0;
5063}
5064
5065/**
5066 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
5067 * @adapter: board private structure
5068 **/
5069static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
5070{
5071 int i;
5072
5073 for (i = 0; i < adapter->num_rx_queues; i++)
5074 if (adapter->rx_ring[i])
5075 igb_clean_rx_ring(adapter->rx_ring[i]);
5076}
5077
5078/**
5079 * igb_set_mac - Change the Ethernet Address of the NIC
5080 * @netdev: network interface device structure
5081 * @p: pointer to an address structure
5082 *
5083 * Returns 0 on success, negative on failure
5084 **/
5085static int igb_set_mac(struct net_device *netdev, void *p)
5086{
5087 struct igb_adapter *adapter = netdev_priv(netdev);
5088 struct e1000_hw *hw = &adapter->hw;
5089 struct sockaddr *addr = p;
5090
5091 if (!is_valid_ether_addr(addr->sa_data))
5092 return -EADDRNOTAVAIL;
5093
5094 eth_hw_addr_set(netdev, addr->sa_data);
5095 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
5096
5097 /* set the correct pool for the new PF MAC address in entry 0 */
5098 igb_set_default_mac_filter(adapter);
5099
5100 return 0;
5101}
5102
5103/**
5104 * igb_write_mc_addr_list - write multicast addresses to MTA
5105 * @netdev: network interface device structure
5106 *
5107 * Writes multicast address list to the MTA hash table.
5108 * Returns: -ENOMEM on failure
5109 * 0 on no addresses written
5110 * X on writing X addresses to MTA
5111 **/
5112static int igb_write_mc_addr_list(struct net_device *netdev)
5113{
5114 struct igb_adapter *adapter = netdev_priv(netdev);
5115 struct e1000_hw *hw = &adapter->hw;
5116 struct netdev_hw_addr *ha;
5117 u8 *mta_list;
5118 int i;
5119
5120 if (netdev_mc_empty(netdev)) {
5121 /* nothing to program, so clear mc list */
5122 igb_update_mc_addr_list(hw, NULL, 0);
5123 igb_restore_vf_multicasts(adapter);
5124 return 0;
5125 }
5126
5127 mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
5128 if (!mta_list)
5129 return -ENOMEM;
5130
5131 /* The shared function expects a packed array of only addresses. */
5132 i = 0;
5133 netdev_for_each_mc_addr(ha, netdev)
5134 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
5135
5136 igb_update_mc_addr_list(hw, mta_list, i);
5137 kfree(mta_list);
5138
5139 return netdev_mc_count(netdev);
5140}
5141
5142static int igb_vlan_promisc_enable(struct igb_adapter *adapter)
5143{
5144 struct e1000_hw *hw = &adapter->hw;
5145 u32 i, pf_id;
5146
5147 switch (hw->mac.type) {
5148 case e1000_i210:
5149 case e1000_i211:
5150 case e1000_i350:
5151 /* VLAN filtering needed for VLAN prio filter */
5152 if (adapter->netdev->features & NETIF_F_NTUPLE)
5153 break;
5154 fallthrough;
5155 case e1000_82576:
5156 case e1000_82580:
5157 case e1000_i354:
5158 /* VLAN filtering needed for pool filtering */
5159 if (adapter->vfs_allocated_count)
5160 break;
5161 fallthrough;
5162 default:
5163 return 1;
5164 }
5165
5166 /* We are already in VLAN promisc, nothing to do */
5167 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
5168 return 0;
5169
5170 if (!adapter->vfs_allocated_count)
5171 goto set_vfta;
5172
5173 /* Add PF to all active pools */
5174 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
5175
5176 for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
5177 u32 vlvf = rd32(E1000_VLVF(i));
5178
5179 vlvf |= BIT(pf_id);
5180 wr32(E1000_VLVF(i), vlvf);
5181 }
5182
5183set_vfta:
5184 /* Set all bits in the VLAN filter table array */
5185 for (i = E1000_VLAN_FILTER_TBL_SIZE; i--;)
5186 hw->mac.ops.write_vfta(hw, i, ~0U);
5187
5188 /* Set flag so we don't redo unnecessary work */
5189 adapter->flags |= IGB_FLAG_VLAN_PROMISC;
5190
5191 return 0;
5192}
5193
5194#define VFTA_BLOCK_SIZE 8
5195static void igb_scrub_vfta(struct igb_adapter *adapter, u32 vfta_offset)
5196{
5197 struct e1000_hw *hw = &adapter->hw;
5198 u32 vfta[VFTA_BLOCK_SIZE] = { 0 };
5199 u32 vid_start = vfta_offset * 32;
5200 u32 vid_end = vid_start + (VFTA_BLOCK_SIZE * 32);
5201 u32 i, vid, word, bits, pf_id;
5202
5203 /* guarantee that we don't scrub out management VLAN */
5204 vid = adapter->mng_vlan_id;
5205 if (vid >= vid_start && vid < vid_end)
5206 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5207
5208 if (!adapter->vfs_allocated_count)
5209 goto set_vfta;
5210
5211 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
5212
5213 for (i = E1000_VLVF_ARRAY_SIZE; --i;) {
5214 u32 vlvf = rd32(E1000_VLVF(i));
5215
5216 /* pull VLAN ID from VLVF */
5217 vid = vlvf & VLAN_VID_MASK;
5218
5219 /* only concern ourselves with a certain range */
5220 if (vid < vid_start || vid >= vid_end)
5221 continue;
5222
5223 if (vlvf & E1000_VLVF_VLANID_ENABLE) {
5224 /* record VLAN ID in VFTA */
5225 vfta[(vid - vid_start) / 32] |= BIT(vid % 32);
5226
5227 /* if PF is part of this then continue */
5228 if (test_bit(vid, adapter->active_vlans))
5229 continue;
5230 }
5231
5232 /* remove PF from the pool */
5233 bits = ~BIT(pf_id);
5234 bits &= rd32(E1000_VLVF(i));
5235 wr32(E1000_VLVF(i), bits);
5236 }
5237
5238set_vfta:
5239 /* extract values from active_vlans and write back to VFTA */
5240 for (i = VFTA_BLOCK_SIZE; i--;) {
5241 vid = (vfta_offset + i) * 32;
5242 word = vid / BITS_PER_LONG;
5243 bits = vid % BITS_PER_LONG;
5244
5245 vfta[i] |= adapter->active_vlans[word] >> bits;
5246
5247 hw->mac.ops.write_vfta(hw, vfta_offset + i, vfta[i]);
5248 }
5249}
5250
5251static void igb_vlan_promisc_disable(struct igb_adapter *adapter)
5252{
5253 u32 i;
5254
5255 /* We are not in VLAN promisc, nothing to do */
5256 if (!(adapter->flags & IGB_FLAG_VLAN_PROMISC))
5257 return;
5258
5259 /* Set flag so we don't redo unnecessary work */
5260 adapter->flags &= ~IGB_FLAG_VLAN_PROMISC;
5261
5262 for (i = 0; i < E1000_VLAN_FILTER_TBL_SIZE; i += VFTA_BLOCK_SIZE)
5263 igb_scrub_vfta(adapter, i);
5264}
5265
5266/**
5267 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
5268 * @netdev: network interface device structure
5269 *
5270 * The set_rx_mode entry point is called whenever the unicast or multicast
5271 * address lists or the network interface flags are updated. This routine is
5272 * responsible for configuring the hardware for proper unicast, multicast,
5273 * promiscuous mode, and all-multi behavior.
5274 **/
5275static void igb_set_rx_mode(struct net_device *netdev)
5276{
5277 struct igb_adapter *adapter = netdev_priv(netdev);
5278 struct e1000_hw *hw = &adapter->hw;
5279 unsigned int vfn = adapter->vfs_allocated_count;
5280 u32 rctl = 0, vmolr = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
5281 int count;
5282
5283 /* Check for Promiscuous and All Multicast modes */
5284 if (netdev->flags & IFF_PROMISC) {
5285 rctl |= E1000_RCTL_UPE | E1000_RCTL_MPE;
5286 vmolr |= E1000_VMOLR_MPME;
5287
5288 /* enable use of UTA filter to force packets to default pool */
5289 if (hw->mac.type == e1000_82576)
5290 vmolr |= E1000_VMOLR_ROPE;
5291 } else {
5292 if (netdev->flags & IFF_ALLMULTI) {
5293 rctl |= E1000_RCTL_MPE;
5294 vmolr |= E1000_VMOLR_MPME;
5295 } else {
5296 /* Write addresses to the MTA, if the attempt fails
5297 * then we should just turn on promiscuous mode so
5298 * that we can at least receive multicast traffic
5299 */
5300 count = igb_write_mc_addr_list(netdev);
5301 if (count < 0) {
5302 rctl |= E1000_RCTL_MPE;
5303 vmolr |= E1000_VMOLR_MPME;
5304 } else if (count) {
5305 vmolr |= E1000_VMOLR_ROMPE;
5306 }
5307 }
5308 }
5309
5310 /* Write addresses to available RAR registers, if there is not
5311 * sufficient space to store all the addresses then enable
5312 * unicast promiscuous mode
5313 */
5314 if (__dev_uc_sync(netdev, igb_uc_sync, igb_uc_unsync)) {
5315 rctl |= E1000_RCTL_UPE;
5316 vmolr |= E1000_VMOLR_ROPE;
5317 }
5318
5319 /* enable VLAN filtering by default */
5320 rctl |= E1000_RCTL_VFE;
5321
5322 /* disable VLAN filtering for modes that require it */
5323 if ((netdev->flags & IFF_PROMISC) ||
5324 (netdev->features & NETIF_F_RXALL)) {
5325 /* if we fail to set all rules then just clear VFE */
5326 if (igb_vlan_promisc_enable(adapter))
5327 rctl &= ~E1000_RCTL_VFE;
5328 } else {
5329 igb_vlan_promisc_disable(adapter);
5330 }
5331
5332 /* update state of unicast, multicast, and VLAN filtering modes */
5333 rctl |= rd32(E1000_RCTL) & ~(E1000_RCTL_UPE | E1000_RCTL_MPE |
5334 E1000_RCTL_VFE);
5335 wr32(E1000_RCTL, rctl);
5336
5337#if (PAGE_SIZE < 8192)
5338 if (!adapter->vfs_allocated_count) {
5339 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5340 rlpml = IGB_MAX_FRAME_BUILD_SKB;
5341 }
5342#endif
5343 wr32(E1000_RLPML, rlpml);
5344
5345 /* In order to support SR-IOV and eventually VMDq it is necessary to set
5346 * the VMOLR to enable the appropriate modes. Without this workaround
5347 * we will have issues with VLAN tag stripping not being done for frames
5348 * that are only arriving because we are the default pool
5349 */
5350 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
5351 return;
5352
5353 /* set UTA to appropriate mode */
5354 igb_set_uta(adapter, !!(vmolr & E1000_VMOLR_ROPE));
5355
5356 vmolr |= rd32(E1000_VMOLR(vfn)) &
5357 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
5358
5359 /* enable Rx jumbo frames, restrict as needed to support build_skb */
5360 vmolr &= ~E1000_VMOLR_RLPML_MASK;
5361#if (PAGE_SIZE < 8192)
5362 if (adapter->max_frame_size <= IGB_MAX_FRAME_BUILD_SKB)
5363 vmolr |= IGB_MAX_FRAME_BUILD_SKB;
5364 else
5365#endif
5366 vmolr |= MAX_JUMBO_FRAME_SIZE;
5367 vmolr |= E1000_VMOLR_LPE;
5368
5369 wr32(E1000_VMOLR(vfn), vmolr);
5370
5371 igb_restore_vf_multicasts(adapter);
5372}
5373
5374static void igb_check_wvbr(struct igb_adapter *adapter)
5375{
5376 struct e1000_hw *hw = &adapter->hw;
5377 u32 wvbr = 0;
5378
5379 switch (hw->mac.type) {
5380 case e1000_82576:
5381 case e1000_i350:
5382 wvbr = rd32(E1000_WVBR);
5383 if (!wvbr)
5384 return;
5385 break;
5386 default:
5387 break;
5388 }
5389
5390 adapter->wvbr |= wvbr;
5391}
5392
5393#define IGB_STAGGERED_QUEUE_OFFSET 8
5394
5395static void igb_spoof_check(struct igb_adapter *adapter)
5396{
5397 int j;
5398
5399 if (!adapter->wvbr)
5400 return;
5401
5402 for (j = 0; j < adapter->vfs_allocated_count; j++) {
5403 if (adapter->wvbr & BIT(j) ||
5404 adapter->wvbr & BIT(j + IGB_STAGGERED_QUEUE_OFFSET)) {
5405 dev_warn(&adapter->pdev->dev,
5406 "Spoof event(s) detected on VF %d\n", j);
5407 adapter->wvbr &=
5408 ~(BIT(j) |
5409 BIT(j + IGB_STAGGERED_QUEUE_OFFSET));
5410 }
5411 }
5412}
5413
5414/* Need to wait a few seconds after link up to get diagnostic information from
5415 * the phy
5416 */
5417static void igb_update_phy_info(struct timer_list *t)
5418{
5419 struct igb_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5420 igb_get_phy_info(&adapter->hw);
5421}
5422
5423/**
5424 * igb_has_link - check shared code for link and determine up/down
5425 * @adapter: pointer to driver private info
5426 **/
5427bool igb_has_link(struct igb_adapter *adapter)
5428{
5429 struct e1000_hw *hw = &adapter->hw;
5430 bool link_active = false;
5431
5432 /* get_link_status is set on LSC (link status) interrupt or
5433 * rx sequence error interrupt. get_link_status will stay
5434 * false until the e1000_check_for_link establishes link
5435 * for copper adapters ONLY
5436 */
5437 switch (hw->phy.media_type) {
5438 case e1000_media_type_copper:
5439 if (!hw->mac.get_link_status)
5440 return true;
5441 fallthrough;
5442 case e1000_media_type_internal_serdes:
5443 hw->mac.ops.check_for_link(hw);
5444 link_active = !hw->mac.get_link_status;
5445 break;
5446 default:
5447 case e1000_media_type_unknown:
5448 break;
5449 }
5450
5451 if (((hw->mac.type == e1000_i210) ||
5452 (hw->mac.type == e1000_i211)) &&
5453 (hw->phy.id == I210_I_PHY_ID)) {
5454 if (!netif_carrier_ok(adapter->netdev)) {
5455 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5456 } else if (!(adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)) {
5457 adapter->flags |= IGB_FLAG_NEED_LINK_UPDATE;
5458 adapter->link_check_timeout = jiffies;
5459 }
5460 }
5461
5462 return link_active;
5463}
5464
5465static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
5466{
5467 bool ret = false;
5468 u32 ctrl_ext, thstat;
5469
5470 /* check for thermal sensor event on i350 copper only */
5471 if (hw->mac.type == e1000_i350) {
5472 thstat = rd32(E1000_THSTAT);
5473 ctrl_ext = rd32(E1000_CTRL_EXT);
5474
5475 if ((hw->phy.media_type == e1000_media_type_copper) &&
5476 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII))
5477 ret = !!(thstat & event);
5478 }
5479
5480 return ret;
5481}
5482
5483/**
5484 * igb_check_lvmmc - check for malformed packets received
5485 * and indicated in LVMMC register
5486 * @adapter: pointer to adapter
5487 **/
5488static void igb_check_lvmmc(struct igb_adapter *adapter)
5489{
5490 struct e1000_hw *hw = &adapter->hw;
5491 u32 lvmmc;
5492
5493 lvmmc = rd32(E1000_LVMMC);
5494 if (lvmmc) {
5495 if (unlikely(net_ratelimit())) {
5496 netdev_warn(adapter->netdev,
5497 "malformed Tx packet detected and dropped, LVMMC:0x%08x\n",
5498 lvmmc);
5499 }
5500 }
5501}
5502
5503/**
5504 * igb_watchdog - Timer Call-back
5505 * @t: pointer to timer_list containing our private info pointer
5506 **/
5507static void igb_watchdog(struct timer_list *t)
5508{
5509 struct igb_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5510 /* Do the rest outside of interrupt context */
5511 schedule_work(&adapter->watchdog_task);
5512}
5513
5514static void igb_watchdog_task(struct work_struct *work)
5515{
5516 struct igb_adapter *adapter = container_of(work,
5517 struct igb_adapter,
5518 watchdog_task);
5519 struct e1000_hw *hw = &adapter->hw;
5520 struct e1000_phy_info *phy = &hw->phy;
5521 struct net_device *netdev = adapter->netdev;
5522 u32 link;
5523 int i;
5524 u32 connsw;
5525 u16 phy_data, retry_count = 20;
5526
5527 link = igb_has_link(adapter);
5528
5529 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE) {
5530 if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5531 adapter->flags &= ~IGB_FLAG_NEED_LINK_UPDATE;
5532 else
5533 link = false;
5534 }
5535
5536 /* Force link down if we have fiber to swap to */
5537 if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5538 if (hw->phy.media_type == e1000_media_type_copper) {
5539 connsw = rd32(E1000_CONNSW);
5540 if (!(connsw & E1000_CONNSW_AUTOSENSE_EN))
5541 link = 0;
5542 }
5543 }
5544 if (link) {
5545 /* Perform a reset if the media type changed. */
5546 if (hw->dev_spec._82575.media_changed) {
5547 hw->dev_spec._82575.media_changed = false;
5548 adapter->flags |= IGB_FLAG_MEDIA_RESET;
5549 igb_reset(adapter);
5550 }
5551 /* Cancel scheduled suspend requests. */
5552 pm_runtime_resume(netdev->dev.parent);
5553
5554 if (!netif_carrier_ok(netdev)) {
5555 u32 ctrl;
5556
5557 hw->mac.ops.get_speed_and_duplex(hw,
5558 &adapter->link_speed,
5559 &adapter->link_duplex);
5560
5561 ctrl = rd32(E1000_CTRL);
5562 /* Links status message must follow this format */
5563 netdev_info(netdev,
5564 "igb: %s NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5565 netdev->name,
5566 adapter->link_speed,
5567 adapter->link_duplex == FULL_DUPLEX ?
5568 "Full" : "Half",
5569 (ctrl & E1000_CTRL_TFCE) &&
5570 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
5571 (ctrl & E1000_CTRL_RFCE) ? "RX" :
5572 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
5573
5574 /* disable EEE if enabled */
5575 if ((adapter->flags & IGB_FLAG_EEE) &&
5576 (adapter->link_duplex == HALF_DUPLEX)) {
5577 dev_info(&adapter->pdev->dev,
5578 "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex.\n");
5579 adapter->hw.dev_spec._82575.eee_disable = true;
5580 adapter->flags &= ~IGB_FLAG_EEE;
5581 }
5582
5583 /* check if SmartSpeed worked */
5584 igb_check_downshift(hw);
5585 if (phy->speed_downgraded)
5586 netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5587
5588 /* check for thermal sensor event */
5589 if (igb_thermal_sensor_event(hw,
5590 E1000_THSTAT_LINK_THROTTLE))
5591 netdev_info(netdev, "The network adapter link speed was downshifted because it overheated\n");
5592
5593 /* adjust timeout factor according to speed/duplex */
5594 adapter->tx_timeout_factor = 1;
5595 switch (adapter->link_speed) {
5596 case SPEED_10:
5597 adapter->tx_timeout_factor = 14;
5598 break;
5599 case SPEED_100:
5600 /* maybe add some timeout factor ? */
5601 break;
5602 }
5603
5604 if (adapter->link_speed != SPEED_1000 ||
5605 !hw->phy.ops.read_reg)
5606 goto no_wait;
5607
5608 /* wait for Remote receiver status OK */
5609retry_read_status:
5610 if (!igb_read_phy_reg(hw, PHY_1000T_STATUS,
5611 &phy_data)) {
5612 if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5613 retry_count) {
5614 msleep(100);
5615 retry_count--;
5616 goto retry_read_status;
5617 } else if (!retry_count) {
5618 dev_err(&adapter->pdev->dev, "exceed max 2 second\n");
5619 }
5620 } else {
5621 dev_err(&adapter->pdev->dev, "read 1000Base-T Status Reg\n");
5622 }
5623no_wait:
5624 netif_carrier_on(netdev);
5625
5626 igb_ping_all_vfs(adapter);
5627 igb_check_vf_rate_limit(adapter);
5628
5629 /* link state has changed, schedule phy info update */
5630 if (!test_bit(__IGB_DOWN, &adapter->state))
5631 mod_timer(&adapter->phy_info_timer,
5632 round_jiffies(jiffies + 2 * HZ));
5633 }
5634 } else {
5635 if (netif_carrier_ok(netdev)) {
5636 adapter->link_speed = 0;
5637 adapter->link_duplex = 0;
5638
5639 /* check for thermal sensor event */
5640 if (igb_thermal_sensor_event(hw,
5641 E1000_THSTAT_PWR_DOWN)) {
5642 netdev_err(netdev, "The network adapter was stopped because it overheated\n");
5643 }
5644
5645 /* Links status message must follow this format */
5646 netdev_info(netdev, "igb: %s NIC Link is Down\n",
5647 netdev->name);
5648 netif_carrier_off(netdev);
5649
5650 igb_ping_all_vfs(adapter);
5651
5652 /* link state has changed, schedule phy info update */
5653 if (!test_bit(__IGB_DOWN, &adapter->state))
5654 mod_timer(&adapter->phy_info_timer,
5655 round_jiffies(jiffies + 2 * HZ));
5656
5657 /* link is down, time to check for alternate media */
5658 if (adapter->flags & IGB_FLAG_MAS_ENABLE) {
5659 igb_check_swap_media(adapter);
5660 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5661 schedule_work(&adapter->reset_task);
5662 /* return immediately */
5663 return;
5664 }
5665 }
5666 pm_schedule_suspend(netdev->dev.parent,
5667 MSEC_PER_SEC * 5);
5668
5669 /* also check for alternate media here */
5670 } else if (!netif_carrier_ok(netdev) &&
5671 (adapter->flags & IGB_FLAG_MAS_ENABLE)) {
5672 igb_check_swap_media(adapter);
5673 if (adapter->flags & IGB_FLAG_MEDIA_RESET) {
5674 schedule_work(&adapter->reset_task);
5675 /* return immediately */
5676 return;
5677 }
5678 }
5679 }
5680
5681 spin_lock(&adapter->stats64_lock);
5682 igb_update_stats(adapter);
5683 spin_unlock(&adapter->stats64_lock);
5684
5685 for (i = 0; i < adapter->num_tx_queues; i++) {
5686 struct igb_ring *tx_ring = adapter->tx_ring[i];
5687 if (!netif_carrier_ok(netdev)) {
5688 /* We've lost link, so the controller stops DMA,
5689 * but we've got queued Tx work that's never going
5690 * to get done, so reset controller to flush Tx.
5691 * (Do the reset outside of interrupt context).
5692 */
5693 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
5694 adapter->tx_timeout_count++;
5695 schedule_work(&adapter->reset_task);
5696 /* return immediately since reset is imminent */
5697 return;
5698 }
5699 }
5700
5701 /* Force detection of hung controller every watchdog period */
5702 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5703 }
5704
5705 /* Cause software interrupt to ensure Rx ring is cleaned */
5706 if (adapter->flags & IGB_FLAG_HAS_MSIX) {
5707 u32 eics = 0;
5708
5709 for (i = 0; i < adapter->num_q_vectors; i++)
5710 eics |= adapter->q_vector[i]->eims_value;
5711 wr32(E1000_EICS, eics);
5712 } else {
5713 wr32(E1000_ICS, E1000_ICS_RXDMT0);
5714 }
5715
5716 igb_spoof_check(adapter);
5717 igb_ptp_rx_hang(adapter);
5718 igb_ptp_tx_hang(adapter);
5719
5720 /* Check LVMMC register on i350/i354 only */
5721 if ((adapter->hw.mac.type == e1000_i350) ||
5722 (adapter->hw.mac.type == e1000_i354))
5723 igb_check_lvmmc(adapter);
5724
5725 /* Reset the timer */
5726 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5727 if (adapter->flags & IGB_FLAG_NEED_LINK_UPDATE)
5728 mod_timer(&adapter->watchdog_timer,
5729 round_jiffies(jiffies + HZ));
5730 else
5731 mod_timer(&adapter->watchdog_timer,
5732 round_jiffies(jiffies + 2 * HZ));
5733 }
5734}
5735
5736enum latency_range {
5737 lowest_latency = 0,
5738 low_latency = 1,
5739 bulk_latency = 2,
5740 latency_invalid = 255
5741};
5742
5743/**
5744 * igb_update_ring_itr - update the dynamic ITR value based on packet size
5745 * @q_vector: pointer to q_vector
5746 *
5747 * Stores a new ITR value based on strictly on packet size. This
5748 * algorithm is less sophisticated than that used in igb_update_itr,
5749 * due to the difficulty of synchronizing statistics across multiple
5750 * receive rings. The divisors and thresholds used by this function
5751 * were determined based on theoretical maximum wire speed and testing
5752 * data, in order to minimize response time while increasing bulk
5753 * throughput.
5754 * This functionality is controlled by ethtool's coalescing settings.
5755 * NOTE: This function is called only when operating in a multiqueue
5756 * receive environment.
5757 **/
5758static void igb_update_ring_itr(struct igb_q_vector *q_vector)
5759{
5760 int new_val = q_vector->itr_val;
5761 int avg_wire_size = 0;
5762 struct igb_adapter *adapter = q_vector->adapter;
5763 unsigned int packets;
5764
5765 /* For non-gigabit speeds, just fix the interrupt rate at 4000
5766 * ints/sec - ITR timer value of 120 ticks.
5767 */
5768 if (adapter->link_speed != SPEED_1000) {
5769 new_val = IGB_4K_ITR;
5770 goto set_itr_val;
5771 }
5772
5773 packets = q_vector->rx.total_packets;
5774 if (packets)
5775 avg_wire_size = q_vector->rx.total_bytes / packets;
5776
5777 packets = q_vector->tx.total_packets;
5778 if (packets)
5779 avg_wire_size = max_t(u32, avg_wire_size,
5780 q_vector->tx.total_bytes / packets);
5781
5782 /* if avg_wire_size isn't set no work was done */
5783 if (!avg_wire_size)
5784 goto clear_counts;
5785
5786 /* Add 24 bytes to size to account for CRC, preamble, and gap */
5787 avg_wire_size += 24;
5788
5789 /* Don't starve jumbo frames */
5790 avg_wire_size = min(avg_wire_size, 3000);
5791
5792 /* Give a little boost to mid-size frames */
5793 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
5794 new_val = avg_wire_size / 3;
5795 else
5796 new_val = avg_wire_size / 2;
5797
5798 /* conservative mode (itr 3) eliminates the lowest_latency setting */
5799 if (new_val < IGB_20K_ITR &&
5800 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5801 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5802 new_val = IGB_20K_ITR;
5803
5804set_itr_val:
5805 if (new_val != q_vector->itr_val) {
5806 q_vector->itr_val = new_val;
5807 q_vector->set_itr = 1;
5808 }
5809clear_counts:
5810 q_vector->rx.total_bytes = 0;
5811 q_vector->rx.total_packets = 0;
5812 q_vector->tx.total_bytes = 0;
5813 q_vector->tx.total_packets = 0;
5814}
5815
5816/**
5817 * igb_update_itr - update the dynamic ITR value based on statistics
5818 * @q_vector: pointer to q_vector
5819 * @ring_container: ring info to update the itr for
5820 *
5821 * Stores a new ITR value based on packets and byte
5822 * counts during the last interrupt. The advantage of per interrupt
5823 * computation is faster updates and more accurate ITR for the current
5824 * traffic pattern. Constants in this function were computed
5825 * based on theoretical maximum wire speed and thresholds were set based
5826 * on testing data as well as attempting to minimize response time
5827 * while increasing bulk throughput.
5828 * This functionality is controlled by ethtool's coalescing settings.
5829 * NOTE: These calculations are only valid when operating in a single-
5830 * queue environment.
5831 **/
5832static void igb_update_itr(struct igb_q_vector *q_vector,
5833 struct igb_ring_container *ring_container)
5834{
5835 unsigned int packets = ring_container->total_packets;
5836 unsigned int bytes = ring_container->total_bytes;
5837 u8 itrval = ring_container->itr;
5838
5839 /* no packets, exit with status unchanged */
5840 if (packets == 0)
5841 return;
5842
5843 switch (itrval) {
5844 case lowest_latency:
5845 /* handle TSO and jumbo frames */
5846 if (bytes/packets > 8000)
5847 itrval = bulk_latency;
5848 else if ((packets < 5) && (bytes > 512))
5849 itrval = low_latency;
5850 break;
5851 case low_latency: /* 50 usec aka 20000 ints/s */
5852 if (bytes > 10000) {
5853 /* this if handles the TSO accounting */
5854 if (bytes/packets > 8000)
5855 itrval = bulk_latency;
5856 else if ((packets < 10) || ((bytes/packets) > 1200))
5857 itrval = bulk_latency;
5858 else if ((packets > 35))
5859 itrval = lowest_latency;
5860 } else if (bytes/packets > 2000) {
5861 itrval = bulk_latency;
5862 } else if (packets <= 2 && bytes < 512) {
5863 itrval = lowest_latency;
5864 }
5865 break;
5866 case bulk_latency: /* 250 usec aka 4000 ints/s */
5867 if (bytes > 25000) {
5868 if (packets > 35)
5869 itrval = low_latency;
5870 } else if (bytes < 1500) {
5871 itrval = low_latency;
5872 }
5873 break;
5874 }
5875
5876 /* clear work counters since we have the values we need */
5877 ring_container->total_bytes = 0;
5878 ring_container->total_packets = 0;
5879
5880 /* write updated itr to ring container */
5881 ring_container->itr = itrval;
5882}
5883
5884static void igb_set_itr(struct igb_q_vector *q_vector)
5885{
5886 struct igb_adapter *adapter = q_vector->adapter;
5887 u32 new_itr = q_vector->itr_val;
5888 u8 current_itr = 0;
5889
5890 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
5891 if (adapter->link_speed != SPEED_1000) {
5892 current_itr = 0;
5893 new_itr = IGB_4K_ITR;
5894 goto set_itr_now;
5895 }
5896
5897 igb_update_itr(q_vector, &q_vector->tx);
5898 igb_update_itr(q_vector, &q_vector->rx);
5899
5900 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
5901
5902 /* conservative mode (itr 3) eliminates the lowest_latency setting */
5903 if (current_itr == lowest_latency &&
5904 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
5905 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
5906 current_itr = low_latency;
5907
5908 switch (current_itr) {
5909 /* counts and packets in update_itr are dependent on these numbers */
5910 case lowest_latency:
5911 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
5912 break;
5913 case low_latency:
5914 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
5915 break;
5916 case bulk_latency:
5917 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
5918 break;
5919 default:
5920 break;
5921 }
5922
5923set_itr_now:
5924 if (new_itr != q_vector->itr_val) {
5925 /* this attempts to bias the interrupt rate towards Bulk
5926 * by adding intermediate steps when interrupt rate is
5927 * increasing
5928 */
5929 new_itr = new_itr > q_vector->itr_val ?
5930 max((new_itr * q_vector->itr_val) /
5931 (new_itr + (q_vector->itr_val >> 2)),
5932 new_itr) : new_itr;
5933 /* Don't write the value here; it resets the adapter's
5934 * internal timer, and causes us to delay far longer than
5935 * we should between interrupts. Instead, we write the ITR
5936 * value at the beginning of the next interrupt so the timing
5937 * ends up being correct.
5938 */
5939 q_vector->itr_val = new_itr;
5940 q_vector->set_itr = 1;
5941 }
5942}
5943
5944static void igb_tx_ctxtdesc(struct igb_ring *tx_ring,
5945 struct igb_tx_buffer *first,
5946 u32 vlan_macip_lens, u32 type_tucmd,
5947 u32 mss_l4len_idx)
5948{
5949 struct e1000_adv_tx_context_desc *context_desc;
5950 u16 i = tx_ring->next_to_use;
5951 struct timespec64 ts;
5952
5953 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
5954
5955 i++;
5956 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
5957
5958 /* set bits to identify this as an advanced context descriptor */
5959 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
5960
5961 /* For 82575, context index must be unique per ring. */
5962 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
5963 mss_l4len_idx |= tx_ring->reg_idx << 4;
5964
5965 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
5966 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
5967 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
5968
5969 /* We assume there is always a valid tx time available. Invalid times
5970 * should have been handled by the upper layers.
5971 */
5972 if (tx_ring->launchtime_enable) {
5973 ts = ktime_to_timespec64(first->skb->tstamp);
5974 skb_txtime_consumed(first->skb);
5975 context_desc->seqnum_seed = cpu_to_le32(ts.tv_nsec / 32);
5976 } else {
5977 context_desc->seqnum_seed = 0;
5978 }
5979}
5980
5981static int igb_tso(struct igb_ring *tx_ring,
5982 struct igb_tx_buffer *first,
5983 u8 *hdr_len)
5984{
5985 u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
5986 struct sk_buff *skb = first->skb;
5987 union {
5988 struct iphdr *v4;
5989 struct ipv6hdr *v6;
5990 unsigned char *hdr;
5991 } ip;
5992 union {
5993 struct tcphdr *tcp;
5994 struct udphdr *udp;
5995 unsigned char *hdr;
5996 } l4;
5997 u32 paylen, l4_offset;
5998 int err;
5999
6000 if (skb->ip_summed != CHECKSUM_PARTIAL)
6001 return 0;
6002
6003 if (!skb_is_gso(skb))
6004 return 0;
6005
6006 err = skb_cow_head(skb, 0);
6007 if (err < 0)
6008 return err;
6009
6010 ip.hdr = skb_network_header(skb);
6011 l4.hdr = skb_checksum_start(skb);
6012
6013 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
6014 type_tucmd = (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) ?
6015 E1000_ADVTXD_TUCMD_L4T_UDP : E1000_ADVTXD_TUCMD_L4T_TCP;
6016
6017 /* initialize outer IP header fields */
6018 if (ip.v4->version == 4) {
6019 unsigned char *csum_start = skb_checksum_start(skb);
6020 unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
6021
6022 /* IP header will have to cancel out any data that
6023 * is not a part of the outer IP header
6024 */
6025 ip.v4->check = csum_fold(csum_partial(trans_start,
6026 csum_start - trans_start,
6027 0));
6028 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
6029
6030 ip.v4->tot_len = 0;
6031 first->tx_flags |= IGB_TX_FLAGS_TSO |
6032 IGB_TX_FLAGS_CSUM |
6033 IGB_TX_FLAGS_IPV4;
6034 } else {
6035 ip.v6->payload_len = 0;
6036 first->tx_flags |= IGB_TX_FLAGS_TSO |
6037 IGB_TX_FLAGS_CSUM;
6038 }
6039
6040 /* determine offset of inner transport header */
6041 l4_offset = l4.hdr - skb->data;
6042
6043 /* remove payload length from inner checksum */
6044 paylen = skb->len - l4_offset;
6045 if (type_tucmd & E1000_ADVTXD_TUCMD_L4T_TCP) {
6046 /* compute length of segmentation header */
6047 *hdr_len = (l4.tcp->doff * 4) + l4_offset;
6048 csum_replace_by_diff(&l4.tcp->check,
6049 (__force __wsum)htonl(paylen));
6050 } else {
6051 /* compute length of segmentation header */
6052 *hdr_len = sizeof(*l4.udp) + l4_offset;
6053 csum_replace_by_diff(&l4.udp->check,
6054 (__force __wsum)htonl(paylen));
6055 }
6056
6057 /* update gso size and bytecount with header size */
6058 first->gso_segs = skb_shinfo(skb)->gso_segs;
6059 first->bytecount += (first->gso_segs - 1) * *hdr_len;
6060
6061 /* MSS L4LEN IDX */
6062 mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
6063 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
6064
6065 /* VLAN MACLEN IPLEN */
6066 vlan_macip_lens = l4.hdr - ip.hdr;
6067 vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
6068 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
6069
6070 igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens,
6071 type_tucmd, mss_l4len_idx);
6072
6073 return 1;
6074}
6075
6076static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
6077{
6078 struct sk_buff *skb = first->skb;
6079 u32 vlan_macip_lens = 0;
6080 u32 type_tucmd = 0;
6081
6082 if (skb->ip_summed != CHECKSUM_PARTIAL) {
6083csum_failed:
6084 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN) &&
6085 !tx_ring->launchtime_enable)
6086 return;
6087 goto no_csum;
6088 }
6089
6090 switch (skb->csum_offset) {
6091 case offsetof(struct tcphdr, check):
6092 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
6093 fallthrough;
6094 case offsetof(struct udphdr, check):
6095 break;
6096 case offsetof(struct sctphdr, checksum):
6097 /* validate that this is actually an SCTP request */
6098 if (skb_csum_is_sctp(skb)) {
6099 type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
6100 break;
6101 }
6102 fallthrough;
6103 default:
6104 skb_checksum_help(skb);
6105 goto csum_failed;
6106 }
6107
6108 /* update TX checksum flag */
6109 first->tx_flags |= IGB_TX_FLAGS_CSUM;
6110 vlan_macip_lens = skb_checksum_start_offset(skb) -
6111 skb_network_offset(skb);
6112no_csum:
6113 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
6114 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
6115
6116 igb_tx_ctxtdesc(tx_ring, first, vlan_macip_lens, type_tucmd, 0);
6117}
6118
6119#define IGB_SET_FLAG(_input, _flag, _result) \
6120 ((_flag <= _result) ? \
6121 ((u32)(_input & _flag) * (_result / _flag)) : \
6122 ((u32)(_input & _flag) / (_flag / _result)))
6123
6124static u32 igb_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
6125{
6126 /* set type for advanced descriptor with frame checksum insertion */
6127 u32 cmd_type = E1000_ADVTXD_DTYP_DATA |
6128 E1000_ADVTXD_DCMD_DEXT |
6129 E1000_ADVTXD_DCMD_IFCS;
6130
6131 /* set HW vlan bit if vlan is present */
6132 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_VLAN,
6133 (E1000_ADVTXD_DCMD_VLE));
6134
6135 /* set segmentation bits for TSO */
6136 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSO,
6137 (E1000_ADVTXD_DCMD_TSE));
6138
6139 /* set timestamp bit if present */
6140 cmd_type |= IGB_SET_FLAG(tx_flags, IGB_TX_FLAGS_TSTAMP,
6141 (E1000_ADVTXD_MAC_TSTAMP));
6142
6143 /* insert frame checksum */
6144 cmd_type ^= IGB_SET_FLAG(skb->no_fcs, 1, E1000_ADVTXD_DCMD_IFCS);
6145
6146 return cmd_type;
6147}
6148
6149static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
6150 union e1000_adv_tx_desc *tx_desc,
6151 u32 tx_flags, unsigned int paylen)
6152{
6153 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
6154
6155 /* 82575 requires a unique index per ring */
6156 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
6157 olinfo_status |= tx_ring->reg_idx << 4;
6158
6159 /* insert L4 checksum */
6160 olinfo_status |= IGB_SET_FLAG(tx_flags,
6161 IGB_TX_FLAGS_CSUM,
6162 (E1000_TXD_POPTS_TXSM << 8));
6163
6164 /* insert IPv4 checksum */
6165 olinfo_status |= IGB_SET_FLAG(tx_flags,
6166 IGB_TX_FLAGS_IPV4,
6167 (E1000_TXD_POPTS_IXSM << 8));
6168
6169 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6170}
6171
6172static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
6173{
6174 struct net_device *netdev = tx_ring->netdev;
6175
6176 netif_stop_subqueue(netdev, tx_ring->queue_index);
6177
6178 /* Herbert's original patch had:
6179 * smp_mb__after_netif_stop_queue();
6180 * but since that doesn't exist yet, just open code it.
6181 */
6182 smp_mb();
6183
6184 /* We need to check again in a case another CPU has just
6185 * made room available.
6186 */
6187 if (igb_desc_unused(tx_ring) < size)
6188 return -EBUSY;
6189
6190 /* A reprieve! */
6191 netif_wake_subqueue(netdev, tx_ring->queue_index);
6192
6193 u64_stats_update_begin(&tx_ring->tx_syncp2);
6194 tx_ring->tx_stats.restart_queue2++;
6195 u64_stats_update_end(&tx_ring->tx_syncp2);
6196
6197 return 0;
6198}
6199
6200static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
6201{
6202 if (igb_desc_unused(tx_ring) >= size)
6203 return 0;
6204 return __igb_maybe_stop_tx(tx_ring, size);
6205}
6206
6207static int igb_tx_map(struct igb_ring *tx_ring,
6208 struct igb_tx_buffer *first,
6209 const u8 hdr_len)
6210{
6211 struct sk_buff *skb = first->skb;
6212 struct igb_tx_buffer *tx_buffer;
6213 union e1000_adv_tx_desc *tx_desc;
6214 skb_frag_t *frag;
6215 dma_addr_t dma;
6216 unsigned int data_len, size;
6217 u32 tx_flags = first->tx_flags;
6218 u32 cmd_type = igb_tx_cmd_type(skb, tx_flags);
6219 u16 i = tx_ring->next_to_use;
6220
6221 tx_desc = IGB_TX_DESC(tx_ring, i);
6222
6223 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
6224
6225 size = skb_headlen(skb);
6226 data_len = skb->data_len;
6227
6228 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
6229
6230 tx_buffer = first;
6231
6232 for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
6233 if (dma_mapping_error(tx_ring->dev, dma))
6234 goto dma_error;
6235
6236 /* record length, and DMA address */
6237 dma_unmap_len_set(tx_buffer, len, size);
6238 dma_unmap_addr_set(tx_buffer, dma, dma);
6239
6240 tx_desc->read.buffer_addr = cpu_to_le64(dma);
6241
6242 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
6243 tx_desc->read.cmd_type_len =
6244 cpu_to_le32(cmd_type ^ IGB_MAX_DATA_PER_TXD);
6245
6246 i++;
6247 tx_desc++;
6248 if (i == tx_ring->count) {
6249 tx_desc = IGB_TX_DESC(tx_ring, 0);
6250 i = 0;
6251 }
6252 tx_desc->read.olinfo_status = 0;
6253
6254 dma += IGB_MAX_DATA_PER_TXD;
6255 size -= IGB_MAX_DATA_PER_TXD;
6256
6257 tx_desc->read.buffer_addr = cpu_to_le64(dma);
6258 }
6259
6260 if (likely(!data_len))
6261 break;
6262
6263 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
6264
6265 i++;
6266 tx_desc++;
6267 if (i == tx_ring->count) {
6268 tx_desc = IGB_TX_DESC(tx_ring, 0);
6269 i = 0;
6270 }
6271 tx_desc->read.olinfo_status = 0;
6272
6273 size = skb_frag_size(frag);
6274 data_len -= size;
6275
6276 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
6277 size, DMA_TO_DEVICE);
6278
6279 tx_buffer = &tx_ring->tx_buffer_info[i];
6280 }
6281
6282 /* write last descriptor with RS and EOP bits */
6283 cmd_type |= size | IGB_TXD_DCMD;
6284 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6285
6286 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
6287
6288 /* set the timestamp */
6289 first->time_stamp = jiffies;
6290
6291 skb_tx_timestamp(skb);
6292
6293 /* Force memory writes to complete before letting h/w know there
6294 * are new descriptors to fetch. (Only applicable for weak-ordered
6295 * memory model archs, such as IA-64).
6296 *
6297 * We also need this memory barrier to make certain all of the
6298 * status bits have been updated before next_to_watch is written.
6299 */
6300 dma_wmb();
6301
6302 /* set next_to_watch value indicating a packet is present */
6303 first->next_to_watch = tx_desc;
6304
6305 i++;
6306 if (i == tx_ring->count)
6307 i = 0;
6308
6309 tx_ring->next_to_use = i;
6310
6311 /* Make sure there is space in the ring for the next send. */
6312 igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6313
6314 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
6315 writel(i, tx_ring->tail);
6316 }
6317 return 0;
6318
6319dma_error:
6320 dev_err(tx_ring->dev, "TX DMA map failed\n");
6321 tx_buffer = &tx_ring->tx_buffer_info[i];
6322
6323 /* clear dma mappings for failed tx_buffer_info map */
6324 while (tx_buffer != first) {
6325 if (dma_unmap_len(tx_buffer, len))
6326 dma_unmap_page(tx_ring->dev,
6327 dma_unmap_addr(tx_buffer, dma),
6328 dma_unmap_len(tx_buffer, len),
6329 DMA_TO_DEVICE);
6330 dma_unmap_len_set(tx_buffer, len, 0);
6331
6332 if (i-- == 0)
6333 i += tx_ring->count;
6334 tx_buffer = &tx_ring->tx_buffer_info[i];
6335 }
6336
6337 if (dma_unmap_len(tx_buffer, len))
6338 dma_unmap_single(tx_ring->dev,
6339 dma_unmap_addr(tx_buffer, dma),
6340 dma_unmap_len(tx_buffer, len),
6341 DMA_TO_DEVICE);
6342 dma_unmap_len_set(tx_buffer, len, 0);
6343
6344 dev_kfree_skb_any(tx_buffer->skb);
6345 tx_buffer->skb = NULL;
6346
6347 tx_ring->next_to_use = i;
6348
6349 return -1;
6350}
6351
6352int igb_xmit_xdp_ring(struct igb_adapter *adapter,
6353 struct igb_ring *tx_ring,
6354 struct xdp_frame *xdpf)
6355{
6356 struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
6357 u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
6358 u16 count, i, index = tx_ring->next_to_use;
6359 struct igb_tx_buffer *tx_head = &tx_ring->tx_buffer_info[index];
6360 struct igb_tx_buffer *tx_buffer = tx_head;
6361 union e1000_adv_tx_desc *tx_desc = IGB_TX_DESC(tx_ring, index);
6362 u32 len = xdpf->len, cmd_type, olinfo_status;
6363 void *data = xdpf->data;
6364
6365 count = TXD_USE_COUNT(len);
6366 for (i = 0; i < nr_frags; i++)
6367 count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
6368
6369 if (igb_maybe_stop_tx(tx_ring, count + 3))
6370 return IGB_XDP_CONSUMED;
6371
6372 i = 0;
6373 /* record the location of the first descriptor for this packet */
6374 tx_head->bytecount = xdp_get_frame_len(xdpf);
6375 tx_head->type = IGB_TYPE_XDP;
6376 tx_head->gso_segs = 1;
6377 tx_head->xdpf = xdpf;
6378
6379 olinfo_status = tx_head->bytecount << E1000_ADVTXD_PAYLEN_SHIFT;
6380 /* 82575 requires a unique index per ring */
6381 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
6382 olinfo_status |= tx_ring->reg_idx << 4;
6383 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
6384
6385 for (;;) {
6386 dma_addr_t dma;
6387
6388 dma = dma_map_single(tx_ring->dev, data, len, DMA_TO_DEVICE);
6389 if (dma_mapping_error(tx_ring->dev, dma))
6390 goto unmap;
6391
6392 /* record length, and DMA address */
6393 dma_unmap_len_set(tx_buffer, len, len);
6394 dma_unmap_addr_set(tx_buffer, dma, dma);
6395
6396 /* put descriptor type bits */
6397 cmd_type = E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_DEXT |
6398 E1000_ADVTXD_DCMD_IFCS | len;
6399
6400 tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
6401 tx_desc->read.buffer_addr = cpu_to_le64(dma);
6402
6403 tx_buffer->protocol = 0;
6404
6405 if (++index == tx_ring->count)
6406 index = 0;
6407
6408 if (i == nr_frags)
6409 break;
6410
6411 tx_buffer = &tx_ring->tx_buffer_info[index];
6412 tx_desc = IGB_TX_DESC(tx_ring, index);
6413 tx_desc->read.olinfo_status = 0;
6414
6415 data = skb_frag_address(&sinfo->frags[i]);
6416 len = skb_frag_size(&sinfo->frags[i]);
6417 i++;
6418 }
6419 tx_desc->read.cmd_type_len |= cpu_to_le32(IGB_TXD_DCMD);
6420
6421 netdev_tx_sent_queue(txring_txq(tx_ring), tx_head->bytecount);
6422 /* set the timestamp */
6423 tx_head->time_stamp = jiffies;
6424
6425 /* Avoid any potential race with xdp_xmit and cleanup */
6426 smp_wmb();
6427
6428 /* set next_to_watch value indicating a packet is present */
6429 tx_head->next_to_watch = tx_desc;
6430 tx_ring->next_to_use = index;
6431
6432 /* Make sure there is space in the ring for the next send. */
6433 igb_maybe_stop_tx(tx_ring, DESC_NEEDED);
6434
6435 if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more())
6436 writel(index, tx_ring->tail);
6437
6438 return IGB_XDP_TX;
6439
6440unmap:
6441 for (;;) {
6442 tx_buffer = &tx_ring->tx_buffer_info[index];
6443 if (dma_unmap_len(tx_buffer, len))
6444 dma_unmap_page(tx_ring->dev,
6445 dma_unmap_addr(tx_buffer, dma),
6446 dma_unmap_len(tx_buffer, len),
6447 DMA_TO_DEVICE);
6448 dma_unmap_len_set(tx_buffer, len, 0);
6449 if (tx_buffer == tx_head)
6450 break;
6451
6452 if (!index)
6453 index += tx_ring->count;
6454 index--;
6455 }
6456
6457 return IGB_XDP_CONSUMED;
6458}
6459
6460netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
6461 struct igb_ring *tx_ring)
6462{
6463 struct igb_tx_buffer *first;
6464 int tso;
6465 u32 tx_flags = 0;
6466 unsigned short f;
6467 u16 count = TXD_USE_COUNT(skb_headlen(skb));
6468 __be16 protocol = vlan_get_protocol(skb);
6469 u8 hdr_len = 0;
6470
6471 /* need: 1 descriptor per page * PAGE_SIZE/IGB_MAX_DATA_PER_TXD,
6472 * + 1 desc for skb_headlen/IGB_MAX_DATA_PER_TXD,
6473 * + 2 desc gap to keep tail from touching head,
6474 * + 1 desc for context descriptor,
6475 * otherwise try next time
6476 */
6477 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
6478 count += TXD_USE_COUNT(skb_frag_size(
6479 &skb_shinfo(skb)->frags[f]));
6480
6481 if (igb_maybe_stop_tx(tx_ring, count + 3)) {
6482 /* this is a hard error */
6483 return NETDEV_TX_BUSY;
6484 }
6485
6486 /* record the location of the first descriptor for this packet */
6487 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
6488 first->type = IGB_TYPE_SKB;
6489 first->skb = skb;
6490 first->bytecount = skb->len;
6491 first->gso_segs = 1;
6492
6493 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
6494 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6495
6496 if (adapter->tstamp_config.tx_type == HWTSTAMP_TX_ON &&
6497 !test_and_set_bit_lock(__IGB_PTP_TX_IN_PROGRESS,
6498 &adapter->state)) {
6499 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
6500 tx_flags |= IGB_TX_FLAGS_TSTAMP;
6501
6502 adapter->ptp_tx_skb = skb_get(skb);
6503 adapter->ptp_tx_start = jiffies;
6504 if (adapter->hw.mac.type == e1000_82576)
6505 schedule_work(&adapter->ptp_tx_work);
6506 } else {
6507 adapter->tx_hwtstamp_skipped++;
6508 }
6509 }
6510
6511 if (skb_vlan_tag_present(skb)) {
6512 tx_flags |= IGB_TX_FLAGS_VLAN;
6513 tx_flags |= (skb_vlan_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
6514 }
6515
6516 /* record initial flags and protocol */
6517 first->tx_flags = tx_flags;
6518 first->protocol = protocol;
6519
6520 tso = igb_tso(tx_ring, first, &hdr_len);
6521 if (tso < 0)
6522 goto out_drop;
6523 else if (!tso)
6524 igb_tx_csum(tx_ring, first);
6525
6526 if (igb_tx_map(tx_ring, first, hdr_len))
6527 goto cleanup_tx_tstamp;
6528
6529 return NETDEV_TX_OK;
6530
6531out_drop:
6532 dev_kfree_skb_any(first->skb);
6533 first->skb = NULL;
6534cleanup_tx_tstamp:
6535 if (unlikely(tx_flags & IGB_TX_FLAGS_TSTAMP)) {
6536 struct igb_adapter *adapter = netdev_priv(tx_ring->netdev);
6537
6538 dev_kfree_skb_any(adapter->ptp_tx_skb);
6539 adapter->ptp_tx_skb = NULL;
6540 if (adapter->hw.mac.type == e1000_82576)
6541 cancel_work_sync(&adapter->ptp_tx_work);
6542 clear_bit_unlock(__IGB_PTP_TX_IN_PROGRESS, &adapter->state);
6543 }
6544
6545 return NETDEV_TX_OK;
6546}
6547
6548static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
6549 struct sk_buff *skb)
6550{
6551 unsigned int r_idx = skb->queue_mapping;
6552
6553 if (r_idx >= adapter->num_tx_queues)
6554 r_idx = r_idx % adapter->num_tx_queues;
6555
6556 return adapter->tx_ring[r_idx];
6557}
6558
6559static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
6560 struct net_device *netdev)
6561{
6562 struct igb_adapter *adapter = netdev_priv(netdev);
6563
6564 /* The minimum packet size with TCTL.PSP set is 17 so pad the skb
6565 * in order to meet this minimum size requirement.
6566 */
6567 if (skb_put_padto(skb, 17))
6568 return NETDEV_TX_OK;
6569
6570 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
6571}
6572
6573/**
6574 * igb_tx_timeout - Respond to a Tx Hang
6575 * @netdev: network interface device structure
6576 * @txqueue: number of the Tx queue that hung (unused)
6577 **/
6578static void igb_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
6579{
6580 struct igb_adapter *adapter = netdev_priv(netdev);
6581 struct e1000_hw *hw = &adapter->hw;
6582
6583 /* Do the reset outside of interrupt context */
6584 adapter->tx_timeout_count++;
6585
6586 if (hw->mac.type >= e1000_82580)
6587 hw->dev_spec._82575.global_device_reset = true;
6588
6589 schedule_work(&adapter->reset_task);
6590 wr32(E1000_EICS,
6591 (adapter->eims_enable_mask & ~adapter->eims_other));
6592}
6593
6594static void igb_reset_task(struct work_struct *work)
6595{
6596 struct igb_adapter *adapter;
6597 adapter = container_of(work, struct igb_adapter, reset_task);
6598
6599 rtnl_lock();
6600 /* If we're already down or resetting, just bail */
6601 if (test_bit(__IGB_DOWN, &adapter->state) ||
6602 test_bit(__IGB_RESETTING, &adapter->state)) {
6603 rtnl_unlock();
6604 return;
6605 }
6606
6607 igb_dump(adapter);
6608 netdev_err(adapter->netdev, "Reset adapter\n");
6609 igb_reinit_locked(adapter);
6610 rtnl_unlock();
6611}
6612
6613/**
6614 * igb_get_stats64 - Get System Network Statistics
6615 * @netdev: network interface device structure
6616 * @stats: rtnl_link_stats64 pointer
6617 **/
6618static void igb_get_stats64(struct net_device *netdev,
6619 struct rtnl_link_stats64 *stats)
6620{
6621 struct igb_adapter *adapter = netdev_priv(netdev);
6622
6623 spin_lock(&adapter->stats64_lock);
6624 igb_update_stats(adapter);
6625 memcpy(stats, &adapter->stats64, sizeof(*stats));
6626 spin_unlock(&adapter->stats64_lock);
6627}
6628
6629/**
6630 * igb_change_mtu - Change the Maximum Transfer Unit
6631 * @netdev: network interface device structure
6632 * @new_mtu: new value for maximum frame size
6633 *
6634 * Returns 0 on success, negative on failure
6635 **/
6636static int igb_change_mtu(struct net_device *netdev, int new_mtu)
6637{
6638 struct igb_adapter *adapter = netdev_priv(netdev);
6639 int max_frame = new_mtu + IGB_ETH_PKT_HDR_PAD;
6640
6641 if (adapter->xdp_prog) {
6642 int i;
6643
6644 for (i = 0; i < adapter->num_rx_queues; i++) {
6645 struct igb_ring *ring = adapter->rx_ring[i];
6646
6647 if (max_frame > igb_rx_bufsz(ring)) {
6648 netdev_warn(adapter->netdev,
6649 "Requested MTU size is not supported with XDP. Max frame size is %d\n",
6650 max_frame);
6651 return -EINVAL;
6652 }
6653 }
6654 }
6655
6656 /* adjust max frame to be at least the size of a standard frame */
6657 if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
6658 max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
6659
6660 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
6661 usleep_range(1000, 2000);
6662
6663 /* igb_down has a dependency on max_frame_size */
6664 adapter->max_frame_size = max_frame;
6665
6666 if (netif_running(netdev))
6667 igb_down(adapter);
6668
6669 netdev_dbg(netdev, "changing MTU from %d to %d\n",
6670 netdev->mtu, new_mtu);
6671 netdev->mtu = new_mtu;
6672
6673 if (netif_running(netdev))
6674 igb_up(adapter);
6675 else
6676 igb_reset(adapter);
6677
6678 clear_bit(__IGB_RESETTING, &adapter->state);
6679
6680 return 0;
6681}
6682
6683/**
6684 * igb_update_stats - Update the board statistics counters
6685 * @adapter: board private structure
6686 **/
6687void igb_update_stats(struct igb_adapter *adapter)
6688{
6689 struct rtnl_link_stats64 *net_stats = &adapter->stats64;
6690 struct e1000_hw *hw = &adapter->hw;
6691 struct pci_dev *pdev = adapter->pdev;
6692 u32 reg, mpc;
6693 int i;
6694 u64 bytes, packets;
6695 unsigned int start;
6696 u64 _bytes, _packets;
6697
6698 /* Prevent stats update while adapter is being reset, or if the pci
6699 * connection is down.
6700 */
6701 if (adapter->link_speed == 0)
6702 return;
6703 if (pci_channel_offline(pdev))
6704 return;
6705
6706 bytes = 0;
6707 packets = 0;
6708
6709 rcu_read_lock();
6710 for (i = 0; i < adapter->num_rx_queues; i++) {
6711 struct igb_ring *ring = adapter->rx_ring[i];
6712 u32 rqdpc = rd32(E1000_RQDPC(i));
6713 if (hw->mac.type >= e1000_i210)
6714 wr32(E1000_RQDPC(i), 0);
6715
6716 if (rqdpc) {
6717 ring->rx_stats.drops += rqdpc;
6718 net_stats->rx_fifo_errors += rqdpc;
6719 }
6720
6721 do {
6722 start = u64_stats_fetch_begin(&ring->rx_syncp);
6723 _bytes = ring->rx_stats.bytes;
6724 _packets = ring->rx_stats.packets;
6725 } while (u64_stats_fetch_retry(&ring->rx_syncp, start));
6726 bytes += _bytes;
6727 packets += _packets;
6728 }
6729
6730 net_stats->rx_bytes = bytes;
6731 net_stats->rx_packets = packets;
6732
6733 bytes = 0;
6734 packets = 0;
6735 for (i = 0; i < adapter->num_tx_queues; i++) {
6736 struct igb_ring *ring = adapter->tx_ring[i];
6737 do {
6738 start = u64_stats_fetch_begin(&ring->tx_syncp);
6739 _bytes = ring->tx_stats.bytes;
6740 _packets = ring->tx_stats.packets;
6741 } while (u64_stats_fetch_retry(&ring->tx_syncp, start));
6742 bytes += _bytes;
6743 packets += _packets;
6744 }
6745 net_stats->tx_bytes = bytes;
6746 net_stats->tx_packets = packets;
6747 rcu_read_unlock();
6748
6749 /* read stats registers */
6750 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
6751 adapter->stats.gprc += rd32(E1000_GPRC);
6752 adapter->stats.gorc += rd32(E1000_GORCL);
6753 rd32(E1000_GORCH); /* clear GORCL */
6754 adapter->stats.bprc += rd32(E1000_BPRC);
6755 adapter->stats.mprc += rd32(E1000_MPRC);
6756 adapter->stats.roc += rd32(E1000_ROC);
6757
6758 adapter->stats.prc64 += rd32(E1000_PRC64);
6759 adapter->stats.prc127 += rd32(E1000_PRC127);
6760 adapter->stats.prc255 += rd32(E1000_PRC255);
6761 adapter->stats.prc511 += rd32(E1000_PRC511);
6762 adapter->stats.prc1023 += rd32(E1000_PRC1023);
6763 adapter->stats.prc1522 += rd32(E1000_PRC1522);
6764 adapter->stats.symerrs += rd32(E1000_SYMERRS);
6765 adapter->stats.sec += rd32(E1000_SEC);
6766
6767 mpc = rd32(E1000_MPC);
6768 adapter->stats.mpc += mpc;
6769 net_stats->rx_fifo_errors += mpc;
6770 adapter->stats.scc += rd32(E1000_SCC);
6771 adapter->stats.ecol += rd32(E1000_ECOL);
6772 adapter->stats.mcc += rd32(E1000_MCC);
6773 adapter->stats.latecol += rd32(E1000_LATECOL);
6774 adapter->stats.dc += rd32(E1000_DC);
6775 adapter->stats.rlec += rd32(E1000_RLEC);
6776 adapter->stats.xonrxc += rd32(E1000_XONRXC);
6777 adapter->stats.xontxc += rd32(E1000_XONTXC);
6778 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
6779 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
6780 adapter->stats.fcruc += rd32(E1000_FCRUC);
6781 adapter->stats.gptc += rd32(E1000_GPTC);
6782 adapter->stats.gotc += rd32(E1000_GOTCL);
6783 rd32(E1000_GOTCH); /* clear GOTCL */
6784 adapter->stats.rnbc += rd32(E1000_RNBC);
6785 adapter->stats.ruc += rd32(E1000_RUC);
6786 adapter->stats.rfc += rd32(E1000_RFC);
6787 adapter->stats.rjc += rd32(E1000_RJC);
6788 adapter->stats.tor += rd32(E1000_TORH);
6789 adapter->stats.tot += rd32(E1000_TOTH);
6790 adapter->stats.tpr += rd32(E1000_TPR);
6791
6792 adapter->stats.ptc64 += rd32(E1000_PTC64);
6793 adapter->stats.ptc127 += rd32(E1000_PTC127);
6794 adapter->stats.ptc255 += rd32(E1000_PTC255);
6795 adapter->stats.ptc511 += rd32(E1000_PTC511);
6796 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
6797 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
6798
6799 adapter->stats.mptc += rd32(E1000_MPTC);
6800 adapter->stats.bptc += rd32(E1000_BPTC);
6801
6802 adapter->stats.tpt += rd32(E1000_TPT);
6803 adapter->stats.colc += rd32(E1000_COLC);
6804
6805 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
6806 /* read internal phy specific stats */
6807 reg = rd32(E1000_CTRL_EXT);
6808 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
6809 adapter->stats.rxerrc += rd32(E1000_RXERRC);
6810
6811 /* this stat has invalid values on i210/i211 */
6812 if ((hw->mac.type != e1000_i210) &&
6813 (hw->mac.type != e1000_i211))
6814 adapter->stats.tncrs += rd32(E1000_TNCRS);
6815 }
6816
6817 adapter->stats.tsctc += rd32(E1000_TSCTC);
6818 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
6819
6820 adapter->stats.iac += rd32(E1000_IAC);
6821 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
6822 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
6823 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
6824 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
6825 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
6826 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
6827 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
6828 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
6829
6830 /* Fill out the OS statistics structure */
6831 net_stats->multicast = adapter->stats.mprc;
6832 net_stats->collisions = adapter->stats.colc;
6833
6834 /* Rx Errors */
6835
6836 /* RLEC on some newer hardware can be incorrect so build
6837 * our own version based on RUC and ROC
6838 */
6839 net_stats->rx_errors = adapter->stats.rxerrc +
6840 adapter->stats.crcerrs + adapter->stats.algnerrc +
6841 adapter->stats.ruc + adapter->stats.roc +
6842 adapter->stats.cexterr;
6843 net_stats->rx_length_errors = adapter->stats.ruc +
6844 adapter->stats.roc;
6845 net_stats->rx_crc_errors = adapter->stats.crcerrs;
6846 net_stats->rx_frame_errors = adapter->stats.algnerrc;
6847 net_stats->rx_missed_errors = adapter->stats.mpc;
6848
6849 /* Tx Errors */
6850 net_stats->tx_errors = adapter->stats.ecol +
6851 adapter->stats.latecol;
6852 net_stats->tx_aborted_errors = adapter->stats.ecol;
6853 net_stats->tx_window_errors = adapter->stats.latecol;
6854 net_stats->tx_carrier_errors = adapter->stats.tncrs;
6855
6856 /* Tx Dropped needs to be maintained elsewhere */
6857
6858 /* Management Stats */
6859 adapter->stats.mgptc += rd32(E1000_MGTPTC);
6860 adapter->stats.mgprc += rd32(E1000_MGTPRC);
6861 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
6862
6863 /* OS2BMC Stats */
6864 reg = rd32(E1000_MANC);
6865 if (reg & E1000_MANC_EN_BMC2OS) {
6866 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
6867 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
6868 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
6869 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
6870 }
6871}
6872
6873static void igb_perout(struct igb_adapter *adapter, int tsintr_tt)
6874{
6875 int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_PEROUT, tsintr_tt);
6876 struct e1000_hw *hw = &adapter->hw;
6877 struct timespec64 ts;
6878 u32 tsauxc;
6879
6880 if (pin < 0 || pin >= IGB_N_SDP)
6881 return;
6882
6883 spin_lock(&adapter->tmreg_lock);
6884
6885 if (hw->mac.type == e1000_82580 ||
6886 hw->mac.type == e1000_i354 ||
6887 hw->mac.type == e1000_i350) {
6888 s64 ns = timespec64_to_ns(&adapter->perout[tsintr_tt].period);
6889 u32 systiml, systimh, level_mask, level, rem;
6890 u64 systim, now;
6891
6892 /* read systim registers in sequence */
6893 rd32(E1000_SYSTIMR);
6894 systiml = rd32(E1000_SYSTIML);
6895 systimh = rd32(E1000_SYSTIMH);
6896 systim = (((u64)(systimh & 0xFF)) << 32) | ((u64)systiml);
6897 now = timecounter_cyc2time(&adapter->tc, systim);
6898
6899 if (pin < 2) {
6900 level_mask = (tsintr_tt == 1) ? 0x80000 : 0x40000;
6901 level = (rd32(E1000_CTRL) & level_mask) ? 1 : 0;
6902 } else {
6903 level_mask = (tsintr_tt == 1) ? 0x80 : 0x40;
6904 level = (rd32(E1000_CTRL_EXT) & level_mask) ? 1 : 0;
6905 }
6906
6907 div_u64_rem(now, ns, &rem);
6908 systim = systim + (ns - rem);
6909
6910 /* synchronize pin level with rising/falling edges */
6911 div_u64_rem(now, ns << 1, &rem);
6912 if (rem < ns) {
6913 /* first half of period */
6914 if (level == 0) {
6915 /* output is already low, skip this period */
6916 systim += ns;
6917 pr_notice("igb: periodic output on %s missed falling edge\n",
6918 adapter->sdp_config[pin].name);
6919 }
6920 } else {
6921 /* second half of period */
6922 if (level == 1) {
6923 /* output is already high, skip this period */
6924 systim += ns;
6925 pr_notice("igb: periodic output on %s missed rising edge\n",
6926 adapter->sdp_config[pin].name);
6927 }
6928 }
6929
6930 /* for this chip family tv_sec is the upper part of the binary value,
6931 * so not seconds
6932 */
6933 ts.tv_nsec = (u32)systim;
6934 ts.tv_sec = ((u32)(systim >> 32)) & 0xFF;
6935 } else {
6936 ts = timespec64_add(adapter->perout[tsintr_tt].start,
6937 adapter->perout[tsintr_tt].period);
6938 }
6939
6940 /* u32 conversion of tv_sec is safe until y2106 */
6941 wr32((tsintr_tt == 1) ? E1000_TRGTTIML1 : E1000_TRGTTIML0, ts.tv_nsec);
6942 wr32((tsintr_tt == 1) ? E1000_TRGTTIMH1 : E1000_TRGTTIMH0, (u32)ts.tv_sec);
6943 tsauxc = rd32(E1000_TSAUXC);
6944 tsauxc |= TSAUXC_EN_TT0;
6945 wr32(E1000_TSAUXC, tsauxc);
6946 adapter->perout[tsintr_tt].start = ts;
6947
6948 spin_unlock(&adapter->tmreg_lock);
6949}
6950
6951static void igb_extts(struct igb_adapter *adapter, int tsintr_tt)
6952{
6953 int pin = ptp_find_pin(adapter->ptp_clock, PTP_PF_EXTTS, tsintr_tt);
6954 int auxstmpl = (tsintr_tt == 1) ? E1000_AUXSTMPL1 : E1000_AUXSTMPL0;
6955 int auxstmph = (tsintr_tt == 1) ? E1000_AUXSTMPH1 : E1000_AUXSTMPH0;
6956 struct e1000_hw *hw = &adapter->hw;
6957 struct ptp_clock_event event;
6958 struct timespec64 ts;
6959 unsigned long flags;
6960
6961 if (pin < 0 || pin >= IGB_N_SDP)
6962 return;
6963
6964 if (hw->mac.type == e1000_82580 ||
6965 hw->mac.type == e1000_i354 ||
6966 hw->mac.type == e1000_i350) {
6967 u64 ns = rd32(auxstmpl);
6968
6969 ns += ((u64)(rd32(auxstmph) & 0xFF)) << 32;
6970 spin_lock_irqsave(&adapter->tmreg_lock, flags);
6971 ns = timecounter_cyc2time(&adapter->tc, ns);
6972 spin_unlock_irqrestore(&adapter->tmreg_lock, flags);
6973 ts = ns_to_timespec64(ns);
6974 } else {
6975 ts.tv_nsec = rd32(auxstmpl);
6976 ts.tv_sec = rd32(auxstmph);
6977 }
6978
6979 event.type = PTP_CLOCK_EXTTS;
6980 event.index = tsintr_tt;
6981 event.timestamp = ts.tv_sec * 1000000000ULL + ts.tv_nsec;
6982 ptp_clock_event(adapter->ptp_clock, &event);
6983}
6984
6985static void igb_tsync_interrupt(struct igb_adapter *adapter)
6986{
6987 struct e1000_hw *hw = &adapter->hw;
6988 u32 ack = 0, tsicr = rd32(E1000_TSICR);
6989 struct ptp_clock_event event;
6990
6991 if (tsicr & TSINTR_SYS_WRAP) {
6992 event.type = PTP_CLOCK_PPS;
6993 if (adapter->ptp_caps.pps)
6994 ptp_clock_event(adapter->ptp_clock, &event);
6995 ack |= TSINTR_SYS_WRAP;
6996 }
6997
6998 if (tsicr & E1000_TSICR_TXTS) {
6999 /* retrieve hardware timestamp */
7000 schedule_work(&adapter->ptp_tx_work);
7001 ack |= E1000_TSICR_TXTS;
7002 }
7003
7004 if (tsicr & TSINTR_TT0) {
7005 igb_perout(adapter, 0);
7006 ack |= TSINTR_TT0;
7007 }
7008
7009 if (tsicr & TSINTR_TT1) {
7010 igb_perout(adapter, 1);
7011 ack |= TSINTR_TT1;
7012 }
7013
7014 if (tsicr & TSINTR_AUTT0) {
7015 igb_extts(adapter, 0);
7016 ack |= TSINTR_AUTT0;
7017 }
7018
7019 if (tsicr & TSINTR_AUTT1) {
7020 igb_extts(adapter, 1);
7021 ack |= TSINTR_AUTT1;
7022 }
7023
7024 /* acknowledge the interrupts */
7025 wr32(E1000_TSICR, ack);
7026}
7027
7028static irqreturn_t igb_msix_other(int irq, void *data)
7029{
7030 struct igb_adapter *adapter = data;
7031 struct e1000_hw *hw = &adapter->hw;
7032 u32 icr = rd32(E1000_ICR);
7033 /* reading ICR causes bit 31 of EICR to be cleared */
7034
7035 if (icr & E1000_ICR_DRSTA)
7036 schedule_work(&adapter->reset_task);
7037
7038 if (icr & E1000_ICR_DOUTSYNC) {
7039 /* HW is reporting DMA is out of sync */
7040 adapter->stats.doosync++;
7041 /* The DMA Out of Sync is also indication of a spoof event
7042 * in IOV mode. Check the Wrong VM Behavior register to
7043 * see if it is really a spoof event.
7044 */
7045 igb_check_wvbr(adapter);
7046 }
7047
7048 /* Check for a mailbox event */
7049 if (icr & E1000_ICR_VMMB)
7050 igb_msg_task(adapter);
7051
7052 if (icr & E1000_ICR_LSC) {
7053 hw->mac.get_link_status = 1;
7054 /* guard against interrupt when we're going down */
7055 if (!test_bit(__IGB_DOWN, &adapter->state))
7056 mod_timer(&adapter->watchdog_timer, jiffies + 1);
7057 }
7058
7059 if (icr & E1000_ICR_TS)
7060 igb_tsync_interrupt(adapter);
7061
7062 wr32(E1000_EIMS, adapter->eims_other);
7063
7064 return IRQ_HANDLED;
7065}
7066
7067static void igb_write_itr(struct igb_q_vector *q_vector)
7068{
7069 struct igb_adapter *adapter = q_vector->adapter;
7070 u32 itr_val = q_vector->itr_val & 0x7FFC;
7071
7072 if (!q_vector->set_itr)
7073 return;
7074
7075 if (!itr_val)
7076 itr_val = 0x4;
7077
7078 if (adapter->hw.mac.type == e1000_82575)
7079 itr_val |= itr_val << 16;
7080 else
7081 itr_val |= E1000_EITR_CNT_IGNR;
7082
7083 writel(itr_val, q_vector->itr_register);
7084 q_vector->set_itr = 0;
7085}
7086
7087static irqreturn_t igb_msix_ring(int irq, void *data)
7088{
7089 struct igb_q_vector *q_vector = data;
7090
7091 /* Write the ITR value calculated from the previous interrupt. */
7092 igb_write_itr(q_vector);
7093
7094 napi_schedule(&q_vector->napi);
7095
7096 return IRQ_HANDLED;
7097}
7098
7099#ifdef CONFIG_IGB_DCA
7100static void igb_update_tx_dca(struct igb_adapter *adapter,
7101 struct igb_ring *tx_ring,
7102 int cpu)
7103{
7104 struct e1000_hw *hw = &adapter->hw;
7105 u32 txctrl = dca3_get_tag(tx_ring->dev, cpu);
7106
7107 if (hw->mac.type != e1000_82575)
7108 txctrl <<= E1000_DCA_TXCTRL_CPUID_SHIFT;
7109
7110 /* We can enable relaxed ordering for reads, but not writes when
7111 * DCA is enabled. This is due to a known issue in some chipsets
7112 * which will cause the DCA tag to be cleared.
7113 */
7114 txctrl |= E1000_DCA_TXCTRL_DESC_RRO_EN |
7115 E1000_DCA_TXCTRL_DATA_RRO_EN |
7116 E1000_DCA_TXCTRL_DESC_DCA_EN;
7117
7118 wr32(E1000_DCA_TXCTRL(tx_ring->reg_idx), txctrl);
7119}
7120
7121static void igb_update_rx_dca(struct igb_adapter *adapter,
7122 struct igb_ring *rx_ring,
7123 int cpu)
7124{
7125 struct e1000_hw *hw = &adapter->hw;
7126 u32 rxctrl = dca3_get_tag(&adapter->pdev->dev, cpu);
7127
7128 if (hw->mac.type != e1000_82575)
7129 rxctrl <<= E1000_DCA_RXCTRL_CPUID_SHIFT;
7130
7131 /* We can enable relaxed ordering for reads, but not writes when
7132 * DCA is enabled. This is due to a known issue in some chipsets
7133 * which will cause the DCA tag to be cleared.
7134 */
7135 rxctrl |= E1000_DCA_RXCTRL_DESC_RRO_EN |
7136 E1000_DCA_RXCTRL_DESC_DCA_EN;
7137
7138 wr32(E1000_DCA_RXCTRL(rx_ring->reg_idx), rxctrl);
7139}
7140
7141static void igb_update_dca(struct igb_q_vector *q_vector)
7142{
7143 struct igb_adapter *adapter = q_vector->adapter;
7144 int cpu = get_cpu();
7145
7146 if (q_vector->cpu == cpu)
7147 goto out_no_update;
7148
7149 if (q_vector->tx.ring)
7150 igb_update_tx_dca(adapter, q_vector->tx.ring, cpu);
7151
7152 if (q_vector->rx.ring)
7153 igb_update_rx_dca(adapter, q_vector->rx.ring, cpu);
7154
7155 q_vector->cpu = cpu;
7156out_no_update:
7157 put_cpu();
7158}
7159
7160static void igb_setup_dca(struct igb_adapter *adapter)
7161{
7162 struct e1000_hw *hw = &adapter->hw;
7163 int i;
7164
7165 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
7166 return;
7167
7168 /* Always use CB2 mode, difference is masked in the CB driver. */
7169 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
7170
7171 for (i = 0; i < adapter->num_q_vectors; i++) {
7172 adapter->q_vector[i]->cpu = -1;
7173 igb_update_dca(adapter->q_vector[i]);
7174 }
7175}
7176
7177static int __igb_notify_dca(struct device *dev, void *data)
7178{
7179 struct net_device *netdev = dev_get_drvdata(dev);
7180 struct igb_adapter *adapter = netdev_priv(netdev);
7181 struct pci_dev *pdev = adapter->pdev;
7182 struct e1000_hw *hw = &adapter->hw;
7183 unsigned long event = *(unsigned long *)data;
7184
7185 switch (event) {
7186 case DCA_PROVIDER_ADD:
7187 /* if already enabled, don't do it again */
7188 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
7189 break;
7190 if (dca_add_requester(dev) == 0) {
7191 adapter->flags |= IGB_FLAG_DCA_ENABLED;
7192 dev_info(&pdev->dev, "DCA enabled\n");
7193 igb_setup_dca(adapter);
7194 break;
7195 }
7196 fallthrough; /* since DCA is disabled. */
7197 case DCA_PROVIDER_REMOVE:
7198 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
7199 /* without this a class_device is left
7200 * hanging around in the sysfs model
7201 */
7202 dca_remove_requester(dev);
7203 dev_info(&pdev->dev, "DCA disabled\n");
7204 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
7205 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
7206 }
7207 break;
7208 }
7209
7210 return 0;
7211}
7212
7213static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
7214 void *p)
7215{
7216 int ret_val;
7217
7218 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
7219 __igb_notify_dca);
7220
7221 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
7222}
7223#endif /* CONFIG_IGB_DCA */
7224
7225#ifdef CONFIG_PCI_IOV
7226static int igb_vf_configure(struct igb_adapter *adapter, int vf)
7227{
7228 unsigned char mac_addr[ETH_ALEN];
7229
7230 eth_zero_addr(mac_addr);
7231 igb_set_vf_mac(adapter, vf, mac_addr);
7232
7233 /* By default spoof check is enabled for all VFs */
7234 adapter->vf_data[vf].spoofchk_enabled = true;
7235
7236 /* By default VFs are not trusted */
7237 adapter->vf_data[vf].trusted = false;
7238
7239 return 0;
7240}
7241
7242#endif
7243static void igb_ping_all_vfs(struct igb_adapter *adapter)
7244{
7245 struct e1000_hw *hw = &adapter->hw;
7246 u32 ping;
7247 int i;
7248
7249 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
7250 ping = E1000_PF_CONTROL_MSG;
7251 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
7252 ping |= E1000_VT_MSGTYPE_CTS;
7253 igb_write_mbx(hw, &ping, 1, i);
7254 }
7255}
7256
7257static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7258{
7259 struct e1000_hw *hw = &adapter->hw;
7260 u32 vmolr = rd32(E1000_VMOLR(vf));
7261 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7262
7263 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
7264 IGB_VF_FLAG_MULTI_PROMISC);
7265 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
7266
7267 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
7268 vmolr |= E1000_VMOLR_MPME;
7269 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
7270 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
7271 } else {
7272 /* if we have hashes and we are clearing a multicast promisc
7273 * flag we need to write the hashes to the MTA as this step
7274 * was previously skipped
7275 */
7276 if (vf_data->num_vf_mc_hashes > 30) {
7277 vmolr |= E1000_VMOLR_MPME;
7278 } else if (vf_data->num_vf_mc_hashes) {
7279 int j;
7280
7281 vmolr |= E1000_VMOLR_ROMPE;
7282 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
7283 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
7284 }
7285 }
7286
7287 wr32(E1000_VMOLR(vf), vmolr);
7288
7289 /* there are flags left unprocessed, likely not supported */
7290 if (*msgbuf & E1000_VT_MSGINFO_MASK)
7291 return -EINVAL;
7292
7293 return 0;
7294}
7295
7296static int igb_set_vf_multicasts(struct igb_adapter *adapter,
7297 u32 *msgbuf, u32 vf)
7298{
7299 int n = FIELD_GET(E1000_VT_MSGINFO_MASK, msgbuf[0]);
7300 u16 *hash_list = (u16 *)&msgbuf[1];
7301 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7302 int i;
7303
7304 /* salt away the number of multicast addresses assigned
7305 * to this VF for later use to restore when the PF multi cast
7306 * list changes
7307 */
7308 vf_data->num_vf_mc_hashes = n;
7309
7310 /* only up to 30 hash values supported */
7311 if (n > 30)
7312 n = 30;
7313
7314 /* store the hashes for later use */
7315 for (i = 0; i < n; i++)
7316 vf_data->vf_mc_hashes[i] = hash_list[i];
7317
7318 /* Flush and reset the mta with the new values */
7319 igb_set_rx_mode(adapter->netdev);
7320
7321 return 0;
7322}
7323
7324static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
7325{
7326 struct e1000_hw *hw = &adapter->hw;
7327 struct vf_data_storage *vf_data;
7328 int i, j;
7329
7330 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7331 u32 vmolr = rd32(E1000_VMOLR(i));
7332
7333 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
7334
7335 vf_data = &adapter->vf_data[i];
7336
7337 if ((vf_data->num_vf_mc_hashes > 30) ||
7338 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
7339 vmolr |= E1000_VMOLR_MPME;
7340 } else if (vf_data->num_vf_mc_hashes) {
7341 vmolr |= E1000_VMOLR_ROMPE;
7342 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
7343 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
7344 }
7345 wr32(E1000_VMOLR(i), vmolr);
7346 }
7347}
7348
7349static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
7350{
7351 struct e1000_hw *hw = &adapter->hw;
7352 u32 pool_mask, vlvf_mask, i;
7353
7354 /* create mask for VF and other pools */
7355 pool_mask = E1000_VLVF_POOLSEL_MASK;
7356 vlvf_mask = BIT(E1000_VLVF_POOLSEL_SHIFT + vf);
7357
7358 /* drop PF from pool bits */
7359 pool_mask &= ~BIT(E1000_VLVF_POOLSEL_SHIFT +
7360 adapter->vfs_allocated_count);
7361
7362 /* Find the vlan filter for this id */
7363 for (i = E1000_VLVF_ARRAY_SIZE; i--;) {
7364 u32 vlvf = rd32(E1000_VLVF(i));
7365 u32 vfta_mask, vid, vfta;
7366
7367 /* remove the vf from the pool */
7368 if (!(vlvf & vlvf_mask))
7369 continue;
7370
7371 /* clear out bit from VLVF */
7372 vlvf ^= vlvf_mask;
7373
7374 /* if other pools are present, just remove ourselves */
7375 if (vlvf & pool_mask)
7376 goto update_vlvfb;
7377
7378 /* if PF is present, leave VFTA */
7379 if (vlvf & E1000_VLVF_POOLSEL_MASK)
7380 goto update_vlvf;
7381
7382 vid = vlvf & E1000_VLVF_VLANID_MASK;
7383 vfta_mask = BIT(vid % 32);
7384
7385 /* clear bit from VFTA */
7386 vfta = adapter->shadow_vfta[vid / 32];
7387 if (vfta & vfta_mask)
7388 hw->mac.ops.write_vfta(hw, vid / 32, vfta ^ vfta_mask);
7389update_vlvf:
7390 /* clear pool selection enable */
7391 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
7392 vlvf &= E1000_VLVF_POOLSEL_MASK;
7393 else
7394 vlvf = 0;
7395update_vlvfb:
7396 /* clear pool bits */
7397 wr32(E1000_VLVF(i), vlvf);
7398 }
7399}
7400
7401static int igb_find_vlvf_entry(struct e1000_hw *hw, u32 vlan)
7402{
7403 u32 vlvf;
7404 int idx;
7405
7406 /* short cut the special case */
7407 if (vlan == 0)
7408 return 0;
7409
7410 /* Search for the VLAN id in the VLVF entries */
7411 for (idx = E1000_VLVF_ARRAY_SIZE; --idx;) {
7412 vlvf = rd32(E1000_VLVF(idx));
7413 if ((vlvf & VLAN_VID_MASK) == vlan)
7414 break;
7415 }
7416
7417 return idx;
7418}
7419
7420static void igb_update_pf_vlvf(struct igb_adapter *adapter, u32 vid)
7421{
7422 struct e1000_hw *hw = &adapter->hw;
7423 u32 bits, pf_id;
7424 int idx;
7425
7426 idx = igb_find_vlvf_entry(hw, vid);
7427 if (!idx)
7428 return;
7429
7430 /* See if any other pools are set for this VLAN filter
7431 * entry other than the PF.
7432 */
7433 pf_id = adapter->vfs_allocated_count + E1000_VLVF_POOLSEL_SHIFT;
7434 bits = ~BIT(pf_id) & E1000_VLVF_POOLSEL_MASK;
7435 bits &= rd32(E1000_VLVF(idx));
7436
7437 /* Disable the filter so this falls into the default pool. */
7438 if (!bits) {
7439 if (adapter->flags & IGB_FLAG_VLAN_PROMISC)
7440 wr32(E1000_VLVF(idx), BIT(pf_id));
7441 else
7442 wr32(E1000_VLVF(idx), 0);
7443 }
7444}
7445
7446static s32 igb_set_vf_vlan(struct igb_adapter *adapter, u32 vid,
7447 bool add, u32 vf)
7448{
7449 int pf_id = adapter->vfs_allocated_count;
7450 struct e1000_hw *hw = &adapter->hw;
7451 int err;
7452
7453 /* If VLAN overlaps with one the PF is currently monitoring make
7454 * sure that we are able to allocate a VLVF entry. This may be
7455 * redundant but it guarantees PF will maintain visibility to
7456 * the VLAN.
7457 */
7458 if (add && test_bit(vid, adapter->active_vlans)) {
7459 err = igb_vfta_set(hw, vid, pf_id, true, false);
7460 if (err)
7461 return err;
7462 }
7463
7464 err = igb_vfta_set(hw, vid, vf, add, false);
7465
7466 if (add && !err)
7467 return err;
7468
7469 /* If we failed to add the VF VLAN or we are removing the VF VLAN
7470 * we may need to drop the PF pool bit in order to allow us to free
7471 * up the VLVF resources.
7472 */
7473 if (test_bit(vid, adapter->active_vlans) ||
7474 (adapter->flags & IGB_FLAG_VLAN_PROMISC))
7475 igb_update_pf_vlvf(adapter, vid);
7476
7477 return err;
7478}
7479
7480static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
7481{
7482 struct e1000_hw *hw = &adapter->hw;
7483
7484 if (vid)
7485 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
7486 else
7487 wr32(E1000_VMVIR(vf), 0);
7488}
7489
7490static int igb_enable_port_vlan(struct igb_adapter *adapter, int vf,
7491 u16 vlan, u8 qos)
7492{
7493 int err;
7494
7495 err = igb_set_vf_vlan(adapter, vlan, true, vf);
7496 if (err)
7497 return err;
7498
7499 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
7500 igb_set_vmolr(adapter, vf, !vlan);
7501
7502 /* revoke access to previous VLAN */
7503 if (vlan != adapter->vf_data[vf].pf_vlan)
7504 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7505 false, vf);
7506
7507 adapter->vf_data[vf].pf_vlan = vlan;
7508 adapter->vf_data[vf].pf_qos = qos;
7509 igb_set_vf_vlan_strip(adapter, vf, true);
7510 dev_info(&adapter->pdev->dev,
7511 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
7512 if (test_bit(__IGB_DOWN, &adapter->state)) {
7513 dev_warn(&adapter->pdev->dev,
7514 "The VF VLAN has been set, but the PF device is not up.\n");
7515 dev_warn(&adapter->pdev->dev,
7516 "Bring the PF device up before attempting to use the VF device.\n");
7517 }
7518
7519 return err;
7520}
7521
7522static int igb_disable_port_vlan(struct igb_adapter *adapter, int vf)
7523{
7524 /* Restore tagless access via VLAN 0 */
7525 igb_set_vf_vlan(adapter, 0, true, vf);
7526
7527 igb_set_vmvir(adapter, 0, vf);
7528 igb_set_vmolr(adapter, vf, true);
7529
7530 /* Remove any PF assigned VLAN */
7531 if (adapter->vf_data[vf].pf_vlan)
7532 igb_set_vf_vlan(adapter, adapter->vf_data[vf].pf_vlan,
7533 false, vf);
7534
7535 adapter->vf_data[vf].pf_vlan = 0;
7536 adapter->vf_data[vf].pf_qos = 0;
7537 igb_set_vf_vlan_strip(adapter, vf, false);
7538
7539 return 0;
7540}
7541
7542static int igb_ndo_set_vf_vlan(struct net_device *netdev, int vf,
7543 u16 vlan, u8 qos, __be16 vlan_proto)
7544{
7545 struct igb_adapter *adapter = netdev_priv(netdev);
7546
7547 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
7548 return -EINVAL;
7549
7550 if (vlan_proto != htons(ETH_P_8021Q))
7551 return -EPROTONOSUPPORT;
7552
7553 return (vlan || qos) ? igb_enable_port_vlan(adapter, vf, vlan, qos) :
7554 igb_disable_port_vlan(adapter, vf);
7555}
7556
7557static int igb_set_vf_vlan_msg(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
7558{
7559 int add = FIELD_GET(E1000_VT_MSGINFO_MASK, msgbuf[0]);
7560 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
7561 int ret;
7562
7563 if (adapter->vf_data[vf].pf_vlan)
7564 return -1;
7565
7566 /* VLAN 0 is a special case, don't allow it to be removed */
7567 if (!vid && !add)
7568 return 0;
7569
7570 ret = igb_set_vf_vlan(adapter, vid, !!add, vf);
7571 if (!ret)
7572 igb_set_vf_vlan_strip(adapter, vf, !!vid);
7573 return ret;
7574}
7575
7576static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
7577{
7578 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7579
7580 /* clear flags - except flag that indicates PF has set the MAC */
7581 vf_data->flags &= IGB_VF_FLAG_PF_SET_MAC;
7582 vf_data->last_nack = jiffies;
7583
7584 /* reset vlans for device */
7585 igb_clear_vf_vfta(adapter, vf);
7586 igb_set_vf_vlan(adapter, vf_data->pf_vlan, true, vf);
7587 igb_set_vmvir(adapter, vf_data->pf_vlan |
7588 (vf_data->pf_qos << VLAN_PRIO_SHIFT), vf);
7589 igb_set_vmolr(adapter, vf, !vf_data->pf_vlan);
7590 igb_set_vf_vlan_strip(adapter, vf, !!(vf_data->pf_vlan));
7591
7592 /* reset multicast table array for vf */
7593 adapter->vf_data[vf].num_vf_mc_hashes = 0;
7594
7595 /* Flush and reset the mta with the new values */
7596 igb_set_rx_mode(adapter->netdev);
7597}
7598
7599static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
7600{
7601 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7602
7603 /* clear mac address as we were hotplug removed/added */
7604 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
7605 eth_zero_addr(vf_mac);
7606
7607 /* process remaining reset events */
7608 igb_vf_reset(adapter, vf);
7609}
7610
7611static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
7612{
7613 struct e1000_hw *hw = &adapter->hw;
7614 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
7615 u32 reg, msgbuf[3] = {};
7616 u8 *addr = (u8 *)(&msgbuf[1]);
7617
7618 /* process all the same items cleared in a function level reset */
7619 igb_vf_reset(adapter, vf);
7620
7621 /* set vf mac address */
7622 igb_set_vf_mac(adapter, vf, vf_mac);
7623
7624 /* enable transmit and receive for vf */
7625 reg = rd32(E1000_VFTE);
7626 wr32(E1000_VFTE, reg | BIT(vf));
7627 reg = rd32(E1000_VFRE);
7628 wr32(E1000_VFRE, reg | BIT(vf));
7629
7630 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
7631
7632 /* reply to reset with ack and vf mac address */
7633 if (!is_zero_ether_addr(vf_mac)) {
7634 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
7635 memcpy(addr, vf_mac, ETH_ALEN);
7636 } else {
7637 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_NACK;
7638 }
7639 igb_write_mbx(hw, msgbuf, 3, vf);
7640}
7641
7642static void igb_flush_mac_table(struct igb_adapter *adapter)
7643{
7644 struct e1000_hw *hw = &adapter->hw;
7645 int i;
7646
7647 for (i = 0; i < hw->mac.rar_entry_count; i++) {
7648 adapter->mac_table[i].state &= ~IGB_MAC_STATE_IN_USE;
7649 eth_zero_addr(adapter->mac_table[i].addr);
7650 adapter->mac_table[i].queue = 0;
7651 igb_rar_set_index(adapter, i);
7652 }
7653}
7654
7655static int igb_available_rars(struct igb_adapter *adapter, u8 queue)
7656{
7657 struct e1000_hw *hw = &adapter->hw;
7658 /* do not count rar entries reserved for VFs MAC addresses */
7659 int rar_entries = hw->mac.rar_entry_count -
7660 adapter->vfs_allocated_count;
7661 int i, count = 0;
7662
7663 for (i = 0; i < rar_entries; i++) {
7664 /* do not count default entries */
7665 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT)
7666 continue;
7667
7668 /* do not count "in use" entries for different queues */
7669 if ((adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE) &&
7670 (adapter->mac_table[i].queue != queue))
7671 continue;
7672
7673 count++;
7674 }
7675
7676 return count;
7677}
7678
7679/* Set default MAC address for the PF in the first RAR entry */
7680static void igb_set_default_mac_filter(struct igb_adapter *adapter)
7681{
7682 struct igb_mac_addr *mac_table = &adapter->mac_table[0];
7683
7684 ether_addr_copy(mac_table->addr, adapter->hw.mac.addr);
7685 mac_table->queue = adapter->vfs_allocated_count;
7686 mac_table->state = IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7687
7688 igb_rar_set_index(adapter, 0);
7689}
7690
7691/* If the filter to be added and an already existing filter express
7692 * the same address and address type, it should be possible to only
7693 * override the other configurations, for example the queue to steer
7694 * traffic.
7695 */
7696static bool igb_mac_entry_can_be_used(const struct igb_mac_addr *entry,
7697 const u8 *addr, const u8 flags)
7698{
7699 if (!(entry->state & IGB_MAC_STATE_IN_USE))
7700 return true;
7701
7702 if ((entry->state & IGB_MAC_STATE_SRC_ADDR) !=
7703 (flags & IGB_MAC_STATE_SRC_ADDR))
7704 return false;
7705
7706 if (!ether_addr_equal(addr, entry->addr))
7707 return false;
7708
7709 return true;
7710}
7711
7712/* Add a MAC filter for 'addr' directing matching traffic to 'queue',
7713 * 'flags' is used to indicate what kind of match is made, match is by
7714 * default for the destination address, if matching by source address
7715 * is desired the flag IGB_MAC_STATE_SRC_ADDR can be used.
7716 */
7717static int igb_add_mac_filter_flags(struct igb_adapter *adapter,
7718 const u8 *addr, const u8 queue,
7719 const u8 flags)
7720{
7721 struct e1000_hw *hw = &adapter->hw;
7722 int rar_entries = hw->mac.rar_entry_count -
7723 adapter->vfs_allocated_count;
7724 int i;
7725
7726 if (is_zero_ether_addr(addr))
7727 return -EINVAL;
7728
7729 /* Search for the first empty entry in the MAC table.
7730 * Do not touch entries at the end of the table reserved for the VF MAC
7731 * addresses.
7732 */
7733 for (i = 0; i < rar_entries; i++) {
7734 if (!igb_mac_entry_can_be_used(&adapter->mac_table[i],
7735 addr, flags))
7736 continue;
7737
7738 ether_addr_copy(adapter->mac_table[i].addr, addr);
7739 adapter->mac_table[i].queue = queue;
7740 adapter->mac_table[i].state |= IGB_MAC_STATE_IN_USE | flags;
7741
7742 igb_rar_set_index(adapter, i);
7743 return i;
7744 }
7745
7746 return -ENOSPC;
7747}
7748
7749static int igb_add_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7750 const u8 queue)
7751{
7752 return igb_add_mac_filter_flags(adapter, addr, queue, 0);
7753}
7754
7755/* Remove a MAC filter for 'addr' directing matching traffic to
7756 * 'queue', 'flags' is used to indicate what kind of match need to be
7757 * removed, match is by default for the destination address, if
7758 * matching by source address is to be removed the flag
7759 * IGB_MAC_STATE_SRC_ADDR can be used.
7760 */
7761static int igb_del_mac_filter_flags(struct igb_adapter *adapter,
7762 const u8 *addr, const u8 queue,
7763 const u8 flags)
7764{
7765 struct e1000_hw *hw = &adapter->hw;
7766 int rar_entries = hw->mac.rar_entry_count -
7767 adapter->vfs_allocated_count;
7768 int i;
7769
7770 if (is_zero_ether_addr(addr))
7771 return -EINVAL;
7772
7773 /* Search for matching entry in the MAC table based on given address
7774 * and queue. Do not touch entries at the end of the table reserved
7775 * for the VF MAC addresses.
7776 */
7777 for (i = 0; i < rar_entries; i++) {
7778 if (!(adapter->mac_table[i].state & IGB_MAC_STATE_IN_USE))
7779 continue;
7780 if ((adapter->mac_table[i].state & flags) != flags)
7781 continue;
7782 if (adapter->mac_table[i].queue != queue)
7783 continue;
7784 if (!ether_addr_equal(adapter->mac_table[i].addr, addr))
7785 continue;
7786
7787 /* When a filter for the default address is "deleted",
7788 * we return it to its initial configuration
7789 */
7790 if (adapter->mac_table[i].state & IGB_MAC_STATE_DEFAULT) {
7791 adapter->mac_table[i].state =
7792 IGB_MAC_STATE_DEFAULT | IGB_MAC_STATE_IN_USE;
7793 adapter->mac_table[i].queue =
7794 adapter->vfs_allocated_count;
7795 } else {
7796 adapter->mac_table[i].state = 0;
7797 adapter->mac_table[i].queue = 0;
7798 eth_zero_addr(adapter->mac_table[i].addr);
7799 }
7800
7801 igb_rar_set_index(adapter, i);
7802 return 0;
7803 }
7804
7805 return -ENOENT;
7806}
7807
7808static int igb_del_mac_filter(struct igb_adapter *adapter, const u8 *addr,
7809 const u8 queue)
7810{
7811 return igb_del_mac_filter_flags(adapter, addr, queue, 0);
7812}
7813
7814int igb_add_mac_steering_filter(struct igb_adapter *adapter,
7815 const u8 *addr, u8 queue, u8 flags)
7816{
7817 struct e1000_hw *hw = &adapter->hw;
7818
7819 /* In theory, this should be supported on 82575 as well, but
7820 * that part wasn't easily accessible during development.
7821 */
7822 if (hw->mac.type != e1000_i210)
7823 return -EOPNOTSUPP;
7824
7825 return igb_add_mac_filter_flags(adapter, addr, queue,
7826 IGB_MAC_STATE_QUEUE_STEERING | flags);
7827}
7828
7829int igb_del_mac_steering_filter(struct igb_adapter *adapter,
7830 const u8 *addr, u8 queue, u8 flags)
7831{
7832 return igb_del_mac_filter_flags(adapter, addr, queue,
7833 IGB_MAC_STATE_QUEUE_STEERING | flags);
7834}
7835
7836static int igb_uc_sync(struct net_device *netdev, const unsigned char *addr)
7837{
7838 struct igb_adapter *adapter = netdev_priv(netdev);
7839 int ret;
7840
7841 ret = igb_add_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7842
7843 return min_t(int, ret, 0);
7844}
7845
7846static int igb_uc_unsync(struct net_device *netdev, const unsigned char *addr)
7847{
7848 struct igb_adapter *adapter = netdev_priv(netdev);
7849
7850 igb_del_mac_filter(adapter, addr, adapter->vfs_allocated_count);
7851
7852 return 0;
7853}
7854
7855static int igb_set_vf_mac_filter(struct igb_adapter *adapter, const int vf,
7856 const u32 info, const u8 *addr)
7857{
7858 struct pci_dev *pdev = adapter->pdev;
7859 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7860 struct vf_mac_filter *entry;
7861 bool found = false;
7862 int ret = 0;
7863
7864 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7865 !vf_data->trusted) {
7866 dev_warn(&pdev->dev,
7867 "VF %d requested MAC filter but is administratively denied\n",
7868 vf);
7869 return -EINVAL;
7870 }
7871 if (!is_valid_ether_addr(addr)) {
7872 dev_warn(&pdev->dev,
7873 "VF %d attempted to set invalid MAC filter\n",
7874 vf);
7875 return -EINVAL;
7876 }
7877
7878 switch (info) {
7879 case E1000_VF_MAC_FILTER_CLR:
7880 /* remove all unicast MAC filters related to the current VF */
7881 list_for_each_entry(entry, &adapter->vf_macs.l, l) {
7882 if (entry->vf == vf) {
7883 entry->vf = -1;
7884 entry->free = true;
7885 igb_del_mac_filter(adapter, entry->vf_mac, vf);
7886 }
7887 }
7888 break;
7889 case E1000_VF_MAC_FILTER_ADD:
7890 /* try to find empty slot in the list */
7891 list_for_each_entry(entry, &adapter->vf_macs.l, l) {
7892 if (entry->free) {
7893 found = true;
7894 break;
7895 }
7896 }
7897
7898 if (found) {
7899 entry->free = false;
7900 entry->vf = vf;
7901 ether_addr_copy(entry->vf_mac, addr);
7902
7903 ret = igb_add_mac_filter(adapter, addr, vf);
7904 ret = min_t(int, ret, 0);
7905 } else {
7906 ret = -ENOSPC;
7907 }
7908
7909 if (ret == -ENOSPC)
7910 dev_warn(&pdev->dev,
7911 "VF %d has requested MAC filter but there is no space for it\n",
7912 vf);
7913 break;
7914 default:
7915 ret = -EINVAL;
7916 break;
7917 }
7918
7919 return ret;
7920}
7921
7922static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
7923{
7924 struct pci_dev *pdev = adapter->pdev;
7925 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7926 u32 info = msg[0] & E1000_VT_MSGINFO_MASK;
7927
7928 /* The VF MAC Address is stored in a packed array of bytes
7929 * starting at the second 32 bit word of the msg array
7930 */
7931 unsigned char *addr = (unsigned char *)&msg[1];
7932 int ret = 0;
7933
7934 if (!info) {
7935 if ((vf_data->flags & IGB_VF_FLAG_PF_SET_MAC) &&
7936 !vf_data->trusted) {
7937 dev_warn(&pdev->dev,
7938 "VF %d attempted to override administratively set MAC address\nReload the VF driver to resume operations\n",
7939 vf);
7940 return -EINVAL;
7941 }
7942
7943 if (!is_valid_ether_addr(addr)) {
7944 dev_warn(&pdev->dev,
7945 "VF %d attempted to set invalid MAC\n",
7946 vf);
7947 return -EINVAL;
7948 }
7949
7950 ret = igb_set_vf_mac(adapter, vf, addr);
7951 } else {
7952 ret = igb_set_vf_mac_filter(adapter, vf, info, addr);
7953 }
7954
7955 return ret;
7956}
7957
7958static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
7959{
7960 struct e1000_hw *hw = &adapter->hw;
7961 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7962 u32 msg = E1000_VT_MSGTYPE_NACK;
7963
7964 /* if device isn't clear to send it shouldn't be reading either */
7965 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
7966 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
7967 igb_write_mbx(hw, &msg, 1, vf);
7968 vf_data->last_nack = jiffies;
7969 }
7970}
7971
7972static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
7973{
7974 struct pci_dev *pdev = adapter->pdev;
7975 u32 msgbuf[E1000_VFMAILBOX_SIZE];
7976 struct e1000_hw *hw = &adapter->hw;
7977 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
7978 s32 retval;
7979
7980 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf, false);
7981
7982 if (retval) {
7983 /* if receive failed revoke VF CTS stats and restart init */
7984 dev_err(&pdev->dev, "Error receiving message from VF\n");
7985 vf_data->flags &= ~IGB_VF_FLAG_CTS;
7986 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
7987 goto unlock;
7988 goto out;
7989 }
7990
7991 /* this is a message we already processed, do nothing */
7992 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
7993 goto unlock;
7994
7995 /* until the vf completes a reset it should not be
7996 * allowed to start any configuration.
7997 */
7998 if (msgbuf[0] == E1000_VF_RESET) {
7999 /* unlocks mailbox */
8000 igb_vf_reset_msg(adapter, vf);
8001 return;
8002 }
8003
8004 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
8005 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
8006 goto unlock;
8007 retval = -1;
8008 goto out;
8009 }
8010
8011 switch ((msgbuf[0] & 0xFFFF)) {
8012 case E1000_VF_SET_MAC_ADDR:
8013 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
8014 break;
8015 case E1000_VF_SET_PROMISC:
8016 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
8017 break;
8018 case E1000_VF_SET_MULTICAST:
8019 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
8020 break;
8021 case E1000_VF_SET_LPE:
8022 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
8023 break;
8024 case E1000_VF_SET_VLAN:
8025 retval = -1;
8026 if (vf_data->pf_vlan)
8027 dev_warn(&pdev->dev,
8028 "VF %d attempted to override administratively set VLAN tag\nReload the VF driver to resume operations\n",
8029 vf);
8030 else
8031 retval = igb_set_vf_vlan_msg(adapter, msgbuf, vf);
8032 break;
8033 default:
8034 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
8035 retval = -1;
8036 break;
8037 }
8038
8039 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
8040out:
8041 /* notify the VF of the results of what it sent us */
8042 if (retval)
8043 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
8044 else
8045 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
8046
8047 /* unlocks mailbox */
8048 igb_write_mbx(hw, msgbuf, 1, vf);
8049 return;
8050
8051unlock:
8052 igb_unlock_mbx(hw, vf);
8053}
8054
8055static void igb_msg_task(struct igb_adapter *adapter)
8056{
8057 struct e1000_hw *hw = &adapter->hw;
8058 unsigned long flags;
8059 u32 vf;
8060
8061 spin_lock_irqsave(&adapter->vfs_lock, flags);
8062 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
8063 /* process any reset requests */
8064 if (!igb_check_for_rst(hw, vf))
8065 igb_vf_reset_event(adapter, vf);
8066
8067 /* process any messages pending */
8068 if (!igb_check_for_msg(hw, vf))
8069 igb_rcv_msg_from_vf(adapter, vf);
8070
8071 /* process any acks */
8072 if (!igb_check_for_ack(hw, vf))
8073 igb_rcv_ack_from_vf(adapter, vf);
8074 }
8075 spin_unlock_irqrestore(&adapter->vfs_lock, flags);
8076}
8077
8078/**
8079 * igb_set_uta - Set unicast filter table address
8080 * @adapter: board private structure
8081 * @set: boolean indicating if we are setting or clearing bits
8082 *
8083 * The unicast table address is a register array of 32-bit registers.
8084 * The table is meant to be used in a way similar to how the MTA is used
8085 * however due to certain limitations in the hardware it is necessary to
8086 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
8087 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
8088 **/
8089static void igb_set_uta(struct igb_adapter *adapter, bool set)
8090{
8091 struct e1000_hw *hw = &adapter->hw;
8092 u32 uta = set ? ~0 : 0;
8093 int i;
8094
8095 /* we only need to do this if VMDq is enabled */
8096 if (!adapter->vfs_allocated_count)
8097 return;
8098
8099 for (i = hw->mac.uta_reg_count; i--;)
8100 array_wr32(E1000_UTA, i, uta);
8101}
8102
8103/**
8104 * igb_intr_msi - Interrupt Handler
8105 * @irq: interrupt number
8106 * @data: pointer to a network interface device structure
8107 **/
8108static irqreturn_t igb_intr_msi(int irq, void *data)
8109{
8110 struct igb_adapter *adapter = data;
8111 struct igb_q_vector *q_vector = adapter->q_vector[0];
8112 struct e1000_hw *hw = &adapter->hw;
8113 /* read ICR disables interrupts using IAM */
8114 u32 icr = rd32(E1000_ICR);
8115
8116 igb_write_itr(q_vector);
8117
8118 if (icr & E1000_ICR_DRSTA)
8119 schedule_work(&adapter->reset_task);
8120
8121 if (icr & E1000_ICR_DOUTSYNC) {
8122 /* HW is reporting DMA is out of sync */
8123 adapter->stats.doosync++;
8124 }
8125
8126 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
8127 hw->mac.get_link_status = 1;
8128 if (!test_bit(__IGB_DOWN, &adapter->state))
8129 mod_timer(&adapter->watchdog_timer, jiffies + 1);
8130 }
8131
8132 if (icr & E1000_ICR_TS)
8133 igb_tsync_interrupt(adapter);
8134
8135 napi_schedule(&q_vector->napi);
8136
8137 return IRQ_HANDLED;
8138}
8139
8140/**
8141 * igb_intr - Legacy Interrupt Handler
8142 * @irq: interrupt number
8143 * @data: pointer to a network interface device structure
8144 **/
8145static irqreturn_t igb_intr(int irq, void *data)
8146{
8147 struct igb_adapter *adapter = data;
8148 struct igb_q_vector *q_vector = adapter->q_vector[0];
8149 struct e1000_hw *hw = &adapter->hw;
8150 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
8151 * need for the IMC write
8152 */
8153 u32 icr = rd32(E1000_ICR);
8154
8155 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
8156 * not set, then the adapter didn't send an interrupt
8157 */
8158 if (!(icr & E1000_ICR_INT_ASSERTED))
8159 return IRQ_NONE;
8160
8161 igb_write_itr(q_vector);
8162
8163 if (icr & E1000_ICR_DRSTA)
8164 schedule_work(&adapter->reset_task);
8165
8166 if (icr & E1000_ICR_DOUTSYNC) {
8167 /* HW is reporting DMA is out of sync */
8168 adapter->stats.doosync++;
8169 }
8170
8171 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
8172 hw->mac.get_link_status = 1;
8173 /* guard against interrupt when we're going down */
8174 if (!test_bit(__IGB_DOWN, &adapter->state))
8175 mod_timer(&adapter->watchdog_timer, jiffies + 1);
8176 }
8177
8178 if (icr & E1000_ICR_TS)
8179 igb_tsync_interrupt(adapter);
8180
8181 napi_schedule(&q_vector->napi);
8182
8183 return IRQ_HANDLED;
8184}
8185
8186static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
8187{
8188 struct igb_adapter *adapter = q_vector->adapter;
8189 struct e1000_hw *hw = &adapter->hw;
8190
8191 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
8192 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
8193 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
8194 igb_set_itr(q_vector);
8195 else
8196 igb_update_ring_itr(q_vector);
8197 }
8198
8199 if (!test_bit(__IGB_DOWN, &adapter->state)) {
8200 if (adapter->flags & IGB_FLAG_HAS_MSIX)
8201 wr32(E1000_EIMS, q_vector->eims_value);
8202 else
8203 igb_irq_enable(adapter);
8204 }
8205}
8206
8207/**
8208 * igb_poll - NAPI Rx polling callback
8209 * @napi: napi polling structure
8210 * @budget: count of how many packets we should handle
8211 **/
8212static int igb_poll(struct napi_struct *napi, int budget)
8213{
8214 struct igb_q_vector *q_vector = container_of(napi,
8215 struct igb_q_vector,
8216 napi);
8217 bool clean_complete = true;
8218 int work_done = 0;
8219
8220#ifdef CONFIG_IGB_DCA
8221 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
8222 igb_update_dca(q_vector);
8223#endif
8224 if (q_vector->tx.ring)
8225 clean_complete = igb_clean_tx_irq(q_vector, budget);
8226
8227 if (q_vector->rx.ring) {
8228 int cleaned = igb_clean_rx_irq(q_vector, budget);
8229
8230 work_done += cleaned;
8231 if (cleaned >= budget)
8232 clean_complete = false;
8233 }
8234
8235 /* If all work not completed, return budget and keep polling */
8236 if (!clean_complete)
8237 return budget;
8238
8239 /* Exit the polling mode, but don't re-enable interrupts if stack might
8240 * poll us due to busy-polling
8241 */
8242 if (likely(napi_complete_done(napi, work_done)))
8243 igb_ring_irq_enable(q_vector);
8244
8245 return work_done;
8246}
8247
8248/**
8249 * igb_clean_tx_irq - Reclaim resources after transmit completes
8250 * @q_vector: pointer to q_vector containing needed info
8251 * @napi_budget: Used to determine if we are in netpoll
8252 *
8253 * returns true if ring is completely cleaned
8254 **/
8255static bool igb_clean_tx_irq(struct igb_q_vector *q_vector, int napi_budget)
8256{
8257 struct igb_adapter *adapter = q_vector->adapter;
8258 struct igb_ring *tx_ring = q_vector->tx.ring;
8259 struct igb_tx_buffer *tx_buffer;
8260 union e1000_adv_tx_desc *tx_desc;
8261 unsigned int total_bytes = 0, total_packets = 0;
8262 unsigned int budget = q_vector->tx.work_limit;
8263 unsigned int i = tx_ring->next_to_clean;
8264
8265 if (test_bit(__IGB_DOWN, &adapter->state))
8266 return true;
8267
8268 tx_buffer = &tx_ring->tx_buffer_info[i];
8269 tx_desc = IGB_TX_DESC(tx_ring, i);
8270 i -= tx_ring->count;
8271
8272 do {
8273 union e1000_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
8274
8275 /* if next_to_watch is not set then there is no work pending */
8276 if (!eop_desc)
8277 break;
8278
8279 /* prevent any other reads prior to eop_desc */
8280 smp_rmb();
8281
8282 /* if DD is not set pending work has not been completed */
8283 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
8284 break;
8285
8286 /* clear next_to_watch to prevent false hangs */
8287 tx_buffer->next_to_watch = NULL;
8288
8289 /* update the statistics for this packet */
8290 total_bytes += tx_buffer->bytecount;
8291 total_packets += tx_buffer->gso_segs;
8292
8293 /* free the skb */
8294 if (tx_buffer->type == IGB_TYPE_SKB)
8295 napi_consume_skb(tx_buffer->skb, napi_budget);
8296 else
8297 xdp_return_frame(tx_buffer->xdpf);
8298
8299 /* unmap skb header data */
8300 dma_unmap_single(tx_ring->dev,
8301 dma_unmap_addr(tx_buffer, dma),
8302 dma_unmap_len(tx_buffer, len),
8303 DMA_TO_DEVICE);
8304
8305 /* clear tx_buffer data */
8306 dma_unmap_len_set(tx_buffer, len, 0);
8307
8308 /* clear last DMA location and unmap remaining buffers */
8309 while (tx_desc != eop_desc) {
8310 tx_buffer++;
8311 tx_desc++;
8312 i++;
8313 if (unlikely(!i)) {
8314 i -= tx_ring->count;
8315 tx_buffer = tx_ring->tx_buffer_info;
8316 tx_desc = IGB_TX_DESC(tx_ring, 0);
8317 }
8318
8319 /* unmap any remaining paged data */
8320 if (dma_unmap_len(tx_buffer, len)) {
8321 dma_unmap_page(tx_ring->dev,
8322 dma_unmap_addr(tx_buffer, dma),
8323 dma_unmap_len(tx_buffer, len),
8324 DMA_TO_DEVICE);
8325 dma_unmap_len_set(tx_buffer, len, 0);
8326 }
8327 }
8328
8329 /* move us one more past the eop_desc for start of next pkt */
8330 tx_buffer++;
8331 tx_desc++;
8332 i++;
8333 if (unlikely(!i)) {
8334 i -= tx_ring->count;
8335 tx_buffer = tx_ring->tx_buffer_info;
8336 tx_desc = IGB_TX_DESC(tx_ring, 0);
8337 }
8338
8339 /* issue prefetch for next Tx descriptor */
8340 prefetch(tx_desc);
8341
8342 /* update budget accounting */
8343 budget--;
8344 } while (likely(budget));
8345
8346 netdev_tx_completed_queue(txring_txq(tx_ring),
8347 total_packets, total_bytes);
8348 i += tx_ring->count;
8349 tx_ring->next_to_clean = i;
8350 u64_stats_update_begin(&tx_ring->tx_syncp);
8351 tx_ring->tx_stats.bytes += total_bytes;
8352 tx_ring->tx_stats.packets += total_packets;
8353 u64_stats_update_end(&tx_ring->tx_syncp);
8354 q_vector->tx.total_bytes += total_bytes;
8355 q_vector->tx.total_packets += total_packets;
8356
8357 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
8358 struct e1000_hw *hw = &adapter->hw;
8359
8360 /* Detect a transmit hang in hardware, this serializes the
8361 * check with the clearing of time_stamp and movement of i
8362 */
8363 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
8364 if (tx_buffer->next_to_watch &&
8365 time_after(jiffies, tx_buffer->time_stamp +
8366 (adapter->tx_timeout_factor * HZ)) &&
8367 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
8368
8369 /* detected Tx unit hang */
8370 dev_err(tx_ring->dev,
8371 "Detected Tx Unit Hang\n"
8372 " Tx Queue <%d>\n"
8373 " TDH <%x>\n"
8374 " TDT <%x>\n"
8375 " next_to_use <%x>\n"
8376 " next_to_clean <%x>\n"
8377 "buffer_info[next_to_clean]\n"
8378 " time_stamp <%lx>\n"
8379 " next_to_watch <%p>\n"
8380 " jiffies <%lx>\n"
8381 " desc.status <%x>\n",
8382 tx_ring->queue_index,
8383 rd32(E1000_TDH(tx_ring->reg_idx)),
8384 readl(tx_ring->tail),
8385 tx_ring->next_to_use,
8386 tx_ring->next_to_clean,
8387 tx_buffer->time_stamp,
8388 tx_buffer->next_to_watch,
8389 jiffies,
8390 tx_buffer->next_to_watch->wb.status);
8391 netif_stop_subqueue(tx_ring->netdev,
8392 tx_ring->queue_index);
8393
8394 /* we are about to reset, no point in enabling stuff */
8395 return true;
8396 }
8397 }
8398
8399#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
8400 if (unlikely(total_packets &&
8401 netif_carrier_ok(tx_ring->netdev) &&
8402 igb_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
8403 /* Make sure that anybody stopping the queue after this
8404 * sees the new next_to_clean.
8405 */
8406 smp_mb();
8407 if (__netif_subqueue_stopped(tx_ring->netdev,
8408 tx_ring->queue_index) &&
8409 !(test_bit(__IGB_DOWN, &adapter->state))) {
8410 netif_wake_subqueue(tx_ring->netdev,
8411 tx_ring->queue_index);
8412
8413 u64_stats_update_begin(&tx_ring->tx_syncp);
8414 tx_ring->tx_stats.restart_queue++;
8415 u64_stats_update_end(&tx_ring->tx_syncp);
8416 }
8417 }
8418
8419 return !!budget;
8420}
8421
8422/**
8423 * igb_reuse_rx_page - page flip buffer and store it back on the ring
8424 * @rx_ring: rx descriptor ring to store buffers on
8425 * @old_buff: donor buffer to have page reused
8426 *
8427 * Synchronizes page for reuse by the adapter
8428 **/
8429static void igb_reuse_rx_page(struct igb_ring *rx_ring,
8430 struct igb_rx_buffer *old_buff)
8431{
8432 struct igb_rx_buffer *new_buff;
8433 u16 nta = rx_ring->next_to_alloc;
8434
8435 new_buff = &rx_ring->rx_buffer_info[nta];
8436
8437 /* update, and store next to alloc */
8438 nta++;
8439 rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
8440
8441 /* Transfer page from old buffer to new buffer.
8442 * Move each member individually to avoid possible store
8443 * forwarding stalls.
8444 */
8445 new_buff->dma = old_buff->dma;
8446 new_buff->page = old_buff->page;
8447 new_buff->page_offset = old_buff->page_offset;
8448 new_buff->pagecnt_bias = old_buff->pagecnt_bias;
8449}
8450
8451static bool igb_can_reuse_rx_page(struct igb_rx_buffer *rx_buffer,
8452 int rx_buf_pgcnt)
8453{
8454 unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
8455 struct page *page = rx_buffer->page;
8456
8457 /* avoid re-using remote and pfmemalloc pages */
8458 if (!dev_page_is_reusable(page))
8459 return false;
8460
8461#if (PAGE_SIZE < 8192)
8462 /* if we are only owner of page we can reuse it */
8463 if (unlikely((rx_buf_pgcnt - pagecnt_bias) > 1))
8464 return false;
8465#else
8466#define IGB_LAST_OFFSET \
8467 (SKB_WITH_OVERHEAD(PAGE_SIZE) - IGB_RXBUFFER_2048)
8468
8469 if (rx_buffer->page_offset > IGB_LAST_OFFSET)
8470 return false;
8471#endif
8472
8473 /* If we have drained the page fragment pool we need to update
8474 * the pagecnt_bias and page count so that we fully restock the
8475 * number of references the driver holds.
8476 */
8477 if (unlikely(pagecnt_bias == 1)) {
8478 page_ref_add(page, USHRT_MAX - 1);
8479 rx_buffer->pagecnt_bias = USHRT_MAX;
8480 }
8481
8482 return true;
8483}
8484
8485/**
8486 * igb_add_rx_frag - Add contents of Rx buffer to sk_buff
8487 * @rx_ring: rx descriptor ring to transact packets on
8488 * @rx_buffer: buffer containing page to add
8489 * @skb: sk_buff to place the data into
8490 * @size: size of buffer to be added
8491 *
8492 * This function will add the data contained in rx_buffer->page to the skb.
8493 **/
8494static void igb_add_rx_frag(struct igb_ring *rx_ring,
8495 struct igb_rx_buffer *rx_buffer,
8496 struct sk_buff *skb,
8497 unsigned int size)
8498{
8499#if (PAGE_SIZE < 8192)
8500 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8501#else
8502 unsigned int truesize = ring_uses_build_skb(rx_ring) ?
8503 SKB_DATA_ALIGN(IGB_SKB_PAD + size) :
8504 SKB_DATA_ALIGN(size);
8505#endif
8506 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
8507 rx_buffer->page_offset, size, truesize);
8508#if (PAGE_SIZE < 8192)
8509 rx_buffer->page_offset ^= truesize;
8510#else
8511 rx_buffer->page_offset += truesize;
8512#endif
8513}
8514
8515static struct sk_buff *igb_construct_skb(struct igb_ring *rx_ring,
8516 struct igb_rx_buffer *rx_buffer,
8517 struct xdp_buff *xdp,
8518 ktime_t timestamp)
8519{
8520#if (PAGE_SIZE < 8192)
8521 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8522#else
8523 unsigned int truesize = SKB_DATA_ALIGN(xdp->data_end -
8524 xdp->data_hard_start);
8525#endif
8526 unsigned int size = xdp->data_end - xdp->data;
8527 unsigned int headlen;
8528 struct sk_buff *skb;
8529
8530 /* prefetch first cache line of first page */
8531 net_prefetch(xdp->data);
8532
8533 /* allocate a skb to store the frags */
8534 skb = napi_alloc_skb(&rx_ring->q_vector->napi, IGB_RX_HDR_LEN);
8535 if (unlikely(!skb))
8536 return NULL;
8537
8538 if (timestamp)
8539 skb_hwtstamps(skb)->hwtstamp = timestamp;
8540
8541 /* Determine available headroom for copy */
8542 headlen = size;
8543 if (headlen > IGB_RX_HDR_LEN)
8544 headlen = eth_get_headlen(skb->dev, xdp->data, IGB_RX_HDR_LEN);
8545
8546 /* align pull length to size of long to optimize memcpy performance */
8547 memcpy(__skb_put(skb, headlen), xdp->data, ALIGN(headlen, sizeof(long)));
8548
8549 /* update all of the pointers */
8550 size -= headlen;
8551 if (size) {
8552 skb_add_rx_frag(skb, 0, rx_buffer->page,
8553 (xdp->data + headlen) - page_address(rx_buffer->page),
8554 size, truesize);
8555#if (PAGE_SIZE < 8192)
8556 rx_buffer->page_offset ^= truesize;
8557#else
8558 rx_buffer->page_offset += truesize;
8559#endif
8560 } else {
8561 rx_buffer->pagecnt_bias++;
8562 }
8563
8564 return skb;
8565}
8566
8567static struct sk_buff *igb_build_skb(struct igb_ring *rx_ring,
8568 struct igb_rx_buffer *rx_buffer,
8569 struct xdp_buff *xdp,
8570 ktime_t timestamp)
8571{
8572#if (PAGE_SIZE < 8192)
8573 unsigned int truesize = igb_rx_pg_size(rx_ring) / 2;
8574#else
8575 unsigned int truesize = SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
8576 SKB_DATA_ALIGN(xdp->data_end -
8577 xdp->data_hard_start);
8578#endif
8579 unsigned int metasize = xdp->data - xdp->data_meta;
8580 struct sk_buff *skb;
8581
8582 /* prefetch first cache line of first page */
8583 net_prefetch(xdp->data_meta);
8584
8585 /* build an skb around the page buffer */
8586 skb = napi_build_skb(xdp->data_hard_start, truesize);
8587 if (unlikely(!skb))
8588 return NULL;
8589
8590 /* update pointers within the skb to store the data */
8591 skb_reserve(skb, xdp->data - xdp->data_hard_start);
8592 __skb_put(skb, xdp->data_end - xdp->data);
8593
8594 if (metasize)
8595 skb_metadata_set(skb, metasize);
8596
8597 if (timestamp)
8598 skb_hwtstamps(skb)->hwtstamp = timestamp;
8599
8600 /* update buffer offset */
8601#if (PAGE_SIZE < 8192)
8602 rx_buffer->page_offset ^= truesize;
8603#else
8604 rx_buffer->page_offset += truesize;
8605#endif
8606
8607 return skb;
8608}
8609
8610static struct sk_buff *igb_run_xdp(struct igb_adapter *adapter,
8611 struct igb_ring *rx_ring,
8612 struct xdp_buff *xdp)
8613{
8614 int err, result = IGB_XDP_PASS;
8615 struct bpf_prog *xdp_prog;
8616 u32 act;
8617
8618 xdp_prog = READ_ONCE(rx_ring->xdp_prog);
8619
8620 if (!xdp_prog)
8621 goto xdp_out;
8622
8623 prefetchw(xdp->data_hard_start); /* xdp_frame write */
8624
8625 act = bpf_prog_run_xdp(xdp_prog, xdp);
8626 switch (act) {
8627 case XDP_PASS:
8628 break;
8629 case XDP_TX:
8630 result = igb_xdp_xmit_back(adapter, xdp);
8631 if (result == IGB_XDP_CONSUMED)
8632 goto out_failure;
8633 break;
8634 case XDP_REDIRECT:
8635 err = xdp_do_redirect(adapter->netdev, xdp, xdp_prog);
8636 if (err)
8637 goto out_failure;
8638 result = IGB_XDP_REDIR;
8639 break;
8640 default:
8641 bpf_warn_invalid_xdp_action(adapter->netdev, xdp_prog, act);
8642 fallthrough;
8643 case XDP_ABORTED:
8644out_failure:
8645 trace_xdp_exception(rx_ring->netdev, xdp_prog, act);
8646 fallthrough;
8647 case XDP_DROP:
8648 result = IGB_XDP_CONSUMED;
8649 break;
8650 }
8651xdp_out:
8652 return ERR_PTR(-result);
8653}
8654
8655static unsigned int igb_rx_frame_truesize(struct igb_ring *rx_ring,
8656 unsigned int size)
8657{
8658 unsigned int truesize;
8659
8660#if (PAGE_SIZE < 8192)
8661 truesize = igb_rx_pg_size(rx_ring) / 2; /* Must be power-of-2 */
8662#else
8663 truesize = ring_uses_build_skb(rx_ring) ?
8664 SKB_DATA_ALIGN(IGB_SKB_PAD + size) +
8665 SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) :
8666 SKB_DATA_ALIGN(size);
8667#endif
8668 return truesize;
8669}
8670
8671static void igb_rx_buffer_flip(struct igb_ring *rx_ring,
8672 struct igb_rx_buffer *rx_buffer,
8673 unsigned int size)
8674{
8675 unsigned int truesize = igb_rx_frame_truesize(rx_ring, size);
8676#if (PAGE_SIZE < 8192)
8677 rx_buffer->page_offset ^= truesize;
8678#else
8679 rx_buffer->page_offset += truesize;
8680#endif
8681}
8682
8683static inline void igb_rx_checksum(struct igb_ring *ring,
8684 union e1000_adv_rx_desc *rx_desc,
8685 struct sk_buff *skb)
8686{
8687 skb_checksum_none_assert(skb);
8688
8689 /* Ignore Checksum bit is set */
8690 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
8691 return;
8692
8693 /* Rx checksum disabled via ethtool */
8694 if (!(ring->netdev->features & NETIF_F_RXCSUM))
8695 return;
8696
8697 /* TCP/UDP checksum error bit is set */
8698 if (igb_test_staterr(rx_desc,
8699 E1000_RXDEXT_STATERR_TCPE |
8700 E1000_RXDEXT_STATERR_IPE)) {
8701 /* work around errata with sctp packets where the TCPE aka
8702 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
8703 * packets, (aka let the stack check the crc32c)
8704 */
8705 if (!((skb->len == 60) &&
8706 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
8707 u64_stats_update_begin(&ring->rx_syncp);
8708 ring->rx_stats.csum_err++;
8709 u64_stats_update_end(&ring->rx_syncp);
8710 }
8711 /* let the stack verify checksum errors */
8712 return;
8713 }
8714 /* It must be a TCP or UDP packet with a valid checksum */
8715 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
8716 E1000_RXD_STAT_UDPCS))
8717 skb->ip_summed = CHECKSUM_UNNECESSARY;
8718
8719 dev_dbg(ring->dev, "cksum success: bits %08X\n",
8720 le32_to_cpu(rx_desc->wb.upper.status_error));
8721}
8722
8723static inline void igb_rx_hash(struct igb_ring *ring,
8724 union e1000_adv_rx_desc *rx_desc,
8725 struct sk_buff *skb)
8726{
8727 if (ring->netdev->features & NETIF_F_RXHASH)
8728 skb_set_hash(skb,
8729 le32_to_cpu(rx_desc->wb.lower.hi_dword.rss),
8730 PKT_HASH_TYPE_L3);
8731}
8732
8733/**
8734 * igb_is_non_eop - process handling of non-EOP buffers
8735 * @rx_ring: Rx ring being processed
8736 * @rx_desc: Rx descriptor for current buffer
8737 *
8738 * This function updates next to clean. If the buffer is an EOP buffer
8739 * this function exits returning false, otherwise it will place the
8740 * sk_buff in the next buffer to be chained and return true indicating
8741 * that this is in fact a non-EOP buffer.
8742 **/
8743static bool igb_is_non_eop(struct igb_ring *rx_ring,
8744 union e1000_adv_rx_desc *rx_desc)
8745{
8746 u32 ntc = rx_ring->next_to_clean + 1;
8747
8748 /* fetch, update, and store next to clean */
8749 ntc = (ntc < rx_ring->count) ? ntc : 0;
8750 rx_ring->next_to_clean = ntc;
8751
8752 prefetch(IGB_RX_DESC(rx_ring, ntc));
8753
8754 if (likely(igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)))
8755 return false;
8756
8757 return true;
8758}
8759
8760/**
8761 * igb_cleanup_headers - Correct corrupted or empty headers
8762 * @rx_ring: rx descriptor ring packet is being transacted on
8763 * @rx_desc: pointer to the EOP Rx descriptor
8764 * @skb: pointer to current skb being fixed
8765 *
8766 * Address the case where we are pulling data in on pages only
8767 * and as such no data is present in the skb header.
8768 *
8769 * In addition if skb is not at least 60 bytes we need to pad it so that
8770 * it is large enough to qualify as a valid Ethernet frame.
8771 *
8772 * Returns true if an error was encountered and skb was freed.
8773 **/
8774static bool igb_cleanup_headers(struct igb_ring *rx_ring,
8775 union e1000_adv_rx_desc *rx_desc,
8776 struct sk_buff *skb)
8777{
8778 /* XDP packets use error pointer so abort at this point */
8779 if (IS_ERR(skb))
8780 return true;
8781
8782 if (unlikely((igb_test_staterr(rx_desc,
8783 E1000_RXDEXT_ERR_FRAME_ERR_MASK)))) {
8784 struct net_device *netdev = rx_ring->netdev;
8785 if (!(netdev->features & NETIF_F_RXALL)) {
8786 dev_kfree_skb_any(skb);
8787 return true;
8788 }
8789 }
8790
8791 /* if eth_skb_pad returns an error the skb was freed */
8792 if (eth_skb_pad(skb))
8793 return true;
8794
8795 return false;
8796}
8797
8798/**
8799 * igb_process_skb_fields - Populate skb header fields from Rx descriptor
8800 * @rx_ring: rx descriptor ring packet is being transacted on
8801 * @rx_desc: pointer to the EOP Rx descriptor
8802 * @skb: pointer to current skb being populated
8803 *
8804 * This function checks the ring, descriptor, and packet information in
8805 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
8806 * other fields within the skb.
8807 **/
8808static void igb_process_skb_fields(struct igb_ring *rx_ring,
8809 union e1000_adv_rx_desc *rx_desc,
8810 struct sk_buff *skb)
8811{
8812 struct net_device *dev = rx_ring->netdev;
8813
8814 igb_rx_hash(rx_ring, rx_desc, skb);
8815
8816 igb_rx_checksum(rx_ring, rx_desc, skb);
8817
8818 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TS) &&
8819 !igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP))
8820 igb_ptp_rx_rgtstamp(rx_ring->q_vector, skb);
8821
8822 if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
8823 igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
8824 u16 vid;
8825
8826 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
8827 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
8828 vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
8829 else
8830 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
8831
8832 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
8833 }
8834
8835 skb_record_rx_queue(skb, rx_ring->queue_index);
8836
8837 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
8838}
8839
8840static unsigned int igb_rx_offset(struct igb_ring *rx_ring)
8841{
8842 return ring_uses_build_skb(rx_ring) ? IGB_SKB_PAD : 0;
8843}
8844
8845static struct igb_rx_buffer *igb_get_rx_buffer(struct igb_ring *rx_ring,
8846 const unsigned int size, int *rx_buf_pgcnt)
8847{
8848 struct igb_rx_buffer *rx_buffer;
8849
8850 rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
8851 *rx_buf_pgcnt =
8852#if (PAGE_SIZE < 8192)
8853 page_count(rx_buffer->page);
8854#else
8855 0;
8856#endif
8857 prefetchw(rx_buffer->page);
8858
8859 /* we are reusing so sync this buffer for CPU use */
8860 dma_sync_single_range_for_cpu(rx_ring->dev,
8861 rx_buffer->dma,
8862 rx_buffer->page_offset,
8863 size,
8864 DMA_FROM_DEVICE);
8865
8866 rx_buffer->pagecnt_bias--;
8867
8868 return rx_buffer;
8869}
8870
8871static void igb_put_rx_buffer(struct igb_ring *rx_ring,
8872 struct igb_rx_buffer *rx_buffer, int rx_buf_pgcnt)
8873{
8874 if (igb_can_reuse_rx_page(rx_buffer, rx_buf_pgcnt)) {
8875 /* hand second half of page back to the ring */
8876 igb_reuse_rx_page(rx_ring, rx_buffer);
8877 } else {
8878 /* We are not reusing the buffer so unmap it and free
8879 * any references we are holding to it
8880 */
8881 dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
8882 igb_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
8883 IGB_RX_DMA_ATTR);
8884 __page_frag_cache_drain(rx_buffer->page,
8885 rx_buffer->pagecnt_bias);
8886 }
8887
8888 /* clear contents of rx_buffer */
8889 rx_buffer->page = NULL;
8890}
8891
8892static int igb_clean_rx_irq(struct igb_q_vector *q_vector, const int budget)
8893{
8894 struct igb_adapter *adapter = q_vector->adapter;
8895 struct igb_ring *rx_ring = q_vector->rx.ring;
8896 struct sk_buff *skb = rx_ring->skb;
8897 unsigned int total_bytes = 0, total_packets = 0;
8898 u16 cleaned_count = igb_desc_unused(rx_ring);
8899 unsigned int xdp_xmit = 0;
8900 struct xdp_buff xdp;
8901 u32 frame_sz = 0;
8902 int rx_buf_pgcnt;
8903
8904 /* Frame size depend on rx_ring setup when PAGE_SIZE=4K */
8905#if (PAGE_SIZE < 8192)
8906 frame_sz = igb_rx_frame_truesize(rx_ring, 0);
8907#endif
8908 xdp_init_buff(&xdp, frame_sz, &rx_ring->xdp_rxq);
8909
8910 while (likely(total_packets < budget)) {
8911 union e1000_adv_rx_desc *rx_desc;
8912 struct igb_rx_buffer *rx_buffer;
8913 ktime_t timestamp = 0;
8914 int pkt_offset = 0;
8915 unsigned int size;
8916 void *pktbuf;
8917
8918 /* return some buffers to hardware, one at a time is too slow */
8919 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
8920 igb_alloc_rx_buffers(rx_ring, cleaned_count);
8921 cleaned_count = 0;
8922 }
8923
8924 rx_desc = IGB_RX_DESC(rx_ring, rx_ring->next_to_clean);
8925 size = le16_to_cpu(rx_desc->wb.upper.length);
8926 if (!size)
8927 break;
8928
8929 /* This memory barrier is needed to keep us from reading
8930 * any other fields out of the rx_desc until we know the
8931 * descriptor has been written back
8932 */
8933 dma_rmb();
8934
8935 rx_buffer = igb_get_rx_buffer(rx_ring, size, &rx_buf_pgcnt);
8936 pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
8937
8938 /* pull rx packet timestamp if available and valid */
8939 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
8940 int ts_hdr_len;
8941
8942 ts_hdr_len = igb_ptp_rx_pktstamp(rx_ring->q_vector,
8943 pktbuf, ×tamp);
8944
8945 pkt_offset += ts_hdr_len;
8946 size -= ts_hdr_len;
8947 }
8948
8949 /* retrieve a buffer from the ring */
8950 if (!skb) {
8951 unsigned char *hard_start = pktbuf - igb_rx_offset(rx_ring);
8952 unsigned int offset = pkt_offset + igb_rx_offset(rx_ring);
8953
8954 xdp_prepare_buff(&xdp, hard_start, offset, size, true);
8955 xdp_buff_clear_frags_flag(&xdp);
8956#if (PAGE_SIZE > 4096)
8957 /* At larger PAGE_SIZE, frame_sz depend on len size */
8958 xdp.frame_sz = igb_rx_frame_truesize(rx_ring, size);
8959#endif
8960 skb = igb_run_xdp(adapter, rx_ring, &xdp);
8961 }
8962
8963 if (IS_ERR(skb)) {
8964 unsigned int xdp_res = -PTR_ERR(skb);
8965
8966 if (xdp_res & (IGB_XDP_TX | IGB_XDP_REDIR)) {
8967 xdp_xmit |= xdp_res;
8968 igb_rx_buffer_flip(rx_ring, rx_buffer, size);
8969 } else {
8970 rx_buffer->pagecnt_bias++;
8971 }
8972 total_packets++;
8973 total_bytes += size;
8974 } else if (skb)
8975 igb_add_rx_frag(rx_ring, rx_buffer, skb, size);
8976 else if (ring_uses_build_skb(rx_ring))
8977 skb = igb_build_skb(rx_ring, rx_buffer, &xdp,
8978 timestamp);
8979 else
8980 skb = igb_construct_skb(rx_ring, rx_buffer,
8981 &xdp, timestamp);
8982
8983 /* exit if we failed to retrieve a buffer */
8984 if (!skb) {
8985 rx_ring->rx_stats.alloc_failed++;
8986 rx_buffer->pagecnt_bias++;
8987 break;
8988 }
8989
8990 igb_put_rx_buffer(rx_ring, rx_buffer, rx_buf_pgcnt);
8991 cleaned_count++;
8992
8993 /* fetch next buffer in frame if non-eop */
8994 if (igb_is_non_eop(rx_ring, rx_desc))
8995 continue;
8996
8997 /* verify the packet layout is correct */
8998 if (igb_cleanup_headers(rx_ring, rx_desc, skb)) {
8999 skb = NULL;
9000 continue;
9001 }
9002
9003 /* probably a little skewed due to removing CRC */
9004 total_bytes += skb->len;
9005
9006 /* populate checksum, timestamp, VLAN, and protocol */
9007 igb_process_skb_fields(rx_ring, rx_desc, skb);
9008
9009 napi_gro_receive(&q_vector->napi, skb);
9010
9011 /* reset skb pointer */
9012 skb = NULL;
9013
9014 /* update budget accounting */
9015 total_packets++;
9016 }
9017
9018 /* place incomplete frames back on ring for completion */
9019 rx_ring->skb = skb;
9020
9021 if (xdp_xmit & IGB_XDP_REDIR)
9022 xdp_do_flush();
9023
9024 if (xdp_xmit & IGB_XDP_TX) {
9025 struct igb_ring *tx_ring = igb_xdp_tx_queue_mapping(adapter);
9026
9027 igb_xdp_ring_update_tail(tx_ring);
9028 }
9029
9030 u64_stats_update_begin(&rx_ring->rx_syncp);
9031 rx_ring->rx_stats.packets += total_packets;
9032 rx_ring->rx_stats.bytes += total_bytes;
9033 u64_stats_update_end(&rx_ring->rx_syncp);
9034 q_vector->rx.total_packets += total_packets;
9035 q_vector->rx.total_bytes += total_bytes;
9036
9037 if (cleaned_count)
9038 igb_alloc_rx_buffers(rx_ring, cleaned_count);
9039
9040 return total_packets;
9041}
9042
9043static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
9044 struct igb_rx_buffer *bi)
9045{
9046 struct page *page = bi->page;
9047 dma_addr_t dma;
9048
9049 /* since we are recycling buffers we should seldom need to alloc */
9050 if (likely(page))
9051 return true;
9052
9053 /* alloc new page for storage */
9054 page = dev_alloc_pages(igb_rx_pg_order(rx_ring));
9055 if (unlikely(!page)) {
9056 rx_ring->rx_stats.alloc_failed++;
9057 return false;
9058 }
9059
9060 /* map page for use */
9061 dma = dma_map_page_attrs(rx_ring->dev, page, 0,
9062 igb_rx_pg_size(rx_ring),
9063 DMA_FROM_DEVICE,
9064 IGB_RX_DMA_ATTR);
9065
9066 /* if mapping failed free memory back to system since
9067 * there isn't much point in holding memory we can't use
9068 */
9069 if (dma_mapping_error(rx_ring->dev, dma)) {
9070 __free_pages(page, igb_rx_pg_order(rx_ring));
9071
9072 rx_ring->rx_stats.alloc_failed++;
9073 return false;
9074 }
9075
9076 bi->dma = dma;
9077 bi->page = page;
9078 bi->page_offset = igb_rx_offset(rx_ring);
9079 page_ref_add(page, USHRT_MAX - 1);
9080 bi->pagecnt_bias = USHRT_MAX;
9081
9082 return true;
9083}
9084
9085/**
9086 * igb_alloc_rx_buffers - Replace used receive buffers
9087 * @rx_ring: rx descriptor ring to allocate new receive buffers
9088 * @cleaned_count: count of buffers to allocate
9089 **/
9090void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
9091{
9092 union e1000_adv_rx_desc *rx_desc;
9093 struct igb_rx_buffer *bi;
9094 u16 i = rx_ring->next_to_use;
9095 u16 bufsz;
9096
9097 /* nothing to do */
9098 if (!cleaned_count)
9099 return;
9100
9101 rx_desc = IGB_RX_DESC(rx_ring, i);
9102 bi = &rx_ring->rx_buffer_info[i];
9103 i -= rx_ring->count;
9104
9105 bufsz = igb_rx_bufsz(rx_ring);
9106
9107 do {
9108 if (!igb_alloc_mapped_page(rx_ring, bi))
9109 break;
9110
9111 /* sync the buffer for use by the device */
9112 dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
9113 bi->page_offset, bufsz,
9114 DMA_FROM_DEVICE);
9115
9116 /* Refresh the desc even if buffer_addrs didn't change
9117 * because each write-back erases this info.
9118 */
9119 rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
9120
9121 rx_desc++;
9122 bi++;
9123 i++;
9124 if (unlikely(!i)) {
9125 rx_desc = IGB_RX_DESC(rx_ring, 0);
9126 bi = rx_ring->rx_buffer_info;
9127 i -= rx_ring->count;
9128 }
9129
9130 /* clear the length for the next_to_use descriptor */
9131 rx_desc->wb.upper.length = 0;
9132
9133 cleaned_count--;
9134 } while (cleaned_count);
9135
9136 i += rx_ring->count;
9137
9138 if (rx_ring->next_to_use != i) {
9139 /* record the next descriptor to use */
9140 rx_ring->next_to_use = i;
9141
9142 /* update next to alloc since we have filled the ring */
9143 rx_ring->next_to_alloc = i;
9144
9145 /* Force memory writes to complete before letting h/w
9146 * know there are new descriptors to fetch. (Only
9147 * applicable for weak-ordered memory model archs,
9148 * such as IA-64).
9149 */
9150 dma_wmb();
9151 writel(i, rx_ring->tail);
9152 }
9153}
9154
9155/**
9156 * igb_mii_ioctl -
9157 * @netdev: pointer to netdev struct
9158 * @ifr: interface structure
9159 * @cmd: ioctl command to execute
9160 **/
9161static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
9162{
9163 struct igb_adapter *adapter = netdev_priv(netdev);
9164 struct mii_ioctl_data *data = if_mii(ifr);
9165
9166 if (adapter->hw.phy.media_type != e1000_media_type_copper)
9167 return -EOPNOTSUPP;
9168
9169 switch (cmd) {
9170 case SIOCGMIIPHY:
9171 data->phy_id = adapter->hw.phy.addr;
9172 break;
9173 case SIOCGMIIREG:
9174 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
9175 &data->val_out))
9176 return -EIO;
9177 break;
9178 case SIOCSMIIREG:
9179 default:
9180 return -EOPNOTSUPP;
9181 }
9182 return 0;
9183}
9184
9185/**
9186 * igb_ioctl -
9187 * @netdev: pointer to netdev struct
9188 * @ifr: interface structure
9189 * @cmd: ioctl command to execute
9190 **/
9191static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
9192{
9193 switch (cmd) {
9194 case SIOCGMIIPHY:
9195 case SIOCGMIIREG:
9196 case SIOCSMIIREG:
9197 return igb_mii_ioctl(netdev, ifr, cmd);
9198 case SIOCGHWTSTAMP:
9199 return igb_ptp_get_ts_config(netdev, ifr);
9200 case SIOCSHWTSTAMP:
9201 return igb_ptp_set_ts_config(netdev, ifr);
9202 default:
9203 return -EOPNOTSUPP;
9204 }
9205}
9206
9207void igb_read_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
9208{
9209 struct igb_adapter *adapter = hw->back;
9210
9211 pci_read_config_word(adapter->pdev, reg, value);
9212}
9213
9214void igb_write_pci_cfg(struct e1000_hw *hw, u32 reg, u16 *value)
9215{
9216 struct igb_adapter *adapter = hw->back;
9217
9218 pci_write_config_word(adapter->pdev, reg, *value);
9219}
9220
9221s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
9222{
9223 struct igb_adapter *adapter = hw->back;
9224
9225 if (pcie_capability_read_word(adapter->pdev, reg, value))
9226 return -E1000_ERR_CONFIG;
9227
9228 return 0;
9229}
9230
9231s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
9232{
9233 struct igb_adapter *adapter = hw->back;
9234
9235 if (pcie_capability_write_word(adapter->pdev, reg, *value))
9236 return -E1000_ERR_CONFIG;
9237
9238 return 0;
9239}
9240
9241static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
9242{
9243 struct igb_adapter *adapter = netdev_priv(netdev);
9244 struct e1000_hw *hw = &adapter->hw;
9245 u32 ctrl, rctl;
9246 bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
9247
9248 if (enable) {
9249 /* enable VLAN tag insert/strip */
9250 ctrl = rd32(E1000_CTRL);
9251 ctrl |= E1000_CTRL_VME;
9252 wr32(E1000_CTRL, ctrl);
9253
9254 /* Disable CFI check */
9255 rctl = rd32(E1000_RCTL);
9256 rctl &= ~E1000_RCTL_CFIEN;
9257 wr32(E1000_RCTL, rctl);
9258 } else {
9259 /* disable VLAN tag insert/strip */
9260 ctrl = rd32(E1000_CTRL);
9261 ctrl &= ~E1000_CTRL_VME;
9262 wr32(E1000_CTRL, ctrl);
9263 }
9264
9265 igb_set_vf_vlan_strip(adapter, adapter->vfs_allocated_count, enable);
9266}
9267
9268static int igb_vlan_rx_add_vid(struct net_device *netdev,
9269 __be16 proto, u16 vid)
9270{
9271 struct igb_adapter *adapter = netdev_priv(netdev);
9272 struct e1000_hw *hw = &adapter->hw;
9273 int pf_id = adapter->vfs_allocated_count;
9274
9275 /* add the filter since PF can receive vlans w/o entry in vlvf */
9276 if (!vid || !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
9277 igb_vfta_set(hw, vid, pf_id, true, !!vid);
9278
9279 set_bit(vid, adapter->active_vlans);
9280
9281 return 0;
9282}
9283
9284static int igb_vlan_rx_kill_vid(struct net_device *netdev,
9285 __be16 proto, u16 vid)
9286{
9287 struct igb_adapter *adapter = netdev_priv(netdev);
9288 int pf_id = adapter->vfs_allocated_count;
9289 struct e1000_hw *hw = &adapter->hw;
9290
9291 /* remove VID from filter table */
9292 if (vid && !(adapter->flags & IGB_FLAG_VLAN_PROMISC))
9293 igb_vfta_set(hw, vid, pf_id, false, true);
9294
9295 clear_bit(vid, adapter->active_vlans);
9296
9297 return 0;
9298}
9299
9300static void igb_restore_vlan(struct igb_adapter *adapter)
9301{
9302 u16 vid = 1;
9303
9304 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
9305 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), 0);
9306
9307 for_each_set_bit_from(vid, adapter->active_vlans, VLAN_N_VID)
9308 igb_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
9309}
9310
9311int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
9312{
9313 struct pci_dev *pdev = adapter->pdev;
9314 struct e1000_mac_info *mac = &adapter->hw.mac;
9315
9316 mac->autoneg = 0;
9317
9318 /* Make sure dplx is at most 1 bit and lsb of speed is not set
9319 * for the switch() below to work
9320 */
9321 if ((spd & 1) || (dplx & ~1))
9322 goto err_inval;
9323
9324 /* Fiber NIC's only allow 1000 gbps Full duplex
9325 * and 100Mbps Full duplex for 100baseFx sfp
9326 */
9327 if (adapter->hw.phy.media_type == e1000_media_type_internal_serdes) {
9328 switch (spd + dplx) {
9329 case SPEED_10 + DUPLEX_HALF:
9330 case SPEED_10 + DUPLEX_FULL:
9331 case SPEED_100 + DUPLEX_HALF:
9332 goto err_inval;
9333 default:
9334 break;
9335 }
9336 }
9337
9338 switch (spd + dplx) {
9339 case SPEED_10 + DUPLEX_HALF:
9340 mac->forced_speed_duplex = ADVERTISE_10_HALF;
9341 break;
9342 case SPEED_10 + DUPLEX_FULL:
9343 mac->forced_speed_duplex = ADVERTISE_10_FULL;
9344 break;
9345 case SPEED_100 + DUPLEX_HALF:
9346 mac->forced_speed_duplex = ADVERTISE_100_HALF;
9347 break;
9348 case SPEED_100 + DUPLEX_FULL:
9349 mac->forced_speed_duplex = ADVERTISE_100_FULL;
9350 break;
9351 case SPEED_1000 + DUPLEX_FULL:
9352 mac->autoneg = 1;
9353 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
9354 break;
9355 case SPEED_1000 + DUPLEX_HALF: /* not supported */
9356 default:
9357 goto err_inval;
9358 }
9359
9360 /* clear MDI, MDI(-X) override is only allowed when autoneg enabled */
9361 adapter->hw.phy.mdix = AUTO_ALL_MODES;
9362
9363 return 0;
9364
9365err_inval:
9366 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
9367 return -EINVAL;
9368}
9369
9370static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
9371 bool runtime)
9372{
9373 struct net_device *netdev = pci_get_drvdata(pdev);
9374 struct igb_adapter *adapter = netdev_priv(netdev);
9375 struct e1000_hw *hw = &adapter->hw;
9376 u32 ctrl, rctl, status;
9377 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
9378 bool wake;
9379
9380 rtnl_lock();
9381 netif_device_detach(netdev);
9382
9383 if (netif_running(netdev))
9384 __igb_close(netdev, true);
9385
9386 igb_ptp_suspend(adapter);
9387
9388 igb_clear_interrupt_scheme(adapter);
9389 rtnl_unlock();
9390
9391 status = rd32(E1000_STATUS);
9392 if (status & E1000_STATUS_LU)
9393 wufc &= ~E1000_WUFC_LNKC;
9394
9395 if (wufc) {
9396 igb_setup_rctl(adapter);
9397 igb_set_rx_mode(netdev);
9398
9399 /* turn on all-multi mode if wake on multicast is enabled */
9400 if (wufc & E1000_WUFC_MC) {
9401 rctl = rd32(E1000_RCTL);
9402 rctl |= E1000_RCTL_MPE;
9403 wr32(E1000_RCTL, rctl);
9404 }
9405
9406 ctrl = rd32(E1000_CTRL);
9407 ctrl |= E1000_CTRL_ADVD3WUC;
9408 wr32(E1000_CTRL, ctrl);
9409
9410 /* Allow time for pending master requests to run */
9411 igb_disable_pcie_master(hw);
9412
9413 wr32(E1000_WUC, E1000_WUC_PME_EN);
9414 wr32(E1000_WUFC, wufc);
9415 } else {
9416 wr32(E1000_WUC, 0);
9417 wr32(E1000_WUFC, 0);
9418 }
9419
9420 wake = wufc || adapter->en_mng_pt;
9421 if (!wake)
9422 igb_power_down_link(adapter);
9423 else
9424 igb_power_up_link(adapter);
9425
9426 if (enable_wake)
9427 *enable_wake = wake;
9428
9429 /* Release control of h/w to f/w. If f/w is AMT enabled, this
9430 * would have already happened in close and is redundant.
9431 */
9432 igb_release_hw_control(adapter);
9433
9434 pci_disable_device(pdev);
9435
9436 return 0;
9437}
9438
9439static void igb_deliver_wake_packet(struct net_device *netdev)
9440{
9441 struct igb_adapter *adapter = netdev_priv(netdev);
9442 struct e1000_hw *hw = &adapter->hw;
9443 struct sk_buff *skb;
9444 u32 wupl;
9445
9446 wupl = rd32(E1000_WUPL) & E1000_WUPL_MASK;
9447
9448 /* WUPM stores only the first 128 bytes of the wake packet.
9449 * Read the packet only if we have the whole thing.
9450 */
9451 if ((wupl == 0) || (wupl > E1000_WUPM_BYTES))
9452 return;
9453
9454 skb = netdev_alloc_skb_ip_align(netdev, E1000_WUPM_BYTES);
9455 if (!skb)
9456 return;
9457
9458 skb_put(skb, wupl);
9459
9460 /* Ensure reads are 32-bit aligned */
9461 wupl = roundup(wupl, 4);
9462
9463 memcpy_fromio(skb->data, hw->hw_addr + E1000_WUPM_REG(0), wupl);
9464
9465 skb->protocol = eth_type_trans(skb, netdev);
9466 netif_rx(skb);
9467}
9468
9469static int __maybe_unused igb_suspend(struct device *dev)
9470{
9471 return __igb_shutdown(to_pci_dev(dev), NULL, 0);
9472}
9473
9474static int __maybe_unused __igb_resume(struct device *dev, bool rpm)
9475{
9476 struct pci_dev *pdev = to_pci_dev(dev);
9477 struct net_device *netdev = pci_get_drvdata(pdev);
9478 struct igb_adapter *adapter = netdev_priv(netdev);
9479 struct e1000_hw *hw = &adapter->hw;
9480 u32 err, val;
9481
9482 pci_set_power_state(pdev, PCI_D0);
9483 pci_restore_state(pdev);
9484 pci_save_state(pdev);
9485
9486 if (!pci_device_is_present(pdev))
9487 return -ENODEV;
9488 err = pci_enable_device_mem(pdev);
9489 if (err) {
9490 dev_err(&pdev->dev,
9491 "igb: Cannot enable PCI device from suspend\n");
9492 return err;
9493 }
9494 pci_set_master(pdev);
9495
9496 pci_enable_wake(pdev, PCI_D3hot, 0);
9497 pci_enable_wake(pdev, PCI_D3cold, 0);
9498
9499 if (igb_init_interrupt_scheme(adapter, true)) {
9500 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
9501 return -ENOMEM;
9502 }
9503
9504 igb_reset(adapter);
9505
9506 /* let the f/w know that the h/w is now under the control of the
9507 * driver.
9508 */
9509 igb_get_hw_control(adapter);
9510
9511 val = rd32(E1000_WUS);
9512 if (val & WAKE_PKT_WUS)
9513 igb_deliver_wake_packet(netdev);
9514
9515 wr32(E1000_WUS, ~0);
9516
9517 if (!rpm)
9518 rtnl_lock();
9519 if (!err && netif_running(netdev))
9520 err = __igb_open(netdev, true);
9521
9522 if (!err)
9523 netif_device_attach(netdev);
9524 if (!rpm)
9525 rtnl_unlock();
9526
9527 return err;
9528}
9529
9530static int __maybe_unused igb_resume(struct device *dev)
9531{
9532 return __igb_resume(dev, false);
9533}
9534
9535static int __maybe_unused igb_runtime_idle(struct device *dev)
9536{
9537 struct net_device *netdev = dev_get_drvdata(dev);
9538 struct igb_adapter *adapter = netdev_priv(netdev);
9539
9540 if (!igb_has_link(adapter))
9541 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
9542
9543 return -EBUSY;
9544}
9545
9546static int __maybe_unused igb_runtime_suspend(struct device *dev)
9547{
9548 return __igb_shutdown(to_pci_dev(dev), NULL, 1);
9549}
9550
9551static int __maybe_unused igb_runtime_resume(struct device *dev)
9552{
9553 return __igb_resume(dev, true);
9554}
9555
9556static void igb_shutdown(struct pci_dev *pdev)
9557{
9558 bool wake;
9559
9560 __igb_shutdown(pdev, &wake, 0);
9561
9562 if (system_state == SYSTEM_POWER_OFF) {
9563 pci_wake_from_d3(pdev, wake);
9564 pci_set_power_state(pdev, PCI_D3hot);
9565 }
9566}
9567
9568static int igb_pci_sriov_configure(struct pci_dev *dev, int num_vfs)
9569{
9570#ifdef CONFIG_PCI_IOV
9571 int err;
9572
9573 if (num_vfs == 0) {
9574 return igb_disable_sriov(dev, true);
9575 } else {
9576 err = igb_enable_sriov(dev, num_vfs, true);
9577 return err ? err : num_vfs;
9578 }
9579#endif
9580 return 0;
9581}
9582
9583/**
9584 * igb_io_error_detected - called when PCI error is detected
9585 * @pdev: Pointer to PCI device
9586 * @state: The current pci connection state
9587 *
9588 * This function is called after a PCI bus error affecting
9589 * this device has been detected.
9590 **/
9591static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
9592 pci_channel_state_t state)
9593{
9594 struct net_device *netdev = pci_get_drvdata(pdev);
9595 struct igb_adapter *adapter = netdev_priv(netdev);
9596
9597 if (state == pci_channel_io_normal) {
9598 dev_warn(&pdev->dev, "Non-correctable non-fatal error reported.\n");
9599 return PCI_ERS_RESULT_CAN_RECOVER;
9600 }
9601
9602 netif_device_detach(netdev);
9603
9604 if (state == pci_channel_io_perm_failure)
9605 return PCI_ERS_RESULT_DISCONNECT;
9606
9607 if (netif_running(netdev))
9608 igb_down(adapter);
9609 pci_disable_device(pdev);
9610
9611 /* Request a slot reset. */
9612 return PCI_ERS_RESULT_NEED_RESET;
9613}
9614
9615/**
9616 * igb_io_slot_reset - called after the pci bus has been reset.
9617 * @pdev: Pointer to PCI device
9618 *
9619 * Restart the card from scratch, as if from a cold-boot. Implementation
9620 * resembles the first-half of the __igb_resume routine.
9621 **/
9622static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
9623{
9624 struct net_device *netdev = pci_get_drvdata(pdev);
9625 struct igb_adapter *adapter = netdev_priv(netdev);
9626 struct e1000_hw *hw = &adapter->hw;
9627 pci_ers_result_t result;
9628
9629 if (pci_enable_device_mem(pdev)) {
9630 dev_err(&pdev->dev,
9631 "Cannot re-enable PCI device after reset.\n");
9632 result = PCI_ERS_RESULT_DISCONNECT;
9633 } else {
9634 pci_set_master(pdev);
9635 pci_restore_state(pdev);
9636 pci_save_state(pdev);
9637
9638 pci_enable_wake(pdev, PCI_D3hot, 0);
9639 pci_enable_wake(pdev, PCI_D3cold, 0);
9640
9641 /* In case of PCI error, adapter lose its HW address
9642 * so we should re-assign it here.
9643 */
9644 hw->hw_addr = adapter->io_addr;
9645
9646 igb_reset(adapter);
9647 wr32(E1000_WUS, ~0);
9648 result = PCI_ERS_RESULT_RECOVERED;
9649 }
9650
9651 return result;
9652}
9653
9654/**
9655 * igb_io_resume - called when traffic can start flowing again.
9656 * @pdev: Pointer to PCI device
9657 *
9658 * This callback is called when the error recovery driver tells us that
9659 * its OK to resume normal operation. Implementation resembles the
9660 * second-half of the __igb_resume routine.
9661 */
9662static void igb_io_resume(struct pci_dev *pdev)
9663{
9664 struct net_device *netdev = pci_get_drvdata(pdev);
9665 struct igb_adapter *adapter = netdev_priv(netdev);
9666
9667 if (netif_running(netdev)) {
9668 if (igb_up(adapter)) {
9669 dev_err(&pdev->dev, "igb_up failed after reset\n");
9670 return;
9671 }
9672 }
9673
9674 netif_device_attach(netdev);
9675
9676 /* let the f/w know that the h/w is now under the control of the
9677 * driver.
9678 */
9679 igb_get_hw_control(adapter);
9680}
9681
9682/**
9683 * igb_rar_set_index - Sync RAL[index] and RAH[index] registers with MAC table
9684 * @adapter: Pointer to adapter structure
9685 * @index: Index of the RAR entry which need to be synced with MAC table
9686 **/
9687static void igb_rar_set_index(struct igb_adapter *adapter, u32 index)
9688{
9689 struct e1000_hw *hw = &adapter->hw;
9690 u32 rar_low, rar_high;
9691 u8 *addr = adapter->mac_table[index].addr;
9692
9693 /* HW expects these to be in network order when they are plugged
9694 * into the registers which are little endian. In order to guarantee
9695 * that ordering we need to do an leXX_to_cpup here in order to be
9696 * ready for the byteswap that occurs with writel
9697 */
9698 rar_low = le32_to_cpup((__le32 *)(addr));
9699 rar_high = le16_to_cpup((__le16 *)(addr + 4));
9700
9701 /* Indicate to hardware the Address is Valid. */
9702 if (adapter->mac_table[index].state & IGB_MAC_STATE_IN_USE) {
9703 if (is_valid_ether_addr(addr))
9704 rar_high |= E1000_RAH_AV;
9705
9706 if (adapter->mac_table[index].state & IGB_MAC_STATE_SRC_ADDR)
9707 rar_high |= E1000_RAH_ASEL_SRC_ADDR;
9708
9709 switch (hw->mac.type) {
9710 case e1000_82575:
9711 case e1000_i210:
9712 if (adapter->mac_table[index].state &
9713 IGB_MAC_STATE_QUEUE_STEERING)
9714 rar_high |= E1000_RAH_QSEL_ENABLE;
9715
9716 rar_high |= E1000_RAH_POOL_1 *
9717 adapter->mac_table[index].queue;
9718 break;
9719 default:
9720 rar_high |= E1000_RAH_POOL_1 <<
9721 adapter->mac_table[index].queue;
9722 break;
9723 }
9724 }
9725
9726 wr32(E1000_RAL(index), rar_low);
9727 wrfl();
9728 wr32(E1000_RAH(index), rar_high);
9729 wrfl();
9730}
9731
9732static int igb_set_vf_mac(struct igb_adapter *adapter,
9733 int vf, unsigned char *mac_addr)
9734{
9735 struct e1000_hw *hw = &adapter->hw;
9736 /* VF MAC addresses start at end of receive addresses and moves
9737 * towards the first, as a result a collision should not be possible
9738 */
9739 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
9740 unsigned char *vf_mac_addr = adapter->vf_data[vf].vf_mac_addresses;
9741
9742 ether_addr_copy(vf_mac_addr, mac_addr);
9743 ether_addr_copy(adapter->mac_table[rar_entry].addr, mac_addr);
9744 adapter->mac_table[rar_entry].queue = vf;
9745 adapter->mac_table[rar_entry].state |= IGB_MAC_STATE_IN_USE;
9746 igb_rar_set_index(adapter, rar_entry);
9747
9748 return 0;
9749}
9750
9751static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
9752{
9753 struct igb_adapter *adapter = netdev_priv(netdev);
9754
9755 if (vf >= adapter->vfs_allocated_count)
9756 return -EINVAL;
9757
9758 /* Setting the VF MAC to 0 reverts the IGB_VF_FLAG_PF_SET_MAC
9759 * flag and allows to overwrite the MAC via VF netdev. This
9760 * is necessary to allow libvirt a way to restore the original
9761 * MAC after unbinding vfio-pci and reloading igbvf after shutting
9762 * down a VM.
9763 */
9764 if (is_zero_ether_addr(mac)) {
9765 adapter->vf_data[vf].flags &= ~IGB_VF_FLAG_PF_SET_MAC;
9766 dev_info(&adapter->pdev->dev,
9767 "remove administratively set MAC on VF %d\n",
9768 vf);
9769 } else if (is_valid_ether_addr(mac)) {
9770 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
9771 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n",
9772 mac, vf);
9773 dev_info(&adapter->pdev->dev,
9774 "Reload the VF driver to make this change effective.");
9775 /* Generate additional warning if PF is down */
9776 if (test_bit(__IGB_DOWN, &adapter->state)) {
9777 dev_warn(&adapter->pdev->dev,
9778 "The VF MAC address has been set, but the PF device is not up.\n");
9779 dev_warn(&adapter->pdev->dev,
9780 "Bring the PF device up before attempting to use the VF device.\n");
9781 }
9782 } else {
9783 return -EINVAL;
9784 }
9785 return igb_set_vf_mac(adapter, vf, mac);
9786}
9787
9788static int igb_link_mbps(int internal_link_speed)
9789{
9790 switch (internal_link_speed) {
9791 case SPEED_100:
9792 return 100;
9793 case SPEED_1000:
9794 return 1000;
9795 default:
9796 return 0;
9797 }
9798}
9799
9800static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
9801 int link_speed)
9802{
9803 int rf_dec, rf_int;
9804 u32 bcnrc_val;
9805
9806 if (tx_rate != 0) {
9807 /* Calculate the rate factor values to set */
9808 rf_int = link_speed / tx_rate;
9809 rf_dec = (link_speed - (rf_int * tx_rate));
9810 rf_dec = (rf_dec * BIT(E1000_RTTBCNRC_RF_INT_SHIFT)) /
9811 tx_rate;
9812
9813 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
9814 bcnrc_val |= FIELD_PREP(E1000_RTTBCNRC_RF_INT_MASK, rf_int);
9815 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
9816 } else {
9817 bcnrc_val = 0;
9818 }
9819
9820 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
9821 /* Set global transmit compensation time to the MMW_SIZE in RTTBCNRM
9822 * register. MMW_SIZE=0x014 if 9728-byte jumbo is supported.
9823 */
9824 wr32(E1000_RTTBCNRM, 0x14);
9825 wr32(E1000_RTTBCNRC, bcnrc_val);
9826}
9827
9828static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
9829{
9830 int actual_link_speed, i;
9831 bool reset_rate = false;
9832
9833 /* VF TX rate limit was not set or not supported */
9834 if ((adapter->vf_rate_link_speed == 0) ||
9835 (adapter->hw.mac.type != e1000_82576))
9836 return;
9837
9838 actual_link_speed = igb_link_mbps(adapter->link_speed);
9839 if (actual_link_speed != adapter->vf_rate_link_speed) {
9840 reset_rate = true;
9841 adapter->vf_rate_link_speed = 0;
9842 dev_info(&adapter->pdev->dev,
9843 "Link speed has been changed. VF Transmit rate is disabled\n");
9844 }
9845
9846 for (i = 0; i < adapter->vfs_allocated_count; i++) {
9847 if (reset_rate)
9848 adapter->vf_data[i].tx_rate = 0;
9849
9850 igb_set_vf_rate_limit(&adapter->hw, i,
9851 adapter->vf_data[i].tx_rate,
9852 actual_link_speed);
9853 }
9854}
9855
9856static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf,
9857 int min_tx_rate, int max_tx_rate)
9858{
9859 struct igb_adapter *adapter = netdev_priv(netdev);
9860 struct e1000_hw *hw = &adapter->hw;
9861 int actual_link_speed;
9862
9863 if (hw->mac.type != e1000_82576)
9864 return -EOPNOTSUPP;
9865
9866 if (min_tx_rate)
9867 return -EINVAL;
9868
9869 actual_link_speed = igb_link_mbps(adapter->link_speed);
9870 if ((vf >= adapter->vfs_allocated_count) ||
9871 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
9872 (max_tx_rate < 0) ||
9873 (max_tx_rate > actual_link_speed))
9874 return -EINVAL;
9875
9876 adapter->vf_rate_link_speed = actual_link_speed;
9877 adapter->vf_data[vf].tx_rate = (u16)max_tx_rate;
9878 igb_set_vf_rate_limit(hw, vf, max_tx_rate, actual_link_speed);
9879
9880 return 0;
9881}
9882
9883static int igb_ndo_set_vf_spoofchk(struct net_device *netdev, int vf,
9884 bool setting)
9885{
9886 struct igb_adapter *adapter = netdev_priv(netdev);
9887 struct e1000_hw *hw = &adapter->hw;
9888 u32 reg_val, reg_offset;
9889
9890 if (!adapter->vfs_allocated_count)
9891 return -EOPNOTSUPP;
9892
9893 if (vf >= adapter->vfs_allocated_count)
9894 return -EINVAL;
9895
9896 reg_offset = (hw->mac.type == e1000_82576) ? E1000_DTXSWC : E1000_TXSWC;
9897 reg_val = rd32(reg_offset);
9898 if (setting)
9899 reg_val |= (BIT(vf) |
9900 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9901 else
9902 reg_val &= ~(BIT(vf) |
9903 BIT(vf + E1000_DTXSWC_VLAN_SPOOF_SHIFT));
9904 wr32(reg_offset, reg_val);
9905
9906 adapter->vf_data[vf].spoofchk_enabled = setting;
9907 return 0;
9908}
9909
9910static int igb_ndo_set_vf_trust(struct net_device *netdev, int vf, bool setting)
9911{
9912 struct igb_adapter *adapter = netdev_priv(netdev);
9913
9914 if (vf >= adapter->vfs_allocated_count)
9915 return -EINVAL;
9916 if (adapter->vf_data[vf].trusted == setting)
9917 return 0;
9918
9919 adapter->vf_data[vf].trusted = setting;
9920
9921 dev_info(&adapter->pdev->dev, "VF %u is %strusted\n",
9922 vf, setting ? "" : "not ");
9923 return 0;
9924}
9925
9926static int igb_ndo_get_vf_config(struct net_device *netdev,
9927 int vf, struct ifla_vf_info *ivi)
9928{
9929 struct igb_adapter *adapter = netdev_priv(netdev);
9930 if (vf >= adapter->vfs_allocated_count)
9931 return -EINVAL;
9932 ivi->vf = vf;
9933 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
9934 ivi->max_tx_rate = adapter->vf_data[vf].tx_rate;
9935 ivi->min_tx_rate = 0;
9936 ivi->vlan = adapter->vf_data[vf].pf_vlan;
9937 ivi->qos = adapter->vf_data[vf].pf_qos;
9938 ivi->spoofchk = adapter->vf_data[vf].spoofchk_enabled;
9939 ivi->trusted = adapter->vf_data[vf].trusted;
9940 return 0;
9941}
9942
9943static void igb_vmm_control(struct igb_adapter *adapter)
9944{
9945 struct e1000_hw *hw = &adapter->hw;
9946 u32 reg;
9947
9948 switch (hw->mac.type) {
9949 case e1000_82575:
9950 case e1000_i210:
9951 case e1000_i211:
9952 case e1000_i354:
9953 default:
9954 /* replication is not supported for 82575 */
9955 return;
9956 case e1000_82576:
9957 /* notify HW that the MAC is adding vlan tags */
9958 reg = rd32(E1000_DTXCTL);
9959 reg |= E1000_DTXCTL_VLAN_ADDED;
9960 wr32(E1000_DTXCTL, reg);
9961 fallthrough;
9962 case e1000_82580:
9963 /* enable replication vlan tag stripping */
9964 reg = rd32(E1000_RPLOLR);
9965 reg |= E1000_RPLOLR_STRVLAN;
9966 wr32(E1000_RPLOLR, reg);
9967 fallthrough;
9968 case e1000_i350:
9969 /* none of the above registers are supported by i350 */
9970 break;
9971 }
9972
9973 if (adapter->vfs_allocated_count) {
9974 igb_vmdq_set_loopback_pf(hw, true);
9975 igb_vmdq_set_replication_pf(hw, true);
9976 igb_vmdq_set_anti_spoofing_pf(hw, true,
9977 adapter->vfs_allocated_count);
9978 } else {
9979 igb_vmdq_set_loopback_pf(hw, false);
9980 igb_vmdq_set_replication_pf(hw, false);
9981 }
9982}
9983
9984static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
9985{
9986 struct e1000_hw *hw = &adapter->hw;
9987 u32 dmac_thr;
9988 u16 hwm;
9989 u32 reg;
9990
9991 if (hw->mac.type > e1000_82580) {
9992 if (adapter->flags & IGB_FLAG_DMAC) {
9993 /* force threshold to 0. */
9994 wr32(E1000_DMCTXTH, 0);
9995
9996 /* DMA Coalescing high water mark needs to be greater
9997 * than the Rx threshold. Set hwm to PBA - max frame
9998 * size in 16B units, capping it at PBA - 6KB.
9999 */
10000 hwm = 64 * (pba - 6);
10001 reg = rd32(E1000_FCRTC);
10002 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
10003 reg |= FIELD_PREP(E1000_FCRTC_RTH_COAL_MASK, hwm);
10004 wr32(E1000_FCRTC, reg);
10005
10006 /* Set the DMA Coalescing Rx threshold to PBA - 2 * max
10007 * frame size, capping it at PBA - 10KB.
10008 */
10009 dmac_thr = pba - 10;
10010 reg = rd32(E1000_DMACR);
10011 reg &= ~E1000_DMACR_DMACTHR_MASK;
10012 reg |= FIELD_PREP(E1000_DMACR_DMACTHR_MASK, dmac_thr);
10013
10014 /* transition to L0x or L1 if available..*/
10015 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
10016
10017 /* watchdog timer= +-1000 usec in 32usec intervals */
10018 reg |= (1000 >> 5);
10019
10020 /* Disable BMC-to-OS Watchdog Enable */
10021 if (hw->mac.type != e1000_i354)
10022 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
10023 wr32(E1000_DMACR, reg);
10024
10025 /* no lower threshold to disable
10026 * coalescing(smart fifb)-UTRESH=0
10027 */
10028 wr32(E1000_DMCRTRH, 0);
10029
10030 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
10031
10032 wr32(E1000_DMCTLX, reg);
10033
10034 /* free space in tx packet buffer to wake from
10035 * DMA coal
10036 */
10037 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
10038 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
10039 }
10040
10041 if (hw->mac.type >= e1000_i210 ||
10042 (adapter->flags & IGB_FLAG_DMAC)) {
10043 reg = rd32(E1000_PCIEMISC);
10044 reg |= E1000_PCIEMISC_LX_DECISION;
10045 wr32(E1000_PCIEMISC, reg);
10046 } /* endif adapter->dmac is not disabled */
10047 } else if (hw->mac.type == e1000_82580) {
10048 u32 reg = rd32(E1000_PCIEMISC);
10049
10050 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
10051 wr32(E1000_DMACR, 0);
10052 }
10053}
10054
10055/**
10056 * igb_read_i2c_byte - Reads 8 bit word over I2C
10057 * @hw: pointer to hardware structure
10058 * @byte_offset: byte offset to read
10059 * @dev_addr: device address
10060 * @data: value read
10061 *
10062 * Performs byte read operation over I2C interface at
10063 * a specified device address.
10064 **/
10065s32 igb_read_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
10066 u8 dev_addr, u8 *data)
10067{
10068 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
10069 struct i2c_client *this_client = adapter->i2c_client;
10070 s32 status;
10071 u16 swfw_mask = 0;
10072
10073 if (!this_client)
10074 return E1000_ERR_I2C;
10075
10076 swfw_mask = E1000_SWFW_PHY0_SM;
10077
10078 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
10079 return E1000_ERR_SWFW_SYNC;
10080
10081 status = i2c_smbus_read_byte_data(this_client, byte_offset);
10082 hw->mac.ops.release_swfw_sync(hw, swfw_mask);
10083
10084 if (status < 0)
10085 return E1000_ERR_I2C;
10086 else {
10087 *data = status;
10088 return 0;
10089 }
10090}
10091
10092/**
10093 * igb_write_i2c_byte - Writes 8 bit word over I2C
10094 * @hw: pointer to hardware structure
10095 * @byte_offset: byte offset to write
10096 * @dev_addr: device address
10097 * @data: value to write
10098 *
10099 * Performs byte write operation over I2C interface at
10100 * a specified device address.
10101 **/
10102s32 igb_write_i2c_byte(struct e1000_hw *hw, u8 byte_offset,
10103 u8 dev_addr, u8 data)
10104{
10105 struct igb_adapter *adapter = container_of(hw, struct igb_adapter, hw);
10106 struct i2c_client *this_client = adapter->i2c_client;
10107 s32 status;
10108 u16 swfw_mask = E1000_SWFW_PHY0_SM;
10109
10110 if (!this_client)
10111 return E1000_ERR_I2C;
10112
10113 if (hw->mac.ops.acquire_swfw_sync(hw, swfw_mask))
10114 return E1000_ERR_SWFW_SYNC;
10115 status = i2c_smbus_write_byte_data(this_client, byte_offset, data);
10116 hw->mac.ops.release_swfw_sync(hw, swfw_mask);
10117
10118 if (status)
10119 return E1000_ERR_I2C;
10120 else
10121 return 0;
10122
10123}
10124
10125int igb_reinit_queues(struct igb_adapter *adapter)
10126{
10127 struct net_device *netdev = adapter->netdev;
10128 struct pci_dev *pdev = adapter->pdev;
10129 int err = 0;
10130
10131 if (netif_running(netdev))
10132 igb_close(netdev);
10133
10134 igb_reset_interrupt_capability(adapter);
10135
10136 if (igb_init_interrupt_scheme(adapter, true)) {
10137 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
10138 return -ENOMEM;
10139 }
10140
10141 if (netif_running(netdev))
10142 err = igb_open(netdev);
10143
10144 return err;
10145}
10146
10147static void igb_nfc_filter_exit(struct igb_adapter *adapter)
10148{
10149 struct igb_nfc_filter *rule;
10150
10151 spin_lock(&adapter->nfc_lock);
10152
10153 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
10154 igb_erase_filter(adapter, rule);
10155
10156 hlist_for_each_entry(rule, &adapter->cls_flower_list, nfc_node)
10157 igb_erase_filter(adapter, rule);
10158
10159 spin_unlock(&adapter->nfc_lock);
10160}
10161
10162static void igb_nfc_filter_restore(struct igb_adapter *adapter)
10163{
10164 struct igb_nfc_filter *rule;
10165
10166 spin_lock(&adapter->nfc_lock);
10167
10168 hlist_for_each_entry(rule, &adapter->nfc_filter_list, nfc_node)
10169 igb_add_filter(adapter, rule);
10170
10171 spin_unlock(&adapter->nfc_lock);
10172}
10173/* igb_main.c */
1/*******************************************************************************
2
3 Intel(R) Gigabit Ethernet Linux driver
4 Copyright(c) 2007-2012 Intel Corporation.
5
6 This program is free software; you can redistribute it and/or modify it
7 under the terms and conditions of the GNU General Public License,
8 version 2, as published by the Free Software Foundation.
9
10 This program is distributed in the hope it will be useful, but WITHOUT
11 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 more details.
14
15 You should have received a copy of the GNU General Public License along with
16 this program; if not, write to the Free Software Foundation, Inc.,
17 51 Franklin St - Fifth Floor, Boston, MA 02110-1301 USA.
18
19 The full GNU General Public License is included in this distribution in
20 the file called "COPYING".
21
22 Contact Information:
23 e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
24 Intel Corporation, 5200 N.E. Elam Young Parkway, Hillsboro, OR 97124-6497
25
26*******************************************************************************/
27
28#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
29
30#include <linux/module.h>
31#include <linux/types.h>
32#include <linux/init.h>
33#include <linux/bitops.h>
34#include <linux/vmalloc.h>
35#include <linux/pagemap.h>
36#include <linux/netdevice.h>
37#include <linux/ipv6.h>
38#include <linux/slab.h>
39#include <net/checksum.h>
40#include <net/ip6_checksum.h>
41#include <linux/net_tstamp.h>
42#include <linux/mii.h>
43#include <linux/ethtool.h>
44#include <linux/if.h>
45#include <linux/if_vlan.h>
46#include <linux/pci.h>
47#include <linux/pci-aspm.h>
48#include <linux/delay.h>
49#include <linux/interrupt.h>
50#include <linux/ip.h>
51#include <linux/tcp.h>
52#include <linux/sctp.h>
53#include <linux/if_ether.h>
54#include <linux/aer.h>
55#include <linux/prefetch.h>
56#include <linux/pm_runtime.h>
57#ifdef CONFIG_IGB_DCA
58#include <linux/dca.h>
59#endif
60#include "igb.h"
61
62#define MAJ 3
63#define MIN 4
64#define BUILD 7
65#define DRV_VERSION __stringify(MAJ) "." __stringify(MIN) "." \
66__stringify(BUILD) "-k"
67char igb_driver_name[] = "igb";
68char igb_driver_version[] = DRV_VERSION;
69static const char igb_driver_string[] =
70 "Intel(R) Gigabit Ethernet Network Driver";
71static const char igb_copyright[] = "Copyright (c) 2007-2012 Intel Corporation.";
72
73static const struct e1000_info *igb_info_tbl[] = {
74 [board_82575] = &e1000_82575_info,
75};
76
77static DEFINE_PCI_DEVICE_TABLE(igb_pci_tbl) = {
78 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I211_COPPER), board_82575 },
79 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_COPPER), board_82575 },
80 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_FIBER), board_82575 },
81 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SERDES), board_82575 },
82 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I210_SGMII), board_82575 },
83 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_COPPER), board_82575 },
84 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_FIBER), board_82575 },
85 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SERDES), board_82575 },
86 { PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_SGMII), board_82575 },
87 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER), board_82575 },
88 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_FIBER), board_82575 },
89 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_QUAD_FIBER), board_82575 },
90 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SERDES), board_82575 },
91 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_SGMII), board_82575 },
92 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82580_COPPER_DUAL), board_82575 },
93 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SGMII), board_82575 },
94 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SERDES), board_82575 },
95 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_BACKPLANE), board_82575 },
96 { PCI_VDEVICE(INTEL, E1000_DEV_ID_DH89XXCC_SFP), board_82575 },
97 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576), board_82575 },
98 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS), board_82575 },
99 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_NS_SERDES), board_82575 },
100 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_FIBER), board_82575 },
101 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES), board_82575 },
102 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_SERDES_QUAD), board_82575 },
103 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER_ET2), board_82575 },
104 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_QUAD_COPPER), board_82575 },
105 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_COPPER), board_82575 },
106 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575EB_FIBER_SERDES), board_82575 },
107 { PCI_VDEVICE(INTEL, E1000_DEV_ID_82575GB_QUAD_COPPER), board_82575 },
108 /* required last entry */
109 {0, }
110};
111
112MODULE_DEVICE_TABLE(pci, igb_pci_tbl);
113
114void igb_reset(struct igb_adapter *);
115static int igb_setup_all_tx_resources(struct igb_adapter *);
116static int igb_setup_all_rx_resources(struct igb_adapter *);
117static void igb_free_all_tx_resources(struct igb_adapter *);
118static void igb_free_all_rx_resources(struct igb_adapter *);
119static void igb_setup_mrqc(struct igb_adapter *);
120static int igb_probe(struct pci_dev *, const struct pci_device_id *);
121static void __devexit igb_remove(struct pci_dev *pdev);
122static int igb_sw_init(struct igb_adapter *);
123static int igb_open(struct net_device *);
124static int igb_close(struct net_device *);
125static void igb_configure_tx(struct igb_adapter *);
126static void igb_configure_rx(struct igb_adapter *);
127static void igb_clean_all_tx_rings(struct igb_adapter *);
128static void igb_clean_all_rx_rings(struct igb_adapter *);
129static void igb_clean_tx_ring(struct igb_ring *);
130static void igb_clean_rx_ring(struct igb_ring *);
131static void igb_set_rx_mode(struct net_device *);
132static void igb_update_phy_info(unsigned long);
133static void igb_watchdog(unsigned long);
134static void igb_watchdog_task(struct work_struct *);
135static netdev_tx_t igb_xmit_frame(struct sk_buff *skb, struct net_device *);
136static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *dev,
137 struct rtnl_link_stats64 *stats);
138static int igb_change_mtu(struct net_device *, int);
139static int igb_set_mac(struct net_device *, void *);
140static void igb_set_uta(struct igb_adapter *adapter);
141static irqreturn_t igb_intr(int irq, void *);
142static irqreturn_t igb_intr_msi(int irq, void *);
143static irqreturn_t igb_msix_other(int irq, void *);
144static irqreturn_t igb_msix_ring(int irq, void *);
145#ifdef CONFIG_IGB_DCA
146static void igb_update_dca(struct igb_q_vector *);
147static void igb_setup_dca(struct igb_adapter *);
148#endif /* CONFIG_IGB_DCA */
149static int igb_poll(struct napi_struct *, int);
150static bool igb_clean_tx_irq(struct igb_q_vector *);
151static bool igb_clean_rx_irq(struct igb_q_vector *, int);
152static int igb_ioctl(struct net_device *, struct ifreq *, int cmd);
153static void igb_tx_timeout(struct net_device *);
154static void igb_reset_task(struct work_struct *);
155static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features);
156static int igb_vlan_rx_add_vid(struct net_device *, u16);
157static int igb_vlan_rx_kill_vid(struct net_device *, u16);
158static void igb_restore_vlan(struct igb_adapter *);
159static void igb_rar_set_qsel(struct igb_adapter *, u8 *, u32 , u8);
160static void igb_ping_all_vfs(struct igb_adapter *);
161static void igb_msg_task(struct igb_adapter *);
162static void igb_vmm_control(struct igb_adapter *);
163static int igb_set_vf_mac(struct igb_adapter *, int, unsigned char *);
164static void igb_restore_vf_multicasts(struct igb_adapter *adapter);
165static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac);
166static int igb_ndo_set_vf_vlan(struct net_device *netdev,
167 int vf, u16 vlan, u8 qos);
168static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate);
169static int igb_ndo_get_vf_config(struct net_device *netdev, int vf,
170 struct ifla_vf_info *ivi);
171static void igb_check_vf_rate_limit(struct igb_adapter *);
172
173#ifdef CONFIG_PCI_IOV
174static int igb_vf_configure(struct igb_adapter *adapter, int vf);
175static int igb_find_enabled_vfs(struct igb_adapter *adapter);
176static int igb_check_vf_assignment(struct igb_adapter *adapter);
177#endif
178
179#ifdef CONFIG_PM
180#ifdef CONFIG_PM_SLEEP
181static int igb_suspend(struct device *);
182#endif
183static int igb_resume(struct device *);
184#ifdef CONFIG_PM_RUNTIME
185static int igb_runtime_suspend(struct device *dev);
186static int igb_runtime_resume(struct device *dev);
187static int igb_runtime_idle(struct device *dev);
188#endif
189static const struct dev_pm_ops igb_pm_ops = {
190 SET_SYSTEM_SLEEP_PM_OPS(igb_suspend, igb_resume)
191 SET_RUNTIME_PM_OPS(igb_runtime_suspend, igb_runtime_resume,
192 igb_runtime_idle)
193};
194#endif
195static void igb_shutdown(struct pci_dev *);
196#ifdef CONFIG_IGB_DCA
197static int igb_notify_dca(struct notifier_block *, unsigned long, void *);
198static struct notifier_block dca_notifier = {
199 .notifier_call = igb_notify_dca,
200 .next = NULL,
201 .priority = 0
202};
203#endif
204#ifdef CONFIG_NET_POLL_CONTROLLER
205/* for netdump / net console */
206static void igb_netpoll(struct net_device *);
207#endif
208#ifdef CONFIG_PCI_IOV
209static unsigned int max_vfs = 0;
210module_param(max_vfs, uint, 0);
211MODULE_PARM_DESC(max_vfs, "Maximum number of virtual functions to allocate "
212 "per physical function");
213#endif /* CONFIG_PCI_IOV */
214
215static pci_ers_result_t igb_io_error_detected(struct pci_dev *,
216 pci_channel_state_t);
217static pci_ers_result_t igb_io_slot_reset(struct pci_dev *);
218static void igb_io_resume(struct pci_dev *);
219
220static struct pci_error_handlers igb_err_handler = {
221 .error_detected = igb_io_error_detected,
222 .slot_reset = igb_io_slot_reset,
223 .resume = igb_io_resume,
224};
225
226static void igb_init_dmac(struct igb_adapter *adapter, u32 pba);
227
228static struct pci_driver igb_driver = {
229 .name = igb_driver_name,
230 .id_table = igb_pci_tbl,
231 .probe = igb_probe,
232 .remove = __devexit_p(igb_remove),
233#ifdef CONFIG_PM
234 .driver.pm = &igb_pm_ops,
235#endif
236 .shutdown = igb_shutdown,
237 .err_handler = &igb_err_handler
238};
239
240MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
241MODULE_DESCRIPTION("Intel(R) Gigabit Ethernet Network Driver");
242MODULE_LICENSE("GPL");
243MODULE_VERSION(DRV_VERSION);
244
245#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
246static int debug = -1;
247module_param(debug, int, 0);
248MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
249
250struct igb_reg_info {
251 u32 ofs;
252 char *name;
253};
254
255static const struct igb_reg_info igb_reg_info_tbl[] = {
256
257 /* General Registers */
258 {E1000_CTRL, "CTRL"},
259 {E1000_STATUS, "STATUS"},
260 {E1000_CTRL_EXT, "CTRL_EXT"},
261
262 /* Interrupt Registers */
263 {E1000_ICR, "ICR"},
264
265 /* RX Registers */
266 {E1000_RCTL, "RCTL"},
267 {E1000_RDLEN(0), "RDLEN"},
268 {E1000_RDH(0), "RDH"},
269 {E1000_RDT(0), "RDT"},
270 {E1000_RXDCTL(0), "RXDCTL"},
271 {E1000_RDBAL(0), "RDBAL"},
272 {E1000_RDBAH(0), "RDBAH"},
273
274 /* TX Registers */
275 {E1000_TCTL, "TCTL"},
276 {E1000_TDBAL(0), "TDBAL"},
277 {E1000_TDBAH(0), "TDBAH"},
278 {E1000_TDLEN(0), "TDLEN"},
279 {E1000_TDH(0), "TDH"},
280 {E1000_TDT(0), "TDT"},
281 {E1000_TXDCTL(0), "TXDCTL"},
282 {E1000_TDFH, "TDFH"},
283 {E1000_TDFT, "TDFT"},
284 {E1000_TDFHS, "TDFHS"},
285 {E1000_TDFPC, "TDFPC"},
286
287 /* List Terminator */
288 {}
289};
290
291/*
292 * igb_regdump - register printout routine
293 */
294static void igb_regdump(struct e1000_hw *hw, struct igb_reg_info *reginfo)
295{
296 int n = 0;
297 char rname[16];
298 u32 regs[8];
299
300 switch (reginfo->ofs) {
301 case E1000_RDLEN(0):
302 for (n = 0; n < 4; n++)
303 regs[n] = rd32(E1000_RDLEN(n));
304 break;
305 case E1000_RDH(0):
306 for (n = 0; n < 4; n++)
307 regs[n] = rd32(E1000_RDH(n));
308 break;
309 case E1000_RDT(0):
310 for (n = 0; n < 4; n++)
311 regs[n] = rd32(E1000_RDT(n));
312 break;
313 case E1000_RXDCTL(0):
314 for (n = 0; n < 4; n++)
315 regs[n] = rd32(E1000_RXDCTL(n));
316 break;
317 case E1000_RDBAL(0):
318 for (n = 0; n < 4; n++)
319 regs[n] = rd32(E1000_RDBAL(n));
320 break;
321 case E1000_RDBAH(0):
322 for (n = 0; n < 4; n++)
323 regs[n] = rd32(E1000_RDBAH(n));
324 break;
325 case E1000_TDBAL(0):
326 for (n = 0; n < 4; n++)
327 regs[n] = rd32(E1000_RDBAL(n));
328 break;
329 case E1000_TDBAH(0):
330 for (n = 0; n < 4; n++)
331 regs[n] = rd32(E1000_TDBAH(n));
332 break;
333 case E1000_TDLEN(0):
334 for (n = 0; n < 4; n++)
335 regs[n] = rd32(E1000_TDLEN(n));
336 break;
337 case E1000_TDH(0):
338 for (n = 0; n < 4; n++)
339 regs[n] = rd32(E1000_TDH(n));
340 break;
341 case E1000_TDT(0):
342 for (n = 0; n < 4; n++)
343 regs[n] = rd32(E1000_TDT(n));
344 break;
345 case E1000_TXDCTL(0):
346 for (n = 0; n < 4; n++)
347 regs[n] = rd32(E1000_TXDCTL(n));
348 break;
349 default:
350 pr_info("%-15s %08x\n", reginfo->name, rd32(reginfo->ofs));
351 return;
352 }
353
354 snprintf(rname, 16, "%s%s", reginfo->name, "[0-3]");
355 pr_info("%-15s %08x %08x %08x %08x\n", rname, regs[0], regs[1],
356 regs[2], regs[3]);
357}
358
359/*
360 * igb_dump - Print registers, tx-rings and rx-rings
361 */
362static void igb_dump(struct igb_adapter *adapter)
363{
364 struct net_device *netdev = adapter->netdev;
365 struct e1000_hw *hw = &adapter->hw;
366 struct igb_reg_info *reginfo;
367 struct igb_ring *tx_ring;
368 union e1000_adv_tx_desc *tx_desc;
369 struct my_u0 { u64 a; u64 b; } *u0;
370 struct igb_ring *rx_ring;
371 union e1000_adv_rx_desc *rx_desc;
372 u32 staterr;
373 u16 i, n;
374
375 if (!netif_msg_hw(adapter))
376 return;
377
378 /* Print netdevice Info */
379 if (netdev) {
380 dev_info(&adapter->pdev->dev, "Net device Info\n");
381 pr_info("Device Name state trans_start "
382 "last_rx\n");
383 pr_info("%-15s %016lX %016lX %016lX\n", netdev->name,
384 netdev->state, netdev->trans_start, netdev->last_rx);
385 }
386
387 /* Print Registers */
388 dev_info(&adapter->pdev->dev, "Register Dump\n");
389 pr_info(" Register Name Value\n");
390 for (reginfo = (struct igb_reg_info *)igb_reg_info_tbl;
391 reginfo->name; reginfo++) {
392 igb_regdump(hw, reginfo);
393 }
394
395 /* Print TX Ring Summary */
396 if (!netdev || !netif_running(netdev))
397 goto exit;
398
399 dev_info(&adapter->pdev->dev, "TX Rings Summary\n");
400 pr_info("Queue [NTU] [NTC] [bi(ntc)->dma ] leng ntw timestamp\n");
401 for (n = 0; n < adapter->num_tx_queues; n++) {
402 struct igb_tx_buffer *buffer_info;
403 tx_ring = adapter->tx_ring[n];
404 buffer_info = &tx_ring->tx_buffer_info[tx_ring->next_to_clean];
405 pr_info(" %5d %5X %5X %016llX %04X %p %016llX\n",
406 n, tx_ring->next_to_use, tx_ring->next_to_clean,
407 (u64)buffer_info->dma,
408 buffer_info->length,
409 buffer_info->next_to_watch,
410 (u64)buffer_info->time_stamp);
411 }
412
413 /* Print TX Rings */
414 if (!netif_msg_tx_done(adapter))
415 goto rx_ring_summary;
416
417 dev_info(&adapter->pdev->dev, "TX Rings Dump\n");
418
419 /* Transmit Descriptor Formats
420 *
421 * Advanced Transmit Descriptor
422 * +--------------------------------------------------------------+
423 * 0 | Buffer Address [63:0] |
424 * +--------------------------------------------------------------+
425 * 8 | PAYLEN | PORTS |CC|IDX | STA | DCMD |DTYP|MAC|RSV| DTALEN |
426 * +--------------------------------------------------------------+
427 * 63 46 45 40 39 38 36 35 32 31 24 15 0
428 */
429
430 for (n = 0; n < adapter->num_tx_queues; n++) {
431 tx_ring = adapter->tx_ring[n];
432 pr_info("------------------------------------\n");
433 pr_info("TX QUEUE INDEX = %d\n", tx_ring->queue_index);
434 pr_info("------------------------------------\n");
435 pr_info("T [desc] [address 63:0 ] [PlPOCIStDDM Ln] "
436 "[bi->dma ] leng ntw timestamp "
437 "bi->skb\n");
438
439 for (i = 0; tx_ring->desc && (i < tx_ring->count); i++) {
440 const char *next_desc;
441 struct igb_tx_buffer *buffer_info;
442 tx_desc = IGB_TX_DESC(tx_ring, i);
443 buffer_info = &tx_ring->tx_buffer_info[i];
444 u0 = (struct my_u0 *)tx_desc;
445 if (i == tx_ring->next_to_use &&
446 i == tx_ring->next_to_clean)
447 next_desc = " NTC/U";
448 else if (i == tx_ring->next_to_use)
449 next_desc = " NTU";
450 else if (i == tx_ring->next_to_clean)
451 next_desc = " NTC";
452 else
453 next_desc = "";
454
455 pr_info("T [0x%03X] %016llX %016llX %016llX"
456 " %04X %p %016llX %p%s\n", i,
457 le64_to_cpu(u0->a),
458 le64_to_cpu(u0->b),
459 (u64)buffer_info->dma,
460 buffer_info->length,
461 buffer_info->next_to_watch,
462 (u64)buffer_info->time_stamp,
463 buffer_info->skb, next_desc);
464
465 if (netif_msg_pktdata(adapter) && buffer_info->dma != 0)
466 print_hex_dump(KERN_INFO, "",
467 DUMP_PREFIX_ADDRESS,
468 16, 1, phys_to_virt(buffer_info->dma),
469 buffer_info->length, true);
470 }
471 }
472
473 /* Print RX Rings Summary */
474rx_ring_summary:
475 dev_info(&adapter->pdev->dev, "RX Rings Summary\n");
476 pr_info("Queue [NTU] [NTC]\n");
477 for (n = 0; n < adapter->num_rx_queues; n++) {
478 rx_ring = adapter->rx_ring[n];
479 pr_info(" %5d %5X %5X\n",
480 n, rx_ring->next_to_use, rx_ring->next_to_clean);
481 }
482
483 /* Print RX Rings */
484 if (!netif_msg_rx_status(adapter))
485 goto exit;
486
487 dev_info(&adapter->pdev->dev, "RX Rings Dump\n");
488
489 /* Advanced Receive Descriptor (Read) Format
490 * 63 1 0
491 * +-----------------------------------------------------+
492 * 0 | Packet Buffer Address [63:1] |A0/NSE|
493 * +----------------------------------------------+------+
494 * 8 | Header Buffer Address [63:1] | DD |
495 * +-----------------------------------------------------+
496 *
497 *
498 * Advanced Receive Descriptor (Write-Back) Format
499 *
500 * 63 48 47 32 31 30 21 20 17 16 4 3 0
501 * +------------------------------------------------------+
502 * 0 | Packet IP |SPH| HDR_LEN | RSV|Packet| RSS |
503 * | Checksum Ident | | | | Type | Type |
504 * +------------------------------------------------------+
505 * 8 | VLAN Tag | Length | Extended Error | Extended Status |
506 * +------------------------------------------------------+
507 * 63 48 47 32 31 20 19 0
508 */
509
510 for (n = 0; n < adapter->num_rx_queues; n++) {
511 rx_ring = adapter->rx_ring[n];
512 pr_info("------------------------------------\n");
513 pr_info("RX QUEUE INDEX = %d\n", rx_ring->queue_index);
514 pr_info("------------------------------------\n");
515 pr_info("R [desc] [ PktBuf A0] [ HeadBuf DD] "
516 "[bi->dma ] [bi->skb] <-- Adv Rx Read format\n");
517 pr_info("RWB[desc] [PcsmIpSHl PtRs] [vl er S cks ln] -----"
518 "----------- [bi->skb] <-- Adv Rx Write-Back format\n");
519
520 for (i = 0; i < rx_ring->count; i++) {
521 const char *next_desc;
522 struct igb_rx_buffer *buffer_info;
523 buffer_info = &rx_ring->rx_buffer_info[i];
524 rx_desc = IGB_RX_DESC(rx_ring, i);
525 u0 = (struct my_u0 *)rx_desc;
526 staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
527
528 if (i == rx_ring->next_to_use)
529 next_desc = " NTU";
530 else if (i == rx_ring->next_to_clean)
531 next_desc = " NTC";
532 else
533 next_desc = "";
534
535 if (staterr & E1000_RXD_STAT_DD) {
536 /* Descriptor Done */
537 pr_info("%s[0x%03X] %016llX %016llX -------"
538 "--------- %p%s\n", "RWB", i,
539 le64_to_cpu(u0->a),
540 le64_to_cpu(u0->b),
541 buffer_info->skb, next_desc);
542 } else {
543 pr_info("%s[0x%03X] %016llX %016llX %016llX"
544 " %p%s\n", "R ", i,
545 le64_to_cpu(u0->a),
546 le64_to_cpu(u0->b),
547 (u64)buffer_info->dma,
548 buffer_info->skb, next_desc);
549
550 if (netif_msg_pktdata(adapter)) {
551 print_hex_dump(KERN_INFO, "",
552 DUMP_PREFIX_ADDRESS,
553 16, 1,
554 phys_to_virt(buffer_info->dma),
555 IGB_RX_HDR_LEN, true);
556 print_hex_dump(KERN_INFO, "",
557 DUMP_PREFIX_ADDRESS,
558 16, 1,
559 phys_to_virt(
560 buffer_info->page_dma +
561 buffer_info->page_offset),
562 PAGE_SIZE/2, true);
563 }
564 }
565 }
566 }
567
568exit:
569 return;
570}
571
572/**
573 * igb_get_hw_dev - return device
574 * used by hardware layer to print debugging information
575 **/
576struct net_device *igb_get_hw_dev(struct e1000_hw *hw)
577{
578 struct igb_adapter *adapter = hw->back;
579 return adapter->netdev;
580}
581
582/**
583 * igb_init_module - Driver Registration Routine
584 *
585 * igb_init_module is the first routine called when the driver is
586 * loaded. All it does is register with the PCI subsystem.
587 **/
588static int __init igb_init_module(void)
589{
590 int ret;
591 pr_info("%s - version %s\n",
592 igb_driver_string, igb_driver_version);
593
594 pr_info("%s\n", igb_copyright);
595
596#ifdef CONFIG_IGB_DCA
597 dca_register_notify(&dca_notifier);
598#endif
599 ret = pci_register_driver(&igb_driver);
600 return ret;
601}
602
603module_init(igb_init_module);
604
605/**
606 * igb_exit_module - Driver Exit Cleanup Routine
607 *
608 * igb_exit_module is called just before the driver is removed
609 * from memory.
610 **/
611static void __exit igb_exit_module(void)
612{
613#ifdef CONFIG_IGB_DCA
614 dca_unregister_notify(&dca_notifier);
615#endif
616 pci_unregister_driver(&igb_driver);
617}
618
619module_exit(igb_exit_module);
620
621#define Q_IDX_82576(i) (((i & 0x1) << 3) + (i >> 1))
622/**
623 * igb_cache_ring_register - Descriptor ring to register mapping
624 * @adapter: board private structure to initialize
625 *
626 * Once we know the feature-set enabled for the device, we'll cache
627 * the register offset the descriptor ring is assigned to.
628 **/
629static void igb_cache_ring_register(struct igb_adapter *adapter)
630{
631 int i = 0, j = 0;
632 u32 rbase_offset = adapter->vfs_allocated_count;
633
634 switch (adapter->hw.mac.type) {
635 case e1000_82576:
636 /* The queues are allocated for virtualization such that VF 0
637 * is allocated queues 0 and 8, VF 1 queues 1 and 9, etc.
638 * In order to avoid collision we start at the first free queue
639 * and continue consuming queues in the same sequence
640 */
641 if (adapter->vfs_allocated_count) {
642 for (; i < adapter->rss_queues; i++)
643 adapter->rx_ring[i]->reg_idx = rbase_offset +
644 Q_IDX_82576(i);
645 }
646 case e1000_82575:
647 case e1000_82580:
648 case e1000_i350:
649 case e1000_i210:
650 case e1000_i211:
651 default:
652 for (; i < adapter->num_rx_queues; i++)
653 adapter->rx_ring[i]->reg_idx = rbase_offset + i;
654 for (; j < adapter->num_tx_queues; j++)
655 adapter->tx_ring[j]->reg_idx = rbase_offset + j;
656 break;
657 }
658}
659
660static void igb_free_queues(struct igb_adapter *adapter)
661{
662 int i;
663
664 for (i = 0; i < adapter->num_tx_queues; i++) {
665 kfree(adapter->tx_ring[i]);
666 adapter->tx_ring[i] = NULL;
667 }
668 for (i = 0; i < adapter->num_rx_queues; i++) {
669 kfree(adapter->rx_ring[i]);
670 adapter->rx_ring[i] = NULL;
671 }
672 adapter->num_rx_queues = 0;
673 adapter->num_tx_queues = 0;
674}
675
676/**
677 * igb_alloc_queues - Allocate memory for all rings
678 * @adapter: board private structure to initialize
679 *
680 * We allocate one ring per queue at run-time since we don't know the
681 * number of queues at compile-time.
682 **/
683static int igb_alloc_queues(struct igb_adapter *adapter)
684{
685 struct igb_ring *ring;
686 int i;
687 int orig_node = adapter->node;
688
689 for (i = 0; i < adapter->num_tx_queues; i++) {
690 if (orig_node == -1) {
691 int cur_node = next_online_node(adapter->node);
692 if (cur_node == MAX_NUMNODES)
693 cur_node = first_online_node;
694 adapter->node = cur_node;
695 }
696 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
697 adapter->node);
698 if (!ring)
699 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
700 if (!ring)
701 goto err;
702 ring->count = adapter->tx_ring_count;
703 ring->queue_index = i;
704 ring->dev = &adapter->pdev->dev;
705 ring->netdev = adapter->netdev;
706 ring->numa_node = adapter->node;
707 /* For 82575, context index must be unique per ring. */
708 if (adapter->hw.mac.type == e1000_82575)
709 set_bit(IGB_RING_FLAG_TX_CTX_IDX, &ring->flags);
710 adapter->tx_ring[i] = ring;
711 }
712 /* Restore the adapter's original node */
713 adapter->node = orig_node;
714
715 for (i = 0; i < adapter->num_rx_queues; i++) {
716 if (orig_node == -1) {
717 int cur_node = next_online_node(adapter->node);
718 if (cur_node == MAX_NUMNODES)
719 cur_node = first_online_node;
720 adapter->node = cur_node;
721 }
722 ring = kzalloc_node(sizeof(struct igb_ring), GFP_KERNEL,
723 adapter->node);
724 if (!ring)
725 ring = kzalloc(sizeof(struct igb_ring), GFP_KERNEL);
726 if (!ring)
727 goto err;
728 ring->count = adapter->rx_ring_count;
729 ring->queue_index = i;
730 ring->dev = &adapter->pdev->dev;
731 ring->netdev = adapter->netdev;
732 ring->numa_node = adapter->node;
733 /* set flag indicating ring supports SCTP checksum offload */
734 if (adapter->hw.mac.type >= e1000_82576)
735 set_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags);
736
737 /*
738 * On i350, i210, and i211, loopback VLAN packets
739 * have the tag byte-swapped.
740 * */
741 if (adapter->hw.mac.type >= e1000_i350)
742 set_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags);
743
744 adapter->rx_ring[i] = ring;
745 }
746 /* Restore the adapter's original node */
747 adapter->node = orig_node;
748
749 igb_cache_ring_register(adapter);
750
751 return 0;
752
753err:
754 /* Restore the adapter's original node */
755 adapter->node = orig_node;
756 igb_free_queues(adapter);
757
758 return -ENOMEM;
759}
760
761/**
762 * igb_write_ivar - configure ivar for given MSI-X vector
763 * @hw: pointer to the HW structure
764 * @msix_vector: vector number we are allocating to a given ring
765 * @index: row index of IVAR register to write within IVAR table
766 * @offset: column offset of in IVAR, should be multiple of 8
767 *
768 * This function is intended to handle the writing of the IVAR register
769 * for adapters 82576 and newer. The IVAR table consists of 2 columns,
770 * each containing an cause allocation for an Rx and Tx ring, and a
771 * variable number of rows depending on the number of queues supported.
772 **/
773static void igb_write_ivar(struct e1000_hw *hw, int msix_vector,
774 int index, int offset)
775{
776 u32 ivar = array_rd32(E1000_IVAR0, index);
777
778 /* clear any bits that are currently set */
779 ivar &= ~((u32)0xFF << offset);
780
781 /* write vector and valid bit */
782 ivar |= (msix_vector | E1000_IVAR_VALID) << offset;
783
784 array_wr32(E1000_IVAR0, index, ivar);
785}
786
787#define IGB_N0_QUEUE -1
788static void igb_assign_vector(struct igb_q_vector *q_vector, int msix_vector)
789{
790 struct igb_adapter *adapter = q_vector->adapter;
791 struct e1000_hw *hw = &adapter->hw;
792 int rx_queue = IGB_N0_QUEUE;
793 int tx_queue = IGB_N0_QUEUE;
794 u32 msixbm = 0;
795
796 if (q_vector->rx.ring)
797 rx_queue = q_vector->rx.ring->reg_idx;
798 if (q_vector->tx.ring)
799 tx_queue = q_vector->tx.ring->reg_idx;
800
801 switch (hw->mac.type) {
802 case e1000_82575:
803 /* The 82575 assigns vectors using a bitmask, which matches the
804 bitmask for the EICR/EIMS/EIMC registers. To assign one
805 or more queues to a vector, we write the appropriate bits
806 into the MSIXBM register for that vector. */
807 if (rx_queue > IGB_N0_QUEUE)
808 msixbm = E1000_EICR_RX_QUEUE0 << rx_queue;
809 if (tx_queue > IGB_N0_QUEUE)
810 msixbm |= E1000_EICR_TX_QUEUE0 << tx_queue;
811 if (!adapter->msix_entries && msix_vector == 0)
812 msixbm |= E1000_EIMS_OTHER;
813 array_wr32(E1000_MSIXBM(0), msix_vector, msixbm);
814 q_vector->eims_value = msixbm;
815 break;
816 case e1000_82576:
817 /*
818 * 82576 uses a table that essentially consists of 2 columns
819 * with 8 rows. The ordering is column-major so we use the
820 * lower 3 bits as the row index, and the 4th bit as the
821 * column offset.
822 */
823 if (rx_queue > IGB_N0_QUEUE)
824 igb_write_ivar(hw, msix_vector,
825 rx_queue & 0x7,
826 (rx_queue & 0x8) << 1);
827 if (tx_queue > IGB_N0_QUEUE)
828 igb_write_ivar(hw, msix_vector,
829 tx_queue & 0x7,
830 ((tx_queue & 0x8) << 1) + 8);
831 q_vector->eims_value = 1 << msix_vector;
832 break;
833 case e1000_82580:
834 case e1000_i350:
835 case e1000_i210:
836 case e1000_i211:
837 /*
838 * On 82580 and newer adapters the scheme is similar to 82576
839 * however instead of ordering column-major we have things
840 * ordered row-major. So we traverse the table by using
841 * bit 0 as the column offset, and the remaining bits as the
842 * row index.
843 */
844 if (rx_queue > IGB_N0_QUEUE)
845 igb_write_ivar(hw, msix_vector,
846 rx_queue >> 1,
847 (rx_queue & 0x1) << 4);
848 if (tx_queue > IGB_N0_QUEUE)
849 igb_write_ivar(hw, msix_vector,
850 tx_queue >> 1,
851 ((tx_queue & 0x1) << 4) + 8);
852 q_vector->eims_value = 1 << msix_vector;
853 break;
854 default:
855 BUG();
856 break;
857 }
858
859 /* add q_vector eims value to global eims_enable_mask */
860 adapter->eims_enable_mask |= q_vector->eims_value;
861
862 /* configure q_vector to set itr on first interrupt */
863 q_vector->set_itr = 1;
864}
865
866/**
867 * igb_configure_msix - Configure MSI-X hardware
868 *
869 * igb_configure_msix sets up the hardware to properly
870 * generate MSI-X interrupts.
871 **/
872static void igb_configure_msix(struct igb_adapter *adapter)
873{
874 u32 tmp;
875 int i, vector = 0;
876 struct e1000_hw *hw = &adapter->hw;
877
878 adapter->eims_enable_mask = 0;
879
880 /* set vector for other causes, i.e. link changes */
881 switch (hw->mac.type) {
882 case e1000_82575:
883 tmp = rd32(E1000_CTRL_EXT);
884 /* enable MSI-X PBA support*/
885 tmp |= E1000_CTRL_EXT_PBA_CLR;
886
887 /* Auto-Mask interrupts upon ICR read. */
888 tmp |= E1000_CTRL_EXT_EIAME;
889 tmp |= E1000_CTRL_EXT_IRCA;
890
891 wr32(E1000_CTRL_EXT, tmp);
892
893 /* enable msix_other interrupt */
894 array_wr32(E1000_MSIXBM(0), vector++,
895 E1000_EIMS_OTHER);
896 adapter->eims_other = E1000_EIMS_OTHER;
897
898 break;
899
900 case e1000_82576:
901 case e1000_82580:
902 case e1000_i350:
903 case e1000_i210:
904 case e1000_i211:
905 /* Turn on MSI-X capability first, or our settings
906 * won't stick. And it will take days to debug. */
907 wr32(E1000_GPIE, E1000_GPIE_MSIX_MODE |
908 E1000_GPIE_PBA | E1000_GPIE_EIAME |
909 E1000_GPIE_NSICR);
910
911 /* enable msix_other interrupt */
912 adapter->eims_other = 1 << vector;
913 tmp = (vector++ | E1000_IVAR_VALID) << 8;
914
915 wr32(E1000_IVAR_MISC, tmp);
916 break;
917 default:
918 /* do nothing, since nothing else supports MSI-X */
919 break;
920 } /* switch (hw->mac.type) */
921
922 adapter->eims_enable_mask |= adapter->eims_other;
923
924 for (i = 0; i < adapter->num_q_vectors; i++)
925 igb_assign_vector(adapter->q_vector[i], vector++);
926
927 wrfl();
928}
929
930/**
931 * igb_request_msix - Initialize MSI-X interrupts
932 *
933 * igb_request_msix allocates MSI-X vectors and requests interrupts from the
934 * kernel.
935 **/
936static int igb_request_msix(struct igb_adapter *adapter)
937{
938 struct net_device *netdev = adapter->netdev;
939 struct e1000_hw *hw = &adapter->hw;
940 int i, err = 0, vector = 0;
941
942 err = request_irq(adapter->msix_entries[vector].vector,
943 igb_msix_other, 0, netdev->name, adapter);
944 if (err)
945 goto out;
946 vector++;
947
948 for (i = 0; i < adapter->num_q_vectors; i++) {
949 struct igb_q_vector *q_vector = adapter->q_vector[i];
950
951 q_vector->itr_register = hw->hw_addr + E1000_EITR(vector);
952
953 if (q_vector->rx.ring && q_vector->tx.ring)
954 sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
955 q_vector->rx.ring->queue_index);
956 else if (q_vector->tx.ring)
957 sprintf(q_vector->name, "%s-tx-%u", netdev->name,
958 q_vector->tx.ring->queue_index);
959 else if (q_vector->rx.ring)
960 sprintf(q_vector->name, "%s-rx-%u", netdev->name,
961 q_vector->rx.ring->queue_index);
962 else
963 sprintf(q_vector->name, "%s-unused", netdev->name);
964
965 err = request_irq(adapter->msix_entries[vector].vector,
966 igb_msix_ring, 0, q_vector->name,
967 q_vector);
968 if (err)
969 goto out;
970 vector++;
971 }
972
973 igb_configure_msix(adapter);
974 return 0;
975out:
976 return err;
977}
978
979static void igb_reset_interrupt_capability(struct igb_adapter *adapter)
980{
981 if (adapter->msix_entries) {
982 pci_disable_msix(adapter->pdev);
983 kfree(adapter->msix_entries);
984 adapter->msix_entries = NULL;
985 } else if (adapter->flags & IGB_FLAG_HAS_MSI) {
986 pci_disable_msi(adapter->pdev);
987 }
988}
989
990/**
991 * igb_free_q_vectors - Free memory allocated for interrupt vectors
992 * @adapter: board private structure to initialize
993 *
994 * This function frees the memory allocated to the q_vectors. In addition if
995 * NAPI is enabled it will delete any references to the NAPI struct prior
996 * to freeing the q_vector.
997 **/
998static void igb_free_q_vectors(struct igb_adapter *adapter)
999{
1000 int v_idx;
1001
1002 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1003 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1004 adapter->q_vector[v_idx] = NULL;
1005 if (!q_vector)
1006 continue;
1007 netif_napi_del(&q_vector->napi);
1008 kfree(q_vector);
1009 }
1010 adapter->num_q_vectors = 0;
1011}
1012
1013/**
1014 * igb_clear_interrupt_scheme - reset the device to a state of no interrupts
1015 *
1016 * This function resets the device so that it has 0 rx queues, tx queues, and
1017 * MSI-X interrupts allocated.
1018 */
1019static void igb_clear_interrupt_scheme(struct igb_adapter *adapter)
1020{
1021 igb_free_queues(adapter);
1022 igb_free_q_vectors(adapter);
1023 igb_reset_interrupt_capability(adapter);
1024}
1025
1026/**
1027 * igb_set_interrupt_capability - set MSI or MSI-X if supported
1028 *
1029 * Attempt to configure interrupts using the best available
1030 * capabilities of the hardware and kernel.
1031 **/
1032static int igb_set_interrupt_capability(struct igb_adapter *adapter)
1033{
1034 int err;
1035 int numvecs, i;
1036
1037 /* Number of supported queues. */
1038 adapter->num_rx_queues = adapter->rss_queues;
1039 if (adapter->vfs_allocated_count)
1040 adapter->num_tx_queues = 1;
1041 else
1042 adapter->num_tx_queues = adapter->rss_queues;
1043
1044 /* start with one vector for every rx queue */
1045 numvecs = adapter->num_rx_queues;
1046
1047 /* if tx handler is separate add 1 for every tx queue */
1048 if (!(adapter->flags & IGB_FLAG_QUEUE_PAIRS))
1049 numvecs += adapter->num_tx_queues;
1050
1051 /* i210 and i211 can only have 4 MSIX vectors for rx/tx queues. */
1052 if ((adapter->hw.mac.type == e1000_i210)
1053 || (adapter->hw.mac.type == e1000_i211))
1054 numvecs = 4;
1055
1056 /* store the number of vectors reserved for queues */
1057 adapter->num_q_vectors = numvecs;
1058
1059 /* add 1 vector for link status interrupts */
1060 numvecs++;
1061 adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
1062 GFP_KERNEL);
1063
1064 if (!adapter->msix_entries)
1065 goto msi_only;
1066
1067 for (i = 0; i < numvecs; i++)
1068 adapter->msix_entries[i].entry = i;
1069
1070 err = pci_enable_msix(adapter->pdev,
1071 adapter->msix_entries,
1072 numvecs);
1073 if (err == 0)
1074 goto out;
1075
1076 igb_reset_interrupt_capability(adapter);
1077
1078 /* If we can't do MSI-X, try MSI */
1079msi_only:
1080#ifdef CONFIG_PCI_IOV
1081 /* disable SR-IOV for non MSI-X configurations */
1082 if (adapter->vf_data) {
1083 struct e1000_hw *hw = &adapter->hw;
1084 /* disable iov and allow time for transactions to clear */
1085 pci_disable_sriov(adapter->pdev);
1086 msleep(500);
1087
1088 kfree(adapter->vf_data);
1089 adapter->vf_data = NULL;
1090 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
1091 wrfl();
1092 msleep(100);
1093 dev_info(&adapter->pdev->dev, "IOV Disabled\n");
1094 }
1095#endif
1096 adapter->vfs_allocated_count = 0;
1097 adapter->rss_queues = 1;
1098 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
1099 adapter->num_rx_queues = 1;
1100 adapter->num_tx_queues = 1;
1101 adapter->num_q_vectors = 1;
1102 if (!pci_enable_msi(adapter->pdev))
1103 adapter->flags |= IGB_FLAG_HAS_MSI;
1104out:
1105 /* Notify the stack of the (possibly) reduced queue counts. */
1106 rtnl_lock();
1107 netif_set_real_num_tx_queues(adapter->netdev, adapter->num_tx_queues);
1108 err = netif_set_real_num_rx_queues(adapter->netdev,
1109 adapter->num_rx_queues);
1110 rtnl_unlock();
1111 return err;
1112}
1113
1114/**
1115 * igb_alloc_q_vectors - Allocate memory for interrupt vectors
1116 * @adapter: board private structure to initialize
1117 *
1118 * We allocate one q_vector per queue interrupt. If allocation fails we
1119 * return -ENOMEM.
1120 **/
1121static int igb_alloc_q_vectors(struct igb_adapter *adapter)
1122{
1123 struct igb_q_vector *q_vector;
1124 struct e1000_hw *hw = &adapter->hw;
1125 int v_idx;
1126 int orig_node = adapter->node;
1127
1128 for (v_idx = 0; v_idx < adapter->num_q_vectors; v_idx++) {
1129 if ((adapter->num_q_vectors == (adapter->num_rx_queues +
1130 adapter->num_tx_queues)) &&
1131 (adapter->num_rx_queues == v_idx))
1132 adapter->node = orig_node;
1133 if (orig_node == -1) {
1134 int cur_node = next_online_node(adapter->node);
1135 if (cur_node == MAX_NUMNODES)
1136 cur_node = first_online_node;
1137 adapter->node = cur_node;
1138 }
1139 q_vector = kzalloc_node(sizeof(struct igb_q_vector), GFP_KERNEL,
1140 adapter->node);
1141 if (!q_vector)
1142 q_vector = kzalloc(sizeof(struct igb_q_vector),
1143 GFP_KERNEL);
1144 if (!q_vector)
1145 goto err_out;
1146 q_vector->adapter = adapter;
1147 q_vector->itr_register = hw->hw_addr + E1000_EITR(0);
1148 q_vector->itr_val = IGB_START_ITR;
1149 netif_napi_add(adapter->netdev, &q_vector->napi, igb_poll, 64);
1150 adapter->q_vector[v_idx] = q_vector;
1151 }
1152 /* Restore the adapter's original node */
1153 adapter->node = orig_node;
1154
1155 return 0;
1156
1157err_out:
1158 /* Restore the adapter's original node */
1159 adapter->node = orig_node;
1160 igb_free_q_vectors(adapter);
1161 return -ENOMEM;
1162}
1163
1164static void igb_map_rx_ring_to_vector(struct igb_adapter *adapter,
1165 int ring_idx, int v_idx)
1166{
1167 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1168
1169 q_vector->rx.ring = adapter->rx_ring[ring_idx];
1170 q_vector->rx.ring->q_vector = q_vector;
1171 q_vector->rx.count++;
1172 q_vector->itr_val = adapter->rx_itr_setting;
1173 if (q_vector->itr_val && q_vector->itr_val <= 3)
1174 q_vector->itr_val = IGB_START_ITR;
1175}
1176
1177static void igb_map_tx_ring_to_vector(struct igb_adapter *adapter,
1178 int ring_idx, int v_idx)
1179{
1180 struct igb_q_vector *q_vector = adapter->q_vector[v_idx];
1181
1182 q_vector->tx.ring = adapter->tx_ring[ring_idx];
1183 q_vector->tx.ring->q_vector = q_vector;
1184 q_vector->tx.count++;
1185 q_vector->itr_val = adapter->tx_itr_setting;
1186 q_vector->tx.work_limit = adapter->tx_work_limit;
1187 if (q_vector->itr_val && q_vector->itr_val <= 3)
1188 q_vector->itr_val = IGB_START_ITR;
1189}
1190
1191/**
1192 * igb_map_ring_to_vector - maps allocated queues to vectors
1193 *
1194 * This function maps the recently allocated queues to vectors.
1195 **/
1196static int igb_map_ring_to_vector(struct igb_adapter *adapter)
1197{
1198 int i;
1199 int v_idx = 0;
1200
1201 if ((adapter->num_q_vectors < adapter->num_rx_queues) ||
1202 (adapter->num_q_vectors < adapter->num_tx_queues))
1203 return -ENOMEM;
1204
1205 if (adapter->num_q_vectors >=
1206 (adapter->num_rx_queues + adapter->num_tx_queues)) {
1207 for (i = 0; i < adapter->num_rx_queues; i++)
1208 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1209 for (i = 0; i < adapter->num_tx_queues; i++)
1210 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1211 } else {
1212 for (i = 0; i < adapter->num_rx_queues; i++) {
1213 if (i < adapter->num_tx_queues)
1214 igb_map_tx_ring_to_vector(adapter, i, v_idx);
1215 igb_map_rx_ring_to_vector(adapter, i, v_idx++);
1216 }
1217 for (; i < adapter->num_tx_queues; i++)
1218 igb_map_tx_ring_to_vector(adapter, i, v_idx++);
1219 }
1220 return 0;
1221}
1222
1223/**
1224 * igb_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
1225 *
1226 * This function initializes the interrupts and allocates all of the queues.
1227 **/
1228static int igb_init_interrupt_scheme(struct igb_adapter *adapter)
1229{
1230 struct pci_dev *pdev = adapter->pdev;
1231 int err;
1232
1233 err = igb_set_interrupt_capability(adapter);
1234 if (err)
1235 return err;
1236
1237 err = igb_alloc_q_vectors(adapter);
1238 if (err) {
1239 dev_err(&pdev->dev, "Unable to allocate memory for vectors\n");
1240 goto err_alloc_q_vectors;
1241 }
1242
1243 err = igb_alloc_queues(adapter);
1244 if (err) {
1245 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
1246 goto err_alloc_queues;
1247 }
1248
1249 err = igb_map_ring_to_vector(adapter);
1250 if (err) {
1251 dev_err(&pdev->dev, "Invalid q_vector to ring mapping\n");
1252 goto err_map_queues;
1253 }
1254
1255
1256 return 0;
1257err_map_queues:
1258 igb_free_queues(adapter);
1259err_alloc_queues:
1260 igb_free_q_vectors(adapter);
1261err_alloc_q_vectors:
1262 igb_reset_interrupt_capability(adapter);
1263 return err;
1264}
1265
1266/**
1267 * igb_request_irq - initialize interrupts
1268 *
1269 * Attempts to configure interrupts using the best available
1270 * capabilities of the hardware and kernel.
1271 **/
1272static int igb_request_irq(struct igb_adapter *adapter)
1273{
1274 struct net_device *netdev = adapter->netdev;
1275 struct pci_dev *pdev = adapter->pdev;
1276 int err = 0;
1277
1278 if (adapter->msix_entries) {
1279 err = igb_request_msix(adapter);
1280 if (!err)
1281 goto request_done;
1282 /* fall back to MSI */
1283 igb_clear_interrupt_scheme(adapter);
1284 if (!pci_enable_msi(pdev))
1285 adapter->flags |= IGB_FLAG_HAS_MSI;
1286 igb_free_all_tx_resources(adapter);
1287 igb_free_all_rx_resources(adapter);
1288 adapter->num_tx_queues = 1;
1289 adapter->num_rx_queues = 1;
1290 adapter->num_q_vectors = 1;
1291 err = igb_alloc_q_vectors(adapter);
1292 if (err) {
1293 dev_err(&pdev->dev,
1294 "Unable to allocate memory for vectors\n");
1295 goto request_done;
1296 }
1297 err = igb_alloc_queues(adapter);
1298 if (err) {
1299 dev_err(&pdev->dev,
1300 "Unable to allocate memory for queues\n");
1301 igb_free_q_vectors(adapter);
1302 goto request_done;
1303 }
1304 igb_setup_all_tx_resources(adapter);
1305 igb_setup_all_rx_resources(adapter);
1306 }
1307
1308 igb_assign_vector(adapter->q_vector[0], 0);
1309
1310 if (adapter->flags & IGB_FLAG_HAS_MSI) {
1311 err = request_irq(pdev->irq, igb_intr_msi, 0,
1312 netdev->name, adapter);
1313 if (!err)
1314 goto request_done;
1315
1316 /* fall back to legacy interrupts */
1317 igb_reset_interrupt_capability(adapter);
1318 adapter->flags &= ~IGB_FLAG_HAS_MSI;
1319 }
1320
1321 err = request_irq(pdev->irq, igb_intr, IRQF_SHARED,
1322 netdev->name, adapter);
1323
1324 if (err)
1325 dev_err(&pdev->dev, "Error %d getting interrupt\n",
1326 err);
1327
1328request_done:
1329 return err;
1330}
1331
1332static void igb_free_irq(struct igb_adapter *adapter)
1333{
1334 if (adapter->msix_entries) {
1335 int vector = 0, i;
1336
1337 free_irq(adapter->msix_entries[vector++].vector, adapter);
1338
1339 for (i = 0; i < adapter->num_q_vectors; i++)
1340 free_irq(adapter->msix_entries[vector++].vector,
1341 adapter->q_vector[i]);
1342 } else {
1343 free_irq(adapter->pdev->irq, adapter);
1344 }
1345}
1346
1347/**
1348 * igb_irq_disable - Mask off interrupt generation on the NIC
1349 * @adapter: board private structure
1350 **/
1351static void igb_irq_disable(struct igb_adapter *adapter)
1352{
1353 struct e1000_hw *hw = &adapter->hw;
1354
1355 /*
1356 * we need to be careful when disabling interrupts. The VFs are also
1357 * mapped into these registers and so clearing the bits can cause
1358 * issues on the VF drivers so we only need to clear what we set
1359 */
1360 if (adapter->msix_entries) {
1361 u32 regval = rd32(E1000_EIAM);
1362 wr32(E1000_EIAM, regval & ~adapter->eims_enable_mask);
1363 wr32(E1000_EIMC, adapter->eims_enable_mask);
1364 regval = rd32(E1000_EIAC);
1365 wr32(E1000_EIAC, regval & ~adapter->eims_enable_mask);
1366 }
1367
1368 wr32(E1000_IAM, 0);
1369 wr32(E1000_IMC, ~0);
1370 wrfl();
1371 if (adapter->msix_entries) {
1372 int i;
1373 for (i = 0; i < adapter->num_q_vectors; i++)
1374 synchronize_irq(adapter->msix_entries[i].vector);
1375 } else {
1376 synchronize_irq(adapter->pdev->irq);
1377 }
1378}
1379
1380/**
1381 * igb_irq_enable - Enable default interrupt generation settings
1382 * @adapter: board private structure
1383 **/
1384static void igb_irq_enable(struct igb_adapter *adapter)
1385{
1386 struct e1000_hw *hw = &adapter->hw;
1387
1388 if (adapter->msix_entries) {
1389 u32 ims = E1000_IMS_LSC | E1000_IMS_DOUTSYNC | E1000_IMS_DRSTA;
1390 u32 regval = rd32(E1000_EIAC);
1391 wr32(E1000_EIAC, regval | adapter->eims_enable_mask);
1392 regval = rd32(E1000_EIAM);
1393 wr32(E1000_EIAM, regval | adapter->eims_enable_mask);
1394 wr32(E1000_EIMS, adapter->eims_enable_mask);
1395 if (adapter->vfs_allocated_count) {
1396 wr32(E1000_MBVFIMR, 0xFF);
1397 ims |= E1000_IMS_VMMB;
1398 }
1399 wr32(E1000_IMS, ims);
1400 } else {
1401 wr32(E1000_IMS, IMS_ENABLE_MASK |
1402 E1000_IMS_DRSTA);
1403 wr32(E1000_IAM, IMS_ENABLE_MASK |
1404 E1000_IMS_DRSTA);
1405 }
1406}
1407
1408static void igb_update_mng_vlan(struct igb_adapter *adapter)
1409{
1410 struct e1000_hw *hw = &adapter->hw;
1411 u16 vid = adapter->hw.mng_cookie.vlan_id;
1412 u16 old_vid = adapter->mng_vlan_id;
1413
1414 if (hw->mng_cookie.status & E1000_MNG_DHCP_COOKIE_STATUS_VLAN) {
1415 /* add VID to filter table */
1416 igb_vfta_set(hw, vid, true);
1417 adapter->mng_vlan_id = vid;
1418 } else {
1419 adapter->mng_vlan_id = IGB_MNG_VLAN_NONE;
1420 }
1421
1422 if ((old_vid != (u16)IGB_MNG_VLAN_NONE) &&
1423 (vid != old_vid) &&
1424 !test_bit(old_vid, adapter->active_vlans)) {
1425 /* remove VID from filter table */
1426 igb_vfta_set(hw, old_vid, false);
1427 }
1428}
1429
1430/**
1431 * igb_release_hw_control - release control of the h/w to f/w
1432 * @adapter: address of board private structure
1433 *
1434 * igb_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
1435 * For ASF and Pass Through versions of f/w this means that the
1436 * driver is no longer loaded.
1437 *
1438 **/
1439static void igb_release_hw_control(struct igb_adapter *adapter)
1440{
1441 struct e1000_hw *hw = &adapter->hw;
1442 u32 ctrl_ext;
1443
1444 /* Let firmware take over control of h/w */
1445 ctrl_ext = rd32(E1000_CTRL_EXT);
1446 wr32(E1000_CTRL_EXT,
1447 ctrl_ext & ~E1000_CTRL_EXT_DRV_LOAD);
1448}
1449
1450/**
1451 * igb_get_hw_control - get control of the h/w from f/w
1452 * @adapter: address of board private structure
1453 *
1454 * igb_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
1455 * For ASF and Pass Through versions of f/w this means that
1456 * the driver is loaded.
1457 *
1458 **/
1459static void igb_get_hw_control(struct igb_adapter *adapter)
1460{
1461 struct e1000_hw *hw = &adapter->hw;
1462 u32 ctrl_ext;
1463
1464 /* Let firmware know the driver has taken over */
1465 ctrl_ext = rd32(E1000_CTRL_EXT);
1466 wr32(E1000_CTRL_EXT,
1467 ctrl_ext | E1000_CTRL_EXT_DRV_LOAD);
1468}
1469
1470/**
1471 * igb_configure - configure the hardware for RX and TX
1472 * @adapter: private board structure
1473 **/
1474static void igb_configure(struct igb_adapter *adapter)
1475{
1476 struct net_device *netdev = adapter->netdev;
1477 int i;
1478
1479 igb_get_hw_control(adapter);
1480 igb_set_rx_mode(netdev);
1481
1482 igb_restore_vlan(adapter);
1483
1484 igb_setup_tctl(adapter);
1485 igb_setup_mrqc(adapter);
1486 igb_setup_rctl(adapter);
1487
1488 igb_configure_tx(adapter);
1489 igb_configure_rx(adapter);
1490
1491 igb_rx_fifo_flush_82575(&adapter->hw);
1492
1493 /* call igb_desc_unused which always leaves
1494 * at least 1 descriptor unused to make sure
1495 * next_to_use != next_to_clean */
1496 for (i = 0; i < adapter->num_rx_queues; i++) {
1497 struct igb_ring *ring = adapter->rx_ring[i];
1498 igb_alloc_rx_buffers(ring, igb_desc_unused(ring));
1499 }
1500}
1501
1502/**
1503 * igb_power_up_link - Power up the phy/serdes link
1504 * @adapter: address of board private structure
1505 **/
1506void igb_power_up_link(struct igb_adapter *adapter)
1507{
1508 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1509 igb_power_up_phy_copper(&adapter->hw);
1510 else
1511 igb_power_up_serdes_link_82575(&adapter->hw);
1512 igb_reset_phy(&adapter->hw);
1513}
1514
1515/**
1516 * igb_power_down_link - Power down the phy/serdes link
1517 * @adapter: address of board private structure
1518 */
1519static void igb_power_down_link(struct igb_adapter *adapter)
1520{
1521 if (adapter->hw.phy.media_type == e1000_media_type_copper)
1522 igb_power_down_phy_copper_82575(&adapter->hw);
1523 else
1524 igb_shutdown_serdes_link_82575(&adapter->hw);
1525}
1526
1527/**
1528 * igb_up - Open the interface and prepare it to handle traffic
1529 * @adapter: board private structure
1530 **/
1531int igb_up(struct igb_adapter *adapter)
1532{
1533 struct e1000_hw *hw = &adapter->hw;
1534 int i;
1535
1536 /* hardware has been reset, we need to reload some things */
1537 igb_configure(adapter);
1538
1539 clear_bit(__IGB_DOWN, &adapter->state);
1540
1541 for (i = 0; i < adapter->num_q_vectors; i++)
1542 napi_enable(&(adapter->q_vector[i]->napi));
1543
1544 if (adapter->msix_entries)
1545 igb_configure_msix(adapter);
1546 else
1547 igb_assign_vector(adapter->q_vector[0], 0);
1548
1549 /* Clear any pending interrupts. */
1550 rd32(E1000_ICR);
1551 igb_irq_enable(adapter);
1552
1553 /* notify VFs that reset has been completed */
1554 if (adapter->vfs_allocated_count) {
1555 u32 reg_data = rd32(E1000_CTRL_EXT);
1556 reg_data |= E1000_CTRL_EXT_PFRSTD;
1557 wr32(E1000_CTRL_EXT, reg_data);
1558 }
1559
1560 netif_tx_start_all_queues(adapter->netdev);
1561
1562 /* start the watchdog. */
1563 hw->mac.get_link_status = 1;
1564 schedule_work(&adapter->watchdog_task);
1565
1566 return 0;
1567}
1568
1569void igb_down(struct igb_adapter *adapter)
1570{
1571 struct net_device *netdev = adapter->netdev;
1572 struct e1000_hw *hw = &adapter->hw;
1573 u32 tctl, rctl;
1574 int i;
1575
1576 /* signal that we're down so the interrupt handler does not
1577 * reschedule our watchdog timer */
1578 set_bit(__IGB_DOWN, &adapter->state);
1579
1580 /* disable receives in the hardware */
1581 rctl = rd32(E1000_RCTL);
1582 wr32(E1000_RCTL, rctl & ~E1000_RCTL_EN);
1583 /* flush and sleep below */
1584
1585 netif_tx_stop_all_queues(netdev);
1586
1587 /* disable transmits in the hardware */
1588 tctl = rd32(E1000_TCTL);
1589 tctl &= ~E1000_TCTL_EN;
1590 wr32(E1000_TCTL, tctl);
1591 /* flush both disables and wait for them to finish */
1592 wrfl();
1593 msleep(10);
1594
1595 for (i = 0; i < adapter->num_q_vectors; i++)
1596 napi_disable(&(adapter->q_vector[i]->napi));
1597
1598 igb_irq_disable(adapter);
1599
1600 del_timer_sync(&adapter->watchdog_timer);
1601 del_timer_sync(&adapter->phy_info_timer);
1602
1603 netif_carrier_off(netdev);
1604
1605 /* record the stats before reset*/
1606 spin_lock(&adapter->stats64_lock);
1607 igb_update_stats(adapter, &adapter->stats64);
1608 spin_unlock(&adapter->stats64_lock);
1609
1610 adapter->link_speed = 0;
1611 adapter->link_duplex = 0;
1612
1613 if (!pci_channel_offline(adapter->pdev))
1614 igb_reset(adapter);
1615 igb_clean_all_tx_rings(adapter);
1616 igb_clean_all_rx_rings(adapter);
1617#ifdef CONFIG_IGB_DCA
1618
1619 /* since we reset the hardware DCA settings were cleared */
1620 igb_setup_dca(adapter);
1621#endif
1622}
1623
1624void igb_reinit_locked(struct igb_adapter *adapter)
1625{
1626 WARN_ON(in_interrupt());
1627 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
1628 msleep(1);
1629 igb_down(adapter);
1630 igb_up(adapter);
1631 clear_bit(__IGB_RESETTING, &adapter->state);
1632}
1633
1634void igb_reset(struct igb_adapter *adapter)
1635{
1636 struct pci_dev *pdev = adapter->pdev;
1637 struct e1000_hw *hw = &adapter->hw;
1638 struct e1000_mac_info *mac = &hw->mac;
1639 struct e1000_fc_info *fc = &hw->fc;
1640 u32 pba = 0, tx_space, min_tx_space, min_rx_space;
1641 u16 hwm;
1642
1643 /* Repartition Pba for greater than 9k mtu
1644 * To take effect CTRL.RST is required.
1645 */
1646 switch (mac->type) {
1647 case e1000_i350:
1648 case e1000_82580:
1649 pba = rd32(E1000_RXPBS);
1650 pba = igb_rxpbs_adjust_82580(pba);
1651 break;
1652 case e1000_82576:
1653 pba = rd32(E1000_RXPBS);
1654 pba &= E1000_RXPBS_SIZE_MASK_82576;
1655 break;
1656 case e1000_82575:
1657 case e1000_i210:
1658 case e1000_i211:
1659 default:
1660 pba = E1000_PBA_34K;
1661 break;
1662 }
1663
1664 if ((adapter->max_frame_size > ETH_FRAME_LEN + ETH_FCS_LEN) &&
1665 (mac->type < e1000_82576)) {
1666 /* adjust PBA for jumbo frames */
1667 wr32(E1000_PBA, pba);
1668
1669 /* To maintain wire speed transmits, the Tx FIFO should be
1670 * large enough to accommodate two full transmit packets,
1671 * rounded up to the next 1KB and expressed in KB. Likewise,
1672 * the Rx FIFO should be large enough to accommodate at least
1673 * one full receive packet and is similarly rounded up and
1674 * expressed in KB. */
1675 pba = rd32(E1000_PBA);
1676 /* upper 16 bits has Tx packet buffer allocation size in KB */
1677 tx_space = pba >> 16;
1678 /* lower 16 bits has Rx packet buffer allocation size in KB */
1679 pba &= 0xffff;
1680 /* the tx fifo also stores 16 bytes of information about the tx
1681 * but don't include ethernet FCS because hardware appends it */
1682 min_tx_space = (adapter->max_frame_size +
1683 sizeof(union e1000_adv_tx_desc) -
1684 ETH_FCS_LEN) * 2;
1685 min_tx_space = ALIGN(min_tx_space, 1024);
1686 min_tx_space >>= 10;
1687 /* software strips receive CRC, so leave room for it */
1688 min_rx_space = adapter->max_frame_size;
1689 min_rx_space = ALIGN(min_rx_space, 1024);
1690 min_rx_space >>= 10;
1691
1692 /* If current Tx allocation is less than the min Tx FIFO size,
1693 * and the min Tx FIFO size is less than the current Rx FIFO
1694 * allocation, take space away from current Rx allocation */
1695 if (tx_space < min_tx_space &&
1696 ((min_tx_space - tx_space) < pba)) {
1697 pba = pba - (min_tx_space - tx_space);
1698
1699 /* if short on rx space, rx wins and must trump tx
1700 * adjustment */
1701 if (pba < min_rx_space)
1702 pba = min_rx_space;
1703 }
1704 wr32(E1000_PBA, pba);
1705 }
1706
1707 /* flow control settings */
1708 /* The high water mark must be low enough to fit one full frame
1709 * (or the size used for early receive) above it in the Rx FIFO.
1710 * Set it to the lower of:
1711 * - 90% of the Rx FIFO size, or
1712 * - the full Rx FIFO size minus one full frame */
1713 hwm = min(((pba << 10) * 9 / 10),
1714 ((pba << 10) - 2 * adapter->max_frame_size));
1715
1716 fc->high_water = hwm & 0xFFF0; /* 16-byte granularity */
1717 fc->low_water = fc->high_water - 16;
1718 fc->pause_time = 0xFFFF;
1719 fc->send_xon = 1;
1720 fc->current_mode = fc->requested_mode;
1721
1722 /* disable receive for all VFs and wait one second */
1723 if (adapter->vfs_allocated_count) {
1724 int i;
1725 for (i = 0 ; i < adapter->vfs_allocated_count; i++)
1726 adapter->vf_data[i].flags &= IGB_VF_FLAG_PF_SET_MAC;
1727
1728 /* ping all the active vfs to let them know we are going down */
1729 igb_ping_all_vfs(adapter);
1730
1731 /* disable transmits and receives */
1732 wr32(E1000_VFRE, 0);
1733 wr32(E1000_VFTE, 0);
1734 }
1735
1736 /* Allow time for pending master requests to run */
1737 hw->mac.ops.reset_hw(hw);
1738 wr32(E1000_WUC, 0);
1739
1740 if (hw->mac.ops.init_hw(hw))
1741 dev_err(&pdev->dev, "Hardware Error\n");
1742
1743 /*
1744 * Flow control settings reset on hardware reset, so guarantee flow
1745 * control is off when forcing speed.
1746 */
1747 if (!hw->mac.autoneg)
1748 igb_force_mac_fc(hw);
1749
1750 igb_init_dmac(adapter, pba);
1751 if (!netif_running(adapter->netdev))
1752 igb_power_down_link(adapter);
1753
1754 igb_update_mng_vlan(adapter);
1755
1756 /* Enable h/w to recognize an 802.1Q VLAN Ethernet packet */
1757 wr32(E1000_VET, ETHERNET_IEEE_VLAN_TYPE);
1758
1759 igb_get_phy_info(hw);
1760}
1761
1762static netdev_features_t igb_fix_features(struct net_device *netdev,
1763 netdev_features_t features)
1764{
1765 /*
1766 * Since there is no support for separate rx/tx vlan accel
1767 * enable/disable make sure tx flag is always in same state as rx.
1768 */
1769 if (features & NETIF_F_HW_VLAN_RX)
1770 features |= NETIF_F_HW_VLAN_TX;
1771 else
1772 features &= ~NETIF_F_HW_VLAN_TX;
1773
1774 return features;
1775}
1776
1777static int igb_set_features(struct net_device *netdev,
1778 netdev_features_t features)
1779{
1780 netdev_features_t changed = netdev->features ^ features;
1781 struct igb_adapter *adapter = netdev_priv(netdev);
1782
1783 if (changed & NETIF_F_HW_VLAN_RX)
1784 igb_vlan_mode(netdev, features);
1785
1786 if (!(changed & NETIF_F_RXALL))
1787 return 0;
1788
1789 netdev->features = features;
1790
1791 if (netif_running(netdev))
1792 igb_reinit_locked(adapter);
1793 else
1794 igb_reset(adapter);
1795
1796 return 0;
1797}
1798
1799static const struct net_device_ops igb_netdev_ops = {
1800 .ndo_open = igb_open,
1801 .ndo_stop = igb_close,
1802 .ndo_start_xmit = igb_xmit_frame,
1803 .ndo_get_stats64 = igb_get_stats64,
1804 .ndo_set_rx_mode = igb_set_rx_mode,
1805 .ndo_set_mac_address = igb_set_mac,
1806 .ndo_change_mtu = igb_change_mtu,
1807 .ndo_do_ioctl = igb_ioctl,
1808 .ndo_tx_timeout = igb_tx_timeout,
1809 .ndo_validate_addr = eth_validate_addr,
1810 .ndo_vlan_rx_add_vid = igb_vlan_rx_add_vid,
1811 .ndo_vlan_rx_kill_vid = igb_vlan_rx_kill_vid,
1812 .ndo_set_vf_mac = igb_ndo_set_vf_mac,
1813 .ndo_set_vf_vlan = igb_ndo_set_vf_vlan,
1814 .ndo_set_vf_tx_rate = igb_ndo_set_vf_bw,
1815 .ndo_get_vf_config = igb_ndo_get_vf_config,
1816#ifdef CONFIG_NET_POLL_CONTROLLER
1817 .ndo_poll_controller = igb_netpoll,
1818#endif
1819 .ndo_fix_features = igb_fix_features,
1820 .ndo_set_features = igb_set_features,
1821};
1822
1823/**
1824 * igb_probe - Device Initialization Routine
1825 * @pdev: PCI device information struct
1826 * @ent: entry in igb_pci_tbl
1827 *
1828 * Returns 0 on success, negative on failure
1829 *
1830 * igb_probe initializes an adapter identified by a pci_dev structure.
1831 * The OS initialization, configuring of the adapter private structure,
1832 * and a hardware reset occur.
1833 **/
1834static int __devinit igb_probe(struct pci_dev *pdev,
1835 const struct pci_device_id *ent)
1836{
1837 struct net_device *netdev;
1838 struct igb_adapter *adapter;
1839 struct e1000_hw *hw;
1840 u16 eeprom_data = 0;
1841 s32 ret_val;
1842 static int global_quad_port_a; /* global quad port a indication */
1843 const struct e1000_info *ei = igb_info_tbl[ent->driver_data];
1844 unsigned long mmio_start, mmio_len;
1845 int err, pci_using_dac;
1846 u16 eeprom_apme_mask = IGB_EEPROM_APME;
1847 u8 part_str[E1000_PBANUM_LENGTH];
1848
1849 /* Catch broken hardware that put the wrong VF device ID in
1850 * the PCIe SR-IOV capability.
1851 */
1852 if (pdev->is_virtfn) {
1853 WARN(1, KERN_ERR "%s (%hx:%hx) should not be a VF!\n",
1854 pci_name(pdev), pdev->vendor, pdev->device);
1855 return -EINVAL;
1856 }
1857
1858 err = pci_enable_device_mem(pdev);
1859 if (err)
1860 return err;
1861
1862 pci_using_dac = 0;
1863 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(64));
1864 if (!err) {
1865 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(64));
1866 if (!err)
1867 pci_using_dac = 1;
1868 } else {
1869 err = dma_set_mask(&pdev->dev, DMA_BIT_MASK(32));
1870 if (err) {
1871 err = dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32));
1872 if (err) {
1873 dev_err(&pdev->dev, "No usable DMA "
1874 "configuration, aborting\n");
1875 goto err_dma;
1876 }
1877 }
1878 }
1879
1880 err = pci_request_selected_regions(pdev, pci_select_bars(pdev,
1881 IORESOURCE_MEM),
1882 igb_driver_name);
1883 if (err)
1884 goto err_pci_reg;
1885
1886 pci_enable_pcie_error_reporting(pdev);
1887
1888 pci_set_master(pdev);
1889 pci_save_state(pdev);
1890
1891 err = -ENOMEM;
1892 netdev = alloc_etherdev_mq(sizeof(struct igb_adapter),
1893 IGB_MAX_TX_QUEUES);
1894 if (!netdev)
1895 goto err_alloc_etherdev;
1896
1897 SET_NETDEV_DEV(netdev, &pdev->dev);
1898
1899 pci_set_drvdata(pdev, netdev);
1900 adapter = netdev_priv(netdev);
1901 adapter->netdev = netdev;
1902 adapter->pdev = pdev;
1903 hw = &adapter->hw;
1904 hw->back = adapter;
1905 adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
1906
1907 mmio_start = pci_resource_start(pdev, 0);
1908 mmio_len = pci_resource_len(pdev, 0);
1909
1910 err = -EIO;
1911 hw->hw_addr = ioremap(mmio_start, mmio_len);
1912 if (!hw->hw_addr)
1913 goto err_ioremap;
1914
1915 netdev->netdev_ops = &igb_netdev_ops;
1916 igb_set_ethtool_ops(netdev);
1917 netdev->watchdog_timeo = 5 * HZ;
1918
1919 strncpy(netdev->name, pci_name(pdev), sizeof(netdev->name) - 1);
1920
1921 netdev->mem_start = mmio_start;
1922 netdev->mem_end = mmio_start + mmio_len;
1923
1924 /* PCI config space info */
1925 hw->vendor_id = pdev->vendor;
1926 hw->device_id = pdev->device;
1927 hw->revision_id = pdev->revision;
1928 hw->subsystem_vendor_id = pdev->subsystem_vendor;
1929 hw->subsystem_device_id = pdev->subsystem_device;
1930
1931 /* Copy the default MAC, PHY and NVM function pointers */
1932 memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
1933 memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
1934 memcpy(&hw->nvm.ops, ei->nvm_ops, sizeof(hw->nvm.ops));
1935 /* Initialize skew-specific constants */
1936 err = ei->get_invariants(hw);
1937 if (err)
1938 goto err_sw_init;
1939
1940 /* setup the private structure */
1941 err = igb_sw_init(adapter);
1942 if (err)
1943 goto err_sw_init;
1944
1945 igb_get_bus_info_pcie(hw);
1946
1947 hw->phy.autoneg_wait_to_complete = false;
1948
1949 /* Copper options */
1950 if (hw->phy.media_type == e1000_media_type_copper) {
1951 hw->phy.mdix = AUTO_ALL_MODES;
1952 hw->phy.disable_polarity_correction = false;
1953 hw->phy.ms_type = e1000_ms_hw_default;
1954 }
1955
1956 if (igb_check_reset_block(hw))
1957 dev_info(&pdev->dev,
1958 "PHY reset is blocked due to SOL/IDER session.\n");
1959
1960 /*
1961 * features is initialized to 0 in allocation, it might have bits
1962 * set by igb_sw_init so we should use an or instead of an
1963 * assignment.
1964 */
1965 netdev->features |= NETIF_F_SG |
1966 NETIF_F_IP_CSUM |
1967 NETIF_F_IPV6_CSUM |
1968 NETIF_F_TSO |
1969 NETIF_F_TSO6 |
1970 NETIF_F_RXHASH |
1971 NETIF_F_RXCSUM |
1972 NETIF_F_HW_VLAN_RX |
1973 NETIF_F_HW_VLAN_TX;
1974
1975 /* copy netdev features into list of user selectable features */
1976 netdev->hw_features |= netdev->features;
1977 netdev->hw_features |= NETIF_F_RXALL;
1978
1979 /* set this bit last since it cannot be part of hw_features */
1980 netdev->features |= NETIF_F_HW_VLAN_FILTER;
1981
1982 netdev->vlan_features |= NETIF_F_TSO |
1983 NETIF_F_TSO6 |
1984 NETIF_F_IP_CSUM |
1985 NETIF_F_IPV6_CSUM |
1986 NETIF_F_SG;
1987
1988 netdev->priv_flags |= IFF_SUPP_NOFCS;
1989
1990 if (pci_using_dac) {
1991 netdev->features |= NETIF_F_HIGHDMA;
1992 netdev->vlan_features |= NETIF_F_HIGHDMA;
1993 }
1994
1995 if (hw->mac.type >= e1000_82576) {
1996 netdev->hw_features |= NETIF_F_SCTP_CSUM;
1997 netdev->features |= NETIF_F_SCTP_CSUM;
1998 }
1999
2000 netdev->priv_flags |= IFF_UNICAST_FLT;
2001
2002 adapter->en_mng_pt = igb_enable_mng_pass_thru(hw);
2003
2004 /* before reading the NVM, reset the controller to put the device in a
2005 * known good starting state */
2006 hw->mac.ops.reset_hw(hw);
2007
2008 /*
2009 * make sure the NVM is good , i211 parts have special NVM that
2010 * doesn't contain a checksum
2011 */
2012 if (hw->mac.type != e1000_i211) {
2013 if (hw->nvm.ops.validate(hw) < 0) {
2014 dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
2015 err = -EIO;
2016 goto err_eeprom;
2017 }
2018 }
2019
2020 /* copy the MAC address out of the NVM */
2021 if (hw->mac.ops.read_mac_addr(hw))
2022 dev_err(&pdev->dev, "NVM Read Error\n");
2023
2024 memcpy(netdev->dev_addr, hw->mac.addr, netdev->addr_len);
2025 memcpy(netdev->perm_addr, hw->mac.addr, netdev->addr_len);
2026
2027 if (!is_valid_ether_addr(netdev->perm_addr)) {
2028 dev_err(&pdev->dev, "Invalid MAC Address\n");
2029 err = -EIO;
2030 goto err_eeprom;
2031 }
2032
2033 setup_timer(&adapter->watchdog_timer, igb_watchdog,
2034 (unsigned long) adapter);
2035 setup_timer(&adapter->phy_info_timer, igb_update_phy_info,
2036 (unsigned long) adapter);
2037
2038 INIT_WORK(&adapter->reset_task, igb_reset_task);
2039 INIT_WORK(&adapter->watchdog_task, igb_watchdog_task);
2040
2041 /* Initialize link properties that are user-changeable */
2042 adapter->fc_autoneg = true;
2043 hw->mac.autoneg = true;
2044 hw->phy.autoneg_advertised = 0x2f;
2045
2046 hw->fc.requested_mode = e1000_fc_default;
2047 hw->fc.current_mode = e1000_fc_default;
2048
2049 igb_validate_mdi_setting(hw);
2050
2051 /* Initial Wake on LAN setting If APM wake is enabled in the EEPROM,
2052 * enable the ACPI Magic Packet filter
2053 */
2054
2055 if (hw->bus.func == 0)
2056 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A, 1, &eeprom_data);
2057 else if (hw->mac.type >= e1000_82580)
2058 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_A +
2059 NVM_82580_LAN_FUNC_OFFSET(hw->bus.func), 1,
2060 &eeprom_data);
2061 else if (hw->bus.func == 1)
2062 hw->nvm.ops.read(hw, NVM_INIT_CONTROL3_PORT_B, 1, &eeprom_data);
2063
2064 if (eeprom_data & eeprom_apme_mask)
2065 adapter->eeprom_wol |= E1000_WUFC_MAG;
2066
2067 /* now that we have the eeprom settings, apply the special cases where
2068 * the eeprom may be wrong or the board simply won't support wake on
2069 * lan on a particular port */
2070 switch (pdev->device) {
2071 case E1000_DEV_ID_82575GB_QUAD_COPPER:
2072 adapter->eeprom_wol = 0;
2073 break;
2074 case E1000_DEV_ID_82575EB_FIBER_SERDES:
2075 case E1000_DEV_ID_82576_FIBER:
2076 case E1000_DEV_ID_82576_SERDES:
2077 /* Wake events only supported on port A for dual fiber
2078 * regardless of eeprom setting */
2079 if (rd32(E1000_STATUS) & E1000_STATUS_FUNC_1)
2080 adapter->eeprom_wol = 0;
2081 break;
2082 case E1000_DEV_ID_82576_QUAD_COPPER:
2083 case E1000_DEV_ID_82576_QUAD_COPPER_ET2:
2084 /* if quad port adapter, disable WoL on all but port A */
2085 if (global_quad_port_a != 0)
2086 adapter->eeprom_wol = 0;
2087 else
2088 adapter->flags |= IGB_FLAG_QUAD_PORT_A;
2089 /* Reset for multiple quad port adapters */
2090 if (++global_quad_port_a == 4)
2091 global_quad_port_a = 0;
2092 break;
2093 }
2094
2095 /* initialize the wol settings based on the eeprom settings */
2096 adapter->wol = adapter->eeprom_wol;
2097 device_set_wakeup_enable(&adapter->pdev->dev, adapter->wol);
2098
2099 /* reset the hardware with the new settings */
2100 igb_reset(adapter);
2101
2102 /* let the f/w know that the h/w is now under the control of the
2103 * driver. */
2104 igb_get_hw_control(adapter);
2105
2106 strcpy(netdev->name, "eth%d");
2107 err = register_netdev(netdev);
2108 if (err)
2109 goto err_register;
2110
2111 /* carrier off reporting is important to ethtool even BEFORE open */
2112 netif_carrier_off(netdev);
2113
2114#ifdef CONFIG_IGB_DCA
2115 if (dca_add_requester(&pdev->dev) == 0) {
2116 adapter->flags |= IGB_FLAG_DCA_ENABLED;
2117 dev_info(&pdev->dev, "DCA enabled\n");
2118 igb_setup_dca(adapter);
2119 }
2120
2121#endif
2122#ifdef CONFIG_IGB_PTP
2123 /* do hw tstamp init after resetting */
2124 igb_ptp_init(adapter);
2125
2126#endif
2127 dev_info(&pdev->dev, "Intel(R) Gigabit Ethernet Network Connection\n");
2128 /* print bus type/speed/width info */
2129 dev_info(&pdev->dev, "%s: (PCIe:%s:%s) %pM\n",
2130 netdev->name,
2131 ((hw->bus.speed == e1000_bus_speed_2500) ? "2.5Gb/s" :
2132 (hw->bus.speed == e1000_bus_speed_5000) ? "5.0Gb/s" :
2133 "unknown"),
2134 ((hw->bus.width == e1000_bus_width_pcie_x4) ? "Width x4" :
2135 (hw->bus.width == e1000_bus_width_pcie_x2) ? "Width x2" :
2136 (hw->bus.width == e1000_bus_width_pcie_x1) ? "Width x1" :
2137 "unknown"),
2138 netdev->dev_addr);
2139
2140 ret_val = igb_read_part_string(hw, part_str, E1000_PBANUM_LENGTH);
2141 if (ret_val)
2142 strcpy(part_str, "Unknown");
2143 dev_info(&pdev->dev, "%s: PBA No: %s\n", netdev->name, part_str);
2144 dev_info(&pdev->dev,
2145 "Using %s interrupts. %d rx queue(s), %d tx queue(s)\n",
2146 adapter->msix_entries ? "MSI-X" :
2147 (adapter->flags & IGB_FLAG_HAS_MSI) ? "MSI" : "legacy",
2148 adapter->num_rx_queues, adapter->num_tx_queues);
2149 switch (hw->mac.type) {
2150 case e1000_i350:
2151 case e1000_i210:
2152 case e1000_i211:
2153 igb_set_eee_i350(hw);
2154 break;
2155 default:
2156 break;
2157 }
2158
2159 pm_runtime_put_noidle(&pdev->dev);
2160 return 0;
2161
2162err_register:
2163 igb_release_hw_control(adapter);
2164err_eeprom:
2165 if (!igb_check_reset_block(hw))
2166 igb_reset_phy(hw);
2167
2168 if (hw->flash_address)
2169 iounmap(hw->flash_address);
2170err_sw_init:
2171 igb_clear_interrupt_scheme(adapter);
2172 iounmap(hw->hw_addr);
2173err_ioremap:
2174 free_netdev(netdev);
2175err_alloc_etherdev:
2176 pci_release_selected_regions(pdev,
2177 pci_select_bars(pdev, IORESOURCE_MEM));
2178err_pci_reg:
2179err_dma:
2180 pci_disable_device(pdev);
2181 return err;
2182}
2183
2184/**
2185 * igb_remove - Device Removal Routine
2186 * @pdev: PCI device information struct
2187 *
2188 * igb_remove is called by the PCI subsystem to alert the driver
2189 * that it should release a PCI device. The could be caused by a
2190 * Hot-Plug event, or because the driver is going to be removed from
2191 * memory.
2192 **/
2193static void __devexit igb_remove(struct pci_dev *pdev)
2194{
2195 struct net_device *netdev = pci_get_drvdata(pdev);
2196 struct igb_adapter *adapter = netdev_priv(netdev);
2197 struct e1000_hw *hw = &adapter->hw;
2198
2199 pm_runtime_get_noresume(&pdev->dev);
2200#ifdef CONFIG_IGB_PTP
2201 igb_ptp_remove(adapter);
2202
2203#endif
2204 /*
2205 * The watchdog timer may be rescheduled, so explicitly
2206 * disable watchdog from being rescheduled.
2207 */
2208 set_bit(__IGB_DOWN, &adapter->state);
2209 del_timer_sync(&adapter->watchdog_timer);
2210 del_timer_sync(&adapter->phy_info_timer);
2211
2212 cancel_work_sync(&adapter->reset_task);
2213 cancel_work_sync(&adapter->watchdog_task);
2214
2215#ifdef CONFIG_IGB_DCA
2216 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
2217 dev_info(&pdev->dev, "DCA disabled\n");
2218 dca_remove_requester(&pdev->dev);
2219 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
2220 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
2221 }
2222#endif
2223
2224 /* Release control of h/w to f/w. If f/w is AMT enabled, this
2225 * would have already happened in close and is redundant. */
2226 igb_release_hw_control(adapter);
2227
2228 unregister_netdev(netdev);
2229
2230 igb_clear_interrupt_scheme(adapter);
2231
2232#ifdef CONFIG_PCI_IOV
2233 /* reclaim resources allocated to VFs */
2234 if (adapter->vf_data) {
2235 /* disable iov and allow time for transactions to clear */
2236 if (!igb_check_vf_assignment(adapter)) {
2237 pci_disable_sriov(pdev);
2238 msleep(500);
2239 } else {
2240 dev_info(&pdev->dev, "VF(s) assigned to guests!\n");
2241 }
2242
2243 kfree(adapter->vf_data);
2244 adapter->vf_data = NULL;
2245 wr32(E1000_IOVCTL, E1000_IOVCTL_REUSE_VFQ);
2246 wrfl();
2247 msleep(100);
2248 dev_info(&pdev->dev, "IOV Disabled\n");
2249 }
2250#endif
2251
2252 iounmap(hw->hw_addr);
2253 if (hw->flash_address)
2254 iounmap(hw->flash_address);
2255 pci_release_selected_regions(pdev,
2256 pci_select_bars(pdev, IORESOURCE_MEM));
2257
2258 kfree(adapter->shadow_vfta);
2259 free_netdev(netdev);
2260
2261 pci_disable_pcie_error_reporting(pdev);
2262
2263 pci_disable_device(pdev);
2264}
2265
2266/**
2267 * igb_probe_vfs - Initialize vf data storage and add VFs to pci config space
2268 * @adapter: board private structure to initialize
2269 *
2270 * This function initializes the vf specific data storage and then attempts to
2271 * allocate the VFs. The reason for ordering it this way is because it is much
2272 * mor expensive time wise to disable SR-IOV than it is to allocate and free
2273 * the memory for the VFs.
2274 **/
2275static void __devinit igb_probe_vfs(struct igb_adapter * adapter)
2276{
2277#ifdef CONFIG_PCI_IOV
2278 struct pci_dev *pdev = adapter->pdev;
2279 struct e1000_hw *hw = &adapter->hw;
2280 int old_vfs = igb_find_enabled_vfs(adapter);
2281 int i;
2282
2283 /* Virtualization features not supported on i210 family. */
2284 if ((hw->mac.type == e1000_i210) || (hw->mac.type == e1000_i211))
2285 return;
2286
2287 if (old_vfs) {
2288 dev_info(&pdev->dev, "%d pre-allocated VFs found - override "
2289 "max_vfs setting of %d\n", old_vfs, max_vfs);
2290 adapter->vfs_allocated_count = old_vfs;
2291 }
2292
2293 if (!adapter->vfs_allocated_count)
2294 return;
2295
2296 adapter->vf_data = kcalloc(adapter->vfs_allocated_count,
2297 sizeof(struct vf_data_storage), GFP_KERNEL);
2298
2299 /* if allocation failed then we do not support SR-IOV */
2300 if (!adapter->vf_data) {
2301 adapter->vfs_allocated_count = 0;
2302 dev_err(&pdev->dev, "Unable to allocate memory for VF "
2303 "Data Storage\n");
2304 goto out;
2305 }
2306
2307 if (!old_vfs) {
2308 if (pci_enable_sriov(pdev, adapter->vfs_allocated_count))
2309 goto err_out;
2310 }
2311 dev_info(&pdev->dev, "%d VFs allocated\n",
2312 adapter->vfs_allocated_count);
2313 for (i = 0; i < adapter->vfs_allocated_count; i++)
2314 igb_vf_configure(adapter, i);
2315
2316 /* DMA Coalescing is not supported in IOV mode. */
2317 adapter->flags &= ~IGB_FLAG_DMAC;
2318 goto out;
2319err_out:
2320 kfree(adapter->vf_data);
2321 adapter->vf_data = NULL;
2322 adapter->vfs_allocated_count = 0;
2323out:
2324 return;
2325#endif /* CONFIG_PCI_IOV */
2326}
2327
2328/**
2329 * igb_sw_init - Initialize general software structures (struct igb_adapter)
2330 * @adapter: board private structure to initialize
2331 *
2332 * igb_sw_init initializes the Adapter private data structure.
2333 * Fields are initialized based on PCI device information and
2334 * OS network device settings (MTU size).
2335 **/
2336static int __devinit igb_sw_init(struct igb_adapter *adapter)
2337{
2338 struct e1000_hw *hw = &adapter->hw;
2339 struct net_device *netdev = adapter->netdev;
2340 struct pci_dev *pdev = adapter->pdev;
2341
2342 pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
2343
2344 /* set default ring sizes */
2345 adapter->tx_ring_count = IGB_DEFAULT_TXD;
2346 adapter->rx_ring_count = IGB_DEFAULT_RXD;
2347
2348 /* set default ITR values */
2349 adapter->rx_itr_setting = IGB_DEFAULT_ITR;
2350 adapter->tx_itr_setting = IGB_DEFAULT_ITR;
2351
2352 /* set default work limits */
2353 adapter->tx_work_limit = IGB_DEFAULT_TX_WORK;
2354
2355 adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
2356 VLAN_HLEN;
2357 adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
2358
2359 adapter->node = -1;
2360
2361 spin_lock_init(&adapter->stats64_lock);
2362#ifdef CONFIG_PCI_IOV
2363 switch (hw->mac.type) {
2364 case e1000_82576:
2365 case e1000_i350:
2366 if (max_vfs > 7) {
2367 dev_warn(&pdev->dev,
2368 "Maximum of 7 VFs per PF, using max\n");
2369 adapter->vfs_allocated_count = 7;
2370 } else
2371 adapter->vfs_allocated_count = max_vfs;
2372 break;
2373 case e1000_i210:
2374 case e1000_i211:
2375 adapter->vfs_allocated_count = 0;
2376 break;
2377 default:
2378 break;
2379 }
2380#endif /* CONFIG_PCI_IOV */
2381 switch (hw->mac.type) {
2382 case e1000_i210:
2383 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES_I210,
2384 num_online_cpus());
2385 break;
2386 case e1000_i211:
2387 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES_I211,
2388 num_online_cpus());
2389 break;
2390 default:
2391 adapter->rss_queues = min_t(u32, IGB_MAX_RX_QUEUES,
2392 num_online_cpus());
2393 break;
2394 }
2395 /* i350 cannot do RSS and SR-IOV at the same time */
2396 if (hw->mac.type == e1000_i350 && adapter->vfs_allocated_count)
2397 adapter->rss_queues = 1;
2398
2399 /*
2400 * if rss_queues > 4 or vfs are going to be allocated with rss_queues
2401 * then we should combine the queues into a queue pair in order to
2402 * conserve interrupts due to limited supply
2403 */
2404 if ((adapter->rss_queues > 4) ||
2405 ((adapter->rss_queues > 1) && (adapter->vfs_allocated_count > 6)))
2406 adapter->flags |= IGB_FLAG_QUEUE_PAIRS;
2407
2408 /* Setup and initialize a copy of the hw vlan table array */
2409 adapter->shadow_vfta = kzalloc(sizeof(u32) *
2410 E1000_VLAN_FILTER_TBL_SIZE,
2411 GFP_ATOMIC);
2412
2413 /* This call may decrease the number of queues */
2414 if (igb_init_interrupt_scheme(adapter)) {
2415 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
2416 return -ENOMEM;
2417 }
2418
2419 igb_probe_vfs(adapter);
2420
2421 /* Explicitly disable IRQ since the NIC can be in any state. */
2422 igb_irq_disable(adapter);
2423
2424 if (hw->mac.type >= e1000_i350)
2425 adapter->flags &= ~IGB_FLAG_DMAC;
2426
2427 set_bit(__IGB_DOWN, &adapter->state);
2428 return 0;
2429}
2430
2431/**
2432 * igb_open - Called when a network interface is made active
2433 * @netdev: network interface device structure
2434 *
2435 * Returns 0 on success, negative value on failure
2436 *
2437 * The open entry point is called when a network interface is made
2438 * active by the system (IFF_UP). At this point all resources needed
2439 * for transmit and receive operations are allocated, the interrupt
2440 * handler is registered with the OS, the watchdog timer is started,
2441 * and the stack is notified that the interface is ready.
2442 **/
2443static int __igb_open(struct net_device *netdev, bool resuming)
2444{
2445 struct igb_adapter *adapter = netdev_priv(netdev);
2446 struct e1000_hw *hw = &adapter->hw;
2447 struct pci_dev *pdev = adapter->pdev;
2448 int err;
2449 int i;
2450
2451 /* disallow open during test */
2452 if (test_bit(__IGB_TESTING, &adapter->state)) {
2453 WARN_ON(resuming);
2454 return -EBUSY;
2455 }
2456
2457 if (!resuming)
2458 pm_runtime_get_sync(&pdev->dev);
2459
2460 netif_carrier_off(netdev);
2461
2462 /* allocate transmit descriptors */
2463 err = igb_setup_all_tx_resources(adapter);
2464 if (err)
2465 goto err_setup_tx;
2466
2467 /* allocate receive descriptors */
2468 err = igb_setup_all_rx_resources(adapter);
2469 if (err)
2470 goto err_setup_rx;
2471
2472 igb_power_up_link(adapter);
2473
2474 /* before we allocate an interrupt, we must be ready to handle it.
2475 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
2476 * as soon as we call pci_request_irq, so we have to setup our
2477 * clean_rx handler before we do so. */
2478 igb_configure(adapter);
2479
2480 err = igb_request_irq(adapter);
2481 if (err)
2482 goto err_req_irq;
2483
2484 /* From here on the code is the same as igb_up() */
2485 clear_bit(__IGB_DOWN, &adapter->state);
2486
2487 for (i = 0; i < adapter->num_q_vectors; i++)
2488 napi_enable(&(adapter->q_vector[i]->napi));
2489
2490 /* Clear any pending interrupts. */
2491 rd32(E1000_ICR);
2492
2493 igb_irq_enable(adapter);
2494
2495 /* notify VFs that reset has been completed */
2496 if (adapter->vfs_allocated_count) {
2497 u32 reg_data = rd32(E1000_CTRL_EXT);
2498 reg_data |= E1000_CTRL_EXT_PFRSTD;
2499 wr32(E1000_CTRL_EXT, reg_data);
2500 }
2501
2502 netif_tx_start_all_queues(netdev);
2503
2504 if (!resuming)
2505 pm_runtime_put(&pdev->dev);
2506
2507 /* start the watchdog. */
2508 hw->mac.get_link_status = 1;
2509 schedule_work(&adapter->watchdog_task);
2510
2511 return 0;
2512
2513err_req_irq:
2514 igb_release_hw_control(adapter);
2515 igb_power_down_link(adapter);
2516 igb_free_all_rx_resources(adapter);
2517err_setup_rx:
2518 igb_free_all_tx_resources(adapter);
2519err_setup_tx:
2520 igb_reset(adapter);
2521 if (!resuming)
2522 pm_runtime_put(&pdev->dev);
2523
2524 return err;
2525}
2526
2527static int igb_open(struct net_device *netdev)
2528{
2529 return __igb_open(netdev, false);
2530}
2531
2532/**
2533 * igb_close - Disables a network interface
2534 * @netdev: network interface device structure
2535 *
2536 * Returns 0, this is not allowed to fail
2537 *
2538 * The close entry point is called when an interface is de-activated
2539 * by the OS. The hardware is still under the driver's control, but
2540 * needs to be disabled. A global MAC reset is issued to stop the
2541 * hardware, and all transmit and receive resources are freed.
2542 **/
2543static int __igb_close(struct net_device *netdev, bool suspending)
2544{
2545 struct igb_adapter *adapter = netdev_priv(netdev);
2546 struct pci_dev *pdev = adapter->pdev;
2547
2548 WARN_ON(test_bit(__IGB_RESETTING, &adapter->state));
2549
2550 if (!suspending)
2551 pm_runtime_get_sync(&pdev->dev);
2552
2553 igb_down(adapter);
2554 igb_free_irq(adapter);
2555
2556 igb_free_all_tx_resources(adapter);
2557 igb_free_all_rx_resources(adapter);
2558
2559 if (!suspending)
2560 pm_runtime_put_sync(&pdev->dev);
2561 return 0;
2562}
2563
2564static int igb_close(struct net_device *netdev)
2565{
2566 return __igb_close(netdev, false);
2567}
2568
2569/**
2570 * igb_setup_tx_resources - allocate Tx resources (Descriptors)
2571 * @tx_ring: tx descriptor ring (for a specific queue) to setup
2572 *
2573 * Return 0 on success, negative on failure
2574 **/
2575int igb_setup_tx_resources(struct igb_ring *tx_ring)
2576{
2577 struct device *dev = tx_ring->dev;
2578 int orig_node = dev_to_node(dev);
2579 int size;
2580
2581 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
2582 tx_ring->tx_buffer_info = vzalloc_node(size, tx_ring->numa_node);
2583 if (!tx_ring->tx_buffer_info)
2584 tx_ring->tx_buffer_info = vzalloc(size);
2585 if (!tx_ring->tx_buffer_info)
2586 goto err;
2587
2588 /* round up to nearest 4K */
2589 tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
2590 tx_ring->size = ALIGN(tx_ring->size, 4096);
2591
2592 set_dev_node(dev, tx_ring->numa_node);
2593 tx_ring->desc = dma_alloc_coherent(dev,
2594 tx_ring->size,
2595 &tx_ring->dma,
2596 GFP_KERNEL);
2597 set_dev_node(dev, orig_node);
2598 if (!tx_ring->desc)
2599 tx_ring->desc = dma_alloc_coherent(dev,
2600 tx_ring->size,
2601 &tx_ring->dma,
2602 GFP_KERNEL);
2603
2604 if (!tx_ring->desc)
2605 goto err;
2606
2607 tx_ring->next_to_use = 0;
2608 tx_ring->next_to_clean = 0;
2609
2610 return 0;
2611
2612err:
2613 vfree(tx_ring->tx_buffer_info);
2614 dev_err(dev,
2615 "Unable to allocate memory for the transmit descriptor ring\n");
2616 return -ENOMEM;
2617}
2618
2619/**
2620 * igb_setup_all_tx_resources - wrapper to allocate Tx resources
2621 * (Descriptors) for all queues
2622 * @adapter: board private structure
2623 *
2624 * Return 0 on success, negative on failure
2625 **/
2626static int igb_setup_all_tx_resources(struct igb_adapter *adapter)
2627{
2628 struct pci_dev *pdev = adapter->pdev;
2629 int i, err = 0;
2630
2631 for (i = 0; i < adapter->num_tx_queues; i++) {
2632 err = igb_setup_tx_resources(adapter->tx_ring[i]);
2633 if (err) {
2634 dev_err(&pdev->dev,
2635 "Allocation for Tx Queue %u failed\n", i);
2636 for (i--; i >= 0; i--)
2637 igb_free_tx_resources(adapter->tx_ring[i]);
2638 break;
2639 }
2640 }
2641
2642 return err;
2643}
2644
2645/**
2646 * igb_setup_tctl - configure the transmit control registers
2647 * @adapter: Board private structure
2648 **/
2649void igb_setup_tctl(struct igb_adapter *adapter)
2650{
2651 struct e1000_hw *hw = &adapter->hw;
2652 u32 tctl;
2653
2654 /* disable queue 0 which is enabled by default on 82575 and 82576 */
2655 wr32(E1000_TXDCTL(0), 0);
2656
2657 /* Program the Transmit Control Register */
2658 tctl = rd32(E1000_TCTL);
2659 tctl &= ~E1000_TCTL_CT;
2660 tctl |= E1000_TCTL_PSP | E1000_TCTL_RTLC |
2661 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT);
2662
2663 igb_config_collision_dist(hw);
2664
2665 /* Enable transmits */
2666 tctl |= E1000_TCTL_EN;
2667
2668 wr32(E1000_TCTL, tctl);
2669}
2670
2671/**
2672 * igb_configure_tx_ring - Configure transmit ring after Reset
2673 * @adapter: board private structure
2674 * @ring: tx ring to configure
2675 *
2676 * Configure a transmit ring after a reset.
2677 **/
2678void igb_configure_tx_ring(struct igb_adapter *adapter,
2679 struct igb_ring *ring)
2680{
2681 struct e1000_hw *hw = &adapter->hw;
2682 u32 txdctl = 0;
2683 u64 tdba = ring->dma;
2684 int reg_idx = ring->reg_idx;
2685
2686 /* disable the queue */
2687 wr32(E1000_TXDCTL(reg_idx), 0);
2688 wrfl();
2689 mdelay(10);
2690
2691 wr32(E1000_TDLEN(reg_idx),
2692 ring->count * sizeof(union e1000_adv_tx_desc));
2693 wr32(E1000_TDBAL(reg_idx),
2694 tdba & 0x00000000ffffffffULL);
2695 wr32(E1000_TDBAH(reg_idx), tdba >> 32);
2696
2697 ring->tail = hw->hw_addr + E1000_TDT(reg_idx);
2698 wr32(E1000_TDH(reg_idx), 0);
2699 writel(0, ring->tail);
2700
2701 txdctl |= IGB_TX_PTHRESH;
2702 txdctl |= IGB_TX_HTHRESH << 8;
2703 txdctl |= IGB_TX_WTHRESH << 16;
2704
2705 txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
2706 wr32(E1000_TXDCTL(reg_idx), txdctl);
2707}
2708
2709/**
2710 * igb_configure_tx - Configure transmit Unit after Reset
2711 * @adapter: board private structure
2712 *
2713 * Configure the Tx unit of the MAC after a reset.
2714 **/
2715static void igb_configure_tx(struct igb_adapter *adapter)
2716{
2717 int i;
2718
2719 for (i = 0; i < adapter->num_tx_queues; i++)
2720 igb_configure_tx_ring(adapter, adapter->tx_ring[i]);
2721}
2722
2723/**
2724 * igb_setup_rx_resources - allocate Rx resources (Descriptors)
2725 * @rx_ring: rx descriptor ring (for a specific queue) to setup
2726 *
2727 * Returns 0 on success, negative on failure
2728 **/
2729int igb_setup_rx_resources(struct igb_ring *rx_ring)
2730{
2731 struct device *dev = rx_ring->dev;
2732 int orig_node = dev_to_node(dev);
2733 int size, desc_len;
2734
2735 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
2736 rx_ring->rx_buffer_info = vzalloc_node(size, rx_ring->numa_node);
2737 if (!rx_ring->rx_buffer_info)
2738 rx_ring->rx_buffer_info = vzalloc(size);
2739 if (!rx_ring->rx_buffer_info)
2740 goto err;
2741
2742 desc_len = sizeof(union e1000_adv_rx_desc);
2743
2744 /* Round up to nearest 4K */
2745 rx_ring->size = rx_ring->count * desc_len;
2746 rx_ring->size = ALIGN(rx_ring->size, 4096);
2747
2748 set_dev_node(dev, rx_ring->numa_node);
2749 rx_ring->desc = dma_alloc_coherent(dev,
2750 rx_ring->size,
2751 &rx_ring->dma,
2752 GFP_KERNEL);
2753 set_dev_node(dev, orig_node);
2754 if (!rx_ring->desc)
2755 rx_ring->desc = dma_alloc_coherent(dev,
2756 rx_ring->size,
2757 &rx_ring->dma,
2758 GFP_KERNEL);
2759
2760 if (!rx_ring->desc)
2761 goto err;
2762
2763 rx_ring->next_to_clean = 0;
2764 rx_ring->next_to_use = 0;
2765
2766 return 0;
2767
2768err:
2769 vfree(rx_ring->rx_buffer_info);
2770 rx_ring->rx_buffer_info = NULL;
2771 dev_err(dev, "Unable to allocate memory for the receive descriptor"
2772 " ring\n");
2773 return -ENOMEM;
2774}
2775
2776/**
2777 * igb_setup_all_rx_resources - wrapper to allocate Rx resources
2778 * (Descriptors) for all queues
2779 * @adapter: board private structure
2780 *
2781 * Return 0 on success, negative on failure
2782 **/
2783static int igb_setup_all_rx_resources(struct igb_adapter *adapter)
2784{
2785 struct pci_dev *pdev = adapter->pdev;
2786 int i, err = 0;
2787
2788 for (i = 0; i < adapter->num_rx_queues; i++) {
2789 err = igb_setup_rx_resources(adapter->rx_ring[i]);
2790 if (err) {
2791 dev_err(&pdev->dev,
2792 "Allocation for Rx Queue %u failed\n", i);
2793 for (i--; i >= 0; i--)
2794 igb_free_rx_resources(adapter->rx_ring[i]);
2795 break;
2796 }
2797 }
2798
2799 return err;
2800}
2801
2802/**
2803 * igb_setup_mrqc - configure the multiple receive queue control registers
2804 * @adapter: Board private structure
2805 **/
2806static void igb_setup_mrqc(struct igb_adapter *adapter)
2807{
2808 struct e1000_hw *hw = &adapter->hw;
2809 u32 mrqc, rxcsum;
2810 u32 j, num_rx_queues, shift = 0, shift2 = 0;
2811 union e1000_reta {
2812 u32 dword;
2813 u8 bytes[4];
2814 } reta;
2815 static const u8 rsshash[40] = {
2816 0x6d, 0x5a, 0x56, 0xda, 0x25, 0x5b, 0x0e, 0xc2, 0x41, 0x67,
2817 0x25, 0x3d, 0x43, 0xa3, 0x8f, 0xb0, 0xd0, 0xca, 0x2b, 0xcb,
2818 0xae, 0x7b, 0x30, 0xb4, 0x77, 0xcb, 0x2d, 0xa3, 0x80, 0x30,
2819 0xf2, 0x0c, 0x6a, 0x42, 0xb7, 0x3b, 0xbe, 0xac, 0x01, 0xfa };
2820
2821 /* Fill out hash function seeds */
2822 for (j = 0; j < 10; j++) {
2823 u32 rsskey = rsshash[(j * 4)];
2824 rsskey |= rsshash[(j * 4) + 1] << 8;
2825 rsskey |= rsshash[(j * 4) + 2] << 16;
2826 rsskey |= rsshash[(j * 4) + 3] << 24;
2827 array_wr32(E1000_RSSRK(0), j, rsskey);
2828 }
2829
2830 num_rx_queues = adapter->rss_queues;
2831
2832 if (adapter->vfs_allocated_count) {
2833 /* 82575 and 82576 supports 2 RSS queues for VMDq */
2834 switch (hw->mac.type) {
2835 case e1000_i350:
2836 case e1000_82580:
2837 num_rx_queues = 1;
2838 shift = 0;
2839 break;
2840 case e1000_82576:
2841 shift = 3;
2842 num_rx_queues = 2;
2843 break;
2844 case e1000_82575:
2845 shift = 2;
2846 shift2 = 6;
2847 default:
2848 break;
2849 }
2850 } else {
2851 if (hw->mac.type == e1000_82575)
2852 shift = 6;
2853 }
2854
2855 for (j = 0; j < (32 * 4); j++) {
2856 reta.bytes[j & 3] = (j % num_rx_queues) << shift;
2857 if (shift2)
2858 reta.bytes[j & 3] |= num_rx_queues << shift2;
2859 if ((j & 3) == 3)
2860 wr32(E1000_RETA(j >> 2), reta.dword);
2861 }
2862
2863 /*
2864 * Disable raw packet checksumming so that RSS hash is placed in
2865 * descriptor on writeback. No need to enable TCP/UDP/IP checksum
2866 * offloads as they are enabled by default
2867 */
2868 rxcsum = rd32(E1000_RXCSUM);
2869 rxcsum |= E1000_RXCSUM_PCSD;
2870
2871 if (adapter->hw.mac.type >= e1000_82576)
2872 /* Enable Receive Checksum Offload for SCTP */
2873 rxcsum |= E1000_RXCSUM_CRCOFL;
2874
2875 /* Don't need to set TUOFL or IPOFL, they default to 1 */
2876 wr32(E1000_RXCSUM, rxcsum);
2877 /*
2878 * Generate RSS hash based on TCP port numbers and/or
2879 * IPv4/v6 src and dst addresses since UDP cannot be
2880 * hashed reliably due to IP fragmentation
2881 */
2882
2883 mrqc = E1000_MRQC_RSS_FIELD_IPV4 |
2884 E1000_MRQC_RSS_FIELD_IPV4_TCP |
2885 E1000_MRQC_RSS_FIELD_IPV6 |
2886 E1000_MRQC_RSS_FIELD_IPV6_TCP |
2887 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX;
2888
2889 /* If VMDq is enabled then we set the appropriate mode for that, else
2890 * we default to RSS so that an RSS hash is calculated per packet even
2891 * if we are only using one queue */
2892 if (adapter->vfs_allocated_count) {
2893 if (hw->mac.type > e1000_82575) {
2894 /* Set the default pool for the PF's first queue */
2895 u32 vtctl = rd32(E1000_VT_CTL);
2896 vtctl &= ~(E1000_VT_CTL_DEFAULT_POOL_MASK |
2897 E1000_VT_CTL_DISABLE_DEF_POOL);
2898 vtctl |= adapter->vfs_allocated_count <<
2899 E1000_VT_CTL_DEFAULT_POOL_SHIFT;
2900 wr32(E1000_VT_CTL, vtctl);
2901 }
2902 if (adapter->rss_queues > 1)
2903 mrqc |= E1000_MRQC_ENABLE_VMDQ_RSS_2Q;
2904 else
2905 mrqc |= E1000_MRQC_ENABLE_VMDQ;
2906 } else {
2907 if (hw->mac.type != e1000_i211)
2908 mrqc |= E1000_MRQC_ENABLE_RSS_4Q;
2909 }
2910 igb_vmm_control(adapter);
2911
2912 wr32(E1000_MRQC, mrqc);
2913}
2914
2915/**
2916 * igb_setup_rctl - configure the receive control registers
2917 * @adapter: Board private structure
2918 **/
2919void igb_setup_rctl(struct igb_adapter *adapter)
2920{
2921 struct e1000_hw *hw = &adapter->hw;
2922 u32 rctl;
2923
2924 rctl = rd32(E1000_RCTL);
2925
2926 rctl &= ~(3 << E1000_RCTL_MO_SHIFT);
2927 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC);
2928
2929 rctl |= E1000_RCTL_EN | E1000_RCTL_BAM | E1000_RCTL_RDMTS_HALF |
2930 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT);
2931
2932 /*
2933 * enable stripping of CRC. It's unlikely this will break BMC
2934 * redirection as it did with e1000. Newer features require
2935 * that the HW strips the CRC.
2936 */
2937 rctl |= E1000_RCTL_SECRC;
2938
2939 /* disable store bad packets and clear size bits. */
2940 rctl &= ~(E1000_RCTL_SBP | E1000_RCTL_SZ_256);
2941
2942 /* enable LPE to prevent packets larger than max_frame_size */
2943 rctl |= E1000_RCTL_LPE;
2944
2945 /* disable queue 0 to prevent tail write w/o re-config */
2946 wr32(E1000_RXDCTL(0), 0);
2947
2948 /* Attention!!! For SR-IOV PF driver operations you must enable
2949 * queue drop for all VF and PF queues to prevent head of line blocking
2950 * if an un-trusted VF does not provide descriptors to hardware.
2951 */
2952 if (adapter->vfs_allocated_count) {
2953 /* set all queue drop enable bits */
2954 wr32(E1000_QDE, ALL_QUEUES);
2955 }
2956
2957 /* This is useful for sniffing bad packets. */
2958 if (adapter->netdev->features & NETIF_F_RXALL) {
2959 /* UPE and MPE will be handled by normal PROMISC logic
2960 * in e1000e_set_rx_mode */
2961 rctl |= (E1000_RCTL_SBP | /* Receive bad packets */
2962 E1000_RCTL_BAM | /* RX All Bcast Pkts */
2963 E1000_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
2964
2965 rctl &= ~(E1000_RCTL_VFE | /* Disable VLAN filter */
2966 E1000_RCTL_DPF | /* Allow filtered pause */
2967 E1000_RCTL_CFIEN); /* Dis VLAN CFIEN Filter */
2968 /* Do not mess with E1000_CTRL_VME, it affects transmit as well,
2969 * and that breaks VLANs.
2970 */
2971 }
2972
2973 wr32(E1000_RCTL, rctl);
2974}
2975
2976static inline int igb_set_vf_rlpml(struct igb_adapter *adapter, int size,
2977 int vfn)
2978{
2979 struct e1000_hw *hw = &adapter->hw;
2980 u32 vmolr;
2981
2982 /* if it isn't the PF check to see if VFs are enabled and
2983 * increase the size to support vlan tags */
2984 if (vfn < adapter->vfs_allocated_count &&
2985 adapter->vf_data[vfn].vlans_enabled)
2986 size += VLAN_TAG_SIZE;
2987
2988 vmolr = rd32(E1000_VMOLR(vfn));
2989 vmolr &= ~E1000_VMOLR_RLPML_MASK;
2990 vmolr |= size | E1000_VMOLR_LPE;
2991 wr32(E1000_VMOLR(vfn), vmolr);
2992
2993 return 0;
2994}
2995
2996/**
2997 * igb_rlpml_set - set maximum receive packet size
2998 * @adapter: board private structure
2999 *
3000 * Configure maximum receivable packet size.
3001 **/
3002static void igb_rlpml_set(struct igb_adapter *adapter)
3003{
3004 u32 max_frame_size = adapter->max_frame_size;
3005 struct e1000_hw *hw = &adapter->hw;
3006 u16 pf_id = adapter->vfs_allocated_count;
3007
3008 if (pf_id) {
3009 igb_set_vf_rlpml(adapter, max_frame_size, pf_id);
3010 /*
3011 * If we're in VMDQ or SR-IOV mode, then set global RLPML
3012 * to our max jumbo frame size, in case we need to enable
3013 * jumbo frames on one of the rings later.
3014 * This will not pass over-length frames into the default
3015 * queue because it's gated by the VMOLR.RLPML.
3016 */
3017 max_frame_size = MAX_JUMBO_FRAME_SIZE;
3018 }
3019
3020 wr32(E1000_RLPML, max_frame_size);
3021}
3022
3023static inline void igb_set_vmolr(struct igb_adapter *adapter,
3024 int vfn, bool aupe)
3025{
3026 struct e1000_hw *hw = &adapter->hw;
3027 u32 vmolr;
3028
3029 /*
3030 * This register exists only on 82576 and newer so if we are older then
3031 * we should exit and do nothing
3032 */
3033 if (hw->mac.type < e1000_82576)
3034 return;
3035
3036 vmolr = rd32(E1000_VMOLR(vfn));
3037 vmolr |= E1000_VMOLR_STRVLAN; /* Strip vlan tags */
3038 if (aupe)
3039 vmolr |= E1000_VMOLR_AUPE; /* Accept untagged packets */
3040 else
3041 vmolr &= ~(E1000_VMOLR_AUPE); /* Tagged packets ONLY */
3042
3043 /* clear all bits that might not be set */
3044 vmolr &= ~(E1000_VMOLR_BAM | E1000_VMOLR_RSSE);
3045
3046 if (adapter->rss_queues > 1 && vfn == adapter->vfs_allocated_count)
3047 vmolr |= E1000_VMOLR_RSSE; /* enable RSS */
3048 /*
3049 * for VMDq only allow the VFs and pool 0 to accept broadcast and
3050 * multicast packets
3051 */
3052 if (vfn <= adapter->vfs_allocated_count)
3053 vmolr |= E1000_VMOLR_BAM; /* Accept broadcast */
3054
3055 wr32(E1000_VMOLR(vfn), vmolr);
3056}
3057
3058/**
3059 * igb_configure_rx_ring - Configure a receive ring after Reset
3060 * @adapter: board private structure
3061 * @ring: receive ring to be configured
3062 *
3063 * Configure the Rx unit of the MAC after a reset.
3064 **/
3065void igb_configure_rx_ring(struct igb_adapter *adapter,
3066 struct igb_ring *ring)
3067{
3068 struct e1000_hw *hw = &adapter->hw;
3069 u64 rdba = ring->dma;
3070 int reg_idx = ring->reg_idx;
3071 u32 srrctl = 0, rxdctl = 0;
3072
3073 /* disable the queue */
3074 wr32(E1000_RXDCTL(reg_idx), 0);
3075
3076 /* Set DMA base address registers */
3077 wr32(E1000_RDBAL(reg_idx),
3078 rdba & 0x00000000ffffffffULL);
3079 wr32(E1000_RDBAH(reg_idx), rdba >> 32);
3080 wr32(E1000_RDLEN(reg_idx),
3081 ring->count * sizeof(union e1000_adv_rx_desc));
3082
3083 /* initialize head and tail */
3084 ring->tail = hw->hw_addr + E1000_RDT(reg_idx);
3085 wr32(E1000_RDH(reg_idx), 0);
3086 writel(0, ring->tail);
3087
3088 /* set descriptor configuration */
3089 srrctl = IGB_RX_HDR_LEN << E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
3090#if (PAGE_SIZE / 2) > IGB_RXBUFFER_16384
3091 srrctl |= IGB_RXBUFFER_16384 >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3092#else
3093 srrctl |= (PAGE_SIZE / 2) >> E1000_SRRCTL_BSIZEPKT_SHIFT;
3094#endif
3095 srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
3096 if (hw->mac.type >= e1000_82580)
3097 srrctl |= E1000_SRRCTL_TIMESTAMP;
3098 /* Only set Drop Enable if we are supporting multiple queues */
3099 if (adapter->vfs_allocated_count || adapter->num_rx_queues > 1)
3100 srrctl |= E1000_SRRCTL_DROP_EN;
3101
3102 wr32(E1000_SRRCTL(reg_idx), srrctl);
3103
3104 /* set filtering for VMDQ pools */
3105 igb_set_vmolr(adapter, reg_idx & 0x7, true);
3106
3107 rxdctl |= IGB_RX_PTHRESH;
3108 rxdctl |= IGB_RX_HTHRESH << 8;
3109 rxdctl |= IGB_RX_WTHRESH << 16;
3110
3111 /* enable receive descriptor fetching */
3112 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
3113 wr32(E1000_RXDCTL(reg_idx), rxdctl);
3114}
3115
3116/**
3117 * igb_configure_rx - Configure receive Unit after Reset
3118 * @adapter: board private structure
3119 *
3120 * Configure the Rx unit of the MAC after a reset.
3121 **/
3122static void igb_configure_rx(struct igb_adapter *adapter)
3123{
3124 int i;
3125
3126 /* set UTA to appropriate mode */
3127 igb_set_uta(adapter);
3128
3129 /* set the correct pool for the PF default MAC address in entry 0 */
3130 igb_rar_set_qsel(adapter, adapter->hw.mac.addr, 0,
3131 adapter->vfs_allocated_count);
3132
3133 /* Setup the HW Rx Head and Tail Descriptor Pointers and
3134 * the Base and Length of the Rx Descriptor Ring */
3135 for (i = 0; i < adapter->num_rx_queues; i++)
3136 igb_configure_rx_ring(adapter, adapter->rx_ring[i]);
3137}
3138
3139/**
3140 * igb_free_tx_resources - Free Tx Resources per Queue
3141 * @tx_ring: Tx descriptor ring for a specific queue
3142 *
3143 * Free all transmit software resources
3144 **/
3145void igb_free_tx_resources(struct igb_ring *tx_ring)
3146{
3147 igb_clean_tx_ring(tx_ring);
3148
3149 vfree(tx_ring->tx_buffer_info);
3150 tx_ring->tx_buffer_info = NULL;
3151
3152 /* if not set, then don't free */
3153 if (!tx_ring->desc)
3154 return;
3155
3156 dma_free_coherent(tx_ring->dev, tx_ring->size,
3157 tx_ring->desc, tx_ring->dma);
3158
3159 tx_ring->desc = NULL;
3160}
3161
3162/**
3163 * igb_free_all_tx_resources - Free Tx Resources for All Queues
3164 * @adapter: board private structure
3165 *
3166 * Free all transmit software resources
3167 **/
3168static void igb_free_all_tx_resources(struct igb_adapter *adapter)
3169{
3170 int i;
3171
3172 for (i = 0; i < adapter->num_tx_queues; i++)
3173 igb_free_tx_resources(adapter->tx_ring[i]);
3174}
3175
3176void igb_unmap_and_free_tx_resource(struct igb_ring *ring,
3177 struct igb_tx_buffer *tx_buffer)
3178{
3179 if (tx_buffer->skb) {
3180 dev_kfree_skb_any(tx_buffer->skb);
3181 if (tx_buffer->dma)
3182 dma_unmap_single(ring->dev,
3183 tx_buffer->dma,
3184 tx_buffer->length,
3185 DMA_TO_DEVICE);
3186 } else if (tx_buffer->dma) {
3187 dma_unmap_page(ring->dev,
3188 tx_buffer->dma,
3189 tx_buffer->length,
3190 DMA_TO_DEVICE);
3191 }
3192 tx_buffer->next_to_watch = NULL;
3193 tx_buffer->skb = NULL;
3194 tx_buffer->dma = 0;
3195 /* buffer_info must be completely set up in the transmit path */
3196}
3197
3198/**
3199 * igb_clean_tx_ring - Free Tx Buffers
3200 * @tx_ring: ring to be cleaned
3201 **/
3202static void igb_clean_tx_ring(struct igb_ring *tx_ring)
3203{
3204 struct igb_tx_buffer *buffer_info;
3205 unsigned long size;
3206 u16 i;
3207
3208 if (!tx_ring->tx_buffer_info)
3209 return;
3210 /* Free all the Tx ring sk_buffs */
3211
3212 for (i = 0; i < tx_ring->count; i++) {
3213 buffer_info = &tx_ring->tx_buffer_info[i];
3214 igb_unmap_and_free_tx_resource(tx_ring, buffer_info);
3215 }
3216
3217 netdev_tx_reset_queue(txring_txq(tx_ring));
3218
3219 size = sizeof(struct igb_tx_buffer) * tx_ring->count;
3220 memset(tx_ring->tx_buffer_info, 0, size);
3221
3222 /* Zero out the descriptor ring */
3223 memset(tx_ring->desc, 0, tx_ring->size);
3224
3225 tx_ring->next_to_use = 0;
3226 tx_ring->next_to_clean = 0;
3227}
3228
3229/**
3230 * igb_clean_all_tx_rings - Free Tx Buffers for all queues
3231 * @adapter: board private structure
3232 **/
3233static void igb_clean_all_tx_rings(struct igb_adapter *adapter)
3234{
3235 int i;
3236
3237 for (i = 0; i < adapter->num_tx_queues; i++)
3238 igb_clean_tx_ring(adapter->tx_ring[i]);
3239}
3240
3241/**
3242 * igb_free_rx_resources - Free Rx Resources
3243 * @rx_ring: ring to clean the resources from
3244 *
3245 * Free all receive software resources
3246 **/
3247void igb_free_rx_resources(struct igb_ring *rx_ring)
3248{
3249 igb_clean_rx_ring(rx_ring);
3250
3251 vfree(rx_ring->rx_buffer_info);
3252 rx_ring->rx_buffer_info = NULL;
3253
3254 /* if not set, then don't free */
3255 if (!rx_ring->desc)
3256 return;
3257
3258 dma_free_coherent(rx_ring->dev, rx_ring->size,
3259 rx_ring->desc, rx_ring->dma);
3260
3261 rx_ring->desc = NULL;
3262}
3263
3264/**
3265 * igb_free_all_rx_resources - Free Rx Resources for All Queues
3266 * @adapter: board private structure
3267 *
3268 * Free all receive software resources
3269 **/
3270static void igb_free_all_rx_resources(struct igb_adapter *adapter)
3271{
3272 int i;
3273
3274 for (i = 0; i < adapter->num_rx_queues; i++)
3275 igb_free_rx_resources(adapter->rx_ring[i]);
3276}
3277
3278/**
3279 * igb_clean_rx_ring - Free Rx Buffers per Queue
3280 * @rx_ring: ring to free buffers from
3281 **/
3282static void igb_clean_rx_ring(struct igb_ring *rx_ring)
3283{
3284 unsigned long size;
3285 u16 i;
3286
3287 if (!rx_ring->rx_buffer_info)
3288 return;
3289
3290 /* Free all the Rx ring sk_buffs */
3291 for (i = 0; i < rx_ring->count; i++) {
3292 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
3293 if (buffer_info->dma) {
3294 dma_unmap_single(rx_ring->dev,
3295 buffer_info->dma,
3296 IGB_RX_HDR_LEN,
3297 DMA_FROM_DEVICE);
3298 buffer_info->dma = 0;
3299 }
3300
3301 if (buffer_info->skb) {
3302 dev_kfree_skb(buffer_info->skb);
3303 buffer_info->skb = NULL;
3304 }
3305 if (buffer_info->page_dma) {
3306 dma_unmap_page(rx_ring->dev,
3307 buffer_info->page_dma,
3308 PAGE_SIZE / 2,
3309 DMA_FROM_DEVICE);
3310 buffer_info->page_dma = 0;
3311 }
3312 if (buffer_info->page) {
3313 put_page(buffer_info->page);
3314 buffer_info->page = NULL;
3315 buffer_info->page_offset = 0;
3316 }
3317 }
3318
3319 size = sizeof(struct igb_rx_buffer) * rx_ring->count;
3320 memset(rx_ring->rx_buffer_info, 0, size);
3321
3322 /* Zero out the descriptor ring */
3323 memset(rx_ring->desc, 0, rx_ring->size);
3324
3325 rx_ring->next_to_clean = 0;
3326 rx_ring->next_to_use = 0;
3327}
3328
3329/**
3330 * igb_clean_all_rx_rings - Free Rx Buffers for all queues
3331 * @adapter: board private structure
3332 **/
3333static void igb_clean_all_rx_rings(struct igb_adapter *adapter)
3334{
3335 int i;
3336
3337 for (i = 0; i < adapter->num_rx_queues; i++)
3338 igb_clean_rx_ring(adapter->rx_ring[i]);
3339}
3340
3341/**
3342 * igb_set_mac - Change the Ethernet Address of the NIC
3343 * @netdev: network interface device structure
3344 * @p: pointer to an address structure
3345 *
3346 * Returns 0 on success, negative on failure
3347 **/
3348static int igb_set_mac(struct net_device *netdev, void *p)
3349{
3350 struct igb_adapter *adapter = netdev_priv(netdev);
3351 struct e1000_hw *hw = &adapter->hw;
3352 struct sockaddr *addr = p;
3353
3354 if (!is_valid_ether_addr(addr->sa_data))
3355 return -EADDRNOTAVAIL;
3356
3357 memcpy(netdev->dev_addr, addr->sa_data, netdev->addr_len);
3358 memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
3359
3360 /* set the correct pool for the new PF MAC address in entry 0 */
3361 igb_rar_set_qsel(adapter, hw->mac.addr, 0,
3362 adapter->vfs_allocated_count);
3363
3364 return 0;
3365}
3366
3367/**
3368 * igb_write_mc_addr_list - write multicast addresses to MTA
3369 * @netdev: network interface device structure
3370 *
3371 * Writes multicast address list to the MTA hash table.
3372 * Returns: -ENOMEM on failure
3373 * 0 on no addresses written
3374 * X on writing X addresses to MTA
3375 **/
3376static int igb_write_mc_addr_list(struct net_device *netdev)
3377{
3378 struct igb_adapter *adapter = netdev_priv(netdev);
3379 struct e1000_hw *hw = &adapter->hw;
3380 struct netdev_hw_addr *ha;
3381 u8 *mta_list;
3382 int i;
3383
3384 if (netdev_mc_empty(netdev)) {
3385 /* nothing to program, so clear mc list */
3386 igb_update_mc_addr_list(hw, NULL, 0);
3387 igb_restore_vf_multicasts(adapter);
3388 return 0;
3389 }
3390
3391 mta_list = kzalloc(netdev_mc_count(netdev) * 6, GFP_ATOMIC);
3392 if (!mta_list)
3393 return -ENOMEM;
3394
3395 /* The shared function expects a packed array of only addresses. */
3396 i = 0;
3397 netdev_for_each_mc_addr(ha, netdev)
3398 memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
3399
3400 igb_update_mc_addr_list(hw, mta_list, i);
3401 kfree(mta_list);
3402
3403 return netdev_mc_count(netdev);
3404}
3405
3406/**
3407 * igb_write_uc_addr_list - write unicast addresses to RAR table
3408 * @netdev: network interface device structure
3409 *
3410 * Writes unicast address list to the RAR table.
3411 * Returns: -ENOMEM on failure/insufficient address space
3412 * 0 on no addresses written
3413 * X on writing X addresses to the RAR table
3414 **/
3415static int igb_write_uc_addr_list(struct net_device *netdev)
3416{
3417 struct igb_adapter *adapter = netdev_priv(netdev);
3418 struct e1000_hw *hw = &adapter->hw;
3419 unsigned int vfn = adapter->vfs_allocated_count;
3420 unsigned int rar_entries = hw->mac.rar_entry_count - (vfn + 1);
3421 int count = 0;
3422
3423 /* return ENOMEM indicating insufficient memory for addresses */
3424 if (netdev_uc_count(netdev) > rar_entries)
3425 return -ENOMEM;
3426
3427 if (!netdev_uc_empty(netdev) && rar_entries) {
3428 struct netdev_hw_addr *ha;
3429
3430 netdev_for_each_uc_addr(ha, netdev) {
3431 if (!rar_entries)
3432 break;
3433 igb_rar_set_qsel(adapter, ha->addr,
3434 rar_entries--,
3435 vfn);
3436 count++;
3437 }
3438 }
3439 /* write the addresses in reverse order to avoid write combining */
3440 for (; rar_entries > 0 ; rar_entries--) {
3441 wr32(E1000_RAH(rar_entries), 0);
3442 wr32(E1000_RAL(rar_entries), 0);
3443 }
3444 wrfl();
3445
3446 return count;
3447}
3448
3449/**
3450 * igb_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3451 * @netdev: network interface device structure
3452 *
3453 * The set_rx_mode entry point is called whenever the unicast or multicast
3454 * address lists or the network interface flags are updated. This routine is
3455 * responsible for configuring the hardware for proper unicast, multicast,
3456 * promiscuous mode, and all-multi behavior.
3457 **/
3458static void igb_set_rx_mode(struct net_device *netdev)
3459{
3460 struct igb_adapter *adapter = netdev_priv(netdev);
3461 struct e1000_hw *hw = &adapter->hw;
3462 unsigned int vfn = adapter->vfs_allocated_count;
3463 u32 rctl, vmolr = 0;
3464 int count;
3465
3466 /* Check for Promiscuous and All Multicast modes */
3467 rctl = rd32(E1000_RCTL);
3468
3469 /* clear the effected bits */
3470 rctl &= ~(E1000_RCTL_UPE | E1000_RCTL_MPE | E1000_RCTL_VFE);
3471
3472 if (netdev->flags & IFF_PROMISC) {
3473 rctl |= (E1000_RCTL_UPE | E1000_RCTL_MPE);
3474 vmolr |= (E1000_VMOLR_ROPE | E1000_VMOLR_MPME);
3475 } else {
3476 if (netdev->flags & IFF_ALLMULTI) {
3477 rctl |= E1000_RCTL_MPE;
3478 vmolr |= E1000_VMOLR_MPME;
3479 } else {
3480 /*
3481 * Write addresses to the MTA, if the attempt fails
3482 * then we should just turn on promiscuous mode so
3483 * that we can at least receive multicast traffic
3484 */
3485 count = igb_write_mc_addr_list(netdev);
3486 if (count < 0) {
3487 rctl |= E1000_RCTL_MPE;
3488 vmolr |= E1000_VMOLR_MPME;
3489 } else if (count) {
3490 vmolr |= E1000_VMOLR_ROMPE;
3491 }
3492 }
3493 /*
3494 * Write addresses to available RAR registers, if there is not
3495 * sufficient space to store all the addresses then enable
3496 * unicast promiscuous mode
3497 */
3498 count = igb_write_uc_addr_list(netdev);
3499 if (count < 0) {
3500 rctl |= E1000_RCTL_UPE;
3501 vmolr |= E1000_VMOLR_ROPE;
3502 }
3503 rctl |= E1000_RCTL_VFE;
3504 }
3505 wr32(E1000_RCTL, rctl);
3506
3507 /*
3508 * In order to support SR-IOV and eventually VMDq it is necessary to set
3509 * the VMOLR to enable the appropriate modes. Without this workaround
3510 * we will have issues with VLAN tag stripping not being done for frames
3511 * that are only arriving because we are the default pool
3512 */
3513 if ((hw->mac.type < e1000_82576) || (hw->mac.type > e1000_i350))
3514 return;
3515
3516 vmolr |= rd32(E1000_VMOLR(vfn)) &
3517 ~(E1000_VMOLR_ROPE | E1000_VMOLR_MPME | E1000_VMOLR_ROMPE);
3518 wr32(E1000_VMOLR(vfn), vmolr);
3519 igb_restore_vf_multicasts(adapter);
3520}
3521
3522static void igb_check_wvbr(struct igb_adapter *adapter)
3523{
3524 struct e1000_hw *hw = &adapter->hw;
3525 u32 wvbr = 0;
3526
3527 switch (hw->mac.type) {
3528 case e1000_82576:
3529 case e1000_i350:
3530 if (!(wvbr = rd32(E1000_WVBR)))
3531 return;
3532 break;
3533 default:
3534 break;
3535 }
3536
3537 adapter->wvbr |= wvbr;
3538}
3539
3540#define IGB_STAGGERED_QUEUE_OFFSET 8
3541
3542static void igb_spoof_check(struct igb_adapter *adapter)
3543{
3544 int j;
3545
3546 if (!adapter->wvbr)
3547 return;
3548
3549 for(j = 0; j < adapter->vfs_allocated_count; j++) {
3550 if (adapter->wvbr & (1 << j) ||
3551 adapter->wvbr & (1 << (j + IGB_STAGGERED_QUEUE_OFFSET))) {
3552 dev_warn(&adapter->pdev->dev,
3553 "Spoof event(s) detected on VF %d\n", j);
3554 adapter->wvbr &=
3555 ~((1 << j) |
3556 (1 << (j + IGB_STAGGERED_QUEUE_OFFSET)));
3557 }
3558 }
3559}
3560
3561/* Need to wait a few seconds after link up to get diagnostic information from
3562 * the phy */
3563static void igb_update_phy_info(unsigned long data)
3564{
3565 struct igb_adapter *adapter = (struct igb_adapter *) data;
3566 igb_get_phy_info(&adapter->hw);
3567}
3568
3569/**
3570 * igb_has_link - check shared code for link and determine up/down
3571 * @adapter: pointer to driver private info
3572 **/
3573bool igb_has_link(struct igb_adapter *adapter)
3574{
3575 struct e1000_hw *hw = &adapter->hw;
3576 bool link_active = false;
3577 s32 ret_val = 0;
3578
3579 /* get_link_status is set on LSC (link status) interrupt or
3580 * rx sequence error interrupt. get_link_status will stay
3581 * false until the e1000_check_for_link establishes link
3582 * for copper adapters ONLY
3583 */
3584 switch (hw->phy.media_type) {
3585 case e1000_media_type_copper:
3586 if (hw->mac.get_link_status) {
3587 ret_val = hw->mac.ops.check_for_link(hw);
3588 link_active = !hw->mac.get_link_status;
3589 } else {
3590 link_active = true;
3591 }
3592 break;
3593 case e1000_media_type_internal_serdes:
3594 ret_val = hw->mac.ops.check_for_link(hw);
3595 link_active = hw->mac.serdes_has_link;
3596 break;
3597 default:
3598 case e1000_media_type_unknown:
3599 break;
3600 }
3601
3602 return link_active;
3603}
3604
3605static bool igb_thermal_sensor_event(struct e1000_hw *hw, u32 event)
3606{
3607 bool ret = false;
3608 u32 ctrl_ext, thstat;
3609
3610 /* check for thermal sensor event on i350 copper only */
3611 if (hw->mac.type == e1000_i350) {
3612 thstat = rd32(E1000_THSTAT);
3613 ctrl_ext = rd32(E1000_CTRL_EXT);
3614
3615 if ((hw->phy.media_type == e1000_media_type_copper) &&
3616 !(ctrl_ext & E1000_CTRL_EXT_LINK_MODE_SGMII)) {
3617 ret = !!(thstat & event);
3618 }
3619 }
3620
3621 return ret;
3622}
3623
3624/**
3625 * igb_watchdog - Timer Call-back
3626 * @data: pointer to adapter cast into an unsigned long
3627 **/
3628static void igb_watchdog(unsigned long data)
3629{
3630 struct igb_adapter *adapter = (struct igb_adapter *)data;
3631 /* Do the rest outside of interrupt context */
3632 schedule_work(&adapter->watchdog_task);
3633}
3634
3635static void igb_watchdog_task(struct work_struct *work)
3636{
3637 struct igb_adapter *adapter = container_of(work,
3638 struct igb_adapter,
3639 watchdog_task);
3640 struct e1000_hw *hw = &adapter->hw;
3641 struct net_device *netdev = adapter->netdev;
3642 u32 link;
3643 int i;
3644
3645 link = igb_has_link(adapter);
3646 if (link) {
3647 /* Cancel scheduled suspend requests. */
3648 pm_runtime_resume(netdev->dev.parent);
3649
3650 if (!netif_carrier_ok(netdev)) {
3651 u32 ctrl;
3652 hw->mac.ops.get_speed_and_duplex(hw,
3653 &adapter->link_speed,
3654 &adapter->link_duplex);
3655
3656 ctrl = rd32(E1000_CTRL);
3657 /* Links status message must follow this format */
3658 printk(KERN_INFO "igb: %s NIC Link is Up %d Mbps %s "
3659 "Duplex, Flow Control: %s\n",
3660 netdev->name,
3661 adapter->link_speed,
3662 adapter->link_duplex == FULL_DUPLEX ?
3663 "Full" : "Half",
3664 (ctrl & E1000_CTRL_TFCE) &&
3665 (ctrl & E1000_CTRL_RFCE) ? "RX/TX" :
3666 (ctrl & E1000_CTRL_RFCE) ? "RX" :
3667 (ctrl & E1000_CTRL_TFCE) ? "TX" : "None");
3668
3669 /* check for thermal sensor event */
3670 if (igb_thermal_sensor_event(hw,
3671 E1000_THSTAT_LINK_THROTTLE)) {
3672 netdev_info(netdev, "The network adapter link "
3673 "speed was downshifted because it "
3674 "overheated\n");
3675 }
3676
3677 /* adjust timeout factor according to speed/duplex */
3678 adapter->tx_timeout_factor = 1;
3679 switch (adapter->link_speed) {
3680 case SPEED_10:
3681 adapter->tx_timeout_factor = 14;
3682 break;
3683 case SPEED_100:
3684 /* maybe add some timeout factor ? */
3685 break;
3686 }
3687
3688 netif_carrier_on(netdev);
3689
3690 igb_ping_all_vfs(adapter);
3691 igb_check_vf_rate_limit(adapter);
3692
3693 /* link state has changed, schedule phy info update */
3694 if (!test_bit(__IGB_DOWN, &adapter->state))
3695 mod_timer(&adapter->phy_info_timer,
3696 round_jiffies(jiffies + 2 * HZ));
3697 }
3698 } else {
3699 if (netif_carrier_ok(netdev)) {
3700 adapter->link_speed = 0;
3701 adapter->link_duplex = 0;
3702
3703 /* check for thermal sensor event */
3704 if (igb_thermal_sensor_event(hw,
3705 E1000_THSTAT_PWR_DOWN)) {
3706 netdev_err(netdev, "The network adapter was "
3707 "stopped because it overheated\n");
3708 }
3709
3710 /* Links status message must follow this format */
3711 printk(KERN_INFO "igb: %s NIC Link is Down\n",
3712 netdev->name);
3713 netif_carrier_off(netdev);
3714
3715 igb_ping_all_vfs(adapter);
3716
3717 /* link state has changed, schedule phy info update */
3718 if (!test_bit(__IGB_DOWN, &adapter->state))
3719 mod_timer(&adapter->phy_info_timer,
3720 round_jiffies(jiffies + 2 * HZ));
3721
3722 pm_schedule_suspend(netdev->dev.parent,
3723 MSEC_PER_SEC * 5);
3724 }
3725 }
3726
3727 spin_lock(&adapter->stats64_lock);
3728 igb_update_stats(adapter, &adapter->stats64);
3729 spin_unlock(&adapter->stats64_lock);
3730
3731 for (i = 0; i < adapter->num_tx_queues; i++) {
3732 struct igb_ring *tx_ring = adapter->tx_ring[i];
3733 if (!netif_carrier_ok(netdev)) {
3734 /* We've lost link, so the controller stops DMA,
3735 * but we've got queued Tx work that's never going
3736 * to get done, so reset controller to flush Tx.
3737 * (Do the reset outside of interrupt context). */
3738 if (igb_desc_unused(tx_ring) + 1 < tx_ring->count) {
3739 adapter->tx_timeout_count++;
3740 schedule_work(&adapter->reset_task);
3741 /* return immediately since reset is imminent */
3742 return;
3743 }
3744 }
3745
3746 /* Force detection of hung controller every watchdog period */
3747 set_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
3748 }
3749
3750 /* Cause software interrupt to ensure rx ring is cleaned */
3751 if (adapter->msix_entries) {
3752 u32 eics = 0;
3753 for (i = 0; i < adapter->num_q_vectors; i++)
3754 eics |= adapter->q_vector[i]->eims_value;
3755 wr32(E1000_EICS, eics);
3756 } else {
3757 wr32(E1000_ICS, E1000_ICS_RXDMT0);
3758 }
3759
3760 igb_spoof_check(adapter);
3761
3762 /* Reset the timer */
3763 if (!test_bit(__IGB_DOWN, &adapter->state))
3764 mod_timer(&adapter->watchdog_timer,
3765 round_jiffies(jiffies + 2 * HZ));
3766}
3767
3768enum latency_range {
3769 lowest_latency = 0,
3770 low_latency = 1,
3771 bulk_latency = 2,
3772 latency_invalid = 255
3773};
3774
3775/**
3776 * igb_update_ring_itr - update the dynamic ITR value based on packet size
3777 *
3778 * Stores a new ITR value based on strictly on packet size. This
3779 * algorithm is less sophisticated than that used in igb_update_itr,
3780 * due to the difficulty of synchronizing statistics across multiple
3781 * receive rings. The divisors and thresholds used by this function
3782 * were determined based on theoretical maximum wire speed and testing
3783 * data, in order to minimize response time while increasing bulk
3784 * throughput.
3785 * This functionality is controlled by the InterruptThrottleRate module
3786 * parameter (see igb_param.c)
3787 * NOTE: This function is called only when operating in a multiqueue
3788 * receive environment.
3789 * @q_vector: pointer to q_vector
3790 **/
3791static void igb_update_ring_itr(struct igb_q_vector *q_vector)
3792{
3793 int new_val = q_vector->itr_val;
3794 int avg_wire_size = 0;
3795 struct igb_adapter *adapter = q_vector->adapter;
3796 unsigned int packets;
3797
3798 /* For non-gigabit speeds, just fix the interrupt rate at 4000
3799 * ints/sec - ITR timer value of 120 ticks.
3800 */
3801 if (adapter->link_speed != SPEED_1000) {
3802 new_val = IGB_4K_ITR;
3803 goto set_itr_val;
3804 }
3805
3806 packets = q_vector->rx.total_packets;
3807 if (packets)
3808 avg_wire_size = q_vector->rx.total_bytes / packets;
3809
3810 packets = q_vector->tx.total_packets;
3811 if (packets)
3812 avg_wire_size = max_t(u32, avg_wire_size,
3813 q_vector->tx.total_bytes / packets);
3814
3815 /* if avg_wire_size isn't set no work was done */
3816 if (!avg_wire_size)
3817 goto clear_counts;
3818
3819 /* Add 24 bytes to size to account for CRC, preamble, and gap */
3820 avg_wire_size += 24;
3821
3822 /* Don't starve jumbo frames */
3823 avg_wire_size = min(avg_wire_size, 3000);
3824
3825 /* Give a little boost to mid-size frames */
3826 if ((avg_wire_size > 300) && (avg_wire_size < 1200))
3827 new_val = avg_wire_size / 3;
3828 else
3829 new_val = avg_wire_size / 2;
3830
3831 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3832 if (new_val < IGB_20K_ITR &&
3833 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3834 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3835 new_val = IGB_20K_ITR;
3836
3837set_itr_val:
3838 if (new_val != q_vector->itr_val) {
3839 q_vector->itr_val = new_val;
3840 q_vector->set_itr = 1;
3841 }
3842clear_counts:
3843 q_vector->rx.total_bytes = 0;
3844 q_vector->rx.total_packets = 0;
3845 q_vector->tx.total_bytes = 0;
3846 q_vector->tx.total_packets = 0;
3847}
3848
3849/**
3850 * igb_update_itr - update the dynamic ITR value based on statistics
3851 * Stores a new ITR value based on packets and byte
3852 * counts during the last interrupt. The advantage of per interrupt
3853 * computation is faster updates and more accurate ITR for the current
3854 * traffic pattern. Constants in this function were computed
3855 * based on theoretical maximum wire speed and thresholds were set based
3856 * on testing data as well as attempting to minimize response time
3857 * while increasing bulk throughput.
3858 * this functionality is controlled by the InterruptThrottleRate module
3859 * parameter (see igb_param.c)
3860 * NOTE: These calculations are only valid when operating in a single-
3861 * queue environment.
3862 * @q_vector: pointer to q_vector
3863 * @ring_container: ring info to update the itr for
3864 **/
3865static void igb_update_itr(struct igb_q_vector *q_vector,
3866 struct igb_ring_container *ring_container)
3867{
3868 unsigned int packets = ring_container->total_packets;
3869 unsigned int bytes = ring_container->total_bytes;
3870 u8 itrval = ring_container->itr;
3871
3872 /* no packets, exit with status unchanged */
3873 if (packets == 0)
3874 return;
3875
3876 switch (itrval) {
3877 case lowest_latency:
3878 /* handle TSO and jumbo frames */
3879 if (bytes/packets > 8000)
3880 itrval = bulk_latency;
3881 else if ((packets < 5) && (bytes > 512))
3882 itrval = low_latency;
3883 break;
3884 case low_latency: /* 50 usec aka 20000 ints/s */
3885 if (bytes > 10000) {
3886 /* this if handles the TSO accounting */
3887 if (bytes/packets > 8000) {
3888 itrval = bulk_latency;
3889 } else if ((packets < 10) || ((bytes/packets) > 1200)) {
3890 itrval = bulk_latency;
3891 } else if ((packets > 35)) {
3892 itrval = lowest_latency;
3893 }
3894 } else if (bytes/packets > 2000) {
3895 itrval = bulk_latency;
3896 } else if (packets <= 2 && bytes < 512) {
3897 itrval = lowest_latency;
3898 }
3899 break;
3900 case bulk_latency: /* 250 usec aka 4000 ints/s */
3901 if (bytes > 25000) {
3902 if (packets > 35)
3903 itrval = low_latency;
3904 } else if (bytes < 1500) {
3905 itrval = low_latency;
3906 }
3907 break;
3908 }
3909
3910 /* clear work counters since we have the values we need */
3911 ring_container->total_bytes = 0;
3912 ring_container->total_packets = 0;
3913
3914 /* write updated itr to ring container */
3915 ring_container->itr = itrval;
3916}
3917
3918static void igb_set_itr(struct igb_q_vector *q_vector)
3919{
3920 struct igb_adapter *adapter = q_vector->adapter;
3921 u32 new_itr = q_vector->itr_val;
3922 u8 current_itr = 0;
3923
3924 /* for non-gigabit speeds, just fix the interrupt rate at 4000 */
3925 if (adapter->link_speed != SPEED_1000) {
3926 current_itr = 0;
3927 new_itr = IGB_4K_ITR;
3928 goto set_itr_now;
3929 }
3930
3931 igb_update_itr(q_vector, &q_vector->tx);
3932 igb_update_itr(q_vector, &q_vector->rx);
3933
3934 current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
3935
3936 /* conservative mode (itr 3) eliminates the lowest_latency setting */
3937 if (current_itr == lowest_latency &&
3938 ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
3939 (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
3940 current_itr = low_latency;
3941
3942 switch (current_itr) {
3943 /* counts and packets in update_itr are dependent on these numbers */
3944 case lowest_latency:
3945 new_itr = IGB_70K_ITR; /* 70,000 ints/sec */
3946 break;
3947 case low_latency:
3948 new_itr = IGB_20K_ITR; /* 20,000 ints/sec */
3949 break;
3950 case bulk_latency:
3951 new_itr = IGB_4K_ITR; /* 4,000 ints/sec */
3952 break;
3953 default:
3954 break;
3955 }
3956
3957set_itr_now:
3958 if (new_itr != q_vector->itr_val) {
3959 /* this attempts to bias the interrupt rate towards Bulk
3960 * by adding intermediate steps when interrupt rate is
3961 * increasing */
3962 new_itr = new_itr > q_vector->itr_val ?
3963 max((new_itr * q_vector->itr_val) /
3964 (new_itr + (q_vector->itr_val >> 2)),
3965 new_itr) :
3966 new_itr;
3967 /* Don't write the value here; it resets the adapter's
3968 * internal timer, and causes us to delay far longer than
3969 * we should between interrupts. Instead, we write the ITR
3970 * value at the beginning of the next interrupt so the timing
3971 * ends up being correct.
3972 */
3973 q_vector->itr_val = new_itr;
3974 q_vector->set_itr = 1;
3975 }
3976}
3977
3978static void igb_tx_ctxtdesc(struct igb_ring *tx_ring, u32 vlan_macip_lens,
3979 u32 type_tucmd, u32 mss_l4len_idx)
3980{
3981 struct e1000_adv_tx_context_desc *context_desc;
3982 u16 i = tx_ring->next_to_use;
3983
3984 context_desc = IGB_TX_CTXTDESC(tx_ring, i);
3985
3986 i++;
3987 tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
3988
3989 /* set bits to identify this as an advanced context descriptor */
3990 type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
3991
3992 /* For 82575, context index must be unique per ring. */
3993 if (test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
3994 mss_l4len_idx |= tx_ring->reg_idx << 4;
3995
3996 context_desc->vlan_macip_lens = cpu_to_le32(vlan_macip_lens);
3997 context_desc->seqnum_seed = 0;
3998 context_desc->type_tucmd_mlhl = cpu_to_le32(type_tucmd);
3999 context_desc->mss_l4len_idx = cpu_to_le32(mss_l4len_idx);
4000}
4001
4002static int igb_tso(struct igb_ring *tx_ring,
4003 struct igb_tx_buffer *first,
4004 u8 *hdr_len)
4005{
4006 struct sk_buff *skb = first->skb;
4007 u32 vlan_macip_lens, type_tucmd;
4008 u32 mss_l4len_idx, l4len;
4009
4010 if (!skb_is_gso(skb))
4011 return 0;
4012
4013 if (skb_header_cloned(skb)) {
4014 int err = pskb_expand_head(skb, 0, 0, GFP_ATOMIC);
4015 if (err)
4016 return err;
4017 }
4018
4019 /* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
4020 type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
4021
4022 if (first->protocol == __constant_htons(ETH_P_IP)) {
4023 struct iphdr *iph = ip_hdr(skb);
4024 iph->tot_len = 0;
4025 iph->check = 0;
4026 tcp_hdr(skb)->check = ~csum_tcpudp_magic(iph->saddr,
4027 iph->daddr, 0,
4028 IPPROTO_TCP,
4029 0);
4030 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4031 first->tx_flags |= IGB_TX_FLAGS_TSO |
4032 IGB_TX_FLAGS_CSUM |
4033 IGB_TX_FLAGS_IPV4;
4034 } else if (skb_is_gso_v6(skb)) {
4035 ipv6_hdr(skb)->payload_len = 0;
4036 tcp_hdr(skb)->check = ~csum_ipv6_magic(&ipv6_hdr(skb)->saddr,
4037 &ipv6_hdr(skb)->daddr,
4038 0, IPPROTO_TCP, 0);
4039 first->tx_flags |= IGB_TX_FLAGS_TSO |
4040 IGB_TX_FLAGS_CSUM;
4041 }
4042
4043 /* compute header lengths */
4044 l4len = tcp_hdrlen(skb);
4045 *hdr_len = skb_transport_offset(skb) + l4len;
4046
4047 /* update gso size and bytecount with header size */
4048 first->gso_segs = skb_shinfo(skb)->gso_segs;
4049 first->bytecount += (first->gso_segs - 1) * *hdr_len;
4050
4051 /* MSS L4LEN IDX */
4052 mss_l4len_idx = l4len << E1000_ADVTXD_L4LEN_SHIFT;
4053 mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
4054
4055 /* VLAN MACLEN IPLEN */
4056 vlan_macip_lens = skb_network_header_len(skb);
4057 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4058 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4059
4060 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4061
4062 return 1;
4063}
4064
4065static void igb_tx_csum(struct igb_ring *tx_ring, struct igb_tx_buffer *first)
4066{
4067 struct sk_buff *skb = first->skb;
4068 u32 vlan_macip_lens = 0;
4069 u32 mss_l4len_idx = 0;
4070 u32 type_tucmd = 0;
4071
4072 if (skb->ip_summed != CHECKSUM_PARTIAL) {
4073 if (!(first->tx_flags & IGB_TX_FLAGS_VLAN))
4074 return;
4075 } else {
4076 u8 l4_hdr = 0;
4077 switch (first->protocol) {
4078 case __constant_htons(ETH_P_IP):
4079 vlan_macip_lens |= skb_network_header_len(skb);
4080 type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
4081 l4_hdr = ip_hdr(skb)->protocol;
4082 break;
4083 case __constant_htons(ETH_P_IPV6):
4084 vlan_macip_lens |= skb_network_header_len(skb);
4085 l4_hdr = ipv6_hdr(skb)->nexthdr;
4086 break;
4087 default:
4088 if (unlikely(net_ratelimit())) {
4089 dev_warn(tx_ring->dev,
4090 "partial checksum but proto=%x!\n",
4091 first->protocol);
4092 }
4093 break;
4094 }
4095
4096 switch (l4_hdr) {
4097 case IPPROTO_TCP:
4098 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_TCP;
4099 mss_l4len_idx = tcp_hdrlen(skb) <<
4100 E1000_ADVTXD_L4LEN_SHIFT;
4101 break;
4102 case IPPROTO_SCTP:
4103 type_tucmd |= E1000_ADVTXD_TUCMD_L4T_SCTP;
4104 mss_l4len_idx = sizeof(struct sctphdr) <<
4105 E1000_ADVTXD_L4LEN_SHIFT;
4106 break;
4107 case IPPROTO_UDP:
4108 mss_l4len_idx = sizeof(struct udphdr) <<
4109 E1000_ADVTXD_L4LEN_SHIFT;
4110 break;
4111 default:
4112 if (unlikely(net_ratelimit())) {
4113 dev_warn(tx_ring->dev,
4114 "partial checksum but l4 proto=%x!\n",
4115 l4_hdr);
4116 }
4117 break;
4118 }
4119
4120 /* update TX checksum flag */
4121 first->tx_flags |= IGB_TX_FLAGS_CSUM;
4122 }
4123
4124 vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
4125 vlan_macip_lens |= first->tx_flags & IGB_TX_FLAGS_VLAN_MASK;
4126
4127 igb_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
4128}
4129
4130static __le32 igb_tx_cmd_type(u32 tx_flags)
4131{
4132 /* set type for advanced descriptor with frame checksum insertion */
4133 __le32 cmd_type = cpu_to_le32(E1000_ADVTXD_DTYP_DATA |
4134 E1000_ADVTXD_DCMD_IFCS |
4135 E1000_ADVTXD_DCMD_DEXT);
4136
4137 /* set HW vlan bit if vlan is present */
4138 if (tx_flags & IGB_TX_FLAGS_VLAN)
4139 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_VLE);
4140
4141 /* set timestamp bit if present */
4142 if (tx_flags & IGB_TX_FLAGS_TSTAMP)
4143 cmd_type |= cpu_to_le32(E1000_ADVTXD_MAC_TSTAMP);
4144
4145 /* set segmentation bits for TSO */
4146 if (tx_flags & IGB_TX_FLAGS_TSO)
4147 cmd_type |= cpu_to_le32(E1000_ADVTXD_DCMD_TSE);
4148
4149 return cmd_type;
4150}
4151
4152static void igb_tx_olinfo_status(struct igb_ring *tx_ring,
4153 union e1000_adv_tx_desc *tx_desc,
4154 u32 tx_flags, unsigned int paylen)
4155{
4156 u32 olinfo_status = paylen << E1000_ADVTXD_PAYLEN_SHIFT;
4157
4158 /* 82575 requires a unique index per ring if any offload is enabled */
4159 if ((tx_flags & (IGB_TX_FLAGS_CSUM | IGB_TX_FLAGS_VLAN)) &&
4160 test_bit(IGB_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
4161 olinfo_status |= tx_ring->reg_idx << 4;
4162
4163 /* insert L4 checksum */
4164 if (tx_flags & IGB_TX_FLAGS_CSUM) {
4165 olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
4166
4167 /* insert IPv4 checksum */
4168 if (tx_flags & IGB_TX_FLAGS_IPV4)
4169 olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
4170 }
4171
4172 tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
4173}
4174
4175/*
4176 * The largest size we can write to the descriptor is 65535. In order to
4177 * maintain a power of two alignment we have to limit ourselves to 32K.
4178 */
4179#define IGB_MAX_TXD_PWR 15
4180#define IGB_MAX_DATA_PER_TXD (1<<IGB_MAX_TXD_PWR)
4181
4182static void igb_tx_map(struct igb_ring *tx_ring,
4183 struct igb_tx_buffer *first,
4184 const u8 hdr_len)
4185{
4186 struct sk_buff *skb = first->skb;
4187 struct igb_tx_buffer *tx_buffer_info;
4188 union e1000_adv_tx_desc *tx_desc;
4189 dma_addr_t dma;
4190 struct skb_frag_struct *frag = &skb_shinfo(skb)->frags[0];
4191 unsigned int data_len = skb->data_len;
4192 unsigned int size = skb_headlen(skb);
4193 unsigned int paylen = skb->len - hdr_len;
4194 __le32 cmd_type;
4195 u32 tx_flags = first->tx_flags;
4196 u16 i = tx_ring->next_to_use;
4197
4198 tx_desc = IGB_TX_DESC(tx_ring, i);
4199
4200 igb_tx_olinfo_status(tx_ring, tx_desc, tx_flags, paylen);
4201 cmd_type = igb_tx_cmd_type(tx_flags);
4202
4203 dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
4204 if (dma_mapping_error(tx_ring->dev, dma))
4205 goto dma_error;
4206
4207 /* record length, and DMA address */
4208 first->length = size;
4209 first->dma = dma;
4210 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4211
4212 for (;;) {
4213 while (unlikely(size > IGB_MAX_DATA_PER_TXD)) {
4214 tx_desc->read.cmd_type_len =
4215 cmd_type | cpu_to_le32(IGB_MAX_DATA_PER_TXD);
4216
4217 i++;
4218 tx_desc++;
4219 if (i == tx_ring->count) {
4220 tx_desc = IGB_TX_DESC(tx_ring, 0);
4221 i = 0;
4222 }
4223
4224 dma += IGB_MAX_DATA_PER_TXD;
4225 size -= IGB_MAX_DATA_PER_TXD;
4226
4227 tx_desc->read.olinfo_status = 0;
4228 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4229 }
4230
4231 if (likely(!data_len))
4232 break;
4233
4234 tx_desc->read.cmd_type_len = cmd_type | cpu_to_le32(size);
4235
4236 i++;
4237 tx_desc++;
4238 if (i == tx_ring->count) {
4239 tx_desc = IGB_TX_DESC(tx_ring, 0);
4240 i = 0;
4241 }
4242
4243 size = skb_frag_size(frag);
4244 data_len -= size;
4245
4246 dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
4247 size, DMA_TO_DEVICE);
4248 if (dma_mapping_error(tx_ring->dev, dma))
4249 goto dma_error;
4250
4251 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4252 tx_buffer_info->length = size;
4253 tx_buffer_info->dma = dma;
4254
4255 tx_desc->read.olinfo_status = 0;
4256 tx_desc->read.buffer_addr = cpu_to_le64(dma);
4257
4258 frag++;
4259 }
4260
4261 netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
4262
4263 /* write last descriptor with RS and EOP bits */
4264 cmd_type |= cpu_to_le32(size) | cpu_to_le32(IGB_TXD_DCMD);
4265 if (unlikely(skb->no_fcs))
4266 cmd_type &= ~(cpu_to_le32(E1000_ADVTXD_DCMD_IFCS));
4267 tx_desc->read.cmd_type_len = cmd_type;
4268
4269 /* set the timestamp */
4270 first->time_stamp = jiffies;
4271
4272 /*
4273 * Force memory writes to complete before letting h/w know there
4274 * are new descriptors to fetch. (Only applicable for weak-ordered
4275 * memory model archs, such as IA-64).
4276 *
4277 * We also need this memory barrier to make certain all of the
4278 * status bits have been updated before next_to_watch is written.
4279 */
4280 wmb();
4281
4282 /* set next_to_watch value indicating a packet is present */
4283 first->next_to_watch = tx_desc;
4284
4285 i++;
4286 if (i == tx_ring->count)
4287 i = 0;
4288
4289 tx_ring->next_to_use = i;
4290
4291 writel(i, tx_ring->tail);
4292
4293 /* we need this if more than one processor can write to our tail
4294 * at a time, it syncronizes IO on IA64/Altix systems */
4295 mmiowb();
4296
4297 return;
4298
4299dma_error:
4300 dev_err(tx_ring->dev, "TX DMA map failed\n");
4301
4302 /* clear dma mappings for failed tx_buffer_info map */
4303 for (;;) {
4304 tx_buffer_info = &tx_ring->tx_buffer_info[i];
4305 igb_unmap_and_free_tx_resource(tx_ring, tx_buffer_info);
4306 if (tx_buffer_info == first)
4307 break;
4308 if (i == 0)
4309 i = tx_ring->count;
4310 i--;
4311 }
4312
4313 tx_ring->next_to_use = i;
4314}
4315
4316static int __igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4317{
4318 struct net_device *netdev = tx_ring->netdev;
4319
4320 netif_stop_subqueue(netdev, tx_ring->queue_index);
4321
4322 /* Herbert's original patch had:
4323 * smp_mb__after_netif_stop_queue();
4324 * but since that doesn't exist yet, just open code it. */
4325 smp_mb();
4326
4327 /* We need to check again in a case another CPU has just
4328 * made room available. */
4329 if (igb_desc_unused(tx_ring) < size)
4330 return -EBUSY;
4331
4332 /* A reprieve! */
4333 netif_wake_subqueue(netdev, tx_ring->queue_index);
4334
4335 u64_stats_update_begin(&tx_ring->tx_syncp2);
4336 tx_ring->tx_stats.restart_queue2++;
4337 u64_stats_update_end(&tx_ring->tx_syncp2);
4338
4339 return 0;
4340}
4341
4342static inline int igb_maybe_stop_tx(struct igb_ring *tx_ring, const u16 size)
4343{
4344 if (igb_desc_unused(tx_ring) >= size)
4345 return 0;
4346 return __igb_maybe_stop_tx(tx_ring, size);
4347}
4348
4349netdev_tx_t igb_xmit_frame_ring(struct sk_buff *skb,
4350 struct igb_ring *tx_ring)
4351{
4352 struct igb_tx_buffer *first;
4353 int tso;
4354 u32 tx_flags = 0;
4355 __be16 protocol = vlan_get_protocol(skb);
4356 u8 hdr_len = 0;
4357
4358 /* need: 1 descriptor per page,
4359 * + 2 desc gap to keep tail from touching head,
4360 * + 1 desc for skb->data,
4361 * + 1 desc for context descriptor,
4362 * otherwise try next time */
4363 if (igb_maybe_stop_tx(tx_ring, skb_shinfo(skb)->nr_frags + 4)) {
4364 /* this is a hard error */
4365 return NETDEV_TX_BUSY;
4366 }
4367
4368 /* record the location of the first descriptor for this packet */
4369 first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
4370 first->skb = skb;
4371 first->bytecount = skb->len;
4372 first->gso_segs = 1;
4373
4374 if (unlikely(skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
4375 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
4376 tx_flags |= IGB_TX_FLAGS_TSTAMP;
4377 }
4378
4379 if (vlan_tx_tag_present(skb)) {
4380 tx_flags |= IGB_TX_FLAGS_VLAN;
4381 tx_flags |= (vlan_tx_tag_get(skb) << IGB_TX_FLAGS_VLAN_SHIFT);
4382 }
4383
4384 /* record initial flags and protocol */
4385 first->tx_flags = tx_flags;
4386 first->protocol = protocol;
4387
4388 tso = igb_tso(tx_ring, first, &hdr_len);
4389 if (tso < 0)
4390 goto out_drop;
4391 else if (!tso)
4392 igb_tx_csum(tx_ring, first);
4393
4394 igb_tx_map(tx_ring, first, hdr_len);
4395
4396 /* Make sure there is space in the ring for the next send. */
4397 igb_maybe_stop_tx(tx_ring, MAX_SKB_FRAGS + 4);
4398
4399 return NETDEV_TX_OK;
4400
4401out_drop:
4402 igb_unmap_and_free_tx_resource(tx_ring, first);
4403
4404 return NETDEV_TX_OK;
4405}
4406
4407static inline struct igb_ring *igb_tx_queue_mapping(struct igb_adapter *adapter,
4408 struct sk_buff *skb)
4409{
4410 unsigned int r_idx = skb->queue_mapping;
4411
4412 if (r_idx >= adapter->num_tx_queues)
4413 r_idx = r_idx % adapter->num_tx_queues;
4414
4415 return adapter->tx_ring[r_idx];
4416}
4417
4418static netdev_tx_t igb_xmit_frame(struct sk_buff *skb,
4419 struct net_device *netdev)
4420{
4421 struct igb_adapter *adapter = netdev_priv(netdev);
4422
4423 if (test_bit(__IGB_DOWN, &adapter->state)) {
4424 dev_kfree_skb_any(skb);
4425 return NETDEV_TX_OK;
4426 }
4427
4428 if (skb->len <= 0) {
4429 dev_kfree_skb_any(skb);
4430 return NETDEV_TX_OK;
4431 }
4432
4433 /*
4434 * The minimum packet size with TCTL.PSP set is 17 so pad the skb
4435 * in order to meet this minimum size requirement.
4436 */
4437 if (skb->len < 17) {
4438 if (skb_padto(skb, 17))
4439 return NETDEV_TX_OK;
4440 skb->len = 17;
4441 }
4442
4443 return igb_xmit_frame_ring(skb, igb_tx_queue_mapping(adapter, skb));
4444}
4445
4446/**
4447 * igb_tx_timeout - Respond to a Tx Hang
4448 * @netdev: network interface device structure
4449 **/
4450static void igb_tx_timeout(struct net_device *netdev)
4451{
4452 struct igb_adapter *adapter = netdev_priv(netdev);
4453 struct e1000_hw *hw = &adapter->hw;
4454
4455 /* Do the reset outside of interrupt context */
4456 adapter->tx_timeout_count++;
4457
4458 if (hw->mac.type >= e1000_82580)
4459 hw->dev_spec._82575.global_device_reset = true;
4460
4461 schedule_work(&adapter->reset_task);
4462 wr32(E1000_EICS,
4463 (adapter->eims_enable_mask & ~adapter->eims_other));
4464}
4465
4466static void igb_reset_task(struct work_struct *work)
4467{
4468 struct igb_adapter *adapter;
4469 adapter = container_of(work, struct igb_adapter, reset_task);
4470
4471 igb_dump(adapter);
4472 netdev_err(adapter->netdev, "Reset adapter\n");
4473 igb_reinit_locked(adapter);
4474}
4475
4476/**
4477 * igb_get_stats64 - Get System Network Statistics
4478 * @netdev: network interface device structure
4479 * @stats: rtnl_link_stats64 pointer
4480 *
4481 **/
4482static struct rtnl_link_stats64 *igb_get_stats64(struct net_device *netdev,
4483 struct rtnl_link_stats64 *stats)
4484{
4485 struct igb_adapter *adapter = netdev_priv(netdev);
4486
4487 spin_lock(&adapter->stats64_lock);
4488 igb_update_stats(adapter, &adapter->stats64);
4489 memcpy(stats, &adapter->stats64, sizeof(*stats));
4490 spin_unlock(&adapter->stats64_lock);
4491
4492 return stats;
4493}
4494
4495/**
4496 * igb_change_mtu - Change the Maximum Transfer Unit
4497 * @netdev: network interface device structure
4498 * @new_mtu: new value for maximum frame size
4499 *
4500 * Returns 0 on success, negative on failure
4501 **/
4502static int igb_change_mtu(struct net_device *netdev, int new_mtu)
4503{
4504 struct igb_adapter *adapter = netdev_priv(netdev);
4505 struct pci_dev *pdev = adapter->pdev;
4506 int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4507
4508 if ((new_mtu < 68) || (max_frame > MAX_JUMBO_FRAME_SIZE)) {
4509 dev_err(&pdev->dev, "Invalid MTU setting\n");
4510 return -EINVAL;
4511 }
4512
4513#define MAX_STD_JUMBO_FRAME_SIZE 9238
4514 if (max_frame > MAX_STD_JUMBO_FRAME_SIZE) {
4515 dev_err(&pdev->dev, "MTU > 9216 not supported.\n");
4516 return -EINVAL;
4517 }
4518
4519 while (test_and_set_bit(__IGB_RESETTING, &adapter->state))
4520 msleep(1);
4521
4522 /* igb_down has a dependency on max_frame_size */
4523 adapter->max_frame_size = max_frame;
4524
4525 if (netif_running(netdev))
4526 igb_down(adapter);
4527
4528 dev_info(&pdev->dev, "changing MTU from %d to %d\n",
4529 netdev->mtu, new_mtu);
4530 netdev->mtu = new_mtu;
4531
4532 if (netif_running(netdev))
4533 igb_up(adapter);
4534 else
4535 igb_reset(adapter);
4536
4537 clear_bit(__IGB_RESETTING, &adapter->state);
4538
4539 return 0;
4540}
4541
4542/**
4543 * igb_update_stats - Update the board statistics counters
4544 * @adapter: board private structure
4545 **/
4546
4547void igb_update_stats(struct igb_adapter *adapter,
4548 struct rtnl_link_stats64 *net_stats)
4549{
4550 struct e1000_hw *hw = &adapter->hw;
4551 struct pci_dev *pdev = adapter->pdev;
4552 u32 reg, mpc;
4553 u16 phy_tmp;
4554 int i;
4555 u64 bytes, packets;
4556 unsigned int start;
4557 u64 _bytes, _packets;
4558
4559#define PHY_IDLE_ERROR_COUNT_MASK 0x00FF
4560
4561 /*
4562 * Prevent stats update while adapter is being reset, or if the pci
4563 * connection is down.
4564 */
4565 if (adapter->link_speed == 0)
4566 return;
4567 if (pci_channel_offline(pdev))
4568 return;
4569
4570 bytes = 0;
4571 packets = 0;
4572 for (i = 0; i < adapter->num_rx_queues; i++) {
4573 u32 rqdpc_tmp = rd32(E1000_RQDPC(i)) & 0x0FFF;
4574 struct igb_ring *ring = adapter->rx_ring[i];
4575
4576 ring->rx_stats.drops += rqdpc_tmp;
4577 net_stats->rx_fifo_errors += rqdpc_tmp;
4578
4579 do {
4580 start = u64_stats_fetch_begin_bh(&ring->rx_syncp);
4581 _bytes = ring->rx_stats.bytes;
4582 _packets = ring->rx_stats.packets;
4583 } while (u64_stats_fetch_retry_bh(&ring->rx_syncp, start));
4584 bytes += _bytes;
4585 packets += _packets;
4586 }
4587
4588 net_stats->rx_bytes = bytes;
4589 net_stats->rx_packets = packets;
4590
4591 bytes = 0;
4592 packets = 0;
4593 for (i = 0; i < adapter->num_tx_queues; i++) {
4594 struct igb_ring *ring = adapter->tx_ring[i];
4595 do {
4596 start = u64_stats_fetch_begin_bh(&ring->tx_syncp);
4597 _bytes = ring->tx_stats.bytes;
4598 _packets = ring->tx_stats.packets;
4599 } while (u64_stats_fetch_retry_bh(&ring->tx_syncp, start));
4600 bytes += _bytes;
4601 packets += _packets;
4602 }
4603 net_stats->tx_bytes = bytes;
4604 net_stats->tx_packets = packets;
4605
4606 /* read stats registers */
4607 adapter->stats.crcerrs += rd32(E1000_CRCERRS);
4608 adapter->stats.gprc += rd32(E1000_GPRC);
4609 adapter->stats.gorc += rd32(E1000_GORCL);
4610 rd32(E1000_GORCH); /* clear GORCL */
4611 adapter->stats.bprc += rd32(E1000_BPRC);
4612 adapter->stats.mprc += rd32(E1000_MPRC);
4613 adapter->stats.roc += rd32(E1000_ROC);
4614
4615 adapter->stats.prc64 += rd32(E1000_PRC64);
4616 adapter->stats.prc127 += rd32(E1000_PRC127);
4617 adapter->stats.prc255 += rd32(E1000_PRC255);
4618 adapter->stats.prc511 += rd32(E1000_PRC511);
4619 adapter->stats.prc1023 += rd32(E1000_PRC1023);
4620 adapter->stats.prc1522 += rd32(E1000_PRC1522);
4621 adapter->stats.symerrs += rd32(E1000_SYMERRS);
4622 adapter->stats.sec += rd32(E1000_SEC);
4623
4624 mpc = rd32(E1000_MPC);
4625 adapter->stats.mpc += mpc;
4626 net_stats->rx_fifo_errors += mpc;
4627 adapter->stats.scc += rd32(E1000_SCC);
4628 adapter->stats.ecol += rd32(E1000_ECOL);
4629 adapter->stats.mcc += rd32(E1000_MCC);
4630 adapter->stats.latecol += rd32(E1000_LATECOL);
4631 adapter->stats.dc += rd32(E1000_DC);
4632 adapter->stats.rlec += rd32(E1000_RLEC);
4633 adapter->stats.xonrxc += rd32(E1000_XONRXC);
4634 adapter->stats.xontxc += rd32(E1000_XONTXC);
4635 adapter->stats.xoffrxc += rd32(E1000_XOFFRXC);
4636 adapter->stats.xofftxc += rd32(E1000_XOFFTXC);
4637 adapter->stats.fcruc += rd32(E1000_FCRUC);
4638 adapter->stats.gptc += rd32(E1000_GPTC);
4639 adapter->stats.gotc += rd32(E1000_GOTCL);
4640 rd32(E1000_GOTCH); /* clear GOTCL */
4641 adapter->stats.rnbc += rd32(E1000_RNBC);
4642 adapter->stats.ruc += rd32(E1000_RUC);
4643 adapter->stats.rfc += rd32(E1000_RFC);
4644 adapter->stats.rjc += rd32(E1000_RJC);
4645 adapter->stats.tor += rd32(E1000_TORH);
4646 adapter->stats.tot += rd32(E1000_TOTH);
4647 adapter->stats.tpr += rd32(E1000_TPR);
4648
4649 adapter->stats.ptc64 += rd32(E1000_PTC64);
4650 adapter->stats.ptc127 += rd32(E1000_PTC127);
4651 adapter->stats.ptc255 += rd32(E1000_PTC255);
4652 adapter->stats.ptc511 += rd32(E1000_PTC511);
4653 adapter->stats.ptc1023 += rd32(E1000_PTC1023);
4654 adapter->stats.ptc1522 += rd32(E1000_PTC1522);
4655
4656 adapter->stats.mptc += rd32(E1000_MPTC);
4657 adapter->stats.bptc += rd32(E1000_BPTC);
4658
4659 adapter->stats.tpt += rd32(E1000_TPT);
4660 adapter->stats.colc += rd32(E1000_COLC);
4661
4662 adapter->stats.algnerrc += rd32(E1000_ALGNERRC);
4663 /* read internal phy specific stats */
4664 reg = rd32(E1000_CTRL_EXT);
4665 if (!(reg & E1000_CTRL_EXT_LINK_MODE_MASK)) {
4666 adapter->stats.rxerrc += rd32(E1000_RXERRC);
4667 adapter->stats.tncrs += rd32(E1000_TNCRS);
4668 }
4669
4670 adapter->stats.tsctc += rd32(E1000_TSCTC);
4671 adapter->stats.tsctfc += rd32(E1000_TSCTFC);
4672
4673 adapter->stats.iac += rd32(E1000_IAC);
4674 adapter->stats.icrxoc += rd32(E1000_ICRXOC);
4675 adapter->stats.icrxptc += rd32(E1000_ICRXPTC);
4676 adapter->stats.icrxatc += rd32(E1000_ICRXATC);
4677 adapter->stats.ictxptc += rd32(E1000_ICTXPTC);
4678 adapter->stats.ictxatc += rd32(E1000_ICTXATC);
4679 adapter->stats.ictxqec += rd32(E1000_ICTXQEC);
4680 adapter->stats.ictxqmtc += rd32(E1000_ICTXQMTC);
4681 adapter->stats.icrxdmtc += rd32(E1000_ICRXDMTC);
4682
4683 /* Fill out the OS statistics structure */
4684 net_stats->multicast = adapter->stats.mprc;
4685 net_stats->collisions = adapter->stats.colc;
4686
4687 /* Rx Errors */
4688
4689 /* RLEC on some newer hardware can be incorrect so build
4690 * our own version based on RUC and ROC */
4691 net_stats->rx_errors = adapter->stats.rxerrc +
4692 adapter->stats.crcerrs + adapter->stats.algnerrc +
4693 adapter->stats.ruc + adapter->stats.roc +
4694 adapter->stats.cexterr;
4695 net_stats->rx_length_errors = adapter->stats.ruc +
4696 adapter->stats.roc;
4697 net_stats->rx_crc_errors = adapter->stats.crcerrs;
4698 net_stats->rx_frame_errors = adapter->stats.algnerrc;
4699 net_stats->rx_missed_errors = adapter->stats.mpc;
4700
4701 /* Tx Errors */
4702 net_stats->tx_errors = adapter->stats.ecol +
4703 adapter->stats.latecol;
4704 net_stats->tx_aborted_errors = adapter->stats.ecol;
4705 net_stats->tx_window_errors = adapter->stats.latecol;
4706 net_stats->tx_carrier_errors = adapter->stats.tncrs;
4707
4708 /* Tx Dropped needs to be maintained elsewhere */
4709
4710 /* Phy Stats */
4711 if (hw->phy.media_type == e1000_media_type_copper) {
4712 if ((adapter->link_speed == SPEED_1000) &&
4713 (!igb_read_phy_reg(hw, PHY_1000T_STATUS, &phy_tmp))) {
4714 phy_tmp &= PHY_IDLE_ERROR_COUNT_MASK;
4715 adapter->phy_stats.idle_errors += phy_tmp;
4716 }
4717 }
4718
4719 /* Management Stats */
4720 adapter->stats.mgptc += rd32(E1000_MGTPTC);
4721 adapter->stats.mgprc += rd32(E1000_MGTPRC);
4722 adapter->stats.mgpdc += rd32(E1000_MGTPDC);
4723
4724 /* OS2BMC Stats */
4725 reg = rd32(E1000_MANC);
4726 if (reg & E1000_MANC_EN_BMC2OS) {
4727 adapter->stats.o2bgptc += rd32(E1000_O2BGPTC);
4728 adapter->stats.o2bspc += rd32(E1000_O2BSPC);
4729 adapter->stats.b2ospc += rd32(E1000_B2OSPC);
4730 adapter->stats.b2ogprc += rd32(E1000_B2OGPRC);
4731 }
4732}
4733
4734static irqreturn_t igb_msix_other(int irq, void *data)
4735{
4736 struct igb_adapter *adapter = data;
4737 struct e1000_hw *hw = &adapter->hw;
4738 u32 icr = rd32(E1000_ICR);
4739 /* reading ICR causes bit 31 of EICR to be cleared */
4740
4741 if (icr & E1000_ICR_DRSTA)
4742 schedule_work(&adapter->reset_task);
4743
4744 if (icr & E1000_ICR_DOUTSYNC) {
4745 /* HW is reporting DMA is out of sync */
4746 adapter->stats.doosync++;
4747 /* The DMA Out of Sync is also indication of a spoof event
4748 * in IOV mode. Check the Wrong VM Behavior register to
4749 * see if it is really a spoof event. */
4750 igb_check_wvbr(adapter);
4751 }
4752
4753 /* Check for a mailbox event */
4754 if (icr & E1000_ICR_VMMB)
4755 igb_msg_task(adapter);
4756
4757 if (icr & E1000_ICR_LSC) {
4758 hw->mac.get_link_status = 1;
4759 /* guard against interrupt when we're going down */
4760 if (!test_bit(__IGB_DOWN, &adapter->state))
4761 mod_timer(&adapter->watchdog_timer, jiffies + 1);
4762 }
4763
4764 wr32(E1000_EIMS, adapter->eims_other);
4765
4766 return IRQ_HANDLED;
4767}
4768
4769static void igb_write_itr(struct igb_q_vector *q_vector)
4770{
4771 struct igb_adapter *adapter = q_vector->adapter;
4772 u32 itr_val = q_vector->itr_val & 0x7FFC;
4773
4774 if (!q_vector->set_itr)
4775 return;
4776
4777 if (!itr_val)
4778 itr_val = 0x4;
4779
4780 if (adapter->hw.mac.type == e1000_82575)
4781 itr_val |= itr_val << 16;
4782 else
4783 itr_val |= E1000_EITR_CNT_IGNR;
4784
4785 writel(itr_val, q_vector->itr_register);
4786 q_vector->set_itr = 0;
4787}
4788
4789static irqreturn_t igb_msix_ring(int irq, void *data)
4790{
4791 struct igb_q_vector *q_vector = data;
4792
4793 /* Write the ITR value calculated from the previous interrupt. */
4794 igb_write_itr(q_vector);
4795
4796 napi_schedule(&q_vector->napi);
4797
4798 return IRQ_HANDLED;
4799}
4800
4801#ifdef CONFIG_IGB_DCA
4802static void igb_update_dca(struct igb_q_vector *q_vector)
4803{
4804 struct igb_adapter *adapter = q_vector->adapter;
4805 struct e1000_hw *hw = &adapter->hw;
4806 int cpu = get_cpu();
4807
4808 if (q_vector->cpu == cpu)
4809 goto out_no_update;
4810
4811 if (q_vector->tx.ring) {
4812 int q = q_vector->tx.ring->reg_idx;
4813 u32 dca_txctrl = rd32(E1000_DCA_TXCTRL(q));
4814 if (hw->mac.type == e1000_82575) {
4815 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK;
4816 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4817 } else {
4818 dca_txctrl &= ~E1000_DCA_TXCTRL_CPUID_MASK_82576;
4819 dca_txctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4820 E1000_DCA_TXCTRL_CPUID_SHIFT;
4821 }
4822 dca_txctrl |= E1000_DCA_TXCTRL_DESC_DCA_EN;
4823 wr32(E1000_DCA_TXCTRL(q), dca_txctrl);
4824 }
4825 if (q_vector->rx.ring) {
4826 int q = q_vector->rx.ring->reg_idx;
4827 u32 dca_rxctrl = rd32(E1000_DCA_RXCTRL(q));
4828 if (hw->mac.type == e1000_82575) {
4829 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK;
4830 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu);
4831 } else {
4832 dca_rxctrl &= ~E1000_DCA_RXCTRL_CPUID_MASK_82576;
4833 dca_rxctrl |= dca3_get_tag(&adapter->pdev->dev, cpu) <<
4834 E1000_DCA_RXCTRL_CPUID_SHIFT;
4835 }
4836 dca_rxctrl |= E1000_DCA_RXCTRL_DESC_DCA_EN;
4837 dca_rxctrl |= E1000_DCA_RXCTRL_HEAD_DCA_EN;
4838 dca_rxctrl |= E1000_DCA_RXCTRL_DATA_DCA_EN;
4839 wr32(E1000_DCA_RXCTRL(q), dca_rxctrl);
4840 }
4841 q_vector->cpu = cpu;
4842out_no_update:
4843 put_cpu();
4844}
4845
4846static void igb_setup_dca(struct igb_adapter *adapter)
4847{
4848 struct e1000_hw *hw = &adapter->hw;
4849 int i;
4850
4851 if (!(adapter->flags & IGB_FLAG_DCA_ENABLED))
4852 return;
4853
4854 /* Always use CB2 mode, difference is masked in the CB driver. */
4855 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_CB2);
4856
4857 for (i = 0; i < adapter->num_q_vectors; i++) {
4858 adapter->q_vector[i]->cpu = -1;
4859 igb_update_dca(adapter->q_vector[i]);
4860 }
4861}
4862
4863static int __igb_notify_dca(struct device *dev, void *data)
4864{
4865 struct net_device *netdev = dev_get_drvdata(dev);
4866 struct igb_adapter *adapter = netdev_priv(netdev);
4867 struct pci_dev *pdev = adapter->pdev;
4868 struct e1000_hw *hw = &adapter->hw;
4869 unsigned long event = *(unsigned long *)data;
4870
4871 switch (event) {
4872 case DCA_PROVIDER_ADD:
4873 /* if already enabled, don't do it again */
4874 if (adapter->flags & IGB_FLAG_DCA_ENABLED)
4875 break;
4876 if (dca_add_requester(dev) == 0) {
4877 adapter->flags |= IGB_FLAG_DCA_ENABLED;
4878 dev_info(&pdev->dev, "DCA enabled\n");
4879 igb_setup_dca(adapter);
4880 break;
4881 }
4882 /* Fall Through since DCA is disabled. */
4883 case DCA_PROVIDER_REMOVE:
4884 if (adapter->flags & IGB_FLAG_DCA_ENABLED) {
4885 /* without this a class_device is left
4886 * hanging around in the sysfs model */
4887 dca_remove_requester(dev);
4888 dev_info(&pdev->dev, "DCA disabled\n");
4889 adapter->flags &= ~IGB_FLAG_DCA_ENABLED;
4890 wr32(E1000_DCA_CTRL, E1000_DCA_CTRL_DCA_MODE_DISABLE);
4891 }
4892 break;
4893 }
4894
4895 return 0;
4896}
4897
4898static int igb_notify_dca(struct notifier_block *nb, unsigned long event,
4899 void *p)
4900{
4901 int ret_val;
4902
4903 ret_val = driver_for_each_device(&igb_driver.driver, NULL, &event,
4904 __igb_notify_dca);
4905
4906 return ret_val ? NOTIFY_BAD : NOTIFY_DONE;
4907}
4908#endif /* CONFIG_IGB_DCA */
4909
4910#ifdef CONFIG_PCI_IOV
4911static int igb_vf_configure(struct igb_adapter *adapter, int vf)
4912{
4913 unsigned char mac_addr[ETH_ALEN];
4914 struct pci_dev *pdev = adapter->pdev;
4915 struct e1000_hw *hw = &adapter->hw;
4916 struct pci_dev *pvfdev;
4917 unsigned int device_id;
4918 u16 thisvf_devfn;
4919
4920 random_ether_addr(mac_addr);
4921 igb_set_vf_mac(adapter, vf, mac_addr);
4922
4923 switch (adapter->hw.mac.type) {
4924 case e1000_82576:
4925 device_id = IGB_82576_VF_DEV_ID;
4926 /* VF Stride for 82576 is 2 */
4927 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 1)) |
4928 (pdev->devfn & 1);
4929 break;
4930 case e1000_i350:
4931 device_id = IGB_I350_VF_DEV_ID;
4932 /* VF Stride for I350 is 4 */
4933 thisvf_devfn = (pdev->devfn + 0x80 + (vf << 2)) |
4934 (pdev->devfn & 3);
4935 break;
4936 default:
4937 device_id = 0;
4938 thisvf_devfn = 0;
4939 break;
4940 }
4941
4942 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
4943 while (pvfdev) {
4944 if (pvfdev->devfn == thisvf_devfn)
4945 break;
4946 pvfdev = pci_get_device(hw->vendor_id,
4947 device_id, pvfdev);
4948 }
4949
4950 if (pvfdev)
4951 adapter->vf_data[vf].vfdev = pvfdev;
4952 else
4953 dev_err(&pdev->dev,
4954 "Couldn't find pci dev ptr for VF %4.4x\n",
4955 thisvf_devfn);
4956 return pvfdev != NULL;
4957}
4958
4959static int igb_find_enabled_vfs(struct igb_adapter *adapter)
4960{
4961 struct e1000_hw *hw = &adapter->hw;
4962 struct pci_dev *pdev = adapter->pdev;
4963 struct pci_dev *pvfdev;
4964 u16 vf_devfn = 0;
4965 u16 vf_stride;
4966 unsigned int device_id;
4967 int vfs_found = 0;
4968
4969 switch (adapter->hw.mac.type) {
4970 case e1000_82576:
4971 device_id = IGB_82576_VF_DEV_ID;
4972 /* VF Stride for 82576 is 2 */
4973 vf_stride = 2;
4974 break;
4975 case e1000_i350:
4976 device_id = IGB_I350_VF_DEV_ID;
4977 /* VF Stride for I350 is 4 */
4978 vf_stride = 4;
4979 break;
4980 default:
4981 device_id = 0;
4982 vf_stride = 0;
4983 break;
4984 }
4985
4986 vf_devfn = pdev->devfn + 0x80;
4987 pvfdev = pci_get_device(hw->vendor_id, device_id, NULL);
4988 while (pvfdev) {
4989 if (pvfdev->devfn == vf_devfn &&
4990 (pvfdev->bus->number >= pdev->bus->number))
4991 vfs_found++;
4992 vf_devfn += vf_stride;
4993 pvfdev = pci_get_device(hw->vendor_id,
4994 device_id, pvfdev);
4995 }
4996
4997 return vfs_found;
4998}
4999
5000static int igb_check_vf_assignment(struct igb_adapter *adapter)
5001{
5002 int i;
5003 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5004 if (adapter->vf_data[i].vfdev) {
5005 if (adapter->vf_data[i].vfdev->dev_flags &
5006 PCI_DEV_FLAGS_ASSIGNED)
5007 return true;
5008 }
5009 }
5010 return false;
5011}
5012
5013#endif
5014static void igb_ping_all_vfs(struct igb_adapter *adapter)
5015{
5016 struct e1000_hw *hw = &adapter->hw;
5017 u32 ping;
5018 int i;
5019
5020 for (i = 0 ; i < adapter->vfs_allocated_count; i++) {
5021 ping = E1000_PF_CONTROL_MSG;
5022 if (adapter->vf_data[i].flags & IGB_VF_FLAG_CTS)
5023 ping |= E1000_VT_MSGTYPE_CTS;
5024 igb_write_mbx(hw, &ping, 1, i);
5025 }
5026}
5027
5028static int igb_set_vf_promisc(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5029{
5030 struct e1000_hw *hw = &adapter->hw;
5031 u32 vmolr = rd32(E1000_VMOLR(vf));
5032 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5033
5034 vf_data->flags &= ~(IGB_VF_FLAG_UNI_PROMISC |
5035 IGB_VF_FLAG_MULTI_PROMISC);
5036 vmolr &= ~(E1000_VMOLR_ROPE | E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5037
5038 if (*msgbuf & E1000_VF_SET_PROMISC_MULTICAST) {
5039 vmolr |= E1000_VMOLR_MPME;
5040 vf_data->flags |= IGB_VF_FLAG_MULTI_PROMISC;
5041 *msgbuf &= ~E1000_VF_SET_PROMISC_MULTICAST;
5042 } else {
5043 /*
5044 * if we have hashes and we are clearing a multicast promisc
5045 * flag we need to write the hashes to the MTA as this step
5046 * was previously skipped
5047 */
5048 if (vf_data->num_vf_mc_hashes > 30) {
5049 vmolr |= E1000_VMOLR_MPME;
5050 } else if (vf_data->num_vf_mc_hashes) {
5051 int j;
5052 vmolr |= E1000_VMOLR_ROMPE;
5053 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5054 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5055 }
5056 }
5057
5058 wr32(E1000_VMOLR(vf), vmolr);
5059
5060 /* there are flags left unprocessed, likely not supported */
5061 if (*msgbuf & E1000_VT_MSGINFO_MASK)
5062 return -EINVAL;
5063
5064 return 0;
5065
5066}
5067
5068static int igb_set_vf_multicasts(struct igb_adapter *adapter,
5069 u32 *msgbuf, u32 vf)
5070{
5071 int n = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5072 u16 *hash_list = (u16 *)&msgbuf[1];
5073 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5074 int i;
5075
5076 /* salt away the number of multicast addresses assigned
5077 * to this VF for later use to restore when the PF multi cast
5078 * list changes
5079 */
5080 vf_data->num_vf_mc_hashes = n;
5081
5082 /* only up to 30 hash values supported */
5083 if (n > 30)
5084 n = 30;
5085
5086 /* store the hashes for later use */
5087 for (i = 0; i < n; i++)
5088 vf_data->vf_mc_hashes[i] = hash_list[i];
5089
5090 /* Flush and reset the mta with the new values */
5091 igb_set_rx_mode(adapter->netdev);
5092
5093 return 0;
5094}
5095
5096static void igb_restore_vf_multicasts(struct igb_adapter *adapter)
5097{
5098 struct e1000_hw *hw = &adapter->hw;
5099 struct vf_data_storage *vf_data;
5100 int i, j;
5101
5102 for (i = 0; i < adapter->vfs_allocated_count; i++) {
5103 u32 vmolr = rd32(E1000_VMOLR(i));
5104 vmolr &= ~(E1000_VMOLR_ROMPE | E1000_VMOLR_MPME);
5105
5106 vf_data = &adapter->vf_data[i];
5107
5108 if ((vf_data->num_vf_mc_hashes > 30) ||
5109 (vf_data->flags & IGB_VF_FLAG_MULTI_PROMISC)) {
5110 vmolr |= E1000_VMOLR_MPME;
5111 } else if (vf_data->num_vf_mc_hashes) {
5112 vmolr |= E1000_VMOLR_ROMPE;
5113 for (j = 0; j < vf_data->num_vf_mc_hashes; j++)
5114 igb_mta_set(hw, vf_data->vf_mc_hashes[j]);
5115 }
5116 wr32(E1000_VMOLR(i), vmolr);
5117 }
5118}
5119
5120static void igb_clear_vf_vfta(struct igb_adapter *adapter, u32 vf)
5121{
5122 struct e1000_hw *hw = &adapter->hw;
5123 u32 pool_mask, reg, vid;
5124 int i;
5125
5126 pool_mask = 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5127
5128 /* Find the vlan filter for this id */
5129 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5130 reg = rd32(E1000_VLVF(i));
5131
5132 /* remove the vf from the pool */
5133 reg &= ~pool_mask;
5134
5135 /* if pool is empty then remove entry from vfta */
5136 if (!(reg & E1000_VLVF_POOLSEL_MASK) &&
5137 (reg & E1000_VLVF_VLANID_ENABLE)) {
5138 reg = 0;
5139 vid = reg & E1000_VLVF_VLANID_MASK;
5140 igb_vfta_set(hw, vid, false);
5141 }
5142
5143 wr32(E1000_VLVF(i), reg);
5144 }
5145
5146 adapter->vf_data[vf].vlans_enabled = 0;
5147}
5148
5149static s32 igb_vlvf_set(struct igb_adapter *adapter, u32 vid, bool add, u32 vf)
5150{
5151 struct e1000_hw *hw = &adapter->hw;
5152 u32 reg, i;
5153
5154 /* The vlvf table only exists on 82576 hardware and newer */
5155 if (hw->mac.type < e1000_82576)
5156 return -1;
5157
5158 /* we only need to do this if VMDq is enabled */
5159 if (!adapter->vfs_allocated_count)
5160 return -1;
5161
5162 /* Find the vlan filter for this id */
5163 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5164 reg = rd32(E1000_VLVF(i));
5165 if ((reg & E1000_VLVF_VLANID_ENABLE) &&
5166 vid == (reg & E1000_VLVF_VLANID_MASK))
5167 break;
5168 }
5169
5170 if (add) {
5171 if (i == E1000_VLVF_ARRAY_SIZE) {
5172 /* Did not find a matching VLAN ID entry that was
5173 * enabled. Search for a free filter entry, i.e.
5174 * one without the enable bit set
5175 */
5176 for (i = 0; i < E1000_VLVF_ARRAY_SIZE; i++) {
5177 reg = rd32(E1000_VLVF(i));
5178 if (!(reg & E1000_VLVF_VLANID_ENABLE))
5179 break;
5180 }
5181 }
5182 if (i < E1000_VLVF_ARRAY_SIZE) {
5183 /* Found an enabled/available entry */
5184 reg |= 1 << (E1000_VLVF_POOLSEL_SHIFT + vf);
5185
5186 /* if !enabled we need to set this up in vfta */
5187 if (!(reg & E1000_VLVF_VLANID_ENABLE)) {
5188 /* add VID to filter table */
5189 igb_vfta_set(hw, vid, true);
5190 reg |= E1000_VLVF_VLANID_ENABLE;
5191 }
5192 reg &= ~E1000_VLVF_VLANID_MASK;
5193 reg |= vid;
5194 wr32(E1000_VLVF(i), reg);
5195
5196 /* do not modify RLPML for PF devices */
5197 if (vf >= adapter->vfs_allocated_count)
5198 return 0;
5199
5200 if (!adapter->vf_data[vf].vlans_enabled) {
5201 u32 size;
5202 reg = rd32(E1000_VMOLR(vf));
5203 size = reg & E1000_VMOLR_RLPML_MASK;
5204 size += 4;
5205 reg &= ~E1000_VMOLR_RLPML_MASK;
5206 reg |= size;
5207 wr32(E1000_VMOLR(vf), reg);
5208 }
5209
5210 adapter->vf_data[vf].vlans_enabled++;
5211 }
5212 } else {
5213 if (i < E1000_VLVF_ARRAY_SIZE) {
5214 /* remove vf from the pool */
5215 reg &= ~(1 << (E1000_VLVF_POOLSEL_SHIFT + vf));
5216 /* if pool is empty then remove entry from vfta */
5217 if (!(reg & E1000_VLVF_POOLSEL_MASK)) {
5218 reg = 0;
5219 igb_vfta_set(hw, vid, false);
5220 }
5221 wr32(E1000_VLVF(i), reg);
5222
5223 /* do not modify RLPML for PF devices */
5224 if (vf >= adapter->vfs_allocated_count)
5225 return 0;
5226
5227 adapter->vf_data[vf].vlans_enabled--;
5228 if (!adapter->vf_data[vf].vlans_enabled) {
5229 u32 size;
5230 reg = rd32(E1000_VMOLR(vf));
5231 size = reg & E1000_VMOLR_RLPML_MASK;
5232 size -= 4;
5233 reg &= ~E1000_VMOLR_RLPML_MASK;
5234 reg |= size;
5235 wr32(E1000_VMOLR(vf), reg);
5236 }
5237 }
5238 }
5239 return 0;
5240}
5241
5242static void igb_set_vmvir(struct igb_adapter *adapter, u32 vid, u32 vf)
5243{
5244 struct e1000_hw *hw = &adapter->hw;
5245
5246 if (vid)
5247 wr32(E1000_VMVIR(vf), (vid | E1000_VMVIR_VLANA_DEFAULT));
5248 else
5249 wr32(E1000_VMVIR(vf), 0);
5250}
5251
5252static int igb_ndo_set_vf_vlan(struct net_device *netdev,
5253 int vf, u16 vlan, u8 qos)
5254{
5255 int err = 0;
5256 struct igb_adapter *adapter = netdev_priv(netdev);
5257
5258 if ((vf >= adapter->vfs_allocated_count) || (vlan > 4095) || (qos > 7))
5259 return -EINVAL;
5260 if (vlan || qos) {
5261 err = igb_vlvf_set(adapter, vlan, !!vlan, vf);
5262 if (err)
5263 goto out;
5264 igb_set_vmvir(adapter, vlan | (qos << VLAN_PRIO_SHIFT), vf);
5265 igb_set_vmolr(adapter, vf, !vlan);
5266 adapter->vf_data[vf].pf_vlan = vlan;
5267 adapter->vf_data[vf].pf_qos = qos;
5268 dev_info(&adapter->pdev->dev,
5269 "Setting VLAN %d, QOS 0x%x on VF %d\n", vlan, qos, vf);
5270 if (test_bit(__IGB_DOWN, &adapter->state)) {
5271 dev_warn(&adapter->pdev->dev,
5272 "The VF VLAN has been set,"
5273 " but the PF device is not up.\n");
5274 dev_warn(&adapter->pdev->dev,
5275 "Bring the PF device up before"
5276 " attempting to use the VF device.\n");
5277 }
5278 } else {
5279 igb_vlvf_set(adapter, adapter->vf_data[vf].pf_vlan,
5280 false, vf);
5281 igb_set_vmvir(adapter, vlan, vf);
5282 igb_set_vmolr(adapter, vf, true);
5283 adapter->vf_data[vf].pf_vlan = 0;
5284 adapter->vf_data[vf].pf_qos = 0;
5285 }
5286out:
5287 return err;
5288}
5289
5290static int igb_set_vf_vlan(struct igb_adapter *adapter, u32 *msgbuf, u32 vf)
5291{
5292 int add = (msgbuf[0] & E1000_VT_MSGINFO_MASK) >> E1000_VT_MSGINFO_SHIFT;
5293 int vid = (msgbuf[1] & E1000_VLVF_VLANID_MASK);
5294
5295 return igb_vlvf_set(adapter, vid, add, vf);
5296}
5297
5298static inline void igb_vf_reset(struct igb_adapter *adapter, u32 vf)
5299{
5300 /* clear flags - except flag that indicates PF has set the MAC */
5301 adapter->vf_data[vf].flags &= IGB_VF_FLAG_PF_SET_MAC;
5302 adapter->vf_data[vf].last_nack = jiffies;
5303
5304 /* reset offloads to defaults */
5305 igb_set_vmolr(adapter, vf, true);
5306
5307 /* reset vlans for device */
5308 igb_clear_vf_vfta(adapter, vf);
5309 if (adapter->vf_data[vf].pf_vlan)
5310 igb_ndo_set_vf_vlan(adapter->netdev, vf,
5311 adapter->vf_data[vf].pf_vlan,
5312 adapter->vf_data[vf].pf_qos);
5313 else
5314 igb_clear_vf_vfta(adapter, vf);
5315
5316 /* reset multicast table array for vf */
5317 adapter->vf_data[vf].num_vf_mc_hashes = 0;
5318
5319 /* Flush and reset the mta with the new values */
5320 igb_set_rx_mode(adapter->netdev);
5321}
5322
5323static void igb_vf_reset_event(struct igb_adapter *adapter, u32 vf)
5324{
5325 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5326
5327 /* generate a new mac address as we were hotplug removed/added */
5328 if (!(adapter->vf_data[vf].flags & IGB_VF_FLAG_PF_SET_MAC))
5329 random_ether_addr(vf_mac);
5330
5331 /* process remaining reset events */
5332 igb_vf_reset(adapter, vf);
5333}
5334
5335static void igb_vf_reset_msg(struct igb_adapter *adapter, u32 vf)
5336{
5337 struct e1000_hw *hw = &adapter->hw;
5338 unsigned char *vf_mac = adapter->vf_data[vf].vf_mac_addresses;
5339 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
5340 u32 reg, msgbuf[3];
5341 u8 *addr = (u8 *)(&msgbuf[1]);
5342
5343 /* process all the same items cleared in a function level reset */
5344 igb_vf_reset(adapter, vf);
5345
5346 /* set vf mac address */
5347 igb_rar_set_qsel(adapter, vf_mac, rar_entry, vf);
5348
5349 /* enable transmit and receive for vf */
5350 reg = rd32(E1000_VFTE);
5351 wr32(E1000_VFTE, reg | (1 << vf));
5352 reg = rd32(E1000_VFRE);
5353 wr32(E1000_VFRE, reg | (1 << vf));
5354
5355 adapter->vf_data[vf].flags |= IGB_VF_FLAG_CTS;
5356
5357 /* reply to reset with ack and vf mac address */
5358 msgbuf[0] = E1000_VF_RESET | E1000_VT_MSGTYPE_ACK;
5359 memcpy(addr, vf_mac, 6);
5360 igb_write_mbx(hw, msgbuf, 3, vf);
5361}
5362
5363static int igb_set_vf_mac_addr(struct igb_adapter *adapter, u32 *msg, int vf)
5364{
5365 /*
5366 * The VF MAC Address is stored in a packed array of bytes
5367 * starting at the second 32 bit word of the msg array
5368 */
5369 unsigned char *addr = (char *)&msg[1];
5370 int err = -1;
5371
5372 if (is_valid_ether_addr(addr))
5373 err = igb_set_vf_mac(adapter, vf, addr);
5374
5375 return err;
5376}
5377
5378static void igb_rcv_ack_from_vf(struct igb_adapter *adapter, u32 vf)
5379{
5380 struct e1000_hw *hw = &adapter->hw;
5381 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5382 u32 msg = E1000_VT_MSGTYPE_NACK;
5383
5384 /* if device isn't clear to send it shouldn't be reading either */
5385 if (!(vf_data->flags & IGB_VF_FLAG_CTS) &&
5386 time_after(jiffies, vf_data->last_nack + (2 * HZ))) {
5387 igb_write_mbx(hw, &msg, 1, vf);
5388 vf_data->last_nack = jiffies;
5389 }
5390}
5391
5392static void igb_rcv_msg_from_vf(struct igb_adapter *adapter, u32 vf)
5393{
5394 struct pci_dev *pdev = adapter->pdev;
5395 u32 msgbuf[E1000_VFMAILBOX_SIZE];
5396 struct e1000_hw *hw = &adapter->hw;
5397 struct vf_data_storage *vf_data = &adapter->vf_data[vf];
5398 s32 retval;
5399
5400 retval = igb_read_mbx(hw, msgbuf, E1000_VFMAILBOX_SIZE, vf);
5401
5402 if (retval) {
5403 /* if receive failed revoke VF CTS stats and restart init */
5404 dev_err(&pdev->dev, "Error receiving message from VF\n");
5405 vf_data->flags &= ~IGB_VF_FLAG_CTS;
5406 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5407 return;
5408 goto out;
5409 }
5410
5411 /* this is a message we already processed, do nothing */
5412 if (msgbuf[0] & (E1000_VT_MSGTYPE_ACK | E1000_VT_MSGTYPE_NACK))
5413 return;
5414
5415 /*
5416 * until the vf completes a reset it should not be
5417 * allowed to start any configuration.
5418 */
5419
5420 if (msgbuf[0] == E1000_VF_RESET) {
5421 igb_vf_reset_msg(adapter, vf);
5422 return;
5423 }
5424
5425 if (!(vf_data->flags & IGB_VF_FLAG_CTS)) {
5426 if (!time_after(jiffies, vf_data->last_nack + (2 * HZ)))
5427 return;
5428 retval = -1;
5429 goto out;
5430 }
5431
5432 switch ((msgbuf[0] & 0xFFFF)) {
5433 case E1000_VF_SET_MAC_ADDR:
5434 retval = -EINVAL;
5435 if (!(vf_data->flags & IGB_VF_FLAG_PF_SET_MAC))
5436 retval = igb_set_vf_mac_addr(adapter, msgbuf, vf);
5437 else
5438 dev_warn(&pdev->dev,
5439 "VF %d attempted to override administratively "
5440 "set MAC address\nReload the VF driver to "
5441 "resume operations\n", vf);
5442 break;
5443 case E1000_VF_SET_PROMISC:
5444 retval = igb_set_vf_promisc(adapter, msgbuf, vf);
5445 break;
5446 case E1000_VF_SET_MULTICAST:
5447 retval = igb_set_vf_multicasts(adapter, msgbuf, vf);
5448 break;
5449 case E1000_VF_SET_LPE:
5450 retval = igb_set_vf_rlpml(adapter, msgbuf[1], vf);
5451 break;
5452 case E1000_VF_SET_VLAN:
5453 retval = -1;
5454 if (vf_data->pf_vlan)
5455 dev_warn(&pdev->dev,
5456 "VF %d attempted to override administratively "
5457 "set VLAN tag\nReload the VF driver to "
5458 "resume operations\n", vf);
5459 else
5460 retval = igb_set_vf_vlan(adapter, msgbuf, vf);
5461 break;
5462 default:
5463 dev_err(&pdev->dev, "Unhandled Msg %08x\n", msgbuf[0]);
5464 retval = -1;
5465 break;
5466 }
5467
5468 msgbuf[0] |= E1000_VT_MSGTYPE_CTS;
5469out:
5470 /* notify the VF of the results of what it sent us */
5471 if (retval)
5472 msgbuf[0] |= E1000_VT_MSGTYPE_NACK;
5473 else
5474 msgbuf[0] |= E1000_VT_MSGTYPE_ACK;
5475
5476 igb_write_mbx(hw, msgbuf, 1, vf);
5477}
5478
5479static void igb_msg_task(struct igb_adapter *adapter)
5480{
5481 struct e1000_hw *hw = &adapter->hw;
5482 u32 vf;
5483
5484 for (vf = 0; vf < adapter->vfs_allocated_count; vf++) {
5485 /* process any reset requests */
5486 if (!igb_check_for_rst(hw, vf))
5487 igb_vf_reset_event(adapter, vf);
5488
5489 /* process any messages pending */
5490 if (!igb_check_for_msg(hw, vf))
5491 igb_rcv_msg_from_vf(adapter, vf);
5492
5493 /* process any acks */
5494 if (!igb_check_for_ack(hw, vf))
5495 igb_rcv_ack_from_vf(adapter, vf);
5496 }
5497}
5498
5499/**
5500 * igb_set_uta - Set unicast filter table address
5501 * @adapter: board private structure
5502 *
5503 * The unicast table address is a register array of 32-bit registers.
5504 * The table is meant to be used in a way similar to how the MTA is used
5505 * however due to certain limitations in the hardware it is necessary to
5506 * set all the hash bits to 1 and use the VMOLR ROPE bit as a promiscuous
5507 * enable bit to allow vlan tag stripping when promiscuous mode is enabled
5508 **/
5509static void igb_set_uta(struct igb_adapter *adapter)
5510{
5511 struct e1000_hw *hw = &adapter->hw;
5512 int i;
5513
5514 /* The UTA table only exists on 82576 hardware and newer */
5515 if (hw->mac.type < e1000_82576)
5516 return;
5517
5518 /* we only need to do this if VMDq is enabled */
5519 if (!adapter->vfs_allocated_count)
5520 return;
5521
5522 for (i = 0; i < hw->mac.uta_reg_count; i++)
5523 array_wr32(E1000_UTA, i, ~0);
5524}
5525
5526/**
5527 * igb_intr_msi - Interrupt Handler
5528 * @irq: interrupt number
5529 * @data: pointer to a network interface device structure
5530 **/
5531static irqreturn_t igb_intr_msi(int irq, void *data)
5532{
5533 struct igb_adapter *adapter = data;
5534 struct igb_q_vector *q_vector = adapter->q_vector[0];
5535 struct e1000_hw *hw = &adapter->hw;
5536 /* read ICR disables interrupts using IAM */
5537 u32 icr = rd32(E1000_ICR);
5538
5539 igb_write_itr(q_vector);
5540
5541 if (icr & E1000_ICR_DRSTA)
5542 schedule_work(&adapter->reset_task);
5543
5544 if (icr & E1000_ICR_DOUTSYNC) {
5545 /* HW is reporting DMA is out of sync */
5546 adapter->stats.doosync++;
5547 }
5548
5549 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5550 hw->mac.get_link_status = 1;
5551 if (!test_bit(__IGB_DOWN, &adapter->state))
5552 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5553 }
5554
5555 napi_schedule(&q_vector->napi);
5556
5557 return IRQ_HANDLED;
5558}
5559
5560/**
5561 * igb_intr - Legacy Interrupt Handler
5562 * @irq: interrupt number
5563 * @data: pointer to a network interface device structure
5564 **/
5565static irqreturn_t igb_intr(int irq, void *data)
5566{
5567 struct igb_adapter *adapter = data;
5568 struct igb_q_vector *q_vector = adapter->q_vector[0];
5569 struct e1000_hw *hw = &adapter->hw;
5570 /* Interrupt Auto-Mask...upon reading ICR, interrupts are masked. No
5571 * need for the IMC write */
5572 u32 icr = rd32(E1000_ICR);
5573
5574 /* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5575 * not set, then the adapter didn't send an interrupt */
5576 if (!(icr & E1000_ICR_INT_ASSERTED))
5577 return IRQ_NONE;
5578
5579 igb_write_itr(q_vector);
5580
5581 if (icr & E1000_ICR_DRSTA)
5582 schedule_work(&adapter->reset_task);
5583
5584 if (icr & E1000_ICR_DOUTSYNC) {
5585 /* HW is reporting DMA is out of sync */
5586 adapter->stats.doosync++;
5587 }
5588
5589 if (icr & (E1000_ICR_RXSEQ | E1000_ICR_LSC)) {
5590 hw->mac.get_link_status = 1;
5591 /* guard against interrupt when we're going down */
5592 if (!test_bit(__IGB_DOWN, &adapter->state))
5593 mod_timer(&adapter->watchdog_timer, jiffies + 1);
5594 }
5595
5596 napi_schedule(&q_vector->napi);
5597
5598 return IRQ_HANDLED;
5599}
5600
5601static void igb_ring_irq_enable(struct igb_q_vector *q_vector)
5602{
5603 struct igb_adapter *adapter = q_vector->adapter;
5604 struct e1000_hw *hw = &adapter->hw;
5605
5606 if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
5607 (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
5608 if ((adapter->num_q_vectors == 1) && !adapter->vf_data)
5609 igb_set_itr(q_vector);
5610 else
5611 igb_update_ring_itr(q_vector);
5612 }
5613
5614 if (!test_bit(__IGB_DOWN, &adapter->state)) {
5615 if (adapter->msix_entries)
5616 wr32(E1000_EIMS, q_vector->eims_value);
5617 else
5618 igb_irq_enable(adapter);
5619 }
5620}
5621
5622/**
5623 * igb_poll - NAPI Rx polling callback
5624 * @napi: napi polling structure
5625 * @budget: count of how many packets we should handle
5626 **/
5627static int igb_poll(struct napi_struct *napi, int budget)
5628{
5629 struct igb_q_vector *q_vector = container_of(napi,
5630 struct igb_q_vector,
5631 napi);
5632 bool clean_complete = true;
5633
5634#ifdef CONFIG_IGB_DCA
5635 if (q_vector->adapter->flags & IGB_FLAG_DCA_ENABLED)
5636 igb_update_dca(q_vector);
5637#endif
5638 if (q_vector->tx.ring)
5639 clean_complete = igb_clean_tx_irq(q_vector);
5640
5641 if (q_vector->rx.ring)
5642 clean_complete &= igb_clean_rx_irq(q_vector, budget);
5643
5644 /* If all work not completed, return budget and keep polling */
5645 if (!clean_complete)
5646 return budget;
5647
5648 /* If not enough Rx work done, exit the polling mode */
5649 napi_complete(napi);
5650 igb_ring_irq_enable(q_vector);
5651
5652 return 0;
5653}
5654
5655#ifdef CONFIG_IGB_PTP
5656/**
5657 * igb_tx_hwtstamp - utility function which checks for TX time stamp
5658 * @q_vector: pointer to q_vector containing needed info
5659 * @buffer: pointer to igb_tx_buffer structure
5660 *
5661 * If we were asked to do hardware stamping and such a time stamp is
5662 * available, then it must have been for this skb here because we only
5663 * allow only one such packet into the queue.
5664 */
5665static void igb_tx_hwtstamp(struct igb_q_vector *q_vector,
5666 struct igb_tx_buffer *buffer_info)
5667{
5668 struct igb_adapter *adapter = q_vector->adapter;
5669 struct e1000_hw *hw = &adapter->hw;
5670 struct skb_shared_hwtstamps shhwtstamps;
5671 u64 regval;
5672
5673 /* if skb does not support hw timestamp or TX stamp not valid exit */
5674 if (likely(!(buffer_info->tx_flags & IGB_TX_FLAGS_TSTAMP)) ||
5675 !(rd32(E1000_TSYNCTXCTL) & E1000_TSYNCTXCTL_VALID))
5676 return;
5677
5678 regval = rd32(E1000_TXSTMPL);
5679 regval |= (u64)rd32(E1000_TXSTMPH) << 32;
5680
5681 igb_systim_to_hwtstamp(adapter, &shhwtstamps, regval);
5682 skb_tstamp_tx(buffer_info->skb, &shhwtstamps);
5683}
5684
5685#endif
5686/**
5687 * igb_clean_tx_irq - Reclaim resources after transmit completes
5688 * @q_vector: pointer to q_vector containing needed info
5689 * returns true if ring is completely cleaned
5690 **/
5691static bool igb_clean_tx_irq(struct igb_q_vector *q_vector)
5692{
5693 struct igb_adapter *adapter = q_vector->adapter;
5694 struct igb_ring *tx_ring = q_vector->tx.ring;
5695 struct igb_tx_buffer *tx_buffer;
5696 union e1000_adv_tx_desc *tx_desc, *eop_desc;
5697 unsigned int total_bytes = 0, total_packets = 0;
5698 unsigned int budget = q_vector->tx.work_limit;
5699 unsigned int i = tx_ring->next_to_clean;
5700
5701 if (test_bit(__IGB_DOWN, &adapter->state))
5702 return true;
5703
5704 tx_buffer = &tx_ring->tx_buffer_info[i];
5705 tx_desc = IGB_TX_DESC(tx_ring, i);
5706 i -= tx_ring->count;
5707
5708 for (; budget; budget--) {
5709 eop_desc = tx_buffer->next_to_watch;
5710
5711 /* prevent any other reads prior to eop_desc */
5712 rmb();
5713
5714 /* if next_to_watch is not set then there is no work pending */
5715 if (!eop_desc)
5716 break;
5717
5718 /* if DD is not set pending work has not been completed */
5719 if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
5720 break;
5721
5722 /* clear next_to_watch to prevent false hangs */
5723 tx_buffer->next_to_watch = NULL;
5724
5725 /* update the statistics for this packet */
5726 total_bytes += tx_buffer->bytecount;
5727 total_packets += tx_buffer->gso_segs;
5728
5729#ifdef CONFIG_IGB_PTP
5730 /* retrieve hardware timestamp */
5731 igb_tx_hwtstamp(q_vector, tx_buffer);
5732
5733#endif
5734 /* free the skb */
5735 dev_kfree_skb_any(tx_buffer->skb);
5736 tx_buffer->skb = NULL;
5737
5738 /* unmap skb header data */
5739 dma_unmap_single(tx_ring->dev,
5740 tx_buffer->dma,
5741 tx_buffer->length,
5742 DMA_TO_DEVICE);
5743
5744 /* clear last DMA location and unmap remaining buffers */
5745 while (tx_desc != eop_desc) {
5746 tx_buffer->dma = 0;
5747
5748 tx_buffer++;
5749 tx_desc++;
5750 i++;
5751 if (unlikely(!i)) {
5752 i -= tx_ring->count;
5753 tx_buffer = tx_ring->tx_buffer_info;
5754 tx_desc = IGB_TX_DESC(tx_ring, 0);
5755 }
5756
5757 /* unmap any remaining paged data */
5758 if (tx_buffer->dma) {
5759 dma_unmap_page(tx_ring->dev,
5760 tx_buffer->dma,
5761 tx_buffer->length,
5762 DMA_TO_DEVICE);
5763 }
5764 }
5765
5766 /* clear last DMA location */
5767 tx_buffer->dma = 0;
5768
5769 /* move us one more past the eop_desc for start of next pkt */
5770 tx_buffer++;
5771 tx_desc++;
5772 i++;
5773 if (unlikely(!i)) {
5774 i -= tx_ring->count;
5775 tx_buffer = tx_ring->tx_buffer_info;
5776 tx_desc = IGB_TX_DESC(tx_ring, 0);
5777 }
5778 }
5779
5780 netdev_tx_completed_queue(txring_txq(tx_ring),
5781 total_packets, total_bytes);
5782 i += tx_ring->count;
5783 tx_ring->next_to_clean = i;
5784 u64_stats_update_begin(&tx_ring->tx_syncp);
5785 tx_ring->tx_stats.bytes += total_bytes;
5786 tx_ring->tx_stats.packets += total_packets;
5787 u64_stats_update_end(&tx_ring->tx_syncp);
5788 q_vector->tx.total_bytes += total_bytes;
5789 q_vector->tx.total_packets += total_packets;
5790
5791 if (test_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
5792 struct e1000_hw *hw = &adapter->hw;
5793
5794 eop_desc = tx_buffer->next_to_watch;
5795
5796 /* Detect a transmit hang in hardware, this serializes the
5797 * check with the clearing of time_stamp and movement of i */
5798 clear_bit(IGB_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5799 if (eop_desc &&
5800 time_after(jiffies, tx_buffer->time_stamp +
5801 (adapter->tx_timeout_factor * HZ)) &&
5802 !(rd32(E1000_STATUS) & E1000_STATUS_TXOFF)) {
5803
5804 /* detected Tx unit hang */
5805 dev_err(tx_ring->dev,
5806 "Detected Tx Unit Hang\n"
5807 " Tx Queue <%d>\n"
5808 " TDH <%x>\n"
5809 " TDT <%x>\n"
5810 " next_to_use <%x>\n"
5811 " next_to_clean <%x>\n"
5812 "buffer_info[next_to_clean]\n"
5813 " time_stamp <%lx>\n"
5814 " next_to_watch <%p>\n"
5815 " jiffies <%lx>\n"
5816 " desc.status <%x>\n",
5817 tx_ring->queue_index,
5818 rd32(E1000_TDH(tx_ring->reg_idx)),
5819 readl(tx_ring->tail),
5820 tx_ring->next_to_use,
5821 tx_ring->next_to_clean,
5822 tx_buffer->time_stamp,
5823 eop_desc,
5824 jiffies,
5825 eop_desc->wb.status);
5826 netif_stop_subqueue(tx_ring->netdev,
5827 tx_ring->queue_index);
5828
5829 /* we are about to reset, no point in enabling stuff */
5830 return true;
5831 }
5832 }
5833
5834 if (unlikely(total_packets &&
5835 netif_carrier_ok(tx_ring->netdev) &&
5836 igb_desc_unused(tx_ring) >= IGB_TX_QUEUE_WAKE)) {
5837 /* Make sure that anybody stopping the queue after this
5838 * sees the new next_to_clean.
5839 */
5840 smp_mb();
5841 if (__netif_subqueue_stopped(tx_ring->netdev,
5842 tx_ring->queue_index) &&
5843 !(test_bit(__IGB_DOWN, &adapter->state))) {
5844 netif_wake_subqueue(tx_ring->netdev,
5845 tx_ring->queue_index);
5846
5847 u64_stats_update_begin(&tx_ring->tx_syncp);
5848 tx_ring->tx_stats.restart_queue++;
5849 u64_stats_update_end(&tx_ring->tx_syncp);
5850 }
5851 }
5852
5853 return !!budget;
5854}
5855
5856static inline void igb_rx_checksum(struct igb_ring *ring,
5857 union e1000_adv_rx_desc *rx_desc,
5858 struct sk_buff *skb)
5859{
5860 skb_checksum_none_assert(skb);
5861
5862 /* Ignore Checksum bit is set */
5863 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_IXSM))
5864 return;
5865
5866 /* Rx checksum disabled via ethtool */
5867 if (!(ring->netdev->features & NETIF_F_RXCSUM))
5868 return;
5869
5870 /* TCP/UDP checksum error bit is set */
5871 if (igb_test_staterr(rx_desc,
5872 E1000_RXDEXT_STATERR_TCPE |
5873 E1000_RXDEXT_STATERR_IPE)) {
5874 /*
5875 * work around errata with sctp packets where the TCPE aka
5876 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
5877 * packets, (aka let the stack check the crc32c)
5878 */
5879 if (!((skb->len == 60) &&
5880 test_bit(IGB_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
5881 u64_stats_update_begin(&ring->rx_syncp);
5882 ring->rx_stats.csum_err++;
5883 u64_stats_update_end(&ring->rx_syncp);
5884 }
5885 /* let the stack verify checksum errors */
5886 return;
5887 }
5888 /* It must be a TCP or UDP packet with a valid checksum */
5889 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_TCPCS |
5890 E1000_RXD_STAT_UDPCS))
5891 skb->ip_summed = CHECKSUM_UNNECESSARY;
5892
5893 dev_dbg(ring->dev, "cksum success: bits %08X\n",
5894 le32_to_cpu(rx_desc->wb.upper.status_error));
5895}
5896
5897static inline void igb_rx_hash(struct igb_ring *ring,
5898 union e1000_adv_rx_desc *rx_desc,
5899 struct sk_buff *skb)
5900{
5901 if (ring->netdev->features & NETIF_F_RXHASH)
5902 skb->rxhash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
5903}
5904
5905#ifdef CONFIG_IGB_PTP
5906static void igb_rx_hwtstamp(struct igb_q_vector *q_vector,
5907 union e1000_adv_rx_desc *rx_desc,
5908 struct sk_buff *skb)
5909{
5910 struct igb_adapter *adapter = q_vector->adapter;
5911 struct e1000_hw *hw = &adapter->hw;
5912 u64 regval;
5913
5914 if (!igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP |
5915 E1000_RXDADV_STAT_TS))
5916 return;
5917
5918 /*
5919 * If this bit is set, then the RX registers contain the time stamp. No
5920 * other packet will be time stamped until we read these registers, so
5921 * read the registers to make them available again. Because only one
5922 * packet can be time stamped at a time, we know that the register
5923 * values must belong to this one here and therefore we don't need to
5924 * compare any of the additional attributes stored for it.
5925 *
5926 * If nothing went wrong, then it should have a shared tx_flags that we
5927 * can turn into a skb_shared_hwtstamps.
5928 */
5929 if (igb_test_staterr(rx_desc, E1000_RXDADV_STAT_TSIP)) {
5930 u32 *stamp = (u32 *)skb->data;
5931 regval = le32_to_cpu(*(stamp + 2));
5932 regval |= (u64)le32_to_cpu(*(stamp + 3)) << 32;
5933 skb_pull(skb, IGB_TS_HDR_LEN);
5934 } else {
5935 if(!(rd32(E1000_TSYNCRXCTL) & E1000_TSYNCRXCTL_VALID))
5936 return;
5937
5938 regval = rd32(E1000_RXSTMPL);
5939 regval |= (u64)rd32(E1000_RXSTMPH) << 32;
5940 }
5941
5942 igb_systim_to_hwtstamp(adapter, skb_hwtstamps(skb), regval);
5943}
5944
5945#endif
5946static void igb_rx_vlan(struct igb_ring *ring,
5947 union e1000_adv_rx_desc *rx_desc,
5948 struct sk_buff *skb)
5949{
5950 if (igb_test_staterr(rx_desc, E1000_RXD_STAT_VP)) {
5951 u16 vid;
5952 if (igb_test_staterr(rx_desc, E1000_RXDEXT_STATERR_LB) &&
5953 test_bit(IGB_RING_FLAG_RX_LB_VLAN_BSWAP, &ring->flags))
5954 vid = be16_to_cpu(rx_desc->wb.upper.vlan);
5955 else
5956 vid = le16_to_cpu(rx_desc->wb.upper.vlan);
5957
5958 __vlan_hwaccel_put_tag(skb, vid);
5959 }
5960}
5961
5962static inline u16 igb_get_hlen(union e1000_adv_rx_desc *rx_desc)
5963{
5964 /* HW will not DMA in data larger than the given buffer, even if it
5965 * parses the (NFS, of course) header to be larger. In that case, it
5966 * fills the header buffer and spills the rest into the page.
5967 */
5968 u16 hlen = (le16_to_cpu(rx_desc->wb.lower.lo_dword.hdr_info) &
5969 E1000_RXDADV_HDRBUFLEN_MASK) >> E1000_RXDADV_HDRBUFLEN_SHIFT;
5970 if (hlen > IGB_RX_HDR_LEN)
5971 hlen = IGB_RX_HDR_LEN;
5972 return hlen;
5973}
5974
5975static bool igb_clean_rx_irq(struct igb_q_vector *q_vector, int budget)
5976{
5977 struct igb_ring *rx_ring = q_vector->rx.ring;
5978 union e1000_adv_rx_desc *rx_desc;
5979 const int current_node = numa_node_id();
5980 unsigned int total_bytes = 0, total_packets = 0;
5981 u16 cleaned_count = igb_desc_unused(rx_ring);
5982 u16 i = rx_ring->next_to_clean;
5983
5984 rx_desc = IGB_RX_DESC(rx_ring, i);
5985
5986 while (igb_test_staterr(rx_desc, E1000_RXD_STAT_DD)) {
5987 struct igb_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
5988 struct sk_buff *skb = buffer_info->skb;
5989 union e1000_adv_rx_desc *next_rxd;
5990
5991 buffer_info->skb = NULL;
5992 prefetch(skb->data);
5993
5994 i++;
5995 if (i == rx_ring->count)
5996 i = 0;
5997
5998 next_rxd = IGB_RX_DESC(rx_ring, i);
5999 prefetch(next_rxd);
6000
6001 /*
6002 * This memory barrier is needed to keep us from reading
6003 * any other fields out of the rx_desc until we know the
6004 * RXD_STAT_DD bit is set
6005 */
6006 rmb();
6007
6008 if (!skb_is_nonlinear(skb)) {
6009 __skb_put(skb, igb_get_hlen(rx_desc));
6010 dma_unmap_single(rx_ring->dev, buffer_info->dma,
6011 IGB_RX_HDR_LEN,
6012 DMA_FROM_DEVICE);
6013 buffer_info->dma = 0;
6014 }
6015
6016 if (rx_desc->wb.upper.length) {
6017 u16 length = le16_to_cpu(rx_desc->wb.upper.length);
6018
6019 skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
6020 buffer_info->page,
6021 buffer_info->page_offset,
6022 length);
6023
6024 skb->len += length;
6025 skb->data_len += length;
6026 skb->truesize += PAGE_SIZE / 2;
6027
6028 if ((page_count(buffer_info->page) != 1) ||
6029 (page_to_nid(buffer_info->page) != current_node))
6030 buffer_info->page = NULL;
6031 else
6032 get_page(buffer_info->page);
6033
6034 dma_unmap_page(rx_ring->dev, buffer_info->page_dma,
6035 PAGE_SIZE / 2, DMA_FROM_DEVICE);
6036 buffer_info->page_dma = 0;
6037 }
6038
6039 if (!igb_test_staterr(rx_desc, E1000_RXD_STAT_EOP)) {
6040 struct igb_rx_buffer *next_buffer;
6041 next_buffer = &rx_ring->rx_buffer_info[i];
6042 buffer_info->skb = next_buffer->skb;
6043 buffer_info->dma = next_buffer->dma;
6044 next_buffer->skb = skb;
6045 next_buffer->dma = 0;
6046 goto next_desc;
6047 }
6048
6049 if (unlikely((igb_test_staterr(rx_desc,
6050 E1000_RXDEXT_ERR_FRAME_ERR_MASK))
6051 && !(rx_ring->netdev->features & NETIF_F_RXALL))) {
6052 dev_kfree_skb_any(skb);
6053 goto next_desc;
6054 }
6055
6056#ifdef CONFIG_IGB_PTP
6057 igb_rx_hwtstamp(q_vector, rx_desc, skb);
6058#endif
6059 igb_rx_hash(rx_ring, rx_desc, skb);
6060 igb_rx_checksum(rx_ring, rx_desc, skb);
6061 igb_rx_vlan(rx_ring, rx_desc, skb);
6062
6063 total_bytes += skb->len;
6064 total_packets++;
6065
6066 skb->protocol = eth_type_trans(skb, rx_ring->netdev);
6067
6068 napi_gro_receive(&q_vector->napi, skb);
6069
6070 budget--;
6071next_desc:
6072 if (!budget)
6073 break;
6074
6075 cleaned_count++;
6076 /* return some buffers to hardware, one at a time is too slow */
6077 if (cleaned_count >= IGB_RX_BUFFER_WRITE) {
6078 igb_alloc_rx_buffers(rx_ring, cleaned_count);
6079 cleaned_count = 0;
6080 }
6081
6082 /* use prefetched values */
6083 rx_desc = next_rxd;
6084 }
6085
6086 rx_ring->next_to_clean = i;
6087 u64_stats_update_begin(&rx_ring->rx_syncp);
6088 rx_ring->rx_stats.packets += total_packets;
6089 rx_ring->rx_stats.bytes += total_bytes;
6090 u64_stats_update_end(&rx_ring->rx_syncp);
6091 q_vector->rx.total_packets += total_packets;
6092 q_vector->rx.total_bytes += total_bytes;
6093
6094 if (cleaned_count)
6095 igb_alloc_rx_buffers(rx_ring, cleaned_count);
6096
6097 return !!budget;
6098}
6099
6100static bool igb_alloc_mapped_skb(struct igb_ring *rx_ring,
6101 struct igb_rx_buffer *bi)
6102{
6103 struct sk_buff *skb = bi->skb;
6104 dma_addr_t dma = bi->dma;
6105
6106 if (dma)
6107 return true;
6108
6109 if (likely(!skb)) {
6110 skb = netdev_alloc_skb_ip_align(rx_ring->netdev,
6111 IGB_RX_HDR_LEN);
6112 bi->skb = skb;
6113 if (!skb) {
6114 rx_ring->rx_stats.alloc_failed++;
6115 return false;
6116 }
6117
6118 /* initialize skb for ring */
6119 skb_record_rx_queue(skb, rx_ring->queue_index);
6120 }
6121
6122 dma = dma_map_single(rx_ring->dev, skb->data,
6123 IGB_RX_HDR_LEN, DMA_FROM_DEVICE);
6124
6125 if (dma_mapping_error(rx_ring->dev, dma)) {
6126 rx_ring->rx_stats.alloc_failed++;
6127 return false;
6128 }
6129
6130 bi->dma = dma;
6131 return true;
6132}
6133
6134static bool igb_alloc_mapped_page(struct igb_ring *rx_ring,
6135 struct igb_rx_buffer *bi)
6136{
6137 struct page *page = bi->page;
6138 dma_addr_t page_dma = bi->page_dma;
6139 unsigned int page_offset = bi->page_offset ^ (PAGE_SIZE / 2);
6140
6141 if (page_dma)
6142 return true;
6143
6144 if (!page) {
6145 page = alloc_page(GFP_ATOMIC | __GFP_COLD);
6146 bi->page = page;
6147 if (unlikely(!page)) {
6148 rx_ring->rx_stats.alloc_failed++;
6149 return false;
6150 }
6151 }
6152
6153 page_dma = dma_map_page(rx_ring->dev, page,
6154 page_offset, PAGE_SIZE / 2,
6155 DMA_FROM_DEVICE);
6156
6157 if (dma_mapping_error(rx_ring->dev, page_dma)) {
6158 rx_ring->rx_stats.alloc_failed++;
6159 return false;
6160 }
6161
6162 bi->page_dma = page_dma;
6163 bi->page_offset = page_offset;
6164 return true;
6165}
6166
6167/**
6168 * igb_alloc_rx_buffers - Replace used receive buffers; packet split
6169 * @adapter: address of board private structure
6170 **/
6171void igb_alloc_rx_buffers(struct igb_ring *rx_ring, u16 cleaned_count)
6172{
6173 union e1000_adv_rx_desc *rx_desc;
6174 struct igb_rx_buffer *bi;
6175 u16 i = rx_ring->next_to_use;
6176
6177 rx_desc = IGB_RX_DESC(rx_ring, i);
6178 bi = &rx_ring->rx_buffer_info[i];
6179 i -= rx_ring->count;
6180
6181 while (cleaned_count--) {
6182 if (!igb_alloc_mapped_skb(rx_ring, bi))
6183 break;
6184
6185 /* Refresh the desc even if buffer_addrs didn't change
6186 * because each write-back erases this info. */
6187 rx_desc->read.hdr_addr = cpu_to_le64(bi->dma);
6188
6189 if (!igb_alloc_mapped_page(rx_ring, bi))
6190 break;
6191
6192 rx_desc->read.pkt_addr = cpu_to_le64(bi->page_dma);
6193
6194 rx_desc++;
6195 bi++;
6196 i++;
6197 if (unlikely(!i)) {
6198 rx_desc = IGB_RX_DESC(rx_ring, 0);
6199 bi = rx_ring->rx_buffer_info;
6200 i -= rx_ring->count;
6201 }
6202
6203 /* clear the hdr_addr for the next_to_use descriptor */
6204 rx_desc->read.hdr_addr = 0;
6205 }
6206
6207 i += rx_ring->count;
6208
6209 if (rx_ring->next_to_use != i) {
6210 rx_ring->next_to_use = i;
6211
6212 /* Force memory writes to complete before letting h/w
6213 * know there are new descriptors to fetch. (Only
6214 * applicable for weak-ordered memory model archs,
6215 * such as IA-64). */
6216 wmb();
6217 writel(i, rx_ring->tail);
6218 }
6219}
6220
6221/**
6222 * igb_mii_ioctl -
6223 * @netdev:
6224 * @ifreq:
6225 * @cmd:
6226 **/
6227static int igb_mii_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6228{
6229 struct igb_adapter *adapter = netdev_priv(netdev);
6230 struct mii_ioctl_data *data = if_mii(ifr);
6231
6232 if (adapter->hw.phy.media_type != e1000_media_type_copper)
6233 return -EOPNOTSUPP;
6234
6235 switch (cmd) {
6236 case SIOCGMIIPHY:
6237 data->phy_id = adapter->hw.phy.addr;
6238 break;
6239 case SIOCGMIIREG:
6240 if (igb_read_phy_reg(&adapter->hw, data->reg_num & 0x1F,
6241 &data->val_out))
6242 return -EIO;
6243 break;
6244 case SIOCSMIIREG:
6245 default:
6246 return -EOPNOTSUPP;
6247 }
6248 return 0;
6249}
6250
6251/**
6252 * igb_hwtstamp_ioctl - control hardware time stamping
6253 * @netdev:
6254 * @ifreq:
6255 * @cmd:
6256 *
6257 * Outgoing time stamping can be enabled and disabled. Play nice and
6258 * disable it when requested, although it shouldn't case any overhead
6259 * when no packet needs it. At most one packet in the queue may be
6260 * marked for time stamping, otherwise it would be impossible to tell
6261 * for sure to which packet the hardware time stamp belongs.
6262 *
6263 * Incoming time stamping has to be configured via the hardware
6264 * filters. Not all combinations are supported, in particular event
6265 * type has to be specified. Matching the kind of event packet is
6266 * not supported, with the exception of "all V2 events regardless of
6267 * level 2 or 4".
6268 *
6269 **/
6270static int igb_hwtstamp_ioctl(struct net_device *netdev,
6271 struct ifreq *ifr, int cmd)
6272{
6273 struct igb_adapter *adapter = netdev_priv(netdev);
6274 struct e1000_hw *hw = &adapter->hw;
6275 struct hwtstamp_config config;
6276 u32 tsync_tx_ctl = E1000_TSYNCTXCTL_ENABLED;
6277 u32 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6278 u32 tsync_rx_cfg = 0;
6279 bool is_l4 = false;
6280 bool is_l2 = false;
6281 u32 regval;
6282
6283 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
6284 return -EFAULT;
6285
6286 /* reserved for future extensions */
6287 if (config.flags)
6288 return -EINVAL;
6289
6290 switch (config.tx_type) {
6291 case HWTSTAMP_TX_OFF:
6292 tsync_tx_ctl = 0;
6293 case HWTSTAMP_TX_ON:
6294 break;
6295 default:
6296 return -ERANGE;
6297 }
6298
6299 switch (config.rx_filter) {
6300 case HWTSTAMP_FILTER_NONE:
6301 tsync_rx_ctl = 0;
6302 break;
6303 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
6304 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
6305 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT:
6306 case HWTSTAMP_FILTER_ALL:
6307 /*
6308 * register TSYNCRXCFG must be set, therefore it is not
6309 * possible to time stamp both Sync and Delay_Req messages
6310 * => fall back to time stamping all packets
6311 */
6312 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6313 config.rx_filter = HWTSTAMP_FILTER_ALL;
6314 break;
6315 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
6316 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6317 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_SYNC_MESSAGE;
6318 is_l4 = true;
6319 break;
6320 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
6321 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L4_V1;
6322 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V1_DELAY_REQ_MESSAGE;
6323 is_l4 = true;
6324 break;
6325 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC:
6326 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
6327 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6328 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_SYNC_MESSAGE;
6329 is_l2 = true;
6330 is_l4 = true;
6331 config.rx_filter = HWTSTAMP_FILTER_SOME;
6332 break;
6333 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ:
6334 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
6335 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_L2_L4_V2;
6336 tsync_rx_cfg = E1000_TSYNCRXCFG_PTP_V2_DELAY_REQ_MESSAGE;
6337 is_l2 = true;
6338 is_l4 = true;
6339 config.rx_filter = HWTSTAMP_FILTER_SOME;
6340 break;
6341 case HWTSTAMP_FILTER_PTP_V2_EVENT:
6342 case HWTSTAMP_FILTER_PTP_V2_SYNC:
6343 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
6344 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_EVENT_V2;
6345 config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
6346 is_l2 = true;
6347 is_l4 = true;
6348 break;
6349 default:
6350 return -ERANGE;
6351 }
6352
6353 if (hw->mac.type == e1000_82575) {
6354 if (tsync_rx_ctl | tsync_tx_ctl)
6355 return -EINVAL;
6356 return 0;
6357 }
6358
6359 /*
6360 * Per-packet timestamping only works if all packets are
6361 * timestamped, so enable timestamping in all packets as
6362 * long as one rx filter was configured.
6363 */
6364 if ((hw->mac.type >= e1000_82580) && tsync_rx_ctl) {
6365 tsync_rx_ctl = E1000_TSYNCRXCTL_ENABLED;
6366 tsync_rx_ctl |= E1000_TSYNCRXCTL_TYPE_ALL;
6367 }
6368
6369 /* enable/disable TX */
6370 regval = rd32(E1000_TSYNCTXCTL);
6371 regval &= ~E1000_TSYNCTXCTL_ENABLED;
6372 regval |= tsync_tx_ctl;
6373 wr32(E1000_TSYNCTXCTL, regval);
6374
6375 /* enable/disable RX */
6376 regval = rd32(E1000_TSYNCRXCTL);
6377 regval &= ~(E1000_TSYNCRXCTL_ENABLED | E1000_TSYNCRXCTL_TYPE_MASK);
6378 regval |= tsync_rx_ctl;
6379 wr32(E1000_TSYNCRXCTL, regval);
6380
6381 /* define which PTP packets are time stamped */
6382 wr32(E1000_TSYNCRXCFG, tsync_rx_cfg);
6383
6384 /* define ethertype filter for timestamped packets */
6385 if (is_l2)
6386 wr32(E1000_ETQF(3),
6387 (E1000_ETQF_FILTER_ENABLE | /* enable filter */
6388 E1000_ETQF_1588 | /* enable timestamping */
6389 ETH_P_1588)); /* 1588 eth protocol type */
6390 else
6391 wr32(E1000_ETQF(3), 0);
6392
6393#define PTP_PORT 319
6394 /* L4 Queue Filter[3]: filter by destination port and protocol */
6395 if (is_l4) {
6396 u32 ftqf = (IPPROTO_UDP /* UDP */
6397 | E1000_FTQF_VF_BP /* VF not compared */
6398 | E1000_FTQF_1588_TIME_STAMP /* Enable Timestamping */
6399 | E1000_FTQF_MASK); /* mask all inputs */
6400 ftqf &= ~E1000_FTQF_MASK_PROTO_BP; /* enable protocol check */
6401
6402 wr32(E1000_IMIR(3), htons(PTP_PORT));
6403 wr32(E1000_IMIREXT(3),
6404 (E1000_IMIREXT_SIZE_BP | E1000_IMIREXT_CTRL_BP));
6405 if (hw->mac.type == e1000_82576) {
6406 /* enable source port check */
6407 wr32(E1000_SPQF(3), htons(PTP_PORT));
6408 ftqf &= ~E1000_FTQF_MASK_SOURCE_PORT_BP;
6409 }
6410 wr32(E1000_FTQF(3), ftqf);
6411 } else {
6412 wr32(E1000_FTQF(3), E1000_FTQF_MASK);
6413 }
6414 wrfl();
6415
6416 adapter->hwtstamp_config = config;
6417
6418 /* clear TX/RX time stamp registers, just to be sure */
6419 regval = rd32(E1000_TXSTMPH);
6420 regval = rd32(E1000_RXSTMPH);
6421
6422 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
6423 -EFAULT : 0;
6424}
6425
6426/**
6427 * igb_ioctl -
6428 * @netdev:
6429 * @ifreq:
6430 * @cmd:
6431 **/
6432static int igb_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6433{
6434 switch (cmd) {
6435 case SIOCGMIIPHY:
6436 case SIOCGMIIREG:
6437 case SIOCSMIIREG:
6438 return igb_mii_ioctl(netdev, ifr, cmd);
6439 case SIOCSHWTSTAMP:
6440 return igb_hwtstamp_ioctl(netdev, ifr, cmd);
6441 default:
6442 return -EOPNOTSUPP;
6443 }
6444}
6445
6446s32 igb_read_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6447{
6448 struct igb_adapter *adapter = hw->back;
6449 u16 cap_offset;
6450
6451 cap_offset = adapter->pdev->pcie_cap;
6452 if (!cap_offset)
6453 return -E1000_ERR_CONFIG;
6454
6455 pci_read_config_word(adapter->pdev, cap_offset + reg, value);
6456
6457 return 0;
6458}
6459
6460s32 igb_write_pcie_cap_reg(struct e1000_hw *hw, u32 reg, u16 *value)
6461{
6462 struct igb_adapter *adapter = hw->back;
6463 u16 cap_offset;
6464
6465 cap_offset = adapter->pdev->pcie_cap;
6466 if (!cap_offset)
6467 return -E1000_ERR_CONFIG;
6468
6469 pci_write_config_word(adapter->pdev, cap_offset + reg, *value);
6470
6471 return 0;
6472}
6473
6474static void igb_vlan_mode(struct net_device *netdev, netdev_features_t features)
6475{
6476 struct igb_adapter *adapter = netdev_priv(netdev);
6477 struct e1000_hw *hw = &adapter->hw;
6478 u32 ctrl, rctl;
6479 bool enable = !!(features & NETIF_F_HW_VLAN_RX);
6480
6481 if (enable) {
6482 /* enable VLAN tag insert/strip */
6483 ctrl = rd32(E1000_CTRL);
6484 ctrl |= E1000_CTRL_VME;
6485 wr32(E1000_CTRL, ctrl);
6486
6487 /* Disable CFI check */
6488 rctl = rd32(E1000_RCTL);
6489 rctl &= ~E1000_RCTL_CFIEN;
6490 wr32(E1000_RCTL, rctl);
6491 } else {
6492 /* disable VLAN tag insert/strip */
6493 ctrl = rd32(E1000_CTRL);
6494 ctrl &= ~E1000_CTRL_VME;
6495 wr32(E1000_CTRL, ctrl);
6496 }
6497
6498 igb_rlpml_set(adapter);
6499}
6500
6501static int igb_vlan_rx_add_vid(struct net_device *netdev, u16 vid)
6502{
6503 struct igb_adapter *adapter = netdev_priv(netdev);
6504 struct e1000_hw *hw = &adapter->hw;
6505 int pf_id = adapter->vfs_allocated_count;
6506
6507 /* attempt to add filter to vlvf array */
6508 igb_vlvf_set(adapter, vid, true, pf_id);
6509
6510 /* add the filter since PF can receive vlans w/o entry in vlvf */
6511 igb_vfta_set(hw, vid, true);
6512
6513 set_bit(vid, adapter->active_vlans);
6514
6515 return 0;
6516}
6517
6518static int igb_vlan_rx_kill_vid(struct net_device *netdev, u16 vid)
6519{
6520 struct igb_adapter *adapter = netdev_priv(netdev);
6521 struct e1000_hw *hw = &adapter->hw;
6522 int pf_id = adapter->vfs_allocated_count;
6523 s32 err;
6524
6525 /* remove vlan from VLVF table array */
6526 err = igb_vlvf_set(adapter, vid, false, pf_id);
6527
6528 /* if vid was not present in VLVF just remove it from table */
6529 if (err)
6530 igb_vfta_set(hw, vid, false);
6531
6532 clear_bit(vid, adapter->active_vlans);
6533
6534 return 0;
6535}
6536
6537static void igb_restore_vlan(struct igb_adapter *adapter)
6538{
6539 u16 vid;
6540
6541 igb_vlan_mode(adapter->netdev, adapter->netdev->features);
6542
6543 for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
6544 igb_vlan_rx_add_vid(adapter->netdev, vid);
6545}
6546
6547int igb_set_spd_dplx(struct igb_adapter *adapter, u32 spd, u8 dplx)
6548{
6549 struct pci_dev *pdev = adapter->pdev;
6550 struct e1000_mac_info *mac = &adapter->hw.mac;
6551
6552 mac->autoneg = 0;
6553
6554 /* Make sure dplx is at most 1 bit and lsb of speed is not set
6555 * for the switch() below to work */
6556 if ((spd & 1) || (dplx & ~1))
6557 goto err_inval;
6558
6559 /* Fiber NIC's only allow 1000 Gbps Full duplex */
6560 if ((adapter->hw.phy.media_type == e1000_media_type_internal_serdes) &&
6561 spd != SPEED_1000 &&
6562 dplx != DUPLEX_FULL)
6563 goto err_inval;
6564
6565 switch (spd + dplx) {
6566 case SPEED_10 + DUPLEX_HALF:
6567 mac->forced_speed_duplex = ADVERTISE_10_HALF;
6568 break;
6569 case SPEED_10 + DUPLEX_FULL:
6570 mac->forced_speed_duplex = ADVERTISE_10_FULL;
6571 break;
6572 case SPEED_100 + DUPLEX_HALF:
6573 mac->forced_speed_duplex = ADVERTISE_100_HALF;
6574 break;
6575 case SPEED_100 + DUPLEX_FULL:
6576 mac->forced_speed_duplex = ADVERTISE_100_FULL;
6577 break;
6578 case SPEED_1000 + DUPLEX_FULL:
6579 mac->autoneg = 1;
6580 adapter->hw.phy.autoneg_advertised = ADVERTISE_1000_FULL;
6581 break;
6582 case SPEED_1000 + DUPLEX_HALF: /* not supported */
6583 default:
6584 goto err_inval;
6585 }
6586 return 0;
6587
6588err_inval:
6589 dev_err(&pdev->dev, "Unsupported Speed/Duplex configuration\n");
6590 return -EINVAL;
6591}
6592
6593static int __igb_shutdown(struct pci_dev *pdev, bool *enable_wake,
6594 bool runtime)
6595{
6596 struct net_device *netdev = pci_get_drvdata(pdev);
6597 struct igb_adapter *adapter = netdev_priv(netdev);
6598 struct e1000_hw *hw = &adapter->hw;
6599 u32 ctrl, rctl, status;
6600 u32 wufc = runtime ? E1000_WUFC_LNKC : adapter->wol;
6601#ifdef CONFIG_PM
6602 int retval = 0;
6603#endif
6604
6605 netif_device_detach(netdev);
6606
6607 if (netif_running(netdev))
6608 __igb_close(netdev, true);
6609
6610 igb_clear_interrupt_scheme(adapter);
6611
6612#ifdef CONFIG_PM
6613 retval = pci_save_state(pdev);
6614 if (retval)
6615 return retval;
6616#endif
6617
6618 status = rd32(E1000_STATUS);
6619 if (status & E1000_STATUS_LU)
6620 wufc &= ~E1000_WUFC_LNKC;
6621
6622 if (wufc) {
6623 igb_setup_rctl(adapter);
6624 igb_set_rx_mode(netdev);
6625
6626 /* turn on all-multi mode if wake on multicast is enabled */
6627 if (wufc & E1000_WUFC_MC) {
6628 rctl = rd32(E1000_RCTL);
6629 rctl |= E1000_RCTL_MPE;
6630 wr32(E1000_RCTL, rctl);
6631 }
6632
6633 ctrl = rd32(E1000_CTRL);
6634 /* advertise wake from D3Cold */
6635 #define E1000_CTRL_ADVD3WUC 0x00100000
6636 /* phy power management enable */
6637 #define E1000_CTRL_EN_PHY_PWR_MGMT 0x00200000
6638 ctrl |= E1000_CTRL_ADVD3WUC;
6639 wr32(E1000_CTRL, ctrl);
6640
6641 /* Allow time for pending master requests to run */
6642 igb_disable_pcie_master(hw);
6643
6644 wr32(E1000_WUC, E1000_WUC_PME_EN);
6645 wr32(E1000_WUFC, wufc);
6646 } else {
6647 wr32(E1000_WUC, 0);
6648 wr32(E1000_WUFC, 0);
6649 }
6650
6651 *enable_wake = wufc || adapter->en_mng_pt;
6652 if (!*enable_wake)
6653 igb_power_down_link(adapter);
6654 else
6655 igb_power_up_link(adapter);
6656
6657 /* Release control of h/w to f/w. If f/w is AMT enabled, this
6658 * would have already happened in close and is redundant. */
6659 igb_release_hw_control(adapter);
6660
6661 pci_disable_device(pdev);
6662
6663 return 0;
6664}
6665
6666#ifdef CONFIG_PM
6667#ifdef CONFIG_PM_SLEEP
6668static int igb_suspend(struct device *dev)
6669{
6670 int retval;
6671 bool wake;
6672 struct pci_dev *pdev = to_pci_dev(dev);
6673
6674 retval = __igb_shutdown(pdev, &wake, 0);
6675 if (retval)
6676 return retval;
6677
6678 if (wake) {
6679 pci_prepare_to_sleep(pdev);
6680 } else {
6681 pci_wake_from_d3(pdev, false);
6682 pci_set_power_state(pdev, PCI_D3hot);
6683 }
6684
6685 return 0;
6686}
6687#endif /* CONFIG_PM_SLEEP */
6688
6689static int igb_resume(struct device *dev)
6690{
6691 struct pci_dev *pdev = to_pci_dev(dev);
6692 struct net_device *netdev = pci_get_drvdata(pdev);
6693 struct igb_adapter *adapter = netdev_priv(netdev);
6694 struct e1000_hw *hw = &adapter->hw;
6695 u32 err;
6696
6697 pci_set_power_state(pdev, PCI_D0);
6698 pci_restore_state(pdev);
6699 pci_save_state(pdev);
6700
6701 err = pci_enable_device_mem(pdev);
6702 if (err) {
6703 dev_err(&pdev->dev,
6704 "igb: Cannot enable PCI device from suspend\n");
6705 return err;
6706 }
6707 pci_set_master(pdev);
6708
6709 pci_enable_wake(pdev, PCI_D3hot, 0);
6710 pci_enable_wake(pdev, PCI_D3cold, 0);
6711
6712 if (igb_init_interrupt_scheme(adapter)) {
6713 dev_err(&pdev->dev, "Unable to allocate memory for queues\n");
6714 return -ENOMEM;
6715 }
6716
6717 igb_reset(adapter);
6718
6719 /* let the f/w know that the h/w is now under the control of the
6720 * driver. */
6721 igb_get_hw_control(adapter);
6722
6723 wr32(E1000_WUS, ~0);
6724
6725 if (netdev->flags & IFF_UP) {
6726 err = __igb_open(netdev, true);
6727 if (err)
6728 return err;
6729 }
6730
6731 netif_device_attach(netdev);
6732 return 0;
6733}
6734
6735#ifdef CONFIG_PM_RUNTIME
6736static int igb_runtime_idle(struct device *dev)
6737{
6738 struct pci_dev *pdev = to_pci_dev(dev);
6739 struct net_device *netdev = pci_get_drvdata(pdev);
6740 struct igb_adapter *adapter = netdev_priv(netdev);
6741
6742 if (!igb_has_link(adapter))
6743 pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
6744
6745 return -EBUSY;
6746}
6747
6748static int igb_runtime_suspend(struct device *dev)
6749{
6750 struct pci_dev *pdev = to_pci_dev(dev);
6751 int retval;
6752 bool wake;
6753
6754 retval = __igb_shutdown(pdev, &wake, 1);
6755 if (retval)
6756 return retval;
6757
6758 if (wake) {
6759 pci_prepare_to_sleep(pdev);
6760 } else {
6761 pci_wake_from_d3(pdev, false);
6762 pci_set_power_state(pdev, PCI_D3hot);
6763 }
6764
6765 return 0;
6766}
6767
6768static int igb_runtime_resume(struct device *dev)
6769{
6770 return igb_resume(dev);
6771}
6772#endif /* CONFIG_PM_RUNTIME */
6773#endif
6774
6775static void igb_shutdown(struct pci_dev *pdev)
6776{
6777 bool wake;
6778
6779 __igb_shutdown(pdev, &wake, 0);
6780
6781 if (system_state == SYSTEM_POWER_OFF) {
6782 pci_wake_from_d3(pdev, wake);
6783 pci_set_power_state(pdev, PCI_D3hot);
6784 }
6785}
6786
6787#ifdef CONFIG_NET_POLL_CONTROLLER
6788/*
6789 * Polling 'interrupt' - used by things like netconsole to send skbs
6790 * without having to re-enable interrupts. It's not called while
6791 * the interrupt routine is executing.
6792 */
6793static void igb_netpoll(struct net_device *netdev)
6794{
6795 struct igb_adapter *adapter = netdev_priv(netdev);
6796 struct e1000_hw *hw = &adapter->hw;
6797 struct igb_q_vector *q_vector;
6798 int i;
6799
6800 for (i = 0; i < adapter->num_q_vectors; i++) {
6801 q_vector = adapter->q_vector[i];
6802 if (adapter->msix_entries)
6803 wr32(E1000_EIMC, q_vector->eims_value);
6804 else
6805 igb_irq_disable(adapter);
6806 napi_schedule(&q_vector->napi);
6807 }
6808}
6809#endif /* CONFIG_NET_POLL_CONTROLLER */
6810
6811/**
6812 * igb_io_error_detected - called when PCI error is detected
6813 * @pdev: Pointer to PCI device
6814 * @state: The current pci connection state
6815 *
6816 * This function is called after a PCI bus error affecting
6817 * this device has been detected.
6818 */
6819static pci_ers_result_t igb_io_error_detected(struct pci_dev *pdev,
6820 pci_channel_state_t state)
6821{
6822 struct net_device *netdev = pci_get_drvdata(pdev);
6823 struct igb_adapter *adapter = netdev_priv(netdev);
6824
6825 netif_device_detach(netdev);
6826
6827 if (state == pci_channel_io_perm_failure)
6828 return PCI_ERS_RESULT_DISCONNECT;
6829
6830 if (netif_running(netdev))
6831 igb_down(adapter);
6832 pci_disable_device(pdev);
6833
6834 /* Request a slot slot reset. */
6835 return PCI_ERS_RESULT_NEED_RESET;
6836}
6837
6838/**
6839 * igb_io_slot_reset - called after the pci bus has been reset.
6840 * @pdev: Pointer to PCI device
6841 *
6842 * Restart the card from scratch, as if from a cold-boot. Implementation
6843 * resembles the first-half of the igb_resume routine.
6844 */
6845static pci_ers_result_t igb_io_slot_reset(struct pci_dev *pdev)
6846{
6847 struct net_device *netdev = pci_get_drvdata(pdev);
6848 struct igb_adapter *adapter = netdev_priv(netdev);
6849 struct e1000_hw *hw = &adapter->hw;
6850 pci_ers_result_t result;
6851 int err;
6852
6853 if (pci_enable_device_mem(pdev)) {
6854 dev_err(&pdev->dev,
6855 "Cannot re-enable PCI device after reset.\n");
6856 result = PCI_ERS_RESULT_DISCONNECT;
6857 } else {
6858 pci_set_master(pdev);
6859 pci_restore_state(pdev);
6860 pci_save_state(pdev);
6861
6862 pci_enable_wake(pdev, PCI_D3hot, 0);
6863 pci_enable_wake(pdev, PCI_D3cold, 0);
6864
6865 igb_reset(adapter);
6866 wr32(E1000_WUS, ~0);
6867 result = PCI_ERS_RESULT_RECOVERED;
6868 }
6869
6870 err = pci_cleanup_aer_uncorrect_error_status(pdev);
6871 if (err) {
6872 dev_err(&pdev->dev, "pci_cleanup_aer_uncorrect_error_status "
6873 "failed 0x%0x\n", err);
6874 /* non-fatal, continue */
6875 }
6876
6877 return result;
6878}
6879
6880/**
6881 * igb_io_resume - called when traffic can start flowing again.
6882 * @pdev: Pointer to PCI device
6883 *
6884 * This callback is called when the error recovery driver tells us that
6885 * its OK to resume normal operation. Implementation resembles the
6886 * second-half of the igb_resume routine.
6887 */
6888static void igb_io_resume(struct pci_dev *pdev)
6889{
6890 struct net_device *netdev = pci_get_drvdata(pdev);
6891 struct igb_adapter *adapter = netdev_priv(netdev);
6892
6893 if (netif_running(netdev)) {
6894 if (igb_up(adapter)) {
6895 dev_err(&pdev->dev, "igb_up failed after reset\n");
6896 return;
6897 }
6898 }
6899
6900 netif_device_attach(netdev);
6901
6902 /* let the f/w know that the h/w is now under the control of the
6903 * driver. */
6904 igb_get_hw_control(adapter);
6905}
6906
6907static void igb_rar_set_qsel(struct igb_adapter *adapter, u8 *addr, u32 index,
6908 u8 qsel)
6909{
6910 u32 rar_low, rar_high;
6911 struct e1000_hw *hw = &adapter->hw;
6912
6913 /* HW expects these in little endian so we reverse the byte order
6914 * from network order (big endian) to little endian
6915 */
6916 rar_low = ((u32) addr[0] | ((u32) addr[1] << 8) |
6917 ((u32) addr[2] << 16) | ((u32) addr[3] << 24));
6918 rar_high = ((u32) addr[4] | ((u32) addr[5] << 8));
6919
6920 /* Indicate to hardware the Address is Valid. */
6921 rar_high |= E1000_RAH_AV;
6922
6923 if (hw->mac.type == e1000_82575)
6924 rar_high |= E1000_RAH_POOL_1 * qsel;
6925 else
6926 rar_high |= E1000_RAH_POOL_1 << qsel;
6927
6928 wr32(E1000_RAL(index), rar_low);
6929 wrfl();
6930 wr32(E1000_RAH(index), rar_high);
6931 wrfl();
6932}
6933
6934static int igb_set_vf_mac(struct igb_adapter *adapter,
6935 int vf, unsigned char *mac_addr)
6936{
6937 struct e1000_hw *hw = &adapter->hw;
6938 /* VF MAC addresses start at end of receive addresses and moves
6939 * torwards the first, as a result a collision should not be possible */
6940 int rar_entry = hw->mac.rar_entry_count - (vf + 1);
6941
6942 memcpy(adapter->vf_data[vf].vf_mac_addresses, mac_addr, ETH_ALEN);
6943
6944 igb_rar_set_qsel(adapter, mac_addr, rar_entry, vf);
6945
6946 return 0;
6947}
6948
6949static int igb_ndo_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
6950{
6951 struct igb_adapter *adapter = netdev_priv(netdev);
6952 if (!is_valid_ether_addr(mac) || (vf >= adapter->vfs_allocated_count))
6953 return -EINVAL;
6954 adapter->vf_data[vf].flags |= IGB_VF_FLAG_PF_SET_MAC;
6955 dev_info(&adapter->pdev->dev, "setting MAC %pM on VF %d\n", mac, vf);
6956 dev_info(&adapter->pdev->dev, "Reload the VF driver to make this"
6957 " change effective.");
6958 if (test_bit(__IGB_DOWN, &adapter->state)) {
6959 dev_warn(&adapter->pdev->dev, "The VF MAC address has been set,"
6960 " but the PF device is not up.\n");
6961 dev_warn(&adapter->pdev->dev, "Bring the PF device up before"
6962 " attempting to use the VF device.\n");
6963 }
6964 return igb_set_vf_mac(adapter, vf, mac);
6965}
6966
6967static int igb_link_mbps(int internal_link_speed)
6968{
6969 switch (internal_link_speed) {
6970 case SPEED_100:
6971 return 100;
6972 case SPEED_1000:
6973 return 1000;
6974 default:
6975 return 0;
6976 }
6977}
6978
6979static void igb_set_vf_rate_limit(struct e1000_hw *hw, int vf, int tx_rate,
6980 int link_speed)
6981{
6982 int rf_dec, rf_int;
6983 u32 bcnrc_val;
6984
6985 if (tx_rate != 0) {
6986 /* Calculate the rate factor values to set */
6987 rf_int = link_speed / tx_rate;
6988 rf_dec = (link_speed - (rf_int * tx_rate));
6989 rf_dec = (rf_dec * (1<<E1000_RTTBCNRC_RF_INT_SHIFT)) / tx_rate;
6990
6991 bcnrc_val = E1000_RTTBCNRC_RS_ENA;
6992 bcnrc_val |= ((rf_int<<E1000_RTTBCNRC_RF_INT_SHIFT) &
6993 E1000_RTTBCNRC_RF_INT_MASK);
6994 bcnrc_val |= (rf_dec & E1000_RTTBCNRC_RF_DEC_MASK);
6995 } else {
6996 bcnrc_val = 0;
6997 }
6998
6999 wr32(E1000_RTTDQSEL, vf); /* vf X uses queue X */
7000 wr32(E1000_RTTBCNRC, bcnrc_val);
7001}
7002
7003static void igb_check_vf_rate_limit(struct igb_adapter *adapter)
7004{
7005 int actual_link_speed, i;
7006 bool reset_rate = false;
7007
7008 /* VF TX rate limit was not set or not supported */
7009 if ((adapter->vf_rate_link_speed == 0) ||
7010 (adapter->hw.mac.type != e1000_82576))
7011 return;
7012
7013 actual_link_speed = igb_link_mbps(adapter->link_speed);
7014 if (actual_link_speed != adapter->vf_rate_link_speed) {
7015 reset_rate = true;
7016 adapter->vf_rate_link_speed = 0;
7017 dev_info(&adapter->pdev->dev,
7018 "Link speed has been changed. VF Transmit "
7019 "rate is disabled\n");
7020 }
7021
7022 for (i = 0; i < adapter->vfs_allocated_count; i++) {
7023 if (reset_rate)
7024 adapter->vf_data[i].tx_rate = 0;
7025
7026 igb_set_vf_rate_limit(&adapter->hw, i,
7027 adapter->vf_data[i].tx_rate,
7028 actual_link_speed);
7029 }
7030}
7031
7032static int igb_ndo_set_vf_bw(struct net_device *netdev, int vf, int tx_rate)
7033{
7034 struct igb_adapter *adapter = netdev_priv(netdev);
7035 struct e1000_hw *hw = &adapter->hw;
7036 int actual_link_speed;
7037
7038 if (hw->mac.type != e1000_82576)
7039 return -EOPNOTSUPP;
7040
7041 actual_link_speed = igb_link_mbps(adapter->link_speed);
7042 if ((vf >= adapter->vfs_allocated_count) ||
7043 (!(rd32(E1000_STATUS) & E1000_STATUS_LU)) ||
7044 (tx_rate < 0) || (tx_rate > actual_link_speed))
7045 return -EINVAL;
7046
7047 adapter->vf_rate_link_speed = actual_link_speed;
7048 adapter->vf_data[vf].tx_rate = (u16)tx_rate;
7049 igb_set_vf_rate_limit(hw, vf, tx_rate, actual_link_speed);
7050
7051 return 0;
7052}
7053
7054static int igb_ndo_get_vf_config(struct net_device *netdev,
7055 int vf, struct ifla_vf_info *ivi)
7056{
7057 struct igb_adapter *adapter = netdev_priv(netdev);
7058 if (vf >= adapter->vfs_allocated_count)
7059 return -EINVAL;
7060 ivi->vf = vf;
7061 memcpy(&ivi->mac, adapter->vf_data[vf].vf_mac_addresses, ETH_ALEN);
7062 ivi->tx_rate = adapter->vf_data[vf].tx_rate;
7063 ivi->vlan = adapter->vf_data[vf].pf_vlan;
7064 ivi->qos = adapter->vf_data[vf].pf_qos;
7065 return 0;
7066}
7067
7068static void igb_vmm_control(struct igb_adapter *adapter)
7069{
7070 struct e1000_hw *hw = &adapter->hw;
7071 u32 reg;
7072
7073 switch (hw->mac.type) {
7074 case e1000_82575:
7075 case e1000_i210:
7076 case e1000_i211:
7077 default:
7078 /* replication is not supported for 82575 */
7079 return;
7080 case e1000_82576:
7081 /* notify HW that the MAC is adding vlan tags */
7082 reg = rd32(E1000_DTXCTL);
7083 reg |= E1000_DTXCTL_VLAN_ADDED;
7084 wr32(E1000_DTXCTL, reg);
7085 case e1000_82580:
7086 /* enable replication vlan tag stripping */
7087 reg = rd32(E1000_RPLOLR);
7088 reg |= E1000_RPLOLR_STRVLAN;
7089 wr32(E1000_RPLOLR, reg);
7090 case e1000_i350:
7091 /* none of the above registers are supported by i350 */
7092 break;
7093 }
7094
7095 if (adapter->vfs_allocated_count) {
7096 igb_vmdq_set_loopback_pf(hw, true);
7097 igb_vmdq_set_replication_pf(hw, true);
7098 igb_vmdq_set_anti_spoofing_pf(hw, true,
7099 adapter->vfs_allocated_count);
7100 } else {
7101 igb_vmdq_set_loopback_pf(hw, false);
7102 igb_vmdq_set_replication_pf(hw, false);
7103 }
7104}
7105
7106static void igb_init_dmac(struct igb_adapter *adapter, u32 pba)
7107{
7108 struct e1000_hw *hw = &adapter->hw;
7109 u32 dmac_thr;
7110 u16 hwm;
7111
7112 if (hw->mac.type > e1000_82580) {
7113 if (adapter->flags & IGB_FLAG_DMAC) {
7114 u32 reg;
7115
7116 /* force threshold to 0. */
7117 wr32(E1000_DMCTXTH, 0);
7118
7119 /*
7120 * DMA Coalescing high water mark needs to be greater
7121 * than the Rx threshold. Set hwm to PBA - max frame
7122 * size in 16B units, capping it at PBA - 6KB.
7123 */
7124 hwm = 64 * pba - adapter->max_frame_size / 16;
7125 if (hwm < 64 * (pba - 6))
7126 hwm = 64 * (pba - 6);
7127 reg = rd32(E1000_FCRTC);
7128 reg &= ~E1000_FCRTC_RTH_COAL_MASK;
7129 reg |= ((hwm << E1000_FCRTC_RTH_COAL_SHIFT)
7130 & E1000_FCRTC_RTH_COAL_MASK);
7131 wr32(E1000_FCRTC, reg);
7132
7133 /*
7134 * Set the DMA Coalescing Rx threshold to PBA - 2 * max
7135 * frame size, capping it at PBA - 10KB.
7136 */
7137 dmac_thr = pba - adapter->max_frame_size / 512;
7138 if (dmac_thr < pba - 10)
7139 dmac_thr = pba - 10;
7140 reg = rd32(E1000_DMACR);
7141 reg &= ~E1000_DMACR_DMACTHR_MASK;
7142 reg |= ((dmac_thr << E1000_DMACR_DMACTHR_SHIFT)
7143 & E1000_DMACR_DMACTHR_MASK);
7144
7145 /* transition to L0x or L1 if available..*/
7146 reg |= (E1000_DMACR_DMAC_EN | E1000_DMACR_DMAC_LX_MASK);
7147
7148 /* watchdog timer= +-1000 usec in 32usec intervals */
7149 reg |= (1000 >> 5);
7150
7151 /* Disable BMC-to-OS Watchdog Enable */
7152 reg &= ~E1000_DMACR_DC_BMC2OSW_EN;
7153 wr32(E1000_DMACR, reg);
7154
7155 /*
7156 * no lower threshold to disable
7157 * coalescing(smart fifb)-UTRESH=0
7158 */
7159 wr32(E1000_DMCRTRH, 0);
7160
7161 reg = (IGB_DMCTLX_DCFLUSH_DIS | 0x4);
7162
7163 wr32(E1000_DMCTLX, reg);
7164
7165 /*
7166 * free space in tx packet buffer to wake from
7167 * DMA coal
7168 */
7169 wr32(E1000_DMCTXTH, (IGB_MIN_TXPBSIZE -
7170 (IGB_TX_BUF_4096 + adapter->max_frame_size)) >> 6);
7171
7172 /*
7173 * make low power state decision controlled
7174 * by DMA coal
7175 */
7176 reg = rd32(E1000_PCIEMISC);
7177 reg &= ~E1000_PCIEMISC_LX_DECISION;
7178 wr32(E1000_PCIEMISC, reg);
7179 } /* endif adapter->dmac is not disabled */
7180 } else if (hw->mac.type == e1000_82580) {
7181 u32 reg = rd32(E1000_PCIEMISC);
7182 wr32(E1000_PCIEMISC, reg & ~E1000_PCIEMISC_LX_DECISION);
7183 wr32(E1000_DMACR, 0);
7184 }
7185}
7186
7187/* igb_main.c */