1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright(c) 2009 - 2018 Intel Corporation. */
   3
   4#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   5
   6#include <linux/bitfield.h>
   7#include <linux/delay.h>
   8#include <linux/ethtool.h>
   9#include <linux/if_vlan.h>
  10#include <linux/init.h>
  11#include <linux/ipv6.h>
  12#include <linux/mii.h>
  13#include <linux/module.h>
  14#include <linux/netdevice.h>
  15#include <linux/pagemap.h>
  16#include <linux/pci.h>
  17#include <linux/prefetch.h>
  18#include <linux/sctp.h>
  19#include <linux/slab.h>
  20#include <linux/tcp.h>
  21#include <linux/types.h>
 
 
  22#include <linux/vmalloc.h>
 
 
 
 
 
 
  23#include <net/checksum.h>
  24#include <net/ip6_checksum.h>
 
 
 
 
 
 
  25#include "igbvf.h"
  26
 
  27char igbvf_driver_name[] = "igbvf";
 
  28static const char igbvf_driver_string[] =
  29		  "Intel(R) Gigabit Virtual Function Network Driver";
  30static const char igbvf_copyright[] =
  31		  "Copyright (c) 2009 - 2012 Intel Corporation.";
  32
  33#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV|NETIF_MSG_PROBE|NETIF_MSG_LINK)
  34static int debug = -1;
  35module_param(debug, int, 0);
  36MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  37
  38static int igbvf_poll(struct napi_struct *napi, int budget);
  39static void igbvf_reset(struct igbvf_adapter *);
  40static void igbvf_set_interrupt_capability(struct igbvf_adapter *);
  41static void igbvf_reset_interrupt_capability(struct igbvf_adapter *);
  42
  43static struct igbvf_info igbvf_vf_info = {
  44	.mac		= e1000_vfadapt,
  45	.flags		= 0,
  46	.pba		= 10,
  47	.init_ops	= e1000_init_function_pointers_vf,
  48};
  49
  50static struct igbvf_info igbvf_i350_vf_info = {
  51	.mac		= e1000_vfadapt_i350,
  52	.flags		= 0,
  53	.pba		= 10,
  54	.init_ops	= e1000_init_function_pointers_vf,
  55};
  56
  57static const struct igbvf_info *igbvf_info_tbl[] = {
  58	[board_vf]	= &igbvf_vf_info,
  59	[board_i350_vf]	= &igbvf_i350_vf_info,
  60};
  61
  62/**
  63 * igbvf_desc_unused - calculate if we have unused descriptors
  64 * @ring: address of receive ring structure
  65 **/
  66static int igbvf_desc_unused(struct igbvf_ring *ring)
  67{
  68	if (ring->next_to_clean > ring->next_to_use)
  69		return ring->next_to_clean - ring->next_to_use - 1;
  70
  71	return ring->count + ring->next_to_clean - ring->next_to_use - 1;
  72}
  73
  74/**
  75 * igbvf_receive_skb - helper function to handle Rx indications
  76 * @adapter: board private structure
  77 * @netdev: pointer to netdev struct
  78 * @skb: skb to indicate to stack
  79 * @status: descriptor status field as written by hardware
  80 * @vlan: descriptor vlan field as written by hardware (no le/be conversion)
  81 * @skb: pointer to sk_buff to be indicated to stack
  82 **/
  83static void igbvf_receive_skb(struct igbvf_adapter *adapter,
  84			      struct net_device *netdev,
  85			      struct sk_buff *skb,
  86			      u32 status, __le16 vlan)
  87{
  88	u16 vid;
  89
  90	if (status & E1000_RXD_STAT_VP) {
  91		if ((adapter->flags & IGBVF_FLAG_RX_LB_VLAN_BSWAP) &&
  92		    (status & E1000_RXDEXT_STATERR_LB))
  93			vid = be16_to_cpu((__force __be16)vlan) & E1000_RXD_SPC_VLAN_MASK;
  94		else
  95			vid = le16_to_cpu(vlan) & E1000_RXD_SPC_VLAN_MASK;
  96		if (test_bit(vid, adapter->active_vlans))
  97			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
  98	}
  99
 100	napi_gro_receive(&adapter->rx_ring->napi, skb);
 101}
 102
 103static inline void igbvf_rx_checksum_adv(struct igbvf_adapter *adapter,
 104					 u32 status_err, struct sk_buff *skb)
 105{
 106	skb_checksum_none_assert(skb);
 107
 108	/* Ignore Checksum bit is set or checksum is disabled through ethtool */
 109	if ((status_err & E1000_RXD_STAT_IXSM) ||
 110	    (adapter->flags & IGBVF_FLAG_RX_CSUM_DISABLED))
 111		return;
 112
 113	/* TCP/UDP checksum error bit is set */
 114	if (status_err &
 115	    (E1000_RXDEXT_STATERR_TCPE | E1000_RXDEXT_STATERR_IPE)) {
 116		/* let the stack verify checksum errors */
 117		adapter->hw_csum_err++;
 118		return;
 119	}
 120
 121	/* It must be a TCP or UDP packet with a valid checksum */
 122	if (status_err & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS))
 123		skb->ip_summed = CHECKSUM_UNNECESSARY;
 124
 125	adapter->hw_csum_good++;
 126}
 127
 128/**
 129 * igbvf_alloc_rx_buffers - Replace used receive buffers; packet split
 130 * @rx_ring: address of ring structure to repopulate
 131 * @cleaned_count: number of buffers to repopulate
 132 **/
 133static void igbvf_alloc_rx_buffers(struct igbvf_ring *rx_ring,
 134				   int cleaned_count)
 135{
 136	struct igbvf_adapter *adapter = rx_ring->adapter;
 137	struct net_device *netdev = adapter->netdev;
 138	struct pci_dev *pdev = adapter->pdev;
 139	union e1000_adv_rx_desc *rx_desc;
 140	struct igbvf_buffer *buffer_info;
 141	struct sk_buff *skb;
 142	unsigned int i;
 143	int bufsz;
 144
 145	i = rx_ring->next_to_use;
 146	buffer_info = &rx_ring->buffer_info[i];
 147
 148	if (adapter->rx_ps_hdr_size)
 149		bufsz = adapter->rx_ps_hdr_size;
 150	else
 151		bufsz = adapter->rx_buffer_len;
 152
 153	while (cleaned_count--) {
 154		rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
 155
 156		if (adapter->rx_ps_hdr_size && !buffer_info->page_dma) {
 157			if (!buffer_info->page) {
 158				buffer_info->page = alloc_page(GFP_ATOMIC);
 159				if (!buffer_info->page) {
 160					adapter->alloc_rx_buff_failed++;
 161					goto no_buffers;
 162				}
 163				buffer_info->page_offset = 0;
 164			} else {
 165				buffer_info->page_offset ^= PAGE_SIZE / 2;
 166			}
 167			buffer_info->page_dma =
 168				dma_map_page(&pdev->dev, buffer_info->page,
 169					     buffer_info->page_offset,
 170					     PAGE_SIZE / 2,
 171					     DMA_FROM_DEVICE);
 172			if (dma_mapping_error(&pdev->dev,
 173					      buffer_info->page_dma)) {
 174				__free_page(buffer_info->page);
 175				buffer_info->page = NULL;
 176				dev_err(&pdev->dev, "RX DMA map failed\n");
 177				break;
 178			}
 179		}
 180
 181		if (!buffer_info->skb) {
 182			skb = netdev_alloc_skb_ip_align(netdev, bufsz);
 183			if (!skb) {
 184				adapter->alloc_rx_buff_failed++;
 185				goto no_buffers;
 186			}
 187
 188			buffer_info->skb = skb;
 189			buffer_info->dma = dma_map_single(&pdev->dev, skb->data,
 190							  bufsz,
 191							  DMA_FROM_DEVICE);
 192			if (dma_mapping_error(&pdev->dev, buffer_info->dma)) {
 193				dev_kfree_skb(buffer_info->skb);
 194				buffer_info->skb = NULL;
 195				dev_err(&pdev->dev, "RX DMA map failed\n");
 196				goto no_buffers;
 197			}
 198		}
 199		/* Refresh the desc even if buffer_addrs didn't change because
 200		 * each write-back erases this info.
 201		 */
 202		if (adapter->rx_ps_hdr_size) {
 203			rx_desc->read.pkt_addr =
 204			     cpu_to_le64(buffer_info->page_dma);
 205			rx_desc->read.hdr_addr = cpu_to_le64(buffer_info->dma);
 206		} else {
 207			rx_desc->read.pkt_addr = cpu_to_le64(buffer_info->dma);
 208			rx_desc->read.hdr_addr = 0;
 209		}
 210
 211		i++;
 212		if (i == rx_ring->count)
 213			i = 0;
 214		buffer_info = &rx_ring->buffer_info[i];
 215	}
 216
 217no_buffers:
 218	if (rx_ring->next_to_use != i) {
 219		rx_ring->next_to_use = i;
 220		if (i == 0)
 221			i = (rx_ring->count - 1);
 222		else
 223			i--;
 224
 225		/* Force memory writes to complete before letting h/w
 226		 * know there are new descriptors to fetch.  (Only
 227		 * applicable for weak-ordered memory model archs,
 228		 * such as IA-64).
 229		*/
 230		wmb();
 231		writel(i, adapter->hw.hw_addr + rx_ring->tail);
 232	}
 233}
 234
 235/**
 236 * igbvf_clean_rx_irq - Send received data up the network stack; legacy
 237 * @adapter: board private structure
 238 * @work_done: output parameter used to indicate completed work
 239 * @work_to_do: input parameter setting limit of work
 240 *
 241 * the return value indicates whether actual cleaning was done, there
 242 * is no guarantee that everything was cleaned
 243 **/
 244static bool igbvf_clean_rx_irq(struct igbvf_adapter *adapter,
 245			       int *work_done, int work_to_do)
 246{
 247	struct igbvf_ring *rx_ring = adapter->rx_ring;
 248	struct net_device *netdev = adapter->netdev;
 249	struct pci_dev *pdev = adapter->pdev;
 250	union e1000_adv_rx_desc *rx_desc, *next_rxd;
 251	struct igbvf_buffer *buffer_info, *next_buffer;
 252	struct sk_buff *skb;
 253	bool cleaned = false;
 254	int cleaned_count = 0;
 255	unsigned int total_bytes = 0, total_packets = 0;
 256	unsigned int i;
 257	u32 length, hlen, staterr;
 258
 259	i = rx_ring->next_to_clean;
 260	rx_desc = IGBVF_RX_DESC_ADV(*rx_ring, i);
 261	staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 262
 263	while (staterr & E1000_RXD_STAT_DD) {
 264		if (*work_done >= work_to_do)
 265			break;
 266		(*work_done)++;
 267		rmb(); /* read descriptor and rx_buffer_info after status DD */
 268
 269		buffer_info = &rx_ring->buffer_info[i];
 270
 271		/* HW will not DMA in data larger than the given buffer, even
 272		 * if it parses the (NFS, of course) header to be larger.  In
 273		 * that case, it fills the header buffer and spills the rest
 274		 * into the page.
 275		 */
 276		hlen = le16_get_bits(rx_desc->wb.lower.lo_dword.hs_rss.hdr_info,
 277				     E1000_RXDADV_HDRBUFLEN_MASK);
 
 278		if (hlen > adapter->rx_ps_hdr_size)
 279			hlen = adapter->rx_ps_hdr_size;
 280
 281		length = le16_to_cpu(rx_desc->wb.upper.length);
 282		cleaned = true;
 283		cleaned_count++;
 284
 285		skb = buffer_info->skb;
 286		prefetch(skb->data - NET_IP_ALIGN);
 287		buffer_info->skb = NULL;
 288		if (!adapter->rx_ps_hdr_size) {
 289			dma_unmap_single(&pdev->dev, buffer_info->dma,
 290					 adapter->rx_buffer_len,
 291					 DMA_FROM_DEVICE);
 292			buffer_info->dma = 0;
 293			skb_put(skb, length);
 294			goto send_up;
 295		}
 296
 297		if (!skb_shinfo(skb)->nr_frags) {
 298			dma_unmap_single(&pdev->dev, buffer_info->dma,
 299					 adapter->rx_ps_hdr_size,
 300					 DMA_FROM_DEVICE);
 301			buffer_info->dma = 0;
 302			skb_put(skb, hlen);
 303		}
 304
 305		if (length) {
 306			dma_unmap_page(&pdev->dev, buffer_info->page_dma,
 307				       PAGE_SIZE / 2,
 308				       DMA_FROM_DEVICE);
 309			buffer_info->page_dma = 0;
 310
 311			skb_fill_page_desc(skb, skb_shinfo(skb)->nr_frags,
 312					   buffer_info->page,
 313					   buffer_info->page_offset,
 314					   length);
 315
 316			if ((adapter->rx_buffer_len > (PAGE_SIZE / 2)) ||
 317			    (page_count(buffer_info->page) != 1))
 318				buffer_info->page = NULL;
 319			else
 320				get_page(buffer_info->page);
 321
 322			skb->len += length;
 323			skb->data_len += length;
 324			skb->truesize += PAGE_SIZE / 2;
 325		}
 326send_up:
 327		i++;
 328		if (i == rx_ring->count)
 329			i = 0;
 330		next_rxd = IGBVF_RX_DESC_ADV(*rx_ring, i);
 331		prefetch(next_rxd);
 332		next_buffer = &rx_ring->buffer_info[i];
 333
 334		if (!(staterr & E1000_RXD_STAT_EOP)) {
 335			buffer_info->skb = next_buffer->skb;
 336			buffer_info->dma = next_buffer->dma;
 337			next_buffer->skb = skb;
 338			next_buffer->dma = 0;
 339			goto next_desc;
 340		}
 341
 342		if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) {
 343			dev_kfree_skb_irq(skb);
 344			goto next_desc;
 345		}
 346
 347		total_bytes += skb->len;
 348		total_packets++;
 349
 350		igbvf_rx_checksum_adv(adapter, staterr, skb);
 351
 352		skb->protocol = eth_type_trans(skb, netdev);
 353
 354		igbvf_receive_skb(adapter, netdev, skb, staterr,
 355				  rx_desc->wb.upper.vlan);
 356
 357next_desc:
 358		rx_desc->wb.upper.status_error = 0;
 359
 360		/* return some buffers to hardware, one at a time is too slow */
 361		if (cleaned_count >= IGBVF_RX_BUFFER_WRITE) {
 362			igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
 363			cleaned_count = 0;
 364		}
 365
 366		/* use prefetched values */
 367		rx_desc = next_rxd;
 368		buffer_info = next_buffer;
 369
 370		staterr = le32_to_cpu(rx_desc->wb.upper.status_error);
 371	}
 372
 373	rx_ring->next_to_clean = i;
 374	cleaned_count = igbvf_desc_unused(rx_ring);
 375
 376	if (cleaned_count)
 377		igbvf_alloc_rx_buffers(rx_ring, cleaned_count);
 378
 379	adapter->total_rx_packets += total_packets;
 380	adapter->total_rx_bytes += total_bytes;
 381	netdev->stats.rx_bytes += total_bytes;
 382	netdev->stats.rx_packets += total_packets;
 383	return cleaned;
 384}
 385
 386static void igbvf_put_txbuf(struct igbvf_adapter *adapter,
 387			    struct igbvf_buffer *buffer_info)
 388{
 389	if (buffer_info->dma) {
 390		if (buffer_info->mapped_as_page)
 391			dma_unmap_page(&adapter->pdev->dev,
 392				       buffer_info->dma,
 393				       buffer_info->length,
 394				       DMA_TO_DEVICE);
 395		else
 396			dma_unmap_single(&adapter->pdev->dev,
 397					 buffer_info->dma,
 398					 buffer_info->length,
 399					 DMA_TO_DEVICE);
 400		buffer_info->dma = 0;
 401	}
 402	if (buffer_info->skb) {
 403		dev_kfree_skb_any(buffer_info->skb);
 404		buffer_info->skb = NULL;
 405	}
 406	buffer_info->time_stamp = 0;
 407}
 408
 409/**
 410 * igbvf_setup_tx_resources - allocate Tx resources (Descriptors)
 411 * @adapter: board private structure
 412 * @tx_ring: ring being initialized
 413 *
 414 * Return 0 on success, negative on failure
 415 **/
 416int igbvf_setup_tx_resources(struct igbvf_adapter *adapter,
 417			     struct igbvf_ring *tx_ring)
 418{
 419	struct pci_dev *pdev = adapter->pdev;
 420	int size;
 421
 422	size = sizeof(struct igbvf_buffer) * tx_ring->count;
 423	tx_ring->buffer_info = vzalloc(size);
 424	if (!tx_ring->buffer_info)
 425		goto err;
 426
 427	/* round up to nearest 4K */
 428	tx_ring->size = tx_ring->count * sizeof(union e1000_adv_tx_desc);
 429	tx_ring->size = ALIGN(tx_ring->size, 4096);
 430
 431	tx_ring->desc = dma_alloc_coherent(&pdev->dev, tx_ring->size,
 432					   &tx_ring->dma, GFP_KERNEL);
 433	if (!tx_ring->desc)
 434		goto err;
 435
 436	tx_ring->adapter = adapter;
 437	tx_ring->next_to_use = 0;
 438	tx_ring->next_to_clean = 0;
 439
 440	return 0;
 441err:
 442	vfree(tx_ring->buffer_info);
 443	dev_err(&adapter->pdev->dev,
 444		"Unable to allocate memory for the transmit descriptor ring\n");
 445	return -ENOMEM;
 446}
 447
 448/**
 449 * igbvf_setup_rx_resources - allocate Rx resources (Descriptors)
 450 * @adapter: board private structure
 451 * @rx_ring: ring being initialized
 452 *
 453 * Returns 0 on success, negative on failure
 454 **/
 455int igbvf_setup_rx_resources(struct igbvf_adapter *adapter,
 456			     struct igbvf_ring *rx_ring)
 457{
 458	struct pci_dev *pdev = adapter->pdev;
 459	int size, desc_len;
 460
 461	size = sizeof(struct igbvf_buffer) * rx_ring->count;
 462	rx_ring->buffer_info = vzalloc(size);
 463	if (!rx_ring->buffer_info)
 464		goto err;
 465
 466	desc_len = sizeof(union e1000_adv_rx_desc);
 467
 468	/* Round up to nearest 4K */
 469	rx_ring->size = rx_ring->count * desc_len;
 470	rx_ring->size = ALIGN(rx_ring->size, 4096);
 471
 472	rx_ring->desc = dma_alloc_coherent(&pdev->dev, rx_ring->size,
 473					   &rx_ring->dma, GFP_KERNEL);
 474	if (!rx_ring->desc)
 475		goto err;
 476
 477	rx_ring->next_to_clean = 0;
 478	rx_ring->next_to_use = 0;
 479
 480	rx_ring->adapter = adapter;
 481
 482	return 0;
 483
 484err:
 485	vfree(rx_ring->buffer_info);
 486	rx_ring->buffer_info = NULL;
 487	dev_err(&adapter->pdev->dev,
 488		"Unable to allocate memory for the receive descriptor ring\n");
 489	return -ENOMEM;
 490}
 491
 492/**
 493 * igbvf_clean_tx_ring - Free Tx Buffers
 494 * @tx_ring: ring to be cleaned
 495 **/
 496static void igbvf_clean_tx_ring(struct igbvf_ring *tx_ring)
 497{
 498	struct igbvf_adapter *adapter = tx_ring->adapter;
 499	struct igbvf_buffer *buffer_info;
 500	unsigned long size;
 501	unsigned int i;
 502
 503	if (!tx_ring->buffer_info)
 504		return;
 505
 506	/* Free all the Tx ring sk_buffs */
 507	for (i = 0; i < tx_ring->count; i++) {
 508		buffer_info = &tx_ring->buffer_info[i];
 509		igbvf_put_txbuf(adapter, buffer_info);
 510	}
 511
 512	size = sizeof(struct igbvf_buffer) * tx_ring->count;
 513	memset(tx_ring->buffer_info, 0, size);
 514
 515	/* Zero out the descriptor ring */
 516	memset(tx_ring->desc, 0, tx_ring->size);
 517
 518	tx_ring->next_to_use = 0;
 519	tx_ring->next_to_clean = 0;
 520
 521	writel(0, adapter->hw.hw_addr + tx_ring->head);
 522	writel(0, adapter->hw.hw_addr + tx_ring->tail);
 523}
 524
 525/**
 526 * igbvf_free_tx_resources - Free Tx Resources per Queue
 527 * @tx_ring: ring to free resources from
 528 *
 529 * Free all transmit software resources
 530 **/
 531void igbvf_free_tx_resources(struct igbvf_ring *tx_ring)
 532{
 533	struct pci_dev *pdev = tx_ring->adapter->pdev;
 534
 535	igbvf_clean_tx_ring(tx_ring);
 536
 537	vfree(tx_ring->buffer_info);
 538	tx_ring->buffer_info = NULL;
 539
 540	dma_free_coherent(&pdev->dev, tx_ring->size, tx_ring->desc,
 541			  tx_ring->dma);
 542
 543	tx_ring->desc = NULL;
 544}
 545
 546/**
 547 * igbvf_clean_rx_ring - Free Rx Buffers per Queue
 548 * @rx_ring: ring structure pointer to free buffers from
 549 **/
 550static void igbvf_clean_rx_ring(struct igbvf_ring *rx_ring)
 551{
 552	struct igbvf_adapter *adapter = rx_ring->adapter;
 553	struct igbvf_buffer *buffer_info;
 554	struct pci_dev *pdev = adapter->pdev;
 555	unsigned long size;
 556	unsigned int i;
 557
 558	if (!rx_ring->buffer_info)
 559		return;
 560
 561	/* Free all the Rx ring sk_buffs */
 562	for (i = 0; i < rx_ring->count; i++) {
 563		buffer_info = &rx_ring->buffer_info[i];
 564		if (buffer_info->dma) {
 565			if (adapter->rx_ps_hdr_size) {
 566				dma_unmap_single(&pdev->dev, buffer_info->dma,
 567						 adapter->rx_ps_hdr_size,
 568						 DMA_FROM_DEVICE);
 569			} else {
 570				dma_unmap_single(&pdev->dev, buffer_info->dma,
 571						 adapter->rx_buffer_len,
 572						 DMA_FROM_DEVICE);
 573			}
 574			buffer_info->dma = 0;
 575		}
 576
 577		if (buffer_info->skb) {
 578			dev_kfree_skb(buffer_info->skb);
 579			buffer_info->skb = NULL;
 580		}
 581
 582		if (buffer_info->page) {
 583			if (buffer_info->page_dma)
 584				dma_unmap_page(&pdev->dev,
 585					       buffer_info->page_dma,
 586					       PAGE_SIZE / 2,
 587					       DMA_FROM_DEVICE);
 588			put_page(buffer_info->page);
 589			buffer_info->page = NULL;
 590			buffer_info->page_dma = 0;
 591			buffer_info->page_offset = 0;
 592		}
 593	}
 594
 595	size = sizeof(struct igbvf_buffer) * rx_ring->count;
 596	memset(rx_ring->buffer_info, 0, size);
 597
 598	/* Zero out the descriptor ring */
 599	memset(rx_ring->desc, 0, rx_ring->size);
 600
 601	rx_ring->next_to_clean = 0;
 602	rx_ring->next_to_use = 0;
 603
 604	writel(0, adapter->hw.hw_addr + rx_ring->head);
 605	writel(0, adapter->hw.hw_addr + rx_ring->tail);
 606}
 607
 608/**
 609 * igbvf_free_rx_resources - Free Rx Resources
 610 * @rx_ring: ring to clean the resources from
 611 *
 612 * Free all receive software resources
 613 **/
 614
 615void igbvf_free_rx_resources(struct igbvf_ring *rx_ring)
 616{
 617	struct pci_dev *pdev = rx_ring->adapter->pdev;
 618
 619	igbvf_clean_rx_ring(rx_ring);
 620
 621	vfree(rx_ring->buffer_info);
 622	rx_ring->buffer_info = NULL;
 623
 624	dma_free_coherent(&pdev->dev, rx_ring->size, rx_ring->desc,
 625			  rx_ring->dma);
 626	rx_ring->desc = NULL;
 627}
 628
 629/**
 630 * igbvf_update_itr - update the dynamic ITR value based on statistics
 631 * @adapter: pointer to adapter
 632 * @itr_setting: current adapter->itr
 633 * @packets: the number of packets during this measurement interval
 634 * @bytes: the number of bytes during this measurement interval
 635 *
 636 * Stores a new ITR value based on packets and byte counts during the last
 637 * interrupt.  The advantage of per interrupt computation is faster updates
 638 * and more accurate ITR for the current traffic pattern.  Constants in this
 639 * function were computed based on theoretical maximum wire speed and thresholds
 640 * were set based on testing data as well as attempting to minimize response
 641 * time while increasing bulk throughput.
 642 **/
 643static enum latency_range igbvf_update_itr(struct igbvf_adapter *adapter,
 644					   enum latency_range itr_setting,
 645					   int packets, int bytes)
 646{
 647	enum latency_range retval = itr_setting;
 648
 649	if (packets == 0)
 650		goto update_itr_done;
 651
 652	switch (itr_setting) {
 653	case lowest_latency:
 654		/* handle TSO and jumbo frames */
 655		if (bytes/packets > 8000)
 656			retval = bulk_latency;
 657		else if ((packets < 5) && (bytes > 512))
 658			retval = low_latency;
 659		break;
 660	case low_latency:  /* 50 usec aka 20000 ints/s */
 661		if (bytes > 10000) {
 662			/* this if handles the TSO accounting */
 663			if (bytes/packets > 8000)
 664				retval = bulk_latency;
 665			else if ((packets < 10) || ((bytes/packets) > 1200))
 666				retval = bulk_latency;
 667			else if ((packets > 35))
 668				retval = lowest_latency;
 669		} else if (bytes/packets > 2000) {
 670			retval = bulk_latency;
 671		} else if (packets <= 2 && bytes < 512) {
 672			retval = lowest_latency;
 673		}
 674		break;
 675	case bulk_latency: /* 250 usec aka 4000 ints/s */
 676		if (bytes > 25000) {
 677			if (packets > 35)
 678				retval = low_latency;
 679		} else if (bytes < 6000) {
 680			retval = low_latency;
 681		}
 682		break;
 683	default:
 684		break;
 685	}
 686
 687update_itr_done:
 688	return retval;
 689}
 690
 691static int igbvf_range_to_itr(enum latency_range current_range)
 692{
 693	int new_itr;
 694
 695	switch (current_range) {
 696	/* counts and packets in update_itr are dependent on these numbers */
 697	case lowest_latency:
 698		new_itr = IGBVF_70K_ITR;
 699		break;
 700	case low_latency:
 701		new_itr = IGBVF_20K_ITR;
 702		break;
 703	case bulk_latency:
 704		new_itr = IGBVF_4K_ITR;
 705		break;
 706	default:
 707		new_itr = IGBVF_START_ITR;
 708		break;
 709	}
 710	return new_itr;
 711}
 712
 713static void igbvf_set_itr(struct igbvf_adapter *adapter)
 714{
 715	u32 new_itr;
 716
 717	adapter->tx_ring->itr_range =
 718			igbvf_update_itr(adapter,
 719					 adapter->tx_ring->itr_val,
 720					 adapter->total_tx_packets,
 721					 adapter->total_tx_bytes);
 722
 723	/* conservative mode (itr 3) eliminates the lowest_latency setting */
 724	if (adapter->requested_itr == 3 &&
 725	    adapter->tx_ring->itr_range == lowest_latency)
 726		adapter->tx_ring->itr_range = low_latency;
 727
 728	new_itr = igbvf_range_to_itr(adapter->tx_ring->itr_range);
 729
 730	if (new_itr != adapter->tx_ring->itr_val) {
 731		u32 current_itr = adapter->tx_ring->itr_val;
 732		/* this attempts to bias the interrupt rate towards Bulk
 733		 * by adding intermediate steps when interrupt rate is
 734		 * increasing
 735		 */
 736		new_itr = new_itr > current_itr ?
 737			  min(current_itr + (new_itr >> 2), new_itr) :
 738			  new_itr;
 739		adapter->tx_ring->itr_val = new_itr;
 740
 741		adapter->tx_ring->set_itr = 1;
 742	}
 743
 744	adapter->rx_ring->itr_range =
 745			igbvf_update_itr(adapter, adapter->rx_ring->itr_val,
 746					 adapter->total_rx_packets,
 747					 adapter->total_rx_bytes);
 748	if (adapter->requested_itr == 3 &&
 749	    adapter->rx_ring->itr_range == lowest_latency)
 750		adapter->rx_ring->itr_range = low_latency;
 751
 752	new_itr = igbvf_range_to_itr(adapter->rx_ring->itr_range);
 753
 754	if (new_itr != adapter->rx_ring->itr_val) {
 755		u32 current_itr = adapter->rx_ring->itr_val;
 756
 757		new_itr = new_itr > current_itr ?
 758			  min(current_itr + (new_itr >> 2), new_itr) :
 759			  new_itr;
 760		adapter->rx_ring->itr_val = new_itr;
 761
 762		adapter->rx_ring->set_itr = 1;
 763	}
 764}
 765
 766/**
 767 * igbvf_clean_tx_irq - Reclaim resources after transmit completes
 768 * @tx_ring: ring structure to clean descriptors from
 769 *
 770 * returns true if ring is completely cleaned
 771 **/
 772static bool igbvf_clean_tx_irq(struct igbvf_ring *tx_ring)
 773{
 774	struct igbvf_adapter *adapter = tx_ring->adapter;
 775	struct net_device *netdev = adapter->netdev;
 776	struct igbvf_buffer *buffer_info;
 777	struct sk_buff *skb;
 778	union e1000_adv_tx_desc *tx_desc, *eop_desc;
 779	unsigned int total_bytes = 0, total_packets = 0;
 780	unsigned int i, count = 0;
 781	bool cleaned = false;
 782
 783	i = tx_ring->next_to_clean;
 784	buffer_info = &tx_ring->buffer_info[i];
 785	eop_desc = buffer_info->next_to_watch;
 786
 787	do {
 788		/* if next_to_watch is not set then there is no work pending */
 789		if (!eop_desc)
 790			break;
 791
 792		/* prevent any other reads prior to eop_desc */
 793		smp_rmb();
 794
 795		/* if DD is not set pending work has not been completed */
 796		if (!(eop_desc->wb.status & cpu_to_le32(E1000_TXD_STAT_DD)))
 797			break;
 798
 799		/* clear next_to_watch to prevent false hangs */
 800		buffer_info->next_to_watch = NULL;
 801
 802		for (cleaned = false; !cleaned; count++) {
 803			tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
 804			cleaned = (tx_desc == eop_desc);
 805			skb = buffer_info->skb;
 806
 807			if (skb) {
 808				unsigned int segs, bytecount;
 809
 810				/* gso_segs is currently only valid for tcp */
 811				segs = skb_shinfo(skb)->gso_segs ?: 1;
 812				/* multiply data chunks by size of headers */
 813				bytecount = ((segs - 1) * skb_headlen(skb)) +
 814					    skb->len;
 815				total_packets += segs;
 816				total_bytes += bytecount;
 817			}
 818
 819			igbvf_put_txbuf(adapter, buffer_info);
 820			tx_desc->wb.status = 0;
 821
 822			i++;
 823			if (i == tx_ring->count)
 824				i = 0;
 825
 826			buffer_info = &tx_ring->buffer_info[i];
 827		}
 828
 829		eop_desc = buffer_info->next_to_watch;
 830	} while (count < tx_ring->count);
 831
 832	tx_ring->next_to_clean = i;
 833
 834	if (unlikely(count && netif_carrier_ok(netdev) &&
 835	    igbvf_desc_unused(tx_ring) >= IGBVF_TX_QUEUE_WAKE)) {
 836		/* Make sure that anybody stopping the queue after this
 837		 * sees the new next_to_clean.
 838		 */
 839		smp_mb();
 840		if (netif_queue_stopped(netdev) &&
 841		    !(test_bit(__IGBVF_DOWN, &adapter->state))) {
 842			netif_wake_queue(netdev);
 843			++adapter->restart_queue;
 844		}
 845	}
 846
 847	netdev->stats.tx_bytes += total_bytes;
 848	netdev->stats.tx_packets += total_packets;
 849	return count < tx_ring->count;
 850}
 851
 852static irqreturn_t igbvf_msix_other(int irq, void *data)
 853{
 854	struct net_device *netdev = data;
 855	struct igbvf_adapter *adapter = netdev_priv(netdev);
 856	struct e1000_hw *hw = &adapter->hw;
 857
 858	adapter->int_counter1++;
 859
 860	hw->mac.get_link_status = 1;
 861	if (!test_bit(__IGBVF_DOWN, &adapter->state))
 862		mod_timer(&adapter->watchdog_timer, jiffies + 1);
 863
 864	ew32(EIMS, adapter->eims_other);
 865
 866	return IRQ_HANDLED;
 867}
 868
 869static irqreturn_t igbvf_intr_msix_tx(int irq, void *data)
 870{
 871	struct net_device *netdev = data;
 872	struct igbvf_adapter *adapter = netdev_priv(netdev);
 873	struct e1000_hw *hw = &adapter->hw;
 874	struct igbvf_ring *tx_ring = adapter->tx_ring;
 875
 876	if (tx_ring->set_itr) {
 877		writel(tx_ring->itr_val,
 878		       adapter->hw.hw_addr + tx_ring->itr_register);
 879		adapter->tx_ring->set_itr = 0;
 880	}
 881
 882	adapter->total_tx_bytes = 0;
 883	adapter->total_tx_packets = 0;
 884
 885	/* auto mask will automatically re-enable the interrupt when we write
 886	 * EICS
 887	 */
 888	if (!igbvf_clean_tx_irq(tx_ring))
 889		/* Ring was not completely cleaned, so fire another interrupt */
 890		ew32(EICS, tx_ring->eims_value);
 891	else
 892		ew32(EIMS, tx_ring->eims_value);
 893
 894	return IRQ_HANDLED;
 895}
 896
 897static irqreturn_t igbvf_intr_msix_rx(int irq, void *data)
 898{
 899	struct net_device *netdev = data;
 900	struct igbvf_adapter *adapter = netdev_priv(netdev);
 901
 902	adapter->int_counter0++;
 903
 904	/* Write the ITR value calculated at the end of the
 905	 * previous interrupt.
 906	 */
 907	if (adapter->rx_ring->set_itr) {
 908		writel(adapter->rx_ring->itr_val,
 909		       adapter->hw.hw_addr + adapter->rx_ring->itr_register);
 910		adapter->rx_ring->set_itr = 0;
 911	}
 912
 913	if (napi_schedule_prep(&adapter->rx_ring->napi)) {
 914		adapter->total_rx_bytes = 0;
 915		adapter->total_rx_packets = 0;
 916		__napi_schedule(&adapter->rx_ring->napi);
 917	}
 918
 919	return IRQ_HANDLED;
 920}
 921
 922#define IGBVF_NO_QUEUE -1
 923
 924static void igbvf_assign_vector(struct igbvf_adapter *adapter, int rx_queue,
 925				int tx_queue, int msix_vector)
 926{
 927	struct e1000_hw *hw = &adapter->hw;
 928	u32 ivar, index;
 929
 930	/* 82576 uses a table-based method for assigning vectors.
 931	 * Each queue has a single entry in the table to which we write
 932	 * a vector number along with a "valid" bit.  Sadly, the layout
 933	 * of the table is somewhat counterintuitive.
 934	 */
 935	if (rx_queue > IGBVF_NO_QUEUE) {
 936		index = (rx_queue >> 1);
 937		ivar = array_er32(IVAR0, index);
 938		if (rx_queue & 0x1) {
 939			/* vector goes into third byte of register */
 940			ivar = ivar & 0xFF00FFFF;
 941			ivar |= (msix_vector | E1000_IVAR_VALID) << 16;
 942		} else {
 943			/* vector goes into low byte of register */
 944			ivar = ivar & 0xFFFFFF00;
 945			ivar |= msix_vector | E1000_IVAR_VALID;
 946		}
 947		adapter->rx_ring[rx_queue].eims_value = BIT(msix_vector);
 948		array_ew32(IVAR0, index, ivar);
 949	}
 950	if (tx_queue > IGBVF_NO_QUEUE) {
 951		index = (tx_queue >> 1);
 952		ivar = array_er32(IVAR0, index);
 953		if (tx_queue & 0x1) {
 954			/* vector goes into high byte of register */
 955			ivar = ivar & 0x00FFFFFF;
 956			ivar |= (msix_vector | E1000_IVAR_VALID) << 24;
 957		} else {
 958			/* vector goes into second byte of register */
 959			ivar = ivar & 0xFFFF00FF;
 960			ivar |= (msix_vector | E1000_IVAR_VALID) << 8;
 961		}
 962		adapter->tx_ring[tx_queue].eims_value = BIT(msix_vector);
 963		array_ew32(IVAR0, index, ivar);
 964	}
 965}
 966
 967/**
 968 * igbvf_configure_msix - Configure MSI-X hardware
 969 * @adapter: board private structure
 970 *
 971 * igbvf_configure_msix sets up the hardware to properly
 972 * generate MSI-X interrupts.
 973 **/
 974static void igbvf_configure_msix(struct igbvf_adapter *adapter)
 975{
 976	u32 tmp;
 977	struct e1000_hw *hw = &adapter->hw;
 978	struct igbvf_ring *tx_ring = adapter->tx_ring;
 979	struct igbvf_ring *rx_ring = adapter->rx_ring;
 980	int vector = 0;
 981
 982	adapter->eims_enable_mask = 0;
 983
 984	igbvf_assign_vector(adapter, IGBVF_NO_QUEUE, 0, vector++);
 985	adapter->eims_enable_mask |= tx_ring->eims_value;
 986	writel(tx_ring->itr_val, hw->hw_addr + tx_ring->itr_register);
 987	igbvf_assign_vector(adapter, 0, IGBVF_NO_QUEUE, vector++);
 988	adapter->eims_enable_mask |= rx_ring->eims_value;
 989	writel(rx_ring->itr_val, hw->hw_addr + rx_ring->itr_register);
 990
 991	/* set vector for other causes, i.e. link changes */
 992
 993	tmp = (vector++ | E1000_IVAR_VALID);
 994
 995	ew32(IVAR_MISC, tmp);
 996
 997	adapter->eims_enable_mask = GENMASK(vector - 1, 0);
 998	adapter->eims_other = BIT(vector - 1);
 999	e1e_flush();
1000}
1001
1002static void igbvf_reset_interrupt_capability(struct igbvf_adapter *adapter)
1003{
1004	if (adapter->msix_entries) {
1005		pci_disable_msix(adapter->pdev);
1006		kfree(adapter->msix_entries);
1007		adapter->msix_entries = NULL;
1008	}
1009}
1010
1011/**
1012 * igbvf_set_interrupt_capability - set MSI or MSI-X if supported
1013 * @adapter: board private structure
1014 *
1015 * Attempt to configure interrupts using the best available
1016 * capabilities of the hardware and kernel.
1017 **/
1018static void igbvf_set_interrupt_capability(struct igbvf_adapter *adapter)
1019{
1020	int err = -ENOMEM;
1021	int i;
1022
1023	/* we allocate 3 vectors, 1 for Tx, 1 for Rx, one for PF messages */
1024	adapter->msix_entries = kcalloc(3, sizeof(struct msix_entry),
1025					GFP_KERNEL);
1026	if (adapter->msix_entries) {
1027		for (i = 0; i < 3; i++)
1028			adapter->msix_entries[i].entry = i;
1029
1030		err = pci_enable_msix_range(adapter->pdev,
1031					    adapter->msix_entries, 3, 3);
1032	}
1033
1034	if (err < 0) {
1035		/* MSI-X failed */
1036		dev_err(&adapter->pdev->dev,
1037			"Failed to initialize MSI-X interrupts.\n");
1038		igbvf_reset_interrupt_capability(adapter);
1039	}
1040}
1041
1042/**
1043 * igbvf_request_msix - Initialize MSI-X interrupts
1044 * @adapter: board private structure
1045 *
1046 * igbvf_request_msix allocates MSI-X vectors and requests interrupts from the
1047 * kernel.
1048 **/
1049static int igbvf_request_msix(struct igbvf_adapter *adapter)
1050{
1051	struct net_device *netdev = adapter->netdev;
1052	int err = 0, vector = 0;
1053
1054	if (strlen(netdev->name) < (IFNAMSIZ - 5)) {
1055		sprintf(adapter->tx_ring->name, "%s-tx-0", netdev->name);
1056		sprintf(adapter->rx_ring->name, "%s-rx-0", netdev->name);
1057	} else {
1058		memcpy(adapter->tx_ring->name, netdev->name, IFNAMSIZ);
1059		memcpy(adapter->rx_ring->name, netdev->name, IFNAMSIZ);
1060	}
1061
1062	err = request_irq(adapter->msix_entries[vector].vector,
1063			  igbvf_intr_msix_tx, 0, adapter->tx_ring->name,
1064			  netdev);
1065	if (err)
1066		goto out;
1067
1068	adapter->tx_ring->itr_register = E1000_EITR(vector);
1069	adapter->tx_ring->itr_val = adapter->current_itr;
1070	vector++;
1071
1072	err = request_irq(adapter->msix_entries[vector].vector,
1073			  igbvf_intr_msix_rx, 0, adapter->rx_ring->name,
1074			  netdev);
1075	if (err)
1076		goto free_irq_tx;
1077
1078	adapter->rx_ring->itr_register = E1000_EITR(vector);
1079	adapter->rx_ring->itr_val = adapter->current_itr;
1080	vector++;
1081
1082	err = request_irq(adapter->msix_entries[vector].vector,
1083			  igbvf_msix_other, 0, netdev->name, netdev);
1084	if (err)
1085		goto free_irq_rx;
1086
1087	igbvf_configure_msix(adapter);
1088	return 0;
1089free_irq_rx:
1090	free_irq(adapter->msix_entries[--vector].vector, netdev);
1091free_irq_tx:
1092	free_irq(adapter->msix_entries[--vector].vector, netdev);
1093out:
1094	return err;
1095}
1096
1097/**
1098 * igbvf_alloc_queues - Allocate memory for all rings
1099 * @adapter: board private structure to initialize
1100 **/
1101static int igbvf_alloc_queues(struct igbvf_adapter *adapter)
1102{
1103	struct net_device *netdev = adapter->netdev;
1104
1105	adapter->tx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1106	if (!adapter->tx_ring)
1107		return -ENOMEM;
1108
1109	adapter->rx_ring = kzalloc(sizeof(struct igbvf_ring), GFP_KERNEL);
1110	if (!adapter->rx_ring) {
1111		kfree(adapter->tx_ring);
1112		return -ENOMEM;
1113	}
1114
1115	netif_napi_add(netdev, &adapter->rx_ring->napi, igbvf_poll);
1116
1117	return 0;
1118}
1119
1120/**
1121 * igbvf_request_irq - initialize interrupts
1122 * @adapter: board private structure
1123 *
1124 * Attempts to configure interrupts using the best available
1125 * capabilities of the hardware and kernel.
1126 **/
1127static int igbvf_request_irq(struct igbvf_adapter *adapter)
1128{
1129	int err = -1;
1130
1131	/* igbvf supports msi-x only */
1132	if (adapter->msix_entries)
1133		err = igbvf_request_msix(adapter);
1134
1135	if (!err)
1136		return err;
1137
1138	dev_err(&adapter->pdev->dev,
1139		"Unable to allocate interrupt, Error: %d\n", err);
1140
1141	return err;
1142}
1143
1144static void igbvf_free_irq(struct igbvf_adapter *adapter)
1145{
1146	struct net_device *netdev = adapter->netdev;
1147	int vector;
1148
1149	if (adapter->msix_entries) {
1150		for (vector = 0; vector < 3; vector++)
1151			free_irq(adapter->msix_entries[vector].vector, netdev);
1152	}
1153}
1154
1155/**
1156 * igbvf_irq_disable - Mask off interrupt generation on the NIC
1157 * @adapter: board private structure
1158 **/
1159static void igbvf_irq_disable(struct igbvf_adapter *adapter)
1160{
1161	struct e1000_hw *hw = &adapter->hw;
1162
1163	ew32(EIMC, ~0);
1164
1165	if (adapter->msix_entries)
1166		ew32(EIAC, 0);
1167}
1168
1169/**
1170 * igbvf_irq_enable - Enable default interrupt generation settings
1171 * @adapter: board private structure
1172 **/
1173static void igbvf_irq_enable(struct igbvf_adapter *adapter)
1174{
1175	struct e1000_hw *hw = &adapter->hw;
1176
1177	ew32(EIAC, adapter->eims_enable_mask);
1178	ew32(EIAM, adapter->eims_enable_mask);
1179	ew32(EIMS, adapter->eims_enable_mask);
1180}
1181
1182/**
1183 * igbvf_poll - NAPI Rx polling callback
1184 * @napi: struct associated with this polling callback
1185 * @budget: amount of packets driver is allowed to process this poll
1186 **/
1187static int igbvf_poll(struct napi_struct *napi, int budget)
1188{
1189	struct igbvf_ring *rx_ring = container_of(napi, struct igbvf_ring, napi);
1190	struct igbvf_adapter *adapter = rx_ring->adapter;
1191	struct e1000_hw *hw = &adapter->hw;
1192	int work_done = 0;
1193
1194	igbvf_clean_rx_irq(adapter, &work_done, budget);
1195
1196	if (work_done == budget)
1197		return budget;
1198
1199	/* Exit the polling mode, but don't re-enable interrupts if stack might
1200	 * poll us due to busy-polling
1201	 */
1202	if (likely(napi_complete_done(napi, work_done))) {
1203		if (adapter->requested_itr & 3)
1204			igbvf_set_itr(adapter);
1205
1206		if (!test_bit(__IGBVF_DOWN, &adapter->state))
1207			ew32(EIMS, adapter->rx_ring->eims_value);
1208	}
1209
1210	return work_done;
1211}
1212
1213/**
1214 * igbvf_set_rlpml - set receive large packet maximum length
1215 * @adapter: board private structure
1216 *
1217 * Configure the maximum size of packets that will be received
1218 */
1219static void igbvf_set_rlpml(struct igbvf_adapter *adapter)
1220{
1221	int max_frame_size;
1222	struct e1000_hw *hw = &adapter->hw;
1223
1224	max_frame_size = adapter->max_frame_size + VLAN_TAG_SIZE;
1225
1226	spin_lock_bh(&hw->mbx_lock);
1227
1228	e1000_rlpml_set_vf(hw, max_frame_size);
1229
1230	spin_unlock_bh(&hw->mbx_lock);
1231}
1232
1233static int igbvf_vlan_rx_add_vid(struct net_device *netdev,
1234				 __be16 proto, u16 vid)
1235{
1236	struct igbvf_adapter *adapter = netdev_priv(netdev);
1237	struct e1000_hw *hw = &adapter->hw;
1238
1239	spin_lock_bh(&hw->mbx_lock);
1240
1241	if (hw->mac.ops.set_vfta(hw, vid, true)) {
1242		dev_warn(&adapter->pdev->dev, "Vlan id %d\n is not added", vid);
1243		spin_unlock_bh(&hw->mbx_lock);
1244		return -EINVAL;
1245	}
1246
1247	spin_unlock_bh(&hw->mbx_lock);
1248
1249	set_bit(vid, adapter->active_vlans);
1250	return 0;
1251}
1252
1253static int igbvf_vlan_rx_kill_vid(struct net_device *netdev,
1254				  __be16 proto, u16 vid)
1255{
1256	struct igbvf_adapter *adapter = netdev_priv(netdev);
1257	struct e1000_hw *hw = &adapter->hw;
1258
1259	spin_lock_bh(&hw->mbx_lock);
1260
1261	if (hw->mac.ops.set_vfta(hw, vid, false)) {
1262		dev_err(&adapter->pdev->dev,
1263			"Failed to remove vlan id %d\n", vid);
1264		spin_unlock_bh(&hw->mbx_lock);
1265		return -EINVAL;
1266	}
1267
1268	spin_unlock_bh(&hw->mbx_lock);
1269
1270	clear_bit(vid, adapter->active_vlans);
1271	return 0;
1272}
1273
1274static void igbvf_restore_vlan(struct igbvf_adapter *adapter)
1275{
1276	u16 vid;
1277
1278	for_each_set_bit(vid, adapter->active_vlans, VLAN_N_VID)
1279		igbvf_vlan_rx_add_vid(adapter->netdev, htons(ETH_P_8021Q), vid);
1280}
1281
1282/**
1283 * igbvf_configure_tx - Configure Transmit Unit after Reset
1284 * @adapter: board private structure
1285 *
1286 * Configure the Tx unit of the MAC after a reset.
1287 **/
1288static void igbvf_configure_tx(struct igbvf_adapter *adapter)
1289{
1290	struct e1000_hw *hw = &adapter->hw;
1291	struct igbvf_ring *tx_ring = adapter->tx_ring;
1292	u64 tdba;
1293	u32 txdctl, dca_txctrl;
1294
1295	/* disable transmits */
1296	txdctl = er32(TXDCTL(0));
1297	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1298	e1e_flush();
1299	msleep(10);
1300
1301	/* Setup the HW Tx Head and Tail descriptor pointers */
1302	ew32(TDLEN(0), tx_ring->count * sizeof(union e1000_adv_tx_desc));
1303	tdba = tx_ring->dma;
1304	ew32(TDBAL(0), (tdba & DMA_BIT_MASK(32)));
1305	ew32(TDBAH(0), (tdba >> 32));
1306	ew32(TDH(0), 0);
1307	ew32(TDT(0), 0);
1308	tx_ring->head = E1000_TDH(0);
1309	tx_ring->tail = E1000_TDT(0);
1310
1311	/* Turn off Relaxed Ordering on head write-backs.  The writebacks
1312	 * MUST be delivered in order or it will completely screw up
1313	 * our bookkeeping.
1314	 */
1315	dca_txctrl = er32(DCA_TXCTRL(0));
1316	dca_txctrl &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN;
1317	ew32(DCA_TXCTRL(0), dca_txctrl);
1318
1319	/* enable transmits */
1320	txdctl |= E1000_TXDCTL_QUEUE_ENABLE;
1321	ew32(TXDCTL(0), txdctl);
1322
1323	/* Setup Transmit Descriptor Settings for eop descriptor */
1324	adapter->txd_cmd = E1000_ADVTXD_DCMD_EOP | E1000_ADVTXD_DCMD_IFCS;
1325
1326	/* enable Report Status bit */
1327	adapter->txd_cmd |= E1000_ADVTXD_DCMD_RS;
1328}
1329
1330/**
1331 * igbvf_setup_srrctl - configure the receive control registers
1332 * @adapter: Board private structure
1333 **/
1334static void igbvf_setup_srrctl(struct igbvf_adapter *adapter)
1335{
1336	struct e1000_hw *hw = &adapter->hw;
1337	u32 srrctl = 0;
1338
1339	srrctl &= ~(E1000_SRRCTL_DESCTYPE_MASK |
1340		    E1000_SRRCTL_BSIZEHDR_MASK |
1341		    E1000_SRRCTL_BSIZEPKT_MASK);
1342
1343	/* Enable queue drop to avoid head of line blocking */
1344	srrctl |= E1000_SRRCTL_DROP_EN;
1345
1346	/* Setup buffer sizes */
1347	srrctl |= ALIGN(adapter->rx_buffer_len, 1024) >>
1348		  E1000_SRRCTL_BSIZEPKT_SHIFT;
1349
1350	if (adapter->rx_buffer_len < 2048) {
1351		adapter->rx_ps_hdr_size = 0;
1352		srrctl |= E1000_SRRCTL_DESCTYPE_ADV_ONEBUF;
1353	} else {
1354		adapter->rx_ps_hdr_size = 128;
1355		srrctl |= adapter->rx_ps_hdr_size <<
1356			  E1000_SRRCTL_BSIZEHDRSIZE_SHIFT;
1357		srrctl |= E1000_SRRCTL_DESCTYPE_HDR_SPLIT_ALWAYS;
1358	}
1359
1360	ew32(SRRCTL(0), srrctl);
1361}
1362
1363/**
1364 * igbvf_configure_rx - Configure Receive Unit after Reset
1365 * @adapter: board private structure
1366 *
1367 * Configure the Rx unit of the MAC after a reset.
1368 **/
1369static void igbvf_configure_rx(struct igbvf_adapter *adapter)
1370{
1371	struct e1000_hw *hw = &adapter->hw;
1372	struct igbvf_ring *rx_ring = adapter->rx_ring;
1373	u64 rdba;
1374	u32 rxdctl;
1375
1376	/* disable receives */
1377	rxdctl = er32(RXDCTL(0));
1378	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1379	e1e_flush();
1380	msleep(10);
1381
1382	/* Setup the HW Rx Head and Tail Descriptor Pointers and
1383	 * the Base and Length of the Rx Descriptor Ring
1384	 */
1385	rdba = rx_ring->dma;
1386	ew32(RDBAL(0), (rdba & DMA_BIT_MASK(32)));
1387	ew32(RDBAH(0), (rdba >> 32));
1388	ew32(RDLEN(0), rx_ring->count * sizeof(union e1000_adv_rx_desc));
1389	rx_ring->head = E1000_RDH(0);
1390	rx_ring->tail = E1000_RDT(0);
1391	ew32(RDH(0), 0);
1392	ew32(RDT(0), 0);
1393
1394	rxdctl |= E1000_RXDCTL_QUEUE_ENABLE;
1395	rxdctl &= 0xFFF00000;
1396	rxdctl |= IGBVF_RX_PTHRESH;
1397	rxdctl |= IGBVF_RX_HTHRESH << 8;
1398	rxdctl |= IGBVF_RX_WTHRESH << 16;
1399
1400	igbvf_set_rlpml(adapter);
1401
1402	/* enable receives */
1403	ew32(RXDCTL(0), rxdctl);
1404}
1405
1406/**
1407 * igbvf_set_multi - Multicast and Promiscuous mode set
1408 * @netdev: network interface device structure
1409 *
1410 * The set_multi entry point is called whenever the multicast address
1411 * list or the network interface flags are updated.  This routine is
1412 * responsible for configuring the hardware for proper multicast,
1413 * promiscuous mode, and all-multi behavior.
1414 **/
1415static void igbvf_set_multi(struct net_device *netdev)
1416{
1417	struct igbvf_adapter *adapter = netdev_priv(netdev);
1418	struct e1000_hw *hw = &adapter->hw;
1419	struct netdev_hw_addr *ha;
1420	u8  *mta_list = NULL;
1421	int i;
1422
1423	if (!netdev_mc_empty(netdev)) {
1424		mta_list = kmalloc_array(netdev_mc_count(netdev), ETH_ALEN,
1425					 GFP_ATOMIC);
1426		if (!mta_list)
1427			return;
1428	}
1429
1430	/* prepare a packed array of only addresses. */
1431	i = 0;
1432	netdev_for_each_mc_addr(ha, netdev)
1433		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
1434
1435	spin_lock_bh(&hw->mbx_lock);
1436
1437	hw->mac.ops.update_mc_addr_list(hw, mta_list, i, 0, 0);
1438
1439	spin_unlock_bh(&hw->mbx_lock);
1440	kfree(mta_list);
1441}
1442
1443/**
1444 * igbvf_set_uni - Configure unicast MAC filters
1445 * @netdev: network interface device structure
1446 *
1447 * This routine is responsible for configuring the hardware for proper
1448 * unicast filters.
1449 **/
1450static int igbvf_set_uni(struct net_device *netdev)
1451{
1452	struct igbvf_adapter *adapter = netdev_priv(netdev);
1453	struct e1000_hw *hw = &adapter->hw;
1454
1455	if (netdev_uc_count(netdev) > IGBVF_MAX_MAC_FILTERS) {
1456		pr_err("Too many unicast filters - No Space\n");
1457		return -ENOSPC;
1458	}
1459
1460	spin_lock_bh(&hw->mbx_lock);
1461
1462	/* Clear all unicast MAC filters */
1463	hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_CLR, NULL);
1464
1465	spin_unlock_bh(&hw->mbx_lock);
1466
1467	if (!netdev_uc_empty(netdev)) {
1468		struct netdev_hw_addr *ha;
1469
1470		/* Add MAC filters one by one */
1471		netdev_for_each_uc_addr(ha, netdev) {
1472			spin_lock_bh(&hw->mbx_lock);
1473
1474			hw->mac.ops.set_uc_addr(hw, E1000_VF_MAC_FILTER_ADD,
1475						ha->addr);
1476
1477			spin_unlock_bh(&hw->mbx_lock);
1478			udelay(200);
1479		}
1480	}
1481
1482	return 0;
1483}
1484
1485static void igbvf_set_rx_mode(struct net_device *netdev)
1486{
1487	igbvf_set_multi(netdev);
1488	igbvf_set_uni(netdev);
1489}
1490
1491/**
1492 * igbvf_configure - configure the hardware for Rx and Tx
1493 * @adapter: private board structure
1494 **/
1495static void igbvf_configure(struct igbvf_adapter *adapter)
1496{
1497	igbvf_set_rx_mode(adapter->netdev);
1498
1499	igbvf_restore_vlan(adapter);
1500
1501	igbvf_configure_tx(adapter);
1502	igbvf_setup_srrctl(adapter);
1503	igbvf_configure_rx(adapter);
1504	igbvf_alloc_rx_buffers(adapter->rx_ring,
1505			       igbvf_desc_unused(adapter->rx_ring));
1506}
1507
1508/* igbvf_reset - bring the hardware into a known good state
1509 * @adapter: private board structure
1510 *
1511 * This function boots the hardware and enables some settings that
1512 * require a configuration cycle of the hardware - those cannot be
1513 * set/changed during runtime. After reset the device needs to be
1514 * properly configured for Rx, Tx etc.
1515 */
1516static void igbvf_reset(struct igbvf_adapter *adapter)
1517{
1518	struct e1000_mac_info *mac = &adapter->hw.mac;
1519	struct net_device *netdev = adapter->netdev;
1520	struct e1000_hw *hw = &adapter->hw;
1521
1522	spin_lock_bh(&hw->mbx_lock);
1523
1524	/* Allow time for pending master requests to run */
1525	if (mac->ops.reset_hw(hw))
1526		dev_info(&adapter->pdev->dev, "PF still resetting\n");
1527
1528	mac->ops.init_hw(hw);
1529
1530	spin_unlock_bh(&hw->mbx_lock);
1531
1532	if (is_valid_ether_addr(adapter->hw.mac.addr)) {
1533		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
 
1534		memcpy(netdev->perm_addr, adapter->hw.mac.addr,
1535		       netdev->addr_len);
1536	}
1537
1538	adapter->last_reset = jiffies;
1539}
1540
1541int igbvf_up(struct igbvf_adapter *adapter)
1542{
1543	struct e1000_hw *hw = &adapter->hw;
1544
1545	/* hardware has been reset, we need to reload some things */
1546	igbvf_configure(adapter);
1547
1548	clear_bit(__IGBVF_DOWN, &adapter->state);
1549
1550	napi_enable(&adapter->rx_ring->napi);
1551	if (adapter->msix_entries)
1552		igbvf_configure_msix(adapter);
1553
1554	/* Clear any pending interrupts. */
1555	er32(EICR);
1556	igbvf_irq_enable(adapter);
1557
1558	/* start the watchdog */
1559	hw->mac.get_link_status = 1;
1560	mod_timer(&adapter->watchdog_timer, jiffies + 1);
1561
1562	return 0;
1563}
1564
1565void igbvf_down(struct igbvf_adapter *adapter)
1566{
1567	struct net_device *netdev = adapter->netdev;
1568	struct e1000_hw *hw = &adapter->hw;
1569	u32 rxdctl, txdctl;
1570
1571	/* signal that we're down so the interrupt handler does not
1572	 * reschedule our watchdog timer
1573	 */
1574	set_bit(__IGBVF_DOWN, &adapter->state);
1575
1576	/* disable receives in the hardware */
1577	rxdctl = er32(RXDCTL(0));
1578	ew32(RXDCTL(0), rxdctl & ~E1000_RXDCTL_QUEUE_ENABLE);
1579
1580	netif_carrier_off(netdev);
1581	netif_stop_queue(netdev);
1582
1583	/* disable transmits in the hardware */
1584	txdctl = er32(TXDCTL(0));
1585	ew32(TXDCTL(0), txdctl & ~E1000_TXDCTL_QUEUE_ENABLE);
1586
1587	/* flush both disables and wait for them to finish */
1588	e1e_flush();
1589	msleep(10);
1590
1591	napi_disable(&adapter->rx_ring->napi);
1592
1593	igbvf_irq_disable(adapter);
1594
1595	del_timer_sync(&adapter->watchdog_timer);
1596
1597	/* record the stats before reset*/
1598	igbvf_update_stats(adapter);
1599
1600	adapter->link_speed = 0;
1601	adapter->link_duplex = 0;
1602
1603	igbvf_reset(adapter);
1604	igbvf_clean_tx_ring(adapter->tx_ring);
1605	igbvf_clean_rx_ring(adapter->rx_ring);
1606}
1607
1608void igbvf_reinit_locked(struct igbvf_adapter *adapter)
1609{
1610	might_sleep();
1611	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
1612		usleep_range(1000, 2000);
1613	igbvf_down(adapter);
1614	igbvf_up(adapter);
1615	clear_bit(__IGBVF_RESETTING, &adapter->state);
1616}
1617
1618/**
1619 * igbvf_sw_init - Initialize general software structures (struct igbvf_adapter)
1620 * @adapter: board private structure to initialize
1621 *
1622 * igbvf_sw_init initializes the Adapter private data structure.
1623 * Fields are initialized based on PCI device information and
1624 * OS network device settings (MTU size).
1625 **/
1626static int igbvf_sw_init(struct igbvf_adapter *adapter)
1627{
1628	struct net_device *netdev = adapter->netdev;
1629	s32 rc;
1630
1631	adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN;
1632	adapter->rx_ps_hdr_size = 0;
1633	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN;
1634	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
1635
1636	adapter->tx_int_delay = 8;
1637	adapter->tx_abs_int_delay = 32;
1638	adapter->rx_int_delay = 0;
1639	adapter->rx_abs_int_delay = 8;
1640	adapter->requested_itr = 3;
1641	adapter->current_itr = IGBVF_START_ITR;
1642
1643	/* Set various function pointers */
1644	adapter->ei->init_ops(&adapter->hw);
1645
1646	rc = adapter->hw.mac.ops.init_params(&adapter->hw);
1647	if (rc)
1648		return rc;
1649
1650	rc = adapter->hw.mbx.ops.init_params(&adapter->hw);
1651	if (rc)
1652		return rc;
1653
1654	igbvf_set_interrupt_capability(adapter);
1655
1656	if (igbvf_alloc_queues(adapter))
1657		return -ENOMEM;
1658
1659	spin_lock_init(&adapter->tx_queue_lock);
1660
1661	/* Explicitly disable IRQ since the NIC can be in any state. */
1662	igbvf_irq_disable(adapter);
1663
1664	spin_lock_init(&adapter->stats_lock);
1665	spin_lock_init(&adapter->hw.mbx_lock);
1666
1667	set_bit(__IGBVF_DOWN, &adapter->state);
1668	return 0;
1669}
1670
1671static void igbvf_initialize_last_counter_stats(struct igbvf_adapter *adapter)
1672{
1673	struct e1000_hw *hw = &adapter->hw;
1674
1675	adapter->stats.last_gprc = er32(VFGPRC);
1676	adapter->stats.last_gorc = er32(VFGORC);
1677	adapter->stats.last_gptc = er32(VFGPTC);
1678	adapter->stats.last_gotc = er32(VFGOTC);
1679	adapter->stats.last_mprc = er32(VFMPRC);
1680	adapter->stats.last_gotlbc = er32(VFGOTLBC);
1681	adapter->stats.last_gptlbc = er32(VFGPTLBC);
1682	adapter->stats.last_gorlbc = er32(VFGORLBC);
1683	adapter->stats.last_gprlbc = er32(VFGPRLBC);
1684
1685	adapter->stats.base_gprc = er32(VFGPRC);
1686	adapter->stats.base_gorc = er32(VFGORC);
1687	adapter->stats.base_gptc = er32(VFGPTC);
1688	adapter->stats.base_gotc = er32(VFGOTC);
1689	adapter->stats.base_mprc = er32(VFMPRC);
1690	adapter->stats.base_gotlbc = er32(VFGOTLBC);
1691	adapter->stats.base_gptlbc = er32(VFGPTLBC);
1692	adapter->stats.base_gorlbc = er32(VFGORLBC);
1693	adapter->stats.base_gprlbc = er32(VFGPRLBC);
1694}
1695
1696/**
1697 * igbvf_open - Called when a network interface is made active
1698 * @netdev: network interface device structure
1699 *
1700 * Returns 0 on success, negative value on failure
1701 *
1702 * The open entry point is called when a network interface is made
1703 * active by the system (IFF_UP).  At this point all resources needed
1704 * for transmit and receive operations are allocated, the interrupt
1705 * handler is registered with the OS, the watchdog timer is started,
1706 * and the stack is notified that the interface is ready.
1707 **/
1708static int igbvf_open(struct net_device *netdev)
1709{
1710	struct igbvf_adapter *adapter = netdev_priv(netdev);
1711	struct e1000_hw *hw = &adapter->hw;
1712	int err;
1713
1714	/* disallow open during test */
1715	if (test_bit(__IGBVF_TESTING, &adapter->state))
1716		return -EBUSY;
1717
1718	/* allocate transmit descriptors */
1719	err = igbvf_setup_tx_resources(adapter, adapter->tx_ring);
1720	if (err)
1721		goto err_setup_tx;
1722
1723	/* allocate receive descriptors */
1724	err = igbvf_setup_rx_resources(adapter, adapter->rx_ring);
1725	if (err)
1726		goto err_setup_rx;
1727
1728	/* before we allocate an interrupt, we must be ready to handle it.
1729	 * Setting DEBUG_SHIRQ in the kernel makes it fire an interrupt
1730	 * as soon as we call pci_request_irq, so we have to setup our
1731	 * clean_rx handler before we do so.
1732	 */
1733	igbvf_configure(adapter);
1734
1735	err = igbvf_request_irq(adapter);
1736	if (err)
1737		goto err_req_irq;
1738
1739	/* From here on the code is the same as igbvf_up() */
1740	clear_bit(__IGBVF_DOWN, &adapter->state);
1741
1742	napi_enable(&adapter->rx_ring->napi);
1743
1744	/* clear any pending interrupts */
1745	er32(EICR);
1746
1747	igbvf_irq_enable(adapter);
1748
1749	/* start the watchdog */
1750	hw->mac.get_link_status = 1;
1751	mod_timer(&adapter->watchdog_timer, jiffies + 1);
1752
1753	return 0;
1754
1755err_req_irq:
1756	igbvf_free_rx_resources(adapter->rx_ring);
1757err_setup_rx:
1758	igbvf_free_tx_resources(adapter->tx_ring);
1759err_setup_tx:
1760	igbvf_reset(adapter);
1761
1762	return err;
1763}
1764
1765/**
1766 * igbvf_close - Disables a network interface
1767 * @netdev: network interface device structure
1768 *
1769 * Returns 0, this is not allowed to fail
1770 *
1771 * The close entry point is called when an interface is de-activated
1772 * by the OS.  The hardware is still under the drivers control, but
1773 * needs to be disabled.  A global MAC reset is issued to stop the
1774 * hardware, and all transmit and receive resources are freed.
1775 **/
1776static int igbvf_close(struct net_device *netdev)
1777{
1778	struct igbvf_adapter *adapter = netdev_priv(netdev);
1779
1780	WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
1781	igbvf_down(adapter);
1782
1783	igbvf_free_irq(adapter);
1784
1785	igbvf_free_tx_resources(adapter->tx_ring);
1786	igbvf_free_rx_resources(adapter->rx_ring);
1787
1788	return 0;
1789}
1790
1791/**
1792 * igbvf_set_mac - Change the Ethernet Address of the NIC
1793 * @netdev: network interface device structure
1794 * @p: pointer to an address structure
1795 *
1796 * Returns 0 on success, negative on failure
1797 **/
1798static int igbvf_set_mac(struct net_device *netdev, void *p)
1799{
1800	struct igbvf_adapter *adapter = netdev_priv(netdev);
1801	struct e1000_hw *hw = &adapter->hw;
1802	struct sockaddr *addr = p;
1803
1804	if (!is_valid_ether_addr(addr->sa_data))
1805		return -EADDRNOTAVAIL;
1806
1807	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
1808
1809	spin_lock_bh(&hw->mbx_lock);
1810
1811	hw->mac.ops.rar_set(hw, hw->mac.addr, 0);
1812
1813	spin_unlock_bh(&hw->mbx_lock);
1814
1815	if (!ether_addr_equal(addr->sa_data, hw->mac.addr))
1816		return -EADDRNOTAVAIL;
1817
1818	eth_hw_addr_set(netdev, addr->sa_data);
1819
1820	return 0;
1821}
1822
1823#define UPDATE_VF_COUNTER(reg, name) \
1824{ \
1825	u32 current_counter = er32(reg); \
1826	if (current_counter < adapter->stats.last_##name) \
1827		adapter->stats.name += 0x100000000LL; \
1828	adapter->stats.last_##name = current_counter; \
1829	adapter->stats.name &= 0xFFFFFFFF00000000LL; \
1830	adapter->stats.name |= current_counter; \
1831}
1832
1833/**
1834 * igbvf_update_stats - Update the board statistics counters
1835 * @adapter: board private structure
1836**/
1837void igbvf_update_stats(struct igbvf_adapter *adapter)
1838{
1839	struct e1000_hw *hw = &adapter->hw;
1840	struct pci_dev *pdev = adapter->pdev;
1841
1842	/* Prevent stats update while adapter is being reset, link is down
1843	 * or if the pci connection is down.
1844	 */
1845	if (adapter->link_speed == 0)
1846		return;
1847
1848	if (test_bit(__IGBVF_RESETTING, &adapter->state))
1849		return;
1850
1851	if (pci_channel_offline(pdev))
1852		return;
1853
1854	UPDATE_VF_COUNTER(VFGPRC, gprc);
1855	UPDATE_VF_COUNTER(VFGORC, gorc);
1856	UPDATE_VF_COUNTER(VFGPTC, gptc);
1857	UPDATE_VF_COUNTER(VFGOTC, gotc);
1858	UPDATE_VF_COUNTER(VFMPRC, mprc);
1859	UPDATE_VF_COUNTER(VFGOTLBC, gotlbc);
1860	UPDATE_VF_COUNTER(VFGPTLBC, gptlbc);
1861	UPDATE_VF_COUNTER(VFGORLBC, gorlbc);
1862	UPDATE_VF_COUNTER(VFGPRLBC, gprlbc);
1863
1864	/* Fill out the OS statistics structure */
1865	adapter->netdev->stats.multicast = adapter->stats.mprc;
1866}
1867
1868static void igbvf_print_link_info(struct igbvf_adapter *adapter)
1869{
1870	dev_info(&adapter->pdev->dev, "Link is Up %d Mbps %s Duplex\n",
1871		 adapter->link_speed,
1872		 adapter->link_duplex == FULL_DUPLEX ? "Full" : "Half");
1873}
1874
1875static bool igbvf_has_link(struct igbvf_adapter *adapter)
1876{
1877	struct e1000_hw *hw = &adapter->hw;
1878	s32 ret_val = E1000_SUCCESS;
1879	bool link_active;
1880
1881	/* If interface is down, stay link down */
1882	if (test_bit(__IGBVF_DOWN, &adapter->state))
1883		return false;
1884
1885	spin_lock_bh(&hw->mbx_lock);
1886
1887	ret_val = hw->mac.ops.check_for_link(hw);
1888
1889	spin_unlock_bh(&hw->mbx_lock);
1890
1891	link_active = !hw->mac.get_link_status;
1892
1893	/* if check for link returns error we will need to reset */
1894	if (ret_val && time_after(jiffies, adapter->last_reset + (10 * HZ)))
1895		schedule_work(&adapter->reset_task);
1896
1897	return link_active;
1898}
1899
1900/**
1901 * igbvf_watchdog - Timer Call-back
1902 * @t: timer list pointer containing private struct
1903 **/
1904static void igbvf_watchdog(struct timer_list *t)
1905{
1906	struct igbvf_adapter *adapter = from_timer(adapter, t, watchdog_timer);
1907
1908	/* Do the rest outside of interrupt context */
1909	schedule_work(&adapter->watchdog_task);
1910}
1911
1912static void igbvf_watchdog_task(struct work_struct *work)
1913{
1914	struct igbvf_adapter *adapter = container_of(work,
1915						     struct igbvf_adapter,
1916						     watchdog_task);
1917	struct net_device *netdev = adapter->netdev;
1918	struct e1000_mac_info *mac = &adapter->hw.mac;
1919	struct igbvf_ring *tx_ring = adapter->tx_ring;
1920	struct e1000_hw *hw = &adapter->hw;
1921	u32 link;
1922	int tx_pending = 0;
1923
1924	link = igbvf_has_link(adapter);
1925
1926	if (link) {
1927		if (!netif_carrier_ok(netdev)) {
1928			mac->ops.get_link_up_info(&adapter->hw,
1929						  &adapter->link_speed,
1930						  &adapter->link_duplex);
1931			igbvf_print_link_info(adapter);
1932
1933			netif_carrier_on(netdev);
1934			netif_wake_queue(netdev);
1935		}
1936	} else {
1937		if (netif_carrier_ok(netdev)) {
1938			adapter->link_speed = 0;
1939			adapter->link_duplex = 0;
1940			dev_info(&adapter->pdev->dev, "Link is Down\n");
1941			netif_carrier_off(netdev);
1942			netif_stop_queue(netdev);
1943		}
1944	}
1945
1946	if (netif_carrier_ok(netdev)) {
1947		igbvf_update_stats(adapter);
1948	} else {
1949		tx_pending = (igbvf_desc_unused(tx_ring) + 1 <
1950			      tx_ring->count);
1951		if (tx_pending) {
1952			/* We've lost link, so the controller stops DMA,
1953			 * but we've got queued Tx work that's never going
1954			 * to get done, so reset controller to flush Tx.
1955			 * (Do the reset outside of interrupt context).
1956			 */
1957			adapter->tx_timeout_count++;
1958			schedule_work(&adapter->reset_task);
1959		}
1960	}
1961
1962	/* Cause software interrupt to ensure Rx ring is cleaned */
1963	ew32(EICS, adapter->rx_ring->eims_value);
1964
1965	/* Reset the timer */
1966	if (!test_bit(__IGBVF_DOWN, &adapter->state))
1967		mod_timer(&adapter->watchdog_timer,
1968			  round_jiffies(jiffies + (2 * HZ)));
1969}
1970
1971#define IGBVF_TX_FLAGS_CSUM		0x00000001
1972#define IGBVF_TX_FLAGS_VLAN		0x00000002
1973#define IGBVF_TX_FLAGS_TSO		0x00000004
1974#define IGBVF_TX_FLAGS_IPV4		0x00000008
1975#define IGBVF_TX_FLAGS_VLAN_MASK	0xffff0000
1976#define IGBVF_TX_FLAGS_VLAN_SHIFT	16
1977
1978static void igbvf_tx_ctxtdesc(struct igbvf_ring *tx_ring, u32 vlan_macip_lens,
1979			      u32 type_tucmd, u32 mss_l4len_idx)
1980{
1981	struct e1000_adv_tx_context_desc *context_desc;
1982	struct igbvf_buffer *buffer_info;
1983	u16 i = tx_ring->next_to_use;
1984
1985	context_desc = IGBVF_TX_CTXTDESC_ADV(*tx_ring, i);
1986	buffer_info = &tx_ring->buffer_info[i];
1987
1988	i++;
1989	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1990
1991	/* set bits to identify this as an advanced context descriptor */
1992	type_tucmd |= E1000_TXD_CMD_DEXT | E1000_ADVTXD_DTYP_CTXT;
1993
1994	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
1995	context_desc->seqnum_seed	= 0;
1996	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
1997	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
1998
1999	buffer_info->time_stamp = jiffies;
2000	buffer_info->dma = 0;
2001}
2002
2003static int igbvf_tso(struct igbvf_ring *tx_ring,
2004		     struct sk_buff *skb, u32 tx_flags, u8 *hdr_len)
2005{
2006	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
2007	union {
2008		struct iphdr *v4;
2009		struct ipv6hdr *v6;
2010		unsigned char *hdr;
2011	} ip;
2012	union {
2013		struct tcphdr *tcp;
2014		unsigned char *hdr;
2015	} l4;
2016	u32 paylen, l4_offset;
2017	int err;
2018
2019	if (skb->ip_summed != CHECKSUM_PARTIAL)
2020		return 0;
2021
2022	if (!skb_is_gso(skb))
2023		return 0;
2024
2025	err = skb_cow_head(skb, 0);
2026	if (err < 0)
2027		return err;
2028
2029	ip.hdr = skb_network_header(skb);
2030	l4.hdr = skb_checksum_start(skb);
2031
2032	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
2033	type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2034
2035	/* initialize outer IP header fields */
2036	if (ip.v4->version == 4) {
2037		unsigned char *csum_start = skb_checksum_start(skb);
2038		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
2039
2040		/* IP header will have to cancel out any data that
2041		 * is not a part of the outer IP header
2042		 */
2043		ip.v4->check = csum_fold(csum_partial(trans_start,
2044						      csum_start - trans_start,
2045						      0));
2046		type_tucmd |= E1000_ADVTXD_TUCMD_IPV4;
2047
2048		ip.v4->tot_len = 0;
2049	} else {
2050		ip.v6->payload_len = 0;
2051	}
2052
2053	/* determine offset of inner transport header */
2054	l4_offset = l4.hdr - skb->data;
2055
2056	/* compute length of segmentation header */
2057	*hdr_len = (l4.tcp->doff * 4) + l4_offset;
2058
2059	/* remove payload length from inner checksum */
2060	paylen = skb->len - l4_offset;
2061	csum_replace_by_diff(&l4.tcp->check, (__force __wsum)htonl(paylen));
2062
2063	/* MSS L4LEN IDX */
2064	mss_l4len_idx = (*hdr_len - l4_offset) << E1000_ADVTXD_L4LEN_SHIFT;
2065	mss_l4len_idx |= skb_shinfo(skb)->gso_size << E1000_ADVTXD_MSS_SHIFT;
2066
2067	/* VLAN MACLEN IPLEN */
2068	vlan_macip_lens = l4.hdr - ip.hdr;
2069	vlan_macip_lens |= (ip.hdr - skb->data) << E1000_ADVTXD_MACLEN_SHIFT;
2070	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2071
2072	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, mss_l4len_idx);
2073
2074	return 1;
2075}
2076
 
 
 
 
 
 
 
 
 
2077static bool igbvf_tx_csum(struct igbvf_ring *tx_ring, struct sk_buff *skb,
2078			  u32 tx_flags, __be16 protocol)
2079{
2080	u32 vlan_macip_lens = 0;
2081	u32 type_tucmd = 0;
2082
2083	if (skb->ip_summed != CHECKSUM_PARTIAL) {
2084csum_failed:
2085		if (!(tx_flags & IGBVF_TX_FLAGS_VLAN))
2086			return false;
2087		goto no_csum;
2088	}
2089
2090	switch (skb->csum_offset) {
2091	case offsetof(struct tcphdr, check):
2092		type_tucmd = E1000_ADVTXD_TUCMD_L4T_TCP;
2093		fallthrough;
2094	case offsetof(struct udphdr, check):
2095		break;
2096	case offsetof(struct sctphdr, checksum):
2097		/* validate that this is actually an SCTP request */
2098		if (skb_csum_is_sctp(skb)) {
 
 
 
2099			type_tucmd = E1000_ADVTXD_TUCMD_L4T_SCTP;
2100			break;
2101		}
2102		fallthrough;
2103	default:
2104		skb_checksum_help(skb);
2105		goto csum_failed;
2106	}
2107
2108	vlan_macip_lens = skb_checksum_start_offset(skb) -
2109			  skb_network_offset(skb);
2110no_csum:
2111	vlan_macip_lens |= skb_network_offset(skb) << E1000_ADVTXD_MACLEN_SHIFT;
2112	vlan_macip_lens |= tx_flags & IGBVF_TX_FLAGS_VLAN_MASK;
2113
2114	igbvf_tx_ctxtdesc(tx_ring, vlan_macip_lens, type_tucmd, 0);
2115	return true;
2116}
2117
2118static int igbvf_maybe_stop_tx(struct net_device *netdev, int size)
2119{
2120	struct igbvf_adapter *adapter = netdev_priv(netdev);
2121
2122	/* there is enough descriptors then we don't need to worry  */
2123	if (igbvf_desc_unused(adapter->tx_ring) >= size)
2124		return 0;
2125
2126	netif_stop_queue(netdev);
2127
2128	/* Herbert's original patch had:
2129	 *  smp_mb__after_netif_stop_queue();
2130	 * but since that doesn't exist yet, just open code it.
2131	 */
2132	smp_mb();
2133
2134	/* We need to check again just in case room has been made available */
2135	if (igbvf_desc_unused(adapter->tx_ring) < size)
2136		return -EBUSY;
2137
2138	netif_wake_queue(netdev);
2139
2140	++adapter->restart_queue;
2141	return 0;
2142}
2143
2144#define IGBVF_MAX_TXD_PWR	16
2145#define IGBVF_MAX_DATA_PER_TXD	(1u << IGBVF_MAX_TXD_PWR)
2146
2147static inline int igbvf_tx_map_adv(struct igbvf_adapter *adapter,
2148				   struct igbvf_ring *tx_ring,
2149				   struct sk_buff *skb)
2150{
2151	struct igbvf_buffer *buffer_info;
2152	struct pci_dev *pdev = adapter->pdev;
2153	unsigned int len = skb_headlen(skb);
2154	unsigned int count = 0, i;
2155	unsigned int f;
2156
2157	i = tx_ring->next_to_use;
2158
2159	buffer_info = &tx_ring->buffer_info[i];
2160	BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2161	buffer_info->length = len;
2162	/* set time_stamp *before* dma to help avoid a possible race */
2163	buffer_info->time_stamp = jiffies;
2164	buffer_info->mapped_as_page = false;
2165	buffer_info->dma = dma_map_single(&pdev->dev, skb->data, len,
2166					  DMA_TO_DEVICE);
2167	if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2168		goto dma_error;
2169
2170	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
2171		const skb_frag_t *frag;
2172
2173		count++;
2174		i++;
2175		if (i == tx_ring->count)
2176			i = 0;
2177
2178		frag = &skb_shinfo(skb)->frags[f];
2179		len = skb_frag_size(frag);
2180
2181		buffer_info = &tx_ring->buffer_info[i];
2182		BUG_ON(len >= IGBVF_MAX_DATA_PER_TXD);
2183		buffer_info->length = len;
2184		buffer_info->time_stamp = jiffies;
2185		buffer_info->mapped_as_page = true;
2186		buffer_info->dma = skb_frag_dma_map(&pdev->dev, frag, 0, len,
2187						    DMA_TO_DEVICE);
2188		if (dma_mapping_error(&pdev->dev, buffer_info->dma))
2189			goto dma_error;
2190	}
2191
2192	tx_ring->buffer_info[i].skb = skb;
2193
2194	return ++count;
2195
2196dma_error:
2197	dev_err(&pdev->dev, "TX DMA map failed\n");
2198
2199	/* clear timestamp and dma mappings for failed buffer_info mapping */
2200	buffer_info->dma = 0;
2201	buffer_info->time_stamp = 0;
2202	buffer_info->length = 0;
2203	buffer_info->mapped_as_page = false;
2204	if (count)
2205		count--;
2206
2207	/* clear timestamp and dma mappings for remaining portion of packet */
2208	while (count--) {
2209		if (i == 0)
2210			i += tx_ring->count;
2211		i--;
2212		buffer_info = &tx_ring->buffer_info[i];
2213		igbvf_put_txbuf(adapter, buffer_info);
2214	}
2215
2216	return 0;
2217}
2218
2219static inline void igbvf_tx_queue_adv(struct igbvf_adapter *adapter,
2220				      struct igbvf_ring *tx_ring,
2221				      int tx_flags, int count,
2222				      unsigned int first, u32 paylen,
2223				      u8 hdr_len)
2224{
2225	union e1000_adv_tx_desc *tx_desc = NULL;
2226	struct igbvf_buffer *buffer_info;
2227	u32 olinfo_status = 0, cmd_type_len;
2228	unsigned int i;
2229
2230	cmd_type_len = (E1000_ADVTXD_DTYP_DATA | E1000_ADVTXD_DCMD_IFCS |
2231			E1000_ADVTXD_DCMD_DEXT);
2232
2233	if (tx_flags & IGBVF_TX_FLAGS_VLAN)
2234		cmd_type_len |= E1000_ADVTXD_DCMD_VLE;
2235
2236	if (tx_flags & IGBVF_TX_FLAGS_TSO) {
2237		cmd_type_len |= E1000_ADVTXD_DCMD_TSE;
2238
2239		/* insert tcp checksum */
2240		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2241
2242		/* insert ip checksum */
2243		if (tx_flags & IGBVF_TX_FLAGS_IPV4)
2244			olinfo_status |= E1000_TXD_POPTS_IXSM << 8;
2245
2246	} else if (tx_flags & IGBVF_TX_FLAGS_CSUM) {
2247		olinfo_status |= E1000_TXD_POPTS_TXSM << 8;
2248	}
2249
2250	olinfo_status |= ((paylen - hdr_len) << E1000_ADVTXD_PAYLEN_SHIFT);
2251
2252	i = tx_ring->next_to_use;
2253	while (count--) {
2254		buffer_info = &tx_ring->buffer_info[i];
2255		tx_desc = IGBVF_TX_DESC_ADV(*tx_ring, i);
2256		tx_desc->read.buffer_addr = cpu_to_le64(buffer_info->dma);
2257		tx_desc->read.cmd_type_len =
2258			 cpu_to_le32(cmd_type_len | buffer_info->length);
2259		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2260		i++;
2261		if (i == tx_ring->count)
2262			i = 0;
2263	}
2264
2265	tx_desc->read.cmd_type_len |= cpu_to_le32(adapter->txd_cmd);
2266	/* Force memory writes to complete before letting h/w
2267	 * know there are new descriptors to fetch.  (Only
2268	 * applicable for weak-ordered memory model archs,
2269	 * such as IA-64).
2270	 */
2271	wmb();
2272
2273	tx_ring->buffer_info[first].next_to_watch = tx_desc;
2274	tx_ring->next_to_use = i;
2275	writel(i, adapter->hw.hw_addr + tx_ring->tail);
2276}
2277
2278static netdev_tx_t igbvf_xmit_frame_ring_adv(struct sk_buff *skb,
2279					     struct net_device *netdev,
2280					     struct igbvf_ring *tx_ring)
2281{
2282	struct igbvf_adapter *adapter = netdev_priv(netdev);
2283	unsigned int first, tx_flags = 0;
2284	u8 hdr_len = 0;
2285	int count = 0;
2286	int tso = 0;
2287	__be16 protocol = vlan_get_protocol(skb);
2288
2289	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2290		dev_kfree_skb_any(skb);
2291		return NETDEV_TX_OK;
2292	}
2293
2294	if (skb->len <= 0) {
2295		dev_kfree_skb_any(skb);
2296		return NETDEV_TX_OK;
2297	}
2298
2299	/* need: count + 4 desc gap to keep tail from touching
2300	 *       + 2 desc gap to keep tail from touching head,
2301	 *       + 1 desc for skb->data,
2302	 *       + 1 desc for context descriptor,
2303	 * head, otherwise try next time
2304	 */
2305	if (igbvf_maybe_stop_tx(netdev, skb_shinfo(skb)->nr_frags + 4)) {
2306		/* this is a hard error */
2307		return NETDEV_TX_BUSY;
2308	}
2309
2310	if (skb_vlan_tag_present(skb)) {
2311		tx_flags |= IGBVF_TX_FLAGS_VLAN;
2312		tx_flags |= (skb_vlan_tag_get(skb) <<
2313			     IGBVF_TX_FLAGS_VLAN_SHIFT);
2314	}
2315
2316	if (protocol == htons(ETH_P_IP))
2317		tx_flags |= IGBVF_TX_FLAGS_IPV4;
2318
2319	first = tx_ring->next_to_use;
2320
2321	tso = igbvf_tso(tx_ring, skb, tx_flags, &hdr_len);
2322	if (unlikely(tso < 0)) {
2323		dev_kfree_skb_any(skb);
2324		return NETDEV_TX_OK;
2325	}
2326
2327	if (tso)
2328		tx_flags |= IGBVF_TX_FLAGS_TSO;
2329	else if (igbvf_tx_csum(tx_ring, skb, tx_flags, protocol) &&
2330		 (skb->ip_summed == CHECKSUM_PARTIAL))
2331		tx_flags |= IGBVF_TX_FLAGS_CSUM;
2332
2333	/* count reflects descriptors mapped, if 0 then mapping error
2334	 * has occurred and we need to rewind the descriptor queue
2335	 */
2336	count = igbvf_tx_map_adv(adapter, tx_ring, skb);
2337
2338	if (count) {
2339		igbvf_tx_queue_adv(adapter, tx_ring, tx_flags, count,
2340				   first, skb->len, hdr_len);
2341		/* Make sure there is space in the ring for the next send. */
2342		igbvf_maybe_stop_tx(netdev, MAX_SKB_FRAGS + 4);
2343	} else {
2344		dev_kfree_skb_any(skb);
2345		tx_ring->buffer_info[first].time_stamp = 0;
2346		tx_ring->next_to_use = first;
2347	}
2348
2349	return NETDEV_TX_OK;
2350}
2351
2352static netdev_tx_t igbvf_xmit_frame(struct sk_buff *skb,
2353				    struct net_device *netdev)
2354{
2355	struct igbvf_adapter *adapter = netdev_priv(netdev);
2356	struct igbvf_ring *tx_ring;
2357
2358	if (test_bit(__IGBVF_DOWN, &adapter->state)) {
2359		dev_kfree_skb_any(skb);
2360		return NETDEV_TX_OK;
2361	}
2362
2363	tx_ring = &adapter->tx_ring[0];
2364
2365	return igbvf_xmit_frame_ring_adv(skb, netdev, tx_ring);
2366}
2367
2368/**
2369 * igbvf_tx_timeout - Respond to a Tx Hang
2370 * @netdev: network interface device structure
2371 * @txqueue: queue timing out (unused)
2372 **/
2373static void igbvf_tx_timeout(struct net_device *netdev, unsigned int __always_unused txqueue)
2374{
2375	struct igbvf_adapter *adapter = netdev_priv(netdev);
2376
2377	/* Do the reset outside of interrupt context */
2378	adapter->tx_timeout_count++;
2379	schedule_work(&adapter->reset_task);
2380}
2381
2382static void igbvf_reset_task(struct work_struct *work)
2383{
2384	struct igbvf_adapter *adapter;
2385
2386	adapter = container_of(work, struct igbvf_adapter, reset_task);
2387
2388	igbvf_reinit_locked(adapter);
2389}
2390
2391/**
2392 * igbvf_change_mtu - Change the Maximum Transfer Unit
2393 * @netdev: network interface device structure
2394 * @new_mtu: new value for maximum frame size
2395 *
2396 * Returns 0 on success, negative on failure
2397 **/
2398static int igbvf_change_mtu(struct net_device *netdev, int new_mtu)
2399{
2400	struct igbvf_adapter *adapter = netdev_priv(netdev);
2401	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN;
2402
2403	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2404		usleep_range(1000, 2000);
2405	/* igbvf_down has a dependency on max_frame_size */
2406	adapter->max_frame_size = max_frame;
2407	if (netif_running(netdev))
2408		igbvf_down(adapter);
2409
2410	/* NOTE: netdev_alloc_skb reserves 16 bytes, and typically NET_IP_ALIGN
2411	 * means we reserve 2 more, this pushes us to allocate from the next
2412	 * larger slab size.
2413	 * i.e. RXBUFFER_2048 --> size-4096 slab
2414	 * However with the new *_jumbo_rx* routines, jumbo receives will use
2415	 * fragmented skbs
2416	 */
2417
2418	if (max_frame <= 1024)
2419		adapter->rx_buffer_len = 1024;
2420	else if (max_frame <= 2048)
2421		adapter->rx_buffer_len = 2048;
2422	else
2423#if (PAGE_SIZE / 2) > 16384
2424		adapter->rx_buffer_len = 16384;
2425#else
2426		adapter->rx_buffer_len = PAGE_SIZE / 2;
2427#endif
2428
2429	/* adjust allocation if LPE protects us, and we aren't using SBP */
2430	if ((max_frame == ETH_FRAME_LEN + ETH_FCS_LEN) ||
2431	    (max_frame == ETH_FRAME_LEN + VLAN_HLEN + ETH_FCS_LEN))
2432		adapter->rx_buffer_len = ETH_FRAME_LEN + VLAN_HLEN +
2433					 ETH_FCS_LEN;
2434
2435	netdev_dbg(netdev, "changing MTU from %d to %d\n",
2436		   netdev->mtu, new_mtu);
2437	netdev->mtu = new_mtu;
2438
2439	if (netif_running(netdev))
2440		igbvf_up(adapter);
2441	else
2442		igbvf_reset(adapter);
2443
2444	clear_bit(__IGBVF_RESETTING, &adapter->state);
2445
2446	return 0;
2447}
2448
2449static int igbvf_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2450{
2451	switch (cmd) {
2452	default:
2453		return -EOPNOTSUPP;
2454	}
2455}
2456
2457static int igbvf_suspend(struct device *dev_d)
2458{
2459	struct net_device *netdev = dev_get_drvdata(dev_d);
2460	struct igbvf_adapter *adapter = netdev_priv(netdev);
 
 
 
2461
2462	netif_device_detach(netdev);
2463
2464	if (netif_running(netdev)) {
2465		WARN_ON(test_bit(__IGBVF_RESETTING, &adapter->state));
2466		igbvf_down(adapter);
2467		igbvf_free_irq(adapter);
2468	}
2469
 
 
 
 
 
 
 
 
2470	return 0;
2471}
2472
2473static int __maybe_unused igbvf_resume(struct device *dev_d)
 
2474{
2475	struct pci_dev *pdev = to_pci_dev(dev_d);
2476	struct net_device *netdev = pci_get_drvdata(pdev);
2477	struct igbvf_adapter *adapter = netdev_priv(netdev);
2478	u32 err;
2479
 
 
 
 
 
 
 
2480	pci_set_master(pdev);
2481
2482	if (netif_running(netdev)) {
2483		err = igbvf_request_irq(adapter);
2484		if (err)
2485			return err;
2486	}
2487
2488	igbvf_reset(adapter);
2489
2490	if (netif_running(netdev))
2491		igbvf_up(adapter);
2492
2493	netif_device_attach(netdev);
2494
2495	return 0;
2496}
 
2497
2498static void igbvf_shutdown(struct pci_dev *pdev)
2499{
2500	igbvf_suspend(&pdev->dev);
2501}
2502
2503#ifdef CONFIG_NET_POLL_CONTROLLER
2504/* Polling 'interrupt' - used by things like netconsole to send skbs
2505 * without having to re-enable interrupts. It's not called while
2506 * the interrupt routine is executing.
2507 */
2508static void igbvf_netpoll(struct net_device *netdev)
2509{
2510	struct igbvf_adapter *adapter = netdev_priv(netdev);
2511
2512	disable_irq(adapter->pdev->irq);
2513
2514	igbvf_clean_tx_irq(adapter->tx_ring);
2515
2516	enable_irq(adapter->pdev->irq);
2517}
2518#endif
2519
2520/**
2521 * igbvf_io_error_detected - called when PCI error is detected
2522 * @pdev: Pointer to PCI device
2523 * @state: The current pci connection state
2524 *
2525 * This function is called after a PCI bus error affecting
2526 * this device has been detected.
2527 */
2528static pci_ers_result_t igbvf_io_error_detected(struct pci_dev *pdev,
2529						pci_channel_state_t state)
2530{
2531	struct net_device *netdev = pci_get_drvdata(pdev);
2532	struct igbvf_adapter *adapter = netdev_priv(netdev);
2533
2534	netif_device_detach(netdev);
2535
2536	if (state == pci_channel_io_perm_failure)
2537		return PCI_ERS_RESULT_DISCONNECT;
2538
2539	if (netif_running(netdev))
2540		igbvf_down(adapter);
2541	pci_disable_device(pdev);
2542
2543	/* Request a slot reset. */
2544	return PCI_ERS_RESULT_NEED_RESET;
2545}
2546
2547/**
2548 * igbvf_io_slot_reset - called after the pci bus has been reset.
2549 * @pdev: Pointer to PCI device
2550 *
2551 * Restart the card from scratch, as if from a cold-boot. Implementation
2552 * resembles the first-half of the igbvf_resume routine.
2553 */
2554static pci_ers_result_t igbvf_io_slot_reset(struct pci_dev *pdev)
2555{
2556	struct net_device *netdev = pci_get_drvdata(pdev);
2557	struct igbvf_adapter *adapter = netdev_priv(netdev);
2558
2559	if (pci_enable_device_mem(pdev)) {
2560		dev_err(&pdev->dev,
2561			"Cannot re-enable PCI device after reset.\n");
2562		return PCI_ERS_RESULT_DISCONNECT;
2563	}
2564	pci_set_master(pdev);
2565
2566	igbvf_reset(adapter);
2567
2568	return PCI_ERS_RESULT_RECOVERED;
2569}
2570
2571/**
2572 * igbvf_io_resume - called when traffic can start flowing again.
2573 * @pdev: Pointer to PCI device
2574 *
2575 * This callback is called when the error recovery driver tells us that
2576 * its OK to resume normal operation. Implementation resembles the
2577 * second-half of the igbvf_resume routine.
2578 */
2579static void igbvf_io_resume(struct pci_dev *pdev)
2580{
2581	struct net_device *netdev = pci_get_drvdata(pdev);
2582	struct igbvf_adapter *adapter = netdev_priv(netdev);
2583
2584	if (netif_running(netdev)) {
2585		if (igbvf_up(adapter)) {
2586			dev_err(&pdev->dev,
2587				"can't bring device back up after reset\n");
2588			return;
2589		}
2590	}
2591
2592	netif_device_attach(netdev);
2593}
2594
2595/**
2596 * igbvf_io_prepare - prepare device driver for PCI reset
2597 * @pdev: PCI device information struct
2598 */
2599static void igbvf_io_prepare(struct pci_dev *pdev)
2600{
2601	struct net_device *netdev = pci_get_drvdata(pdev);
2602	struct igbvf_adapter *adapter = netdev_priv(netdev);
2603
2604	while (test_and_set_bit(__IGBVF_RESETTING, &adapter->state))
2605		usleep_range(1000, 2000);
2606	igbvf_down(adapter);
2607}
2608
2609/**
2610 * igbvf_io_reset_done - PCI reset done, device driver reset can begin
2611 * @pdev: PCI device information struct
2612 */
2613static void igbvf_io_reset_done(struct pci_dev *pdev)
2614{
2615	struct net_device *netdev = pci_get_drvdata(pdev);
2616	struct igbvf_adapter *adapter = netdev_priv(netdev);
2617
2618	igbvf_up(adapter);
2619	clear_bit(__IGBVF_RESETTING, &adapter->state);
2620}
2621
2622static void igbvf_print_device_info(struct igbvf_adapter *adapter)
2623{
2624	struct e1000_hw *hw = &adapter->hw;
2625	struct net_device *netdev = adapter->netdev;
2626	struct pci_dev *pdev = adapter->pdev;
2627
2628	if (hw->mac.type == e1000_vfadapt_i350)
2629		dev_info(&pdev->dev, "Intel(R) I350 Virtual Function\n");
2630	else
2631		dev_info(&pdev->dev, "Intel(R) 82576 Virtual Function\n");
2632	dev_info(&pdev->dev, "Address: %pM\n", netdev->dev_addr);
2633}
2634
2635static int igbvf_set_features(struct net_device *netdev,
2636			      netdev_features_t features)
2637{
2638	struct igbvf_adapter *adapter = netdev_priv(netdev);
2639
2640	if (features & NETIF_F_RXCSUM)
2641		adapter->flags &= ~IGBVF_FLAG_RX_CSUM_DISABLED;
2642	else
2643		adapter->flags |= IGBVF_FLAG_RX_CSUM_DISABLED;
2644
2645	return 0;
2646}
2647
2648#define IGBVF_MAX_MAC_HDR_LEN		127
2649#define IGBVF_MAX_NETWORK_HDR_LEN	511
2650
2651static netdev_features_t
2652igbvf_features_check(struct sk_buff *skb, struct net_device *dev,
2653		     netdev_features_t features)
2654{
2655	unsigned int network_hdr_len, mac_hdr_len;
2656
2657	/* Make certain the headers can be described by a context descriptor */
2658	mac_hdr_len = skb_network_header(skb) - skb->data;
2659	if (unlikely(mac_hdr_len > IGBVF_MAX_MAC_HDR_LEN))
2660		return features & ~(NETIF_F_HW_CSUM |
2661				    NETIF_F_SCTP_CRC |
2662				    NETIF_F_HW_VLAN_CTAG_TX |
2663				    NETIF_F_TSO |
2664				    NETIF_F_TSO6);
2665
2666	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
2667	if (unlikely(network_hdr_len >  IGBVF_MAX_NETWORK_HDR_LEN))
2668		return features & ~(NETIF_F_HW_CSUM |
2669				    NETIF_F_SCTP_CRC |
2670				    NETIF_F_TSO |
2671				    NETIF_F_TSO6);
2672
2673	/* We can only support IPV4 TSO in tunnels if we can mangle the
2674	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
2675	 */
2676	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
2677		features &= ~NETIF_F_TSO;
2678
2679	return features;
2680}
2681
2682static const struct net_device_ops igbvf_netdev_ops = {
2683	.ndo_open		= igbvf_open,
2684	.ndo_stop		= igbvf_close,
2685	.ndo_start_xmit		= igbvf_xmit_frame,
2686	.ndo_set_rx_mode	= igbvf_set_rx_mode,
2687	.ndo_set_mac_address	= igbvf_set_mac,
2688	.ndo_change_mtu		= igbvf_change_mtu,
2689	.ndo_eth_ioctl		= igbvf_ioctl,
2690	.ndo_tx_timeout		= igbvf_tx_timeout,
2691	.ndo_vlan_rx_add_vid	= igbvf_vlan_rx_add_vid,
2692	.ndo_vlan_rx_kill_vid	= igbvf_vlan_rx_kill_vid,
2693#ifdef CONFIG_NET_POLL_CONTROLLER
2694	.ndo_poll_controller	= igbvf_netpoll,
2695#endif
2696	.ndo_set_features	= igbvf_set_features,
2697	.ndo_features_check	= igbvf_features_check,
2698};
2699
2700/**
2701 * igbvf_probe - Device Initialization Routine
2702 * @pdev: PCI device information struct
2703 * @ent: entry in igbvf_pci_tbl
2704 *
2705 * Returns 0 on success, negative on failure
2706 *
2707 * igbvf_probe initializes an adapter identified by a pci_dev structure.
2708 * The OS initialization, configuring of the adapter private structure,
2709 * and a hardware reset occur.
2710 **/
2711static int igbvf_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2712{
2713	struct net_device *netdev;
2714	struct igbvf_adapter *adapter;
2715	struct e1000_hw *hw;
2716	const struct igbvf_info *ei = igbvf_info_tbl[ent->driver_data];
 
2717	static int cards_found;
2718	int err;
2719
2720	err = pci_enable_device_mem(pdev);
2721	if (err)
2722		return err;
2723
 
2724	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
2725	if (err) {
2726		dev_err(&pdev->dev,
2727			"No usable DMA configuration, aborting\n");
2728		goto err_dma;
 
 
 
 
 
2729	}
2730
2731	err = pci_request_regions(pdev, igbvf_driver_name);
2732	if (err)
2733		goto err_pci_reg;
2734
2735	pci_set_master(pdev);
2736
2737	err = -ENOMEM;
2738	netdev = alloc_etherdev(sizeof(struct igbvf_adapter));
2739	if (!netdev)
2740		goto err_alloc_etherdev;
2741
2742	SET_NETDEV_DEV(netdev, &pdev->dev);
2743
2744	pci_set_drvdata(pdev, netdev);
2745	adapter = netdev_priv(netdev);
2746	hw = &adapter->hw;
2747	adapter->netdev = netdev;
2748	adapter->pdev = pdev;
2749	adapter->ei = ei;
2750	adapter->pba = ei->pba;
2751	adapter->flags = ei->flags;
2752	adapter->hw.back = adapter;
2753	adapter->hw.mac.type = ei->mac;
2754	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
2755
2756	/* PCI config space info */
2757
2758	hw->vendor_id = pdev->vendor;
2759	hw->device_id = pdev->device;
2760	hw->subsystem_vendor_id = pdev->subsystem_vendor;
2761	hw->subsystem_device_id = pdev->subsystem_device;
2762	hw->revision_id = pdev->revision;
2763
2764	err = -EIO;
2765	adapter->hw.hw_addr = ioremap(pci_resource_start(pdev, 0),
2766				      pci_resource_len(pdev, 0));
2767
2768	if (!adapter->hw.hw_addr)
2769		goto err_ioremap;
2770
2771	if (ei->get_variants) {
2772		err = ei->get_variants(adapter);
2773		if (err)
2774			goto err_get_variants;
2775	}
2776
2777	/* setup adapter struct */
2778	err = igbvf_sw_init(adapter);
2779	if (err)
2780		goto err_sw_init;
2781
2782	/* construct the net_device struct */
2783	netdev->netdev_ops = &igbvf_netdev_ops;
2784
2785	igbvf_set_ethtool_ops(netdev);
2786	netdev->watchdog_timeo = 5 * HZ;
2787	strscpy(netdev->name, pci_name(pdev), sizeof(netdev->name));
2788
2789	adapter->bd_number = cards_found++;
2790
2791	netdev->hw_features = NETIF_F_SG |
2792			      NETIF_F_TSO |
2793			      NETIF_F_TSO6 |
2794			      NETIF_F_RXCSUM |
2795			      NETIF_F_HW_CSUM |
2796			      NETIF_F_SCTP_CRC;
2797
2798#define IGBVF_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
2799				    NETIF_F_GSO_GRE_CSUM | \
2800				    NETIF_F_GSO_IPXIP4 | \
2801				    NETIF_F_GSO_IPXIP6 | \
2802				    NETIF_F_GSO_UDP_TUNNEL | \
2803				    NETIF_F_GSO_UDP_TUNNEL_CSUM)
2804
2805	netdev->gso_partial_features = IGBVF_GSO_PARTIAL_FEATURES;
2806	netdev->hw_features |= NETIF_F_GSO_PARTIAL |
2807			       IGBVF_GSO_PARTIAL_FEATURES;
2808
2809	netdev->features = netdev->hw_features | NETIF_F_HIGHDMA;
 
 
 
2810
2811	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
2812	netdev->mpls_features |= NETIF_F_HW_CSUM;
2813	netdev->hw_enc_features |= netdev->vlan_features;
2814
2815	/* set this bit last since it cannot be part of vlan_features */
2816	netdev->features |= NETIF_F_HW_VLAN_CTAG_FILTER |
2817			    NETIF_F_HW_VLAN_CTAG_RX |
2818			    NETIF_F_HW_VLAN_CTAG_TX;
2819
2820	/* MTU range: 68 - 9216 */
2821	netdev->min_mtu = ETH_MIN_MTU;
2822	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
2823
2824	spin_lock_bh(&hw->mbx_lock);
2825
2826	/*reset the controller to put the device in a known good state */
2827	err = hw->mac.ops.reset_hw(hw);
2828	if (err) {
2829		dev_info(&pdev->dev,
2830			 "PF still in reset state. Is the PF interface up?\n");
2831	} else {
2832		err = hw->mac.ops.read_mac_addr(hw);
2833		if (err)
2834			dev_info(&pdev->dev, "Error reading MAC address.\n");
2835		else if (is_zero_ether_addr(adapter->hw.mac.addr))
2836			dev_info(&pdev->dev,
2837				 "MAC address not assigned by administrator.\n");
2838		eth_hw_addr_set(netdev, adapter->hw.mac.addr);
 
2839	}
2840
2841	spin_unlock_bh(&hw->mbx_lock);
2842
2843	if (!is_valid_ether_addr(netdev->dev_addr)) {
2844		dev_info(&pdev->dev, "Assigning random MAC address.\n");
2845		eth_hw_addr_random(netdev);
2846		memcpy(adapter->hw.mac.addr, netdev->dev_addr,
2847		       netdev->addr_len);
2848	}
2849
2850	timer_setup(&adapter->watchdog_timer, igbvf_watchdog, 0);
2851
2852	INIT_WORK(&adapter->reset_task, igbvf_reset_task);
2853	INIT_WORK(&adapter->watchdog_task, igbvf_watchdog_task);
2854
2855	/* ring size defaults */
2856	adapter->rx_ring->count = 1024;
2857	adapter->tx_ring->count = 1024;
2858
2859	/* reset the hardware with the new settings */
2860	igbvf_reset(adapter);
2861
2862	/* set hardware-specific flags */
2863	if (adapter->hw.mac.type == e1000_vfadapt_i350)
2864		adapter->flags |= IGBVF_FLAG_RX_LB_VLAN_BSWAP;
2865
2866	strcpy(netdev->name, "eth%d");
2867	err = register_netdev(netdev);
2868	if (err)
2869		goto err_hw_init;
2870
2871	/* tell the stack to leave us alone until igbvf_open() is called */
2872	netif_carrier_off(netdev);
2873	netif_stop_queue(netdev);
2874
2875	igbvf_print_device_info(adapter);
2876
2877	igbvf_initialize_last_counter_stats(adapter);
2878
2879	return 0;
2880
2881err_hw_init:
2882	netif_napi_del(&adapter->rx_ring->napi);
2883	kfree(adapter->tx_ring);
2884	kfree(adapter->rx_ring);
2885err_sw_init:
2886	igbvf_reset_interrupt_capability(adapter);
2887err_get_variants:
2888	iounmap(adapter->hw.hw_addr);
2889err_ioremap:
2890	free_netdev(netdev);
2891err_alloc_etherdev:
2892	pci_release_regions(pdev);
2893err_pci_reg:
2894err_dma:
2895	pci_disable_device(pdev);
2896	return err;
2897}
2898
2899/**
2900 * igbvf_remove - Device Removal Routine
2901 * @pdev: PCI device information struct
2902 *
2903 * igbvf_remove is called by the PCI subsystem to alert the driver
2904 * that it should release a PCI device.  The could be caused by a
2905 * Hot-Plug event, or because the driver is going to be removed from
2906 * memory.
2907 **/
2908static void igbvf_remove(struct pci_dev *pdev)
2909{
2910	struct net_device *netdev = pci_get_drvdata(pdev);
2911	struct igbvf_adapter *adapter = netdev_priv(netdev);
2912	struct e1000_hw *hw = &adapter->hw;
2913
2914	/* The watchdog timer may be rescheduled, so explicitly
2915	 * disable it from being rescheduled.
2916	 */
2917	set_bit(__IGBVF_DOWN, &adapter->state);
2918	del_timer_sync(&adapter->watchdog_timer);
2919
2920	cancel_work_sync(&adapter->reset_task);
2921	cancel_work_sync(&adapter->watchdog_task);
2922
2923	unregister_netdev(netdev);
2924
2925	igbvf_reset_interrupt_capability(adapter);
2926
2927	/* it is important to delete the NAPI struct prior to freeing the
2928	 * Rx ring so that you do not end up with null pointer refs
2929	 */
2930	netif_napi_del(&adapter->rx_ring->napi);
2931	kfree(adapter->tx_ring);
2932	kfree(adapter->rx_ring);
2933
2934	iounmap(hw->hw_addr);
2935	if (hw->flash_address)
2936		iounmap(hw->flash_address);
2937	pci_release_regions(pdev);
2938
2939	free_netdev(netdev);
2940
2941	pci_disable_device(pdev);
2942}
2943
2944/* PCI Error Recovery (ERS) */
2945static const struct pci_error_handlers igbvf_err_handler = {
2946	.error_detected = igbvf_io_error_detected,
2947	.slot_reset = igbvf_io_slot_reset,
2948	.resume = igbvf_io_resume,
2949	.reset_prepare = igbvf_io_prepare,
2950	.reset_done = igbvf_io_reset_done,
2951};
2952
2953static const struct pci_device_id igbvf_pci_tbl[] = {
2954	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_82576_VF), board_vf },
2955	{ PCI_VDEVICE(INTEL, E1000_DEV_ID_I350_VF), board_i350_vf },
2956	{ } /* terminate list */
2957};
2958MODULE_DEVICE_TABLE(pci, igbvf_pci_tbl);
2959
2960static SIMPLE_DEV_PM_OPS(igbvf_pm_ops, igbvf_suspend, igbvf_resume);
2961
2962/* PCI Device API Driver */
2963static struct pci_driver igbvf_driver = {
2964	.name		= igbvf_driver_name,
2965	.id_table	= igbvf_pci_tbl,
2966	.probe		= igbvf_probe,
2967	.remove		= igbvf_remove,
2968	.driver.pm	= &igbvf_pm_ops,
 
 
 
 
2969	.shutdown	= igbvf_shutdown,
2970	.err_handler	= &igbvf_err_handler
2971};
2972
2973/**
2974 * igbvf_init_module - Driver Registration Routine
2975 *
2976 * igbvf_init_module is the first routine called when the driver is
2977 * loaded. All it does is register with the PCI subsystem.
2978 **/
2979static int __init igbvf_init_module(void)
2980{
2981	int ret;
2982
2983	pr_info("%s\n", igbvf_driver_string);
2984	pr_info("%s\n", igbvf_copyright);
2985
2986	ret = pci_register_driver(&igbvf_driver);
2987
2988	return ret;
2989}
2990module_init(igbvf_init_module);
2991
2992/**
2993 * igbvf_exit_module - Driver Exit Cleanup Routine
2994 *
2995 * igbvf_exit_module is called just before the driver is removed
2996 * from memory.
2997 **/
2998static void __exit igbvf_exit_module(void)
2999{
3000	pci_unregister_driver(&igbvf_driver);
3001}
3002module_exit(igbvf_exit_module);
3003
3004MODULE_AUTHOR("Intel Corporation, <e1000-devel@lists.sourceforge.net>");
3005MODULE_DESCRIPTION("Intel(R) Gigabit Virtual Function Network Driver");
3006MODULE_LICENSE("GPL v2");
 
3007
3008/* netdev.c */