1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c)  2018 Intel Corporation */
   3
   4#include <linux/module.h>
   5#include <linux/types.h>
   6#include <linux/if_vlan.h>
   7#include <linux/tcp.h>
   8#include <linux/udp.h>
   9#include <linux/ip.h>
  10#include <linux/pm_runtime.h>
  11#include <net/pkt_sched.h>
  12#include <linux/bpf_trace.h>
  13#include <net/xdp_sock_drv.h>
  14#include <linux/pci.h>
  15#include <linux/mdio.h>
  16
  17#include <net/ipv6.h>
  18
  19#include "igc.h"
  20#include "igc_hw.h"
  21#include "igc_tsn.h"
  22#include "igc_xdp.h"
  23
  24#define DRV_SUMMARY	"Intel(R) 2.5G Ethernet Linux Driver"
  25
  26#define DEFAULT_MSG_ENABLE (NETIF_MSG_DRV | NETIF_MSG_PROBE | NETIF_MSG_LINK)
  27
  28#define IGC_XDP_PASS		0
  29#define IGC_XDP_CONSUMED	BIT(0)
  30#define IGC_XDP_TX		BIT(1)
  31#define IGC_XDP_REDIRECT	BIT(2)
  32
  33static int debug = -1;
  34
  35MODULE_DESCRIPTION(DRV_SUMMARY);
  36MODULE_LICENSE("GPL v2");
  37module_param(debug, int, 0);
  38MODULE_PARM_DESC(debug, "Debug level (0=none,...,16=all)");
  39
  40char igc_driver_name[] = "igc";
  41static const char igc_driver_string[] = DRV_SUMMARY;
  42static const char igc_copyright[] =
  43	"Copyright(c) 2018 Intel Corporation.";
  44
  45static const struct igc_info *igc_info_tbl[] = {
  46	[board_base] = &igc_base_info,
  47};
  48
  49static const struct pci_device_id igc_pci_tbl[] = {
  50	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LM), board_base },
  51	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_V), board_base },
  52	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_I), board_base },
  53	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I220_V), board_base },
  54	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K), board_base },
  55	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_K2), board_base },
  56	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_K), board_base },
  57	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_LMVP), board_base },
  58	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LMVP), board_base },
  59	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_IT), board_base },
  60	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_LM), board_base },
  61	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_V), board_base },
  62	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_IT), board_base },
  63	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I221_V), board_base },
  64	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I226_BLANK_NVM), board_base },
  65	{ PCI_VDEVICE(INTEL, IGC_DEV_ID_I225_BLANK_NVM), board_base },
  66	/* required last entry */
  67	{0, }
  68};
  69
  70MODULE_DEVICE_TABLE(pci, igc_pci_tbl);
  71
  72enum latency_range {
  73	lowest_latency = 0,
  74	low_latency = 1,
  75	bulk_latency = 2,
  76	latency_invalid = 255
  77};
  78
  79void igc_reset(struct igc_adapter *adapter)
  80{
  81	struct net_device *dev = adapter->netdev;
  82	struct igc_hw *hw = &adapter->hw;
  83	struct igc_fc_info *fc = &hw->fc;
  84	u32 pba, hwm;
  85
  86	/* Repartition PBA for greater than 9k MTU if required */
  87	pba = IGC_PBA_34K;
  88
  89	/* flow control settings
  90	 * The high water mark must be low enough to fit one full frame
  91	 * after transmitting the pause frame.  As such we must have enough
  92	 * space to allow for us to complete our current transmit and then
  93	 * receive the frame that is in progress from the link partner.
  94	 * Set it to:
  95	 * - the full Rx FIFO size minus one full Tx plus one full Rx frame
  96	 */
  97	hwm = (pba << 10) - (adapter->max_frame_size + MAX_JUMBO_FRAME_SIZE);
  98
  99	fc->high_water = hwm & 0xFFFFFFF0;	/* 16-byte granularity */
 100	fc->low_water = fc->high_water - 16;
 101	fc->pause_time = 0xFFFF;
 102	fc->send_xon = 1;
 103	fc->current_mode = fc->requested_mode;
 104
 105	hw->mac.ops.reset_hw(hw);
 106
 107	if (hw->mac.ops.init_hw(hw))
 108		netdev_err(dev, "Error on hardware initialization\n");
 109
 110	/* Re-establish EEE setting */
 111	igc_set_eee_i225(hw, true, true, true);
 112
 113	if (!netif_running(adapter->netdev))
 114		igc_power_down_phy_copper_base(&adapter->hw);
 115
 116	/* Enable HW to recognize an 802.1Q VLAN Ethernet packet */
 117	wr32(IGC_VET, ETH_P_8021Q);
 118
 119	/* Re-enable PTP, where applicable. */
 120	igc_ptp_reset(adapter);
 121
 122	/* Re-enable TSN offloading, where applicable. */
 123	igc_tsn_reset(adapter);
 124
 125	igc_get_phy_info(hw);
 126}
 127
 128/**
 129 * igc_power_up_link - Power up the phy link
 130 * @adapter: address of board private structure
 131 */
 132static void igc_power_up_link(struct igc_adapter *adapter)
 133{
 134	igc_reset_phy(&adapter->hw);
 135
 136	igc_power_up_phy_copper(&adapter->hw);
 137
 138	igc_setup_link(&adapter->hw);
 139}
 140
 141/**
 142 * igc_release_hw_control - release control of the h/w to f/w
 143 * @adapter: address of board private structure
 144 *
 145 * igc_release_hw_control resets CTRL_EXT:DRV_LOAD bit.
 146 * For ASF and Pass Through versions of f/w this means that the
 147 * driver is no longer loaded.
 148 */
 149static void igc_release_hw_control(struct igc_adapter *adapter)
 150{
 151	struct igc_hw *hw = &adapter->hw;
 152	u32 ctrl_ext;
 153
 154	if (!pci_device_is_present(adapter->pdev))
 155		return;
 156
 157	/* Let firmware take over control of h/w */
 158	ctrl_ext = rd32(IGC_CTRL_EXT);
 159	wr32(IGC_CTRL_EXT,
 160	     ctrl_ext & ~IGC_CTRL_EXT_DRV_LOAD);
 161}
 162
 163/**
 164 * igc_get_hw_control - get control of the h/w from f/w
 165 * @adapter: address of board private structure
 166 *
 167 * igc_get_hw_control sets CTRL_EXT:DRV_LOAD bit.
 168 * For ASF and Pass Through versions of f/w this means that
 169 * the driver is loaded.
 170 */
 171static void igc_get_hw_control(struct igc_adapter *adapter)
 172{
 173	struct igc_hw *hw = &adapter->hw;
 174	u32 ctrl_ext;
 175
 176	/* Let firmware know the driver has taken over */
 177	ctrl_ext = rd32(IGC_CTRL_EXT);
 178	wr32(IGC_CTRL_EXT,
 179	     ctrl_ext | IGC_CTRL_EXT_DRV_LOAD);
 180}
 181
 182static void igc_unmap_tx_buffer(struct device *dev, struct igc_tx_buffer *buf)
 183{
 184	dma_unmap_single(dev, dma_unmap_addr(buf, dma),
 185			 dma_unmap_len(buf, len), DMA_TO_DEVICE);
 186
 187	dma_unmap_len_set(buf, len, 0);
 188}
 189
 190/**
 191 * igc_clean_tx_ring - Free Tx Buffers
 192 * @tx_ring: ring to be cleaned
 193 */
 194static void igc_clean_tx_ring(struct igc_ring *tx_ring)
 195{
 196	u16 i = tx_ring->next_to_clean;
 197	struct igc_tx_buffer *tx_buffer = &tx_ring->tx_buffer_info[i];
 198	u32 xsk_frames = 0;
 199
 200	while (i != tx_ring->next_to_use) {
 201		union igc_adv_tx_desc *eop_desc, *tx_desc;
 202
 203		switch (tx_buffer->type) {
 204		case IGC_TX_BUFFER_TYPE_XSK:
 205			xsk_frames++;
 206			break;
 207		case IGC_TX_BUFFER_TYPE_XDP:
 208			xdp_return_frame(tx_buffer->xdpf);
 209			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
 210			break;
 211		case IGC_TX_BUFFER_TYPE_SKB:
 212			dev_kfree_skb_any(tx_buffer->skb);
 213			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
 214			break;
 215		default:
 216			netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
 217			break;
 218		}
 219
 220		/* check for eop_desc to determine the end of the packet */
 221		eop_desc = tx_buffer->next_to_watch;
 222		tx_desc = IGC_TX_DESC(tx_ring, i);
 223
 224		/* unmap remaining buffers */
 225		while (tx_desc != eop_desc) {
 226			tx_buffer++;
 227			tx_desc++;
 228			i++;
 229			if (unlikely(i == tx_ring->count)) {
 230				i = 0;
 231				tx_buffer = tx_ring->tx_buffer_info;
 232				tx_desc = IGC_TX_DESC(tx_ring, 0);
 233			}
 234
 235			/* unmap any remaining paged data */
 236			if (dma_unmap_len(tx_buffer, len))
 237				igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
 238		}
 239
 240		tx_buffer->next_to_watch = NULL;
 241
 242		/* move us one more past the eop_desc for start of next pkt */
 243		tx_buffer++;
 244		i++;
 245		if (unlikely(i == tx_ring->count)) {
 246			i = 0;
 247			tx_buffer = tx_ring->tx_buffer_info;
 248		}
 249	}
 250
 251	if (tx_ring->xsk_pool && xsk_frames)
 252		xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
 253
 254	/* reset BQL for queue */
 255	netdev_tx_reset_queue(txring_txq(tx_ring));
 256
 257	/* Zero out the buffer ring */
 258	memset(tx_ring->tx_buffer_info, 0,
 259	       sizeof(*tx_ring->tx_buffer_info) * tx_ring->count);
 260
 261	/* Zero out the descriptor ring */
 262	memset(tx_ring->desc, 0, tx_ring->size);
 263
 264	/* reset next_to_use and next_to_clean */
 265	tx_ring->next_to_use = 0;
 266	tx_ring->next_to_clean = 0;
 267}
 268
 269/**
 270 * igc_free_tx_resources - Free Tx Resources per Queue
 271 * @tx_ring: Tx descriptor ring for a specific queue
 272 *
 273 * Free all transmit software resources
 274 */
 275void igc_free_tx_resources(struct igc_ring *tx_ring)
 276{
 277	igc_disable_tx_ring(tx_ring);
 278
 279	vfree(tx_ring->tx_buffer_info);
 280	tx_ring->tx_buffer_info = NULL;
 281
 282	/* if not set, then don't free */
 283	if (!tx_ring->desc)
 284		return;
 285
 286	dma_free_coherent(tx_ring->dev, tx_ring->size,
 287			  tx_ring->desc, tx_ring->dma);
 288
 289	tx_ring->desc = NULL;
 290}
 291
 292/**
 293 * igc_free_all_tx_resources - Free Tx Resources for All Queues
 294 * @adapter: board private structure
 295 *
 296 * Free all transmit software resources
 297 */
 298static void igc_free_all_tx_resources(struct igc_adapter *adapter)
 299{
 300	int i;
 301
 302	for (i = 0; i < adapter->num_tx_queues; i++)
 303		igc_free_tx_resources(adapter->tx_ring[i]);
 304}
 305
 306/**
 307 * igc_clean_all_tx_rings - Free Tx Buffers for all queues
 308 * @adapter: board private structure
 309 */
 310static void igc_clean_all_tx_rings(struct igc_adapter *adapter)
 311{
 312	int i;
 313
 314	for (i = 0; i < adapter->num_tx_queues; i++)
 315		if (adapter->tx_ring[i])
 316			igc_clean_tx_ring(adapter->tx_ring[i]);
 317}
 318
 319static void igc_disable_tx_ring_hw(struct igc_ring *ring)
 320{
 321	struct igc_hw *hw = &ring->q_vector->adapter->hw;
 322	u8 idx = ring->reg_idx;
 323	u32 txdctl;
 324
 325	txdctl = rd32(IGC_TXDCTL(idx));
 326	txdctl &= ~IGC_TXDCTL_QUEUE_ENABLE;
 327	txdctl |= IGC_TXDCTL_SWFLUSH;
 328	wr32(IGC_TXDCTL(idx), txdctl);
 329}
 330
 331/**
 332 * igc_disable_all_tx_rings_hw - Disable all transmit queue operation
 333 * @adapter: board private structure
 334 */
 335static void igc_disable_all_tx_rings_hw(struct igc_adapter *adapter)
 336{
 337	int i;
 338
 339	for (i = 0; i < adapter->num_tx_queues; i++) {
 340		struct igc_ring *tx_ring = adapter->tx_ring[i];
 341
 342		igc_disable_tx_ring_hw(tx_ring);
 343	}
 344}
 345
 346/**
 347 * igc_setup_tx_resources - allocate Tx resources (Descriptors)
 348 * @tx_ring: tx descriptor ring (for a specific queue) to setup
 349 *
 350 * Return 0 on success, negative on failure
 351 */
 352int igc_setup_tx_resources(struct igc_ring *tx_ring)
 353{
 354	struct net_device *ndev = tx_ring->netdev;
 355	struct device *dev = tx_ring->dev;
 356	int size = 0;
 357
 358	size = sizeof(struct igc_tx_buffer) * tx_ring->count;
 359	tx_ring->tx_buffer_info = vzalloc(size);
 360	if (!tx_ring->tx_buffer_info)
 361		goto err;
 362
 363	/* round up to nearest 4K */
 364	tx_ring->size = tx_ring->count * sizeof(union igc_adv_tx_desc);
 365	tx_ring->size = ALIGN(tx_ring->size, 4096);
 366
 367	tx_ring->desc = dma_alloc_coherent(dev, tx_ring->size,
 368					   &tx_ring->dma, GFP_KERNEL);
 369
 370	if (!tx_ring->desc)
 371		goto err;
 372
 373	tx_ring->next_to_use = 0;
 374	tx_ring->next_to_clean = 0;
 375
 376	return 0;
 377
 378err:
 379	vfree(tx_ring->tx_buffer_info);
 380	netdev_err(ndev, "Unable to allocate memory for Tx descriptor ring\n");
 381	return -ENOMEM;
 382}
 383
 384/**
 385 * igc_setup_all_tx_resources - wrapper to allocate Tx resources for all queues
 386 * @adapter: board private structure
 387 *
 388 * Return 0 on success, negative on failure
 389 */
 390static int igc_setup_all_tx_resources(struct igc_adapter *adapter)
 391{
 392	struct net_device *dev = adapter->netdev;
 393	int i, err = 0;
 394
 395	for (i = 0; i < adapter->num_tx_queues; i++) {
 396		err = igc_setup_tx_resources(adapter->tx_ring[i]);
 397		if (err) {
 398			netdev_err(dev, "Error on Tx queue %u setup\n", i);
 399			for (i--; i >= 0; i--)
 400				igc_free_tx_resources(adapter->tx_ring[i]);
 401			break;
 402		}
 403	}
 404
 405	return err;
 406}
 407
 408static void igc_clean_rx_ring_page_shared(struct igc_ring *rx_ring)
 409{
 410	u16 i = rx_ring->next_to_clean;
 411
 412	dev_kfree_skb(rx_ring->skb);
 413	rx_ring->skb = NULL;
 414
 415	/* Free all the Rx ring sk_buffs */
 416	while (i != rx_ring->next_to_alloc) {
 417		struct igc_rx_buffer *buffer_info = &rx_ring->rx_buffer_info[i];
 418
 419		/* Invalidate cache lines that may have been written to by
 420		 * device so that we avoid corrupting memory.
 421		 */
 422		dma_sync_single_range_for_cpu(rx_ring->dev,
 423					      buffer_info->dma,
 424					      buffer_info->page_offset,
 425					      igc_rx_bufsz(rx_ring),
 426					      DMA_FROM_DEVICE);
 427
 428		/* free resources associated with mapping */
 429		dma_unmap_page_attrs(rx_ring->dev,
 430				     buffer_info->dma,
 431				     igc_rx_pg_size(rx_ring),
 432				     DMA_FROM_DEVICE,
 433				     IGC_RX_DMA_ATTR);
 434		__page_frag_cache_drain(buffer_info->page,
 435					buffer_info->pagecnt_bias);
 436
 437		i++;
 438		if (i == rx_ring->count)
 439			i = 0;
 440	}
 441}
 442
 443static void igc_clean_rx_ring_xsk_pool(struct igc_ring *ring)
 444{
 445	struct igc_rx_buffer *bi;
 446	u16 i;
 447
 448	for (i = 0; i < ring->count; i++) {
 449		bi = &ring->rx_buffer_info[i];
 450		if (!bi->xdp)
 451			continue;
 452
 453		xsk_buff_free(bi->xdp);
 454		bi->xdp = NULL;
 455	}
 456}
 457
 458/**
 459 * igc_clean_rx_ring - Free Rx Buffers per Queue
 460 * @ring: ring to free buffers from
 461 */
 462static void igc_clean_rx_ring(struct igc_ring *ring)
 463{
 464	if (ring->xsk_pool)
 465		igc_clean_rx_ring_xsk_pool(ring);
 466	else
 467		igc_clean_rx_ring_page_shared(ring);
 468
 469	clear_ring_uses_large_buffer(ring);
 470
 471	ring->next_to_alloc = 0;
 472	ring->next_to_clean = 0;
 473	ring->next_to_use = 0;
 474}
 475
 476/**
 477 * igc_clean_all_rx_rings - Free Rx Buffers for all queues
 478 * @adapter: board private structure
 479 */
 480static void igc_clean_all_rx_rings(struct igc_adapter *adapter)
 481{
 482	int i;
 483
 484	for (i = 0; i < adapter->num_rx_queues; i++)
 485		if (adapter->rx_ring[i])
 486			igc_clean_rx_ring(adapter->rx_ring[i]);
 487}
 488
 489/**
 490 * igc_free_rx_resources - Free Rx Resources
 491 * @rx_ring: ring to clean the resources from
 492 *
 493 * Free all receive software resources
 494 */
 495void igc_free_rx_resources(struct igc_ring *rx_ring)
 496{
 497	igc_clean_rx_ring(rx_ring);
 498
 499	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 500
 501	vfree(rx_ring->rx_buffer_info);
 502	rx_ring->rx_buffer_info = NULL;
 503
 504	/* if not set, then don't free */
 505	if (!rx_ring->desc)
 506		return;
 507
 508	dma_free_coherent(rx_ring->dev, rx_ring->size,
 509			  rx_ring->desc, rx_ring->dma);
 510
 511	rx_ring->desc = NULL;
 512}
 513
 514/**
 515 * igc_free_all_rx_resources - Free Rx Resources for All Queues
 516 * @adapter: board private structure
 517 *
 518 * Free all receive software resources
 519 */
 520static void igc_free_all_rx_resources(struct igc_adapter *adapter)
 521{
 522	int i;
 523
 524	for (i = 0; i < adapter->num_rx_queues; i++)
 525		igc_free_rx_resources(adapter->rx_ring[i]);
 526}
 527
 528/**
 529 * igc_setup_rx_resources - allocate Rx resources (Descriptors)
 530 * @rx_ring:    rx descriptor ring (for a specific queue) to setup
 531 *
 532 * Returns 0 on success, negative on failure
 533 */
 534int igc_setup_rx_resources(struct igc_ring *rx_ring)
 535{
 536	struct net_device *ndev = rx_ring->netdev;
 537	struct device *dev = rx_ring->dev;
 538	u8 index = rx_ring->queue_index;
 539	int size, desc_len, res;
 540
 541	/* XDP RX-queue info */
 542	if (xdp_rxq_info_is_reg(&rx_ring->xdp_rxq))
 543		xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 544	res = xdp_rxq_info_reg(&rx_ring->xdp_rxq, ndev, index,
 545			       rx_ring->q_vector->napi.napi_id);
 546	if (res < 0) {
 547		netdev_err(ndev, "Failed to register xdp_rxq index %u\n",
 548			   index);
 549		return res;
 550	}
 551
 552	size = sizeof(struct igc_rx_buffer) * rx_ring->count;
 553	rx_ring->rx_buffer_info = vzalloc(size);
 554	if (!rx_ring->rx_buffer_info)
 555		goto err;
 556
 557	desc_len = sizeof(union igc_adv_rx_desc);
 558
 559	/* Round up to nearest 4K */
 560	rx_ring->size = rx_ring->count * desc_len;
 561	rx_ring->size = ALIGN(rx_ring->size, 4096);
 562
 563	rx_ring->desc = dma_alloc_coherent(dev, rx_ring->size,
 564					   &rx_ring->dma, GFP_KERNEL);
 565
 566	if (!rx_ring->desc)
 567		goto err;
 568
 569	rx_ring->next_to_alloc = 0;
 570	rx_ring->next_to_clean = 0;
 571	rx_ring->next_to_use = 0;
 572
 573	return 0;
 574
 575err:
 576	xdp_rxq_info_unreg(&rx_ring->xdp_rxq);
 577	vfree(rx_ring->rx_buffer_info);
 578	rx_ring->rx_buffer_info = NULL;
 579	netdev_err(ndev, "Unable to allocate memory for Rx descriptor ring\n");
 580	return -ENOMEM;
 581}
 582
 583/**
 584 * igc_setup_all_rx_resources - wrapper to allocate Rx resources
 585 *                                (Descriptors) for all queues
 586 * @adapter: board private structure
 587 *
 588 * Return 0 on success, negative on failure
 589 */
 590static int igc_setup_all_rx_resources(struct igc_adapter *adapter)
 591{
 592	struct net_device *dev = adapter->netdev;
 593	int i, err = 0;
 594
 595	for (i = 0; i < adapter->num_rx_queues; i++) {
 596		err = igc_setup_rx_resources(adapter->rx_ring[i]);
 597		if (err) {
 598			netdev_err(dev, "Error on Rx queue %u setup\n", i);
 599			for (i--; i >= 0; i--)
 600				igc_free_rx_resources(adapter->rx_ring[i]);
 601			break;
 602		}
 603	}
 604
 605	return err;
 606}
 607
 608static struct xsk_buff_pool *igc_get_xsk_pool(struct igc_adapter *adapter,
 609					      struct igc_ring *ring)
 610{
 611	if (!igc_xdp_is_enabled(adapter) ||
 612	    !test_bit(IGC_RING_FLAG_AF_XDP_ZC, &ring->flags))
 613		return NULL;
 614
 615	return xsk_get_pool_from_qid(ring->netdev, ring->queue_index);
 616}
 617
 618/**
 619 * igc_configure_rx_ring - Configure a receive ring after Reset
 620 * @adapter: board private structure
 621 * @ring: receive ring to be configured
 622 *
 623 * Configure the Rx unit of the MAC after a reset.
 624 */
 625static void igc_configure_rx_ring(struct igc_adapter *adapter,
 626				  struct igc_ring *ring)
 627{
 628	struct igc_hw *hw = &adapter->hw;
 629	union igc_adv_rx_desc *rx_desc;
 630	int reg_idx = ring->reg_idx;
 631	u32 srrctl = 0, rxdctl = 0;
 632	u64 rdba = ring->dma;
 633	u32 buf_size;
 634
 635	xdp_rxq_info_unreg_mem_model(&ring->xdp_rxq);
 636	ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
 637	if (ring->xsk_pool) {
 638		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
 639						   MEM_TYPE_XSK_BUFF_POOL,
 640						   NULL));
 641		xsk_pool_set_rxq_info(ring->xsk_pool, &ring->xdp_rxq);
 642	} else {
 643		WARN_ON(xdp_rxq_info_reg_mem_model(&ring->xdp_rxq,
 644						   MEM_TYPE_PAGE_SHARED,
 645						   NULL));
 646	}
 647
 648	if (igc_xdp_is_enabled(adapter))
 649		set_ring_uses_large_buffer(ring);
 650
 651	/* disable the queue */
 652	wr32(IGC_RXDCTL(reg_idx), 0);
 653
 654	/* Set DMA base address registers */
 655	wr32(IGC_RDBAL(reg_idx),
 656	     rdba & 0x00000000ffffffffULL);
 657	wr32(IGC_RDBAH(reg_idx), rdba >> 32);
 658	wr32(IGC_RDLEN(reg_idx),
 659	     ring->count * sizeof(union igc_adv_rx_desc));
 660
 661	/* initialize head and tail */
 662	ring->tail = adapter->io_addr + IGC_RDT(reg_idx);
 663	wr32(IGC_RDH(reg_idx), 0);
 664	writel(0, ring->tail);
 665
 666	/* reset next-to- use/clean to place SW in sync with hardware */
 667	ring->next_to_clean = 0;
 668	ring->next_to_use = 0;
 669
 670	if (ring->xsk_pool)
 671		buf_size = xsk_pool_get_rx_frame_size(ring->xsk_pool);
 672	else if (ring_uses_large_buffer(ring))
 673		buf_size = IGC_RXBUFFER_3072;
 674	else
 675		buf_size = IGC_RXBUFFER_2048;
 676
 677	srrctl = rd32(IGC_SRRCTL(reg_idx));
 678	srrctl &= ~(IGC_SRRCTL_BSIZEPKT_MASK | IGC_SRRCTL_BSIZEHDR_MASK |
 679		    IGC_SRRCTL_DESCTYPE_MASK);
 680	srrctl |= IGC_SRRCTL_BSIZEHDR(IGC_RX_HDR_LEN);
 681	srrctl |= IGC_SRRCTL_BSIZEPKT(buf_size);
 682	srrctl |= IGC_SRRCTL_DESCTYPE_ADV_ONEBUF;
 683
 684	wr32(IGC_SRRCTL(reg_idx), srrctl);
 685
 686	rxdctl |= IGC_RX_PTHRESH;
 687	rxdctl |= IGC_RX_HTHRESH << 8;
 688	rxdctl |= IGC_RX_WTHRESH << 16;
 689
 690	/* initialize rx_buffer_info */
 691	memset(ring->rx_buffer_info, 0,
 692	       sizeof(struct igc_rx_buffer) * ring->count);
 693
 694	/* initialize Rx descriptor 0 */
 695	rx_desc = IGC_RX_DESC(ring, 0);
 696	rx_desc->wb.upper.length = 0;
 697
 698	/* enable receive descriptor fetching */
 699	rxdctl |= IGC_RXDCTL_QUEUE_ENABLE;
 700
 701	wr32(IGC_RXDCTL(reg_idx), rxdctl);
 702}
 703
 704/**
 705 * igc_configure_rx - Configure receive Unit after Reset
 706 * @adapter: board private structure
 707 *
 708 * Configure the Rx unit of the MAC after a reset.
 709 */
 710static void igc_configure_rx(struct igc_adapter *adapter)
 711{
 712	int i;
 713
 714	/* Setup the HW Rx Head and Tail Descriptor Pointers and
 715	 * the Base and Length of the Rx Descriptor Ring
 716	 */
 717	for (i = 0; i < adapter->num_rx_queues; i++)
 718		igc_configure_rx_ring(adapter, adapter->rx_ring[i]);
 719}
 720
 721/**
 722 * igc_configure_tx_ring - Configure transmit ring after Reset
 723 * @adapter: board private structure
 724 * @ring: tx ring to configure
 725 *
 726 * Configure a transmit ring after a reset.
 727 */
 728static void igc_configure_tx_ring(struct igc_adapter *adapter,
 729				  struct igc_ring *ring)
 730{
 731	struct igc_hw *hw = &adapter->hw;
 732	int reg_idx = ring->reg_idx;
 733	u64 tdba = ring->dma;
 734	u32 txdctl = 0;
 735
 736	ring->xsk_pool = igc_get_xsk_pool(adapter, ring);
 737
 738	/* disable the queue */
 739	wr32(IGC_TXDCTL(reg_idx), 0);
 740	wrfl();
 741
 742	wr32(IGC_TDLEN(reg_idx),
 743	     ring->count * sizeof(union igc_adv_tx_desc));
 744	wr32(IGC_TDBAL(reg_idx),
 745	     tdba & 0x00000000ffffffffULL);
 746	wr32(IGC_TDBAH(reg_idx), tdba >> 32);
 747
 748	ring->tail = adapter->io_addr + IGC_TDT(reg_idx);
 749	wr32(IGC_TDH(reg_idx), 0);
 750	writel(0, ring->tail);
 751
 752	txdctl |= IGC_TX_PTHRESH;
 753	txdctl |= IGC_TX_HTHRESH << 8;
 754	txdctl |= IGC_TX_WTHRESH << 16;
 755
 756	txdctl |= IGC_TXDCTL_QUEUE_ENABLE;
 757	wr32(IGC_TXDCTL(reg_idx), txdctl);
 758}
 759
 760/**
 761 * igc_configure_tx - Configure transmit Unit after Reset
 762 * @adapter: board private structure
 763 *
 764 * Configure the Tx unit of the MAC after a reset.
 765 */
 766static void igc_configure_tx(struct igc_adapter *adapter)
 767{
 768	int i;
 769
 770	for (i = 0; i < adapter->num_tx_queues; i++)
 771		igc_configure_tx_ring(adapter, adapter->tx_ring[i]);
 772}
 773
 774/**
 775 * igc_setup_mrqc - configure the multiple receive queue control registers
 776 * @adapter: Board private structure
 777 */
 778static void igc_setup_mrqc(struct igc_adapter *adapter)
 779{
 780	struct igc_hw *hw = &adapter->hw;
 781	u32 j, num_rx_queues;
 782	u32 mrqc, rxcsum;
 783	u32 rss_key[10];
 784
 785	netdev_rss_key_fill(rss_key, sizeof(rss_key));
 786	for (j = 0; j < 10; j++)
 787		wr32(IGC_RSSRK(j), rss_key[j]);
 788
 789	num_rx_queues = adapter->rss_queues;
 790
 791	if (adapter->rss_indir_tbl_init != num_rx_queues) {
 792		for (j = 0; j < IGC_RETA_SIZE; j++)
 793			adapter->rss_indir_tbl[j] =
 794			(j * num_rx_queues) / IGC_RETA_SIZE;
 795		adapter->rss_indir_tbl_init = num_rx_queues;
 796	}
 797	igc_write_rss_indir_tbl(adapter);
 798
 799	/* Disable raw packet checksumming so that RSS hash is placed in
 800	 * descriptor on writeback.  No need to enable TCP/UDP/IP checksum
 801	 * offloads as they are enabled by default
 802	 */
 803	rxcsum = rd32(IGC_RXCSUM);
 804	rxcsum |= IGC_RXCSUM_PCSD;
 805
 806	/* Enable Receive Checksum Offload for SCTP */
 807	rxcsum |= IGC_RXCSUM_CRCOFL;
 808
 809	/* Don't need to set TUOFL or IPOFL, they default to 1 */
 810	wr32(IGC_RXCSUM, rxcsum);
 811
 812	/* Generate RSS hash based on packet types, TCP/UDP
 813	 * port numbers and/or IPv4/v6 src and dst addresses
 814	 */
 815	mrqc = IGC_MRQC_RSS_FIELD_IPV4 |
 816	       IGC_MRQC_RSS_FIELD_IPV4_TCP |
 817	       IGC_MRQC_RSS_FIELD_IPV6 |
 818	       IGC_MRQC_RSS_FIELD_IPV6_TCP |
 819	       IGC_MRQC_RSS_FIELD_IPV6_TCP_EX;
 820
 821	if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV4_UDP)
 822		mrqc |= IGC_MRQC_RSS_FIELD_IPV4_UDP;
 823	if (adapter->flags & IGC_FLAG_RSS_FIELD_IPV6_UDP)
 824		mrqc |= IGC_MRQC_RSS_FIELD_IPV6_UDP;
 825
 826	mrqc |= IGC_MRQC_ENABLE_RSS_MQ;
 827
 828	wr32(IGC_MRQC, mrqc);
 829}
 830
 831/**
 832 * igc_setup_rctl - configure the receive control registers
 833 * @adapter: Board private structure
 834 */
 835static void igc_setup_rctl(struct igc_adapter *adapter)
 836{
 837	struct igc_hw *hw = &adapter->hw;
 838	u32 rctl;
 839
 840	rctl = rd32(IGC_RCTL);
 841
 842	rctl &= ~(3 << IGC_RCTL_MO_SHIFT);
 843	rctl &= ~(IGC_RCTL_LBM_TCVR | IGC_RCTL_LBM_MAC);
 844
 845	rctl |= IGC_RCTL_EN | IGC_RCTL_BAM | IGC_RCTL_RDMTS_HALF |
 846		(hw->mac.mc_filter_type << IGC_RCTL_MO_SHIFT);
 847
 848	/* enable stripping of CRC. Newer features require
 849	 * that the HW strips the CRC.
 850	 */
 851	rctl |= IGC_RCTL_SECRC;
 852
 853	/* disable store bad packets and clear size bits. */
 854	rctl &= ~(IGC_RCTL_SBP | IGC_RCTL_SZ_256);
 855
 856	/* enable LPE to allow for reception of jumbo frames */
 857	rctl |= IGC_RCTL_LPE;
 858
 859	/* disable queue 0 to prevent tail write w/o re-config */
 860	wr32(IGC_RXDCTL(0), 0);
 861
 862	/* This is useful for sniffing bad packets. */
 863	if (adapter->netdev->features & NETIF_F_RXALL) {
 864		/* UPE and MPE will be handled by normal PROMISC logic
 865		 * in set_rx_mode
 866		 */
 867		rctl |= (IGC_RCTL_SBP | /* Receive bad packets */
 868			 IGC_RCTL_BAM | /* RX All Bcast Pkts */
 869			 IGC_RCTL_PMCF); /* RX All MAC Ctrl Pkts */
 870
 871		rctl &= ~(IGC_RCTL_DPF | /* Allow filtered pause */
 872			  IGC_RCTL_CFIEN); /* Disable VLAN CFIEN Filter */
 873	}
 874
 875	wr32(IGC_RCTL, rctl);
 876}
 877
 878/**
 879 * igc_setup_tctl - configure the transmit control registers
 880 * @adapter: Board private structure
 881 */
 882static void igc_setup_tctl(struct igc_adapter *adapter)
 883{
 884	struct igc_hw *hw = &adapter->hw;
 885	u32 tctl;
 886
 887	/* disable queue 0 which icould be enabled by default */
 888	wr32(IGC_TXDCTL(0), 0);
 889
 890	/* Program the Transmit Control Register */
 891	tctl = rd32(IGC_TCTL);
 892	tctl &= ~IGC_TCTL_CT;
 893	tctl |= IGC_TCTL_PSP | IGC_TCTL_RTLC |
 894		(IGC_COLLISION_THRESHOLD << IGC_CT_SHIFT);
 895
 896	/* Enable transmits */
 897	tctl |= IGC_TCTL_EN;
 898
 899	wr32(IGC_TCTL, tctl);
 900}
 901
 902/**
 903 * igc_set_mac_filter_hw() - Set MAC address filter in hardware
 904 * @adapter: Pointer to adapter where the filter should be set
 905 * @index: Filter index
 906 * @type: MAC address filter type (source or destination)
 907 * @addr: MAC address
 908 * @queue: If non-negative, queue assignment feature is enabled and frames
 909 *         matching the filter are enqueued onto 'queue'. Otherwise, queue
 910 *         assignment is disabled.
 911 */
 912static void igc_set_mac_filter_hw(struct igc_adapter *adapter, int index,
 913				  enum igc_mac_filter_type type,
 914				  const u8 *addr, int queue)
 915{
 916	struct net_device *dev = adapter->netdev;
 917	struct igc_hw *hw = &adapter->hw;
 918	u32 ral, rah;
 919
 920	if (WARN_ON(index >= hw->mac.rar_entry_count))
 921		return;
 922
 923	ral = le32_to_cpup((__le32 *)(addr));
 924	rah = le16_to_cpup((__le16 *)(addr + 4));
 925
 926	if (type == IGC_MAC_FILTER_TYPE_SRC) {
 927		rah &= ~IGC_RAH_ASEL_MASK;
 928		rah |= IGC_RAH_ASEL_SRC_ADDR;
 929	}
 930
 931	if (queue >= 0) {
 932		rah &= ~IGC_RAH_QSEL_MASK;
 933		rah |= (queue << IGC_RAH_QSEL_SHIFT);
 934		rah |= IGC_RAH_QSEL_ENABLE;
 935	}
 936
 937	rah |= IGC_RAH_AV;
 938
 939	wr32(IGC_RAL(index), ral);
 940	wr32(IGC_RAH(index), rah);
 941
 942	netdev_dbg(dev, "MAC address filter set in HW: index %d", index);
 943}
 944
 945/**
 946 * igc_clear_mac_filter_hw() - Clear MAC address filter in hardware
 947 * @adapter: Pointer to adapter where the filter should be cleared
 948 * @index: Filter index
 949 */
 950static void igc_clear_mac_filter_hw(struct igc_adapter *adapter, int index)
 951{
 952	struct net_device *dev = adapter->netdev;
 953	struct igc_hw *hw = &adapter->hw;
 954
 955	if (WARN_ON(index >= hw->mac.rar_entry_count))
 956		return;
 957
 958	wr32(IGC_RAL(index), 0);
 959	wr32(IGC_RAH(index), 0);
 960
 961	netdev_dbg(dev, "MAC address filter cleared in HW: index %d", index);
 962}
 963
 964/* Set default MAC address for the PF in the first RAR entry */
 965static void igc_set_default_mac_filter(struct igc_adapter *adapter)
 966{
 967	struct net_device *dev = adapter->netdev;
 968	u8 *addr = adapter->hw.mac.addr;
 969
 970	netdev_dbg(dev, "Set default MAC address filter: address %pM", addr);
 971
 972	igc_set_mac_filter_hw(adapter, 0, IGC_MAC_FILTER_TYPE_DST, addr, -1);
 973}
 974
 975/**
 976 * igc_set_mac - Change the Ethernet Address of the NIC
 977 * @netdev: network interface device structure
 978 * @p: pointer to an address structure
 979 *
 980 * Returns 0 on success, negative on failure
 981 */
 982static int igc_set_mac(struct net_device *netdev, void *p)
 983{
 984	struct igc_adapter *adapter = netdev_priv(netdev);
 985	struct igc_hw *hw = &adapter->hw;
 986	struct sockaddr *addr = p;
 987
 988	if (!is_valid_ether_addr(addr->sa_data))
 989		return -EADDRNOTAVAIL;
 990
 991	eth_hw_addr_set(netdev, addr->sa_data);
 992	memcpy(hw->mac.addr, addr->sa_data, netdev->addr_len);
 993
 994	/* set the correct pool for the new PF MAC address in entry 0 */
 995	igc_set_default_mac_filter(adapter);
 996
 997	return 0;
 998}
 999
1000/**
1001 *  igc_write_mc_addr_list - write multicast addresses to MTA
1002 *  @netdev: network interface device structure
1003 *
1004 *  Writes multicast address list to the MTA hash table.
1005 *  Returns: -ENOMEM on failure
1006 *           0 on no addresses written
1007 *           X on writing X addresses to MTA
1008 **/
1009static int igc_write_mc_addr_list(struct net_device *netdev)
1010{
1011	struct igc_adapter *adapter = netdev_priv(netdev);
1012	struct igc_hw *hw = &adapter->hw;
1013	struct netdev_hw_addr *ha;
1014	u8  *mta_list;
1015	int i;
1016
1017	if (netdev_mc_empty(netdev)) {
1018		/* nothing to program, so clear mc list */
1019		igc_update_mc_addr_list(hw, NULL, 0);
1020		return 0;
1021	}
1022
1023	mta_list = kcalloc(netdev_mc_count(netdev), 6, GFP_ATOMIC);
1024	if (!mta_list)
1025		return -ENOMEM;
1026
1027	/* The shared function expects a packed array of only addresses. */
1028	i = 0;
1029	netdev_for_each_mc_addr(ha, netdev)
1030		memcpy(mta_list + (i++ * ETH_ALEN), ha->addr, ETH_ALEN);
1031
1032	igc_update_mc_addr_list(hw, mta_list, i);
1033	kfree(mta_list);
1034
1035	return netdev_mc_count(netdev);
1036}
1037
1038static __le32 igc_tx_launchtime(struct igc_ring *ring, ktime_t txtime,
1039				bool *first_flag, bool *insert_empty)
1040{
1041	struct igc_adapter *adapter = netdev_priv(ring->netdev);
1042	ktime_t cycle_time = adapter->cycle_time;
1043	ktime_t base_time = adapter->base_time;
1044	ktime_t now = ktime_get_clocktai();
1045	ktime_t baset_est, end_of_cycle;
1046	s32 launchtime;
1047	s64 n;
1048
1049	n = div64_s64(ktime_sub_ns(now, base_time), cycle_time);
1050
1051	baset_est = ktime_add_ns(base_time, cycle_time * (n));
1052	end_of_cycle = ktime_add_ns(baset_est, cycle_time);
1053
1054	if (ktime_compare(txtime, end_of_cycle) >= 0) {
1055		if (baset_est != ring->last_ff_cycle) {
1056			*first_flag = true;
1057			ring->last_ff_cycle = baset_est;
1058
1059			if (ktime_compare(end_of_cycle, ring->last_tx_cycle) > 0)
1060				*insert_empty = true;
1061		}
1062	}
1063
1064	/* Introducing a window at end of cycle on which packets
1065	 * potentially not honor launchtime. Window of 5us chosen
1066	 * considering software update the tail pointer and packets
1067	 * are dma'ed to packet buffer.
1068	 */
1069	if ((ktime_sub_ns(end_of_cycle, now) < 5 * NSEC_PER_USEC))
1070		netdev_warn(ring->netdev, "Packet with txtime=%llu may not be honoured\n",
1071			    txtime);
1072
1073	ring->last_tx_cycle = end_of_cycle;
1074
1075	launchtime = ktime_sub_ns(txtime, baset_est);
1076	if (launchtime > 0)
1077		div_s64_rem(launchtime, cycle_time, &launchtime);
1078	else
1079		launchtime = 0;
1080
1081	return cpu_to_le32(launchtime);
1082}
1083
1084static int igc_init_empty_frame(struct igc_ring *ring,
1085				struct igc_tx_buffer *buffer,
1086				struct sk_buff *skb)
1087{
1088	unsigned int size;
1089	dma_addr_t dma;
1090
1091	size = skb_headlen(skb);
1092
1093	dma = dma_map_single(ring->dev, skb->data, size, DMA_TO_DEVICE);
1094	if (dma_mapping_error(ring->dev, dma)) {
1095		netdev_err_once(ring->netdev, "Failed to map DMA for TX\n");
1096		return -ENOMEM;
1097	}
1098
1099	buffer->type = IGC_TX_BUFFER_TYPE_SKB;
1100	buffer->skb = skb;
1101	buffer->protocol = 0;
1102	buffer->bytecount = skb->len;
1103	buffer->gso_segs = 1;
1104	buffer->time_stamp = jiffies;
1105	dma_unmap_len_set(buffer, len, skb->len);
1106	dma_unmap_addr_set(buffer, dma, dma);
1107
1108	return 0;
1109}
1110
1111static int igc_init_tx_empty_descriptor(struct igc_ring *ring,
1112					struct sk_buff *skb,
1113					struct igc_tx_buffer *first)
1114{
1115	union igc_adv_tx_desc *desc;
1116	u32 cmd_type, olinfo_status;
1117	int err;
1118
1119	if (!igc_desc_unused(ring))
1120		return -EBUSY;
1121
1122	err = igc_init_empty_frame(ring, first, skb);
1123	if (err)
1124		return err;
1125
1126	cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
1127		   IGC_ADVTXD_DCMD_IFCS | IGC_TXD_DCMD |
1128		   first->bytecount;
1129	olinfo_status = first->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
1130
1131	desc = IGC_TX_DESC(ring, ring->next_to_use);
1132	desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1133	desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1134	desc->read.buffer_addr = cpu_to_le64(dma_unmap_addr(first, dma));
1135
1136	netdev_tx_sent_queue(txring_txq(ring), skb->len);
1137
1138	first->next_to_watch = desc;
1139
1140	ring->next_to_use++;
1141	if (ring->next_to_use == ring->count)
1142		ring->next_to_use = 0;
1143
1144	return 0;
1145}
1146
1147#define IGC_EMPTY_FRAME_SIZE 60
1148
1149static void igc_tx_ctxtdesc(struct igc_ring *tx_ring,
1150			    __le32 launch_time, bool first_flag,
1151			    u32 vlan_macip_lens, u32 type_tucmd,
1152			    u32 mss_l4len_idx)
1153{
1154	struct igc_adv_tx_context_desc *context_desc;
1155	u16 i = tx_ring->next_to_use;
1156
1157	context_desc = IGC_TX_CTXTDESC(tx_ring, i);
1158
1159	i++;
1160	tx_ring->next_to_use = (i < tx_ring->count) ? i : 0;
1161
1162	/* set bits to identify this as an advanced context descriptor */
1163	type_tucmd |= IGC_TXD_CMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
1164
1165	/* For i225, context index must be unique per ring. */
1166	if (test_bit(IGC_RING_FLAG_TX_CTX_IDX, &tx_ring->flags))
1167		mss_l4len_idx |= tx_ring->reg_idx << 4;
1168
1169	if (first_flag)
1170		mss_l4len_idx |= IGC_ADVTXD_TSN_CNTX_FIRST;
1171
1172	context_desc->vlan_macip_lens	= cpu_to_le32(vlan_macip_lens);
1173	context_desc->type_tucmd_mlhl	= cpu_to_le32(type_tucmd);
1174	context_desc->mss_l4len_idx	= cpu_to_le32(mss_l4len_idx);
1175	context_desc->launch_time	= launch_time;
1176}
1177
1178static void igc_tx_csum(struct igc_ring *tx_ring, struct igc_tx_buffer *first,
1179			__le32 launch_time, bool first_flag)
1180{
1181	struct sk_buff *skb = first->skb;
1182	u32 vlan_macip_lens = 0;
1183	u32 type_tucmd = 0;
1184
1185	if (skb->ip_summed != CHECKSUM_PARTIAL) {
1186csum_failed:
1187		if (!(first->tx_flags & IGC_TX_FLAGS_VLAN) &&
1188		    !tx_ring->launchtime_enable)
1189			return;
1190		goto no_csum;
1191	}
1192
1193	switch (skb->csum_offset) {
1194	case offsetof(struct tcphdr, check):
1195		type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1196		fallthrough;
1197	case offsetof(struct udphdr, check):
1198		break;
1199	case offsetof(struct sctphdr, checksum):
1200		/* validate that this is actually an SCTP request */
1201		if (skb_csum_is_sctp(skb)) {
1202			type_tucmd = IGC_ADVTXD_TUCMD_L4T_SCTP;
1203			break;
1204		}
1205		fallthrough;
1206	default:
1207		skb_checksum_help(skb);
1208		goto csum_failed;
1209	}
1210
1211	/* update TX checksum flag */
1212	first->tx_flags |= IGC_TX_FLAGS_CSUM;
1213	vlan_macip_lens = skb_checksum_start_offset(skb) -
1214			  skb_network_offset(skb);
1215no_csum:
1216	vlan_macip_lens |= skb_network_offset(skb) << IGC_ADVTXD_MACLEN_SHIFT;
1217	vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1218
1219	igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1220			vlan_macip_lens, type_tucmd, 0);
1221}
1222
1223static int __igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1224{
1225	struct net_device *netdev = tx_ring->netdev;
1226
1227	netif_stop_subqueue(netdev, tx_ring->queue_index);
1228
1229	/* memory barriier comment */
1230	smp_mb();
1231
1232	/* We need to check again in a case another CPU has just
1233	 * made room available.
1234	 */
1235	if (igc_desc_unused(tx_ring) < size)
1236		return -EBUSY;
1237
1238	/* A reprieve! */
1239	netif_wake_subqueue(netdev, tx_ring->queue_index);
1240
1241	u64_stats_update_begin(&tx_ring->tx_syncp2);
1242	tx_ring->tx_stats.restart_queue2++;
1243	u64_stats_update_end(&tx_ring->tx_syncp2);
1244
1245	return 0;
1246}
1247
1248static inline int igc_maybe_stop_tx(struct igc_ring *tx_ring, const u16 size)
1249{
1250	if (igc_desc_unused(tx_ring) >= size)
1251		return 0;
1252	return __igc_maybe_stop_tx(tx_ring, size);
1253}
1254
1255#define IGC_SET_FLAG(_input, _flag, _result) \
1256	(((_flag) <= (_result)) ?				\
1257	 ((u32)((_input) & (_flag)) * ((_result) / (_flag))) :	\
1258	 ((u32)((_input) & (_flag)) / ((_flag) / (_result))))
1259
1260static u32 igc_tx_cmd_type(struct sk_buff *skb, u32 tx_flags)
1261{
1262	/* set type for advanced descriptor with frame checksum insertion */
1263	u32 cmd_type = IGC_ADVTXD_DTYP_DATA |
1264		       IGC_ADVTXD_DCMD_DEXT |
1265		       IGC_ADVTXD_DCMD_IFCS;
1266
1267	/* set HW vlan bit if vlan is present */
1268	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_VLAN,
1269				 IGC_ADVTXD_DCMD_VLE);
1270
1271	/* set segmentation bits for TSO */
1272	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSO,
1273				 (IGC_ADVTXD_DCMD_TSE));
1274
1275	/* set timestamp bit if present, will select the register set
1276	 * based on the _TSTAMP(_X) bit.
1277	 */
1278	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP,
1279				 (IGC_ADVTXD_MAC_TSTAMP));
1280
1281	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_1,
1282				 (IGC_ADVTXD_TSTAMP_REG_1));
1283
1284	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_2,
1285				 (IGC_ADVTXD_TSTAMP_REG_2));
1286
1287	cmd_type |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_3,
1288				 (IGC_ADVTXD_TSTAMP_REG_3));
1289
1290	/* insert frame checksum */
1291	cmd_type ^= IGC_SET_FLAG(skb->no_fcs, 1, IGC_ADVTXD_DCMD_IFCS);
1292
1293	return cmd_type;
1294}
1295
1296static void igc_tx_olinfo_status(struct igc_ring *tx_ring,
1297				 union igc_adv_tx_desc *tx_desc,
1298				 u32 tx_flags, unsigned int paylen)
1299{
1300	u32 olinfo_status = paylen << IGC_ADVTXD_PAYLEN_SHIFT;
1301
1302	/* insert L4 checksum */
1303	olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_CSUM,
1304				      (IGC_TXD_POPTS_TXSM << 8));
1305
1306	/* insert IPv4 checksum */
1307	olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_IPV4,
1308				      (IGC_TXD_POPTS_IXSM << 8));
1309
1310	/* Use the second timer (free running, in general) for the timestamp */
1311	olinfo_status |= IGC_SET_FLAG(tx_flags, IGC_TX_FLAGS_TSTAMP_TIMER_1,
1312				      IGC_TXD_PTP2_TIMER_1);
1313
1314	tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
1315}
1316
1317static int igc_tx_map(struct igc_ring *tx_ring,
1318		      struct igc_tx_buffer *first,
1319		      const u8 hdr_len)
1320{
1321	struct sk_buff *skb = first->skb;
1322	struct igc_tx_buffer *tx_buffer;
1323	union igc_adv_tx_desc *tx_desc;
1324	u32 tx_flags = first->tx_flags;
1325	skb_frag_t *frag;
1326	u16 i = tx_ring->next_to_use;
1327	unsigned int data_len, size;
1328	dma_addr_t dma;
1329	u32 cmd_type;
1330
1331	cmd_type = igc_tx_cmd_type(skb, tx_flags);
1332	tx_desc = IGC_TX_DESC(tx_ring, i);
1333
1334	igc_tx_olinfo_status(tx_ring, tx_desc, tx_flags, skb->len - hdr_len);
1335
1336	size = skb_headlen(skb);
1337	data_len = skb->data_len;
1338
1339	dma = dma_map_single(tx_ring->dev, skb->data, size, DMA_TO_DEVICE);
1340
1341	tx_buffer = first;
1342
1343	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
1344		if (dma_mapping_error(tx_ring->dev, dma))
1345			goto dma_error;
1346
1347		/* record length, and DMA address */
1348		dma_unmap_len_set(tx_buffer, len, size);
1349		dma_unmap_addr_set(tx_buffer, dma, dma);
1350
1351		tx_desc->read.buffer_addr = cpu_to_le64(dma);
1352
1353		while (unlikely(size > IGC_MAX_DATA_PER_TXD)) {
1354			tx_desc->read.cmd_type_len =
1355				cpu_to_le32(cmd_type ^ IGC_MAX_DATA_PER_TXD);
1356
1357			i++;
1358			tx_desc++;
1359			if (i == tx_ring->count) {
1360				tx_desc = IGC_TX_DESC(tx_ring, 0);
1361				i = 0;
1362			}
1363			tx_desc->read.olinfo_status = 0;
1364
1365			dma += IGC_MAX_DATA_PER_TXD;
1366			size -= IGC_MAX_DATA_PER_TXD;
1367
1368			tx_desc->read.buffer_addr = cpu_to_le64(dma);
1369		}
1370
1371		if (likely(!data_len))
1372			break;
1373
1374		tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type ^ size);
1375
1376		i++;
1377		tx_desc++;
1378		if (i == tx_ring->count) {
1379			tx_desc = IGC_TX_DESC(tx_ring, 0);
1380			i = 0;
1381		}
1382		tx_desc->read.olinfo_status = 0;
1383
1384		size = skb_frag_size(frag);
1385		data_len -= size;
1386
1387		dma = skb_frag_dma_map(tx_ring->dev, frag, 0,
1388				       size, DMA_TO_DEVICE);
1389
1390		tx_buffer = &tx_ring->tx_buffer_info[i];
1391	}
1392
1393	/* write last descriptor with RS and EOP bits */
1394	cmd_type |= size | IGC_TXD_DCMD;
1395	tx_desc->read.cmd_type_len = cpu_to_le32(cmd_type);
1396
1397	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1398
1399	/* set the timestamp */
1400	first->time_stamp = jiffies;
1401
1402	skb_tx_timestamp(skb);
1403
1404	/* Force memory writes to complete before letting h/w know there
1405	 * are new descriptors to fetch.  (Only applicable for weak-ordered
1406	 * memory model archs, such as IA-64).
1407	 *
1408	 * We also need this memory barrier to make certain all of the
1409	 * status bits have been updated before next_to_watch is written.
1410	 */
1411	wmb();
1412
1413	/* set next_to_watch value indicating a packet is present */
1414	first->next_to_watch = tx_desc;
1415
1416	i++;
1417	if (i == tx_ring->count)
1418		i = 0;
1419
1420	tx_ring->next_to_use = i;
1421
1422	/* Make sure there is space in the ring for the next send. */
1423	igc_maybe_stop_tx(tx_ring, DESC_NEEDED);
1424
1425	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1426		writel(i, tx_ring->tail);
1427	}
1428
1429	return 0;
1430dma_error:
1431	netdev_err(tx_ring->netdev, "TX DMA map failed\n");
1432	tx_buffer = &tx_ring->tx_buffer_info[i];
1433
1434	/* clear dma mappings for failed tx_buffer_info map */
1435	while (tx_buffer != first) {
1436		if (dma_unmap_len(tx_buffer, len))
1437			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1438
1439		if (i-- == 0)
1440			i += tx_ring->count;
1441		tx_buffer = &tx_ring->tx_buffer_info[i];
1442	}
1443
1444	if (dma_unmap_len(tx_buffer, len))
1445		igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
1446
1447	dev_kfree_skb_any(tx_buffer->skb);
1448	tx_buffer->skb = NULL;
1449
1450	tx_ring->next_to_use = i;
1451
1452	return -1;
1453}
1454
1455static int igc_tso(struct igc_ring *tx_ring,
1456		   struct igc_tx_buffer *first,
1457		   __le32 launch_time, bool first_flag,
1458		   u8 *hdr_len)
1459{
1460	u32 vlan_macip_lens, type_tucmd, mss_l4len_idx;
1461	struct sk_buff *skb = first->skb;
1462	union {
1463		struct iphdr *v4;
1464		struct ipv6hdr *v6;
1465		unsigned char *hdr;
1466	} ip;
1467	union {
1468		struct tcphdr *tcp;
1469		struct udphdr *udp;
1470		unsigned char *hdr;
1471	} l4;
1472	u32 paylen, l4_offset;
1473	int err;
1474
1475	if (skb->ip_summed != CHECKSUM_PARTIAL)
1476		return 0;
1477
1478	if (!skb_is_gso(skb))
1479		return 0;
1480
1481	err = skb_cow_head(skb, 0);
1482	if (err < 0)
1483		return err;
1484
1485	ip.hdr = skb_network_header(skb);
1486	l4.hdr = skb_checksum_start(skb);
1487
1488	/* ADV DTYP TUCMD MKRLOC/ISCSIHEDLEN */
1489	type_tucmd = IGC_ADVTXD_TUCMD_L4T_TCP;
1490
1491	/* initialize outer IP header fields */
1492	if (ip.v4->version == 4) {
1493		unsigned char *csum_start = skb_checksum_start(skb);
1494		unsigned char *trans_start = ip.hdr + (ip.v4->ihl * 4);
1495
1496		/* IP header will have to cancel out any data that
1497		 * is not a part of the outer IP header
1498		 */
1499		ip.v4->check = csum_fold(csum_partial(trans_start,
1500						      csum_start - trans_start,
1501						      0));
1502		type_tucmd |= IGC_ADVTXD_TUCMD_IPV4;
1503
1504		ip.v4->tot_len = 0;
1505		first->tx_flags |= IGC_TX_FLAGS_TSO |
1506				   IGC_TX_FLAGS_CSUM |
1507				   IGC_TX_FLAGS_IPV4;
1508	} else {
1509		ip.v6->payload_len = 0;
1510		first->tx_flags |= IGC_TX_FLAGS_TSO |
1511				   IGC_TX_FLAGS_CSUM;
1512	}
1513
1514	/* determine offset of inner transport header */
1515	l4_offset = l4.hdr - skb->data;
1516
1517	/* remove payload length from inner checksum */
1518	paylen = skb->len - l4_offset;
1519	if (type_tucmd & IGC_ADVTXD_TUCMD_L4T_TCP) {
1520		/* compute length of segmentation header */
1521		*hdr_len = (l4.tcp->doff * 4) + l4_offset;
1522		csum_replace_by_diff(&l4.tcp->check,
1523				     (__force __wsum)htonl(paylen));
1524	} else {
1525		/* compute length of segmentation header */
1526		*hdr_len = sizeof(*l4.udp) + l4_offset;
1527		csum_replace_by_diff(&l4.udp->check,
1528				     (__force __wsum)htonl(paylen));
1529	}
1530
1531	/* update gso size and bytecount with header size */
1532	first->gso_segs = skb_shinfo(skb)->gso_segs;
1533	first->bytecount += (first->gso_segs - 1) * *hdr_len;
1534
1535	/* MSS L4LEN IDX */
1536	mss_l4len_idx = (*hdr_len - l4_offset) << IGC_ADVTXD_L4LEN_SHIFT;
1537	mss_l4len_idx |= skb_shinfo(skb)->gso_size << IGC_ADVTXD_MSS_SHIFT;
1538
1539	/* VLAN MACLEN IPLEN */
1540	vlan_macip_lens = l4.hdr - ip.hdr;
1541	vlan_macip_lens |= (ip.hdr - skb->data) << IGC_ADVTXD_MACLEN_SHIFT;
1542	vlan_macip_lens |= first->tx_flags & IGC_TX_FLAGS_VLAN_MASK;
1543
1544	igc_tx_ctxtdesc(tx_ring, launch_time, first_flag,
1545			vlan_macip_lens, type_tucmd, mss_l4len_idx);
1546
1547	return 1;
1548}
1549
1550static bool igc_request_tx_tstamp(struct igc_adapter *adapter, struct sk_buff *skb, u32 *flags)
1551{
1552	int i;
1553
1554	for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
1555		struct igc_tx_timestamp_request *tstamp = &adapter->tx_tstamp[i];
1556
1557		if (tstamp->skb)
1558			continue;
1559
1560		tstamp->skb = skb_get(skb);
1561		tstamp->start = jiffies;
1562		*flags = tstamp->flags;
1563
1564		return true;
1565	}
1566
1567	return false;
1568}
1569
1570static netdev_tx_t igc_xmit_frame_ring(struct sk_buff *skb,
1571				       struct igc_ring *tx_ring)
1572{
1573	struct igc_adapter *adapter = netdev_priv(tx_ring->netdev);
1574	bool first_flag = false, insert_empty = false;
1575	u16 count = TXD_USE_COUNT(skb_headlen(skb));
1576	__be16 protocol = vlan_get_protocol(skb);
1577	struct igc_tx_buffer *first;
1578	__le32 launch_time = 0;
1579	u32 tx_flags = 0;
1580	unsigned short f;
1581	ktime_t txtime;
1582	u8 hdr_len = 0;
1583	int tso = 0;
1584
1585	/* need: 1 descriptor per page * PAGE_SIZE/IGC_MAX_DATA_PER_TXD,
1586	 *	+ 1 desc for skb_headlen/IGC_MAX_DATA_PER_TXD,
1587	 *	+ 2 desc gap to keep tail from touching head,
1588	 *	+ 1 desc for context descriptor,
1589	 * otherwise try next time
1590	 */
1591	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++)
1592		count += TXD_USE_COUNT(skb_frag_size(
1593						&skb_shinfo(skb)->frags[f]));
1594
1595	if (igc_maybe_stop_tx(tx_ring, count + 5)) {
1596		/* this is a hard error */
1597		return NETDEV_TX_BUSY;
1598	}
1599
1600	if (!tx_ring->launchtime_enable)
1601		goto done;
1602
1603	txtime = skb->tstamp;
1604	skb->tstamp = ktime_set(0, 0);
1605	launch_time = igc_tx_launchtime(tx_ring, txtime, &first_flag, &insert_empty);
1606
1607	if (insert_empty) {
1608		struct igc_tx_buffer *empty_info;
1609		struct sk_buff *empty;
1610		void *data;
1611
1612		empty_info = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1613		empty = alloc_skb(IGC_EMPTY_FRAME_SIZE, GFP_ATOMIC);
1614		if (!empty)
1615			goto done;
1616
1617		data = skb_put(empty, IGC_EMPTY_FRAME_SIZE);
1618		memset(data, 0, IGC_EMPTY_FRAME_SIZE);
1619
1620		igc_tx_ctxtdesc(tx_ring, 0, false, 0, 0, 0);
1621
1622		if (igc_init_tx_empty_descriptor(tx_ring,
1623						 empty,
1624						 empty_info) < 0)
1625			dev_kfree_skb_any(empty);
1626	}
1627
1628done:
1629	/* record the location of the first descriptor for this packet */
1630	first = &tx_ring->tx_buffer_info[tx_ring->next_to_use];
1631	first->type = IGC_TX_BUFFER_TYPE_SKB;
1632	first->skb = skb;
1633	first->bytecount = skb->len;
1634	first->gso_segs = 1;
1635
1636	if (adapter->qbv_transition || tx_ring->oper_gate_closed)
1637		goto out_drop;
1638
1639	if (tx_ring->max_sdu > 0 && first->bytecount > tx_ring->max_sdu) {
1640		adapter->stats.txdrop++;
1641		goto out_drop;
1642	}
1643
1644	if (unlikely(test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags) &&
1645		     skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP)) {
1646		unsigned long flags;
1647		u32 tstamp_flags;
1648
1649		spin_lock_irqsave(&adapter->ptp_tx_lock, flags);
1650		if (igc_request_tx_tstamp(adapter, skb, &tstamp_flags)) {
1651			skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
1652			tx_flags |= IGC_TX_FLAGS_TSTAMP | tstamp_flags;
1653			if (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP_USE_CYCLES)
1654				tx_flags |= IGC_TX_FLAGS_TSTAMP_TIMER_1;
1655		} else {
1656			adapter->tx_hwtstamp_skipped++;
1657		}
1658
1659		spin_unlock_irqrestore(&adapter->ptp_tx_lock, flags);
1660	}
1661
1662	if (skb_vlan_tag_present(skb)) {
1663		tx_flags |= IGC_TX_FLAGS_VLAN;
1664		tx_flags |= (skb_vlan_tag_get(skb) << IGC_TX_FLAGS_VLAN_SHIFT);
1665	}
1666
1667	/* record initial flags and protocol */
1668	first->tx_flags = tx_flags;
1669	first->protocol = protocol;
1670
1671	tso = igc_tso(tx_ring, first, launch_time, first_flag, &hdr_len);
1672	if (tso < 0)
1673		goto out_drop;
1674	else if (!tso)
1675		igc_tx_csum(tx_ring, first, launch_time, first_flag);
1676
1677	igc_tx_map(tx_ring, first, hdr_len);
1678
1679	return NETDEV_TX_OK;
1680
1681out_drop:
1682	dev_kfree_skb_any(first->skb);
1683	first->skb = NULL;
1684
1685	return NETDEV_TX_OK;
1686}
1687
1688static inline struct igc_ring *igc_tx_queue_mapping(struct igc_adapter *adapter,
1689						    struct sk_buff *skb)
1690{
1691	unsigned int r_idx = skb->queue_mapping;
1692
1693	if (r_idx >= adapter->num_tx_queues)
1694		r_idx = r_idx % adapter->num_tx_queues;
1695
1696	return adapter->tx_ring[r_idx];
1697}
1698
1699static netdev_tx_t igc_xmit_frame(struct sk_buff *skb,
1700				  struct net_device *netdev)
1701{
1702	struct igc_adapter *adapter = netdev_priv(netdev);
1703
1704	/* The minimum packet size with TCTL.PSP set is 17 so pad the skb
1705	 * in order to meet this minimum size requirement.
1706	 */
1707	if (skb->len < 17) {
1708		if (skb_padto(skb, 17))
1709			return NETDEV_TX_OK;
1710		skb->len = 17;
1711	}
1712
1713	return igc_xmit_frame_ring(skb, igc_tx_queue_mapping(adapter, skb));
1714}
1715
1716static void igc_rx_checksum(struct igc_ring *ring,
1717			    union igc_adv_rx_desc *rx_desc,
1718			    struct sk_buff *skb)
1719{
1720	skb_checksum_none_assert(skb);
1721
1722	/* Ignore Checksum bit is set */
1723	if (igc_test_staterr(rx_desc, IGC_RXD_STAT_IXSM))
1724		return;
1725
1726	/* Rx checksum disabled via ethtool */
1727	if (!(ring->netdev->features & NETIF_F_RXCSUM))
1728		return;
1729
1730	/* TCP/UDP checksum error bit is set */
1731	if (igc_test_staterr(rx_desc,
1732			     IGC_RXDEXT_STATERR_L4E |
1733			     IGC_RXDEXT_STATERR_IPE)) {
1734		/* work around errata with sctp packets where the TCPE aka
1735		 * L4E bit is set incorrectly on 64 byte (60 byte w/o crc)
1736		 * packets (aka let the stack check the crc32c)
1737		 */
1738		if (!(skb->len == 60 &&
1739		      test_bit(IGC_RING_FLAG_RX_SCTP_CSUM, &ring->flags))) {
1740			u64_stats_update_begin(&ring->rx_syncp);
1741			ring->rx_stats.csum_err++;
1742			u64_stats_update_end(&ring->rx_syncp);
1743		}
1744		/* let the stack verify checksum errors */
1745		return;
1746	}
1747	/* It must be a TCP or UDP packet with a valid checksum */
1748	if (igc_test_staterr(rx_desc, IGC_RXD_STAT_TCPCS |
1749				      IGC_RXD_STAT_UDPCS))
1750		skb->ip_summed = CHECKSUM_UNNECESSARY;
1751
1752	netdev_dbg(ring->netdev, "cksum success: bits %08X\n",
1753		   le32_to_cpu(rx_desc->wb.upper.status_error));
1754}
1755
1756/* Mapping HW RSS Type to enum pkt_hash_types */
1757static const enum pkt_hash_types igc_rss_type_table[IGC_RSS_TYPE_MAX_TABLE] = {
1758	[IGC_RSS_TYPE_NO_HASH]		= PKT_HASH_TYPE_L2,
1759	[IGC_RSS_TYPE_HASH_TCP_IPV4]	= PKT_HASH_TYPE_L4,
1760	[IGC_RSS_TYPE_HASH_IPV4]	= PKT_HASH_TYPE_L3,
1761	[IGC_RSS_TYPE_HASH_TCP_IPV6]	= PKT_HASH_TYPE_L4,
1762	[IGC_RSS_TYPE_HASH_IPV6_EX]	= PKT_HASH_TYPE_L3,
1763	[IGC_RSS_TYPE_HASH_IPV6]	= PKT_HASH_TYPE_L3,
1764	[IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = PKT_HASH_TYPE_L4,
1765	[IGC_RSS_TYPE_HASH_UDP_IPV4]	= PKT_HASH_TYPE_L4,
1766	[IGC_RSS_TYPE_HASH_UDP_IPV6]	= PKT_HASH_TYPE_L4,
1767	[IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = PKT_HASH_TYPE_L4,
1768	[10] = PKT_HASH_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW  */
1769	[11] = PKT_HASH_TYPE_NONE, /* keep array sized for SW bit-mask   */
1770	[12] = PKT_HASH_TYPE_NONE, /* to handle future HW revisons       */
1771	[13] = PKT_HASH_TYPE_NONE,
1772	[14] = PKT_HASH_TYPE_NONE,
1773	[15] = PKT_HASH_TYPE_NONE,
1774};
1775
1776static inline void igc_rx_hash(struct igc_ring *ring,
1777			       union igc_adv_rx_desc *rx_desc,
1778			       struct sk_buff *skb)
1779{
1780	if (ring->netdev->features & NETIF_F_RXHASH) {
1781		u32 rss_hash = le32_to_cpu(rx_desc->wb.lower.hi_dword.rss);
1782		u32 rss_type = igc_rss_type(rx_desc);
1783
1784		skb_set_hash(skb, rss_hash, igc_rss_type_table[rss_type]);
1785	}
1786}
1787
1788static void igc_rx_vlan(struct igc_ring *rx_ring,
1789			union igc_adv_rx_desc *rx_desc,
1790			struct sk_buff *skb)
1791{
1792	struct net_device *dev = rx_ring->netdev;
1793	u16 vid;
1794
1795	if ((dev->features & NETIF_F_HW_VLAN_CTAG_RX) &&
1796	    igc_test_staterr(rx_desc, IGC_RXD_STAT_VP)) {
1797		if (igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_LB) &&
1798		    test_bit(IGC_RING_FLAG_RX_LB_VLAN_BSWAP, &rx_ring->flags))
1799			vid = be16_to_cpu((__force __be16)rx_desc->wb.upper.vlan);
1800		else
1801			vid = le16_to_cpu(rx_desc->wb.upper.vlan);
1802
1803		__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
1804	}
1805}
1806
1807/**
1808 * igc_process_skb_fields - Populate skb header fields from Rx descriptor
1809 * @rx_ring: rx descriptor ring packet is being transacted on
1810 * @rx_desc: pointer to the EOP Rx descriptor
1811 * @skb: pointer to current skb being populated
1812 *
1813 * This function checks the ring, descriptor, and packet information in order
1814 * to populate the hash, checksum, VLAN, protocol, and other fields within the
1815 * skb.
1816 */
1817static void igc_process_skb_fields(struct igc_ring *rx_ring,
1818				   union igc_adv_rx_desc *rx_desc,
1819				   struct sk_buff *skb)
1820{
1821	igc_rx_hash(rx_ring, rx_desc, skb);
1822
1823	igc_rx_checksum(rx_ring, rx_desc, skb);
1824
1825	igc_rx_vlan(rx_ring, rx_desc, skb);
1826
1827	skb_record_rx_queue(skb, rx_ring->queue_index);
1828
1829	skb->protocol = eth_type_trans(skb, rx_ring->netdev);
1830}
1831
1832static void igc_vlan_mode(struct net_device *netdev, netdev_features_t features)
1833{
1834	bool enable = !!(features & NETIF_F_HW_VLAN_CTAG_RX);
1835	struct igc_adapter *adapter = netdev_priv(netdev);
1836	struct igc_hw *hw = &adapter->hw;
1837	u32 ctrl;
1838
1839	ctrl = rd32(IGC_CTRL);
1840
1841	if (enable) {
1842		/* enable VLAN tag insert/strip */
1843		ctrl |= IGC_CTRL_VME;
1844	} else {
1845		/* disable VLAN tag insert/strip */
1846		ctrl &= ~IGC_CTRL_VME;
1847	}
1848	wr32(IGC_CTRL, ctrl);
1849}
1850
1851static void igc_restore_vlan(struct igc_adapter *adapter)
1852{
1853	igc_vlan_mode(adapter->netdev, adapter->netdev->features);
1854}
1855
1856static struct igc_rx_buffer *igc_get_rx_buffer(struct igc_ring *rx_ring,
1857					       const unsigned int size,
1858					       int *rx_buffer_pgcnt)
1859{
1860	struct igc_rx_buffer *rx_buffer;
1861
1862	rx_buffer = &rx_ring->rx_buffer_info[rx_ring->next_to_clean];
1863	*rx_buffer_pgcnt =
1864#if (PAGE_SIZE < 8192)
1865		page_count(rx_buffer->page);
1866#else
1867		0;
1868#endif
1869	prefetchw(rx_buffer->page);
1870
1871	/* we are reusing so sync this buffer for CPU use */
1872	dma_sync_single_range_for_cpu(rx_ring->dev,
1873				      rx_buffer->dma,
1874				      rx_buffer->page_offset,
1875				      size,
1876				      DMA_FROM_DEVICE);
1877
1878	rx_buffer->pagecnt_bias--;
1879
1880	return rx_buffer;
1881}
1882
1883static void igc_rx_buffer_flip(struct igc_rx_buffer *buffer,
1884			       unsigned int truesize)
1885{
1886#if (PAGE_SIZE < 8192)
1887	buffer->page_offset ^= truesize;
1888#else
1889	buffer->page_offset += truesize;
1890#endif
1891}
1892
1893static unsigned int igc_get_rx_frame_truesize(struct igc_ring *ring,
1894					      unsigned int size)
1895{
1896	unsigned int truesize;
1897
1898#if (PAGE_SIZE < 8192)
1899	truesize = igc_rx_pg_size(ring) / 2;
1900#else
1901	truesize = ring_uses_build_skb(ring) ?
1902		   SKB_DATA_ALIGN(sizeof(struct skb_shared_info)) +
1903		   SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1904		   SKB_DATA_ALIGN(size);
1905#endif
1906	return truesize;
1907}
1908
1909/**
1910 * igc_add_rx_frag - Add contents of Rx buffer to sk_buff
1911 * @rx_ring: rx descriptor ring to transact packets on
1912 * @rx_buffer: buffer containing page to add
1913 * @skb: sk_buff to place the data into
1914 * @size: size of buffer to be added
1915 *
1916 * This function will add the data contained in rx_buffer->page to the skb.
1917 */
1918static void igc_add_rx_frag(struct igc_ring *rx_ring,
1919			    struct igc_rx_buffer *rx_buffer,
1920			    struct sk_buff *skb,
1921			    unsigned int size)
1922{
1923	unsigned int truesize;
1924
1925#if (PAGE_SIZE < 8192)
1926	truesize = igc_rx_pg_size(rx_ring) / 2;
1927#else
1928	truesize = ring_uses_build_skb(rx_ring) ?
1929		   SKB_DATA_ALIGN(IGC_SKB_PAD + size) :
1930		   SKB_DATA_ALIGN(size);
1931#endif
1932	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_buffer->page,
1933			rx_buffer->page_offset, size, truesize);
1934
1935	igc_rx_buffer_flip(rx_buffer, truesize);
1936}
1937
1938static struct sk_buff *igc_build_skb(struct igc_ring *rx_ring,
1939				     struct igc_rx_buffer *rx_buffer,
1940				     struct xdp_buff *xdp)
1941{
1942	unsigned int size = xdp->data_end - xdp->data;
1943	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1944	unsigned int metasize = xdp->data - xdp->data_meta;
1945	struct sk_buff *skb;
1946
1947	/* prefetch first cache line of first page */
1948	net_prefetch(xdp->data_meta);
1949
1950	/* build an skb around the page buffer */
1951	skb = napi_build_skb(xdp->data_hard_start, truesize);
1952	if (unlikely(!skb))
1953		return NULL;
1954
1955	/* update pointers within the skb to store the data */
1956	skb_reserve(skb, xdp->data - xdp->data_hard_start);
1957	__skb_put(skb, size);
1958	if (metasize)
1959		skb_metadata_set(skb, metasize);
1960
1961	igc_rx_buffer_flip(rx_buffer, truesize);
1962	return skb;
1963}
1964
1965static struct sk_buff *igc_construct_skb(struct igc_ring *rx_ring,
1966					 struct igc_rx_buffer *rx_buffer,
1967					 struct igc_xdp_buff *ctx)
1968{
1969	struct xdp_buff *xdp = &ctx->xdp;
1970	unsigned int metasize = xdp->data - xdp->data_meta;
1971	unsigned int size = xdp->data_end - xdp->data;
1972	unsigned int truesize = igc_get_rx_frame_truesize(rx_ring, size);
1973	void *va = xdp->data;
1974	unsigned int headlen;
1975	struct sk_buff *skb;
1976
1977	/* prefetch first cache line of first page */
1978	net_prefetch(xdp->data_meta);
1979
1980	/* allocate a skb to store the frags */
1981	skb = napi_alloc_skb(&rx_ring->q_vector->napi,
1982			     IGC_RX_HDR_LEN + metasize);
1983	if (unlikely(!skb))
1984		return NULL;
1985
1986	if (ctx->rx_ts) {
1987		skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP_NETDEV;
1988		skb_hwtstamps(skb)->netdev_data = ctx->rx_ts;
1989	}
1990
1991	/* Determine available headroom for copy */
1992	headlen = size;
1993	if (headlen > IGC_RX_HDR_LEN)
1994		headlen = eth_get_headlen(skb->dev, va, IGC_RX_HDR_LEN);
1995
1996	/* align pull length to size of long to optimize memcpy performance */
1997	memcpy(__skb_put(skb, headlen + metasize), xdp->data_meta,
1998	       ALIGN(headlen + metasize, sizeof(long)));
1999
2000	if (metasize) {
2001		skb_metadata_set(skb, metasize);
2002		__skb_pull(skb, metasize);
2003	}
2004
2005	/* update all of the pointers */
2006	size -= headlen;
2007	if (size) {
2008		skb_add_rx_frag(skb, 0, rx_buffer->page,
2009				(va + headlen) - page_address(rx_buffer->page),
2010				size, truesize);
2011		igc_rx_buffer_flip(rx_buffer, truesize);
2012	} else {
2013		rx_buffer->pagecnt_bias++;
2014	}
2015
2016	return skb;
2017}
2018
2019/**
2020 * igc_reuse_rx_page - page flip buffer and store it back on the ring
2021 * @rx_ring: rx descriptor ring to store buffers on
2022 * @old_buff: donor buffer to have page reused
2023 *
2024 * Synchronizes page for reuse by the adapter
2025 */
2026static void igc_reuse_rx_page(struct igc_ring *rx_ring,
2027			      struct igc_rx_buffer *old_buff)
2028{
2029	u16 nta = rx_ring->next_to_alloc;
2030	struct igc_rx_buffer *new_buff;
2031
2032	new_buff = &rx_ring->rx_buffer_info[nta];
2033
2034	/* update, and store next to alloc */
2035	nta++;
2036	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
2037
2038	/* Transfer page from old buffer to new buffer.
2039	 * Move each member individually to avoid possible store
2040	 * forwarding stalls.
2041	 */
2042	new_buff->dma		= old_buff->dma;
2043	new_buff->page		= old_buff->page;
2044	new_buff->page_offset	= old_buff->page_offset;
2045	new_buff->pagecnt_bias	= old_buff->pagecnt_bias;
2046}
2047
2048static bool igc_can_reuse_rx_page(struct igc_rx_buffer *rx_buffer,
2049				  int rx_buffer_pgcnt)
2050{
2051	unsigned int pagecnt_bias = rx_buffer->pagecnt_bias;
2052	struct page *page = rx_buffer->page;
2053
2054	/* avoid re-using remote and pfmemalloc pages */
2055	if (!dev_page_is_reusable(page))
2056		return false;
2057
2058#if (PAGE_SIZE < 8192)
2059	/* if we are only owner of page we can reuse it */
2060	if (unlikely((rx_buffer_pgcnt - pagecnt_bias) > 1))
2061		return false;
2062#else
2063#define IGC_LAST_OFFSET \
2064	(SKB_WITH_OVERHEAD(PAGE_SIZE) - IGC_RXBUFFER_2048)
2065
2066	if (rx_buffer->page_offset > IGC_LAST_OFFSET)
2067		return false;
2068#endif
2069
2070	/* If we have drained the page fragment pool we need to update
2071	 * the pagecnt_bias and page count so that we fully restock the
2072	 * number of references the driver holds.
2073	 */
2074	if (unlikely(pagecnt_bias == 1)) {
2075		page_ref_add(page, USHRT_MAX - 1);
2076		rx_buffer->pagecnt_bias = USHRT_MAX;
2077	}
2078
2079	return true;
2080}
2081
2082/**
2083 * igc_is_non_eop - process handling of non-EOP buffers
2084 * @rx_ring: Rx ring being processed
2085 * @rx_desc: Rx descriptor for current buffer
2086 *
2087 * This function updates next to clean.  If the buffer is an EOP buffer
2088 * this function exits returning false, otherwise it will place the
2089 * sk_buff in the next buffer to be chained and return true indicating
2090 * that this is in fact a non-EOP buffer.
2091 */
2092static bool igc_is_non_eop(struct igc_ring *rx_ring,
2093			   union igc_adv_rx_desc *rx_desc)
2094{
2095	u32 ntc = rx_ring->next_to_clean + 1;
2096
2097	/* fetch, update, and store next to clean */
2098	ntc = (ntc < rx_ring->count) ? ntc : 0;
2099	rx_ring->next_to_clean = ntc;
2100
2101	prefetch(IGC_RX_DESC(rx_ring, ntc));
2102
2103	if (likely(igc_test_staterr(rx_desc, IGC_RXD_STAT_EOP)))
2104		return false;
2105
2106	return true;
2107}
2108
2109/**
2110 * igc_cleanup_headers - Correct corrupted or empty headers
2111 * @rx_ring: rx descriptor ring packet is being transacted on
2112 * @rx_desc: pointer to the EOP Rx descriptor
2113 * @skb: pointer to current skb being fixed
2114 *
2115 * Address the case where we are pulling data in on pages only
2116 * and as such no data is present in the skb header.
2117 *
2118 * In addition if skb is not at least 60 bytes we need to pad it so that
2119 * it is large enough to qualify as a valid Ethernet frame.
2120 *
2121 * Returns true if an error was encountered and skb was freed.
2122 */
2123static bool igc_cleanup_headers(struct igc_ring *rx_ring,
2124				union igc_adv_rx_desc *rx_desc,
2125				struct sk_buff *skb)
2126{
2127	/* XDP packets use error pointer so abort at this point */
2128	if (IS_ERR(skb))
2129		return true;
2130
2131	if (unlikely(igc_test_staterr(rx_desc, IGC_RXDEXT_STATERR_RXE))) {
2132		struct net_device *netdev = rx_ring->netdev;
2133
2134		if (!(netdev->features & NETIF_F_RXALL)) {
2135			dev_kfree_skb_any(skb);
2136			return true;
2137		}
2138	}
2139
2140	/* if eth_skb_pad returns an error the skb was freed */
2141	if (eth_skb_pad(skb))
2142		return true;
2143
2144	return false;
2145}
2146
2147static void igc_put_rx_buffer(struct igc_ring *rx_ring,
2148			      struct igc_rx_buffer *rx_buffer,
2149			      int rx_buffer_pgcnt)
2150{
2151	if (igc_can_reuse_rx_page(rx_buffer, rx_buffer_pgcnt)) {
2152		/* hand second half of page back to the ring */
2153		igc_reuse_rx_page(rx_ring, rx_buffer);
2154	} else {
2155		/* We are not reusing the buffer so unmap it and free
2156		 * any references we are holding to it
2157		 */
2158		dma_unmap_page_attrs(rx_ring->dev, rx_buffer->dma,
2159				     igc_rx_pg_size(rx_ring), DMA_FROM_DEVICE,
2160				     IGC_RX_DMA_ATTR);
2161		__page_frag_cache_drain(rx_buffer->page,
2162					rx_buffer->pagecnt_bias);
2163	}
2164
2165	/* clear contents of rx_buffer */
2166	rx_buffer->page = NULL;
2167}
2168
2169static inline unsigned int igc_rx_offset(struct igc_ring *rx_ring)
2170{
2171	struct igc_adapter *adapter = rx_ring->q_vector->adapter;
2172
2173	if (ring_uses_build_skb(rx_ring))
2174		return IGC_SKB_PAD;
2175	if (igc_xdp_is_enabled(adapter))
2176		return XDP_PACKET_HEADROOM;
2177
2178	return 0;
2179}
2180
2181static bool igc_alloc_mapped_page(struct igc_ring *rx_ring,
2182				  struct igc_rx_buffer *bi)
2183{
2184	struct page *page = bi->page;
2185	dma_addr_t dma;
2186
2187	/* since we are recycling buffers we should seldom need to alloc */
2188	if (likely(page))
2189		return true;
2190
2191	/* alloc new page for storage */
2192	page = dev_alloc_pages(igc_rx_pg_order(rx_ring));
2193	if (unlikely(!page)) {
2194		rx_ring->rx_stats.alloc_failed++;
2195		set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags);
2196		return false;
2197	}
2198
2199	/* map page for use */
2200	dma = dma_map_page_attrs(rx_ring->dev, page, 0,
2201				 igc_rx_pg_size(rx_ring),
2202				 DMA_FROM_DEVICE,
2203				 IGC_RX_DMA_ATTR);
2204
2205	/* if mapping failed free memory back to system since
2206	 * there isn't much point in holding memory we can't use
2207	 */
2208	if (dma_mapping_error(rx_ring->dev, dma)) {
2209		__free_page(page);
2210
2211		rx_ring->rx_stats.alloc_failed++;
2212		set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags);
2213		return false;
2214	}
2215
2216	bi->dma = dma;
2217	bi->page = page;
2218	bi->page_offset = igc_rx_offset(rx_ring);
2219	page_ref_add(page, USHRT_MAX - 1);
2220	bi->pagecnt_bias = USHRT_MAX;
2221
2222	return true;
2223}
2224
2225/**
2226 * igc_alloc_rx_buffers - Replace used receive buffers; packet split
2227 * @rx_ring: rx descriptor ring
2228 * @cleaned_count: number of buffers to clean
2229 */
2230static void igc_alloc_rx_buffers(struct igc_ring *rx_ring, u16 cleaned_count)
2231{
2232	union igc_adv_rx_desc *rx_desc;
2233	u16 i = rx_ring->next_to_use;
2234	struct igc_rx_buffer *bi;
2235	u16 bufsz;
2236
2237	/* nothing to do */
2238	if (!cleaned_count)
2239		return;
2240
2241	rx_desc = IGC_RX_DESC(rx_ring, i);
2242	bi = &rx_ring->rx_buffer_info[i];
2243	i -= rx_ring->count;
2244
2245	bufsz = igc_rx_bufsz(rx_ring);
2246
2247	do {
2248		if (!igc_alloc_mapped_page(rx_ring, bi))
2249			break;
2250
2251		/* sync the buffer for use by the device */
2252		dma_sync_single_range_for_device(rx_ring->dev, bi->dma,
2253						 bi->page_offset, bufsz,
2254						 DMA_FROM_DEVICE);
2255
2256		/* Refresh the desc even if buffer_addrs didn't change
2257		 * because each write-back erases this info.
2258		 */
2259		rx_desc->read.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
2260
2261		rx_desc++;
2262		bi++;
2263		i++;
2264		if (unlikely(!i)) {
2265			rx_desc = IGC_RX_DESC(rx_ring, 0);
2266			bi = rx_ring->rx_buffer_info;
2267			i -= rx_ring->count;
2268		}
2269
2270		/* clear the length for the next_to_use descriptor */
2271		rx_desc->wb.upper.length = 0;
2272
2273		cleaned_count--;
2274	} while (cleaned_count);
2275
2276	i += rx_ring->count;
2277
2278	if (rx_ring->next_to_use != i) {
2279		/* record the next descriptor to use */
2280		rx_ring->next_to_use = i;
2281
2282		/* update next to alloc since we have filled the ring */
2283		rx_ring->next_to_alloc = i;
2284
2285		/* Force memory writes to complete before letting h/w
2286		 * know there are new descriptors to fetch.  (Only
2287		 * applicable for weak-ordered memory model archs,
2288		 * such as IA-64).
2289		 */
2290		wmb();
2291		writel(i, rx_ring->tail);
2292	}
2293}
2294
2295static bool igc_alloc_rx_buffers_zc(struct igc_ring *ring, u16 count)
2296{
2297	union igc_adv_rx_desc *desc;
2298	u16 i = ring->next_to_use;
2299	struct igc_rx_buffer *bi;
2300	dma_addr_t dma;
2301	bool ok = true;
2302
2303	if (!count)
2304		return ok;
2305
2306	XSK_CHECK_PRIV_TYPE(struct igc_xdp_buff);
2307
2308	desc = IGC_RX_DESC(ring, i);
2309	bi = &ring->rx_buffer_info[i];
2310	i -= ring->count;
2311
2312	do {
2313		bi->xdp = xsk_buff_alloc(ring->xsk_pool);
2314		if (!bi->xdp) {
2315			ok = false;
2316			break;
2317		}
2318
2319		dma = xsk_buff_xdp_get_dma(bi->xdp);
2320		desc->read.pkt_addr = cpu_to_le64(dma);
2321
2322		desc++;
2323		bi++;
2324		i++;
2325		if (unlikely(!i)) {
2326			desc = IGC_RX_DESC(ring, 0);
2327			bi = ring->rx_buffer_info;
2328			i -= ring->count;
2329		}
2330
2331		/* Clear the length for the next_to_use descriptor. */
2332		desc->wb.upper.length = 0;
2333
2334		count--;
2335	} while (count);
2336
2337	i += ring->count;
2338
2339	if (ring->next_to_use != i) {
2340		ring->next_to_use = i;
2341
2342		/* Force memory writes to complete before letting h/w
2343		 * know there are new descriptors to fetch.  (Only
2344		 * applicable for weak-ordered memory model archs,
2345		 * such as IA-64).
2346		 */
2347		wmb();
2348		writel(i, ring->tail);
2349	}
2350
2351	return ok;
2352}
2353
2354/* This function requires __netif_tx_lock is held by the caller. */
2355static int igc_xdp_init_tx_descriptor(struct igc_ring *ring,
2356				      struct xdp_frame *xdpf)
2357{
2358	struct skb_shared_info *sinfo = xdp_get_shared_info_from_frame(xdpf);
2359	u8 nr_frags = unlikely(xdp_frame_has_frags(xdpf)) ? sinfo->nr_frags : 0;
2360	u16 count, index = ring->next_to_use;
2361	struct igc_tx_buffer *head = &ring->tx_buffer_info[index];
2362	struct igc_tx_buffer *buffer = head;
2363	union igc_adv_tx_desc *desc = IGC_TX_DESC(ring, index);
2364	u32 olinfo_status, len = xdpf->len, cmd_type;
2365	void *data = xdpf->data;
2366	u16 i;
2367
2368	count = TXD_USE_COUNT(len);
2369	for (i = 0; i < nr_frags; i++)
2370		count += TXD_USE_COUNT(skb_frag_size(&sinfo->frags[i]));
2371
2372	if (igc_maybe_stop_tx(ring, count + 3)) {
2373		/* this is a hard error */
2374		return -EBUSY;
2375	}
2376
2377	i = 0;
2378	head->bytecount = xdp_get_frame_len(xdpf);
2379	head->type = IGC_TX_BUFFER_TYPE_XDP;
2380	head->gso_segs = 1;
2381	head->xdpf = xdpf;
2382
2383	olinfo_status = head->bytecount << IGC_ADVTXD_PAYLEN_SHIFT;
2384	desc->read.olinfo_status = cpu_to_le32(olinfo_status);
2385
2386	for (;;) {
2387		dma_addr_t dma;
2388
2389		dma = dma_map_single(ring->dev, data, len, DMA_TO_DEVICE);
2390		if (dma_mapping_error(ring->dev, dma)) {
2391			netdev_err_once(ring->netdev,
2392					"Failed to map DMA for TX\n");
2393			goto unmap;
2394		}
2395
2396		dma_unmap_len_set(buffer, len, len);
2397		dma_unmap_addr_set(buffer, dma, dma);
2398
2399		cmd_type = IGC_ADVTXD_DTYP_DATA | IGC_ADVTXD_DCMD_DEXT |
2400			   IGC_ADVTXD_DCMD_IFCS | len;
2401
2402		desc->read.cmd_type_len = cpu_to_le32(cmd_type);
2403		desc->read.buffer_addr = cpu_to_le64(dma);
2404
2405		buffer->protocol = 0;
2406
2407		if (++index == ring->count)
2408			index = 0;
2409
2410		if (i == nr_frags)
2411			break;
2412
2413		buffer = &ring->tx_buffer_info[index];
2414		desc = IGC_TX_DESC(ring, index);
2415		desc->read.olinfo_status = 0;
2416
2417		data = skb_frag_address(&sinfo->frags[i]);
2418		len = skb_frag_size(&sinfo->frags[i]);
2419		i++;
2420	}
2421	desc->read.cmd_type_len |= cpu_to_le32(IGC_TXD_DCMD);
2422
2423	netdev_tx_sent_queue(txring_txq(ring), head->bytecount);
2424	/* set the timestamp */
2425	head->time_stamp = jiffies;
2426	/* set next_to_watch value indicating a packet is present */
2427	head->next_to_watch = desc;
2428	ring->next_to_use = index;
2429
2430	return 0;
2431
2432unmap:
2433	for (;;) {
2434		buffer = &ring->tx_buffer_info[index];
2435		if (dma_unmap_len(buffer, len))
2436			dma_unmap_page(ring->dev,
2437				       dma_unmap_addr(buffer, dma),
2438				       dma_unmap_len(buffer, len),
2439				       DMA_TO_DEVICE);
2440		dma_unmap_len_set(buffer, len, 0);
2441		if (buffer == head)
2442			break;
2443
2444		if (!index)
2445			index += ring->count;
2446		index--;
2447	}
2448
2449	return -ENOMEM;
2450}
2451
2452static struct igc_ring *igc_xdp_get_tx_ring(struct igc_adapter *adapter,
2453					    int cpu)
2454{
2455	int index = cpu;
2456
2457	if (unlikely(index < 0))
2458		index = 0;
2459
2460	while (index >= adapter->num_tx_queues)
2461		index -= adapter->num_tx_queues;
2462
2463	return adapter->tx_ring[index];
2464}
2465
2466static int igc_xdp_xmit_back(struct igc_adapter *adapter, struct xdp_buff *xdp)
2467{
2468	struct xdp_frame *xdpf = xdp_convert_buff_to_frame(xdp);
2469	int cpu = smp_processor_id();
2470	struct netdev_queue *nq;
2471	struct igc_ring *ring;
2472	int res;
2473
2474	if (unlikely(!xdpf))
2475		return -EFAULT;
2476
2477	ring = igc_xdp_get_tx_ring(adapter, cpu);
2478	nq = txring_txq(ring);
2479
2480	__netif_tx_lock(nq, cpu);
2481	/* Avoid transmit queue timeout since we share it with the slow path */
2482	txq_trans_cond_update(nq);
2483	res = igc_xdp_init_tx_descriptor(ring, xdpf);
2484	__netif_tx_unlock(nq);
2485	return res;
2486}
2487
2488/* This function assumes rcu_read_lock() is held by the caller. */
2489static int __igc_xdp_run_prog(struct igc_adapter *adapter,
2490			      struct bpf_prog *prog,
2491			      struct xdp_buff *xdp)
2492{
2493	u32 act = bpf_prog_run_xdp(prog, xdp);
2494
2495	switch (act) {
2496	case XDP_PASS:
2497		return IGC_XDP_PASS;
2498	case XDP_TX:
2499		if (igc_xdp_xmit_back(adapter, xdp) < 0)
2500			goto out_failure;
2501		return IGC_XDP_TX;
2502	case XDP_REDIRECT:
2503		if (xdp_do_redirect(adapter->netdev, xdp, prog) < 0)
2504			goto out_failure;
2505		return IGC_XDP_REDIRECT;
2506		break;
2507	default:
2508		bpf_warn_invalid_xdp_action(adapter->netdev, prog, act);
2509		fallthrough;
2510	case XDP_ABORTED:
2511out_failure:
2512		trace_xdp_exception(adapter->netdev, prog, act);
2513		fallthrough;
2514	case XDP_DROP:
2515		return IGC_XDP_CONSUMED;
2516	}
2517}
2518
2519static struct sk_buff *igc_xdp_run_prog(struct igc_adapter *adapter,
2520					struct xdp_buff *xdp)
2521{
2522	struct bpf_prog *prog;
2523	int res;
2524
2525	prog = READ_ONCE(adapter->xdp_prog);
2526	if (!prog) {
2527		res = IGC_XDP_PASS;
2528		goto out;
2529	}
2530
2531	res = __igc_xdp_run_prog(adapter, prog, xdp);
2532
2533out:
2534	return ERR_PTR(-res);
2535}
2536
2537/* This function assumes __netif_tx_lock is held by the caller. */
2538static void igc_flush_tx_descriptors(struct igc_ring *ring)
2539{
2540	/* Once tail pointer is updated, hardware can fetch the descriptors
2541	 * any time so we issue a write membar here to ensure all memory
2542	 * writes are complete before the tail pointer is updated.
2543	 */
2544	wmb();
2545	writel(ring->next_to_use, ring->tail);
2546}
2547
2548static void igc_finalize_xdp(struct igc_adapter *adapter, int status)
2549{
2550	int cpu = smp_processor_id();
2551	struct netdev_queue *nq;
2552	struct igc_ring *ring;
2553
2554	if (status & IGC_XDP_TX) {
2555		ring = igc_xdp_get_tx_ring(adapter, cpu);
2556		nq = txring_txq(ring);
2557
2558		__netif_tx_lock(nq, cpu);
2559		igc_flush_tx_descriptors(ring);
2560		__netif_tx_unlock(nq);
2561	}
2562
2563	if (status & IGC_XDP_REDIRECT)
2564		xdp_do_flush();
2565}
2566
2567static void igc_update_rx_stats(struct igc_q_vector *q_vector,
2568				unsigned int packets, unsigned int bytes)
2569{
2570	struct igc_ring *ring = q_vector->rx.ring;
2571
2572	u64_stats_update_begin(&ring->rx_syncp);
2573	ring->rx_stats.packets += packets;
2574	ring->rx_stats.bytes += bytes;
2575	u64_stats_update_end(&ring->rx_syncp);
2576
2577	q_vector->rx.total_packets += packets;
2578	q_vector->rx.total_bytes += bytes;
2579}
2580
2581static int igc_clean_rx_irq(struct igc_q_vector *q_vector, const int budget)
2582{
2583	unsigned int total_bytes = 0, total_packets = 0;
2584	struct igc_adapter *adapter = q_vector->adapter;
2585	struct igc_ring *rx_ring = q_vector->rx.ring;
2586	struct sk_buff *skb = rx_ring->skb;
2587	u16 cleaned_count = igc_desc_unused(rx_ring);
2588	int xdp_status = 0, rx_buffer_pgcnt;
2589
2590	while (likely(total_packets < budget)) {
2591		struct igc_xdp_buff ctx = { .rx_ts = NULL };
2592		struct igc_rx_buffer *rx_buffer;
2593		union igc_adv_rx_desc *rx_desc;
2594		unsigned int size, truesize;
2595		int pkt_offset = 0;
2596		void *pktbuf;
2597
2598		/* return some buffers to hardware, one at a time is too slow */
2599		if (cleaned_count >= IGC_RX_BUFFER_WRITE) {
2600			igc_alloc_rx_buffers(rx_ring, cleaned_count);
2601			cleaned_count = 0;
2602		}
2603
2604		rx_desc = IGC_RX_DESC(rx_ring, rx_ring->next_to_clean);
2605		size = le16_to_cpu(rx_desc->wb.upper.length);
2606		if (!size)
2607			break;
2608
2609		/* This memory barrier is needed to keep us from reading
2610		 * any other fields out of the rx_desc until we know the
2611		 * descriptor has been written back
2612		 */
2613		dma_rmb();
2614
2615		rx_buffer = igc_get_rx_buffer(rx_ring, size, &rx_buffer_pgcnt);
2616		truesize = igc_get_rx_frame_truesize(rx_ring, size);
2617
2618		pktbuf = page_address(rx_buffer->page) + rx_buffer->page_offset;
2619
2620		if (igc_test_staterr(rx_desc, IGC_RXDADV_STAT_TSIP)) {
2621			ctx.rx_ts = pktbuf;
2622			pkt_offset = IGC_TS_HDR_LEN;
2623			size -= IGC_TS_HDR_LEN;
2624		}
2625
2626		if (!skb) {
2627			xdp_init_buff(&ctx.xdp, truesize, &rx_ring->xdp_rxq);
2628			xdp_prepare_buff(&ctx.xdp, pktbuf - igc_rx_offset(rx_ring),
2629					 igc_rx_offset(rx_ring) + pkt_offset,
2630					 size, true);
2631			xdp_buff_clear_frags_flag(&ctx.xdp);
2632			ctx.rx_desc = rx_desc;
2633
2634			skb = igc_xdp_run_prog(adapter, &ctx.xdp);
2635		}
2636
2637		if (IS_ERR(skb)) {
2638			unsigned int xdp_res = -PTR_ERR(skb);
2639
2640			switch (xdp_res) {
2641			case IGC_XDP_CONSUMED:
2642				rx_buffer->pagecnt_bias++;
2643				break;
2644			case IGC_XDP_TX:
2645			case IGC_XDP_REDIRECT:
2646				igc_rx_buffer_flip(rx_buffer, truesize);
2647				xdp_status |= xdp_res;
2648				break;
2649			}
2650
2651			total_packets++;
2652			total_bytes += size;
2653		} else if (skb)
2654			igc_add_rx_frag(rx_ring, rx_buffer, skb, size);
2655		else if (ring_uses_build_skb(rx_ring))
2656			skb = igc_build_skb(rx_ring, rx_buffer, &ctx.xdp);
2657		else
2658			skb = igc_construct_skb(rx_ring, rx_buffer, &ctx);
2659
2660		/* exit if we failed to retrieve a buffer */
2661		if (!skb) {
2662			rx_ring->rx_stats.alloc_failed++;
2663			rx_buffer->pagecnt_bias++;
2664			set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags);
2665			break;
2666		}
2667
2668		igc_put_rx_buffer(rx_ring, rx_buffer, rx_buffer_pgcnt);
2669		cleaned_count++;
2670
2671		/* fetch next buffer in frame if non-eop */
2672		if (igc_is_non_eop(rx_ring, rx_desc))
2673			continue;
2674
2675		/* verify the packet layout is correct */
2676		if (igc_cleanup_headers(rx_ring, rx_desc, skb)) {
2677			skb = NULL;
2678			continue;
2679		}
2680
2681		/* probably a little skewed due to removing CRC */
2682		total_bytes += skb->len;
2683
2684		/* populate checksum, VLAN, and protocol */
2685		igc_process_skb_fields(rx_ring, rx_desc, skb);
2686
2687		napi_gro_receive(&q_vector->napi, skb);
2688
2689		/* reset skb pointer */
2690		skb = NULL;
2691
2692		/* update budget accounting */
2693		total_packets++;
2694	}
2695
2696	if (xdp_status)
2697		igc_finalize_xdp(adapter, xdp_status);
2698
2699	/* place incomplete frames back on ring for completion */
2700	rx_ring->skb = skb;
2701
2702	igc_update_rx_stats(q_vector, total_packets, total_bytes);
2703
2704	if (cleaned_count)
2705		igc_alloc_rx_buffers(rx_ring, cleaned_count);
2706
2707	return total_packets;
2708}
2709
2710static struct sk_buff *igc_construct_skb_zc(struct igc_ring *ring,
2711					    struct igc_xdp_buff *ctx)
2712{
2713	struct xdp_buff *xdp = &ctx->xdp;
2714	unsigned int totalsize = xdp->data_end - xdp->data_meta;
2715	unsigned int metasize = xdp->data - xdp->data_meta;
2716	struct sk_buff *skb;
2717
2718	net_prefetch(xdp->data_meta);
2719
2720	skb = napi_alloc_skb(&ring->q_vector->napi, totalsize);
2721	if (unlikely(!skb))
2722		return NULL;
2723
2724	memcpy(__skb_put(skb, totalsize), xdp->data_meta,
2725	       ALIGN(totalsize, sizeof(long)));
2726
2727	if (metasize) {
2728		skb_metadata_set(skb, metasize);
2729		__skb_pull(skb, metasize);
2730	}
2731
2732	if (ctx->rx_ts) {
2733		skb_shinfo(skb)->tx_flags |= SKBTX_HW_TSTAMP_NETDEV;
2734		skb_hwtstamps(skb)->netdev_data = ctx->rx_ts;
2735	}
2736
2737	return skb;
2738}
2739
2740static void igc_dispatch_skb_zc(struct igc_q_vector *q_vector,
2741				union igc_adv_rx_desc *desc,
2742				struct igc_xdp_buff *ctx)
2743{
2744	struct igc_ring *ring = q_vector->rx.ring;
2745	struct sk_buff *skb;
2746
2747	skb = igc_construct_skb_zc(ring, ctx);
2748	if (!skb) {
2749		ring->rx_stats.alloc_failed++;
2750		set_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &ring->flags);
2751		return;
2752	}
2753
2754	if (igc_cleanup_headers(ring, desc, skb))
2755		return;
2756
2757	igc_process_skb_fields(ring, desc, skb);
2758	napi_gro_receive(&q_vector->napi, skb);
2759}
2760
2761static struct igc_xdp_buff *xsk_buff_to_igc_ctx(struct xdp_buff *xdp)
2762{
2763	/* xdp_buff pointer used by ZC code path is alloc as xdp_buff_xsk. The
2764	 * igc_xdp_buff shares its layout with xdp_buff_xsk and private
2765	 * igc_xdp_buff fields fall into xdp_buff_xsk->cb
2766	 */
2767       return (struct igc_xdp_buff *)xdp;
2768}
2769
2770static int igc_clean_rx_irq_zc(struct igc_q_vector *q_vector, const int budget)
2771{
2772	struct igc_adapter *adapter = q_vector->adapter;
2773	struct igc_ring *ring = q_vector->rx.ring;
2774	u16 cleaned_count = igc_desc_unused(ring);
2775	int total_bytes = 0, total_packets = 0;
2776	u16 ntc = ring->next_to_clean;
2777	struct bpf_prog *prog;
2778	bool failure = false;
2779	int xdp_status = 0;
2780
2781	rcu_read_lock();
2782
2783	prog = READ_ONCE(adapter->xdp_prog);
2784
2785	while (likely(total_packets < budget)) {
2786		union igc_adv_rx_desc *desc;
2787		struct igc_rx_buffer *bi;
2788		struct igc_xdp_buff *ctx;
2789		unsigned int size;
2790		int res;
2791
2792		desc = IGC_RX_DESC(ring, ntc);
2793		size = le16_to_cpu(desc->wb.upper.length);
2794		if (!size)
2795			break;
2796
2797		/* This memory barrier is needed to keep us from reading
2798		 * any other fields out of the rx_desc until we know the
2799		 * descriptor has been written back
2800		 */
2801		dma_rmb();
2802
2803		bi = &ring->rx_buffer_info[ntc];
2804
2805		ctx = xsk_buff_to_igc_ctx(bi->xdp);
2806		ctx->rx_desc = desc;
2807
2808		if (igc_test_staterr(desc, IGC_RXDADV_STAT_TSIP)) {
2809			ctx->rx_ts = bi->xdp->data;
2810
2811			bi->xdp->data += IGC_TS_HDR_LEN;
2812
2813			/* HW timestamp has been copied into local variable. Metadata
2814			 * length when XDP program is called should be 0.
2815			 */
2816			bi->xdp->data_meta += IGC_TS_HDR_LEN;
2817			size -= IGC_TS_HDR_LEN;
2818		} else {
2819			ctx->rx_ts = NULL;
2820		}
2821
2822		bi->xdp->data_end = bi->xdp->data + size;
2823		xsk_buff_dma_sync_for_cpu(bi->xdp);
2824
2825		res = __igc_xdp_run_prog(adapter, prog, bi->xdp);
2826		switch (res) {
2827		case IGC_XDP_PASS:
2828			igc_dispatch_skb_zc(q_vector, desc, ctx);
2829			fallthrough;
2830		case IGC_XDP_CONSUMED:
2831			xsk_buff_free(bi->xdp);
2832			break;
2833		case IGC_XDP_TX:
2834		case IGC_XDP_REDIRECT:
2835			xdp_status |= res;
2836			break;
2837		}
2838
2839		bi->xdp = NULL;
2840		total_bytes += size;
2841		total_packets++;
2842		cleaned_count++;
2843		ntc++;
2844		if (ntc == ring->count)
2845			ntc = 0;
2846	}
2847
2848	ring->next_to_clean = ntc;
2849	rcu_read_unlock();
2850
2851	if (cleaned_count >= IGC_RX_BUFFER_WRITE)
2852		failure = !igc_alloc_rx_buffers_zc(ring, cleaned_count);
2853
2854	if (xdp_status)
2855		igc_finalize_xdp(adapter, xdp_status);
2856
2857	igc_update_rx_stats(q_vector, total_packets, total_bytes);
2858
2859	if (xsk_uses_need_wakeup(ring->xsk_pool)) {
2860		if (failure || ring->next_to_clean == ring->next_to_use)
2861			xsk_set_rx_need_wakeup(ring->xsk_pool);
2862		else
2863			xsk_clear_rx_need_wakeup(ring->xsk_pool);
2864		return total_packets;
2865	}
2866
2867	return failure ? budget : total_packets;
2868}
2869
2870static void igc_update_tx_stats(struct igc_q_vector *q_vector,
2871				unsigned int packets, unsigned int bytes)
2872{
2873	struct igc_ring *ring = q_vector->tx.ring;
2874
2875	u64_stats_update_begin(&ring->tx_syncp);
2876	ring->tx_stats.bytes += bytes;
2877	ring->tx_stats.packets += packets;
2878	u64_stats_update_end(&ring->tx_syncp);
2879
2880	q_vector->tx.total_bytes += bytes;
2881	q_vector->tx.total_packets += packets;
2882}
2883
2884static void igc_xsk_request_timestamp(void *_priv)
2885{
2886	struct igc_metadata_request *meta_req = _priv;
2887	struct igc_ring *tx_ring = meta_req->tx_ring;
2888	struct igc_tx_timestamp_request *tstamp;
2889	u32 tx_flags = IGC_TX_FLAGS_TSTAMP;
2890	struct igc_adapter *adapter;
2891	unsigned long lock_flags;
2892	bool found = false;
2893	int i;
2894
2895	if (test_bit(IGC_RING_FLAG_TX_HWTSTAMP, &tx_ring->flags)) {
2896		adapter = netdev_priv(tx_ring->netdev);
2897
2898		spin_lock_irqsave(&adapter->ptp_tx_lock, lock_flags);
2899
2900		/* Search for available tstamp regs */
2901		for (i = 0; i < IGC_MAX_TX_TSTAMP_REGS; i++) {
2902			tstamp = &adapter->tx_tstamp[i];
2903
2904			/* tstamp->skb and tstamp->xsk_tx_buffer are in union.
2905			 * When tstamp->skb is equal to NULL,
2906			 * tstamp->xsk_tx_buffer is equal to NULL as well.
2907			 * This condition means that the particular tstamp reg
2908			 * is not occupied by other packet.
2909			 */
2910			if (!tstamp->skb) {
2911				found = true;
2912				break;
2913			}
2914		}
2915
2916		/* Return if no available tstamp regs */
2917		if (!found) {
2918			adapter->tx_hwtstamp_skipped++;
2919			spin_unlock_irqrestore(&adapter->ptp_tx_lock,
2920					       lock_flags);
2921			return;
2922		}
2923
2924		tstamp->start = jiffies;
2925		tstamp->xsk_queue_index = tx_ring->queue_index;
2926		tstamp->xsk_tx_buffer = meta_req->tx_buffer;
2927		tstamp->buffer_type = IGC_TX_BUFFER_TYPE_XSK;
2928
2929		/* Hold the transmit completion until timestamp is ready */
2930		meta_req->tx_buffer->xsk_pending_ts = true;
2931
2932		/* Keep the pointer to tx_timestamp, which is located in XDP
2933		 * metadata area. It is the location to store the value of
2934		 * tx hardware timestamp.
2935		 */
2936		xsk_tx_metadata_to_compl(meta_req->meta, &tstamp->xsk_meta);
2937
2938		/* Set timestamp bit based on the _TSTAMP(_X) bit. */
2939		tx_flags |= tstamp->flags;
2940		meta_req->cmd_type |= IGC_SET_FLAG(tx_flags,
2941						   IGC_TX_FLAGS_TSTAMP,
2942						   (IGC_ADVTXD_MAC_TSTAMP));
2943		meta_req->cmd_type |= IGC_SET_FLAG(tx_flags,
2944						   IGC_TX_FLAGS_TSTAMP_1,
2945						   (IGC_ADVTXD_TSTAMP_REG_1));
2946		meta_req->cmd_type |= IGC_SET_FLAG(tx_flags,
2947						   IGC_TX_FLAGS_TSTAMP_2,
2948						   (IGC_ADVTXD_TSTAMP_REG_2));
2949		meta_req->cmd_type |= IGC_SET_FLAG(tx_flags,
2950						   IGC_TX_FLAGS_TSTAMP_3,
2951						   (IGC_ADVTXD_TSTAMP_REG_3));
2952
2953		spin_unlock_irqrestore(&adapter->ptp_tx_lock, lock_flags);
2954	}
2955}
2956
2957static u64 igc_xsk_fill_timestamp(void *_priv)
2958{
2959	return *(u64 *)_priv;
2960}
2961
2962const struct xsk_tx_metadata_ops igc_xsk_tx_metadata_ops = {
2963	.tmo_request_timestamp		= igc_xsk_request_timestamp,
2964	.tmo_fill_timestamp		= igc_xsk_fill_timestamp,
2965};
2966
2967static void igc_xdp_xmit_zc(struct igc_ring *ring)
2968{
2969	struct xsk_buff_pool *pool = ring->xsk_pool;
2970	struct netdev_queue *nq = txring_txq(ring);
2971	union igc_adv_tx_desc *tx_desc = NULL;
2972	int cpu = smp_processor_id();
2973	struct xdp_desc xdp_desc;
2974	u16 budget, ntu;
2975
2976	if (!netif_carrier_ok(ring->netdev))
2977		return;
2978
2979	__netif_tx_lock(nq, cpu);
2980
2981	/* Avoid transmit queue timeout since we share it with the slow path */
2982	txq_trans_cond_update(nq);
2983
2984	ntu = ring->next_to_use;
2985	budget = igc_desc_unused(ring);
2986
2987	while (xsk_tx_peek_desc(pool, &xdp_desc) && budget--) {
2988		struct igc_metadata_request meta_req;
2989		struct xsk_tx_metadata *meta = NULL;
2990		struct igc_tx_buffer *bi;
2991		u32 olinfo_status;
2992		dma_addr_t dma;
2993
2994		meta_req.cmd_type = IGC_ADVTXD_DTYP_DATA |
2995				    IGC_ADVTXD_DCMD_DEXT |
2996				    IGC_ADVTXD_DCMD_IFCS |
2997				    IGC_TXD_DCMD | xdp_desc.len;
2998		olinfo_status = xdp_desc.len << IGC_ADVTXD_PAYLEN_SHIFT;
2999
3000		dma = xsk_buff_raw_get_dma(pool, xdp_desc.addr);
3001		meta = xsk_buff_get_metadata(pool, xdp_desc.addr);
3002		xsk_buff_raw_dma_sync_for_device(pool, dma, xdp_desc.len);
3003		bi = &ring->tx_buffer_info[ntu];
3004
3005		meta_req.tx_ring = ring;
3006		meta_req.tx_buffer = bi;
3007		meta_req.meta = meta;
3008		xsk_tx_metadata_request(meta, &igc_xsk_tx_metadata_ops,
3009					&meta_req);
3010
3011		tx_desc = IGC_TX_DESC(ring, ntu);
3012		tx_desc->read.cmd_type_len = cpu_to_le32(meta_req.cmd_type);
3013		tx_desc->read.olinfo_status = cpu_to_le32(olinfo_status);
3014		tx_desc->read.buffer_addr = cpu_to_le64(dma);
3015
3016		bi->type = IGC_TX_BUFFER_TYPE_XSK;
3017		bi->protocol = 0;
3018		bi->bytecount = xdp_desc.len;
3019		bi->gso_segs = 1;
3020		bi->time_stamp = jiffies;
3021		bi->next_to_watch = tx_desc;
3022
3023		netdev_tx_sent_queue(txring_txq(ring), xdp_desc.len);
3024
3025		ntu++;
3026		if (ntu == ring->count)
3027			ntu = 0;
3028	}
3029
3030	ring->next_to_use = ntu;
3031	if (tx_desc) {
3032		igc_flush_tx_descriptors(ring);
3033		xsk_tx_release(pool);
3034	}
3035
3036	__netif_tx_unlock(nq);
3037}
3038
3039/**
3040 * igc_clean_tx_irq - Reclaim resources after transmit completes
3041 * @q_vector: pointer to q_vector containing needed info
3042 * @napi_budget: Used to determine if we are in netpoll
3043 *
3044 * returns true if ring is completely cleaned
3045 */
3046static bool igc_clean_tx_irq(struct igc_q_vector *q_vector, int napi_budget)
3047{
3048	struct igc_adapter *adapter = q_vector->adapter;
3049	unsigned int total_bytes = 0, total_packets = 0;
3050	unsigned int budget = q_vector->tx.work_limit;
3051	struct igc_ring *tx_ring = q_vector->tx.ring;
3052	unsigned int i = tx_ring->next_to_clean;
3053	struct igc_tx_buffer *tx_buffer;
3054	union igc_adv_tx_desc *tx_desc;
3055	u32 xsk_frames = 0;
3056
3057	if (test_bit(__IGC_DOWN, &adapter->state))
3058		return true;
3059
3060	tx_buffer = &tx_ring->tx_buffer_info[i];
3061	tx_desc = IGC_TX_DESC(tx_ring, i);
3062	i -= tx_ring->count;
3063
3064	do {
3065		union igc_adv_tx_desc *eop_desc = tx_buffer->next_to_watch;
3066
3067		/* if next_to_watch is not set then there is no work pending */
3068		if (!eop_desc)
3069			break;
3070
3071		/* prevent any other reads prior to eop_desc */
3072		smp_rmb();
3073
3074		/* if DD is not set pending work has not been completed */
3075		if (!(eop_desc->wb.status & cpu_to_le32(IGC_TXD_STAT_DD)))
3076			break;
3077
3078		/* Hold the completions while there's a pending tx hardware
3079		 * timestamp request from XDP Tx metadata.
3080		 */
3081		if (tx_buffer->type == IGC_TX_BUFFER_TYPE_XSK &&
3082		    tx_buffer->xsk_pending_ts)
3083			break;
3084
3085		/* clear next_to_watch to prevent false hangs */
3086		tx_buffer->next_to_watch = NULL;
3087
3088		/* update the statistics for this packet */
3089		total_bytes += tx_buffer->bytecount;
3090		total_packets += tx_buffer->gso_segs;
3091
3092		switch (tx_buffer->type) {
3093		case IGC_TX_BUFFER_TYPE_XSK:
3094			xsk_frames++;
3095			break;
3096		case IGC_TX_BUFFER_TYPE_XDP:
3097			xdp_return_frame(tx_buffer->xdpf);
3098			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
3099			break;
3100		case IGC_TX_BUFFER_TYPE_SKB:
3101			napi_consume_skb(tx_buffer->skb, napi_budget);
3102			igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
3103			break;
3104		default:
3105			netdev_warn_once(tx_ring->netdev, "Unknown Tx buffer type\n");
3106			break;
3107		}
3108
3109		/* clear last DMA location and unmap remaining buffers */
3110		while (tx_desc != eop_desc) {
3111			tx_buffer++;
3112			tx_desc++;
3113			i++;
3114			if (unlikely(!i)) {
3115				i -= tx_ring->count;
3116				tx_buffer = tx_ring->tx_buffer_info;
3117				tx_desc = IGC_TX_DESC(tx_ring, 0);
3118			}
3119
3120			/* unmap any remaining paged data */
3121			if (dma_unmap_len(tx_buffer, len))
3122				igc_unmap_tx_buffer(tx_ring->dev, tx_buffer);
3123		}
3124
3125		/* move us one more past the eop_desc for start of next pkt */
3126		tx_buffer++;
3127		tx_desc++;
3128		i++;
3129		if (unlikely(!i)) {
3130			i -= tx_ring->count;
3131			tx_buffer = tx_ring->tx_buffer_info;
3132			tx_desc = IGC_TX_DESC(tx_ring, 0);
3133		}
3134
3135		/* issue prefetch for next Tx descriptor */
3136		prefetch(tx_desc);
3137
3138		/* update budget accounting */
3139		budget--;
3140	} while (likely(budget));
3141
3142	netdev_tx_completed_queue(txring_txq(tx_ring),
3143				  total_packets, total_bytes);
3144
3145	i += tx_ring->count;
3146	tx_ring->next_to_clean = i;
3147
3148	igc_update_tx_stats(q_vector, total_packets, total_bytes);
3149
3150	if (tx_ring->xsk_pool) {
3151		if (xsk_frames)
3152			xsk_tx_completed(tx_ring->xsk_pool, xsk_frames);
3153		if (xsk_uses_need_wakeup(tx_ring->xsk_pool))
3154			xsk_set_tx_need_wakeup(tx_ring->xsk_pool);
3155		igc_xdp_xmit_zc(tx_ring);
3156	}
3157
3158	if (test_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags)) {
3159		struct igc_hw *hw = &adapter->hw;
3160
3161		/* Detect a transmit hang in hardware, this serializes the
3162		 * check with the clearing of time_stamp and movement of i
3163		 */
3164		clear_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
3165		if (tx_buffer->next_to_watch &&
3166		    time_after(jiffies, tx_buffer->time_stamp +
3167		    (adapter->tx_timeout_factor * HZ)) &&
3168		    !(rd32(IGC_STATUS) & IGC_STATUS_TXOFF) &&
3169		    (rd32(IGC_TDH(tx_ring->reg_idx)) != readl(tx_ring->tail)) &&
3170		    !tx_ring->oper_gate_closed) {
3171			/* detected Tx unit hang */
3172			netdev_err(tx_ring->netdev,
3173				   "Detected Tx Unit Hang\n"
3174				   "  Tx Queue             <%d>\n"
3175				   "  TDH                  <%x>\n"
3176				   "  TDT                  <%x>\n"
3177				   "  next_to_use          <%x>\n"
3178				   "  next_to_clean        <%x>\n"
3179				   "buffer_info[next_to_clean]\n"
3180				   "  time_stamp           <%lx>\n"
3181				   "  next_to_watch        <%p>\n"
3182				   "  jiffies              <%lx>\n"
3183				   "  desc.status          <%x>\n",
3184				   tx_ring->queue_index,
3185				   rd32(IGC_TDH(tx_ring->reg_idx)),
3186				   readl(tx_ring->tail),
3187				   tx_ring->next_to_use,
3188				   tx_ring->next_to_clean,
3189				   tx_buffer->time_stamp,
3190				   tx_buffer->next_to_watch,
3191				   jiffies,
3192				   tx_buffer->next_to_watch->wb.status);
3193			netif_stop_subqueue(tx_ring->netdev,
3194					    tx_ring->queue_index);
3195
3196			/* we are about to reset, no point in enabling stuff */
3197			return true;
3198		}
3199	}
3200
3201#define TX_WAKE_THRESHOLD (DESC_NEEDED * 2)
3202	if (unlikely(total_packets &&
3203		     netif_carrier_ok(tx_ring->netdev) &&
3204		     igc_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD)) {
3205		/* Make sure that anybody stopping the queue after this
3206		 * sees the new next_to_clean.
3207		 */
3208		smp_mb();
3209		if (__netif_subqueue_stopped(tx_ring->netdev,
3210					     tx_ring->queue_index) &&
3211		    !(test_bit(__IGC_DOWN, &adapter->state))) {
3212			netif_wake_subqueue(tx_ring->netdev,
3213					    tx_ring->queue_index);
3214
3215			u64_stats_update_begin(&tx_ring->tx_syncp);
3216			tx_ring->tx_stats.restart_queue++;
3217			u64_stats_update_end(&tx_ring->tx_syncp);
3218		}
3219	}
3220
3221	return !!budget;
3222}
3223
3224static int igc_find_mac_filter(struct igc_adapter *adapter,
3225			       enum igc_mac_filter_type type, const u8 *addr)
3226{
3227	struct igc_hw *hw = &adapter->hw;
3228	int max_entries = hw->mac.rar_entry_count;
3229	u32 ral, rah;
3230	int i;
3231
3232	for (i = 0; i < max_entries; i++) {
3233		ral = rd32(IGC_RAL(i));
3234		rah = rd32(IGC_RAH(i));
3235
3236		if (!(rah & IGC_RAH_AV))
3237			continue;
3238		if (!!(rah & IGC_RAH_ASEL_SRC_ADDR) != type)
3239			continue;
3240		if ((rah & IGC_RAH_RAH_MASK) !=
3241		    le16_to_cpup((__le16 *)(addr + 4)))
3242			continue;
3243		if (ral != le32_to_cpup((__le32 *)(addr)))
3244			continue;
3245
3246		return i;
3247	}
3248
3249	return -1;
3250}
3251
3252static int igc_get_avail_mac_filter_slot(struct igc_adapter *adapter)
3253{
3254	struct igc_hw *hw = &adapter->hw;
3255	int max_entries = hw->mac.rar_entry_count;
3256	u32 rah;
3257	int i;
3258
3259	for (i = 0; i < max_entries; i++) {
3260		rah = rd32(IGC_RAH(i));
3261
3262		if (!(rah & IGC_RAH_AV))
3263			return i;
3264	}
3265
3266	return -1;
3267}
3268
3269/**
3270 * igc_add_mac_filter() - Add MAC address filter
3271 * @adapter: Pointer to adapter where the filter should be added
3272 * @type: MAC address filter type (source or destination)
3273 * @addr: MAC address
3274 * @queue: If non-negative, queue assignment feature is enabled and frames
3275 *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3276 *         assignment is disabled.
3277 *
3278 * Return: 0 in case of success, negative errno code otherwise.
3279 */
3280static int igc_add_mac_filter(struct igc_adapter *adapter,
3281			      enum igc_mac_filter_type type, const u8 *addr,
3282			      int queue)
3283{
3284	struct net_device *dev = adapter->netdev;
3285	int index;
3286
3287	index = igc_find_mac_filter(adapter, type, addr);
3288	if (index >= 0)
3289		goto update_filter;
3290
3291	index = igc_get_avail_mac_filter_slot(adapter);
3292	if (index < 0)
3293		return -ENOSPC;
3294
3295	netdev_dbg(dev, "Add MAC address filter: index %d type %s address %pM queue %d\n",
3296		   index, type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3297		   addr, queue);
3298
3299update_filter:
3300	igc_set_mac_filter_hw(adapter, index, type, addr, queue);
3301	return 0;
3302}
3303
3304/**
3305 * igc_del_mac_filter() - Delete MAC address filter
3306 * @adapter: Pointer to adapter where the filter should be deleted from
3307 * @type: MAC address filter type (source or destination)
3308 * @addr: MAC address
3309 */
3310static void igc_del_mac_filter(struct igc_adapter *adapter,
3311			       enum igc_mac_filter_type type, const u8 *addr)
3312{
3313	struct net_device *dev = adapter->netdev;
3314	int index;
3315
3316	index = igc_find_mac_filter(adapter, type, addr);
3317	if (index < 0)
3318		return;
3319
3320	if (index == 0) {
3321		/* If this is the default filter, we don't actually delete it.
3322		 * We just reset to its default value i.e. disable queue
3323		 * assignment.
3324		 */
3325		netdev_dbg(dev, "Disable default MAC filter queue assignment");
3326
3327		igc_set_mac_filter_hw(adapter, 0, type, addr, -1);
3328	} else {
3329		netdev_dbg(dev, "Delete MAC address filter: index %d type %s address %pM\n",
3330			   index,
3331			   type == IGC_MAC_FILTER_TYPE_DST ? "dst" : "src",
3332			   addr);
3333
3334		igc_clear_mac_filter_hw(adapter, index);
3335	}
3336}
3337
3338/**
3339 * igc_add_vlan_prio_filter() - Add VLAN priority filter
3340 * @adapter: Pointer to adapter where the filter should be added
3341 * @prio: VLAN priority value
3342 * @queue: Queue number which matching frames are assigned to
3343 *
3344 * Return: 0 in case of success, negative errno code otherwise.
3345 */
3346static int igc_add_vlan_prio_filter(struct igc_adapter *adapter, int prio,
3347				    int queue)
3348{
3349	struct net_device *dev = adapter->netdev;
3350	struct igc_hw *hw = &adapter->hw;
3351	u32 vlanpqf;
3352
3353	vlanpqf = rd32(IGC_VLANPQF);
3354
3355	if (vlanpqf & IGC_VLANPQF_VALID(prio)) {
3356		netdev_dbg(dev, "VLAN priority filter already in use\n");
3357		return -EEXIST;
3358	}
3359
3360	vlanpqf |= IGC_VLANPQF_QSEL(prio, queue);
3361	vlanpqf |= IGC_VLANPQF_VALID(prio);
3362
3363	wr32(IGC_VLANPQF, vlanpqf);
3364
3365	netdev_dbg(dev, "Add VLAN priority filter: prio %d queue %d\n",
3366		   prio, queue);
3367	return 0;
3368}
3369
3370/**
3371 * igc_del_vlan_prio_filter() - Delete VLAN priority filter
3372 * @adapter: Pointer to adapter where the filter should be deleted from
3373 * @prio: VLAN priority value
3374 */
3375static void igc_del_vlan_prio_filter(struct igc_adapter *adapter, int prio)
3376{
3377	struct igc_hw *hw = &adapter->hw;
3378	u32 vlanpqf;
3379
3380	vlanpqf = rd32(IGC_VLANPQF);
3381
3382	vlanpqf &= ~IGC_VLANPQF_VALID(prio);
3383	vlanpqf &= ~IGC_VLANPQF_QSEL(prio, IGC_VLANPQF_QUEUE_MASK);
3384
3385	wr32(IGC_VLANPQF, vlanpqf);
3386
3387	netdev_dbg(adapter->netdev, "Delete VLAN priority filter: prio %d\n",
3388		   prio);
3389}
3390
3391static int igc_get_avail_etype_filter_slot(struct igc_adapter *adapter)
3392{
3393	struct igc_hw *hw = &adapter->hw;
3394	int i;
3395
3396	for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3397		u32 etqf = rd32(IGC_ETQF(i));
3398
3399		if (!(etqf & IGC_ETQF_FILTER_ENABLE))
3400			return i;
3401	}
3402
3403	return -1;
3404}
3405
3406/**
3407 * igc_add_etype_filter() - Add ethertype filter
3408 * @adapter: Pointer to adapter where the filter should be added
3409 * @etype: Ethertype value
3410 * @queue: If non-negative, queue assignment feature is enabled and frames
3411 *         matching the filter are enqueued onto 'queue'. Otherwise, queue
3412 *         assignment is disabled.
3413 *
3414 * Return: 0 in case of success, negative errno code otherwise.
3415 */
3416static int igc_add_etype_filter(struct igc_adapter *adapter, u16 etype,
3417				int queue)
3418{
3419	struct igc_hw *hw = &adapter->hw;
3420	int index;
3421	u32 etqf;
3422
3423	index = igc_get_avail_etype_filter_slot(adapter);
3424	if (index < 0)
3425		return -ENOSPC;
3426
3427	etqf = rd32(IGC_ETQF(index));
3428
3429	etqf &= ~IGC_ETQF_ETYPE_MASK;
3430	etqf |= etype;
3431
3432	if (queue >= 0) {
3433		etqf &= ~IGC_ETQF_QUEUE_MASK;
3434		etqf |= (queue << IGC_ETQF_QUEUE_SHIFT);
3435		etqf |= IGC_ETQF_QUEUE_ENABLE;
3436	}
3437
3438	etqf |= IGC_ETQF_FILTER_ENABLE;
3439
3440	wr32(IGC_ETQF(index), etqf);
3441
3442	netdev_dbg(adapter->netdev, "Add ethertype filter: etype %04x queue %d\n",
3443		   etype, queue);
3444	return 0;
3445}
3446
3447static int igc_find_etype_filter(struct igc_adapter *adapter, u16 etype)
3448{
3449	struct igc_hw *hw = &adapter->hw;
3450	int i;
3451
3452	for (i = 0; i < MAX_ETYPE_FILTER; i++) {
3453		u32 etqf = rd32(IGC_ETQF(i));
3454
3455		if ((etqf & IGC_ETQF_ETYPE_MASK) == etype)
3456			return i;
3457	}
3458
3459	return -1;
3460}
3461
3462/**
3463 * igc_del_etype_filter() - Delete ethertype filter
3464 * @adapter: Pointer to adapter where the filter should be deleted from
3465 * @etype: Ethertype value
3466 */
3467static void igc_del_etype_filter(struct igc_adapter *adapter, u16 etype)
3468{
3469	struct igc_hw *hw = &adapter->hw;
3470	int index;
3471
3472	index = igc_find_etype_filter(adapter, etype);
3473	if (index < 0)
3474		return;
3475
3476	wr32(IGC_ETQF(index), 0);
3477
3478	netdev_dbg(adapter->netdev, "Delete ethertype filter: etype %04x\n",
3479		   etype);
3480}
3481
3482static int igc_flex_filter_select(struct igc_adapter *adapter,
3483				  struct igc_flex_filter *input,
3484				  u32 *fhft)
3485{
3486	struct igc_hw *hw = &adapter->hw;
3487	u8 fhft_index;
3488	u32 fhftsl;
3489
3490	if (input->index >= MAX_FLEX_FILTER) {
3491		netdev_err(adapter->netdev, "Wrong Flex Filter index selected!\n");
3492		return -EINVAL;
3493	}
3494
3495	/* Indirect table select register */
3496	fhftsl = rd32(IGC_FHFTSL);
3497	fhftsl &= ~IGC_FHFTSL_FTSL_MASK;
3498	switch (input->index) {
3499	case 0 ... 7:
3500		fhftsl |= 0x00;
3501		break;
3502	case 8 ... 15:
3503		fhftsl |= 0x01;
3504		break;
3505	case 16 ... 23:
3506		fhftsl |= 0x02;
3507		break;
3508	case 24 ... 31:
3509		fhftsl |= 0x03;
3510		break;
3511	}
3512	wr32(IGC_FHFTSL, fhftsl);
3513
3514	/* Normalize index down to host table register */
3515	fhft_index = input->index % 8;
3516
3517	*fhft = (fhft_index < 4) ? IGC_FHFT(fhft_index) :
3518		IGC_FHFT_EXT(fhft_index - 4);
3519
3520	return 0;
3521}
3522
3523static int igc_write_flex_filter_ll(struct igc_adapter *adapter,
3524				    struct igc_flex_filter *input)
3525{
3526	struct igc_hw *hw = &adapter->hw;
3527	u8 *data = input->data;
3528	u8 *mask = input->mask;
3529	u32 queuing;
3530	u32 fhft;
3531	u32 wufc;
3532	int ret;
3533	int i;
3534
3535	/* Length has to be aligned to 8. Otherwise the filter will fail. Bail
3536	 * out early to avoid surprises later.
3537	 */
3538	if (input->length % 8 != 0) {
3539		netdev_err(adapter->netdev, "The length of a flex filter has to be 8 byte aligned!\n");
3540		return -EINVAL;
3541	}
3542
3543	/* Select corresponding flex filter register and get base for host table. */
3544	ret = igc_flex_filter_select(adapter, input, &fhft);
3545	if (ret)
3546		return ret;
3547
3548	/* When adding a filter globally disable flex filter feature. That is
3549	 * recommended within the datasheet.
3550	 */
3551	wufc = rd32(IGC_WUFC);
3552	wufc &= ~IGC_WUFC_FLEX_HQ;
3553	wr32(IGC_WUFC, wufc);
3554
3555	/* Configure filter */
3556	queuing = input->length & IGC_FHFT_LENGTH_MASK;
3557	queuing |= FIELD_PREP(IGC_FHFT_QUEUE_MASK, input->rx_queue);
3558	queuing |= FIELD_PREP(IGC_FHFT_PRIO_MASK, input->prio);
3559
3560	if (input->immediate_irq)
3561		queuing |= IGC_FHFT_IMM_INT;
3562
3563	if (input->drop)
3564		queuing |= IGC_FHFT_DROP;
3565
3566	wr32(fhft + 0xFC, queuing);
3567
3568	/* Write data (128 byte) and mask (128 bit) */
3569	for (i = 0; i < 16; ++i) {
3570		const size_t data_idx = i * 8;
3571		const size_t row_idx = i * 16;
3572		u32 dw0 =
3573			(data[data_idx + 0] << 0) |
3574			(data[data_idx + 1] << 8) |
3575			(data[data_idx + 2] << 16) |
3576			(data[data_idx + 3] << 24);
3577		u32 dw1 =
3578			(data[data_idx + 4] << 0) |
3579			(data[data_idx + 5] << 8) |
3580			(data[data_idx + 6] << 16) |
3581			(data[data_idx + 7] << 24);
3582		u32 tmp;
3583
3584		/* Write row: dw0, dw1 and mask */
3585		wr32(fhft + row_idx, dw0);
3586		wr32(fhft + row_idx + 4, dw1);
3587
3588		/* mask is only valid for MASK(7, 0) */
3589		tmp = rd32(fhft + row_idx + 8);
3590		tmp &= ~GENMASK(7, 0);
3591		tmp |= mask[i];
3592		wr32(fhft + row_idx + 8, tmp);
3593	}
3594
3595	/* Enable filter. */
3596	wufc |= IGC_WUFC_FLEX_HQ;
3597	if (input->index > 8) {
3598		/* Filter 0-7 are enabled via WUFC. The other 24 filters are not. */
3599		u32 wufc_ext = rd32(IGC_WUFC_EXT);
3600
3601		wufc_ext |= (IGC_WUFC_EXT_FLX8 << (input->index - 8));
3602
3603		wr32(IGC_WUFC_EXT, wufc_ext);
3604	} else {
3605		wufc |= (IGC_WUFC_FLX0 << input->index);
3606	}
3607	wr32(IGC_WUFC, wufc);
3608
3609	netdev_dbg(adapter->netdev, "Added flex filter %u to HW.\n",
3610		   input->index);
3611
3612	return 0;
3613}
3614
3615static void igc_flex_filter_add_field(struct igc_flex_filter *flex,
3616				      const void *src, unsigned int offset,
3617				      size_t len, const void *mask)
3618{
3619	int i;
3620
3621	/* data */
3622	memcpy(&flex->data[offset], src, len);
3623
3624	/* mask */
3625	for (i = 0; i < len; ++i) {
3626		const unsigned int idx = i + offset;
3627		const u8 *ptr = mask;
3628
3629		if (mask) {
3630			if (ptr[i] & 0xff)
3631				flex->mask[idx / 8] |= BIT(idx % 8);
3632
3633			continue;
3634		}
3635
3636		flex->mask[idx / 8] |= BIT(idx % 8);
3637	}
3638}
3639
3640static int igc_find_avail_flex_filter_slot(struct igc_adapter *adapter)
3641{
3642	struct igc_hw *hw = &adapter->hw;
3643	u32 wufc, wufc_ext;
3644	int i;
3645
3646	wufc = rd32(IGC_WUFC);
3647	wufc_ext = rd32(IGC_WUFC_EXT);
3648
3649	for (i = 0; i < MAX_FLEX_FILTER; i++) {
3650		if (i < 8) {
3651			if (!(wufc & (IGC_WUFC_FLX0 << i)))
3652				return i;
3653		} else {
3654			if (!(wufc_ext & (IGC_WUFC_EXT_FLX8 << (i - 8))))
3655				return i;
3656		}
3657	}
3658
3659	return -ENOSPC;
3660}
3661
3662static bool igc_flex_filter_in_use(struct igc_adapter *adapter)
3663{
3664	struct igc_hw *hw = &adapter->hw;
3665	u32 wufc, wufc_ext;
3666
3667	wufc = rd32(IGC_WUFC);
3668	wufc_ext = rd32(IGC_WUFC_EXT);
3669
3670	if (wufc & IGC_WUFC_FILTER_MASK)
3671		return true;
3672
3673	if (wufc_ext & IGC_WUFC_EXT_FILTER_MASK)
3674		return true;
3675
3676	return false;
3677}
3678
3679static int igc_add_flex_filter(struct igc_adapter *adapter,
3680			       struct igc_nfc_rule *rule)
3681{
3682	struct igc_nfc_filter *filter = &rule->filter;
3683	unsigned int eth_offset, user_offset;
3684	struct igc_flex_filter flex = { };
3685	int ret, index;
3686	bool vlan;
3687
3688	index = igc_find_avail_flex_filter_slot(adapter);
3689	if (index < 0)
3690		return -ENOSPC;
3691
3692	/* Construct the flex filter:
3693	 *  -> dest_mac [6]
3694	 *  -> src_mac [6]
3695	 *  -> tpid [2]
3696	 *  -> vlan tci [2]
3697	 *  -> ether type [2]
3698	 *  -> user data [8]
3699	 *  -> = 26 bytes => 32 length
3700	 */
3701	flex.index    = index;
3702	flex.length   = 32;
3703	flex.rx_queue = rule->action;
3704
3705	vlan = rule->filter.vlan_tci || rule->filter.vlan_etype;
3706	eth_offset = vlan ? 16 : 12;
3707	user_offset = vlan ? 18 : 14;
3708
3709	/* Add destination MAC  */
3710	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3711		igc_flex_filter_add_field(&flex, &filter->dst_addr, 0,
3712					  ETH_ALEN, NULL);
3713
3714	/* Add source MAC */
3715	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3716		igc_flex_filter_add_field(&flex, &filter->src_addr, 6,
3717					  ETH_ALEN, NULL);
3718
3719	/* Add VLAN etype */
3720	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_ETYPE) {
3721		__be16 vlan_etype = cpu_to_be16(filter->vlan_etype);
3722
3723		igc_flex_filter_add_field(&flex, &vlan_etype, 12,
3724					  sizeof(vlan_etype), NULL);
3725	}
3726
3727	/* Add VLAN TCI */
3728	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI)
3729		igc_flex_filter_add_field(&flex, &filter->vlan_tci, 14,
3730					  sizeof(filter->vlan_tci), NULL);
3731
3732	/* Add Ether type */
3733	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3734		__be16 etype = cpu_to_be16(filter->etype);
3735
3736		igc_flex_filter_add_field(&flex, &etype, eth_offset,
3737					  sizeof(etype), NULL);
3738	}
3739
3740	/* Add user data */
3741	if (rule->filter.match_flags & IGC_FILTER_FLAG_USER_DATA)
3742		igc_flex_filter_add_field(&flex, &filter->user_data,
3743					  user_offset,
3744					  sizeof(filter->user_data),
3745					  filter->user_mask);
3746
3747	/* Add it down to the hardware and enable it. */
3748	ret = igc_write_flex_filter_ll(adapter, &flex);
3749	if (ret)
3750		return ret;
3751
3752	filter->flex_index = index;
3753
3754	return 0;
3755}
3756
3757static void igc_del_flex_filter(struct igc_adapter *adapter,
3758				u16 reg_index)
3759{
3760	struct igc_hw *hw = &adapter->hw;
3761	u32 wufc;
3762
3763	/* Just disable the filter. The filter table itself is kept
3764	 * intact. Another flex_filter_add() should override the "old" data
3765	 * then.
3766	 */
3767	if (reg_index > 8) {
3768		u32 wufc_ext = rd32(IGC_WUFC_EXT);
3769
3770		wufc_ext &= ~(IGC_WUFC_EXT_FLX8 << (reg_index - 8));
3771		wr32(IGC_WUFC_EXT, wufc_ext);
3772	} else {
3773		wufc = rd32(IGC_WUFC);
3774
3775		wufc &= ~(IGC_WUFC_FLX0 << reg_index);
3776		wr32(IGC_WUFC, wufc);
3777	}
3778
3779	if (igc_flex_filter_in_use(adapter))
3780		return;
3781
3782	/* No filters are in use, we may disable flex filters */
3783	wufc = rd32(IGC_WUFC);
3784	wufc &= ~IGC_WUFC_FLEX_HQ;
3785	wr32(IGC_WUFC, wufc);
3786}
3787
3788static int igc_enable_nfc_rule(struct igc_adapter *adapter,
3789			       struct igc_nfc_rule *rule)
3790{
3791	int err;
3792
3793	if (rule->flex) {
3794		return igc_add_flex_filter(adapter, rule);
3795	}
3796
3797	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE) {
3798		err = igc_add_etype_filter(adapter, rule->filter.etype,
3799					   rule->action);
3800		if (err)
3801			return err;
3802	}
3803
3804	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR) {
3805		err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3806					 rule->filter.src_addr, rule->action);
3807		if (err)
3808			return err;
3809	}
3810
3811	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR) {
3812		err = igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3813					 rule->filter.dst_addr, rule->action);
3814		if (err)
3815			return err;
3816	}
3817
3818	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3819		int prio = FIELD_GET(VLAN_PRIO_MASK, rule->filter.vlan_tci);
3820
3821		err = igc_add_vlan_prio_filter(adapter, prio, rule->action);
3822		if (err)
3823			return err;
3824	}
3825
3826	return 0;
3827}
3828
3829static void igc_disable_nfc_rule(struct igc_adapter *adapter,
3830				 const struct igc_nfc_rule *rule)
3831{
3832	if (rule->flex) {
3833		igc_del_flex_filter(adapter, rule->filter.flex_index);
3834		return;
3835	}
3836
3837	if (rule->filter.match_flags & IGC_FILTER_FLAG_ETHER_TYPE)
3838		igc_del_etype_filter(adapter, rule->filter.etype);
3839
3840	if (rule->filter.match_flags & IGC_FILTER_FLAG_VLAN_TCI) {
3841		int prio = FIELD_GET(VLAN_PRIO_MASK, rule->filter.vlan_tci);
3842
3843		igc_del_vlan_prio_filter(adapter, prio);
3844	}
3845
3846	if (rule->filter.match_flags & IGC_FILTER_FLAG_SRC_MAC_ADDR)
3847		igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_SRC,
3848				   rule->filter.src_addr);
3849
3850	if (rule->filter.match_flags & IGC_FILTER_FLAG_DST_MAC_ADDR)
3851		igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST,
3852				   rule->filter.dst_addr);
3853}
3854
3855/**
3856 * igc_get_nfc_rule() - Get NFC rule
3857 * @adapter: Pointer to adapter
3858 * @location: Rule location
3859 *
3860 * Context: Expects adapter->nfc_rule_lock to be held by caller.
3861 *
3862 * Return: Pointer to NFC rule at @location. If not found, NULL.
3863 */
3864struct igc_nfc_rule *igc_get_nfc_rule(struct igc_adapter *adapter,
3865				      u32 location)
3866{
3867	struct igc_nfc_rule *rule;
3868
3869	list_for_each_entry(rule, &adapter->nfc_rule_list, list) {
3870		if (rule->location == location)
3871			return rule;
3872		if (rule->location > location)
3873			break;
3874	}
3875
3876	return NULL;
3877}
3878
3879/**
3880 * igc_del_nfc_rule() - Delete NFC rule
3881 * @adapter: Pointer to adapter
3882 * @rule: Pointer to rule to be deleted
3883 *
3884 * Disable NFC rule in hardware and delete it from adapter.
3885 *
3886 * Context: Expects adapter->nfc_rule_lock to be held by caller.
3887 */
3888void igc_del_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3889{
3890	igc_disable_nfc_rule(adapter, rule);
3891
3892	list_del(&rule->list);
3893	adapter->nfc_rule_count--;
3894
3895	kfree(rule);
3896}
3897
3898static void igc_flush_nfc_rules(struct igc_adapter *adapter)
3899{
3900	struct igc_nfc_rule *rule, *tmp;
3901
3902	mutex_lock(&adapter->nfc_rule_lock);
3903
3904	list_for_each_entry_safe(rule, tmp, &adapter->nfc_rule_list, list)
3905		igc_del_nfc_rule(adapter, rule);
3906
3907	mutex_unlock(&adapter->nfc_rule_lock);
3908}
3909
3910/**
3911 * igc_add_nfc_rule() - Add NFC rule
3912 * @adapter: Pointer to adapter
3913 * @rule: Pointer to rule to be added
3914 *
3915 * Enable NFC rule in hardware and add it to adapter.
3916 *
3917 * Context: Expects adapter->nfc_rule_lock to be held by caller.
3918 *
3919 * Return: 0 on success, negative errno on failure.
3920 */
3921int igc_add_nfc_rule(struct igc_adapter *adapter, struct igc_nfc_rule *rule)
3922{
3923	struct igc_nfc_rule *pred, *cur;
3924	int err;
3925
3926	err = igc_enable_nfc_rule(adapter, rule);
3927	if (err)
3928		return err;
3929
3930	pred = NULL;
3931	list_for_each_entry(cur, &adapter->nfc_rule_list, list) {
3932		if (cur->location >= rule->location)
3933			break;
3934		pred = cur;
3935	}
3936
3937	list_add(&rule->list, pred ? &pred->list : &adapter->nfc_rule_list);
3938	adapter->nfc_rule_count++;
3939	return 0;
3940}
3941
3942static void igc_restore_nfc_rules(struct igc_adapter *adapter)
3943{
3944	struct igc_nfc_rule *rule;
3945
3946	mutex_lock(&adapter->nfc_rule_lock);
3947
3948	list_for_each_entry_reverse(rule, &adapter->nfc_rule_list, list)
3949		igc_enable_nfc_rule(adapter, rule);
3950
3951	mutex_unlock(&adapter->nfc_rule_lock);
3952}
3953
3954static int igc_uc_sync(struct net_device *netdev, const unsigned char *addr)
3955{
3956	struct igc_adapter *adapter = netdev_priv(netdev);
3957
3958	return igc_add_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr, -1);
3959}
3960
3961static int igc_uc_unsync(struct net_device *netdev, const unsigned char *addr)
3962{
3963	struct igc_adapter *adapter = netdev_priv(netdev);
3964
3965	igc_del_mac_filter(adapter, IGC_MAC_FILTER_TYPE_DST, addr);
3966	return 0;
3967}
3968
3969/**
3970 * igc_set_rx_mode - Secondary Unicast, Multicast and Promiscuous mode set
3971 * @netdev: network interface device structure
3972 *
3973 * The set_rx_mode entry point is called whenever the unicast or multicast
3974 * address lists or the network interface flags are updated.  This routine is
3975 * responsible for configuring the hardware for proper unicast, multicast,
3976 * promiscuous mode, and all-multi behavior.
3977 */
3978static void igc_set_rx_mode(struct net_device *netdev)
3979{
3980	struct igc_adapter *adapter = netdev_priv(netdev);
3981	struct igc_hw *hw = &adapter->hw;
3982	u32 rctl = 0, rlpml = MAX_JUMBO_FRAME_SIZE;
3983	int count;
3984
3985	/* Check for Promiscuous and All Multicast modes */
3986	if (netdev->flags & IFF_PROMISC) {
3987		rctl |= IGC_RCTL_UPE | IGC_RCTL_MPE;
3988	} else {
3989		if (netdev->flags & IFF_ALLMULTI) {
3990			rctl |= IGC_RCTL_MPE;
3991		} else {
3992			/* Write addresses to the MTA, if the attempt fails
3993			 * then we should just turn on promiscuous mode so
3994			 * that we can at least receive multicast traffic
3995			 */
3996			count = igc_write_mc_addr_list(netdev);
3997			if (count < 0)
3998				rctl |= IGC_RCTL_MPE;
3999		}
4000	}
4001
4002	/* Write addresses to available RAR registers, if there is not
4003	 * sufficient space to store all the addresses then enable
4004	 * unicast promiscuous mode
4005	 */
4006	if (__dev_uc_sync(netdev, igc_uc_sync, igc_uc_unsync))
4007		rctl |= IGC_RCTL_UPE;
4008
4009	/* update state of unicast and multicast */
4010	rctl |= rd32(IGC_RCTL) & ~(IGC_RCTL_UPE | IGC_RCTL_MPE);
4011	wr32(IGC_RCTL, rctl);
4012
4013#if (PAGE_SIZE < 8192)
4014	if (adapter->max_frame_size <= IGC_MAX_FRAME_BUILD_SKB)
4015		rlpml = IGC_MAX_FRAME_BUILD_SKB;
4016#endif
4017	wr32(IGC_RLPML, rlpml);
4018}
4019
4020/**
4021 * igc_configure - configure the hardware for RX and TX
4022 * @adapter: private board structure
4023 */
4024static void igc_configure(struct igc_adapter *adapter)
4025{
4026	struct net_device *netdev = adapter->netdev;
4027	int i = 0;
4028
4029	igc_get_hw_control(adapter);
4030	igc_set_rx_mode(netdev);
4031
4032	igc_restore_vlan(adapter);
4033
4034	igc_setup_tctl(adapter);
4035	igc_setup_mrqc(adapter);
4036	igc_setup_rctl(adapter);
4037
4038	igc_set_default_mac_filter(adapter);
4039	igc_restore_nfc_rules(adapter);
4040
4041	igc_configure_tx(adapter);
4042	igc_configure_rx(adapter);
4043
4044	igc_rx_fifo_flush_base(&adapter->hw);
4045
4046	/* call igc_desc_unused which always leaves
4047	 * at least 1 descriptor unused to make sure
4048	 * next_to_use != next_to_clean
4049	 */
4050	for (i = 0; i < adapter->num_rx_queues; i++) {
4051		struct igc_ring *ring = adapter->rx_ring[i];
4052
4053		if (ring->xsk_pool)
4054			igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
4055		else
4056			igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
4057	}
4058}
4059
4060/**
4061 * igc_write_ivar - configure ivar for given MSI-X vector
4062 * @hw: pointer to the HW structure
4063 * @msix_vector: vector number we are allocating to a given ring
4064 * @index: row index of IVAR register to write within IVAR table
4065 * @offset: column offset of in IVAR, should be multiple of 8
4066 *
4067 * The IVAR table consists of 2 columns,
4068 * each containing an cause allocation for an Rx and Tx ring, and a
4069 * variable number of rows depending on the number of queues supported.
4070 */
4071static void igc_write_ivar(struct igc_hw *hw, int msix_vector,
4072			   int index, int offset)
4073{
4074	u32 ivar = array_rd32(IGC_IVAR0, index);
4075
4076	/* clear any bits that are currently set */
4077	ivar &= ~((u32)0xFF << offset);
4078
4079	/* write vector and valid bit */
4080	ivar |= (msix_vector | IGC_IVAR_VALID) << offset;
4081
4082	array_wr32(IGC_IVAR0, index, ivar);
4083}
4084
4085static void igc_assign_vector(struct igc_q_vector *q_vector, int msix_vector)
4086{
4087	struct igc_adapter *adapter = q_vector->adapter;
4088	struct igc_hw *hw = &adapter->hw;
4089	int rx_queue = IGC_N0_QUEUE;
4090	int tx_queue = IGC_N0_QUEUE;
4091
4092	if (q_vector->rx.ring)
4093		rx_queue = q_vector->rx.ring->reg_idx;
4094	if (q_vector->tx.ring)
4095		tx_queue = q_vector->tx.ring->reg_idx;
4096
4097	switch (hw->mac.type) {
4098	case igc_i225:
4099		if (rx_queue > IGC_N0_QUEUE)
4100			igc_write_ivar(hw, msix_vector,
4101				       rx_queue >> 1,
4102				       (rx_queue & 0x1) << 4);
4103		if (tx_queue > IGC_N0_QUEUE)
4104			igc_write_ivar(hw, msix_vector,
4105				       tx_queue >> 1,
4106				       ((tx_queue & 0x1) << 4) + 8);
4107		q_vector->eims_value = BIT(msix_vector);
4108		break;
4109	default:
4110		WARN_ONCE(hw->mac.type != igc_i225, "Wrong MAC type\n");
4111		break;
4112	}
4113
4114	/* add q_vector eims value to global eims_enable_mask */
4115	adapter->eims_enable_mask |= q_vector->eims_value;
4116
4117	/* configure q_vector to set itr on first interrupt */
4118	q_vector->set_itr = 1;
4119}
4120
4121/**
4122 * igc_configure_msix - Configure MSI-X hardware
4123 * @adapter: Pointer to adapter structure
4124 *
4125 * igc_configure_msix sets up the hardware to properly
4126 * generate MSI-X interrupts.
4127 */
4128static void igc_configure_msix(struct igc_adapter *adapter)
4129{
4130	struct igc_hw *hw = &adapter->hw;
4131	int i, vector = 0;
4132	u32 tmp;
4133
4134	adapter->eims_enable_mask = 0;
4135
4136	/* set vector for other causes, i.e. link changes */
4137	switch (hw->mac.type) {
4138	case igc_i225:
4139		/* Turn on MSI-X capability first, or our settings
4140		 * won't stick.  And it will take days to debug.
4141		 */
4142		wr32(IGC_GPIE, IGC_GPIE_MSIX_MODE |
4143		     IGC_GPIE_PBA | IGC_GPIE_EIAME |
4144		     IGC_GPIE_NSICR);
4145
4146		/* enable msix_other interrupt */
4147		adapter->eims_other = BIT(vector);
4148		tmp = (vector++ | IGC_IVAR_VALID) << 8;
4149
4150		wr32(IGC_IVAR_MISC, tmp);
4151		break;
4152	default:
4153		/* do nothing, since nothing else supports MSI-X */
4154		break;
4155	} /* switch (hw->mac.type) */
4156
4157	adapter->eims_enable_mask |= adapter->eims_other;
4158
4159	for (i = 0; i < adapter->num_q_vectors; i++)
4160		igc_assign_vector(adapter->q_vector[i], vector++);
4161
4162	wrfl();
4163}
4164
4165/**
4166 * igc_irq_enable - Enable default interrupt generation settings
4167 * @adapter: board private structure
4168 */
4169static void igc_irq_enable(struct igc_adapter *adapter)
4170{
4171	struct igc_hw *hw = &adapter->hw;
4172
4173	if (adapter->msix_entries) {
4174		u32 ims = IGC_IMS_LSC | IGC_IMS_DOUTSYNC | IGC_IMS_DRSTA;
4175		u32 regval = rd32(IGC_EIAC);
4176
4177		wr32(IGC_EIAC, regval | adapter->eims_enable_mask);
4178		regval = rd32(IGC_EIAM);
4179		wr32(IGC_EIAM, regval | adapter->eims_enable_mask);
4180		wr32(IGC_EIMS, adapter->eims_enable_mask);
4181		wr32(IGC_IMS, ims);
4182	} else {
4183		wr32(IGC_IMS, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
4184		wr32(IGC_IAM, IMS_ENABLE_MASK | IGC_IMS_DRSTA);
4185	}
4186}
4187
4188/**
4189 * igc_irq_disable - Mask off interrupt generation on the NIC
4190 * @adapter: board private structure
4191 */
4192static void igc_irq_disable(struct igc_adapter *adapter)
4193{
4194	struct igc_hw *hw = &adapter->hw;
4195
4196	if (adapter->msix_entries) {
4197		u32 regval = rd32(IGC_EIAM);
4198
4199		wr32(IGC_EIAM, regval & ~adapter->eims_enable_mask);
4200		wr32(IGC_EIMC, adapter->eims_enable_mask);
4201		regval = rd32(IGC_EIAC);
4202		wr32(IGC_EIAC, regval & ~adapter->eims_enable_mask);
4203	}
4204
4205	wr32(IGC_IAM, 0);
4206	wr32(IGC_IMC, ~0);
4207	wrfl();
4208
4209	if (adapter->msix_entries) {
4210		int vector = 0, i;
4211
4212		synchronize_irq(adapter->msix_entries[vector++].vector);
4213
4214		for (i = 0; i < adapter->num_q_vectors; i++)
4215			synchronize_irq(adapter->msix_entries[vector++].vector);
4216	} else {
4217		synchronize_irq(adapter->pdev->irq);
4218	}
4219}
4220
4221void igc_set_flag_queue_pairs(struct igc_adapter *adapter,
4222			      const u32 max_rss_queues)
4223{
4224	/* Determine if we need to pair queues. */
4225	/* If rss_queues > half of max_rss_queues, pair the queues in
4226	 * order to conserve interrupts due to limited supply.
4227	 */
4228	if (adapter->rss_queues > (max_rss_queues / 2))
4229		adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4230	else
4231		adapter->flags &= ~IGC_FLAG_QUEUE_PAIRS;
4232}
4233
4234unsigned int igc_get_max_rss_queues(struct igc_adapter *adapter)
4235{
4236	return IGC_MAX_RX_QUEUES;
4237}
4238
4239static void igc_init_queue_configuration(struct igc_adapter *adapter)
4240{
4241	u32 max_rss_queues;
4242
4243	max_rss_queues = igc_get_max_rss_queues(adapter);
4244	adapter->rss_queues = min_t(u32, max_rss_queues, num_online_cpus());
4245
4246	igc_set_flag_queue_pairs(adapter, max_rss_queues);
4247}
4248
4249/**
4250 * igc_reset_q_vector - Reset config for interrupt vector
4251 * @adapter: board private structure to initialize
4252 * @v_idx: Index of vector to be reset
4253 *
4254 * If NAPI is enabled it will delete any references to the
4255 * NAPI struct. This is preparation for igc_free_q_vector.
4256 */
4257static void igc_reset_q_vector(struct igc_adapter *adapter, int v_idx)
4258{
4259	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
4260
4261	/* if we're coming from igc_set_interrupt_capability, the vectors are
4262	 * not yet allocated
4263	 */
4264	if (!q_vector)
4265		return;
4266
4267	if (q_vector->tx.ring)
4268		adapter->tx_ring[q_vector->tx.ring->queue_index] = NULL;
4269
4270	if (q_vector->rx.ring)
4271		adapter->rx_ring[q_vector->rx.ring->queue_index] = NULL;
4272
4273	netif_napi_del(&q_vector->napi);
4274}
4275
4276/**
4277 * igc_free_q_vector - Free memory allocated for specific interrupt vector
4278 * @adapter: board private structure to initialize
4279 * @v_idx: Index of vector to be freed
4280 *
4281 * This function frees the memory allocated to the q_vector.
4282 */
4283static void igc_free_q_vector(struct igc_adapter *adapter, int v_idx)
4284{
4285	struct igc_q_vector *q_vector = adapter->q_vector[v_idx];
4286
4287	adapter->q_vector[v_idx] = NULL;
4288
4289	/* igc_get_stats64() might access the rings on this vector,
4290	 * we must wait a grace period before freeing it.
4291	 */
4292	if (q_vector)
4293		kfree_rcu(q_vector, rcu);
4294}
4295
4296/**
4297 * igc_free_q_vectors - Free memory allocated for interrupt vectors
4298 * @adapter: board private structure to initialize
4299 *
4300 * This function frees the memory allocated to the q_vectors.  In addition if
4301 * NAPI is enabled it will delete any references to the NAPI struct prior
4302 * to freeing the q_vector.
4303 */
4304static void igc_free_q_vectors(struct igc_adapter *adapter)
4305{
4306	int v_idx = adapter->num_q_vectors;
4307
4308	adapter->num_tx_queues = 0;
4309	adapter->num_rx_queues = 0;
4310	adapter->num_q_vectors = 0;
4311
4312	while (v_idx--) {
4313		igc_reset_q_vector(adapter, v_idx);
4314		igc_free_q_vector(adapter, v_idx);
4315	}
4316}
4317
4318/**
4319 * igc_update_itr - update the dynamic ITR value based on statistics
4320 * @q_vector: pointer to q_vector
4321 * @ring_container: ring info to update the itr for
4322 *
4323 * Stores a new ITR value based on packets and byte
4324 * counts during the last interrupt.  The advantage of per interrupt
4325 * computation is faster updates and more accurate ITR for the current
4326 * traffic pattern.  Constants in this function were computed
4327 * based on theoretical maximum wire speed and thresholds were set based
4328 * on testing data as well as attempting to minimize response time
4329 * while increasing bulk throughput.
4330 * NOTE: These calculations are only valid when operating in a single-
4331 * queue environment.
4332 */
4333static void igc_update_itr(struct igc_q_vector *q_vector,
4334			   struct igc_ring_container *ring_container)
4335{
4336	unsigned int packets = ring_container->total_packets;
4337	unsigned int bytes = ring_container->total_bytes;
4338	u8 itrval = ring_container->itr;
4339
4340	/* no packets, exit with status unchanged */
4341	if (packets == 0)
4342		return;
4343
4344	switch (itrval) {
4345	case lowest_latency:
4346		/* handle TSO and jumbo frames */
4347		if (bytes / packets > 8000)
4348			itrval = bulk_latency;
4349		else if ((packets < 5) && (bytes > 512))
4350			itrval = low_latency;
4351		break;
4352	case low_latency:  /* 50 usec aka 20000 ints/s */
4353		if (bytes > 10000) {
4354			/* this if handles the TSO accounting */
4355			if (bytes / packets > 8000)
4356				itrval = bulk_latency;
4357			else if ((packets < 10) || ((bytes / packets) > 1200))
4358				itrval = bulk_latency;
4359			else if ((packets > 35))
4360				itrval = lowest_latency;
4361		} else if (bytes / packets > 2000) {
4362			itrval = bulk_latency;
4363		} else if (packets <= 2 && bytes < 512) {
4364			itrval = lowest_latency;
4365		}
4366		break;
4367	case bulk_latency: /* 250 usec aka 4000 ints/s */
4368		if (bytes > 25000) {
4369			if (packets > 35)
4370				itrval = low_latency;
4371		} else if (bytes < 1500) {
4372			itrval = low_latency;
4373		}
4374		break;
4375	}
4376
4377	/* clear work counters since we have the values we need */
4378	ring_container->total_bytes = 0;
4379	ring_container->total_packets = 0;
4380
4381	/* write updated itr to ring container */
4382	ring_container->itr = itrval;
4383}
4384
4385static void igc_set_itr(struct igc_q_vector *q_vector)
4386{
4387	struct igc_adapter *adapter = q_vector->adapter;
4388	u32 new_itr = q_vector->itr_val;
4389	u8 current_itr = 0;
4390
4391	/* for non-gigabit speeds, just fix the interrupt rate at 4000 */
4392	switch (adapter->link_speed) {
4393	case SPEED_10:
4394	case SPEED_100:
4395		current_itr = 0;
4396		new_itr = IGC_4K_ITR;
4397		goto set_itr_now;
4398	default:
4399		break;
4400	}
4401
4402	igc_update_itr(q_vector, &q_vector->tx);
4403	igc_update_itr(q_vector, &q_vector->rx);
4404
4405	current_itr = max(q_vector->rx.itr, q_vector->tx.itr);
4406
4407	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4408	if (current_itr == lowest_latency &&
4409	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4410	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4411		current_itr = low_latency;
4412
4413	switch (current_itr) {
4414	/* counts and packets in update_itr are dependent on these numbers */
4415	case lowest_latency:
4416		new_itr = IGC_70K_ITR; /* 70,000 ints/sec */
4417		break;
4418	case low_latency:
4419		new_itr = IGC_20K_ITR; /* 20,000 ints/sec */
4420		break;
4421	case bulk_latency:
4422		new_itr = IGC_4K_ITR;  /* 4,000 ints/sec */
4423		break;
4424	default:
4425		break;
4426	}
4427
4428set_itr_now:
4429	if (new_itr != q_vector->itr_val) {
4430		/* this attempts to bias the interrupt rate towards Bulk
4431		 * by adding intermediate steps when interrupt rate is
4432		 * increasing
4433		 */
4434		new_itr = new_itr > q_vector->itr_val ?
4435			  max((new_itr * q_vector->itr_val) /
4436			  (new_itr + (q_vector->itr_val >> 2)),
4437			  new_itr) : new_itr;
4438		/* Don't write the value here; it resets the adapter's
4439		 * internal timer, and causes us to delay far longer than
4440		 * we should between interrupts.  Instead, we write the ITR
4441		 * value at the beginning of the next interrupt so the timing
4442		 * ends up being correct.
4443		 */
4444		q_vector->itr_val = new_itr;
4445		q_vector->set_itr = 1;
4446	}
4447}
4448
4449static void igc_reset_interrupt_capability(struct igc_adapter *adapter)
4450{
4451	int v_idx = adapter->num_q_vectors;
4452
4453	if (adapter->msix_entries) {
4454		pci_disable_msix(adapter->pdev);
4455		kfree(adapter->msix_entries);
4456		adapter->msix_entries = NULL;
4457	} else if (adapter->flags & IGC_FLAG_HAS_MSI) {
4458		pci_disable_msi(adapter->pdev);
4459	}
4460
4461	while (v_idx--)
4462		igc_reset_q_vector(adapter, v_idx);
4463}
4464
4465/**
4466 * igc_set_interrupt_capability - set MSI or MSI-X if supported
4467 * @adapter: Pointer to adapter structure
4468 * @msix: boolean value for MSI-X capability
4469 *
4470 * Attempt to configure interrupts using the best available
4471 * capabilities of the hardware and kernel.
4472 */
4473static void igc_set_interrupt_capability(struct igc_adapter *adapter,
4474					 bool msix)
4475{
4476	int numvecs, i;
4477	int err;
4478
4479	if (!msix)
4480		goto msi_only;
4481	adapter->flags |= IGC_FLAG_HAS_MSIX;
4482
4483	/* Number of supported queues. */
4484	adapter->num_rx_queues = adapter->rss_queues;
4485
4486	adapter->num_tx_queues = adapter->rss_queues;
4487
4488	/* start with one vector for every Rx queue */
4489	numvecs = adapter->num_rx_queues;
4490
4491	/* if Tx handler is separate add 1 for every Tx queue */
4492	if (!(adapter->flags & IGC_FLAG_QUEUE_PAIRS))
4493		numvecs += adapter->num_tx_queues;
4494
4495	/* store the number of vectors reserved for queues */
4496	adapter->num_q_vectors = numvecs;
4497
4498	/* add 1 vector for link status interrupts */
4499	numvecs++;
4500
4501	adapter->msix_entries = kcalloc(numvecs, sizeof(struct msix_entry),
4502					GFP_KERNEL);
4503
4504	if (!adapter->msix_entries)
4505		return;
4506
4507	/* populate entry values */
4508	for (i = 0; i < numvecs; i++)
4509		adapter->msix_entries[i].entry = i;
4510
4511	err = pci_enable_msix_range(adapter->pdev,
4512				    adapter->msix_entries,
4513				    numvecs,
4514				    numvecs);
4515	if (err > 0)
4516		return;
4517
4518	kfree(adapter->msix_entries);
4519	adapter->msix_entries = NULL;
4520
4521	igc_reset_interrupt_capability(adapter);
4522
4523msi_only:
4524	adapter->flags &= ~IGC_FLAG_HAS_MSIX;
4525
4526	adapter->rss_queues = 1;
4527	adapter->flags |= IGC_FLAG_QUEUE_PAIRS;
4528	adapter->num_rx_queues = 1;
4529	adapter->num_tx_queues = 1;
4530	adapter->num_q_vectors = 1;
4531	if (!pci_enable_msi(adapter->pdev))
4532		adapter->flags |= IGC_FLAG_HAS_MSI;
4533}
4534
4535/**
4536 * igc_update_ring_itr - update the dynamic ITR value based on packet size
4537 * @q_vector: pointer to q_vector
4538 *
4539 * Stores a new ITR value based on strictly on packet size.  This
4540 * algorithm is less sophisticated than that used in igc_update_itr,
4541 * due to the difficulty of synchronizing statistics across multiple
4542 * receive rings.  The divisors and thresholds used by this function
4543 * were determined based on theoretical maximum wire speed and testing
4544 * data, in order to minimize response time while increasing bulk
4545 * throughput.
4546 * NOTE: This function is called only when operating in a multiqueue
4547 * receive environment.
4548 */
4549static void igc_update_ring_itr(struct igc_q_vector *q_vector)
4550{
4551	struct igc_adapter *adapter = q_vector->adapter;
4552	int new_val = q_vector->itr_val;
4553	int avg_wire_size = 0;
4554	unsigned int packets;
4555
4556	/* For non-gigabit speeds, just fix the interrupt rate at 4000
4557	 * ints/sec - ITR timer value of 120 ticks.
4558	 */
4559	switch (adapter->link_speed) {
4560	case SPEED_10:
4561	case SPEED_100:
4562		new_val = IGC_4K_ITR;
4563		goto set_itr_val;
4564	default:
4565		break;
4566	}
4567
4568	packets = q_vector->rx.total_packets;
4569	if (packets)
4570		avg_wire_size = q_vector->rx.total_bytes / packets;
4571
4572	packets = q_vector->tx.total_packets;
4573	if (packets)
4574		avg_wire_size = max_t(u32, avg_wire_size,
4575				      q_vector->tx.total_bytes / packets);
4576
4577	/* if avg_wire_size isn't set no work was done */
4578	if (!avg_wire_size)
4579		goto clear_counts;
4580
4581	/* Add 24 bytes to size to account for CRC, preamble, and gap */
4582	avg_wire_size += 24;
4583
4584	/* Don't starve jumbo frames */
4585	avg_wire_size = min(avg_wire_size, 3000);
4586
4587	/* Give a little boost to mid-size frames */
4588	if (avg_wire_size > 300 && avg_wire_size < 1200)
4589		new_val = avg_wire_size / 3;
4590	else
4591		new_val = avg_wire_size / 2;
4592
4593	/* conservative mode (itr 3) eliminates the lowest_latency setting */
4594	if (new_val < IGC_20K_ITR &&
4595	    ((q_vector->rx.ring && adapter->rx_itr_setting == 3) ||
4596	    (!q_vector->rx.ring && adapter->tx_itr_setting == 3)))
4597		new_val = IGC_20K_ITR;
4598
4599set_itr_val:
4600	if (new_val != q_vector->itr_val) {
4601		q_vector->itr_val = new_val;
4602		q_vector->set_itr = 1;
4603	}
4604clear_counts:
4605	q_vector->rx.total_bytes = 0;
4606	q_vector->rx.total_packets = 0;
4607	q_vector->tx.total_bytes = 0;
4608	q_vector->tx.total_packets = 0;
4609}
4610
4611static void igc_ring_irq_enable(struct igc_q_vector *q_vector)
4612{
4613	struct igc_adapter *adapter = q_vector->adapter;
4614	struct igc_hw *hw = &adapter->hw;
4615
4616	if ((q_vector->rx.ring && (adapter->rx_itr_setting & 3)) ||
4617	    (!q_vector->rx.ring && (adapter->tx_itr_setting & 3))) {
4618		if (adapter->num_q_vectors == 1)
4619			igc_set_itr(q_vector);
4620		else
4621			igc_update_ring_itr(q_vector);
4622	}
4623
4624	if (!test_bit(__IGC_DOWN, &adapter->state)) {
4625		if (adapter->msix_entries)
4626			wr32(IGC_EIMS, q_vector->eims_value);
4627		else
4628			igc_irq_enable(adapter);
4629	}
4630}
4631
4632static void igc_add_ring(struct igc_ring *ring,
4633			 struct igc_ring_container *head)
4634{
4635	head->ring = ring;
4636	head->count++;
4637}
4638
4639/**
4640 * igc_cache_ring_register - Descriptor ring to register mapping
4641 * @adapter: board private structure to initialize
4642 *
4643 * Once we know the feature-set enabled for the device, we'll cache
4644 * the register offset the descriptor ring is assigned to.
4645 */
4646static void igc_cache_ring_register(struct igc_adapter *adapter)
4647{
4648	int i = 0, j = 0;
4649
4650	switch (adapter->hw.mac.type) {
4651	case igc_i225:
4652	default:
4653		for (; i < adapter->num_rx_queues; i++)
4654			adapter->rx_ring[i]->reg_idx = i;
4655		for (; j < adapter->num_tx_queues; j++)
4656			adapter->tx_ring[j]->reg_idx = j;
4657		break;
4658	}
4659}
4660
4661/**
4662 * igc_poll - NAPI Rx polling callback
4663 * @napi: napi polling structure
4664 * @budget: count of how many packets we should handle
4665 */
4666static int igc_poll(struct napi_struct *napi, int budget)
4667{
4668	struct igc_q_vector *q_vector = container_of(napi,
4669						     struct igc_q_vector,
4670						     napi);
4671	struct igc_ring *rx_ring = q_vector->rx.ring;
4672	bool clean_complete = true;
4673	int work_done = 0;
4674
4675	if (q_vector->tx.ring)
4676		clean_complete = igc_clean_tx_irq(q_vector, budget);
4677
4678	if (rx_ring) {
4679		int cleaned = rx_ring->xsk_pool ?
4680			      igc_clean_rx_irq_zc(q_vector, budget) :
4681			      igc_clean_rx_irq(q_vector, budget);
4682
4683		work_done += cleaned;
4684		if (cleaned >= budget)
4685			clean_complete = false;
4686	}
4687
4688	/* If all work not completed, return budget and keep polling */
4689	if (!clean_complete)
4690		return budget;
4691
4692	/* Exit the polling mode, but don't re-enable interrupts if stack might
4693	 * poll us due to busy-polling
4694	 */
4695	if (likely(napi_complete_done(napi, work_done)))
4696		igc_ring_irq_enable(q_vector);
4697
4698	return min(work_done, budget - 1);
4699}
4700
4701/**
4702 * igc_alloc_q_vector - Allocate memory for a single interrupt vector
4703 * @adapter: board private structure to initialize
4704 * @v_count: q_vectors allocated on adapter, used for ring interleaving
4705 * @v_idx: index of vector in adapter struct
4706 * @txr_count: total number of Tx rings to allocate
4707 * @txr_idx: index of first Tx ring to allocate
4708 * @rxr_count: total number of Rx rings to allocate
4709 * @rxr_idx: index of first Rx ring to allocate
4710 *
4711 * We allocate one q_vector.  If allocation fails we return -ENOMEM.
4712 */
4713static int igc_alloc_q_vector(struct igc_adapter *adapter,
4714			      unsigned int v_count, unsigned int v_idx,
4715			      unsigned int txr_count, unsigned int txr_idx,
4716			      unsigned int rxr_count, unsigned int rxr_idx)
4717{
4718	struct igc_q_vector *q_vector;
4719	struct igc_ring *ring;
4720	int ring_count;
4721
4722	/* igc only supports 1 Tx and/or 1 Rx queue per vector */
4723	if (txr_count > 1 || rxr_count > 1)
4724		return -ENOMEM;
4725
4726	ring_count = txr_count + rxr_count;
4727
4728	/* allocate q_vector and rings */
4729	q_vector = adapter->q_vector[v_idx];
4730	if (!q_vector)
4731		q_vector = kzalloc(struct_size(q_vector, ring, ring_count),
4732				   GFP_KERNEL);
4733	else
4734		memset(q_vector, 0, struct_size(q_vector, ring, ring_count));
4735	if (!q_vector)
4736		return -ENOMEM;
4737
4738	/* initialize NAPI */
4739	netif_napi_add(adapter->netdev, &q_vector->napi, igc_poll);
4740
4741	/* tie q_vector and adapter together */
4742	adapter->q_vector[v_idx] = q_vector;
4743	q_vector->adapter = adapter;
4744
4745	/* initialize work limits */
4746	q_vector->tx.work_limit = adapter->tx_work_limit;
4747
4748	/* initialize ITR configuration */
4749	q_vector->itr_register = adapter->io_addr + IGC_EITR(0);
4750	q_vector->itr_val = IGC_START_ITR;
4751
4752	/* initialize pointer to rings */
4753	ring = q_vector->ring;
4754
4755	/* initialize ITR */
4756	if (rxr_count) {
4757		/* rx or rx/tx vector */
4758		if (!adapter->rx_itr_setting || adapter->rx_itr_setting > 3)
4759			q_vector->itr_val = adapter->rx_itr_setting;
4760	} else {
4761		/* tx only vector */
4762		if (!adapter->tx_itr_setting || adapter->tx_itr_setting > 3)
4763			q_vector->itr_val = adapter->tx_itr_setting;
4764	}
4765
4766	if (txr_count) {
4767		/* assign generic ring traits */
4768		ring->dev = &adapter->pdev->dev;
4769		ring->netdev = adapter->netdev;
4770
4771		/* configure backlink on ring */
4772		ring->q_vector = q_vector;
4773
4774		/* update q_vector Tx values */
4775		igc_add_ring(ring, &q_vector->tx);
4776
4777		/* apply Tx specific ring traits */
4778		ring->count = adapter->tx_ring_count;
4779		ring->queue_index = txr_idx;
4780
4781		/* assign ring to adapter */
4782		adapter->tx_ring[txr_idx] = ring;
4783
4784		/* push pointer to next ring */
4785		ring++;
4786	}
4787
4788	if (rxr_count) {
4789		/* assign generic ring traits */
4790		ring->dev = &adapter->pdev->dev;
4791		ring->netdev = adapter->netdev;
4792
4793		/* configure backlink on ring */
4794		ring->q_vector = q_vector;
4795
4796		/* update q_vector Rx values */
4797		igc_add_ring(ring, &q_vector->rx);
4798
4799		/* apply Rx specific ring traits */
4800		ring->count = adapter->rx_ring_count;
4801		ring->queue_index = rxr_idx;
4802
4803		/* assign ring to adapter */
4804		adapter->rx_ring[rxr_idx] = ring;
4805	}
4806
4807	return 0;
4808}
4809
4810/**
4811 * igc_alloc_q_vectors - Allocate memory for interrupt vectors
4812 * @adapter: board private structure to initialize
4813 *
4814 * We allocate one q_vector per queue interrupt.  If allocation fails we
4815 * return -ENOMEM.
4816 */
4817static int igc_alloc_q_vectors(struct igc_adapter *adapter)
4818{
4819	int rxr_remaining = adapter->num_rx_queues;
4820	int txr_remaining = adapter->num_tx_queues;
4821	int rxr_idx = 0, txr_idx = 0, v_idx = 0;
4822	int q_vectors = adapter->num_q_vectors;
4823	int err;
4824
4825	if (q_vectors >= (rxr_remaining + txr_remaining)) {
4826		for (; rxr_remaining; v_idx++) {
4827			err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4828						 0, 0, 1, rxr_idx);
4829
4830			if (err)
4831				goto err_out;
4832
4833			/* update counts and index */
4834			rxr_remaining--;
4835			rxr_idx++;
4836		}
4837	}
4838
4839	for (; v_idx < q_vectors; v_idx++) {
4840		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
4841		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
4842
4843		err = igc_alloc_q_vector(adapter, q_vectors, v_idx,
4844					 tqpv, txr_idx, rqpv, rxr_idx);
4845
4846		if (err)
4847			goto err_out;
4848
4849		/* update counts and index */
4850		rxr_remaining -= rqpv;
4851		txr_remaining -= tqpv;
4852		rxr_idx++;
4853		txr_idx++;
4854	}
4855
4856	return 0;
4857
4858err_out:
4859	adapter->num_tx_queues = 0;
4860	adapter->num_rx_queues = 0;
4861	adapter->num_q_vectors = 0;
4862
4863	while (v_idx--)
4864		igc_free_q_vector(adapter, v_idx);
4865
4866	return -ENOMEM;
4867}
4868
4869/**
4870 * igc_init_interrupt_scheme - initialize interrupts, allocate queues/vectors
4871 * @adapter: Pointer to adapter structure
4872 * @msix: boolean for MSI-X capability
4873 *
4874 * This function initializes the interrupts and allocates all of the queues.
4875 */
4876static int igc_init_interrupt_scheme(struct igc_adapter *adapter, bool msix)
4877{
4878	struct net_device *dev = adapter->netdev;
4879	int err = 0;
4880
4881	igc_set_interrupt_capability(adapter, msix);
4882
4883	err = igc_alloc_q_vectors(adapter);
4884	if (err) {
4885		netdev_err(dev, "Unable to allocate memory for vectors\n");
4886		goto err_alloc_q_vectors;
4887	}
4888
4889	igc_cache_ring_register(adapter);
4890
4891	return 0;
4892
4893err_alloc_q_vectors:
4894	igc_reset_interrupt_capability(adapter);
4895	return err;
4896}
4897
4898/**
4899 * igc_sw_init - Initialize general software structures (struct igc_adapter)
4900 * @adapter: board private structure to initialize
4901 *
4902 * igc_sw_init initializes the Adapter private data structure.
4903 * Fields are initialized based on PCI device information and
4904 * OS network device settings (MTU size).
4905 */
4906static int igc_sw_init(struct igc_adapter *adapter)
4907{
4908	struct net_device *netdev = adapter->netdev;
4909	struct pci_dev *pdev = adapter->pdev;
4910	struct igc_hw *hw = &adapter->hw;
4911
4912	pci_read_config_word(pdev, PCI_COMMAND, &hw->bus.pci_cmd_word);
4913
4914	/* set default ring sizes */
4915	adapter->tx_ring_count = IGC_DEFAULT_TXD;
4916	adapter->rx_ring_count = IGC_DEFAULT_RXD;
4917
4918	/* set default ITR values */
4919	adapter->rx_itr_setting = IGC_DEFAULT_ITR;
4920	adapter->tx_itr_setting = IGC_DEFAULT_ITR;
4921
4922	/* set default work limits */
4923	adapter->tx_work_limit = IGC_DEFAULT_TX_WORK;
4924
4925	/* adjust max frame to be at least the size of a standard frame */
4926	adapter->max_frame_size = netdev->mtu + ETH_HLEN + ETH_FCS_LEN +
4927				VLAN_HLEN;
4928	adapter->min_frame_size = ETH_ZLEN + ETH_FCS_LEN;
4929
4930	mutex_init(&adapter->nfc_rule_lock);
4931	INIT_LIST_HEAD(&adapter->nfc_rule_list);
4932	adapter->nfc_rule_count = 0;
4933
4934	spin_lock_init(&adapter->stats64_lock);
4935	spin_lock_init(&adapter->qbv_tx_lock);
4936	/* Assume MSI-X interrupts, will be checked during IRQ allocation */
4937	adapter->flags |= IGC_FLAG_HAS_MSIX;
4938
4939	igc_init_queue_configuration(adapter);
4940
4941	/* This call may decrease the number of queues */
4942	if (igc_init_interrupt_scheme(adapter, true)) {
4943		netdev_err(netdev, "Unable to allocate memory for queues\n");
4944		return -ENOMEM;
4945	}
4946
4947	/* Explicitly disable IRQ since the NIC can be in any state. */
4948	igc_irq_disable(adapter);
4949
4950	set_bit(__IGC_DOWN, &adapter->state);
4951
4952	return 0;
4953}
4954
4955/**
4956 * igc_up - Open the interface and prepare it to handle traffic
4957 * @adapter: board private structure
4958 */
4959void igc_up(struct igc_adapter *adapter)
4960{
4961	struct igc_hw *hw = &adapter->hw;
4962	int i = 0;
4963
4964	/* hardware has been reset, we need to reload some things */
4965	igc_configure(adapter);
4966
4967	clear_bit(__IGC_DOWN, &adapter->state);
4968
4969	for (i = 0; i < adapter->num_q_vectors; i++)
4970		napi_enable(&adapter->q_vector[i]->napi);
4971
4972	if (adapter->msix_entries)
4973		igc_configure_msix(adapter);
4974	else
4975		igc_assign_vector(adapter->q_vector[0], 0);
4976
4977	/* Clear any pending interrupts. */
4978	rd32(IGC_ICR);
4979	igc_irq_enable(adapter);
4980
4981	netif_tx_start_all_queues(adapter->netdev);
4982
4983	/* start the watchdog. */
4984	hw->mac.get_link_status = true;
4985	schedule_work(&adapter->watchdog_task);
4986}
4987
4988/**
4989 * igc_update_stats - Update the board statistics counters
4990 * @adapter: board private structure
4991 */
4992void igc_update_stats(struct igc_adapter *adapter)
4993{
4994	struct rtnl_link_stats64 *net_stats = &adapter->stats64;
4995	struct pci_dev *pdev = adapter->pdev;
4996	struct igc_hw *hw = &adapter->hw;
4997	u64 _bytes, _packets;
4998	u64 bytes, packets;
4999	unsigned int start;
5000	u32 mpc;
5001	int i;
5002
5003	/* Prevent stats update while adapter is being reset, or if the pci
5004	 * connection is down.
5005	 */
5006	if (adapter->link_speed == 0)
5007		return;
5008	if (pci_channel_offline(pdev))
5009		return;
5010
5011	packets = 0;
5012	bytes = 0;
5013
5014	rcu_read_lock();
5015	for (i = 0; i < adapter->num_rx_queues; i++) {
5016		struct igc_ring *ring = adapter->rx_ring[i];
5017		u32 rqdpc = rd32(IGC_RQDPC(i));
5018
5019		if (hw->mac.type >= igc_i225)
5020			wr32(IGC_RQDPC(i), 0);
5021
5022		if (rqdpc) {
5023			ring->rx_stats.drops += rqdpc;
5024			net_stats->rx_fifo_errors += rqdpc;
5025		}
5026
5027		do {
5028			start = u64_stats_fetch_begin(&ring->rx_syncp);
5029			_bytes = ring->rx_stats.bytes;
5030			_packets = ring->rx_stats.packets;
5031		} while (u64_stats_fetch_retry(&ring->rx_syncp, start));
5032		bytes += _bytes;
5033		packets += _packets;
5034	}
5035
5036	net_stats->rx_bytes = bytes;
5037	net_stats->rx_packets = packets;
5038
5039	packets = 0;
5040	bytes = 0;
5041	for (i = 0; i < adapter->num_tx_queues; i++) {
5042		struct igc_ring *ring = adapter->tx_ring[i];
5043
5044		do {
5045			start = u64_stats_fetch_begin(&ring->tx_syncp);
5046			_bytes = ring->tx_stats.bytes;
5047			_packets = ring->tx_stats.packets;
5048		} while (u64_stats_fetch_retry(&ring->tx_syncp, start));
5049		bytes += _bytes;
5050		packets += _packets;
5051	}
5052	net_stats->tx_bytes = bytes;
5053	net_stats->tx_packets = packets;
5054	rcu_read_unlock();
5055
5056	/* read stats registers */
5057	adapter->stats.crcerrs += rd32(IGC_CRCERRS);
5058	adapter->stats.gprc += rd32(IGC_GPRC);
5059	adapter->stats.gorc += rd32(IGC_GORCL);
5060	rd32(IGC_GORCH); /* clear GORCL */
5061	adapter->stats.bprc += rd32(IGC_BPRC);
5062	adapter->stats.mprc += rd32(IGC_MPRC);
5063	adapter->stats.roc += rd32(IGC_ROC);
5064
5065	adapter->stats.prc64 += rd32(IGC_PRC64);
5066	adapter->stats.prc127 += rd32(IGC_PRC127);
5067	adapter->stats.prc255 += rd32(IGC_PRC255);
5068	adapter->stats.prc511 += rd32(IGC_PRC511);
5069	adapter->stats.prc1023 += rd32(IGC_PRC1023);
5070	adapter->stats.prc1522 += rd32(IGC_PRC1522);
5071	adapter->stats.tlpic += rd32(IGC_TLPIC);
5072	adapter->stats.rlpic += rd32(IGC_RLPIC);
5073	adapter->stats.hgptc += rd32(IGC_HGPTC);
5074
5075	mpc = rd32(IGC_MPC);
5076	adapter->stats.mpc += mpc;
5077	net_stats->rx_fifo_errors += mpc;
5078	adapter->stats.scc += rd32(IGC_SCC);
5079	adapter->stats.ecol += rd32(IGC_ECOL);
5080	adapter->stats.mcc += rd32(IGC_MCC);
5081	adapter->stats.latecol += rd32(IGC_LATECOL);
5082	adapter->stats.dc += rd32(IGC_DC);
5083	adapter->stats.rlec += rd32(IGC_RLEC);
5084	adapter->stats.xonrxc += rd32(IGC_XONRXC);
5085	adapter->stats.xontxc += rd32(IGC_XONTXC);
5086	adapter->stats.xoffrxc += rd32(IGC_XOFFRXC);
5087	adapter->stats.xofftxc += rd32(IGC_XOFFTXC);
5088	adapter->stats.fcruc += rd32(IGC_FCRUC);
5089	adapter->stats.gptc += rd32(IGC_GPTC);
5090	adapter->stats.gotc += rd32(IGC_GOTCL);
5091	rd32(IGC_GOTCH); /* clear GOTCL */
5092	adapter->stats.rnbc += rd32(IGC_RNBC);
5093	adapter->stats.ruc += rd32(IGC_RUC);
5094	adapter->stats.rfc += rd32(IGC_RFC);
5095	adapter->stats.rjc += rd32(IGC_RJC);
5096	adapter->stats.tor += rd32(IGC_TORH);
5097	adapter->stats.tot += rd32(IGC_TOTH);
5098	adapter->stats.tpr += rd32(IGC_TPR);
5099
5100	adapter->stats.ptc64 += rd32(IGC_PTC64);
5101	adapter->stats.ptc127 += rd32(IGC_PTC127);
5102	adapter->stats.ptc255 += rd32(IGC_PTC255);
5103	adapter->stats.ptc511 += rd32(IGC_PTC511);
5104	adapter->stats.ptc1023 += rd32(IGC_PTC1023);
5105	adapter->stats.ptc1522 += rd32(IGC_PTC1522);
5106
5107	adapter->stats.mptc += rd32(IGC_MPTC);
5108	adapter->stats.bptc += rd32(IGC_BPTC);
5109
5110	adapter->stats.tpt += rd32(IGC_TPT);
5111	adapter->stats.colc += rd32(IGC_COLC);
5112	adapter->stats.colc += rd32(IGC_RERC);
5113
5114	adapter->stats.algnerrc += rd32(IGC_ALGNERRC);
5115
5116	adapter->stats.tsctc += rd32(IGC_TSCTC);
5117
5118	adapter->stats.iac += rd32(IGC_IAC);
5119
5120	/* Fill out the OS statistics structure */
5121	net_stats->multicast = adapter->stats.mprc;
5122	net_stats->collisions = adapter->stats.colc;
5123
5124	/* Rx Errors */
5125
5126	/* RLEC on some newer hardware can be incorrect so build
5127	 * our own version based on RUC and ROC
5128	 */
5129	net_stats->rx_errors = adapter->stats.rxerrc +
5130		adapter->stats.crcerrs + adapter->stats.algnerrc +
5131		adapter->stats.ruc + adapter->stats.roc +
5132		adapter->stats.cexterr;
5133	net_stats->rx_length_errors = adapter->stats.ruc +
5134				      adapter->stats.roc;
5135	net_stats->rx_crc_errors = adapter->stats.crcerrs;
5136	net_stats->rx_frame_errors = adapter->stats.algnerrc;
5137	net_stats->rx_missed_errors = adapter->stats.mpc;
5138
5139	/* Tx Errors */
5140	net_stats->tx_errors = adapter->stats.ecol +
5141			       adapter->stats.latecol;
5142	net_stats->tx_aborted_errors = adapter->stats.ecol;
5143	net_stats->tx_window_errors = adapter->stats.latecol;
5144	net_stats->tx_carrier_errors = adapter->stats.tncrs;
5145
5146	/* Tx Dropped */
5147	net_stats->tx_dropped = adapter->stats.txdrop;
5148
5149	/* Management Stats */
5150	adapter->stats.mgptc += rd32(IGC_MGTPTC);
5151	adapter->stats.mgprc += rd32(IGC_MGTPRC);
5152	adapter->stats.mgpdc += rd32(IGC_MGTPDC);
5153}
5154
5155/**
5156 * igc_down - Close the interface
5157 * @adapter: board private structure
5158 */
5159void igc_down(struct igc_adapter *adapter)
5160{
5161	struct net_device *netdev = adapter->netdev;
5162	struct igc_hw *hw = &adapter->hw;
5163	u32 tctl, rctl;
5164	int i = 0;
5165
5166	set_bit(__IGC_DOWN, &adapter->state);
5167
5168	igc_ptp_suspend(adapter);
5169
5170	if (pci_device_is_present(adapter->pdev)) {
5171		/* disable receives in the hardware */
5172		rctl = rd32(IGC_RCTL);
5173		wr32(IGC_RCTL, rctl & ~IGC_RCTL_EN);
5174		/* flush and sleep below */
5175	}
5176	/* set trans_start so we don't get spurious watchdogs during reset */
5177	netif_trans_update(netdev);
5178
5179	netif_carrier_off(netdev);
5180	netif_tx_stop_all_queues(netdev);
5181
5182	if (pci_device_is_present(adapter->pdev)) {
5183		/* disable transmits in the hardware */
5184		tctl = rd32(IGC_TCTL);
5185		tctl &= ~IGC_TCTL_EN;
5186		wr32(IGC_TCTL, tctl);
5187		/* flush both disables and wait for them to finish */
5188		wrfl();
5189		usleep_range(10000, 20000);
5190
5191		igc_irq_disable(adapter);
5192	}
5193
5194	adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5195
5196	for (i = 0; i < adapter->num_q_vectors; i++) {
5197		if (adapter->q_vector[i]) {
5198			napi_synchronize(&adapter->q_vector[i]->napi);
5199			napi_disable(&adapter->q_vector[i]->napi);
5200		}
5201	}
5202
5203	del_timer_sync(&adapter->watchdog_timer);
5204	del_timer_sync(&adapter->phy_info_timer);
5205
5206	/* record the stats before reset*/
5207	spin_lock(&adapter->stats64_lock);
5208	igc_update_stats(adapter);
5209	spin_unlock(&adapter->stats64_lock);
5210
5211	adapter->link_speed = 0;
5212	adapter->link_duplex = 0;
5213
5214	if (!pci_channel_offline(adapter->pdev))
5215		igc_reset(adapter);
5216
5217	/* clear VLAN promisc flag so VFTA will be updated if necessary */
5218	adapter->flags &= ~IGC_FLAG_VLAN_PROMISC;
5219
5220	igc_disable_all_tx_rings_hw(adapter);
5221	igc_clean_all_tx_rings(adapter);
5222	igc_clean_all_rx_rings(adapter);
5223}
5224
5225void igc_reinit_locked(struct igc_adapter *adapter)
5226{
5227	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5228		usleep_range(1000, 2000);
5229	igc_down(adapter);
5230	igc_up(adapter);
5231	clear_bit(__IGC_RESETTING, &adapter->state);
5232}
5233
5234static void igc_reset_task(struct work_struct *work)
5235{
5236	struct igc_adapter *adapter;
5237
5238	adapter = container_of(work, struct igc_adapter, reset_task);
5239
5240	rtnl_lock();
5241	/* If we're already down or resetting, just bail */
5242	if (test_bit(__IGC_DOWN, &adapter->state) ||
5243	    test_bit(__IGC_RESETTING, &adapter->state)) {
5244		rtnl_unlock();
5245		return;
5246	}
5247
5248	igc_rings_dump(adapter);
5249	igc_regs_dump(adapter);
5250	netdev_err(adapter->netdev, "Reset adapter\n");
5251	igc_reinit_locked(adapter);
5252	rtnl_unlock();
5253}
5254
5255/**
5256 * igc_change_mtu - Change the Maximum Transfer Unit
5257 * @netdev: network interface device structure
5258 * @new_mtu: new value for maximum frame size
5259 *
5260 * Returns 0 on success, negative on failure
5261 */
5262static int igc_change_mtu(struct net_device *netdev, int new_mtu)
5263{
5264	int max_frame = new_mtu + ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
5265	struct igc_adapter *adapter = netdev_priv(netdev);
5266
5267	if (igc_xdp_is_enabled(adapter) && new_mtu > ETH_DATA_LEN) {
5268		netdev_dbg(netdev, "Jumbo frames not supported with XDP");
5269		return -EINVAL;
5270	}
5271
5272	/* adjust max frame to be at least the size of a standard frame */
5273	if (max_frame < (ETH_FRAME_LEN + ETH_FCS_LEN))
5274		max_frame = ETH_FRAME_LEN + ETH_FCS_LEN;
5275
5276	while (test_and_set_bit(__IGC_RESETTING, &adapter->state))
5277		usleep_range(1000, 2000);
5278
5279	/* igc_down has a dependency on max_frame_size */
5280	adapter->max_frame_size = max_frame;
5281
5282	if (netif_running(netdev))
5283		igc_down(adapter);
5284
5285	netdev_dbg(netdev, "changing MTU from %d to %d\n", netdev->mtu, new_mtu);
5286	WRITE_ONCE(netdev->mtu, new_mtu);
5287
5288	if (netif_running(netdev))
5289		igc_up(adapter);
5290	else
5291		igc_reset(adapter);
5292
5293	clear_bit(__IGC_RESETTING, &adapter->state);
5294
5295	return 0;
5296}
5297
5298/**
5299 * igc_tx_timeout - Respond to a Tx Hang
5300 * @netdev: network interface device structure
5301 * @txqueue: queue number that timed out
5302 **/
5303static void igc_tx_timeout(struct net_device *netdev,
5304			   unsigned int __always_unused txqueue)
5305{
5306	struct igc_adapter *adapter = netdev_priv(netdev);
5307	struct igc_hw *hw = &adapter->hw;
5308
5309	/* Do the reset outside of interrupt context */
5310	adapter->tx_timeout_count++;
5311	schedule_work(&adapter->reset_task);
5312	wr32(IGC_EICS,
5313	     (adapter->eims_enable_mask & ~adapter->eims_other));
5314}
5315
5316/**
5317 * igc_get_stats64 - Get System Network Statistics
5318 * @netdev: network interface device structure
5319 * @stats: rtnl_link_stats64 pointer
5320 *
5321 * Returns the address of the device statistics structure.
5322 * The statistics are updated here and also from the timer callback.
5323 */
5324static void igc_get_stats64(struct net_device *netdev,
5325			    struct rtnl_link_stats64 *stats)
5326{
5327	struct igc_adapter *adapter = netdev_priv(netdev);
5328
5329	spin_lock(&adapter->stats64_lock);
5330	if (!test_bit(__IGC_RESETTING, &adapter->state))
5331		igc_update_stats(adapter);
5332	memcpy(stats, &adapter->stats64, sizeof(*stats));
5333	spin_unlock(&adapter->stats64_lock);
5334}
5335
5336static netdev_features_t igc_fix_features(struct net_device *netdev,
5337					  netdev_features_t features)
5338{
5339	/* Since there is no support for separate Rx/Tx vlan accel
5340	 * enable/disable make sure Tx flag is always in same state as Rx.
5341	 */
5342	if (features & NETIF_F_HW_VLAN_CTAG_RX)
5343		features |= NETIF_F_HW_VLAN_CTAG_TX;
5344	else
5345		features &= ~NETIF_F_HW_VLAN_CTAG_TX;
5346
5347	return features;
5348}
5349
5350static int igc_set_features(struct net_device *netdev,
5351			    netdev_features_t features)
5352{
5353	netdev_features_t changed = netdev->features ^ features;
5354	struct igc_adapter *adapter = netdev_priv(netdev);
5355
5356	if (changed & NETIF_F_HW_VLAN_CTAG_RX)
5357		igc_vlan_mode(netdev, features);
5358
5359	/* Add VLAN support */
5360	if (!(changed & (NETIF_F_RXALL | NETIF_F_NTUPLE)))
5361		return 0;
5362
5363	if (!(features & NETIF_F_NTUPLE))
5364		igc_flush_nfc_rules(adapter);
5365
5366	netdev->features = features;
5367
5368	if (netif_running(netdev))
5369		igc_reinit_locked(adapter);
5370	else
5371		igc_reset(adapter);
5372
5373	return 1;
5374}
5375
5376static netdev_features_t
5377igc_features_check(struct sk_buff *skb, struct net_device *dev,
5378		   netdev_features_t features)
5379{
5380	unsigned int network_hdr_len, mac_hdr_len;
5381
5382	/* Make certain the headers can be described by a context descriptor */
5383	mac_hdr_len = skb_network_offset(skb);
5384	if (unlikely(mac_hdr_len > IGC_MAX_MAC_HDR_LEN))
5385		return features & ~(NETIF_F_HW_CSUM |
5386				    NETIF_F_SCTP_CRC |
5387				    NETIF_F_HW_VLAN_CTAG_TX |
5388				    NETIF_F_TSO |
5389				    NETIF_F_TSO6);
5390
5391	network_hdr_len = skb_checksum_start(skb) - skb_network_header(skb);
5392	if (unlikely(network_hdr_len >  IGC_MAX_NETWORK_HDR_LEN))
5393		return features & ~(NETIF_F_HW_CSUM |
5394				    NETIF_F_SCTP_CRC |
5395				    NETIF_F_TSO |
5396				    NETIF_F_TSO6);
5397
5398	/* We can only support IPv4 TSO in tunnels if we can mangle the
5399	 * inner IP ID field, so strip TSO if MANGLEID is not supported.
5400	 */
5401	if (skb->encapsulation && !(features & NETIF_F_TSO_MANGLEID))
5402		features &= ~NETIF_F_TSO;
5403
5404	return features;
5405}
5406
5407static void igc_tsync_interrupt(struct igc_adapter *adapter)
5408{
5409	struct igc_hw *hw = &adapter->hw;
5410	u32 tsauxc, sec, nsec, tsicr;
5411	struct ptp_clock_event event;
5412	struct timespec64 ts;
5413
5414	tsicr = rd32(IGC_TSICR);
5415
5416	if (tsicr & IGC_TSICR_SYS_WRAP) {
5417		event.type = PTP_CLOCK_PPS;
5418		if (adapter->ptp_caps.pps)
5419			ptp_clock_event(adapter->ptp_clock, &event);
5420	}
5421
5422	if (tsicr & IGC_TSICR_TXTS) {
5423		/* retrieve hardware timestamp */
5424		igc_ptp_tx_tstamp_event(adapter);
5425	}
5426
5427	if (tsicr & IGC_TSICR_TT0) {
5428		spin_lock(&adapter->tmreg_lock);
5429		ts = timespec64_add(adapter->perout[0].start,
5430				    adapter->perout[0].period);
5431		wr32(IGC_TRGTTIML0, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5432		wr32(IGC_TRGTTIMH0, (u32)ts.tv_sec);
5433		tsauxc = rd32(IGC_TSAUXC);
5434		tsauxc |= IGC_TSAUXC_EN_TT0;
5435		wr32(IGC_TSAUXC, tsauxc);
5436		adapter->perout[0].start = ts;
5437		spin_unlock(&adapter->tmreg_lock);
5438	}
5439
5440	if (tsicr & IGC_TSICR_TT1) {
5441		spin_lock(&adapter->tmreg_lock);
5442		ts = timespec64_add(adapter->perout[1].start,
5443				    adapter->perout[1].period);
5444		wr32(IGC_TRGTTIML1, ts.tv_nsec | IGC_TT_IO_TIMER_SEL_SYSTIM0);
5445		wr32(IGC_TRGTTIMH1, (u32)ts.tv_sec);
5446		tsauxc = rd32(IGC_TSAUXC);
5447		tsauxc |= IGC_TSAUXC_EN_TT1;
5448		wr32(IGC_TSAUXC, tsauxc);
5449		adapter->perout[1].start = ts;
5450		spin_unlock(&adapter->tmreg_lock);
5451	}
5452
5453	if (tsicr & IGC_TSICR_AUTT0) {
5454		nsec = rd32(IGC_AUXSTMPL0);
5455		sec  = rd32(IGC_AUXSTMPH0);
5456		event.type = PTP_CLOCK_EXTTS;
5457		event.index = 0;
5458		event.timestamp = sec * NSEC_PER_SEC + nsec;
5459		ptp_clock_event(adapter->ptp_clock, &event);
5460	}
5461
5462	if (tsicr & IGC_TSICR_AUTT1) {
5463		nsec = rd32(IGC_AUXSTMPL1);
5464		sec  = rd32(IGC_AUXSTMPH1);
5465		event.type = PTP_CLOCK_EXTTS;
5466		event.index = 1;
5467		event.timestamp = sec * NSEC_PER_SEC + nsec;
5468		ptp_clock_event(adapter->ptp_clock, &event);
5469	}
5470}
5471
5472/**
5473 * igc_msix_other - msix other interrupt handler
5474 * @irq: interrupt number
5475 * @data: pointer to a q_vector
5476 */
5477static irqreturn_t igc_msix_other(int irq, void *data)
5478{
5479	struct igc_adapter *adapter = data;
5480	struct igc_hw *hw = &adapter->hw;
5481	u32 icr = rd32(IGC_ICR);
5482
5483	/* reading ICR causes bit 31 of EICR to be cleared */
5484	if (icr & IGC_ICR_DRSTA)
5485		schedule_work(&adapter->reset_task);
5486
5487	if (icr & IGC_ICR_DOUTSYNC) {
5488		/* HW is reporting DMA is out of sync */
5489		adapter->stats.doosync++;
5490	}
5491
5492	if (icr & IGC_ICR_LSC) {
5493		hw->mac.get_link_status = true;
5494		/* guard against interrupt when we're going down */
5495		if (!test_bit(__IGC_DOWN, &adapter->state))
5496			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5497	}
5498
5499	if (icr & IGC_ICR_TS)
5500		igc_tsync_interrupt(adapter);
5501
5502	wr32(IGC_EIMS, adapter->eims_other);
5503
5504	return IRQ_HANDLED;
5505}
5506
5507static void igc_write_itr(struct igc_q_vector *q_vector)
5508{
5509	u32 itr_val = q_vector->itr_val & IGC_QVECTOR_MASK;
5510
5511	if (!q_vector->set_itr)
5512		return;
5513
5514	if (!itr_val)
5515		itr_val = IGC_ITR_VAL_MASK;
5516
5517	itr_val |= IGC_EITR_CNT_IGNR;
5518
5519	writel(itr_val, q_vector->itr_register);
5520	q_vector->set_itr = 0;
5521}
5522
5523static irqreturn_t igc_msix_ring(int irq, void *data)
5524{
5525	struct igc_q_vector *q_vector = data;
5526
5527	/* Write the ITR value calculated from the previous interrupt. */
5528	igc_write_itr(q_vector);
5529
5530	napi_schedule(&q_vector->napi);
5531
5532	return IRQ_HANDLED;
5533}
5534
5535/**
5536 * igc_request_msix - Initialize MSI-X interrupts
5537 * @adapter: Pointer to adapter structure
5538 *
5539 * igc_request_msix allocates MSI-X vectors and requests interrupts from the
5540 * kernel.
5541 */
5542static int igc_request_msix(struct igc_adapter *adapter)
5543{
5544	unsigned int num_q_vectors = adapter->num_q_vectors;
5545	int i = 0, err = 0, vector = 0, free_vector = 0;
5546	struct net_device *netdev = adapter->netdev;
5547
5548	err = request_irq(adapter->msix_entries[vector].vector,
5549			  &igc_msix_other, 0, netdev->name, adapter);
5550	if (err)
5551		goto err_out;
5552
5553	if (num_q_vectors > MAX_Q_VECTORS) {
5554		num_q_vectors = MAX_Q_VECTORS;
5555		dev_warn(&adapter->pdev->dev,
5556			 "The number of queue vectors (%d) is higher than max allowed (%d)\n",
5557			 adapter->num_q_vectors, MAX_Q_VECTORS);
5558	}
5559	for (i = 0; i < num_q_vectors; i++) {
5560		struct igc_q_vector *q_vector = adapter->q_vector[i];
5561
5562		vector++;
5563
5564		q_vector->itr_register = adapter->io_addr + IGC_EITR(vector);
5565
5566		if (q_vector->rx.ring && q_vector->tx.ring)
5567			sprintf(q_vector->name, "%s-TxRx-%u", netdev->name,
5568				q_vector->rx.ring->queue_index);
5569		else if (q_vector->tx.ring)
5570			sprintf(q_vector->name, "%s-tx-%u", netdev->name,
5571				q_vector->tx.ring->queue_index);
5572		else if (q_vector->rx.ring)
5573			sprintf(q_vector->name, "%s-rx-%u", netdev->name,
5574				q_vector->rx.ring->queue_index);
5575		else
5576			sprintf(q_vector->name, "%s-unused", netdev->name);
5577
5578		err = request_irq(adapter->msix_entries[vector].vector,
5579				  igc_msix_ring, 0, q_vector->name,
5580				  q_vector);
5581		if (err)
5582			goto err_free;
5583	}
5584
5585	igc_configure_msix(adapter);
5586	return 0;
5587
5588err_free:
5589	/* free already assigned IRQs */
5590	free_irq(adapter->msix_entries[free_vector++].vector, adapter);
5591
5592	vector--;
5593	for (i = 0; i < vector; i++) {
5594		free_irq(adapter->msix_entries[free_vector++].vector,
5595			 adapter->q_vector[i]);
5596	}
5597err_out:
5598	return err;
5599}
5600
5601/**
5602 * igc_clear_interrupt_scheme - reset the device to a state of no interrupts
5603 * @adapter: Pointer to adapter structure
5604 *
5605 * This function resets the device so that it has 0 rx queues, tx queues, and
5606 * MSI-X interrupts allocated.
5607 */
5608static void igc_clear_interrupt_scheme(struct igc_adapter *adapter)
5609{
5610	igc_free_q_vectors(adapter);
5611	igc_reset_interrupt_capability(adapter);
5612}
5613
5614/* Need to wait a few seconds after link up to get diagnostic information from
5615 * the phy
5616 */
5617static void igc_update_phy_info(struct timer_list *t)
5618{
5619	struct igc_adapter *adapter = from_timer(adapter, t, phy_info_timer);
5620
5621	igc_get_phy_info(&adapter->hw);
5622}
5623
5624/**
5625 * igc_has_link - check shared code for link and determine up/down
5626 * @adapter: pointer to driver private info
5627 */
5628bool igc_has_link(struct igc_adapter *adapter)
5629{
5630	struct igc_hw *hw = &adapter->hw;
5631	bool link_active = false;
5632
5633	/* get_link_status is set on LSC (link status) interrupt or
5634	 * rx sequence error interrupt.  get_link_status will stay
5635	 * false until the igc_check_for_link establishes link
5636	 * for copper adapters ONLY
5637	 */
5638	if (!hw->mac.get_link_status)
5639		return true;
5640	hw->mac.ops.check_for_link(hw);
5641	link_active = !hw->mac.get_link_status;
5642
5643	if (hw->mac.type == igc_i225) {
5644		if (!netif_carrier_ok(adapter->netdev)) {
5645			adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5646		} else if (!(adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)) {
5647			adapter->flags |= IGC_FLAG_NEED_LINK_UPDATE;
5648			adapter->link_check_timeout = jiffies;
5649		}
5650	}
5651
5652	return link_active;
5653}
5654
5655/**
5656 * igc_watchdog - Timer Call-back
5657 * @t: timer for the watchdog
5658 */
5659static void igc_watchdog(struct timer_list *t)
5660{
5661	struct igc_adapter *adapter = from_timer(adapter, t, watchdog_timer);
5662	/* Do the rest outside of interrupt context */
5663	schedule_work(&adapter->watchdog_task);
5664}
5665
5666static void igc_watchdog_task(struct work_struct *work)
5667{
5668	struct igc_adapter *adapter = container_of(work,
5669						   struct igc_adapter,
5670						   watchdog_task);
5671	struct net_device *netdev = adapter->netdev;
5672	struct igc_hw *hw = &adapter->hw;
5673	struct igc_phy_info *phy = &hw->phy;
5674	u16 phy_data, retry_count = 20;
5675	u32 link;
5676	int i;
5677
5678	link = igc_has_link(adapter);
5679
5680	if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE) {
5681		if (time_after(jiffies, (adapter->link_check_timeout + HZ)))
5682			adapter->flags &= ~IGC_FLAG_NEED_LINK_UPDATE;
5683		else
5684			link = false;
5685	}
5686
5687	if (link) {
5688		/* Cancel scheduled suspend requests. */
5689		pm_runtime_resume(netdev->dev.parent);
5690
5691		if (!netif_carrier_ok(netdev)) {
5692			u32 ctrl;
5693
5694			hw->mac.ops.get_speed_and_duplex(hw,
5695							 &adapter->link_speed,
5696							 &adapter->link_duplex);
5697
5698			ctrl = rd32(IGC_CTRL);
5699			/* Link status message must follow this format */
5700			netdev_info(netdev,
5701				    "NIC Link is Up %d Mbps %s Duplex, Flow Control: %s\n",
5702				    adapter->link_speed,
5703				    adapter->link_duplex == FULL_DUPLEX ?
5704				    "Full" : "Half",
5705				    (ctrl & IGC_CTRL_TFCE) &&
5706				    (ctrl & IGC_CTRL_RFCE) ? "RX/TX" :
5707				    (ctrl & IGC_CTRL_RFCE) ?  "RX" :
5708				    (ctrl & IGC_CTRL_TFCE) ?  "TX" : "None");
5709
5710			/* disable EEE if enabled */
5711			if ((adapter->flags & IGC_FLAG_EEE) &&
5712			    adapter->link_duplex == HALF_DUPLEX) {
5713				netdev_info(netdev,
5714					    "EEE Disabled: unsupported at half duplex. Re-enable using ethtool when at full duplex\n");
5715				adapter->hw.dev_spec._base.eee_enable = false;
5716				adapter->flags &= ~IGC_FLAG_EEE;
5717			}
5718
5719			/* check if SmartSpeed worked */
5720			igc_check_downshift(hw);
5721			if (phy->speed_downgraded)
5722				netdev_warn(netdev, "Link Speed was downgraded by SmartSpeed\n");
5723
5724			/* adjust timeout factor according to speed/duplex */
5725			adapter->tx_timeout_factor = 1;
5726			switch (adapter->link_speed) {
5727			case SPEED_10:
5728				adapter->tx_timeout_factor = 14;
5729				break;
5730			case SPEED_100:
5731			case SPEED_1000:
5732			case SPEED_2500:
5733				adapter->tx_timeout_factor = 1;
5734				break;
5735			}
5736
5737			/* Once the launch time has been set on the wire, there
5738			 * is a delay before the link speed can be determined
5739			 * based on link-up activity. Write into the register
5740			 * as soon as we know the correct link speed.
5741			 */
5742			igc_tsn_adjust_txtime_offset(adapter);
5743
5744			if (adapter->link_speed != SPEED_1000)
5745				goto no_wait;
5746
5747			/* wait for Remote receiver status OK */
5748retry_read_status:
5749			if (!igc_read_phy_reg(hw, PHY_1000T_STATUS,
5750					      &phy_data)) {
5751				if (!(phy_data & SR_1000T_REMOTE_RX_STATUS) &&
5752				    retry_count) {
5753					msleep(100);
5754					retry_count--;
5755					goto retry_read_status;
5756				} else if (!retry_count) {
5757					netdev_err(netdev, "exceed max 2 second\n");
5758				}
5759			} else {
5760				netdev_err(netdev, "read 1000Base-T Status Reg\n");
5761			}
5762no_wait:
5763			netif_carrier_on(netdev);
5764
5765			/* link state has changed, schedule phy info update */
5766			if (!test_bit(__IGC_DOWN, &adapter->state))
5767				mod_timer(&adapter->phy_info_timer,
5768					  round_jiffies(jiffies + 2 * HZ));
5769		}
5770	} else {
5771		if (netif_carrier_ok(netdev)) {
5772			adapter->link_speed = 0;
5773			adapter->link_duplex = 0;
5774
5775			/* Links status message must follow this format */
5776			netdev_info(netdev, "NIC Link is Down\n");
5777			netif_carrier_off(netdev);
5778
5779			/* link state has changed, schedule phy info update */
5780			if (!test_bit(__IGC_DOWN, &adapter->state))
5781				mod_timer(&adapter->phy_info_timer,
5782					  round_jiffies(jiffies + 2 * HZ));
5783
5784			pm_schedule_suspend(netdev->dev.parent,
5785					    MSEC_PER_SEC * 5);
5786		}
5787	}
5788
5789	spin_lock(&adapter->stats64_lock);
5790	igc_update_stats(adapter);
5791	spin_unlock(&adapter->stats64_lock);
5792
5793	for (i = 0; i < adapter->num_tx_queues; i++) {
5794		struct igc_ring *tx_ring = adapter->tx_ring[i];
5795
5796		if (!netif_carrier_ok(netdev)) {
5797			/* We've lost link, so the controller stops DMA,
5798			 * but we've got queued Tx work that's never going
5799			 * to get done, so reset controller to flush Tx.
5800			 * (Do the reset outside of interrupt context).
5801			 */
5802			if (igc_desc_unused(tx_ring) + 1 < tx_ring->count) {
5803				adapter->tx_timeout_count++;
5804				schedule_work(&adapter->reset_task);
5805				/* return immediately since reset is imminent */
5806				return;
5807			}
5808		}
5809
5810		/* Force detection of hung controller every watchdog period */
5811		set_bit(IGC_RING_FLAG_TX_DETECT_HANG, &tx_ring->flags);
5812	}
5813
5814	/* Cause software interrupt to ensure Rx ring is cleaned */
5815	if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5816		u32 eics = 0;
5817
5818		for (i = 0; i < adapter->num_q_vectors; i++) {
5819			struct igc_q_vector *q_vector = adapter->q_vector[i];
5820			struct igc_ring *rx_ring;
5821
5822			if (!q_vector->rx.ring)
5823				continue;
5824
5825			rx_ring = adapter->rx_ring[q_vector->rx.ring->queue_index];
5826
5827			if (test_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags)) {
5828				eics |= q_vector->eims_value;
5829				clear_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags);
5830			}
5831		}
5832		if (eics)
5833			wr32(IGC_EICS, eics);
5834	} else {
5835		struct igc_ring *rx_ring = adapter->rx_ring[0];
5836
5837		if (test_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags)) {
5838			clear_bit(IGC_RING_FLAG_RX_ALLOC_FAILED, &rx_ring->flags);
5839			wr32(IGC_ICS, IGC_ICS_RXDMT0);
5840		}
5841	}
5842
5843	igc_ptp_tx_hang(adapter);
5844
5845	/* Reset the timer */
5846	if (!test_bit(__IGC_DOWN, &adapter->state)) {
5847		if (adapter->flags & IGC_FLAG_NEED_LINK_UPDATE)
5848			mod_timer(&adapter->watchdog_timer,
5849				  round_jiffies(jiffies +  HZ));
5850		else
5851			mod_timer(&adapter->watchdog_timer,
5852				  round_jiffies(jiffies + 2 * HZ));
5853	}
5854}
5855
5856/**
5857 * igc_intr_msi - Interrupt Handler
5858 * @irq: interrupt number
5859 * @data: pointer to a network interface device structure
5860 */
5861static irqreturn_t igc_intr_msi(int irq, void *data)
5862{
5863	struct igc_adapter *adapter = data;
5864	struct igc_q_vector *q_vector = adapter->q_vector[0];
5865	struct igc_hw *hw = &adapter->hw;
5866	/* read ICR disables interrupts using IAM */
5867	u32 icr = rd32(IGC_ICR);
5868
5869	igc_write_itr(q_vector);
5870
5871	if (icr & IGC_ICR_DRSTA)
5872		schedule_work(&adapter->reset_task);
5873
5874	if (icr & IGC_ICR_DOUTSYNC) {
5875		/* HW is reporting DMA is out of sync */
5876		adapter->stats.doosync++;
5877	}
5878
5879	if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5880		hw->mac.get_link_status = true;
5881		if (!test_bit(__IGC_DOWN, &adapter->state))
5882			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5883	}
5884
5885	if (icr & IGC_ICR_TS)
5886		igc_tsync_interrupt(adapter);
5887
5888	napi_schedule(&q_vector->napi);
5889
5890	return IRQ_HANDLED;
5891}
5892
5893/**
5894 * igc_intr - Legacy Interrupt Handler
5895 * @irq: interrupt number
5896 * @data: pointer to a network interface device structure
5897 */
5898static irqreturn_t igc_intr(int irq, void *data)
5899{
5900	struct igc_adapter *adapter = data;
5901	struct igc_q_vector *q_vector = adapter->q_vector[0];
5902	struct igc_hw *hw = &adapter->hw;
5903	/* Interrupt Auto-Mask...upon reading ICR, interrupts are masked.  No
5904	 * need for the IMC write
5905	 */
5906	u32 icr = rd32(IGC_ICR);
5907
5908	/* IMS will not auto-mask if INT_ASSERTED is not set, and if it is
5909	 * not set, then the adapter didn't send an interrupt
5910	 */
5911	if (!(icr & IGC_ICR_INT_ASSERTED))
5912		return IRQ_NONE;
5913
5914	igc_write_itr(q_vector);
5915
5916	if (icr & IGC_ICR_DRSTA)
5917		schedule_work(&adapter->reset_task);
5918
5919	if (icr & IGC_ICR_DOUTSYNC) {
5920		/* HW is reporting DMA is out of sync */
5921		adapter->stats.doosync++;
5922	}
5923
5924	if (icr & (IGC_ICR_RXSEQ | IGC_ICR_LSC)) {
5925		hw->mac.get_link_status = true;
5926		/* guard against interrupt when we're going down */
5927		if (!test_bit(__IGC_DOWN, &adapter->state))
5928			mod_timer(&adapter->watchdog_timer, jiffies + 1);
5929	}
5930
5931	if (icr & IGC_ICR_TS)
5932		igc_tsync_interrupt(adapter);
5933
5934	napi_schedule(&q_vector->napi);
5935
5936	return IRQ_HANDLED;
5937}
5938
5939static void igc_free_irq(struct igc_adapter *adapter)
5940{
5941	if (adapter->msix_entries) {
5942		int vector = 0, i;
5943
5944		free_irq(adapter->msix_entries[vector++].vector, adapter);
5945
5946		for (i = 0; i < adapter->num_q_vectors; i++)
5947			free_irq(adapter->msix_entries[vector++].vector,
5948				 adapter->q_vector[i]);
5949	} else {
5950		free_irq(adapter->pdev->irq, adapter);
5951	}
5952}
5953
5954/**
5955 * igc_request_irq - initialize interrupts
5956 * @adapter: Pointer to adapter structure
5957 *
5958 * Attempts to configure interrupts using the best available
5959 * capabilities of the hardware and kernel.
5960 */
5961static int igc_request_irq(struct igc_adapter *adapter)
5962{
5963	struct net_device *netdev = adapter->netdev;
5964	struct pci_dev *pdev = adapter->pdev;
5965	int err = 0;
5966
5967	if (adapter->flags & IGC_FLAG_HAS_MSIX) {
5968		err = igc_request_msix(adapter);
5969		if (!err)
5970			goto request_done;
5971		/* fall back to MSI */
5972		igc_free_all_tx_resources(adapter);
5973		igc_free_all_rx_resources(adapter);
5974
5975		igc_clear_interrupt_scheme(adapter);
5976		err = igc_init_interrupt_scheme(adapter, false);
5977		if (err)
5978			goto request_done;
5979		igc_setup_all_tx_resources(adapter);
5980		igc_setup_all_rx_resources(adapter);
5981		igc_configure(adapter);
5982	}
5983
5984	igc_assign_vector(adapter->q_vector[0], 0);
5985
5986	if (adapter->flags & IGC_FLAG_HAS_MSI) {
5987		err = request_irq(pdev->irq, &igc_intr_msi, 0,
5988				  netdev->name, adapter);
5989		if (!err)
5990			goto request_done;
5991
5992		/* fall back to legacy interrupts */
5993		igc_reset_interrupt_capability(adapter);
5994		adapter->flags &= ~IGC_FLAG_HAS_MSI;
5995	}
5996
5997	err = request_irq(pdev->irq, &igc_intr, IRQF_SHARED,
5998			  netdev->name, adapter);
5999
6000	if (err)
6001		netdev_err(netdev, "Error %d getting interrupt\n", err);
6002
6003request_done:
6004	return err;
6005}
6006
6007/**
6008 * __igc_open - Called when a network interface is made active
6009 * @netdev: network interface device structure
6010 * @resuming: boolean indicating if the device is resuming
6011 *
6012 * Returns 0 on success, negative value on failure
6013 *
6014 * The open entry point is called when a network interface is made
6015 * active by the system (IFF_UP).  At this point all resources needed
6016 * for transmit and receive operations are allocated, the interrupt
6017 * handler is registered with the OS, the watchdog timer is started,
6018 * and the stack is notified that the interface is ready.
6019 */
6020static int __igc_open(struct net_device *netdev, bool resuming)
6021{
6022	struct igc_adapter *adapter = netdev_priv(netdev);
6023	struct pci_dev *pdev = adapter->pdev;
6024	struct igc_hw *hw = &adapter->hw;
6025	int err = 0;
6026	int i = 0;
6027
6028	/* disallow open during test */
6029
6030	if (test_bit(__IGC_TESTING, &adapter->state)) {
6031		WARN_ON(resuming);
6032		return -EBUSY;
6033	}
6034
6035	if (!resuming)
6036		pm_runtime_get_sync(&pdev->dev);
6037
6038	netif_carrier_off(netdev);
6039
6040	/* allocate transmit descriptors */
6041	err = igc_setup_all_tx_resources(adapter);
6042	if (err)
6043		goto err_setup_tx;
6044
6045	/* allocate receive descriptors */
6046	err = igc_setup_all_rx_resources(adapter);
6047	if (err)
6048		goto err_setup_rx;
6049
6050	igc_power_up_link(adapter);
6051
6052	igc_configure(adapter);
6053
6054	err = igc_request_irq(adapter);
6055	if (err)
6056		goto err_req_irq;
6057
6058	clear_bit(__IGC_DOWN, &adapter->state);
6059
6060	for (i = 0; i < adapter->num_q_vectors; i++)
6061		napi_enable(&adapter->q_vector[i]->napi);
6062
6063	/* Clear any pending interrupts. */
6064	rd32(IGC_ICR);
6065	igc_irq_enable(adapter);
6066
6067	if (!resuming)
6068		pm_runtime_put(&pdev->dev);
6069
6070	netif_tx_start_all_queues(netdev);
6071
6072	/* start the watchdog. */
6073	hw->mac.get_link_status = true;
6074	schedule_work(&adapter->watchdog_task);
6075
6076	return IGC_SUCCESS;
6077
6078err_req_irq:
6079	igc_release_hw_control(adapter);
6080	igc_power_down_phy_copper_base(&adapter->hw);
6081	igc_free_all_rx_resources(adapter);
6082err_setup_rx:
6083	igc_free_all_tx_resources(adapter);
6084err_setup_tx:
6085	igc_reset(adapter);
6086	if (!resuming)
6087		pm_runtime_put(&pdev->dev);
6088
6089	return err;
6090}
6091
6092int igc_open(struct net_device *netdev)
6093{
6094	struct igc_adapter *adapter = netdev_priv(netdev);
6095	int err;
6096
6097	/* Notify the stack of the actual queue counts. */
6098	err = netif_set_real_num_queues(netdev, adapter->num_tx_queues,
6099					adapter->num_rx_queues);
6100	if (err) {
6101		netdev_err(netdev, "error setting real queue count\n");
6102		return err;
6103	}
6104
6105	return __igc_open(netdev, false);
6106}
6107
6108/**
6109 * __igc_close - Disables a network interface
6110 * @netdev: network interface device structure
6111 * @suspending: boolean indicating the device is suspending
6112 *
6113 * Returns 0, this is not allowed to fail
6114 *
6115 * The close entry point is called when an interface is de-activated
6116 * by the OS.  The hardware is still under the driver's control, but
6117 * needs to be disabled.  A global MAC reset is issued to stop the
6118 * hardware, and all transmit and receive resources are freed.
6119 */
6120static int __igc_close(struct net_device *netdev, bool suspending)
6121{
6122	struct igc_adapter *adapter = netdev_priv(netdev);
6123	struct pci_dev *pdev = adapter->pdev;
6124
6125	WARN_ON(test_bit(__IGC_RESETTING, &adapter->state));
6126
6127	if (!suspending)
6128		pm_runtime_get_sync(&pdev->dev);
6129
6130	igc_down(adapter);
6131
6132	igc_release_hw_control(adapter);
6133
6134	igc_free_irq(adapter);
6135
6136	igc_free_all_tx_resources(adapter);
6137	igc_free_all_rx_resources(adapter);
6138
6139	if (!suspending)
6140		pm_runtime_put_sync(&pdev->dev);
6141
6142	return 0;
6143}
6144
6145int igc_close(struct net_device *netdev)
6146{
6147	if (netif_device_present(netdev) || netdev->dismantle)
6148		return __igc_close(netdev, false);
6149	return 0;
6150}
6151
6152/**
6153 * igc_ioctl - Access the hwtstamp interface
6154 * @netdev: network interface device structure
6155 * @ifr: interface request data
6156 * @cmd: ioctl command
6157 **/
6158static int igc_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
6159{
6160	switch (cmd) {
6161	case SIOCGHWTSTAMP:
6162		return igc_ptp_get_ts_config(netdev, ifr);
6163	case SIOCSHWTSTAMP:
6164		return igc_ptp_set_ts_config(netdev, ifr);
6165	default:
6166		return -EOPNOTSUPP;
6167	}
6168}
6169
6170static int igc_save_launchtime_params(struct igc_adapter *adapter, int queue,
6171				      bool enable)
6172{
6173	struct igc_ring *ring;
6174
6175	if (queue < 0 || queue >= adapter->num_tx_queues)
6176		return -EINVAL;
6177
6178	ring = adapter->tx_ring[queue];
6179	ring->launchtime_enable = enable;
6180
6181	return 0;
6182}
6183
6184static bool is_base_time_past(ktime_t base_time, const struct timespec64 *now)
6185{
6186	struct timespec64 b;
6187
6188	b = ktime_to_timespec64(base_time);
6189
6190	return timespec64_compare(now, &b) > 0;
6191}
6192
6193static bool validate_schedule(struct igc_adapter *adapter,
6194			      const struct tc_taprio_qopt_offload *qopt)
6195{
6196	int queue_uses[IGC_MAX_TX_QUEUES] = { };
6197	struct igc_hw *hw = &adapter->hw;
6198	struct timespec64 now;
6199	size_t n;
6200
6201	if (qopt->cycle_time_extension)
6202		return false;
6203
6204	igc_ptp_read(adapter, &now);
6205
6206	/* If we program the controller's BASET registers with a time
6207	 * in the future, it will hold all the packets until that
6208	 * time, causing a lot of TX Hangs, so to avoid that, we
6209	 * reject schedules that would start in the future.
6210	 * Note: Limitation above is no longer in i226.
6211	 */
6212	if (!is_base_time_past(qopt->base_time, &now) &&
6213	    igc_is_device_id_i225(hw))
6214		return false;
6215
6216	for (n = 0; n < qopt->num_entries; n++) {
6217		const struct tc_taprio_sched_entry *e, *prev;
6218		int i;
6219
6220		prev = n ? &qopt->entries[n - 1] : NULL;
6221		e = &qopt->entries[n];
6222
6223		/* i225 only supports "global" frame preemption
6224		 * settings.
6225		 */
6226		if (e->command != TC_TAPRIO_CMD_SET_GATES)
6227			return false;
6228
6229		for (i = 0; i < adapter->num_tx_queues; i++)
6230			if (e->gate_mask & BIT(i)) {
6231				queue_uses[i]++;
6232
6233				/* There are limitations: A single queue cannot
6234				 * be opened and closed multiple times per cycle
6235				 * unless the gate stays open. Check for it.
6236				 */
6237				if (queue_uses[i] > 1 &&
6238				    !(prev->gate_mask & BIT(i)))
6239					return false;
6240			}
6241	}
6242
6243	return true;
6244}
6245
6246static int igc_tsn_enable_launchtime(struct igc_adapter *adapter,
6247				     struct tc_etf_qopt_offload *qopt)
6248{
6249	struct igc_hw *hw = &adapter->hw;
6250	int err;
6251
6252	if (hw->mac.type != igc_i225)
6253		return -EOPNOTSUPP;
6254
6255	err = igc_save_launchtime_params(adapter, qopt->queue, qopt->enable);
6256	if (err)
6257		return err;
6258
6259	return igc_tsn_offload_apply(adapter);
6260}
6261
6262static int igc_qbv_clear_schedule(struct igc_adapter *adapter)
6263{
6264	unsigned long flags;
6265	int i;
6266
6267	adapter->base_time = 0;
6268	adapter->cycle_time = NSEC_PER_SEC;
6269	adapter->taprio_offload_enable = false;
6270	adapter->qbv_config_change_errors = 0;
6271	adapter->qbv_count = 0;
6272
6273	for (i = 0; i < adapter->num_tx_queues; i++) {
6274		struct igc_ring *ring = adapter->tx_ring[i];
6275
6276		ring->start_time = 0;
6277		ring->end_time = NSEC_PER_SEC;
6278		ring->max_sdu = 0;
6279	}
6280
6281	spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
6282
6283	adapter->qbv_transition = false;
6284
6285	for (i = 0; i < adapter->num_tx_queues; i++) {
6286		struct igc_ring *ring = adapter->tx_ring[i];
6287
6288		ring->oper_gate_closed = false;
6289		ring->admin_gate_closed = false;
6290	}
6291
6292	spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
6293
6294	return 0;
6295}
6296
6297static int igc_tsn_clear_schedule(struct igc_adapter *adapter)
6298{
6299	igc_qbv_clear_schedule(adapter);
6300
6301	return 0;
6302}
6303
6304static void igc_taprio_stats(struct net_device *dev,
6305			     struct tc_taprio_qopt_stats *stats)
6306{
6307	/* When Strict_End is enabled, the tx_overruns counter
6308	 * will always be zero.
6309	 */
6310	stats->tx_overruns = 0;
6311}
6312
6313static void igc_taprio_queue_stats(struct net_device *dev,
6314				   struct tc_taprio_qopt_queue_stats *queue_stats)
6315{
6316	struct tc_taprio_qopt_stats *stats = &queue_stats->stats;
6317
6318	/* When Strict_End is enabled, the tx_overruns counter
6319	 * will always be zero.
6320	 */
6321	stats->tx_overruns = 0;
6322}
6323
6324static int igc_save_qbv_schedule(struct igc_adapter *adapter,
6325				 struct tc_taprio_qopt_offload *qopt)
6326{
6327	bool queue_configured[IGC_MAX_TX_QUEUES] = { };
6328	struct igc_hw *hw = &adapter->hw;
6329	u32 start_time = 0, end_time = 0;
6330	struct timespec64 now;
6331	unsigned long flags;
6332	size_t n;
6333	int i;
6334
6335	if (qopt->base_time < 0)
6336		return -ERANGE;
6337
6338	if (igc_is_device_id_i225(hw) && adapter->taprio_offload_enable)
6339		return -EALREADY;
6340
6341	if (!validate_schedule(adapter, qopt))
6342		return -EINVAL;
6343
6344	igc_ptp_read(adapter, &now);
6345
6346	if (igc_tsn_is_taprio_activated_by_user(adapter) &&
6347	    is_base_time_past(qopt->base_time, &now))
6348		adapter->qbv_config_change_errors++;
6349
6350	adapter->cycle_time = qopt->cycle_time;
6351	adapter->base_time = qopt->base_time;
6352	adapter->taprio_offload_enable = true;
6353
6354	for (n = 0; n < qopt->num_entries; n++) {
6355		struct tc_taprio_sched_entry *e = &qopt->entries[n];
6356
6357		end_time += e->interval;
6358
6359		/* If any of the conditions below are true, we need to manually
6360		 * control the end time of the cycle.
6361		 * 1. Qbv users can specify a cycle time that is not equal
6362		 * to the total GCL intervals. Hence, recalculation is
6363		 * necessary here to exclude the time interval that
6364		 * exceeds the cycle time.
6365		 * 2. According to IEEE Std. 802.1Q-2018 section 8.6.9.2,
6366		 * once the end of the list is reached, it will switch
6367		 * to the END_OF_CYCLE state and leave the gates in the
6368		 * same state until the next cycle is started.
6369		 */
6370		if (end_time > adapter->cycle_time ||
6371		    n + 1 == qopt->num_entries)
6372			end_time = adapter->cycle_time;
6373
6374		for (i = 0; i < adapter->num_tx_queues; i++) {
6375			struct igc_ring *ring = adapter->tx_ring[i];
6376
6377			if (!(e->gate_mask & BIT(i)))
6378				continue;
6379
6380			/* Check whether a queue stays open for more than one
6381			 * entry. If so, keep the start and advance the end
6382			 * time.
6383			 */
6384			if (!queue_configured[i])
6385				ring->start_time = start_time;
6386			ring->end_time = end_time;
6387
6388			if (ring->start_time >= adapter->cycle_time)
6389				queue_configured[i] = false;
6390			else
6391				queue_configured[i] = true;
6392		}
6393
6394		start_time += e->interval;
6395	}
6396
6397	spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
6398
6399	/* Check whether a queue gets configured.
6400	 * If not, set the start and end time to be end time.
6401	 */
6402	for (i = 0; i < adapter->num_tx_queues; i++) {
6403		struct igc_ring *ring = adapter->tx_ring[i];
6404
6405		if (!is_base_time_past(qopt->base_time, &now)) {
6406			ring->admin_gate_closed = false;
6407		} else {
6408			ring->oper_gate_closed = false;
6409			ring->admin_gate_closed = false;
6410		}
6411
6412		if (!queue_configured[i]) {
6413			if (!is_base_time_past(qopt->base_time, &now))
6414				ring->admin_gate_closed = true;
6415			else
6416				ring->oper_gate_closed = true;
6417
6418			ring->start_time = end_time;
6419			ring->end_time = end_time;
6420		}
6421	}
6422
6423	spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
6424
6425	for (i = 0; i < adapter->num_tx_queues; i++) {
6426		struct igc_ring *ring = adapter->tx_ring[i];
6427		struct net_device *dev = adapter->netdev;
6428
6429		if (qopt->max_sdu[i])
6430			ring->max_sdu = qopt->max_sdu[i] + dev->hard_header_len - ETH_TLEN;
6431		else
6432			ring->max_sdu = 0;
6433	}
6434
6435	return 0;
6436}
6437
6438static int igc_tsn_enable_qbv_scheduling(struct igc_adapter *adapter,
6439					 struct tc_taprio_qopt_offload *qopt)
6440{
6441	struct igc_hw *hw = &adapter->hw;
6442	int err;
6443
6444	if (hw->mac.type != igc_i225)
6445		return -EOPNOTSUPP;
6446
6447	switch (qopt->cmd) {
6448	case TAPRIO_CMD_REPLACE:
6449		err = igc_save_qbv_schedule(adapter, qopt);
6450		break;
6451	case TAPRIO_CMD_DESTROY:
6452		err = igc_tsn_clear_schedule(adapter);
6453		break;
6454	case TAPRIO_CMD_STATS:
6455		igc_taprio_stats(adapter->netdev, &qopt->stats);
6456		return 0;
6457	case TAPRIO_CMD_QUEUE_STATS:
6458		igc_taprio_queue_stats(adapter->netdev, &qopt->queue_stats);
6459		return 0;
6460	default:
6461		return -EOPNOTSUPP;
6462	}
6463
6464	if (err)
6465		return err;
6466
6467	return igc_tsn_offload_apply(adapter);
6468}
6469
6470static int igc_save_cbs_params(struct igc_adapter *adapter, int queue,
6471			       bool enable, int idleslope, int sendslope,
6472			       int hicredit, int locredit)
6473{
6474	bool cbs_status[IGC_MAX_SR_QUEUES] = { false };
6475	struct net_device *netdev = adapter->netdev;
6476	struct igc_ring *ring;
6477	int i;
6478
6479	/* i225 has two sets of credit-based shaper logic.
6480	 * Supporting it only on the top two priority queues
6481	 */
6482	if (queue < 0 || queue > 1)
6483		return -EINVAL;
6484
6485	ring = adapter->tx_ring[queue];
6486
6487	for (i = 0; i < IGC_MAX_SR_QUEUES; i++)
6488		if (adapter->tx_ring[i])
6489			cbs_status[i] = adapter->tx_ring[i]->cbs_enable;
6490
6491	/* CBS should be enabled on the highest priority queue first in order
6492	 * for the CBS algorithm to operate as intended.
6493	 */
6494	if (enable) {
6495		if (queue == 1 && !cbs_status[0]) {
6496			netdev_err(netdev,
6497				   "Enabling CBS on queue1 before queue0\n");
6498			return -EINVAL;
6499		}
6500	} else {
6501		if (queue == 0 && cbs_status[1]) {
6502			netdev_err(netdev,
6503				   "Disabling CBS on queue0 before queue1\n");
6504			return -EINVAL;
6505		}
6506	}
6507
6508	ring->cbs_enable = enable;
6509	ring->idleslope = idleslope;
6510	ring->sendslope = sendslope;
6511	ring->hicredit = hicredit;
6512	ring->locredit = locredit;
6513
6514	return 0;
6515}
6516
6517static int igc_tsn_enable_cbs(struct igc_adapter *adapter,
6518			      struct tc_cbs_qopt_offload *qopt)
6519{
6520	struct igc_hw *hw = &adapter->hw;
6521	int err;
6522
6523	if (hw->mac.type != igc_i225)
6524		return -EOPNOTSUPP;
6525
6526	if (qopt->queue < 0 || qopt->queue > 1)
6527		return -EINVAL;
6528
6529	err = igc_save_cbs_params(adapter, qopt->queue, qopt->enable,
6530				  qopt->idleslope, qopt->sendslope,
6531				  qopt->hicredit, qopt->locredit);
6532	if (err)
6533		return err;
6534
6535	return igc_tsn_offload_apply(adapter);
6536}
6537
6538static int igc_tc_query_caps(struct igc_adapter *adapter,
6539			     struct tc_query_caps_base *base)
6540{
6541	struct igc_hw *hw = &adapter->hw;
6542
6543	switch (base->type) {
6544	case TC_SETUP_QDISC_MQPRIO: {
6545		struct tc_mqprio_caps *caps = base->caps;
6546
6547		caps->validate_queue_counts = true;
6548
6549		return 0;
6550	}
6551	case TC_SETUP_QDISC_TAPRIO: {
6552		struct tc_taprio_caps *caps = base->caps;
6553
6554		caps->broken_mqprio = true;
6555
6556		if (hw->mac.type == igc_i225) {
6557			caps->supports_queue_max_sdu = true;
6558			caps->gate_mask_per_txq = true;
6559		}
6560
6561		return 0;
6562	}
6563	default:
6564		return -EOPNOTSUPP;
6565	}
6566}
6567
6568static void igc_save_mqprio_params(struct igc_adapter *adapter, u8 num_tc,
6569				   u16 *offset)
6570{
6571	int i;
6572
6573	adapter->strict_priority_enable = true;
6574	adapter->num_tc = num_tc;
6575
6576	for (i = 0; i < num_tc; i++)
6577		adapter->queue_per_tc[i] = offset[i];
6578}
6579
6580static int igc_tsn_enable_mqprio(struct igc_adapter *adapter,
6581				 struct tc_mqprio_qopt_offload *mqprio)
6582{
6583	struct igc_hw *hw = &adapter->hw;
6584	int i;
6585
6586	if (hw->mac.type != igc_i225)
6587		return -EOPNOTSUPP;
6588
6589	if (!mqprio->qopt.num_tc) {
6590		adapter->strict_priority_enable = false;
6591		goto apply;
6592	}
6593
6594	/* There are as many TCs as Tx queues. */
6595	if (mqprio->qopt.num_tc != adapter->num_tx_queues) {
6596		NL_SET_ERR_MSG_FMT_MOD(mqprio->extack,
6597				       "Only %d traffic classes supported",
6598				       adapter->num_tx_queues);
6599		return -EOPNOTSUPP;
6600	}
6601
6602	/* Only one queue per TC is supported. */
6603	for (i = 0; i < mqprio->qopt.num_tc; i++) {
6604		if (mqprio->qopt.count[i] != 1) {
6605			NL_SET_ERR_MSG_MOD(mqprio->extack,
6606					   "Only one queue per TC supported");
6607			return -EOPNOTSUPP;
6608		}
6609	}
6610
6611	/* Preemption is not supported yet. */
6612	if (mqprio->preemptible_tcs) {
6613		NL_SET_ERR_MSG_MOD(mqprio->extack,
6614				   "Preemption is not supported yet");
6615		return -EOPNOTSUPP;
6616	}
6617
6618	igc_save_mqprio_params(adapter, mqprio->qopt.num_tc,
6619			       mqprio->qopt.offset);
6620
6621	mqprio->qopt.hw = TC_MQPRIO_HW_OFFLOAD_TCS;
6622
6623apply:
6624	return igc_tsn_offload_apply(adapter);
6625}
6626
6627static int igc_setup_tc(struct net_device *dev, enum tc_setup_type type,
6628			void *type_data)
6629{
6630	struct igc_adapter *adapter = netdev_priv(dev);
6631
6632	adapter->tc_setup_type = type;
6633
6634	switch (type) {
6635	case TC_QUERY_CAPS:
6636		return igc_tc_query_caps(adapter, type_data);
6637	case TC_SETUP_QDISC_TAPRIO:
6638		return igc_tsn_enable_qbv_scheduling(adapter, type_data);
6639
6640	case TC_SETUP_QDISC_ETF:
6641		return igc_tsn_enable_launchtime(adapter, type_data);
6642
6643	case TC_SETUP_QDISC_CBS:
6644		return igc_tsn_enable_cbs(adapter, type_data);
6645
6646	case TC_SETUP_QDISC_MQPRIO:
6647		return igc_tsn_enable_mqprio(adapter, type_data);
6648
6649	default:
6650		return -EOPNOTSUPP;
6651	}
6652}
6653
6654static int igc_bpf(struct net_device *dev, struct netdev_bpf *bpf)
6655{
6656	struct igc_adapter *adapter = netdev_priv(dev);
6657
6658	switch (bpf->command) {
6659	case XDP_SETUP_PROG:
6660		return igc_xdp_set_prog(adapter, bpf->prog, bpf->extack);
6661	case XDP_SETUP_XSK_POOL:
6662		return igc_xdp_setup_pool(adapter, bpf->xsk.pool,
6663					  bpf->xsk.queue_id);
6664	default:
6665		return -EOPNOTSUPP;
6666	}
6667}
6668
6669static int igc_xdp_xmit(struct net_device *dev, int num_frames,
6670			struct xdp_frame **frames, u32 flags)
6671{
6672	struct igc_adapter *adapter = netdev_priv(dev);
6673	int cpu = smp_processor_id();
6674	struct netdev_queue *nq;
6675	struct igc_ring *ring;
6676	int i, nxmit;
6677
6678	if (unlikely(!netif_carrier_ok(dev)))
6679		return -ENETDOWN;
6680
6681	if (unlikely(flags & ~XDP_XMIT_FLAGS_MASK))
6682		return -EINVAL;
6683
6684	ring = igc_xdp_get_tx_ring(adapter, cpu);
6685	nq = txring_txq(ring);
6686
6687	__netif_tx_lock(nq, cpu);
6688
6689	/* Avoid transmit queue timeout since we share it with the slow path */
6690	txq_trans_cond_update(nq);
6691
6692	nxmit = 0;
6693	for (i = 0; i < num_frames; i++) {
6694		int err;
6695		struct xdp_frame *xdpf = frames[i];
6696
6697		err = igc_xdp_init_tx_descriptor(ring, xdpf);
6698		if (err)
6699			break;
6700		nxmit++;
6701	}
6702
6703	if (flags & XDP_XMIT_FLUSH)
6704		igc_flush_tx_descriptors(ring);
6705
6706	__netif_tx_unlock(nq);
6707
6708	return nxmit;
6709}
6710
6711static void igc_trigger_rxtxq_interrupt(struct igc_adapter *adapter,
6712					struct igc_q_vector *q_vector)
6713{
6714	struct igc_hw *hw = &adapter->hw;
6715	u32 eics = 0;
6716
6717	eics |= q_vector->eims_value;
6718	wr32(IGC_EICS, eics);
6719}
6720
6721int igc_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags)
6722{
6723	struct igc_adapter *adapter = netdev_priv(dev);
6724	struct igc_q_vector *q_vector;
6725	struct igc_ring *ring;
6726
6727	if (test_bit(__IGC_DOWN, &adapter->state))
6728		return -ENETDOWN;
6729
6730	if (!igc_xdp_is_enabled(adapter))
6731		return -ENXIO;
6732
6733	if (queue_id >= adapter->num_rx_queues)
6734		return -EINVAL;
6735
6736	ring = adapter->rx_ring[queue_id];
6737
6738	if (!ring->xsk_pool)
6739		return -ENXIO;
6740
6741	q_vector = adapter->q_vector[queue_id];
6742	if (!napi_if_scheduled_mark_missed(&q_vector->napi))
6743		igc_trigger_rxtxq_interrupt(adapter, q_vector);
6744
6745	return 0;
6746}
6747
6748static ktime_t igc_get_tstamp(struct net_device *dev,
6749			      const struct skb_shared_hwtstamps *hwtstamps,
6750			      bool cycles)
6751{
6752	struct igc_adapter *adapter = netdev_priv(dev);
6753	struct igc_inline_rx_tstamps *tstamp;
6754	ktime_t timestamp;
6755
6756	tstamp = hwtstamps->netdev_data;
6757
6758	if (cycles)
6759		timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer1);
6760	else
6761		timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer0);
6762
6763	return timestamp;
6764}
6765
6766static const struct net_device_ops igc_netdev_ops = {
6767	.ndo_open		= igc_open,
6768	.ndo_stop		= igc_close,
6769	.ndo_start_xmit		= igc_xmit_frame,
6770	.ndo_set_rx_mode	= igc_set_rx_mode,
6771	.ndo_set_mac_address	= igc_set_mac,
6772	.ndo_change_mtu		= igc_change_mtu,
6773	.ndo_tx_timeout		= igc_tx_timeout,
6774	.ndo_get_stats64	= igc_get_stats64,
6775	.ndo_fix_features	= igc_fix_features,
6776	.ndo_set_features	= igc_set_features,
6777	.ndo_features_check	= igc_features_check,
6778	.ndo_eth_ioctl		= igc_ioctl,
6779	.ndo_setup_tc		= igc_setup_tc,
6780	.ndo_bpf		= igc_bpf,
6781	.ndo_xdp_xmit		= igc_xdp_xmit,
6782	.ndo_xsk_wakeup		= igc_xsk_wakeup,
6783	.ndo_get_tstamp		= igc_get_tstamp,
6784};
6785
6786/* PCIe configuration access */
6787void igc_read_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6788{
6789	struct igc_adapter *adapter = hw->back;
6790
6791	pci_read_config_word(adapter->pdev, reg, value);
6792}
6793
6794void igc_write_pci_cfg(struct igc_hw *hw, u32 reg, u16 *value)
6795{
6796	struct igc_adapter *adapter = hw->back;
6797
6798	pci_write_config_word(adapter->pdev, reg, *value);
6799}
6800
6801s32 igc_read_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6802{
6803	struct igc_adapter *adapter = hw->back;
6804
6805	if (!pci_is_pcie(adapter->pdev))
6806		return -IGC_ERR_CONFIG;
6807
6808	pcie_capability_read_word(adapter->pdev, reg, value);
6809
6810	return IGC_SUCCESS;
6811}
6812
6813s32 igc_write_pcie_cap_reg(struct igc_hw *hw, u32 reg, u16 *value)
6814{
6815	struct igc_adapter *adapter = hw->back;
6816
6817	if (!pci_is_pcie(adapter->pdev))
6818		return -IGC_ERR_CONFIG;
6819
6820	pcie_capability_write_word(adapter->pdev, reg, *value);
6821
6822	return IGC_SUCCESS;
6823}
6824
6825u32 igc_rd32(struct igc_hw *hw, u32 reg)
6826{
6827	struct igc_adapter *igc = container_of(hw, struct igc_adapter, hw);
6828	u8 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
6829	u32 value = 0;
6830
6831	if (IGC_REMOVED(hw_addr))
6832		return ~value;
6833
6834	value = readl(&hw_addr[reg]);
6835
6836	/* reads should not return all F's */
6837	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
6838		struct net_device *netdev = igc->netdev;
6839
6840		hw->hw_addr = NULL;
6841		netif_device_detach(netdev);
6842		netdev_err(netdev, "PCIe link lost, device now detached\n");
6843		WARN(pci_device_is_present(igc->pdev),
6844		     "igc: Failed to read reg 0x%x!\n", reg);
6845	}
6846
6847	return value;
6848}
6849
6850/* Mapping HW RSS Type to enum xdp_rss_hash_type */
6851static enum xdp_rss_hash_type igc_xdp_rss_type[IGC_RSS_TYPE_MAX_TABLE] = {
6852	[IGC_RSS_TYPE_NO_HASH]		= XDP_RSS_TYPE_L2,
6853	[IGC_RSS_TYPE_HASH_TCP_IPV4]	= XDP_RSS_TYPE_L4_IPV4_TCP,
6854	[IGC_RSS_TYPE_HASH_IPV4]	= XDP_RSS_TYPE_L3_IPV4,
6855	[IGC_RSS_TYPE_HASH_TCP_IPV6]	= XDP_RSS_TYPE_L4_IPV6_TCP,
6856	[IGC_RSS_TYPE_HASH_IPV6_EX]	= XDP_RSS_TYPE_L3_IPV6_EX,
6857	[IGC_RSS_TYPE_HASH_IPV6]	= XDP_RSS_TYPE_L3_IPV6,
6858	[IGC_RSS_TYPE_HASH_TCP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_TCP_EX,
6859	[IGC_RSS_TYPE_HASH_UDP_IPV4]	= XDP_RSS_TYPE_L4_IPV4_UDP,
6860	[IGC_RSS_TYPE_HASH_UDP_IPV6]	= XDP_RSS_TYPE_L4_IPV6_UDP,
6861	[IGC_RSS_TYPE_HASH_UDP_IPV6_EX] = XDP_RSS_TYPE_L4_IPV6_UDP_EX,
6862	[10] = XDP_RSS_TYPE_NONE, /* RSS Type above 9 "Reserved" by HW  */
6863	[11] = XDP_RSS_TYPE_NONE, /* keep array sized for SW bit-mask   */
6864	[12] = XDP_RSS_TYPE_NONE, /* to handle future HW revisons       */
6865	[13] = XDP_RSS_TYPE_NONE,
6866	[14] = XDP_RSS_TYPE_NONE,
6867	[15] = XDP_RSS_TYPE_NONE,
6868};
6869
6870static int igc_xdp_rx_hash(const struct xdp_md *_ctx, u32 *hash,
6871			   enum xdp_rss_hash_type *rss_type)
6872{
6873	const struct igc_xdp_buff *ctx = (void *)_ctx;
6874
6875	if (!(ctx->xdp.rxq->dev->features & NETIF_F_RXHASH))
6876		return -ENODATA;
6877
6878	*hash = le32_to_cpu(ctx->rx_desc->wb.lower.hi_dword.rss);
6879	*rss_type = igc_xdp_rss_type[igc_rss_type(ctx->rx_desc)];
6880
6881	return 0;
6882}
6883
6884static int igc_xdp_rx_timestamp(const struct xdp_md *_ctx, u64 *timestamp)
6885{
6886	const struct igc_xdp_buff *ctx = (void *)_ctx;
6887	struct igc_adapter *adapter = netdev_priv(ctx->xdp.rxq->dev);
6888	struct igc_inline_rx_tstamps *tstamp = ctx->rx_ts;
6889
6890	if (igc_test_staterr(ctx->rx_desc, IGC_RXDADV_STAT_TSIP)) {
6891		*timestamp = igc_ptp_rx_pktstamp(adapter, tstamp->timer0);
6892
6893		return 0;
6894	}
6895
6896	return -ENODATA;
6897}
6898
6899static const struct xdp_metadata_ops igc_xdp_metadata_ops = {
6900	.xmo_rx_hash			= igc_xdp_rx_hash,
6901	.xmo_rx_timestamp		= igc_xdp_rx_timestamp,
6902};
6903
6904static enum hrtimer_restart igc_qbv_scheduling_timer(struct hrtimer *timer)
6905{
6906	struct igc_adapter *adapter = container_of(timer, struct igc_adapter,
6907						   hrtimer);
6908	unsigned long flags;
6909	unsigned int i;
6910
6911	spin_lock_irqsave(&adapter->qbv_tx_lock, flags);
6912
6913	adapter->qbv_transition = true;
6914	for (i = 0; i < adapter->num_tx_queues; i++) {
6915		struct igc_ring *tx_ring = adapter->tx_ring[i];
6916
6917		if (tx_ring->admin_gate_closed) {
6918			tx_ring->admin_gate_closed = false;
6919			tx_ring->oper_gate_closed = true;
6920		} else {
6921			tx_ring->oper_gate_closed = false;
6922		}
6923	}
6924	adapter->qbv_transition = false;
6925
6926	spin_unlock_irqrestore(&adapter->qbv_tx_lock, flags);
6927
6928	return HRTIMER_NORESTART;
6929}
6930
6931/**
6932 * igc_probe - Device Initialization Routine
6933 * @pdev: PCI device information struct
6934 * @ent: entry in igc_pci_tbl
6935 *
6936 * Returns 0 on success, negative on failure
6937 *
6938 * igc_probe initializes an adapter identified by a pci_dev structure.
6939 * The OS initialization, configuring the adapter private structure,
6940 * and a hardware reset occur.
6941 */
6942static int igc_probe(struct pci_dev *pdev,
6943		     const struct pci_device_id *ent)
6944{
6945	struct igc_adapter *adapter;
6946	struct net_device *netdev;
6947	struct igc_hw *hw;
6948	const struct igc_info *ei = igc_info_tbl[ent->driver_data];
6949	int err;
6950
6951	err = pci_enable_device_mem(pdev);
6952	if (err)
6953		return err;
6954
6955	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
6956	if (err) {
6957		dev_err(&pdev->dev,
6958			"No usable DMA configuration, aborting\n");
6959		goto err_dma;
6960	}
6961
6962	err = pci_request_mem_regions(pdev, igc_driver_name);
6963	if (err)
6964		goto err_pci_reg;
6965
6966	err = pci_enable_ptm(pdev, NULL);
6967	if (err < 0)
6968		dev_info(&pdev->dev, "PCIe PTM not supported by PCIe bus/controller\n");
6969
6970	pci_set_master(pdev);
6971
6972	err = -ENOMEM;
6973	netdev = alloc_etherdev_mq(sizeof(struct igc_adapter),
6974				   IGC_MAX_TX_QUEUES);
6975
6976	if (!netdev)
6977		goto err_alloc_etherdev;
6978
6979	SET_NETDEV_DEV(netdev, &pdev->dev);
6980
6981	pci_set_drvdata(pdev, netdev);
6982	adapter = netdev_priv(netdev);
6983	adapter->netdev = netdev;
6984	adapter->pdev = pdev;
6985	hw = &adapter->hw;
6986	hw->back = adapter;
6987	adapter->port_num = hw->bus.func;
6988	adapter->msg_enable = netif_msg_init(debug, DEFAULT_MSG_ENABLE);
6989
6990	err = pci_save_state(pdev);
6991	if (err)
6992		goto err_ioremap;
6993
6994	err = -EIO;
6995	adapter->io_addr = ioremap(pci_resource_start(pdev, 0),
6996				   pci_resource_len(pdev, 0));
6997	if (!adapter->io_addr)
6998		goto err_ioremap;
6999
7000	/* hw->hw_addr can be zeroed, so use adapter->io_addr for unmap */
7001	hw->hw_addr = adapter->io_addr;
7002
7003	netdev->netdev_ops = &igc_netdev_ops;
7004	netdev->xdp_metadata_ops = &igc_xdp_metadata_ops;
7005	netdev->xsk_tx_metadata_ops = &igc_xsk_tx_metadata_ops;
7006	igc_ethtool_set_ops(netdev);
7007	netdev->watchdog_timeo = 5 * HZ;
7008
7009	netdev->mem_start = pci_resource_start(pdev, 0);
7010	netdev->mem_end = pci_resource_end(pdev, 0);
7011
7012	/* PCI config space info */
7013	hw->vendor_id = pdev->vendor;
7014	hw->device_id = pdev->device;
7015	hw->revision_id = pdev->revision;
7016	hw->subsystem_vendor_id = pdev->subsystem_vendor;
7017	hw->subsystem_device_id = pdev->subsystem_device;
7018
7019	/* Copy the default MAC and PHY function pointers */
7020	memcpy(&hw->mac.ops, ei->mac_ops, sizeof(hw->mac.ops));
7021	memcpy(&hw->phy.ops, ei->phy_ops, sizeof(hw->phy.ops));
7022
7023	/* Initialize skew-specific constants */
7024	err = ei->get_invariants(hw);
7025	if (err)
7026		goto err_sw_init;
7027
7028	/* Add supported features to the features list*/
7029	netdev->features |= NETIF_F_SG;
7030	netdev->features |= NETIF_F_TSO;
7031	netdev->features |= NETIF_F_TSO6;
7032	netdev->features |= NETIF_F_TSO_ECN;
7033	netdev->features |= NETIF_F_RXHASH;
7034	netdev->features |= NETIF_F_RXCSUM;
7035	netdev->features |= NETIF_F_HW_CSUM;
7036	netdev->features |= NETIF_F_SCTP_CRC;
7037	netdev->features |= NETIF_F_HW_TC;
7038
7039#define IGC_GSO_PARTIAL_FEATURES (NETIF_F_GSO_GRE | \
7040				  NETIF_F_GSO_GRE_CSUM | \
7041				  NETIF_F_GSO_IPXIP4 | \
7042				  NETIF_F_GSO_IPXIP6 | \
7043				  NETIF_F_GSO_UDP_TUNNEL | \
7044				  NETIF_F_GSO_UDP_TUNNEL_CSUM)
7045
7046	netdev->gso_partial_features = IGC_GSO_PARTIAL_FEATURES;
7047	netdev->features |= NETIF_F_GSO_PARTIAL | IGC_GSO_PARTIAL_FEATURES;
7048
7049	/* setup the private structure */
7050	err = igc_sw_init(adapter);
7051	if (err)
7052		goto err_sw_init;
7053
7054	/* copy netdev features into list of user selectable features */
7055	netdev->hw_features |= NETIF_F_NTUPLE;
7056	netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
7057	netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
7058	netdev->hw_features |= netdev->features;
7059
7060	netdev->features |= NETIF_F_HIGHDMA;
7061
7062	netdev->vlan_features |= netdev->features | NETIF_F_TSO_MANGLEID;
7063	netdev->mpls_features |= NETIF_F_HW_CSUM;
7064	netdev->hw_enc_features |= netdev->vlan_features;
7065
7066	netdev->xdp_features = NETDEV_XDP_ACT_BASIC | NETDEV_XDP_ACT_REDIRECT |
7067			       NETDEV_XDP_ACT_XSK_ZEROCOPY;
7068
7069	/* MTU range: 68 - 9216 */
7070	netdev->min_mtu = ETH_MIN_MTU;
7071	netdev->max_mtu = MAX_STD_JUMBO_FRAME_SIZE;
7072
7073	/* before reading the NVM, reset the controller to put the device in a
7074	 * known good starting state
7075	 */
7076	hw->mac.ops.reset_hw(hw);
7077
7078	if (igc_get_flash_presence_i225(hw)) {
7079		if (hw->nvm.ops.validate(hw) < 0) {
7080			dev_err(&pdev->dev, "The NVM Checksum Is Not Valid\n");
7081			err = -EIO;
7082			goto err_eeprom;
7083		}
7084	}
7085
7086	if (eth_platform_get_mac_address(&pdev->dev, hw->mac.addr)) {
7087		/* copy the MAC address out of the NVM */
7088		if (hw->mac.ops.read_mac_addr(hw))
7089			dev_err(&pdev->dev, "NVM Read Error\n");
7090	}
7091
7092	eth_hw_addr_set(netdev, hw->mac.addr);
7093
7094	if (!is_valid_ether_addr(netdev->dev_addr)) {
7095		dev_err(&pdev->dev, "Invalid MAC Address\n");
7096		err = -EIO;
7097		goto err_eeprom;
7098	}
7099
7100	/* configure RXPBSIZE and TXPBSIZE */
7101	wr32(IGC_RXPBS, I225_RXPBSIZE_DEFAULT);
7102	wr32(IGC_TXPBS, I225_TXPBSIZE_DEFAULT);
7103
7104	timer_setup(&adapter->watchdog_timer, igc_watchdog, 0);
7105	timer_setup(&adapter->phy_info_timer, igc_update_phy_info, 0);
7106
7107	INIT_WORK(&adapter->reset_task, igc_reset_task);
7108	INIT_WORK(&adapter->watchdog_task, igc_watchdog_task);
7109
7110	hrtimer_init(&adapter->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
7111	adapter->hrtimer.function = &igc_qbv_scheduling_timer;
7112
7113	/* Initialize link properties that are user-changeable */
7114	adapter->fc_autoneg = true;
7115	hw->phy.autoneg_advertised = 0xaf;
7116
7117	hw->fc.requested_mode = igc_fc_default;
7118	hw->fc.current_mode = igc_fc_default;
7119
7120	/* By default, support wake on port A */
7121	adapter->flags |= IGC_FLAG_WOL_SUPPORTED;
7122
7123	/* initialize the wol settings based on the eeprom settings */
7124	if (adapter->flags & IGC_FLAG_WOL_SUPPORTED)
7125		adapter->wol |= IGC_WUFC_MAG;
7126
7127	device_set_wakeup_enable(&adapter->pdev->dev,
7128				 adapter->flags & IGC_FLAG_WOL_SUPPORTED);
7129
7130	igc_ptp_init(adapter);
7131
7132	igc_tsn_clear_schedule(adapter);
7133
7134	/* reset the hardware with the new settings */
7135	igc_reset(adapter);
7136
7137	/* let the f/w know that the h/w is now under the control of the
7138	 * driver.
7139	 */
7140	igc_get_hw_control(adapter);
7141
7142	strscpy(netdev->name, "eth%d", sizeof(netdev->name));
7143	err = register_netdev(netdev);
7144	if (err)
7145		goto err_register;
7146
7147	 /* carrier off reporting is important to ethtool even BEFORE open */
7148	netif_carrier_off(netdev);
7149
7150	/* Check if Media Autosense is enabled */
7151	adapter->ei = *ei;
7152
7153	/* print pcie link status and MAC address */
7154	pcie_print_link_status(pdev);
7155	netdev_info(netdev, "MAC: %pM\n", netdev->dev_addr);
7156
7157	dev_pm_set_driver_flags(&pdev->dev, DPM_FLAG_NO_DIRECT_COMPLETE);
7158	/* Disable EEE for internal PHY devices */
7159	hw->dev_spec._base.eee_enable = false;
7160	adapter->flags &= ~IGC_FLAG_EEE;
7161	igc_set_eee_i225(hw, false, false, false);
7162
7163	pm_runtime_put_noidle(&pdev->dev);
7164
7165	if (IS_ENABLED(CONFIG_IGC_LEDS)) {
7166		err = igc_led_setup(adapter);
7167		if (err)
7168			goto err_register;
7169	}
7170
7171	return 0;
7172
7173err_register:
7174	igc_release_hw_control(adapter);
7175err_eeprom:
7176	if (!igc_check_reset_block(hw))
7177		igc_reset_phy(hw);
7178err_sw_init:
7179	igc_clear_interrupt_scheme(adapter);
7180	iounmap(adapter->io_addr);
7181err_ioremap:
7182	free_netdev(netdev);
7183err_alloc_etherdev:
7184	pci_release_mem_regions(pdev);
7185err_pci_reg:
7186err_dma:
7187	pci_disable_device(pdev);
7188	return err;
7189}
7190
7191/**
7192 * igc_remove - Device Removal Routine
7193 * @pdev: PCI device information struct
7194 *
7195 * igc_remove is called by the PCI subsystem to alert the driver
7196 * that it should release a PCI device.  This could be caused by a
7197 * Hot-Plug event, or because the driver is going to be removed from
7198 * memory.
7199 */
7200static void igc_remove(struct pci_dev *pdev)
7201{
7202	struct net_device *netdev = pci_get_drvdata(pdev);
7203	struct igc_adapter *adapter = netdev_priv(netdev);
7204
7205	pm_runtime_get_noresume(&pdev->dev);
7206
7207	igc_flush_nfc_rules(adapter);
7208
7209	igc_ptp_stop(adapter);
7210
7211	pci_disable_ptm(pdev);
7212	pci_clear_master(pdev);
7213
7214	set_bit(__IGC_DOWN, &adapter->state);
7215
7216	del_timer_sync(&adapter->watchdog_timer);
7217	del_timer_sync(&adapter->phy_info_timer);
7218
7219	cancel_work_sync(&adapter->reset_task);
7220	cancel_work_sync(&adapter->watchdog_task);
7221	hrtimer_cancel(&adapter->hrtimer);
7222
7223	if (IS_ENABLED(CONFIG_IGC_LEDS))
7224		igc_led_free(adapter);
7225
7226	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
7227	 * would have already happened in close and is redundant.
7228	 */
7229	igc_release_hw_control(adapter);
7230	unregister_netdev(netdev);
7231
7232	igc_clear_interrupt_scheme(adapter);
7233	pci_iounmap(pdev, adapter->io_addr);
7234	pci_release_mem_regions(pdev);
7235
7236	free_netdev(netdev);
7237
7238	pci_disable_device(pdev);
7239}
7240
7241static int __igc_shutdown(struct pci_dev *pdev, bool *enable_wake,
7242			  bool runtime)
7243{
7244	struct net_device *netdev = pci_get_drvdata(pdev);
7245	struct igc_adapter *adapter = netdev_priv(netdev);
7246	u32 wufc = runtime ? IGC_WUFC_LNKC : adapter->wol;
7247	struct igc_hw *hw = &adapter->hw;
7248	u32 ctrl, rctl, status;
7249	bool wake;
7250
7251	rtnl_lock();
7252	netif_device_detach(netdev);
7253
7254	if (netif_running(netdev))
7255		__igc_close(netdev, true);
7256
7257	igc_ptp_suspend(adapter);
7258
7259	igc_clear_interrupt_scheme(adapter);
7260	rtnl_unlock();
7261
7262	status = rd32(IGC_STATUS);
7263	if (status & IGC_STATUS_LU)
7264		wufc &= ~IGC_WUFC_LNKC;
7265
7266	if (wufc) {
7267		igc_setup_rctl(adapter);
7268		igc_set_rx_mode(netdev);
7269
7270		/* turn on all-multi mode if wake on multicast is enabled */
7271		if (wufc & IGC_WUFC_MC) {
7272			rctl = rd32(IGC_RCTL);
7273			rctl |= IGC_RCTL_MPE;
7274			wr32(IGC_RCTL, rctl);
7275		}
7276
7277		ctrl = rd32(IGC_CTRL);
7278		ctrl |= IGC_CTRL_ADVD3WUC;
7279		wr32(IGC_CTRL, ctrl);
7280
7281		/* Allow time for pending master requests to run */
7282		igc_disable_pcie_master(hw);
7283
7284		wr32(IGC_WUC, IGC_WUC_PME_EN);
7285		wr32(IGC_WUFC, wufc);
7286	} else {
7287		wr32(IGC_WUC, 0);
7288		wr32(IGC_WUFC, 0);
7289	}
7290
7291	wake = wufc || adapter->en_mng_pt;
7292	if (!wake)
7293		igc_power_down_phy_copper_base(&adapter->hw);
7294	else
7295		igc_power_up_link(adapter);
7296
7297	if (enable_wake)
7298		*enable_wake = wake;
7299
7300	/* Release control of h/w to f/w.  If f/w is AMT enabled, this
7301	 * would have already happened in close and is redundant.
7302	 */
7303	igc_release_hw_control(adapter);
7304
7305	pci_disable_device(pdev);
7306
7307	return 0;
7308}
7309
7310static int igc_runtime_suspend(struct device *dev)
7311{
7312	return __igc_shutdown(to_pci_dev(dev), NULL, 1);
7313}
7314
7315static void igc_deliver_wake_packet(struct net_device *netdev)
7316{
7317	struct igc_adapter *adapter = netdev_priv(netdev);
7318	struct igc_hw *hw = &adapter->hw;
7319	struct sk_buff *skb;
7320	u32 wupl;
7321
7322	wupl = rd32(IGC_WUPL) & IGC_WUPL_MASK;
7323
7324	/* WUPM stores only the first 128 bytes of the wake packet.
7325	 * Read the packet only if we have the whole thing.
7326	 */
7327	if (wupl == 0 || wupl > IGC_WUPM_BYTES)
7328		return;
7329
7330	skb = netdev_alloc_skb_ip_align(netdev, IGC_WUPM_BYTES);
7331	if (!skb)
7332		return;
7333
7334	skb_put(skb, wupl);
7335
7336	/* Ensure reads are 32-bit aligned */
7337	wupl = roundup(wupl, 4);
7338
7339	memcpy_fromio(skb->data, hw->hw_addr + IGC_WUPM_REG(0), wupl);
7340
7341	skb->protocol = eth_type_trans(skb, netdev);
7342	netif_rx(skb);
7343}
7344
7345static int igc_resume(struct device *dev)
7346{
7347	struct pci_dev *pdev = to_pci_dev(dev);
7348	struct net_device *netdev = pci_get_drvdata(pdev);
7349	struct igc_adapter *adapter = netdev_priv(netdev);
7350	struct igc_hw *hw = &adapter->hw;
7351	u32 err, val;
7352
7353	pci_set_power_state(pdev, PCI_D0);
7354	pci_restore_state(pdev);
7355	pci_save_state(pdev);
7356
7357	if (!pci_device_is_present(pdev))
7358		return -ENODEV;
7359	err = pci_enable_device_mem(pdev);
7360	if (err) {
7361		netdev_err(netdev, "Cannot enable PCI device from suspend\n");
7362		return err;
7363	}
7364	pci_set_master(pdev);
7365
7366	pci_enable_wake(pdev, PCI_D3hot, 0);
7367	pci_enable_wake(pdev, PCI_D3cold, 0);
7368
7369	if (igc_init_interrupt_scheme(adapter, true)) {
7370		netdev_err(netdev, "Unable to allocate memory for queues\n");
7371		return -ENOMEM;
7372	}
7373
7374	igc_reset(adapter);
7375
7376	/* let the f/w know that the h/w is now under the control of the
7377	 * driver.
7378	 */
7379	igc_get_hw_control(adapter);
7380
7381	val = rd32(IGC_WUS);
7382	if (val & WAKE_PKT_WUS)
7383		igc_deliver_wake_packet(netdev);
7384
7385	wr32(IGC_WUS, ~0);
7386
7387	if (netif_running(netdev)) {
7388		err = __igc_open(netdev, true);
7389		if (!err)
7390			netif_device_attach(netdev);
7391	}
7392
7393	return err;
7394}
7395
7396static int igc_runtime_resume(struct device *dev)
7397{
7398	return igc_resume(dev);
7399}
7400
7401static int igc_suspend(struct device *dev)
7402{
7403	return __igc_shutdown(to_pci_dev(dev), NULL, 0);
7404}
7405
7406static int __maybe_unused igc_runtime_idle(struct device *dev)
7407{
7408	struct net_device *netdev = dev_get_drvdata(dev);
7409	struct igc_adapter *adapter = netdev_priv(netdev);
7410
7411	if (!igc_has_link(adapter))
7412		pm_schedule_suspend(dev, MSEC_PER_SEC * 5);
7413
7414	return -EBUSY;
7415}
7416
7417static void igc_shutdown(struct pci_dev *pdev)
7418{
7419	bool wake;
7420
7421	__igc_shutdown(pdev, &wake, 0);
7422
7423	if (system_state == SYSTEM_POWER_OFF) {
7424		pci_wake_from_d3(pdev, wake);
7425		pci_set_power_state(pdev, PCI_D3hot);
7426	}
7427}
7428
7429/**
7430 *  igc_io_error_detected - called when PCI error is detected
7431 *  @pdev: Pointer to PCI device
7432 *  @state: The current PCI connection state
7433 *
7434 *  This function is called after a PCI bus error affecting
7435 *  this device has been detected.
7436 **/
7437static pci_ers_result_t igc_io_error_detected(struct pci_dev *pdev,
7438					      pci_channel_state_t state)
7439{
7440	struct net_device *netdev = pci_get_drvdata(pdev);
7441	struct igc_adapter *adapter = netdev_priv(netdev);
7442
7443	netif_device_detach(netdev);
7444
7445	if (state == pci_channel_io_perm_failure)
7446		return PCI_ERS_RESULT_DISCONNECT;
7447
7448	if (netif_running(netdev))
7449		igc_down(adapter);
7450	pci_disable_device(pdev);
7451
7452	/* Request a slot reset. */
7453	return PCI_ERS_RESULT_NEED_RESET;
7454}
7455
7456/**
7457 *  igc_io_slot_reset - called after the PCI bus has been reset.
7458 *  @pdev: Pointer to PCI device
7459 *
7460 *  Restart the card from scratch, as if from a cold-boot. Implementation
7461 *  resembles the first-half of the igc_resume routine.
7462 **/
7463static pci_ers_result_t igc_io_slot_reset(struct pci_dev *pdev)
7464{
7465	struct net_device *netdev = pci_get_drvdata(pdev);
7466	struct igc_adapter *adapter = netdev_priv(netdev);
7467	struct igc_hw *hw = &adapter->hw;
7468	pci_ers_result_t result;
7469
7470	if (pci_enable_device_mem(pdev)) {
7471		netdev_err(netdev, "Could not re-enable PCI device after reset\n");
7472		result = PCI_ERS_RESULT_DISCONNECT;
7473	} else {
7474		pci_set_master(pdev);
7475		pci_restore_state(pdev);
7476		pci_save_state(pdev);
7477
7478		pci_enable_wake(pdev, PCI_D3hot, 0);
7479		pci_enable_wake(pdev, PCI_D3cold, 0);
7480
7481		/* In case of PCI error, adapter loses its HW address
7482		 * so we should re-assign it here.
7483		 */
7484		hw->hw_addr = adapter->io_addr;
7485
7486		igc_reset(adapter);
7487		wr32(IGC_WUS, ~0);
7488		result = PCI_ERS_RESULT_RECOVERED;
7489	}
7490
7491	return result;
7492}
7493
7494/**
7495 *  igc_io_resume - called when traffic can start to flow again.
7496 *  @pdev: Pointer to PCI device
7497 *
7498 *  This callback is called when the error recovery driver tells us that
7499 *  its OK to resume normal operation. Implementation resembles the
7500 *  second-half of the igc_resume routine.
7501 */
7502static void igc_io_resume(struct pci_dev *pdev)
7503{
7504	struct net_device *netdev = pci_get_drvdata(pdev);
7505	struct igc_adapter *adapter = netdev_priv(netdev);
7506
7507	rtnl_lock();
7508	if (netif_running(netdev)) {
7509		if (igc_open(netdev)) {
7510			rtnl_unlock();
7511			netdev_err(netdev, "igc_open failed after reset\n");
7512			return;
7513		}
7514	}
7515
7516	netif_device_attach(netdev);
7517
7518	/* let the f/w know that the h/w is now under the control of the
7519	 * driver.
7520	 */
7521	igc_get_hw_control(adapter);
7522	rtnl_unlock();
7523}
7524
7525static const struct pci_error_handlers igc_err_handler = {
7526	.error_detected = igc_io_error_detected,
7527	.slot_reset = igc_io_slot_reset,
7528	.resume = igc_io_resume,
7529};
7530
7531static _DEFINE_DEV_PM_OPS(igc_pm_ops, igc_suspend, igc_resume,
7532			  igc_runtime_suspend, igc_runtime_resume,
7533			  igc_runtime_idle);
7534
7535static struct pci_driver igc_driver = {
7536	.name     = igc_driver_name,
7537	.id_table = igc_pci_tbl,
7538	.probe    = igc_probe,
7539	.remove   = igc_remove,
7540	.driver.pm = pm_ptr(&igc_pm_ops),
7541	.shutdown = igc_shutdown,
7542	.err_handler = &igc_err_handler,
7543};
7544
7545/**
7546 * igc_reinit_queues - return error
7547 * @adapter: pointer to adapter structure
7548 */
7549int igc_reinit_queues(struct igc_adapter *adapter)
7550{
7551	struct net_device *netdev = adapter->netdev;
7552	int err = 0;
7553
7554	if (netif_running(netdev))
7555		igc_close(netdev);
7556
7557	igc_reset_interrupt_capability(adapter);
7558
7559	if (igc_init_interrupt_scheme(adapter, true)) {
7560		netdev_err(netdev, "Unable to allocate memory for queues\n");
7561		return -ENOMEM;
7562	}
7563
7564	if (netif_running(netdev))
7565		err = igc_open(netdev);
7566
7567	return err;
7568}
7569
7570/**
7571 * igc_get_hw_dev - return device
7572 * @hw: pointer to hardware structure
7573 *
7574 * used by hardware layer to print debugging information
7575 */
7576struct net_device *igc_get_hw_dev(struct igc_hw *hw)
7577{
7578	struct igc_adapter *adapter = hw->back;
7579
7580	return adapter->netdev;
7581}
7582
7583static void igc_disable_rx_ring_hw(struct igc_ring *ring)
7584{
7585	struct igc_hw *hw = &ring->q_vector->adapter->hw;
7586	u8 idx = ring->reg_idx;
7587	u32 rxdctl;
7588
7589	rxdctl = rd32(IGC_RXDCTL(idx));
7590	rxdctl &= ~IGC_RXDCTL_QUEUE_ENABLE;
7591	rxdctl |= IGC_RXDCTL_SWFLUSH;
7592	wr32(IGC_RXDCTL(idx), rxdctl);
7593}
7594
7595void igc_disable_rx_ring(struct igc_ring *ring)
7596{
7597	igc_disable_rx_ring_hw(ring);
7598	igc_clean_rx_ring(ring);
7599}
7600
7601void igc_enable_rx_ring(struct igc_ring *ring)
7602{
7603	struct igc_adapter *adapter = ring->q_vector->adapter;
7604
7605	igc_configure_rx_ring(adapter, ring);
7606
7607	if (ring->xsk_pool)
7608		igc_alloc_rx_buffers_zc(ring, igc_desc_unused(ring));
7609	else
7610		igc_alloc_rx_buffers(ring, igc_desc_unused(ring));
7611}
7612
7613void igc_disable_tx_ring(struct igc_ring *ring)
7614{
7615	igc_disable_tx_ring_hw(ring);
7616	igc_clean_tx_ring(ring);
7617}
7618
7619void igc_enable_tx_ring(struct igc_ring *ring)
7620{
7621	struct igc_adapter *adapter = ring->q_vector->adapter;
7622
7623	igc_configure_tx_ring(adapter, ring);
7624}
7625
7626/**
7627 * igc_init_module - Driver Registration Routine
7628 *
7629 * igc_init_module is the first routine called when the driver is
7630 * loaded. All it does is register with the PCI subsystem.
7631 */
7632static int __init igc_init_module(void)
7633{
7634	int ret;
7635
7636	pr_info("%s\n", igc_driver_string);
7637	pr_info("%s\n", igc_copyright);
7638
7639	ret = pci_register_driver(&igc_driver);
7640	return ret;
7641}
7642
7643module_init(igc_init_module);
7644
7645/**
7646 * igc_exit_module - Driver Exit Cleanup Routine
7647 *
7648 * igc_exit_module is called just before the driver is removed
7649 * from memory.
7650 */
7651static void __exit igc_exit_module(void)
7652{
7653	pci_unregister_driver(&igc_driver);
7654}
7655
7656module_exit(igc_exit_module);
7657/* igc_main.c */