1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright(c) 2013 - 2019 Intel Corporation. */
   3
   4#include <linux/module.h>
   5#include <linux/interrupt.h>
   6#include <linux/aer.h>
   7
   8#include "fm10k.h"
   9
  10static const struct fm10k_info *fm10k_info_tbl[] = {
  11	[fm10k_device_pf] = &fm10k_pf_info,
  12	[fm10k_device_vf] = &fm10k_vf_info,
  13};
  14
  15/*
  16 * fm10k_pci_tbl - PCI Device ID Table
  17 *
  18 * Wildcard entries (PCI_ANY_ID) should come last
  19 * Last entry must be all 0s
  20 *
  21 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
  22 *   Class, Class Mask, private data (not used) }
  23 */
  24static const struct pci_device_id fm10k_pci_tbl[] = {
  25	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_PF), fm10k_device_pf },
  26	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_SDI_FM10420_QDA2), fm10k_device_pf },
  27	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_SDI_FM10420_DA2), fm10k_device_pf },
  28	{ PCI_VDEVICE(INTEL, FM10K_DEV_ID_VF), fm10k_device_vf },
  29	/* required last entry */
  30	{ 0, }
  31};
  32MODULE_DEVICE_TABLE(pci, fm10k_pci_tbl);
  33
  34u16 fm10k_read_pci_cfg_word(struct fm10k_hw *hw, u32 reg)
  35{
  36	struct fm10k_intfc *interface = hw->back;
  37	u16 value = 0;
  38
  39	if (FM10K_REMOVED(hw->hw_addr))
  40		return ~value;
  41
  42	pci_read_config_word(interface->pdev, reg, &value);
  43	if (value == 0xFFFF)
  44		fm10k_write_flush(hw);
  45
  46	return value;
  47}
  48
  49u32 fm10k_read_reg(struct fm10k_hw *hw, int reg)
  50{
  51	u32 __iomem *hw_addr = READ_ONCE(hw->hw_addr);
  52	u32 value = 0;
  53
  54	if (FM10K_REMOVED(hw_addr))
  55		return ~value;
  56
  57	value = readl(&hw_addr[reg]);
  58	if (!(~value) && (!reg || !(~readl(hw_addr)))) {
  59		struct fm10k_intfc *interface = hw->back;
  60		struct net_device *netdev = interface->netdev;
  61
  62		hw->hw_addr = NULL;
  63		netif_device_detach(netdev);
  64		netdev_err(netdev, "PCIe link lost, device now detached\n");
  65	}
  66
  67	return value;
  68}
  69
  70static int fm10k_hw_ready(struct fm10k_intfc *interface)
  71{
  72	struct fm10k_hw *hw = &interface->hw;
  73
  74	fm10k_write_flush(hw);
  75
  76	return FM10K_REMOVED(hw->hw_addr) ? -ENODEV : 0;
  77}
  78
  79/**
  80 * fm10k_macvlan_schedule - Schedule MAC/VLAN queue task
  81 * @interface: fm10k private interface structure
  82 *
  83 * Schedule the MAC/VLAN queue monitor task. If the MAC/VLAN task cannot be
  84 * started immediately, request that it be restarted when possible.
  85 */
  86void fm10k_macvlan_schedule(struct fm10k_intfc *interface)
  87{
  88	/* Avoid processing the MAC/VLAN queue when the service task is
  89	 * disabled, or when we're resetting the device.
  90	 */
  91	if (!test_bit(__FM10K_MACVLAN_DISABLE, interface->state) &&
  92	    !test_and_set_bit(__FM10K_MACVLAN_SCHED, interface->state)) {
  93		clear_bit(__FM10K_MACVLAN_REQUEST, interface->state);
  94		/* We delay the actual start of execution in order to allow
  95		 * multiple MAC/VLAN updates to accumulate before handling
  96		 * them, and to allow some time to let the mailbox drain
  97		 * between runs.
  98		 */
  99		queue_delayed_work(fm10k_workqueue,
 100				   &interface->macvlan_task, 10);
 101	} else {
 102		set_bit(__FM10K_MACVLAN_REQUEST, interface->state);
 103	}
 104}
 105
 106/**
 107 * fm10k_stop_macvlan_task - Stop the MAC/VLAN queue monitor
 108 * @interface: fm10k private interface structure
 109 *
 110 * Wait until the MAC/VLAN queue task has stopped, and cancel any future
 111 * requests.
 112 */
 113static void fm10k_stop_macvlan_task(struct fm10k_intfc *interface)
 114{
 115	/* Disable the MAC/VLAN work item */
 116	set_bit(__FM10K_MACVLAN_DISABLE, interface->state);
 117
 118	/* Make sure we waited until any current invocations have stopped */
 119	cancel_delayed_work_sync(&interface->macvlan_task);
 120
 121	/* We set the __FM10K_MACVLAN_SCHED bit when we schedule the task.
 122	 * However, it may not be unset of the MAC/VLAN task never actually
 123	 * got a chance to run. Since we've canceled the task here, and it
 124	 * cannot be rescheuled right now, we need to ensure the scheduled bit
 125	 * gets unset.
 126	 */
 127	clear_bit(__FM10K_MACVLAN_SCHED, interface->state);
 128}
 129
 130/**
 131 * fm10k_resume_macvlan_task - Restart the MAC/VLAN queue monitor
 132 * @interface: fm10k private interface structure
 133 *
 134 * Clear the __FM10K_MACVLAN_DISABLE bit and, if a request occurred, schedule
 135 * the MAC/VLAN work monitor.
 136 */
 137static void fm10k_resume_macvlan_task(struct fm10k_intfc *interface)
 138{
 139	/* Re-enable the MAC/VLAN work item */
 140	clear_bit(__FM10K_MACVLAN_DISABLE, interface->state);
 141
 142	/* We might have received a MAC/VLAN request while disabled. If so,
 143	 * kick off the queue now.
 144	 */
 145	if (test_bit(__FM10K_MACVLAN_REQUEST, interface->state))
 146		fm10k_macvlan_schedule(interface);
 147}
 148
 149void fm10k_service_event_schedule(struct fm10k_intfc *interface)
 150{
 151	if (!test_bit(__FM10K_SERVICE_DISABLE, interface->state) &&
 152	    !test_and_set_bit(__FM10K_SERVICE_SCHED, interface->state)) {
 153		clear_bit(__FM10K_SERVICE_REQUEST, interface->state);
 154		queue_work(fm10k_workqueue, &interface->service_task);
 155	} else {
 156		set_bit(__FM10K_SERVICE_REQUEST, interface->state);
 157	}
 158}
 159
 160static void fm10k_service_event_complete(struct fm10k_intfc *interface)
 161{
 162	WARN_ON(!test_bit(__FM10K_SERVICE_SCHED, interface->state));
 163
 164	/* flush memory to make sure state is correct before next watchog */
 165	smp_mb__before_atomic();
 166	clear_bit(__FM10K_SERVICE_SCHED, interface->state);
 167
 168	/* If a service event was requested since we started, immediately
 169	 * re-schedule now. This ensures we don't drop a request until the
 170	 * next timer event.
 171	 */
 172	if (test_bit(__FM10K_SERVICE_REQUEST, interface->state))
 173		fm10k_service_event_schedule(interface);
 174}
 175
 176static void fm10k_stop_service_event(struct fm10k_intfc *interface)
 177{
 178	set_bit(__FM10K_SERVICE_DISABLE, interface->state);
 179	cancel_work_sync(&interface->service_task);
 180
 181	/* It's possible that cancel_work_sync stopped the service task from
 182	 * running before it could actually start. In this case the
 183	 * __FM10K_SERVICE_SCHED bit will never be cleared. Since we know that
 184	 * the service task cannot be running at this point, we need to clear
 185	 * the scheduled bit, as otherwise the service task may never be
 186	 * restarted.
 187	 */
 188	clear_bit(__FM10K_SERVICE_SCHED, interface->state);
 189}
 190
 191static void fm10k_start_service_event(struct fm10k_intfc *interface)
 192{
 193	clear_bit(__FM10K_SERVICE_DISABLE, interface->state);
 194	fm10k_service_event_schedule(interface);
 195}
 196
 197/**
 198 * fm10k_service_timer - Timer Call-back
 199 * @t: pointer to timer data
 200 **/
 201static void fm10k_service_timer(struct timer_list *t)
 202{
 203	struct fm10k_intfc *interface = from_timer(interface, t,
 204						   service_timer);
 205
 206	/* Reset the timer */
 207	mod_timer(&interface->service_timer, (HZ * 2) + jiffies);
 208
 209	fm10k_service_event_schedule(interface);
 210}
 211
 212/**
 213 * fm10k_prepare_for_reset - Prepare the driver and device for a pending reset
 214 * @interface: fm10k private data structure
 215 *
 216 * This function prepares for a device reset by shutting as much down as we
 217 * can. It does nothing and returns false if __FM10K_RESETTING was already set
 218 * prior to calling this function. It returns true if it actually did work.
 219 */
 220static bool fm10k_prepare_for_reset(struct fm10k_intfc *interface)
 221{
 222	struct net_device *netdev = interface->netdev;
 223
 224	WARN_ON(in_interrupt());
 225
 226	/* put off any impending NetWatchDogTimeout */
 227	netif_trans_update(netdev);
 228
 229	/* Nothing to do if a reset is already in progress */
 230	if (test_and_set_bit(__FM10K_RESETTING, interface->state))
 231		return false;
 232
 233	/* As the MAC/VLAN task will be accessing registers it must not be
 234	 * running while we reset. Although the task will not be scheduled
 235	 * once we start resetting it may already be running
 236	 */
 237	fm10k_stop_macvlan_task(interface);
 238
 239	rtnl_lock();
 240
 241	fm10k_iov_suspend(interface->pdev);
 242
 243	if (netif_running(netdev))
 244		fm10k_close(netdev);
 245
 246	fm10k_mbx_free_irq(interface);
 247
 248	/* free interrupts */
 249	fm10k_clear_queueing_scheme(interface);
 250
 251	/* delay any future reset requests */
 252	interface->last_reset = jiffies + (10 * HZ);
 253
 254	rtnl_unlock();
 255
 256	return true;
 257}
 258
 259static int fm10k_handle_reset(struct fm10k_intfc *interface)
 260{
 261	struct net_device *netdev = interface->netdev;
 262	struct fm10k_hw *hw = &interface->hw;
 263	int err;
 264
 265	WARN_ON(!test_bit(__FM10K_RESETTING, interface->state));
 266
 267	rtnl_lock();
 268
 269	pci_set_master(interface->pdev);
 270
 271	/* reset and initialize the hardware so it is in a known state */
 272	err = hw->mac.ops.reset_hw(hw);
 273	if (err) {
 274		dev_err(&interface->pdev->dev, "reset_hw failed: %d\n", err);
 275		goto reinit_err;
 276	}
 277
 278	err = hw->mac.ops.init_hw(hw);
 279	if (err) {
 280		dev_err(&interface->pdev->dev, "init_hw failed: %d\n", err);
 281		goto reinit_err;
 282	}
 283
 284	err = fm10k_init_queueing_scheme(interface);
 285	if (err) {
 286		dev_err(&interface->pdev->dev,
 287			"init_queueing_scheme failed: %d\n", err);
 288		goto reinit_err;
 289	}
 290
 291	/* re-associate interrupts */
 292	err = fm10k_mbx_request_irq(interface);
 293	if (err)
 294		goto err_mbx_irq;
 295
 296	err = fm10k_hw_ready(interface);
 297	if (err)
 298		goto err_open;
 299
 300	/* update hardware address for VFs if perm_addr has changed */
 301	if (hw->mac.type == fm10k_mac_vf) {
 302		if (is_valid_ether_addr(hw->mac.perm_addr)) {
 303			ether_addr_copy(hw->mac.addr, hw->mac.perm_addr);
 304			ether_addr_copy(netdev->perm_addr, hw->mac.perm_addr);
 305			ether_addr_copy(netdev->dev_addr, hw->mac.perm_addr);
 306			netdev->addr_assign_type &= ~NET_ADDR_RANDOM;
 307		}
 308
 309		if (hw->mac.vlan_override)
 310			netdev->features &= ~NETIF_F_HW_VLAN_CTAG_RX;
 311		else
 312			netdev->features |= NETIF_F_HW_VLAN_CTAG_RX;
 313	}
 314
 315	err = netif_running(netdev) ? fm10k_open(netdev) : 0;
 316	if (err)
 317		goto err_open;
 318
 319	fm10k_iov_resume(interface->pdev);
 320
 321	rtnl_unlock();
 322
 323	fm10k_resume_macvlan_task(interface);
 324
 325	clear_bit(__FM10K_RESETTING, interface->state);
 326
 327	return err;
 328err_open:
 329	fm10k_mbx_free_irq(interface);
 330err_mbx_irq:
 331	fm10k_clear_queueing_scheme(interface);
 332reinit_err:
 333	netif_device_detach(netdev);
 334
 335	rtnl_unlock();
 336
 337	clear_bit(__FM10K_RESETTING, interface->state);
 338
 339	return err;
 340}
 341
 342static void fm10k_detach_subtask(struct fm10k_intfc *interface)
 343{
 344	struct net_device *netdev = interface->netdev;
 345	u32 __iomem *hw_addr;
 346	u32 value;
 347
 348	/* do nothing if netdev is still present or hw_addr is set */
 349	if (netif_device_present(netdev) || interface->hw.hw_addr)
 350		return;
 351
 352	/* We've lost the PCIe register space, and can no longer access the
 353	 * device. Shut everything except the detach subtask down and prepare
 354	 * to reset the device in case we recover. If we actually prepare for
 355	 * reset, indicate that we're detached.
 356	 */
 357	if (fm10k_prepare_for_reset(interface))
 358		set_bit(__FM10K_RESET_DETACHED, interface->state);
 359
 360	/* check the real address space to see if we've recovered */
 361	hw_addr = READ_ONCE(interface->uc_addr);
 362	value = readl(hw_addr);
 363	if (~value) {
 364		int err;
 365
 366		/* Make sure the reset was initiated because we detached,
 367		 * otherwise we might race with a different reset flow.
 368		 */
 369		if (!test_and_clear_bit(__FM10K_RESET_DETACHED,
 370					interface->state))
 371			return;
 372
 373		/* Restore the hardware address */
 374		interface->hw.hw_addr = interface->uc_addr;
 375
 376		/* PCIe link has been restored, and the device is active
 377		 * again. Restore everything and reset the device.
 378		 */
 379		err = fm10k_handle_reset(interface);
 380		if (err) {
 381			netdev_err(netdev, "Unable to reset device: %d\n", err);
 382			interface->hw.hw_addr = NULL;
 383			return;
 384		}
 385
 386		/* Re-attach the netdev */
 387		netif_device_attach(netdev);
 388		netdev_warn(netdev, "PCIe link restored, device now attached\n");
 389		return;
 390	}
 391}
 392
 393static void fm10k_reset_subtask(struct fm10k_intfc *interface)
 394{
 395	int err;
 396
 397	if (!test_and_clear_bit(FM10K_FLAG_RESET_REQUESTED,
 398				interface->flags))
 399		return;
 400
 401	/* If another thread has already prepared to reset the device, we
 402	 * should not attempt to handle a reset here, since we'd race with
 403	 * that thread. This may happen if we suspend the device or if the
 404	 * PCIe link is lost. In this case, we'll just ignore the RESET
 405	 * request, as it will (eventually) be taken care of when the thread
 406	 * which actually started the reset is finished.
 407	 */
 408	if (!fm10k_prepare_for_reset(interface))
 409		return;
 410
 411	netdev_err(interface->netdev, "Reset interface\n");
 412
 413	err = fm10k_handle_reset(interface);
 414	if (err)
 415		dev_err(&interface->pdev->dev,
 416			"fm10k_handle_reset failed: %d\n", err);
 417}
 418
 419/**
 420 * fm10k_configure_swpri_map - Configure Receive SWPRI to PC mapping
 421 * @interface: board private structure
 422 *
 423 * Configure the SWPRI to PC mapping for the port.
 424 **/
 425static void fm10k_configure_swpri_map(struct fm10k_intfc *interface)
 426{
 427	struct net_device *netdev = interface->netdev;
 428	struct fm10k_hw *hw = &interface->hw;
 429	int i;
 430
 431	/* clear flag indicating update is needed */
 432	clear_bit(FM10K_FLAG_SWPRI_CONFIG, interface->flags);
 433
 434	/* these registers are only available on the PF */
 435	if (hw->mac.type != fm10k_mac_pf)
 436		return;
 437
 438	/* configure SWPRI to PC map */
 439	for (i = 0; i < FM10K_SWPRI_MAX; i++)
 440		fm10k_write_reg(hw, FM10K_SWPRI_MAP(i),
 441				netdev_get_prio_tc_map(netdev, i));
 442}
 443
 444/**
 445 * fm10k_watchdog_update_host_state - Update the link status based on host.
 446 * @interface: board private structure
 447 **/
 448static void fm10k_watchdog_update_host_state(struct fm10k_intfc *interface)
 449{
 450	struct fm10k_hw *hw = &interface->hw;
 451	s32 err;
 452
 453	if (test_bit(__FM10K_LINK_DOWN, interface->state)) {
 454		interface->host_ready = false;
 455		if (time_is_after_jiffies(interface->link_down_event))
 456			return;
 457		clear_bit(__FM10K_LINK_DOWN, interface->state);
 458	}
 459
 460	if (test_bit(FM10K_FLAG_SWPRI_CONFIG, interface->flags)) {
 461		if (rtnl_trylock()) {
 462			fm10k_configure_swpri_map(interface);
 463			rtnl_unlock();
 464		}
 465	}
 466
 467	/* lock the mailbox for transmit and receive */
 468	fm10k_mbx_lock(interface);
 469
 470	err = hw->mac.ops.get_host_state(hw, &interface->host_ready);
 471	if (err && time_is_before_jiffies(interface->last_reset))
 472		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
 473
 474	/* free the lock */
 475	fm10k_mbx_unlock(interface);
 476}
 477
 478/**
 479 * fm10k_mbx_subtask - Process upstream and downstream mailboxes
 480 * @interface: board private structure
 481 *
 482 * This function will process both the upstream and downstream mailboxes.
 483 **/
 484static void fm10k_mbx_subtask(struct fm10k_intfc *interface)
 485{
 486	/* If we're resetting, bail out */
 487	if (test_bit(__FM10K_RESETTING, interface->state))
 488		return;
 489
 490	/* process upstream mailbox and update device state */
 491	fm10k_watchdog_update_host_state(interface);
 492
 493	/* process downstream mailboxes */
 494	fm10k_iov_mbx(interface);
 495}
 496
 497/**
 498 * fm10k_watchdog_host_is_ready - Update netdev status based on host ready
 499 * @interface: board private structure
 500 **/
 501static void fm10k_watchdog_host_is_ready(struct fm10k_intfc *interface)
 502{
 503	struct net_device *netdev = interface->netdev;
 504
 505	/* only continue if link state is currently down */
 506	if (netif_carrier_ok(netdev))
 507		return;
 508
 509	netif_info(interface, drv, netdev, "NIC Link is up\n");
 510
 511	netif_carrier_on(netdev);
 512	netif_tx_wake_all_queues(netdev);
 513}
 514
 515/**
 516 * fm10k_watchdog_host_not_ready - Update netdev status based on host not ready
 517 * @interface: board private structure
 518 **/
 519static void fm10k_watchdog_host_not_ready(struct fm10k_intfc *interface)
 520{
 521	struct net_device *netdev = interface->netdev;
 522
 523	/* only continue if link state is currently up */
 524	if (!netif_carrier_ok(netdev))
 525		return;
 526
 527	netif_info(interface, drv, netdev, "NIC Link is down\n");
 528
 529	netif_carrier_off(netdev);
 530	netif_tx_stop_all_queues(netdev);
 531}
 532
 533/**
 534 * fm10k_update_stats - Update the board statistics counters.
 535 * @interface: board private structure
 536 **/
 537void fm10k_update_stats(struct fm10k_intfc *interface)
 538{
 539	struct net_device_stats *net_stats = &interface->netdev->stats;
 540	struct fm10k_hw *hw = &interface->hw;
 541	u64 hw_csum_tx_good = 0, hw_csum_rx_good = 0, rx_length_errors = 0;
 542	u64 rx_switch_errors = 0, rx_drops = 0, rx_pp_errors = 0;
 543	u64 rx_link_errors = 0;
 544	u64 rx_errors = 0, rx_csum_errors = 0, tx_csum_errors = 0;
 545	u64 restart_queue = 0, tx_busy = 0, alloc_failed = 0;
 546	u64 rx_bytes_nic = 0, rx_pkts_nic = 0, rx_drops_nic = 0;
 547	u64 tx_bytes_nic = 0, tx_pkts_nic = 0;
 548	u64 bytes, pkts;
 549	int i;
 550
 551	/* ensure only one thread updates stats at a time */
 552	if (test_and_set_bit(__FM10K_UPDATING_STATS, interface->state))
 553		return;
 554
 555	/* do not allow stats update via service task for next second */
 556	interface->next_stats_update = jiffies + HZ;
 557
 558	/* gather some stats to the interface struct that are per queue */
 559	for (bytes = 0, pkts = 0, i = 0; i < interface->num_tx_queues; i++) {
 560		struct fm10k_ring *tx_ring = READ_ONCE(interface->tx_ring[i]);
 561
 562		if (!tx_ring)
 563			continue;
 564
 565		restart_queue += tx_ring->tx_stats.restart_queue;
 566		tx_busy += tx_ring->tx_stats.tx_busy;
 567		tx_csum_errors += tx_ring->tx_stats.csum_err;
 568		bytes += tx_ring->stats.bytes;
 569		pkts += tx_ring->stats.packets;
 570		hw_csum_tx_good += tx_ring->tx_stats.csum_good;
 571	}
 572
 573	interface->restart_queue = restart_queue;
 574	interface->tx_busy = tx_busy;
 575	net_stats->tx_bytes = bytes;
 576	net_stats->tx_packets = pkts;
 577	interface->tx_csum_errors = tx_csum_errors;
 578	interface->hw_csum_tx_good = hw_csum_tx_good;
 579
 580	/* gather some stats to the interface struct that are per queue */
 581	for (bytes = 0, pkts = 0, i = 0; i < interface->num_rx_queues; i++) {
 582		struct fm10k_ring *rx_ring = READ_ONCE(interface->rx_ring[i]);
 583
 584		if (!rx_ring)
 585			continue;
 586
 587		bytes += rx_ring->stats.bytes;
 588		pkts += rx_ring->stats.packets;
 589		alloc_failed += rx_ring->rx_stats.alloc_failed;
 590		rx_csum_errors += rx_ring->rx_stats.csum_err;
 591		rx_errors += rx_ring->rx_stats.errors;
 592		hw_csum_rx_good += rx_ring->rx_stats.csum_good;
 593		rx_switch_errors += rx_ring->rx_stats.switch_errors;
 594		rx_drops += rx_ring->rx_stats.drops;
 595		rx_pp_errors += rx_ring->rx_stats.pp_errors;
 596		rx_link_errors += rx_ring->rx_stats.link_errors;
 597		rx_length_errors += rx_ring->rx_stats.length_errors;
 598	}
 599
 600	net_stats->rx_bytes = bytes;
 601	net_stats->rx_packets = pkts;
 602	interface->alloc_failed = alloc_failed;
 603	interface->rx_csum_errors = rx_csum_errors;
 604	interface->hw_csum_rx_good = hw_csum_rx_good;
 605	interface->rx_switch_errors = rx_switch_errors;
 606	interface->rx_drops = rx_drops;
 607	interface->rx_pp_errors = rx_pp_errors;
 608	interface->rx_link_errors = rx_link_errors;
 609	interface->rx_length_errors = rx_length_errors;
 610
 611	hw->mac.ops.update_hw_stats(hw, &interface->stats);
 612
 613	for (i = 0; i < hw->mac.max_queues; i++) {
 614		struct fm10k_hw_stats_q *q = &interface->stats.q[i];
 615
 616		tx_bytes_nic += q->tx_bytes.count;
 617		tx_pkts_nic += q->tx_packets.count;
 618		rx_bytes_nic += q->rx_bytes.count;
 619		rx_pkts_nic += q->rx_packets.count;
 620		rx_drops_nic += q->rx_drops.count;
 621	}
 622
 623	interface->tx_bytes_nic = tx_bytes_nic;
 624	interface->tx_packets_nic = tx_pkts_nic;
 625	interface->rx_bytes_nic = rx_bytes_nic;
 626	interface->rx_packets_nic = rx_pkts_nic;
 627	interface->rx_drops_nic = rx_drops_nic;
 628
 629	/* Fill out the OS statistics structure */
 630	net_stats->rx_errors = rx_errors;
 631	net_stats->rx_dropped = interface->stats.nodesc_drop.count;
 632
 633	/* Update VF statistics */
 634	fm10k_iov_update_stats(interface);
 635
 636	clear_bit(__FM10K_UPDATING_STATS, interface->state);
 637}
 638
 639/**
 640 * fm10k_watchdog_flush_tx - flush queues on host not ready
 641 * @interface: pointer to the device interface structure
 642 **/
 643static void fm10k_watchdog_flush_tx(struct fm10k_intfc *interface)
 644{
 645	int some_tx_pending = 0;
 646	int i;
 647
 648	/* nothing to do if carrier is up */
 649	if (netif_carrier_ok(interface->netdev))
 650		return;
 651
 652	for (i = 0; i < interface->num_tx_queues; i++) {
 653		struct fm10k_ring *tx_ring = interface->tx_ring[i];
 654
 655		if (tx_ring->next_to_use != tx_ring->next_to_clean) {
 656			some_tx_pending = 1;
 657			break;
 658		}
 659	}
 660
 661	/* We've lost link, so the controller stops DMA, but we've got
 662	 * queued Tx work that's never going to get done, so reset
 663	 * controller to flush Tx.
 664	 */
 665	if (some_tx_pending)
 666		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
 667}
 668
 669/**
 670 * fm10k_watchdog_subtask - check and bring link up
 671 * @interface: pointer to the device interface structure
 672 **/
 673static void fm10k_watchdog_subtask(struct fm10k_intfc *interface)
 674{
 675	/* if interface is down do nothing */
 676	if (test_bit(__FM10K_DOWN, interface->state) ||
 677	    test_bit(__FM10K_RESETTING, interface->state))
 678		return;
 679
 680	if (interface->host_ready)
 681		fm10k_watchdog_host_is_ready(interface);
 682	else
 683		fm10k_watchdog_host_not_ready(interface);
 684
 685	/* update stats only once every second */
 686	if (time_is_before_jiffies(interface->next_stats_update))
 687		fm10k_update_stats(interface);
 688
 689	/* flush any uncompleted work */
 690	fm10k_watchdog_flush_tx(interface);
 691}
 692
 693/**
 694 * fm10k_check_hang_subtask - check for hung queues and dropped interrupts
 695 * @interface: pointer to the device interface structure
 696 *
 697 * This function serves two purposes.  First it strobes the interrupt lines
 698 * in order to make certain interrupts are occurring.  Secondly it sets the
 699 * bits needed to check for TX hangs.  As a result we should immediately
 700 * determine if a hang has occurred.
 701 */
 702static void fm10k_check_hang_subtask(struct fm10k_intfc *interface)
 703{
 704	/* If we're down or resetting, just bail */
 705	if (test_bit(__FM10K_DOWN, interface->state) ||
 706	    test_bit(__FM10K_RESETTING, interface->state))
 707		return;
 708
 709	/* rate limit tx hang checks to only once every 2 seconds */
 710	if (time_is_after_eq_jiffies(interface->next_tx_hang_check))
 711		return;
 712	interface->next_tx_hang_check = jiffies + (2 * HZ);
 713
 714	if (netif_carrier_ok(interface->netdev)) {
 715		int i;
 716
 717		/* Force detection of hung controller */
 718		for (i = 0; i < interface->num_tx_queues; i++)
 719			set_check_for_tx_hang(interface->tx_ring[i]);
 720
 721		/* Rearm all in-use q_vectors for immediate firing */
 722		for (i = 0; i < interface->num_q_vectors; i++) {
 723			struct fm10k_q_vector *qv = interface->q_vector[i];
 724
 725			if (!qv->tx.count && !qv->rx.count)
 726				continue;
 727			writel(FM10K_ITR_ENABLE | FM10K_ITR_PENDING2, qv->itr);
 728		}
 729	}
 730}
 731
 732/**
 733 * fm10k_service_task - manages and runs subtasks
 734 * @work: pointer to work_struct containing our data
 735 **/
 736static void fm10k_service_task(struct work_struct *work)
 737{
 738	struct fm10k_intfc *interface;
 739
 740	interface = container_of(work, struct fm10k_intfc, service_task);
 741
 742	/* Check whether we're detached first */
 743	fm10k_detach_subtask(interface);
 744
 745	/* tasks run even when interface is down */
 746	fm10k_mbx_subtask(interface);
 747	fm10k_reset_subtask(interface);
 748
 749	/* tasks only run when interface is up */
 750	fm10k_watchdog_subtask(interface);
 751	fm10k_check_hang_subtask(interface);
 752
 753	/* release lock on service events to allow scheduling next event */
 754	fm10k_service_event_complete(interface);
 755}
 756
 757/**
 758 * fm10k_macvlan_task - send queued MAC/VLAN requests to switch manager
 759 * @work: pointer to work_struct containing our data
 760 *
 761 * This work item handles sending MAC/VLAN updates to the switch manager. When
 762 * the interface is up, it will attempt to queue mailbox messages to the
 763 * switch manager requesting updates for MAC/VLAN pairs. If the Tx fifo of the
 764 * mailbox is full, it will reschedule itself to try again in a short while.
 765 * This ensures that the driver does not overload the switch mailbox with too
 766 * many simultaneous requests, causing an unnecessary reset.
 767 **/
 768static void fm10k_macvlan_task(struct work_struct *work)
 769{
 770	struct fm10k_macvlan_request *item;
 771	struct fm10k_intfc *interface;
 772	struct delayed_work *dwork;
 773	struct list_head *requests;
 774	struct fm10k_hw *hw;
 775	unsigned long flags;
 776
 777	dwork = to_delayed_work(work);
 778	interface = container_of(dwork, struct fm10k_intfc, macvlan_task);
 779	hw = &interface->hw;
 780	requests = &interface->macvlan_requests;
 781
 782	do {
 783		/* Pop the first item off the list */
 784		spin_lock_irqsave(&interface->macvlan_lock, flags);
 785		item = list_first_entry_or_null(requests,
 786						struct fm10k_macvlan_request,
 787						list);
 788		if (item)
 789			list_del_init(&item->list);
 790
 791		spin_unlock_irqrestore(&interface->macvlan_lock, flags);
 792
 793		/* We have no more items to process */
 794		if (!item)
 795			goto done;
 796
 797		fm10k_mbx_lock(interface);
 798
 799		/* Check that we have plenty of space to send the message. We
 800		 * want to ensure that the mailbox stays low enough to avoid a
 801		 * change in the host state, otherwise we may see spurious
 802		 * link up / link down notifications.
 803		 */
 804		if (!hw->mbx.ops.tx_ready(&hw->mbx, FM10K_VFMBX_MSG_MTU + 5)) {
 805			hw->mbx.ops.process(hw, &hw->mbx);
 806			set_bit(__FM10K_MACVLAN_REQUEST, interface->state);
 807			fm10k_mbx_unlock(interface);
 808
 809			/* Put the request back on the list */
 810			spin_lock_irqsave(&interface->macvlan_lock, flags);
 811			list_add(&item->list, requests);
 812			spin_unlock_irqrestore(&interface->macvlan_lock, flags);
 813			break;
 814		}
 815
 816		switch (item->type) {
 817		case FM10K_MC_MAC_REQUEST:
 818			hw->mac.ops.update_mc_addr(hw,
 819						   item->mac.glort,
 820						   item->mac.addr,
 821						   item->mac.vid,
 822						   item->set);
 823			break;
 824		case FM10K_UC_MAC_REQUEST:
 825			hw->mac.ops.update_uc_addr(hw,
 826						   item->mac.glort,
 827						   item->mac.addr,
 828						   item->mac.vid,
 829						   item->set,
 830						   0);
 831			break;
 832		case FM10K_VLAN_REQUEST:
 833			hw->mac.ops.update_vlan(hw,
 834						item->vlan.vid,
 835						item->vlan.vsi,
 836						item->set);
 837			break;
 838		default:
 839			break;
 840		}
 841
 842		fm10k_mbx_unlock(interface);
 843
 844		/* Free the item now that we've sent the update */
 845		kfree(item);
 846	} while (true);
 847
 848done:
 849	WARN_ON(!test_bit(__FM10K_MACVLAN_SCHED, interface->state));
 850
 851	/* flush memory to make sure state is correct */
 852	smp_mb__before_atomic();
 853	clear_bit(__FM10K_MACVLAN_SCHED, interface->state);
 854
 855	/* If a MAC/VLAN request was scheduled since we started, we should
 856	 * re-schedule. However, there is no reason to re-schedule if there is
 857	 * no work to do.
 858	 */
 859	if (test_bit(__FM10K_MACVLAN_REQUEST, interface->state))
 860		fm10k_macvlan_schedule(interface);
 861}
 862
 863/**
 864 * fm10k_configure_tx_ring - Configure Tx ring after Reset
 865 * @interface: board private structure
 866 * @ring: structure containing ring specific data
 867 *
 868 * Configure the Tx descriptor ring after a reset.
 869 **/
 870static void fm10k_configure_tx_ring(struct fm10k_intfc *interface,
 871				    struct fm10k_ring *ring)
 872{
 873	struct fm10k_hw *hw = &interface->hw;
 874	u64 tdba = ring->dma;
 875	u32 size = ring->count * sizeof(struct fm10k_tx_desc);
 876	u32 txint = FM10K_INT_MAP_DISABLE;
 877	u32 txdctl = BIT(FM10K_TXDCTL_MAX_TIME_SHIFT) | FM10K_TXDCTL_ENABLE;
 878	u8 reg_idx = ring->reg_idx;
 879
 880	/* disable queue to avoid issues while updating state */
 881	fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), 0);
 882	fm10k_write_flush(hw);
 883
 884	/* possible poll here to verify ring resources have been cleaned */
 885
 886	/* set location and size for descriptor ring */
 887	fm10k_write_reg(hw, FM10K_TDBAL(reg_idx), tdba & DMA_BIT_MASK(32));
 888	fm10k_write_reg(hw, FM10K_TDBAH(reg_idx), tdba >> 32);
 889	fm10k_write_reg(hw, FM10K_TDLEN(reg_idx), size);
 890
 891	/* reset head and tail pointers */
 892	fm10k_write_reg(hw, FM10K_TDH(reg_idx), 0);
 893	fm10k_write_reg(hw, FM10K_TDT(reg_idx), 0);
 894
 895	/* store tail pointer */
 896	ring->tail = &interface->uc_addr[FM10K_TDT(reg_idx)];
 897
 898	/* reset ntu and ntc to place SW in sync with hardware */
 899	ring->next_to_clean = 0;
 900	ring->next_to_use = 0;
 901
 902	/* Map interrupt */
 903	if (ring->q_vector) {
 904		txint = ring->q_vector->v_idx + NON_Q_VECTORS;
 905		txint |= FM10K_INT_MAP_TIMER0;
 906	}
 907
 908	fm10k_write_reg(hw, FM10K_TXINT(reg_idx), txint);
 909
 910	/* enable use of FTAG bit in Tx descriptor, register is RO for VF */
 911	fm10k_write_reg(hw, FM10K_PFVTCTL(reg_idx),
 912			FM10K_PFVTCTL_FTAG_DESC_ENABLE);
 913
 914	/* Initialize XPS */
 915	if (!test_and_set_bit(__FM10K_TX_XPS_INIT_DONE, ring->state) &&
 916	    ring->q_vector)
 917		netif_set_xps_queue(ring->netdev,
 918				    &ring->q_vector->affinity_mask,
 919				    ring->queue_index);
 920
 921	/* enable queue */
 922	fm10k_write_reg(hw, FM10K_TXDCTL(reg_idx), txdctl);
 923}
 924
 925/**
 926 * fm10k_enable_tx_ring - Verify Tx ring is enabled after configuration
 927 * @interface: board private structure
 928 * @ring: structure containing ring specific data
 929 *
 930 * Verify the Tx descriptor ring is ready for transmit.
 931 **/
 932static void fm10k_enable_tx_ring(struct fm10k_intfc *interface,
 933				 struct fm10k_ring *ring)
 934{
 935	struct fm10k_hw *hw = &interface->hw;
 936	int wait_loop = 10;
 937	u32 txdctl;
 938	u8 reg_idx = ring->reg_idx;
 939
 940	/* if we are already enabled just exit */
 941	if (fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx)) & FM10K_TXDCTL_ENABLE)
 942		return;
 943
 944	/* poll to verify queue is enabled */
 945	do {
 946		usleep_range(1000, 2000);
 947		txdctl = fm10k_read_reg(hw, FM10K_TXDCTL(reg_idx));
 948	} while (!(txdctl & FM10K_TXDCTL_ENABLE) && --wait_loop);
 949	if (!wait_loop)
 950		netif_err(interface, drv, interface->netdev,
 951			  "Could not enable Tx Queue %d\n", reg_idx);
 952}
 953
 954/**
 955 * fm10k_configure_tx - Configure Transmit Unit after Reset
 956 * @interface: board private structure
 957 *
 958 * Configure the Tx unit of the MAC after a reset.
 959 **/
 960static void fm10k_configure_tx(struct fm10k_intfc *interface)
 961{
 962	int i;
 963
 964	/* Setup the HW Tx Head and Tail descriptor pointers */
 965	for (i = 0; i < interface->num_tx_queues; i++)
 966		fm10k_configure_tx_ring(interface, interface->tx_ring[i]);
 967
 968	/* poll here to verify that Tx rings are now enabled */
 969	for (i = 0; i < interface->num_tx_queues; i++)
 970		fm10k_enable_tx_ring(interface, interface->tx_ring[i]);
 971}
 972
 973/**
 974 * fm10k_configure_rx_ring - Configure Rx ring after Reset
 975 * @interface: board private structure
 976 * @ring: structure containing ring specific data
 977 *
 978 * Configure the Rx descriptor ring after a reset.
 979 **/
 980static void fm10k_configure_rx_ring(struct fm10k_intfc *interface,
 981				    struct fm10k_ring *ring)
 982{
 983	u64 rdba = ring->dma;
 984	struct fm10k_hw *hw = &interface->hw;
 985	u32 size = ring->count * sizeof(union fm10k_rx_desc);
 986	u32 rxqctl, rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
 987	u32 srrctl = FM10K_SRRCTL_BUFFER_CHAINING_EN;
 988	u32 rxint = FM10K_INT_MAP_DISABLE;
 989	u8 rx_pause = interface->rx_pause;
 990	u8 reg_idx = ring->reg_idx;
 991
 992	/* disable queue to avoid issues while updating state */
 993	rxqctl = fm10k_read_reg(hw, FM10K_RXQCTL(reg_idx));
 994	rxqctl &= ~FM10K_RXQCTL_ENABLE;
 995	fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl);
 996	fm10k_write_flush(hw);
 997
 998	/* possible poll here to verify ring resources have been cleaned */
 999
1000	/* set location and size for descriptor ring */
1001	fm10k_write_reg(hw, FM10K_RDBAL(reg_idx), rdba & DMA_BIT_MASK(32));
1002	fm10k_write_reg(hw, FM10K_RDBAH(reg_idx), rdba >> 32);
1003	fm10k_write_reg(hw, FM10K_RDLEN(reg_idx), size);
1004
1005	/* reset head and tail pointers */
1006	fm10k_write_reg(hw, FM10K_RDH(reg_idx), 0);
1007	fm10k_write_reg(hw, FM10K_RDT(reg_idx), 0);
1008
1009	/* store tail pointer */
1010	ring->tail = &interface->uc_addr[FM10K_RDT(reg_idx)];
1011
1012	/* reset ntu and ntc to place SW in sync with hardware */
1013	ring->next_to_clean = 0;
1014	ring->next_to_use = 0;
1015	ring->next_to_alloc = 0;
1016
1017	/* Configure the Rx buffer size for one buff without split */
1018	srrctl |= FM10K_RX_BUFSZ >> FM10K_SRRCTL_BSIZEPKT_SHIFT;
1019
1020	/* Configure the Rx ring to suppress loopback packets */
1021	srrctl |= FM10K_SRRCTL_LOOPBACK_SUPPRESS;
1022	fm10k_write_reg(hw, FM10K_SRRCTL(reg_idx), srrctl);
1023
1024	/* Enable drop on empty */
1025#ifdef CONFIG_DCB
1026	if (interface->pfc_en)
1027		rx_pause = interface->pfc_en;
1028#endif
1029	if (!(rx_pause & BIT(ring->qos_pc)))
1030		rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
1031
1032	fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl);
1033
1034	/* assign default VLAN to queue */
1035	ring->vid = hw->mac.default_vid;
1036
1037	/* if we have an active VLAN, disable default VLAN ID */
1038	if (test_bit(hw->mac.default_vid, interface->active_vlans))
1039		ring->vid |= FM10K_VLAN_CLEAR;
1040
1041	/* Map interrupt */
1042	if (ring->q_vector) {
1043		rxint = ring->q_vector->v_idx + NON_Q_VECTORS;
1044		rxint |= FM10K_INT_MAP_TIMER1;
1045	}
1046
1047	fm10k_write_reg(hw, FM10K_RXINT(reg_idx), rxint);
1048
1049	/* enable queue */
1050	rxqctl = fm10k_read_reg(hw, FM10K_RXQCTL(reg_idx));
1051	rxqctl |= FM10K_RXQCTL_ENABLE;
1052	fm10k_write_reg(hw, FM10K_RXQCTL(reg_idx), rxqctl);
1053
1054	/* place buffers on ring for receive data */
1055	fm10k_alloc_rx_buffers(ring, fm10k_desc_unused(ring));
1056}
1057
1058/**
1059 * fm10k_update_rx_drop_en - Configures the drop enable bits for Rx rings
1060 * @interface: board private structure
1061 *
1062 * Configure the drop enable bits for the Rx rings.
1063 **/
1064void fm10k_update_rx_drop_en(struct fm10k_intfc *interface)
1065{
1066	struct fm10k_hw *hw = &interface->hw;
1067	u8 rx_pause = interface->rx_pause;
1068	int i;
1069
1070#ifdef CONFIG_DCB
1071	if (interface->pfc_en)
1072		rx_pause = interface->pfc_en;
1073
1074#endif
1075	for (i = 0; i < interface->num_rx_queues; i++) {
1076		struct fm10k_ring *ring = interface->rx_ring[i];
1077		u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
1078		u8 reg_idx = ring->reg_idx;
1079
1080		if (!(rx_pause & BIT(ring->qos_pc)))
1081			rxdctl |= FM10K_RXDCTL_DROP_ON_EMPTY;
1082
1083		fm10k_write_reg(hw, FM10K_RXDCTL(reg_idx), rxdctl);
1084	}
1085}
1086
1087/**
1088 * fm10k_configure_dglort - Configure Receive DGLORT after reset
1089 * @interface: board private structure
1090 *
1091 * Configure the DGLORT description and RSS tables.
1092 **/
1093static void fm10k_configure_dglort(struct fm10k_intfc *interface)
1094{
1095	struct fm10k_dglort_cfg dglort = { 0 };
1096	struct fm10k_hw *hw = &interface->hw;
1097	int i;
1098	u32 mrqc;
1099
1100	/* Fill out hash function seeds */
1101	for (i = 0; i < FM10K_RSSRK_SIZE; i++)
1102		fm10k_write_reg(hw, FM10K_RSSRK(0, i), interface->rssrk[i]);
1103
1104	/* Write RETA table to hardware */
1105	for (i = 0; i < FM10K_RETA_SIZE; i++)
1106		fm10k_write_reg(hw, FM10K_RETA(0, i), interface->reta[i]);
1107
1108	/* Generate RSS hash based on packet types, TCP/UDP
1109	 * port numbers and/or IPv4/v6 src and dst addresses
1110	 */
1111	mrqc = FM10K_MRQC_IPV4 |
1112	       FM10K_MRQC_TCP_IPV4 |
1113	       FM10K_MRQC_IPV6 |
1114	       FM10K_MRQC_TCP_IPV6;
1115
1116	if (test_bit(FM10K_FLAG_RSS_FIELD_IPV4_UDP, interface->flags))
1117		mrqc |= FM10K_MRQC_UDP_IPV4;
1118	if (test_bit(FM10K_FLAG_RSS_FIELD_IPV6_UDP, interface->flags))
1119		mrqc |= FM10K_MRQC_UDP_IPV6;
1120
1121	fm10k_write_reg(hw, FM10K_MRQC(0), mrqc);
1122
1123	/* configure default DGLORT mapping for RSS/DCB */
1124	dglort.inner_rss = 1;
1125	dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask);
1126	dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask);
1127	hw->mac.ops.configure_dglort_map(hw, &dglort);
1128
1129	/* assign GLORT per queue for queue mapped testing */
1130	if (interface->glort_count > 64) {
1131		memset(&dglort, 0, sizeof(dglort));
1132		dglort.inner_rss = 1;
1133		dglort.glort = interface->glort + 64;
1134		dglort.idx = fm10k_dglort_pf_queue;
1135		dglort.queue_l = fls(interface->num_rx_queues - 1);
1136		hw->mac.ops.configure_dglort_map(hw, &dglort);
1137	}
1138
1139	/* assign glort value for RSS/DCB specific to this interface */
1140	memset(&dglort, 0, sizeof(dglort));
1141	dglort.inner_rss = 1;
1142	dglort.glort = interface->glort;
1143	dglort.rss_l = fls(interface->ring_feature[RING_F_RSS].mask);
1144	dglort.pc_l = fls(interface->ring_feature[RING_F_QOS].mask);
1145	/* configure DGLORT mapping for RSS/DCB */
1146	dglort.idx = fm10k_dglort_pf_rss;
1147	if (interface->l2_accel)
1148		dglort.shared_l = fls(interface->l2_accel->size);
1149	hw->mac.ops.configure_dglort_map(hw, &dglort);
1150}
1151
1152/**
1153 * fm10k_configure_rx - Configure Receive Unit after Reset
1154 * @interface: board private structure
1155 *
1156 * Configure the Rx unit of the MAC after a reset.
1157 **/
1158static void fm10k_configure_rx(struct fm10k_intfc *interface)
1159{
1160	int i;
1161
1162	/* Configure SWPRI to PC map */
1163	fm10k_configure_swpri_map(interface);
1164
1165	/* Configure RSS and DGLORT map */
1166	fm10k_configure_dglort(interface);
1167
1168	/* Setup the HW Rx Head and Tail descriptor pointers */
1169	for (i = 0; i < interface->num_rx_queues; i++)
1170		fm10k_configure_rx_ring(interface, interface->rx_ring[i]);
1171
1172	/* possible poll here to verify that Rx rings are now enabled */
1173}
1174
1175static void fm10k_napi_enable_all(struct fm10k_intfc *interface)
1176{
1177	struct fm10k_q_vector *q_vector;
1178	int q_idx;
1179
1180	for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) {
1181		q_vector = interface->q_vector[q_idx];
1182		napi_enable(&q_vector->napi);
1183	}
1184}
1185
1186static irqreturn_t fm10k_msix_clean_rings(int __always_unused irq, void *data)
1187{
1188	struct fm10k_q_vector *q_vector = data;
1189
1190	if (q_vector->rx.count || q_vector->tx.count)
1191		napi_schedule_irqoff(&q_vector->napi);
1192
1193	return IRQ_HANDLED;
1194}
1195
1196static irqreturn_t fm10k_msix_mbx_vf(int __always_unused irq, void *data)
1197{
1198	struct fm10k_intfc *interface = data;
1199	struct fm10k_hw *hw = &interface->hw;
1200	struct fm10k_mbx_info *mbx = &hw->mbx;
1201
1202	/* re-enable mailbox interrupt and indicate 20us delay */
1203	fm10k_write_reg(hw, FM10K_VFITR(FM10K_MBX_VECTOR),
1204			(FM10K_MBX_INT_DELAY >> hw->mac.itr_scale) |
1205			FM10K_ITR_ENABLE);
1206
1207	/* service upstream mailbox */
1208	if (fm10k_mbx_trylock(interface)) {
1209		mbx->ops.process(hw, mbx);
1210		fm10k_mbx_unlock(interface);
1211	}
1212
1213	hw->mac.get_host_state = true;
1214	fm10k_service_event_schedule(interface);
1215
1216	return IRQ_HANDLED;
1217}
1218
1219#define FM10K_ERR_MSG(type) case (type): error = #type; break
1220static void fm10k_handle_fault(struct fm10k_intfc *interface, int type,
1221			       struct fm10k_fault *fault)
1222{
1223	struct pci_dev *pdev = interface->pdev;
1224	struct fm10k_hw *hw = &interface->hw;
1225	struct fm10k_iov_data *iov_data = interface->iov_data;
1226	char *error;
1227
1228	switch (type) {
1229	case FM10K_PCA_FAULT:
1230		switch (fault->type) {
1231		default:
1232			error = "Unknown PCA error";
1233			break;
1234		FM10K_ERR_MSG(PCA_NO_FAULT);
1235		FM10K_ERR_MSG(PCA_UNMAPPED_ADDR);
1236		FM10K_ERR_MSG(PCA_BAD_QACCESS_PF);
1237		FM10K_ERR_MSG(PCA_BAD_QACCESS_VF);
1238		FM10K_ERR_MSG(PCA_MALICIOUS_REQ);
1239		FM10K_ERR_MSG(PCA_POISONED_TLP);
1240		FM10K_ERR_MSG(PCA_TLP_ABORT);
1241		}
1242		break;
1243	case FM10K_THI_FAULT:
1244		switch (fault->type) {
1245		default:
1246			error = "Unknown THI error";
1247			break;
1248		FM10K_ERR_MSG(THI_NO_FAULT);
1249		FM10K_ERR_MSG(THI_MAL_DIS_Q_FAULT);
1250		}
1251		break;
1252	case FM10K_FUM_FAULT:
1253		switch (fault->type) {
1254		default:
1255			error = "Unknown FUM error";
1256			break;
1257		FM10K_ERR_MSG(FUM_NO_FAULT);
1258		FM10K_ERR_MSG(FUM_UNMAPPED_ADDR);
1259		FM10K_ERR_MSG(FUM_BAD_VF_QACCESS);
1260		FM10K_ERR_MSG(FUM_ADD_DECODE_ERR);
1261		FM10K_ERR_MSG(FUM_RO_ERROR);
1262		FM10K_ERR_MSG(FUM_QPRC_CRC_ERROR);
1263		FM10K_ERR_MSG(FUM_CSR_TIMEOUT);
1264		FM10K_ERR_MSG(FUM_INVALID_TYPE);
1265		FM10K_ERR_MSG(FUM_INVALID_LENGTH);
1266		FM10K_ERR_MSG(FUM_INVALID_BE);
1267		FM10K_ERR_MSG(FUM_INVALID_ALIGN);
1268		}
1269		break;
1270	default:
1271		error = "Undocumented fault";
1272		break;
1273	}
1274
1275	dev_warn(&pdev->dev,
1276		 "%s Address: 0x%llx SpecInfo: 0x%x Func: %02x.%0x\n",
1277		 error, fault->address, fault->specinfo,
1278		 PCI_SLOT(fault->func), PCI_FUNC(fault->func));
1279
1280	/* For VF faults, clear out the respective LPORT, reset the queue
1281	 * resources, and then reconnect to the mailbox. This allows the
1282	 * VF in question to resume behavior. For transient faults that are
1283	 * the result of non-malicious behavior this will log the fault and
1284	 * allow the VF to resume functionality. Obviously for malicious VFs
1285	 * they will be able to attempt malicious behavior again. In this
1286	 * case, the system administrator will need to step in and manually
1287	 * remove or disable the VF in question.
1288	 */
1289	if (fault->func && iov_data) {
1290		int vf = fault->func - 1;
1291		struct fm10k_vf_info *vf_info = &iov_data->vf_info[vf];
1292
1293		hw->iov.ops.reset_lport(hw, vf_info);
1294		hw->iov.ops.reset_resources(hw, vf_info);
1295
1296		/* reset_lport disables the VF, so re-enable it */
1297		hw->iov.ops.set_lport(hw, vf_info, vf,
1298				      FM10K_VF_FLAG_MULTI_CAPABLE);
1299
1300		/* reset_resources will disconnect from the mbx  */
1301		vf_info->mbx.ops.connect(hw, &vf_info->mbx);
1302	}
1303}
1304
1305static void fm10k_report_fault(struct fm10k_intfc *interface, u32 eicr)
1306{
1307	struct fm10k_hw *hw = &interface->hw;
1308	struct fm10k_fault fault = { 0 };
1309	int type, err;
1310
1311	for (eicr &= FM10K_EICR_FAULT_MASK, type = FM10K_PCA_FAULT;
1312	     eicr;
1313	     eicr >>= 1, type += FM10K_FAULT_SIZE) {
1314		/* only check if there is an error reported */
1315		if (!(eicr & 0x1))
1316			continue;
1317
1318		/* retrieve fault info */
1319		err = hw->mac.ops.get_fault(hw, type, &fault);
1320		if (err) {
1321			dev_err(&interface->pdev->dev,
1322				"error reading fault\n");
1323			continue;
1324		}
1325
1326		fm10k_handle_fault(interface, type, &fault);
1327	}
1328}
1329
1330static void fm10k_reset_drop_on_empty(struct fm10k_intfc *interface, u32 eicr)
1331{
1332	struct fm10k_hw *hw = &interface->hw;
1333	const u32 rxdctl = FM10K_RXDCTL_WRITE_BACK_MIN_DELAY;
1334	u32 maxholdq;
1335	int q;
1336
1337	if (!(eicr & FM10K_EICR_MAXHOLDTIME))
1338		return;
1339
1340	maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(7));
1341	if (maxholdq)
1342		fm10k_write_reg(hw, FM10K_MAXHOLDQ(7), maxholdq);
1343	for (q = 255;;) {
1344		if (maxholdq & BIT(31)) {
1345			if (q < FM10K_MAX_QUEUES_PF) {
1346				interface->rx_overrun_pf++;
1347				fm10k_write_reg(hw, FM10K_RXDCTL(q), rxdctl);
1348			} else {
1349				interface->rx_overrun_vf++;
1350			}
1351		}
1352
1353		maxholdq *= 2;
1354		if (!maxholdq)
1355			q &= ~(32 - 1);
1356
1357		if (!q)
1358			break;
1359
1360		if (q-- % 32)
1361			continue;
1362
1363		maxholdq = fm10k_read_reg(hw, FM10K_MAXHOLDQ(q / 32));
1364		if (maxholdq)
1365			fm10k_write_reg(hw, FM10K_MAXHOLDQ(q / 32), maxholdq);
1366	}
1367}
1368
1369static irqreturn_t fm10k_msix_mbx_pf(int __always_unused irq, void *data)
1370{
1371	struct fm10k_intfc *interface = data;
1372	struct fm10k_hw *hw = &interface->hw;
1373	struct fm10k_mbx_info *mbx = &hw->mbx;
1374	u32 eicr;
1375	s32 err = 0;
1376
1377	/* unmask any set bits related to this interrupt */
1378	eicr = fm10k_read_reg(hw, FM10K_EICR);
1379	fm10k_write_reg(hw, FM10K_EICR, eicr & (FM10K_EICR_MAILBOX |
1380						FM10K_EICR_SWITCHREADY |
1381						FM10K_EICR_SWITCHNOTREADY));
1382
1383	/* report any faults found to the message log */
1384	fm10k_report_fault(interface, eicr);
1385
1386	/* reset any queues disabled due to receiver overrun */
1387	fm10k_reset_drop_on_empty(interface, eicr);
1388
1389	/* service mailboxes */
1390	if (fm10k_mbx_trylock(interface)) {
1391		err = mbx->ops.process(hw, mbx);
1392		/* handle VFLRE events */
1393		fm10k_iov_event(interface);
1394		fm10k_mbx_unlock(interface);
1395	}
1396
1397	if (err == FM10K_ERR_RESET_REQUESTED)
1398		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1399
1400	/* if switch toggled state we should reset GLORTs */
1401	if (eicr & FM10K_EICR_SWITCHNOTREADY) {
1402		/* force link down for at least 4 seconds */
1403		interface->link_down_event = jiffies + (4 * HZ);
1404		set_bit(__FM10K_LINK_DOWN, interface->state);
1405
1406		/* reset dglort_map back to no config */
1407		hw->mac.dglort_map = FM10K_DGLORTMAP_NONE;
1408	}
1409
1410	/* we should validate host state after interrupt event */
1411	hw->mac.get_host_state = true;
1412
1413	/* validate host state, and handle VF mailboxes in the service task */
1414	fm10k_service_event_schedule(interface);
1415
1416	/* re-enable mailbox interrupt and indicate 20us delay */
1417	fm10k_write_reg(hw, FM10K_ITR(FM10K_MBX_VECTOR),
1418			(FM10K_MBX_INT_DELAY >> hw->mac.itr_scale) |
1419			FM10K_ITR_ENABLE);
1420
1421	return IRQ_HANDLED;
1422}
1423
1424void fm10k_mbx_free_irq(struct fm10k_intfc *interface)
1425{
1426	struct fm10k_hw *hw = &interface->hw;
1427	struct msix_entry *entry;
1428	int itr_reg;
1429
1430	/* no mailbox IRQ to free if MSI-X is not enabled */
1431	if (!interface->msix_entries)
1432		return;
1433
1434	entry = &interface->msix_entries[FM10K_MBX_VECTOR];
1435
1436	/* disconnect the mailbox */
1437	hw->mbx.ops.disconnect(hw, &hw->mbx);
1438
1439	/* disable Mailbox cause */
1440	if (hw->mac.type == fm10k_mac_pf) {
1441		fm10k_write_reg(hw, FM10K_EIMR,
1442				FM10K_EIMR_DISABLE(PCA_FAULT) |
1443				FM10K_EIMR_DISABLE(FUM_FAULT) |
1444				FM10K_EIMR_DISABLE(MAILBOX) |
1445				FM10K_EIMR_DISABLE(SWITCHREADY) |
1446				FM10K_EIMR_DISABLE(SWITCHNOTREADY) |
1447				FM10K_EIMR_DISABLE(SRAMERROR) |
1448				FM10K_EIMR_DISABLE(VFLR) |
1449				FM10K_EIMR_DISABLE(MAXHOLDTIME));
1450		itr_reg = FM10K_ITR(FM10K_MBX_VECTOR);
1451	} else {
1452		itr_reg = FM10K_VFITR(FM10K_MBX_VECTOR);
1453	}
1454
1455	fm10k_write_reg(hw, itr_reg, FM10K_ITR_MASK_SET);
1456
1457	free_irq(entry->vector, interface);
1458}
1459
1460static s32 fm10k_mbx_mac_addr(struct fm10k_hw *hw, u32 **results,
1461			      struct fm10k_mbx_info *mbx)
1462{
1463	bool vlan_override = hw->mac.vlan_override;
1464	u16 default_vid = hw->mac.default_vid;
1465	struct fm10k_intfc *interface;
1466	s32 err;
1467
1468	err = fm10k_msg_mac_vlan_vf(hw, results, mbx);
1469	if (err)
1470		return err;
1471
1472	interface = container_of(hw, struct fm10k_intfc, hw);
1473
1474	/* MAC was changed so we need reset */
1475	if (is_valid_ether_addr(hw->mac.perm_addr) &&
1476	    !ether_addr_equal(hw->mac.perm_addr, hw->mac.addr))
1477		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1478
1479	/* VLAN override was changed, or default VLAN changed */
1480	if ((vlan_override != hw->mac.vlan_override) ||
1481	    (default_vid != hw->mac.default_vid))
1482		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1483
1484	return 0;
1485}
1486
1487/* generic error handler for mailbox issues */
1488static s32 fm10k_mbx_error(struct fm10k_hw *hw, u32 **results,
1489			   struct fm10k_mbx_info __always_unused *mbx)
1490{
1491	struct fm10k_intfc *interface;
1492	struct pci_dev *pdev;
1493
1494	interface = container_of(hw, struct fm10k_intfc, hw);
1495	pdev = interface->pdev;
1496
1497	dev_err(&pdev->dev, "Unknown message ID %u\n",
1498		**results & FM10K_TLV_ID_MASK);
1499
1500	return 0;
1501}
1502
1503static const struct fm10k_msg_data vf_mbx_data[] = {
1504	FM10K_TLV_MSG_TEST_HANDLER(fm10k_tlv_msg_test),
1505	FM10K_VF_MSG_MAC_VLAN_HANDLER(fm10k_mbx_mac_addr),
1506	FM10K_VF_MSG_LPORT_STATE_HANDLER(fm10k_msg_lport_state_vf),
1507	FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error),
1508};
1509
1510static int fm10k_mbx_request_irq_vf(struct fm10k_intfc *interface)
1511{
1512	struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR];
1513	struct net_device *dev = interface->netdev;
1514	struct fm10k_hw *hw = &interface->hw;
1515	int err;
1516
1517	/* Use timer0 for interrupt moderation on the mailbox */
1518	u32 itr = entry->entry | FM10K_INT_MAP_TIMER0;
1519
1520	/* register mailbox handlers */
1521	err = hw->mbx.ops.register_handlers(&hw->mbx, vf_mbx_data);
1522	if (err)
1523		return err;
1524
1525	/* request the IRQ */
1526	err = request_irq(entry->vector, fm10k_msix_mbx_vf, 0,
1527			  dev->name, interface);
1528	if (err) {
1529		netif_err(interface, probe, dev,
1530			  "request_irq for msix_mbx failed: %d\n", err);
1531		return err;
1532	}
1533
1534	/* map all of the interrupt sources */
1535	fm10k_write_reg(hw, FM10K_VFINT_MAP, itr);
1536
1537	/* enable interrupt */
1538	fm10k_write_reg(hw, FM10K_VFITR(entry->entry), FM10K_ITR_ENABLE);
1539
1540	return 0;
1541}
1542
1543static s32 fm10k_lport_map(struct fm10k_hw *hw, u32 **results,
1544			   struct fm10k_mbx_info *mbx)
1545{
1546	struct fm10k_intfc *interface;
1547	u32 dglort_map = hw->mac.dglort_map;
1548	s32 err;
1549
1550	interface = container_of(hw, struct fm10k_intfc, hw);
1551
1552	err = fm10k_msg_err_pf(hw, results, mbx);
1553	if (!err && hw->swapi.status) {
1554		/* force link down for a reasonable delay */
1555		interface->link_down_event = jiffies + (2 * HZ);
1556		set_bit(__FM10K_LINK_DOWN, interface->state);
1557
1558		/* reset dglort_map back to no config */
1559		hw->mac.dglort_map = FM10K_DGLORTMAP_NONE;
1560
1561		fm10k_service_event_schedule(interface);
1562
1563		/* prevent overloading kernel message buffer */
1564		if (interface->lport_map_failed)
1565			return 0;
1566
1567		interface->lport_map_failed = true;
1568
1569		if (hw->swapi.status == FM10K_MSG_ERR_PEP_NOT_SCHEDULED)
1570			dev_warn(&interface->pdev->dev,
1571				 "cannot obtain link because the host interface is configured for a PCIe host interface bandwidth of zero\n");
1572		dev_warn(&interface->pdev->dev,
1573			 "request logical port map failed: %d\n",
1574			 hw->swapi.status);
1575
1576		return 0;
1577	}
1578
1579	err = fm10k_msg_lport_map_pf(hw, results, mbx);
1580	if (err)
1581		return err;
1582
1583	interface->lport_map_failed = false;
1584
1585	/* we need to reset if port count was just updated */
1586	if (dglort_map != hw->mac.dglort_map)
1587		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1588
1589	return 0;
1590}
1591
1592static s32 fm10k_update_pvid(struct fm10k_hw *hw, u32 **results,
1593			     struct fm10k_mbx_info __always_unused *mbx)
1594{
1595	struct fm10k_intfc *interface;
1596	u16 glort, pvid;
1597	u32 pvid_update;
1598	s32 err;
1599
1600	err = fm10k_tlv_attr_get_u32(results[FM10K_PF_ATTR_ID_UPDATE_PVID],
1601				     &pvid_update);
1602	if (err)
1603		return err;
1604
1605	/* extract values from the pvid update */
1606	glort = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_GLORT);
1607	pvid = FM10K_MSG_HDR_FIELD_GET(pvid_update, UPDATE_PVID_PVID);
1608
1609	/* if glort is not valid return error */
1610	if (!fm10k_glort_valid_pf(hw, glort))
1611		return FM10K_ERR_PARAM;
1612
1613	/* verify VLAN ID is valid */
1614	if (pvid >= FM10K_VLAN_TABLE_VID_MAX)
1615		return FM10K_ERR_PARAM;
1616
1617	interface = container_of(hw, struct fm10k_intfc, hw);
1618
1619	/* check to see if this belongs to one of the VFs */
1620	err = fm10k_iov_update_pvid(interface, glort, pvid);
1621	if (!err)
1622		return 0;
1623
1624	/* we need to reset if default VLAN was just updated */
1625	if (pvid != hw->mac.default_vid)
1626		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1627
1628	hw->mac.default_vid = pvid;
1629
1630	return 0;
1631}
1632
1633static const struct fm10k_msg_data pf_mbx_data[] = {
1634	FM10K_PF_MSG_ERR_HANDLER(XCAST_MODES, fm10k_msg_err_pf),
1635	FM10K_PF_MSG_ERR_HANDLER(UPDATE_MAC_FWD_RULE, fm10k_msg_err_pf),
1636	FM10K_PF_MSG_LPORT_MAP_HANDLER(fm10k_lport_map),
1637	FM10K_PF_MSG_ERR_HANDLER(LPORT_CREATE, fm10k_msg_err_pf),
1638	FM10K_PF_MSG_ERR_HANDLER(LPORT_DELETE, fm10k_msg_err_pf),
1639	FM10K_PF_MSG_UPDATE_PVID_HANDLER(fm10k_update_pvid),
1640	FM10K_TLV_MSG_ERROR_HANDLER(fm10k_mbx_error),
1641};
1642
1643static int fm10k_mbx_request_irq_pf(struct fm10k_intfc *interface)
1644{
1645	struct msix_entry *entry = &interface->msix_entries[FM10K_MBX_VECTOR];
1646	struct net_device *dev = interface->netdev;
1647	struct fm10k_hw *hw = &interface->hw;
1648	int err;
1649
1650	/* Use timer0 for interrupt moderation on the mailbox */
1651	u32 mbx_itr = entry->entry | FM10K_INT_MAP_TIMER0;
1652	u32 other_itr = entry->entry | FM10K_INT_MAP_IMMEDIATE;
1653
1654	/* register mailbox handlers */
1655	err = hw->mbx.ops.register_handlers(&hw->mbx, pf_mbx_data);
1656	if (err)
1657		return err;
1658
1659	/* request the IRQ */
1660	err = request_irq(entry->vector, fm10k_msix_mbx_pf, 0,
1661			  dev->name, interface);
1662	if (err) {
1663		netif_err(interface, probe, dev,
1664			  "request_irq for msix_mbx failed: %d\n", err);
1665		return err;
1666	}
1667
1668	/* Enable interrupts w/ no moderation for "other" interrupts */
1669	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_pcie_fault), other_itr);
1670	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_switch_up_down), other_itr);
1671	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_sram), other_itr);
1672	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_max_hold_time), other_itr);
1673	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_vflr), other_itr);
1674
1675	/* Enable interrupts w/ moderation for mailbox */
1676	fm10k_write_reg(hw, FM10K_INT_MAP(fm10k_int_mailbox), mbx_itr);
1677
1678	/* Enable individual interrupt causes */
1679	fm10k_write_reg(hw, FM10K_EIMR, FM10K_EIMR_ENABLE(PCA_FAULT) |
1680					FM10K_EIMR_ENABLE(FUM_FAULT) |
1681					FM10K_EIMR_ENABLE(MAILBOX) |
1682					FM10K_EIMR_ENABLE(SWITCHREADY) |
1683					FM10K_EIMR_ENABLE(SWITCHNOTREADY) |
1684					FM10K_EIMR_ENABLE(SRAMERROR) |
1685					FM10K_EIMR_ENABLE(VFLR) |
1686					FM10K_EIMR_ENABLE(MAXHOLDTIME));
1687
1688	/* enable interrupt */
1689	fm10k_write_reg(hw, FM10K_ITR(entry->entry), FM10K_ITR_ENABLE);
1690
1691	return 0;
1692}
1693
1694int fm10k_mbx_request_irq(struct fm10k_intfc *interface)
1695{
1696	struct fm10k_hw *hw = &interface->hw;
1697	int err;
1698
1699	/* enable Mailbox cause */
1700	if (hw->mac.type == fm10k_mac_pf)
1701		err = fm10k_mbx_request_irq_pf(interface);
1702	else
1703		err = fm10k_mbx_request_irq_vf(interface);
1704	if (err)
1705		return err;
1706
1707	/* connect mailbox */
1708	err = hw->mbx.ops.connect(hw, &hw->mbx);
1709
1710	/* if the mailbox failed to connect, then free IRQ */
1711	if (err)
1712		fm10k_mbx_free_irq(interface);
1713
1714	return err;
1715}
1716
1717/**
1718 * fm10k_qv_free_irq - release interrupts associated with queue vectors
1719 * @interface: board private structure
1720 *
1721 * Release all interrupts associated with this interface
1722 **/
1723void fm10k_qv_free_irq(struct fm10k_intfc *interface)
1724{
1725	int vector = interface->num_q_vectors;
1726	struct msix_entry *entry;
1727
1728	entry = &interface->msix_entries[NON_Q_VECTORS + vector];
1729
1730	while (vector) {
1731		struct fm10k_q_vector *q_vector;
1732
1733		vector--;
1734		entry--;
1735		q_vector = interface->q_vector[vector];
1736
1737		if (!q_vector->tx.count && !q_vector->rx.count)
1738			continue;
1739
1740		/* clear the affinity_mask in the IRQ descriptor */
1741		irq_set_affinity_hint(entry->vector, NULL);
1742
1743		/* disable interrupts */
1744		writel(FM10K_ITR_MASK_SET, q_vector->itr);
1745
1746		free_irq(entry->vector, q_vector);
1747	}
1748}
1749
1750/**
1751 * fm10k_qv_request_irq - initialize interrupts for queue vectors
1752 * @interface: board private structure
1753 *
1754 * Attempts to configure interrupts using the best available
1755 * capabilities of the hardware and kernel.
1756 **/
1757int fm10k_qv_request_irq(struct fm10k_intfc *interface)
1758{
1759	struct net_device *dev = interface->netdev;
1760	struct fm10k_hw *hw = &interface->hw;
1761	struct msix_entry *entry;
1762	unsigned int ri = 0, ti = 0;
1763	int vector, err;
1764
1765	entry = &interface->msix_entries[NON_Q_VECTORS];
1766
1767	for (vector = 0; vector < interface->num_q_vectors; vector++) {
1768		struct fm10k_q_vector *q_vector = interface->q_vector[vector];
1769
1770		/* name the vector */
1771		if (q_vector->tx.count && q_vector->rx.count) {
1772			snprintf(q_vector->name, sizeof(q_vector->name),
1773				 "%s-TxRx-%u", dev->name, ri++);
1774			ti++;
1775		} else if (q_vector->rx.count) {
1776			snprintf(q_vector->name, sizeof(q_vector->name),
1777				 "%s-rx-%u", dev->name, ri++);
1778		} else if (q_vector->tx.count) {
1779			snprintf(q_vector->name, sizeof(q_vector->name),
1780				 "%s-tx-%u", dev->name, ti++);
1781		} else {
1782			/* skip this unused q_vector */
1783			continue;
1784		}
1785
1786		/* Assign ITR register to q_vector */
1787		q_vector->itr = (hw->mac.type == fm10k_mac_pf) ?
1788				&interface->uc_addr[FM10K_ITR(entry->entry)] :
1789				&interface->uc_addr[FM10K_VFITR(entry->entry)];
1790
1791		/* request the IRQ */
1792		err = request_irq(entry->vector, &fm10k_msix_clean_rings, 0,
1793				  q_vector->name, q_vector);
1794		if (err) {
1795			netif_err(interface, probe, dev,
1796				  "request_irq failed for MSIX interrupt Error: %d\n",
1797				  err);
1798			goto err_out;
1799		}
1800
1801		/* assign the mask for this irq */
1802		irq_set_affinity_hint(entry->vector, &q_vector->affinity_mask);
1803
1804		/* Enable q_vector */
1805		writel(FM10K_ITR_ENABLE, q_vector->itr);
1806
1807		entry++;
1808	}
1809
1810	return 0;
1811
1812err_out:
1813	/* wind through the ring freeing all entries and vectors */
1814	while (vector) {
1815		struct fm10k_q_vector *q_vector;
1816
1817		entry--;
1818		vector--;
1819		q_vector = interface->q_vector[vector];
1820
1821		if (!q_vector->tx.count && !q_vector->rx.count)
1822			continue;
1823
1824		/* clear the affinity_mask in the IRQ descriptor */
1825		irq_set_affinity_hint(entry->vector, NULL);
1826
1827		/* disable interrupts */
1828		writel(FM10K_ITR_MASK_SET, q_vector->itr);
1829
1830		free_irq(entry->vector, q_vector);
1831	}
1832
1833	return err;
1834}
1835
1836void fm10k_up(struct fm10k_intfc *interface)
1837{
1838	struct fm10k_hw *hw = &interface->hw;
1839
1840	/* Enable Tx/Rx DMA */
1841	hw->mac.ops.start_hw(hw);
1842
1843	/* configure Tx descriptor rings */
1844	fm10k_configure_tx(interface);
1845
1846	/* configure Rx descriptor rings */
1847	fm10k_configure_rx(interface);
1848
1849	/* configure interrupts */
1850	hw->mac.ops.update_int_moderator(hw);
1851
1852	/* enable statistics capture again */
1853	clear_bit(__FM10K_UPDATING_STATS, interface->state);
1854
1855	/* clear down bit to indicate we are ready to go */
1856	clear_bit(__FM10K_DOWN, interface->state);
1857
1858	/* enable polling cleanups */
1859	fm10k_napi_enable_all(interface);
1860
1861	/* re-establish Rx filters */
1862	fm10k_restore_rx_state(interface);
1863
1864	/* enable transmits */
1865	netif_tx_start_all_queues(interface->netdev);
1866
1867	/* kick off the service timer now */
1868	hw->mac.get_host_state = true;
1869	mod_timer(&interface->service_timer, jiffies);
1870}
1871
1872static void fm10k_napi_disable_all(struct fm10k_intfc *interface)
1873{
1874	struct fm10k_q_vector *q_vector;
1875	int q_idx;
1876
1877	for (q_idx = 0; q_idx < interface->num_q_vectors; q_idx++) {
1878		q_vector = interface->q_vector[q_idx];
1879		napi_disable(&q_vector->napi);
1880	}
1881}
1882
1883void fm10k_down(struct fm10k_intfc *interface)
1884{
1885	struct net_device *netdev = interface->netdev;
1886	struct fm10k_hw *hw = &interface->hw;
1887	int err, i = 0, count = 0;
1888
1889	/* signal that we are down to the interrupt handler and service task */
1890	if (test_and_set_bit(__FM10K_DOWN, interface->state))
1891		return;
1892
1893	/* call carrier off first to avoid false dev_watchdog timeouts */
1894	netif_carrier_off(netdev);
1895
1896	/* disable transmits */
1897	netif_tx_stop_all_queues(netdev);
1898	netif_tx_disable(netdev);
1899
1900	/* reset Rx filters */
1901	fm10k_reset_rx_state(interface);
1902
1903	/* disable polling routines */
1904	fm10k_napi_disable_all(interface);
1905
1906	/* capture stats one last time before stopping interface */
1907	fm10k_update_stats(interface);
1908
1909	/* prevent updating statistics while we're down */
1910	while (test_and_set_bit(__FM10K_UPDATING_STATS, interface->state))
1911		usleep_range(1000, 2000);
1912
1913	/* skip waiting for TX DMA if we lost PCIe link */
1914	if (FM10K_REMOVED(hw->hw_addr))
1915		goto skip_tx_dma_drain;
1916
1917	/* In some rare circumstances it can take a while for Tx queues to
1918	 * quiesce and be fully disabled. Attempt to .stop_hw() first, and
1919	 * then if we get ERR_REQUESTS_PENDING, go ahead and wait in a loop
1920	 * until the Tx queues have emptied, or until a number of retries. If
1921	 * we fail to clear within the retry loop, we will issue a warning
1922	 * indicating that Tx DMA is probably hung. Note this means we call
1923	 * .stop_hw() twice but this shouldn't cause any problems.
1924	 */
1925	err = hw->mac.ops.stop_hw(hw);
1926	if (err != FM10K_ERR_REQUESTS_PENDING)
1927		goto skip_tx_dma_drain;
1928
1929#define TX_DMA_DRAIN_RETRIES 25
1930	for (count = 0; count < TX_DMA_DRAIN_RETRIES; count++) {
1931		usleep_range(10000, 20000);
1932
1933		/* start checking at the last ring to have pending Tx */
1934		for (; i < interface->num_tx_queues; i++)
1935			if (fm10k_get_tx_pending(interface->tx_ring[i], false))
1936				break;
1937
1938		/* if all the queues are drained, we can break now */
1939		if (i == interface->num_tx_queues)
1940			break;
1941	}
1942
1943	if (count >= TX_DMA_DRAIN_RETRIES)
1944		dev_err(&interface->pdev->dev,
1945			"Tx queues failed to drain after %d tries. Tx DMA is probably hung.\n",
1946			count);
1947skip_tx_dma_drain:
1948	/* Disable DMA engine for Tx/Rx */
1949	err = hw->mac.ops.stop_hw(hw);
1950	if (err == FM10K_ERR_REQUESTS_PENDING)
1951		dev_err(&interface->pdev->dev,
1952			"due to pending requests hw was not shut down gracefully\n");
1953	else if (err)
1954		dev_err(&interface->pdev->dev, "stop_hw failed: %d\n", err);
1955
1956	/* free any buffers still on the rings */
1957	fm10k_clean_all_tx_rings(interface);
1958	fm10k_clean_all_rx_rings(interface);
1959}
1960
1961/**
1962 * fm10k_sw_init - Initialize general software structures
1963 * @interface: host interface private structure to initialize
1964 * @ent: PCI device ID entry
1965 *
1966 * fm10k_sw_init initializes the interface private data structure.
1967 * Fields are initialized based on PCI device information and
1968 * OS network device settings (MTU size).
1969 **/
1970static int fm10k_sw_init(struct fm10k_intfc *interface,
1971			 const struct pci_device_id *ent)
1972{
1973	const struct fm10k_info *fi = fm10k_info_tbl[ent->driver_data];
1974	struct fm10k_hw *hw = &interface->hw;
1975	struct pci_dev *pdev = interface->pdev;
1976	struct net_device *netdev = interface->netdev;
1977	u32 rss_key[FM10K_RSSRK_SIZE];
1978	unsigned int rss;
1979	int err;
1980
1981	/* initialize back pointer */
1982	hw->back = interface;
1983	hw->hw_addr = interface->uc_addr;
1984
1985	/* PCI config space info */
1986	hw->vendor_id = pdev->vendor;
1987	hw->device_id = pdev->device;
1988	hw->revision_id = pdev->revision;
1989	hw->subsystem_vendor_id = pdev->subsystem_vendor;
1990	hw->subsystem_device_id = pdev->subsystem_device;
1991
1992	/* Setup hw api */
1993	memcpy(&hw->mac.ops, fi->mac_ops, sizeof(hw->mac.ops));
1994	hw->mac.type = fi->mac;
1995
1996	/* Setup IOV handlers */
1997	if (fi->iov_ops)
1998		memcpy(&hw->iov.ops, fi->iov_ops, sizeof(hw->iov.ops));
1999
2000	/* Set common capability flags and settings */
2001	rss = min_t(int, FM10K_MAX_RSS_INDICES, num_online_cpus());
2002	interface->ring_feature[RING_F_RSS].limit = rss;
2003	fi->get_invariants(hw);
2004
2005	/* pick up the PCIe bus settings for reporting later */
2006	if (hw->mac.ops.get_bus_info)
2007		hw->mac.ops.get_bus_info(hw);
2008
2009	/* limit the usable DMA range */
2010	if (hw->mac.ops.set_dma_mask)
2011		hw->mac.ops.set_dma_mask(hw, dma_get_mask(&pdev->dev));
2012
2013	/* update netdev with DMA restrictions */
2014	if (dma_get_mask(&pdev->dev) > DMA_BIT_MASK(32)) {
2015		netdev->features |= NETIF_F_HIGHDMA;
2016		netdev->vlan_features |= NETIF_F_HIGHDMA;
2017	}
2018
2019	/* reset and initialize the hardware so it is in a known state */
2020	err = hw->mac.ops.reset_hw(hw);
2021	if (err) {
2022		dev_err(&pdev->dev, "reset_hw failed: %d\n", err);
2023		return err;
2024	}
2025
2026	err = hw->mac.ops.init_hw(hw);
2027	if (err) {
2028		dev_err(&pdev->dev, "init_hw failed: %d\n", err);
2029		return err;
2030	}
2031
2032	/* initialize hardware statistics */
2033	hw->mac.ops.update_hw_stats(hw, &interface->stats);
2034
2035	/* Set upper limit on IOV VFs that can be allocated */
2036	pci_sriov_set_totalvfs(pdev, hw->iov.total_vfs);
2037
2038	/* Start with random Ethernet address */
2039	eth_random_addr(hw->mac.addr);
2040
2041	/* Initialize MAC address from hardware */
2042	err = hw->mac.ops.read_mac_addr(hw);
2043	if (err) {
2044		dev_warn(&pdev->dev,
2045			 "Failed to obtain MAC address defaulting to random\n");
2046		/* tag address assignment as random */
2047		netdev->addr_assign_type |= NET_ADDR_RANDOM;
2048	}
2049
2050	ether_addr_copy(netdev->dev_addr, hw->mac.addr);
2051	ether_addr_copy(netdev->perm_addr, hw->mac.addr);
2052
2053	if (!is_valid_ether_addr(netdev->perm_addr)) {
2054		dev_err(&pdev->dev, "Invalid MAC Address\n");
2055		return -EIO;
2056	}
2057
2058	/* initialize DCBNL interface */
2059	fm10k_dcbnl_set_ops(netdev);
2060
2061	/* set default ring sizes */
2062	interface->tx_ring_count = FM10K_DEFAULT_TXD;
2063	interface->rx_ring_count = FM10K_DEFAULT_RXD;
2064
2065	/* set default interrupt moderation */
2066	interface->tx_itr = FM10K_TX_ITR_DEFAULT;
2067	interface->rx_itr = FM10K_ITR_ADAPTIVE | FM10K_RX_ITR_DEFAULT;
2068
2069	/* Initialize the MAC/VLAN queue */
2070	INIT_LIST_HEAD(&interface->macvlan_requests);
2071
2072	netdev_rss_key_fill(rss_key, sizeof(rss_key));
2073	memcpy(interface->rssrk, rss_key, sizeof(rss_key));
2074
2075	/* Initialize the mailbox lock */
2076	spin_lock_init(&interface->mbx_lock);
2077	spin_lock_init(&interface->macvlan_lock);
2078
2079	/* Start off interface as being down */
2080	set_bit(__FM10K_DOWN, interface->state);
2081	set_bit(__FM10K_UPDATING_STATS, interface->state);
2082
2083	return 0;
2084}
2085
2086/**
2087 * fm10k_probe - Device Initialization Routine
2088 * @pdev: PCI device information struct
2089 * @ent: entry in fm10k_pci_tbl
2090 *
2091 * Returns 0 on success, negative on failure
2092 *
2093 * fm10k_probe initializes an interface identified by a pci_dev structure.
2094 * The OS initialization, configuring of the interface private structure,
2095 * and a hardware reset occur.
2096 **/
2097static int fm10k_probe(struct pci_dev *pdev, const struct pci_device_id *ent)
2098{
2099	struct net_device *netdev;
2100	struct fm10k_intfc *interface;
2101	int err;
2102
2103	if (pdev->error_state != pci_channel_io_normal) {
2104		dev_err(&pdev->dev,
2105			"PCI device still in an error state. Unable to load...\n");
2106		return -EIO;
2107	}
2108
2109	err = pci_enable_device_mem(pdev);
2110	if (err) {
2111		dev_err(&pdev->dev,
2112			"PCI enable device failed: %d\n", err);
2113		return err;
2114	}
2115
2116	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(48));
2117	if (err)
2118		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
2119	if (err) {
2120		dev_err(&pdev->dev,
2121			"DMA configuration failed: %d\n", err);
2122		goto err_dma;
2123	}
2124
2125	err = pci_request_mem_regions(pdev, fm10k_driver_name);
2126	if (err) {
2127		dev_err(&pdev->dev,
2128			"pci_request_selected_regions failed: %d\n", err);
2129		goto err_pci_reg;
2130	}
2131
2132	pci_enable_pcie_error_reporting(pdev);
2133
2134	pci_set_master(pdev);
2135	pci_save_state(pdev);
2136
2137	netdev = fm10k_alloc_netdev(fm10k_info_tbl[ent->driver_data]);
2138	if (!netdev) {
2139		err = -ENOMEM;
2140		goto err_alloc_netdev;
2141	}
2142
2143	SET_NETDEV_DEV(netdev, &pdev->dev);
2144
2145	interface = netdev_priv(netdev);
2146	pci_set_drvdata(pdev, interface);
2147
2148	interface->netdev = netdev;
2149	interface->pdev = pdev;
2150
2151	interface->uc_addr = ioremap(pci_resource_start(pdev, 0),
2152				     FM10K_UC_ADDR_SIZE);
2153	if (!interface->uc_addr) {
2154		err = -EIO;
2155		goto err_ioremap;
2156	}
2157
2158	err = fm10k_sw_init(interface, ent);
2159	if (err)
2160		goto err_sw_init;
2161
2162	/* enable debugfs support */
2163	fm10k_dbg_intfc_init(interface);
2164
2165	err = fm10k_init_queueing_scheme(interface);
2166	if (err)
2167		goto err_sw_init;
2168
2169	/* the mbx interrupt might attempt to schedule the service task, so we
2170	 * must ensure it is disabled since we haven't yet requested the timer
2171	 * or work item.
2172	 */
2173	set_bit(__FM10K_SERVICE_DISABLE, interface->state);
2174
2175	err = fm10k_mbx_request_irq(interface);
2176	if (err)
2177		goto err_mbx_interrupt;
2178
2179	/* final check of hardware state before registering the interface */
2180	err = fm10k_hw_ready(interface);
2181	if (err)
2182		goto err_register;
2183
2184	err = register_netdev(netdev);
2185	if (err)
2186		goto err_register;
2187
2188	/* carrier off reporting is important to ethtool even BEFORE open */
2189	netif_carrier_off(netdev);
2190
2191	/* stop all the transmit queues from transmitting until link is up */
2192	netif_tx_stop_all_queues(netdev);
2193
2194	/* Initialize service timer and service task late in order to avoid
2195	 * cleanup issues.
2196	 */
2197	timer_setup(&interface->service_timer, fm10k_service_timer, 0);
2198	INIT_WORK(&interface->service_task, fm10k_service_task);
2199
2200	/* Setup the MAC/VLAN queue */
2201	INIT_DELAYED_WORK(&interface->macvlan_task, fm10k_macvlan_task);
2202
2203	/* kick off service timer now, even when interface is down */
2204	mod_timer(&interface->service_timer, (HZ * 2) + jiffies);
2205
2206	/* print warning for non-optimal configurations */
2207	pcie_print_link_status(interface->pdev);
2208
2209	/* report MAC address for logging */
2210	dev_info(&pdev->dev, "%pM\n", netdev->dev_addr);
2211
2212	/* enable SR-IOV after registering netdev to enforce PF/VF ordering */
2213	fm10k_iov_configure(pdev, 0);
2214
2215	/* clear the service task disable bit and kick off service task */
2216	clear_bit(__FM10K_SERVICE_DISABLE, interface->state);
2217	fm10k_service_event_schedule(interface);
2218
2219	return 0;
2220
2221err_register:
2222	fm10k_mbx_free_irq(interface);
2223err_mbx_interrupt:
2224	fm10k_clear_queueing_scheme(interface);
2225err_sw_init:
2226	if (interface->sw_addr)
2227		iounmap(interface->sw_addr);
2228	iounmap(interface->uc_addr);
2229err_ioremap:
2230	free_netdev(netdev);
2231err_alloc_netdev:
2232	pci_release_mem_regions(pdev);
2233err_pci_reg:
2234err_dma:
2235	pci_disable_device(pdev);
2236	return err;
2237}
2238
2239/**
2240 * fm10k_remove - Device Removal Routine
2241 * @pdev: PCI device information struct
2242 *
2243 * fm10k_remove is called by the PCI subsystem to alert the driver
2244 * that it should release a PCI device.  The could be caused by a
2245 * Hot-Plug event, or because the driver is going to be removed from
2246 * memory.
2247 **/
2248static void fm10k_remove(struct pci_dev *pdev)
2249{
2250	struct fm10k_intfc *interface = pci_get_drvdata(pdev);
2251	struct net_device *netdev = interface->netdev;
2252
2253	del_timer_sync(&interface->service_timer);
2254
2255	fm10k_stop_service_event(interface);
2256	fm10k_stop_macvlan_task(interface);
2257
2258	/* Remove all pending MAC/VLAN requests */
2259	fm10k_clear_macvlan_queue(interface, interface->glort, true);
2260
2261	/* free netdev, this may bounce the interrupts due to setup_tc */
2262	if (netdev->reg_state == NETREG_REGISTERED)
2263		unregister_netdev(netdev);
2264
2265	/* release VFs */
2266	fm10k_iov_disable(pdev);
2267
2268	/* disable mailbox interrupt */
2269	fm10k_mbx_free_irq(interface);
2270
2271	/* free interrupts */
2272	fm10k_clear_queueing_scheme(interface);
2273
2274	/* remove any debugfs interfaces */
2275	fm10k_dbg_intfc_exit(interface);
2276
2277	if (interface->sw_addr)
2278		iounmap(interface->sw_addr);
2279	iounmap(interface->uc_addr);
2280
2281	free_netdev(netdev);
2282
2283	pci_release_mem_regions(pdev);
2284
2285	pci_disable_pcie_error_reporting(pdev);
2286
2287	pci_disable_device(pdev);
2288}
2289
2290static void fm10k_prepare_suspend(struct fm10k_intfc *interface)
2291{
2292	/* the watchdog task reads from registers, which might appear like
2293	 * a surprise remove if the PCIe device is disabled while we're
2294	 * stopped. We stop the watchdog task until after we resume software
2295	 * activity.
2296	 *
2297	 * Note that the MAC/VLAN task will be stopped as part of preparing
2298	 * for reset so we don't need to handle it here.
2299	 */
2300	fm10k_stop_service_event(interface);
2301
2302	if (fm10k_prepare_for_reset(interface))
2303		set_bit(__FM10K_RESET_SUSPENDED, interface->state);
2304}
2305
2306static int fm10k_handle_resume(struct fm10k_intfc *interface)
2307{
2308	struct fm10k_hw *hw = &interface->hw;
2309	int err;
2310
2311	/* Even if we didn't properly prepare for reset in
2312	 * fm10k_prepare_suspend, we'll attempt to resume anyways.
2313	 */
2314	if (!test_and_clear_bit(__FM10K_RESET_SUSPENDED, interface->state))
2315		dev_warn(&interface->pdev->dev,
2316			 "Device was shut down as part of suspend... Attempting to recover\n");
2317
2318	/* reset statistics starting values */
2319	hw->mac.ops.rebind_hw_stats(hw, &interface->stats);
2320
2321	err = fm10k_handle_reset(interface);
2322	if (err)
2323		return err;
2324
2325	/* assume host is not ready, to prevent race with watchdog in case we
2326	 * actually don't have connection to the switch
2327	 */
2328	interface->host_ready = false;
2329	fm10k_watchdog_host_not_ready(interface);
2330
2331	/* force link to stay down for a second to prevent link flutter */
2332	interface->link_down_event = jiffies + (HZ);
2333	set_bit(__FM10K_LINK_DOWN, interface->state);
2334
2335	/* restart the service task */
2336	fm10k_start_service_event(interface);
2337
2338	/* Restart the MAC/VLAN request queue in-case of outstanding events */
2339	fm10k_macvlan_schedule(interface);
2340
2341	return 0;
2342}
2343
2344/**
2345 * fm10k_resume - Generic PM resume hook
2346 * @dev: generic device structure
2347 *
2348 * Generic PM hook used when waking the device from a low power state after
2349 * suspend or hibernation. This function does not need to handle lower PCIe
2350 * device state as the stack takes care of that for us.
2351 **/
2352static int __maybe_unused fm10k_resume(struct device *dev)
2353{
2354	struct fm10k_intfc *interface = dev_get_drvdata(dev);
2355	struct net_device *netdev = interface->netdev;
2356	struct fm10k_hw *hw = &interface->hw;
2357	int err;
2358
2359	/* refresh hw_addr in case it was dropped */
2360	hw->hw_addr = interface->uc_addr;
2361
2362	err = fm10k_handle_resume(interface);
2363	if (err)
2364		return err;
2365
2366	netif_device_attach(netdev);
2367
2368	return 0;
2369}
2370
2371/**
2372 * fm10k_suspend - Generic PM suspend hook
2373 * @dev: generic device structure
2374 *
2375 * Generic PM hook used when setting the device into a low power state for
2376 * system suspend or hibernation. This function does not need to handle lower
2377 * PCIe device state as the stack takes care of that for us.
2378 **/
2379static int __maybe_unused fm10k_suspend(struct device *dev)
2380{
2381	struct fm10k_intfc *interface = dev_get_drvdata(dev);
2382	struct net_device *netdev = interface->netdev;
2383
2384	netif_device_detach(netdev);
2385
2386	fm10k_prepare_suspend(interface);
2387
2388	return 0;
2389}
2390
2391/**
2392 * fm10k_io_error_detected - called when PCI error is detected
2393 * @pdev: Pointer to PCI device
2394 * @state: The current pci connection state
2395 *
2396 * This function is called after a PCI bus error affecting
2397 * this device has been detected.
2398 */
2399static pci_ers_result_t fm10k_io_error_detected(struct pci_dev *pdev,
2400						pci_channel_state_t state)
2401{
2402	struct fm10k_intfc *interface = pci_get_drvdata(pdev);
2403	struct net_device *netdev = interface->netdev;
2404
2405	netif_device_detach(netdev);
2406
2407	if (state == pci_channel_io_perm_failure)
2408		return PCI_ERS_RESULT_DISCONNECT;
2409
2410	fm10k_prepare_suspend(interface);
2411
2412	/* Request a slot reset. */
2413	return PCI_ERS_RESULT_NEED_RESET;
2414}
2415
2416/**
2417 * fm10k_io_slot_reset - called after the pci bus has been reset.
2418 * @pdev: Pointer to PCI device
2419 *
2420 * Restart the card from scratch, as if from a cold-boot.
2421 */
2422static pci_ers_result_t fm10k_io_slot_reset(struct pci_dev *pdev)
2423{
2424	pci_ers_result_t result;
2425
2426	if (pci_reenable_device(pdev)) {
2427		dev_err(&pdev->dev,
2428			"Cannot re-enable PCI device after reset.\n");
2429		result = PCI_ERS_RESULT_DISCONNECT;
2430	} else {
2431		pci_set_master(pdev);
2432		pci_restore_state(pdev);
2433
2434		/* After second error pci->state_saved is false, this
2435		 * resets it so EEH doesn't break.
2436		 */
2437		pci_save_state(pdev);
2438
2439		pci_wake_from_d3(pdev, false);
2440
2441		result = PCI_ERS_RESULT_RECOVERED;
2442	}
2443
2444	return result;
2445}
2446
2447/**
2448 * fm10k_io_resume - called when traffic can start flowing again.
2449 * @pdev: Pointer to PCI device
2450 *
2451 * This callback is called when the error recovery driver tells us that
2452 * its OK to resume normal operation.
2453 */
2454static void fm10k_io_resume(struct pci_dev *pdev)
2455{
2456	struct fm10k_intfc *interface = pci_get_drvdata(pdev);
2457	struct net_device *netdev = interface->netdev;
2458	int err;
2459
2460	err = fm10k_handle_resume(interface);
2461
2462	if (err)
2463		dev_warn(&pdev->dev,
2464			 "%s failed: %d\n", __func__, err);
2465	else
2466		netif_device_attach(netdev);
2467}
2468
2469/**
2470 * fm10k_io_reset_prepare - called when PCI function is about to be reset
2471 * @pdev: Pointer to PCI device
2472 *
2473 * This callback is called when the PCI function is about to be reset,
2474 * allowing the device driver to prepare for it.
2475 */
2476static void fm10k_io_reset_prepare(struct pci_dev *pdev)
2477{
2478	/* warn incase we have any active VF devices */
2479	if (pci_num_vf(pdev))
2480		dev_warn(&pdev->dev,
2481			 "PCIe FLR may cause issues for any active VF devices\n");
2482	fm10k_prepare_suspend(pci_get_drvdata(pdev));
2483}
2484
2485/**
2486 * fm10k_io_reset_done - called when PCI function has finished resetting
2487 * @pdev: Pointer to PCI device
2488 *
2489 * This callback is called just after the PCI function is reset, such as via
2490 * /sys/class/net/<enpX>/device/reset or similar.
2491 */
2492static void fm10k_io_reset_done(struct pci_dev *pdev)
2493{
2494	struct fm10k_intfc *interface = pci_get_drvdata(pdev);
2495	int err = fm10k_handle_resume(interface);
2496
2497	if (err) {
2498		dev_warn(&pdev->dev,
2499			 "%s failed: %d\n", __func__, err);
2500		netif_device_detach(interface->netdev);
2501	}
2502}
2503
2504static const struct pci_error_handlers fm10k_err_handler = {
2505	.error_detected = fm10k_io_error_detected,
2506	.slot_reset = fm10k_io_slot_reset,
2507	.resume = fm10k_io_resume,
2508	.reset_prepare = fm10k_io_reset_prepare,
2509	.reset_done = fm10k_io_reset_done,
2510};
2511
2512static SIMPLE_DEV_PM_OPS(fm10k_pm_ops, fm10k_suspend, fm10k_resume);
2513
2514static struct pci_driver fm10k_driver = {
2515	.name			= fm10k_driver_name,
2516	.id_table		= fm10k_pci_tbl,
2517	.probe			= fm10k_probe,
2518	.remove			= fm10k_remove,
2519	.driver = {
2520		.pm		= &fm10k_pm_ops,
2521	},
2522	.sriov_configure	= fm10k_iov_configure,
2523	.err_handler		= &fm10k_err_handler
2524};
2525
2526/**
2527 * fm10k_register_pci_driver - register driver interface
2528 *
2529 * This function is called on module load in order to register the driver.
2530 **/
2531int fm10k_register_pci_driver(void)
2532{
2533	return pci_register_driver(&fm10k_driver);
2534}
2535
2536/**
2537 * fm10k_unregister_pci_driver - unregister driver interface
2538 *
2539 * This function is called on module unload in order to remove the driver.
2540 **/
2541void fm10k_unregister_pci_driver(void)
2542{
2543	pci_unregister_driver(&fm10k_driver);
2544}