1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright (c) 2018, Intel Corporation. */
   3
   4/* Intel(R) Ethernet Connection E800 Series Linux Driver */
   5
   6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7
   8#include "ice.h"
   9
  10#define DRV_VERSION	"ice-0.7.0-k"
  11#define DRV_SUMMARY	"Intel(R) Ethernet Connection E800 Series Linux Driver"
  12const char ice_drv_ver[] = DRV_VERSION;
  13static const char ice_driver_string[] = DRV_SUMMARY;
  14static const char ice_copyright[] = "Copyright (c) 2018, Intel Corporation.";
  15
  16MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  17MODULE_DESCRIPTION(DRV_SUMMARY);
  18MODULE_LICENSE("GPL");
  19MODULE_VERSION(DRV_VERSION);
  20
  21static int debug = -1;
  22module_param(debug, int, 0644);
  23#ifndef CONFIG_DYNAMIC_DEBUG
  24MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all), hw debug_mask (0x8XXXXXXX)");
  25#else
  26MODULE_PARM_DESC(debug, "netif level (0=none,...,16=all)");
  27#endif /* !CONFIG_DYNAMIC_DEBUG */
  28
  29static struct workqueue_struct *ice_wq;
  30static const struct net_device_ops ice_netdev_ops;
  31
  32static void ice_pf_dis_all_vsi(struct ice_pf *pf);
  33static void ice_rebuild(struct ice_pf *pf);
  34static int ice_vsi_release(struct ice_vsi *vsi);
  35static void ice_update_vsi_stats(struct ice_vsi *vsi);
  36static void ice_update_pf_stats(struct ice_pf *pf);
  37
  38/**
  39 * ice_get_free_slot - get the next non-NULL location index in array
  40 * @array: array to search
  41 * @size: size of the array
  42 * @curr: last known occupied index to be used as a search hint
  43 *
  44 * void * is being used to keep the functionality generic. This lets us use this
  45 * function on any array of pointers.
  46 */
  47static int ice_get_free_slot(void *array, int size, int curr)
  48{
  49	int **tmp_array = (int **)array;
  50	int next;
  51
  52	if (curr < (size - 1) && !tmp_array[curr + 1]) {
  53		next = curr + 1;
  54	} else {
  55		int i = 0;
  56
  57		while ((i < size) && (tmp_array[i]))
  58			i++;
  59		if (i == size)
  60			next = ICE_NO_VSI;
  61		else
  62			next = i;
  63	}
  64	return next;
  65}
  66
  67/**
  68 * ice_search_res - Search the tracker for a block of resources
  69 * @res: pointer to the resource
  70 * @needed: size of the block needed
  71 * @id: identifier to track owner
  72 * Returns the base item index of the block, or -ENOMEM for error
  73 */
  74static int ice_search_res(struct ice_res_tracker *res, u16 needed, u16 id)
  75{
  76	int start = res->search_hint;
  77	int end = start;
  78
  79	id |= ICE_RES_VALID_BIT;
  80
  81	do {
  82		/* skip already allocated entries */
  83		if (res->list[end++] & ICE_RES_VALID_BIT) {
  84			start = end;
  85			if ((start + needed) > res->num_entries)
  86				break;
  87		}
  88
  89		if (end == (start + needed)) {
  90			int i = start;
  91
  92			/* there was enough, so assign it to the requestor */
  93			while (i != end)
  94				res->list[i++] = id;
  95
  96			if (end == res->num_entries)
  97				end = 0;
  98
  99			res->search_hint = end;
 100			return start;
 101		}
 102	} while (1);
 103
 104	return -ENOMEM;
 105}
 106
 107/**
 108 * ice_get_res - get a block of resources
 109 * @pf: board private structure
 110 * @res: pointer to the resource
 111 * @needed: size of the block needed
 112 * @id: identifier to track owner
 113 *
 114 * Returns the base item index of the block, or -ENOMEM for error
 115 * The search_hint trick and lack of advanced fit-finding only works
 116 * because we're highly likely to have all the same sized requests.
 117 * Linear search time and any fragmentation should be minimal.
 118 */
 119static int
 120ice_get_res(struct ice_pf *pf, struct ice_res_tracker *res, u16 needed, u16 id)
 121{
 122	int ret;
 123
 124	if (!res || !pf)
 125		return -EINVAL;
 126
 127	if (!needed || needed > res->num_entries || id >= ICE_RES_VALID_BIT) {
 128		dev_err(&pf->pdev->dev,
 129			"param err: needed=%d, num_entries = %d id=0x%04x\n",
 130			needed, res->num_entries, id);
 131		return -EINVAL;
 132	}
 133
 134	/* search based on search_hint */
 135	ret = ice_search_res(res, needed, id);
 136
 137	if (ret < 0) {
 138		/* previous search failed. Reset search hint and try again */
 139		res->search_hint = 0;
 140		ret = ice_search_res(res, needed, id);
 141	}
 142
 143	return ret;
 144}
 145
 146/**
 147 * ice_free_res - free a block of resources
 148 * @res: pointer to the resource
 149 * @index: starting index previously returned by ice_get_res
 150 * @id: identifier to track owner
 151 * Returns number of resources freed
 152 */
 153static int ice_free_res(struct ice_res_tracker *res, u16 index, u16 id)
 154{
 155	int count = 0;
 156	int i;
 157
 158	if (!res || index >= res->num_entries)
 159		return -EINVAL;
 160
 161	id |= ICE_RES_VALID_BIT;
 162	for (i = index; i < res->num_entries && res->list[i] == id; i++) {
 163		res->list[i] = 0;
 164		count++;
 165	}
 166
 167	return count;
 168}
 169
 170/**
 171 * ice_add_mac_to_list - Add a mac address filter entry to the list
 172 * @vsi: the VSI to be forwarded to
 173 * @add_list: pointer to the list which contains MAC filter entries
 174 * @macaddr: the MAC address to be added.
 175 *
 176 * Adds mac address filter entry to the temp list
 177 *
 178 * Returns 0 on success or ENOMEM on failure.
 179 */
 180static int ice_add_mac_to_list(struct ice_vsi *vsi, struct list_head *add_list,
 181			       const u8 *macaddr)
 182{
 183	struct ice_fltr_list_entry *tmp;
 184	struct ice_pf *pf = vsi->back;
 185
 186	tmp = devm_kzalloc(&pf->pdev->dev, sizeof(*tmp), GFP_ATOMIC);
 187	if (!tmp)
 188		return -ENOMEM;
 189
 190	tmp->fltr_info.flag = ICE_FLTR_TX;
 191	tmp->fltr_info.src = vsi->vsi_num;
 192	tmp->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
 193	tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI;
 194	tmp->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
 195	ether_addr_copy(tmp->fltr_info.l_data.mac.mac_addr, macaddr);
 196
 197	INIT_LIST_HEAD(&tmp->list_entry);
 198	list_add(&tmp->list_entry, add_list);
 199
 200	return 0;
 201}
 202
 203/**
 204 * ice_add_mac_to_sync_list - creates list of mac addresses to be synced
 205 * @netdev: the net device on which the sync is happening
 206 * @addr: mac address to sync
 207 *
 208 * This is a callback function which is called by the in kernel device sync
 209 * functions (like __dev_uc_sync, __dev_mc_sync, etc). This function only
 210 * populates the tmp_sync_list, which is later used by ice_add_mac to add the
 211 * mac filters from the hardware.
 212 */
 213static int ice_add_mac_to_sync_list(struct net_device *netdev, const u8 *addr)
 214{
 215	struct ice_netdev_priv *np = netdev_priv(netdev);
 216	struct ice_vsi *vsi = np->vsi;
 217
 218	if (ice_add_mac_to_list(vsi, &vsi->tmp_sync_list, addr))
 219		return -EINVAL;
 220
 221	return 0;
 222}
 223
 224/**
 225 * ice_add_mac_to_unsync_list - creates list of mac addresses to be unsynced
 226 * @netdev: the net device on which the unsync is happening
 227 * @addr: mac address to unsync
 228 *
 229 * This is a callback function which is called by the in kernel device unsync
 230 * functions (like __dev_uc_unsync, __dev_mc_unsync, etc). This function only
 231 * populates the tmp_unsync_list, which is later used by ice_remove_mac to
 232 * delete the mac filters from the hardware.
 233 */
 234static int ice_add_mac_to_unsync_list(struct net_device *netdev, const u8 *addr)
 235{
 236	struct ice_netdev_priv *np = netdev_priv(netdev);
 237	struct ice_vsi *vsi = np->vsi;
 238
 239	if (ice_add_mac_to_list(vsi, &vsi->tmp_unsync_list, addr))
 240		return -EINVAL;
 241
 242	return 0;
 243}
 244
 245/**
 246 * ice_free_fltr_list - free filter lists helper
 247 * @dev: pointer to the device struct
 248 * @h: pointer to the list head to be freed
 249 *
 250 * Helper function to free filter lists previously created using
 251 * ice_add_mac_to_list
 252 */
 253static void ice_free_fltr_list(struct device *dev, struct list_head *h)
 254{
 255	struct ice_fltr_list_entry *e, *tmp;
 256
 257	list_for_each_entry_safe(e, tmp, h, list_entry) {
 258		list_del(&e->list_entry);
 259		devm_kfree(dev, e);
 260	}
 261}
 262
 263/**
 264 * ice_vsi_fltr_changed - check if filter state changed
 265 * @vsi: VSI to be checked
 266 *
 267 * returns true if filter state has changed, false otherwise.
 268 */
 269static bool ice_vsi_fltr_changed(struct ice_vsi *vsi)
 270{
 271	return test_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags) ||
 272	       test_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags) ||
 273	       test_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags);
 274}
 275
 276/**
 277 * ice_vsi_sync_fltr - Update the VSI filter list to the HW
 278 * @vsi: ptr to the VSI
 279 *
 280 * Push any outstanding VSI filter changes through the AdminQ.
 281 */
 282static int ice_vsi_sync_fltr(struct ice_vsi *vsi)
 283{
 284	struct device *dev = &vsi->back->pdev->dev;
 285	struct net_device *netdev = vsi->netdev;
 286	bool promisc_forced_on = false;
 287	struct ice_pf *pf = vsi->back;
 288	struct ice_hw *hw = &pf->hw;
 289	enum ice_status status = 0;
 290	u32 changed_flags = 0;
 291	int err = 0;
 292
 293	if (!vsi->netdev)
 294		return -EINVAL;
 295
 296	while (test_and_set_bit(__ICE_CFG_BUSY, vsi->state))
 297		usleep_range(1000, 2000);
 298
 299	changed_flags = vsi->current_netdev_flags ^ vsi->netdev->flags;
 300	vsi->current_netdev_flags = vsi->netdev->flags;
 301
 302	INIT_LIST_HEAD(&vsi->tmp_sync_list);
 303	INIT_LIST_HEAD(&vsi->tmp_unsync_list);
 304
 305	if (ice_vsi_fltr_changed(vsi)) {
 306		clear_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
 307		clear_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
 308		clear_bit(ICE_VSI_FLAG_VLAN_FLTR_CHANGED, vsi->flags);
 309
 310		/* grab the netdev's addr_list_lock */
 311		netif_addr_lock_bh(netdev);
 312		__dev_uc_sync(netdev, ice_add_mac_to_sync_list,
 313			      ice_add_mac_to_unsync_list);
 314		__dev_mc_sync(netdev, ice_add_mac_to_sync_list,
 315			      ice_add_mac_to_unsync_list);
 316		/* our temp lists are populated. release lock */
 317		netif_addr_unlock_bh(netdev);
 318	}
 319
 320	/* Remove mac addresses in the unsync list */
 321	status = ice_remove_mac(hw, &vsi->tmp_unsync_list);
 322	ice_free_fltr_list(dev, &vsi->tmp_unsync_list);
 323	if (status) {
 324		netdev_err(netdev, "Failed to delete MAC filters\n");
 325		/* if we failed because of alloc failures, just bail */
 326		if (status == ICE_ERR_NO_MEMORY) {
 327			err = -ENOMEM;
 328			goto out;
 329		}
 330	}
 331
 332	/* Add mac addresses in the sync list */
 333	status = ice_add_mac(hw, &vsi->tmp_sync_list);
 334	ice_free_fltr_list(dev, &vsi->tmp_sync_list);
 335	if (status) {
 336		netdev_err(netdev, "Failed to add MAC filters\n");
 337		/* If there is no more space for new umac filters, vsi
 338		 * should go into promiscuous mode. There should be some
 339		 * space reserved for promiscuous filters.
 340		 */
 341		if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOSPC &&
 342		    !test_and_set_bit(__ICE_FLTR_OVERFLOW_PROMISC,
 343				      vsi->state)) {
 344			promisc_forced_on = true;
 345			netdev_warn(netdev,
 346				    "Reached MAC filter limit, forcing promisc mode on VSI %d\n",
 347				    vsi->vsi_num);
 348		} else {
 349			err = -EIO;
 350			goto out;
 351		}
 352	}
 353	/* check for changes in promiscuous modes */
 354	if (changed_flags & IFF_ALLMULTI)
 355		netdev_warn(netdev, "Unsupported configuration\n");
 356
 357	if (((changed_flags & IFF_PROMISC) || promisc_forced_on) ||
 358	    test_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags)) {
 359		clear_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags);
 360		if (vsi->current_netdev_flags & IFF_PROMISC) {
 361			/* Apply TX filter rule to get traffic from VMs */
 362			status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, true,
 363						  ICE_FLTR_TX);
 364			if (status) {
 365				netdev_err(netdev, "Error setting default VSI %i tx rule\n",
 366					   vsi->vsi_num);
 367				vsi->current_netdev_flags &= ~IFF_PROMISC;
 368				err = -EIO;
 369				goto out_promisc;
 370			}
 371			/* Apply RX filter rule to get traffic from wire */
 372			status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, true,
 373						  ICE_FLTR_RX);
 374			if (status) {
 375				netdev_err(netdev, "Error setting default VSI %i rx rule\n",
 376					   vsi->vsi_num);
 377				vsi->current_netdev_flags &= ~IFF_PROMISC;
 378				err = -EIO;
 379				goto out_promisc;
 380			}
 381		} else {
 382			/* Clear TX filter rule to stop traffic from VMs */
 383			status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, false,
 384						  ICE_FLTR_TX);
 385			if (status) {
 386				netdev_err(netdev, "Error clearing default VSI %i tx rule\n",
 387					   vsi->vsi_num);
 388				vsi->current_netdev_flags |= IFF_PROMISC;
 389				err = -EIO;
 390				goto out_promisc;
 391			}
 392			/* Clear filter RX to remove traffic from wire */
 393			status = ice_cfg_dflt_vsi(hw, vsi->vsi_num, false,
 394						  ICE_FLTR_RX);
 395			if (status) {
 396				netdev_err(netdev, "Error clearing default VSI %i rx rule\n",
 397					   vsi->vsi_num);
 398				vsi->current_netdev_flags |= IFF_PROMISC;
 399				err = -EIO;
 400				goto out_promisc;
 401			}
 402		}
 403	}
 404	goto exit;
 405
 406out_promisc:
 407	set_bit(ICE_VSI_FLAG_PROMISC_CHANGED, vsi->flags);
 408	goto exit;
 409out:
 410	/* if something went wrong then set the changed flag so we try again */
 411	set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
 412	set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
 413exit:
 414	clear_bit(__ICE_CFG_BUSY, vsi->state);
 415	return err;
 416}
 417
 418/**
 419 * ice_sync_fltr_subtask - Sync the VSI filter list with HW
 420 * @pf: board private structure
 421 */
 422static void ice_sync_fltr_subtask(struct ice_pf *pf)
 423{
 424	int v;
 425
 426	if (!pf || !(test_bit(ICE_FLAG_FLTR_SYNC, pf->flags)))
 427		return;
 428
 429	clear_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
 430
 431	for (v = 0; v < pf->num_alloc_vsi; v++)
 432		if (pf->vsi[v] && ice_vsi_fltr_changed(pf->vsi[v]) &&
 433		    ice_vsi_sync_fltr(pf->vsi[v])) {
 434			/* come back and try again later */
 435			set_bit(ICE_FLAG_FLTR_SYNC, pf->flags);
 436			break;
 437		}
 438}
 439
 440/**
 441 * ice_is_reset_recovery_pending - schedule a reset
 442 * @state: pf state field
 443 */
 444static bool ice_is_reset_recovery_pending(unsigned long int *state)
 445{
 446	return test_bit(__ICE_RESET_RECOVERY_PENDING, state);
 447}
 448
 449/**
 450 * ice_prepare_for_reset - prep for the core to reset
 451 * @pf: board private structure
 452 *
 453 * Inform or close all dependent features in prep for reset.
 454 */
 455static void
 456ice_prepare_for_reset(struct ice_pf *pf)
 457{
 458	struct ice_hw *hw = &pf->hw;
 459	u32 v;
 460
 461	ice_for_each_vsi(pf, v)
 462		if (pf->vsi[v])
 463			ice_remove_vsi_fltr(hw, pf->vsi[v]->vsi_num);
 464
 465	dev_dbg(&pf->pdev->dev, "Tearing down internal switch for reset\n");
 466
 467	/* disable the VSIs and their queues that are not already DOWN */
 468	/* pf_dis_all_vsi modifies netdev structures -rtnl_lock needed */
 469	ice_pf_dis_all_vsi(pf);
 470
 471	ice_for_each_vsi(pf, v)
 472		if (pf->vsi[v])
 473			pf->vsi[v]->vsi_num = 0;
 474
 475	ice_shutdown_all_ctrlq(hw);
 476}
 477
 478/**
 479 * ice_do_reset - Initiate one of many types of resets
 480 * @pf: board private structure
 481 * @reset_type: reset type requested
 482 * before this function was called.
 483 */
 484static void ice_do_reset(struct ice_pf *pf, enum ice_reset_req reset_type)
 485{
 486	struct device *dev = &pf->pdev->dev;
 487	struct ice_hw *hw = &pf->hw;
 488
 489	dev_dbg(dev, "reset_type 0x%x requested\n", reset_type);
 490	WARN_ON(in_interrupt());
 491
 492	/* PFR is a bit of a special case because it doesn't result in an OICR
 493	 * interrupt. So for PFR, we prepare for reset, issue the reset and
 494	 * rebuild sequentially.
 495	 */
 496	if (reset_type == ICE_RESET_PFR) {
 497		set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
 498		ice_prepare_for_reset(pf);
 499	}
 500
 501	/* trigger the reset */
 502	if (ice_reset(hw, reset_type)) {
 503		dev_err(dev, "reset %d failed\n", reset_type);
 504		set_bit(__ICE_RESET_FAILED, pf->state);
 505		clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
 506		return;
 507	}
 508
 509	if (reset_type == ICE_RESET_PFR) {
 510		pf->pfr_count++;
 511		ice_rebuild(pf);
 512		clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
 513	}
 514}
 515
 516/**
 517 * ice_reset_subtask - Set up for resetting the device and driver
 518 * @pf: board private structure
 519 */
 520static void ice_reset_subtask(struct ice_pf *pf)
 521{
 522	enum ice_reset_req reset_type;
 523
 524	rtnl_lock();
 525
 526	/* When a CORER/GLOBR/EMPR is about to happen, the hardware triggers an
 527	 * OICR interrupt. The OICR handler (ice_misc_intr) determines what
 528	 * type of reset happened and sets __ICE_RESET_RECOVERY_PENDING bit in
 529	 * pf->state. So if reset/recovery is pending (as indicated by this bit)
 530	 * we do a rebuild and return.
 531	 */
 532	if (ice_is_reset_recovery_pending(pf->state)) {
 533		clear_bit(__ICE_GLOBR_RECV, pf->state);
 534		clear_bit(__ICE_CORER_RECV, pf->state);
 535		ice_prepare_for_reset(pf);
 536
 537		/* make sure we are ready to rebuild */
 538		if (ice_check_reset(&pf->hw))
 539			set_bit(__ICE_RESET_FAILED, pf->state);
 540		else
 541			ice_rebuild(pf);
 542		clear_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
 543		goto unlock;
 544	}
 545
 546	/* No pending resets to finish processing. Check for new resets */
 547	if (test_and_clear_bit(__ICE_GLOBR_REQ, pf->state))
 548		reset_type = ICE_RESET_GLOBR;
 549	else if (test_and_clear_bit(__ICE_CORER_REQ, pf->state))
 550		reset_type = ICE_RESET_CORER;
 551	else if (test_and_clear_bit(__ICE_PFR_REQ, pf->state))
 552		reset_type = ICE_RESET_PFR;
 553	else
 554		goto unlock;
 555
 556	/* reset if not already down or resetting */
 557	if (!test_bit(__ICE_DOWN, pf->state) &&
 558	    !test_bit(__ICE_CFG_BUSY, pf->state)) {
 559		ice_do_reset(pf, reset_type);
 560	}
 561
 562unlock:
 563	rtnl_unlock();
 564}
 565
 566/**
 567 * ice_watchdog_subtask - periodic tasks not using event driven scheduling
 568 * @pf: board private structure
 569 */
 570static void ice_watchdog_subtask(struct ice_pf *pf)
 571{
 572	int i;
 573
 574	/* if interface is down do nothing */
 575	if (test_bit(__ICE_DOWN, pf->state) ||
 576	    test_bit(__ICE_CFG_BUSY, pf->state))
 577		return;
 578
 579	/* make sure we don't do these things too often */
 580	if (time_before(jiffies,
 581			pf->serv_tmr_prev + pf->serv_tmr_period))
 582		return;
 583
 584	pf->serv_tmr_prev = jiffies;
 585
 586	/* Update the stats for active netdevs so the network stack
 587	 * can look at updated numbers whenever it cares to
 588	 */
 589	ice_update_pf_stats(pf);
 590	for (i = 0; i < pf->num_alloc_vsi; i++)
 591		if (pf->vsi[i] && pf->vsi[i]->netdev)
 592			ice_update_vsi_stats(pf->vsi[i]);
 593}
 594
 595/**
 596 * ice_print_link_msg - print link up or down message
 597 * @vsi: the VSI whose link status is being queried
 598 * @isup: boolean for if the link is now up or down
 599 */
 600void ice_print_link_msg(struct ice_vsi *vsi, bool isup)
 601{
 602	const char *speed;
 603	const char *fc;
 604
 605	if (vsi->current_isup == isup)
 606		return;
 607
 608	vsi->current_isup = isup;
 609
 610	if (!isup) {
 611		netdev_info(vsi->netdev, "NIC Link is Down\n");
 612		return;
 613	}
 614
 615	switch (vsi->port_info->phy.link_info.link_speed) {
 616	case ICE_AQ_LINK_SPEED_40GB:
 617		speed = "40 G";
 618		break;
 619	case ICE_AQ_LINK_SPEED_25GB:
 620		speed = "25 G";
 621		break;
 622	case ICE_AQ_LINK_SPEED_20GB:
 623		speed = "20 G";
 624		break;
 625	case ICE_AQ_LINK_SPEED_10GB:
 626		speed = "10 G";
 627		break;
 628	case ICE_AQ_LINK_SPEED_5GB:
 629		speed = "5 G";
 630		break;
 631	case ICE_AQ_LINK_SPEED_2500MB:
 632		speed = "2.5 G";
 633		break;
 634	case ICE_AQ_LINK_SPEED_1000MB:
 635		speed = "1 G";
 636		break;
 637	case ICE_AQ_LINK_SPEED_100MB:
 638		speed = "100 M";
 639		break;
 640	default:
 641		speed = "Unknown";
 642		break;
 643	}
 644
 645	switch (vsi->port_info->fc.current_mode) {
 646	case ICE_FC_FULL:
 647		fc = "RX/TX";
 648		break;
 649	case ICE_FC_TX_PAUSE:
 650		fc = "TX";
 651		break;
 652	case ICE_FC_RX_PAUSE:
 653		fc = "RX";
 654		break;
 655	default:
 656		fc = "Unknown";
 657		break;
 658	}
 659
 660	netdev_info(vsi->netdev, "NIC Link is up %sbps, Flow Control: %s\n",
 661		    speed, fc);
 662}
 663
 664/**
 665 * ice_init_link_events - enable/initialize link events
 666 * @pi: pointer to the port_info instance
 667 *
 668 * Returns -EIO on failure, 0 on success
 669 */
 670static int ice_init_link_events(struct ice_port_info *pi)
 671{
 672	u16 mask;
 673
 674	mask = ~((u16)(ICE_AQ_LINK_EVENT_UPDOWN | ICE_AQ_LINK_EVENT_MEDIA_NA |
 675		       ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL));
 676
 677	if (ice_aq_set_event_mask(pi->hw, pi->lport, mask, NULL)) {
 678		dev_dbg(ice_hw_to_dev(pi->hw),
 679			"Failed to set link event mask for port %d\n",
 680			pi->lport);
 681		return -EIO;
 682	}
 683
 684	if (ice_aq_get_link_info(pi, true, NULL, NULL)) {
 685		dev_dbg(ice_hw_to_dev(pi->hw),
 686			"Failed to enable link events for port %d\n",
 687			pi->lport);
 688		return -EIO;
 689	}
 690
 691	return 0;
 692}
 693
 694/**
 695 * ice_vsi_link_event - update the vsi's netdev
 696 * @vsi: the vsi on which the link event occurred
 697 * @link_up: whether or not the vsi needs to be set up or down
 698 */
 699static void ice_vsi_link_event(struct ice_vsi *vsi, bool link_up)
 700{
 701	if (!vsi || test_bit(__ICE_DOWN, vsi->state))
 702		return;
 703
 704	if (vsi->type == ICE_VSI_PF) {
 705		if (!vsi->netdev) {
 706			dev_dbg(&vsi->back->pdev->dev,
 707				"vsi->netdev is not initialized!\n");
 708			return;
 709		}
 710		if (link_up) {
 711			netif_carrier_on(vsi->netdev);
 712			netif_tx_wake_all_queues(vsi->netdev);
 713		} else {
 714			netif_carrier_off(vsi->netdev);
 715			netif_tx_stop_all_queues(vsi->netdev);
 716		}
 717	}
 718}
 719
 720/**
 721 * ice_link_event - process the link event
 722 * @pf: pf that the link event is associated with
 723 * @pi: port_info for the port that the link event is associated with
 724 *
 725 * Returns -EIO if ice_get_link_status() fails
 726 * Returns 0 on success
 727 */
 728static int
 729ice_link_event(struct ice_pf *pf, struct ice_port_info *pi)
 730{
 731	u8 new_link_speed, old_link_speed;
 732	struct ice_phy_info *phy_info;
 733	bool new_link_same_as_old;
 734	bool new_link, old_link;
 735	u8 lport;
 736	u16 v;
 737
 738	phy_info = &pi->phy;
 739	phy_info->link_info_old = phy_info->link_info;
 740	/* Force ice_get_link_status() to update link info */
 741	phy_info->get_link_info = true;
 742
 743	old_link = (phy_info->link_info_old.link_info & ICE_AQ_LINK_UP);
 744	old_link_speed = phy_info->link_info_old.link_speed;
 745
 746	lport = pi->lport;
 747	if (ice_get_link_status(pi, &new_link)) {
 748		dev_dbg(&pf->pdev->dev,
 749			"Could not get link status for port %d\n", lport);
 750		return -EIO;
 751	}
 752
 753	new_link_speed = phy_info->link_info.link_speed;
 754
 755	new_link_same_as_old = (new_link == old_link &&
 756				new_link_speed == old_link_speed);
 757
 758	ice_for_each_vsi(pf, v) {
 759		struct ice_vsi *vsi = pf->vsi[v];
 760
 761		if (!vsi || !vsi->port_info)
 762			continue;
 763
 764		if (new_link_same_as_old &&
 765		    (test_bit(__ICE_DOWN, vsi->state) ||
 766		    new_link == netif_carrier_ok(vsi->netdev)))
 767			continue;
 768
 769		if (vsi->port_info->lport == lport) {
 770			ice_print_link_msg(vsi, new_link);
 771			ice_vsi_link_event(vsi, new_link);
 772		}
 773	}
 774
 775	return 0;
 776}
 777
 778/**
 779 * ice_handle_link_event - handle link event via ARQ
 780 * @pf: pf that the link event is associated with
 781 *
 782 * Return -EINVAL if port_info is null
 783 * Return status on succes
 784 */
 785static int ice_handle_link_event(struct ice_pf *pf)
 786{
 787	struct ice_port_info *port_info;
 788	int status;
 789
 790	port_info = pf->hw.port_info;
 791	if (!port_info)
 792		return -EINVAL;
 793
 794	status = ice_link_event(pf, port_info);
 795	if (status)
 796		dev_dbg(&pf->pdev->dev,
 797			"Could not process link event, error %d\n", status);
 798
 799	return status;
 800}
 801
 802/**
 803 * __ice_clean_ctrlq - helper function to clean controlq rings
 804 * @pf: ptr to struct ice_pf
 805 * @q_type: specific Control queue type
 806 */
 807static int __ice_clean_ctrlq(struct ice_pf *pf, enum ice_ctl_q q_type)
 808{
 809	struct ice_rq_event_info event;
 810	struct ice_hw *hw = &pf->hw;
 811	struct ice_ctl_q_info *cq;
 812	u16 pending, i = 0;
 813	const char *qtype;
 814	u32 oldval, val;
 815
 816	/* Do not clean control queue if/when PF reset fails */
 817	if (test_bit(__ICE_RESET_FAILED, pf->state))
 818		return 0;
 819
 820	switch (q_type) {
 821	case ICE_CTL_Q_ADMIN:
 822		cq = &hw->adminq;
 823		qtype = "Admin";
 824		break;
 825	default:
 826		dev_warn(&pf->pdev->dev, "Unknown control queue type 0x%x\n",
 827			 q_type);
 828		return 0;
 829	}
 830
 831	/* check for error indications - PF_xx_AxQLEN register layout for
 832	 * FW/MBX/SB are identical so just use defines for PF_FW_AxQLEN.
 833	 */
 834	val = rd32(hw, cq->rq.len);
 835	if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
 836		   PF_FW_ARQLEN_ARQCRIT_M)) {
 837		oldval = val;
 838		if (val & PF_FW_ARQLEN_ARQVFE_M)
 839			dev_dbg(&pf->pdev->dev,
 840				"%s Receive Queue VF Error detected\n", qtype);
 841		if (val & PF_FW_ARQLEN_ARQOVFL_M) {
 842			dev_dbg(&pf->pdev->dev,
 843				"%s Receive Queue Overflow Error detected\n",
 844				qtype);
 845		}
 846		if (val & PF_FW_ARQLEN_ARQCRIT_M)
 847			dev_dbg(&pf->pdev->dev,
 848				"%s Receive Queue Critical Error detected\n",
 849				qtype);
 850		val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
 851			 PF_FW_ARQLEN_ARQCRIT_M);
 852		if (oldval != val)
 853			wr32(hw, cq->rq.len, val);
 854	}
 855
 856	val = rd32(hw, cq->sq.len);
 857	if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
 858		   PF_FW_ATQLEN_ATQCRIT_M)) {
 859		oldval = val;
 860		if (val & PF_FW_ATQLEN_ATQVFE_M)
 861			dev_dbg(&pf->pdev->dev,
 862				"%s Send Queue VF Error detected\n", qtype);
 863		if (val & PF_FW_ATQLEN_ATQOVFL_M) {
 864			dev_dbg(&pf->pdev->dev,
 865				"%s Send Queue Overflow Error detected\n",
 866				qtype);
 867		}
 868		if (val & PF_FW_ATQLEN_ATQCRIT_M)
 869			dev_dbg(&pf->pdev->dev,
 870				"%s Send Queue Critical Error detected\n",
 871				qtype);
 872		val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
 873			 PF_FW_ATQLEN_ATQCRIT_M);
 874		if (oldval != val)
 875			wr32(hw, cq->sq.len, val);
 876	}
 877
 878	event.buf_len = cq->rq_buf_size;
 879	event.msg_buf = devm_kzalloc(&pf->pdev->dev, event.buf_len,
 880				     GFP_KERNEL);
 881	if (!event.msg_buf)
 882		return 0;
 883
 884	do {
 885		enum ice_status ret;
 886		u16 opcode;
 887
 888		ret = ice_clean_rq_elem(hw, cq, &event, &pending);
 889		if (ret == ICE_ERR_AQ_NO_WORK)
 890			break;
 891		if (ret) {
 892			dev_err(&pf->pdev->dev,
 893				"%s Receive Queue event error %d\n", qtype,
 894				ret);
 895			break;
 896		}
 897
 898		opcode = le16_to_cpu(event.desc.opcode);
 899
 900		switch (opcode) {
 901		case ice_aqc_opc_get_link_status:
 902			if (ice_handle_link_event(pf))
 903				dev_err(&pf->pdev->dev,
 904					"Could not handle link event");
 905			break;
 906		default:
 907			dev_dbg(&pf->pdev->dev,
 908				"%s Receive Queue unknown event 0x%04x ignored\n",
 909				qtype, opcode);
 910			break;
 911		}
 912	} while (pending && (i++ < ICE_DFLT_IRQ_WORK));
 913
 914	devm_kfree(&pf->pdev->dev, event.msg_buf);
 915
 916	return pending && (i == ICE_DFLT_IRQ_WORK);
 917}
 918
 919/**
 920 * ice_clean_adminq_subtask - clean the AdminQ rings
 921 * @pf: board private structure
 922 */
 923static void ice_clean_adminq_subtask(struct ice_pf *pf)
 924{
 925	struct ice_hw *hw = &pf->hw;
 926	u32 val;
 927
 928	if (!test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state))
 929		return;
 930
 931	if (__ice_clean_ctrlq(pf, ICE_CTL_Q_ADMIN))
 932		return;
 933
 934	clear_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state);
 935
 936	/* re-enable Admin queue interrupt causes */
 937	val = rd32(hw, PFINT_FW_CTL);
 938	wr32(hw, PFINT_FW_CTL, (val | PFINT_FW_CTL_CAUSE_ENA_M));
 939
 940	ice_flush(hw);
 941}
 942
 943/**
 944 * ice_service_task_schedule - schedule the service task to wake up
 945 * @pf: board private structure
 946 *
 947 * If not already scheduled, this puts the task into the work queue.
 948 */
 949static void ice_service_task_schedule(struct ice_pf *pf)
 950{
 951	if (!test_bit(__ICE_DOWN, pf->state) &&
 952	    !test_and_set_bit(__ICE_SERVICE_SCHED, pf->state))
 953		queue_work(ice_wq, &pf->serv_task);
 954}
 955
 956/**
 957 * ice_service_task_complete - finish up the service task
 958 * @pf: board private structure
 959 */
 960static void ice_service_task_complete(struct ice_pf *pf)
 961{
 962	WARN_ON(!test_bit(__ICE_SERVICE_SCHED, pf->state));
 963
 964	/* force memory (pf->state) to sync before next service task */
 965	smp_mb__before_atomic();
 966	clear_bit(__ICE_SERVICE_SCHED, pf->state);
 967}
 968
 969/**
 970 * ice_service_timer - timer callback to schedule service task
 971 * @t: pointer to timer_list
 972 */
 973static void ice_service_timer(struct timer_list *t)
 974{
 975	struct ice_pf *pf = from_timer(pf, t, serv_tmr);
 976
 977	mod_timer(&pf->serv_tmr, round_jiffies(pf->serv_tmr_period + jiffies));
 978	ice_service_task_schedule(pf);
 979}
 980
 981/**
 982 * ice_service_task - manage and run subtasks
 983 * @work: pointer to work_struct contained by the PF struct
 984 */
 985static void ice_service_task(struct work_struct *work)
 986{
 987	struct ice_pf *pf = container_of(work, struct ice_pf, serv_task);
 988	unsigned long start_time = jiffies;
 989
 990	/* subtasks */
 991
 992	/* process reset requests first */
 993	ice_reset_subtask(pf);
 994
 995	/* bail if a reset/recovery cycle is pending */
 996	if (ice_is_reset_recovery_pending(pf->state) ||
 997	    test_bit(__ICE_SUSPENDED, pf->state)) {
 998		ice_service_task_complete(pf);
 999		return;
1000	}
1001
1002	ice_sync_fltr_subtask(pf);
1003	ice_watchdog_subtask(pf);
1004	ice_clean_adminq_subtask(pf);
1005
1006	/* Clear __ICE_SERVICE_SCHED flag to allow scheduling next event */
1007	ice_service_task_complete(pf);
1008
1009	/* If the tasks have taken longer than one service timer period
1010	 * or there is more work to be done, reset the service timer to
1011	 * schedule the service task now.
1012	 */
1013	if (time_after(jiffies, (start_time + pf->serv_tmr_period)) ||
1014	    test_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state))
1015		mod_timer(&pf->serv_tmr, jiffies);
1016}
1017
1018/**
1019 * ice_set_ctrlq_len - helper function to set controlq length
1020 * @hw: pointer to the hw instance
1021 */
1022static void ice_set_ctrlq_len(struct ice_hw *hw)
1023{
1024	hw->adminq.num_rq_entries = ICE_AQ_LEN;
1025	hw->adminq.num_sq_entries = ICE_AQ_LEN;
1026	hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
1027	hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;
1028}
1029
1030/**
1031 * ice_irq_affinity_notify - Callback for affinity changes
1032 * @notify: context as to what irq was changed
1033 * @mask: the new affinity mask
1034 *
1035 * This is a callback function used by the irq_set_affinity_notifier function
1036 * so that we may register to receive changes to the irq affinity masks.
1037 */
1038static void ice_irq_affinity_notify(struct irq_affinity_notify *notify,
1039				    const cpumask_t *mask)
1040{
1041	struct ice_q_vector *q_vector =
1042		container_of(notify, struct ice_q_vector, affinity_notify);
1043
1044	cpumask_copy(&q_vector->affinity_mask, mask);
1045}
1046
1047/**
1048 * ice_irq_affinity_release - Callback for affinity notifier release
1049 * @ref: internal core kernel usage
1050 *
1051 * This is a callback function used by the irq_set_affinity_notifier function
1052 * to inform the current notification subscriber that they will no longer
1053 * receive notifications.
1054 */
1055static void ice_irq_affinity_release(struct kref __always_unused *ref) {}
1056
1057/**
1058 * ice_vsi_dis_irq - Mask off queue interrupt generation on the VSI
1059 * @vsi: the VSI being un-configured
1060 */
1061static void ice_vsi_dis_irq(struct ice_vsi *vsi)
1062{
1063	struct ice_pf *pf = vsi->back;
1064	struct ice_hw *hw = &pf->hw;
1065	int base = vsi->base_vector;
1066	u32 val;
1067	int i;
1068
1069	/* disable interrupt causation from each queue */
1070	if (vsi->tx_rings) {
1071		ice_for_each_txq(vsi, i) {
1072			if (vsi->tx_rings[i]) {
1073				u16 reg;
1074
1075				reg = vsi->tx_rings[i]->reg_idx;
1076				val = rd32(hw, QINT_TQCTL(reg));
1077				val &= ~QINT_TQCTL_CAUSE_ENA_M;
1078				wr32(hw, QINT_TQCTL(reg), val);
1079			}
1080		}
1081	}
1082
1083	if (vsi->rx_rings) {
1084		ice_for_each_rxq(vsi, i) {
1085			if (vsi->rx_rings[i]) {
1086				u16 reg;
1087
1088				reg = vsi->rx_rings[i]->reg_idx;
1089				val = rd32(hw, QINT_RQCTL(reg));
1090				val &= ~QINT_RQCTL_CAUSE_ENA_M;
1091				wr32(hw, QINT_RQCTL(reg), val);
1092			}
1093		}
1094	}
1095
1096	/* disable each interrupt */
1097	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1098		for (i = vsi->base_vector;
1099		     i < (vsi->num_q_vectors + vsi->base_vector); i++)
1100			wr32(hw, GLINT_DYN_CTL(i), 0);
1101
1102		ice_flush(hw);
1103		for (i = 0; i < vsi->num_q_vectors; i++)
1104			synchronize_irq(pf->msix_entries[i + base].vector);
1105	}
1106}
1107
1108/**
1109 * ice_vsi_ena_irq - Enable IRQ for the given VSI
1110 * @vsi: the VSI being configured
1111 */
1112static int ice_vsi_ena_irq(struct ice_vsi *vsi)
1113{
1114	struct ice_pf *pf = vsi->back;
1115	struct ice_hw *hw = &pf->hw;
1116
1117	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1118		int i;
1119
1120		for (i = 0; i < vsi->num_q_vectors; i++)
1121			ice_irq_dynamic_ena(hw, vsi, vsi->q_vectors[i]);
1122	}
1123
1124	ice_flush(hw);
1125	return 0;
1126}
1127
1128/**
1129 * ice_vsi_delete - delete a VSI from the switch
1130 * @vsi: pointer to VSI being removed
1131 */
1132static void ice_vsi_delete(struct ice_vsi *vsi)
1133{
1134	struct ice_pf *pf = vsi->back;
1135	struct ice_vsi_ctx ctxt;
1136	enum ice_status status;
1137
1138	ctxt.vsi_num = vsi->vsi_num;
1139
1140	memcpy(&ctxt.info, &vsi->info, sizeof(struct ice_aqc_vsi_props));
1141
1142	status = ice_aq_free_vsi(&pf->hw, &ctxt, false, NULL);
1143	if (status)
1144		dev_err(&pf->pdev->dev, "Failed to delete VSI %i in FW\n",
1145			vsi->vsi_num);
1146}
1147
1148/**
1149 * ice_vsi_req_irq_msix - get MSI-X vectors from the OS for the VSI
1150 * @vsi: the VSI being configured
1151 * @basename: name for the vector
1152 */
1153static int ice_vsi_req_irq_msix(struct ice_vsi *vsi, char *basename)
1154{
1155	int q_vectors = vsi->num_q_vectors;
1156	struct ice_pf *pf = vsi->back;
1157	int base = vsi->base_vector;
1158	int rx_int_idx = 0;
1159	int tx_int_idx = 0;
1160	int vector, err;
1161	int irq_num;
1162
1163	for (vector = 0; vector < q_vectors; vector++) {
1164		struct ice_q_vector *q_vector = vsi->q_vectors[vector];
1165
1166		irq_num = pf->msix_entries[base + vector].vector;
1167
1168		if (q_vector->tx.ring && q_vector->rx.ring) {
1169			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1170				 "%s-%s-%d", basename, "TxRx", rx_int_idx++);
1171			tx_int_idx++;
1172		} else if (q_vector->rx.ring) {
1173			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1174				 "%s-%s-%d", basename, "rx", rx_int_idx++);
1175		} else if (q_vector->tx.ring) {
1176			snprintf(q_vector->name, sizeof(q_vector->name) - 1,
1177				 "%s-%s-%d", basename, "tx", tx_int_idx++);
1178		} else {
1179			/* skip this unused q_vector */
1180			continue;
1181		}
1182		err = devm_request_irq(&pf->pdev->dev,
1183				       pf->msix_entries[base + vector].vector,
1184				       vsi->irq_handler, 0, q_vector->name,
1185				       q_vector);
1186		if (err) {
1187			netdev_err(vsi->netdev,
1188				   "MSIX request_irq failed, error: %d\n", err);
1189			goto free_q_irqs;
1190		}
1191
1192		/* register for affinity change notifications */
1193		q_vector->affinity_notify.notify = ice_irq_affinity_notify;
1194		q_vector->affinity_notify.release = ice_irq_affinity_release;
1195		irq_set_affinity_notifier(irq_num, &q_vector->affinity_notify);
1196
1197		/* assign the mask for this irq */
1198		irq_set_affinity_hint(irq_num, &q_vector->affinity_mask);
1199	}
1200
1201	vsi->irqs_ready = true;
1202	return 0;
1203
1204free_q_irqs:
1205	while (vector) {
1206		vector--;
1207		irq_num = pf->msix_entries[base + vector].vector,
1208		irq_set_affinity_notifier(irq_num, NULL);
1209		irq_set_affinity_hint(irq_num, NULL);
1210		devm_free_irq(&pf->pdev->dev, irq_num, &vsi->q_vectors[vector]);
1211	}
1212	return err;
1213}
1214
1215/**
1216 * ice_vsi_set_rss_params - Setup RSS capabilities per VSI type
1217 * @vsi: the VSI being configured
1218 */
1219static void ice_vsi_set_rss_params(struct ice_vsi *vsi)
1220{
1221	struct ice_hw_common_caps *cap;
1222	struct ice_pf *pf = vsi->back;
1223
1224	if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags)) {
1225		vsi->rss_size = 1;
1226		return;
1227	}
1228
1229	cap = &pf->hw.func_caps.common_cap;
1230	switch (vsi->type) {
1231	case ICE_VSI_PF:
1232		/* PF VSI will inherit RSS instance of PF */
1233		vsi->rss_table_size = cap->rss_table_size;
1234		vsi->rss_size = min_t(int, num_online_cpus(),
1235				      BIT(cap->rss_table_entry_width));
1236		vsi->rss_lut_type = ICE_AQC_GSET_RSS_LUT_TABLE_TYPE_PF;
1237		break;
1238	default:
1239		dev_warn(&pf->pdev->dev, "Unknown VSI type %d\n", vsi->type);
1240		break;
1241	}
1242}
1243
1244/**
1245 * ice_vsi_setup_q_map - Setup a VSI queue map
1246 * @vsi: the VSI being configured
1247 * @ctxt: VSI context structure
1248 */
1249static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt)
1250{
1251	u16 offset = 0, qmap = 0, numq_tc;
1252	u16 pow = 0, max_rss = 0, qcount;
1253	u16 qcount_tx = vsi->alloc_txq;
1254	u16 qcount_rx = vsi->alloc_rxq;
1255	bool ena_tc0 = false;
1256	int i;
1257
1258	/* at least TC0 should be enabled by default */
1259	if (vsi->tc_cfg.numtc) {
1260		if (!(vsi->tc_cfg.ena_tc & BIT(0)))
1261			ena_tc0 =  true;
1262	} else {
1263		ena_tc0 =  true;
1264	}
1265
1266	if (ena_tc0) {
1267		vsi->tc_cfg.numtc++;
1268		vsi->tc_cfg.ena_tc |= 1;
1269	}
1270
1271	numq_tc = qcount_rx / vsi->tc_cfg.numtc;
1272
1273	/* TC mapping is a function of the number of Rx queues assigned to the
1274	 * VSI for each traffic class and the offset of these queues.
1275	 * The first 10 bits are for queue offset for TC0, next 4 bits for no:of
1276	 * queues allocated to TC0. No:of queues is a power-of-2.
1277	 *
1278	 * If TC is not enabled, the queue offset is set to 0, and allocate one
1279	 * queue, this way, traffic for the given TC will be sent to the default
1280	 * queue.
1281	 *
1282	 * Setup number and offset of Rx queues for all TCs for the VSI
1283	 */
1284
1285	/* qcount will change if RSS is enabled */
1286	if (test_bit(ICE_FLAG_RSS_ENA, vsi->back->flags)) {
1287		if (vsi->type == ICE_VSI_PF)
1288			max_rss = ICE_MAX_LG_RSS_QS;
1289		else
1290			max_rss = ICE_MAX_SMALL_RSS_QS;
1291
1292		qcount = min_t(int, numq_tc, max_rss);
1293		qcount = min_t(int, qcount, vsi->rss_size);
1294	} else {
1295		qcount = numq_tc;
1296	}
1297
1298	/* find higher power-of-2 of qcount */
1299	pow = ilog2(qcount);
1300
1301	if (!is_power_of_2(qcount))
1302		pow++;
1303
1304	for (i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
1305		if (!(vsi->tc_cfg.ena_tc & BIT(i))) {
1306			/* TC is not enabled */
1307			vsi->tc_cfg.tc_info[i].qoffset = 0;
1308			vsi->tc_cfg.tc_info[i].qcount = 1;
1309			ctxt->info.tc_mapping[i] = 0;
1310			continue;
1311		}
1312
1313		/* TC is enabled */
1314		vsi->tc_cfg.tc_info[i].qoffset = offset;
1315		vsi->tc_cfg.tc_info[i].qcount = qcount;
1316
1317		qmap = ((offset << ICE_AQ_VSI_TC_Q_OFFSET_S) &
1318			ICE_AQ_VSI_TC_Q_OFFSET_M) |
1319			((pow << ICE_AQ_VSI_TC_Q_NUM_S) &
1320			 ICE_AQ_VSI_TC_Q_NUM_M);
1321		offset += qcount;
1322		ctxt->info.tc_mapping[i] = cpu_to_le16(qmap);
1323	}
1324
1325	vsi->num_txq = qcount_tx;
1326	vsi->num_rxq = offset;
1327
1328	/* Rx queue mapping */
1329	ctxt->info.mapping_flags |= cpu_to_le16(ICE_AQ_VSI_Q_MAP_CONTIG);
1330	/* q_mapping buffer holds the info for the first queue allocated for
1331	 * this VSI in the PF space and also the number of queues associated
1332	 * with this VSI.
1333	 */
1334	ctxt->info.q_mapping[0] = cpu_to_le16(vsi->rxq_map[0]);
1335	ctxt->info.q_mapping[1] = cpu_to_le16(vsi->num_rxq);
1336}
1337
1338/**
1339 * ice_set_dflt_vsi_ctx - Set default VSI context before adding a VSI
1340 * @ctxt: the VSI context being set
1341 *
1342 * This initializes a default VSI context for all sections except the Queues.
1343 */
1344static void ice_set_dflt_vsi_ctx(struct ice_vsi_ctx *ctxt)
1345{
1346	u32 table = 0;
1347
1348	memset(&ctxt->info, 0, sizeof(ctxt->info));
1349	/* VSI's should be allocated from shared pool */
1350	ctxt->alloc_from_pool = true;
1351	/* Src pruning enabled by default */
1352	ctxt->info.sw_flags = ICE_AQ_VSI_SW_FLAG_SRC_PRUNE;
1353	/* Traffic from VSI can be sent to LAN */
1354	ctxt->info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA;
1355	/* Allow all packets untagged/tagged */
1356	ctxt->info.port_vlan_flags = ((ICE_AQ_VSI_PVLAN_MODE_ALL &
1357				       ICE_AQ_VSI_PVLAN_MODE_M) >>
1358				      ICE_AQ_VSI_PVLAN_MODE_S);
1359	/* Show VLAN/UP from packets in Rx descriptors */
1360	ctxt->info.port_vlan_flags |= ((ICE_AQ_VSI_PVLAN_EMOD_STR_BOTH &
1361					ICE_AQ_VSI_PVLAN_EMOD_M) >>
1362				       ICE_AQ_VSI_PVLAN_EMOD_S);
1363	/* Have 1:1 UP mapping for both ingress/egress tables */
1364	table |= ICE_UP_TABLE_TRANSLATE(0, 0);
1365	table |= ICE_UP_TABLE_TRANSLATE(1, 1);
1366	table |= ICE_UP_TABLE_TRANSLATE(2, 2);
1367	table |= ICE_UP_TABLE_TRANSLATE(3, 3);
1368	table |= ICE_UP_TABLE_TRANSLATE(4, 4);
1369	table |= ICE_UP_TABLE_TRANSLATE(5, 5);
1370	table |= ICE_UP_TABLE_TRANSLATE(6, 6);
1371	table |= ICE_UP_TABLE_TRANSLATE(7, 7);
1372	ctxt->info.ingress_table = cpu_to_le32(table);
1373	ctxt->info.egress_table = cpu_to_le32(table);
1374	/* Have 1:1 UP mapping for outer to inner UP table */
1375	ctxt->info.outer_up_table = cpu_to_le32(table);
1376	/* No Outer tag support outer_tag_flags remains to zero */
1377}
1378
1379/**
1380 * ice_set_rss_vsi_ctx - Set RSS VSI context before adding a VSI
1381 * @ctxt: the VSI context being set
1382 * @vsi: the VSI being configured
1383 */
1384static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctxt, struct ice_vsi *vsi)
1385{
1386	u8 lut_type, hash_type;
1387
1388	switch (vsi->type) {
1389	case ICE_VSI_PF:
1390		/* PF VSI will inherit RSS instance of PF */
1391		lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF;
1392		hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
1393		break;
1394	default:
1395		dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
1396			 vsi->type);
1397		return;
1398	}
1399
1400	ctxt->info.q_opt_rss = ((lut_type << ICE_AQ_VSI_Q_OPT_RSS_LUT_S) &
1401				ICE_AQ_VSI_Q_OPT_RSS_LUT_M) |
1402				((hash_type << ICE_AQ_VSI_Q_OPT_RSS_HASH_S) &
1403				 ICE_AQ_VSI_Q_OPT_RSS_HASH_M);
1404}
1405
1406/**
1407 * ice_vsi_add - Create a new VSI or fetch preallocated VSI
1408 * @vsi: the VSI being configured
1409 *
1410 * This initializes a VSI context depending on the VSI type to be added and
1411 * passes it down to the add_vsi aq command to create a new VSI.
1412 */
1413static int ice_vsi_add(struct ice_vsi *vsi)
1414{
1415	struct ice_vsi_ctx ctxt = { 0 };
1416	struct ice_pf *pf = vsi->back;
1417	struct ice_hw *hw = &pf->hw;
1418	int ret = 0;
1419
1420	switch (vsi->type) {
1421	case ICE_VSI_PF:
1422		ctxt.flags = ICE_AQ_VSI_TYPE_PF;
1423		break;
1424	default:
1425		return -ENODEV;
1426	}
1427
1428	ice_set_dflt_vsi_ctx(&ctxt);
1429	/* if the switch is in VEB mode, allow VSI loopback */
1430	if (vsi->vsw->bridge_mode == BRIDGE_MODE_VEB)
1431		ctxt.info.sw_flags |= ICE_AQ_VSI_SW_FLAG_ALLOW_LB;
1432
1433	/* Set LUT type and HASH type if RSS is enabled */
1434	if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
1435		ice_set_rss_vsi_ctx(&ctxt, vsi);
1436
1437	ctxt.info.sw_id = vsi->port_info->sw_id;
1438	ice_vsi_setup_q_map(vsi, &ctxt);
1439
1440	ret = ice_aq_add_vsi(hw, &ctxt, NULL);
1441	if (ret) {
1442		dev_err(&vsi->back->pdev->dev,
1443			"Add VSI AQ call failed, err %d\n", ret);
1444		return -EIO;
1445	}
1446	vsi->info = ctxt.info;
1447	vsi->vsi_num = ctxt.vsi_num;
1448
1449	return ret;
1450}
1451
1452/**
1453 * ice_vsi_release_msix - Clear the queue to Interrupt mapping in HW
1454 * @vsi: the VSI being cleaned up
1455 */
1456static void ice_vsi_release_msix(struct ice_vsi *vsi)
1457{
1458	struct ice_pf *pf = vsi->back;
1459	u16 vector = vsi->base_vector;
1460	struct ice_hw *hw = &pf->hw;
1461	u32 txq = 0;
1462	u32 rxq = 0;
1463	int i, q;
1464
1465	for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
1466		struct ice_q_vector *q_vector = vsi->q_vectors[i];
1467
1468		wr32(hw, GLINT_ITR(ICE_RX_ITR, vector), 0);
1469		wr32(hw, GLINT_ITR(ICE_TX_ITR, vector), 0);
1470		for (q = 0; q < q_vector->num_ring_tx; q++) {
1471			wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), 0);
1472			txq++;
1473		}
1474
1475		for (q = 0; q < q_vector->num_ring_rx; q++) {
1476			wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), 0);
1477			rxq++;
1478		}
1479	}
1480
1481	ice_flush(hw);
1482}
1483
1484/**
1485 * ice_vsi_clear_rings - Deallocates the Tx and Rx rings for VSI
1486 * @vsi: the VSI having rings deallocated
1487 */
1488static void ice_vsi_clear_rings(struct ice_vsi *vsi)
1489{
1490	int i;
1491
1492	if (vsi->tx_rings) {
1493		for (i = 0; i < vsi->alloc_txq; i++) {
1494			if (vsi->tx_rings[i]) {
1495				kfree_rcu(vsi->tx_rings[i], rcu);
1496				vsi->tx_rings[i] = NULL;
1497			}
1498		}
1499	}
1500	if (vsi->rx_rings) {
1501		for (i = 0; i < vsi->alloc_rxq; i++) {
1502			if (vsi->rx_rings[i]) {
1503				kfree_rcu(vsi->rx_rings[i], rcu);
1504				vsi->rx_rings[i] = NULL;
1505			}
1506		}
1507	}
1508}
1509
1510/**
1511 * ice_vsi_alloc_rings - Allocates Tx and Rx rings for the VSI
1512 * @vsi: VSI which is having rings allocated
1513 */
1514static int ice_vsi_alloc_rings(struct ice_vsi *vsi)
1515{
1516	struct ice_pf *pf = vsi->back;
1517	int i;
1518
1519	/* Allocate tx_rings */
1520	for (i = 0; i < vsi->alloc_txq; i++) {
1521		struct ice_ring *ring;
1522
1523		/* allocate with kzalloc(), free with kfree_rcu() */
1524		ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1525
1526		if (!ring)
1527			goto err_out;
1528
1529		ring->q_index = i;
1530		ring->reg_idx = vsi->txq_map[i];
1531		ring->ring_active = false;
1532		ring->vsi = vsi;
1533		ring->netdev = vsi->netdev;
1534		ring->dev = &pf->pdev->dev;
1535		ring->count = vsi->num_desc;
1536
1537		vsi->tx_rings[i] = ring;
1538	}
1539
1540	/* Allocate rx_rings */
1541	for (i = 0; i < vsi->alloc_rxq; i++) {
1542		struct ice_ring *ring;
1543
1544		/* allocate with kzalloc(), free with kfree_rcu() */
1545		ring = kzalloc(sizeof(*ring), GFP_KERNEL);
1546		if (!ring)
1547			goto err_out;
1548
1549		ring->q_index = i;
1550		ring->reg_idx = vsi->rxq_map[i];
1551		ring->ring_active = false;
1552		ring->vsi = vsi;
1553		ring->netdev = vsi->netdev;
1554		ring->dev = &pf->pdev->dev;
1555		ring->count = vsi->num_desc;
1556		vsi->rx_rings[i] = ring;
1557	}
1558
1559	return 0;
1560
1561err_out:
1562	ice_vsi_clear_rings(vsi);
1563	return -ENOMEM;
1564}
1565
1566/**
1567 * ice_vsi_free_irq - Free the irq association with the OS
1568 * @vsi: the VSI being configured
1569 */
1570static void ice_vsi_free_irq(struct ice_vsi *vsi)
1571{
1572	struct ice_pf *pf = vsi->back;
1573	int base = vsi->base_vector;
1574
1575	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
1576		int i;
1577
1578		if (!vsi->q_vectors || !vsi->irqs_ready)
1579			return;
1580
1581		vsi->irqs_ready = false;
1582		for (i = 0; i < vsi->num_q_vectors; i++) {
1583			u16 vector = i + base;
1584			int irq_num;
1585
1586			irq_num = pf->msix_entries[vector].vector;
1587
1588			/* free only the irqs that were actually requested */
1589			if (!vsi->q_vectors[i] ||
1590			    !(vsi->q_vectors[i]->num_ring_tx ||
1591			      vsi->q_vectors[i]->num_ring_rx))
1592				continue;
1593
1594			/* clear the affinity notifier in the IRQ descriptor */
1595			irq_set_affinity_notifier(irq_num, NULL);
1596
1597			/* clear the affinity_mask in the IRQ descriptor */
1598			irq_set_affinity_hint(irq_num, NULL);
1599			synchronize_irq(irq_num);
1600			devm_free_irq(&pf->pdev->dev, irq_num,
1601				      vsi->q_vectors[i]);
1602		}
1603		ice_vsi_release_msix(vsi);
1604	}
1605}
1606
1607/**
1608 * ice_vsi_cfg_msix - MSIX mode Interrupt Config in the HW
1609 * @vsi: the VSI being configured
1610 */
1611static void ice_vsi_cfg_msix(struct ice_vsi *vsi)
1612{
1613	struct ice_pf *pf = vsi->back;
1614	u16 vector = vsi->base_vector;
1615	struct ice_hw *hw = &pf->hw;
1616	u32 txq = 0, rxq = 0;
1617	int i, q, itr;
1618	u8 itr_gran;
1619
1620	for (i = 0; i < vsi->num_q_vectors; i++, vector++) {
1621		struct ice_q_vector *q_vector = vsi->q_vectors[i];
1622
1623		itr_gran = hw->itr_gran_200;
1624
1625		if (q_vector->num_ring_rx) {
1626			q_vector->rx.itr =
1627				ITR_TO_REG(vsi->rx_rings[rxq]->rx_itr_setting,
1628					   itr_gran);
1629			q_vector->rx.latency_range = ICE_LOW_LATENCY;
1630		}
1631
1632		if (q_vector->num_ring_tx) {
1633			q_vector->tx.itr =
1634				ITR_TO_REG(vsi->tx_rings[txq]->tx_itr_setting,
1635					   itr_gran);
1636			q_vector->tx.latency_range = ICE_LOW_LATENCY;
1637		}
1638		wr32(hw, GLINT_ITR(ICE_RX_ITR, vector), q_vector->rx.itr);
1639		wr32(hw, GLINT_ITR(ICE_TX_ITR, vector), q_vector->tx.itr);
1640
1641		/* Both Transmit Queue Interrupt Cause Control register
1642		 * and Receive Queue Interrupt Cause control register
1643		 * expects MSIX_INDX field to be the vector index
1644		 * within the function space and not the absolute
1645		 * vector index across PF or across device.
1646		 * For SR-IOV VF VSIs queue vector index always starts
1647		 * with 1 since first vector index(0) is used for OICR
1648		 * in VF space. Since VMDq and other PF VSIs are withtin
1649		 * the PF function space, use the vector index thats
1650		 * tracked for this PF.
1651		 */
1652		for (q = 0; q < q_vector->num_ring_tx; q++) {
1653			u32 val;
1654
1655			itr = ICE_TX_ITR;
1656			val = QINT_TQCTL_CAUSE_ENA_M |
1657			      (itr << QINT_TQCTL_ITR_INDX_S)  |
1658			      (vector << QINT_TQCTL_MSIX_INDX_S);
1659			wr32(hw, QINT_TQCTL(vsi->txq_map[txq]), val);
1660			txq++;
1661		}
1662
1663		for (q = 0; q < q_vector->num_ring_rx; q++) {
1664			u32 val;
1665
1666			itr = ICE_RX_ITR;
1667			val = QINT_RQCTL_CAUSE_ENA_M |
1668			      (itr << QINT_RQCTL_ITR_INDX_S)  |
1669			      (vector << QINT_RQCTL_MSIX_INDX_S);
1670			wr32(hw, QINT_RQCTL(vsi->rxq_map[rxq]), val);
1671			rxq++;
1672		}
1673	}
1674
1675	ice_flush(hw);
1676}
1677
1678/**
1679 * ice_ena_misc_vector - enable the non-queue interrupts
1680 * @pf: board private structure
1681 */
1682static void ice_ena_misc_vector(struct ice_pf *pf)
1683{
1684	struct ice_hw *hw = &pf->hw;
1685	u32 val;
1686
1687	/* clear things first */
1688	wr32(hw, PFINT_OICR_ENA, 0);	/* disable all */
1689	rd32(hw, PFINT_OICR);		/* read to clear */
1690
1691	val = (PFINT_OICR_HLP_RDY_M |
1692	       PFINT_OICR_CPM_RDY_M |
1693	       PFINT_OICR_ECC_ERR_M |
1694	       PFINT_OICR_MAL_DETECT_M |
1695	       PFINT_OICR_GRST_M |
1696	       PFINT_OICR_PCI_EXCEPTION_M |
1697	       PFINT_OICR_GPIO_M |
1698	       PFINT_OICR_STORM_DETECT_M |
1699	       PFINT_OICR_HMC_ERR_M);
1700
1701	wr32(hw, PFINT_OICR_ENA, val);
1702
1703	/* SW_ITR_IDX = 0, but don't change INTENA */
1704	wr32(hw, GLINT_DYN_CTL(pf->oicr_idx),
1705	     GLINT_DYN_CTL_SW_ITR_INDX_M | GLINT_DYN_CTL_INTENA_MSK_M);
1706}
1707
1708/**
1709 * ice_misc_intr - misc interrupt handler
1710 * @irq: interrupt number
1711 * @data: pointer to a q_vector
1712 */
1713static irqreturn_t ice_misc_intr(int __always_unused irq, void *data)
1714{
1715	struct ice_pf *pf = (struct ice_pf *)data;
1716	struct ice_hw *hw = &pf->hw;
1717	irqreturn_t ret = IRQ_NONE;
1718	u32 oicr, ena_mask;
1719
1720	set_bit(__ICE_ADMINQ_EVENT_PENDING, pf->state);
1721
1722	oicr = rd32(hw, PFINT_OICR);
1723	ena_mask = rd32(hw, PFINT_OICR_ENA);
1724
1725	if (oicr & PFINT_OICR_GRST_M) {
1726		u32 reset;
1727		/* we have a reset warning */
1728		ena_mask &= ~PFINT_OICR_GRST_M;
1729		reset = (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_RESET_TYPE_M) >>
1730			GLGEN_RSTAT_RESET_TYPE_S;
1731
1732		if (reset == ICE_RESET_CORER)
1733			pf->corer_count++;
1734		else if (reset == ICE_RESET_GLOBR)
1735			pf->globr_count++;
1736		else
1737			pf->empr_count++;
1738
1739		/* If a reset cycle isn't already in progress, we set a bit in
1740		 * pf->state so that the service task can start a reset/rebuild.
1741		 * We also make note of which reset happened so that peer
1742		 * devices/drivers can be informed.
1743		 */
1744		if (!test_bit(__ICE_RESET_RECOVERY_PENDING, pf->state)) {
1745			if (reset == ICE_RESET_CORER)
1746				set_bit(__ICE_CORER_RECV, pf->state);
1747			else if (reset == ICE_RESET_GLOBR)
1748				set_bit(__ICE_GLOBR_RECV, pf->state);
1749			else
1750				set_bit(__ICE_EMPR_RECV, pf->state);
1751
1752			set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
1753		}
1754	}
1755
1756	if (oicr & PFINT_OICR_HMC_ERR_M) {
1757		ena_mask &= ~PFINT_OICR_HMC_ERR_M;
1758		dev_dbg(&pf->pdev->dev,
1759			"HMC Error interrupt - info 0x%x, data 0x%x\n",
1760			rd32(hw, PFHMC_ERRORINFO),
1761			rd32(hw, PFHMC_ERRORDATA));
1762	}
1763
1764	/* Report and mask off any remaining unexpected interrupts */
1765	oicr &= ena_mask;
1766	if (oicr) {
1767		dev_dbg(&pf->pdev->dev, "unhandled interrupt oicr=0x%08x\n",
1768			oicr);
1769		/* If a critical error is pending there is no choice but to
1770		 * reset the device.
1771		 */
1772		if (oicr & (PFINT_OICR_PE_CRITERR_M |
1773			    PFINT_OICR_PCI_EXCEPTION_M |
1774			    PFINT_OICR_ECC_ERR_M)) {
1775			set_bit(__ICE_PFR_REQ, pf->state);
1776			ice_service_task_schedule(pf);
1777		}
1778		ena_mask &= ~oicr;
1779	}
1780	ret = IRQ_HANDLED;
1781
1782	/* re-enable interrupt causes that are not handled during this pass */
1783	wr32(hw, PFINT_OICR_ENA, ena_mask);
1784	if (!test_bit(__ICE_DOWN, pf->state)) {
1785		ice_service_task_schedule(pf);
1786		ice_irq_dynamic_ena(hw, NULL, NULL);
1787	}
1788
1789	return ret;
1790}
1791
1792/**
1793 * ice_vsi_map_rings_to_vectors - Map VSI rings to interrupt vectors
1794 * @vsi: the VSI being configured
1795 *
1796 * This function maps descriptor rings to the queue-specific vectors allotted
1797 * through the MSI-X enabling code. On a constrained vector budget, we map Tx
1798 * and Rx rings to the vector as "efficiently" as possible.
1799 */
1800static void ice_vsi_map_rings_to_vectors(struct ice_vsi *vsi)
1801{
1802	int q_vectors = vsi->num_q_vectors;
1803	int tx_rings_rem, rx_rings_rem;
1804	int v_id;
1805
1806	/* initially assigning remaining rings count to VSIs num queue value */
1807	tx_rings_rem = vsi->num_txq;
1808	rx_rings_rem = vsi->num_rxq;
1809
1810	for (v_id = 0; v_id < q_vectors; v_id++) {
1811		struct ice_q_vector *q_vector = vsi->q_vectors[v_id];
1812		int tx_rings_per_v, rx_rings_per_v, q_id, q_base;
1813
1814		/* Tx rings mapping to vector */
1815		tx_rings_per_v = DIV_ROUND_UP(tx_rings_rem, q_vectors - v_id);
1816		q_vector->num_ring_tx = tx_rings_per_v;
1817		q_vector->tx.ring = NULL;
1818		q_base = vsi->num_txq - tx_rings_rem;
1819
1820		for (q_id = q_base; q_id < (q_base + tx_rings_per_v); q_id++) {
1821			struct ice_ring *tx_ring = vsi->tx_rings[q_id];
1822
1823			tx_ring->q_vector = q_vector;
1824			tx_ring->next = q_vector->tx.ring;
1825			q_vector->tx.ring = tx_ring;
1826		}
1827		tx_rings_rem -= tx_rings_per_v;
1828
1829		/* Rx rings mapping to vector */
1830		rx_rings_per_v = DIV_ROUND_UP(rx_rings_rem, q_vectors - v_id);
1831		q_vector->num_ring_rx = rx_rings_per_v;
1832		q_vector->rx.ring = NULL;
1833		q_base = vsi->num_rxq - rx_rings_rem;
1834
1835		for (q_id = q_base; q_id < (q_base + rx_rings_per_v); q_id++) {
1836			struct ice_ring *rx_ring = vsi->rx_rings[q_id];
1837
1838			rx_ring->q_vector = q_vector;
1839			rx_ring->next = q_vector->rx.ring;
1840			q_vector->rx.ring = rx_ring;
1841		}
1842		rx_rings_rem -= rx_rings_per_v;
1843	}
1844}
1845
1846/**
1847 * ice_vsi_set_num_qs - Set num queues, descriptors and vectors for a VSI
1848 * @vsi: the VSI being configured
1849 *
1850 * Return 0 on success and a negative value on error
1851 */
1852static void ice_vsi_set_num_qs(struct ice_vsi *vsi)
1853{
1854	struct ice_pf *pf = vsi->back;
1855
1856	switch (vsi->type) {
1857	case ICE_VSI_PF:
1858		vsi->alloc_txq = pf->num_lan_tx;
1859		vsi->alloc_rxq = pf->num_lan_rx;
1860		vsi->num_desc = ALIGN(ICE_DFLT_NUM_DESC, ICE_REQ_DESC_MULTIPLE);
1861		vsi->num_q_vectors = max_t(int, pf->num_lan_rx, pf->num_lan_tx);
1862		break;
1863	default:
1864		dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
1865			 vsi->type);
1866		break;
1867	}
1868}
1869
1870/**
1871 * ice_vsi_alloc_arrays - Allocate queue and vector pointer arrays for the vsi
1872 * @vsi: VSI pointer
1873 * @alloc_qvectors: a bool to specify if q_vectors need to be allocated.
1874 *
1875 * On error: returns error code (negative)
1876 * On success: returns 0
1877 */
1878static int ice_vsi_alloc_arrays(struct ice_vsi *vsi, bool alloc_qvectors)
1879{
1880	struct ice_pf *pf = vsi->back;
1881
1882	/* allocate memory for both Tx and Rx ring pointers */
1883	vsi->tx_rings = devm_kcalloc(&pf->pdev->dev, vsi->alloc_txq,
1884				     sizeof(struct ice_ring *), GFP_KERNEL);
1885	if (!vsi->tx_rings)
1886		goto err_txrings;
1887
1888	vsi->rx_rings = devm_kcalloc(&pf->pdev->dev, vsi->alloc_rxq,
1889				     sizeof(struct ice_ring *), GFP_KERNEL);
1890	if (!vsi->rx_rings)
1891		goto err_rxrings;
1892
1893	if (alloc_qvectors) {
1894		/* allocate memory for q_vector pointers */
1895		vsi->q_vectors = devm_kcalloc(&pf->pdev->dev,
1896					      vsi->num_q_vectors,
1897					      sizeof(struct ice_q_vector *),
1898					      GFP_KERNEL);
1899		if (!vsi->q_vectors)
1900			goto err_vectors;
1901	}
1902
1903	return 0;
1904
1905err_vectors:
1906	devm_kfree(&pf->pdev->dev, vsi->rx_rings);
1907err_rxrings:
1908	devm_kfree(&pf->pdev->dev, vsi->tx_rings);
1909err_txrings:
1910	return -ENOMEM;
1911}
1912
1913/**
1914 * ice_msix_clean_rings - MSIX mode Interrupt Handler
1915 * @irq: interrupt number
1916 * @data: pointer to a q_vector
1917 */
1918static irqreturn_t ice_msix_clean_rings(int __always_unused irq, void *data)
1919{
1920	struct ice_q_vector *q_vector = (struct ice_q_vector *)data;
1921
1922	if (!q_vector->tx.ring && !q_vector->rx.ring)
1923		return IRQ_HANDLED;
1924
1925	napi_schedule(&q_vector->napi);
1926
1927	return IRQ_HANDLED;
1928}
1929
1930/**
1931 * ice_vsi_alloc - Allocates the next available struct vsi in the PF
1932 * @pf: board private structure
1933 * @type: type of VSI
1934 *
1935 * returns a pointer to a VSI on success, NULL on failure.
1936 */
1937static struct ice_vsi *ice_vsi_alloc(struct ice_pf *pf, enum ice_vsi_type type)
1938{
1939	struct ice_vsi *vsi = NULL;
1940
1941	/* Need to protect the allocation of the VSIs at the PF level */
1942	mutex_lock(&pf->sw_mutex);
1943
1944	/* If we have already allocated our maximum number of VSIs,
1945	 * pf->next_vsi will be ICE_NO_VSI. If not, pf->next_vsi index
1946	 * is available to be populated
1947	 */
1948	if (pf->next_vsi == ICE_NO_VSI) {
1949		dev_dbg(&pf->pdev->dev, "out of VSI slots!\n");
1950		goto unlock_pf;
1951	}
1952
1953	vsi = devm_kzalloc(&pf->pdev->dev, sizeof(*vsi), GFP_KERNEL);
1954	if (!vsi)
1955		goto unlock_pf;
1956
1957	vsi->type = type;
1958	vsi->back = pf;
1959	set_bit(__ICE_DOWN, vsi->state);
1960	vsi->idx = pf->next_vsi;
1961	vsi->work_lmt = ICE_DFLT_IRQ_WORK;
1962
1963	ice_vsi_set_num_qs(vsi);
1964
1965	switch (vsi->type) {
1966	case ICE_VSI_PF:
1967		if (ice_vsi_alloc_arrays(vsi, true))
1968			goto err_rings;
1969
1970		/* Setup default MSIX irq handler for VSI */
1971		vsi->irq_handler = ice_msix_clean_rings;
1972		break;
1973	default:
1974		dev_warn(&pf->pdev->dev, "Unknown VSI type %d\n", vsi->type);
1975		goto unlock_pf;
1976	}
1977
1978	/* fill VSI slot in the PF struct */
1979	pf->vsi[pf->next_vsi] = vsi;
1980
1981	/* prepare pf->next_vsi for next use */
1982	pf->next_vsi = ice_get_free_slot(pf->vsi, pf->num_alloc_vsi,
1983					 pf->next_vsi);
1984	goto unlock_pf;
1985
1986err_rings:
1987	devm_kfree(&pf->pdev->dev, vsi);
1988	vsi = NULL;
1989unlock_pf:
1990	mutex_unlock(&pf->sw_mutex);
1991	return vsi;
1992}
1993
1994/**
1995 * ice_free_irq_msix_misc - Unroll misc vector setup
1996 * @pf: board private structure
1997 */
1998static void ice_free_irq_msix_misc(struct ice_pf *pf)
1999{
2000	/* disable OICR interrupt */
2001	wr32(&pf->hw, PFINT_OICR_ENA, 0);
2002	ice_flush(&pf->hw);
2003
2004	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags) && pf->msix_entries) {
2005		synchronize_irq(pf->msix_entries[pf->oicr_idx].vector);
2006		devm_free_irq(&pf->pdev->dev,
2007			      pf->msix_entries[pf->oicr_idx].vector, pf);
2008	}
2009
2010	ice_free_res(pf->irq_tracker, pf->oicr_idx, ICE_RES_MISC_VEC_ID);
2011}
2012
2013/**
2014 * ice_req_irq_msix_misc - Setup the misc vector to handle non queue events
2015 * @pf: board private structure
2016 *
2017 * This sets up the handler for MSIX 0, which is used to manage the
2018 * non-queue interrupts, e.g. AdminQ and errors.  This is not used
2019 * when in MSI or Legacy interrupt mode.
2020 */
2021static int ice_req_irq_msix_misc(struct ice_pf *pf)
2022{
2023	struct ice_hw *hw = &pf->hw;
2024	int oicr_idx, err = 0;
2025	u8 itr_gran;
2026	u32 val;
2027
2028	if (!pf->int_name[0])
2029		snprintf(pf->int_name, sizeof(pf->int_name) - 1, "%s-%s:misc",
2030			 dev_driver_string(&pf->pdev->dev),
2031			 dev_name(&pf->pdev->dev));
2032
2033	/* Do not request IRQ but do enable OICR interrupt since settings are
2034	 * lost during reset. Note that this function is called only during
2035	 * rebuild path and not while reset is in progress.
2036	 */
2037	if (ice_is_reset_recovery_pending(pf->state))
2038		goto skip_req_irq;
2039
2040	/* reserve one vector in irq_tracker for misc interrupts */
2041	oicr_idx = ice_get_res(pf, pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
2042	if (oicr_idx < 0)
2043		return oicr_idx;
2044
2045	pf->oicr_idx = oicr_idx;
2046
2047	err = devm_request_irq(&pf->pdev->dev,
2048			       pf->msix_entries[pf->oicr_idx].vector,
2049			       ice_misc_intr, 0, pf->int_name, pf);
2050	if (err) {
2051		dev_err(&pf->pdev->dev,
2052			"devm_request_irq for %s failed: %d\n",
2053			pf->int_name, err);
2054		ice_free_res(pf->irq_tracker, 1, ICE_RES_MISC_VEC_ID);
2055		return err;
2056	}
2057
2058skip_req_irq:
2059	ice_ena_misc_vector(pf);
2060
2061	val = (pf->oicr_idx & PFINT_OICR_CTL_MSIX_INDX_M) |
2062	      (ICE_RX_ITR & PFINT_OICR_CTL_ITR_INDX_M) |
2063	      PFINT_OICR_CTL_CAUSE_ENA_M;
2064	wr32(hw, PFINT_OICR_CTL, val);
2065
2066	/* This enables Admin queue Interrupt causes */
2067	val = (pf->oicr_idx & PFINT_FW_CTL_MSIX_INDX_M) |
2068	      (ICE_RX_ITR & PFINT_FW_CTL_ITR_INDX_M) |
2069	      PFINT_FW_CTL_CAUSE_ENA_M;
2070	wr32(hw, PFINT_FW_CTL, val);
2071
2072	itr_gran = hw->itr_gran_200;
2073
2074	wr32(hw, GLINT_ITR(ICE_RX_ITR, pf->oicr_idx),
2075	     ITR_TO_REG(ICE_ITR_8K, itr_gran));
2076
2077	ice_flush(hw);
2078	ice_irq_dynamic_ena(hw, NULL, NULL);
2079
2080	return 0;
2081}
2082
2083/**
2084 * ice_vsi_get_qs_contig - Assign a contiguous chunk of queues to VSI
2085 * @vsi: the VSI getting queues
2086 *
2087 * Return 0 on success and a negative value on error
2088 */
2089static int ice_vsi_get_qs_contig(struct ice_vsi *vsi)
2090{
2091	struct ice_pf *pf = vsi->back;
2092	int offset, ret = 0;
2093
2094	mutex_lock(&pf->avail_q_mutex);
2095	/* look for contiguous block of queues for tx */
2096	offset = bitmap_find_next_zero_area(pf->avail_txqs, ICE_MAX_TXQS,
2097					    0, vsi->alloc_txq, 0);
2098	if (offset < ICE_MAX_TXQS) {
2099		int i;
2100
2101		bitmap_set(pf->avail_txqs, offset, vsi->alloc_txq);
2102		for (i = 0; i < vsi->alloc_txq; i++)
2103			vsi->txq_map[i] = i + offset;
2104	} else {
2105		ret = -ENOMEM;
2106		vsi->tx_mapping_mode = ICE_VSI_MAP_SCATTER;
2107	}
2108
2109	/* look for contiguous block of queues for rx */
2110	offset = bitmap_find_next_zero_area(pf->avail_rxqs, ICE_MAX_RXQS,
2111					    0, vsi->alloc_rxq, 0);
2112	if (offset < ICE_MAX_RXQS) {
2113		int i;
2114
2115		bitmap_set(pf->avail_rxqs, offset, vsi->alloc_rxq);
2116		for (i = 0; i < vsi->alloc_rxq; i++)
2117			vsi->rxq_map[i] = i + offset;
2118	} else {
2119		ret = -ENOMEM;
2120		vsi->rx_mapping_mode = ICE_VSI_MAP_SCATTER;
2121	}
2122	mutex_unlock(&pf->avail_q_mutex);
2123
2124	return ret;
2125}
2126
2127/**
2128 * ice_vsi_get_qs_scatter - Assign a scattered queues to VSI
2129 * @vsi: the VSI getting queues
2130 *
2131 * Return 0 on success and a negative value on error
2132 */
2133static int ice_vsi_get_qs_scatter(struct ice_vsi *vsi)
2134{
2135	struct ice_pf *pf = vsi->back;
2136	int i, index = 0;
2137
2138	mutex_lock(&pf->avail_q_mutex);
2139
2140	if (vsi->tx_mapping_mode == ICE_VSI_MAP_SCATTER) {
2141		for (i = 0; i < vsi->alloc_txq; i++) {
2142			index = find_next_zero_bit(pf->avail_txqs,
2143						   ICE_MAX_TXQS, index);
2144			if (index < ICE_MAX_TXQS) {
2145				set_bit(index, pf->avail_txqs);
2146				vsi->txq_map[i] = index;
2147			} else {
2148				goto err_scatter_tx;
2149			}
2150		}
2151	}
2152
2153	if (vsi->rx_mapping_mode == ICE_VSI_MAP_SCATTER) {
2154		for (i = 0; i < vsi->alloc_rxq; i++) {
2155			index = find_next_zero_bit(pf->avail_rxqs,
2156						   ICE_MAX_RXQS, index);
2157			if (index < ICE_MAX_RXQS) {
2158				set_bit(index, pf->avail_rxqs);
2159				vsi->rxq_map[i] = index;
2160			} else {
2161				goto err_scatter_rx;
2162			}
2163		}
2164	}
2165
2166	mutex_unlock(&pf->avail_q_mutex);
2167	return 0;
2168
2169err_scatter_rx:
2170	/* unflag any queues we have grabbed (i is failed position) */
2171	for (index = 0; index < i; index++) {
2172		clear_bit(vsi->rxq_map[index], pf->avail_rxqs);
2173		vsi->rxq_map[index] = 0;
2174	}
2175	i = vsi->alloc_txq;
2176err_scatter_tx:
2177	/* i is either position of failed attempt or vsi->alloc_txq */
2178	for (index = 0; index < i; index++) {
2179		clear_bit(vsi->txq_map[index], pf->avail_txqs);
2180		vsi->txq_map[index] = 0;
2181	}
2182
2183	mutex_unlock(&pf->avail_q_mutex);
2184	return -ENOMEM;
2185}
2186
2187/**
2188 * ice_vsi_get_qs - Assign queues from PF to VSI
2189 * @vsi: the VSI to assign queues to
2190 *
2191 * Returns 0 on success and a negative value on error
2192 */
2193static int ice_vsi_get_qs(struct ice_vsi *vsi)
2194{
2195	int ret = 0;
2196
2197	vsi->tx_mapping_mode = ICE_VSI_MAP_CONTIG;
2198	vsi->rx_mapping_mode = ICE_VSI_MAP_CONTIG;
2199
2200	/* NOTE: ice_vsi_get_qs_contig() will set the rx/tx mapping
2201	 * modes individually to scatter if assigning contiguous queues
2202	 * to rx or tx fails
2203	 */
2204	ret = ice_vsi_get_qs_contig(vsi);
2205	if (ret < 0) {
2206		if (vsi->tx_mapping_mode == ICE_VSI_MAP_SCATTER)
2207			vsi->alloc_txq = max_t(u16, vsi->alloc_txq,
2208					       ICE_MAX_SCATTER_TXQS);
2209		if (vsi->rx_mapping_mode == ICE_VSI_MAP_SCATTER)
2210			vsi->alloc_rxq = max_t(u16, vsi->alloc_rxq,
2211					       ICE_MAX_SCATTER_RXQS);
2212		ret = ice_vsi_get_qs_scatter(vsi);
2213	}
2214
2215	return ret;
2216}
2217
2218/**
2219 * ice_vsi_put_qs - Release queues from VSI to PF
2220 * @vsi: the VSI thats going to release queues
2221 */
2222static void ice_vsi_put_qs(struct ice_vsi *vsi)
2223{
2224	struct ice_pf *pf = vsi->back;
2225	int i;
2226
2227	mutex_lock(&pf->avail_q_mutex);
2228
2229	for (i = 0; i < vsi->alloc_txq; i++) {
2230		clear_bit(vsi->txq_map[i], pf->avail_txqs);
2231		vsi->txq_map[i] = ICE_INVAL_Q_INDEX;
2232	}
2233
2234	for (i = 0; i < vsi->alloc_rxq; i++) {
2235		clear_bit(vsi->rxq_map[i], pf->avail_rxqs);
2236		vsi->rxq_map[i] = ICE_INVAL_Q_INDEX;
2237	}
2238
2239	mutex_unlock(&pf->avail_q_mutex);
2240}
2241
2242/**
2243 * ice_free_q_vector - Free memory allocated for a specific interrupt vector
2244 * @vsi: VSI having the memory freed
2245 * @v_idx: index of the vector to be freed
2246 */
2247static void ice_free_q_vector(struct ice_vsi *vsi, int v_idx)
2248{
2249	struct ice_q_vector *q_vector;
2250	struct ice_ring *ring;
2251
2252	if (!vsi->q_vectors[v_idx]) {
2253		dev_dbg(&vsi->back->pdev->dev, "Queue vector at index %d not found\n",
2254			v_idx);
2255		return;
2256	}
2257	q_vector = vsi->q_vectors[v_idx];
2258
2259	ice_for_each_ring(ring, q_vector->tx)
2260		ring->q_vector = NULL;
2261	ice_for_each_ring(ring, q_vector->rx)
2262		ring->q_vector = NULL;
2263
2264	/* only VSI with an associated netdev is set up with NAPI */
2265	if (vsi->netdev)
2266		netif_napi_del(&q_vector->napi);
2267
2268	devm_kfree(&vsi->back->pdev->dev, q_vector);
2269	vsi->q_vectors[v_idx] = NULL;
2270}
2271
2272/**
2273 * ice_vsi_free_q_vectors - Free memory allocated for interrupt vectors
2274 * @vsi: the VSI having memory freed
2275 */
2276static void ice_vsi_free_q_vectors(struct ice_vsi *vsi)
2277{
2278	int v_idx;
2279
2280	for (v_idx = 0; v_idx < vsi->num_q_vectors; v_idx++)
2281		ice_free_q_vector(vsi, v_idx);
2282}
2283
2284/**
2285 * ice_cfg_netdev - Setup the netdev flags
2286 * @vsi: the VSI being configured
2287 *
2288 * Returns 0 on success, negative value on failure
2289 */
2290static int ice_cfg_netdev(struct ice_vsi *vsi)
2291{
2292	netdev_features_t csumo_features;
2293	netdev_features_t vlano_features;
2294	netdev_features_t dflt_features;
2295	netdev_features_t tso_features;
2296	struct ice_netdev_priv *np;
2297	struct net_device *netdev;
2298	u8 mac_addr[ETH_ALEN];
2299
2300	netdev = alloc_etherdev_mqs(sizeof(struct ice_netdev_priv),
2301				    vsi->alloc_txq, vsi->alloc_rxq);
2302	if (!netdev)
2303		return -ENOMEM;
2304
2305	vsi->netdev = netdev;
2306	np = netdev_priv(netdev);
2307	np->vsi = vsi;
2308
2309	dflt_features = NETIF_F_SG	|
2310			NETIF_F_HIGHDMA	|
2311			NETIF_F_RXHASH;
2312
2313	csumo_features = NETIF_F_RXCSUM	  |
2314			 NETIF_F_IP_CSUM  |
2315			 NETIF_F_IPV6_CSUM;
2316
2317	vlano_features = NETIF_F_HW_VLAN_CTAG_FILTER |
2318			 NETIF_F_HW_VLAN_CTAG_TX     |
2319			 NETIF_F_HW_VLAN_CTAG_RX;
2320
2321	tso_features = NETIF_F_TSO;
2322
2323	/* set features that user can change */
2324	netdev->hw_features = dflt_features | csumo_features |
2325			      vlano_features | tso_features;
2326
2327	/* enable features */
2328	netdev->features |= netdev->hw_features;
2329	/* encap and VLAN devices inherit default, csumo and tso features */
2330	netdev->hw_enc_features |= dflt_features | csumo_features |
2331				   tso_features;
2332	netdev->vlan_features |= dflt_features | csumo_features |
2333				 tso_features;
2334
2335	if (vsi->type == ICE_VSI_PF) {
2336		SET_NETDEV_DEV(netdev, &vsi->back->pdev->dev);
2337		ether_addr_copy(mac_addr, vsi->port_info->mac.perm_addr);
2338
2339		ether_addr_copy(netdev->dev_addr, mac_addr);
2340		ether_addr_copy(netdev->perm_addr, mac_addr);
2341	}
2342
2343	netdev->priv_flags |= IFF_UNICAST_FLT;
2344
2345	/* assign netdev_ops */
2346	netdev->netdev_ops = &ice_netdev_ops;
2347
2348	/* setup watchdog timeout value to be 5 second */
2349	netdev->watchdog_timeo = 5 * HZ;
2350
2351	ice_set_ethtool_ops(netdev);
2352
2353	netdev->min_mtu = ETH_MIN_MTU;
2354	netdev->max_mtu = ICE_MAX_MTU;
2355
2356	return 0;
2357}
2358
2359/**
2360 * ice_vsi_free_arrays - clean up vsi resources
2361 * @vsi: pointer to VSI being cleared
2362 * @free_qvectors: bool to specify if q_vectors should be deallocated
2363 */
2364static void ice_vsi_free_arrays(struct ice_vsi *vsi, bool free_qvectors)
2365{
2366	struct ice_pf *pf = vsi->back;
2367
2368	/* free the ring and vector containers */
2369	if (free_qvectors && vsi->q_vectors) {
2370		devm_kfree(&pf->pdev->dev, vsi->q_vectors);
2371		vsi->q_vectors = NULL;
2372	}
2373	if (vsi->tx_rings) {
2374		devm_kfree(&pf->pdev->dev, vsi->tx_rings);
2375		vsi->tx_rings = NULL;
2376	}
2377	if (vsi->rx_rings) {
2378		devm_kfree(&pf->pdev->dev, vsi->rx_rings);
2379		vsi->rx_rings = NULL;
2380	}
2381}
2382
2383/**
2384 * ice_vsi_clear - clean up and deallocate the provided vsi
2385 * @vsi: pointer to VSI being cleared
2386 *
2387 * This deallocates the vsi's queue resources, removes it from the PF's
2388 * VSI array if necessary, and deallocates the VSI
2389 *
2390 * Returns 0 on success, negative on failure
2391 */
2392static int ice_vsi_clear(struct ice_vsi *vsi)
2393{
2394	struct ice_pf *pf = NULL;
2395
2396	if (!vsi)
2397		return 0;
2398
2399	if (!vsi->back)
2400		return -EINVAL;
2401
2402	pf = vsi->back;
2403
2404	if (!pf->vsi[vsi->idx] || pf->vsi[vsi->idx] != vsi) {
2405		dev_dbg(&pf->pdev->dev, "vsi does not exist at pf->vsi[%d]\n",
2406			vsi->idx);
2407		return -EINVAL;
2408	}
2409
2410	mutex_lock(&pf->sw_mutex);
2411	/* updates the PF for this cleared vsi */
2412
2413	pf->vsi[vsi->idx] = NULL;
2414	if (vsi->idx < pf->next_vsi)
2415		pf->next_vsi = vsi->idx;
2416
2417	ice_vsi_free_arrays(vsi, true);
2418	mutex_unlock(&pf->sw_mutex);
2419	devm_kfree(&pf->pdev->dev, vsi);
2420
2421	return 0;
2422}
2423
2424/**
2425 * ice_vsi_alloc_q_vector - Allocate memory for a single interrupt vector
2426 * @vsi: the VSI being configured
2427 * @v_idx: index of the vector in the vsi struct
2428 *
2429 * We allocate one q_vector.  If allocation fails we return -ENOMEM.
2430 */
2431static int ice_vsi_alloc_q_vector(struct ice_vsi *vsi, int v_idx)
2432{
2433	struct ice_pf *pf = vsi->back;
2434	struct ice_q_vector *q_vector;
2435
2436	/* allocate q_vector */
2437	q_vector = devm_kzalloc(&pf->pdev->dev, sizeof(*q_vector), GFP_KERNEL);
2438	if (!q_vector)
2439		return -ENOMEM;
2440
2441	q_vector->vsi = vsi;
2442	q_vector->v_idx = v_idx;
2443	/* only set affinity_mask if the CPU is online */
2444	if (cpu_online(v_idx))
2445		cpumask_set_cpu(v_idx, &q_vector->affinity_mask);
2446
2447	if (vsi->netdev)
2448		netif_napi_add(vsi->netdev, &q_vector->napi, ice_napi_poll,
2449			       NAPI_POLL_WEIGHT);
2450	/* tie q_vector and vsi together */
2451	vsi->q_vectors[v_idx] = q_vector;
2452
2453	return 0;
2454}
2455
2456/**
2457 * ice_vsi_alloc_q_vectors - Allocate memory for interrupt vectors
2458 * @vsi: the VSI being configured
2459 *
2460 * We allocate one q_vector per queue interrupt.  If allocation fails we
2461 * return -ENOMEM.
2462 */
2463static int ice_vsi_alloc_q_vectors(struct ice_vsi *vsi)
2464{
2465	struct ice_pf *pf = vsi->back;
2466	int v_idx = 0, num_q_vectors;
2467	int err;
2468
2469	if (vsi->q_vectors[0]) {
2470		dev_dbg(&pf->pdev->dev, "VSI %d has existing q_vectors\n",
2471			vsi->vsi_num);
2472		return -EEXIST;
2473	}
2474
2475	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
2476		num_q_vectors = vsi->num_q_vectors;
2477	} else {
2478		err = -EINVAL;
2479		goto err_out;
2480	}
2481
2482	for (v_idx = 0; v_idx < num_q_vectors; v_idx++) {
2483		err = ice_vsi_alloc_q_vector(vsi, v_idx);
2484		if (err)
2485			goto err_out;
2486	}
2487
2488	return 0;
2489
2490err_out:
2491	while (v_idx--)
2492		ice_free_q_vector(vsi, v_idx);
2493
2494	dev_err(&pf->pdev->dev,
2495		"Failed to allocate %d q_vector for VSI %d, ret=%d\n",
2496		vsi->num_q_vectors, vsi->vsi_num, err);
2497	vsi->num_q_vectors = 0;
2498	return err;
2499}
2500
2501/**
2502 * ice_vsi_setup_vector_base - Set up the base vector for the given VSI
2503 * @vsi: ptr to the VSI
2504 *
2505 * This should only be called after ice_vsi_alloc() which allocates the
2506 * corresponding SW VSI structure and initializes num_queue_pairs for the
2507 * newly allocated VSI.
2508 *
2509 * Returns 0 on success or negative on failure
2510 */
2511static int ice_vsi_setup_vector_base(struct ice_vsi *vsi)
2512{
2513	struct ice_pf *pf = vsi->back;
2514	int num_q_vectors = 0;
2515
2516	if (vsi->base_vector) {
2517		dev_dbg(&pf->pdev->dev, "VSI %d has non-zero base vector %d\n",
2518			vsi->vsi_num, vsi->base_vector);
2519		return -EEXIST;
2520	}
2521
2522	if (!test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
2523		return -ENOENT;
2524
2525	switch (vsi->type) {
2526	case ICE_VSI_PF:
2527		num_q_vectors = vsi->num_q_vectors;
2528		break;
2529	default:
2530		dev_warn(&vsi->back->pdev->dev, "Unknown VSI type %d\n",
2531			 vsi->type);
2532		break;
2533	}
2534
2535	if (num_q_vectors)
2536		vsi->base_vector = ice_get_res(pf, pf->irq_tracker,
2537					       num_q_vectors, vsi->idx);
2538
2539	if (vsi->base_vector < 0) {
2540		dev_err(&pf->pdev->dev,
2541			"Failed to get tracking for %d vectors for VSI %d, err=%d\n",
2542			num_q_vectors, vsi->vsi_num, vsi->base_vector);
2543		return -ENOENT;
2544	}
2545
2546	return 0;
2547}
2548
2549/**
2550 * ice_fill_rss_lut - Fill the RSS lookup table with default values
2551 * @lut: Lookup table
2552 * @rss_table_size: Lookup table size
2553 * @rss_size: Range of queue number for hashing
2554 */
2555void ice_fill_rss_lut(u8 *lut, u16 rss_table_size, u16 rss_size)
2556{
2557	u16 i;
2558
2559	for (i = 0; i < rss_table_size; i++)
2560		lut[i] = i % rss_size;
2561}
2562
2563/**
2564 * ice_vsi_cfg_rss - Configure RSS params for a VSI
2565 * @vsi: VSI to be configured
2566 */
2567static int ice_vsi_cfg_rss(struct ice_vsi *vsi)
2568{
2569	u8 seed[ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE];
2570	struct ice_aqc_get_set_rss_keys *key;
2571	struct ice_pf *pf = vsi->back;
2572	enum ice_status status;
2573	int err = 0;
2574	u8 *lut;
2575
2576	vsi->rss_size = min_t(int, vsi->rss_size, vsi->num_rxq);
2577
2578	lut = devm_kzalloc(&pf->pdev->dev, vsi->rss_table_size, GFP_KERNEL);
2579	if (!lut)
2580		return -ENOMEM;
2581
2582	if (vsi->rss_lut_user)
2583		memcpy(lut, vsi->rss_lut_user, vsi->rss_table_size);
2584	else
2585		ice_fill_rss_lut(lut, vsi->rss_table_size, vsi->rss_size);
2586
2587	status = ice_aq_set_rss_lut(&pf->hw, vsi->vsi_num, vsi->rss_lut_type,
2588				    lut, vsi->rss_table_size);
2589
2590	if (status) {
2591		dev_err(&vsi->back->pdev->dev,
2592			"set_rss_lut failed, error %d\n", status);
2593		err = -EIO;
2594		goto ice_vsi_cfg_rss_exit;
2595	}
2596
2597	key = devm_kzalloc(&vsi->back->pdev->dev, sizeof(*key), GFP_KERNEL);
2598	if (!key) {
2599		err = -ENOMEM;
2600		goto ice_vsi_cfg_rss_exit;
2601	}
2602
2603	if (vsi->rss_hkey_user)
2604		memcpy(seed, vsi->rss_hkey_user,
2605		       ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2606	else
2607		netdev_rss_key_fill((void *)seed,
2608				    ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2609	memcpy(&key->standard_rss_key, seed,
2610	       ICE_AQC_GET_SET_RSS_KEY_DATA_RSS_KEY_SIZE);
2611
2612	status = ice_aq_set_rss_key(&pf->hw, vsi->vsi_num, key);
2613
2614	if (status) {
2615		dev_err(&vsi->back->pdev->dev, "set_rss_key failed, error %d\n",
2616			status);
2617		err = -EIO;
2618	}
2619
2620	devm_kfree(&pf->pdev->dev, key);
2621ice_vsi_cfg_rss_exit:
2622	devm_kfree(&pf->pdev->dev, lut);
2623	return err;
2624}
2625
2626/**
2627 * ice_vsi_reinit_setup - return resource and reallocate resource for a VSI
2628 * @vsi: pointer to the ice_vsi
2629 *
2630 * This reallocates the VSIs queue resources
2631 *
2632 * Returns 0 on success and negative value on failure
2633 */
2634static int ice_vsi_reinit_setup(struct ice_vsi *vsi)
2635{
2636	u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2637	int ret, i;
2638
2639	if (!vsi)
2640		return -EINVAL;
2641
2642	ice_vsi_free_q_vectors(vsi);
2643	ice_free_res(vsi->back->irq_tracker, vsi->base_vector, vsi->idx);
2644	vsi->base_vector = 0;
2645	ice_vsi_clear_rings(vsi);
2646	ice_vsi_free_arrays(vsi, false);
2647	ice_vsi_set_num_qs(vsi);
2648
2649	/* Initialize VSI struct elements and create VSI in FW */
2650	ret = ice_vsi_add(vsi);
2651	if (ret < 0)
2652		goto err_vsi;
2653
2654	ret = ice_vsi_alloc_arrays(vsi, false);
2655	if (ret < 0)
2656		goto err_vsi;
2657
2658	switch (vsi->type) {
2659	case ICE_VSI_PF:
2660		if (!vsi->netdev) {
2661			ret = ice_cfg_netdev(vsi);
2662			if (ret)
2663				goto err_rings;
2664
2665			ret = register_netdev(vsi->netdev);
2666			if (ret)
2667				goto err_rings;
2668
2669			netif_carrier_off(vsi->netdev);
2670			netif_tx_stop_all_queues(vsi->netdev);
2671		}
2672
2673		ret = ice_vsi_alloc_q_vectors(vsi);
2674		if (ret)
2675			goto err_rings;
2676
2677		ret = ice_vsi_setup_vector_base(vsi);
2678		if (ret)
2679			goto err_vectors;
2680
2681		ret = ice_vsi_alloc_rings(vsi);
2682		if (ret)
2683			goto err_vectors;
2684
2685		ice_vsi_map_rings_to_vectors(vsi);
2686		break;
2687	default:
2688		break;
2689	}
2690
2691	ice_vsi_set_tc_cfg(vsi);
2692
2693	/* configure VSI nodes based on number of queues and TC's */
2694	for (i = 0; i < vsi->tc_cfg.numtc; i++)
2695		max_txqs[i] = vsi->num_txq;
2696
2697	ret = ice_cfg_vsi_lan(vsi->port_info, vsi->vsi_num,
2698			      vsi->tc_cfg.ena_tc, max_txqs);
2699	if (ret) {
2700		dev_info(&vsi->back->pdev->dev,
2701			 "Failed VSI lan queue config\n");
2702		goto err_vectors;
2703	}
2704	return 0;
2705
2706err_vectors:
2707	ice_vsi_free_q_vectors(vsi);
2708err_rings:
2709	if (vsi->netdev) {
2710		vsi->current_netdev_flags = 0;
2711		unregister_netdev(vsi->netdev);
2712		free_netdev(vsi->netdev);
2713		vsi->netdev = NULL;
2714	}
2715err_vsi:
2716	ice_vsi_clear(vsi);
2717	set_bit(__ICE_RESET_FAILED, vsi->back->state);
2718	return ret;
2719}
2720
2721/**
2722 * ice_vsi_setup - Set up a VSI by a given type
2723 * @pf: board private structure
2724 * @type: VSI type
2725 * @pi: pointer to the port_info instance
2726 *
2727 * This allocates the sw VSI structure and its queue resources.
2728 *
2729 * Returns pointer to the successfully allocated and configure VSI sw struct on
2730 * success, otherwise returns NULL on failure.
2731 */
2732static struct ice_vsi *
2733ice_vsi_setup(struct ice_pf *pf, enum ice_vsi_type type,
2734	      struct ice_port_info *pi)
2735{
2736	u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
2737	struct device *dev = &pf->pdev->dev;
2738	struct ice_vsi_ctx ctxt = { 0 };
2739	struct ice_vsi *vsi;
2740	int ret, i;
2741
2742	vsi = ice_vsi_alloc(pf, type);
2743	if (!vsi) {
2744		dev_err(dev, "could not allocate VSI\n");
2745		return NULL;
2746	}
2747
2748	vsi->port_info = pi;
2749	vsi->vsw = pf->first_sw;
2750
2751	if (ice_vsi_get_qs(vsi)) {
2752		dev_err(dev, "Failed to allocate queues. vsi->idx = %d\n",
2753			vsi->idx);
2754		goto err_get_qs;
2755	}
2756
2757	/* set RSS capabilities */
2758	ice_vsi_set_rss_params(vsi);
2759
2760	/* create the VSI */
2761	ret = ice_vsi_add(vsi);
2762	if (ret)
2763		goto err_vsi;
2764
2765	ctxt.vsi_num = vsi->vsi_num;
2766
2767	switch (vsi->type) {
2768	case ICE_VSI_PF:
2769		ret = ice_cfg_netdev(vsi);
2770		if (ret)
2771			goto err_cfg_netdev;
2772
2773		ret = register_netdev(vsi->netdev);
2774		if (ret)
2775			goto err_register_netdev;
2776
2777		netif_carrier_off(vsi->netdev);
2778
2779		/* make sure transmit queues start off as stopped */
2780		netif_tx_stop_all_queues(vsi->netdev);
2781		ret = ice_vsi_alloc_q_vectors(vsi);
2782		if (ret)
2783			goto err_msix;
2784
2785		ret = ice_vsi_setup_vector_base(vsi);
2786		if (ret)
2787			goto err_rings;
2788
2789		ret = ice_vsi_alloc_rings(vsi);
2790		if (ret)
2791			goto err_rings;
2792
2793		ice_vsi_map_rings_to_vectors(vsi);
2794
2795		/* Do not exit if configuring RSS had an issue, at least
2796		 * receive traffic on first queue. Hence no need to capture
2797		 * return value
2798		 */
2799		if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
2800			ice_vsi_cfg_rss(vsi);
2801		break;
2802	default:
2803		/* if vsi type is not recognized, clean up the resources and
2804		 * exit
2805		 */
2806		goto err_rings;
2807	}
2808
2809	ice_vsi_set_tc_cfg(vsi);
2810
2811	/* configure VSI nodes based on number of queues and TC's */
2812	for (i = 0; i < vsi->tc_cfg.numtc; i++)
2813		max_txqs[i] = vsi->num_txq;
2814
2815	ret = ice_cfg_vsi_lan(vsi->port_info, vsi->vsi_num,
2816			      vsi->tc_cfg.ena_tc, max_txqs);
2817	if (ret) {
2818		dev_info(&pf->pdev->dev, "Failed VSI lan queue config\n");
2819		goto err_rings;
2820	}
2821
2822	return vsi;
2823
2824err_rings:
2825	ice_vsi_free_q_vectors(vsi);
2826err_msix:
2827	if (vsi->netdev && vsi->netdev->reg_state == NETREG_REGISTERED)
2828		unregister_netdev(vsi->netdev);
2829err_register_netdev:
2830	if (vsi->netdev) {
2831		free_netdev(vsi->netdev);
2832		vsi->netdev = NULL;
2833	}
2834err_cfg_netdev:
2835	ret = ice_aq_free_vsi(&pf->hw, &ctxt, false, NULL);
2836	if (ret)
2837		dev_err(&vsi->back->pdev->dev,
2838			"Free VSI AQ call failed, err %d\n", ret);
2839err_vsi:
2840	ice_vsi_put_qs(vsi);
2841err_get_qs:
2842	pf->q_left_tx += vsi->alloc_txq;
2843	pf->q_left_rx += vsi->alloc_rxq;
2844	ice_vsi_clear(vsi);
2845
2846	return NULL;
2847}
2848
2849/**
2850 * ice_vsi_add_vlan - Add vsi membership for given vlan
2851 * @vsi: the vsi being configured
2852 * @vid: vlan id to be added
2853 */
2854static int ice_vsi_add_vlan(struct ice_vsi *vsi, u16 vid)
2855{
2856	struct ice_fltr_list_entry *tmp;
2857	struct ice_pf *pf = vsi->back;
2858	LIST_HEAD(tmp_add_list);
2859	enum ice_status status;
2860	int err = 0;
2861
2862	tmp = devm_kzalloc(&pf->pdev->dev, sizeof(*tmp), GFP_KERNEL);
2863	if (!tmp)
2864		return -ENOMEM;
2865
2866	tmp->fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
2867	tmp->fltr_info.fltr_act = ICE_FWD_TO_VSI;
2868	tmp->fltr_info.flag = ICE_FLTR_TX;
2869	tmp->fltr_info.src = vsi->vsi_num;
2870	tmp->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
2871	tmp->fltr_info.l_data.vlan.vlan_id = vid;
2872
2873	INIT_LIST_HEAD(&tmp->list_entry);
2874	list_add(&tmp->list_entry, &tmp_add_list);
2875
2876	status = ice_add_vlan(&pf->hw, &tmp_add_list);
2877	if (status) {
2878		err = -ENODEV;
2879		dev_err(&pf->pdev->dev, "Failure Adding VLAN %d on VSI %i\n",
2880			vid, vsi->vsi_num);
2881	}
2882
2883	ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
2884	return err;
2885}
2886
2887/**
2888 * ice_vlan_rx_add_vid - Add a vlan id filter to HW offload
2889 * @netdev: network interface to be adjusted
2890 * @proto: unused protocol
2891 * @vid: vlan id to be added
2892 *
2893 * net_device_ops implementation for adding vlan ids
2894 */
2895static int ice_vlan_rx_add_vid(struct net_device *netdev,
2896			       __always_unused __be16 proto, u16 vid)
2897{
2898	struct ice_netdev_priv *np = netdev_priv(netdev);
2899	struct ice_vsi *vsi = np->vsi;
2900	int ret = 0;
2901
2902	if (vid >= VLAN_N_VID) {
2903		netdev_err(netdev, "VLAN id requested %d is out of range %d\n",
2904			   vid, VLAN_N_VID);
2905		return -EINVAL;
2906	}
2907
2908	if (vsi->info.pvid)
2909		return -EINVAL;
2910
2911	/* Add all VLAN ids including 0 to the switch filter. VLAN id 0 is
2912	 * needed to continue allowing all untagged packets since VLAN prune
2913	 * list is applied to all packets by the switch
2914	 */
2915	ret = ice_vsi_add_vlan(vsi, vid);
2916
2917	if (!ret)
2918		set_bit(vid, vsi->active_vlans);
2919
2920	return ret;
2921}
2922
2923/**
2924 * ice_vsi_kill_vlan - Remove VSI membership for a given VLAN
2925 * @vsi: the VSI being configured
2926 * @vid: VLAN id to be removed
2927 */
2928static void ice_vsi_kill_vlan(struct ice_vsi *vsi, u16 vid)
2929{
2930	struct ice_fltr_list_entry *list;
2931	struct ice_pf *pf = vsi->back;
2932	LIST_HEAD(tmp_add_list);
2933
2934	list = devm_kzalloc(&pf->pdev->dev, sizeof(*list), GFP_KERNEL);
2935	if (!list)
2936		return;
2937
2938	list->fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
2939	list->fltr_info.fwd_id.vsi_id = vsi->vsi_num;
2940	list->fltr_info.fltr_act = ICE_FWD_TO_VSI;
2941	list->fltr_info.l_data.vlan.vlan_id = vid;
2942	list->fltr_info.flag = ICE_FLTR_TX;
2943	list->fltr_info.src = vsi->vsi_num;
2944
2945	INIT_LIST_HEAD(&list->list_entry);
2946	list_add(&list->list_entry, &tmp_add_list);
2947
2948	if (ice_remove_vlan(&pf->hw, &tmp_add_list))
2949		dev_err(&pf->pdev->dev, "Error removing VLAN %d on vsi %i\n",
2950			vid, vsi->vsi_num);
2951
2952	ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
2953}
2954
2955/**
2956 * ice_vlan_rx_kill_vid - Remove a vlan id filter from HW offload
2957 * @netdev: network interface to be adjusted
2958 * @proto: unused protocol
2959 * @vid: vlan id to be removed
2960 *
2961 * net_device_ops implementation for removing vlan ids
2962 */
2963static int ice_vlan_rx_kill_vid(struct net_device *netdev,
2964				__always_unused __be16 proto, u16 vid)
2965{
2966	struct ice_netdev_priv *np = netdev_priv(netdev);
2967	struct ice_vsi *vsi = np->vsi;
2968
2969	if (vsi->info.pvid)
2970		return -EINVAL;
2971
2972	/* return code is ignored as there is nothing a user
2973	 * can do about failure to remove and a log message was
2974	 * already printed from the other function
2975	 */
2976	ice_vsi_kill_vlan(vsi, vid);
2977
2978	clear_bit(vid, vsi->active_vlans);
2979
2980	return 0;
2981}
2982
2983/**
2984 * ice_setup_pf_sw - Setup the HW switch on startup or after reset
2985 * @pf: board private structure
2986 *
2987 * Returns 0 on success, negative value on failure
2988 */
2989static int ice_setup_pf_sw(struct ice_pf *pf)
2990{
2991	LIST_HEAD(tmp_add_list);
2992	u8 broadcast[ETH_ALEN];
2993	struct ice_vsi *vsi;
2994	int status = 0;
2995
2996	if (!ice_is_reset_recovery_pending(pf->state)) {
2997		vsi = ice_vsi_setup(pf, ICE_VSI_PF, pf->hw.port_info);
2998		if (!vsi) {
2999			status = -ENOMEM;
3000			goto error_exit;
3001		}
3002	} else {
3003		vsi = pf->vsi[0];
3004		status = ice_vsi_reinit_setup(vsi);
3005		if (status < 0)
3006			return -EIO;
3007	}
3008
3009	/* tmp_add_list contains a list of MAC addresses for which MAC
3010	 * filters need to be programmed. Add the VSI's unicast MAC to
3011	 * this list
3012	 */
3013	status = ice_add_mac_to_list(vsi, &tmp_add_list,
3014				     vsi->port_info->mac.perm_addr);
3015	if (status)
3016		goto error_exit;
3017
3018	/* VSI needs to receive broadcast traffic, so add the broadcast
3019	 * MAC address to the list.
3020	 */
3021	eth_broadcast_addr(broadcast);
3022	status = ice_add_mac_to_list(vsi, &tmp_add_list, broadcast);
3023	if (status)
3024		goto error_exit;
3025
3026	/* program MAC filters for entries in tmp_add_list */
3027	status = ice_add_mac(&pf->hw, &tmp_add_list);
3028	if (status) {
3029		dev_err(&pf->pdev->dev, "Could not add MAC filters\n");
3030		status = -ENOMEM;
3031		goto error_exit;
3032	}
3033
3034	ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
3035	return status;
3036
3037error_exit:
3038	ice_free_fltr_list(&pf->pdev->dev, &tmp_add_list);
3039
3040	if (vsi) {
3041		ice_vsi_free_q_vectors(vsi);
3042		if (vsi->netdev && vsi->netdev->reg_state == NETREG_REGISTERED)
3043			unregister_netdev(vsi->netdev);
3044		if (vsi->netdev) {
3045			free_netdev(vsi->netdev);
3046			vsi->netdev = NULL;
3047		}
3048
3049		ice_vsi_delete(vsi);
3050		ice_vsi_put_qs(vsi);
3051		pf->q_left_tx += vsi->alloc_txq;
3052		pf->q_left_rx += vsi->alloc_rxq;
3053		ice_vsi_clear(vsi);
3054	}
3055	return status;
3056}
3057
3058/**
3059 * ice_determine_q_usage - Calculate queue distribution
3060 * @pf: board private structure
3061 *
3062 * Return -ENOMEM if we don't get enough queues for all ports
3063 */
3064static void ice_determine_q_usage(struct ice_pf *pf)
3065{
3066	u16 q_left_tx, q_left_rx;
3067
3068	q_left_tx = pf->hw.func_caps.common_cap.num_txq;
3069	q_left_rx = pf->hw.func_caps.common_cap.num_rxq;
3070
3071	pf->num_lan_tx = min_t(int, q_left_tx, num_online_cpus());
3072
3073	/* only 1 rx queue unless RSS is enabled */
3074	if (!test_bit(ICE_FLAG_RSS_ENA, pf->flags))
3075		pf->num_lan_rx = 1;
3076	else
3077		pf->num_lan_rx = min_t(int, q_left_rx, num_online_cpus());
3078
3079	pf->q_left_tx = q_left_tx - pf->num_lan_tx;
3080	pf->q_left_rx = q_left_rx - pf->num_lan_rx;
3081}
3082
3083/**
3084 * ice_deinit_pf - Unrolls initialziations done by ice_init_pf
3085 * @pf: board private structure to initialize
3086 */
3087static void ice_deinit_pf(struct ice_pf *pf)
3088{
3089	if (pf->serv_tmr.function)
3090		del_timer_sync(&pf->serv_tmr);
3091	if (pf->serv_task.func)
3092		cancel_work_sync(&pf->serv_task);
3093	mutex_destroy(&pf->sw_mutex);
3094	mutex_destroy(&pf->avail_q_mutex);
3095}
3096
3097/**
3098 * ice_init_pf - Initialize general software structures (struct ice_pf)
3099 * @pf: board private structure to initialize
3100 */
3101static void ice_init_pf(struct ice_pf *pf)
3102{
3103	bitmap_zero(pf->flags, ICE_PF_FLAGS_NBITS);
3104	set_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3105
3106	mutex_init(&pf->sw_mutex);
3107	mutex_init(&pf->avail_q_mutex);
3108
3109	/* Clear avail_[t|r]x_qs bitmaps (set all to avail) */
3110	mutex_lock(&pf->avail_q_mutex);
3111	bitmap_zero(pf->avail_txqs, ICE_MAX_TXQS);
3112	bitmap_zero(pf->avail_rxqs, ICE_MAX_RXQS);
3113	mutex_unlock(&pf->avail_q_mutex);
3114
3115	if (pf->hw.func_caps.common_cap.rss_table_size)
3116		set_bit(ICE_FLAG_RSS_ENA, pf->flags);
3117
3118	/* setup service timer and periodic service task */
3119	timer_setup(&pf->serv_tmr, ice_service_timer, 0);
3120	pf->serv_tmr_period = HZ;
3121	INIT_WORK(&pf->serv_task, ice_service_task);
3122	clear_bit(__ICE_SERVICE_SCHED, pf->state);
3123}
3124
3125/**
3126 * ice_ena_msix_range - Request a range of MSIX vectors from the OS
3127 * @pf: board private structure
3128 *
3129 * compute the number of MSIX vectors required (v_budget) and request from
3130 * the OS. Return the number of vectors reserved or negative on failure
3131 */
3132static int ice_ena_msix_range(struct ice_pf *pf)
3133{
3134	int v_left, v_actual, v_budget = 0;
3135	int needed, err, i;
3136
3137	v_left = pf->hw.func_caps.common_cap.num_msix_vectors;
3138
3139	/* reserve one vector for miscellaneous handler */
3140	needed = 1;
3141	v_budget += needed;
3142	v_left -= needed;
3143
3144	/* reserve vectors for LAN traffic */
3145	pf->num_lan_msix = min_t(int, num_online_cpus(), v_left);
3146	v_budget += pf->num_lan_msix;
3147
3148	pf->msix_entries = devm_kcalloc(&pf->pdev->dev, v_budget,
3149					sizeof(struct msix_entry), GFP_KERNEL);
3150
3151	if (!pf->msix_entries) {
3152		err = -ENOMEM;
3153		goto exit_err;
3154	}
3155
3156	for (i = 0; i < v_budget; i++)
3157		pf->msix_entries[i].entry = i;
3158
3159	/* actually reserve the vectors */
3160	v_actual = pci_enable_msix_range(pf->pdev, pf->msix_entries,
3161					 ICE_MIN_MSIX, v_budget);
3162
3163	if (v_actual < 0) {
3164		dev_err(&pf->pdev->dev, "unable to reserve MSI-X vectors\n");
3165		err = v_actual;
3166		goto msix_err;
3167	}
3168
3169	if (v_actual < v_budget) {
3170		dev_warn(&pf->pdev->dev,
3171			 "not enough vectors. requested = %d, obtained = %d\n",
3172			 v_budget, v_actual);
3173		if (v_actual >= (pf->num_lan_msix + 1)) {
3174			pf->num_avail_msix = v_actual - (pf->num_lan_msix + 1);
3175		} else if (v_actual >= 2) {
3176			pf->num_lan_msix = 1;
3177			pf->num_avail_msix = v_actual - 2;
3178		} else {
3179			pci_disable_msix(pf->pdev);
3180			err = -ERANGE;
3181			goto msix_err;
3182		}
3183	}
3184
3185	return v_actual;
3186
3187msix_err:
3188	devm_kfree(&pf->pdev->dev, pf->msix_entries);
3189	goto exit_err;
3190
3191exit_err:
3192	pf->num_lan_msix = 0;
3193	clear_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3194	return err;
3195}
3196
3197/**
3198 * ice_dis_msix - Disable MSI-X interrupt setup in OS
3199 * @pf: board private structure
3200 */
3201static void ice_dis_msix(struct ice_pf *pf)
3202{
3203	pci_disable_msix(pf->pdev);
3204	devm_kfree(&pf->pdev->dev, pf->msix_entries);
3205	pf->msix_entries = NULL;
3206	clear_bit(ICE_FLAG_MSIX_ENA, pf->flags);
3207}
3208
3209/**
3210 * ice_init_interrupt_scheme - Determine proper interrupt scheme
3211 * @pf: board private structure to initialize
3212 */
3213static int ice_init_interrupt_scheme(struct ice_pf *pf)
3214{
3215	int vectors = 0;
3216	ssize_t size;
3217
3218	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
3219		vectors = ice_ena_msix_range(pf);
3220	else
3221		return -ENODEV;
3222
3223	if (vectors < 0)
3224		return vectors;
3225
3226	/* set up vector assignment tracking */
3227	size = sizeof(struct ice_res_tracker) + (sizeof(u16) * vectors);
3228
3229	pf->irq_tracker = devm_kzalloc(&pf->pdev->dev, size, GFP_KERNEL);
3230	if (!pf->irq_tracker) {
3231		ice_dis_msix(pf);
3232		return -ENOMEM;
3233	}
3234
3235	pf->irq_tracker->num_entries = vectors;
3236
3237	return 0;
3238}
3239
3240/**
3241 * ice_clear_interrupt_scheme - Undo things done by ice_init_interrupt_scheme
3242 * @pf: board private structure
3243 */
3244static void ice_clear_interrupt_scheme(struct ice_pf *pf)
3245{
3246	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
3247		ice_dis_msix(pf);
3248
3249	devm_kfree(&pf->pdev->dev, pf->irq_tracker);
3250	pf->irq_tracker = NULL;
3251}
3252
3253/**
3254 * ice_probe - Device initialization routine
3255 * @pdev: PCI device information struct
3256 * @ent: entry in ice_pci_tbl
3257 *
3258 * Returns 0 on success, negative on failure
3259 */
3260static int ice_probe(struct pci_dev *pdev,
3261		     const struct pci_device_id __always_unused *ent)
3262{
3263	struct ice_pf *pf;
3264	struct ice_hw *hw;
3265	int err;
3266
3267	/* this driver uses devres, see Documentation/driver-model/devres.txt */
3268	err = pcim_enable_device(pdev);
3269	if (err)
3270		return err;
3271
3272	err = pcim_iomap_regions(pdev, BIT(ICE_BAR0), pci_name(pdev));
3273	if (err) {
3274		dev_err(&pdev->dev, "I/O map error %d\n", err);
3275		return err;
3276	}
3277
3278	pf = devm_kzalloc(&pdev->dev, sizeof(*pf), GFP_KERNEL);
3279	if (!pf)
3280		return -ENOMEM;
3281
3282	/* set up for high or low dma */
3283	err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
3284	if (err)
3285		err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
3286	if (err) {
3287		dev_err(&pdev->dev, "DMA configuration failed: 0x%x\n", err);
3288		return err;
3289	}
3290
3291	pci_enable_pcie_error_reporting(pdev);
3292	pci_set_master(pdev);
3293
3294	pf->pdev = pdev;
3295	pci_set_drvdata(pdev, pf);
3296	set_bit(__ICE_DOWN, pf->state);
3297
3298	hw = &pf->hw;
3299	hw->hw_addr = pcim_iomap_table(pdev)[ICE_BAR0];
3300	hw->back = pf;
3301	hw->vendor_id = pdev->vendor;
3302	hw->device_id = pdev->device;
3303	pci_read_config_byte(pdev, PCI_REVISION_ID, &hw->revision_id);
3304	hw->subsystem_vendor_id = pdev->subsystem_vendor;
3305	hw->subsystem_device_id = pdev->subsystem_device;
3306	hw->bus.device = PCI_SLOT(pdev->devfn);
3307	hw->bus.func = PCI_FUNC(pdev->devfn);
3308	ice_set_ctrlq_len(hw);
3309
3310	pf->msg_enable = netif_msg_init(debug, ICE_DFLT_NETIF_M);
3311
3312#ifndef CONFIG_DYNAMIC_DEBUG
3313	if (debug < -1)
3314		hw->debug_mask = debug;
3315#endif
3316
3317	err = ice_init_hw(hw);
3318	if (err) {
3319		dev_err(&pdev->dev, "ice_init_hw failed: %d\n", err);
3320		err = -EIO;
3321		goto err_exit_unroll;
3322	}
3323
3324	dev_info(&pdev->dev, "firmware %d.%d.%05d api %d.%d\n",
3325		 hw->fw_maj_ver, hw->fw_min_ver, hw->fw_build,
3326		 hw->api_maj_ver, hw->api_min_ver);
3327
3328	ice_init_pf(pf);
3329
3330	ice_determine_q_usage(pf);
3331
3332	pf->num_alloc_vsi = min_t(u16, ICE_MAX_VSI_ALLOC,
3333				  hw->func_caps.guaranteed_num_vsi);
3334	if (!pf->num_alloc_vsi) {
3335		err = -EIO;
3336		goto err_init_pf_unroll;
3337	}
3338
3339	pf->vsi = devm_kcalloc(&pdev->dev, pf->num_alloc_vsi,
3340			       sizeof(struct ice_vsi *), GFP_KERNEL);
3341	if (!pf->vsi) {
3342		err = -ENOMEM;
3343		goto err_init_pf_unroll;
3344	}
3345
3346	err = ice_init_interrupt_scheme(pf);
3347	if (err) {
3348		dev_err(&pdev->dev,
3349			"ice_init_interrupt_scheme failed: %d\n", err);
3350		err = -EIO;
3351		goto err_init_interrupt_unroll;
3352	}
3353
3354	/* In case of MSIX we are going to setup the misc vector right here
3355	 * to handle admin queue events etc. In case of legacy and MSI
3356	 * the misc functionality and queue processing is combined in
3357	 * the same vector and that gets setup at open.
3358	 */
3359	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
3360		err = ice_req_irq_msix_misc(pf);
3361		if (err) {
3362			dev_err(&pdev->dev,
3363				"setup of misc vector failed: %d\n", err);
3364			goto err_init_interrupt_unroll;
3365		}
3366	}
3367
3368	/* create switch struct for the switch element created by FW on boot */
3369	pf->first_sw = devm_kzalloc(&pdev->dev, sizeof(struct ice_sw),
3370				    GFP_KERNEL);
3371	if (!pf->first_sw) {
3372		err = -ENOMEM;
3373		goto err_msix_misc_unroll;
3374	}
3375
3376	pf->first_sw->bridge_mode = BRIDGE_MODE_VEB;
3377	pf->first_sw->pf = pf;
3378
3379	/* record the sw_id available for later use */
3380	pf->first_sw->sw_id = hw->port_info->sw_id;
3381
3382	err = ice_setup_pf_sw(pf);
3383	if (err) {
3384		dev_err(&pdev->dev,
3385			"probe failed due to setup pf switch:%d\n", err);
3386		goto err_alloc_sw_unroll;
3387	}
3388
3389	/* Driver is mostly up */
3390	clear_bit(__ICE_DOWN, pf->state);
3391
3392	/* since everything is good, start the service timer */
3393	mod_timer(&pf->serv_tmr, round_jiffies(jiffies + pf->serv_tmr_period));
3394
3395	err = ice_init_link_events(pf->hw.port_info);
3396	if (err) {
3397		dev_err(&pdev->dev, "ice_init_link_events failed: %d\n", err);
3398		goto err_alloc_sw_unroll;
3399	}
3400
3401	return 0;
3402
3403err_alloc_sw_unroll:
3404	set_bit(__ICE_DOWN, pf->state);
3405	devm_kfree(&pf->pdev->dev, pf->first_sw);
3406err_msix_misc_unroll:
3407	ice_free_irq_msix_misc(pf);
3408err_init_interrupt_unroll:
3409	ice_clear_interrupt_scheme(pf);
3410	devm_kfree(&pdev->dev, pf->vsi);
3411err_init_pf_unroll:
3412	ice_deinit_pf(pf);
3413	ice_deinit_hw(hw);
3414err_exit_unroll:
3415	pci_disable_pcie_error_reporting(pdev);
3416	return err;
3417}
3418
3419/**
3420 * ice_remove - Device removal routine
3421 * @pdev: PCI device information struct
3422 */
3423static void ice_remove(struct pci_dev *pdev)
3424{
3425	struct ice_pf *pf = pci_get_drvdata(pdev);
3426	int i = 0;
3427	int err;
3428
3429	if (!pf)
3430		return;
3431
3432	set_bit(__ICE_DOWN, pf->state);
3433
3434	for (i = 0; i < pf->num_alloc_vsi; i++) {
3435		if (!pf->vsi[i])
3436			continue;
3437
3438		err = ice_vsi_release(pf->vsi[i]);
3439		if (err)
3440			dev_dbg(&pf->pdev->dev, "Failed to release VSI index %d (err %d)\n",
3441				i, err);
3442	}
3443
3444	ice_free_irq_msix_misc(pf);
3445	ice_clear_interrupt_scheme(pf);
3446	ice_deinit_pf(pf);
3447	ice_deinit_hw(&pf->hw);
3448	pci_disable_pcie_error_reporting(pdev);
3449}
3450
3451/* ice_pci_tbl - PCI Device ID Table
3452 *
3453 * Wildcard entries (PCI_ANY_ID) should come last
3454 * Last entry must be all 0s
3455 *
3456 * { Vendor ID, Device ID, SubVendor ID, SubDevice ID,
3457 *   Class, Class Mask, private data (not used) }
3458 */
3459static const struct pci_device_id ice_pci_tbl[] = {
3460	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_BACKPLANE), 0 },
3461	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_QSFP), 0 },
3462	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_SFP), 0 },
3463	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_10G_BASE_T), 0 },
3464	{ PCI_VDEVICE(INTEL, ICE_DEV_ID_C810_SGMII), 0 },
3465	/* required last entry */
3466	{ 0, }
3467};
3468MODULE_DEVICE_TABLE(pci, ice_pci_tbl);
3469
3470static struct pci_driver ice_driver = {
3471	.name = KBUILD_MODNAME,
3472	.id_table = ice_pci_tbl,
3473	.probe = ice_probe,
3474	.remove = ice_remove,
3475};
3476
3477/**
3478 * ice_module_init - Driver registration routine
3479 *
3480 * ice_module_init is the first routine called when the driver is
3481 * loaded. All it does is register with the PCI subsystem.
3482 */
3483static int __init ice_module_init(void)
3484{
3485	int status;
3486
3487	pr_info("%s - version %s\n", ice_driver_string, ice_drv_ver);
3488	pr_info("%s\n", ice_copyright);
3489
3490	ice_wq = alloc_ordered_workqueue("%s", WQ_MEM_RECLAIM, KBUILD_MODNAME);
3491	if (!ice_wq) {
3492		pr_err("Failed to create workqueue\n");
3493		return -ENOMEM;
3494	}
3495
3496	status = pci_register_driver(&ice_driver);
3497	if (status) {
3498		pr_err("failed to register pci driver, err %d\n", status);
3499		destroy_workqueue(ice_wq);
3500	}
3501
3502	return status;
3503}
3504module_init(ice_module_init);
3505
3506/**
3507 * ice_module_exit - Driver exit cleanup routine
3508 *
3509 * ice_module_exit is called just before the driver is removed
3510 * from memory.
3511 */
3512static void __exit ice_module_exit(void)
3513{
3514	pci_unregister_driver(&ice_driver);
3515	destroy_workqueue(ice_wq);
3516	pr_info("module unloaded\n");
3517}
3518module_exit(ice_module_exit);
3519
3520/**
3521 * ice_set_mac_address - NDO callback to set mac address
3522 * @netdev: network interface device structure
3523 * @pi: pointer to an address structure
3524 *
3525 * Returns 0 on success, negative on failure
3526 */
3527static int ice_set_mac_address(struct net_device *netdev, void *pi)
3528{
3529	struct ice_netdev_priv *np = netdev_priv(netdev);
3530	struct ice_vsi *vsi = np->vsi;
3531	struct ice_pf *pf = vsi->back;
3532	struct ice_hw *hw = &pf->hw;
3533	struct sockaddr *addr = pi;
3534	enum ice_status status;
3535	LIST_HEAD(a_mac_list);
3536	LIST_HEAD(r_mac_list);
3537	u8 flags = 0;
3538	int err;
3539	u8 *mac;
3540
3541	mac = (u8 *)addr->sa_data;
3542
3543	if (!is_valid_ether_addr(mac))
3544		return -EADDRNOTAVAIL;
3545
3546	if (ether_addr_equal(netdev->dev_addr, mac)) {
3547		netdev_warn(netdev, "already using mac %pM\n", mac);
3548		return 0;
3549	}
3550
3551	if (test_bit(__ICE_DOWN, pf->state) ||
3552	    ice_is_reset_recovery_pending(pf->state)) {
3553		netdev_err(netdev, "can't set mac %pM. device not ready\n",
3554			   mac);
3555		return -EBUSY;
3556	}
3557
3558	/* When we change the mac address we also have to change the mac address
3559	 * based filter rules that were created previously for the old mac
3560	 * address. So first, we remove the old filter rule using ice_remove_mac
3561	 * and then create a new filter rule using ice_add_mac. Note that for
3562	 * both these operations, we first need to form a "list" of mac
3563	 * addresses (even though in this case, we have only 1 mac address to be
3564	 * added/removed) and this done using ice_add_mac_to_list. Depending on
3565	 * the ensuing operation this "list" of mac addresses is either to be
3566	 * added or removed from the filter.
3567	 */
3568	err = ice_add_mac_to_list(vsi, &r_mac_list, netdev->dev_addr);
3569	if (err) {
3570		err = -EADDRNOTAVAIL;
3571		goto free_lists;
3572	}
3573
3574	status = ice_remove_mac(hw, &r_mac_list);
3575	if (status) {
3576		err = -EADDRNOTAVAIL;
3577		goto free_lists;
3578	}
3579
3580	err = ice_add_mac_to_list(vsi, &a_mac_list, mac);
3581	if (err) {
3582		err = -EADDRNOTAVAIL;
3583		goto free_lists;
3584	}
3585
3586	status = ice_add_mac(hw, &a_mac_list);
3587	if (status) {
3588		err = -EADDRNOTAVAIL;
3589		goto free_lists;
3590	}
3591
3592free_lists:
3593	/* free list entries */
3594	ice_free_fltr_list(&pf->pdev->dev, &r_mac_list);
3595	ice_free_fltr_list(&pf->pdev->dev, &a_mac_list);
3596
3597	if (err) {
3598		netdev_err(netdev, "can't set mac %pM. filter update failed\n",
3599			   mac);
3600		return err;
3601	}
3602
3603	/* change the netdev's mac address */
3604	memcpy(netdev->dev_addr, mac, netdev->addr_len);
3605	netdev_dbg(vsi->netdev, "updated mac address to %pM\n",
3606		   netdev->dev_addr);
3607
3608	/* write new mac address to the firmware */
3609	flags = ICE_AQC_MAN_MAC_UPDATE_LAA_WOL;
3610	status = ice_aq_manage_mac_write(hw, mac, flags, NULL);
3611	if (status) {
3612		netdev_err(netdev, "can't set mac %pM. write to firmware failed.\n",
3613			   mac);
3614	}
3615	return 0;
3616}
3617
3618/**
3619 * ice_set_rx_mode - NDO callback to set the netdev filters
3620 * @netdev: network interface device structure
3621 */
3622static void ice_set_rx_mode(struct net_device *netdev)
3623{
3624	struct ice_netdev_priv *np = netdev_priv(netdev);
3625	struct ice_vsi *vsi = np->vsi;
3626
3627	if (!vsi)
3628		return;
3629
3630	/* Set the flags to synchronize filters
3631	 * ndo_set_rx_mode may be triggered even without a change in netdev
3632	 * flags
3633	 */
3634	set_bit(ICE_VSI_FLAG_UMAC_FLTR_CHANGED, vsi->flags);
3635	set_bit(ICE_VSI_FLAG_MMAC_FLTR_CHANGED, vsi->flags);
3636	set_bit(ICE_FLAG_FLTR_SYNC, vsi->back->flags);
3637
3638	/* schedule our worker thread which will take care of
3639	 * applying the new filter changes
3640	 */
3641	ice_service_task_schedule(vsi->back);
3642}
3643
3644/**
3645 * ice_fdb_add - add an entry to the hardware database
3646 * @ndm: the input from the stack
3647 * @tb: pointer to array of nladdr (unused)
3648 * @dev: the net device pointer
3649 * @addr: the MAC address entry being added
3650 * @vid: VLAN id
3651 * @flags: instructions from stack about fdb operation
3652 */
3653static int ice_fdb_add(struct ndmsg *ndm, struct nlattr __always_unused *tb[],
3654		       struct net_device *dev, const unsigned char *addr,
3655		       u16 vid, u16 flags)
3656{
3657	int err;
3658
3659	if (vid) {
3660		netdev_err(dev, "VLANs aren't supported yet for dev_uc|mc_add()\n");
3661		return -EINVAL;
3662	}
3663	if (ndm->ndm_state && !(ndm->ndm_state & NUD_PERMANENT)) {
3664		netdev_err(dev, "FDB only supports static addresses\n");
3665		return -EINVAL;
3666	}
3667
3668	if (is_unicast_ether_addr(addr) || is_link_local_ether_addr(addr))
3669		err = dev_uc_add_excl(dev, addr);
3670	else if (is_multicast_ether_addr(addr))
3671		err = dev_mc_add_excl(dev, addr);
3672	else
3673		err = -EINVAL;
3674
3675	/* Only return duplicate errors if NLM_F_EXCL is set */
3676	if (err == -EEXIST && !(flags & NLM_F_EXCL))
3677		err = 0;
3678
3679	return err;
3680}
3681
3682/**
3683 * ice_fdb_del - delete an entry from the hardware database
3684 * @ndm: the input from the stack
3685 * @tb: pointer to array of nladdr (unused)
3686 * @dev: the net device pointer
3687 * @addr: the MAC address entry being added
3688 * @vid: VLAN id
3689 */
3690static int ice_fdb_del(struct ndmsg *ndm, __always_unused struct nlattr *tb[],
3691		       struct net_device *dev, const unsigned char *addr,
3692		       __always_unused u16 vid)
3693{
3694	int err;
3695
3696	if (ndm->ndm_state & NUD_PERMANENT) {
3697		netdev_err(dev, "FDB only supports static addresses\n");
3698		return -EINVAL;
3699	}
3700
3701	if (is_unicast_ether_addr(addr))
3702		err = dev_uc_del(dev, addr);
3703	else if (is_multicast_ether_addr(addr))
3704		err = dev_mc_del(dev, addr);
3705	else
3706		err = -EINVAL;
3707
3708	return err;
3709}
3710
3711/**
3712 * ice_vsi_manage_vlan_insertion - Manage VLAN insertion for the VSI for Tx
3713 * @vsi: the vsi being changed
3714 */
3715static int ice_vsi_manage_vlan_insertion(struct ice_vsi *vsi)
3716{
3717	struct device *dev = &vsi->back->pdev->dev;
3718	struct ice_hw *hw = &vsi->back->hw;
3719	struct ice_vsi_ctx ctxt = { 0 };
3720	enum ice_status status;
3721
3722	/* Here we are configuring the VSI to let the driver add VLAN tags by
3723	 * setting port_vlan_flags to ICE_AQ_VSI_PVLAN_MODE_ALL. The actual VLAN
3724	 * tag insertion happens in the Tx hot path, in ice_tx_map.
3725	 */
3726	ctxt.info.port_vlan_flags = ICE_AQ_VSI_PVLAN_MODE_ALL;
3727
3728	ctxt.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID);
3729	ctxt.vsi_num = vsi->vsi_num;
3730
3731	status = ice_aq_update_vsi(hw, &ctxt, NULL);
3732	if (status) {
3733		dev_err(dev, "update VSI for VLAN insert failed, err %d aq_err %d\n",
3734			status, hw->adminq.sq_last_status);
3735		return -EIO;
3736	}
3737
3738	vsi->info.port_vlan_flags = ctxt.info.port_vlan_flags;
3739	return 0;
3740}
3741
3742/**
3743 * ice_vsi_manage_vlan_stripping - Manage VLAN stripping for the VSI for Rx
3744 * @vsi: the vsi being changed
3745 * @ena: boolean value indicating if this is a enable or disable request
3746 */
3747static int ice_vsi_manage_vlan_stripping(struct ice_vsi *vsi, bool ena)
3748{
3749	struct device *dev = &vsi->back->pdev->dev;
3750	struct ice_hw *hw = &vsi->back->hw;
3751	struct ice_vsi_ctx ctxt = { 0 };
3752	enum ice_status status;
3753
3754	/* Here we are configuring what the VSI should do with the VLAN tag in
3755	 * the Rx packet. We can either leave the tag in the packet or put it in
3756	 * the Rx descriptor.
3757	 */
3758	if (ena) {
3759		/* Strip VLAN tag from Rx packet and put it in the desc */
3760		ctxt.info.port_vlan_flags = ICE_AQ_VSI_PVLAN_EMOD_STR_BOTH;
3761	} else {
3762		/* Disable stripping. Leave tag in packet */
3763		ctxt.info.port_vlan_flags = ICE_AQ_VSI_PVLAN_EMOD_NOTHING;
3764	}
3765
3766	ctxt.info.valid_sections = cpu_to_le16(ICE_AQ_VSI_PROP_VLAN_VALID);
3767	ctxt.vsi_num = vsi->vsi_num;
3768
3769	status = ice_aq_update_vsi(hw, &ctxt, NULL);
3770	if (status) {
3771		dev_err(dev, "update VSI for VALN strip failed, ena = %d err %d aq_err %d\n",
3772			ena, status, hw->adminq.sq_last_status);
3773		return -EIO;
3774	}
3775
3776	vsi->info.port_vlan_flags = ctxt.info.port_vlan_flags;
3777	return 0;
3778}
3779
3780/**
3781 * ice_set_features - set the netdev feature flags
3782 * @netdev: ptr to the netdev being adjusted
3783 * @features: the feature set that the stack is suggesting
3784 */
3785static int ice_set_features(struct net_device *netdev,
3786			    netdev_features_t features)
3787{
3788	struct ice_netdev_priv *np = netdev_priv(netdev);
3789	struct ice_vsi *vsi = np->vsi;
3790	int ret = 0;
3791
3792	if ((features & NETIF_F_HW_VLAN_CTAG_RX) &&
3793	    !(netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
3794		ret = ice_vsi_manage_vlan_stripping(vsi, true);
3795	else if (!(features & NETIF_F_HW_VLAN_CTAG_RX) &&
3796		 (netdev->features & NETIF_F_HW_VLAN_CTAG_RX))
3797		ret = ice_vsi_manage_vlan_stripping(vsi, false);
3798	else if ((features & NETIF_F_HW_VLAN_CTAG_TX) &&
3799		 !(netdev->features & NETIF_F_HW_VLAN_CTAG_TX))
3800		ret = ice_vsi_manage_vlan_insertion(vsi);
3801	else if (!(features & NETIF_F_HW_VLAN_CTAG_TX) &&
3802		 (netdev->features & NETIF_F_HW_VLAN_CTAG_TX))
3803		ret = ice_vsi_manage_vlan_insertion(vsi);
3804
3805	return ret;
3806}
3807
3808/**
3809 * ice_vsi_vlan_setup - Setup vlan offload properties on a VSI
3810 * @vsi: VSI to setup vlan properties for
3811 */
3812static int ice_vsi_vlan_setup(struct ice_vsi *vsi)
3813{
3814	int ret = 0;
3815
3816	if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_RX)
3817		ret = ice_vsi_manage_vlan_stripping(vsi, true);
3818	if (vsi->netdev->features & NETIF_F_HW_VLAN_CTAG_TX)
3819		ret = ice_vsi_manage_vlan_insertion(vsi);
3820
3821	return ret;
3822}
3823
3824/**
3825 * ice_restore_vlan - Reinstate VLANs when vsi/netdev comes back up
3826 * @vsi: the VSI being brought back up
3827 */
3828static int ice_restore_vlan(struct ice_vsi *vsi)
3829{
3830	int err;
3831	u16 vid;
3832
3833	if (!vsi->netdev)
3834		return -EINVAL;
3835
3836	err = ice_vsi_vlan_setup(vsi);
3837	if (err)
3838		return err;
3839
3840	for_each_set_bit(vid, vsi->active_vlans, VLAN_N_VID) {
3841		err = ice_vlan_rx_add_vid(vsi->netdev, htons(ETH_P_8021Q), vid);
3842		if (err)
3843			break;
3844	}
3845
3846	return err;
3847}
3848
3849/**
3850 * ice_setup_tx_ctx - setup a struct ice_tlan_ctx instance
3851 * @ring: The Tx ring to configure
3852 * @tlan_ctx: Pointer to the Tx LAN queue context structure to be initialized
3853 * @pf_q: queue index in the PF space
3854 *
3855 * Configure the Tx descriptor ring in TLAN context.
3856 */
3857static void
3858ice_setup_tx_ctx(struct ice_ring *ring, struct ice_tlan_ctx *tlan_ctx, u16 pf_q)
3859{
3860	struct ice_vsi *vsi = ring->vsi;
3861	struct ice_hw *hw = &vsi->back->hw;
3862
3863	tlan_ctx->base = ring->dma >> ICE_TLAN_CTX_BASE_S;
3864
3865	tlan_ctx->port_num = vsi->port_info->lport;
3866
3867	/* Transmit Queue Length */
3868	tlan_ctx->qlen = ring->count;
3869
3870	/* PF number */
3871	tlan_ctx->pf_num = hw->pf_id;
3872
3873	/* queue belongs to a specific VSI type
3874	 * VF / VM index should be programmed per vmvf_type setting:
3875	 * for vmvf_type = VF, it is VF number between 0-256
3876	 * for vmvf_type = VM, it is VM number between 0-767
3877	 * for PF or EMP this field should be set to zero
3878	 */
3879	switch (vsi->type) {
3880	case ICE_VSI_PF:
3881		tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
3882		break;
3883	default:
3884		return;
3885	}
3886
3887	/* make sure the context is associated with the right VSI */
3888	tlan_ctx->src_vsi = vsi->vsi_num;
3889
3890	tlan_ctx->tso_ena = ICE_TX_LEGACY;
3891	tlan_ctx->tso_qnum = pf_q;
3892
3893	/* Legacy or Advanced Host Interface:
3894	 * 0: Advanced Host Interface
3895	 * 1: Legacy Host Interface
3896	 */
3897	tlan_ctx->legacy_int = ICE_TX_LEGACY;
3898}
3899
3900/**
3901 * ice_vsi_cfg_txqs - Configure the VSI for Tx
3902 * @vsi: the VSI being configured
3903 *
3904 * Return 0 on success and a negative value on error
3905 * Configure the Tx VSI for operation.
3906 */
3907static int ice_vsi_cfg_txqs(struct ice_vsi *vsi)
3908{
3909	struct ice_aqc_add_tx_qgrp *qg_buf;
3910	struct ice_aqc_add_txqs_perq *txq;
3911	struct ice_pf *pf = vsi->back;
3912	enum ice_status status;
3913	u16 buf_len, i, pf_q;
3914	int err = 0, tc = 0;
3915	u8 num_q_grps;
3916
3917	buf_len = sizeof(struct ice_aqc_add_tx_qgrp);
3918	qg_buf = devm_kzalloc(&pf->pdev->dev, buf_len, GFP_KERNEL);
3919	if (!qg_buf)
3920		return -ENOMEM;
3921
3922	if (vsi->num_txq > ICE_MAX_TXQ_PER_TXQG) {
3923		err = -EINVAL;
3924		goto err_cfg_txqs;
3925	}
3926	qg_buf->num_txqs = 1;
3927	num_q_grps = 1;
3928
3929	/* set up and configure the tx queues */
3930	ice_for_each_txq(vsi, i) {
3931		struct ice_tlan_ctx tlan_ctx = { 0 };
3932
3933		pf_q = vsi->txq_map[i];
3934		ice_setup_tx_ctx(vsi->tx_rings[i], &tlan_ctx, pf_q);
3935		/* copy context contents into the qg_buf */
3936		qg_buf->txqs[0].txq_id = cpu_to_le16(pf_q);
3937		ice_set_ctx((u8 *)&tlan_ctx, qg_buf->txqs[0].txq_ctx,
3938			    ice_tlan_ctx_info);
3939
3940		/* init queue specific tail reg. It is referred as transmit
3941		 * comm scheduler queue doorbell.
3942		 */
3943		vsi->tx_rings[i]->tail = pf->hw.hw_addr + QTX_COMM_DBELL(pf_q);
3944		status = ice_ena_vsi_txq(vsi->port_info, vsi->vsi_num, tc,
3945					 num_q_grps, qg_buf, buf_len, NULL);
3946		if (status) {
3947			dev_err(&vsi->back->pdev->dev,
3948				"Failed to set LAN Tx queue context, error: %d\n",
3949				status);
3950			err = -ENODEV;
3951			goto err_cfg_txqs;
3952		}
3953
3954		/* Add Tx Queue TEID into the VSI tx ring from the response
3955		 * This will complete configuring and enabling the queue.
3956		 */
3957		txq = &qg_buf->txqs[0];
3958		if (pf_q == le16_to_cpu(txq->txq_id))
3959			vsi->tx_rings[i]->txq_teid =
3960				le32_to_cpu(txq->q_teid);
3961	}
3962err_cfg_txqs:
3963	devm_kfree(&pf->pdev->dev, qg_buf);
3964	return err;
3965}
3966
3967/**
3968 * ice_setup_rx_ctx - Configure a receive ring context
3969 * @ring: The Rx ring to configure
3970 *
3971 * Configure the Rx descriptor ring in RLAN context.
3972 */
3973static int ice_setup_rx_ctx(struct ice_ring *ring)
3974{
3975	struct ice_vsi *vsi = ring->vsi;
3976	struct ice_hw *hw = &vsi->back->hw;
3977	u32 rxdid = ICE_RXDID_FLEX_NIC;
3978	struct ice_rlan_ctx rlan_ctx;
3979	u32 regval;
3980	u16 pf_q;
3981	int err;
3982
3983	/* what is RX queue number in global space of 2K rx queues */
3984	pf_q = vsi->rxq_map[ring->q_index];
3985
3986	/* clear the context structure first */
3987	memset(&rlan_ctx, 0, sizeof(rlan_ctx));
3988
3989	rlan_ctx.base = ring->dma >> 7;
3990
3991	rlan_ctx.qlen = ring->count;
3992
3993	/* Receive Packet Data Buffer Size.
3994	 * The Packet Data Buffer Size is defined in 128 byte units.
3995	 */
3996	rlan_ctx.dbuf = vsi->rx_buf_len >> ICE_RLAN_CTX_DBUF_S;
3997
3998	/* use 32 byte descriptors */
3999	rlan_ctx.dsize = 1;
4000
4001	/* Strip the Ethernet CRC bytes before the packet is posted to host
4002	 * memory.
4003	 */
4004	rlan_ctx.crcstrip = 1;
4005
4006	/* L2TSEL flag defines the reported L2 Tags in the receive descriptor */
4007	rlan_ctx.l2tsel = 1;
4008
4009	rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT;
4010	rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT;
4011	rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT;
4012
4013	/* This controls whether VLAN is stripped from inner headers
4014	 * The VLAN in the inner L2 header is stripped to the receive
4015	 * descriptor if enabled by this flag.
4016	 */
4017	rlan_ctx.showiv = 0;
4018
4019	/* Max packet size for this queue - must not be set to a larger value
4020	 * than 5 x DBUF
4021	 */
4022	rlan_ctx.rxmax = min_t(u16, vsi->max_frame,
4023			       ICE_MAX_CHAINED_RX_BUFS * vsi->rx_buf_len);
4024
4025	/* Rx queue threshold in units of 64 */
4026	rlan_ctx.lrxqthresh = 1;
4027
4028	 /* Enable Flexible Descriptors in the queue context which
4029	  * allows this driver to select a specific receive descriptor format
4030	  */
4031	regval = rd32(hw, QRXFLXP_CNTXT(pf_q));
4032	regval |= (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
4033		QRXFLXP_CNTXT_RXDID_IDX_M;
4034
4035	/* increasing context priority to pick up profile id;
4036	 * default is 0x01; setting to 0x03 to ensure profile
4037	 * is programming if prev context is of same priority
4038	 */
4039	regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
4040		QRXFLXP_CNTXT_RXDID_PRIO_M;
4041
4042	wr32(hw, QRXFLXP_CNTXT(pf_q), regval);
4043
4044	/* Absolute queue number out of 2K needs to be passed */
4045	err = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q);
4046	if (err) {
4047		dev_err(&vsi->back->pdev->dev,
4048			"Failed to set LAN Rx queue context for absolute Rx queue %d error: %d\n",
4049			pf_q, err);
4050		return -EIO;
4051	}
4052
4053	/* init queue specific tail register */
4054	ring->tail = hw->hw_addr + QRX_TAIL(pf_q);
4055	writel(0, ring->tail);
4056	ice_alloc_rx_bufs(ring, ICE_DESC_UNUSED(ring));
4057
4058	return 0;
4059}
4060
4061/**
4062 * ice_vsi_cfg_rxqs - Configure the VSI for Rx
4063 * @vsi: the VSI being configured
4064 *
4065 * Return 0 on success and a negative value on error
4066 * Configure the Rx VSI for operation.
4067 */
4068static int ice_vsi_cfg_rxqs(struct ice_vsi *vsi)
4069{
4070	int err = 0;
4071	u16 i;
4072
4073	if (vsi->netdev && vsi->netdev->mtu > ETH_DATA_LEN)
4074		vsi->max_frame = vsi->netdev->mtu +
4075			ETH_HLEN + ETH_FCS_LEN + VLAN_HLEN;
4076	else
4077		vsi->max_frame = ICE_RXBUF_2048;
4078
4079	vsi->rx_buf_len = ICE_RXBUF_2048;
4080	/* set up individual rings */
4081	for (i = 0; i < vsi->num_rxq && !err; i++)
4082		err = ice_setup_rx_ctx(vsi->rx_rings[i]);
4083
4084	if (err) {
4085		dev_err(&vsi->back->pdev->dev, "ice_setup_rx_ctx failed\n");
4086		return -EIO;
4087	}
4088	return err;
4089}
4090
4091/**
4092 * ice_vsi_cfg - Setup the VSI
4093 * @vsi: the VSI being configured
4094 *
4095 * Return 0 on success and negative value on error
4096 */
4097static int ice_vsi_cfg(struct ice_vsi *vsi)
4098{
4099	int err;
4100
4101	ice_set_rx_mode(vsi->netdev);
4102
4103	err = ice_restore_vlan(vsi);
4104	if (err)
4105		return err;
4106
4107	err = ice_vsi_cfg_txqs(vsi);
4108	if (!err)
4109		err = ice_vsi_cfg_rxqs(vsi);
4110
4111	return err;
4112}
4113
4114/**
4115 * ice_vsi_stop_tx_rings - Disable Tx rings
4116 * @vsi: the VSI being configured
4117 */
4118static int ice_vsi_stop_tx_rings(struct ice_vsi *vsi)
4119{
4120	struct ice_pf *pf = vsi->back;
4121	struct ice_hw *hw = &pf->hw;
4122	enum ice_status status;
4123	u32 *q_teids, val;
4124	u16 *q_ids, i;
4125	int err = 0;
4126
4127	if (vsi->num_txq > ICE_LAN_TXQ_MAX_QDIS)
4128		return -EINVAL;
4129
4130	q_teids = devm_kcalloc(&pf->pdev->dev, vsi->num_txq, sizeof(*q_teids),
4131			       GFP_KERNEL);
4132	if (!q_teids)
4133		return -ENOMEM;
4134
4135	q_ids = devm_kcalloc(&pf->pdev->dev, vsi->num_txq, sizeof(*q_ids),
4136			     GFP_KERNEL);
4137	if (!q_ids) {
4138		err = -ENOMEM;
4139		goto err_alloc_q_ids;
4140	}
4141
4142	/* set up the tx queue list to be disabled */
4143	ice_for_each_txq(vsi, i) {
4144		u16 v_idx;
4145
4146		if (!vsi->tx_rings || !vsi->tx_rings[i]) {
4147			err = -EINVAL;
4148			goto err_out;
4149		}
4150
4151		q_ids[i] = vsi->txq_map[i];
4152		q_teids[i] = vsi->tx_rings[i]->txq_teid;
4153
4154		/* clear cause_ena bit for disabled queues */
4155		val = rd32(hw, QINT_TQCTL(vsi->tx_rings[i]->reg_idx));
4156		val &= ~QINT_TQCTL_CAUSE_ENA_M;
4157		wr32(hw, QINT_TQCTL(vsi->tx_rings[i]->reg_idx), val);
4158
4159		/* software is expected to wait for 100 ns */
4160		ndelay(100);
4161
4162		/* trigger a software interrupt for the vector associated to
4163		 * the queue to schedule napi handler
4164		 */
4165		v_idx = vsi->tx_rings[i]->q_vector->v_idx;
4166		wr32(hw, GLINT_DYN_CTL(vsi->base_vector + v_idx),
4167		     GLINT_DYN_CTL_SWINT_TRIG_M | GLINT_DYN_CTL_INTENA_MSK_M);
4168	}
4169	status = ice_dis_vsi_txq(vsi->port_info, vsi->num_txq, q_ids, q_teids,
4170				 NULL);
4171	if (status) {
4172		dev_err(&pf->pdev->dev,
4173			"Failed to disable LAN Tx queues, error: %d\n",
4174			status);
4175		err = -ENODEV;
4176	}
4177
4178err_out:
4179	devm_kfree(&pf->pdev->dev, q_ids);
4180
4181err_alloc_q_ids:
4182	devm_kfree(&pf->pdev->dev, q_teids);
4183
4184	return err;
4185}
4186
4187/**
4188 * ice_pf_rxq_wait - Wait for a PF's Rx queue to be enabled or disabled
4189 * @pf: the PF being configured
4190 * @pf_q: the PF queue
4191 * @ena: enable or disable state of the queue
4192 *
4193 * This routine will wait for the given Rx queue of the PF to reach the
4194 * enabled or disabled state.
4195 * Returns -ETIMEDOUT in case of failing to reach the requested state after
4196 * multiple retries; else will return 0 in case of success.
4197 */
4198static int ice_pf_rxq_wait(struct ice_pf *pf, int pf_q, bool ena)
4199{
4200	int i;
4201
4202	for (i = 0; i < ICE_Q_WAIT_RETRY_LIMIT; i++) {
4203		u32 rx_reg = rd32(&pf->hw, QRX_CTRL(pf_q));
4204
4205		if (ena == !!(rx_reg & QRX_CTRL_QENA_STAT_M))
4206			break;
4207
4208		usleep_range(10, 20);
4209	}
4210	if (i >= ICE_Q_WAIT_RETRY_LIMIT)
4211		return -ETIMEDOUT;
4212
4213	return 0;
4214}
4215
4216/**
4217 * ice_vsi_ctrl_rx_rings - Start or stop a VSI's rx rings
4218 * @vsi: the VSI being configured
4219 * @ena: start or stop the rx rings
4220 */
4221static int ice_vsi_ctrl_rx_rings(struct ice_vsi *vsi, bool ena)
4222{
4223	struct ice_pf *pf = vsi->back;
4224	struct ice_hw *hw = &pf->hw;
4225	int i, j, ret = 0;
4226
4227	for (i = 0; i < vsi->num_rxq; i++) {
4228		int pf_q = vsi->rxq_map[i];
4229		u32 rx_reg;
4230
4231		for (j = 0; j < ICE_Q_WAIT_MAX_RETRY; j++) {
4232			rx_reg = rd32(hw, QRX_CTRL(pf_q));
4233			if (((rx_reg >> QRX_CTRL_QENA_REQ_S) & 1) ==
4234			    ((rx_reg >> QRX_CTRL_QENA_STAT_S) & 1))
4235				break;
4236			usleep_range(1000, 2000);
4237		}
4238
4239		/* Skip if the queue is already in the requested state */
4240		if (ena == !!(rx_reg & QRX_CTRL_QENA_STAT_M))
4241			continue;
4242
4243		/* turn on/off the queue */
4244		if (ena)
4245			rx_reg |= QRX_CTRL_QENA_REQ_M;
4246		else
4247			rx_reg &= ~QRX_CTRL_QENA_REQ_M;
4248		wr32(hw, QRX_CTRL(pf_q), rx_reg);
4249
4250		/* wait for the change to finish */
4251		ret = ice_pf_rxq_wait(pf, pf_q, ena);
4252		if (ret) {
4253			dev_err(&pf->pdev->dev,
4254				"VSI idx %d Rx ring %d %sable timeout\n",
4255				vsi->idx, pf_q, (ena ? "en" : "dis"));
4256			break;
4257		}
4258	}
4259
4260	return ret;
4261}
4262
4263/**
4264 * ice_vsi_start_rx_rings - start VSI's rx rings
4265 * @vsi: the VSI whose rings are to be started
4266 *
4267 * Returns 0 on success and a negative value on error
4268 */
4269static int ice_vsi_start_rx_rings(struct ice_vsi *vsi)
4270{
4271	return ice_vsi_ctrl_rx_rings(vsi, true);
4272}
4273
4274/**
4275 * ice_vsi_stop_rx_rings - stop VSI's rx rings
4276 * @vsi: the VSI
4277 *
4278 * Returns 0 on success and a negative value on error
4279 */
4280static int ice_vsi_stop_rx_rings(struct ice_vsi *vsi)
4281{
4282	return ice_vsi_ctrl_rx_rings(vsi, false);
4283}
4284
4285/**
4286 * ice_vsi_stop_tx_rx_rings - stop VSI's tx and rx rings
4287 * @vsi: the VSI
4288 * Returns 0 on success and a negative value on error
4289 */
4290static int ice_vsi_stop_tx_rx_rings(struct ice_vsi *vsi)
4291{
4292	int err_tx, err_rx;
4293
4294	err_tx = ice_vsi_stop_tx_rings(vsi);
4295	if (err_tx)
4296		dev_dbg(&vsi->back->pdev->dev, "Failed to disable Tx rings\n");
4297
4298	err_rx = ice_vsi_stop_rx_rings(vsi);
4299	if (err_rx)
4300		dev_dbg(&vsi->back->pdev->dev, "Failed to disable Rx rings\n");
4301
4302	if (err_tx || err_rx)
4303		return -EIO;
4304
4305	return 0;
4306}
4307
4308/**
4309 * ice_napi_enable_all - Enable NAPI for all q_vectors in the VSI
4310 * @vsi: the VSI being configured
4311 */
4312static void ice_napi_enable_all(struct ice_vsi *vsi)
4313{
4314	int q_idx;
4315
4316	if (!vsi->netdev)
4317		return;
4318
4319	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4320		napi_enable(&vsi->q_vectors[q_idx]->napi);
4321}
4322
4323/**
4324 * ice_up_complete - Finish the last steps of bringing up a connection
4325 * @vsi: The VSI being configured
4326 *
4327 * Return 0 on success and negative value on error
4328 */
4329static int ice_up_complete(struct ice_vsi *vsi)
4330{
4331	struct ice_pf *pf = vsi->back;
4332	int err;
4333
4334	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
4335		ice_vsi_cfg_msix(vsi);
4336	else
4337		return -ENOTSUPP;
4338
4339	/* Enable only Rx rings, Tx rings were enabled by the FW when the
4340	 * Tx queue group list was configured and the context bits were
4341	 * programmed using ice_vsi_cfg_txqs
4342	 */
4343	err = ice_vsi_start_rx_rings(vsi);
4344	if (err)
4345		return err;
4346
4347	clear_bit(__ICE_DOWN, vsi->state);
4348	ice_napi_enable_all(vsi);
4349	ice_vsi_ena_irq(vsi);
4350
4351	if (vsi->port_info &&
4352	    (vsi->port_info->phy.link_info.link_info & ICE_AQ_LINK_UP) &&
4353	    vsi->netdev) {
4354		ice_print_link_msg(vsi, true);
4355		netif_tx_start_all_queues(vsi->netdev);
4356		netif_carrier_on(vsi->netdev);
4357	}
4358
4359	ice_service_task_schedule(pf);
4360
4361	return err;
4362}
4363
4364/**
4365 * ice_up - Bring the connection back up after being down
4366 * @vsi: VSI being configured
4367 */
4368int ice_up(struct ice_vsi *vsi)
4369{
4370	int err;
4371
4372	err = ice_vsi_cfg(vsi);
4373	if (!err)
4374		err = ice_up_complete(vsi);
4375
4376	return err;
4377}
4378
4379/**
4380 * ice_fetch_u64_stats_per_ring - get packets and bytes stats per ring
4381 * @ring: Tx or Rx ring to read stats from
4382 * @pkts: packets stats counter
4383 * @bytes: bytes stats counter
4384 *
4385 * This function fetches stats from the ring considering the atomic operations
4386 * that needs to be performed to read u64 values in 32 bit machine.
4387 */
4388static void ice_fetch_u64_stats_per_ring(struct ice_ring *ring, u64 *pkts,
4389					 u64 *bytes)
4390{
4391	unsigned int start;
4392	*pkts = 0;
4393	*bytes = 0;
4394
4395	if (!ring)
4396		return;
4397	do {
4398		start = u64_stats_fetch_begin_irq(&ring->syncp);
4399		*pkts = ring->stats.pkts;
4400		*bytes = ring->stats.bytes;
4401	} while (u64_stats_fetch_retry_irq(&ring->syncp, start));
4402}
4403
4404/**
4405 * ice_stat_update40 - read 40 bit stat from the chip and update stat values
4406 * @hw: ptr to the hardware info
4407 * @hireg: high 32 bit HW register to read from
4408 * @loreg: low 32 bit HW register to read from
4409 * @prev_stat_loaded: bool to specify if previous stats are loaded
4410 * @prev_stat: ptr to previous loaded stat value
4411 * @cur_stat: ptr to current stat value
4412 */
4413static void ice_stat_update40(struct ice_hw *hw, u32 hireg, u32 loreg,
4414			      bool prev_stat_loaded, u64 *prev_stat,
4415			      u64 *cur_stat)
4416{
4417	u64 new_data;
4418
4419	new_data = rd32(hw, loreg);
4420	new_data |= ((u64)(rd32(hw, hireg) & 0xFFFF)) << 32;
4421
4422	/* device stats are not reset at PFR, they likely will not be zeroed
4423	 * when the driver starts. So save the first values read and use them as
4424	 * offsets to be subtracted from the raw values in order to report stats
4425	 * that count from zero.
4426	 */
4427	if (!prev_stat_loaded)
4428		*prev_stat = new_data;
4429	if (likely(new_data >= *prev_stat))
4430		*cur_stat = new_data - *prev_stat;
4431	else
4432		/* to manage the potential roll-over */
4433		*cur_stat = (new_data + BIT_ULL(40)) - *prev_stat;
4434	*cur_stat &= 0xFFFFFFFFFFULL;
4435}
4436
4437/**
4438 * ice_stat_update32 - read 32 bit stat from the chip and update stat values
4439 * @hw: ptr to the hardware info
4440 * @reg: HW register to read from
4441 * @prev_stat_loaded: bool to specify if previous stats are loaded
4442 * @prev_stat: ptr to previous loaded stat value
4443 * @cur_stat: ptr to current stat value
4444 */
4445static void ice_stat_update32(struct ice_hw *hw, u32 reg, bool prev_stat_loaded,
4446			      u64 *prev_stat, u64 *cur_stat)
4447{
4448	u32 new_data;
4449
4450	new_data = rd32(hw, reg);
4451
4452	/* device stats are not reset at PFR, they likely will not be zeroed
4453	 * when the driver starts. So save the first values read and use them as
4454	 * offsets to be subtracted from the raw values in order to report stats
4455	 * that count from zero.
4456	 */
4457	if (!prev_stat_loaded)
4458		*prev_stat = new_data;
4459	if (likely(new_data >= *prev_stat))
4460		*cur_stat = new_data - *prev_stat;
4461	else
4462		/* to manage the potential roll-over */
4463		*cur_stat = (new_data + BIT_ULL(32)) - *prev_stat;
4464}
4465
4466/**
4467 * ice_update_eth_stats - Update VSI-specific ethernet statistics counters
4468 * @vsi: the VSI to be updated
4469 */
4470static void ice_update_eth_stats(struct ice_vsi *vsi)
4471{
4472	struct ice_eth_stats *prev_es, *cur_es;
4473	struct ice_hw *hw = &vsi->back->hw;
4474	u16 vsi_num = vsi->vsi_num;    /* HW absolute index of a VSI */
4475
4476	prev_es = &vsi->eth_stats_prev;
4477	cur_es = &vsi->eth_stats;
4478
4479	ice_stat_update40(hw, GLV_GORCH(vsi_num), GLV_GORCL(vsi_num),
4480			  vsi->stat_offsets_loaded, &prev_es->rx_bytes,
4481			  &cur_es->rx_bytes);
4482
4483	ice_stat_update40(hw, GLV_UPRCH(vsi_num), GLV_UPRCL(vsi_num),
4484			  vsi->stat_offsets_loaded, &prev_es->rx_unicast,
4485			  &cur_es->rx_unicast);
4486
4487	ice_stat_update40(hw, GLV_MPRCH(vsi_num), GLV_MPRCL(vsi_num),
4488			  vsi->stat_offsets_loaded, &prev_es->rx_multicast,
4489			  &cur_es->rx_multicast);
4490
4491	ice_stat_update40(hw, GLV_BPRCH(vsi_num), GLV_BPRCL(vsi_num),
4492			  vsi->stat_offsets_loaded, &prev_es->rx_broadcast,
4493			  &cur_es->rx_broadcast);
4494
4495	ice_stat_update32(hw, GLV_RDPC(vsi_num), vsi->stat_offsets_loaded,
4496			  &prev_es->rx_discards, &cur_es->rx_discards);
4497
4498	ice_stat_update40(hw, GLV_GOTCH(vsi_num), GLV_GOTCL(vsi_num),
4499			  vsi->stat_offsets_loaded, &prev_es->tx_bytes,
4500			  &cur_es->tx_bytes);
4501
4502	ice_stat_update40(hw, GLV_UPTCH(vsi_num), GLV_UPTCL(vsi_num),
4503			  vsi->stat_offsets_loaded, &prev_es->tx_unicast,
4504			  &cur_es->tx_unicast);
4505
4506	ice_stat_update40(hw, GLV_MPTCH(vsi_num), GLV_MPTCL(vsi_num),
4507			  vsi->stat_offsets_loaded, &prev_es->tx_multicast,
4508			  &cur_es->tx_multicast);
4509
4510	ice_stat_update40(hw, GLV_BPTCH(vsi_num), GLV_BPTCL(vsi_num),
4511			  vsi->stat_offsets_loaded, &prev_es->tx_broadcast,
4512			  &cur_es->tx_broadcast);
4513
4514	ice_stat_update32(hw, GLV_TEPC(vsi_num), vsi->stat_offsets_loaded,
4515			  &prev_es->tx_errors, &cur_es->tx_errors);
4516
4517	vsi->stat_offsets_loaded = true;
4518}
4519
4520/**
4521 * ice_update_vsi_ring_stats - Update VSI stats counters
4522 * @vsi: the VSI to be updated
4523 */
4524static void ice_update_vsi_ring_stats(struct ice_vsi *vsi)
4525{
4526	struct rtnl_link_stats64 *vsi_stats = &vsi->net_stats;
4527	struct ice_ring *ring;
4528	u64 pkts, bytes;
4529	int i;
4530
4531	/* reset netdev stats */
4532	vsi_stats->tx_packets = 0;
4533	vsi_stats->tx_bytes = 0;
4534	vsi_stats->rx_packets = 0;
4535	vsi_stats->rx_bytes = 0;
4536
4537	/* reset non-netdev (extended) stats */
4538	vsi->tx_restart = 0;
4539	vsi->tx_busy = 0;
4540	vsi->tx_linearize = 0;
4541	vsi->rx_buf_failed = 0;
4542	vsi->rx_page_failed = 0;
4543
4544	rcu_read_lock();
4545
4546	/* update Tx rings counters */
4547	ice_for_each_txq(vsi, i) {
4548		ring = READ_ONCE(vsi->tx_rings[i]);
4549		ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes);
4550		vsi_stats->tx_packets += pkts;
4551		vsi_stats->tx_bytes += bytes;
4552		vsi->tx_restart += ring->tx_stats.restart_q;
4553		vsi->tx_busy += ring->tx_stats.tx_busy;
4554		vsi->tx_linearize += ring->tx_stats.tx_linearize;
4555	}
4556
4557	/* update Rx rings counters */
4558	ice_for_each_rxq(vsi, i) {
4559		ring = READ_ONCE(vsi->rx_rings[i]);
4560		ice_fetch_u64_stats_per_ring(ring, &pkts, &bytes);
4561		vsi_stats->rx_packets += pkts;
4562		vsi_stats->rx_bytes += bytes;
4563		vsi->rx_buf_failed += ring->rx_stats.alloc_buf_failed;
4564		vsi->rx_page_failed += ring->rx_stats.alloc_page_failed;
4565	}
4566
4567	rcu_read_unlock();
4568}
4569
4570/**
4571 * ice_update_vsi_stats - Update VSI stats counters
4572 * @vsi: the VSI to be updated
4573 */
4574static void ice_update_vsi_stats(struct ice_vsi *vsi)
4575{
4576	struct rtnl_link_stats64 *cur_ns = &vsi->net_stats;
4577	struct ice_eth_stats *cur_es = &vsi->eth_stats;
4578	struct ice_pf *pf = vsi->back;
4579
4580	if (test_bit(__ICE_DOWN, vsi->state) ||
4581	    test_bit(__ICE_CFG_BUSY, pf->state))
4582		return;
4583
4584	/* get stats as recorded by Tx/Rx rings */
4585	ice_update_vsi_ring_stats(vsi);
4586
4587	/* get VSI stats as recorded by the hardware */
4588	ice_update_eth_stats(vsi);
4589
4590	cur_ns->tx_errors = cur_es->tx_errors;
4591	cur_ns->rx_dropped = cur_es->rx_discards;
4592	cur_ns->tx_dropped = cur_es->tx_discards;
4593	cur_ns->multicast = cur_es->rx_multicast;
4594
4595	/* update some more netdev stats if this is main VSI */
4596	if (vsi->type == ICE_VSI_PF) {
4597		cur_ns->rx_crc_errors = pf->stats.crc_errors;
4598		cur_ns->rx_errors = pf->stats.crc_errors +
4599				    pf->stats.illegal_bytes;
4600		cur_ns->rx_length_errors = pf->stats.rx_len_errors;
4601	}
4602}
4603
4604/**
4605 * ice_update_pf_stats - Update PF port stats counters
4606 * @pf: PF whose stats needs to be updated
4607 */
4608static void ice_update_pf_stats(struct ice_pf *pf)
4609{
4610	struct ice_hw_port_stats *prev_ps, *cur_ps;
4611	struct ice_hw *hw = &pf->hw;
4612	u8 pf_id;
4613
4614	prev_ps = &pf->stats_prev;
4615	cur_ps = &pf->stats;
4616	pf_id = hw->pf_id;
4617
4618	ice_stat_update40(hw, GLPRT_GORCH(pf_id), GLPRT_GORCL(pf_id),
4619			  pf->stat_prev_loaded, &prev_ps->eth.rx_bytes,
4620			  &cur_ps->eth.rx_bytes);
4621
4622	ice_stat_update40(hw, GLPRT_UPRCH(pf_id), GLPRT_UPRCL(pf_id),
4623			  pf->stat_prev_loaded, &prev_ps->eth.rx_unicast,
4624			  &cur_ps->eth.rx_unicast);
4625
4626	ice_stat_update40(hw, GLPRT_MPRCH(pf_id), GLPRT_MPRCL(pf_id),
4627			  pf->stat_prev_loaded, &prev_ps->eth.rx_multicast,
4628			  &cur_ps->eth.rx_multicast);
4629
4630	ice_stat_update40(hw, GLPRT_BPRCH(pf_id), GLPRT_BPRCL(pf_id),
4631			  pf->stat_prev_loaded, &prev_ps->eth.rx_broadcast,
4632			  &cur_ps->eth.rx_broadcast);
4633
4634	ice_stat_update40(hw, GLPRT_GOTCH(pf_id), GLPRT_GOTCL(pf_id),
4635			  pf->stat_prev_loaded, &prev_ps->eth.tx_bytes,
4636			  &cur_ps->eth.tx_bytes);
4637
4638	ice_stat_update40(hw, GLPRT_UPTCH(pf_id), GLPRT_UPTCL(pf_id),
4639			  pf->stat_prev_loaded, &prev_ps->eth.tx_unicast,
4640			  &cur_ps->eth.tx_unicast);
4641
4642	ice_stat_update40(hw, GLPRT_MPTCH(pf_id), GLPRT_MPTCL(pf_id),
4643			  pf->stat_prev_loaded, &prev_ps->eth.tx_multicast,
4644			  &cur_ps->eth.tx_multicast);
4645
4646	ice_stat_update40(hw, GLPRT_BPTCH(pf_id), GLPRT_BPTCL(pf_id),
4647			  pf->stat_prev_loaded, &prev_ps->eth.tx_broadcast,
4648			  &cur_ps->eth.tx_broadcast);
4649
4650	ice_stat_update32(hw, GLPRT_TDOLD(pf_id), pf->stat_prev_loaded,
4651			  &prev_ps->tx_dropped_link_down,
4652			  &cur_ps->tx_dropped_link_down);
4653
4654	ice_stat_update40(hw, GLPRT_PRC64H(pf_id), GLPRT_PRC64L(pf_id),
4655			  pf->stat_prev_loaded, &prev_ps->rx_size_64,
4656			  &cur_ps->rx_size_64);
4657
4658	ice_stat_update40(hw, GLPRT_PRC127H(pf_id), GLPRT_PRC127L(pf_id),
4659			  pf->stat_prev_loaded, &prev_ps->rx_size_127,
4660			  &cur_ps->rx_size_127);
4661
4662	ice_stat_update40(hw, GLPRT_PRC255H(pf_id), GLPRT_PRC255L(pf_id),
4663			  pf->stat_prev_loaded, &prev_ps->rx_size_255,
4664			  &cur_ps->rx_size_255);
4665
4666	ice_stat_update40(hw, GLPRT_PRC511H(pf_id), GLPRT_PRC511L(pf_id),
4667			  pf->stat_prev_loaded, &prev_ps->rx_size_511,
4668			  &cur_ps->rx_size_511);
4669
4670	ice_stat_update40(hw, GLPRT_PRC1023H(pf_id),
4671			  GLPRT_PRC1023L(pf_id), pf->stat_prev_loaded,
4672			  &prev_ps->rx_size_1023, &cur_ps->rx_size_1023);
4673
4674	ice_stat_update40(hw, GLPRT_PRC1522H(pf_id),
4675			  GLPRT_PRC1522L(pf_id), pf->stat_prev_loaded,
4676			  &prev_ps->rx_size_1522, &cur_ps->rx_size_1522);
4677
4678	ice_stat_update40(hw, GLPRT_PRC9522H(pf_id),
4679			  GLPRT_PRC9522L(pf_id), pf->stat_prev_loaded,
4680			  &prev_ps->rx_size_big, &cur_ps->rx_size_big);
4681
4682	ice_stat_update40(hw, GLPRT_PTC64H(pf_id), GLPRT_PTC64L(pf_id),
4683			  pf->stat_prev_loaded, &prev_ps->tx_size_64,
4684			  &cur_ps->tx_size_64);
4685
4686	ice_stat_update40(hw, GLPRT_PTC127H(pf_id), GLPRT_PTC127L(pf_id),
4687			  pf->stat_prev_loaded, &prev_ps->tx_size_127,
4688			  &cur_ps->tx_size_127);
4689
4690	ice_stat_update40(hw, GLPRT_PTC255H(pf_id), GLPRT_PTC255L(pf_id),
4691			  pf->stat_prev_loaded, &prev_ps->tx_size_255,
4692			  &cur_ps->tx_size_255);
4693
4694	ice_stat_update40(hw, GLPRT_PTC511H(pf_id), GLPRT_PTC511L(pf_id),
4695			  pf->stat_prev_loaded, &prev_ps->tx_size_511,
4696			  &cur_ps->tx_size_511);
4697
4698	ice_stat_update40(hw, GLPRT_PTC1023H(pf_id),
4699			  GLPRT_PTC1023L(pf_id), pf->stat_prev_loaded,
4700			  &prev_ps->tx_size_1023, &cur_ps->tx_size_1023);
4701
4702	ice_stat_update40(hw, GLPRT_PTC1522H(pf_id),
4703			  GLPRT_PTC1522L(pf_id), pf->stat_prev_loaded,
4704			  &prev_ps->tx_size_1522, &cur_ps->tx_size_1522);
4705
4706	ice_stat_update40(hw, GLPRT_PTC9522H(pf_id),
4707			  GLPRT_PTC9522L(pf_id), pf->stat_prev_loaded,
4708			  &prev_ps->tx_size_big, &cur_ps->tx_size_big);
4709
4710	ice_stat_update32(hw, GLPRT_LXONRXC(pf_id), pf->stat_prev_loaded,
4711			  &prev_ps->link_xon_rx, &cur_ps->link_xon_rx);
4712
4713	ice_stat_update32(hw, GLPRT_LXOFFRXC(pf_id), pf->stat_prev_loaded,
4714			  &prev_ps->link_xoff_rx, &cur_ps->link_xoff_rx);
4715
4716	ice_stat_update32(hw, GLPRT_LXONTXC(pf_id), pf->stat_prev_loaded,
4717			  &prev_ps->link_xon_tx, &cur_ps->link_xon_tx);
4718
4719	ice_stat_update32(hw, GLPRT_LXOFFTXC(pf_id), pf->stat_prev_loaded,
4720			  &prev_ps->link_xoff_tx, &cur_ps->link_xoff_tx);
4721
4722	ice_stat_update32(hw, GLPRT_CRCERRS(pf_id), pf->stat_prev_loaded,
4723			  &prev_ps->crc_errors, &cur_ps->crc_errors);
4724
4725	ice_stat_update32(hw, GLPRT_ILLERRC(pf_id), pf->stat_prev_loaded,
4726			  &prev_ps->illegal_bytes, &cur_ps->illegal_bytes);
4727
4728	ice_stat_update32(hw, GLPRT_MLFC(pf_id), pf->stat_prev_loaded,
4729			  &prev_ps->mac_local_faults,
4730			  &cur_ps->mac_local_faults);
4731
4732	ice_stat_update32(hw, GLPRT_MRFC(pf_id), pf->stat_prev_loaded,
4733			  &prev_ps->mac_remote_faults,
4734			  &cur_ps->mac_remote_faults);
4735
4736	ice_stat_update32(hw, GLPRT_RLEC(pf_id), pf->stat_prev_loaded,
4737			  &prev_ps->rx_len_errors, &cur_ps->rx_len_errors);
4738
4739	ice_stat_update32(hw, GLPRT_RUC(pf_id), pf->stat_prev_loaded,
4740			  &prev_ps->rx_undersize, &cur_ps->rx_undersize);
4741
4742	ice_stat_update32(hw, GLPRT_RFC(pf_id), pf->stat_prev_loaded,
4743			  &prev_ps->rx_fragments, &cur_ps->rx_fragments);
4744
4745	ice_stat_update32(hw, GLPRT_ROC(pf_id), pf->stat_prev_loaded,
4746			  &prev_ps->rx_oversize, &cur_ps->rx_oversize);
4747
4748	ice_stat_update32(hw, GLPRT_RJC(pf_id), pf->stat_prev_loaded,
4749			  &prev_ps->rx_jabber, &cur_ps->rx_jabber);
4750
4751	pf->stat_prev_loaded = true;
4752}
4753
4754/**
4755 * ice_get_stats64 - get statistics for network device structure
4756 * @netdev: network interface device structure
4757 * @stats: main device statistics structure
4758 */
4759static
4760void ice_get_stats64(struct net_device *netdev, struct rtnl_link_stats64 *stats)
4761{
4762	struct ice_netdev_priv *np = netdev_priv(netdev);
4763	struct rtnl_link_stats64 *vsi_stats;
4764	struct ice_vsi *vsi = np->vsi;
4765
4766	vsi_stats = &vsi->net_stats;
4767
4768	if (test_bit(__ICE_DOWN, vsi->state) || !vsi->num_txq || !vsi->num_rxq)
4769		return;
4770	/* netdev packet/byte stats come from ring counter. These are obtained
4771	 * by summing up ring counters (done by ice_update_vsi_ring_stats).
4772	 */
4773	ice_update_vsi_ring_stats(vsi);
4774	stats->tx_packets = vsi_stats->tx_packets;
4775	stats->tx_bytes = vsi_stats->tx_bytes;
4776	stats->rx_packets = vsi_stats->rx_packets;
4777	stats->rx_bytes = vsi_stats->rx_bytes;
4778
4779	/* The rest of the stats can be read from the hardware but instead we
4780	 * just return values that the watchdog task has already obtained from
4781	 * the hardware.
4782	 */
4783	stats->multicast = vsi_stats->multicast;
4784	stats->tx_errors = vsi_stats->tx_errors;
4785	stats->tx_dropped = vsi_stats->tx_dropped;
4786	stats->rx_errors = vsi_stats->rx_errors;
4787	stats->rx_dropped = vsi_stats->rx_dropped;
4788	stats->rx_crc_errors = vsi_stats->rx_crc_errors;
4789	stats->rx_length_errors = vsi_stats->rx_length_errors;
4790}
4791
4792#ifdef CONFIG_NET_POLL_CONTROLLER
4793/**
4794 * ice_netpoll - polling "interrupt" handler
4795 * @netdev: network interface device structure
4796 *
4797 * Used by netconsole to send skbs without having to re-enable interrupts.
4798 * This is not called in the normal interrupt path.
4799 */
4800static void ice_netpoll(struct net_device *netdev)
4801{
4802	struct ice_netdev_priv *np = netdev_priv(netdev);
4803	struct ice_vsi *vsi = np->vsi;
4804	struct ice_pf *pf = vsi->back;
4805	int i;
4806
4807	if (test_bit(__ICE_DOWN, vsi->state) ||
4808	    !test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
4809		return;
4810
4811	for (i = 0; i < vsi->num_q_vectors; i++)
4812		ice_msix_clean_rings(0, vsi->q_vectors[i]);
4813}
4814#endif /* CONFIG_NET_POLL_CONTROLLER */
4815
4816/**
4817 * ice_napi_disable_all - Disable NAPI for all q_vectors in the VSI
4818 * @vsi: VSI having NAPI disabled
4819 */
4820static void ice_napi_disable_all(struct ice_vsi *vsi)
4821{
4822	int q_idx;
4823
4824	if (!vsi->netdev)
4825		return;
4826
4827	for (q_idx = 0; q_idx < vsi->num_q_vectors; q_idx++)
4828		napi_disable(&vsi->q_vectors[q_idx]->napi);
4829}
4830
4831/**
4832 * ice_down - Shutdown the connection
4833 * @vsi: The VSI being stopped
4834 */
4835int ice_down(struct ice_vsi *vsi)
4836{
4837	int i, err;
4838
4839	/* Caller of this function is expected to set the
4840	 * vsi->state __ICE_DOWN bit
4841	 */
4842	if (vsi->netdev) {
4843		netif_carrier_off(vsi->netdev);
4844		netif_tx_disable(vsi->netdev);
4845	}
4846
4847	ice_vsi_dis_irq(vsi);
4848	err = ice_vsi_stop_tx_rx_rings(vsi);
4849	ice_napi_disable_all(vsi);
4850
4851	ice_for_each_txq(vsi, i)
4852		ice_clean_tx_ring(vsi->tx_rings[i]);
4853
4854	ice_for_each_rxq(vsi, i)
4855		ice_clean_rx_ring(vsi->rx_rings[i]);
4856
4857	if (err)
4858		netdev_err(vsi->netdev, "Failed to close VSI 0x%04X on switch 0x%04X\n",
4859			   vsi->vsi_num, vsi->vsw->sw_id);
4860	return err;
4861}
4862
4863/**
4864 * ice_vsi_setup_tx_rings - Allocate VSI Tx queue resources
4865 * @vsi: VSI having resources allocated
4866 *
4867 * Return 0 on success, negative on failure
4868 */
4869static int ice_vsi_setup_tx_rings(struct ice_vsi *vsi)
4870{
4871	int i, err;
4872
4873	if (!vsi->num_txq) {
4874		dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Tx queues\n",
4875			vsi->vsi_num);
4876		return -EINVAL;
4877	}
4878
4879	ice_for_each_txq(vsi, i) {
4880		err = ice_setup_tx_ring(vsi->tx_rings[i]);
4881		if (err)
4882			break;
4883	}
4884
4885	return err;
4886}
4887
4888/**
4889 * ice_vsi_setup_rx_rings - Allocate VSI Rx queue resources
4890 * @vsi: VSI having resources allocated
4891 *
4892 * Return 0 on success, negative on failure
4893 */
4894static int ice_vsi_setup_rx_rings(struct ice_vsi *vsi)
4895{
4896	int i, err;
4897
4898	if (!vsi->num_rxq) {
4899		dev_err(&vsi->back->pdev->dev, "VSI %d has 0 Rx queues\n",
4900			vsi->vsi_num);
4901		return -EINVAL;
4902	}
4903
4904	ice_for_each_rxq(vsi, i) {
4905		err = ice_setup_rx_ring(vsi->rx_rings[i]);
4906		if (err)
4907			break;
4908	}
4909
4910	return err;
4911}
4912
4913/**
4914 * ice_vsi_req_irq - Request IRQ from the OS
4915 * @vsi: The VSI IRQ is being requested for
4916 * @basename: name for the vector
4917 *
4918 * Return 0 on success and a negative value on error
4919 */
4920static int ice_vsi_req_irq(struct ice_vsi *vsi, char *basename)
4921{
4922	struct ice_pf *pf = vsi->back;
4923	int err = -EINVAL;
4924
4925	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags))
4926		err = ice_vsi_req_irq_msix(vsi, basename);
4927
4928	return err;
4929}
4930
4931/**
4932 * ice_vsi_free_tx_rings - Free Tx resources for VSI queues
4933 * @vsi: the VSI having resources freed
4934 */
4935static void ice_vsi_free_tx_rings(struct ice_vsi *vsi)
4936{
4937	int i;
4938
4939	if (!vsi->tx_rings)
4940		return;
4941
4942	ice_for_each_txq(vsi, i)
4943		if (vsi->tx_rings[i] && vsi->tx_rings[i]->desc)
4944			ice_free_tx_ring(vsi->tx_rings[i]);
4945}
4946
4947/**
4948 * ice_vsi_free_rx_rings - Free Rx resources for VSI queues
4949 * @vsi: the VSI having resources freed
4950 */
4951static void ice_vsi_free_rx_rings(struct ice_vsi *vsi)
4952{
4953	int i;
4954
4955	if (!vsi->rx_rings)
4956		return;
4957
4958	ice_for_each_rxq(vsi, i)
4959		if (vsi->rx_rings[i] && vsi->rx_rings[i]->desc)
4960			ice_free_rx_ring(vsi->rx_rings[i]);
4961}
4962
4963/**
4964 * ice_vsi_open - Called when a network interface is made active
4965 * @vsi: the VSI to open
4966 *
4967 * Initialization of the VSI
4968 *
4969 * Returns 0 on success, negative value on error
4970 */
4971static int ice_vsi_open(struct ice_vsi *vsi)
4972{
4973	char int_name[ICE_INT_NAME_STR_LEN];
4974	struct ice_pf *pf = vsi->back;
4975	int err;
4976
4977	/* allocate descriptors */
4978	err = ice_vsi_setup_tx_rings(vsi);
4979	if (err)
4980		goto err_setup_tx;
4981
4982	err = ice_vsi_setup_rx_rings(vsi);
4983	if (err)
4984		goto err_setup_rx;
4985
4986	err = ice_vsi_cfg(vsi);
4987	if (err)
4988		goto err_setup_rx;
4989
4990	snprintf(int_name, sizeof(int_name) - 1, "%s-%s",
4991		 dev_driver_string(&pf->pdev->dev), vsi->netdev->name);
4992	err = ice_vsi_req_irq(vsi, int_name);
4993	if (err)
4994		goto err_setup_rx;
4995
4996	/* Notify the stack of the actual queue counts. */
4997	err = netif_set_real_num_tx_queues(vsi->netdev, vsi->num_txq);
4998	if (err)
4999		goto err_set_qs;
5000
5001	err = netif_set_real_num_rx_queues(vsi->netdev, vsi->num_rxq);
5002	if (err)
5003		goto err_set_qs;
5004
5005	err = ice_up_complete(vsi);
5006	if (err)
5007		goto err_up_complete;
5008
5009	return 0;
5010
5011err_up_complete:
5012	ice_down(vsi);
5013err_set_qs:
5014	ice_vsi_free_irq(vsi);
5015err_setup_rx:
5016	ice_vsi_free_rx_rings(vsi);
5017err_setup_tx:
5018	ice_vsi_free_tx_rings(vsi);
5019
5020	return err;
5021}
5022
5023/**
5024 * ice_vsi_close - Shut down a VSI
5025 * @vsi: the VSI being shut down
5026 */
5027static void ice_vsi_close(struct ice_vsi *vsi)
5028{
5029	if (!test_and_set_bit(__ICE_DOWN, vsi->state))
5030		ice_down(vsi);
5031
5032	ice_vsi_free_irq(vsi);
5033	ice_vsi_free_tx_rings(vsi);
5034	ice_vsi_free_rx_rings(vsi);
5035}
5036
5037/**
5038 * ice_rss_clean - Delete RSS related VSI structures that hold user inputs
5039 * @vsi: the VSI being removed
5040 */
5041static void ice_rss_clean(struct ice_vsi *vsi)
5042{
5043	struct ice_pf *pf;
5044
5045	pf = vsi->back;
5046
5047	if (vsi->rss_hkey_user)
5048		devm_kfree(&pf->pdev->dev, vsi->rss_hkey_user);
5049	if (vsi->rss_lut_user)
5050		devm_kfree(&pf->pdev->dev, vsi->rss_lut_user);
5051}
5052
5053/**
5054 * ice_vsi_release - Delete a VSI and free its resources
5055 * @vsi: the VSI being removed
5056 *
5057 * Returns 0 on success or < 0 on error
5058 */
5059static int ice_vsi_release(struct ice_vsi *vsi)
5060{
5061	struct ice_pf *pf;
5062
5063	if (!vsi->back)
5064		return -ENODEV;
5065	pf = vsi->back;
5066
5067	if (vsi->netdev) {
5068		unregister_netdev(vsi->netdev);
5069		free_netdev(vsi->netdev);
5070		vsi->netdev = NULL;
5071	}
5072
5073	if (test_bit(ICE_FLAG_RSS_ENA, pf->flags))
5074		ice_rss_clean(vsi);
5075
5076	/* Disable VSI and free resources */
5077	ice_vsi_dis_irq(vsi);
5078	ice_vsi_close(vsi);
5079
5080	/* reclaim interrupt vectors back to PF */
5081	ice_free_res(vsi->back->irq_tracker, vsi->base_vector, vsi->idx);
5082	pf->num_avail_msix += vsi->num_q_vectors;
5083
5084	ice_remove_vsi_fltr(&pf->hw, vsi->vsi_num);
5085	ice_vsi_delete(vsi);
5086	ice_vsi_free_q_vectors(vsi);
5087	ice_vsi_clear_rings(vsi);
5088
5089	ice_vsi_put_qs(vsi);
5090	pf->q_left_tx += vsi->alloc_txq;
5091	pf->q_left_rx += vsi->alloc_rxq;
5092
5093	ice_vsi_clear(vsi);
5094
5095	return 0;
5096}
5097
5098/**
5099 * ice_dis_vsi - pause a VSI
5100 * @vsi: the VSI being paused
5101 */
5102static void ice_dis_vsi(struct ice_vsi *vsi)
5103{
5104	if (test_bit(__ICE_DOWN, vsi->state))
5105		return;
5106
5107	set_bit(__ICE_NEEDS_RESTART, vsi->state);
5108
5109	if (vsi->netdev && netif_running(vsi->netdev) &&
5110	    vsi->type == ICE_VSI_PF)
5111		vsi->netdev->netdev_ops->ndo_stop(vsi->netdev);
5112
5113	ice_vsi_close(vsi);
5114}
5115
5116/**
5117 * ice_ena_vsi - resume a VSI
5118 * @vsi: the VSI being resume
5119 */
5120static void ice_ena_vsi(struct ice_vsi *vsi)
5121{
5122	if (!test_and_clear_bit(__ICE_NEEDS_RESTART, vsi->state))
5123		return;
5124
5125	if (vsi->netdev && netif_running(vsi->netdev))
5126		vsi->netdev->netdev_ops->ndo_open(vsi->netdev);
5127	else if (ice_vsi_open(vsi))
5128		/* this clears the DOWN bit */
5129		dev_dbg(&vsi->back->pdev->dev, "Failed open VSI 0x%04X on switch 0x%04X\n",
5130			vsi->vsi_num, vsi->vsw->sw_id);
5131}
5132
5133/**
5134 * ice_pf_dis_all_vsi - Pause all VSIs on a PF
5135 * @pf: the PF
5136 */
5137static void ice_pf_dis_all_vsi(struct ice_pf *pf)
5138{
5139	int v;
5140
5141	ice_for_each_vsi(pf, v)
5142		if (pf->vsi[v])
5143			ice_dis_vsi(pf->vsi[v]);
5144}
5145
5146/**
5147 * ice_pf_ena_all_vsi - Resume all VSIs on a PF
5148 * @pf: the PF
5149 */
5150static void ice_pf_ena_all_vsi(struct ice_pf *pf)
5151{
5152	int v;
5153
5154	ice_for_each_vsi(pf, v)
5155		if (pf->vsi[v])
5156			ice_ena_vsi(pf->vsi[v]);
5157}
5158
5159/**
5160 * ice_rebuild - rebuild after reset
5161 * @pf: pf to rebuild
5162 */
5163static void ice_rebuild(struct ice_pf *pf)
5164{
5165	struct device *dev = &pf->pdev->dev;
5166	struct ice_hw *hw = &pf->hw;
5167	enum ice_status ret;
5168	int err;
5169
5170	if (test_bit(__ICE_DOWN, pf->state))
5171		goto clear_recovery;
5172
5173	dev_dbg(dev, "rebuilding pf\n");
5174
5175	ret = ice_init_all_ctrlq(hw);
5176	if (ret) {
5177		dev_err(dev, "control queues init failed %d\n", ret);
5178		goto fail_reset;
5179	}
5180
5181	ret = ice_clear_pf_cfg(hw);
5182	if (ret) {
5183		dev_err(dev, "clear PF configuration failed %d\n", ret);
5184		goto fail_reset;
5185	}
5186
5187	ice_clear_pxe_mode(hw);
5188
5189	ret = ice_get_caps(hw);
5190	if (ret) {
5191		dev_err(dev, "ice_get_caps failed %d\n", ret);
5192		goto fail_reset;
5193	}
5194
5195	/* basic nic switch setup */
5196	err = ice_setup_pf_sw(pf);
5197	if (err) {
5198		dev_err(dev, "ice_setup_pf_sw failed\n");
5199		goto fail_reset;
5200	}
5201
5202	/* start misc vector */
5203	if (test_bit(ICE_FLAG_MSIX_ENA, pf->flags)) {
5204		err = ice_req_irq_msix_misc(pf);
5205		if (err) {
5206			dev_err(dev, "misc vector setup failed: %d\n", err);
5207			goto fail_reset;
5208		}
5209	}
5210
5211	/* restart the VSIs that were rebuilt and running before the reset */
5212	ice_pf_ena_all_vsi(pf);
5213
5214	return;
5215
5216fail_reset:
5217	ice_shutdown_all_ctrlq(hw);
5218	set_bit(__ICE_RESET_FAILED, pf->state);
5219clear_recovery:
5220	set_bit(__ICE_RESET_RECOVERY_PENDING, pf->state);
5221}
5222
5223/**
5224 * ice_change_mtu - NDO callback to change the MTU
5225 * @netdev: network interface device structure
5226 * @new_mtu: new value for maximum frame size
5227 *
5228 * Returns 0 on success, negative on failure
5229 */
5230static int ice_change_mtu(struct net_device *netdev, int new_mtu)
5231{
5232	struct ice_netdev_priv *np = netdev_priv(netdev);
5233	struct ice_vsi *vsi = np->vsi;
5234	struct ice_pf *pf = vsi->back;
5235	u8 count = 0;
5236
5237	if (new_mtu == netdev->mtu) {
5238		netdev_warn(netdev, "mtu is already %d\n", netdev->mtu);
5239		return 0;
5240	}
5241
5242	if (new_mtu < netdev->min_mtu) {
5243		netdev_err(netdev, "new mtu invalid. min_mtu is %d\n",
5244			   netdev->min_mtu);
5245		return -EINVAL;
5246	} else if (new_mtu > netdev->max_mtu) {
5247		netdev_err(netdev, "new mtu invalid. max_mtu is %d\n",
5248			   netdev->min_mtu);
5249		return -EINVAL;
5250	}
5251	/* if a reset is in progress, wait for some time for it to complete */
5252	do {
5253		if (ice_is_reset_recovery_pending(pf->state)) {
5254			count++;
5255			usleep_range(1000, 2000);
5256		} else {
5257			break;
5258		}
5259
5260	} while (count < 100);
5261
5262	if (count == 100) {
5263		netdev_err(netdev, "can't change mtu. Device is busy\n");
5264		return -EBUSY;
5265	}
5266
5267	netdev->mtu = new_mtu;
5268
5269	/* if VSI is up, bring it down and then back up */
5270	if (!test_and_set_bit(__ICE_DOWN, vsi->state)) {
5271		int err;
5272
5273		err = ice_down(vsi);
5274		if (err) {
5275			netdev_err(netdev, "change mtu if_up err %d\n", err);
5276			return err;
5277		}
5278
5279		err = ice_up(vsi);
5280		if (err) {
5281			netdev_err(netdev, "change mtu if_up err %d\n", err);
5282			return err;
5283		}
5284	}
5285
5286	netdev_dbg(netdev, "changed mtu to %d\n", new_mtu);
5287	return 0;
5288}
5289
5290/**
5291 * ice_set_rss - Set RSS keys and lut
5292 * @vsi: Pointer to VSI structure
5293 * @seed: RSS hash seed
5294 * @lut: Lookup table
5295 * @lut_size: Lookup table size
5296 *
5297 * Returns 0 on success, negative on failure
5298 */
5299int ice_set_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
5300{
5301	struct ice_pf *pf = vsi->back;
5302	struct ice_hw *hw = &pf->hw;
5303	enum ice_status status;
5304
5305	if (seed) {
5306		struct ice_aqc_get_set_rss_keys *buf =
5307				  (struct ice_aqc_get_set_rss_keys *)seed;
5308
5309		status = ice_aq_set_rss_key(hw, vsi->vsi_num, buf);
5310
5311		if (status) {
5312			dev_err(&pf->pdev->dev,
5313				"Cannot set RSS key, err %d aq_err %d\n",
5314				status, hw->adminq.rq_last_status);
5315			return -EIO;
5316		}
5317	}
5318
5319	if (lut) {
5320		status = ice_aq_set_rss_lut(hw, vsi->vsi_num,
5321					    vsi->rss_lut_type, lut, lut_size);
5322		if (status) {
5323			dev_err(&pf->pdev->dev,
5324				"Cannot set RSS lut, err %d aq_err %d\n",
5325				status, hw->adminq.rq_last_status);
5326			return -EIO;
5327		}
5328	}
5329
5330	return 0;
5331}
5332
5333/**
5334 * ice_get_rss - Get RSS keys and lut
5335 * @vsi: Pointer to VSI structure
5336 * @seed: Buffer to store the keys
5337 * @lut: Buffer to store the lookup table entries
5338 * @lut_size: Size of buffer to store the lookup table entries
5339 *
5340 * Returns 0 on success, negative on failure
5341 */
5342int ice_get_rss(struct ice_vsi *vsi, u8 *seed, u8 *lut, u16 lut_size)
5343{
5344	struct ice_pf *pf = vsi->back;
5345	struct ice_hw *hw = &pf->hw;
5346	enum ice_status status;
5347
5348	if (seed) {
5349		struct ice_aqc_get_set_rss_keys *buf =
5350				  (struct ice_aqc_get_set_rss_keys *)seed;
5351
5352		status = ice_aq_get_rss_key(hw, vsi->vsi_num, buf);
5353		if (status) {
5354			dev_err(&pf->pdev->dev,
5355				"Cannot get RSS key, err %d aq_err %d\n",
5356				status, hw->adminq.rq_last_status);
5357			return -EIO;
5358		}
5359	}
5360
5361	if (lut) {
5362		status = ice_aq_get_rss_lut(hw, vsi->vsi_num,
5363					    vsi->rss_lut_type, lut, lut_size);
5364		if (status) {
5365			dev_err(&pf->pdev->dev,
5366				"Cannot get RSS lut, err %d aq_err %d\n",
5367				status, hw->adminq.rq_last_status);
5368			return -EIO;
5369		}
5370	}
5371
5372	return 0;
5373}
5374
5375/**
5376 * ice_open - Called when a network interface becomes active
5377 * @netdev: network interface device structure
5378 *
5379 * The open entry point is called when a network interface is made
5380 * active by the system (IFF_UP).  At this point all resources needed
5381 * for transmit and receive operations are allocated, the interrupt
5382 * handler is registered with the OS, the netdev watchdog is enabled,
5383 * and the stack is notified that the interface is ready.
5384 *
5385 * Returns 0 on success, negative value on failure
5386 */
5387static int ice_open(struct net_device *netdev)
5388{
5389	struct ice_netdev_priv *np = netdev_priv(netdev);
5390	struct ice_vsi *vsi = np->vsi;
5391	int err;
5392
5393	netif_carrier_off(netdev);
5394
5395	err = ice_vsi_open(vsi);
5396
5397	if (err)
5398		netdev_err(netdev, "Failed to open VSI 0x%04X on switch 0x%04X\n",
5399			   vsi->vsi_num, vsi->vsw->sw_id);
5400	return err;
5401}
5402
5403/**
5404 * ice_stop - Disables a network interface
5405 * @netdev: network interface device structure
5406 *
5407 * The stop entry point is called when an interface is de-activated by the OS,
5408 * and the netdevice enters the DOWN state.  The hardware is still under the
5409 * driver's control, but the netdev interface is disabled.
5410 *
5411 * Returns success only - not allowed to fail
5412 */
5413static int ice_stop(struct net_device *netdev)
5414{
5415	struct ice_netdev_priv *np = netdev_priv(netdev);
5416	struct ice_vsi *vsi = np->vsi;
5417
5418	ice_vsi_close(vsi);
5419
5420	return 0;
5421}
5422
5423/**
5424 * ice_features_check - Validate encapsulated packet conforms to limits
5425 * @skb: skb buffer
5426 * @netdev: This port's netdev
5427 * @features: Offload features that the stack believes apply
5428 */
5429static netdev_features_t
5430ice_features_check(struct sk_buff *skb,
5431		   struct net_device __always_unused *netdev,
5432		   netdev_features_t features)
5433{
5434	size_t len;
5435
5436	/* No point in doing any of this if neither checksum nor GSO are
5437	 * being requested for this frame.  We can rule out both by just
5438	 * checking for CHECKSUM_PARTIAL
5439	 */
5440	if (skb->ip_summed != CHECKSUM_PARTIAL)
5441		return features;
5442
5443	/* We cannot support GSO if the MSS is going to be less than
5444	 * 64 bytes.  If it is then we need to drop support for GSO.
5445	 */
5446	if (skb_is_gso(skb) && (skb_shinfo(skb)->gso_size < 64))
5447		features &= ~NETIF_F_GSO_MASK;
5448
5449	len = skb_network_header(skb) - skb->data;
5450	if (len & ~(ICE_TXD_MACLEN_MAX))
5451		goto out_rm_features;
5452
5453	len = skb_transport_header(skb) - skb_network_header(skb);
5454	if (len & ~(ICE_TXD_IPLEN_MAX))
5455		goto out_rm_features;
5456
5457	if (skb->encapsulation) {
5458		len = skb_inner_network_header(skb) - skb_transport_header(skb);
5459		if (len & ~(ICE_TXD_L4LEN_MAX))
5460			goto out_rm_features;
5461
5462		len = skb_inner_transport_header(skb) -
5463		      skb_inner_network_header(skb);
5464		if (len & ~(ICE_TXD_IPLEN_MAX))
5465			goto out_rm_features;
5466	}
5467
5468	return features;
5469out_rm_features:
5470	return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK);
5471}
5472
5473static const struct net_device_ops ice_netdev_ops = {
5474	.ndo_open = ice_open,
5475	.ndo_stop = ice_stop,
5476	.ndo_start_xmit = ice_start_xmit,
5477	.ndo_features_check = ice_features_check,
5478	.ndo_set_rx_mode = ice_set_rx_mode,
5479	.ndo_set_mac_address = ice_set_mac_address,
5480	.ndo_validate_addr = eth_validate_addr,
5481	.ndo_change_mtu = ice_change_mtu,
5482	.ndo_get_stats64 = ice_get_stats64,
5483#ifdef CONFIG_NET_POLL_CONTROLLER
5484	.ndo_poll_controller = ice_netpoll,
5485#endif /* CONFIG_NET_POLL_CONTROLLER */
5486	.ndo_vlan_rx_add_vid = ice_vlan_rx_add_vid,
5487	.ndo_vlan_rx_kill_vid = ice_vlan_rx_kill_vid,
5488	.ndo_set_features = ice_set_features,
5489	.ndo_fdb_add = ice_fdb_add,
5490	.ndo_fdb_del = ice_fdb_del,
5491};