1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright(c) 2013 - 2019 Intel Corporation. */
   3
   4#include <linux/types.h>
   5#include <linux/module.h>
   6#include <net/ipv6.h>
   7#include <net/ip.h>
   8#include <net/tcp.h>
   9#include <linux/if_macvlan.h>
  10#include <linux/prefetch.h>
  11
  12#include "fm10k.h"
  13
  14#define DRV_SUMMARY	"Intel(R) Ethernet Switch Host Interface Driver"
  15char fm10k_driver_name[] = "fm10k";
  16static const char fm10k_driver_string[] = DRV_SUMMARY;
  17static const char fm10k_copyright[] =
  18	"Copyright(c) 2013 - 2019 Intel Corporation.";
  19
  20MODULE_AUTHOR("Intel Corporation, <linux.nics@intel.com>");
  21MODULE_DESCRIPTION(DRV_SUMMARY);
  22MODULE_LICENSE("GPL v2");
  23
  24/* single workqueue for entire fm10k driver */
  25struct workqueue_struct *fm10k_workqueue;
  26
  27/**
  28 * fm10k_init_module - Driver Registration Routine
  29 *
  30 * fm10k_init_module is the first routine called when the driver is
  31 * loaded.  All it does is register with the PCI subsystem.
  32 **/
  33static int __init fm10k_init_module(void)
  34{
  35	int ret;
  36
  37	pr_info("%s\n", fm10k_driver_string);
  38	pr_info("%s\n", fm10k_copyright);
  39
  40	/* create driver workqueue */
  41	fm10k_workqueue = alloc_workqueue("%s", WQ_MEM_RECLAIM, 0,
  42					  fm10k_driver_name);
  43	if (!fm10k_workqueue)
  44		return -ENOMEM;
  45
  46	fm10k_dbg_init();
  47
  48	ret = fm10k_register_pci_driver();
  49	if (ret) {
  50		fm10k_dbg_exit();
  51		destroy_workqueue(fm10k_workqueue);
  52	}
  53
  54	return ret;
  55}
  56module_init(fm10k_init_module);
  57
  58/**
  59 * fm10k_exit_module - Driver Exit Cleanup Routine
  60 *
  61 * fm10k_exit_module is called just before the driver is removed
  62 * from memory.
  63 **/
  64static void __exit fm10k_exit_module(void)
  65{
  66	fm10k_unregister_pci_driver();
  67
  68	fm10k_dbg_exit();
  69
  70	/* destroy driver workqueue */
  71	destroy_workqueue(fm10k_workqueue);
  72}
  73module_exit(fm10k_exit_module);
  74
  75static bool fm10k_alloc_mapped_page(struct fm10k_ring *rx_ring,
  76				    struct fm10k_rx_buffer *bi)
  77{
  78	struct page *page = bi->page;
  79	dma_addr_t dma;
  80
  81	/* Only page will be NULL if buffer was consumed */
  82	if (likely(page))
  83		return true;
  84
  85	/* alloc new page for storage */
  86	page = dev_alloc_page();
  87	if (unlikely(!page)) {
  88		rx_ring->rx_stats.alloc_failed++;
  89		return false;
  90	}
  91
  92	/* map page for use */
  93	dma = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
  94
  95	/* if mapping failed free memory back to system since
  96	 * there isn't much point in holding memory we can't use
  97	 */
  98	if (dma_mapping_error(rx_ring->dev, dma)) {
  99		__free_page(page);
 100
 101		rx_ring->rx_stats.alloc_failed++;
 102		return false;
 103	}
 104
 105	bi->dma = dma;
 106	bi->page = page;
 107	bi->page_offset = 0;
 108
 109	return true;
 110}
 111
 112/**
 113 * fm10k_alloc_rx_buffers - Replace used receive buffers
 114 * @rx_ring: ring to place buffers on
 115 * @cleaned_count: number of buffers to replace
 116 **/
 117void fm10k_alloc_rx_buffers(struct fm10k_ring *rx_ring, u16 cleaned_count)
 118{
 119	union fm10k_rx_desc *rx_desc;
 120	struct fm10k_rx_buffer *bi;
 121	u16 i = rx_ring->next_to_use;
 122
 123	/* nothing to do */
 124	if (!cleaned_count)
 125		return;
 126
 127	rx_desc = FM10K_RX_DESC(rx_ring, i);
 128	bi = &rx_ring->rx_buffer[i];
 129	i -= rx_ring->count;
 130
 131	do {
 132		if (!fm10k_alloc_mapped_page(rx_ring, bi))
 133			break;
 134
 135		/* Refresh the desc even if buffer_addrs didn't change
 136		 * because each write-back erases this info.
 137		 */
 138		rx_desc->q.pkt_addr = cpu_to_le64(bi->dma + bi->page_offset);
 139
 140		rx_desc++;
 141		bi++;
 142		i++;
 143		if (unlikely(!i)) {
 144			rx_desc = FM10K_RX_DESC(rx_ring, 0);
 145			bi = rx_ring->rx_buffer;
 146			i -= rx_ring->count;
 147		}
 148
 149		/* clear the status bits for the next_to_use descriptor */
 150		rx_desc->d.staterr = 0;
 151
 152		cleaned_count--;
 153	} while (cleaned_count);
 154
 155	i += rx_ring->count;
 156
 157	if (rx_ring->next_to_use != i) {
 158		/* record the next descriptor to use */
 159		rx_ring->next_to_use = i;
 160
 161		/* update next to alloc since we have filled the ring */
 162		rx_ring->next_to_alloc = i;
 163
 164		/* Force memory writes to complete before letting h/w
 165		 * know there are new descriptors to fetch.  (Only
 166		 * applicable for weak-ordered memory model archs,
 167		 * such as IA-64).
 168		 */
 169		wmb();
 170
 171		/* notify hardware of new descriptors */
 172		writel(i, rx_ring->tail);
 173	}
 174}
 175
 176/**
 177 * fm10k_reuse_rx_page - page flip buffer and store it back on the ring
 178 * @rx_ring: rx descriptor ring to store buffers on
 179 * @old_buff: donor buffer to have page reused
 180 *
 181 * Synchronizes page for reuse by the interface
 182 **/
 183static void fm10k_reuse_rx_page(struct fm10k_ring *rx_ring,
 184				struct fm10k_rx_buffer *old_buff)
 185{
 186	struct fm10k_rx_buffer *new_buff;
 187	u16 nta = rx_ring->next_to_alloc;
 188
 189	new_buff = &rx_ring->rx_buffer[nta];
 190
 191	/* update, and store next to alloc */
 192	nta++;
 193	rx_ring->next_to_alloc = (nta < rx_ring->count) ? nta : 0;
 194
 195	/* transfer page from old buffer to new buffer */
 196	*new_buff = *old_buff;
 197
 198	/* sync the buffer for use by the device */
 199	dma_sync_single_range_for_device(rx_ring->dev, old_buff->dma,
 200					 old_buff->page_offset,
 201					 FM10K_RX_BUFSZ,
 202					 DMA_FROM_DEVICE);
 203}
 204
 205static bool fm10k_can_reuse_rx_page(struct fm10k_rx_buffer *rx_buffer,
 206				    struct page *page,
 207				    unsigned int __maybe_unused truesize)
 208{
 209	/* avoid re-using remote and pfmemalloc pages */
 210	if (!dev_page_is_reusable(page))
 211		return false;
 212
 213#if (PAGE_SIZE < 8192)
 214	/* if we are only owner of page we can reuse it */
 215	if (unlikely(page_count(page) != 1))
 216		return false;
 217
 218	/* flip page offset to other buffer */
 219	rx_buffer->page_offset ^= FM10K_RX_BUFSZ;
 220#else
 221	/* move offset up to the next cache line */
 222	rx_buffer->page_offset += truesize;
 223
 224	if (rx_buffer->page_offset > (PAGE_SIZE - FM10K_RX_BUFSZ))
 225		return false;
 226#endif
 227
 228	/* Even if we own the page, we are not allowed to use atomic_set()
 229	 * This would break get_page_unless_zero() users.
 230	 */
 231	page_ref_inc(page);
 232
 233	return true;
 234}
 235
 236/**
 237 * fm10k_add_rx_frag - Add contents of Rx buffer to sk_buff
 238 * @rx_buffer: buffer containing page to add
 239 * @size: packet size from rx_desc
 240 * @rx_desc: descriptor containing length of buffer written by hardware
 241 * @skb: sk_buff to place the data into
 242 *
 243 * This function will add the data contained in rx_buffer->page to the skb.
 244 * This is done either through a direct copy if the data in the buffer is
 245 * less than the skb header size, otherwise it will just attach the page as
 246 * a frag to the skb.
 247 *
 248 * The function will then update the page offset if necessary and return
 249 * true if the buffer can be reused by the interface.
 250 **/
 251static bool fm10k_add_rx_frag(struct fm10k_rx_buffer *rx_buffer,
 252			      unsigned int size,
 253			      union fm10k_rx_desc *rx_desc,
 254			      struct sk_buff *skb)
 255{
 256	struct page *page = rx_buffer->page;
 257	unsigned char *va = page_address(page) + rx_buffer->page_offset;
 258#if (PAGE_SIZE < 8192)
 259	unsigned int truesize = FM10K_RX_BUFSZ;
 260#else
 261	unsigned int truesize = ALIGN(size, 512);
 262#endif
 263	unsigned int pull_len;
 264
 265	if (unlikely(skb_is_nonlinear(skb)))
 266		goto add_tail_frag;
 267
 268	if (likely(size <= FM10K_RX_HDR_LEN)) {
 269		memcpy(__skb_put(skb, size), va, ALIGN(size, sizeof(long)));
 270
 271		/* page is reusable, we can reuse buffer as-is */
 272		if (dev_page_is_reusable(page))
 273			return true;
 274
 275		/* this page cannot be reused so discard it */
 276		__free_page(page);
 277		return false;
 278	}
 279
 280	/* we need the header to contain the greater of either ETH_HLEN or
 281	 * 60 bytes if the skb->len is less than 60 for skb_pad.
 282	 */
 283	pull_len = eth_get_headlen(skb->dev, va, FM10K_RX_HDR_LEN);
 284
 285	/* align pull length to size of long to optimize memcpy performance */
 286	memcpy(__skb_put(skb, pull_len), va, ALIGN(pull_len, sizeof(long)));
 287
 288	/* update all of the pointers */
 289	va += pull_len;
 290	size -= pull_len;
 291
 292add_tail_frag:
 293	skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, page,
 294			(unsigned long)va & ~PAGE_MASK, size, truesize);
 295
 296	return fm10k_can_reuse_rx_page(rx_buffer, page, truesize);
 297}
 298
 299static struct sk_buff *fm10k_fetch_rx_buffer(struct fm10k_ring *rx_ring,
 300					     union fm10k_rx_desc *rx_desc,
 301					     struct sk_buff *skb)
 302{
 303	unsigned int size = le16_to_cpu(rx_desc->w.length);
 304	struct fm10k_rx_buffer *rx_buffer;
 305	struct page *page;
 306
 307	rx_buffer = &rx_ring->rx_buffer[rx_ring->next_to_clean];
 308	page = rx_buffer->page;
 309	prefetchw(page);
 310
 311	if (likely(!skb)) {
 312		void *page_addr = page_address(page) +
 313				  rx_buffer->page_offset;
 314
 315		/* prefetch first cache line of first page */
 316		net_prefetch(page_addr);
 317
 318		/* allocate a skb to store the frags */
 319		skb = napi_alloc_skb(&rx_ring->q_vector->napi,
 320				     FM10K_RX_HDR_LEN);
 321		if (unlikely(!skb)) {
 322			rx_ring->rx_stats.alloc_failed++;
 323			return NULL;
 324		}
 325
 326		/* we will be copying header into skb->data in
 327		 * pskb_may_pull so it is in our interest to prefetch
 328		 * it now to avoid a possible cache miss
 329		 */
 330		prefetchw(skb->data);
 331	}
 332
 333	/* we are reusing so sync this buffer for CPU use */
 334	dma_sync_single_range_for_cpu(rx_ring->dev,
 335				      rx_buffer->dma,
 336				      rx_buffer->page_offset,
 337				      size,
 338				      DMA_FROM_DEVICE);
 339
 340	/* pull page into skb */
 341	if (fm10k_add_rx_frag(rx_buffer, size, rx_desc, skb)) {
 342		/* hand second half of page back to the ring */
 343		fm10k_reuse_rx_page(rx_ring, rx_buffer);
 344	} else {
 345		/* we are not reusing the buffer so unmap it */
 346		dma_unmap_page(rx_ring->dev, rx_buffer->dma,
 347			       PAGE_SIZE, DMA_FROM_DEVICE);
 348	}
 349
 350	/* clear contents of rx_buffer */
 351	rx_buffer->page = NULL;
 352
 353	return skb;
 354}
 355
 356static inline void fm10k_rx_checksum(struct fm10k_ring *ring,
 357				     union fm10k_rx_desc *rx_desc,
 358				     struct sk_buff *skb)
 359{
 360	skb_checksum_none_assert(skb);
 361
 362	/* Rx checksum disabled via ethtool */
 363	if (!(ring->netdev->features & NETIF_F_RXCSUM))
 364		return;
 365
 366	/* TCP/UDP checksum error bit is set */
 367	if (fm10k_test_staterr(rx_desc,
 368			       FM10K_RXD_STATUS_L4E |
 369			       FM10K_RXD_STATUS_L4E2 |
 370			       FM10K_RXD_STATUS_IPE |
 371			       FM10K_RXD_STATUS_IPE2)) {
 372		ring->rx_stats.csum_err++;
 373		return;
 374	}
 375
 376	/* It must be a TCP or UDP packet with a valid checksum */
 377	if (fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS2))
 378		skb->encapsulation = true;
 379	else if (!fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_L4CS))
 380		return;
 381
 382	skb->ip_summed = CHECKSUM_UNNECESSARY;
 383
 384	ring->rx_stats.csum_good++;
 385}
 386
 387#define FM10K_RSS_L4_TYPES_MASK \
 388	(BIT(FM10K_RSSTYPE_IPV4_TCP) | \
 389	 BIT(FM10K_RSSTYPE_IPV4_UDP) | \
 390	 BIT(FM10K_RSSTYPE_IPV6_TCP) | \
 391	 BIT(FM10K_RSSTYPE_IPV6_UDP))
 392
 393static inline void fm10k_rx_hash(struct fm10k_ring *ring,
 394				 union fm10k_rx_desc *rx_desc,
 395				 struct sk_buff *skb)
 396{
 397	u16 rss_type;
 398
 399	if (!(ring->netdev->features & NETIF_F_RXHASH))
 400		return;
 401
 402	rss_type = le16_to_cpu(rx_desc->w.pkt_info) & FM10K_RXD_RSSTYPE_MASK;
 403	if (!rss_type)
 404		return;
 405
 406	skb_set_hash(skb, le32_to_cpu(rx_desc->d.rss),
 407		     (BIT(rss_type) & FM10K_RSS_L4_TYPES_MASK) ?
 408		     PKT_HASH_TYPE_L4 : PKT_HASH_TYPE_L3);
 409}
 410
 411static void fm10k_type_trans(struct fm10k_ring *rx_ring,
 412			     union fm10k_rx_desc __maybe_unused *rx_desc,
 413			     struct sk_buff *skb)
 414{
 415	struct net_device *dev = rx_ring->netdev;
 416	struct fm10k_l2_accel *l2_accel = rcu_dereference_bh(rx_ring->l2_accel);
 417
 418	/* check to see if DGLORT belongs to a MACVLAN */
 419	if (l2_accel) {
 420		u16 idx = le16_to_cpu(FM10K_CB(skb)->fi.w.dglort) - 1;
 421
 422		idx -= l2_accel->dglort;
 423		if (idx < l2_accel->size && l2_accel->macvlan[idx])
 424			dev = l2_accel->macvlan[idx];
 425		else
 426			l2_accel = NULL;
 427	}
 428
 429	/* Record Rx queue, or update macvlan statistics */
 430	if (!l2_accel)
 431		skb_record_rx_queue(skb, rx_ring->queue_index);
 432	else
 433		macvlan_count_rx(netdev_priv(dev), skb->len + ETH_HLEN, true,
 434				 false);
 435
 436	skb->protocol = eth_type_trans(skb, dev);
 437}
 438
 439/**
 440 * fm10k_process_skb_fields - Populate skb header fields from Rx descriptor
 441 * @rx_ring: rx descriptor ring packet is being transacted on
 442 * @rx_desc: pointer to the EOP Rx descriptor
 443 * @skb: pointer to current skb being populated
 444 *
 445 * This function checks the ring, descriptor, and packet information in
 446 * order to populate the hash, checksum, VLAN, timestamp, protocol, and
 447 * other fields within the skb.
 448 **/
 449static unsigned int fm10k_process_skb_fields(struct fm10k_ring *rx_ring,
 450					     union fm10k_rx_desc *rx_desc,
 451					     struct sk_buff *skb)
 452{
 453	unsigned int len = skb->len;
 454
 455	fm10k_rx_hash(rx_ring, rx_desc, skb);
 456
 457	fm10k_rx_checksum(rx_ring, rx_desc, skb);
 458
 459	FM10K_CB(skb)->tstamp = rx_desc->q.timestamp;
 460
 461	FM10K_CB(skb)->fi.w.vlan = rx_desc->w.vlan;
 462
 463	FM10K_CB(skb)->fi.d.glort = rx_desc->d.glort;
 464
 465	if (rx_desc->w.vlan) {
 466		u16 vid = le16_to_cpu(rx_desc->w.vlan);
 467
 468		if ((vid & VLAN_VID_MASK) != rx_ring->vid)
 469			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q), vid);
 470		else if (vid & VLAN_PRIO_MASK)
 471			__vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
 472					       vid & VLAN_PRIO_MASK);
 473	}
 474
 475	fm10k_type_trans(rx_ring, rx_desc, skb);
 476
 477	return len;
 478}
 479
 480/**
 481 * fm10k_is_non_eop - process handling of non-EOP buffers
 482 * @rx_ring: Rx ring being processed
 483 * @rx_desc: Rx descriptor for current buffer
 484 *
 485 * This function updates next to clean.  If the buffer is an EOP buffer
 486 * this function exits returning false, otherwise it will place the
 487 * sk_buff in the next buffer to be chained and return true indicating
 488 * that this is in fact a non-EOP buffer.
 489 **/
 490static bool fm10k_is_non_eop(struct fm10k_ring *rx_ring,
 491			     union fm10k_rx_desc *rx_desc)
 492{
 493	u32 ntc = rx_ring->next_to_clean + 1;
 494
 495	/* fetch, update, and store next to clean */
 496	ntc = (ntc < rx_ring->count) ? ntc : 0;
 497	rx_ring->next_to_clean = ntc;
 498
 499	prefetch(FM10K_RX_DESC(rx_ring, ntc));
 500
 501	if (likely(fm10k_test_staterr(rx_desc, FM10K_RXD_STATUS_EOP)))
 502		return false;
 503
 504	return true;
 505}
 506
 507/**
 508 * fm10k_cleanup_headers - Correct corrupted or empty headers
 509 * @rx_ring: rx descriptor ring packet is being transacted on
 510 * @rx_desc: pointer to the EOP Rx descriptor
 511 * @skb: pointer to current skb being fixed
 512 *
 513 * Address the case where we are pulling data in on pages only
 514 * and as such no data is present in the skb header.
 515 *
 516 * In addition if skb is not at least 60 bytes we need to pad it so that
 517 * it is large enough to qualify as a valid Ethernet frame.
 518 *
 519 * Returns true if an error was encountered and skb was freed.
 520 **/
 521static bool fm10k_cleanup_headers(struct fm10k_ring *rx_ring,
 522				  union fm10k_rx_desc *rx_desc,
 523				  struct sk_buff *skb)
 524{
 525	if (unlikely((fm10k_test_staterr(rx_desc,
 526					 FM10K_RXD_STATUS_RXE)))) {
 527#define FM10K_TEST_RXD_BIT(rxd, bit) \
 528	((rxd)->w.csum_err & cpu_to_le16(bit))
 529		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_SWITCH_ERROR))
 530			rx_ring->rx_stats.switch_errors++;
 531		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_NO_DESCRIPTOR))
 532			rx_ring->rx_stats.drops++;
 533		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_PP_ERROR))
 534			rx_ring->rx_stats.pp_errors++;
 535		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_SWITCH_READY))
 536			rx_ring->rx_stats.link_errors++;
 537		if (FM10K_TEST_RXD_BIT(rx_desc, FM10K_RXD_ERR_TOO_BIG))
 538			rx_ring->rx_stats.length_errors++;
 539		dev_kfree_skb_any(skb);
 540		rx_ring->rx_stats.errors++;
 541		return true;
 542	}
 543
 544	/* if eth_skb_pad returns an error the skb was freed */
 545	if (eth_skb_pad(skb))
 546		return true;
 547
 548	return false;
 549}
 550
 551/**
 552 * fm10k_receive_skb - helper function to handle rx indications
 553 * @q_vector: structure containing interrupt and ring information
 554 * @skb: packet to send up
 555 **/
 556static void fm10k_receive_skb(struct fm10k_q_vector *q_vector,
 557			      struct sk_buff *skb)
 558{
 559	napi_gro_receive(&q_vector->napi, skb);
 560}
 561
 562static int fm10k_clean_rx_irq(struct fm10k_q_vector *q_vector,
 563			      struct fm10k_ring *rx_ring,
 564			      int budget)
 565{
 566	struct sk_buff *skb = rx_ring->skb;
 567	unsigned int total_bytes = 0, total_packets = 0;
 568	u16 cleaned_count = fm10k_desc_unused(rx_ring);
 569
 570	while (likely(total_packets < budget)) {
 571		union fm10k_rx_desc *rx_desc;
 572
 573		/* return some buffers to hardware, one at a time is too slow */
 574		if (cleaned_count >= FM10K_RX_BUFFER_WRITE) {
 575			fm10k_alloc_rx_buffers(rx_ring, cleaned_count);
 576			cleaned_count = 0;
 577		}
 578
 579		rx_desc = FM10K_RX_DESC(rx_ring, rx_ring->next_to_clean);
 580
 581		if (!rx_desc->d.staterr)
 582			break;
 583
 584		/* This memory barrier is needed to keep us from reading
 585		 * any other fields out of the rx_desc until we know the
 586		 * descriptor has been written back
 587		 */
 588		dma_rmb();
 589
 590		/* retrieve a buffer from the ring */
 591		skb = fm10k_fetch_rx_buffer(rx_ring, rx_desc, skb);
 592
 593		/* exit if we failed to retrieve a buffer */
 594		if (!skb)
 595			break;
 596
 597		cleaned_count++;
 598
 599		/* fetch next buffer in frame if non-eop */
 600		if (fm10k_is_non_eop(rx_ring, rx_desc))
 601			continue;
 602
 603		/* verify the packet layout is correct */
 604		if (fm10k_cleanup_headers(rx_ring, rx_desc, skb)) {
 605			skb = NULL;
 606			continue;
 607		}
 608
 609		/* populate checksum, timestamp, VLAN, and protocol */
 610		total_bytes += fm10k_process_skb_fields(rx_ring, rx_desc, skb);
 611
 612		fm10k_receive_skb(q_vector, skb);
 613
 614		/* reset skb pointer */
 615		skb = NULL;
 616
 617		/* update budget accounting */
 618		total_packets++;
 619	}
 620
 621	/* place incomplete frames back on ring for completion */
 622	rx_ring->skb = skb;
 623
 624	u64_stats_update_begin(&rx_ring->syncp);
 625	rx_ring->stats.packets += total_packets;
 626	rx_ring->stats.bytes += total_bytes;
 627	u64_stats_update_end(&rx_ring->syncp);
 628	q_vector->rx.total_packets += total_packets;
 629	q_vector->rx.total_bytes += total_bytes;
 630
 631	return total_packets;
 632}
 633
 634#define VXLAN_HLEN (sizeof(struct udphdr) + 8)
 635static struct ethhdr *fm10k_port_is_vxlan(struct sk_buff *skb)
 636{
 637	struct fm10k_intfc *interface = netdev_priv(skb->dev);
 638
 639	if (interface->vxlan_port != udp_hdr(skb)->dest)
 640		return NULL;
 641
 642	/* return offset of udp_hdr plus 8 bytes for VXLAN header */
 643	return (struct ethhdr *)(skb_transport_header(skb) + VXLAN_HLEN);
 644}
 645
 646#define FM10K_NVGRE_RESERVED0_FLAGS htons(0x9FFF)
 647#define NVGRE_TNI htons(0x2000)
 648struct fm10k_nvgre_hdr {
 649	__be16 flags;
 650	__be16 proto;
 651	__be32 tni;
 652};
 653
 654static struct ethhdr *fm10k_gre_is_nvgre(struct sk_buff *skb)
 655{
 656	struct fm10k_nvgre_hdr *nvgre_hdr;
 657	int hlen = ip_hdrlen(skb);
 658
 659	/* currently only IPv4 is supported due to hlen above */
 660	if (vlan_get_protocol(skb) != htons(ETH_P_IP))
 661		return NULL;
 662
 663	/* our transport header should be NVGRE */
 664	nvgre_hdr = (struct fm10k_nvgre_hdr *)(skb_network_header(skb) + hlen);
 665
 666	/* verify all reserved flags are 0 */
 667	if (nvgre_hdr->flags & FM10K_NVGRE_RESERVED0_FLAGS)
 668		return NULL;
 669
 670	/* report start of ethernet header */
 671	if (nvgre_hdr->flags & NVGRE_TNI)
 672		return (struct ethhdr *)(nvgre_hdr + 1);
 673
 674	return (struct ethhdr *)(&nvgre_hdr->tni);
 675}
 676
 677__be16 fm10k_tx_encap_offload(struct sk_buff *skb)
 678{
 679	u8 l4_hdr = 0, inner_l4_hdr = 0, inner_l4_hlen;
 680	struct ethhdr *eth_hdr;
 681
 682	if (skb->inner_protocol_type != ENCAP_TYPE_ETHER ||
 683	    skb->inner_protocol != htons(ETH_P_TEB))
 684		return 0;
 685
 686	switch (vlan_get_protocol(skb)) {
 687	case htons(ETH_P_IP):
 688		l4_hdr = ip_hdr(skb)->protocol;
 689		break;
 690	case htons(ETH_P_IPV6):
 691		l4_hdr = ipv6_hdr(skb)->nexthdr;
 692		break;
 693	default:
 694		return 0;
 695	}
 696
 697	switch (l4_hdr) {
 698	case IPPROTO_UDP:
 699		eth_hdr = fm10k_port_is_vxlan(skb);
 700		break;
 701	case IPPROTO_GRE:
 702		eth_hdr = fm10k_gre_is_nvgre(skb);
 703		break;
 704	default:
 705		return 0;
 706	}
 707
 708	if (!eth_hdr)
 709		return 0;
 710
 711	switch (eth_hdr->h_proto) {
 712	case htons(ETH_P_IP):
 713		inner_l4_hdr = inner_ip_hdr(skb)->protocol;
 714		break;
 715	case htons(ETH_P_IPV6):
 716		inner_l4_hdr = inner_ipv6_hdr(skb)->nexthdr;
 717		break;
 718	default:
 719		return 0;
 720	}
 721
 722	switch (inner_l4_hdr) {
 723	case IPPROTO_TCP:
 724		inner_l4_hlen = inner_tcp_hdrlen(skb);
 725		break;
 726	case IPPROTO_UDP:
 727		inner_l4_hlen = 8;
 728		break;
 729	default:
 730		return 0;
 731	}
 732
 733	/* The hardware allows tunnel offloads only if the combined inner and
 734	 * outer header is 184 bytes or less
 735	 */
 736	if (skb_inner_transport_header(skb) + inner_l4_hlen -
 737	    skb_mac_header(skb) > FM10K_TUNNEL_HEADER_LENGTH)
 738		return 0;
 739
 740	return eth_hdr->h_proto;
 741}
 742
 743static int fm10k_tso(struct fm10k_ring *tx_ring,
 744		     struct fm10k_tx_buffer *first)
 745{
 746	struct sk_buff *skb = first->skb;
 747	struct fm10k_tx_desc *tx_desc;
 748	unsigned char *th;
 749	u8 hdrlen;
 750
 751	if (skb->ip_summed != CHECKSUM_PARTIAL)
 752		return 0;
 753
 754	if (!skb_is_gso(skb))
 755		return 0;
 756
 757	/* compute header lengths */
 758	if (skb->encapsulation) {
 759		if (!fm10k_tx_encap_offload(skb))
 760			goto err_vxlan;
 761		th = skb_inner_transport_header(skb);
 762	} else {
 763		th = skb_transport_header(skb);
 764	}
 765
 766	/* compute offset from SOF to transport header and add header len */
 767	hdrlen = (th - skb->data) + (((struct tcphdr *)th)->doff << 2);
 768
 769	first->tx_flags |= FM10K_TX_FLAGS_CSUM;
 770
 771	/* update gso size and bytecount with header size */
 772	first->gso_segs = skb_shinfo(skb)->gso_segs;
 773	first->bytecount += (first->gso_segs - 1) * hdrlen;
 774
 775	/* populate Tx descriptor header size and mss */
 776	tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use);
 777	tx_desc->hdrlen = hdrlen;
 778	tx_desc->mss = cpu_to_le16(skb_shinfo(skb)->gso_size);
 779
 780	return 1;
 781
 782err_vxlan:
 783	tx_ring->netdev->features &= ~NETIF_F_GSO_UDP_TUNNEL;
 784	if (net_ratelimit())
 785		netdev_err(tx_ring->netdev,
 786			   "TSO requested for unsupported tunnel, disabling offload\n");
 787	return -1;
 788}
 789
 790static void fm10k_tx_csum(struct fm10k_ring *tx_ring,
 791			  struct fm10k_tx_buffer *first)
 792{
 793	struct sk_buff *skb = first->skb;
 794	struct fm10k_tx_desc *tx_desc;
 795	union {
 796		struct iphdr *ipv4;
 797		struct ipv6hdr *ipv6;
 798		u8 *raw;
 799	} network_hdr;
 800	u8 *transport_hdr;
 801	__be16 frag_off;
 802	__be16 protocol;
 803	u8 l4_hdr = 0;
 804
 805	if (skb->ip_summed != CHECKSUM_PARTIAL)
 806		goto no_csum;
 807
 808	if (skb->encapsulation) {
 809		protocol = fm10k_tx_encap_offload(skb);
 810		if (!protocol) {
 811			if (skb_checksum_help(skb)) {
 812				dev_warn(tx_ring->dev,
 813					 "failed to offload encap csum!\n");
 814				tx_ring->tx_stats.csum_err++;
 815			}
 816			goto no_csum;
 817		}
 818		network_hdr.raw = skb_inner_network_header(skb);
 819		transport_hdr = skb_inner_transport_header(skb);
 820	} else {
 821		protocol = vlan_get_protocol(skb);
 822		network_hdr.raw = skb_network_header(skb);
 823		transport_hdr = skb_transport_header(skb);
 824	}
 825
 826	switch (protocol) {
 827	case htons(ETH_P_IP):
 828		l4_hdr = network_hdr.ipv4->protocol;
 829		break;
 830	case htons(ETH_P_IPV6):
 831		l4_hdr = network_hdr.ipv6->nexthdr;
 832		if (likely((transport_hdr - network_hdr.raw) ==
 833			   sizeof(struct ipv6hdr)))
 834			break;
 835		ipv6_skip_exthdr(skb, network_hdr.raw - skb->data +
 836				      sizeof(struct ipv6hdr),
 837				 &l4_hdr, &frag_off);
 838		if (unlikely(frag_off))
 839			l4_hdr = NEXTHDR_FRAGMENT;
 840		break;
 841	default:
 842		break;
 843	}
 844
 845	switch (l4_hdr) {
 846	case IPPROTO_TCP:
 847	case IPPROTO_UDP:
 848		break;
 849	case IPPROTO_GRE:
 850		if (skb->encapsulation)
 851			break;
 852		fallthrough;
 853	default:
 854		if (unlikely(net_ratelimit())) {
 855			dev_warn(tx_ring->dev,
 856				 "partial checksum, version=%d l4 proto=%x\n",
 857				 protocol, l4_hdr);
 858		}
 859		skb_checksum_help(skb);
 860		tx_ring->tx_stats.csum_err++;
 861		goto no_csum;
 862	}
 863
 864	/* update TX checksum flag */
 865	first->tx_flags |= FM10K_TX_FLAGS_CSUM;
 866	tx_ring->tx_stats.csum_good++;
 867
 868no_csum:
 869	/* populate Tx descriptor header size and mss */
 870	tx_desc = FM10K_TX_DESC(tx_ring, tx_ring->next_to_use);
 871	tx_desc->hdrlen = 0;
 872	tx_desc->mss = 0;
 873}
 874
 875#define FM10K_SET_FLAG(_input, _flag, _result) \
 876	((_flag <= _result) ? \
 877	 ((u32)(_input & _flag) * (_result / _flag)) : \
 878	 ((u32)(_input & _flag) / (_flag / _result)))
 879
 880static u8 fm10k_tx_desc_flags(struct sk_buff *skb, u32 tx_flags)
 881{
 882	/* set type for advanced descriptor with frame checksum insertion */
 883	u32 desc_flags = 0;
 884
 885	/* set checksum offload bits */
 886	desc_flags |= FM10K_SET_FLAG(tx_flags, FM10K_TX_FLAGS_CSUM,
 887				     FM10K_TXD_FLAG_CSUM);
 888
 889	return desc_flags;
 890}
 891
 892static bool fm10k_tx_desc_push(struct fm10k_ring *tx_ring,
 893			       struct fm10k_tx_desc *tx_desc, u16 i,
 894			       dma_addr_t dma, unsigned int size, u8 desc_flags)
 895{
 896	/* set RS and INT for last frame in a cache line */
 897	if ((++i & (FM10K_TXD_WB_FIFO_SIZE - 1)) == 0)
 898		desc_flags |= FM10K_TXD_FLAG_RS | FM10K_TXD_FLAG_INT;
 899
 900	/* record values to descriptor */
 901	tx_desc->buffer_addr = cpu_to_le64(dma);
 902	tx_desc->flags = desc_flags;
 903	tx_desc->buflen = cpu_to_le16(size);
 904
 905	/* return true if we just wrapped the ring */
 906	return i == tx_ring->count;
 907}
 908
 909static int __fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size)
 910{
 911	netif_stop_subqueue(tx_ring->netdev, tx_ring->queue_index);
 912
 913	/* Memory barrier before checking head and tail */
 914	smp_mb();
 915
 916	/* Check again in a case another CPU has just made room available */
 917	if (likely(fm10k_desc_unused(tx_ring) < size))
 918		return -EBUSY;
 919
 920	/* A reprieve! - use start_queue because it doesn't call schedule */
 921	netif_start_subqueue(tx_ring->netdev, tx_ring->queue_index);
 922	++tx_ring->tx_stats.restart_queue;
 923	return 0;
 924}
 925
 926static inline int fm10k_maybe_stop_tx(struct fm10k_ring *tx_ring, u16 size)
 927{
 928	if (likely(fm10k_desc_unused(tx_ring) >= size))
 929		return 0;
 930	return __fm10k_maybe_stop_tx(tx_ring, size);
 931}
 932
 933static void fm10k_tx_map(struct fm10k_ring *tx_ring,
 934			 struct fm10k_tx_buffer *first)
 935{
 936	struct sk_buff *skb = first->skb;
 937	struct fm10k_tx_buffer *tx_buffer;
 938	struct fm10k_tx_desc *tx_desc;
 939	skb_frag_t *frag;
 940	unsigned char *data;
 941	dma_addr_t dma;
 942	unsigned int data_len, size;
 943	u32 tx_flags = first->tx_flags;
 944	u16 i = tx_ring->next_to_use;
 945	u8 flags = fm10k_tx_desc_flags(skb, tx_flags);
 946
 947	tx_desc = FM10K_TX_DESC(tx_ring, i);
 948
 949	/* add HW VLAN tag */
 950	if (skb_vlan_tag_present(skb))
 951		tx_desc->vlan = cpu_to_le16(skb_vlan_tag_get(skb));
 952	else
 953		tx_desc->vlan = 0;
 954
 955	size = skb_headlen(skb);
 956	data = skb->data;
 957
 958	dma = dma_map_single(tx_ring->dev, data, size, DMA_TO_DEVICE);
 959
 960	data_len = skb->data_len;
 961	tx_buffer = first;
 962
 963	for (frag = &skb_shinfo(skb)->frags[0];; frag++) {
 964		if (dma_mapping_error(tx_ring->dev, dma))
 965			goto dma_error;
 966
 967		/* record length, and DMA address */
 968		dma_unmap_len_set(tx_buffer, len, size);
 969		dma_unmap_addr_set(tx_buffer, dma, dma);
 970
 971		while (unlikely(size > FM10K_MAX_DATA_PER_TXD)) {
 972			if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++, dma,
 973					       FM10K_MAX_DATA_PER_TXD, flags)) {
 974				tx_desc = FM10K_TX_DESC(tx_ring, 0);
 975				i = 0;
 976			}
 977
 978			dma += FM10K_MAX_DATA_PER_TXD;
 979			size -= FM10K_MAX_DATA_PER_TXD;
 980		}
 981
 982		if (likely(!data_len))
 983			break;
 984
 985		if (fm10k_tx_desc_push(tx_ring, tx_desc++, i++,
 986				       dma, size, flags)) {
 987			tx_desc = FM10K_TX_DESC(tx_ring, 0);
 988			i = 0;
 989		}
 990
 991		size = skb_frag_size(frag);
 992		data_len -= size;
 993
 994		dma = skb_frag_dma_map(tx_ring->dev, frag, 0, size,
 995				       DMA_TO_DEVICE);
 996
 997		tx_buffer = &tx_ring->tx_buffer[i];
 998	}
 999
1000	/* write last descriptor with LAST bit set */
1001	flags |= FM10K_TXD_FLAG_LAST;
1002
1003	if (fm10k_tx_desc_push(tx_ring, tx_desc, i++, dma, size, flags))
1004		i = 0;
1005
1006	/* record bytecount for BQL */
1007	netdev_tx_sent_queue(txring_txq(tx_ring), first->bytecount);
1008
1009	/* record SW timestamp if HW timestamp is not available */
1010	skb_tx_timestamp(first->skb);
1011
1012	/* Force memory writes to complete before letting h/w know there
1013	 * are new descriptors to fetch.  (Only applicable for weak-ordered
1014	 * memory model archs, such as IA-64).
1015	 *
1016	 * We also need this memory barrier to make certain all of the
1017	 * status bits have been updated before next_to_watch is written.
1018	 */
1019	wmb();
1020
1021	/* set next_to_watch value indicating a packet is present */
1022	first->next_to_watch = tx_desc;
1023
1024	tx_ring->next_to_use = i;
1025
1026	/* Make sure there is space in the ring for the next send. */
1027	fm10k_maybe_stop_tx(tx_ring, DESC_NEEDED);
1028
1029	/* notify HW of packet */
1030	if (netif_xmit_stopped(txring_txq(tx_ring)) || !netdev_xmit_more()) {
1031		writel(i, tx_ring->tail);
1032	}
1033
1034	return;
1035dma_error:
1036	dev_err(tx_ring->dev, "TX DMA map failed\n");
1037
1038	/* clear dma mappings for failed tx_buffer map */
1039	for (;;) {
1040		tx_buffer = &tx_ring->tx_buffer[i];
1041		fm10k_unmap_and_free_tx_resource(tx_ring, tx_buffer);
1042		if (tx_buffer == first)
1043			break;
1044		if (i == 0)
1045			i = tx_ring->count;
1046		i--;
1047	}
1048
1049	tx_ring->next_to_use = i;
1050}
1051
1052netdev_tx_t fm10k_xmit_frame_ring(struct sk_buff *skb,
1053				  struct fm10k_ring *tx_ring)
1054{
1055	u16 count = TXD_USE_COUNT(skb_headlen(skb));
1056	struct fm10k_tx_buffer *first;
1057	unsigned short f;
1058	u32 tx_flags = 0;
1059	int tso;
1060
1061	/* need: 1 descriptor per page * PAGE_SIZE/FM10K_MAX_DATA_PER_TXD,
1062	 *       + 1 desc for skb_headlen/FM10K_MAX_DATA_PER_TXD,
1063	 *       + 2 desc gap to keep tail from touching head
1064	 * otherwise try next time
1065	 */
1066	for (f = 0; f < skb_shinfo(skb)->nr_frags; f++) {
1067		skb_frag_t *frag = &skb_shinfo(skb)->frags[f];
1068
1069		count += TXD_USE_COUNT(skb_frag_size(frag));
1070	}
1071
1072	if (fm10k_maybe_stop_tx(tx_ring, count + 3)) {
1073		tx_ring->tx_stats.tx_busy++;
1074		return NETDEV_TX_BUSY;
1075	}
1076
1077	/* record the location of the first descriptor for this packet */
1078	first = &tx_ring->tx_buffer[tx_ring->next_to_use];
1079	first->skb = skb;
1080	first->bytecount = max_t(unsigned int, skb->len, ETH_ZLEN);
1081	first->gso_segs = 1;
1082
1083	/* record initial flags and protocol */
1084	first->tx_flags = tx_flags;
1085
1086	tso = fm10k_tso(tx_ring, first);
1087	if (tso < 0)
1088		goto out_drop;
1089	else if (!tso)
1090		fm10k_tx_csum(tx_ring, first);
1091
1092	fm10k_tx_map(tx_ring, first);
1093
1094	return NETDEV_TX_OK;
1095
1096out_drop:
1097	dev_kfree_skb_any(first->skb);
1098	first->skb = NULL;
1099
1100	return NETDEV_TX_OK;
1101}
1102
1103static u64 fm10k_get_tx_completed(struct fm10k_ring *ring)
1104{
1105	return ring->stats.packets;
1106}
1107
1108/**
1109 * fm10k_get_tx_pending - how many Tx descriptors not processed
1110 * @ring: the ring structure
1111 * @in_sw: is tx_pending being checked in SW or in HW?
1112 */
1113u64 fm10k_get_tx_pending(struct fm10k_ring *ring, bool in_sw)
1114{
1115	struct fm10k_intfc *interface = ring->q_vector->interface;
1116	struct fm10k_hw *hw = &interface->hw;
1117	u32 head, tail;
1118
1119	if (likely(in_sw)) {
1120		head = ring->next_to_clean;
1121		tail = ring->next_to_use;
1122	} else {
1123		head = fm10k_read_reg(hw, FM10K_TDH(ring->reg_idx));
1124		tail = fm10k_read_reg(hw, FM10K_TDT(ring->reg_idx));
1125	}
1126
1127	return ((head <= tail) ? tail : tail + ring->count) - head;
1128}
1129
1130bool fm10k_check_tx_hang(struct fm10k_ring *tx_ring)
1131{
1132	u32 tx_done = fm10k_get_tx_completed(tx_ring);
1133	u32 tx_done_old = tx_ring->tx_stats.tx_done_old;
1134	u32 tx_pending = fm10k_get_tx_pending(tx_ring, true);
1135
1136	clear_check_for_tx_hang(tx_ring);
1137
1138	/* Check for a hung queue, but be thorough. This verifies
1139	 * that a transmit has been completed since the previous
1140	 * check AND there is at least one packet pending. By
1141	 * requiring this to fail twice we avoid races with
1142	 * clearing the ARMED bit and conditions where we
1143	 * run the check_tx_hang logic with a transmit completion
1144	 * pending but without time to complete it yet.
1145	 */
1146	if (!tx_pending || (tx_done_old != tx_done)) {
1147		/* update completed stats and continue */
1148		tx_ring->tx_stats.tx_done_old = tx_done;
1149		/* reset the countdown */
1150		clear_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state);
1151
1152		return false;
1153	}
1154
1155	/* make sure it is true for two checks in a row */
1156	return test_and_set_bit(__FM10K_HANG_CHECK_ARMED, tx_ring->state);
1157}
1158
1159/**
1160 * fm10k_tx_timeout_reset - initiate reset due to Tx timeout
1161 * @interface: driver private struct
1162 **/
1163void fm10k_tx_timeout_reset(struct fm10k_intfc *interface)
1164{
1165	/* Do the reset outside of interrupt context */
1166	if (!test_bit(__FM10K_DOWN, interface->state)) {
1167		interface->tx_timeout_count++;
1168		set_bit(FM10K_FLAG_RESET_REQUESTED, interface->flags);
1169		fm10k_service_event_schedule(interface);
1170	}
1171}
1172
1173/**
1174 * fm10k_clean_tx_irq - Reclaim resources after transmit completes
1175 * @q_vector: structure containing interrupt and ring information
1176 * @tx_ring: tx ring to clean
1177 * @napi_budget: Used to determine if we are in netpoll
1178 **/
1179static bool fm10k_clean_tx_irq(struct fm10k_q_vector *q_vector,
1180			       struct fm10k_ring *tx_ring, int napi_budget)
1181{
1182	struct fm10k_intfc *interface = q_vector->interface;
1183	struct fm10k_tx_buffer *tx_buffer;
1184	struct fm10k_tx_desc *tx_desc;
1185	unsigned int total_bytes = 0, total_packets = 0;
1186	unsigned int budget = q_vector->tx.work_limit;
1187	unsigned int i = tx_ring->next_to_clean;
1188
1189	if (test_bit(__FM10K_DOWN, interface->state))
1190		return true;
1191
1192	tx_buffer = &tx_ring->tx_buffer[i];
1193	tx_desc = FM10K_TX_DESC(tx_ring, i);
1194	i -= tx_ring->count;
1195
1196	do {
1197		struct fm10k_tx_desc *eop_desc = tx_buffer->next_to_watch;
1198
1199		/* if next_to_watch is not set then there is no work pending */
1200		if (!eop_desc)
1201			break;
1202
1203		/* prevent any other reads prior to eop_desc */
1204		smp_rmb();
1205
1206		/* if DD is not set pending work has not been completed */
1207		if (!(eop_desc->flags & FM10K_TXD_FLAG_DONE))
1208			break;
1209
1210		/* clear next_to_watch to prevent false hangs */
1211		tx_buffer->next_to_watch = NULL;
1212
1213		/* update the statistics for this packet */
1214		total_bytes += tx_buffer->bytecount;
1215		total_packets += tx_buffer->gso_segs;
1216
1217		/* free the skb */
1218		napi_consume_skb(tx_buffer->skb, napi_budget);
1219
1220		/* unmap skb header data */
1221		dma_unmap_single(tx_ring->dev,
1222				 dma_unmap_addr(tx_buffer, dma),
1223				 dma_unmap_len(tx_buffer, len),
1224				 DMA_TO_DEVICE);
1225
1226		/* clear tx_buffer data */
1227		tx_buffer->skb = NULL;
1228		dma_unmap_len_set(tx_buffer, len, 0);
1229
1230		/* unmap remaining buffers */
1231		while (tx_desc != eop_desc) {
1232			tx_buffer++;
1233			tx_desc++;
1234			i++;
1235			if (unlikely(!i)) {
1236				i -= tx_ring->count;
1237				tx_buffer = tx_ring->tx_buffer;
1238				tx_desc = FM10K_TX_DESC(tx_ring, 0);
1239			}
1240
1241			/* unmap any remaining paged data */
1242			if (dma_unmap_len(tx_buffer, len)) {
1243				dma_unmap_page(tx_ring->dev,
1244					       dma_unmap_addr(tx_buffer, dma),
1245					       dma_unmap_len(tx_buffer, len),
1246					       DMA_TO_DEVICE);
1247				dma_unmap_len_set(tx_buffer, len, 0);
1248			}
1249		}
1250
1251		/* move us one more past the eop_desc for start of next pkt */
1252		tx_buffer++;
1253		tx_desc++;
1254		i++;
1255		if (unlikely(!i)) {
1256			i -= tx_ring->count;
1257			tx_buffer = tx_ring->tx_buffer;
1258			tx_desc = FM10K_TX_DESC(tx_ring, 0);
1259		}
1260
1261		/* issue prefetch for next Tx descriptor */
1262		prefetch(tx_desc);
1263
1264		/* update budget accounting */
1265		budget--;
1266	} while (likely(budget));
1267
1268	i += tx_ring->count;
1269	tx_ring->next_to_clean = i;
1270	u64_stats_update_begin(&tx_ring->syncp);
1271	tx_ring->stats.bytes += total_bytes;
1272	tx_ring->stats.packets += total_packets;
1273	u64_stats_update_end(&tx_ring->syncp);
1274	q_vector->tx.total_bytes += total_bytes;
1275	q_vector->tx.total_packets += total_packets;
1276
1277	if (check_for_tx_hang(tx_ring) && fm10k_check_tx_hang(tx_ring)) {
1278		/* schedule immediate reset if we believe we hung */
1279		struct fm10k_hw *hw = &interface->hw;
1280
1281		netif_err(interface, drv, tx_ring->netdev,
1282			  "Detected Tx Unit Hang\n"
1283			  "  Tx Queue             <%d>\n"
1284			  "  TDH, TDT             <%x>, <%x>\n"
1285			  "  next_to_use          <%x>\n"
1286			  "  next_to_clean        <%x>\n",
1287			  tx_ring->queue_index,
1288			  fm10k_read_reg(hw, FM10K_TDH(tx_ring->reg_idx)),
1289			  fm10k_read_reg(hw, FM10K_TDT(tx_ring->reg_idx)),
1290			  tx_ring->next_to_use, i);
1291
1292		netif_stop_subqueue(tx_ring->netdev,
1293				    tx_ring->queue_index);
1294
1295		netif_info(interface, probe, tx_ring->netdev,
1296			   "tx hang %d detected on queue %d, resetting interface\n",
1297			   interface->tx_timeout_count + 1,
1298			   tx_ring->queue_index);
1299
1300		fm10k_tx_timeout_reset(interface);
1301
1302		/* the netdev is about to reset, no point in enabling stuff */
1303		return true;
1304	}
1305
1306	/* notify netdev of completed buffers */
1307	netdev_tx_completed_queue(txring_txq(tx_ring),
1308				  total_packets, total_bytes);
1309
1310#define TX_WAKE_THRESHOLD min_t(u16, FM10K_MIN_TXD - 1, DESC_NEEDED * 2)
1311	if (unlikely(total_packets && netif_carrier_ok(tx_ring->netdev) &&
1312		     (fm10k_desc_unused(tx_ring) >= TX_WAKE_THRESHOLD))) {
1313		/* Make sure that anybody stopping the queue after this
1314		 * sees the new next_to_clean.
1315		 */
1316		smp_mb();
1317		if (__netif_subqueue_stopped(tx_ring->netdev,
1318					     tx_ring->queue_index) &&
1319		    !test_bit(__FM10K_DOWN, interface->state)) {
1320			netif_wake_subqueue(tx_ring->netdev,
1321					    tx_ring->queue_index);
1322			++tx_ring->tx_stats.restart_queue;
1323		}
1324	}
1325
1326	return !!budget;
1327}
1328
1329/**
1330 * fm10k_update_itr - update the dynamic ITR value based on packet size
1331 *
1332 *      Stores a new ITR value based on strictly on packet size.  The
1333 *      divisors and thresholds used by this function were determined based
1334 *      on theoretical maximum wire speed and testing data, in order to
1335 *      minimize response time while increasing bulk throughput.
1336 *
1337 * @ring_container: Container for rings to have ITR updated
1338 **/
1339static void fm10k_update_itr(struct fm10k_ring_container *ring_container)
1340{
1341	unsigned int avg_wire_size, packets, itr_round;
1342
1343	/* Only update ITR if we are using adaptive setting */
1344	if (!ITR_IS_ADAPTIVE(ring_container->itr))
1345		goto clear_counts;
1346
1347	packets = ring_container->total_packets;
1348	if (!packets)
1349		goto clear_counts;
1350
1351	avg_wire_size = ring_container->total_bytes / packets;
1352
1353	/* The following is a crude approximation of:
1354	 *  wmem_default / (size + overhead) = desired_pkts_per_int
1355	 *  rate / bits_per_byte / (size + ethernet overhead) = pkt_rate
1356	 *  (desired_pkt_rate / pkt_rate) * usecs_per_sec = ITR value
1357	 *
1358	 * Assuming wmem_default is 212992 and overhead is 640 bytes per
1359	 * packet, (256 skb, 64 headroom, 320 shared info), we can reduce the
1360	 * formula down to
1361	 *
1362	 *  (34 * (size + 24)) / (size + 640) = ITR
1363	 *
1364	 * We first do some math on the packet size and then finally bitshift
1365	 * by 8 after rounding up. We also have to account for PCIe link speed
1366	 * difference as ITR scales based on this.
1367	 */
1368	if (avg_wire_size <= 360) {
1369		/* Start at 250K ints/sec and gradually drop to 77K ints/sec */
1370		avg_wire_size *= 8;
1371		avg_wire_size += 376;
1372	} else if (avg_wire_size <= 1152) {
1373		/* 77K ints/sec to 45K ints/sec */
1374		avg_wire_size *= 3;
1375		avg_wire_size += 2176;
1376	} else if (avg_wire_size <= 1920) {
1377		/* 45K ints/sec to 38K ints/sec */
1378		avg_wire_size += 4480;
1379	} else {
1380		/* plateau at a limit of 38K ints/sec */
1381		avg_wire_size = 6656;
1382	}
1383
1384	/* Perform final bitshift for division after rounding up to ensure
1385	 * that the calculation will never get below a 1. The bit shift
1386	 * accounts for changes in the ITR due to PCIe link speed.
1387	 */
1388	itr_round = READ_ONCE(ring_container->itr_scale) + 8;
1389	avg_wire_size += BIT(itr_round) - 1;
1390	avg_wire_size >>= itr_round;
1391
1392	/* write back value and retain adaptive flag */
1393	ring_container->itr = avg_wire_size | FM10K_ITR_ADAPTIVE;
1394
1395clear_counts:
1396	ring_container->total_bytes = 0;
1397	ring_container->total_packets = 0;
1398}
1399
1400static void fm10k_qv_enable(struct fm10k_q_vector *q_vector)
1401{
1402	/* Enable auto-mask and clear the current mask */
1403	u32 itr = FM10K_ITR_ENABLE;
1404
1405	/* Update Tx ITR */
1406	fm10k_update_itr(&q_vector->tx);
1407
1408	/* Update Rx ITR */
1409	fm10k_update_itr(&q_vector->rx);
1410
1411	/* Store Tx itr in timer slot 0 */
1412	itr |= (q_vector->tx.itr & FM10K_ITR_MAX);
1413
1414	/* Shift Rx itr to timer slot 1 */
1415	itr |= (q_vector->rx.itr & FM10K_ITR_MAX) << FM10K_ITR_INTERVAL1_SHIFT;
1416
1417	/* Write the final value to the ITR register */
1418	writel(itr, q_vector->itr);
1419}
1420
1421static int fm10k_poll(struct napi_struct *napi, int budget)
1422{
1423	struct fm10k_q_vector *q_vector =
1424			       container_of(napi, struct fm10k_q_vector, napi);
1425	struct fm10k_ring *ring;
1426	int per_ring_budget, work_done = 0;
1427	bool clean_complete = true;
1428
1429	fm10k_for_each_ring(ring, q_vector->tx) {
1430		if (!fm10k_clean_tx_irq(q_vector, ring, budget))
1431			clean_complete = false;
1432	}
1433
1434	/* Handle case where we are called by netpoll with a budget of 0 */
1435	if (budget <= 0)
1436		return budget;
1437
1438	/* attempt to distribute budget to each queue fairly, but don't
1439	 * allow the budget to go below 1 because we'll exit polling
1440	 */
1441	if (q_vector->rx.count > 1)
1442		per_ring_budget = max(budget / q_vector->rx.count, 1);
1443	else
1444		per_ring_budget = budget;
1445
1446	fm10k_for_each_ring(ring, q_vector->rx) {
1447		int work = fm10k_clean_rx_irq(q_vector, ring, per_ring_budget);
1448
1449		work_done += work;
1450		if (work >= per_ring_budget)
1451			clean_complete = false;
1452	}
1453
1454	/* If all work not completed, return budget and keep polling */
1455	if (!clean_complete)
1456		return budget;
1457
1458	/* Exit the polling mode, but don't re-enable interrupts if stack might
1459	 * poll us due to busy-polling
1460	 */
1461	if (likely(napi_complete_done(napi, work_done)))
1462		fm10k_qv_enable(q_vector);
1463
1464	return min(work_done, budget - 1);
1465}
1466
1467/**
1468 * fm10k_set_qos_queues: Allocate queues for a QOS-enabled device
1469 * @interface: board private structure to initialize
1470 *
1471 * When QoS (Quality of Service) is enabled, allocate queues for
1472 * each traffic class.  If multiqueue isn't available,then abort QoS
1473 * initialization.
1474 *
1475 * This function handles all combinations of Qos and RSS.
1476 *
1477 **/
1478static bool fm10k_set_qos_queues(struct fm10k_intfc *interface)
1479{
1480	struct net_device *dev = interface->netdev;
1481	struct fm10k_ring_feature *f;
1482	int rss_i, i;
1483	int pcs;
1484
1485	/* Map queue offset and counts onto allocated tx queues */
1486	pcs = netdev_get_num_tc(dev);
1487
1488	if (pcs <= 1)
1489		return false;
1490
1491	/* set QoS mask and indices */
1492	f = &interface->ring_feature[RING_F_QOS];
1493	f->indices = pcs;
1494	f->mask = BIT(fls(pcs - 1)) - 1;
1495
1496	/* determine the upper limit for our current DCB mode */
1497	rss_i = interface->hw.mac.max_queues / pcs;
1498	rss_i = BIT(fls(rss_i) - 1);
1499
1500	/* set RSS mask and indices */
1501	f = &interface->ring_feature[RING_F_RSS];
1502	rss_i = min_t(u16, rss_i, f->limit);
1503	f->indices = rss_i;
1504	f->mask = BIT(fls(rss_i - 1)) - 1;
1505
1506	/* configure pause class to queue mapping */
1507	for (i = 0; i < pcs; i++)
1508		netdev_set_tc_queue(dev, i, rss_i, rss_i * i);
1509
1510	interface->num_rx_queues = rss_i * pcs;
1511	interface->num_tx_queues = rss_i * pcs;
1512
1513	return true;
1514}
1515
1516/**
1517 * fm10k_set_rss_queues: Allocate queues for RSS
1518 * @interface: board private structure to initialize
1519 *
1520 * This is our "base" multiqueue mode.  RSS (Receive Side Scaling) will try
1521 * to allocate one Rx queue per CPU, and if available, one Tx queue per CPU.
1522 *
1523 **/
1524static bool fm10k_set_rss_queues(struct fm10k_intfc *interface)
1525{
1526	struct fm10k_ring_feature *f;
1527	u16 rss_i;
1528
1529	f = &interface->ring_feature[RING_F_RSS];
1530	rss_i = min_t(u16, interface->hw.mac.max_queues, f->limit);
1531
1532	/* record indices and power of 2 mask for RSS */
1533	f->indices = rss_i;
1534	f->mask = BIT(fls(rss_i - 1)) - 1;
1535
1536	interface->num_rx_queues = rss_i;
1537	interface->num_tx_queues = rss_i;
1538
1539	return true;
1540}
1541
1542/**
1543 * fm10k_set_num_queues: Allocate queues for device, feature dependent
1544 * @interface: board private structure to initialize
1545 *
1546 * This is the top level queue allocation routine.  The order here is very
1547 * important, starting with the "most" number of features turned on at once,
1548 * and ending with the smallest set of features.  This way large combinations
1549 * can be allocated if they're turned on, and smaller combinations are the
1550 * fall through conditions.
1551 *
1552 **/
1553static void fm10k_set_num_queues(struct fm10k_intfc *interface)
1554{
1555	/* Attempt to setup QoS and RSS first */
1556	if (fm10k_set_qos_queues(interface))
1557		return;
1558
1559	/* If we don't have QoS, just fallback to only RSS. */
1560	fm10k_set_rss_queues(interface);
1561}
1562
1563/**
1564 * fm10k_reset_num_queues - Reset the number of queues to zero
1565 * @interface: board private structure
1566 *
1567 * This function should be called whenever we need to reset the number of
1568 * queues after an error condition.
1569 */
1570static void fm10k_reset_num_queues(struct fm10k_intfc *interface)
1571{
1572	interface->num_tx_queues = 0;
1573	interface->num_rx_queues = 0;
1574	interface->num_q_vectors = 0;
1575}
1576
1577/**
1578 * fm10k_alloc_q_vector - Allocate memory for a single interrupt vector
1579 * @interface: board private structure to initialize
1580 * @v_count: q_vectors allocated on interface, used for ring interleaving
1581 * @v_idx: index of vector in interface struct
1582 * @txr_count: total number of Tx rings to allocate
1583 * @txr_idx: index of first Tx ring to allocate
1584 * @rxr_count: total number of Rx rings to allocate
1585 * @rxr_idx: index of first Rx ring to allocate
1586 *
1587 * We allocate one q_vector.  If allocation fails we return -ENOMEM.
1588 **/
1589static int fm10k_alloc_q_vector(struct fm10k_intfc *interface,
1590				unsigned int v_count, unsigned int v_idx,
1591				unsigned int txr_count, unsigned int txr_idx,
1592				unsigned int rxr_count, unsigned int rxr_idx)
1593{
1594	struct fm10k_q_vector *q_vector;
1595	struct fm10k_ring *ring;
1596	int ring_count;
1597
1598	ring_count = txr_count + rxr_count;
1599
1600	/* allocate q_vector and rings */
1601	q_vector = kzalloc(struct_size(q_vector, ring, ring_count), GFP_KERNEL);
1602	if (!q_vector)
1603		return -ENOMEM;
1604
1605	/* initialize NAPI */
1606	netif_napi_add(interface->netdev, &q_vector->napi, fm10k_poll);
 
1607
1608	/* tie q_vector and interface together */
1609	interface->q_vector[v_idx] = q_vector;
1610	q_vector->interface = interface;
1611	q_vector->v_idx = v_idx;
1612
1613	/* initialize pointer to rings */
1614	ring = q_vector->ring;
1615
1616	/* save Tx ring container info */
1617	q_vector->tx.ring = ring;
1618	q_vector->tx.work_limit = FM10K_DEFAULT_TX_WORK;
1619	q_vector->tx.itr = interface->tx_itr;
1620	q_vector->tx.itr_scale = interface->hw.mac.itr_scale;
1621	q_vector->tx.count = txr_count;
1622
1623	while (txr_count) {
1624		/* assign generic ring traits */
1625		ring->dev = &interface->pdev->dev;
1626		ring->netdev = interface->netdev;
1627
1628		/* configure backlink on ring */
1629		ring->q_vector = q_vector;
1630
1631		/* apply Tx specific ring traits */
1632		ring->count = interface->tx_ring_count;
1633		ring->queue_index = txr_idx;
1634
1635		/* assign ring to interface */
1636		interface->tx_ring[txr_idx] = ring;
1637
1638		/* update count and index */
1639		txr_count--;
1640		txr_idx += v_count;
1641
1642		/* push pointer to next ring */
1643		ring++;
1644	}
1645
1646	/* save Rx ring container info */
1647	q_vector->rx.ring = ring;
1648	q_vector->rx.itr = interface->rx_itr;
1649	q_vector->rx.itr_scale = interface->hw.mac.itr_scale;
1650	q_vector->rx.count = rxr_count;
1651
1652	while (rxr_count) {
1653		/* assign generic ring traits */
1654		ring->dev = &interface->pdev->dev;
1655		ring->netdev = interface->netdev;
1656		rcu_assign_pointer(ring->l2_accel, interface->l2_accel);
1657
1658		/* configure backlink on ring */
1659		ring->q_vector = q_vector;
1660
1661		/* apply Rx specific ring traits */
1662		ring->count = interface->rx_ring_count;
1663		ring->queue_index = rxr_idx;
1664
1665		/* assign ring to interface */
1666		interface->rx_ring[rxr_idx] = ring;
1667
1668		/* update count and index */
1669		rxr_count--;
1670		rxr_idx += v_count;
1671
1672		/* push pointer to next ring */
1673		ring++;
1674	}
1675
1676	fm10k_dbg_q_vector_init(q_vector);
1677
1678	return 0;
1679}
1680
1681/**
1682 * fm10k_free_q_vector - Free memory allocated for specific interrupt vector
1683 * @interface: board private structure to initialize
1684 * @v_idx: Index of vector to be freed
1685 *
1686 * This function frees the memory allocated to the q_vector.  In addition if
1687 * NAPI is enabled it will delete any references to the NAPI struct prior
1688 * to freeing the q_vector.
1689 **/
1690static void fm10k_free_q_vector(struct fm10k_intfc *interface, int v_idx)
1691{
1692	struct fm10k_q_vector *q_vector = interface->q_vector[v_idx];
1693	struct fm10k_ring *ring;
1694
1695	fm10k_dbg_q_vector_exit(q_vector);
1696
1697	fm10k_for_each_ring(ring, q_vector->tx)
1698		interface->tx_ring[ring->queue_index] = NULL;
1699
1700	fm10k_for_each_ring(ring, q_vector->rx)
1701		interface->rx_ring[ring->queue_index] = NULL;
1702
1703	interface->q_vector[v_idx] = NULL;
1704	netif_napi_del(&q_vector->napi);
1705	kfree_rcu(q_vector, rcu);
1706}
1707
1708/**
1709 * fm10k_alloc_q_vectors - Allocate memory for interrupt vectors
1710 * @interface: board private structure to initialize
1711 *
1712 * We allocate one q_vector per queue interrupt.  If allocation fails we
1713 * return -ENOMEM.
1714 **/
1715static int fm10k_alloc_q_vectors(struct fm10k_intfc *interface)
1716{
1717	unsigned int q_vectors = interface->num_q_vectors;
1718	unsigned int rxr_remaining = interface->num_rx_queues;
1719	unsigned int txr_remaining = interface->num_tx_queues;
1720	unsigned int rxr_idx = 0, txr_idx = 0, v_idx = 0;
1721	int err;
1722
1723	if (q_vectors >= (rxr_remaining + txr_remaining)) {
1724		for (; rxr_remaining; v_idx++) {
1725			err = fm10k_alloc_q_vector(interface, q_vectors, v_idx,
1726						   0, 0, 1, rxr_idx);
1727			if (err)
1728				goto err_out;
1729
1730			/* update counts and index */
1731			rxr_remaining--;
1732			rxr_idx++;
1733		}
1734	}
1735
1736	for (; v_idx < q_vectors; v_idx++) {
1737		int rqpv = DIV_ROUND_UP(rxr_remaining, q_vectors - v_idx);
1738		int tqpv = DIV_ROUND_UP(txr_remaining, q_vectors - v_idx);
1739
1740		err = fm10k_alloc_q_vector(interface, q_vectors, v_idx,
1741					   tqpv, txr_idx,
1742					   rqpv, rxr_idx);
1743
1744		if (err)
1745			goto err_out;
1746
1747		/* update counts and index */
1748		rxr_remaining -= rqpv;
1749		txr_remaining -= tqpv;
1750		rxr_idx++;
1751		txr_idx++;
1752	}
1753
1754	return 0;
1755
1756err_out:
1757	fm10k_reset_num_queues(interface);
1758
1759	while (v_idx--)
1760		fm10k_free_q_vector(interface, v_idx);
1761
1762	return -ENOMEM;
1763}
1764
1765/**
1766 * fm10k_free_q_vectors - Free memory allocated for interrupt vectors
1767 * @interface: board private structure to initialize
1768 *
1769 * This function frees the memory allocated to the q_vectors.  In addition if
1770 * NAPI is enabled it will delete any references to the NAPI struct prior
1771 * to freeing the q_vector.
1772 **/
1773static void fm10k_free_q_vectors(struct fm10k_intfc *interface)
1774{
1775	int v_idx = interface->num_q_vectors;
1776
1777	fm10k_reset_num_queues(interface);
1778
1779	while (v_idx--)
1780		fm10k_free_q_vector(interface, v_idx);
1781}
1782
1783/**
1784 * fm10k_reset_msix_capability - reset MSI-X capability
1785 * @interface: board private structure to initialize
1786 *
1787 * Reset the MSI-X capability back to its starting state
1788 **/
1789static void fm10k_reset_msix_capability(struct fm10k_intfc *interface)
1790{
1791	pci_disable_msix(interface->pdev);
1792	kfree(interface->msix_entries);
1793	interface->msix_entries = NULL;
1794}
1795
1796/**
1797 * fm10k_init_msix_capability - configure MSI-X capability
1798 * @interface: board private structure to initialize
1799 *
1800 * Attempt to configure the interrupts using the best available
1801 * capabilities of the hardware and the kernel.
1802 **/
1803static int fm10k_init_msix_capability(struct fm10k_intfc *interface)
1804{
1805	struct fm10k_hw *hw = &interface->hw;
1806	int v_budget, vector;
1807
1808	/* It's easy to be greedy for MSI-X vectors, but it really
1809	 * doesn't do us much good if we have a lot more vectors
1810	 * than CPU's.  So let's be conservative and only ask for
1811	 * (roughly) the same number of vectors as there are CPU's.
1812	 * the default is to use pairs of vectors
1813	 */
1814	v_budget = max(interface->num_rx_queues, interface->num_tx_queues);
1815	v_budget = min_t(u16, v_budget, num_online_cpus());
1816
1817	/* account for vectors not related to queues */
1818	v_budget += NON_Q_VECTORS;
1819
1820	/* At the same time, hardware can only support a maximum of
1821	 * hw.mac->max_msix_vectors vectors.  With features
1822	 * such as RSS and VMDq, we can easily surpass the number of Rx and Tx
1823	 * descriptor queues supported by our device.  Thus, we cap it off in
1824	 * those rare cases where the cpu count also exceeds our vector limit.
1825	 */
1826	v_budget = min_t(int, v_budget, hw->mac.max_msix_vectors);
1827
1828	/* A failure in MSI-X entry allocation is fatal. */
1829	interface->msix_entries = kcalloc(v_budget, sizeof(struct msix_entry),
1830					  GFP_KERNEL);
1831	if (!interface->msix_entries)
1832		return -ENOMEM;
1833
1834	/* populate entry values */
1835	for (vector = 0; vector < v_budget; vector++)
1836		interface->msix_entries[vector].entry = vector;
1837
1838	/* Attempt to enable MSI-X with requested value */
1839	v_budget = pci_enable_msix_range(interface->pdev,
1840					 interface->msix_entries,
1841					 MIN_MSIX_COUNT(hw),
1842					 v_budget);
1843	if (v_budget < 0) {
1844		kfree(interface->msix_entries);
1845		interface->msix_entries = NULL;
1846		return v_budget;
1847	}
1848
1849	/* record the number of queues available for q_vectors */
1850	interface->num_q_vectors = v_budget - NON_Q_VECTORS;
1851
1852	return 0;
1853}
1854
1855/**
1856 * fm10k_cache_ring_qos - Descriptor ring to register mapping for QoS
1857 * @interface: Interface structure continaining rings and devices
1858 *
1859 * Cache the descriptor ring offsets for Qos
1860 **/
1861static bool fm10k_cache_ring_qos(struct fm10k_intfc *interface)
1862{
1863	struct net_device *dev = interface->netdev;
1864	int pc, offset, rss_i, i;
1865	u16 pc_stride = interface->ring_feature[RING_F_QOS].mask + 1;
1866	u8 num_pcs = netdev_get_num_tc(dev);
1867
1868	if (num_pcs <= 1)
1869		return false;
1870
1871	rss_i = interface->ring_feature[RING_F_RSS].indices;
1872
1873	for (pc = 0, offset = 0; pc < num_pcs; pc++, offset += rss_i) {
1874		int q_idx = pc;
1875
1876		for (i = 0; i < rss_i; i++) {
1877			interface->tx_ring[offset + i]->reg_idx = q_idx;
1878			interface->tx_ring[offset + i]->qos_pc = pc;
1879			interface->rx_ring[offset + i]->reg_idx = q_idx;
1880			interface->rx_ring[offset + i]->qos_pc = pc;
1881			q_idx += pc_stride;
1882		}
1883	}
1884
1885	return true;
1886}
1887
1888/**
1889 * fm10k_cache_ring_rss - Descriptor ring to register mapping for RSS
1890 * @interface: Interface structure continaining rings and devices
1891 *
1892 * Cache the descriptor ring offsets for RSS
1893 **/
1894static void fm10k_cache_ring_rss(struct fm10k_intfc *interface)
1895{
1896	int i;
1897
1898	for (i = 0; i < interface->num_rx_queues; i++)
1899		interface->rx_ring[i]->reg_idx = i;
1900
1901	for (i = 0; i < interface->num_tx_queues; i++)
1902		interface->tx_ring[i]->reg_idx = i;
1903}
1904
1905/**
1906 * fm10k_assign_rings - Map rings to network devices
1907 * @interface: Interface structure containing rings and devices
1908 *
1909 * This function is meant to go though and configure both the network
1910 * devices so that they contain rings, and configure the rings so that
1911 * they function with their network devices.
1912 **/
1913static void fm10k_assign_rings(struct fm10k_intfc *interface)
1914{
1915	if (fm10k_cache_ring_qos(interface))
1916		return;
1917
1918	fm10k_cache_ring_rss(interface);
1919}
1920
1921static void fm10k_init_reta(struct fm10k_intfc *interface)
1922{
1923	u16 i, rss_i = interface->ring_feature[RING_F_RSS].indices;
1924	u32 reta;
1925
1926	/* If the Rx flow indirection table has been configured manually, we
1927	 * need to maintain it when possible.
1928	 */
1929	if (netif_is_rxfh_configured(interface->netdev)) {
1930		for (i = FM10K_RETA_SIZE; i--;) {
1931			reta = interface->reta[i];
1932			if ((((reta << 24) >> 24) < rss_i) &&
1933			    (((reta << 16) >> 24) < rss_i) &&
1934			    (((reta <<  8) >> 24) < rss_i) &&
1935			    (((reta)       >> 24) < rss_i))
1936				continue;
1937
1938			/* this should never happen */
1939			dev_err(&interface->pdev->dev,
1940				"RSS indirection table assigned flows out of queue bounds. Reconfiguring.\n");
1941			goto repopulate_reta;
1942		}
1943
1944		/* do nothing if all of the elements are in bounds */
1945		return;
1946	}
1947
1948repopulate_reta:
1949	fm10k_write_reta(interface, NULL);
1950}
1951
1952/**
1953 * fm10k_init_queueing_scheme - Determine proper queueing scheme
1954 * @interface: board private structure to initialize
1955 *
1956 * We determine which queueing scheme to use based on...
1957 * - Hardware queue count (num_*_queues)
1958 *   - defined by miscellaneous hardware support/features (RSS, etc.)
1959 **/
1960int fm10k_init_queueing_scheme(struct fm10k_intfc *interface)
1961{
1962	int err;
1963
1964	/* Number of supported queues */
1965	fm10k_set_num_queues(interface);
1966
1967	/* Configure MSI-X capability */
1968	err = fm10k_init_msix_capability(interface);
1969	if (err) {
1970		dev_err(&interface->pdev->dev,
1971			"Unable to initialize MSI-X capability\n");
1972		goto err_init_msix;
1973	}
1974
1975	/* Allocate memory for queues */
1976	err = fm10k_alloc_q_vectors(interface);
1977	if (err) {
1978		dev_err(&interface->pdev->dev,
1979			"Unable to allocate queue vectors\n");
1980		goto err_alloc_q_vectors;
1981	}
1982
1983	/* Map rings to devices, and map devices to physical queues */
1984	fm10k_assign_rings(interface);
1985
1986	/* Initialize RSS redirection table */
1987	fm10k_init_reta(interface);
1988
1989	return 0;
1990
1991err_alloc_q_vectors:
1992	fm10k_reset_msix_capability(interface);
1993err_init_msix:
1994	fm10k_reset_num_queues(interface);
1995	return err;
1996}
1997
1998/**
1999 * fm10k_clear_queueing_scheme - Clear the current queueing scheme settings
2000 * @interface: board private structure to clear queueing scheme on
2001 *
2002 * We go through and clear queueing specific resources and reset the structure
2003 * to pre-load conditions
2004 **/
2005void fm10k_clear_queueing_scheme(struct fm10k_intfc *interface)
2006{
2007	fm10k_free_q_vectors(interface);
2008	fm10k_reset_msix_capability(interface);
2009}