1// SPDX-License-Identifier: GPL-2.0
   2/* Copyright(c) 2017 - 2019 Pensando Systems, Inc */
   3
   4#include <linux/ethtool.h>
   5#include <linux/printk.h>
   6#include <linux/dynamic_debug.h>
   7#include <linux/netdevice.h>
   8#include <linux/etherdevice.h>
   9#include <linux/if_vlan.h>
  10#include <linux/rtnetlink.h>
  11#include <linux/interrupt.h>
  12#include <linux/pci.h>
  13#include <linux/cpumask.h>
  14
  15#include "ionic.h"
  16#include "ionic_bus.h"
  17#include "ionic_lif.h"
  18#include "ionic_txrx.h"
  19#include "ionic_ethtool.h"
  20#include "ionic_debugfs.h"
  21
  22/* queuetype support level */
  23static const u8 ionic_qtype_versions[IONIC_QTYPE_MAX] = {
  24	[IONIC_QTYPE_ADMINQ]  = 0,   /* 0 = Base version with CQ support */
  25	[IONIC_QTYPE_NOTIFYQ] = 0,   /* 0 = Base version */
  26	[IONIC_QTYPE_RXQ]     = 0,   /* 0 = Base version with CQ+SG support */
  27	[IONIC_QTYPE_TXQ]     = 1,   /* 0 = Base version with CQ+SG support
  28				      * 1 =   ... with Tx SG version 1
  29				      */
  30};
  31
  32static void ionic_lif_rx_mode(struct ionic_lif *lif);
  33static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr);
  34static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr);
  35static void ionic_link_status_check(struct ionic_lif *lif);
  36static void ionic_lif_handle_fw_down(struct ionic_lif *lif);
  37static void ionic_lif_handle_fw_up(struct ionic_lif *lif);
  38static void ionic_lif_set_netdev_info(struct ionic_lif *lif);
  39
  40static void ionic_txrx_deinit(struct ionic_lif *lif);
  41static int ionic_txrx_init(struct ionic_lif *lif);
  42static int ionic_start_queues(struct ionic_lif *lif);
  43static void ionic_stop_queues(struct ionic_lif *lif);
  44static void ionic_lif_queue_identify(struct ionic_lif *lif);
  45
  46static void ionic_dim_work(struct work_struct *work)
  47{
  48	struct dim *dim = container_of(work, struct dim, work);
  49	struct dim_cq_moder cur_moder;
  50	struct ionic_qcq *qcq;
  51	u32 new_coal;
  52
  53	cur_moder = net_dim_get_rx_moderation(dim->mode, dim->profile_ix);
  54	qcq = container_of(dim, struct ionic_qcq, dim);
  55	new_coal = ionic_coal_usec_to_hw(qcq->q.lif->ionic, cur_moder.usec);
  56	new_coal = new_coal ? new_coal : 1;
  57
  58	if (qcq->intr.dim_coal_hw != new_coal) {
  59		unsigned int qi = qcq->cq.bound_q->index;
  60		struct ionic_lif *lif = qcq->q.lif;
  61
  62		qcq->intr.dim_coal_hw = new_coal;
  63
  64		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
  65				     lif->rxqcqs[qi]->intr.index,
  66				     qcq->intr.dim_coal_hw);
  67	}
  68
  69	dim->state = DIM_START_MEASURE;
  70}
  71
  72static void ionic_lif_deferred_work(struct work_struct *work)
  73{
  74	struct ionic_lif *lif = container_of(work, struct ionic_lif, deferred.work);
  75	struct ionic_deferred *def = &lif->deferred;
  76	struct ionic_deferred_work *w = NULL;
  77
  78	do {
  79		spin_lock_bh(&def->lock);
  80		if (!list_empty(&def->list)) {
  81			w = list_first_entry(&def->list,
  82					     struct ionic_deferred_work, list);
  83			list_del(&w->list);
  84		}
  85		spin_unlock_bh(&def->lock);
  86
  87		if (!w)
  88			break;
  89
  90		switch (w->type) {
  91		case IONIC_DW_TYPE_RX_MODE:
  92			ionic_lif_rx_mode(lif);
  93			break;
  94		case IONIC_DW_TYPE_RX_ADDR_ADD:
  95			ionic_lif_addr_add(lif, w->addr);
  96			break;
  97		case IONIC_DW_TYPE_RX_ADDR_DEL:
  98			ionic_lif_addr_del(lif, w->addr);
  99			break;
 100		case IONIC_DW_TYPE_LINK_STATUS:
 101			ionic_link_status_check(lif);
 102			break;
 103		case IONIC_DW_TYPE_LIF_RESET:
 104			if (w->fw_status)
 105				ionic_lif_handle_fw_up(lif);
 106			else
 107				ionic_lif_handle_fw_down(lif);
 108			break;
 109		default:
 110			break;
 111		}
 112		kfree(w);
 113		w = NULL;
 114	} while (true);
 115}
 116
 117void ionic_lif_deferred_enqueue(struct ionic_deferred *def,
 118				struct ionic_deferred_work *work)
 119{
 120	spin_lock_bh(&def->lock);
 121	list_add_tail(&work->list, &def->list);
 122	spin_unlock_bh(&def->lock);
 123	schedule_work(&def->work);
 124}
 125
 126static void ionic_link_status_check(struct ionic_lif *lif)
 127{
 128	struct net_device *netdev = lif->netdev;
 129	u16 link_status;
 130	bool link_up;
 131
 132	if (!test_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 133		return;
 134
 135	/* Don't put carrier back up if we're in a broken state */
 136	if (test_bit(IONIC_LIF_F_BROKEN, lif->state)) {
 137		clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 138		return;
 139	}
 140
 141	link_status = le16_to_cpu(lif->info->status.link_status);
 142	link_up = link_status == IONIC_PORT_OPER_STATUS_UP;
 143
 144	if (link_up) {
 145		int err = 0;
 146
 147		if (netdev->flags & IFF_UP && netif_running(netdev)) {
 148			mutex_lock(&lif->queue_lock);
 149			err = ionic_start_queues(lif);
 150			if (err && err != -EBUSY) {
 151				netdev_err(lif->netdev,
 152					   "Failed to start queues: %d\n", err);
 153				set_bit(IONIC_LIF_F_BROKEN, lif->state);
 154				netif_carrier_off(lif->netdev);
 155			}
 156			mutex_unlock(&lif->queue_lock);
 157		}
 158
 159		if (!err && !netif_carrier_ok(netdev)) {
 160			ionic_port_identify(lif->ionic);
 161			netdev_info(netdev, "Link up - %d Gbps\n",
 162				    le32_to_cpu(lif->info->status.link_speed) / 1000);
 163			netif_carrier_on(netdev);
 164		}
 165	} else {
 166		if (netif_carrier_ok(netdev)) {
 167			netdev_info(netdev, "Link down\n");
 168			netif_carrier_off(netdev);
 169		}
 170
 171		if (netdev->flags & IFF_UP && netif_running(netdev)) {
 172			mutex_lock(&lif->queue_lock);
 173			ionic_stop_queues(lif);
 174			mutex_unlock(&lif->queue_lock);
 175		}
 176	}
 177
 178	clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 179}
 180
 181void ionic_link_status_check_request(struct ionic_lif *lif, bool can_sleep)
 182{
 183	struct ionic_deferred_work *work;
 184
 185	/* we only need one request outstanding at a time */
 186	if (test_and_set_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state))
 187		return;
 188
 189	if (!can_sleep) {
 190		work = kzalloc(sizeof(*work), GFP_ATOMIC);
 191		if (!work) {
 192			clear_bit(IONIC_LIF_F_LINK_CHECK_REQUESTED, lif->state);
 193			return;
 194		}
 195
 196		work->type = IONIC_DW_TYPE_LINK_STATUS;
 197		ionic_lif_deferred_enqueue(&lif->deferred, work);
 198	} else {
 199		ionic_link_status_check(lif);
 200	}
 201}
 202
 203static irqreturn_t ionic_isr(int irq, void *data)
 204{
 205	struct napi_struct *napi = data;
 206
 207	napi_schedule_irqoff(napi);
 208
 209	return IRQ_HANDLED;
 210}
 211
 212static int ionic_request_irq(struct ionic_lif *lif, struct ionic_qcq *qcq)
 213{
 214	struct ionic_intr_info *intr = &qcq->intr;
 215	struct device *dev = lif->ionic->dev;
 216	struct ionic_queue *q = &qcq->q;
 217	const char *name;
 218
 219	if (lif->registered)
 220		name = lif->netdev->name;
 221	else
 222		name = dev_name(dev);
 223
 224	snprintf(intr->name, sizeof(intr->name),
 225		 "%s-%s-%s", IONIC_DRV_NAME, name, q->name);
 226
 227	return devm_request_irq(dev, intr->vector, ionic_isr,
 228				0, intr->name, &qcq->napi);
 229}
 230
 231static int ionic_intr_alloc(struct ionic_lif *lif, struct ionic_intr_info *intr)
 232{
 233	struct ionic *ionic = lif->ionic;
 234	int index;
 235
 236	index = find_first_zero_bit(ionic->intrs, ionic->nintrs);
 237	if (index == ionic->nintrs) {
 238		netdev_warn(lif->netdev, "%s: no intr, index=%d nintrs=%d\n",
 239			    __func__, index, ionic->nintrs);
 240		return -ENOSPC;
 241	}
 242
 243	set_bit(index, ionic->intrs);
 244	ionic_intr_init(&ionic->idev, intr, index);
 245
 246	return 0;
 247}
 248
 249static void ionic_intr_free(struct ionic *ionic, int index)
 250{
 251	if (index != IONIC_INTR_INDEX_NOT_ASSIGNED && index < ionic->nintrs)
 252		clear_bit(index, ionic->intrs);
 253}
 254
 255static int ionic_qcq_enable(struct ionic_qcq *qcq)
 256{
 257	struct ionic_queue *q = &qcq->q;
 258	struct ionic_lif *lif = q->lif;
 259	struct ionic_dev *idev;
 260	struct device *dev;
 261
 262	struct ionic_admin_ctx ctx = {
 263		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 264		.cmd.q_control = {
 265			.opcode = IONIC_CMD_Q_CONTROL,
 266			.lif_index = cpu_to_le16(lif->index),
 267			.type = q->type,
 268			.index = cpu_to_le32(q->index),
 269			.oper = IONIC_Q_ENABLE,
 270		},
 271	};
 272
 273	idev = &lif->ionic->idev;
 274	dev = lif->ionic->dev;
 275
 276	dev_dbg(dev, "q_enable.index %d q_enable.qtype %d\n",
 277		ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 278
 279	if (qcq->flags & IONIC_QCQ_F_INTR) {
 280		irq_set_affinity_hint(qcq->intr.vector,
 281				      &qcq->intr.affinity_mask);
 282		napi_enable(&qcq->napi);
 283		ionic_intr_clean(idev->intr_ctrl, qcq->intr.index);
 284		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 285				IONIC_INTR_MASK_CLEAR);
 286	}
 287
 288	return ionic_adminq_post_wait(lif, &ctx);
 289}
 290
 291static int ionic_qcq_disable(struct ionic_qcq *qcq, bool send_to_hw)
 292{
 293	struct ionic_queue *q;
 294	struct ionic_lif *lif;
 295	int err = 0;
 296
 297	struct ionic_admin_ctx ctx = {
 298		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 299		.cmd.q_control = {
 300			.opcode = IONIC_CMD_Q_CONTROL,
 301			.oper = IONIC_Q_DISABLE,
 302		},
 303	};
 304
 305	if (!qcq)
 306		return -ENXIO;
 307
 308	q = &qcq->q;
 309	lif = q->lif;
 310
 311	if (qcq->flags & IONIC_QCQ_F_INTR) {
 312		struct ionic_dev *idev = &lif->ionic->idev;
 313
 314		cancel_work_sync(&qcq->dim.work);
 315		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 316				IONIC_INTR_MASK_SET);
 317		synchronize_irq(qcq->intr.vector);
 318		irq_set_affinity_hint(qcq->intr.vector, NULL);
 319		napi_disable(&qcq->napi);
 320	}
 321
 322	if (send_to_hw) {
 323		ctx.cmd.q_control.lif_index = cpu_to_le16(lif->index);
 324		ctx.cmd.q_control.type = q->type;
 325		ctx.cmd.q_control.index = cpu_to_le32(q->index);
 326		dev_dbg(lif->ionic->dev, "q_disable.index %d q_disable.qtype %d\n",
 327			ctx.cmd.q_control.index, ctx.cmd.q_control.type);
 328
 329		err = ionic_adminq_post_wait(lif, &ctx);
 330	}
 331
 332	return err;
 333}
 334
 335static void ionic_lif_qcq_deinit(struct ionic_lif *lif, struct ionic_qcq *qcq)
 336{
 337	struct ionic_dev *idev = &lif->ionic->idev;
 338
 339	if (!qcq)
 340		return;
 341
 342	if (!(qcq->flags & IONIC_QCQ_F_INITED))
 343		return;
 344
 345	if (qcq->flags & IONIC_QCQ_F_INTR) {
 346		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
 347				IONIC_INTR_MASK_SET);
 348		netif_napi_del(&qcq->napi);
 349	}
 350
 351	qcq->flags &= ~IONIC_QCQ_F_INITED;
 352}
 353
 354static void ionic_qcq_intr_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 355{
 356	if (!(qcq->flags & IONIC_QCQ_F_INTR) || qcq->intr.vector == 0)
 357		return;
 358
 359	irq_set_affinity_hint(qcq->intr.vector, NULL);
 360	devm_free_irq(lif->ionic->dev, qcq->intr.vector, &qcq->napi);
 361	qcq->intr.vector = 0;
 362	ionic_intr_free(lif->ionic, qcq->intr.index);
 363	qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 364}
 365
 366static void ionic_qcq_free(struct ionic_lif *lif, struct ionic_qcq *qcq)
 367{
 368	struct device *dev = lif->ionic->dev;
 369
 370	if (!qcq)
 371		return;
 372
 373	ionic_debugfs_del_qcq(qcq);
 374
 375	if (qcq->q_base) {
 376		dma_free_coherent(dev, qcq->q_size, qcq->q_base, qcq->q_base_pa);
 377		qcq->q_base = NULL;
 378		qcq->q_base_pa = 0;
 379	}
 380
 381	if (qcq->cq_base) {
 382		dma_free_coherent(dev, qcq->cq_size, qcq->cq_base, qcq->cq_base_pa);
 383		qcq->cq_base = NULL;
 384		qcq->cq_base_pa = 0;
 385	}
 386
 387	if (qcq->sg_base) {
 388		dma_free_coherent(dev, qcq->sg_size, qcq->sg_base, qcq->sg_base_pa);
 389		qcq->sg_base = NULL;
 390		qcq->sg_base_pa = 0;
 391	}
 392
 393	ionic_qcq_intr_free(lif, qcq);
 394
 395	if (qcq->cq.info) {
 396		devm_kfree(dev, qcq->cq.info);
 397		qcq->cq.info = NULL;
 398	}
 399	if (qcq->q.info) {
 400		devm_kfree(dev, qcq->q.info);
 401		qcq->q.info = NULL;
 402	}
 403}
 404
 405static void ionic_qcqs_free(struct ionic_lif *lif)
 406{
 407	struct device *dev = lif->ionic->dev;
 408	struct ionic_qcq *adminqcq;
 409	unsigned long irqflags;
 410
 411	if (lif->notifyqcq) {
 412		ionic_qcq_free(lif, lif->notifyqcq);
 413		devm_kfree(dev, lif->notifyqcq);
 414		lif->notifyqcq = NULL;
 415	}
 416
 417	if (lif->adminqcq) {
 418		spin_lock_irqsave(&lif->adminq_lock, irqflags);
 419		adminqcq = READ_ONCE(lif->adminqcq);
 420		lif->adminqcq = NULL;
 421		spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
 422		if (adminqcq) {
 423			ionic_qcq_free(lif, adminqcq);
 424			devm_kfree(dev, adminqcq);
 425		}
 426	}
 427
 428	if (lif->rxqcqs) {
 429		devm_kfree(dev, lif->rxqstats);
 430		lif->rxqstats = NULL;
 431		devm_kfree(dev, lif->rxqcqs);
 432		lif->rxqcqs = NULL;
 433	}
 434
 435	if (lif->txqcqs) {
 436		devm_kfree(dev, lif->txqstats);
 437		lif->txqstats = NULL;
 438		devm_kfree(dev, lif->txqcqs);
 439		lif->txqcqs = NULL;
 440	}
 441}
 442
 443static void ionic_link_qcq_interrupts(struct ionic_qcq *src_qcq,
 444				      struct ionic_qcq *n_qcq)
 445{
 446	if (WARN_ON(n_qcq->flags & IONIC_QCQ_F_INTR)) {
 447		ionic_intr_free(n_qcq->cq.lif->ionic, n_qcq->intr.index);
 448		n_qcq->flags &= ~IONIC_QCQ_F_INTR;
 449	}
 450
 451	n_qcq->intr.vector = src_qcq->intr.vector;
 452	n_qcq->intr.index = src_qcq->intr.index;
 453}
 454
 455static int ionic_alloc_qcq_interrupt(struct ionic_lif *lif, struct ionic_qcq *qcq)
 456{
 457	int err;
 458
 459	if (!(qcq->flags & IONIC_QCQ_F_INTR)) {
 460		qcq->intr.index = IONIC_INTR_INDEX_NOT_ASSIGNED;
 461		return 0;
 462	}
 463
 464	err = ionic_intr_alloc(lif, &qcq->intr);
 465	if (err) {
 466		netdev_warn(lif->netdev, "no intr for %s: %d\n",
 467			    qcq->q.name, err);
 468		goto err_out;
 469	}
 470
 471	err = ionic_bus_get_irq(lif->ionic, qcq->intr.index);
 472	if (err < 0) {
 473		netdev_warn(lif->netdev, "no vector for %s: %d\n",
 474			    qcq->q.name, err);
 475		goto err_out_free_intr;
 476	}
 477	qcq->intr.vector = err;
 478	ionic_intr_mask_assert(lif->ionic->idev.intr_ctrl, qcq->intr.index,
 479			       IONIC_INTR_MASK_SET);
 480
 481	err = ionic_request_irq(lif, qcq);
 482	if (err) {
 483		netdev_warn(lif->netdev, "irq request failed %d\n", err);
 484		goto err_out_free_intr;
 485	}
 486
 487	/* try to get the irq on the local numa node first */
 488	qcq->intr.cpu = cpumask_local_spread(qcq->intr.index,
 489					     dev_to_node(lif->ionic->dev));
 490	if (qcq->intr.cpu != -1)
 491		cpumask_set_cpu(qcq->intr.cpu, &qcq->intr.affinity_mask);
 492
 493	netdev_dbg(lif->netdev, "%s: Interrupt index %d\n", qcq->q.name, qcq->intr.index);
 494	return 0;
 495
 496err_out_free_intr:
 497	ionic_intr_free(lif->ionic, qcq->intr.index);
 498err_out:
 499	return err;
 500}
 501
 502static int ionic_qcq_alloc(struct ionic_lif *lif, unsigned int type,
 503			   unsigned int index,
 504			   const char *name, unsigned int flags,
 505			   unsigned int num_descs, unsigned int desc_size,
 506			   unsigned int cq_desc_size,
 507			   unsigned int sg_desc_size,
 508			   unsigned int pid, struct ionic_qcq **qcq)
 509{
 510	struct ionic_dev *idev = &lif->ionic->idev;
 511	struct device *dev = lif->ionic->dev;
 512	void *q_base, *cq_base, *sg_base;
 513	dma_addr_t cq_base_pa = 0;
 514	dma_addr_t sg_base_pa = 0;
 515	dma_addr_t q_base_pa = 0;
 516	struct ionic_qcq *new;
 517	int err;
 518
 519	*qcq = NULL;
 520
 521	new = devm_kzalloc(dev, sizeof(*new), GFP_KERNEL);
 522	if (!new) {
 523		netdev_err(lif->netdev, "Cannot allocate queue structure\n");
 524		err = -ENOMEM;
 525		goto err_out;
 526	}
 527
 528	new->q.dev = dev;
 529	new->flags = flags;
 530
 531	new->q.info = devm_kcalloc(dev, num_descs, sizeof(*new->q.info),
 532				   GFP_KERNEL);
 533	if (!new->q.info) {
 534		netdev_err(lif->netdev, "Cannot allocate queue info\n");
 535		err = -ENOMEM;
 536		goto err_out_free_qcq;
 537	}
 538
 539	new->q.type = type;
 540	new->q.max_sg_elems = lif->qtype_info[type].max_sg_elems;
 541
 542	err = ionic_q_init(lif, idev, &new->q, index, name, num_descs,
 543			   desc_size, sg_desc_size, pid);
 544	if (err) {
 545		netdev_err(lif->netdev, "Cannot initialize queue\n");
 546		goto err_out_free_q_info;
 547	}
 548
 549	err = ionic_alloc_qcq_interrupt(lif, new);
 550	if (err)
 551		goto err_out;
 552
 553	new->cq.info = devm_kcalloc(dev, num_descs, sizeof(*new->cq.info),
 554				    GFP_KERNEL);
 555	if (!new->cq.info) {
 556		netdev_err(lif->netdev, "Cannot allocate completion queue info\n");
 557		err = -ENOMEM;
 558		goto err_out_free_irq;
 559	}
 560
 561	err = ionic_cq_init(lif, &new->cq, &new->intr, num_descs, cq_desc_size);
 562	if (err) {
 563		netdev_err(lif->netdev, "Cannot initialize completion queue\n");
 564		goto err_out_free_cq_info;
 565	}
 566
 567	if (flags & IONIC_QCQ_F_NOTIFYQ) {
 568		int q_size, cq_size;
 569
 570		/* q & cq need to be contiguous in case of notifyq */
 571		q_size = ALIGN(num_descs * desc_size, PAGE_SIZE);
 572		cq_size = ALIGN(num_descs * cq_desc_size, PAGE_SIZE);
 573
 574		new->q_size = PAGE_SIZE + q_size + cq_size;
 575		new->q_base = dma_alloc_coherent(dev, new->q_size,
 576						 &new->q_base_pa, GFP_KERNEL);
 577		if (!new->q_base) {
 578			netdev_err(lif->netdev, "Cannot allocate qcq DMA memory\n");
 579			err = -ENOMEM;
 580			goto err_out_free_cq_info;
 581		}
 582		q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 583		q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 584		ionic_q_map(&new->q, q_base, q_base_pa);
 585
 586		cq_base = PTR_ALIGN(q_base + q_size, PAGE_SIZE);
 587		cq_base_pa = ALIGN(new->q_base_pa + q_size, PAGE_SIZE);
 588		ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 589		ionic_cq_bind(&new->cq, &new->q);
 590	} else {
 591		new->q_size = PAGE_SIZE + (num_descs * desc_size);
 592		new->q_base = dma_alloc_coherent(dev, new->q_size, &new->q_base_pa,
 593						 GFP_KERNEL);
 594		if (!new->q_base) {
 595			netdev_err(lif->netdev, "Cannot allocate queue DMA memory\n");
 596			err = -ENOMEM;
 597			goto err_out_free_cq_info;
 598		}
 599		q_base = PTR_ALIGN(new->q_base, PAGE_SIZE);
 600		q_base_pa = ALIGN(new->q_base_pa, PAGE_SIZE);
 601		ionic_q_map(&new->q, q_base, q_base_pa);
 602
 603		new->cq_size = PAGE_SIZE + (num_descs * cq_desc_size);
 604		new->cq_base = dma_alloc_coherent(dev, new->cq_size, &new->cq_base_pa,
 605						  GFP_KERNEL);
 606		if (!new->cq_base) {
 607			netdev_err(lif->netdev, "Cannot allocate cq DMA memory\n");
 608			err = -ENOMEM;
 609			goto err_out_free_q;
 610		}
 611		cq_base = PTR_ALIGN(new->cq_base, PAGE_SIZE);
 612		cq_base_pa = ALIGN(new->cq_base_pa, PAGE_SIZE);
 613		ionic_cq_map(&new->cq, cq_base, cq_base_pa);
 614		ionic_cq_bind(&new->cq, &new->q);
 615	}
 616
 617	if (flags & IONIC_QCQ_F_SG) {
 618		new->sg_size = PAGE_SIZE + (num_descs * sg_desc_size);
 619		new->sg_base = dma_alloc_coherent(dev, new->sg_size, &new->sg_base_pa,
 620						  GFP_KERNEL);
 621		if (!new->sg_base) {
 622			netdev_err(lif->netdev, "Cannot allocate sg DMA memory\n");
 623			err = -ENOMEM;
 624			goto err_out_free_cq;
 625		}
 626		sg_base = PTR_ALIGN(new->sg_base, PAGE_SIZE);
 627		sg_base_pa = ALIGN(new->sg_base_pa, PAGE_SIZE);
 628		ionic_q_sg_map(&new->q, sg_base, sg_base_pa);
 629	}
 630
 631	INIT_WORK(&new->dim.work, ionic_dim_work);
 632	new->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE;
 633
 634	*qcq = new;
 635
 636	return 0;
 637
 638err_out_free_cq:
 639	dma_free_coherent(dev, new->cq_size, new->cq_base, new->cq_base_pa);
 640err_out_free_q:
 641	dma_free_coherent(dev, new->q_size, new->q_base, new->q_base_pa);
 642err_out_free_cq_info:
 643	devm_kfree(dev, new->cq.info);
 644err_out_free_irq:
 645	if (flags & IONIC_QCQ_F_INTR) {
 646		devm_free_irq(dev, new->intr.vector, &new->napi);
 647		ionic_intr_free(lif->ionic, new->intr.index);
 648	}
 649err_out_free_q_info:
 650	devm_kfree(dev, new->q.info);
 651err_out_free_qcq:
 652	devm_kfree(dev, new);
 653err_out:
 654	dev_err(dev, "qcq alloc of %s%d failed %d\n", name, index, err);
 655	return err;
 656}
 657
 658static int ionic_qcqs_alloc(struct ionic_lif *lif)
 659{
 660	struct device *dev = lif->ionic->dev;
 661	unsigned int flags;
 662	int err;
 663
 664	flags = IONIC_QCQ_F_INTR;
 665	err = ionic_qcq_alloc(lif, IONIC_QTYPE_ADMINQ, 0, "admin", flags,
 666			      IONIC_ADMINQ_LENGTH,
 667			      sizeof(struct ionic_admin_cmd),
 668			      sizeof(struct ionic_admin_comp),
 669			      0, lif->kern_pid, &lif->adminqcq);
 670	if (err)
 671		return err;
 672	ionic_debugfs_add_qcq(lif, lif->adminqcq);
 673
 674	if (lif->ionic->nnqs_per_lif) {
 675		flags = IONIC_QCQ_F_NOTIFYQ;
 676		err = ionic_qcq_alloc(lif, IONIC_QTYPE_NOTIFYQ, 0, "notifyq",
 677				      flags, IONIC_NOTIFYQ_LENGTH,
 678				      sizeof(struct ionic_notifyq_cmd),
 679				      sizeof(union ionic_notifyq_comp),
 680				      0, lif->kern_pid, &lif->notifyqcq);
 681		if (err)
 682			goto err_out;
 683		ionic_debugfs_add_qcq(lif, lif->notifyqcq);
 684
 685		/* Let the notifyq ride on the adminq interrupt */
 686		ionic_link_qcq_interrupts(lif->adminqcq, lif->notifyqcq);
 687	}
 688
 689	err = -ENOMEM;
 690	lif->txqcqs = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif,
 691				   sizeof(*lif->txqcqs), GFP_KERNEL);
 692	if (!lif->txqcqs)
 693		goto err_out;
 694	lif->rxqcqs = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif,
 695				   sizeof(*lif->rxqcqs), GFP_KERNEL);
 696	if (!lif->rxqcqs)
 697		goto err_out;
 698
 699	lif->txqstats = devm_kcalloc(dev, lif->ionic->ntxqs_per_lif + 1,
 700				     sizeof(*lif->txqstats), GFP_KERNEL);
 701	if (!lif->txqstats)
 702		goto err_out;
 703	lif->rxqstats = devm_kcalloc(dev, lif->ionic->nrxqs_per_lif + 1,
 704				     sizeof(*lif->rxqstats), GFP_KERNEL);
 705	if (!lif->rxqstats)
 706		goto err_out;
 707
 708	return 0;
 709
 710err_out:
 711	ionic_qcqs_free(lif);
 712	return err;
 713}
 714
 715static void ionic_qcq_sanitize(struct ionic_qcq *qcq)
 716{
 717	qcq->q.tail_idx = 0;
 718	qcq->q.head_idx = 0;
 719	qcq->cq.tail_idx = 0;
 720	qcq->cq.done_color = 1;
 721	memset(qcq->q_base, 0, qcq->q_size);
 722	memset(qcq->cq_base, 0, qcq->cq_size);
 723	memset(qcq->sg_base, 0, qcq->sg_size);
 724}
 725
 726static int ionic_lif_txq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 727{
 728	struct device *dev = lif->ionic->dev;
 729	struct ionic_queue *q = &qcq->q;
 730	struct ionic_cq *cq = &qcq->cq;
 731	struct ionic_admin_ctx ctx = {
 732		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 733		.cmd.q_init = {
 734			.opcode = IONIC_CMD_Q_INIT,
 735			.lif_index = cpu_to_le16(lif->index),
 736			.type = q->type,
 737			.ver = lif->qtype_info[q->type].version,
 738			.index = cpu_to_le32(q->index),
 739			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 740					     IONIC_QINIT_F_SG),
 741			.pid = cpu_to_le16(q->pid),
 742			.ring_size = ilog2(q->num_descs),
 743			.ring_base = cpu_to_le64(q->base_pa),
 744			.cq_ring_base = cpu_to_le64(cq->base_pa),
 745			.sg_ring_base = cpu_to_le64(q->sg_base_pa),
 746			.features = cpu_to_le64(q->features),
 747		},
 748	};
 749	unsigned int intr_index;
 750	int err;
 751
 752	intr_index = qcq->intr.index;
 753
 754	ctx.cmd.q_init.intr_index = cpu_to_le16(intr_index);
 755
 756	dev_dbg(dev, "txq_init.pid %d\n", ctx.cmd.q_init.pid);
 757	dev_dbg(dev, "txq_init.index %d\n", ctx.cmd.q_init.index);
 758	dev_dbg(dev, "txq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 759	dev_dbg(dev, "txq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 760	dev_dbg(dev, "txq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 761	dev_dbg(dev, "txq_init.ver %d\n", ctx.cmd.q_init.ver);
 762	dev_dbg(dev, "txq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 763
 764	ionic_qcq_sanitize(qcq);
 765
 766	err = ionic_adminq_post_wait(lif, &ctx);
 767	if (err)
 768		return err;
 769
 770	q->hw_type = ctx.comp.q_init.hw_type;
 771	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 772	q->dbval = IONIC_DBELL_QID(q->hw_index);
 773
 774	dev_dbg(dev, "txq->hw_type %d\n", q->hw_type);
 775	dev_dbg(dev, "txq->hw_index %d\n", q->hw_index);
 776
 777	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 778		netif_napi_add(lif->netdev, &qcq->napi, ionic_tx_napi,
 779			       NAPI_POLL_WEIGHT);
 780
 781	qcq->flags |= IONIC_QCQ_F_INITED;
 782
 783	return 0;
 784}
 785
 786static int ionic_lif_rxq_init(struct ionic_lif *lif, struct ionic_qcq *qcq)
 787{
 788	struct device *dev = lif->ionic->dev;
 789	struct ionic_queue *q = &qcq->q;
 790	struct ionic_cq *cq = &qcq->cq;
 791	struct ionic_admin_ctx ctx = {
 792		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
 793		.cmd.q_init = {
 794			.opcode = IONIC_CMD_Q_INIT,
 795			.lif_index = cpu_to_le16(lif->index),
 796			.type = q->type,
 797			.ver = lif->qtype_info[q->type].version,
 798			.index = cpu_to_le32(q->index),
 799			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
 800					     IONIC_QINIT_F_SG),
 801			.intr_index = cpu_to_le16(cq->bound_intr->index),
 802			.pid = cpu_to_le16(q->pid),
 803			.ring_size = ilog2(q->num_descs),
 804			.ring_base = cpu_to_le64(q->base_pa),
 805			.cq_ring_base = cpu_to_le64(cq->base_pa),
 806			.sg_ring_base = cpu_to_le64(q->sg_base_pa),
 807			.features = cpu_to_le64(q->features),
 808		},
 809	};
 810	int err;
 811
 812	dev_dbg(dev, "rxq_init.pid %d\n", ctx.cmd.q_init.pid);
 813	dev_dbg(dev, "rxq_init.index %d\n", ctx.cmd.q_init.index);
 814	dev_dbg(dev, "rxq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
 815	dev_dbg(dev, "rxq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
 816	dev_dbg(dev, "rxq_init.flags 0x%x\n", ctx.cmd.q_init.flags);
 817	dev_dbg(dev, "rxq_init.ver %d\n", ctx.cmd.q_init.ver);
 818	dev_dbg(dev, "rxq_init.intr_index %d\n", ctx.cmd.q_init.intr_index);
 819
 820	ionic_qcq_sanitize(qcq);
 821
 822	err = ionic_adminq_post_wait(lif, &ctx);
 823	if (err)
 824		return err;
 825
 826	q->hw_type = ctx.comp.q_init.hw_type;
 827	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
 828	q->dbval = IONIC_DBELL_QID(q->hw_index);
 829
 830	dev_dbg(dev, "rxq->hw_type %d\n", q->hw_type);
 831	dev_dbg(dev, "rxq->hw_index %d\n", q->hw_index);
 832
 833	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
 834		netif_napi_add(lif->netdev, &qcq->napi, ionic_rx_napi,
 835			       NAPI_POLL_WEIGHT);
 836	else
 837		netif_napi_add(lif->netdev, &qcq->napi, ionic_txrx_napi,
 838			       NAPI_POLL_WEIGHT);
 839
 840	qcq->flags |= IONIC_QCQ_F_INITED;
 841
 842	return 0;
 843}
 844
 845int ionic_lif_create_hwstamp_txq(struct ionic_lif *lif)
 846{
 847	unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 848	unsigned int txq_i, flags;
 849	struct ionic_qcq *txq;
 850	u64 features;
 851	int err;
 852
 853	mutex_lock(&lif->queue_lock);
 854
 855	if (lif->hwstamp_txq)
 856		goto out;
 857
 858	features = IONIC_Q_F_2X_CQ_DESC | IONIC_TXQ_F_HWSTAMP;
 859
 860	num_desc = IONIC_MIN_TXRX_DESC;
 861	desc_sz = sizeof(struct ionic_txq_desc);
 862	comp_sz = 2 * sizeof(struct ionic_txq_comp);
 863
 864	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
 865	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz == sizeof(struct ionic_txq_sg_desc_v1))
 866		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
 867	else
 868		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
 869
 870	txq_i = lif->ionic->ntxqs_per_lif;
 871	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
 872
 873	err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, txq_i, "hwstamp_tx", flags,
 874			      num_desc, desc_sz, comp_sz, sg_desc_sz,
 875			      lif->kern_pid, &txq);
 876	if (err)
 877		goto err_qcq_alloc;
 878
 879	txq->q.features = features;
 880
 881	ionic_link_qcq_interrupts(lif->adminqcq, txq);
 882	ionic_debugfs_add_qcq(lif, txq);
 883
 884	lif->hwstamp_txq = txq;
 885
 886	if (netif_running(lif->netdev)) {
 887		err = ionic_lif_txq_init(lif, txq);
 888		if (err)
 889			goto err_qcq_init;
 890
 891		if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 892			err = ionic_qcq_enable(txq);
 893			if (err)
 894				goto err_qcq_enable;
 895		}
 896	}
 897
 898out:
 899	mutex_unlock(&lif->queue_lock);
 900
 901	return 0;
 902
 903err_qcq_enable:
 904	ionic_lif_qcq_deinit(lif, txq);
 905err_qcq_init:
 906	lif->hwstamp_txq = NULL;
 907	ionic_debugfs_del_qcq(txq);
 908	ionic_qcq_free(lif, txq);
 909	devm_kfree(lif->ionic->dev, txq);
 910err_qcq_alloc:
 911	mutex_unlock(&lif->queue_lock);
 912	return err;
 913}
 914
 915int ionic_lif_create_hwstamp_rxq(struct ionic_lif *lif)
 916{
 917	unsigned int num_desc, desc_sz, comp_sz, sg_desc_sz;
 918	unsigned int rxq_i, flags;
 919	struct ionic_qcq *rxq;
 920	u64 features;
 921	int err;
 922
 923	mutex_lock(&lif->queue_lock);
 924
 925	if (lif->hwstamp_rxq)
 926		goto out;
 927
 928	features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 929
 930	num_desc = IONIC_MIN_TXRX_DESC;
 931	desc_sz = sizeof(struct ionic_rxq_desc);
 932	comp_sz = 2 * sizeof(struct ionic_rxq_comp);
 933	sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
 934
 935	rxq_i = lif->ionic->nrxqs_per_lif;
 936	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG;
 937
 938	err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, rxq_i, "hwstamp_rx", flags,
 939			      num_desc, desc_sz, comp_sz, sg_desc_sz,
 940			      lif->kern_pid, &rxq);
 941	if (err)
 942		goto err_qcq_alloc;
 943
 944	rxq->q.features = features;
 945
 946	ionic_link_qcq_interrupts(lif->adminqcq, rxq);
 947	ionic_debugfs_add_qcq(lif, rxq);
 948
 949	lif->hwstamp_rxq = rxq;
 950
 951	if (netif_running(lif->netdev)) {
 952		err = ionic_lif_rxq_init(lif, rxq);
 953		if (err)
 954			goto err_qcq_init;
 955
 956		if (test_bit(IONIC_LIF_F_UP, lif->state)) {
 957			ionic_rx_fill(&rxq->q);
 958			err = ionic_qcq_enable(rxq);
 959			if (err)
 960				goto err_qcq_enable;
 961		}
 962	}
 963
 964out:
 965	mutex_unlock(&lif->queue_lock);
 966
 967	return 0;
 968
 969err_qcq_enable:
 970	ionic_lif_qcq_deinit(lif, rxq);
 971err_qcq_init:
 972	lif->hwstamp_rxq = NULL;
 973	ionic_debugfs_del_qcq(rxq);
 974	ionic_qcq_free(lif, rxq);
 975	devm_kfree(lif->ionic->dev, rxq);
 976err_qcq_alloc:
 977	mutex_unlock(&lif->queue_lock);
 978	return err;
 979}
 980
 981int ionic_lif_config_hwstamp_rxq_all(struct ionic_lif *lif, bool rx_all)
 982{
 983	struct ionic_queue_params qparam;
 984
 985	ionic_init_queue_params(lif, &qparam);
 986
 987	if (rx_all)
 988		qparam.rxq_features = IONIC_Q_F_2X_CQ_DESC | IONIC_RXQ_F_HWSTAMP;
 989	else
 990		qparam.rxq_features = 0;
 991
 992	/* if we're not running, just set the values and return */
 993	if (!netif_running(lif->netdev)) {
 994		lif->rxq_features = qparam.rxq_features;
 995		return 0;
 996	}
 997
 998	return ionic_reconfigure_queues(lif, &qparam);
 999}
1000
1001int ionic_lif_set_hwstamp_txmode(struct ionic_lif *lif, u16 txstamp_mode)
1002{
1003	struct ionic_admin_ctx ctx = {
1004		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1005		.cmd.lif_setattr = {
1006			.opcode = IONIC_CMD_LIF_SETATTR,
1007			.index = cpu_to_le16(lif->index),
1008			.attr = IONIC_LIF_ATTR_TXSTAMP,
1009			.txstamp_mode = cpu_to_le16(txstamp_mode),
1010		},
1011	};
1012
1013	return ionic_adminq_post_wait(lif, &ctx);
1014}
1015
1016static void ionic_lif_del_hwstamp_rxfilt(struct ionic_lif *lif)
1017{
1018	struct ionic_admin_ctx ctx = {
1019		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1020		.cmd.rx_filter_del = {
1021			.opcode = IONIC_CMD_RX_FILTER_DEL,
1022			.lif_index = cpu_to_le16(lif->index),
1023		},
1024	};
1025	struct ionic_rx_filter *f;
1026	u32 filter_id;
1027	int err;
1028
1029	spin_lock_bh(&lif->rx_filters.lock);
1030
1031	f = ionic_rx_filter_rxsteer(lif);
1032	if (!f) {
1033		spin_unlock_bh(&lif->rx_filters.lock);
1034		return;
1035	}
1036
1037	filter_id = f->filter_id;
1038	ionic_rx_filter_free(lif, f);
1039
1040	spin_unlock_bh(&lif->rx_filters.lock);
1041
1042	netdev_dbg(lif->netdev, "rx_filter del RXSTEER (id %d)\n", filter_id);
1043
1044	ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(filter_id);
1045
1046	err = ionic_adminq_post_wait(lif, &ctx);
1047	if (err && err != -EEXIST)
1048		netdev_dbg(lif->netdev, "failed to delete rx_filter RXSTEER (id %d)\n", filter_id);
1049}
1050
1051static int ionic_lif_add_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1052{
1053	struct ionic_admin_ctx ctx = {
1054		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1055		.cmd.rx_filter_add = {
1056			.opcode = IONIC_CMD_RX_FILTER_ADD,
1057			.lif_index = cpu_to_le16(lif->index),
1058			.match = cpu_to_le16(IONIC_RX_FILTER_STEER_PKTCLASS),
1059			.pkt_class = cpu_to_le64(pkt_class),
1060		},
1061	};
1062	u8 qtype;
1063	u32 qid;
1064	int err;
1065
1066	if (!lif->hwstamp_rxq)
1067		return -EINVAL;
1068
1069	qtype = lif->hwstamp_rxq->q.type;
1070	ctx.cmd.rx_filter_add.qtype = qtype;
1071
1072	qid = lif->hwstamp_rxq->q.index;
1073	ctx.cmd.rx_filter_add.qid = cpu_to_le32(qid);
1074
1075	netdev_dbg(lif->netdev, "rx_filter add RXSTEER\n");
1076	err = ionic_adminq_post_wait(lif, &ctx);
1077	if (err && err != -EEXIST)
1078		return err;
1079
1080	return ionic_rx_filter_save(lif, 0, qid, 0, &ctx);
1081}
1082
1083int ionic_lif_set_hwstamp_rxfilt(struct ionic_lif *lif, u64 pkt_class)
1084{
1085	ionic_lif_del_hwstamp_rxfilt(lif);
1086
1087	if (!pkt_class)
1088		return 0;
1089
1090	return ionic_lif_add_hwstamp_rxfilt(lif, pkt_class);
1091}
1092
1093static bool ionic_notifyq_service(struct ionic_cq *cq,
1094				  struct ionic_cq_info *cq_info)
1095{
1096	union ionic_notifyq_comp *comp = cq_info->cq_desc;
1097	struct ionic_deferred_work *work;
1098	struct net_device *netdev;
1099	struct ionic_queue *q;
1100	struct ionic_lif *lif;
1101	u64 eid;
1102
1103	q = cq->bound_q;
1104	lif = q->info[0].cb_arg;
1105	netdev = lif->netdev;
1106	eid = le64_to_cpu(comp->event.eid);
1107
1108	/* Have we run out of new completions to process? */
1109	if ((s64)(eid - lif->last_eid) <= 0)
1110		return false;
1111
1112	lif->last_eid = eid;
1113
1114	dev_dbg(lif->ionic->dev, "notifyq event:\n");
1115	dynamic_hex_dump("event ", DUMP_PREFIX_OFFSET, 16, 1,
1116			 comp, sizeof(*comp), true);
1117
1118	switch (le16_to_cpu(comp->event.ecode)) {
1119	case IONIC_EVENT_LINK_CHANGE:
1120		ionic_link_status_check_request(lif, CAN_NOT_SLEEP);
1121		break;
1122	case IONIC_EVENT_RESET:
1123		work = kzalloc(sizeof(*work), GFP_ATOMIC);
1124		if (!work) {
1125			netdev_err(lif->netdev, "Reset event dropped\n");
1126		} else {
1127			work->type = IONIC_DW_TYPE_LIF_RESET;
1128			ionic_lif_deferred_enqueue(&lif->deferred, work);
1129		}
1130		break;
1131	default:
1132		netdev_warn(netdev, "Notifyq event ecode=%d eid=%lld\n",
1133			    comp->event.ecode, eid);
1134		break;
1135	}
1136
1137	return true;
1138}
1139
1140static bool ionic_adminq_service(struct ionic_cq *cq,
1141				 struct ionic_cq_info *cq_info)
1142{
1143	struct ionic_admin_comp *comp = cq_info->cq_desc;
1144
1145	if (!color_match(comp->color, cq->done_color))
1146		return false;
1147
1148	ionic_q_service(cq->bound_q, cq_info, le16_to_cpu(comp->comp_index));
1149
1150	return true;
1151}
1152
1153static int ionic_adminq_napi(struct napi_struct *napi, int budget)
1154{
1155	struct ionic_intr_info *intr = napi_to_cq(napi)->bound_intr;
1156	struct ionic_lif *lif = napi_to_cq(napi)->lif;
1157	struct ionic_dev *idev = &lif->ionic->idev;
1158	unsigned long irqflags;
1159	unsigned int flags = 0;
1160	int rx_work = 0;
1161	int tx_work = 0;
1162	int n_work = 0;
1163	int a_work = 0;
1164	int work_done;
1165	int credits;
1166
1167	if (lif->notifyqcq && lif->notifyqcq->flags & IONIC_QCQ_F_INITED)
1168		n_work = ionic_cq_service(&lif->notifyqcq->cq, budget,
1169					  ionic_notifyq_service, NULL, NULL);
1170
1171	spin_lock_irqsave(&lif->adminq_lock, irqflags);
1172	if (lif->adminqcq && lif->adminqcq->flags & IONIC_QCQ_F_INITED)
1173		a_work = ionic_cq_service(&lif->adminqcq->cq, budget,
1174					  ionic_adminq_service, NULL, NULL);
1175	spin_unlock_irqrestore(&lif->adminq_lock, irqflags);
1176
1177	if (lif->hwstamp_rxq)
1178		rx_work = ionic_cq_service(&lif->hwstamp_rxq->cq, budget,
1179					   ionic_rx_service, NULL, NULL);
1180
1181	if (lif->hwstamp_txq)
1182		tx_work = ionic_cq_service(&lif->hwstamp_txq->cq, budget,
1183					   ionic_tx_service, NULL, NULL);
1184
1185	work_done = max(max(n_work, a_work), max(rx_work, tx_work));
1186	if (work_done < budget && napi_complete_done(napi, work_done)) {
1187		flags |= IONIC_INTR_CRED_UNMASK;
1188		intr->rearm_count++;
1189	}
1190
1191	if (work_done || flags) {
1192		flags |= IONIC_INTR_CRED_RESET_COALESCE;
1193		credits = n_work + a_work + rx_work + tx_work;
1194		ionic_intr_credits(idev->intr_ctrl, intr->index, credits, flags);
1195	}
1196
1197	return work_done;
1198}
1199
1200void ionic_get_stats64(struct net_device *netdev,
1201		       struct rtnl_link_stats64 *ns)
1202{
1203	struct ionic_lif *lif = netdev_priv(netdev);
1204	struct ionic_lif_stats *ls;
1205
1206	memset(ns, 0, sizeof(*ns));
1207	ls = &lif->info->stats;
1208
1209	ns->rx_packets = le64_to_cpu(ls->rx_ucast_packets) +
1210			 le64_to_cpu(ls->rx_mcast_packets) +
1211			 le64_to_cpu(ls->rx_bcast_packets);
1212
1213	ns->tx_packets = le64_to_cpu(ls->tx_ucast_packets) +
1214			 le64_to_cpu(ls->tx_mcast_packets) +
1215			 le64_to_cpu(ls->tx_bcast_packets);
1216
1217	ns->rx_bytes = le64_to_cpu(ls->rx_ucast_bytes) +
1218		       le64_to_cpu(ls->rx_mcast_bytes) +
1219		       le64_to_cpu(ls->rx_bcast_bytes);
1220
1221	ns->tx_bytes = le64_to_cpu(ls->tx_ucast_bytes) +
1222		       le64_to_cpu(ls->tx_mcast_bytes) +
1223		       le64_to_cpu(ls->tx_bcast_bytes);
1224
1225	ns->rx_dropped = le64_to_cpu(ls->rx_ucast_drop_packets) +
1226			 le64_to_cpu(ls->rx_mcast_drop_packets) +
1227			 le64_to_cpu(ls->rx_bcast_drop_packets);
1228
1229	ns->tx_dropped = le64_to_cpu(ls->tx_ucast_drop_packets) +
1230			 le64_to_cpu(ls->tx_mcast_drop_packets) +
1231			 le64_to_cpu(ls->tx_bcast_drop_packets);
1232
1233	ns->multicast = le64_to_cpu(ls->rx_mcast_packets);
1234
1235	ns->rx_over_errors = le64_to_cpu(ls->rx_queue_empty);
1236
1237	ns->rx_missed_errors = le64_to_cpu(ls->rx_dma_error) +
1238			       le64_to_cpu(ls->rx_queue_disabled) +
1239			       le64_to_cpu(ls->rx_desc_fetch_error) +
1240			       le64_to_cpu(ls->rx_desc_data_error);
1241
1242	ns->tx_aborted_errors = le64_to_cpu(ls->tx_dma_error) +
1243				le64_to_cpu(ls->tx_queue_disabled) +
1244				le64_to_cpu(ls->tx_desc_fetch_error) +
1245				le64_to_cpu(ls->tx_desc_data_error);
1246
1247	ns->rx_errors = ns->rx_over_errors +
1248			ns->rx_missed_errors;
1249
1250	ns->tx_errors = ns->tx_aborted_errors;
1251}
1252
1253static int ionic_lif_addr_add(struct ionic_lif *lif, const u8 *addr)
1254{
1255	struct ionic_admin_ctx ctx = {
1256		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1257		.cmd.rx_filter_add = {
1258			.opcode = IONIC_CMD_RX_FILTER_ADD,
1259			.lif_index = cpu_to_le16(lif->index),
1260			.match = cpu_to_le16(IONIC_RX_FILTER_MATCH_MAC),
1261		},
1262	};
1263	struct ionic_rx_filter *f;
1264	int err;
1265
1266	/* don't bother if we already have it */
1267	spin_lock_bh(&lif->rx_filters.lock);
1268	f = ionic_rx_filter_by_addr(lif, addr);
1269	spin_unlock_bh(&lif->rx_filters.lock);
1270	if (f)
1271		return 0;
1272
1273	netdev_dbg(lif->netdev, "rx_filter add ADDR %pM\n", addr);
1274
1275	memcpy(ctx.cmd.rx_filter_add.mac.addr, addr, ETH_ALEN);
1276	err = ionic_adminq_post_wait(lif, &ctx);
1277	if (err && err != -EEXIST)
1278		return err;
1279
1280	return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1281}
1282
1283static int ionic_lif_addr_del(struct ionic_lif *lif, const u8 *addr)
1284{
1285	struct ionic_admin_ctx ctx = {
1286		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1287		.cmd.rx_filter_del = {
1288			.opcode = IONIC_CMD_RX_FILTER_DEL,
1289			.lif_index = cpu_to_le16(lif->index),
1290		},
1291	};
1292	struct ionic_rx_filter *f;
1293	int err;
1294
1295	spin_lock_bh(&lif->rx_filters.lock);
1296	f = ionic_rx_filter_by_addr(lif, addr);
1297	if (!f) {
1298		spin_unlock_bh(&lif->rx_filters.lock);
1299		return -ENOENT;
1300	}
1301
1302	netdev_dbg(lif->netdev, "rx_filter del ADDR %pM (id %d)\n",
1303		   addr, f->filter_id);
1304
1305	ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1306	ionic_rx_filter_free(lif, f);
1307	spin_unlock_bh(&lif->rx_filters.lock);
1308
1309	err = ionic_adminq_post_wait(lif, &ctx);
1310	if (err && err != -EEXIST)
1311		return err;
1312
1313	return 0;
1314}
1315
1316static int ionic_lif_addr(struct ionic_lif *lif, const u8 *addr, bool add)
1317{
1318	unsigned int nmfilters;
1319	unsigned int nufilters;
1320
1321	if (add) {
1322		/* Do we have space for this filter?  We test the counters
1323		 * here before checking the need for deferral so that we
1324		 * can return an overflow error to the stack.
1325		 */
1326		nmfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1327		nufilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1328
1329		if ((is_multicast_ether_addr(addr) && lif->nmcast < nmfilters))
1330			lif->nmcast++;
1331		else if (!is_multicast_ether_addr(addr) &&
1332			 lif->nucast < nufilters)
1333			lif->nucast++;
1334		else
1335			return -ENOSPC;
1336	} else {
1337		if (is_multicast_ether_addr(addr) && lif->nmcast)
1338			lif->nmcast--;
1339		else if (!is_multicast_ether_addr(addr) && lif->nucast)
1340			lif->nucast--;
1341	}
1342
1343	netdev_dbg(lif->netdev, "rx_filter %s %pM\n",
1344		   add ? "add" : "del", addr);
1345	if (add)
1346		return ionic_lif_addr_add(lif, addr);
1347	else
1348		return ionic_lif_addr_del(lif, addr);
1349
1350	return 0;
1351}
1352
1353static int ionic_addr_add(struct net_device *netdev, const u8 *addr)
1354{
1355	return ionic_lif_addr(netdev_priv(netdev), addr, ADD_ADDR);
1356}
1357
1358static int ionic_addr_del(struct net_device *netdev, const u8 *addr)
1359{
1360	/* Don't delete our own address from the uc list */
1361	if (ether_addr_equal(addr, netdev->dev_addr))
1362		return 0;
1363
1364	return ionic_lif_addr(netdev_priv(netdev), addr, DEL_ADDR);
1365}
1366
1367static void ionic_lif_rx_mode(struct ionic_lif *lif)
1368{
1369	struct net_device *netdev = lif->netdev;
1370	unsigned int nfilters;
1371	unsigned int nd_flags;
1372	char buf[128];
1373	u16 rx_mode;
1374	int i;
1375#define REMAIN(__x) (sizeof(buf) - (__x))
1376
1377	mutex_lock(&lif->config_lock);
1378
1379	/* grab the flags once for local use */
1380	nd_flags = netdev->flags;
1381
1382	rx_mode = IONIC_RX_MODE_F_UNICAST;
1383	rx_mode |= (nd_flags & IFF_MULTICAST) ? IONIC_RX_MODE_F_MULTICAST : 0;
1384	rx_mode |= (nd_flags & IFF_BROADCAST) ? IONIC_RX_MODE_F_BROADCAST : 0;
1385	rx_mode |= (nd_flags & IFF_PROMISC) ? IONIC_RX_MODE_F_PROMISC : 0;
1386	rx_mode |= (nd_flags & IFF_ALLMULTI) ? IONIC_RX_MODE_F_ALLMULTI : 0;
1387
1388	/* sync unicast addresses
1389	 * next check to see if we're in an overflow state
1390	 *    if so, we track that we overflowed and enable NIC PROMISC
1391	 *    else if the overflow is set and not needed
1392	 *       we remove our overflow flag and check the netdev flags
1393	 *       to see if we can disable NIC PROMISC
1394	 */
1395	__dev_uc_sync(netdev, ionic_addr_add, ionic_addr_del);
1396	nfilters = le32_to_cpu(lif->identity->eth.max_ucast_filters);
1397	if (netdev_uc_count(netdev) + 1 > nfilters) {
1398		rx_mode |= IONIC_RX_MODE_F_PROMISC;
1399		lif->uc_overflow = true;
1400	} else if (lif->uc_overflow) {
1401		lif->uc_overflow = false;
1402		if (!(nd_flags & IFF_PROMISC))
1403			rx_mode &= ~IONIC_RX_MODE_F_PROMISC;
1404	}
1405
1406	/* same for multicast */
1407	__dev_mc_sync(netdev, ionic_addr_add, ionic_addr_del);
1408	nfilters = le32_to_cpu(lif->identity->eth.max_mcast_filters);
1409	if (netdev_mc_count(netdev) > nfilters) {
1410		rx_mode |= IONIC_RX_MODE_F_ALLMULTI;
1411		lif->mc_overflow = true;
1412	} else if (lif->mc_overflow) {
1413		lif->mc_overflow = false;
1414		if (!(nd_flags & IFF_ALLMULTI))
1415			rx_mode &= ~IONIC_RX_MODE_F_ALLMULTI;
1416	}
1417
1418	i = scnprintf(buf, sizeof(buf), "rx_mode 0x%04x -> 0x%04x:",
1419		      lif->rx_mode, rx_mode);
1420	if (rx_mode & IONIC_RX_MODE_F_UNICAST)
1421		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_UNICAST");
1422	if (rx_mode & IONIC_RX_MODE_F_MULTICAST)
1423		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_MULTICAST");
1424	if (rx_mode & IONIC_RX_MODE_F_BROADCAST)
1425		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_BROADCAST");
1426	if (rx_mode & IONIC_RX_MODE_F_PROMISC)
1427		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_PROMISC");
1428	if (rx_mode & IONIC_RX_MODE_F_ALLMULTI)
1429		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_ALLMULTI");
1430	if (rx_mode & IONIC_RX_MODE_F_RDMA_SNIFFER)
1431		i += scnprintf(&buf[i], REMAIN(i), " RX_MODE_F_RDMA_SNIFFER");
1432	netdev_dbg(netdev, "lif%d %s\n", lif->index, buf);
1433
1434	if (lif->rx_mode != rx_mode) {
1435		struct ionic_admin_ctx ctx = {
1436			.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1437			.cmd.rx_mode_set = {
1438				.opcode = IONIC_CMD_RX_MODE_SET,
1439				.lif_index = cpu_to_le16(lif->index),
1440			},
1441		};
1442		int err;
1443
1444		ctx.cmd.rx_mode_set.rx_mode = cpu_to_le16(rx_mode);
1445		err = ionic_adminq_post_wait(lif, &ctx);
1446		if (err)
1447			netdev_warn(netdev, "set rx_mode 0x%04x failed: %d\n",
1448				    rx_mode, err);
1449		else
1450			lif->rx_mode = rx_mode;
1451	}
1452
1453	mutex_unlock(&lif->config_lock);
1454}
1455
1456static void ionic_set_rx_mode(struct net_device *netdev, bool can_sleep)
1457{
1458	struct ionic_lif *lif = netdev_priv(netdev);
1459	struct ionic_deferred_work *work;
1460
1461	if (!can_sleep) {
1462		work = kzalloc(sizeof(*work), GFP_ATOMIC);
1463		if (!work) {
1464			netdev_err(lif->netdev, "rxmode change dropped\n");
1465			return;
1466		}
1467		work->type = IONIC_DW_TYPE_RX_MODE;
1468		netdev_dbg(lif->netdev, "deferred: rx_mode\n");
1469		ionic_lif_deferred_enqueue(&lif->deferred, work);
1470	} else {
1471		ionic_lif_rx_mode(lif);
1472	}
1473}
1474
1475static void ionic_ndo_set_rx_mode(struct net_device *netdev)
1476{
1477	ionic_set_rx_mode(netdev, CAN_NOT_SLEEP);
1478}
1479
1480static __le64 ionic_netdev_features_to_nic(netdev_features_t features)
1481{
1482	u64 wanted = 0;
1483
1484	if (features & NETIF_F_HW_VLAN_CTAG_TX)
1485		wanted |= IONIC_ETH_HW_VLAN_TX_TAG;
1486	if (features & NETIF_F_HW_VLAN_CTAG_RX)
1487		wanted |= IONIC_ETH_HW_VLAN_RX_STRIP;
1488	if (features & NETIF_F_HW_VLAN_CTAG_FILTER)
1489		wanted |= IONIC_ETH_HW_VLAN_RX_FILTER;
1490	if (features & NETIF_F_RXHASH)
1491		wanted |= IONIC_ETH_HW_RX_HASH;
1492	if (features & NETIF_F_RXCSUM)
1493		wanted |= IONIC_ETH_HW_RX_CSUM;
1494	if (features & NETIF_F_SG)
1495		wanted |= IONIC_ETH_HW_TX_SG;
1496	if (features & NETIF_F_HW_CSUM)
1497		wanted |= IONIC_ETH_HW_TX_CSUM;
1498	if (features & NETIF_F_TSO)
1499		wanted |= IONIC_ETH_HW_TSO;
1500	if (features & NETIF_F_TSO6)
1501		wanted |= IONIC_ETH_HW_TSO_IPV6;
1502	if (features & NETIF_F_TSO_ECN)
1503		wanted |= IONIC_ETH_HW_TSO_ECN;
1504	if (features & NETIF_F_GSO_GRE)
1505		wanted |= IONIC_ETH_HW_TSO_GRE;
1506	if (features & NETIF_F_GSO_GRE_CSUM)
1507		wanted |= IONIC_ETH_HW_TSO_GRE_CSUM;
1508	if (features & NETIF_F_GSO_IPXIP4)
1509		wanted |= IONIC_ETH_HW_TSO_IPXIP4;
1510	if (features & NETIF_F_GSO_IPXIP6)
1511		wanted |= IONIC_ETH_HW_TSO_IPXIP6;
1512	if (features & NETIF_F_GSO_UDP_TUNNEL)
1513		wanted |= IONIC_ETH_HW_TSO_UDP;
1514	if (features & NETIF_F_GSO_UDP_TUNNEL_CSUM)
1515		wanted |= IONIC_ETH_HW_TSO_UDP_CSUM;
1516
1517	return cpu_to_le64(wanted);
1518}
1519
1520static int ionic_set_nic_features(struct ionic_lif *lif,
1521				  netdev_features_t features)
1522{
1523	struct device *dev = lif->ionic->dev;
1524	struct ionic_admin_ctx ctx = {
1525		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1526		.cmd.lif_setattr = {
1527			.opcode = IONIC_CMD_LIF_SETATTR,
1528			.index = cpu_to_le16(lif->index),
1529			.attr = IONIC_LIF_ATTR_FEATURES,
1530		},
1531	};
1532	u64 vlan_flags = IONIC_ETH_HW_VLAN_TX_TAG |
1533			 IONIC_ETH_HW_VLAN_RX_STRIP |
1534			 IONIC_ETH_HW_VLAN_RX_FILTER;
1535	u64 old_hw_features;
1536	int err;
1537
1538	ctx.cmd.lif_setattr.features = ionic_netdev_features_to_nic(features);
1539
1540	if (lif->phc)
1541		ctx.cmd.lif_setattr.features |= cpu_to_le64(IONIC_ETH_HW_TIMESTAMP);
1542
1543	err = ionic_adminq_post_wait(lif, &ctx);
1544	if (err)
1545		return err;
1546
1547	old_hw_features = lif->hw_features;
1548	lif->hw_features = le64_to_cpu(ctx.cmd.lif_setattr.features &
1549				       ctx.comp.lif_setattr.features);
1550
1551	if ((old_hw_features ^ lif->hw_features) & IONIC_ETH_HW_RX_HASH)
1552		ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1553
1554	if ((vlan_flags & features) &&
1555	    !(vlan_flags & le64_to_cpu(ctx.comp.lif_setattr.features)))
1556		dev_info_once(lif->ionic->dev, "NIC is not supporting vlan offload, likely in SmartNIC mode\n");
1557
1558	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1559		dev_dbg(dev, "feature ETH_HW_VLAN_TX_TAG\n");
1560	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1561		dev_dbg(dev, "feature ETH_HW_VLAN_RX_STRIP\n");
1562	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1563		dev_dbg(dev, "feature ETH_HW_VLAN_RX_FILTER\n");
1564	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1565		dev_dbg(dev, "feature ETH_HW_RX_HASH\n");
1566	if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1567		dev_dbg(dev, "feature ETH_HW_TX_SG\n");
1568	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1569		dev_dbg(dev, "feature ETH_HW_TX_CSUM\n");
1570	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1571		dev_dbg(dev, "feature ETH_HW_RX_CSUM\n");
1572	if (lif->hw_features & IONIC_ETH_HW_TSO)
1573		dev_dbg(dev, "feature ETH_HW_TSO\n");
1574	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1575		dev_dbg(dev, "feature ETH_HW_TSO_IPV6\n");
1576	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1577		dev_dbg(dev, "feature ETH_HW_TSO_ECN\n");
1578	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1579		dev_dbg(dev, "feature ETH_HW_TSO_GRE\n");
1580	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1581		dev_dbg(dev, "feature ETH_HW_TSO_GRE_CSUM\n");
1582	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1583		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP4\n");
1584	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1585		dev_dbg(dev, "feature ETH_HW_TSO_IPXIP6\n");
1586	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1587		dev_dbg(dev, "feature ETH_HW_TSO_UDP\n");
1588	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1589		dev_dbg(dev, "feature ETH_HW_TSO_UDP_CSUM\n");
1590	if (lif->hw_features & IONIC_ETH_HW_TIMESTAMP)
1591		dev_dbg(dev, "feature ETH_HW_TIMESTAMP\n");
1592
1593	return 0;
1594}
1595
1596static int ionic_init_nic_features(struct ionic_lif *lif)
1597{
1598	struct net_device *netdev = lif->netdev;
1599	netdev_features_t features;
1600	int err;
1601
1602	/* set up what we expect to support by default */
1603	features = NETIF_F_HW_VLAN_CTAG_TX |
1604		   NETIF_F_HW_VLAN_CTAG_RX |
1605		   NETIF_F_HW_VLAN_CTAG_FILTER |
1606		   NETIF_F_RXHASH |
1607		   NETIF_F_SG |
1608		   NETIF_F_HW_CSUM |
1609		   NETIF_F_RXCSUM |
1610		   NETIF_F_TSO |
1611		   NETIF_F_TSO6 |
1612		   NETIF_F_TSO_ECN;
1613
1614	err = ionic_set_nic_features(lif, features);
1615	if (err)
1616		return err;
1617
1618	/* tell the netdev what we actually can support */
1619	netdev->features |= NETIF_F_HIGHDMA;
1620
1621	if (lif->hw_features & IONIC_ETH_HW_VLAN_TX_TAG)
1622		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX;
1623	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_STRIP)
1624		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX;
1625	if (lif->hw_features & IONIC_ETH_HW_VLAN_RX_FILTER)
1626		netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER;
1627	if (lif->hw_features & IONIC_ETH_HW_RX_HASH)
1628		netdev->hw_features |= NETIF_F_RXHASH;
1629	if (lif->hw_features & IONIC_ETH_HW_TX_SG)
1630		netdev->hw_features |= NETIF_F_SG;
1631
1632	if (lif->hw_features & IONIC_ETH_HW_TX_CSUM)
1633		netdev->hw_enc_features |= NETIF_F_HW_CSUM;
1634	if (lif->hw_features & IONIC_ETH_HW_RX_CSUM)
1635		netdev->hw_enc_features |= NETIF_F_RXCSUM;
1636	if (lif->hw_features & IONIC_ETH_HW_TSO)
1637		netdev->hw_enc_features |= NETIF_F_TSO;
1638	if (lif->hw_features & IONIC_ETH_HW_TSO_IPV6)
1639		netdev->hw_enc_features |= NETIF_F_TSO6;
1640	if (lif->hw_features & IONIC_ETH_HW_TSO_ECN)
1641		netdev->hw_enc_features |= NETIF_F_TSO_ECN;
1642	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE)
1643		netdev->hw_enc_features |= NETIF_F_GSO_GRE;
1644	if (lif->hw_features & IONIC_ETH_HW_TSO_GRE_CSUM)
1645		netdev->hw_enc_features |= NETIF_F_GSO_GRE_CSUM;
1646	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP4)
1647		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4;
1648	if (lif->hw_features & IONIC_ETH_HW_TSO_IPXIP6)
1649		netdev->hw_enc_features |= NETIF_F_GSO_IPXIP6;
1650	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP)
1651		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL;
1652	if (lif->hw_features & IONIC_ETH_HW_TSO_UDP_CSUM)
1653		netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL_CSUM;
1654
1655	netdev->hw_features |= netdev->hw_enc_features;
1656	netdev->features |= netdev->hw_features;
1657	netdev->vlan_features |= netdev->features & ~NETIF_F_VLAN_FEATURES;
1658
1659	netdev->priv_flags |= IFF_UNICAST_FLT |
1660			      IFF_LIVE_ADDR_CHANGE;
1661
1662	return 0;
1663}
1664
1665static int ionic_set_features(struct net_device *netdev,
1666			      netdev_features_t features)
1667{
1668	struct ionic_lif *lif = netdev_priv(netdev);
1669	int err;
1670
1671	netdev_dbg(netdev, "%s: lif->features=0x%08llx new_features=0x%08llx\n",
1672		   __func__, (u64)lif->netdev->features, (u64)features);
1673
1674	err = ionic_set_nic_features(lif, features);
1675
1676	return err;
1677}
1678
1679static int ionic_set_mac_address(struct net_device *netdev, void *sa)
1680{
1681	struct sockaddr *addr = sa;
1682	u8 *mac;
1683	int err;
1684
1685	mac = (u8 *)addr->sa_data;
1686	if (ether_addr_equal(netdev->dev_addr, mac))
1687		return 0;
1688
1689	err = eth_prepare_mac_addr_change(netdev, addr);
1690	if (err)
1691		return err;
1692
1693	if (!is_zero_ether_addr(netdev->dev_addr)) {
1694		netdev_info(netdev, "deleting mac addr %pM\n",
1695			    netdev->dev_addr);
1696		ionic_addr_del(netdev, netdev->dev_addr);
1697	}
1698
1699	eth_commit_mac_addr_change(netdev, addr);
1700	netdev_info(netdev, "updating mac addr %pM\n", mac);
1701
1702	return ionic_addr_add(netdev, mac);
1703}
1704
1705static void ionic_stop_queues_reconfig(struct ionic_lif *lif)
1706{
1707	/* Stop and clean the queues before reconfiguration */
1708	mutex_lock(&lif->queue_lock);
1709	netif_device_detach(lif->netdev);
1710	ionic_stop_queues(lif);
1711	ionic_txrx_deinit(lif);
1712}
1713
1714static int ionic_start_queues_reconfig(struct ionic_lif *lif)
1715{
1716	int err;
1717
1718	/* Re-init the queues after reconfiguration */
1719
1720	/* The only way txrx_init can fail here is if communication
1721	 * with FW is suddenly broken.  There's not much we can do
1722	 * at this point - error messages have already been printed,
1723	 * so we can continue on and the user can eventually do a
1724	 * DOWN and UP to try to reset and clear the issue.
1725	 */
1726	err = ionic_txrx_init(lif);
1727	mutex_unlock(&lif->queue_lock);
1728	ionic_link_status_check_request(lif, CAN_SLEEP);
1729	netif_device_attach(lif->netdev);
1730
1731	return err;
1732}
1733
1734static int ionic_change_mtu(struct net_device *netdev, int new_mtu)
1735{
1736	struct ionic_lif *lif = netdev_priv(netdev);
1737	struct ionic_admin_ctx ctx = {
1738		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1739		.cmd.lif_setattr = {
1740			.opcode = IONIC_CMD_LIF_SETATTR,
1741			.index = cpu_to_le16(lif->index),
1742			.attr = IONIC_LIF_ATTR_MTU,
1743			.mtu = cpu_to_le32(new_mtu),
1744		},
1745	};
1746	int err;
1747
1748	err = ionic_adminq_post_wait(lif, &ctx);
1749	if (err)
1750		return err;
1751
1752	/* if we're not running, nothing more to do */
1753	if (!netif_running(netdev)) {
1754		netdev->mtu = new_mtu;
1755		return 0;
1756	}
1757
1758	ionic_stop_queues_reconfig(lif);
1759	netdev->mtu = new_mtu;
1760	return ionic_start_queues_reconfig(lif);
1761}
1762
1763static void ionic_tx_timeout_work(struct work_struct *ws)
1764{
1765	struct ionic_lif *lif = container_of(ws, struct ionic_lif, tx_timeout_work);
1766
1767	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
1768		return;
1769
1770	/* if we were stopped before this scheduled job was launched,
1771	 * don't bother the queues as they are already stopped.
1772	 */
1773	if (!netif_running(lif->netdev))
1774		return;
1775
1776	ionic_stop_queues_reconfig(lif);
1777	ionic_start_queues_reconfig(lif);
1778}
1779
1780static void ionic_tx_timeout(struct net_device *netdev, unsigned int txqueue)
1781{
1782	struct ionic_lif *lif = netdev_priv(netdev);
1783
1784	netdev_info(lif->netdev, "Tx Timeout triggered - txq %d\n", txqueue);
1785	schedule_work(&lif->tx_timeout_work);
1786}
1787
1788static int ionic_vlan_rx_add_vid(struct net_device *netdev, __be16 proto,
1789				 u16 vid)
1790{
1791	struct ionic_lif *lif = netdev_priv(netdev);
1792	struct ionic_admin_ctx ctx = {
1793		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1794		.cmd.rx_filter_add = {
1795			.opcode = IONIC_CMD_RX_FILTER_ADD,
1796			.lif_index = cpu_to_le16(lif->index),
1797			.match = cpu_to_le16(IONIC_RX_FILTER_MATCH_VLAN),
1798			.vlan.vlan = cpu_to_le16(vid),
1799		},
1800	};
1801	int err;
1802
1803	netdev_dbg(netdev, "rx_filter add VLAN %d\n", vid);
1804	err = ionic_adminq_post_wait(lif, &ctx);
1805	if (err)
1806		return err;
1807
1808	return ionic_rx_filter_save(lif, 0, IONIC_RXQ_INDEX_ANY, 0, &ctx);
1809}
1810
1811static int ionic_vlan_rx_kill_vid(struct net_device *netdev, __be16 proto,
1812				  u16 vid)
1813{
1814	struct ionic_lif *lif = netdev_priv(netdev);
1815	struct ionic_admin_ctx ctx = {
1816		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1817		.cmd.rx_filter_del = {
1818			.opcode = IONIC_CMD_RX_FILTER_DEL,
1819			.lif_index = cpu_to_le16(lif->index),
1820		},
1821	};
1822	struct ionic_rx_filter *f;
1823
1824	spin_lock_bh(&lif->rx_filters.lock);
1825
1826	f = ionic_rx_filter_by_vlan(lif, vid);
1827	if (!f) {
1828		spin_unlock_bh(&lif->rx_filters.lock);
1829		return -ENOENT;
1830	}
1831
1832	netdev_dbg(netdev, "rx_filter del VLAN %d (id %d)\n",
1833		   vid, f->filter_id);
1834
1835	ctx.cmd.rx_filter_del.filter_id = cpu_to_le32(f->filter_id);
1836	ionic_rx_filter_free(lif, f);
1837	spin_unlock_bh(&lif->rx_filters.lock);
1838
1839	return ionic_adminq_post_wait(lif, &ctx);
1840}
1841
1842int ionic_lif_rss_config(struct ionic_lif *lif, const u16 types,
1843			 const u8 *key, const u32 *indir)
1844{
1845	struct ionic_admin_ctx ctx = {
1846		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1847		.cmd.lif_setattr = {
1848			.opcode = IONIC_CMD_LIF_SETATTR,
1849			.attr = IONIC_LIF_ATTR_RSS,
1850			.rss.addr = cpu_to_le64(lif->rss_ind_tbl_pa),
1851		},
1852	};
1853	unsigned int i, tbl_sz;
1854
1855	if (lif->hw_features & IONIC_ETH_HW_RX_HASH) {
1856		lif->rss_types = types;
1857		ctx.cmd.lif_setattr.rss.types = cpu_to_le16(types);
1858	}
1859
1860	if (key)
1861		memcpy(lif->rss_hash_key, key, IONIC_RSS_HASH_KEY_SIZE);
1862
1863	if (indir) {
1864		tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1865		for (i = 0; i < tbl_sz; i++)
1866			lif->rss_ind_tbl[i] = indir[i];
1867	}
1868
1869	memcpy(ctx.cmd.lif_setattr.rss.key, lif->rss_hash_key,
1870	       IONIC_RSS_HASH_KEY_SIZE);
1871
1872	return ionic_adminq_post_wait(lif, &ctx);
1873}
1874
1875static int ionic_lif_rss_init(struct ionic_lif *lif)
1876{
1877	unsigned int tbl_sz;
1878	unsigned int i;
1879
1880	lif->rss_types = IONIC_RSS_TYPE_IPV4     |
1881			 IONIC_RSS_TYPE_IPV4_TCP |
1882			 IONIC_RSS_TYPE_IPV4_UDP |
1883			 IONIC_RSS_TYPE_IPV6     |
1884			 IONIC_RSS_TYPE_IPV6_TCP |
1885			 IONIC_RSS_TYPE_IPV6_UDP;
1886
1887	/* Fill indirection table with 'default' values */
1888	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1889	for (i = 0; i < tbl_sz; i++)
1890		lif->rss_ind_tbl[i] = ethtool_rxfh_indir_default(i, lif->nxqs);
1891
1892	return ionic_lif_rss_config(lif, lif->rss_types, NULL, NULL);
1893}
1894
1895static void ionic_lif_rss_deinit(struct ionic_lif *lif)
1896{
1897	int tbl_sz;
1898
1899	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
1900	memset(lif->rss_ind_tbl, 0, tbl_sz);
1901	memset(lif->rss_hash_key, 0, IONIC_RSS_HASH_KEY_SIZE);
1902
1903	ionic_lif_rss_config(lif, 0x0, NULL, NULL);
1904}
1905
1906static void ionic_lif_quiesce(struct ionic_lif *lif)
1907{
1908	struct ionic_admin_ctx ctx = {
1909		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
1910		.cmd.lif_setattr = {
1911			.opcode = IONIC_CMD_LIF_SETATTR,
1912			.index = cpu_to_le16(lif->index),
1913			.attr = IONIC_LIF_ATTR_STATE,
1914			.state = IONIC_LIF_QUIESCE,
1915		},
1916	};
1917	int err;
1918
1919	err = ionic_adminq_post_wait(lif, &ctx);
1920	if (err)
1921		netdev_err(lif->netdev, "lif quiesce failed %d\n", err);
1922}
1923
1924static void ionic_txrx_disable(struct ionic_lif *lif)
1925{
1926	unsigned int i;
1927	int err = 0;
1928
1929	if (lif->txqcqs) {
1930		for (i = 0; i < lif->nxqs; i++)
1931			err = ionic_qcq_disable(lif->txqcqs[i], (err != -ETIMEDOUT));
1932	}
1933
1934	if (lif->hwstamp_txq)
1935		err = ionic_qcq_disable(lif->hwstamp_txq, (err != -ETIMEDOUT));
1936
1937	if (lif->rxqcqs) {
1938		for (i = 0; i < lif->nxqs; i++)
1939			err = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
1940	}
1941
1942	if (lif->hwstamp_rxq)
1943		err = ionic_qcq_disable(lif->hwstamp_rxq, (err != -ETIMEDOUT));
1944
1945	ionic_lif_quiesce(lif);
1946}
1947
1948static void ionic_txrx_deinit(struct ionic_lif *lif)
1949{
1950	unsigned int i;
1951
1952	if (lif->txqcqs) {
1953		for (i = 0; i < lif->nxqs && lif->txqcqs[i]; i++) {
1954			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
1955			ionic_tx_flush(&lif->txqcqs[i]->cq);
1956			ionic_tx_empty(&lif->txqcqs[i]->q);
1957		}
1958	}
1959
1960	if (lif->rxqcqs) {
1961		for (i = 0; i < lif->nxqs && lif->rxqcqs[i]; i++) {
1962			ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
1963			ionic_rx_empty(&lif->rxqcqs[i]->q);
1964		}
1965	}
1966	lif->rx_mode = 0;
1967
1968	if (lif->hwstamp_txq) {
1969		ionic_lif_qcq_deinit(lif, lif->hwstamp_txq);
1970		ionic_tx_flush(&lif->hwstamp_txq->cq);
1971		ionic_tx_empty(&lif->hwstamp_txq->q);
1972	}
1973
1974	if (lif->hwstamp_rxq) {
1975		ionic_lif_qcq_deinit(lif, lif->hwstamp_rxq);
1976		ionic_rx_empty(&lif->hwstamp_rxq->q);
1977	}
1978}
1979
1980static void ionic_txrx_free(struct ionic_lif *lif)
1981{
1982	unsigned int i;
1983
1984	if (lif->txqcqs) {
1985		for (i = 0; i < lif->ionic->ntxqs_per_lif && lif->txqcqs[i]; i++) {
1986			ionic_qcq_free(lif, lif->txqcqs[i]);
1987			devm_kfree(lif->ionic->dev, lif->txqcqs[i]);
1988			lif->txqcqs[i] = NULL;
1989		}
1990	}
1991
1992	if (lif->rxqcqs) {
1993		for (i = 0; i < lif->ionic->nrxqs_per_lif && lif->rxqcqs[i]; i++) {
1994			ionic_qcq_free(lif, lif->rxqcqs[i]);
1995			devm_kfree(lif->ionic->dev, lif->rxqcqs[i]);
1996			lif->rxqcqs[i] = NULL;
1997		}
1998	}
1999
2000	if (lif->hwstamp_txq) {
2001		ionic_qcq_free(lif, lif->hwstamp_txq);
2002		devm_kfree(lif->ionic->dev, lif->hwstamp_txq);
2003		lif->hwstamp_txq = NULL;
2004	}
2005
2006	if (lif->hwstamp_rxq) {
2007		ionic_qcq_free(lif, lif->hwstamp_rxq);
2008		devm_kfree(lif->ionic->dev, lif->hwstamp_rxq);
2009		lif->hwstamp_rxq = NULL;
2010	}
2011}
2012
2013static int ionic_txrx_alloc(struct ionic_lif *lif)
2014{
2015	unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2016	unsigned int flags, i;
2017	int err = 0;
2018
2019	num_desc = lif->ntxq_descs;
2020	desc_sz = sizeof(struct ionic_txq_desc);
2021	comp_sz = sizeof(struct ionic_txq_comp);
2022
2023	if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2024	    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2025					  sizeof(struct ionic_txq_sg_desc_v1))
2026		sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2027	else
2028		sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2029
2030	flags = IONIC_QCQ_F_TX_STATS | IONIC_QCQ_F_SG;
2031	if (test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2032		flags |= IONIC_QCQ_F_INTR;
2033	for (i = 0; i < lif->nxqs; i++) {
2034		err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2035				      num_desc, desc_sz, comp_sz, sg_desc_sz,
2036				      lif->kern_pid, &lif->txqcqs[i]);
2037		if (err)
2038			goto err_out;
2039
2040		if (flags & IONIC_QCQ_F_INTR) {
2041			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2042					     lif->txqcqs[i]->intr.index,
2043					     lif->tx_coalesce_hw);
2044			if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2045				lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2046		}
2047
2048		ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2049	}
2050
2051	flags = IONIC_QCQ_F_RX_STATS | IONIC_QCQ_F_SG | IONIC_QCQ_F_INTR;
2052
2053	num_desc = lif->nrxq_descs;
2054	desc_sz = sizeof(struct ionic_rxq_desc);
2055	comp_sz = sizeof(struct ionic_rxq_comp);
2056	sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2057
2058	if (lif->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2059		comp_sz *= 2;
2060
2061	for (i = 0; i < lif->nxqs; i++) {
2062		err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2063				      num_desc, desc_sz, comp_sz, sg_desc_sz,
2064				      lif->kern_pid, &lif->rxqcqs[i]);
2065		if (err)
2066			goto err_out;
2067
2068		lif->rxqcqs[i]->q.features = lif->rxq_features;
2069
2070		ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2071				     lif->rxqcqs[i]->intr.index,
2072				     lif->rx_coalesce_hw);
2073		if (test_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state))
2074			lif->rxqcqs[i]->intr.dim_coal_hw = lif->rx_coalesce_hw;
2075
2076		if (!test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state))
2077			ionic_link_qcq_interrupts(lif->rxqcqs[i],
2078						  lif->txqcqs[i]);
2079
2080		ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2081	}
2082
2083	return 0;
2084
2085err_out:
2086	ionic_txrx_free(lif);
2087
2088	return err;
2089}
2090
2091static int ionic_txrx_init(struct ionic_lif *lif)
2092{
2093	unsigned int i;
2094	int err;
2095
2096	for (i = 0; i < lif->nxqs; i++) {
2097		err = ionic_lif_txq_init(lif, lif->txqcqs[i]);
2098		if (err)
2099			goto err_out;
2100
2101		err = ionic_lif_rxq_init(lif, lif->rxqcqs[i]);
2102		if (err) {
2103			ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2104			goto err_out;
2105		}
2106	}
2107
2108	if (lif->netdev->features & NETIF_F_RXHASH)
2109		ionic_lif_rss_init(lif);
2110
2111	ionic_set_rx_mode(lif->netdev, CAN_SLEEP);
2112
2113	return 0;
2114
2115err_out:
2116	while (i--) {
2117		ionic_lif_qcq_deinit(lif, lif->txqcqs[i]);
2118		ionic_lif_qcq_deinit(lif, lif->rxqcqs[i]);
2119	}
2120
2121	return err;
2122}
2123
2124static int ionic_txrx_enable(struct ionic_lif *lif)
2125{
2126	int derr = 0;
2127	int i, err;
2128
2129	for (i = 0; i < lif->nxqs; i++) {
2130		if (!(lif->rxqcqs[i] && lif->txqcqs[i])) {
2131			dev_err(lif->ionic->dev, "%s: bad qcq %d\n", __func__, i);
2132			err = -ENXIO;
2133			goto err_out;
2134		}
2135
2136		ionic_rx_fill(&lif->rxqcqs[i]->q);
2137		err = ionic_qcq_enable(lif->rxqcqs[i]);
2138		if (err)
2139			goto err_out;
2140
2141		err = ionic_qcq_enable(lif->txqcqs[i]);
2142		if (err) {
2143			derr = ionic_qcq_disable(lif->rxqcqs[i], (err != -ETIMEDOUT));
2144			goto err_out;
2145		}
2146	}
2147
2148	if (lif->hwstamp_rxq) {
2149		ionic_rx_fill(&lif->hwstamp_rxq->q);
2150		err = ionic_qcq_enable(lif->hwstamp_rxq);
2151		if (err)
2152			goto err_out_hwstamp_rx;
2153	}
2154
2155	if (lif->hwstamp_txq) {
2156		err = ionic_qcq_enable(lif->hwstamp_txq);
2157		if (err)
2158			goto err_out_hwstamp_tx;
2159	}
2160
2161	return 0;
2162
2163err_out_hwstamp_tx:
2164	if (lif->hwstamp_rxq)
2165		derr = ionic_qcq_disable(lif->hwstamp_rxq, (derr != -ETIMEDOUT));
2166err_out_hwstamp_rx:
2167	i = lif->nxqs;
2168err_out:
2169	while (i--) {
2170		derr = ionic_qcq_disable(lif->txqcqs[i], (derr != -ETIMEDOUT));
2171		derr = ionic_qcq_disable(lif->rxqcqs[i], (derr != -ETIMEDOUT));
2172	}
2173
2174	return err;
2175}
2176
2177static int ionic_start_queues(struct ionic_lif *lif)
2178{
2179	int err;
2180
2181	if (test_bit(IONIC_LIF_F_BROKEN, lif->state))
2182		return -EIO;
2183
2184	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2185		return -EBUSY;
2186
2187	if (test_and_set_bit(IONIC_LIF_F_UP, lif->state))
2188		return 0;
2189
2190	err = ionic_txrx_enable(lif);
2191	if (err) {
2192		clear_bit(IONIC_LIF_F_UP, lif->state);
2193		return err;
2194	}
2195	netif_tx_wake_all_queues(lif->netdev);
2196
2197	return 0;
2198}
2199
2200static int ionic_open(struct net_device *netdev)
2201{
2202	struct ionic_lif *lif = netdev_priv(netdev);
2203	int err;
2204
2205	/* If recovering from a broken state, clear the bit and we'll try again */
2206	if (test_and_clear_bit(IONIC_LIF_F_BROKEN, lif->state))
2207		netdev_info(netdev, "clearing broken state\n");
2208
2209	err = ionic_txrx_alloc(lif);
2210	if (err)
2211		return err;
2212
2213	err = ionic_txrx_init(lif);
2214	if (err)
2215		goto err_txrx_free;
2216
2217	err = netif_set_real_num_tx_queues(netdev, lif->nxqs);
2218	if (err)
2219		goto err_txrx_deinit;
2220
2221	err = netif_set_real_num_rx_queues(netdev, lif->nxqs);
2222	if (err)
2223		goto err_txrx_deinit;
2224
2225	/* don't start the queues until we have link */
2226	if (netif_carrier_ok(netdev)) {
2227		err = ionic_start_queues(lif);
2228		if (err)
2229			goto err_txrx_deinit;
2230	}
2231
2232	return 0;
2233
2234err_txrx_deinit:
2235	ionic_txrx_deinit(lif);
2236err_txrx_free:
2237	ionic_txrx_free(lif);
2238	return err;
2239}
2240
2241static void ionic_stop_queues(struct ionic_lif *lif)
2242{
2243	if (!test_and_clear_bit(IONIC_LIF_F_UP, lif->state))
2244		return;
2245
2246	netif_tx_disable(lif->netdev);
2247	ionic_txrx_disable(lif);
2248}
2249
2250static int ionic_stop(struct net_device *netdev)
2251{
2252	struct ionic_lif *lif = netdev_priv(netdev);
2253
2254	if (test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2255		return 0;
2256
2257	ionic_stop_queues(lif);
2258	ionic_txrx_deinit(lif);
2259	ionic_txrx_free(lif);
2260
2261	return 0;
2262}
2263
2264static int ionic_do_ioctl(struct net_device *netdev, struct ifreq *ifr, int cmd)
2265{
2266	struct ionic_lif *lif = netdev_priv(netdev);
2267
2268	switch (cmd) {
2269	case SIOCSHWTSTAMP:
2270		return ionic_lif_hwstamp_set(lif, ifr);
2271	case SIOCGHWTSTAMP:
2272		return ionic_lif_hwstamp_get(lif, ifr);
2273	default:
2274		return -EOPNOTSUPP;
2275	}
2276}
2277
2278static int ionic_get_vf_config(struct net_device *netdev,
2279			       int vf, struct ifla_vf_info *ivf)
2280{
2281	struct ionic_lif *lif = netdev_priv(netdev);
2282	struct ionic *ionic = lif->ionic;
2283	int ret = 0;
2284
2285	if (!netif_device_present(netdev))
2286		return -EBUSY;
2287
2288	down_read(&ionic->vf_op_lock);
2289
2290	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2291		ret = -EINVAL;
2292	} else {
2293		ivf->vf           = vf;
2294		ivf->vlan         = le16_to_cpu(ionic->vfs[vf].vlanid);
2295		ivf->qos	  = 0;
2296		ivf->spoofchk     = ionic->vfs[vf].spoofchk;
2297		ivf->linkstate    = ionic->vfs[vf].linkstate;
2298		ivf->max_tx_rate  = le32_to_cpu(ionic->vfs[vf].maxrate);
2299		ivf->trusted      = ionic->vfs[vf].trusted;
2300		ether_addr_copy(ivf->mac, ionic->vfs[vf].macaddr);
2301	}
2302
2303	up_read(&ionic->vf_op_lock);
2304	return ret;
2305}
2306
2307static int ionic_get_vf_stats(struct net_device *netdev, int vf,
2308			      struct ifla_vf_stats *vf_stats)
2309{
2310	struct ionic_lif *lif = netdev_priv(netdev);
2311	struct ionic *ionic = lif->ionic;
2312	struct ionic_lif_stats *vs;
2313	int ret = 0;
2314
2315	if (!netif_device_present(netdev))
2316		return -EBUSY;
2317
2318	down_read(&ionic->vf_op_lock);
2319
2320	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2321		ret = -EINVAL;
2322	} else {
2323		memset(vf_stats, 0, sizeof(*vf_stats));
2324		vs = &ionic->vfs[vf].stats;
2325
2326		vf_stats->rx_packets = le64_to_cpu(vs->rx_ucast_packets);
2327		vf_stats->tx_packets = le64_to_cpu(vs->tx_ucast_packets);
2328		vf_stats->rx_bytes   = le64_to_cpu(vs->rx_ucast_bytes);
2329		vf_stats->tx_bytes   = le64_to_cpu(vs->tx_ucast_bytes);
2330		vf_stats->broadcast  = le64_to_cpu(vs->rx_bcast_packets);
2331		vf_stats->multicast  = le64_to_cpu(vs->rx_mcast_packets);
2332		vf_stats->rx_dropped = le64_to_cpu(vs->rx_ucast_drop_packets) +
2333				       le64_to_cpu(vs->rx_mcast_drop_packets) +
2334				       le64_to_cpu(vs->rx_bcast_drop_packets);
2335		vf_stats->tx_dropped = le64_to_cpu(vs->tx_ucast_drop_packets) +
2336				       le64_to_cpu(vs->tx_mcast_drop_packets) +
2337				       le64_to_cpu(vs->tx_bcast_drop_packets);
2338	}
2339
2340	up_read(&ionic->vf_op_lock);
2341	return ret;
2342}
2343
2344static int ionic_set_vf_mac(struct net_device *netdev, int vf, u8 *mac)
2345{
2346	struct ionic_lif *lif = netdev_priv(netdev);
2347	struct ionic *ionic = lif->ionic;
2348	int ret;
2349
2350	if (!(is_zero_ether_addr(mac) || is_valid_ether_addr(mac)))
2351		return -EINVAL;
2352
2353	if (!netif_device_present(netdev))
2354		return -EBUSY;
2355
2356	down_write(&ionic->vf_op_lock);
2357
2358	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2359		ret = -EINVAL;
2360	} else {
2361		ret = ionic_set_vf_config(ionic, vf, IONIC_VF_ATTR_MAC, mac);
2362		if (!ret)
2363			ether_addr_copy(ionic->vfs[vf].macaddr, mac);
2364	}
2365
2366	up_write(&ionic->vf_op_lock);
2367	return ret;
2368}
2369
2370static int ionic_set_vf_vlan(struct net_device *netdev, int vf, u16 vlan,
2371			     u8 qos, __be16 proto)
2372{
2373	struct ionic_lif *lif = netdev_priv(netdev);
2374	struct ionic *ionic = lif->ionic;
2375	int ret;
2376
2377	/* until someday when we support qos */
2378	if (qos)
2379		return -EINVAL;
2380
2381	if (vlan > 4095)
2382		return -EINVAL;
2383
2384	if (proto != htons(ETH_P_8021Q))
2385		return -EPROTONOSUPPORT;
2386
2387	if (!netif_device_present(netdev))
2388		return -EBUSY;
2389
2390	down_write(&ionic->vf_op_lock);
2391
2392	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2393		ret = -EINVAL;
2394	} else {
2395		ret = ionic_set_vf_config(ionic, vf,
2396					  IONIC_VF_ATTR_VLAN, (u8 *)&vlan);
2397		if (!ret)
2398			ionic->vfs[vf].vlanid = cpu_to_le16(vlan);
2399	}
2400
2401	up_write(&ionic->vf_op_lock);
2402	return ret;
2403}
2404
2405static int ionic_set_vf_rate(struct net_device *netdev, int vf,
2406			     int tx_min, int tx_max)
2407{
2408	struct ionic_lif *lif = netdev_priv(netdev);
2409	struct ionic *ionic = lif->ionic;
2410	int ret;
2411
2412	/* setting the min just seems silly */
2413	if (tx_min)
2414		return -EINVAL;
2415
2416	if (!netif_device_present(netdev))
2417		return -EBUSY;
2418
2419	down_write(&ionic->vf_op_lock);
2420
2421	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2422		ret = -EINVAL;
2423	} else {
2424		ret = ionic_set_vf_config(ionic, vf,
2425					  IONIC_VF_ATTR_RATE, (u8 *)&tx_max);
2426		if (!ret)
2427			lif->ionic->vfs[vf].maxrate = cpu_to_le32(tx_max);
2428	}
2429
2430	up_write(&ionic->vf_op_lock);
2431	return ret;
2432}
2433
2434static int ionic_set_vf_spoofchk(struct net_device *netdev, int vf, bool set)
2435{
2436	struct ionic_lif *lif = netdev_priv(netdev);
2437	struct ionic *ionic = lif->ionic;
2438	u8 data = set;  /* convert to u8 for config */
2439	int ret;
2440
2441	if (!netif_device_present(netdev))
2442		return -EBUSY;
2443
2444	down_write(&ionic->vf_op_lock);
2445
2446	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2447		ret = -EINVAL;
2448	} else {
2449		ret = ionic_set_vf_config(ionic, vf,
2450					  IONIC_VF_ATTR_SPOOFCHK, &data);
2451		if (!ret)
2452			ionic->vfs[vf].spoofchk = data;
2453	}
2454
2455	up_write(&ionic->vf_op_lock);
2456	return ret;
2457}
2458
2459static int ionic_set_vf_trust(struct net_device *netdev, int vf, bool set)
2460{
2461	struct ionic_lif *lif = netdev_priv(netdev);
2462	struct ionic *ionic = lif->ionic;
2463	u8 data = set;  /* convert to u8 for config */
2464	int ret;
2465
2466	if (!netif_device_present(netdev))
2467		return -EBUSY;
2468
2469	down_write(&ionic->vf_op_lock);
2470
2471	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2472		ret = -EINVAL;
2473	} else {
2474		ret = ionic_set_vf_config(ionic, vf,
2475					  IONIC_VF_ATTR_TRUST, &data);
2476		if (!ret)
2477			ionic->vfs[vf].trusted = data;
2478	}
2479
2480	up_write(&ionic->vf_op_lock);
2481	return ret;
2482}
2483
2484static int ionic_set_vf_link_state(struct net_device *netdev, int vf, int set)
2485{
2486	struct ionic_lif *lif = netdev_priv(netdev);
2487	struct ionic *ionic = lif->ionic;
2488	u8 data;
2489	int ret;
2490
2491	switch (set) {
2492	case IFLA_VF_LINK_STATE_ENABLE:
2493		data = IONIC_VF_LINK_STATUS_UP;
2494		break;
2495	case IFLA_VF_LINK_STATE_DISABLE:
2496		data = IONIC_VF_LINK_STATUS_DOWN;
2497		break;
2498	case IFLA_VF_LINK_STATE_AUTO:
2499		data = IONIC_VF_LINK_STATUS_AUTO;
2500		break;
2501	default:
2502		return -EINVAL;
2503	}
2504
2505	if (!netif_device_present(netdev))
2506		return -EBUSY;
2507
2508	down_write(&ionic->vf_op_lock);
2509
2510	if (vf >= pci_num_vf(ionic->pdev) || !ionic->vfs) {
2511		ret = -EINVAL;
2512	} else {
2513		ret = ionic_set_vf_config(ionic, vf,
2514					  IONIC_VF_ATTR_LINKSTATE, &data);
2515		if (!ret)
2516			ionic->vfs[vf].linkstate = set;
2517	}
2518
2519	up_write(&ionic->vf_op_lock);
2520	return ret;
2521}
2522
2523static const struct net_device_ops ionic_netdev_ops = {
2524	.ndo_open               = ionic_open,
2525	.ndo_stop               = ionic_stop,
2526	.ndo_do_ioctl		= ionic_do_ioctl,
2527	.ndo_start_xmit		= ionic_start_xmit,
2528	.ndo_get_stats64	= ionic_get_stats64,
2529	.ndo_set_rx_mode	= ionic_ndo_set_rx_mode,
2530	.ndo_set_features	= ionic_set_features,
2531	.ndo_set_mac_address	= ionic_set_mac_address,
2532	.ndo_validate_addr	= eth_validate_addr,
2533	.ndo_tx_timeout         = ionic_tx_timeout,
2534	.ndo_change_mtu         = ionic_change_mtu,
2535	.ndo_vlan_rx_add_vid    = ionic_vlan_rx_add_vid,
2536	.ndo_vlan_rx_kill_vid   = ionic_vlan_rx_kill_vid,
2537	.ndo_set_vf_vlan	= ionic_set_vf_vlan,
2538	.ndo_set_vf_trust	= ionic_set_vf_trust,
2539	.ndo_set_vf_mac		= ionic_set_vf_mac,
2540	.ndo_set_vf_rate	= ionic_set_vf_rate,
2541	.ndo_set_vf_spoofchk	= ionic_set_vf_spoofchk,
2542	.ndo_get_vf_config	= ionic_get_vf_config,
2543	.ndo_set_vf_link_state	= ionic_set_vf_link_state,
2544	.ndo_get_vf_stats       = ionic_get_vf_stats,
2545};
2546
2547static void ionic_swap_queues(struct ionic_qcq *a, struct ionic_qcq *b)
2548{
2549	/* only swapping the queues, not the napi, flags, or other stuff */
2550	swap(a->q.features,   b->q.features);
2551	swap(a->q.num_descs,  b->q.num_descs);
2552	swap(a->q.desc_size,  b->q.desc_size);
2553	swap(a->q.base,       b->q.base);
2554	swap(a->q.base_pa,    b->q.base_pa);
2555	swap(a->q.info,       b->q.info);
2556	swap(a->q_base,       b->q_base);
2557	swap(a->q_base_pa,    b->q_base_pa);
2558	swap(a->q_size,       b->q_size);
2559
2560	swap(a->q.sg_desc_size, b->q.sg_desc_size);
2561	swap(a->q.sg_base,    b->q.sg_base);
2562	swap(a->q.sg_base_pa, b->q.sg_base_pa);
2563	swap(a->sg_base,      b->sg_base);
2564	swap(a->sg_base_pa,   b->sg_base_pa);
2565	swap(a->sg_size,      b->sg_size);
2566
2567	swap(a->cq.num_descs, b->cq.num_descs);
2568	swap(a->cq.desc_size, b->cq.desc_size);
2569	swap(a->cq.base,      b->cq.base);
2570	swap(a->cq.base_pa,   b->cq.base_pa);
2571	swap(a->cq.info,      b->cq.info);
2572	swap(a->cq_base,      b->cq_base);
2573	swap(a->cq_base_pa,   b->cq_base_pa);
2574	swap(a->cq_size,      b->cq_size);
2575
2576	ionic_debugfs_del_qcq(a);
2577	ionic_debugfs_add_qcq(a->q.lif, a);
2578}
2579
2580int ionic_reconfigure_queues(struct ionic_lif *lif,
2581			     struct ionic_queue_params *qparam)
2582{
2583	unsigned int comp_sz, desc_sz, num_desc, sg_desc_sz;
2584	struct ionic_qcq **tx_qcqs = NULL;
2585	struct ionic_qcq **rx_qcqs = NULL;
2586	unsigned int flags, i;
2587	int err = -ENOMEM;
2588
2589	/* allocate temporary qcq arrays to hold new queue structs */
2590	if (qparam->nxqs != lif->nxqs || qparam->ntxq_descs != lif->ntxq_descs) {
2591		tx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->ntxqs_per_lif,
2592				       sizeof(struct ionic_qcq *), GFP_KERNEL);
2593		if (!tx_qcqs)
2594			goto err_out;
2595	}
2596	if (qparam->nxqs != lif->nxqs ||
2597	    qparam->nrxq_descs != lif->nrxq_descs ||
2598	    qparam->rxq_features != lif->rxq_features) {
2599		rx_qcqs = devm_kcalloc(lif->ionic->dev, lif->ionic->nrxqs_per_lif,
2600				       sizeof(struct ionic_qcq *), GFP_KERNEL);
2601		if (!rx_qcqs)
2602			goto err_out;
2603	}
2604
2605	/* allocate new desc_info and rings, but leave the interrupt setup
2606	 * until later so as to not mess with the still-running queues
2607	 */
2608	if (tx_qcqs) {
2609		num_desc = qparam->ntxq_descs;
2610		desc_sz = sizeof(struct ionic_txq_desc);
2611		comp_sz = sizeof(struct ionic_txq_comp);
2612
2613		if (lif->qtype_info[IONIC_QTYPE_TXQ].version >= 1 &&
2614		    lif->qtype_info[IONIC_QTYPE_TXQ].sg_desc_sz ==
2615		    sizeof(struct ionic_txq_sg_desc_v1))
2616			sg_desc_sz = sizeof(struct ionic_txq_sg_desc_v1);
2617		else
2618			sg_desc_sz = sizeof(struct ionic_txq_sg_desc);
2619
2620		for (i = 0; i < qparam->nxqs; i++) {
2621			flags = lif->txqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2622			err = ionic_qcq_alloc(lif, IONIC_QTYPE_TXQ, i, "tx", flags,
2623					      num_desc, desc_sz, comp_sz, sg_desc_sz,
2624					      lif->kern_pid, &tx_qcqs[i]);
2625			if (err)
2626				goto err_out;
2627		}
2628	}
2629
2630	if (rx_qcqs) {
2631		num_desc = qparam->nrxq_descs;
2632		desc_sz = sizeof(struct ionic_rxq_desc);
2633		comp_sz = sizeof(struct ionic_rxq_comp);
2634		sg_desc_sz = sizeof(struct ionic_rxq_sg_desc);
2635
2636		if (qparam->rxq_features & IONIC_Q_F_2X_CQ_DESC)
2637			comp_sz *= 2;
2638
2639		for (i = 0; i < qparam->nxqs; i++) {
2640			flags = lif->rxqcqs[i]->flags & ~IONIC_QCQ_F_INTR;
2641			err = ionic_qcq_alloc(lif, IONIC_QTYPE_RXQ, i, "rx", flags,
2642					      num_desc, desc_sz, comp_sz, sg_desc_sz,
2643					      lif->kern_pid, &rx_qcqs[i]);
2644			if (err)
2645				goto err_out;
2646
2647			rx_qcqs[i]->q.features = qparam->rxq_features;
2648		}
2649	}
2650
2651	/* stop and clean the queues */
2652	ionic_stop_queues_reconfig(lif);
2653
2654	if (qparam->nxqs != lif->nxqs) {
2655		err = netif_set_real_num_tx_queues(lif->netdev, qparam->nxqs);
2656		if (err)
2657			goto err_out_reinit_unlock;
2658		err = netif_set_real_num_rx_queues(lif->netdev, qparam->nxqs);
2659		if (err) {
2660			netif_set_real_num_tx_queues(lif->netdev, lif->nxqs);
2661			goto err_out_reinit_unlock;
2662		}
2663	}
2664
2665	/* swap new desc_info and rings, keeping existing interrupt config */
2666	if (tx_qcqs) {
2667		lif->ntxq_descs = qparam->ntxq_descs;
2668		for (i = 0; i < qparam->nxqs; i++)
2669			ionic_swap_queues(lif->txqcqs[i], tx_qcqs[i]);
2670	}
2671
2672	if (rx_qcqs) {
2673		lif->nrxq_descs = qparam->nrxq_descs;
2674		for (i = 0; i < qparam->nxqs; i++)
2675			ionic_swap_queues(lif->rxqcqs[i], rx_qcqs[i]);
2676	}
2677
2678	/* if we need to change the interrupt layout, this is the time */
2679	if (qparam->intr_split != test_bit(IONIC_LIF_F_SPLIT_INTR, lif->state) ||
2680	    qparam->nxqs != lif->nxqs) {
2681		if (qparam->intr_split) {
2682			set_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2683		} else {
2684			clear_bit(IONIC_LIF_F_SPLIT_INTR, lif->state);
2685			lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2686			lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2687		}
2688
2689		/* clear existing interrupt assignments */
2690		for (i = 0; i < lif->ionic->ntxqs_per_lif; i++) {
2691			ionic_qcq_intr_free(lif, lif->txqcqs[i]);
2692			ionic_qcq_intr_free(lif, lif->rxqcqs[i]);
2693		}
2694
2695		/* re-assign the interrupts */
2696		for (i = 0; i < qparam->nxqs; i++) {
2697			lif->rxqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2698			err = ionic_alloc_qcq_interrupt(lif, lif->rxqcqs[i]);
2699			ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2700					     lif->rxqcqs[i]->intr.index,
2701					     lif->rx_coalesce_hw);
2702
2703			if (qparam->intr_split) {
2704				lif->txqcqs[i]->flags |= IONIC_QCQ_F_INTR;
2705				err = ionic_alloc_qcq_interrupt(lif, lif->txqcqs[i]);
2706				ionic_intr_coal_init(lif->ionic->idev.intr_ctrl,
2707						     lif->txqcqs[i]->intr.index,
2708						     lif->tx_coalesce_hw);
2709				if (test_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state))
2710					lif->txqcqs[i]->intr.dim_coal_hw = lif->tx_coalesce_hw;
2711			} else {
2712				lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2713				ionic_link_qcq_interrupts(lif->rxqcqs[i], lif->txqcqs[i]);
2714			}
2715		}
2716	}
2717
2718	/* now we can rework the debugfs mappings */
2719	if (tx_qcqs) {
2720		for (i = 0; i < qparam->nxqs; i++) {
2721			ionic_debugfs_del_qcq(lif->txqcqs[i]);
2722			ionic_debugfs_add_qcq(lif, lif->txqcqs[i]);
2723		}
2724	}
2725
2726	if (rx_qcqs) {
2727		for (i = 0; i < qparam->nxqs; i++) {
2728			ionic_debugfs_del_qcq(lif->rxqcqs[i]);
2729			ionic_debugfs_add_qcq(lif, lif->rxqcqs[i]);
2730		}
2731	}
2732
2733	swap(lif->nxqs, qparam->nxqs);
2734	swap(lif->rxq_features, qparam->rxq_features);
2735
2736err_out_reinit_unlock:
2737	/* re-init the queues, but don't lose an error code */
2738	if (err)
2739		ionic_start_queues_reconfig(lif);
2740	else
2741		err = ionic_start_queues_reconfig(lif);
2742
2743err_out:
2744	/* free old allocs without cleaning intr */
2745	for (i = 0; i < qparam->nxqs; i++) {
2746		if (tx_qcqs && tx_qcqs[i]) {
2747			tx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2748			ionic_qcq_free(lif, tx_qcqs[i]);
2749			devm_kfree(lif->ionic->dev, tx_qcqs[i]);
2750			tx_qcqs[i] = NULL;
2751		}
2752		if (rx_qcqs && rx_qcqs[i]) {
2753			rx_qcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2754			ionic_qcq_free(lif, rx_qcqs[i]);
2755			devm_kfree(lif->ionic->dev, rx_qcqs[i]);
2756			rx_qcqs[i] = NULL;
2757		}
2758	}
2759
2760	/* free q array */
2761	if (rx_qcqs) {
2762		devm_kfree(lif->ionic->dev, rx_qcqs);
2763		rx_qcqs = NULL;
2764	}
2765	if (tx_qcqs) {
2766		devm_kfree(lif->ionic->dev, tx_qcqs);
2767		tx_qcqs = NULL;
2768	}
2769
2770	/* clean the unused dma and info allocations when new set is smaller
2771	 * than the full array, but leave the qcq shells in place
2772	 */
2773	for (i = lif->nxqs; i < lif->ionic->ntxqs_per_lif; i++) {
2774		lif->txqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2775		ionic_qcq_free(lif, lif->txqcqs[i]);
2776
2777		lif->rxqcqs[i]->flags &= ~IONIC_QCQ_F_INTR;
2778		ionic_qcq_free(lif, lif->rxqcqs[i]);
2779	}
2780
2781	return err;
2782}
2783
2784int ionic_lif_alloc(struct ionic *ionic)
2785{
2786	struct device *dev = ionic->dev;
2787	union ionic_lif_identity *lid;
2788	struct net_device *netdev;
2789	struct ionic_lif *lif;
2790	int tbl_sz;
2791	int err;
2792
2793	lid = kzalloc(sizeof(*lid), GFP_KERNEL);
2794	if (!lid)
2795		return -ENOMEM;
2796
2797	netdev = alloc_etherdev_mqs(sizeof(*lif),
2798				    ionic->ntxqs_per_lif, ionic->ntxqs_per_lif);
2799	if (!netdev) {
2800		dev_err(dev, "Cannot allocate netdev, aborting\n");
2801		err = -ENOMEM;
2802		goto err_out_free_lid;
2803	}
2804
2805	SET_NETDEV_DEV(netdev, dev);
2806
2807	lif = netdev_priv(netdev);
2808	lif->netdev = netdev;
2809	ionic->lif = lif;
2810	netdev->netdev_ops = &ionic_netdev_ops;
2811	ionic_ethtool_set_ops(netdev);
2812
2813	netdev->watchdog_timeo = 2 * HZ;
2814	netif_carrier_off(netdev);
2815
2816	lif->identity = lid;
2817	lif->lif_type = IONIC_LIF_TYPE_CLASSIC;
2818	err = ionic_lif_identify(ionic, lif->lif_type, lif->identity);
2819	if (err) {
2820		dev_err(ionic->dev, "Cannot identify type %d: %d\n",
2821			lif->lif_type, err);
2822		goto err_out_free_netdev;
2823	}
2824	lif->netdev->min_mtu = max_t(unsigned int, ETH_MIN_MTU,
2825				     le32_to_cpu(lif->identity->eth.min_frame_size));
2826	lif->netdev->max_mtu =
2827		le32_to_cpu(lif->identity->eth.max_frame_size) - ETH_HLEN - VLAN_HLEN;
2828
2829	lif->neqs = ionic->neqs_per_lif;
2830	lif->nxqs = ionic->ntxqs_per_lif;
2831
2832	lif->ionic = ionic;
2833	lif->index = 0;
2834	lif->ntxq_descs = IONIC_DEF_TXRX_DESC;
2835	lif->nrxq_descs = IONIC_DEF_TXRX_DESC;
2836
2837	/* Convert the default coalesce value to actual hw resolution */
2838	lif->rx_coalesce_usecs = IONIC_ITR_COAL_USEC_DEFAULT;
2839	lif->rx_coalesce_hw = ionic_coal_usec_to_hw(lif->ionic,
2840						    lif->rx_coalesce_usecs);
2841	lif->tx_coalesce_usecs = lif->rx_coalesce_usecs;
2842	lif->tx_coalesce_hw = lif->rx_coalesce_hw;
2843	set_bit(IONIC_LIF_F_RX_DIM_INTR, lif->state);
2844	set_bit(IONIC_LIF_F_TX_DIM_INTR, lif->state);
2845
2846	snprintf(lif->name, sizeof(lif->name), "lif%u", lif->index);
2847
2848	spin_lock_init(&lif->adminq_lock);
2849
2850	spin_lock_init(&lif->deferred.lock);
2851	INIT_LIST_HEAD(&lif->deferred.list);
2852	INIT_WORK(&lif->deferred.work, ionic_lif_deferred_work);
2853
2854	/* allocate lif info */
2855	lif->info_sz = ALIGN(sizeof(*lif->info), PAGE_SIZE);
2856	lif->info = dma_alloc_coherent(dev, lif->info_sz,
2857				       &lif->info_pa, GFP_KERNEL);
2858	if (!lif->info) {
2859		dev_err(dev, "Failed to allocate lif info, aborting\n");
2860		err = -ENOMEM;
2861		goto err_out_free_netdev;
2862	}
2863
2864	ionic_debugfs_add_lif(lif);
2865
2866	/* allocate control queues and txrx queue arrays */
2867	ionic_lif_queue_identify(lif);
2868	err = ionic_qcqs_alloc(lif);
2869	if (err)
2870		goto err_out_free_lif_info;
2871
2872	/* allocate rss indirection table */
2873	tbl_sz = le16_to_cpu(lif->ionic->ident.lif.eth.rss_ind_tbl_sz);
2874	lif->rss_ind_tbl_sz = sizeof(*lif->rss_ind_tbl) * tbl_sz;
2875	lif->rss_ind_tbl = dma_alloc_coherent(dev, lif->rss_ind_tbl_sz,
2876					      &lif->rss_ind_tbl_pa,
2877					      GFP_KERNEL);
2878
2879	if (!lif->rss_ind_tbl) {
2880		err = -ENOMEM;
2881		dev_err(dev, "Failed to allocate rss indirection table, aborting\n");
2882		goto err_out_free_qcqs;
2883	}
2884	netdev_rss_key_fill(lif->rss_hash_key, IONIC_RSS_HASH_KEY_SIZE);
2885
2886	ionic_lif_alloc_phc(lif);
2887
2888	return 0;
2889
2890err_out_free_qcqs:
2891	ionic_qcqs_free(lif);
2892err_out_free_lif_info:
2893	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
2894	lif->info = NULL;
2895	lif->info_pa = 0;
2896err_out_free_netdev:
2897	free_netdev(lif->netdev);
2898	lif = NULL;
2899err_out_free_lid:
2900	kfree(lid);
2901
2902	return err;
2903}
2904
2905static void ionic_lif_reset(struct ionic_lif *lif)
2906{
2907	struct ionic_dev *idev = &lif->ionic->idev;
2908
2909	mutex_lock(&lif->ionic->dev_cmd_lock);
2910	ionic_dev_cmd_lif_reset(idev, lif->index);
2911	ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
2912	mutex_unlock(&lif->ionic->dev_cmd_lock);
2913}
2914
2915static void ionic_lif_handle_fw_down(struct ionic_lif *lif)
2916{
2917	struct ionic *ionic = lif->ionic;
2918
2919	if (test_and_set_bit(IONIC_LIF_F_FW_RESET, lif->state))
2920		return;
2921
2922	dev_info(ionic->dev, "FW Down: Stopping LIFs\n");
2923
2924	netif_device_detach(lif->netdev);
2925
2926	if (test_bit(IONIC_LIF_F_UP, lif->state)) {
2927		dev_info(ionic->dev, "Surprise FW stop, stopping queues\n");
2928		mutex_lock(&lif->queue_lock);
2929		ionic_stop_queues(lif);
2930		mutex_unlock(&lif->queue_lock);
2931	}
2932
2933	if (netif_running(lif->netdev)) {
2934		ionic_txrx_deinit(lif);
2935		ionic_txrx_free(lif);
2936	}
2937	ionic_lif_deinit(lif);
2938	ionic_reset(ionic);
2939	ionic_qcqs_free(lif);
2940
2941	dev_info(ionic->dev, "FW Down: LIFs stopped\n");
2942}
2943
2944static void ionic_lif_handle_fw_up(struct ionic_lif *lif)
2945{
2946	struct ionic *ionic = lif->ionic;
2947	int err;
2948
2949	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
2950		return;
2951
2952	dev_info(ionic->dev, "FW Up: restarting LIFs\n");
2953
2954	ionic_init_devinfo(ionic);
2955	err = ionic_identify(ionic);
2956	if (err)
2957		goto err_out;
2958	err = ionic_port_identify(ionic);
2959	if (err)
2960		goto err_out;
2961	err = ionic_port_init(ionic);
2962	if (err)
2963		goto err_out;
2964	err = ionic_qcqs_alloc(lif);
2965	if (err)
2966		goto err_out;
2967
2968	err = ionic_lif_init(lif);
2969	if (err)
2970		goto err_qcqs_free;
2971
2972	if (lif->registered)
2973		ionic_lif_set_netdev_info(lif);
2974
2975	ionic_rx_filter_replay(lif);
2976
2977	if (netif_running(lif->netdev)) {
2978		err = ionic_txrx_alloc(lif);
2979		if (err)
2980			goto err_lifs_deinit;
2981
2982		err = ionic_txrx_init(lif);
2983		if (err)
2984			goto err_txrx_free;
2985	}
2986
2987	clear_bit(IONIC_LIF_F_FW_RESET, lif->state);
2988	ionic_link_status_check_request(lif, CAN_SLEEP);
2989	netif_device_attach(lif->netdev);
2990	dev_info(ionic->dev, "FW Up: LIFs restarted\n");
2991
2992	/* restore the hardware timestamping queues */
2993	ionic_lif_hwstamp_replay(lif);
2994
2995	return;
2996
2997err_txrx_free:
2998	ionic_txrx_free(lif);
2999err_lifs_deinit:
3000	ionic_lif_deinit(lif);
3001err_qcqs_free:
3002	ionic_qcqs_free(lif);
3003err_out:
3004	dev_err(ionic->dev, "FW Up: LIFs restart failed - err %d\n", err);
3005}
3006
3007void ionic_lif_free(struct ionic_lif *lif)
3008{
3009	struct device *dev = lif->ionic->dev;
3010
3011	ionic_lif_free_phc(lif);
3012
3013	/* free rss indirection table */
3014	dma_free_coherent(dev, lif->rss_ind_tbl_sz, lif->rss_ind_tbl,
3015			  lif->rss_ind_tbl_pa);
3016	lif->rss_ind_tbl = NULL;
3017	lif->rss_ind_tbl_pa = 0;
3018
3019	/* free queues */
3020	ionic_qcqs_free(lif);
3021	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state))
3022		ionic_lif_reset(lif);
3023
3024	/* free lif info */
3025	kfree(lif->identity);
3026	dma_free_coherent(dev, lif->info_sz, lif->info, lif->info_pa);
3027	lif->info = NULL;
3028	lif->info_pa = 0;
3029
3030	/* unmap doorbell page */
3031	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3032	lif->kern_dbpage = NULL;
3033	kfree(lif->dbid_inuse);
3034	lif->dbid_inuse = NULL;
3035
3036	/* free netdev & lif */
3037	ionic_debugfs_del_lif(lif);
3038	free_netdev(lif->netdev);
3039}
3040
3041void ionic_lif_deinit(struct ionic_lif *lif)
3042{
3043	if (!test_and_clear_bit(IONIC_LIF_F_INITED, lif->state))
3044		return;
3045
3046	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3047		cancel_work_sync(&lif->deferred.work);
3048		cancel_work_sync(&lif->tx_timeout_work);
3049		ionic_rx_filters_deinit(lif);
3050		if (lif->netdev->features & NETIF_F_RXHASH)
3051			ionic_lif_rss_deinit(lif);
3052	}
3053
3054	napi_disable(&lif->adminqcq->napi);
3055	ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3056	ionic_lif_qcq_deinit(lif, lif->adminqcq);
3057
3058	mutex_destroy(&lif->config_lock);
3059	mutex_destroy(&lif->queue_lock);
3060	ionic_lif_reset(lif);
3061}
3062
3063static int ionic_lif_adminq_init(struct ionic_lif *lif)
3064{
3065	struct device *dev = lif->ionic->dev;
3066	struct ionic_q_init_comp comp;
3067	struct ionic_dev *idev;
3068	struct ionic_qcq *qcq;
3069	struct ionic_queue *q;
3070	int err;
3071
3072	idev = &lif->ionic->idev;
3073	qcq = lif->adminqcq;
3074	q = &qcq->q;
3075
3076	mutex_lock(&lif->ionic->dev_cmd_lock);
3077	ionic_dev_cmd_adminq_init(idev, qcq, lif->index, qcq->intr.index);
3078	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3079	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3080	mutex_unlock(&lif->ionic->dev_cmd_lock);
3081	if (err) {
3082		netdev_err(lif->netdev, "adminq init failed %d\n", err);
3083		return err;
3084	}
3085
3086	q->hw_type = comp.hw_type;
3087	q->hw_index = le32_to_cpu(comp.hw_index);
3088	q->dbval = IONIC_DBELL_QID(q->hw_index);
3089
3090	dev_dbg(dev, "adminq->hw_type %d\n", q->hw_type);
3091	dev_dbg(dev, "adminq->hw_index %d\n", q->hw_index);
3092
3093	netif_napi_add(lif->netdev, &qcq->napi, ionic_adminq_napi,
3094		       NAPI_POLL_WEIGHT);
3095
3096	napi_enable(&qcq->napi);
3097
3098	if (qcq->flags & IONIC_QCQ_F_INTR)
3099		ionic_intr_mask(idev->intr_ctrl, qcq->intr.index,
3100				IONIC_INTR_MASK_CLEAR);
3101
3102	qcq->flags |= IONIC_QCQ_F_INITED;
3103
3104	return 0;
3105}
3106
3107static int ionic_lif_notifyq_init(struct ionic_lif *lif)
3108{
3109	struct ionic_qcq *qcq = lif->notifyqcq;
3110	struct device *dev = lif->ionic->dev;
3111	struct ionic_queue *q = &qcq->q;
3112	int err;
3113
3114	struct ionic_admin_ctx ctx = {
3115		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3116		.cmd.q_init = {
3117			.opcode = IONIC_CMD_Q_INIT,
3118			.lif_index = cpu_to_le16(lif->index),
3119			.type = q->type,
3120			.ver = lif->qtype_info[q->type].version,
3121			.index = cpu_to_le32(q->index),
3122			.flags = cpu_to_le16(IONIC_QINIT_F_IRQ |
3123					     IONIC_QINIT_F_ENA),
3124			.intr_index = cpu_to_le16(lif->adminqcq->intr.index),
3125			.pid = cpu_to_le16(q->pid),
3126			.ring_size = ilog2(q->num_descs),
3127			.ring_base = cpu_to_le64(q->base_pa),
3128		}
3129	};
3130
3131	dev_dbg(dev, "notifyq_init.pid %d\n", ctx.cmd.q_init.pid);
3132	dev_dbg(dev, "notifyq_init.index %d\n", ctx.cmd.q_init.index);
3133	dev_dbg(dev, "notifyq_init.ring_base 0x%llx\n", ctx.cmd.q_init.ring_base);
3134	dev_dbg(dev, "notifyq_init.ring_size %d\n", ctx.cmd.q_init.ring_size);
3135
3136	err = ionic_adminq_post_wait(lif, &ctx);
3137	if (err)
3138		return err;
3139
3140	lif->last_eid = 0;
3141	q->hw_type = ctx.comp.q_init.hw_type;
3142	q->hw_index = le32_to_cpu(ctx.comp.q_init.hw_index);
3143	q->dbval = IONIC_DBELL_QID(q->hw_index);
3144
3145	dev_dbg(dev, "notifyq->hw_type %d\n", q->hw_type);
3146	dev_dbg(dev, "notifyq->hw_index %d\n", q->hw_index);
3147
3148	/* preset the callback info */
3149	q->info[0].cb_arg = lif;
3150
3151	qcq->flags |= IONIC_QCQ_F_INITED;
3152
3153	return 0;
3154}
3155
3156static int ionic_station_set(struct ionic_lif *lif)
3157{
3158	struct net_device *netdev = lif->netdev;
3159	struct ionic_admin_ctx ctx = {
3160		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3161		.cmd.lif_getattr = {
3162			.opcode = IONIC_CMD_LIF_GETATTR,
3163			.index = cpu_to_le16(lif->index),
3164			.attr = IONIC_LIF_ATTR_MAC,
3165		},
3166	};
3167	struct sockaddr addr;
3168	int err;
3169
3170	err = ionic_adminq_post_wait(lif, &ctx);
3171	if (err)
3172		return err;
3173	netdev_dbg(lif->netdev, "found initial MAC addr %pM\n",
3174		   ctx.comp.lif_getattr.mac);
3175	if (is_zero_ether_addr(ctx.comp.lif_getattr.mac))
3176		return 0;
3177
3178	if (!is_zero_ether_addr(netdev->dev_addr)) {
3179		/* If the netdev mac is non-zero and doesn't match the default
3180		 * device address, it was set by something earlier and we're
3181		 * likely here again after a fw-upgrade reset.  We need to be
3182		 * sure the netdev mac is in our filter list.
3183		 */
3184		if (!ether_addr_equal(ctx.comp.lif_getattr.mac,
3185				      netdev->dev_addr))
3186			ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR);
3187	} else {
3188		/* Update the netdev mac with the device's mac */
3189		memcpy(addr.sa_data, ctx.comp.lif_getattr.mac, netdev->addr_len);
3190		addr.sa_family = AF_INET;
3191		err = eth_prepare_mac_addr_change(netdev, &addr);
3192		if (err) {
3193			netdev_warn(lif->netdev, "ignoring bad MAC addr from NIC %pM - err %d\n",
3194				    addr.sa_data, err);
3195			return 0;
3196		}
3197
3198		eth_commit_mac_addr_change(netdev, &addr);
3199	}
3200
3201	netdev_dbg(lif->netdev, "adding station MAC addr %pM\n",
3202		   netdev->dev_addr);
3203	ionic_lif_addr(lif, netdev->dev_addr, ADD_ADDR);
3204
3205	return 0;
3206}
3207
3208int ionic_lif_init(struct ionic_lif *lif)
3209{
3210	struct ionic_dev *idev = &lif->ionic->idev;
3211	struct device *dev = lif->ionic->dev;
3212	struct ionic_lif_init_comp comp;
3213	int dbpage_num;
3214	int err;
3215
3216	mutex_lock(&lif->ionic->dev_cmd_lock);
3217	ionic_dev_cmd_lif_init(idev, lif->index, lif->info_pa);
3218	err = ionic_dev_cmd_wait(lif->ionic, DEVCMD_TIMEOUT);
3219	ionic_dev_cmd_comp(idev, (union ionic_dev_cmd_comp *)&comp);
3220	mutex_unlock(&lif->ionic->dev_cmd_lock);
3221	if (err)
3222		return err;
3223
3224	lif->hw_index = le16_to_cpu(comp.hw_index);
3225	mutex_init(&lif->queue_lock);
3226	mutex_init(&lif->config_lock);
3227
3228	/* now that we have the hw_index we can figure out our doorbell page */
3229	lif->dbid_count = le32_to_cpu(lif->ionic->ident.dev.ndbpgs_per_lif);
3230	if (!lif->dbid_count) {
3231		dev_err(dev, "No doorbell pages, aborting\n");
3232		return -EINVAL;
3233	}
3234
3235	lif->dbid_inuse = bitmap_alloc(lif->dbid_count, GFP_KERNEL);
3236	if (!lif->dbid_inuse) {
3237		dev_err(dev, "Failed alloc doorbell id bitmap, aborting\n");
3238		return -ENOMEM;
3239	}
3240
3241	/* first doorbell id reserved for kernel (dbid aka pid == zero) */
3242	set_bit(0, lif->dbid_inuse);
3243	lif->kern_pid = 0;
3244
3245	dbpage_num = ionic_db_page_num(lif, lif->kern_pid);
3246	lif->kern_dbpage = ionic_bus_map_dbpage(lif->ionic, dbpage_num);
3247	if (!lif->kern_dbpage) {
3248		dev_err(dev, "Cannot map dbpage, aborting\n");
3249		err = -ENOMEM;
3250		goto err_out_free_dbid;
3251	}
3252
3253	err = ionic_lif_adminq_init(lif);
3254	if (err)
3255		goto err_out_adminq_deinit;
3256
3257	if (lif->ionic->nnqs_per_lif) {
3258		err = ionic_lif_notifyq_init(lif);
3259		if (err)
3260			goto err_out_notifyq_deinit;
3261	}
3262
3263	err = ionic_init_nic_features(lif);
3264	if (err)
3265		goto err_out_notifyq_deinit;
3266
3267	if (!test_bit(IONIC_LIF_F_FW_RESET, lif->state)) {
3268		err = ionic_rx_filters_init(lif);
3269		if (err)
3270			goto err_out_notifyq_deinit;
3271	}
3272
3273	err = ionic_station_set(lif);
3274	if (err)
3275		goto err_out_notifyq_deinit;
3276
3277	lif->rx_copybreak = IONIC_RX_COPYBREAK_DEFAULT;
3278
3279	set_bit(IONIC_LIF_F_INITED, lif->state);
3280
3281	INIT_WORK(&lif->tx_timeout_work, ionic_tx_timeout_work);
3282
3283	return 0;
3284
3285err_out_notifyq_deinit:
3286	ionic_lif_qcq_deinit(lif, lif->notifyqcq);
3287err_out_adminq_deinit:
3288	ionic_lif_qcq_deinit(lif, lif->adminqcq);
3289	ionic_lif_reset(lif);
3290	ionic_bus_unmap_dbpage(lif->ionic, lif->kern_dbpage);
3291	lif->kern_dbpage = NULL;
3292err_out_free_dbid:
3293	kfree(lif->dbid_inuse);
3294	lif->dbid_inuse = NULL;
3295
3296	return err;
3297}
3298
3299static void ionic_lif_notify_work(struct work_struct *ws)
3300{
3301}
3302
3303static void ionic_lif_set_netdev_info(struct ionic_lif *lif)
3304{
3305	struct ionic_admin_ctx ctx = {
3306		.work = COMPLETION_INITIALIZER_ONSTACK(ctx.work),
3307		.cmd.lif_setattr = {
3308			.opcode = IONIC_CMD_LIF_SETATTR,
3309			.index = cpu_to_le16(lif->index),
3310			.attr = IONIC_LIF_ATTR_NAME,
3311		},
3312	};
3313
3314	strlcpy(ctx.cmd.lif_setattr.name, lif->netdev->name,
3315		sizeof(ctx.cmd.lif_setattr.name));
3316
3317	ionic_adminq_post_wait(lif, &ctx);
3318}
3319
3320static struct ionic_lif *ionic_netdev_lif(struct net_device *netdev)
3321{
3322	if (!netdev || netdev->netdev_ops->ndo_start_xmit != ionic_start_xmit)
3323		return NULL;
3324
3325	return netdev_priv(netdev);
3326}
3327
3328static int ionic_lif_notify(struct notifier_block *nb,
3329			    unsigned long event, void *info)
3330{
3331	struct net_device *ndev = netdev_notifier_info_to_dev(info);
3332	struct ionic *ionic = container_of(nb, struct ionic, nb);
3333	struct ionic_lif *lif = ionic_netdev_lif(ndev);
3334
3335	if (!lif || lif->ionic != ionic)
3336		return NOTIFY_DONE;
3337
3338	switch (event) {
3339	case NETDEV_CHANGENAME:
3340		ionic_lif_set_netdev_info(lif);
3341		break;
3342	}
3343
3344	return NOTIFY_DONE;
3345}
3346
3347int ionic_lif_register(struct ionic_lif *lif)
3348{
3349	int err;
3350
3351	ionic_lif_register_phc(lif);
3352
3353	INIT_WORK(&lif->ionic->nb_work, ionic_lif_notify_work);
3354
3355	lif->ionic->nb.notifier_call = ionic_lif_notify;
3356
3357	err = register_netdevice_notifier(&lif->ionic->nb);
3358	if (err)
3359		lif->ionic->nb.notifier_call = NULL;
3360
3361	/* only register LIF0 for now */
3362	err = register_netdev(lif->netdev);
3363	if (err) {
3364		dev_err(lif->ionic->dev, "Cannot register net device, aborting\n");
3365		ionic_lif_unregister_phc(lif);
3366		return err;
3367	}
3368
3369	ionic_link_status_check_request(lif, CAN_SLEEP);
3370	lif->registered = true;
3371	ionic_lif_set_netdev_info(lif);
3372
3373	return 0;
3374}
3375
3376void ionic_lif_unregister(struct ionic_lif *lif)
3377{
3378	if (lif->ionic->nb.notifier_call) {
3379		unregister_netdevice_notifier(&lif->ionic->nb);
3380		cancel_work_sync(&lif->ionic->nb_work);
3381		lif->ionic->nb.notifier_call = NULL;
3382	}
3383
3384	if (lif->netdev->reg_state == NETREG_REGISTERED)
3385		unregister_netdev(lif->netdev);
3386
3387	ionic_lif_unregister_phc(lif);
3388
3389	lif->registered = false;
3390}
3391
3392static void ionic_lif_queue_identify(struct ionic_lif *lif)
3393{
3394	union ionic_q_identity __iomem *q_ident;
3395	struct ionic *ionic = lif->ionic;
3396	struct ionic_dev *idev;
3397	int qtype;
3398	int err;
3399
3400	idev = &lif->ionic->idev;
3401	q_ident = (union ionic_q_identity __iomem *)&idev->dev_cmd_regs->data;
3402
3403	for (qtype = 0; qtype < ARRAY_SIZE(ionic_qtype_versions); qtype++) {
3404		struct ionic_qtype_info *qti = &lif->qtype_info[qtype];
3405
3406		/* filter out the ones we know about */
3407		switch (qtype) {
3408		case IONIC_QTYPE_ADMINQ:
3409		case IONIC_QTYPE_NOTIFYQ:
3410		case IONIC_QTYPE_RXQ:
3411		case IONIC_QTYPE_TXQ:
3412			break;
3413		default:
3414			continue;
3415		}
3416
3417		memset(qti, 0, sizeof(*qti));
3418
3419		mutex_lock(&ionic->dev_cmd_lock);
3420		ionic_dev_cmd_queue_identify(idev, lif->lif_type, qtype,
3421					     ionic_qtype_versions[qtype]);
3422		err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3423		if (!err) {
3424			qti->version   = readb(&q_ident->version);
3425			qti->supported = readb(&q_ident->supported);
3426			qti->features  = readq(&q_ident->features);
3427			qti->desc_sz   = readw(&q_ident->desc_sz);
3428			qti->comp_sz   = readw(&q_ident->comp_sz);
3429			qti->sg_desc_sz   = readw(&q_ident->sg_desc_sz);
3430			qti->max_sg_elems = readw(&q_ident->max_sg_elems);
3431			qti->sg_desc_stride = readw(&q_ident->sg_desc_stride);
3432		}
3433		mutex_unlock(&ionic->dev_cmd_lock);
3434
3435		if (err == -EINVAL) {
3436			dev_err(ionic->dev, "qtype %d not supported\n", qtype);
3437			continue;
3438		} else if (err == -EIO) {
3439			dev_err(ionic->dev, "q_ident failed, not supported on older FW\n");
3440			return;
3441		} else if (err) {
3442			dev_err(ionic->dev, "q_ident failed, qtype %d: %d\n",
3443				qtype, err);
3444			return;
3445		}
3446
3447		dev_dbg(ionic->dev, " qtype[%d].version = %d\n",
3448			qtype, qti->version);
3449		dev_dbg(ionic->dev, " qtype[%d].supported = 0x%02x\n",
3450			qtype, qti->supported);
3451		dev_dbg(ionic->dev, " qtype[%d].features = 0x%04llx\n",
3452			qtype, qti->features);
3453		dev_dbg(ionic->dev, " qtype[%d].desc_sz = %d\n",
3454			qtype, qti->desc_sz);
3455		dev_dbg(ionic->dev, " qtype[%d].comp_sz = %d\n",
3456			qtype, qti->comp_sz);
3457		dev_dbg(ionic->dev, " qtype[%d].sg_desc_sz = %d\n",
3458			qtype, qti->sg_desc_sz);
3459		dev_dbg(ionic->dev, " qtype[%d].max_sg_elems = %d\n",
3460			qtype, qti->max_sg_elems);
3461		dev_dbg(ionic->dev, " qtype[%d].sg_desc_stride = %d\n",
3462			qtype, qti->sg_desc_stride);
3463	}
3464}
3465
3466int ionic_lif_identify(struct ionic *ionic, u8 lif_type,
3467		       union ionic_lif_identity *lid)
3468{
3469	struct ionic_dev *idev = &ionic->idev;
3470	size_t sz;
3471	int err;
3472
3473	sz = min(sizeof(*lid), sizeof(idev->dev_cmd_regs->data));
3474
3475	mutex_lock(&ionic->dev_cmd_lock);
3476	ionic_dev_cmd_lif_identify(idev, lif_type, IONIC_IDENTITY_VERSION_1);
3477	err = ionic_dev_cmd_wait(ionic, DEVCMD_TIMEOUT);
3478	memcpy_fromio(lid, &idev->dev_cmd_regs->data, sz);
3479	mutex_unlock(&ionic->dev_cmd_lock);
3480	if (err)
3481		return (err);
3482
3483	dev_dbg(ionic->dev, "capabilities 0x%llx\n",
3484		le64_to_cpu(lid->capabilities));
3485
3486	dev_dbg(ionic->dev, "eth.max_ucast_filters %d\n",
3487		le32_to_cpu(lid->eth.max_ucast_filters));
3488	dev_dbg(ionic->dev, "eth.max_mcast_filters %d\n",
3489		le32_to_cpu(lid->eth.max_mcast_filters));
3490	dev_dbg(ionic->dev, "eth.features 0x%llx\n",
3491		le64_to_cpu(lid->eth.config.features));
3492	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_ADMINQ] %d\n",
3493		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_ADMINQ]));
3494	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_NOTIFYQ] %d\n",
3495		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_NOTIFYQ]));
3496	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_RXQ] %d\n",
3497		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_RXQ]));
3498	dev_dbg(ionic->dev, "eth.queue_count[IONIC_QTYPE_TXQ] %d\n",
3499		le32_to_cpu(lid->eth.config.queue_count[IONIC_QTYPE_TXQ]));
3500	dev_dbg(ionic->dev, "eth.config.name %s\n", lid->eth.config.name);
3501	dev_dbg(ionic->dev, "eth.config.mac %pM\n", lid->eth.config.mac);
3502	dev_dbg(ionic->dev, "eth.config.mtu %d\n",
3503		le32_to_cpu(lid->eth.config.mtu));
3504
3505	return 0;
3506}
3507
3508int ionic_lif_size(struct ionic *ionic)
3509{
3510	struct ionic_identity *ident = &ionic->ident;
3511	unsigned int nintrs, dev_nintrs;
3512	union ionic_lif_config *lc;
3513	unsigned int ntxqs_per_lif;
3514	unsigned int nrxqs_per_lif;
3515	unsigned int neqs_per_lif;
3516	unsigned int nnqs_per_lif;
3517	unsigned int nxqs, neqs;
3518	unsigned int min_intrs;
3519	int err;
3520
3521	lc = &ident->lif.eth.config;
3522	dev_nintrs = le32_to_cpu(ident->dev.nintrs);
3523	neqs_per_lif = le32_to_cpu(ident->lif.rdma.eq_qtype.qid_count);
3524	nnqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_NOTIFYQ]);
3525	ntxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_TXQ]);
3526	nrxqs_per_lif = le32_to_cpu(lc->queue_count[IONIC_QTYPE_RXQ]);
3527
3528	/* reserve last queue id for hardware timestamping */
3529	if (lc->features & cpu_to_le64(IONIC_ETH_HW_TIMESTAMP)) {
3530		if (ntxqs_per_lif <= 1 || nrxqs_per_lif <= 1) {
3531			lc->features &= cpu_to_le64(~IONIC_ETH_HW_TIMESTAMP);
3532		} else {
3533			ntxqs_per_lif -= 1;
3534			nrxqs_per_lif -= 1;
3535		}
3536	}
3537
3538	nxqs = min(ntxqs_per_lif, nrxqs_per_lif);
3539	nxqs = min(nxqs, num_online_cpus());
3540	neqs = min(neqs_per_lif, num_online_cpus());
3541
3542try_again:
3543	/* interrupt usage:
3544	 *    1 for master lif adminq/notifyq
3545	 *    1 for each CPU for master lif TxRx queue pairs
3546	 *    whatever's left is for RDMA queues
3547	 */
3548	nintrs = 1 + nxqs + neqs;
3549	min_intrs = 2;  /* adminq + 1 TxRx queue pair */
3550
3551	if (nintrs > dev_nintrs)
3552		goto try_fewer;
3553
3554	err = ionic_bus_alloc_irq_vectors(ionic, nintrs);
3555	if (err < 0 && err != -ENOSPC) {
3556		dev_err(ionic->dev, "Can't get intrs from OS: %d\n", err);
3557		return err;
3558	}
3559	if (err == -ENOSPC)
3560		goto try_fewer;
3561
3562	if (err != nintrs) {
3563		ionic_bus_free_irq_vectors(ionic);
3564		goto try_fewer;
3565	}
3566
3567	ionic->nnqs_per_lif = nnqs_per_lif;
3568	ionic->neqs_per_lif = neqs;
3569	ionic->ntxqs_per_lif = nxqs;
3570	ionic->nrxqs_per_lif = nxqs;
3571	ionic->nintrs = nintrs;
3572
3573	ionic_debugfs_add_sizes(ionic);
3574
3575	return 0;
3576
3577try_fewer:
3578	if (nnqs_per_lif > 1) {
3579		nnqs_per_lif >>= 1;
3580		goto try_again;
3581	}
3582	if (neqs > 1) {
3583		neqs >>= 1;
3584		goto try_again;
3585	}
3586	if (nxqs > 1) {
3587		nxqs >>= 1;
3588		goto try_again;
3589	}
3590	dev_err(ionic->dev, "Can't get minimum %d intrs from OS\n", min_intrs);
3591	return -ENOSPC;
3592}