1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * QLogic iSCSI Offload Driver
   4 * Copyright (c) 2016 Cavium Inc.
   5 */
   6
   7#include <linux/module.h>
   8#include <linux/pci.h>
   9#include <linux/kernel.h>
  10#include <linux/if_arp.h>
  11#include <scsi/iscsi_if.h>
  12#include <linux/inet.h>
  13#include <net/arp.h>
  14#include <linux/list.h>
  15#include <linux/kthread.h>
  16#include <linux/mm.h>
  17#include <linux/if_vlan.h>
  18#include <linux/cpu.h>
  19#include <linux/iscsi_boot_sysfs.h>
  20
  21#include <scsi/scsi_cmnd.h>
  22#include <scsi/scsi_device.h>
  23#include <scsi/scsi_eh.h>
  24#include <scsi/scsi_host.h>
  25#include <scsi/scsi.h>
  26
  27#include "qedi.h"
  28#include "qedi_gbl.h"
  29#include "qedi_iscsi.h"
  30
  31static uint qedi_qed_debug;
  32module_param(qedi_qed_debug, uint, 0644);
  33MODULE_PARM_DESC(qedi_qed_debug, " QED debug level 0 (default)");
  34
  35static uint qedi_fw_debug;
  36module_param(qedi_fw_debug, uint, 0644);
  37MODULE_PARM_DESC(qedi_fw_debug, " Firmware debug level 0(default) to 3");
  38
  39uint qedi_dbg_log = QEDI_LOG_WARN | QEDI_LOG_SCSI_TM;
  40module_param(qedi_dbg_log, uint, 0644);
  41MODULE_PARM_DESC(qedi_dbg_log, " Default debug level");
  42
  43uint qedi_io_tracing;
  44module_param(qedi_io_tracing, uint, 0644);
  45MODULE_PARM_DESC(qedi_io_tracing,
  46		 " Enable logging of SCSI requests/completions into trace buffer. (default off).");
  47
  48static uint qedi_ll2_buf_size = 0x400;
  49module_param(qedi_ll2_buf_size, uint, 0644);
  50MODULE_PARM_DESC(qedi_ll2_buf_size,
  51		 "parameter to set ping packet size, default - 0x400, Jumbo packets - 0x2400.");
  52
  53static uint qedi_flags_override;
  54module_param(qedi_flags_override, uint, 0644);
  55MODULE_PARM_DESC(qedi_flags_override, "Disable/Enable MFW error flags bits action.");
  56
  57const struct qed_iscsi_ops *qedi_ops;
  58static struct scsi_transport_template *qedi_scsi_transport;
  59static struct pci_driver qedi_pci_driver;
  60static DEFINE_PER_CPU(struct qedi_percpu_s, qedi_percpu);
  61static LIST_HEAD(qedi_udev_list);
  62/* Static function declaration */
  63static int qedi_alloc_global_queues(struct qedi_ctx *qedi);
  64static void qedi_free_global_queues(struct qedi_ctx *qedi);
  65static struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid);
  66static void qedi_reset_uio_rings(struct qedi_uio_dev *udev);
  67static void qedi_ll2_free_skbs(struct qedi_ctx *qedi);
  68static struct nvm_iscsi_block *qedi_get_nvram_block(struct qedi_ctx *qedi);
  69static void qedi_recovery_handler(struct work_struct *work);
  70static void qedi_schedule_hw_err_handler(void *dev,
  71					 enum qed_hw_err_type err_type);
  72static int qedi_suspend(struct pci_dev *pdev, pm_message_t state);
  73
  74static int qedi_iscsi_event_cb(void *context, u8 fw_event_code, void *fw_handle)
  75{
  76	struct qedi_ctx *qedi;
  77	struct qedi_endpoint *qedi_ep;
  78	struct iscsi_eqe_data *data;
  79	int rval = 0;
  80
  81	if (!context || !fw_handle) {
  82		QEDI_ERR(NULL, "Recv event with ctx NULL\n");
  83		return -EINVAL;
  84	}
  85
  86	qedi = (struct qedi_ctx *)context;
  87	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
  88		  "Recv Event %d fw_handle %p\n", fw_event_code, fw_handle);
  89
  90	data = (struct iscsi_eqe_data *)fw_handle;
  91	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
  92		  "icid=0x%x conn_id=0x%x err-code=0x%x error-pdu-opcode-reserved=0x%x\n",
  93		   data->icid, data->conn_id, data->error_code,
  94		   data->error_pdu_opcode_reserved);
  95
  96	qedi_ep = qedi->ep_tbl[data->icid];
  97
  98	if (!qedi_ep) {
  99		QEDI_WARN(&qedi->dbg_ctx,
 100			  "Cannot process event, ep already disconnected, cid=0x%x\n",
 101			   data->icid);
 102		WARN_ON(1);
 103		return -ENODEV;
 104	}
 105
 106	switch (fw_event_code) {
 107	case ISCSI_EVENT_TYPE_ASYN_CONNECT_COMPLETE:
 108		if (qedi_ep->state == EP_STATE_OFLDCONN_START)
 109			qedi_ep->state = EP_STATE_OFLDCONN_COMPL;
 110
 111		wake_up_interruptible(&qedi_ep->tcp_ofld_wait);
 112		break;
 113	case ISCSI_EVENT_TYPE_ASYN_TERMINATE_DONE:
 114		qedi_ep->state = EP_STATE_DISCONN_COMPL;
 115		wake_up_interruptible(&qedi_ep->tcp_ofld_wait);
 116		break;
 117	case ISCSI_EVENT_TYPE_ISCSI_CONN_ERROR:
 118		qedi_process_iscsi_error(qedi_ep, data);
 119		break;
 120	case ISCSI_EVENT_TYPE_ASYN_ABORT_RCVD:
 121	case ISCSI_EVENT_TYPE_ASYN_SYN_RCVD:
 122	case ISCSI_EVENT_TYPE_ASYN_MAX_RT_TIME:
 123	case ISCSI_EVENT_TYPE_ASYN_MAX_RT_CNT:
 124	case ISCSI_EVENT_TYPE_ASYN_MAX_KA_PROBES_CNT:
 125	case ISCSI_EVENT_TYPE_ASYN_FIN_WAIT2:
 126	case ISCSI_EVENT_TYPE_TCP_CONN_ERROR:
 127		qedi_process_tcp_error(qedi_ep, data);
 128		break;
 129	default:
 130		QEDI_ERR(&qedi->dbg_ctx, "Recv Unknown Event %u\n",
 131			 fw_event_code);
 132	}
 133
 134	return rval;
 135}
 136
 137static int qedi_uio_open(struct uio_info *uinfo, struct inode *inode)
 138{
 139	struct qedi_uio_dev *udev = uinfo->priv;
 140	struct qedi_ctx *qedi = udev->qedi;
 141
 142	if (!capable(CAP_NET_ADMIN))
 143		return -EPERM;
 144
 145	if (udev->uio_dev != -1)
 146		return -EBUSY;
 147
 148	rtnl_lock();
 149	udev->uio_dev = iminor(inode);
 150	qedi_reset_uio_rings(udev);
 151	set_bit(UIO_DEV_OPENED, &qedi->flags);
 152	rtnl_unlock();
 153
 154	return 0;
 155}
 156
 157static int qedi_uio_close(struct uio_info *uinfo, struct inode *inode)
 158{
 159	struct qedi_uio_dev *udev = uinfo->priv;
 160	struct qedi_ctx *qedi = udev->qedi;
 161
 162	udev->uio_dev = -1;
 163	clear_bit(UIO_DEV_OPENED, &qedi->flags);
 164	qedi_ll2_free_skbs(qedi);
 165	return 0;
 166}
 167
 168static void __qedi_free_uio_rings(struct qedi_uio_dev *udev)
 169{
 170	if (udev->uctrl) {
 171		free_page((unsigned long)udev->uctrl);
 172		udev->uctrl = NULL;
 173	}
 174
 175	if (udev->ll2_ring) {
 176		free_page((unsigned long)udev->ll2_ring);
 177		udev->ll2_ring = NULL;
 178	}
 179
 180	if (udev->ll2_buf) {
 181		free_pages((unsigned long)udev->ll2_buf, 2);
 182		udev->ll2_buf = NULL;
 183	}
 184}
 185
 186static void __qedi_free_uio(struct qedi_uio_dev *udev)
 187{
 188	uio_unregister_device(&udev->qedi_uinfo);
 189
 190	__qedi_free_uio_rings(udev);
 191
 192	pci_dev_put(udev->pdev);
 193	kfree(udev);
 194}
 195
 196static void qedi_free_uio(struct qedi_uio_dev *udev)
 197{
 198	if (!udev)
 199		return;
 200
 201	list_del_init(&udev->list);
 202	__qedi_free_uio(udev);
 203}
 204
 205static void qedi_reset_uio_rings(struct qedi_uio_dev *udev)
 206{
 207	struct qedi_ctx *qedi = NULL;
 208	struct qedi_uio_ctrl *uctrl = NULL;
 209
 210	qedi = udev->qedi;
 211	uctrl = udev->uctrl;
 212
 213	spin_lock_bh(&qedi->ll2_lock);
 214	uctrl->host_rx_cons = 0;
 215	uctrl->hw_rx_prod = 0;
 216	uctrl->hw_rx_bd_prod = 0;
 217	uctrl->host_rx_bd_cons = 0;
 218
 219	memset(udev->ll2_ring, 0, udev->ll2_ring_size);
 220	memset(udev->ll2_buf, 0, udev->ll2_buf_size);
 221	spin_unlock_bh(&qedi->ll2_lock);
 222}
 223
 224static int __qedi_alloc_uio_rings(struct qedi_uio_dev *udev)
 225{
 226	int rc = 0;
 227
 228	if (udev->ll2_ring || udev->ll2_buf)
 229		return rc;
 230
 231	/* Memory for control area.  */
 232	udev->uctrl = (void *)get_zeroed_page(GFP_KERNEL);
 233	if (!udev->uctrl)
 234		return -ENOMEM;
 235
 236	/* Allocating memory for LL2 ring  */
 237	udev->ll2_ring_size = QEDI_PAGE_SIZE;
 238	udev->ll2_ring = (void *)get_zeroed_page(GFP_KERNEL | __GFP_COMP);
 239	if (!udev->ll2_ring) {
 240		rc = -ENOMEM;
 241		goto exit_alloc_ring;
 242	}
 243
 244	/* Allocating memory for Tx/Rx pkt buffer */
 245	udev->ll2_buf_size = TX_RX_RING * qedi_ll2_buf_size;
 246	udev->ll2_buf_size = QEDI_PAGE_ALIGN(udev->ll2_buf_size);
 247	udev->ll2_buf = (void *)__get_free_pages(GFP_KERNEL | __GFP_COMP |
 248						 __GFP_ZERO, 2);
 249	if (!udev->ll2_buf) {
 250		rc = -ENOMEM;
 251		goto exit_alloc_buf;
 252	}
 253	return rc;
 254
 255exit_alloc_buf:
 256	free_page((unsigned long)udev->ll2_ring);
 257	udev->ll2_ring = NULL;
 258exit_alloc_ring:
 259	return rc;
 260}
 261
 262static int qedi_alloc_uio_rings(struct qedi_ctx *qedi)
 263{
 264	struct qedi_uio_dev *udev = NULL;
 265	int rc = 0;
 266
 267	list_for_each_entry(udev, &qedi_udev_list, list) {
 268		if (udev->pdev == qedi->pdev) {
 269			udev->qedi = qedi;
 270			if (__qedi_alloc_uio_rings(udev)) {
 271				udev->qedi = NULL;
 272				return -ENOMEM;
 273			}
 274			qedi->udev = udev;
 275			return 0;
 276		}
 277	}
 278
 279	udev = kzalloc(sizeof(*udev), GFP_KERNEL);
 280	if (!udev)
 281		goto err_udev;
 282
 283	udev->uio_dev = -1;
 284
 285	udev->qedi = qedi;
 286	udev->pdev = qedi->pdev;
 287
 288	rc = __qedi_alloc_uio_rings(udev);
 289	if (rc)
 290		goto err_uctrl;
 291
 292	list_add(&udev->list, &qedi_udev_list);
 293
 294	pci_dev_get(udev->pdev);
 295	qedi->udev = udev;
 296
 297	udev->tx_pkt = udev->ll2_buf;
 298	udev->rx_pkt = udev->ll2_buf + qedi_ll2_buf_size;
 299	return 0;
 300
 301 err_uctrl:
 302	kfree(udev);
 303 err_udev:
 304	return -ENOMEM;
 305}
 306
 307static int qedi_init_uio(struct qedi_ctx *qedi)
 308{
 309	struct qedi_uio_dev *udev = qedi->udev;
 310	struct uio_info *uinfo;
 311	int ret = 0;
 312
 313	if (!udev)
 314		return -ENOMEM;
 315
 316	uinfo = &udev->qedi_uinfo;
 317
 318	uinfo->mem[0].addr = (unsigned long)udev->uctrl;
 319	uinfo->mem[0].size = sizeof(struct qedi_uio_ctrl);
 320	uinfo->mem[0].memtype = UIO_MEM_LOGICAL;
 321
 322	uinfo->mem[1].addr = (unsigned long)udev->ll2_ring;
 323	uinfo->mem[1].size = udev->ll2_ring_size;
 324	uinfo->mem[1].memtype = UIO_MEM_LOGICAL;
 325
 326	uinfo->mem[2].addr = (unsigned long)udev->ll2_buf;
 327	uinfo->mem[2].size = udev->ll2_buf_size;
 328	uinfo->mem[2].memtype = UIO_MEM_LOGICAL;
 329
 330	uinfo->name = "qedi_uio";
 331	uinfo->version = QEDI_MODULE_VERSION;
 332	uinfo->irq = UIO_IRQ_CUSTOM;
 333
 334	uinfo->open = qedi_uio_open;
 335	uinfo->release = qedi_uio_close;
 336
 337	if (udev->uio_dev == -1) {
 338		if (!uinfo->priv) {
 339			uinfo->priv = udev;
 340
 341			ret = uio_register_device(&udev->pdev->dev, uinfo);
 342			if (ret) {
 343				QEDI_ERR(&qedi->dbg_ctx,
 344					 "UIO registration failed\n");
 345			}
 346		}
 347	}
 348
 349	return ret;
 350}
 351
 352static int qedi_alloc_and_init_sb(struct qedi_ctx *qedi,
 353				  struct qed_sb_info *sb_info, u16 sb_id)
 354{
 355	struct status_block *sb_virt;
 356	dma_addr_t sb_phys;
 357	int ret;
 358
 359	sb_virt = dma_alloc_coherent(&qedi->pdev->dev,
 360				     sizeof(struct status_block), &sb_phys,
 361				     GFP_KERNEL);
 362	if (!sb_virt) {
 363		QEDI_ERR(&qedi->dbg_ctx,
 364			 "Status block allocation failed for id = %d.\n",
 365			  sb_id);
 366		return -ENOMEM;
 367	}
 368
 369	ret = qedi_ops->common->sb_init(qedi->cdev, sb_info, sb_virt, sb_phys,
 370				       sb_id, QED_SB_TYPE_STORAGE);
 371	if (ret) {
 372		QEDI_ERR(&qedi->dbg_ctx,
 373			 "Status block initialization failed for id = %d.\n",
 374			  sb_id);
 375		return ret;
 376	}
 377
 378	return 0;
 379}
 380
 381static void qedi_free_sb(struct qedi_ctx *qedi)
 382{
 383	struct qed_sb_info *sb_info;
 384	int id;
 385
 386	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
 387		sb_info = &qedi->sb_array[id];
 388		if (sb_info->sb_virt)
 389			dma_free_coherent(&qedi->pdev->dev,
 390					  sizeof(*sb_info->sb_virt),
 391					  (void *)sb_info->sb_virt,
 392					  sb_info->sb_phys);
 393	}
 394}
 395
 396static void qedi_free_fp(struct qedi_ctx *qedi)
 397{
 398	kfree(qedi->fp_array);
 399	kfree(qedi->sb_array);
 400}
 401
 402static void qedi_destroy_fp(struct qedi_ctx *qedi)
 403{
 404	qedi_free_sb(qedi);
 405	qedi_free_fp(qedi);
 406}
 407
 408static int qedi_alloc_fp(struct qedi_ctx *qedi)
 409{
 410	int ret = 0;
 411
 412	qedi->fp_array = kcalloc(MIN_NUM_CPUS_MSIX(qedi),
 413				 sizeof(struct qedi_fastpath), GFP_KERNEL);
 414	if (!qedi->fp_array) {
 415		QEDI_ERR(&qedi->dbg_ctx,
 416			 "fastpath fp array allocation failed.\n");
 417		return -ENOMEM;
 418	}
 419
 420	qedi->sb_array = kcalloc(MIN_NUM_CPUS_MSIX(qedi),
 421				 sizeof(struct qed_sb_info), GFP_KERNEL);
 422	if (!qedi->sb_array) {
 423		QEDI_ERR(&qedi->dbg_ctx,
 424			 "fastpath sb array allocation failed.\n");
 425		ret = -ENOMEM;
 426		goto free_fp;
 427	}
 428
 429	return ret;
 430
 431free_fp:
 432	qedi_free_fp(qedi);
 433	return ret;
 434}
 435
 436static void qedi_int_fp(struct qedi_ctx *qedi)
 437{
 438	struct qedi_fastpath *fp;
 439	int id;
 440
 441	memset(qedi->fp_array, 0, MIN_NUM_CPUS_MSIX(qedi) *
 442	       sizeof(*qedi->fp_array));
 443	memset(qedi->sb_array, 0, MIN_NUM_CPUS_MSIX(qedi) *
 444	       sizeof(*qedi->sb_array));
 445
 446	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
 447		fp = &qedi->fp_array[id];
 448		fp->sb_info = &qedi->sb_array[id];
 449		fp->sb_id = id;
 450		fp->qedi = qedi;
 451		snprintf(fp->name, sizeof(fp->name), "%s-fp-%d",
 452			 "qedi", id);
 453
 454		/* fp_array[i] ---- irq cookie
 455		 * So init data which is needed in int ctx
 456		 */
 457	}
 458}
 459
 460static int qedi_prepare_fp(struct qedi_ctx *qedi)
 461{
 462	struct qedi_fastpath *fp;
 463	int id, ret = 0;
 464
 465	ret = qedi_alloc_fp(qedi);
 466	if (ret)
 467		goto err;
 468
 469	qedi_int_fp(qedi);
 470
 471	for (id = 0; id < MIN_NUM_CPUS_MSIX(qedi); id++) {
 472		fp = &qedi->fp_array[id];
 473		ret = qedi_alloc_and_init_sb(qedi, fp->sb_info, fp->sb_id);
 474		if (ret) {
 475			QEDI_ERR(&qedi->dbg_ctx,
 476				 "SB allocation and initialization failed.\n");
 477			ret = -EIO;
 478			goto err_init;
 479		}
 480	}
 481
 482	return 0;
 483
 484err_init:
 485	qedi_free_sb(qedi);
 486	qedi_free_fp(qedi);
 487err:
 488	return ret;
 489}
 490
 491static int qedi_setup_cid_que(struct qedi_ctx *qedi)
 492{
 493	int i;
 494
 495	qedi->cid_que.cid_que_base = kmalloc_array(qedi->max_active_conns,
 496						   sizeof(u32), GFP_KERNEL);
 497	if (!qedi->cid_que.cid_que_base)
 498		return -ENOMEM;
 499
 500	qedi->cid_que.conn_cid_tbl = kmalloc_array(qedi->max_active_conns,
 501						   sizeof(struct qedi_conn *),
 502						   GFP_KERNEL);
 503	if (!qedi->cid_que.conn_cid_tbl) {
 504		kfree(qedi->cid_que.cid_que_base);
 505		qedi->cid_que.cid_que_base = NULL;
 506		return -ENOMEM;
 507	}
 508
 509	qedi->cid_que.cid_que = (u32 *)qedi->cid_que.cid_que_base;
 510	qedi->cid_que.cid_q_prod_idx = 0;
 511	qedi->cid_que.cid_q_cons_idx = 0;
 512	qedi->cid_que.cid_q_max_idx = qedi->max_active_conns;
 513	qedi->cid_que.cid_free_cnt = qedi->max_active_conns;
 514
 515	for (i = 0; i < qedi->max_active_conns; i++) {
 516		qedi->cid_que.cid_que[i] = i;
 517		qedi->cid_que.conn_cid_tbl[i] = NULL;
 518	}
 519
 520	return 0;
 521}
 522
 523static void qedi_release_cid_que(struct qedi_ctx *qedi)
 524{
 525	kfree(qedi->cid_que.cid_que_base);
 526	qedi->cid_que.cid_que_base = NULL;
 527
 528	kfree(qedi->cid_que.conn_cid_tbl);
 529	qedi->cid_que.conn_cid_tbl = NULL;
 530}
 531
 532static int qedi_init_id_tbl(struct qedi_portid_tbl *id_tbl, u16 size,
 533			    u16 start_id, u16 next)
 534{
 535	id_tbl->start = start_id;
 536	id_tbl->max = size;
 537	id_tbl->next = next;
 538	spin_lock_init(&id_tbl->lock);
 539	id_tbl->table = kcalloc(BITS_TO_LONGS(size), sizeof(long), GFP_KERNEL);
 540	if (!id_tbl->table)
 541		return -ENOMEM;
 542
 543	return 0;
 544}
 545
 546static void qedi_free_id_tbl(struct qedi_portid_tbl *id_tbl)
 547{
 548	kfree(id_tbl->table);
 549	id_tbl->table = NULL;
 550}
 551
 552int qedi_alloc_id(struct qedi_portid_tbl *id_tbl, u16 id)
 553{
 554	int ret = -1;
 555
 556	id -= id_tbl->start;
 557	if (id >= id_tbl->max)
 558		return ret;
 559
 560	spin_lock(&id_tbl->lock);
 561	if (!test_bit(id, id_tbl->table)) {
 562		set_bit(id, id_tbl->table);
 563		ret = 0;
 564	}
 565	spin_unlock(&id_tbl->lock);
 566	return ret;
 567}
 568
 569u16 qedi_alloc_new_id(struct qedi_portid_tbl *id_tbl)
 570{
 571	u16 id;
 572
 573	spin_lock(&id_tbl->lock);
 574	id = find_next_zero_bit(id_tbl->table, id_tbl->max, id_tbl->next);
 575	if (id >= id_tbl->max) {
 576		id = QEDI_LOCAL_PORT_INVALID;
 577		if (id_tbl->next != 0) {
 578			id = find_first_zero_bit(id_tbl->table, id_tbl->next);
 579			if (id >= id_tbl->next)
 580				id = QEDI_LOCAL_PORT_INVALID;
 581		}
 582	}
 583
 584	if (id < id_tbl->max) {
 585		set_bit(id, id_tbl->table);
 586		id_tbl->next = (id + 1) & (id_tbl->max - 1);
 587		id += id_tbl->start;
 588	}
 589
 590	spin_unlock(&id_tbl->lock);
 591
 592	return id;
 593}
 594
 595void qedi_free_id(struct qedi_portid_tbl *id_tbl, u16 id)
 596{
 597	if (id == QEDI_LOCAL_PORT_INVALID)
 598		return;
 599
 600	id -= id_tbl->start;
 601	if (id >= id_tbl->max)
 602		return;
 603
 604	clear_bit(id, id_tbl->table);
 605}
 606
 607static void qedi_cm_free_mem(struct qedi_ctx *qedi)
 608{
 609	kfree(qedi->ep_tbl);
 610	qedi->ep_tbl = NULL;
 611	qedi_free_id_tbl(&qedi->lcl_port_tbl);
 612}
 613
 614static int qedi_cm_alloc_mem(struct qedi_ctx *qedi)
 615{
 616	u16 port_id;
 617
 618	qedi->ep_tbl = kzalloc((qedi->max_active_conns *
 619				sizeof(struct qedi_endpoint *)), GFP_KERNEL);
 620	if (!qedi->ep_tbl)
 621		return -ENOMEM;
 622	port_id = get_random_u32_below(QEDI_LOCAL_PORT_RANGE);
 623	if (qedi_init_id_tbl(&qedi->lcl_port_tbl, QEDI_LOCAL_PORT_RANGE,
 624			     QEDI_LOCAL_PORT_MIN, port_id)) {
 625		qedi_cm_free_mem(qedi);
 626		return -ENOMEM;
 627	}
 628
 629	return 0;
 630}
 631
 632static struct qedi_ctx *qedi_host_alloc(struct pci_dev *pdev)
 633{
 634	struct Scsi_Host *shost;
 635	struct qedi_ctx *qedi = NULL;
 636
 637	shost = iscsi_host_alloc(&qedi_host_template,
 638				 sizeof(struct qedi_ctx), 0);
 639	if (!shost) {
 640		QEDI_ERR(NULL, "Could not allocate shost\n");
 641		goto exit_setup_shost;
 642	}
 643
 644	shost->max_id = QEDI_MAX_ISCSI_CONNS_PER_HBA - 1;
 645	shost->max_channel = 0;
 646	shost->max_lun = ~0;
 647	shost->max_cmd_len = 16;
 648	shost->transportt = qedi_scsi_transport;
 649
 650	qedi = iscsi_host_priv(shost);
 651	memset(qedi, 0, sizeof(*qedi));
 652	qedi->shost = shost;
 653	qedi->dbg_ctx.host_no = shost->host_no;
 654	qedi->pdev = pdev;
 655	qedi->dbg_ctx.pdev = pdev;
 656	qedi->max_active_conns = ISCSI_MAX_SESS_PER_HBA;
 657	qedi->max_sqes = QEDI_SQ_SIZE;
 658
 659	shost->nr_hw_queues = MIN_NUM_CPUS_MSIX(qedi);
 660
 661	pci_set_drvdata(pdev, qedi);
 662
 663exit_setup_shost:
 664	return qedi;
 665}
 666
 667static int qedi_ll2_rx(void *cookie, struct sk_buff *skb, u32 arg1, u32 arg2)
 668{
 669	struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;
 670	struct skb_work_list *work;
 671	struct ethhdr *eh;
 672
 673	if (!qedi) {
 674		QEDI_ERR(NULL, "qedi is NULL\n");
 675		return -1;
 676	}
 677
 678	if (!test_bit(UIO_DEV_OPENED, &qedi->flags)) {
 679		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_UIO,
 680			  "UIO DEV is not opened\n");
 681		kfree_skb(skb);
 682		return 0;
 683	}
 684
 685	eh = (struct ethhdr *)skb->data;
 686	/* Undo VLAN encapsulation */
 687	if (eh->h_proto == htons(ETH_P_8021Q)) {
 688		memmove((u8 *)eh + VLAN_HLEN, eh, ETH_ALEN * 2);
 689		eh = (struct ethhdr *)skb_pull(skb, VLAN_HLEN);
 690		skb_reset_mac_header(skb);
 691	}
 692
 693	/* Filter out non FIP/FCoE frames here to free them faster */
 694	if (eh->h_proto != htons(ETH_P_ARP) &&
 695	    eh->h_proto != htons(ETH_P_IP) &&
 696	    eh->h_proto != htons(ETH_P_IPV6)) {
 697		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
 698			  "Dropping frame ethertype [0x%x] len [0x%x].\n",
 699			  eh->h_proto, skb->len);
 700		kfree_skb(skb);
 701		return 0;
 702	}
 703
 704	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
 705		  "Allowed frame ethertype [0x%x] len [0x%x].\n",
 706		  eh->h_proto, skb->len);
 707
 708	work = kzalloc(sizeof(*work), GFP_ATOMIC);
 709	if (!work) {
 710		QEDI_WARN(&qedi->dbg_ctx,
 711			  "Could not allocate work so dropping frame.\n");
 712		kfree_skb(skb);
 713		return 0;
 714	}
 715
 716	INIT_LIST_HEAD(&work->list);
 717	work->skb = skb;
 718
 719	if (skb_vlan_tag_present(skb))
 720		work->vlan_id = skb_vlan_tag_get(skb);
 721
 722	if (work->vlan_id)
 723		__vlan_insert_tag(work->skb, htons(ETH_P_8021Q), work->vlan_id);
 724
 725	spin_lock_bh(&qedi->ll2_lock);
 726	list_add_tail(&work->list, &qedi->ll2_skb_list);
 727	spin_unlock_bh(&qedi->ll2_lock);
 728
 729	wake_up_process(qedi->ll2_recv_thread);
 730
 731	return 0;
 732}
 733
 734/* map this skb to iscsiuio mmaped region */
 735static int qedi_ll2_process_skb(struct qedi_ctx *qedi, struct sk_buff *skb,
 736				u16 vlan_id)
 737{
 738	struct qedi_uio_dev *udev = NULL;
 739	struct qedi_uio_ctrl *uctrl = NULL;
 740	struct qedi_rx_bd rxbd;
 741	struct qedi_rx_bd *p_rxbd;
 742	u32 rx_bd_prod;
 743	void *pkt;
 744	int len = 0;
 745	u32 prod;
 746
 747	if (!qedi) {
 748		QEDI_ERR(NULL, "qedi is NULL\n");
 749		return -1;
 750	}
 751
 752	udev = qedi->udev;
 753	uctrl = udev->uctrl;
 754
 755	++uctrl->hw_rx_prod_cnt;
 756	prod = (uctrl->hw_rx_prod + 1) % RX_RING;
 757
 758	pkt = udev->rx_pkt + (prod * qedi_ll2_buf_size);
 759	len = min_t(u32, skb->len, (u32)qedi_ll2_buf_size);
 760	memcpy(pkt, skb->data, len);
 761
 762	memset(&rxbd, 0, sizeof(rxbd));
 763	rxbd.rx_pkt_index = prod;
 764	rxbd.rx_pkt_len = len;
 765	rxbd.vlan_id = vlan_id;
 766
 767	uctrl->hw_rx_bd_prod = (uctrl->hw_rx_bd_prod + 1) % QEDI_NUM_RX_BD;
 768	rx_bd_prod = uctrl->hw_rx_bd_prod;
 769	p_rxbd = (struct qedi_rx_bd *)udev->ll2_ring;
 770	p_rxbd += rx_bd_prod;
 771
 772	memcpy(p_rxbd, &rxbd, sizeof(rxbd));
 773
 774	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
 775		  "hw_rx_prod [%d] prod [%d] hw_rx_bd_prod [%d] rx_pkt_idx [%d] rx_len [%d].\n",
 776		  uctrl->hw_rx_prod, prod, uctrl->hw_rx_bd_prod,
 777		  rxbd.rx_pkt_index, rxbd.rx_pkt_len);
 778	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_LL2,
 779		  "host_rx_cons [%d] hw_rx_bd_cons [%d].\n",
 780		  uctrl->host_rx_cons, uctrl->host_rx_bd_cons);
 781
 782	uctrl->hw_rx_prod = prod;
 783
 784	/* notify the iscsiuio about new packet */
 785	uio_event_notify(&udev->qedi_uinfo);
 786
 787	return 0;
 788}
 789
 790static void qedi_ll2_free_skbs(struct qedi_ctx *qedi)
 791{
 792	struct skb_work_list *work, *work_tmp;
 793
 794	spin_lock_bh(&qedi->ll2_lock);
 795	list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list, list) {
 796		list_del(&work->list);
 797		kfree_skb(work->skb);
 798		kfree(work);
 799	}
 800	spin_unlock_bh(&qedi->ll2_lock);
 801}
 802
 803static int qedi_ll2_recv_thread(void *arg)
 804{
 805	struct qedi_ctx *qedi = (struct qedi_ctx *)arg;
 806	struct skb_work_list *work, *work_tmp;
 807
 808	set_user_nice(current, -20);
 809
 810	while (!kthread_should_stop()) {
 811		spin_lock_bh(&qedi->ll2_lock);
 812		list_for_each_entry_safe(work, work_tmp, &qedi->ll2_skb_list,
 813					 list) {
 814			list_del(&work->list);
 815			qedi_ll2_process_skb(qedi, work->skb, work->vlan_id);
 816			kfree_skb(work->skb);
 817			kfree(work);
 818		}
 819		set_current_state(TASK_INTERRUPTIBLE);
 820		spin_unlock_bh(&qedi->ll2_lock);
 821		schedule();
 822	}
 823
 824	__set_current_state(TASK_RUNNING);
 825	return 0;
 826}
 827
 828static int qedi_set_iscsi_pf_param(struct qedi_ctx *qedi)
 829{
 830	u8 num_sq_pages;
 831	u32 log_page_size;
 832	int rval = 0;
 833
 834
 835	num_sq_pages = (MAX_OUTSTANDING_TASKS_PER_CON * 8) / QEDI_PAGE_SIZE;
 836
 837	qedi->num_queues = MIN_NUM_CPUS_MSIX(qedi);
 838
 839	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
 840		  "Number of CQ count is %d\n", qedi->num_queues);
 841
 842	memset(&qedi->pf_params.iscsi_pf_params, 0,
 843	       sizeof(qedi->pf_params.iscsi_pf_params));
 844
 845	qedi->p_cpuq = dma_alloc_coherent(&qedi->pdev->dev,
 846			qedi->num_queues * sizeof(struct qedi_glbl_q_params),
 847			&qedi->hw_p_cpuq, GFP_KERNEL);
 848	if (!qedi->p_cpuq) {
 849		QEDI_ERR(&qedi->dbg_ctx, "dma_alloc_coherent fail\n");
 850		rval = -1;
 851		goto err_alloc_mem;
 852	}
 853
 854	rval = qedi_alloc_global_queues(qedi);
 855	if (rval) {
 856		QEDI_ERR(&qedi->dbg_ctx, "Global queue allocation failed.\n");
 857		rval = -1;
 858		goto err_alloc_mem;
 859	}
 860
 861	qedi->pf_params.iscsi_pf_params.num_cons = QEDI_MAX_ISCSI_CONNS_PER_HBA;
 862	qedi->pf_params.iscsi_pf_params.num_tasks = QEDI_MAX_ISCSI_TASK;
 863	qedi->pf_params.iscsi_pf_params.half_way_close_timeout = 10;
 864	qedi->pf_params.iscsi_pf_params.num_sq_pages_in_ring = num_sq_pages;
 865	qedi->pf_params.iscsi_pf_params.num_r2tq_pages_in_ring = num_sq_pages;
 866	qedi->pf_params.iscsi_pf_params.num_uhq_pages_in_ring = num_sq_pages;
 867	qedi->pf_params.iscsi_pf_params.num_queues = qedi->num_queues;
 868	qedi->pf_params.iscsi_pf_params.debug_mode = qedi_fw_debug;
 869	qedi->pf_params.iscsi_pf_params.two_msl_timer = QED_TWO_MSL_TIMER_DFLT;
 870	qedi->pf_params.iscsi_pf_params.tx_sws_timer = QED_TX_SWS_TIMER_DFLT;
 871	qedi->pf_params.iscsi_pf_params.max_fin_rt = 2;
 872
 873	for (log_page_size = 0 ; log_page_size < 32 ; log_page_size++) {
 874		if ((1 << log_page_size) == QEDI_PAGE_SIZE)
 875			break;
 876	}
 877	qedi->pf_params.iscsi_pf_params.log_page_size = log_page_size;
 878
 879	qedi->pf_params.iscsi_pf_params.glbl_q_params_addr =
 880							   (u64)qedi->hw_p_cpuq;
 881
 882	/* RQ BDQ initializations.
 883	 * rq_num_entries: suggested value for Initiator is 16 (4KB RQ)
 884	 * rqe_log_size: 8 for 256B RQE
 885	 */
 886	qedi->pf_params.iscsi_pf_params.rqe_log_size = 8;
 887	/* BDQ address and size */
 888	qedi->pf_params.iscsi_pf_params.bdq_pbl_base_addr[BDQ_ID_RQ] =
 889							qedi->bdq_pbl_list_dma;
 890	qedi->pf_params.iscsi_pf_params.bdq_pbl_num_entries[BDQ_ID_RQ] =
 891						qedi->bdq_pbl_list_num_entries;
 892	qedi->pf_params.iscsi_pf_params.rq_buffer_size = QEDI_BDQ_BUF_SIZE;
 893
 894	/* cq_num_entries: num_tasks + rq_num_entries */
 895	qedi->pf_params.iscsi_pf_params.cq_num_entries = 2048;
 896
 897	qedi->pf_params.iscsi_pf_params.gl_rq_pi = QEDI_PROTO_CQ_PROD_IDX;
 898	qedi->pf_params.iscsi_pf_params.gl_cmd_pi = 1;
 899
 900err_alloc_mem:
 901	return rval;
 902}
 903
 904/* Free DMA coherent memory for array of queue pointers we pass to qed */
 905static void qedi_free_iscsi_pf_param(struct qedi_ctx *qedi)
 906{
 907	size_t size = 0;
 908
 909	if (qedi->p_cpuq) {
 910		size = qedi->num_queues * sizeof(struct qedi_glbl_q_params);
 911		dma_free_coherent(&qedi->pdev->dev, size, qedi->p_cpuq,
 912				    qedi->hw_p_cpuq);
 913	}
 914
 915	qedi_free_global_queues(qedi);
 916
 917	kfree(qedi->global_queues);
 918}
 919
 920static void qedi_get_boot_tgt_info(struct nvm_iscsi_block *block,
 921				   struct qedi_boot_target *tgt, u8 index)
 922{
 923	u32 ipv6_en;
 924
 925	ipv6_en = !!(block->generic.ctrl_flags &
 926		     NVM_ISCSI_CFG_GEN_IPV6_ENABLED);
 927
 928	snprintf(tgt->iscsi_name, sizeof(tgt->iscsi_name), "%s",
 929		 block->target[index].target_name.byte);
 930
 931	tgt->ipv6_en = ipv6_en;
 932
 933	if (ipv6_en)
 934		snprintf(tgt->ip_addr, IPV6_LEN, "%pI6\n",
 935			 block->target[index].ipv6_addr.byte);
 936	else
 937		snprintf(tgt->ip_addr, IPV4_LEN, "%pI4\n",
 938			 block->target[index].ipv4_addr.byte);
 939}
 940
 941static int qedi_find_boot_info(struct qedi_ctx *qedi,
 942			       struct qed_mfw_tlv_iscsi *iscsi,
 943			       struct nvm_iscsi_block *block)
 944{
 945	struct qedi_boot_target *pri_tgt = NULL, *sec_tgt = NULL;
 946	u32 pri_ctrl_flags = 0, sec_ctrl_flags = 0, found = 0;
 947	struct iscsi_cls_session *cls_sess;
 948	struct iscsi_cls_conn *cls_conn;
 949	struct qedi_conn *qedi_conn;
 950	struct iscsi_session *sess;
 951	struct iscsi_conn *conn;
 952	char ep_ip_addr[64];
 953	int i, ret = 0;
 954
 955	pri_ctrl_flags = !!(block->target[0].ctrl_flags &
 956					NVM_ISCSI_CFG_TARGET_ENABLED);
 957	if (pri_ctrl_flags) {
 958		pri_tgt = kzalloc(sizeof(*pri_tgt), GFP_KERNEL);
 959		if (!pri_tgt)
 960			return -1;
 961		qedi_get_boot_tgt_info(block, pri_tgt, 0);
 962	}
 963
 964	sec_ctrl_flags = !!(block->target[1].ctrl_flags &
 965					NVM_ISCSI_CFG_TARGET_ENABLED);
 966	if (sec_ctrl_flags) {
 967		sec_tgt = kzalloc(sizeof(*sec_tgt), GFP_KERNEL);
 968		if (!sec_tgt) {
 969			ret = -1;
 970			goto free_tgt;
 971		}
 972		qedi_get_boot_tgt_info(block, sec_tgt, 1);
 973	}
 974
 975	for (i = 0; i < qedi->max_active_conns; i++) {
 976		qedi_conn = qedi_get_conn_from_id(qedi, i);
 977		if (!qedi_conn)
 978			continue;
 979
 980		if (qedi_conn->ep->ip_type == TCP_IPV4)
 981			snprintf(ep_ip_addr, IPV4_LEN, "%pI4\n",
 982				 qedi_conn->ep->dst_addr);
 983		else
 984			snprintf(ep_ip_addr, IPV6_LEN, "%pI6\n",
 985				 qedi_conn->ep->dst_addr);
 986
 987		cls_conn = qedi_conn->cls_conn;
 988		conn = cls_conn->dd_data;
 989		cls_sess = iscsi_conn_to_session(cls_conn);
 990		sess = cls_sess->dd_data;
 991
 992		if (!iscsi_is_session_online(cls_sess))
 993			continue;
 994
 995		if (!sess->targetname)
 996			continue;
 997
 998		if (pri_ctrl_flags) {
 999			if (!strcmp(pri_tgt->iscsi_name, sess->targetname) &&
1000			    !strcmp(pri_tgt->ip_addr, ep_ip_addr)) {
1001				found = 1;
1002				break;
1003			}
1004		}
1005
1006		if (sec_ctrl_flags) {
1007			if (!strcmp(sec_tgt->iscsi_name, sess->targetname) &&
1008			    !strcmp(sec_tgt->ip_addr, ep_ip_addr)) {
1009				found = 1;
1010				break;
1011			}
1012		}
1013	}
1014
1015	if (found) {
1016		if (conn->hdrdgst_en) {
1017			iscsi->header_digest_set = true;
1018			iscsi->header_digest = 1;
1019		}
1020
1021		if (conn->datadgst_en) {
1022			iscsi->data_digest_set = true;
1023			iscsi->data_digest = 1;
1024		}
1025		iscsi->boot_taget_portal_set = true;
1026		iscsi->boot_taget_portal = sess->tpgt;
1027
1028	} else {
1029		ret = -1;
1030	}
1031
1032	if (sec_ctrl_flags)
1033		kfree(sec_tgt);
1034free_tgt:
1035	if (pri_ctrl_flags)
1036		kfree(pri_tgt);
1037
1038	return ret;
1039}
1040
1041static void qedi_get_generic_tlv_data(void *dev, struct qed_generic_tlvs *data)
1042{
1043	struct qedi_ctx *qedi;
1044
1045	if (!dev) {
1046		QEDI_INFO(NULL, QEDI_LOG_EVT,
1047			  "dev is NULL so ignoring get_generic_tlv_data request.\n");
1048		return;
1049	}
1050	qedi = (struct qedi_ctx *)dev;
1051
1052	memset(data, 0, sizeof(struct qed_generic_tlvs));
1053	ether_addr_copy(data->mac[0], qedi->mac);
1054}
1055
1056/*
1057 * Protocol TLV handler
1058 */
1059static void qedi_get_protocol_tlv_data(void *dev, void *data)
1060{
1061	struct qed_mfw_tlv_iscsi *iscsi = data;
1062	struct qed_iscsi_stats *fw_iscsi_stats;
1063	struct nvm_iscsi_block *block = NULL;
1064	u32 chap_en = 0, mchap_en = 0;
1065	struct qedi_ctx *qedi = dev;
1066	int rval = 0;
1067
1068	fw_iscsi_stats = kmalloc(sizeof(*fw_iscsi_stats), GFP_KERNEL);
1069	if (!fw_iscsi_stats) {
1070		QEDI_ERR(&qedi->dbg_ctx,
1071			 "Could not allocate memory for fw_iscsi_stats.\n");
1072		goto exit_get_data;
1073	}
1074
1075	mutex_lock(&qedi->stats_lock);
1076	/* Query firmware for offload stats */
1077	qedi_ops->get_stats(qedi->cdev, fw_iscsi_stats);
1078	mutex_unlock(&qedi->stats_lock);
1079
1080	iscsi->rx_frames_set = true;
1081	iscsi->rx_frames = fw_iscsi_stats->iscsi_rx_packet_cnt;
1082	iscsi->rx_bytes_set = true;
1083	iscsi->rx_bytes = fw_iscsi_stats->iscsi_rx_bytes_cnt;
1084	iscsi->tx_frames_set = true;
1085	iscsi->tx_frames = fw_iscsi_stats->iscsi_tx_packet_cnt;
1086	iscsi->tx_bytes_set = true;
1087	iscsi->tx_bytes = fw_iscsi_stats->iscsi_tx_bytes_cnt;
1088	iscsi->frame_size_set = true;
1089	iscsi->frame_size = qedi->ll2_mtu;
1090	block = qedi_get_nvram_block(qedi);
1091	if (block) {
1092		chap_en = !!(block->generic.ctrl_flags &
1093			     NVM_ISCSI_CFG_GEN_CHAP_ENABLED);
1094		mchap_en = !!(block->generic.ctrl_flags &
1095			      NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED);
1096
1097		iscsi->auth_method_set = (chap_en || mchap_en) ? true : false;
1098		iscsi->auth_method = 1;
1099		if (chap_en)
1100			iscsi->auth_method = 2;
1101		if (mchap_en)
1102			iscsi->auth_method = 3;
1103
1104		iscsi->tx_desc_size_set = true;
1105		iscsi->tx_desc_size = QEDI_SQ_SIZE;
1106		iscsi->rx_desc_size_set = true;
1107		iscsi->rx_desc_size = QEDI_CQ_SIZE;
1108
1109		/* tpgt, hdr digest, data digest */
1110		rval = qedi_find_boot_info(qedi, iscsi, block);
1111		if (rval)
1112			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1113				  "Boot target not set");
1114	}
1115
1116	kfree(fw_iscsi_stats);
1117exit_get_data:
1118	return;
1119}
1120
1121void qedi_schedule_hw_err_handler(void *dev,
1122				  enum qed_hw_err_type err_type)
1123{
1124	struct qedi_ctx *qedi = (struct qedi_ctx *)dev;
1125	unsigned long override_flags = qedi_flags_override;
1126
1127	if (override_flags && test_bit(QEDI_ERR_OVERRIDE_EN, &override_flags))
1128		qedi->qedi_err_flags = qedi_flags_override;
1129
1130	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1131		  "HW error handler scheduled, err=%d err_flags=0x%x\n",
1132		  err_type, qedi->qedi_err_flags);
1133
1134	switch (err_type) {
1135	case QED_HW_ERR_FAN_FAIL:
1136		schedule_delayed_work(&qedi->board_disable_work, 0);
1137		break;
1138	case QED_HW_ERR_MFW_RESP_FAIL:
1139	case QED_HW_ERR_HW_ATTN:
1140	case QED_HW_ERR_DMAE_FAIL:
1141	case QED_HW_ERR_RAMROD_FAIL:
1142	case QED_HW_ERR_FW_ASSERT:
1143		/* Prevent HW attentions from being reasserted */
1144		if (test_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags))
1145			qedi_ops->common->attn_clr_enable(qedi->cdev, true);
1146
1147		if (err_type == QED_HW_ERR_RAMROD_FAIL &&
1148		    test_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags))
1149			qedi_ops->common->recovery_process(qedi->cdev);
1150
1151		break;
1152	default:
1153		break;
1154	}
1155}
1156
1157static void qedi_schedule_recovery_handler(void *dev)
1158{
1159	struct qedi_ctx *qedi = dev;
1160
1161	QEDI_ERR(&qedi->dbg_ctx, "Recovery handler scheduled.\n");
1162
1163	if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags))
1164		return;
1165
1166	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
1167
1168	schedule_delayed_work(&qedi->recovery_work, 0);
1169}
1170
1171static void qedi_set_conn_recovery(struct iscsi_cls_session *cls_session)
1172{
1173	struct iscsi_session *session = cls_session->dd_data;
1174	struct iscsi_conn *conn = session->leadconn;
1175	struct qedi_conn *qedi_conn = conn->dd_data;
1176
1177	qedi_start_conn_recovery(qedi_conn->qedi, qedi_conn);
1178}
1179
1180static void qedi_link_update(void *dev, struct qed_link_output *link)
1181{
1182	struct qedi_ctx *qedi = (struct qedi_ctx *)dev;
1183
1184	if (link->link_up) {
1185		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, "Link Up event.\n");
1186		atomic_set(&qedi->link_state, QEDI_LINK_UP);
1187	} else {
1188		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1189			  "Link Down event.\n");
1190		atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
1191		iscsi_host_for_each_session(qedi->shost, qedi_set_conn_recovery);
1192	}
1193}
1194
1195static struct qed_iscsi_cb_ops qedi_cb_ops = {
1196	{
1197		.link_update =		qedi_link_update,
1198		.schedule_recovery_handler = qedi_schedule_recovery_handler,
1199		.schedule_hw_err_handler = qedi_schedule_hw_err_handler,
1200		.get_protocol_tlv_data = qedi_get_protocol_tlv_data,
1201		.get_generic_tlv_data = qedi_get_generic_tlv_data,
1202	}
1203};
1204
1205static int qedi_queue_cqe(struct qedi_ctx *qedi, union iscsi_cqe *cqe,
1206			  u16 que_idx, struct qedi_percpu_s *p)
1207{
1208	struct qedi_work *qedi_work;
1209	struct qedi_conn *q_conn;
1210	struct qedi_cmd *qedi_cmd;
1211	u32 iscsi_cid;
1212	int rc = 0;
1213
1214	iscsi_cid  = cqe->cqe_common.conn_id;
1215	q_conn = qedi->cid_que.conn_cid_tbl[iscsi_cid];
1216	if (!q_conn) {
1217		QEDI_WARN(&qedi->dbg_ctx,
1218			  "Session no longer exists for cid=0x%x!!\n",
1219			  iscsi_cid);
1220		return -1;
1221	}
1222
1223	switch (cqe->cqe_common.cqe_type) {
1224	case ISCSI_CQE_TYPE_SOLICITED:
1225	case ISCSI_CQE_TYPE_SOLICITED_WITH_SENSE:
1226		qedi_cmd = qedi_get_cmd_from_tid(qedi, cqe->cqe_solicited.itid);
1227		if (!qedi_cmd) {
1228			rc = -1;
1229			break;
1230		}
1231		INIT_LIST_HEAD(&qedi_cmd->cqe_work.list);
1232		qedi_cmd->cqe_work.qedi = qedi;
1233		memcpy(&qedi_cmd->cqe_work.cqe, cqe, sizeof(union iscsi_cqe));
1234		qedi_cmd->cqe_work.que_idx = que_idx;
1235		qedi_cmd->cqe_work.is_solicited = true;
1236		list_add_tail(&qedi_cmd->cqe_work.list, &p->work_list);
1237		break;
1238	case ISCSI_CQE_TYPE_UNSOLICITED:
1239	case ISCSI_CQE_TYPE_DUMMY:
1240	case ISCSI_CQE_TYPE_TASK_CLEANUP:
1241		qedi_work = kzalloc(sizeof(*qedi_work), GFP_ATOMIC);
1242		if (!qedi_work) {
1243			rc = -1;
1244			break;
1245		}
1246		INIT_LIST_HEAD(&qedi_work->list);
1247		qedi_work->qedi = qedi;
1248		memcpy(&qedi_work->cqe, cqe, sizeof(union iscsi_cqe));
1249		qedi_work->que_idx = que_idx;
1250		qedi_work->is_solicited = false;
1251		list_add_tail(&qedi_work->list, &p->work_list);
1252		break;
1253	default:
1254		rc = -1;
1255		QEDI_ERR(&qedi->dbg_ctx, "FW Error cqe.\n");
1256	}
1257	return rc;
1258}
1259
1260static bool qedi_process_completions(struct qedi_fastpath *fp)
1261{
1262	struct qedi_ctx *qedi = fp->qedi;
1263	struct qed_sb_info *sb_info = fp->sb_info;
1264	struct status_block *sb = sb_info->sb_virt;
1265	struct qedi_percpu_s *p = NULL;
1266	struct global_queue *que;
1267	u16 prod_idx;
1268	unsigned long flags;
1269	union iscsi_cqe *cqe;
1270	int cpu;
1271	int ret;
1272
1273	/* Get the current firmware producer index */
1274	prod_idx = sb->pi_array[QEDI_PROTO_CQ_PROD_IDX];
1275
1276	if (prod_idx >= QEDI_CQ_SIZE)
1277		prod_idx = prod_idx % QEDI_CQ_SIZE;
1278
1279	que = qedi->global_queues[fp->sb_id];
1280	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
1281		  "Before: global queue=%p prod_idx=%d cons_idx=%d, sb_id=%d\n",
1282		  que, prod_idx, que->cq_cons_idx, fp->sb_id);
1283
1284	qedi->intr_cpu = fp->sb_id;
1285	cpu = smp_processor_id();
1286	p = &per_cpu(qedi_percpu, cpu);
1287
1288	if (unlikely(!p->iothread))
1289		WARN_ON(1);
1290
1291	spin_lock_irqsave(&p->p_work_lock, flags);
1292	while (que->cq_cons_idx != prod_idx) {
1293		cqe = &que->cq[que->cq_cons_idx];
1294
1295		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_IO,
1296			  "cqe=%p prod_idx=%d cons_idx=%d.\n",
1297			  cqe, prod_idx, que->cq_cons_idx);
1298
1299		ret = qedi_queue_cqe(qedi, cqe, fp->sb_id, p);
1300		if (ret)
1301			QEDI_WARN(&qedi->dbg_ctx,
1302				  "Dropping CQE 0x%x for cid=0x%x.\n",
1303				  que->cq_cons_idx, cqe->cqe_common.conn_id);
1304
1305		que->cq_cons_idx++;
1306		if (que->cq_cons_idx == QEDI_CQ_SIZE)
1307			que->cq_cons_idx = 0;
1308	}
1309	wake_up_process(p->iothread);
1310	spin_unlock_irqrestore(&p->p_work_lock, flags);
1311
1312	return true;
1313}
1314
1315static bool qedi_fp_has_work(struct qedi_fastpath *fp)
1316{
1317	struct qedi_ctx *qedi = fp->qedi;
1318	struct global_queue *que;
1319	struct qed_sb_info *sb_info = fp->sb_info;
1320	struct status_block *sb = sb_info->sb_virt;
1321	u16 prod_idx;
1322
1323	barrier();
1324
1325	/* Get the current firmware producer index */
1326	prod_idx = sb->pi_array[QEDI_PROTO_CQ_PROD_IDX];
1327
1328	/* Get the pointer to the global CQ this completion is on */
1329	que = qedi->global_queues[fp->sb_id];
1330
1331	/* prod idx wrap around uint16 */
1332	if (prod_idx >= QEDI_CQ_SIZE)
1333		prod_idx = prod_idx % QEDI_CQ_SIZE;
1334
1335	return (que->cq_cons_idx != prod_idx);
1336}
1337
1338/* MSI-X fastpath handler code */
1339static irqreturn_t qedi_msix_handler(int irq, void *dev_id)
1340{
1341	struct qedi_fastpath *fp = dev_id;
1342	struct qedi_ctx *qedi = fp->qedi;
1343	bool wake_io_thread = true;
1344
1345	qed_sb_ack(fp->sb_info, IGU_INT_DISABLE, 0);
1346
1347process_again:
1348	wake_io_thread = qedi_process_completions(fp);
1349	if (wake_io_thread) {
1350		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1351			  "process already running\n");
1352	}
1353
1354	if (!qedi_fp_has_work(fp))
1355		qed_sb_update_sb_idx(fp->sb_info);
1356
1357	/* Check for more work */
1358	rmb();
1359
1360	if (!qedi_fp_has_work(fp))
1361		qed_sb_ack(fp->sb_info, IGU_INT_ENABLE, 1);
1362	else
1363		goto process_again;
1364
1365	return IRQ_HANDLED;
1366}
1367
1368/* simd handler for MSI/INTa */
1369static void qedi_simd_int_handler(void *cookie)
1370{
1371	/* Cookie is qedi_ctx struct */
1372	struct qedi_ctx *qedi = (struct qedi_ctx *)cookie;
1373
1374	QEDI_WARN(&qedi->dbg_ctx, "qedi=%p.\n", qedi);
1375}
1376
1377#define QEDI_SIMD_HANDLER_NUM		0
1378static void qedi_sync_free_irqs(struct qedi_ctx *qedi)
1379{
1380	int i;
1381	u16 idx;
1382
1383	if (qedi->int_info.msix_cnt) {
1384		for (i = 0; i < qedi->int_info.used_cnt; i++) {
1385			idx = i * qedi->dev_info.common.num_hwfns +
1386			qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
1387
1388			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1389				  "Freeing IRQ #%d vector_idx=%d.\n", i, idx);
1390
1391			synchronize_irq(qedi->int_info.msix[idx].vector);
1392			irq_set_affinity_hint(qedi->int_info.msix[idx].vector,
1393					      NULL);
1394			free_irq(qedi->int_info.msix[idx].vector,
1395				 &qedi->fp_array[i]);
1396		}
1397	} else {
1398		qedi_ops->common->simd_handler_clean(qedi->cdev,
1399						     QEDI_SIMD_HANDLER_NUM);
1400	}
1401
1402	qedi->int_info.used_cnt = 0;
1403	qedi_ops->common->set_fp_int(qedi->cdev, 0);
1404}
1405
1406static int qedi_request_msix_irq(struct qedi_ctx *qedi)
1407{
1408	int i, rc, cpu;
1409	u16 idx;
1410
1411	cpu = cpumask_first(cpu_online_mask);
1412	for (i = 0; i < qedi->msix_count; i++) {
1413		idx = i * qedi->dev_info.common.num_hwfns +
1414			  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev);
1415
1416		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1417			  "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",
1418			  qedi->dev_info.common.num_hwfns,
1419			  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));
1420
1421		rc = request_irq(qedi->int_info.msix[idx].vector,
1422				 qedi_msix_handler, 0, "qedi",
1423				 &qedi->fp_array[i]);
1424		if (rc) {
1425			QEDI_WARN(&qedi->dbg_ctx, "request_irq failed.\n");
1426			qedi_sync_free_irqs(qedi);
1427			return rc;
1428		}
1429		qedi->int_info.used_cnt++;
1430		rc = irq_set_affinity_hint(qedi->int_info.msix[idx].vector,
1431					   get_cpu_mask(cpu));
1432		cpu = cpumask_next(cpu, cpu_online_mask);
1433	}
1434
1435	return 0;
1436}
1437
1438static int qedi_setup_int(struct qedi_ctx *qedi)
1439{
1440	int rc = 0;
1441
1442	rc = qedi_ops->common->set_fp_int(qedi->cdev, qedi->num_queues);
1443	if (rc < 0)
1444		goto exit_setup_int;
1445
1446	qedi->msix_count = rc;
1447
1448	rc = qedi_ops->common->get_fp_int(qedi->cdev, &qedi->int_info);
1449	if (rc)
1450		goto exit_setup_int;
1451
1452	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1453		  "Number of msix_cnt = 0x%x num of cpus = 0x%x\n",
1454		   qedi->int_info.msix_cnt, num_online_cpus());
1455
1456	if (qedi->int_info.msix_cnt) {
1457		rc = qedi_request_msix_irq(qedi);
1458		goto exit_setup_int;
1459	} else {
1460		qedi_ops->common->simd_handler_config(qedi->cdev, &qedi,
1461						      QEDI_SIMD_HANDLER_NUM,
1462						      qedi_simd_int_handler);
1463		qedi->int_info.used_cnt = 1;
1464	}
1465
1466exit_setup_int:
1467	return rc;
1468}
1469
1470static void qedi_free_nvm_iscsi_cfg(struct qedi_ctx *qedi)
1471{
1472	if (qedi->iscsi_image)
1473		dma_free_coherent(&qedi->pdev->dev,
1474				  sizeof(struct qedi_nvm_iscsi_image),
1475				  qedi->iscsi_image, qedi->nvm_buf_dma);
1476}
1477
1478static int qedi_alloc_nvm_iscsi_cfg(struct qedi_ctx *qedi)
1479{
1480	qedi->iscsi_image = dma_alloc_coherent(&qedi->pdev->dev,
1481					       sizeof(struct qedi_nvm_iscsi_image),
1482					       &qedi->nvm_buf_dma, GFP_KERNEL);
1483	if (!qedi->iscsi_image) {
1484		QEDI_ERR(&qedi->dbg_ctx, "Could not allocate NVM BUF.\n");
1485		return -ENOMEM;
1486	}
1487	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
1488		  "NVM BUF addr=0x%p dma=0x%llx.\n", qedi->iscsi_image,
1489		  qedi->nvm_buf_dma);
1490
1491	return 0;
1492}
1493
1494static void qedi_free_bdq(struct qedi_ctx *qedi)
1495{
1496	int i;
1497
1498	if (qedi->bdq_pbl_list)
1499		dma_free_coherent(&qedi->pdev->dev, QEDI_PAGE_SIZE,
1500				  qedi->bdq_pbl_list, qedi->bdq_pbl_list_dma);
1501
1502	if (qedi->bdq_pbl)
1503		dma_free_coherent(&qedi->pdev->dev, qedi->bdq_pbl_mem_size,
1504				  qedi->bdq_pbl, qedi->bdq_pbl_dma);
1505
1506	for (i = 0; i < QEDI_BDQ_NUM; i++) {
1507		if (qedi->bdq[i].buf_addr) {
1508			dma_free_coherent(&qedi->pdev->dev, QEDI_BDQ_BUF_SIZE,
1509					  qedi->bdq[i].buf_addr,
1510					  qedi->bdq[i].buf_dma);
1511		}
1512	}
1513}
1514
1515static void qedi_free_global_queues(struct qedi_ctx *qedi)
1516{
1517	int i;
1518	struct global_queue **gl = qedi->global_queues;
1519
1520	for (i = 0; i < qedi->num_queues; i++) {
1521		if (!gl[i])
1522			continue;
1523
1524		if (gl[i]->cq)
1525			dma_free_coherent(&qedi->pdev->dev, gl[i]->cq_mem_size,
1526					  gl[i]->cq, gl[i]->cq_dma);
1527		if (gl[i]->cq_pbl)
1528			dma_free_coherent(&qedi->pdev->dev, gl[i]->cq_pbl_size,
1529					  gl[i]->cq_pbl, gl[i]->cq_pbl_dma);
1530
1531		kfree(gl[i]);
1532	}
1533	qedi_free_bdq(qedi);
1534	qedi_free_nvm_iscsi_cfg(qedi);
1535}
1536
1537static int qedi_alloc_bdq(struct qedi_ctx *qedi)
1538{
1539	int i;
1540	struct scsi_bd *pbl;
1541	u64 *list;
1542
1543	/* Alloc dma memory for BDQ buffers */
1544	for (i = 0; i < QEDI_BDQ_NUM; i++) {
1545		qedi->bdq[i].buf_addr =
1546				dma_alloc_coherent(&qedi->pdev->dev,
1547						   QEDI_BDQ_BUF_SIZE,
1548						   &qedi->bdq[i].buf_dma,
1549						   GFP_KERNEL);
1550		if (!qedi->bdq[i].buf_addr) {
1551			QEDI_ERR(&qedi->dbg_ctx,
1552				 "Could not allocate BDQ buffer %d.\n", i);
1553			return -ENOMEM;
1554		}
1555	}
1556
1557	/* Alloc dma memory for BDQ page buffer list */
1558	qedi->bdq_pbl_mem_size = QEDI_BDQ_NUM * sizeof(struct scsi_bd);
1559	qedi->bdq_pbl_mem_size = ALIGN(qedi->bdq_pbl_mem_size, QEDI_PAGE_SIZE);
1560	qedi->rq_num_entries = qedi->bdq_pbl_mem_size / sizeof(struct scsi_bd);
1561
1562	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN, "rq_num_entries = %d.\n",
1563		  qedi->rq_num_entries);
1564
1565	qedi->bdq_pbl = dma_alloc_coherent(&qedi->pdev->dev,
1566					   qedi->bdq_pbl_mem_size,
1567					   &qedi->bdq_pbl_dma, GFP_KERNEL);
1568	if (!qedi->bdq_pbl) {
1569		QEDI_ERR(&qedi->dbg_ctx, "Could not allocate BDQ PBL.\n");
1570		return -ENOMEM;
1571	}
1572
1573	/*
1574	 * Populate BDQ PBL with physical and virtual address of individual
1575	 * BDQ buffers
1576	 */
1577	pbl = (struct scsi_bd  *)qedi->bdq_pbl;
1578	for (i = 0; i < QEDI_BDQ_NUM; i++) {
1579		pbl->address.hi =
1580				cpu_to_le32(QEDI_U64_HI(qedi->bdq[i].buf_dma));
1581		pbl->address.lo =
1582				cpu_to_le32(QEDI_U64_LO(qedi->bdq[i].buf_dma));
1583		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1584			  "pbl [0x%p] pbl->address hi [0x%llx] lo [0x%llx], idx [%d]\n",
1585			  pbl, pbl->address.hi, pbl->address.lo, i);
1586		pbl->opaque.iscsi_opaque.reserved_zero[0] = 0;
1587		pbl->opaque.iscsi_opaque.reserved_zero[1] = 0;
1588		pbl->opaque.iscsi_opaque.reserved_zero[2] = 0;
1589		pbl->opaque.iscsi_opaque.opaque = cpu_to_le16(i);
1590		pbl++;
1591	}
1592
1593	/* Allocate list of PBL pages */
1594	qedi->bdq_pbl_list = dma_alloc_coherent(&qedi->pdev->dev,
1595						QEDI_PAGE_SIZE,
1596						&qedi->bdq_pbl_list_dma,
1597						GFP_KERNEL);
1598	if (!qedi->bdq_pbl_list) {
1599		QEDI_ERR(&qedi->dbg_ctx,
1600			 "Could not allocate list of PBL pages.\n");
1601		return -ENOMEM;
1602	}
1603
1604	/*
1605	 * Now populate PBL list with pages that contain pointers to the
1606	 * individual buffers.
1607	 */
1608	qedi->bdq_pbl_list_num_entries = qedi->bdq_pbl_mem_size /
1609					 QEDI_PAGE_SIZE;
1610	list = (u64 *)qedi->bdq_pbl_list;
1611	for (i = 0; i < qedi->bdq_pbl_list_num_entries; i++) {
1612		*list = qedi->bdq_pbl_dma;
1613		list++;
1614	}
1615
1616	return 0;
1617}
1618
1619static int qedi_alloc_global_queues(struct qedi_ctx *qedi)
1620{
1621	u32 *list;
1622	int i;
1623	int status;
1624	u32 *pbl;
1625	dma_addr_t page;
1626	int num_pages;
1627
1628	/*
1629	 * Number of global queues (CQ / RQ). This should
1630	 * be <= number of available MSIX vectors for the PF
1631	 */
1632	if (!qedi->num_queues) {
1633		QEDI_ERR(&qedi->dbg_ctx, "No MSI-X vectors available!\n");
1634		return -ENOMEM;
1635	}
1636
1637	/* Make sure we allocated the PBL that will contain the physical
1638	 * addresses of our queues
1639	 */
1640	if (!qedi->p_cpuq) {
1641		status = -EINVAL;
1642		goto mem_alloc_failure;
1643	}
1644
1645	qedi->global_queues = kzalloc((sizeof(struct global_queue *) *
1646				       qedi->num_queues), GFP_KERNEL);
1647	if (!qedi->global_queues) {
1648		QEDI_ERR(&qedi->dbg_ctx,
1649			 "Unable to allocate global queues array ptr memory\n");
1650		return -ENOMEM;
1651	}
1652	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
1653		  "qedi->global_queues=%p.\n", qedi->global_queues);
1654
1655	/* Allocate DMA coherent buffers for BDQ */
1656	status = qedi_alloc_bdq(qedi);
1657	if (status)
1658		goto mem_alloc_failure;
1659
1660	/* Allocate DMA coherent buffers for NVM_ISCSI_CFG */
1661	status = qedi_alloc_nvm_iscsi_cfg(qedi);
1662	if (status)
1663		goto mem_alloc_failure;
1664
1665	/* Allocate a CQ and an associated PBL for each MSI-X
1666	 * vector.
1667	 */
1668	for (i = 0; i < qedi->num_queues; i++) {
1669		qedi->global_queues[i] =
1670					kzalloc(sizeof(*qedi->global_queues[0]),
1671						GFP_KERNEL);
1672		if (!qedi->global_queues[i]) {
1673			QEDI_ERR(&qedi->dbg_ctx,
1674				 "Unable to allocation global queue %d.\n", i);
1675			status = -ENOMEM;
1676			goto mem_alloc_failure;
1677		}
1678
1679		qedi->global_queues[i]->cq_mem_size =
1680		    (QEDI_CQ_SIZE + 8) * sizeof(union iscsi_cqe);
1681		qedi->global_queues[i]->cq_mem_size =
1682		    (qedi->global_queues[i]->cq_mem_size +
1683		    (QEDI_PAGE_SIZE - 1));
1684
1685		qedi->global_queues[i]->cq_pbl_size =
1686		    (qedi->global_queues[i]->cq_mem_size /
1687		    QEDI_PAGE_SIZE) * sizeof(void *);
1688		qedi->global_queues[i]->cq_pbl_size =
1689		    (qedi->global_queues[i]->cq_pbl_size +
1690		    (QEDI_PAGE_SIZE - 1));
1691
1692		qedi->global_queues[i]->cq = dma_alloc_coherent(&qedi->pdev->dev,
1693								qedi->global_queues[i]->cq_mem_size,
1694								&qedi->global_queues[i]->cq_dma,
1695								GFP_KERNEL);
1696
1697		if (!qedi->global_queues[i]->cq) {
1698			QEDI_WARN(&qedi->dbg_ctx,
1699				  "Could not allocate cq.\n");
1700			status = -ENOMEM;
1701			goto mem_alloc_failure;
1702		}
1703		qedi->global_queues[i]->cq_pbl = dma_alloc_coherent(&qedi->pdev->dev,
1704								    qedi->global_queues[i]->cq_pbl_size,
1705								    &qedi->global_queues[i]->cq_pbl_dma,
1706								    GFP_KERNEL);
1707
1708		if (!qedi->global_queues[i]->cq_pbl) {
1709			QEDI_WARN(&qedi->dbg_ctx,
1710				  "Could not allocate cq PBL.\n");
1711			status = -ENOMEM;
1712			goto mem_alloc_failure;
1713		}
1714
1715		/* Create PBL */
1716		num_pages = qedi->global_queues[i]->cq_mem_size /
1717		    QEDI_PAGE_SIZE;
1718		page = qedi->global_queues[i]->cq_dma;
1719		pbl = (u32 *)qedi->global_queues[i]->cq_pbl;
1720
1721		while (num_pages--) {
1722			*pbl = (u32)page;
1723			pbl++;
1724			*pbl = (u32)((u64)page >> 32);
1725			pbl++;
1726			page += QEDI_PAGE_SIZE;
1727		}
1728	}
1729
1730	list = (u32 *)qedi->p_cpuq;
1731
1732	/*
1733	 * The list is built as follows: CQ#0 PBL pointer, RQ#0 PBL pointer,
1734	 * CQ#1 PBL pointer, RQ#1 PBL pointer, etc.  Each PBL pointer points
1735	 * to the physical address which contains an array of pointers to the
1736	 * physical addresses of the specific queue pages.
1737	 */
1738	for (i = 0; i < qedi->num_queues; i++) {
1739		*list = (u32)qedi->global_queues[i]->cq_pbl_dma;
1740		list++;
1741		*list = (u32)((u64)qedi->global_queues[i]->cq_pbl_dma >> 32);
1742		list++;
1743
1744		*list = (u32)0;
1745		list++;
1746		*list = (u32)((u64)0 >> 32);
1747		list++;
1748	}
1749
1750	return 0;
1751
1752mem_alloc_failure:
1753	qedi_free_global_queues(qedi);
1754	return status;
1755}
1756
1757int qedi_alloc_sq(struct qedi_ctx *qedi, struct qedi_endpoint *ep)
1758{
1759	int rval = 0;
1760	u32 *pbl;
1761	dma_addr_t page;
1762	int num_pages;
1763
1764	if (!ep)
1765		return -EIO;
1766
1767	/* Calculate appropriate queue and PBL sizes */
1768	ep->sq_mem_size = QEDI_SQ_SIZE * sizeof(struct iscsi_wqe);
1769	ep->sq_mem_size += QEDI_PAGE_SIZE - 1;
1770
1771	ep->sq_pbl_size = (ep->sq_mem_size / QEDI_PAGE_SIZE) * sizeof(void *);
1772	ep->sq_pbl_size = ep->sq_pbl_size + QEDI_PAGE_SIZE;
1773
1774	ep->sq = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_mem_size,
1775				    &ep->sq_dma, GFP_KERNEL);
1776	if (!ep->sq) {
1777		QEDI_WARN(&qedi->dbg_ctx,
1778			  "Could not allocate send queue.\n");
1779		rval = -ENOMEM;
1780		goto out;
1781	}
1782	ep->sq_pbl = dma_alloc_coherent(&qedi->pdev->dev, ep->sq_pbl_size,
1783					&ep->sq_pbl_dma, GFP_KERNEL);
1784	if (!ep->sq_pbl) {
1785		QEDI_WARN(&qedi->dbg_ctx,
1786			  "Could not allocate send queue PBL.\n");
1787		rval = -ENOMEM;
1788		goto out_free_sq;
1789	}
1790
1791	/* Create PBL */
1792	num_pages = ep->sq_mem_size / QEDI_PAGE_SIZE;
1793	page = ep->sq_dma;
1794	pbl = (u32 *)ep->sq_pbl;
1795
1796	while (num_pages--) {
1797		*pbl = (u32)page;
1798		pbl++;
1799		*pbl = (u32)((u64)page >> 32);
1800		pbl++;
1801		page += QEDI_PAGE_SIZE;
1802	}
1803
1804	return rval;
1805
1806out_free_sq:
1807	dma_free_coherent(&qedi->pdev->dev, ep->sq_mem_size, ep->sq,
1808			  ep->sq_dma);
1809out:
1810	return rval;
1811}
1812
1813void qedi_free_sq(struct qedi_ctx *qedi, struct qedi_endpoint *ep)
1814{
1815	if (ep->sq_pbl)
1816		dma_free_coherent(&qedi->pdev->dev, ep->sq_pbl_size, ep->sq_pbl,
1817				  ep->sq_pbl_dma);
1818	if (ep->sq)
1819		dma_free_coherent(&qedi->pdev->dev, ep->sq_mem_size, ep->sq,
1820				  ep->sq_dma);
1821}
1822
1823int qedi_get_task_idx(struct qedi_ctx *qedi)
1824{
1825	s16 tmp_idx;
1826
1827again:
1828	tmp_idx = find_first_zero_bit(qedi->task_idx_map,
1829				      MAX_ISCSI_TASK_ENTRIES);
1830
1831	if (tmp_idx >= MAX_ISCSI_TASK_ENTRIES) {
1832		QEDI_ERR(&qedi->dbg_ctx, "FW task context pool is full.\n");
1833		tmp_idx = -1;
1834		goto err_idx;
1835	}
1836
1837	if (test_and_set_bit(tmp_idx, qedi->task_idx_map))
1838		goto again;
1839
1840err_idx:
1841	return tmp_idx;
1842}
1843
1844void qedi_clear_task_idx(struct qedi_ctx *qedi, int idx)
1845{
1846	if (!test_and_clear_bit(idx, qedi->task_idx_map))
1847		QEDI_ERR(&qedi->dbg_ctx,
1848			 "FW task context, already cleared, tid=0x%x\n", idx);
1849}
1850
1851void qedi_update_itt_map(struct qedi_ctx *qedi, u32 tid, u32 proto_itt,
1852			 struct qedi_cmd *cmd)
1853{
1854	qedi->itt_map[tid].itt = proto_itt;
1855	qedi->itt_map[tid].p_cmd = cmd;
1856
1857	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1858		  "update itt map tid=0x%x, with proto itt=0x%x\n", tid,
1859		  qedi->itt_map[tid].itt);
1860}
1861
1862void qedi_get_task_tid(struct qedi_ctx *qedi, u32 itt, s16 *tid)
1863{
1864	u16 i;
1865
1866	for (i = 0; i < MAX_ISCSI_TASK_ENTRIES; i++) {
1867		if (qedi->itt_map[i].itt == itt) {
1868			*tid = i;
1869			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1870				  "Ref itt=0x%x, found at tid=0x%x\n",
1871				  itt, *tid);
1872			return;
1873		}
1874	}
1875
1876	WARN_ON(1);
1877}
1878
1879void qedi_get_proto_itt(struct qedi_ctx *qedi, u32 tid, u32 *proto_itt)
1880{
1881	*proto_itt = qedi->itt_map[tid].itt;
1882	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_CONN,
1883		  "Get itt map tid [0x%x with proto itt[0x%x]",
1884		  tid, *proto_itt);
1885}
1886
1887struct qedi_cmd *qedi_get_cmd_from_tid(struct qedi_ctx *qedi, u32 tid)
1888{
1889	struct qedi_cmd *cmd = NULL;
1890
1891	if (tid >= MAX_ISCSI_TASK_ENTRIES)
1892		return NULL;
1893
1894	cmd = qedi->itt_map[tid].p_cmd;
1895	if (cmd->task_id != tid)
1896		return NULL;
1897
1898	qedi->itt_map[tid].p_cmd = NULL;
1899
1900	return cmd;
1901}
1902
1903static int qedi_alloc_itt(struct qedi_ctx *qedi)
1904{
1905	qedi->itt_map = kcalloc(MAX_ISCSI_TASK_ENTRIES,
1906				sizeof(struct qedi_itt_map), GFP_KERNEL);
1907	if (!qedi->itt_map) {
1908		QEDI_ERR(&qedi->dbg_ctx,
1909			 "Unable to allocate itt map array memory\n");
1910		return -ENOMEM;
1911	}
1912	return 0;
1913}
1914
1915static void qedi_free_itt(struct qedi_ctx *qedi)
1916{
1917	kfree(qedi->itt_map);
1918}
1919
1920static struct qed_ll2_cb_ops qedi_ll2_cb_ops = {
1921	.rx_cb = qedi_ll2_rx,
1922	.tx_cb = NULL,
1923};
1924
1925static int qedi_percpu_io_thread(void *arg)
1926{
1927	struct qedi_percpu_s *p = arg;
1928	struct qedi_work *work, *tmp;
1929	unsigned long flags;
1930	LIST_HEAD(work_list);
1931
1932	set_user_nice(current, -20);
1933
1934	while (!kthread_should_stop()) {
1935		spin_lock_irqsave(&p->p_work_lock, flags);
1936		while (!list_empty(&p->work_list)) {
1937			list_splice_init(&p->work_list, &work_list);
1938			spin_unlock_irqrestore(&p->p_work_lock, flags);
1939
1940			list_for_each_entry_safe(work, tmp, &work_list, list) {
1941				list_del_init(&work->list);
1942				qedi_fp_process_cqes(work);
1943				if (!work->is_solicited)
1944					kfree(work);
1945			}
1946			cond_resched();
1947			spin_lock_irqsave(&p->p_work_lock, flags);
1948		}
1949		set_current_state(TASK_INTERRUPTIBLE);
1950		spin_unlock_irqrestore(&p->p_work_lock, flags);
1951		schedule();
1952	}
1953	__set_current_state(TASK_RUNNING);
1954
1955	return 0;
1956}
1957
1958static int qedi_cpu_online(unsigned int cpu)
1959{
1960	struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);
1961	struct task_struct *thread;
1962
1963	thread = kthread_create_on_node(qedi_percpu_io_thread, (void *)p,
1964					cpu_to_node(cpu),
1965					"qedi_thread/%d", cpu);
1966	if (IS_ERR(thread))
1967		return PTR_ERR(thread);
1968
1969	kthread_bind(thread, cpu);
1970	p->iothread = thread;
1971	wake_up_process(thread);
1972	return 0;
1973}
1974
1975static int qedi_cpu_offline(unsigned int cpu)
1976{
1977	struct qedi_percpu_s *p = this_cpu_ptr(&qedi_percpu);
1978	struct qedi_work *work, *tmp;
1979	struct task_struct *thread;
1980	unsigned long flags;
1981
1982	spin_lock_irqsave(&p->p_work_lock, flags);
1983	thread = p->iothread;
1984	p->iothread = NULL;
1985
1986	list_for_each_entry_safe(work, tmp, &p->work_list, list) {
1987		list_del_init(&work->list);
1988		qedi_fp_process_cqes(work);
1989		if (!work->is_solicited)
1990			kfree(work);
1991	}
1992
1993	spin_unlock_irqrestore(&p->p_work_lock, flags);
1994	if (thread)
1995		kthread_stop(thread);
1996	return 0;
1997}
1998
1999void qedi_reset_host_mtu(struct qedi_ctx *qedi, u16 mtu)
2000{
2001	struct qed_ll2_params params;
2002
2003	qedi_recover_all_conns(qedi);
2004
2005	qedi_ops->ll2->stop(qedi->cdev);
2006	qedi_ll2_free_skbs(qedi);
2007
2008	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO, "old MTU %u, new MTU %u\n",
2009		  qedi->ll2_mtu, mtu);
2010	memset(&params, 0, sizeof(params));
2011	qedi->ll2_mtu = mtu;
2012	params.mtu = qedi->ll2_mtu + IPV6_HDR_LEN + TCP_HDR_LEN;
2013	params.drop_ttl0_packets = 0;
2014	params.rx_vlan_stripping = 1;
2015	ether_addr_copy(params.ll2_mac_address, qedi->dev_info.common.hw_mac);
2016	qedi_ops->ll2->start(qedi->cdev, &params);
2017}
2018
2019/*
2020 * qedi_get_nvram_block: - Scan through the iSCSI NVRAM block (while accounting
2021 * for gaps) for the matching absolute-pf-id of the QEDI device.
2022 */
2023static struct nvm_iscsi_block *
2024qedi_get_nvram_block(struct qedi_ctx *qedi)
2025{
2026	int i;
2027	u8 pf;
2028	u32 flags;
2029	struct nvm_iscsi_block *block;
2030
2031	pf = qedi->dev_info.common.abs_pf_id;
2032	block = &qedi->iscsi_image->iscsi_cfg.block[0];
2033	for (i = 0; i < NUM_OF_ISCSI_PF_SUPPORTED; i++, block++) {
2034		flags = ((block->id) & NVM_ISCSI_CFG_BLK_CTRL_FLAG_MASK) >>
2035			NVM_ISCSI_CFG_BLK_CTRL_FLAG_OFFSET;
2036		if (flags & (NVM_ISCSI_CFG_BLK_CTRL_FLAG_IS_NOT_EMPTY |
2037				NVM_ISCSI_CFG_BLK_CTRL_FLAG_PF_MAPPED) &&
2038			(pf == (block->id & NVM_ISCSI_CFG_BLK_MAPPED_PF_ID_MASK)
2039				>> NVM_ISCSI_CFG_BLK_MAPPED_PF_ID_OFFSET))
2040			return block;
2041	}
2042	return NULL;
2043}
2044
2045static ssize_t qedi_show_boot_eth_info(void *data, int type, char *buf)
2046{
2047	struct qedi_ctx *qedi = data;
2048	struct nvm_iscsi_initiator *initiator;
2049	int rc = 1;
2050	u32 ipv6_en, dhcp_en, ip_len;
2051	struct nvm_iscsi_block *block;
2052	char *fmt, *ip, *sub, *gw;
2053
2054	block = qedi_get_nvram_block(qedi);
2055	if (!block)
2056		return 0;
2057
2058	initiator = &block->initiator;
2059	ipv6_en = block->generic.ctrl_flags &
2060		  NVM_ISCSI_CFG_GEN_IPV6_ENABLED;
2061	dhcp_en = block->generic.ctrl_flags &
2062		  NVM_ISCSI_CFG_GEN_DHCP_TCPIP_CONFIG_ENABLED;
2063	/* Static IP assignments. */
2064	fmt = ipv6_en ? "%pI6\n" : "%pI4\n";
2065	ip = ipv6_en ? initiator->ipv6.addr.byte : initiator->ipv4.addr.byte;
2066	ip_len = ipv6_en ? IPV6_LEN : IPV4_LEN;
2067	sub = ipv6_en ? initiator->ipv6.subnet_mask.byte :
2068	      initiator->ipv4.subnet_mask.byte;
2069	gw = ipv6_en ? initiator->ipv6.gateway.byte :
2070	     initiator->ipv4.gateway.byte;
2071	/* DHCP IP adjustments. */
2072	fmt = dhcp_en ? "%s\n" : fmt;
2073	if (dhcp_en) {
2074		ip = ipv6_en ? "0::0" : "0.0.0.0";
2075		sub = ip;
2076		gw = ip;
2077		ip_len = ipv6_en ? 5 : 8;
2078	}
2079
2080	switch (type) {
2081	case ISCSI_BOOT_ETH_IP_ADDR:
2082		rc = snprintf(buf, ip_len, fmt, ip);
2083		break;
2084	case ISCSI_BOOT_ETH_SUBNET_MASK:
2085		rc = snprintf(buf, ip_len, fmt, sub);
2086		break;
2087	case ISCSI_BOOT_ETH_GATEWAY:
2088		rc = snprintf(buf, ip_len, fmt, gw);
2089		break;
2090	case ISCSI_BOOT_ETH_FLAGS:
2091		rc = snprintf(buf, 3, "%d\n", (char)SYSFS_FLAG_FW_SEL_BOOT);
2092		break;
2093	case ISCSI_BOOT_ETH_INDEX:
2094		rc = snprintf(buf, 3, "0\n");
2095		break;
2096	case ISCSI_BOOT_ETH_MAC:
2097		rc = sysfs_format_mac(buf, qedi->mac, ETH_ALEN);
2098		break;
2099	case ISCSI_BOOT_ETH_VLAN:
2100		rc = snprintf(buf, 12, "%d\n",
2101			      GET_FIELD2(initiator->generic_cont0,
2102					 NVM_ISCSI_CFG_INITIATOR_VLAN));
2103		break;
2104	case ISCSI_BOOT_ETH_ORIGIN:
2105		if (dhcp_en)
2106			rc = snprintf(buf, 3, "3\n");
2107		break;
2108	default:
2109		rc = 0;
2110		break;
2111	}
2112
2113	return rc;
2114}
2115
2116static umode_t qedi_eth_get_attr_visibility(void *data, int type)
2117{
2118	int rc = 1;
2119
2120	switch (type) {
2121	case ISCSI_BOOT_ETH_FLAGS:
2122	case ISCSI_BOOT_ETH_MAC:
2123	case ISCSI_BOOT_ETH_INDEX:
2124	case ISCSI_BOOT_ETH_IP_ADDR:
2125	case ISCSI_BOOT_ETH_SUBNET_MASK:
2126	case ISCSI_BOOT_ETH_GATEWAY:
2127	case ISCSI_BOOT_ETH_ORIGIN:
2128	case ISCSI_BOOT_ETH_VLAN:
2129		rc = 0444;
2130		break;
2131	default:
2132		rc = 0;
2133		break;
2134	}
2135	return rc;
2136}
2137
2138static ssize_t qedi_show_boot_ini_info(void *data, int type, char *buf)
2139{
2140	struct qedi_ctx *qedi = data;
2141	struct nvm_iscsi_initiator *initiator;
2142	int rc;
2143	struct nvm_iscsi_block *block;
2144
2145	block = qedi_get_nvram_block(qedi);
2146	if (!block)
2147		return 0;
2148
2149	initiator = &block->initiator;
2150
2151	switch (type) {
2152	case ISCSI_BOOT_INI_INITIATOR_NAME:
2153		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN,
2154			     initiator->initiator_name.byte);
2155		break;
2156	default:
2157		rc = 0;
2158		break;
2159	}
2160	return rc;
2161}
2162
2163static umode_t qedi_ini_get_attr_visibility(void *data, int type)
2164{
2165	int rc;
2166
2167	switch (type) {
2168	case ISCSI_BOOT_INI_INITIATOR_NAME:
2169		rc = 0444;
2170		break;
2171	default:
2172		rc = 0;
2173		break;
2174	}
2175	return rc;
2176}
2177
2178static ssize_t
2179qedi_show_boot_tgt_info(struct qedi_ctx *qedi, int type,
2180			char *buf, enum qedi_nvm_tgts idx)
2181{
2182	int rc = 1;
2183	u32 ctrl_flags, ipv6_en, chap_en, mchap_en, ip_len;
2184	struct nvm_iscsi_block *block;
2185	char *chap_name, *chap_secret;
2186	char *mchap_name, *mchap_secret;
2187
2188	block = qedi_get_nvram_block(qedi);
2189	if (!block)
2190		goto exit_show_tgt_info;
2191
2192	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_EVT,
2193		  "Port:%d, tgt_idx:%d\n",
2194		  GET_FIELD2(block->id, NVM_ISCSI_CFG_BLK_MAPPED_PF_ID), idx);
2195
2196	ctrl_flags = block->target[idx].ctrl_flags &
2197		     NVM_ISCSI_CFG_TARGET_ENABLED;
2198
2199	if (!ctrl_flags) {
2200		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_EVT,
2201			  "Target disabled\n");
2202		goto exit_show_tgt_info;
2203	}
2204
2205	ipv6_en = block->generic.ctrl_flags &
2206		  NVM_ISCSI_CFG_GEN_IPV6_ENABLED;
2207	ip_len = ipv6_en ? IPV6_LEN : IPV4_LEN;
2208	chap_en = block->generic.ctrl_flags &
2209		  NVM_ISCSI_CFG_GEN_CHAP_ENABLED;
2210	chap_name = chap_en ? block->initiator.chap_name.byte : NULL;
2211	chap_secret = chap_en ? block->initiator.chap_password.byte : NULL;
2212
2213	mchap_en = block->generic.ctrl_flags &
2214		  NVM_ISCSI_CFG_GEN_CHAP_MUTUAL_ENABLED;
2215	mchap_name = mchap_en ? block->target[idx].chap_name.byte : NULL;
2216	mchap_secret = mchap_en ? block->target[idx].chap_password.byte : NULL;
2217
2218	switch (type) {
2219	case ISCSI_BOOT_TGT_NAME:
2220		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_ISCSI_NAME_MAX_LEN,
2221			     block->target[idx].target_name.byte);
2222		break;
2223	case ISCSI_BOOT_TGT_IP_ADDR:
2224		if (ipv6_en)
2225			rc = snprintf(buf, ip_len, "%pI6\n",
2226				      block->target[idx].ipv6_addr.byte);
2227		else
2228			rc = snprintf(buf, ip_len, "%pI4\n",
2229				      block->target[idx].ipv4_addr.byte);
2230		break;
2231	case ISCSI_BOOT_TGT_PORT:
2232		rc = snprintf(buf, 12, "%d\n",
2233			      GET_FIELD2(block->target[idx].generic_cont0,
2234					 NVM_ISCSI_CFG_TARGET_TCP_PORT));
2235		break;
2236	case ISCSI_BOOT_TGT_LUN:
2237		rc = snprintf(buf, 22, "%.*d\n",
2238			      block->target[idx].lun.value[1],
2239			      block->target[idx].lun.value[0]);
2240		break;
2241	case ISCSI_BOOT_TGT_CHAP_NAME:
2242		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,
2243			     chap_name);
2244		break;
2245	case ISCSI_BOOT_TGT_CHAP_SECRET:
2246		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,
2247			     chap_secret);
2248		break;
2249	case ISCSI_BOOT_TGT_REV_CHAP_NAME:
2250		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_NAME_MAX_LEN,
2251			     mchap_name);
2252		break;
2253	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
2254		rc = sprintf(buf, "%.*s\n", NVM_ISCSI_CFG_CHAP_PWD_MAX_LEN,
2255			     mchap_secret);
2256		break;
2257	case ISCSI_BOOT_TGT_FLAGS:
2258		rc = snprintf(buf, 3, "%d\n", (char)SYSFS_FLAG_FW_SEL_BOOT);
2259		break;
2260	case ISCSI_BOOT_TGT_NIC_ASSOC:
2261		rc = snprintf(buf, 3, "0\n");
2262		break;
2263	default:
2264		rc = 0;
2265		break;
2266	}
2267
2268exit_show_tgt_info:
2269	return rc;
2270}
2271
2272static ssize_t qedi_show_boot_tgt_pri_info(void *data, int type, char *buf)
2273{
2274	struct qedi_ctx *qedi = data;
2275
2276	return qedi_show_boot_tgt_info(qedi, type, buf, QEDI_NVM_TGT_PRI);
2277}
2278
2279static ssize_t qedi_show_boot_tgt_sec_info(void *data, int type, char *buf)
2280{
2281	struct qedi_ctx *qedi = data;
2282
2283	return qedi_show_boot_tgt_info(qedi, type, buf, QEDI_NVM_TGT_SEC);
2284}
2285
2286static umode_t qedi_tgt_get_attr_visibility(void *data, int type)
2287{
2288	int rc;
2289
2290	switch (type) {
2291	case ISCSI_BOOT_TGT_NAME:
2292	case ISCSI_BOOT_TGT_IP_ADDR:
2293	case ISCSI_BOOT_TGT_PORT:
2294	case ISCSI_BOOT_TGT_LUN:
2295	case ISCSI_BOOT_TGT_CHAP_NAME:
2296	case ISCSI_BOOT_TGT_CHAP_SECRET:
2297	case ISCSI_BOOT_TGT_REV_CHAP_NAME:
2298	case ISCSI_BOOT_TGT_REV_CHAP_SECRET:
2299	case ISCSI_BOOT_TGT_NIC_ASSOC:
2300	case ISCSI_BOOT_TGT_FLAGS:
2301		rc = 0444;
2302		break;
2303	default:
2304		rc = 0;
2305		break;
2306	}
2307	return rc;
2308}
2309
2310static void qedi_boot_release(void *data)
2311{
2312	struct qedi_ctx *qedi = data;
2313
2314	scsi_host_put(qedi->shost);
2315}
2316
2317static int qedi_get_boot_info(struct qedi_ctx *qedi)
2318{
2319	int ret = 1;
2320
2321	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2322		  "Get NVM iSCSI CFG image\n");
2323	ret = qedi_ops->common->nvm_get_image(qedi->cdev,
2324					      QED_NVM_IMAGE_ISCSI_CFG,
2325					      (char *)qedi->iscsi_image,
2326					      sizeof(struct qedi_nvm_iscsi_image));
2327	if (ret)
2328		QEDI_ERR(&qedi->dbg_ctx,
2329			 "Could not get NVM image. ret = %d\n", ret);
2330
2331	return ret;
2332}
2333
2334static int qedi_setup_boot_info(struct qedi_ctx *qedi)
2335{
2336	struct iscsi_boot_kobj *boot_kobj;
2337
2338	if (qedi_get_boot_info(qedi))
2339		return -EPERM;
2340
2341	qedi->boot_kset = iscsi_boot_create_host_kset(qedi->shost->host_no);
2342	if (!qedi->boot_kset)
2343		goto kset_free;
2344
2345	if (!scsi_host_get(qedi->shost))
2346		goto kset_free;
2347
2348	boot_kobj = iscsi_boot_create_target(qedi->boot_kset, 0, qedi,
2349					     qedi_show_boot_tgt_pri_info,
2350					     qedi_tgt_get_attr_visibility,
2351					     qedi_boot_release);
2352	if (!boot_kobj)
2353		goto put_host;
2354
2355	if (!scsi_host_get(qedi->shost))
2356		goto kset_free;
2357
2358	boot_kobj = iscsi_boot_create_target(qedi->boot_kset, 1, qedi,
2359					     qedi_show_boot_tgt_sec_info,
2360					     qedi_tgt_get_attr_visibility,
2361					     qedi_boot_release);
2362	if (!boot_kobj)
2363		goto put_host;
2364
2365	if (!scsi_host_get(qedi->shost))
2366		goto kset_free;
2367
2368	boot_kobj = iscsi_boot_create_initiator(qedi->boot_kset, 0, qedi,
2369						qedi_show_boot_ini_info,
2370						qedi_ini_get_attr_visibility,
2371						qedi_boot_release);
2372	if (!boot_kobj)
2373		goto put_host;
2374
2375	if (!scsi_host_get(qedi->shost))
2376		goto kset_free;
2377
2378	boot_kobj = iscsi_boot_create_ethernet(qedi->boot_kset, 0, qedi,
2379					       qedi_show_boot_eth_info,
2380					       qedi_eth_get_attr_visibility,
2381					       qedi_boot_release);
2382	if (!boot_kobj)
2383		goto put_host;
2384
2385	return 0;
2386
2387put_host:
2388	scsi_host_put(qedi->shost);
2389kset_free:
2390	iscsi_boot_destroy_kset(qedi->boot_kset);
2391	return -ENOMEM;
2392}
2393
2394static pci_ers_result_t qedi_io_error_detected(struct pci_dev *pdev,
2395					       pci_channel_state_t state)
2396{
2397	struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2398
2399	QEDI_ERR(&qedi->dbg_ctx, "%s: PCI error detected [%d]\n",
2400		 __func__, state);
2401
2402	if (test_and_set_bit(QEDI_IN_RECOVERY, &qedi->flags)) {
2403		QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2404			  "Recovery already in progress.\n");
2405		return PCI_ERS_RESULT_NONE;
2406	}
2407
2408	qedi_ops->common->recovery_process(qedi->cdev);
2409
2410	return PCI_ERS_RESULT_CAN_RECOVER;
2411}
2412
2413static void __qedi_remove(struct pci_dev *pdev, int mode)
2414{
2415	struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2416	int rval;
2417	u16 retry = 10;
2418
2419	if (mode == QEDI_MODE_NORMAL)
2420		iscsi_host_remove(qedi->shost, false);
2421	else if (mode == QEDI_MODE_SHUTDOWN)
2422		iscsi_host_remove(qedi->shost, true);
2423
2424	if (mode == QEDI_MODE_NORMAL || mode == QEDI_MODE_SHUTDOWN) {
2425		if (qedi->tmf_thread) {
2426			destroy_workqueue(qedi->tmf_thread);
2427			qedi->tmf_thread = NULL;
2428		}
2429
2430		if (qedi->offload_thread) {
2431			destroy_workqueue(qedi->offload_thread);
2432			qedi->offload_thread = NULL;
2433		}
2434	}
2435
2436#ifdef CONFIG_DEBUG_FS
2437	qedi_dbg_host_exit(&qedi->dbg_ctx);
2438#endif
2439	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags))
2440		qedi_ops->common->set_power_state(qedi->cdev, PCI_D0);
2441
2442	qedi_sync_free_irqs(qedi);
2443
2444	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {
2445		while (retry--) {
2446			rval = qedi_ops->stop(qedi->cdev);
2447			if (rval < 0)
2448				msleep(1000);
2449			else
2450				break;
2451		}
2452		qedi_ops->ll2->stop(qedi->cdev);
2453	}
2454
2455	cancel_delayed_work_sync(&qedi->recovery_work);
2456	cancel_delayed_work_sync(&qedi->board_disable_work);
2457
2458	qedi_free_iscsi_pf_param(qedi);
2459
2460	rval = qedi_ops->common->update_drv_state(qedi->cdev, false);
2461	if (rval)
2462		QEDI_ERR(&qedi->dbg_ctx, "Failed to send drv state to MFW\n");
2463
2464	if (!test_bit(QEDI_IN_OFFLINE, &qedi->flags)) {
2465		qedi_ops->common->slowpath_stop(qedi->cdev);
2466		qedi_ops->common->remove(qedi->cdev);
2467	}
2468
2469	qedi_destroy_fp(qedi);
2470
2471	if (mode == QEDI_MODE_NORMAL || mode == QEDI_MODE_SHUTDOWN) {
2472		qedi_release_cid_que(qedi);
2473		qedi_cm_free_mem(qedi);
2474		qedi_free_uio(qedi->udev);
2475		qedi_free_itt(qedi);
2476
2477		if (qedi->ll2_recv_thread) {
2478			kthread_stop(qedi->ll2_recv_thread);
2479			qedi->ll2_recv_thread = NULL;
2480		}
2481		qedi_ll2_free_skbs(qedi);
2482
2483		if (qedi->boot_kset)
2484			iscsi_boot_destroy_kset(qedi->boot_kset);
2485
2486		iscsi_host_free(qedi->shost);
2487	}
2488}
2489
2490static void qedi_board_disable_work(struct work_struct *work)
2491{
2492	struct qedi_ctx *qedi =
2493			container_of(work, struct qedi_ctx,
2494				     board_disable_work.work);
2495
2496	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2497		  "Fan failure, Unloading firmware context.\n");
2498
2499	if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))
2500		return;
2501
2502	__qedi_remove(qedi->pdev, QEDI_MODE_NORMAL);
2503}
2504
2505static void qedi_shutdown(struct pci_dev *pdev)
2506{
2507	struct qedi_ctx *qedi = pci_get_drvdata(pdev);
2508
2509	QEDI_ERR(&qedi->dbg_ctx, "%s: Shutdown qedi\n", __func__);
2510	if (test_and_set_bit(QEDI_IN_SHUTDOWN, &qedi->flags))
2511		return;
2512	__qedi_remove(pdev, QEDI_MODE_SHUTDOWN);
2513}
2514
2515static int qedi_suspend(struct pci_dev *pdev, pm_message_t state)
2516{
2517	struct qedi_ctx *qedi;
2518
2519	if (!pdev) {
2520		QEDI_ERR(NULL, "pdev is NULL.\n");
2521		return -ENODEV;
2522	}
2523
2524	qedi = pci_get_drvdata(pdev);
2525
2526	QEDI_ERR(&qedi->dbg_ctx, "%s: Device does not support suspend operation\n", __func__);
2527
2528	return -EPERM;
2529}
2530
2531static int __qedi_probe(struct pci_dev *pdev, int mode)
2532{
2533	struct qedi_ctx *qedi;
2534	struct qed_ll2_params params;
2535	u8 dp_level = 0;
2536	bool is_vf = false;
2537	char host_buf[16];
2538	struct qed_link_params link_params;
2539	struct qed_slowpath_params sp_params;
2540	struct qed_probe_params qed_params;
2541	void *task_start, *task_end;
2542	int rc;
2543	u16 retry = 10;
2544
2545	if (mode != QEDI_MODE_RECOVERY) {
2546		qedi = qedi_host_alloc(pdev);
2547		if (!qedi) {
2548			rc = -ENOMEM;
2549			goto exit_probe;
2550		}
2551	} else {
2552		qedi = pci_get_drvdata(pdev);
2553	}
2554
2555retry_probe:
2556	if (mode == QEDI_MODE_RECOVERY)
2557		msleep(2000);
2558
2559	memset(&qed_params, 0, sizeof(qed_params));
2560	qed_params.protocol = QED_PROTOCOL_ISCSI;
2561	qed_params.dp_module = qedi_qed_debug;
2562	qed_params.dp_level = dp_level;
2563	qed_params.is_vf = is_vf;
2564	qedi->cdev = qedi_ops->common->probe(pdev, &qed_params);
2565	if (!qedi->cdev) {
2566		if (mode == QEDI_MODE_RECOVERY && retry) {
2567			QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2568				  "Retry %d initialize hardware\n", retry);
2569			retry--;
2570			goto retry_probe;
2571		}
2572
2573		rc = -ENODEV;
2574		QEDI_ERR(&qedi->dbg_ctx, "Cannot initialize hardware\n");
2575		goto free_host;
2576	}
2577
2578	set_bit(QEDI_ERR_ATTN_CLR_EN, &qedi->qedi_err_flags);
2579	set_bit(QEDI_ERR_IS_RECOVERABLE, &qedi->qedi_err_flags);
2580	atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
2581
2582	rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);
2583	if (rc)
2584		goto free_host;
2585
2586	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2587		  "dev_info: num_hwfns=%d affin_hwfn_idx=%d.\n",
2588		  qedi->dev_info.common.num_hwfns,
2589		  qedi_ops->common->get_affin_hwfn_idx(qedi->cdev));
2590
2591	rc = qedi_set_iscsi_pf_param(qedi);
2592	if (rc) {
2593		rc = -ENOMEM;
2594		QEDI_ERR(&qedi->dbg_ctx,
2595			 "Set iSCSI pf param fail\n");
2596		goto free_host;
2597	}
2598
2599	qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);
2600
2601	rc = qedi_prepare_fp(qedi);
2602	if (rc) {
2603		QEDI_ERR(&qedi->dbg_ctx, "Cannot start slowpath.\n");
2604		goto free_pf_params;
2605	}
2606
2607	/* Start the Slowpath-process */
2608	memset(&sp_params, 0, sizeof(struct qed_slowpath_params));
2609	sp_params.int_mode = QED_INT_MODE_MSIX;
2610	sp_params.drv_major = QEDI_DRIVER_MAJOR_VER;
2611	sp_params.drv_minor = QEDI_DRIVER_MINOR_VER;
2612	sp_params.drv_rev = QEDI_DRIVER_REV_VER;
2613	sp_params.drv_eng = QEDI_DRIVER_ENG_VER;
2614	strscpy(sp_params.name, "qedi iSCSI", QED_DRV_VER_STR_SIZE);
2615	rc = qedi_ops->common->slowpath_start(qedi->cdev, &sp_params);
2616	if (rc) {
2617		QEDI_ERR(&qedi->dbg_ctx, "Cannot start slowpath\n");
2618		goto stop_hw;
2619	}
2620
2621	/* update_pf_params needs to be called before and after slowpath
2622	 * start
2623	 */
2624	qedi_ops->common->update_pf_params(qedi->cdev, &qedi->pf_params);
2625
2626	rc = qedi_setup_int(qedi);
2627	if (rc)
2628		goto stop_iscsi_func;
2629
2630	qedi_ops->common->set_power_state(qedi->cdev, PCI_D0);
2631
2632	/* Learn information crucial for qedi to progress */
2633	rc = qedi_ops->fill_dev_info(qedi->cdev, &qedi->dev_info);
2634	if (rc)
2635		goto stop_iscsi_func;
2636
2637	/* Record BDQ producer doorbell addresses */
2638	qedi->bdq_primary_prod = qedi->dev_info.primary_dbq_rq_addr;
2639	qedi->bdq_secondary_prod = qedi->dev_info.secondary_bdq_rq_addr;
2640	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2641		  "BDQ primary_prod=%p secondary_prod=%p.\n",
2642		  qedi->bdq_primary_prod,
2643		  qedi->bdq_secondary_prod);
2644
2645	/*
2646	 * We need to write the number of BDs in the BDQ we've preallocated so
2647	 * the f/w will do a prefetch and we'll get an unsolicited CQE when a
2648	 * packet arrives.
2649	 */
2650	qedi->bdq_prod_idx = QEDI_BDQ_NUM;
2651	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2652		  "Writing %d to primary and secondary BDQ doorbell registers.\n",
2653		  qedi->bdq_prod_idx);
2654	writew(qedi->bdq_prod_idx, qedi->bdq_primary_prod);
2655	readw(qedi->bdq_primary_prod);
2656	writew(qedi->bdq_prod_idx, qedi->bdq_secondary_prod);
2657	readw(qedi->bdq_secondary_prod);
2658
2659	ether_addr_copy(qedi->mac, qedi->dev_info.common.hw_mac);
2660	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC, "MAC address is %pM.\n",
2661		  qedi->mac);
2662
2663	snprintf(host_buf, sizeof(host_buf), "host_%d", qedi->shost->host_no);
2664	qedi_ops->common->set_name(qedi->cdev, host_buf);
2665
2666	qedi_ops->register_ops(qedi->cdev, &qedi_cb_ops, qedi);
2667
2668	memset(&params, 0, sizeof(params));
2669	params.mtu = DEF_PATH_MTU + IPV6_HDR_LEN + TCP_HDR_LEN;
2670	qedi->ll2_mtu = DEF_PATH_MTU;
2671	params.drop_ttl0_packets = 0;
2672	params.rx_vlan_stripping = 1;
2673	ether_addr_copy(params.ll2_mac_address, qedi->dev_info.common.hw_mac);
2674
2675	if (mode != QEDI_MODE_RECOVERY) {
2676		/* set up rx path */
2677		INIT_LIST_HEAD(&qedi->ll2_skb_list);
2678		spin_lock_init(&qedi->ll2_lock);
2679		/* start qedi context */
2680		spin_lock_init(&qedi->hba_lock);
2681		spin_lock_init(&qedi->task_idx_lock);
2682		mutex_init(&qedi->stats_lock);
2683	}
2684	qedi_ops->ll2->register_cb_ops(qedi->cdev, &qedi_ll2_cb_ops, qedi);
2685	qedi_ops->ll2->start(qedi->cdev, &params);
2686
2687	if (mode != QEDI_MODE_RECOVERY) {
2688		qedi->ll2_recv_thread = kthread_run(qedi_ll2_recv_thread,
2689						    (void *)qedi,
2690						    "qedi_ll2_thread");
2691	}
2692
2693	rc = qedi_ops->start(qedi->cdev, &qedi->tasks,
2694			     qedi, qedi_iscsi_event_cb);
2695	if (rc) {
2696		rc = -ENODEV;
2697		QEDI_ERR(&qedi->dbg_ctx, "Cannot start iSCSI function\n");
2698		goto stop_slowpath;
2699	}
2700
2701	task_start = qedi_get_task_mem(&qedi->tasks, 0);
2702	task_end = qedi_get_task_mem(&qedi->tasks, MAX_TID_BLOCKS_ISCSI - 1);
2703	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_DISC,
2704		  "Task context start=%p, end=%p block_size=%u.\n",
2705		   task_start, task_end, qedi->tasks.size);
2706
2707	memset(&link_params, 0, sizeof(link_params));
2708	link_params.link_up = true;
2709	rc = qedi_ops->common->set_link(qedi->cdev, &link_params);
2710	if (rc) {
2711		QEDI_WARN(&qedi->dbg_ctx, "Link set up failed.\n");
2712		atomic_set(&qedi->link_state, QEDI_LINK_DOWN);
2713	}
2714
2715#ifdef CONFIG_DEBUG_FS
2716	qedi_dbg_host_init(&qedi->dbg_ctx, qedi_debugfs_ops,
2717			   qedi_dbg_fops);
2718#endif
2719	QEDI_INFO(&qedi->dbg_ctx, QEDI_LOG_INFO,
2720		  "QLogic FastLinQ iSCSI Module qedi %s, FW %d.%d.%d.%d\n",
2721		  QEDI_MODULE_VERSION, FW_MAJOR_VERSION, FW_MINOR_VERSION,
2722		  FW_REVISION_VERSION, FW_ENGINEERING_VERSION);
2723
2724	if (mode == QEDI_MODE_NORMAL) {
2725		if (iscsi_host_add(qedi->shost, &pdev->dev)) {
2726			QEDI_ERR(&qedi->dbg_ctx,
2727				 "Could not add iscsi host\n");
2728			rc = -ENOMEM;
2729			goto remove_host;
2730		}
2731
2732		/* Allocate uio buffers */
2733		rc = qedi_alloc_uio_rings(qedi);
2734		if (rc) {
2735			QEDI_ERR(&qedi->dbg_ctx,
2736				 "UIO alloc ring failed err=%d\n", rc);
2737			goto remove_host;
2738		}
2739
2740		rc = qedi_init_uio(qedi);
2741		if (rc) {
2742			QEDI_ERR(&qedi->dbg_ctx,
2743				 "UIO init failed, err=%d\n", rc);
2744			goto free_uio;
2745		}
2746
2747		/* host the array on iscsi_conn */
2748		rc = qedi_setup_cid_que(qedi);
2749		if (rc) {
2750			QEDI_ERR(&qedi->dbg_ctx,
2751				 "Could not setup cid que\n");
2752			goto free_uio;
2753		}
2754
2755		rc = qedi_cm_alloc_mem(qedi);
2756		if (rc) {
2757			QEDI_ERR(&qedi->dbg_ctx,
2758				 "Could not alloc cm memory\n");
2759			goto free_cid_que;
2760		}
2761
2762		rc = qedi_alloc_itt(qedi);
2763		if (rc) {
2764			QEDI_ERR(&qedi->dbg_ctx,
2765				 "Could not alloc itt memory\n");
2766			goto free_cid_que;
2767		}
2768
2769		sprintf(host_buf, "host_%d", qedi->shost->host_no);
2770		qedi->tmf_thread = create_singlethread_workqueue(host_buf);
2771		if (!qedi->tmf_thread) {
2772			QEDI_ERR(&qedi->dbg_ctx,
2773				 "Unable to start tmf thread!\n");
2774			rc = -ENODEV;
2775			goto free_cid_que;
2776		}
2777
2778		sprintf(host_buf, "qedi_ofld%d", qedi->shost->host_no);
2779		qedi->offload_thread = create_workqueue(host_buf);
2780		if (!qedi->offload_thread) {
2781			QEDI_ERR(&qedi->dbg_ctx,
2782				 "Unable to start offload thread!\n");
2783			rc = -ENODEV;
2784			goto free_tmf_thread;
2785		}
2786
2787		INIT_DELAYED_WORK(&qedi->recovery_work, qedi_recovery_handler);
2788		INIT_DELAYED_WORK(&qedi->board_disable_work,
2789				  qedi_board_disable_work);
2790
2791		/* F/w needs 1st task context memory entry for performance */
2792		set_bit(QEDI_RESERVE_TASK_ID, qedi->task_idx_map);
2793		atomic_set(&qedi->num_offloads, 0);
2794
2795		if (qedi_setup_boot_info(qedi))
2796			QEDI_ERR(&qedi->dbg_ctx,
2797				 "No iSCSI boot target configured\n");
2798
2799		rc = qedi_ops->common->update_drv_state(qedi->cdev, true);
2800		if (rc)
2801			QEDI_ERR(&qedi->dbg_ctx,
2802				 "Failed to send drv state to MFW\n");
2803
2804	}
2805
2806	return 0;
2807
2808free_tmf_thread:
2809	destroy_workqueue(qedi->tmf_thread);
2810free_cid_que:
2811	qedi_release_cid_que(qedi);
2812free_uio:
2813	qedi_free_uio(qedi->udev);
2814remove_host:
2815#ifdef CONFIG_DEBUG_FS
2816	qedi_dbg_host_exit(&qedi->dbg_ctx);
2817#endif
2818	iscsi_host_remove(qedi->shost, false);
2819stop_iscsi_func:
2820	qedi_ops->stop(qedi->cdev);
2821stop_slowpath:
2822	qedi_ops->common->slowpath_stop(qedi->cdev);
2823stop_hw:
2824	qedi_ops->common->remove(qedi->cdev);
2825free_pf_params:
2826	qedi_free_iscsi_pf_param(qedi);
2827free_host:
2828	iscsi_host_free(qedi->shost);
2829exit_probe:
2830	return rc;
2831}
2832
2833static void qedi_mark_conn_recovery(struct iscsi_cls_session *cls_session)
2834{
2835	struct iscsi_session *session = cls_session->dd_data;
2836	struct iscsi_conn *conn = session->leadconn;
2837	struct qedi_conn *qedi_conn = conn->dd_data;
2838
2839	iscsi_conn_failure(qedi_conn->cls_conn->dd_data, ISCSI_ERR_CONN_FAILED);
2840}
2841
2842static void qedi_recovery_handler(struct work_struct *work)
2843{
2844	struct qedi_ctx *qedi =
2845			container_of(work, struct qedi_ctx, recovery_work.work);
2846
2847	iscsi_host_for_each_session(qedi->shost, qedi_mark_conn_recovery);
2848
2849	/* Call common_ops->recovery_prolog to allow the MFW to quiesce
2850	 * any PCI transactions.
2851	 */
2852	qedi_ops->common->recovery_prolog(qedi->cdev);
2853
2854	__qedi_remove(qedi->pdev, QEDI_MODE_RECOVERY);
2855	__qedi_probe(qedi->pdev, QEDI_MODE_RECOVERY);
2856	clear_bit(QEDI_IN_RECOVERY, &qedi->flags);
2857}
2858
2859static int qedi_probe(struct pci_dev *pdev, const struct pci_device_id *id)
2860{
2861	return __qedi_probe(pdev, QEDI_MODE_NORMAL);
2862}
2863
2864static void qedi_remove(struct pci_dev *pdev)
2865{
2866	__qedi_remove(pdev, QEDI_MODE_NORMAL);
2867}
2868
2869static struct pci_device_id qedi_pci_tbl[] = {
2870	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x165E) },
2871	{ PCI_DEVICE(PCI_VENDOR_ID_QLOGIC, 0x8084) },
2872	{ 0 },
2873};
2874MODULE_DEVICE_TABLE(pci, qedi_pci_tbl);
2875
2876static enum cpuhp_state qedi_cpuhp_state;
2877
2878static struct pci_error_handlers qedi_err_handler = {
2879	.error_detected = qedi_io_error_detected,
2880};
2881
2882static struct pci_driver qedi_pci_driver = {
2883	.name = QEDI_MODULE_NAME,
2884	.id_table = qedi_pci_tbl,
2885	.probe = qedi_probe,
2886	.remove = qedi_remove,
2887	.shutdown = qedi_shutdown,
2888	.err_handler = &qedi_err_handler,
2889	.suspend = qedi_suspend,
2890};
2891
2892static int __init qedi_init(void)
2893{
2894	struct qedi_percpu_s *p;
2895	int cpu, rc = 0;
2896
2897	qedi_ops = qed_get_iscsi_ops();
2898	if (!qedi_ops) {
2899		QEDI_ERR(NULL, "Failed to get qed iSCSI operations\n");
2900		return -EINVAL;
2901	}
2902
2903#ifdef CONFIG_DEBUG_FS
2904	qedi_dbg_init("qedi");
2905#endif
2906
2907	qedi_scsi_transport = iscsi_register_transport(&qedi_iscsi_transport);
2908	if (!qedi_scsi_transport) {
2909		QEDI_ERR(NULL, "Could not register qedi transport");
2910		rc = -ENOMEM;
2911		goto exit_qedi_init_1;
2912	}
2913
2914	for_each_possible_cpu(cpu) {
2915		p = &per_cpu(qedi_percpu, cpu);
2916		INIT_LIST_HEAD(&p->work_list);
2917		spin_lock_init(&p->p_work_lock);
2918		p->iothread = NULL;
2919	}
2920
2921	rc = cpuhp_setup_state(CPUHP_AP_ONLINE_DYN, "scsi/qedi:online",
2922			       qedi_cpu_online, qedi_cpu_offline);
2923	if (rc < 0)
2924		goto exit_qedi_init_2;
2925	qedi_cpuhp_state = rc;
2926
2927	rc = pci_register_driver(&qedi_pci_driver);
2928	if (rc) {
2929		QEDI_ERR(NULL, "Failed to register driver\n");
2930		goto exit_qedi_hp;
2931	}
2932
2933	return 0;
2934
2935exit_qedi_hp:
2936	cpuhp_remove_state(qedi_cpuhp_state);
2937exit_qedi_init_2:
2938	iscsi_unregister_transport(&qedi_iscsi_transport);
2939exit_qedi_init_1:
2940#ifdef CONFIG_DEBUG_FS
2941	qedi_dbg_exit();
2942#endif
2943	qed_put_iscsi_ops();
2944	return rc;
2945}
2946
2947static void __exit qedi_cleanup(void)
2948{
2949	pci_unregister_driver(&qedi_pci_driver);
2950	cpuhp_remove_state(qedi_cpuhp_state);
2951	iscsi_unregister_transport(&qedi_iscsi_transport);
2952
2953#ifdef CONFIG_DEBUG_FS
2954	qedi_dbg_exit();
2955#endif
2956	qed_put_iscsi_ops();
2957}
2958
2959MODULE_DESCRIPTION("QLogic FastLinQ 4xxxx iSCSI Module");
2960MODULE_LICENSE("GPL");
2961MODULE_AUTHOR("QLogic Corporation");
2962MODULE_VERSION(QEDI_MODULE_VERSION);
2963module_init(qedi_init);
2964module_exit(qedi_cleanup);