1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Driver for Broadcom MPI3 Storage Controllers
   4 *
   5 * Copyright (C) 2017-2023 Broadcom Inc.
   6 *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
   7 *
   8 */
   9
  10#include "mpi3mr.h"
  11#include <linux/io-64-nonatomic-lo-hi.h>
  12
  13static int
  14mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type, u32 reset_reason);
  15static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc);
  16static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
  17	struct mpi3_ioc_facts_data *facts_data);
  18static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
  19	struct mpi3mr_drv_cmd *drv_cmd);
  20
  21static int poll_queues;
  22module_param(poll_queues, int, 0444);
  23MODULE_PARM_DESC(poll_queues, "Number of queues for io_uring poll mode. (Range 1 - 126)");
  24
  25#if defined(writeq) && defined(CONFIG_64BIT)
  26static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  27{
  28	writeq(b, addr);
  29}
  30#else
  31static inline void mpi3mr_writeq(__u64 b, volatile void __iomem *addr)
  32{
  33	__u64 data_out = b;
  34
  35	writel((u32)(data_out), addr);
  36	writel((u32)(data_out >> 32), (addr + 4));
  37}
  38#endif
  39
  40static inline bool
  41mpi3mr_check_req_qfull(struct op_req_qinfo *op_req_q)
  42{
  43	u16 pi, ci, max_entries;
  44	bool is_qfull = false;
  45
  46	pi = op_req_q->pi;
  47	ci = READ_ONCE(op_req_q->ci);
  48	max_entries = op_req_q->num_requests;
  49
  50	if ((ci == (pi + 1)) || ((!ci) && (pi == (max_entries - 1))))
  51		is_qfull = true;
  52
  53	return is_qfull;
  54}
  55
  56static void mpi3mr_sync_irqs(struct mpi3mr_ioc *mrioc)
  57{
  58	u16 i, max_vectors;
  59
  60	max_vectors = mrioc->intr_info_count;
  61
  62	for (i = 0; i < max_vectors; i++)
  63		synchronize_irq(pci_irq_vector(mrioc->pdev, i));
  64}
  65
  66void mpi3mr_ioc_disable_intr(struct mpi3mr_ioc *mrioc)
  67{
  68	mrioc->intr_enabled = 0;
  69	mpi3mr_sync_irqs(mrioc);
  70}
  71
  72void mpi3mr_ioc_enable_intr(struct mpi3mr_ioc *mrioc)
  73{
  74	mrioc->intr_enabled = 1;
  75}
  76
  77static void mpi3mr_cleanup_isr(struct mpi3mr_ioc *mrioc)
  78{
  79	u16 i;
  80
  81	mpi3mr_ioc_disable_intr(mrioc);
  82
  83	if (!mrioc->intr_info)
  84		return;
  85
  86	for (i = 0; i < mrioc->intr_info_count; i++)
  87		free_irq(pci_irq_vector(mrioc->pdev, i),
  88		    (mrioc->intr_info + i));
  89
  90	kfree(mrioc->intr_info);
  91	mrioc->intr_info = NULL;
  92	mrioc->intr_info_count = 0;
  93	mrioc->is_intr_info_set = false;
  94	pci_free_irq_vectors(mrioc->pdev);
  95}
  96
  97void mpi3mr_add_sg_single(void *paddr, u8 flags, u32 length,
  98	dma_addr_t dma_addr)
  99{
 100	struct mpi3_sge_common *sgel = paddr;
 101
 102	sgel->flags = flags;
 103	sgel->length = cpu_to_le32(length);
 104	sgel->address = cpu_to_le64(dma_addr);
 105}
 106
 107void mpi3mr_build_zero_len_sge(void *paddr)
 108{
 109	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
 110
 111	mpi3mr_add_sg_single(paddr, sgl_flags, 0, -1);
 112}
 113
 114void *mpi3mr_get_reply_virt_addr(struct mpi3mr_ioc *mrioc,
 115	dma_addr_t phys_addr)
 116{
 117	if (!phys_addr)
 118		return NULL;
 119
 120	if ((phys_addr < mrioc->reply_buf_dma) ||
 121	    (phys_addr > mrioc->reply_buf_dma_max_address))
 122		return NULL;
 123
 124	return mrioc->reply_buf + (phys_addr - mrioc->reply_buf_dma);
 125}
 126
 127void *mpi3mr_get_sensebuf_virt_addr(struct mpi3mr_ioc *mrioc,
 128	dma_addr_t phys_addr)
 129{
 130	if (!phys_addr)
 131		return NULL;
 132
 133	return mrioc->sense_buf + (phys_addr - mrioc->sense_buf_dma);
 134}
 135
 136static void mpi3mr_repost_reply_buf(struct mpi3mr_ioc *mrioc,
 137	u64 reply_dma)
 138{
 139	u32 old_idx = 0;
 140	unsigned long flags;
 141
 142	spin_lock_irqsave(&mrioc->reply_free_queue_lock, flags);
 143	old_idx  =  mrioc->reply_free_queue_host_index;
 144	mrioc->reply_free_queue_host_index = (
 145	    (mrioc->reply_free_queue_host_index ==
 146	    (mrioc->reply_free_qsz - 1)) ? 0 :
 147	    (mrioc->reply_free_queue_host_index + 1));
 148	mrioc->reply_free_q[old_idx] = cpu_to_le64(reply_dma);
 149	writel(mrioc->reply_free_queue_host_index,
 150	    &mrioc->sysif_regs->reply_free_host_index);
 151	spin_unlock_irqrestore(&mrioc->reply_free_queue_lock, flags);
 152}
 153
 154void mpi3mr_repost_sense_buf(struct mpi3mr_ioc *mrioc,
 155	u64 sense_buf_dma)
 156{
 157	u32 old_idx = 0;
 158	unsigned long flags;
 159
 160	spin_lock_irqsave(&mrioc->sbq_lock, flags);
 161	old_idx  =  mrioc->sbq_host_index;
 162	mrioc->sbq_host_index = ((mrioc->sbq_host_index ==
 163	    (mrioc->sense_buf_q_sz - 1)) ? 0 :
 164	    (mrioc->sbq_host_index + 1));
 165	mrioc->sense_buf_q[old_idx] = cpu_to_le64(sense_buf_dma);
 166	writel(mrioc->sbq_host_index,
 167	    &mrioc->sysif_regs->sense_buffer_free_host_index);
 168	spin_unlock_irqrestore(&mrioc->sbq_lock, flags);
 169}
 170
 171static void mpi3mr_print_event_data(struct mpi3mr_ioc *mrioc,
 172	struct mpi3_event_notification_reply *event_reply)
 173{
 174	char *desc = NULL;
 175	u16 event;
 176
 177	event = event_reply->event;
 178
 179	switch (event) {
 180	case MPI3_EVENT_LOG_DATA:
 181		desc = "Log Data";
 182		break;
 183	case MPI3_EVENT_CHANGE:
 184		desc = "Event Change";
 185		break;
 186	case MPI3_EVENT_GPIO_INTERRUPT:
 187		desc = "GPIO Interrupt";
 188		break;
 189	case MPI3_EVENT_CABLE_MGMT:
 190		desc = "Cable Management";
 191		break;
 192	case MPI3_EVENT_ENERGY_PACK_CHANGE:
 193		desc = "Energy Pack Change";
 194		break;
 195	case MPI3_EVENT_DEVICE_ADDED:
 196	{
 197		struct mpi3_device_page0 *event_data =
 198		    (struct mpi3_device_page0 *)event_reply->event_data;
 199		ioc_info(mrioc, "Device Added: dev=0x%04x Form=0x%x\n",
 200		    event_data->dev_handle, event_data->device_form);
 201		return;
 202	}
 203	case MPI3_EVENT_DEVICE_INFO_CHANGED:
 204	{
 205		struct mpi3_device_page0 *event_data =
 206		    (struct mpi3_device_page0 *)event_reply->event_data;
 207		ioc_info(mrioc, "Device Info Changed: dev=0x%04x Form=0x%x\n",
 208		    event_data->dev_handle, event_data->device_form);
 209		return;
 210	}
 211	case MPI3_EVENT_DEVICE_STATUS_CHANGE:
 212	{
 213		struct mpi3_event_data_device_status_change *event_data =
 214		    (struct mpi3_event_data_device_status_change *)event_reply->event_data;
 215		ioc_info(mrioc, "Device status Change: dev=0x%04x RC=0x%x\n",
 216		    event_data->dev_handle, event_data->reason_code);
 217		return;
 218	}
 219	case MPI3_EVENT_SAS_DISCOVERY:
 220	{
 221		struct mpi3_event_data_sas_discovery *event_data =
 222		    (struct mpi3_event_data_sas_discovery *)event_reply->event_data;
 223		ioc_info(mrioc, "SAS Discovery: (%s) status (0x%08x)\n",
 224		    (event_data->reason_code == MPI3_EVENT_SAS_DISC_RC_STARTED) ?
 225		    "start" : "stop",
 226		    le32_to_cpu(event_data->discovery_status));
 227		return;
 228	}
 229	case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
 230		desc = "SAS Broadcast Primitive";
 231		break;
 232	case MPI3_EVENT_SAS_NOTIFY_PRIMITIVE:
 233		desc = "SAS Notify Primitive";
 234		break;
 235	case MPI3_EVENT_SAS_INIT_DEVICE_STATUS_CHANGE:
 236		desc = "SAS Init Device Status Change";
 237		break;
 238	case MPI3_EVENT_SAS_INIT_TABLE_OVERFLOW:
 239		desc = "SAS Init Table Overflow";
 240		break;
 241	case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
 242		desc = "SAS Topology Change List";
 243		break;
 244	case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
 245		desc = "Enclosure Device Status Change";
 246		break;
 247	case MPI3_EVENT_ENCL_DEVICE_ADDED:
 248		desc = "Enclosure Added";
 249		break;
 250	case MPI3_EVENT_HARD_RESET_RECEIVED:
 251		desc = "Hard Reset Received";
 252		break;
 253	case MPI3_EVENT_SAS_PHY_COUNTER:
 254		desc = "SAS PHY Counter";
 255		break;
 256	case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
 257		desc = "SAS Device Discovery Error";
 258		break;
 259	case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
 260		desc = "PCIE Topology Change List";
 261		break;
 262	case MPI3_EVENT_PCIE_ENUMERATION:
 263	{
 264		struct mpi3_event_data_pcie_enumeration *event_data =
 265		    (struct mpi3_event_data_pcie_enumeration *)event_reply->event_data;
 266		ioc_info(mrioc, "PCIE Enumeration: (%s)",
 267		    (event_data->reason_code ==
 268		    MPI3_EVENT_PCIE_ENUM_RC_STARTED) ? "start" : "stop");
 269		if (event_data->enumeration_status)
 270			ioc_info(mrioc, "enumeration_status(0x%08x)\n",
 271			    le32_to_cpu(event_data->enumeration_status));
 272		return;
 273	}
 274	case MPI3_EVENT_PREPARE_FOR_RESET:
 275		desc = "Prepare For Reset";
 276		break;
 277	}
 278
 279	if (!desc)
 280		return;
 281
 282	ioc_info(mrioc, "%s\n", desc);
 283}
 284
 285static void mpi3mr_handle_events(struct mpi3mr_ioc *mrioc,
 286	struct mpi3_default_reply *def_reply)
 287{
 288	struct mpi3_event_notification_reply *event_reply =
 289	    (struct mpi3_event_notification_reply *)def_reply;
 290
 291	mrioc->change_count = le16_to_cpu(event_reply->ioc_change_count);
 292	mpi3mr_print_event_data(mrioc, event_reply);
 293	mpi3mr_os_handle_events(mrioc, event_reply);
 294}
 295
 296static struct mpi3mr_drv_cmd *
 297mpi3mr_get_drv_cmd(struct mpi3mr_ioc *mrioc, u16 host_tag,
 298	struct mpi3_default_reply *def_reply)
 299{
 300	u16 idx;
 301
 302	switch (host_tag) {
 303	case MPI3MR_HOSTTAG_INITCMDS:
 304		return &mrioc->init_cmds;
 305	case MPI3MR_HOSTTAG_CFG_CMDS:
 306		return &mrioc->cfg_cmds;
 307	case MPI3MR_HOSTTAG_BSG_CMDS:
 308		return &mrioc->bsg_cmds;
 309	case MPI3MR_HOSTTAG_BLK_TMS:
 310		return &mrioc->host_tm_cmds;
 311	case MPI3MR_HOSTTAG_PEL_ABORT:
 312		return &mrioc->pel_abort_cmd;
 313	case MPI3MR_HOSTTAG_PEL_WAIT:
 314		return &mrioc->pel_cmds;
 315	case MPI3MR_HOSTTAG_TRANSPORT_CMDS:
 316		return &mrioc->transport_cmds;
 317	case MPI3MR_HOSTTAG_INVALID:
 318		if (def_reply && def_reply->function ==
 319		    MPI3_FUNCTION_EVENT_NOTIFICATION)
 320			mpi3mr_handle_events(mrioc, def_reply);
 321		return NULL;
 322	default:
 323		break;
 324	}
 325	if (host_tag >= MPI3MR_HOSTTAG_DEVRMCMD_MIN &&
 326	    host_tag <= MPI3MR_HOSTTAG_DEVRMCMD_MAX) {
 327		idx = host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
 328		return &mrioc->dev_rmhs_cmds[idx];
 329	}
 330
 331	if (host_tag >= MPI3MR_HOSTTAG_EVTACKCMD_MIN &&
 332	    host_tag <= MPI3MR_HOSTTAG_EVTACKCMD_MAX) {
 333		idx = host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
 334		return &mrioc->evtack_cmds[idx];
 335	}
 336
 337	return NULL;
 338}
 339
 340static void mpi3mr_process_admin_reply_desc(struct mpi3mr_ioc *mrioc,
 341	struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma)
 342{
 343	u16 reply_desc_type, host_tag = 0;
 344	u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
 345	u32 ioc_loginfo = 0;
 346	struct mpi3_status_reply_descriptor *status_desc;
 347	struct mpi3_address_reply_descriptor *addr_desc;
 348	struct mpi3_success_reply_descriptor *success_desc;
 349	struct mpi3_default_reply *def_reply = NULL;
 350	struct mpi3mr_drv_cmd *cmdptr = NULL;
 351	struct mpi3_scsi_io_reply *scsi_reply;
 352	u8 *sense_buf = NULL;
 353
 354	*reply_dma = 0;
 355	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
 356	    MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
 357	switch (reply_desc_type) {
 358	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
 359		status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
 360		host_tag = le16_to_cpu(status_desc->host_tag);
 361		ioc_status = le16_to_cpu(status_desc->ioc_status);
 362		if (ioc_status &
 363		    MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 364			ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
 365		ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 366		break;
 367	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
 368		addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
 369		*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
 370		def_reply = mpi3mr_get_reply_virt_addr(mrioc, *reply_dma);
 371		if (!def_reply)
 372			goto out;
 373		host_tag = le16_to_cpu(def_reply->host_tag);
 374		ioc_status = le16_to_cpu(def_reply->ioc_status);
 375		if (ioc_status &
 376		    MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
 377			ioc_loginfo = le32_to_cpu(def_reply->ioc_log_info);
 378		ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
 379		if (def_reply->function == MPI3_FUNCTION_SCSI_IO) {
 380			scsi_reply = (struct mpi3_scsi_io_reply *)def_reply;
 381			sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
 382			    le64_to_cpu(scsi_reply->sense_data_buffer_address));
 383		}
 384		break;
 385	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
 386		success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
 387		host_tag = le16_to_cpu(success_desc->host_tag);
 388		break;
 389	default:
 390		break;
 391	}
 392
 393	cmdptr = mpi3mr_get_drv_cmd(mrioc, host_tag, def_reply);
 394	if (cmdptr) {
 395		if (cmdptr->state & MPI3MR_CMD_PENDING) {
 396			cmdptr->state |= MPI3MR_CMD_COMPLETE;
 397			cmdptr->ioc_loginfo = ioc_loginfo;
 398			cmdptr->ioc_status = ioc_status;
 399			cmdptr->state &= ~MPI3MR_CMD_PENDING;
 400			if (def_reply) {
 401				cmdptr->state |= MPI3MR_CMD_REPLY_VALID;
 402				memcpy((u8 *)cmdptr->reply, (u8 *)def_reply,
 403				    mrioc->reply_sz);
 404			}
 405			if (sense_buf && cmdptr->sensebuf) {
 406				cmdptr->is_sense = 1;
 407				memcpy(cmdptr->sensebuf, sense_buf,
 408				       MPI3MR_SENSE_BUF_SZ);
 409			}
 410			if (cmdptr->is_waiting) {
 411				complete(&cmdptr->done);
 412				cmdptr->is_waiting = 0;
 413			} else if (cmdptr->callback)
 414				cmdptr->callback(mrioc, cmdptr);
 415		}
 416	}
 417out:
 418	if (sense_buf)
 419		mpi3mr_repost_sense_buf(mrioc,
 420		    le64_to_cpu(scsi_reply->sense_data_buffer_address));
 421}
 422
 423int mpi3mr_process_admin_reply_q(struct mpi3mr_ioc *mrioc)
 424{
 425	u32 exp_phase = mrioc->admin_reply_ephase;
 426	u32 admin_reply_ci = mrioc->admin_reply_ci;
 427	u32 num_admin_replies = 0;
 428	u64 reply_dma = 0;
 429	struct mpi3_default_reply_descriptor *reply_desc;
 430
 431	if (!atomic_add_unless(&mrioc->admin_reply_q_in_use, 1, 1))
 432		return 0;
 433
 434	reply_desc = (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 435	    admin_reply_ci;
 436
 437	if ((le16_to_cpu(reply_desc->reply_flags) &
 438	    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 439		atomic_dec(&mrioc->admin_reply_q_in_use);
 440		return 0;
 441	}
 442
 443	do {
 444		if (mrioc->unrecoverable)
 445			break;
 446
 447		mrioc->admin_req_ci = le16_to_cpu(reply_desc->request_queue_ci);
 448		mpi3mr_process_admin_reply_desc(mrioc, reply_desc, &reply_dma);
 449		if (reply_dma)
 450			mpi3mr_repost_reply_buf(mrioc, reply_dma);
 451		num_admin_replies++;
 452		if (++admin_reply_ci == mrioc->num_admin_replies) {
 453			admin_reply_ci = 0;
 454			exp_phase ^= 1;
 455		}
 456		reply_desc =
 457		    (struct mpi3_default_reply_descriptor *)mrioc->admin_reply_base +
 458		    admin_reply_ci;
 459		if ((le16_to_cpu(reply_desc->reply_flags) &
 460		    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 461			break;
 462	} while (1);
 463
 464	writel(admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
 465	mrioc->admin_reply_ci = admin_reply_ci;
 466	mrioc->admin_reply_ephase = exp_phase;
 467	atomic_dec(&mrioc->admin_reply_q_in_use);
 468
 469	return num_admin_replies;
 470}
 471
 472/**
 473 * mpi3mr_get_reply_desc - get reply descriptor frame corresponding to
 474 *	queue's consumer index from operational reply descriptor queue.
 475 * @op_reply_q: op_reply_qinfo object
 476 * @reply_ci: operational reply descriptor's queue consumer index
 477 *
 478 * Returns: reply descriptor frame address
 479 */
 480static inline struct mpi3_default_reply_descriptor *
 481mpi3mr_get_reply_desc(struct op_reply_qinfo *op_reply_q, u32 reply_ci)
 482{
 483	void *segment_base_addr;
 484	struct segments *segments = op_reply_q->q_segments;
 485	struct mpi3_default_reply_descriptor *reply_desc = NULL;
 486
 487	segment_base_addr =
 488	    segments[reply_ci / op_reply_q->segment_qd].segment;
 489	reply_desc = (struct mpi3_default_reply_descriptor *)segment_base_addr +
 490	    (reply_ci % op_reply_q->segment_qd);
 491	return reply_desc;
 492}
 493
 494/**
 495 * mpi3mr_process_op_reply_q - Operational reply queue handler
 496 * @mrioc: Adapter instance reference
 497 * @op_reply_q: Operational reply queue info
 498 *
 499 * Checks the specific operational reply queue and drains the
 500 * reply queue entries until the queue is empty and process the
 501 * individual reply descriptors.
 502 *
 503 * Return: 0 if queue is already processed,or number of reply
 504 *	    descriptors processed.
 505 */
 506int mpi3mr_process_op_reply_q(struct mpi3mr_ioc *mrioc,
 507	struct op_reply_qinfo *op_reply_q)
 508{
 509	struct op_req_qinfo *op_req_q;
 510	u32 exp_phase;
 511	u32 reply_ci;
 512	u32 num_op_reply = 0;
 513	u64 reply_dma = 0;
 514	struct mpi3_default_reply_descriptor *reply_desc;
 515	u16 req_q_idx = 0, reply_qidx;
 516
 517	reply_qidx = op_reply_q->qid - 1;
 518
 519	if (!atomic_add_unless(&op_reply_q->in_use, 1, 1))
 520		return 0;
 521
 522	exp_phase = op_reply_q->ephase;
 523	reply_ci = op_reply_q->ci;
 524
 525	reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 526	if ((le16_to_cpu(reply_desc->reply_flags) &
 527	    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase) {
 528		atomic_dec(&op_reply_q->in_use);
 529		return 0;
 530	}
 531
 532	do {
 533		if (mrioc->unrecoverable)
 534			break;
 535
 536		req_q_idx = le16_to_cpu(reply_desc->request_queue_id) - 1;
 537		op_req_q = &mrioc->req_qinfo[req_q_idx];
 538
 539		WRITE_ONCE(op_req_q->ci, le16_to_cpu(reply_desc->request_queue_ci));
 540		mpi3mr_process_op_reply_desc(mrioc, reply_desc, &reply_dma,
 541		    reply_qidx);
 542		atomic_dec(&op_reply_q->pend_ios);
 543		if (reply_dma)
 544			mpi3mr_repost_reply_buf(mrioc, reply_dma);
 545		num_op_reply++;
 546
 547		if (++reply_ci == op_reply_q->num_replies) {
 548			reply_ci = 0;
 549			exp_phase ^= 1;
 550		}
 551
 552		reply_desc = mpi3mr_get_reply_desc(op_reply_q, reply_ci);
 553
 554		if ((le16_to_cpu(reply_desc->reply_flags) &
 555		    MPI3_REPLY_DESCRIPT_FLAGS_PHASE_MASK) != exp_phase)
 556			break;
 557#ifndef CONFIG_PREEMPT_RT
 558		/*
 559		 * Exit completion loop to avoid CPU lockup
 560		 * Ensure remaining completion happens from threaded ISR.
 561		 */
 562		if (num_op_reply > mrioc->max_host_ios) {
 563			op_reply_q->enable_irq_poll = true;
 564			break;
 565		}
 566#endif
 567	} while (1);
 568
 569	writel(reply_ci,
 570	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].consumer_index);
 571	op_reply_q->ci = reply_ci;
 572	op_reply_q->ephase = exp_phase;
 573
 574	atomic_dec(&op_reply_q->in_use);
 575	return num_op_reply;
 576}
 577
 578/**
 579 * mpi3mr_blk_mq_poll - Operational reply queue handler
 580 * @shost: SCSI Host reference
 581 * @queue_num: Request queue number (w.r.t OS it is hardware context number)
 582 *
 583 * Checks the specific operational reply queue and drains the
 584 * reply queue entries until the queue is empty and process the
 585 * individual reply descriptors.
 586 *
 587 * Return: 0 if queue is already processed,or number of reply
 588 *	    descriptors processed.
 589 */
 590int mpi3mr_blk_mq_poll(struct Scsi_Host *shost, unsigned int queue_num)
 591{
 592	int num_entries = 0;
 593	struct mpi3mr_ioc *mrioc;
 594
 595	mrioc = (struct mpi3mr_ioc *)shost->hostdata;
 596
 597	if ((mrioc->reset_in_progress || mrioc->prepare_for_reset ||
 598	    mrioc->unrecoverable))
 599		return 0;
 600
 601	num_entries = mpi3mr_process_op_reply_q(mrioc,
 602			&mrioc->op_reply_qinfo[queue_num]);
 603
 604	return num_entries;
 605}
 606
 607static irqreturn_t mpi3mr_isr_primary(int irq, void *privdata)
 608{
 609	struct mpi3mr_intr_info *intr_info = privdata;
 610	struct mpi3mr_ioc *mrioc;
 611	u16 midx;
 612	u32 num_admin_replies = 0, num_op_reply = 0;
 613
 614	if (!intr_info)
 615		return IRQ_NONE;
 616
 617	mrioc = intr_info->mrioc;
 618
 619	if (!mrioc->intr_enabled)
 620		return IRQ_NONE;
 621
 622	midx = intr_info->msix_index;
 623
 624	if (!midx)
 625		num_admin_replies = mpi3mr_process_admin_reply_q(mrioc);
 626	if (intr_info->op_reply_q)
 627		num_op_reply = mpi3mr_process_op_reply_q(mrioc,
 628		    intr_info->op_reply_q);
 629
 630	if (num_admin_replies || num_op_reply)
 631		return IRQ_HANDLED;
 632	else
 633		return IRQ_NONE;
 634}
 635
 636#ifndef CONFIG_PREEMPT_RT
 637
 638static irqreturn_t mpi3mr_isr(int irq, void *privdata)
 639{
 640	struct mpi3mr_intr_info *intr_info = privdata;
 641	int ret;
 642
 643	if (!intr_info)
 644		return IRQ_NONE;
 645
 646	/* Call primary ISR routine */
 647	ret = mpi3mr_isr_primary(irq, privdata);
 648
 649	/*
 650	 * If more IOs are expected, schedule IRQ polling thread.
 651	 * Otherwise exit from ISR.
 652	 */
 653	if (!intr_info->op_reply_q)
 654		return ret;
 655
 656	if (!intr_info->op_reply_q->enable_irq_poll ||
 657	    !atomic_read(&intr_info->op_reply_q->pend_ios))
 658		return ret;
 659
 660	disable_irq_nosync(intr_info->os_irq);
 661
 662	return IRQ_WAKE_THREAD;
 663}
 664
 665/**
 666 * mpi3mr_isr_poll - Reply queue polling routine
 667 * @irq: IRQ
 668 * @privdata: Interrupt info
 669 *
 670 * poll for pending I/O completions in a loop until pending I/Os
 671 * present or controller queue depth I/Os are processed.
 672 *
 673 * Return: IRQ_NONE or IRQ_HANDLED
 674 */
 675static irqreturn_t mpi3mr_isr_poll(int irq, void *privdata)
 676{
 677	struct mpi3mr_intr_info *intr_info = privdata;
 678	struct mpi3mr_ioc *mrioc;
 679	u16 midx;
 680	u32 num_op_reply = 0;
 681
 682	if (!intr_info || !intr_info->op_reply_q)
 683		return IRQ_NONE;
 684
 685	mrioc = intr_info->mrioc;
 686	midx = intr_info->msix_index;
 687
 688	/* Poll for pending IOs completions */
 689	do {
 690		if (!mrioc->intr_enabled || mrioc->unrecoverable)
 691			break;
 692
 693		if (!midx)
 694			mpi3mr_process_admin_reply_q(mrioc);
 695		if (intr_info->op_reply_q)
 696			num_op_reply +=
 697			    mpi3mr_process_op_reply_q(mrioc,
 698				intr_info->op_reply_q);
 699
 700		usleep_range(MPI3MR_IRQ_POLL_SLEEP, 10 * MPI3MR_IRQ_POLL_SLEEP);
 701
 702	} while (atomic_read(&intr_info->op_reply_q->pend_ios) &&
 703	    (num_op_reply < mrioc->max_host_ios));
 704
 705	intr_info->op_reply_q->enable_irq_poll = false;
 706	enable_irq(intr_info->os_irq);
 707
 708	return IRQ_HANDLED;
 709}
 710
 711#endif
 712
 713/**
 714 * mpi3mr_request_irq - Request IRQ and register ISR
 715 * @mrioc: Adapter instance reference
 716 * @index: IRQ vector index
 717 *
 718 * Request threaded ISR with primary ISR and secondary
 719 *
 720 * Return: 0 on success and non zero on failures.
 721 */
 722static inline int mpi3mr_request_irq(struct mpi3mr_ioc *mrioc, u16 index)
 723{
 724	struct pci_dev *pdev = mrioc->pdev;
 725	struct mpi3mr_intr_info *intr_info = mrioc->intr_info + index;
 726	int retval = 0;
 727
 728	intr_info->mrioc = mrioc;
 729	intr_info->msix_index = index;
 730	intr_info->op_reply_q = NULL;
 731
 732	snprintf(intr_info->name, MPI3MR_NAME_LENGTH, "%s%d-msix%d",
 733	    mrioc->driver_name, mrioc->id, index);
 734
 735#ifndef CONFIG_PREEMPT_RT
 736	retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr,
 737	    mpi3mr_isr_poll, IRQF_SHARED, intr_info->name, intr_info);
 738#else
 739	retval = request_threaded_irq(pci_irq_vector(pdev, index), mpi3mr_isr_primary,
 740	    NULL, IRQF_SHARED, intr_info->name, intr_info);
 741#endif
 742	if (retval) {
 743		ioc_err(mrioc, "%s: Unable to allocate interrupt %d!\n",
 744		    intr_info->name, pci_irq_vector(pdev, index));
 745		return retval;
 746	}
 747
 748	intr_info->os_irq = pci_irq_vector(pdev, index);
 749	return retval;
 750}
 751
 752static void mpi3mr_calc_poll_queues(struct mpi3mr_ioc *mrioc, u16 max_vectors)
 753{
 754	if (!mrioc->requested_poll_qcount)
 755		return;
 756
 757	/* Reserved for Admin and Default Queue */
 758	if (max_vectors > 2 &&
 759		(mrioc->requested_poll_qcount < max_vectors - 2)) {
 760		ioc_info(mrioc,
 761		    "enabled polled queues (%d) msix (%d)\n",
 762		    mrioc->requested_poll_qcount, max_vectors);
 763	} else {
 764		ioc_info(mrioc,
 765		    "disabled polled queues (%d) msix (%d) because of no resources for default queue\n",
 766		    mrioc->requested_poll_qcount, max_vectors);
 767		mrioc->requested_poll_qcount = 0;
 768	}
 769}
 770
 771/**
 772 * mpi3mr_setup_isr - Setup ISR for the controller
 773 * @mrioc: Adapter instance reference
 774 * @setup_one: Request one IRQ or more
 775 *
 776 * Allocate IRQ vectors and call mpi3mr_request_irq to setup ISR
 777 *
 778 * Return: 0 on success and non zero on failures.
 779 */
 780static int mpi3mr_setup_isr(struct mpi3mr_ioc *mrioc, u8 setup_one)
 781{
 782	unsigned int irq_flags = PCI_IRQ_MSIX;
 783	int max_vectors, min_vec;
 784	int retval;
 785	int i;
 786	struct irq_affinity desc = { .pre_vectors =  1, .post_vectors = 1 };
 787
 788	if (mrioc->is_intr_info_set)
 789		return 0;
 790
 791	mpi3mr_cleanup_isr(mrioc);
 792
 793	if (setup_one || reset_devices) {
 794		max_vectors = 1;
 795		retval = pci_alloc_irq_vectors(mrioc->pdev,
 796		    1, max_vectors, irq_flags);
 797		if (retval < 0) {
 798			ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
 799			    retval);
 800			goto out_failed;
 801		}
 802	} else {
 803		max_vectors =
 804		    min_t(int, mrioc->cpu_count + 1 +
 805			mrioc->requested_poll_qcount, mrioc->msix_count);
 806
 807		mpi3mr_calc_poll_queues(mrioc, max_vectors);
 808
 809		ioc_info(mrioc,
 810		    "MSI-X vectors supported: %d, no of cores: %d,",
 811		    mrioc->msix_count, mrioc->cpu_count);
 812		ioc_info(mrioc,
 813		    "MSI-x vectors requested: %d poll_queues %d\n",
 814		    max_vectors, mrioc->requested_poll_qcount);
 815
 816		desc.post_vectors = mrioc->requested_poll_qcount;
 817		min_vec = desc.pre_vectors + desc.post_vectors;
 818		irq_flags |= PCI_IRQ_AFFINITY | PCI_IRQ_ALL_TYPES;
 819
 820		retval = pci_alloc_irq_vectors_affinity(mrioc->pdev,
 821			min_vec, max_vectors, irq_flags, &desc);
 822
 823		if (retval < 0) {
 824			ioc_err(mrioc, "cannot allocate irq vectors, ret %d\n",
 825			    retval);
 826			goto out_failed;
 827		}
 828
 829
 830		/*
 831		 * If only one MSI-x is allocated, then MSI-x 0 will be shared
 832		 * between Admin queue and operational queue
 833		 */
 834		if (retval == min_vec)
 835			mrioc->op_reply_q_offset = 0;
 836		else if (retval != (max_vectors)) {
 837			ioc_info(mrioc,
 838			    "allocated vectors (%d) are less than configured (%d)\n",
 839			    retval, max_vectors);
 840		}
 841
 842		max_vectors = retval;
 843		mrioc->op_reply_q_offset = (max_vectors > 1) ? 1 : 0;
 844
 845		mpi3mr_calc_poll_queues(mrioc, max_vectors);
 846
 847	}
 848
 849	mrioc->intr_info = kzalloc(sizeof(struct mpi3mr_intr_info) * max_vectors,
 850	    GFP_KERNEL);
 851	if (!mrioc->intr_info) {
 852		retval = -ENOMEM;
 853		pci_free_irq_vectors(mrioc->pdev);
 854		goto out_failed;
 855	}
 856	for (i = 0; i < max_vectors; i++) {
 857		retval = mpi3mr_request_irq(mrioc, i);
 858		if (retval) {
 859			mrioc->intr_info_count = i;
 860			goto out_failed;
 861		}
 862	}
 863	if (reset_devices || !setup_one)
 864		mrioc->is_intr_info_set = true;
 865	mrioc->intr_info_count = max_vectors;
 866	mpi3mr_ioc_enable_intr(mrioc);
 867	return 0;
 868
 869out_failed:
 870	mpi3mr_cleanup_isr(mrioc);
 871
 872	return retval;
 873}
 874
 875static const struct {
 876	enum mpi3mr_iocstate value;
 877	char *name;
 878} mrioc_states[] = {
 879	{ MRIOC_STATE_READY, "ready" },
 880	{ MRIOC_STATE_FAULT, "fault" },
 881	{ MRIOC_STATE_RESET, "reset" },
 882	{ MRIOC_STATE_BECOMING_READY, "becoming ready" },
 883	{ MRIOC_STATE_RESET_REQUESTED, "reset requested" },
 884	{ MRIOC_STATE_UNRECOVERABLE, "unrecoverable error" },
 885};
 886
 887static const char *mpi3mr_iocstate_name(enum mpi3mr_iocstate mrioc_state)
 888{
 889	int i;
 890	char *name = NULL;
 891
 892	for (i = 0; i < ARRAY_SIZE(mrioc_states); i++) {
 893		if (mrioc_states[i].value == mrioc_state) {
 894			name = mrioc_states[i].name;
 895			break;
 896		}
 897	}
 898	return name;
 899}
 900
 901/* Reset reason to name mapper structure*/
 902static const struct {
 903	enum mpi3mr_reset_reason value;
 904	char *name;
 905} mpi3mr_reset_reason_codes[] = {
 906	{ MPI3MR_RESET_FROM_BRINGUP, "timeout in bringup" },
 907	{ MPI3MR_RESET_FROM_FAULT_WATCH, "fault" },
 908	{ MPI3MR_RESET_FROM_APP, "application invocation" },
 909	{ MPI3MR_RESET_FROM_EH_HOS, "error handling" },
 910	{ MPI3MR_RESET_FROM_TM_TIMEOUT, "TM timeout" },
 911	{ MPI3MR_RESET_FROM_APP_TIMEOUT, "application command timeout" },
 912	{ MPI3MR_RESET_FROM_MUR_FAILURE, "MUR failure" },
 913	{ MPI3MR_RESET_FROM_CTLR_CLEANUP, "timeout in controller cleanup" },
 914	{ MPI3MR_RESET_FROM_CIACTIV_FAULT, "component image activation fault" },
 915	{ MPI3MR_RESET_FROM_PE_TIMEOUT, "port enable timeout" },
 916	{ MPI3MR_RESET_FROM_TSU_TIMEOUT, "time stamp update timeout" },
 917	{ MPI3MR_RESET_FROM_DELREQQ_TIMEOUT, "delete request queue timeout" },
 918	{ MPI3MR_RESET_FROM_DELREPQ_TIMEOUT, "delete reply queue timeout" },
 919	{
 920		MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT,
 921		"create request queue timeout"
 922	},
 923	{
 924		MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT,
 925		"create reply queue timeout"
 926	},
 927	{ MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT, "IOC facts timeout" },
 928	{ MPI3MR_RESET_FROM_IOCINIT_TIMEOUT, "IOC init timeout" },
 929	{ MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT, "event notify timeout" },
 930	{ MPI3MR_RESET_FROM_EVTACK_TIMEOUT, "event acknowledgment timeout" },
 931	{
 932		MPI3MR_RESET_FROM_CIACTVRST_TIMER,
 933		"component image activation timeout"
 934	},
 935	{
 936		MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT,
 937		"get package version timeout"
 938	},
 939	{ MPI3MR_RESET_FROM_SYSFS, "sysfs invocation" },
 940	{ MPI3MR_RESET_FROM_SYSFS_TIMEOUT, "sysfs TM timeout" },
 941	{ MPI3MR_RESET_FROM_FIRMWARE, "firmware asynchronous reset" },
 942	{ MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT, "configuration request timeout"},
 943	{ MPI3MR_RESET_FROM_SAS_TRANSPORT_TIMEOUT, "timeout of a SAS transport layer request" },
 944};
 945
 946/**
 947 * mpi3mr_reset_rc_name - get reset reason code name
 948 * @reason_code: reset reason code value
 949 *
 950 * Map reset reason to an NULL terminated ASCII string
 951 *
 952 * Return: name corresponding to reset reason value or NULL.
 953 */
 954static const char *mpi3mr_reset_rc_name(enum mpi3mr_reset_reason reason_code)
 955{
 956	int i;
 957	char *name = NULL;
 958
 959	for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_reason_codes); i++) {
 960		if (mpi3mr_reset_reason_codes[i].value == reason_code) {
 961			name = mpi3mr_reset_reason_codes[i].name;
 962			break;
 963		}
 964	}
 965	return name;
 966}
 967
 968/* Reset type to name mapper structure*/
 969static const struct {
 970	u16 reset_type;
 971	char *name;
 972} mpi3mr_reset_types[] = {
 973	{ MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, "soft" },
 974	{ MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, "diag fault" },
 975};
 976
 977/**
 978 * mpi3mr_reset_type_name - get reset type name
 979 * @reset_type: reset type value
 980 *
 981 * Map reset type to an NULL terminated ASCII string
 982 *
 983 * Return: name corresponding to reset type value or NULL.
 984 */
 985static const char *mpi3mr_reset_type_name(u16 reset_type)
 986{
 987	int i;
 988	char *name = NULL;
 989
 990	for (i = 0; i < ARRAY_SIZE(mpi3mr_reset_types); i++) {
 991		if (mpi3mr_reset_types[i].reset_type == reset_type) {
 992			name = mpi3mr_reset_types[i].name;
 993			break;
 994		}
 995	}
 996	return name;
 997}
 998
 999/**
1000 * mpi3mr_print_fault_info - Display fault information
1001 * @mrioc: Adapter instance reference
1002 *
1003 * Display the controller fault information if there is a
1004 * controller fault.
1005 *
1006 * Return: Nothing.
1007 */
1008void mpi3mr_print_fault_info(struct mpi3mr_ioc *mrioc)
1009{
1010	u32 ioc_status, code, code1, code2, code3;
1011
1012	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1013
1014	if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1015		code = readl(&mrioc->sysif_regs->fault);
1016		code1 = readl(&mrioc->sysif_regs->fault_info[0]);
1017		code2 = readl(&mrioc->sysif_regs->fault_info[1]);
1018		code3 = readl(&mrioc->sysif_regs->fault_info[2]);
1019
1020		ioc_info(mrioc,
1021		    "fault code(0x%08X): Additional code: (0x%08X:0x%08X:0x%08X)\n",
1022		    code, code1, code2, code3);
1023	}
1024}
1025
1026/**
1027 * mpi3mr_get_iocstate - Get IOC State
1028 * @mrioc: Adapter instance reference
1029 *
1030 * Return a proper IOC state enum based on the IOC status and
1031 * IOC configuration and unrcoverable state of the controller.
1032 *
1033 * Return: Current IOC state.
1034 */
1035enum mpi3mr_iocstate mpi3mr_get_iocstate(struct mpi3mr_ioc *mrioc)
1036{
1037	u32 ioc_status, ioc_config;
1038	u8 ready, enabled;
1039
1040	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1041	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1042
1043	if (mrioc->unrecoverable)
1044		return MRIOC_STATE_UNRECOVERABLE;
1045	if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)
1046		return MRIOC_STATE_FAULT;
1047
1048	ready = (ioc_status & MPI3_SYSIF_IOC_STATUS_READY);
1049	enabled = (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC);
1050
1051	if (ready && enabled)
1052		return MRIOC_STATE_READY;
1053	if ((!ready) && (!enabled))
1054		return MRIOC_STATE_RESET;
1055	if ((!ready) && (enabled))
1056		return MRIOC_STATE_BECOMING_READY;
1057
1058	return MRIOC_STATE_RESET_REQUESTED;
1059}
1060
1061/**
1062 * mpi3mr_free_ioctl_dma_memory - free memory for ioctl dma
1063 * @mrioc: Adapter instance reference
1064 *
1065 * Free the DMA memory allocated for IOCTL handling purpose.
1066 *
1067 * Return: None
1068 */
1069static void mpi3mr_free_ioctl_dma_memory(struct mpi3mr_ioc *mrioc)
1070{
1071	struct dma_memory_desc *mem_desc;
1072	u16 i;
1073
1074	if (!mrioc->ioctl_dma_pool)
1075		return;
1076
1077	for (i = 0; i < MPI3MR_NUM_IOCTL_SGE; i++) {
1078		mem_desc = &mrioc->ioctl_sge[i];
1079		if (mem_desc->addr) {
1080			dma_pool_free(mrioc->ioctl_dma_pool,
1081				      mem_desc->addr,
1082				      mem_desc->dma_addr);
1083			mem_desc->addr = NULL;
1084		}
1085	}
1086	dma_pool_destroy(mrioc->ioctl_dma_pool);
1087	mrioc->ioctl_dma_pool = NULL;
1088	mem_desc = &mrioc->ioctl_chain_sge;
1089
1090	if (mem_desc->addr) {
1091		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,
1092				  mem_desc->addr, mem_desc->dma_addr);
1093		mem_desc->addr = NULL;
1094	}
1095	mem_desc = &mrioc->ioctl_resp_sge;
1096	if (mem_desc->addr) {
1097		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,
1098				  mem_desc->addr, mem_desc->dma_addr);
1099		mem_desc->addr = NULL;
1100	}
1101
1102	mrioc->ioctl_sges_allocated = false;
1103}
1104
1105/**
1106 * mpi3mr_alloc_ioctl_dma_memory - Alloc memory for ioctl dma
1107 * @mrioc: Adapter instance reference
1108 *
1109 * This function allocates dmaable memory required to handle the
1110 * application issued MPI3 IOCTL requests.
1111 *
1112 * Return: None
1113 */
1114static void mpi3mr_alloc_ioctl_dma_memory(struct mpi3mr_ioc *mrioc)
1115
1116{
1117	struct dma_memory_desc *mem_desc;
1118	u16 i;
1119
1120	mrioc->ioctl_dma_pool = dma_pool_create("ioctl dma pool",
1121						&mrioc->pdev->dev,
1122						MPI3MR_IOCTL_SGE_SIZE,
1123						MPI3MR_PAGE_SIZE_4K, 0);
1124
1125	if (!mrioc->ioctl_dma_pool) {
1126		ioc_err(mrioc, "ioctl_dma_pool: dma_pool_create failed\n");
1127		goto out_failed;
1128	}
1129
1130	for (i = 0; i < MPI3MR_NUM_IOCTL_SGE; i++) {
1131		mem_desc = &mrioc->ioctl_sge[i];
1132		mem_desc->size = MPI3MR_IOCTL_SGE_SIZE;
1133		mem_desc->addr = dma_pool_zalloc(mrioc->ioctl_dma_pool,
1134						 GFP_KERNEL,
1135						 &mem_desc->dma_addr);
1136		if (!mem_desc->addr)
1137			goto out_failed;
1138	}
1139
1140	mem_desc = &mrioc->ioctl_chain_sge;
1141	mem_desc->size = MPI3MR_PAGE_SIZE_4K;
1142	mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,
1143					    mem_desc->size,
1144					    &mem_desc->dma_addr,
1145					    GFP_KERNEL);
1146	if (!mem_desc->addr)
1147		goto out_failed;
1148
1149	mem_desc = &mrioc->ioctl_resp_sge;
1150	mem_desc->size = MPI3MR_PAGE_SIZE_4K;
1151	mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,
1152					    mem_desc->size,
1153					    &mem_desc->dma_addr,
1154					    GFP_KERNEL);
1155	if (!mem_desc->addr)
1156		goto out_failed;
1157
1158	mrioc->ioctl_sges_allocated = true;
1159
1160	return;
1161out_failed:
1162	ioc_warn(mrioc, "cannot allocate DMA memory for the mpt commands\n"
1163		 "from the applications, application interface for MPT command is disabled\n");
1164	mpi3mr_free_ioctl_dma_memory(mrioc);
1165}
1166
1167/**
1168 * mpi3mr_clear_reset_history - clear reset history
1169 * @mrioc: Adapter instance reference
1170 *
1171 * Write the reset history bit in IOC status to clear the bit,
1172 * if it is already set.
1173 *
1174 * Return: Nothing.
1175 */
1176static inline void mpi3mr_clear_reset_history(struct mpi3mr_ioc *mrioc)
1177{
1178	u32 ioc_status;
1179
1180	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1181	if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1182		writel(ioc_status, &mrioc->sysif_regs->ioc_status);
1183}
1184
1185/**
1186 * mpi3mr_issue_and_process_mur - Message unit Reset handler
1187 * @mrioc: Adapter instance reference
1188 * @reset_reason: Reset reason code
1189 *
1190 * Issue Message unit Reset to the controller and wait for it to
1191 * be complete.
1192 *
1193 * Return: 0 on success, -1 on failure.
1194 */
1195static int mpi3mr_issue_and_process_mur(struct mpi3mr_ioc *mrioc,
1196	u32 reset_reason)
1197{
1198	u32 ioc_config, timeout, ioc_status;
1199	int retval = -1;
1200
1201	ioc_info(mrioc, "Issuing Message unit Reset(MUR)\n");
1202	if (mrioc->unrecoverable) {
1203		ioc_info(mrioc, "IOC is unrecoverable MUR not issued\n");
1204		return retval;
1205	}
1206	mpi3mr_clear_reset_history(mrioc);
1207	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1208	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1209	ioc_config &= ~MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1210	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1211
1212	timeout = MPI3MR_MUR_TIMEOUT * 10;
1213	do {
1214		ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1215		if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)) {
1216			mpi3mr_clear_reset_history(mrioc);
1217			break;
1218		}
1219		if (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) {
1220			mpi3mr_print_fault_info(mrioc);
1221			break;
1222		}
1223		msleep(100);
1224	} while (--timeout);
1225
1226	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1227	if (timeout && !((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1228	      (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT) ||
1229	      (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1230		retval = 0;
1231
1232	ioc_info(mrioc, "Base IOC Sts/Config after %s MUR is (0x%x)/(0x%x)\n",
1233	    (!retval) ? "successful" : "failed", ioc_status, ioc_config);
1234	return retval;
1235}
1236
1237/**
1238 * mpi3mr_revalidate_factsdata - validate IOCFacts parameters
1239 * during reset/resume
1240 * @mrioc: Adapter instance reference
1241 *
1242 * Return: zero if the new IOCFacts parameters value is compatible with
1243 * older values else return -EPERM
1244 */
1245static int
1246mpi3mr_revalidate_factsdata(struct mpi3mr_ioc *mrioc)
1247{
1248	unsigned long *removepend_bitmap;
 
1249
1250	if (mrioc->facts.reply_sz > mrioc->reply_sz) {
1251		ioc_err(mrioc,
1252		    "cannot increase reply size from %d to %d\n",
1253		    mrioc->reply_sz, mrioc->facts.reply_sz);
1254		return -EPERM;
1255	}
1256
1257	if (mrioc->facts.max_op_reply_q < mrioc->num_op_reply_q) {
1258		ioc_err(mrioc,
1259		    "cannot reduce number of operational reply queues from %d to %d\n",
1260		    mrioc->num_op_reply_q,
1261		    mrioc->facts.max_op_reply_q);
1262		return -EPERM;
1263	}
1264
1265	if (mrioc->facts.max_op_req_q < mrioc->num_op_req_q) {
1266		ioc_err(mrioc,
1267		    "cannot reduce number of operational request queues from %d to %d\n",
1268		    mrioc->num_op_req_q, mrioc->facts.max_op_req_q);
1269		return -EPERM;
1270	}
1271
1272	if (mrioc->shost->max_sectors != (mrioc->facts.max_data_length / 512))
1273		ioc_err(mrioc, "Warning: The maximum data transfer length\n"
1274			    "\tchanged after reset: previous(%d), new(%d),\n"
1275			    "the driver cannot change this at run time\n",
1276			    mrioc->shost->max_sectors * 512, mrioc->facts.max_data_length);
1277
1278	if ((mrioc->sas_transport_enabled) && (mrioc->facts.ioc_capabilities &
1279	    MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED))
1280		ioc_err(mrioc,
1281		    "critical error: multipath capability is enabled at the\n"
1282		    "\tcontroller while sas transport support is enabled at the\n"
1283		    "\tdriver, please reboot the system or reload the driver\n");
1284
1285	if (mrioc->facts.max_devhandle > mrioc->dev_handle_bitmap_bits) {
1286		removepend_bitmap = bitmap_zalloc(mrioc->facts.max_devhandle,
1287						  GFP_KERNEL);
 
 
 
1288		if (!removepend_bitmap) {
1289			ioc_err(mrioc,
1290				"failed to increase removepend_bitmap bits from %d to %d\n",
1291				mrioc->dev_handle_bitmap_bits,
1292				mrioc->facts.max_devhandle);
1293			return -EPERM;
1294		}
1295		bitmap_free(mrioc->removepend_bitmap);
 
1296		mrioc->removepend_bitmap = removepend_bitmap;
1297		ioc_info(mrioc,
1298			 "increased bits of dev_handle_bitmap from %d to %d\n",
1299			 mrioc->dev_handle_bitmap_bits,
1300			 mrioc->facts.max_devhandle);
1301		mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
1302	}
1303
1304	return 0;
1305}
1306
1307/**
1308 * mpi3mr_bring_ioc_ready - Bring controller to ready state
1309 * @mrioc: Adapter instance reference
1310 *
1311 * Set Enable IOC bit in IOC configuration register and wait for
1312 * the controller to become ready.
1313 *
1314 * Return: 0 on success, appropriate error on failure.
1315 */
1316static int mpi3mr_bring_ioc_ready(struct mpi3mr_ioc *mrioc)
1317{
1318	u32 ioc_config, ioc_status, timeout, host_diagnostic;
1319	int retval = 0;
1320	enum mpi3mr_iocstate ioc_state;
1321	u64 base_info;
1322
1323	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1324	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1325	base_info = lo_hi_readq(&mrioc->sysif_regs->ioc_information);
1326	ioc_info(mrioc, "ioc_status(0x%08x), ioc_config(0x%08x), ioc_info(0x%016llx) at the bringup\n",
1327	    ioc_status, ioc_config, base_info);
1328
1329	/*The timeout value is in 2sec unit, changing it to seconds*/
1330	mrioc->ready_timeout =
1331	    ((base_info & MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_MASK) >>
1332	    MPI3_SYSIF_IOC_INFO_LOW_TIMEOUT_SHIFT) * 2;
1333
1334	ioc_info(mrioc, "ready timeout: %d seconds\n", mrioc->ready_timeout);
1335
1336	ioc_state = mpi3mr_get_iocstate(mrioc);
1337	ioc_info(mrioc, "controller is in %s state during detection\n",
1338	    mpi3mr_iocstate_name(ioc_state));
1339
1340	if (ioc_state == MRIOC_STATE_BECOMING_READY ||
1341	    ioc_state == MRIOC_STATE_RESET_REQUESTED) {
1342		timeout = mrioc->ready_timeout * 10;
1343		do {
1344			msleep(100);
1345		} while (--timeout);
1346
1347		if (!pci_device_is_present(mrioc->pdev)) {
1348			mrioc->unrecoverable = 1;
1349			ioc_err(mrioc,
1350			    "controller is not present while waiting to reset\n");
1351			retval = -1;
1352			goto out_device_not_present;
1353		}
1354
1355		ioc_state = mpi3mr_get_iocstate(mrioc);
1356		ioc_info(mrioc,
1357		    "controller is in %s state after waiting to reset\n",
1358		    mpi3mr_iocstate_name(ioc_state));
1359	}
1360
1361	if (ioc_state == MRIOC_STATE_READY) {
1362		ioc_info(mrioc, "issuing message unit reset (MUR) to bring to reset state\n");
1363		retval = mpi3mr_issue_and_process_mur(mrioc,
1364		    MPI3MR_RESET_FROM_BRINGUP);
1365		ioc_state = mpi3mr_get_iocstate(mrioc);
1366		if (retval)
1367			ioc_err(mrioc,
1368			    "message unit reset failed with error %d current state %s\n",
1369			    retval, mpi3mr_iocstate_name(ioc_state));
1370	}
1371	if (ioc_state != MRIOC_STATE_RESET) {
1372		if (ioc_state == MRIOC_STATE_FAULT) {
1373			timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
1374			mpi3mr_print_fault_info(mrioc);
1375			do {
1376				host_diagnostic =
1377					readl(&mrioc->sysif_regs->host_diagnostic);
1378				if (!(host_diagnostic &
1379				      MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
1380					break;
1381				if (!pci_device_is_present(mrioc->pdev)) {
1382					mrioc->unrecoverable = 1;
1383					ioc_err(mrioc, "controller is not present at the bringup\n");
1384					goto out_device_not_present;
1385				}
1386				msleep(100);
1387			} while (--timeout);
1388		}
1389		mpi3mr_print_fault_info(mrioc);
1390		ioc_info(mrioc, "issuing soft reset to bring to reset state\n");
1391		retval = mpi3mr_issue_reset(mrioc,
1392		    MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
1393		    MPI3MR_RESET_FROM_BRINGUP);
1394		if (retval) {
1395			ioc_err(mrioc,
1396			    "soft reset failed with error %d\n", retval);
1397			goto out_failed;
1398		}
1399	}
1400	ioc_state = mpi3mr_get_iocstate(mrioc);
1401	if (ioc_state != MRIOC_STATE_RESET) {
1402		ioc_err(mrioc,
1403		    "cannot bring controller to reset state, current state: %s\n",
1404		    mpi3mr_iocstate_name(ioc_state));
1405		goto out_failed;
1406	}
1407	mpi3mr_clear_reset_history(mrioc);
1408	retval = mpi3mr_setup_admin_qpair(mrioc);
1409	if (retval) {
1410		ioc_err(mrioc, "failed to setup admin queues: error %d\n",
1411		    retval);
1412		goto out_failed;
1413	}
1414
1415	ioc_info(mrioc, "bringing controller to ready state\n");
1416	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1417	ioc_config |= MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC;
1418	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1419
1420	timeout = mrioc->ready_timeout * 10;
1421	do {
1422		ioc_state = mpi3mr_get_iocstate(mrioc);
1423		if (ioc_state == MRIOC_STATE_READY) {
1424			ioc_info(mrioc,
1425			    "successfully transitioned to %s state\n",
1426			    mpi3mr_iocstate_name(ioc_state));
1427			return 0;
1428		}
1429		if (!pci_device_is_present(mrioc->pdev)) {
1430			mrioc->unrecoverable = 1;
1431			ioc_err(mrioc,
1432			    "controller is not present at the bringup\n");
1433			retval = -1;
1434			goto out_device_not_present;
1435		}
1436		msleep(100);
1437	} while (--timeout);
1438
1439out_failed:
1440	ioc_state = mpi3mr_get_iocstate(mrioc);
1441	ioc_err(mrioc,
1442	    "failed to bring to ready state,  current state: %s\n",
1443	    mpi3mr_iocstate_name(ioc_state));
1444out_device_not_present:
1445	return retval;
1446}
1447
1448/**
1449 * mpi3mr_soft_reset_success - Check softreset is success or not
1450 * @ioc_status: IOC status register value
1451 * @ioc_config: IOC config register value
1452 *
1453 * Check whether the soft reset is successful or not based on
1454 * IOC status and IOC config register values.
1455 *
1456 * Return: True when the soft reset is success, false otherwise.
1457 */
1458static inline bool
1459mpi3mr_soft_reset_success(u32 ioc_status, u32 ioc_config)
1460{
1461	if (!((ioc_status & MPI3_SYSIF_IOC_STATUS_READY) ||
1462	    (ioc_config & MPI3_SYSIF_IOC_CONFIG_ENABLE_IOC)))
1463		return true;
1464	return false;
1465}
1466
1467/**
1468 * mpi3mr_diagfault_success - Check diag fault is success or not
1469 * @mrioc: Adapter reference
1470 * @ioc_status: IOC status register value
1471 *
1472 * Check whether the controller hit diag reset fault code.
1473 *
1474 * Return: True when there is diag fault, false otherwise.
1475 */
1476static inline bool mpi3mr_diagfault_success(struct mpi3mr_ioc *mrioc,
1477	u32 ioc_status)
1478{
1479	u32 fault;
1480
1481	if (!(ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT))
1482		return false;
1483	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
1484	if (fault == MPI3_SYSIF_FAULT_CODE_DIAG_FAULT_RESET) {
1485		mpi3mr_print_fault_info(mrioc);
1486		return true;
1487	}
1488	return false;
1489}
1490
1491/**
1492 * mpi3mr_set_diagsave - Set diag save bit for snapdump
1493 * @mrioc: Adapter reference
1494 *
1495 * Set diag save bit in IOC configuration register to enable
1496 * snapdump.
1497 *
1498 * Return: Nothing.
1499 */
1500static inline void mpi3mr_set_diagsave(struct mpi3mr_ioc *mrioc)
1501{
1502	u32 ioc_config;
1503
1504	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1505	ioc_config |= MPI3_SYSIF_IOC_CONFIG_DIAG_SAVE;
1506	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
1507}
1508
1509/**
1510 * mpi3mr_issue_reset - Issue reset to the controller
1511 * @mrioc: Adapter reference
1512 * @reset_type: Reset type
1513 * @reset_reason: Reset reason code
1514 *
1515 * Unlock the host diagnostic registers and write the specific
1516 * reset type to that, wait for reset acknowledgment from the
1517 * controller, if the reset is not successful retry for the
1518 * predefined number of times.
1519 *
1520 * Return: 0 on success, non-zero on failure.
1521 */
1522static int mpi3mr_issue_reset(struct mpi3mr_ioc *mrioc, u16 reset_type,
1523	u32 reset_reason)
1524{
1525	int retval = -1;
1526	u8 unlock_retry_count = 0;
1527	u32 host_diagnostic, ioc_status, ioc_config;
1528	u32 timeout = MPI3MR_RESET_ACK_TIMEOUT * 10;
1529
1530	if ((reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET) &&
1531	    (reset_type != MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT))
1532		return retval;
1533	if (mrioc->unrecoverable)
1534		return retval;
1535	if (reset_reason == MPI3MR_RESET_FROM_FIRMWARE) {
1536		retval = 0;
1537		return retval;
1538	}
1539
1540	ioc_info(mrioc, "%s reset due to %s(0x%x)\n",
1541	    mpi3mr_reset_type_name(reset_type),
1542	    mpi3mr_reset_rc_name(reset_reason), reset_reason);
1543
1544	mpi3mr_clear_reset_history(mrioc);
1545	do {
1546		ioc_info(mrioc,
1547		    "Write magic sequence to unlock host diag register (retry=%d)\n",
1548		    ++unlock_retry_count);
1549		if (unlock_retry_count >= MPI3MR_HOSTDIAG_UNLOCK_RETRY_COUNT) {
1550			ioc_err(mrioc,
1551			    "%s reset failed due to unlock failure, host_diagnostic(0x%08x)\n",
1552			    mpi3mr_reset_type_name(reset_type),
1553			    host_diagnostic);
1554			mrioc->unrecoverable = 1;
1555			return retval;
1556		}
1557
1558		writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_FLUSH,
1559		    &mrioc->sysif_regs->write_sequence);
1560		writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_1ST,
1561		    &mrioc->sysif_regs->write_sequence);
1562		writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1563		    &mrioc->sysif_regs->write_sequence);
1564		writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_3RD,
1565		    &mrioc->sysif_regs->write_sequence);
1566		writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_4TH,
1567		    &mrioc->sysif_regs->write_sequence);
1568		writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_5TH,
1569		    &mrioc->sysif_regs->write_sequence);
1570		writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_6TH,
1571		    &mrioc->sysif_regs->write_sequence);
1572		usleep_range(1000, 1100);
1573		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
1574		ioc_info(mrioc,
1575		    "wrote magic sequence: retry_count(%d), host_diagnostic(0x%08x)\n",
1576		    unlock_retry_count, host_diagnostic);
1577	} while (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_DIAG_WRITE_ENABLE));
1578
1579	writel(reset_reason, &mrioc->sysif_regs->scratchpad[0]);
1580	writel(host_diagnostic | reset_type,
1581	    &mrioc->sysif_regs->host_diagnostic);
1582	switch (reset_type) {
1583	case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET:
1584		do {
1585			ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1586			ioc_config =
1587			    readl(&mrioc->sysif_regs->ioc_configuration);
1588			if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY)
1589			    && mpi3mr_soft_reset_success(ioc_status, ioc_config)
1590			    ) {
1591				mpi3mr_clear_reset_history(mrioc);
1592				retval = 0;
1593				break;
1594			}
1595			msleep(100);
1596		} while (--timeout);
1597		mpi3mr_print_fault_info(mrioc);
1598		break;
1599	case MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT:
1600		do {
1601			ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1602			if (mpi3mr_diagfault_success(mrioc, ioc_status)) {
1603				retval = 0;
1604				break;
1605			}
1606			msleep(100);
1607		} while (--timeout);
1608		break;
1609	default:
1610		break;
1611	}
1612
1613	writel(MPI3_SYSIF_WRITE_SEQUENCE_KEY_VALUE_2ND,
1614	    &mrioc->sysif_regs->write_sequence);
1615
1616	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
1617	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
1618	ioc_info(mrioc,
1619	    "ioc_status/ioc_onfig after %s reset is (0x%x)/(0x%x)\n",
1620	    (!retval)?"successful":"failed", ioc_status,
1621	    ioc_config);
1622	if (retval)
1623		mrioc->unrecoverable = 1;
1624	return retval;
1625}
1626
1627/**
1628 * mpi3mr_admin_request_post - Post request to admin queue
1629 * @mrioc: Adapter reference
1630 * @admin_req: MPI3 request
1631 * @admin_req_sz: Request size
1632 * @ignore_reset: Ignore reset in process
1633 *
1634 * Post the MPI3 request into admin request queue and
1635 * inform the controller, if the queue is full return
1636 * appropriate error.
1637 *
1638 * Return: 0 on success, non-zero on failure.
1639 */
1640int mpi3mr_admin_request_post(struct mpi3mr_ioc *mrioc, void *admin_req,
1641	u16 admin_req_sz, u8 ignore_reset)
1642{
1643	u16 areq_pi = 0, areq_ci = 0, max_entries = 0;
1644	int retval = 0;
1645	unsigned long flags;
1646	u8 *areq_entry;
1647
1648	if (mrioc->unrecoverable) {
1649		ioc_err(mrioc, "%s : Unrecoverable controller\n", __func__);
1650		return -EFAULT;
1651	}
1652
1653	spin_lock_irqsave(&mrioc->admin_req_lock, flags);
1654	areq_pi = mrioc->admin_req_pi;
1655	areq_ci = mrioc->admin_req_ci;
1656	max_entries = mrioc->num_admin_req;
1657	if ((areq_ci == (areq_pi + 1)) || ((!areq_ci) &&
1658	    (areq_pi == (max_entries - 1)))) {
1659		ioc_err(mrioc, "AdminReqQ full condition detected\n");
1660		retval = -EAGAIN;
1661		goto out;
1662	}
1663	if (!ignore_reset && mrioc->reset_in_progress) {
1664		ioc_err(mrioc, "AdminReqQ submit reset in progress\n");
1665		retval = -EAGAIN;
1666		goto out;
1667	}
1668	areq_entry = (u8 *)mrioc->admin_req_base +
1669	    (areq_pi * MPI3MR_ADMIN_REQ_FRAME_SZ);
1670	memset(areq_entry, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
1671	memcpy(areq_entry, (u8 *)admin_req, admin_req_sz);
1672
1673	if (++areq_pi == max_entries)
1674		areq_pi = 0;
1675	mrioc->admin_req_pi = areq_pi;
1676
1677	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
1678
1679out:
1680	spin_unlock_irqrestore(&mrioc->admin_req_lock, flags);
1681
1682	return retval;
1683}
1684
1685/**
1686 * mpi3mr_free_op_req_q_segments - free request memory segments
1687 * @mrioc: Adapter instance reference
1688 * @q_idx: operational request queue index
1689 *
1690 * Free memory segments allocated for operational request queue
1691 *
1692 * Return: Nothing.
1693 */
1694static void mpi3mr_free_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1695{
1696	u16 j;
1697	int size;
1698	struct segments *segments;
1699
1700	segments = mrioc->req_qinfo[q_idx].q_segments;
1701	if (!segments)
1702		return;
1703
1704	if (mrioc->enable_segqueue) {
1705		size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1706		if (mrioc->req_qinfo[q_idx].q_segment_list) {
1707			dma_free_coherent(&mrioc->pdev->dev,
1708			    MPI3MR_MAX_SEG_LIST_SIZE,
1709			    mrioc->req_qinfo[q_idx].q_segment_list,
1710			    mrioc->req_qinfo[q_idx].q_segment_list_dma);
1711			mrioc->req_qinfo[q_idx].q_segment_list = NULL;
1712		}
1713	} else
1714		size = mrioc->req_qinfo[q_idx].segment_qd *
1715		    mrioc->facts.op_req_sz;
1716
1717	for (j = 0; j < mrioc->req_qinfo[q_idx].num_segments; j++) {
1718		if (!segments[j].segment)
1719			continue;
1720		dma_free_coherent(&mrioc->pdev->dev,
1721		    size, segments[j].segment, segments[j].segment_dma);
1722		segments[j].segment = NULL;
1723	}
1724	kfree(mrioc->req_qinfo[q_idx].q_segments);
1725	mrioc->req_qinfo[q_idx].q_segments = NULL;
1726	mrioc->req_qinfo[q_idx].qid = 0;
1727}
1728
1729/**
1730 * mpi3mr_free_op_reply_q_segments - free reply memory segments
1731 * @mrioc: Adapter instance reference
1732 * @q_idx: operational reply queue index
1733 *
1734 * Free memory segments allocated for operational reply queue
1735 *
1736 * Return: Nothing.
1737 */
1738static void mpi3mr_free_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 q_idx)
1739{
1740	u16 j;
1741	int size;
1742	struct segments *segments;
1743
1744	segments = mrioc->op_reply_qinfo[q_idx].q_segments;
1745	if (!segments)
1746		return;
1747
1748	if (mrioc->enable_segqueue) {
1749		size = MPI3MR_OP_REP_Q_SEG_SIZE;
1750		if (mrioc->op_reply_qinfo[q_idx].q_segment_list) {
1751			dma_free_coherent(&mrioc->pdev->dev,
1752			    MPI3MR_MAX_SEG_LIST_SIZE,
1753			    mrioc->op_reply_qinfo[q_idx].q_segment_list,
1754			    mrioc->op_reply_qinfo[q_idx].q_segment_list_dma);
1755			mrioc->op_reply_qinfo[q_idx].q_segment_list = NULL;
1756		}
1757	} else
1758		size = mrioc->op_reply_qinfo[q_idx].segment_qd *
1759		    mrioc->op_reply_desc_sz;
1760
1761	for (j = 0; j < mrioc->op_reply_qinfo[q_idx].num_segments; j++) {
1762		if (!segments[j].segment)
1763			continue;
1764		dma_free_coherent(&mrioc->pdev->dev,
1765		    size, segments[j].segment, segments[j].segment_dma);
1766		segments[j].segment = NULL;
1767	}
1768
1769	kfree(mrioc->op_reply_qinfo[q_idx].q_segments);
1770	mrioc->op_reply_qinfo[q_idx].q_segments = NULL;
1771	mrioc->op_reply_qinfo[q_idx].qid = 0;
1772}
1773
1774/**
1775 * mpi3mr_delete_op_reply_q - delete operational reply queue
1776 * @mrioc: Adapter instance reference
1777 * @qidx: operational reply queue index
1778 *
1779 * Delete operatinal reply queue by issuing MPI request
1780 * through admin queue.
1781 *
1782 * Return:  0 on success, non-zero on failure.
1783 */
1784static int mpi3mr_delete_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1785{
1786	struct mpi3_delete_reply_queue_request delq_req;
1787	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1788	int retval = 0;
1789	u16 reply_qid = 0, midx;
1790
1791	reply_qid = op_reply_q->qid;
1792
1793	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1794
1795	if (!reply_qid)	{
1796		retval = -1;
1797		ioc_err(mrioc, "Issue DelRepQ: called with invalid ReqQID\n");
1798		goto out;
1799	}
1800
1801	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount-- :
1802	    mrioc->active_poll_qcount--;
1803
1804	memset(&delq_req, 0, sizeof(delq_req));
1805	mutex_lock(&mrioc->init_cmds.mutex);
1806	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
1807		retval = -1;
1808		ioc_err(mrioc, "Issue DelRepQ: Init command is in use\n");
1809		mutex_unlock(&mrioc->init_cmds.mutex);
1810		goto out;
1811	}
1812	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
1813	mrioc->init_cmds.is_waiting = 1;
1814	mrioc->init_cmds.callback = NULL;
1815	delq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
1816	delq_req.function = MPI3_FUNCTION_DELETE_REPLY_QUEUE;
1817	delq_req.queue_id = cpu_to_le16(reply_qid);
1818
1819	init_completion(&mrioc->init_cmds.done);
1820	retval = mpi3mr_admin_request_post(mrioc, &delq_req, sizeof(delq_req),
1821	    1);
1822	if (retval) {
1823		ioc_err(mrioc, "Issue DelRepQ: Admin Post failed\n");
1824		goto out_unlock;
1825	}
1826	wait_for_completion_timeout(&mrioc->init_cmds.done,
1827	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
1828	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
1829		ioc_err(mrioc, "delete reply queue timed out\n");
1830		mpi3mr_check_rh_fault_ioc(mrioc,
1831		    MPI3MR_RESET_FROM_DELREPQ_TIMEOUT);
1832		retval = -1;
1833		goto out_unlock;
1834	}
1835	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
1836	    != MPI3_IOCSTATUS_SUCCESS) {
1837		ioc_err(mrioc,
1838		    "Issue DelRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
1839		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
1840		    mrioc->init_cmds.ioc_loginfo);
1841		retval = -1;
1842		goto out_unlock;
1843	}
1844	mrioc->intr_info[midx].op_reply_q = NULL;
1845
1846	mpi3mr_free_op_reply_q_segments(mrioc, qidx);
1847out_unlock:
1848	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
1849	mutex_unlock(&mrioc->init_cmds.mutex);
1850out:
1851
1852	return retval;
1853}
1854
1855/**
1856 * mpi3mr_alloc_op_reply_q_segments -Alloc segmented reply pool
1857 * @mrioc: Adapter instance reference
1858 * @qidx: request queue index
1859 *
1860 * Allocate segmented memory pools for operational reply
1861 * queue.
1862 *
1863 * Return: 0 on success, non-zero on failure.
1864 */
1865static int mpi3mr_alloc_op_reply_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1866{
1867	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1868	int i, size;
1869	u64 *q_segment_list_entry = NULL;
1870	struct segments *segments;
1871
1872	if (mrioc->enable_segqueue) {
1873		op_reply_q->segment_qd =
1874		    MPI3MR_OP_REP_Q_SEG_SIZE / mrioc->op_reply_desc_sz;
1875
1876		size = MPI3MR_OP_REP_Q_SEG_SIZE;
1877
1878		op_reply_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1879		    MPI3MR_MAX_SEG_LIST_SIZE, &op_reply_q->q_segment_list_dma,
1880		    GFP_KERNEL);
1881		if (!op_reply_q->q_segment_list)
1882			return -ENOMEM;
1883		q_segment_list_entry = (u64 *)op_reply_q->q_segment_list;
1884	} else {
1885		op_reply_q->segment_qd = op_reply_q->num_replies;
1886		size = op_reply_q->num_replies * mrioc->op_reply_desc_sz;
1887	}
1888
1889	op_reply_q->num_segments = DIV_ROUND_UP(op_reply_q->num_replies,
1890	    op_reply_q->segment_qd);
1891
1892	op_reply_q->q_segments = kcalloc(op_reply_q->num_segments,
1893	    sizeof(struct segments), GFP_KERNEL);
1894	if (!op_reply_q->q_segments)
1895		return -ENOMEM;
1896
1897	segments = op_reply_q->q_segments;
1898	for (i = 0; i < op_reply_q->num_segments; i++) {
1899		segments[i].segment =
1900		    dma_alloc_coherent(&mrioc->pdev->dev,
1901		    size, &segments[i].segment_dma, GFP_KERNEL);
1902		if (!segments[i].segment)
1903			return -ENOMEM;
1904		if (mrioc->enable_segqueue)
1905			q_segment_list_entry[i] =
1906			    (unsigned long)segments[i].segment_dma;
1907	}
1908
1909	return 0;
1910}
1911
1912/**
1913 * mpi3mr_alloc_op_req_q_segments - Alloc segmented req pool.
1914 * @mrioc: Adapter instance reference
1915 * @qidx: request queue index
1916 *
1917 * Allocate segmented memory pools for operational request
1918 * queue.
1919 *
1920 * Return: 0 on success, non-zero on failure.
1921 */
1922static int mpi3mr_alloc_op_req_q_segments(struct mpi3mr_ioc *mrioc, u16 qidx)
1923{
1924	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
1925	int i, size;
1926	u64 *q_segment_list_entry = NULL;
1927	struct segments *segments;
1928
1929	if (mrioc->enable_segqueue) {
1930		op_req_q->segment_qd =
1931		    MPI3MR_OP_REQ_Q_SEG_SIZE / mrioc->facts.op_req_sz;
1932
1933		size = MPI3MR_OP_REQ_Q_SEG_SIZE;
1934
1935		op_req_q->q_segment_list = dma_alloc_coherent(&mrioc->pdev->dev,
1936		    MPI3MR_MAX_SEG_LIST_SIZE, &op_req_q->q_segment_list_dma,
1937		    GFP_KERNEL);
1938		if (!op_req_q->q_segment_list)
1939			return -ENOMEM;
1940		q_segment_list_entry = (u64 *)op_req_q->q_segment_list;
1941
1942	} else {
1943		op_req_q->segment_qd = op_req_q->num_requests;
1944		size = op_req_q->num_requests * mrioc->facts.op_req_sz;
1945	}
1946
1947	op_req_q->num_segments = DIV_ROUND_UP(op_req_q->num_requests,
1948	    op_req_q->segment_qd);
1949
1950	op_req_q->q_segments = kcalloc(op_req_q->num_segments,
1951	    sizeof(struct segments), GFP_KERNEL);
1952	if (!op_req_q->q_segments)
1953		return -ENOMEM;
1954
1955	segments = op_req_q->q_segments;
1956	for (i = 0; i < op_req_q->num_segments; i++) {
1957		segments[i].segment =
1958		    dma_alloc_coherent(&mrioc->pdev->dev,
1959		    size, &segments[i].segment_dma, GFP_KERNEL);
1960		if (!segments[i].segment)
1961			return -ENOMEM;
1962		if (mrioc->enable_segqueue)
1963			q_segment_list_entry[i] =
1964			    (unsigned long)segments[i].segment_dma;
1965	}
1966
1967	return 0;
1968}
1969
1970/**
1971 * mpi3mr_create_op_reply_q - create operational reply queue
1972 * @mrioc: Adapter instance reference
1973 * @qidx: operational reply queue index
1974 *
1975 * Create operatinal reply queue by issuing MPI request
1976 * through admin queue.
1977 *
1978 * Return:  0 on success, non-zero on failure.
1979 */
1980static int mpi3mr_create_op_reply_q(struct mpi3mr_ioc *mrioc, u16 qidx)
1981{
1982	struct mpi3_create_reply_queue_request create_req;
1983	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
1984	int retval = 0;
1985	u16 reply_qid = 0, midx;
1986
1987	reply_qid = op_reply_q->qid;
1988
1989	midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(qidx, mrioc->op_reply_q_offset);
1990
1991	if (reply_qid) {
1992		retval = -1;
1993		ioc_err(mrioc, "CreateRepQ: called for duplicate qid %d\n",
1994		    reply_qid);
1995
1996		return retval;
1997	}
1998
1999	reply_qid = qidx + 1;
2000	op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD;
2001	if ((mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) &&
2002		!mrioc->pdev->revision)
2003		op_reply_q->num_replies = MPI3MR_OP_REP_Q_QD4K;
2004	op_reply_q->ci = 0;
2005	op_reply_q->ephase = 1;
2006	atomic_set(&op_reply_q->pend_ios, 0);
2007	atomic_set(&op_reply_q->in_use, 0);
2008	op_reply_q->enable_irq_poll = false;
2009
2010	if (!op_reply_q->q_segments) {
2011		retval = mpi3mr_alloc_op_reply_q_segments(mrioc, qidx);
2012		if (retval) {
2013			mpi3mr_free_op_reply_q_segments(mrioc, qidx);
2014			goto out;
2015		}
2016	}
2017
2018	memset(&create_req, 0, sizeof(create_req));
2019	mutex_lock(&mrioc->init_cmds.mutex);
2020	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2021		retval = -1;
2022		ioc_err(mrioc, "CreateRepQ: Init command is in use\n");
2023		goto out_unlock;
2024	}
2025	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2026	mrioc->init_cmds.is_waiting = 1;
2027	mrioc->init_cmds.callback = NULL;
2028	create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2029	create_req.function = MPI3_FUNCTION_CREATE_REPLY_QUEUE;
2030	create_req.queue_id = cpu_to_le16(reply_qid);
2031
2032	if (midx < (mrioc->intr_info_count - mrioc->requested_poll_qcount))
2033		op_reply_q->qtype = MPI3MR_DEFAULT_QUEUE;
2034	else
2035		op_reply_q->qtype = MPI3MR_POLL_QUEUE;
2036
2037	if (op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) {
2038		create_req.flags =
2039			MPI3_CREATE_REPLY_QUEUE_FLAGS_INT_ENABLE_ENABLE;
2040		create_req.msix_index =
2041			cpu_to_le16(mrioc->intr_info[midx].msix_index);
2042	} else {
2043		create_req.msix_index = cpu_to_le16(mrioc->intr_info_count - 1);
2044		ioc_info(mrioc, "create reply queue(polled): for qid(%d), midx(%d)\n",
2045			reply_qid, midx);
2046		if (!mrioc->active_poll_qcount)
2047			disable_irq_nosync(pci_irq_vector(mrioc->pdev,
2048			    mrioc->intr_info_count - 1));
2049	}
2050
2051	if (mrioc->enable_segqueue) {
2052		create_req.flags |=
2053		    MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
2054		create_req.base_address = cpu_to_le64(
2055		    op_reply_q->q_segment_list_dma);
2056	} else
2057		create_req.base_address = cpu_to_le64(
2058		    op_reply_q->q_segments[0].segment_dma);
2059
2060	create_req.size = cpu_to_le16(op_reply_q->num_replies);
2061
2062	init_completion(&mrioc->init_cmds.done);
2063	retval = mpi3mr_admin_request_post(mrioc, &create_req,
2064	    sizeof(create_req), 1);
2065	if (retval) {
2066		ioc_err(mrioc, "CreateRepQ: Admin Post failed\n");
2067		goto out_unlock;
2068	}
2069	wait_for_completion_timeout(&mrioc->init_cmds.done,
2070	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2071	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2072		ioc_err(mrioc, "create reply queue timed out\n");
2073		mpi3mr_check_rh_fault_ioc(mrioc,
2074		    MPI3MR_RESET_FROM_CREATEREPQ_TIMEOUT);
2075		retval = -1;
2076		goto out_unlock;
2077	}
2078	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2079	    != MPI3_IOCSTATUS_SUCCESS) {
2080		ioc_err(mrioc,
2081		    "CreateRepQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2082		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2083		    mrioc->init_cmds.ioc_loginfo);
2084		retval = -1;
2085		goto out_unlock;
2086	}
2087	op_reply_q->qid = reply_qid;
2088	if (midx < mrioc->intr_info_count)
2089		mrioc->intr_info[midx].op_reply_q = op_reply_q;
2090
2091	(op_reply_q->qtype == MPI3MR_DEFAULT_QUEUE) ? mrioc->default_qcount++ :
2092	    mrioc->active_poll_qcount++;
2093
2094out_unlock:
2095	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2096	mutex_unlock(&mrioc->init_cmds.mutex);
2097out:
2098
2099	return retval;
2100}
2101
2102/**
2103 * mpi3mr_create_op_req_q - create operational request queue
2104 * @mrioc: Adapter instance reference
2105 * @idx: operational request queue index
2106 * @reply_qid: Reply queue ID
2107 *
2108 * Create operatinal request queue by issuing MPI request
2109 * through admin queue.
2110 *
2111 * Return:  0 on success, non-zero on failure.
2112 */
2113static int mpi3mr_create_op_req_q(struct mpi3mr_ioc *mrioc, u16 idx,
2114	u16 reply_qid)
2115{
2116	struct mpi3_create_request_queue_request create_req;
2117	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + idx;
2118	int retval = 0;
2119	u16 req_qid = 0;
2120
2121	req_qid = op_req_q->qid;
2122
2123	if (req_qid) {
2124		retval = -1;
2125		ioc_err(mrioc, "CreateReqQ: called for duplicate qid %d\n",
2126		    req_qid);
2127
2128		return retval;
2129	}
2130	req_qid = idx + 1;
2131
2132	op_req_q->num_requests = MPI3MR_OP_REQ_Q_QD;
2133	op_req_q->ci = 0;
2134	op_req_q->pi = 0;
2135	op_req_q->reply_qid = reply_qid;
2136	spin_lock_init(&op_req_q->q_lock);
2137
2138	if (!op_req_q->q_segments) {
2139		retval = mpi3mr_alloc_op_req_q_segments(mrioc, idx);
2140		if (retval) {
2141			mpi3mr_free_op_req_q_segments(mrioc, idx);
2142			goto out;
2143		}
2144	}
2145
2146	memset(&create_req, 0, sizeof(create_req));
2147	mutex_lock(&mrioc->init_cmds.mutex);
2148	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2149		retval = -1;
2150		ioc_err(mrioc, "CreateReqQ: Init command is in use\n");
2151		goto out_unlock;
2152	}
2153	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2154	mrioc->init_cmds.is_waiting = 1;
2155	mrioc->init_cmds.callback = NULL;
2156	create_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2157	create_req.function = MPI3_FUNCTION_CREATE_REQUEST_QUEUE;
2158	create_req.queue_id = cpu_to_le16(req_qid);
2159	if (mrioc->enable_segqueue) {
2160		create_req.flags =
2161		    MPI3_CREATE_REQUEST_QUEUE_FLAGS_SEGMENTED_SEGMENTED;
2162		create_req.base_address = cpu_to_le64(
2163		    op_req_q->q_segment_list_dma);
2164	} else
2165		create_req.base_address = cpu_to_le64(
2166		    op_req_q->q_segments[0].segment_dma);
2167	create_req.reply_queue_id = cpu_to_le16(reply_qid);
2168	create_req.size = cpu_to_le16(op_req_q->num_requests);
2169
2170	init_completion(&mrioc->init_cmds.done);
2171	retval = mpi3mr_admin_request_post(mrioc, &create_req,
2172	    sizeof(create_req), 1);
2173	if (retval) {
2174		ioc_err(mrioc, "CreateReqQ: Admin Post failed\n");
2175		goto out_unlock;
2176	}
2177	wait_for_completion_timeout(&mrioc->init_cmds.done,
2178	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2179	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2180		ioc_err(mrioc, "create request queue timed out\n");
2181		mpi3mr_check_rh_fault_ioc(mrioc,
2182		    MPI3MR_RESET_FROM_CREATEREQQ_TIMEOUT);
2183		retval = -1;
2184		goto out_unlock;
2185	}
2186	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2187	    != MPI3_IOCSTATUS_SUCCESS) {
2188		ioc_err(mrioc,
2189		    "CreateReqQ: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2190		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2191		    mrioc->init_cmds.ioc_loginfo);
2192		retval = -1;
2193		goto out_unlock;
2194	}
2195	op_req_q->qid = req_qid;
2196
2197out_unlock:
2198	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2199	mutex_unlock(&mrioc->init_cmds.mutex);
2200out:
2201
2202	return retval;
2203}
2204
2205/**
2206 * mpi3mr_create_op_queues - create operational queue pairs
2207 * @mrioc: Adapter instance reference
2208 *
2209 * Allocate memory for operational queue meta data and call
2210 * create request and reply queue functions.
2211 *
2212 * Return: 0 on success, non-zero on failures.
2213 */
2214static int mpi3mr_create_op_queues(struct mpi3mr_ioc *mrioc)
2215{
2216	int retval = 0;
2217	u16 num_queues = 0, i = 0, msix_count_op_q = 1;
2218
2219	num_queues = min_t(int, mrioc->facts.max_op_reply_q,
2220	    mrioc->facts.max_op_req_q);
2221
2222	msix_count_op_q =
2223	    mrioc->intr_info_count - mrioc->op_reply_q_offset;
2224	if (!mrioc->num_queues)
2225		mrioc->num_queues = min_t(int, num_queues, msix_count_op_q);
2226	/*
2227	 * During reset set the num_queues to the number of queues
2228	 * that was set before the reset.
2229	 */
2230	num_queues = mrioc->num_op_reply_q ?
2231	    mrioc->num_op_reply_q : mrioc->num_queues;
2232	ioc_info(mrioc, "trying to create %d operational queue pairs\n",
2233	    num_queues);
2234
2235	if (!mrioc->req_qinfo) {
2236		mrioc->req_qinfo = kcalloc(num_queues,
2237		    sizeof(struct op_req_qinfo), GFP_KERNEL);
2238		if (!mrioc->req_qinfo) {
2239			retval = -1;
2240			goto out_failed;
2241		}
2242
2243		mrioc->op_reply_qinfo = kzalloc(sizeof(struct op_reply_qinfo) *
2244		    num_queues, GFP_KERNEL);
2245		if (!mrioc->op_reply_qinfo) {
2246			retval = -1;
2247			goto out_failed;
2248		}
2249	}
2250
2251	if (mrioc->enable_segqueue)
2252		ioc_info(mrioc,
2253		    "allocating operational queues through segmented queues\n");
2254
2255	for (i = 0; i < num_queues; i++) {
2256		if (mpi3mr_create_op_reply_q(mrioc, i)) {
2257			ioc_err(mrioc, "Cannot create OP RepQ %d\n", i);
2258			break;
2259		}
2260		if (mpi3mr_create_op_req_q(mrioc, i,
2261		    mrioc->op_reply_qinfo[i].qid)) {
2262			ioc_err(mrioc, "Cannot create OP ReqQ %d\n", i);
2263			mpi3mr_delete_op_reply_q(mrioc, i);
2264			break;
2265		}
2266	}
2267
2268	if (i == 0) {
2269		/* Not even one queue is created successfully*/
2270		retval = -1;
2271		goto out_failed;
2272	}
2273	mrioc->num_op_reply_q = mrioc->num_op_req_q = i;
2274	ioc_info(mrioc,
2275	    "successfully created %d operational queue pairs(default/polled) queue = (%d/%d)\n",
2276	    mrioc->num_op_reply_q, mrioc->default_qcount,
2277	    mrioc->active_poll_qcount);
2278
2279	return retval;
2280out_failed:
2281	kfree(mrioc->req_qinfo);
2282	mrioc->req_qinfo = NULL;
2283
2284	kfree(mrioc->op_reply_qinfo);
2285	mrioc->op_reply_qinfo = NULL;
2286
2287	return retval;
2288}
2289
2290/**
2291 * mpi3mr_op_request_post - Post request to operational queue
2292 * @mrioc: Adapter reference
2293 * @op_req_q: Operational request queue info
2294 * @req: MPI3 request
2295 *
2296 * Post the MPI3 request into operational request queue and
2297 * inform the controller, if the queue is full return
2298 * appropriate error.
2299 *
2300 * Return: 0 on success, non-zero on failure.
2301 */
2302int mpi3mr_op_request_post(struct mpi3mr_ioc *mrioc,
2303	struct op_req_qinfo *op_req_q, u8 *req)
2304{
2305	u16 pi = 0, max_entries, reply_qidx = 0, midx;
2306	int retval = 0;
2307	unsigned long flags;
2308	u8 *req_entry;
2309	void *segment_base_addr;
2310	u16 req_sz = mrioc->facts.op_req_sz;
2311	struct segments *segments = op_req_q->q_segments;
2312
2313	reply_qidx = op_req_q->reply_qid - 1;
2314
2315	if (mrioc->unrecoverable)
2316		return -EFAULT;
2317
2318	spin_lock_irqsave(&op_req_q->q_lock, flags);
2319	pi = op_req_q->pi;
2320	max_entries = op_req_q->num_requests;
2321
2322	if (mpi3mr_check_req_qfull(op_req_q)) {
2323		midx = REPLY_QUEUE_IDX_TO_MSIX_IDX(
2324		    reply_qidx, mrioc->op_reply_q_offset);
2325		mpi3mr_process_op_reply_q(mrioc, mrioc->intr_info[midx].op_reply_q);
2326
2327		if (mpi3mr_check_req_qfull(op_req_q)) {
2328			retval = -EAGAIN;
2329			goto out;
2330		}
2331	}
2332
2333	if (mrioc->reset_in_progress) {
2334		ioc_err(mrioc, "OpReqQ submit reset in progress\n");
2335		retval = -EAGAIN;
2336		goto out;
2337	}
2338
2339	segment_base_addr = segments[pi / op_req_q->segment_qd].segment;
2340	req_entry = (u8 *)segment_base_addr +
2341	    ((pi % op_req_q->segment_qd) * req_sz);
2342
2343	memset(req_entry, 0, req_sz);
2344	memcpy(req_entry, req, MPI3MR_ADMIN_REQ_FRAME_SZ);
2345
2346	if (++pi == max_entries)
2347		pi = 0;
2348	op_req_q->pi = pi;
2349
2350#ifndef CONFIG_PREEMPT_RT
2351	if (atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios)
2352	    > MPI3MR_IRQ_POLL_TRIGGER_IOCOUNT)
2353		mrioc->op_reply_qinfo[reply_qidx].enable_irq_poll = true;
2354#else
2355	atomic_inc_return(&mrioc->op_reply_qinfo[reply_qidx].pend_ios);
2356#endif
2357
2358	writel(op_req_q->pi,
2359	    &mrioc->sysif_regs->oper_queue_indexes[reply_qidx].producer_index);
2360
2361out:
2362	spin_unlock_irqrestore(&op_req_q->q_lock, flags);
2363	return retval;
2364}
2365
2366/**
2367 * mpi3mr_check_rh_fault_ioc - check reset history and fault
2368 * controller
2369 * @mrioc: Adapter instance reference
2370 * @reason_code: reason code for the fault.
2371 *
2372 * This routine will save snapdump and fault the controller with
2373 * the given reason code if it is not already in the fault or
2374 * not asynchronosuly reset. This will be used to handle
2375 * initilaization time faults/resets/timeout as in those cases
2376 * immediate soft reset invocation is not required.
2377 *
2378 * Return:  None.
2379 */
2380void mpi3mr_check_rh_fault_ioc(struct mpi3mr_ioc *mrioc, u32 reason_code)
2381{
2382	u32 ioc_status, host_diagnostic, timeout;
2383
2384	if (mrioc->unrecoverable) {
2385		ioc_err(mrioc, "controller is unrecoverable\n");
2386		return;
2387	}
2388
2389	if (!pci_device_is_present(mrioc->pdev)) {
2390		mrioc->unrecoverable = 1;
2391		ioc_err(mrioc, "controller is not present\n");
2392		return;
2393	}
2394
2395	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2396	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
2397	    (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
2398		mpi3mr_print_fault_info(mrioc);
2399		return;
2400	}
2401	mpi3mr_set_diagsave(mrioc);
2402	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
2403	    reason_code);
2404	timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
2405	do {
2406		host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2407		if (!(host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
2408			break;
2409		msleep(100);
2410	} while (--timeout);
2411}
2412
2413/**
2414 * mpi3mr_sync_timestamp - Issue time stamp sync request
2415 * @mrioc: Adapter reference
2416 *
2417 * Issue IO unit control MPI request to synchornize firmware
2418 * timestamp with host time.
2419 *
2420 * Return: 0 on success, non-zero on failure.
2421 */
2422static int mpi3mr_sync_timestamp(struct mpi3mr_ioc *mrioc)
2423{
2424	ktime_t current_time;
2425	struct mpi3_iounit_control_request iou_ctrl;
2426	int retval = 0;
2427
2428	memset(&iou_ctrl, 0, sizeof(iou_ctrl));
2429	mutex_lock(&mrioc->init_cmds.mutex);
2430	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2431		retval = -1;
2432		ioc_err(mrioc, "Issue IOUCTL time_stamp: command is in use\n");
2433		mutex_unlock(&mrioc->init_cmds.mutex);
2434		goto out;
2435	}
2436	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2437	mrioc->init_cmds.is_waiting = 1;
2438	mrioc->init_cmds.callback = NULL;
2439	iou_ctrl.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2440	iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
2441	iou_ctrl.operation = MPI3_CTRL_OP_UPDATE_TIMESTAMP;
2442	current_time = ktime_get_real();
2443	iou_ctrl.param64[0] = cpu_to_le64(ktime_to_ms(current_time));
2444
2445	init_completion(&mrioc->init_cmds.done);
2446	retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl,
2447	    sizeof(iou_ctrl), 0);
2448	if (retval) {
2449		ioc_err(mrioc, "Issue IOUCTL time_stamp: Admin Post failed\n");
2450		goto out_unlock;
2451	}
2452
2453	wait_for_completion_timeout(&mrioc->init_cmds.done,
2454	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2455	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2456		ioc_err(mrioc, "Issue IOUCTL time_stamp: command timed out\n");
2457		mrioc->init_cmds.is_waiting = 0;
2458		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
2459			mpi3mr_check_rh_fault_ioc(mrioc,
2460			    MPI3MR_RESET_FROM_TSU_TIMEOUT);
2461		retval = -1;
2462		goto out_unlock;
2463	}
2464	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2465	    != MPI3_IOCSTATUS_SUCCESS) {
2466		ioc_err(mrioc,
2467		    "Issue IOUCTL time_stamp: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2468		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2469		    mrioc->init_cmds.ioc_loginfo);
2470		retval = -1;
2471		goto out_unlock;
2472	}
2473
2474out_unlock:
2475	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2476	mutex_unlock(&mrioc->init_cmds.mutex);
2477
2478out:
2479	return retval;
2480}
2481
2482/**
2483 * mpi3mr_print_pkg_ver - display controller fw package version
2484 * @mrioc: Adapter reference
2485 *
2486 * Retrieve firmware package version from the component image
2487 * header of the controller flash and display it.
2488 *
2489 * Return: 0 on success and non-zero on failure.
2490 */
2491static int mpi3mr_print_pkg_ver(struct mpi3mr_ioc *mrioc)
2492{
2493	struct mpi3_ci_upload_request ci_upload;
2494	int retval = -1;
2495	void *data = NULL;
2496	dma_addr_t data_dma;
2497	struct mpi3_ci_manifest_mpi *manifest;
2498	u32 data_len = sizeof(struct mpi3_ci_manifest_mpi);
2499	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2500
2501	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2502	    GFP_KERNEL);
2503	if (!data)
2504		return -ENOMEM;
2505
2506	memset(&ci_upload, 0, sizeof(ci_upload));
2507	mutex_lock(&mrioc->init_cmds.mutex);
2508	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2509		ioc_err(mrioc, "sending get package version failed due to command in use\n");
2510		mutex_unlock(&mrioc->init_cmds.mutex);
2511		goto out;
2512	}
2513	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2514	mrioc->init_cmds.is_waiting = 1;
2515	mrioc->init_cmds.callback = NULL;
2516	ci_upload.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2517	ci_upload.function = MPI3_FUNCTION_CI_UPLOAD;
2518	ci_upload.msg_flags = MPI3_CI_UPLOAD_MSGFLAGS_LOCATION_PRIMARY;
2519	ci_upload.signature1 = cpu_to_le32(MPI3_IMAGE_HEADER_SIGNATURE1_MANIFEST);
2520	ci_upload.image_offset = cpu_to_le32(MPI3_IMAGE_HEADER_SIZE);
2521	ci_upload.segment_size = cpu_to_le32(data_len);
2522
2523	mpi3mr_add_sg_single(&ci_upload.sgl, sgl_flags, data_len,
2524	    data_dma);
2525	init_completion(&mrioc->init_cmds.done);
2526	retval = mpi3mr_admin_request_post(mrioc, &ci_upload,
2527	    sizeof(ci_upload), 1);
2528	if (retval) {
2529		ioc_err(mrioc, "posting get package version failed\n");
2530		goto out_unlock;
2531	}
2532	wait_for_completion_timeout(&mrioc->init_cmds.done,
2533	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2534	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2535		ioc_err(mrioc, "get package version timed out\n");
2536		mpi3mr_check_rh_fault_ioc(mrioc,
2537		    MPI3MR_RESET_FROM_GETPKGVER_TIMEOUT);
2538		retval = -1;
2539		goto out_unlock;
2540	}
2541	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2542	    == MPI3_IOCSTATUS_SUCCESS) {
2543		manifest = (struct mpi3_ci_manifest_mpi *) data;
2544		if (manifest->manifest_type == MPI3_CI_MANIFEST_TYPE_MPI) {
2545			ioc_info(mrioc,
2546			    "firmware package version(%d.%d.%d.%d.%05d-%05d)\n",
2547			    manifest->package_version.gen_major,
2548			    manifest->package_version.gen_minor,
2549			    manifest->package_version.phase_major,
2550			    manifest->package_version.phase_minor,
2551			    manifest->package_version.customer_id,
2552			    manifest->package_version.build_num);
2553		}
2554	}
2555	retval = 0;
2556out_unlock:
2557	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2558	mutex_unlock(&mrioc->init_cmds.mutex);
2559
2560out:
2561	if (data)
2562		dma_free_coherent(&mrioc->pdev->dev, data_len, data,
2563		    data_dma);
2564	return retval;
2565}
2566
2567/**
2568 * mpi3mr_watchdog_work - watchdog thread to monitor faults
2569 * @work: work struct
2570 *
2571 * Watch dog work periodically executed (1 second interval) to
2572 * monitor firmware fault and to issue periodic timer sync to
2573 * the firmware.
2574 *
2575 * Return: Nothing.
2576 */
2577static void mpi3mr_watchdog_work(struct work_struct *work)
2578{
2579	struct mpi3mr_ioc *mrioc =
2580	    container_of(work, struct mpi3mr_ioc, watchdog_work.work);
2581	unsigned long flags;
2582	enum mpi3mr_iocstate ioc_state;
2583	u32 fault, host_diagnostic, ioc_status;
2584	u32 reset_reason = MPI3MR_RESET_FROM_FAULT_WATCH;
2585
2586	if (mrioc->reset_in_progress)
2587		return;
2588
2589	if (!mrioc->unrecoverable && !pci_device_is_present(mrioc->pdev)) {
2590		ioc_err(mrioc, "watchdog could not detect the controller\n");
2591		mrioc->unrecoverable = 1;
2592	}
2593
2594	if (mrioc->unrecoverable) {
2595		ioc_err(mrioc,
2596		    "flush pending commands for unrecoverable controller\n");
2597		mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
2598		return;
2599	}
2600
2601	if (mrioc->ts_update_counter++ >= MPI3MR_TSUPDATE_INTERVAL) {
2602		mrioc->ts_update_counter = 0;
2603		mpi3mr_sync_timestamp(mrioc);
2604	}
2605
2606	if ((mrioc->prepare_for_reset) &&
2607	    ((mrioc->prepare_for_reset_timeout_counter++) >=
2608	     MPI3MR_PREPARE_FOR_RESET_TIMEOUT)) {
2609		mpi3mr_soft_reset_handler(mrioc,
2610		    MPI3MR_RESET_FROM_CIACTVRST_TIMER, 1);
2611		return;
2612	}
2613
2614	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
2615	if (ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) {
2616		mpi3mr_soft_reset_handler(mrioc, MPI3MR_RESET_FROM_FIRMWARE, 0);
2617		return;
2618	}
2619
2620	/*Check for fault state every one second and issue Soft reset*/
2621	ioc_state = mpi3mr_get_iocstate(mrioc);
2622	if (ioc_state != MRIOC_STATE_FAULT)
2623		goto schedule_work;
2624
2625	fault = readl(&mrioc->sysif_regs->fault) & MPI3_SYSIF_FAULT_CODE_MASK;
2626	host_diagnostic = readl(&mrioc->sysif_regs->host_diagnostic);
2627	if (host_diagnostic & MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS) {
2628		if (!mrioc->diagsave_timeout) {
2629			mpi3mr_print_fault_info(mrioc);
2630			ioc_warn(mrioc, "diag save in progress\n");
2631		}
2632		if ((mrioc->diagsave_timeout++) <= MPI3_SYSIF_DIAG_SAVE_TIMEOUT)
2633			goto schedule_work;
2634	}
2635
2636	mpi3mr_print_fault_info(mrioc);
2637	mrioc->diagsave_timeout = 0;
2638
2639	switch (fault) {
2640	case MPI3_SYSIF_FAULT_CODE_COMPLETE_RESET_NEEDED:
2641	case MPI3_SYSIF_FAULT_CODE_POWER_CYCLE_REQUIRED:
2642		ioc_warn(mrioc,
2643		    "controller requires system power cycle, marking controller as unrecoverable\n");
2644		mrioc->unrecoverable = 1;
2645		goto schedule_work;
2646	case MPI3_SYSIF_FAULT_CODE_SOFT_RESET_IN_PROGRESS:
2647		goto schedule_work;
2648	case MPI3_SYSIF_FAULT_CODE_CI_ACTIVATION_RESET:
2649		reset_reason = MPI3MR_RESET_FROM_CIACTIV_FAULT;
2650		break;
2651	default:
2652		break;
2653	}
2654	mpi3mr_soft_reset_handler(mrioc, reset_reason, 0);
2655	return;
2656
2657schedule_work:
2658	spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2659	if (mrioc->watchdog_work_q)
2660		queue_delayed_work(mrioc->watchdog_work_q,
2661		    &mrioc->watchdog_work,
2662		    msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2663	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2664	return;
2665}
2666
2667/**
2668 * mpi3mr_start_watchdog - Start watchdog
2669 * @mrioc: Adapter instance reference
2670 *
2671 * Create and start the watchdog thread to monitor controller
2672 * faults.
2673 *
2674 * Return: Nothing.
2675 */
2676void mpi3mr_start_watchdog(struct mpi3mr_ioc *mrioc)
2677{
2678	if (mrioc->watchdog_work_q)
2679		return;
2680
2681	INIT_DELAYED_WORK(&mrioc->watchdog_work, mpi3mr_watchdog_work);
2682	snprintf(mrioc->watchdog_work_q_name,
2683	    sizeof(mrioc->watchdog_work_q_name), "watchdog_%s%d", mrioc->name,
2684	    mrioc->id);
2685	mrioc->watchdog_work_q =
2686	    create_singlethread_workqueue(mrioc->watchdog_work_q_name);
2687	if (!mrioc->watchdog_work_q) {
2688		ioc_err(mrioc, "%s: failed (line=%d)\n", __func__, __LINE__);
2689		return;
2690	}
2691
2692	if (mrioc->watchdog_work_q)
2693		queue_delayed_work(mrioc->watchdog_work_q,
2694		    &mrioc->watchdog_work,
2695		    msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
2696}
2697
2698/**
2699 * mpi3mr_stop_watchdog - Stop watchdog
2700 * @mrioc: Adapter instance reference
2701 *
2702 * Stop the watchdog thread created to monitor controller
2703 * faults.
2704 *
2705 * Return: Nothing.
2706 */
2707void mpi3mr_stop_watchdog(struct mpi3mr_ioc *mrioc)
2708{
2709	unsigned long flags;
2710	struct workqueue_struct *wq;
2711
2712	spin_lock_irqsave(&mrioc->watchdog_lock, flags);
2713	wq = mrioc->watchdog_work_q;
2714	mrioc->watchdog_work_q = NULL;
2715	spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
2716	if (wq) {
2717		if (!cancel_delayed_work_sync(&mrioc->watchdog_work))
2718			flush_workqueue(wq);
2719		destroy_workqueue(wq);
2720	}
2721}
2722
2723/**
2724 * mpi3mr_setup_admin_qpair - Setup admin queue pair
2725 * @mrioc: Adapter instance reference
2726 *
2727 * Allocate memory for admin queue pair if required and register
2728 * the admin queue with the controller.
2729 *
2730 * Return: 0 on success, non-zero on failures.
2731 */
2732static int mpi3mr_setup_admin_qpair(struct mpi3mr_ioc *mrioc)
2733{
2734	int retval = 0;
2735	u32 num_admin_entries = 0;
2736
2737	mrioc->admin_req_q_sz = MPI3MR_ADMIN_REQ_Q_SIZE;
2738	mrioc->num_admin_req = mrioc->admin_req_q_sz /
2739	    MPI3MR_ADMIN_REQ_FRAME_SZ;
2740	mrioc->admin_req_ci = mrioc->admin_req_pi = 0;
 
2741
2742	mrioc->admin_reply_q_sz = MPI3MR_ADMIN_REPLY_Q_SIZE;
2743	mrioc->num_admin_replies = mrioc->admin_reply_q_sz /
2744	    MPI3MR_ADMIN_REPLY_FRAME_SZ;
2745	mrioc->admin_reply_ci = 0;
2746	mrioc->admin_reply_ephase = 1;
2747	atomic_set(&mrioc->admin_reply_q_in_use, 0);
2748
2749	if (!mrioc->admin_req_base) {
2750		mrioc->admin_req_base = dma_alloc_coherent(&mrioc->pdev->dev,
2751		    mrioc->admin_req_q_sz, &mrioc->admin_req_dma, GFP_KERNEL);
2752
2753		if (!mrioc->admin_req_base) {
2754			retval = -1;
2755			goto out_failed;
2756		}
2757
2758		mrioc->admin_reply_base = dma_alloc_coherent(&mrioc->pdev->dev,
2759		    mrioc->admin_reply_q_sz, &mrioc->admin_reply_dma,
2760		    GFP_KERNEL);
2761
2762		if (!mrioc->admin_reply_base) {
2763			retval = -1;
2764			goto out_failed;
2765		}
2766	}
2767
2768	num_admin_entries = (mrioc->num_admin_replies << 16) |
2769	    (mrioc->num_admin_req);
2770	writel(num_admin_entries, &mrioc->sysif_regs->admin_queue_num_entries);
2771	mpi3mr_writeq(mrioc->admin_req_dma,
2772	    &mrioc->sysif_regs->admin_request_queue_address);
2773	mpi3mr_writeq(mrioc->admin_reply_dma,
2774	    &mrioc->sysif_regs->admin_reply_queue_address);
2775	writel(mrioc->admin_req_pi, &mrioc->sysif_regs->admin_request_queue_pi);
2776	writel(mrioc->admin_reply_ci, &mrioc->sysif_regs->admin_reply_queue_ci);
2777	return retval;
2778
2779out_failed:
2780
2781	if (mrioc->admin_reply_base) {
2782		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
2783		    mrioc->admin_reply_base, mrioc->admin_reply_dma);
2784		mrioc->admin_reply_base = NULL;
2785	}
2786	if (mrioc->admin_req_base) {
2787		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
2788		    mrioc->admin_req_base, mrioc->admin_req_dma);
2789		mrioc->admin_req_base = NULL;
2790	}
2791	return retval;
2792}
2793
2794/**
2795 * mpi3mr_issue_iocfacts - Send IOC Facts
2796 * @mrioc: Adapter instance reference
2797 * @facts_data: Cached IOC facts data
2798 *
2799 * Issue IOC Facts MPI request through admin queue and wait for
2800 * the completion of it or time out.
2801 *
2802 * Return: 0 on success, non-zero on failures.
2803 */
2804static int mpi3mr_issue_iocfacts(struct mpi3mr_ioc *mrioc,
2805	struct mpi3_ioc_facts_data *facts_data)
2806{
2807	struct mpi3_ioc_facts_request iocfacts_req;
2808	void *data = NULL;
2809	dma_addr_t data_dma;
2810	u32 data_len = sizeof(*facts_data);
2811	int retval = 0;
2812	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
2813
2814	data = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
2815	    GFP_KERNEL);
2816
2817	if (!data) {
2818		retval = -1;
2819		goto out;
2820	}
2821
2822	memset(&iocfacts_req, 0, sizeof(iocfacts_req));
2823	mutex_lock(&mrioc->init_cmds.mutex);
2824	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
2825		retval = -1;
2826		ioc_err(mrioc, "Issue IOCFacts: Init command is in use\n");
2827		mutex_unlock(&mrioc->init_cmds.mutex);
2828		goto out;
2829	}
2830	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
2831	mrioc->init_cmds.is_waiting = 1;
2832	mrioc->init_cmds.callback = NULL;
2833	iocfacts_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
2834	iocfacts_req.function = MPI3_FUNCTION_IOC_FACTS;
2835
2836	mpi3mr_add_sg_single(&iocfacts_req.sgl, sgl_flags, data_len,
2837	    data_dma);
2838
2839	init_completion(&mrioc->init_cmds.done);
2840	retval = mpi3mr_admin_request_post(mrioc, &iocfacts_req,
2841	    sizeof(iocfacts_req), 1);
2842	if (retval) {
2843		ioc_err(mrioc, "Issue IOCFacts: Admin Post failed\n");
2844		goto out_unlock;
2845	}
2846	wait_for_completion_timeout(&mrioc->init_cmds.done,
2847	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
2848	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
2849		ioc_err(mrioc, "ioc_facts timed out\n");
2850		mpi3mr_check_rh_fault_ioc(mrioc,
2851		    MPI3MR_RESET_FROM_IOCFACTS_TIMEOUT);
2852		retval = -1;
2853		goto out_unlock;
2854	}
2855	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
2856	    != MPI3_IOCSTATUS_SUCCESS) {
2857		ioc_err(mrioc,
2858		    "Issue IOCFacts: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
2859		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2860		    mrioc->init_cmds.ioc_loginfo);
2861		retval = -1;
2862		goto out_unlock;
2863	}
2864	memcpy(facts_data, (u8 *)data, data_len);
2865	mpi3mr_process_factsdata(mrioc, facts_data);
2866out_unlock:
2867	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
2868	mutex_unlock(&mrioc->init_cmds.mutex);
2869
2870out:
2871	if (data)
2872		dma_free_coherent(&mrioc->pdev->dev, data_len, data, data_dma);
2873
2874	return retval;
2875}
2876
2877/**
2878 * mpi3mr_check_reset_dma_mask - Process IOC facts data
2879 * @mrioc: Adapter instance reference
2880 *
2881 * Check whether the new DMA mask requested through IOCFacts by
2882 * firmware needs to be set, if so set it .
2883 *
2884 * Return: 0 on success, non-zero on failure.
2885 */
2886static inline int mpi3mr_check_reset_dma_mask(struct mpi3mr_ioc *mrioc)
2887{
2888	struct pci_dev *pdev = mrioc->pdev;
2889	int r;
2890	u64 facts_dma_mask = DMA_BIT_MASK(mrioc->facts.dma_mask);
2891
2892	if (!mrioc->facts.dma_mask || (mrioc->dma_mask <= facts_dma_mask))
2893		return 0;
2894
2895	ioc_info(mrioc, "Changing DMA mask from 0x%016llx to 0x%016llx\n",
2896	    mrioc->dma_mask, facts_dma_mask);
2897
2898	r = dma_set_mask_and_coherent(&pdev->dev, facts_dma_mask);
2899	if (r) {
2900		ioc_err(mrioc, "Setting DMA mask to 0x%016llx failed: %d\n",
2901		    facts_dma_mask, r);
2902		return r;
2903	}
2904	mrioc->dma_mask = facts_dma_mask;
2905	return r;
2906}
2907
2908/**
2909 * mpi3mr_process_factsdata - Process IOC facts data
2910 * @mrioc: Adapter instance reference
2911 * @facts_data: Cached IOC facts data
2912 *
2913 * Convert IOC facts data into cpu endianness and cache it in
2914 * the driver .
2915 *
2916 * Return: Nothing.
2917 */
2918static void mpi3mr_process_factsdata(struct mpi3mr_ioc *mrioc,
2919	struct mpi3_ioc_facts_data *facts_data)
2920{
2921	u32 ioc_config, req_sz, facts_flags;
2922
2923	if ((le16_to_cpu(facts_data->ioc_facts_data_length)) !=
2924	    (sizeof(*facts_data) / 4)) {
2925		ioc_warn(mrioc,
2926		    "IOCFactsdata length mismatch driver_sz(%zu) firmware_sz(%d)\n",
2927		    sizeof(*facts_data),
2928		    le16_to_cpu(facts_data->ioc_facts_data_length) * 4);
2929	}
2930
2931	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
2932	req_sz = 1 << ((ioc_config & MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ) >>
2933	    MPI3_SYSIF_IOC_CONFIG_OPER_REQ_ENT_SZ_SHIFT);
2934	if (le16_to_cpu(facts_data->ioc_request_frame_size) != (req_sz / 4)) {
2935		ioc_err(mrioc,
2936		    "IOCFacts data reqFrameSize mismatch hw_size(%d) firmware_sz(%d)\n",
2937		    req_sz / 4, le16_to_cpu(facts_data->ioc_request_frame_size));
2938	}
2939
2940	memset(&mrioc->facts, 0, sizeof(mrioc->facts));
2941
2942	facts_flags = le32_to_cpu(facts_data->flags);
2943	mrioc->facts.op_req_sz = req_sz;
2944	mrioc->op_reply_desc_sz = 1 << ((ioc_config &
2945	    MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ) >>
2946	    MPI3_SYSIF_IOC_CONFIG_OPER_RPY_ENT_SZ_SHIFT);
2947
2948	mrioc->facts.ioc_num = facts_data->ioc_number;
2949	mrioc->facts.who_init = facts_data->who_init;
2950	mrioc->facts.max_msix_vectors = le16_to_cpu(facts_data->max_msix_vectors);
2951	mrioc->facts.personality = (facts_flags &
2952	    MPI3_IOCFACTS_FLAGS_PERSONALITY_MASK);
2953	mrioc->facts.dma_mask = (facts_flags &
2954	    MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_MASK) >>
2955	    MPI3_IOCFACTS_FLAGS_DMA_ADDRESS_WIDTH_SHIFT;
2956	mrioc->facts.protocol_flags = facts_data->protocol_flags;
2957	mrioc->facts.mpi_version = le32_to_cpu(facts_data->mpi_version.word);
2958	mrioc->facts.max_reqs = le16_to_cpu(facts_data->max_outstanding_requests);
2959	mrioc->facts.product_id = le16_to_cpu(facts_data->product_id);
2960	mrioc->facts.reply_sz = le16_to_cpu(facts_data->reply_frame_size) * 4;
2961	mrioc->facts.exceptions = le16_to_cpu(facts_data->ioc_exceptions);
2962	mrioc->facts.max_perids = le16_to_cpu(facts_data->max_persistent_id);
2963	mrioc->facts.max_vds = le16_to_cpu(facts_data->max_vds);
2964	mrioc->facts.max_hpds = le16_to_cpu(facts_data->max_host_pds);
2965	mrioc->facts.max_advhpds = le16_to_cpu(facts_data->max_adv_host_pds);
2966	mrioc->facts.max_raid_pds = le16_to_cpu(facts_data->max_raid_pds);
2967	mrioc->facts.max_nvme = le16_to_cpu(facts_data->max_nvme);
2968	mrioc->facts.max_pcie_switches =
2969	    le16_to_cpu(facts_data->max_pcie_switches);
2970	mrioc->facts.max_sasexpanders =
2971	    le16_to_cpu(facts_data->max_sas_expanders);
2972	mrioc->facts.max_data_length = le16_to_cpu(facts_data->max_data_length);
2973	mrioc->facts.max_sasinitiators =
2974	    le16_to_cpu(facts_data->max_sas_initiators);
2975	mrioc->facts.max_enclosures = le16_to_cpu(facts_data->max_enclosures);
2976	mrioc->facts.min_devhandle = le16_to_cpu(facts_data->min_dev_handle);
2977	mrioc->facts.max_devhandle = le16_to_cpu(facts_data->max_dev_handle);
2978	mrioc->facts.max_op_req_q =
2979	    le16_to_cpu(facts_data->max_operational_request_queues);
2980	mrioc->facts.max_op_reply_q =
2981	    le16_to_cpu(facts_data->max_operational_reply_queues);
2982	mrioc->facts.ioc_capabilities =
2983	    le32_to_cpu(facts_data->ioc_capabilities);
2984	mrioc->facts.fw_ver.build_num =
2985	    le16_to_cpu(facts_data->fw_version.build_num);
2986	mrioc->facts.fw_ver.cust_id =
2987	    le16_to_cpu(facts_data->fw_version.customer_id);
2988	mrioc->facts.fw_ver.ph_minor = facts_data->fw_version.phase_minor;
2989	mrioc->facts.fw_ver.ph_major = facts_data->fw_version.phase_major;
2990	mrioc->facts.fw_ver.gen_minor = facts_data->fw_version.gen_minor;
2991	mrioc->facts.fw_ver.gen_major = facts_data->fw_version.gen_major;
2992	mrioc->msix_count = min_t(int, mrioc->msix_count,
2993	    mrioc->facts.max_msix_vectors);
2994	mrioc->facts.sge_mod_mask = facts_data->sge_modifier_mask;
2995	mrioc->facts.sge_mod_value = facts_data->sge_modifier_value;
2996	mrioc->facts.sge_mod_shift = facts_data->sge_modifier_shift;
2997	mrioc->facts.shutdown_timeout =
2998	    le16_to_cpu(facts_data->shutdown_timeout);
2999
3000	mrioc->facts.max_dev_per_tg =
3001	    facts_data->max_devices_per_throttle_group;
3002	mrioc->facts.io_throttle_data_length =
3003	    le16_to_cpu(facts_data->io_throttle_data_length);
3004	mrioc->facts.max_io_throttle_group =
3005	    le16_to_cpu(facts_data->max_io_throttle_group);
3006	mrioc->facts.io_throttle_low = le16_to_cpu(facts_data->io_throttle_low);
3007	mrioc->facts.io_throttle_high =
3008	    le16_to_cpu(facts_data->io_throttle_high);
3009
3010	if (mrioc->facts.max_data_length ==
3011	    MPI3_IOCFACTS_MAX_DATA_LENGTH_NOT_REPORTED)
3012		mrioc->facts.max_data_length = MPI3MR_DEFAULT_MAX_IO_SIZE;
3013	else
3014		mrioc->facts.max_data_length *= MPI3MR_PAGE_SIZE_4K;
3015	/* Store in 512b block count */
3016	if (mrioc->facts.io_throttle_data_length)
3017		mrioc->io_throttle_data_length =
3018		    (mrioc->facts.io_throttle_data_length * 2 * 4);
3019	else
3020		/* set the length to 1MB + 1K to disable throttle */
3021		mrioc->io_throttle_data_length = (mrioc->facts.max_data_length / 512) + 2;
3022
3023	mrioc->io_throttle_high = (mrioc->facts.io_throttle_high * 2 * 1024);
3024	mrioc->io_throttle_low = (mrioc->facts.io_throttle_low * 2 * 1024);
3025
3026	ioc_info(mrioc, "ioc_num(%d), maxopQ(%d), maxopRepQ(%d), maxdh(%d),",
3027	    mrioc->facts.ioc_num, mrioc->facts.max_op_req_q,
3028	    mrioc->facts.max_op_reply_q, mrioc->facts.max_devhandle);
3029	ioc_info(mrioc,
3030	    "maxreqs(%d), mindh(%d) maxvectors(%d) maxperids(%d)\n",
3031	    mrioc->facts.max_reqs, mrioc->facts.min_devhandle,
3032	    mrioc->facts.max_msix_vectors, mrioc->facts.max_perids);
3033	ioc_info(mrioc, "SGEModMask 0x%x SGEModVal 0x%x SGEModShift 0x%x ",
3034	    mrioc->facts.sge_mod_mask, mrioc->facts.sge_mod_value,
3035	    mrioc->facts.sge_mod_shift);
3036	ioc_info(mrioc, "DMA mask %d InitialPE status 0x%x max_data_len (%d)\n",
3037	    mrioc->facts.dma_mask, (facts_flags &
3038	    MPI3_IOCFACTS_FLAGS_INITIAL_PORT_ENABLE_MASK), mrioc->facts.max_data_length);
3039	ioc_info(mrioc,
3040	    "max_dev_per_throttle_group(%d), max_throttle_groups(%d)\n",
3041	    mrioc->facts.max_dev_per_tg, mrioc->facts.max_io_throttle_group);
3042	ioc_info(mrioc,
3043	   "io_throttle_data_len(%dKiB), io_throttle_high(%dMiB), io_throttle_low(%dMiB)\n",
3044	   mrioc->facts.io_throttle_data_length * 4,
3045	   mrioc->facts.io_throttle_high, mrioc->facts.io_throttle_low);
3046}
3047
3048/**
3049 * mpi3mr_alloc_reply_sense_bufs - Send IOC Init
3050 * @mrioc: Adapter instance reference
3051 *
3052 * Allocate and initialize the reply free buffers, sense
3053 * buffers, reply free queue and sense buffer queue.
3054 *
3055 * Return: 0 on success, non-zero on failures.
3056 */
3057static int mpi3mr_alloc_reply_sense_bufs(struct mpi3mr_ioc *mrioc)
3058{
3059	int retval = 0;
3060	u32 sz, i;
3061
3062	if (mrioc->init_cmds.reply)
3063		return retval;
3064
3065	mrioc->init_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
3066	if (!mrioc->init_cmds.reply)
3067		goto out_failed;
3068
3069	mrioc->bsg_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
3070	if (!mrioc->bsg_cmds.reply)
3071		goto out_failed;
3072
3073	mrioc->transport_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
3074	if (!mrioc->transport_cmds.reply)
3075		goto out_failed;
3076
3077	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
3078		mrioc->dev_rmhs_cmds[i].reply = kzalloc(mrioc->reply_sz,
3079		    GFP_KERNEL);
3080		if (!mrioc->dev_rmhs_cmds[i].reply)
3081			goto out_failed;
3082	}
3083
3084	for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
3085		mrioc->evtack_cmds[i].reply = kzalloc(mrioc->reply_sz,
3086		    GFP_KERNEL);
3087		if (!mrioc->evtack_cmds[i].reply)
3088			goto out_failed;
3089	}
3090
3091	mrioc->host_tm_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
3092	if (!mrioc->host_tm_cmds.reply)
3093		goto out_failed;
3094
3095	mrioc->pel_cmds.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
3096	if (!mrioc->pel_cmds.reply)
3097		goto out_failed;
3098
3099	mrioc->pel_abort_cmd.reply = kzalloc(mrioc->reply_sz, GFP_KERNEL);
3100	if (!mrioc->pel_abort_cmd.reply)
3101		goto out_failed;
3102
3103	mrioc->dev_handle_bitmap_bits = mrioc->facts.max_devhandle;
3104	mrioc->removepend_bitmap = bitmap_zalloc(mrioc->dev_handle_bitmap_bits,
3105						 GFP_KERNEL);
 
 
3106	if (!mrioc->removepend_bitmap)
3107		goto out_failed;
3108
3109	mrioc->devrem_bitmap = bitmap_zalloc(MPI3MR_NUM_DEVRMCMD, GFP_KERNEL);
 
 
 
 
3110	if (!mrioc->devrem_bitmap)
3111		goto out_failed;
3112
3113	mrioc->evtack_cmds_bitmap = bitmap_zalloc(MPI3MR_NUM_EVTACKCMD,
3114						  GFP_KERNEL);
 
 
 
3115	if (!mrioc->evtack_cmds_bitmap)
3116		goto out_failed;
3117
3118	mrioc->num_reply_bufs = mrioc->facts.max_reqs + MPI3MR_NUM_EVT_REPLIES;
3119	mrioc->reply_free_qsz = mrioc->num_reply_bufs + 1;
3120	mrioc->num_sense_bufs = mrioc->facts.max_reqs / MPI3MR_SENSEBUF_FACTOR;
3121	mrioc->sense_buf_q_sz = mrioc->num_sense_bufs + 1;
3122
3123	/* reply buffer pool, 16 byte align */
3124	sz = mrioc->num_reply_bufs * mrioc->reply_sz;
3125	mrioc->reply_buf_pool = dma_pool_create("reply_buf pool",
3126	    &mrioc->pdev->dev, sz, 16, 0);
3127	if (!mrioc->reply_buf_pool) {
3128		ioc_err(mrioc, "reply buf pool: dma_pool_create failed\n");
3129		goto out_failed;
3130	}
3131
3132	mrioc->reply_buf = dma_pool_zalloc(mrioc->reply_buf_pool, GFP_KERNEL,
3133	    &mrioc->reply_buf_dma);
3134	if (!mrioc->reply_buf)
3135		goto out_failed;
3136
3137	mrioc->reply_buf_dma_max_address = mrioc->reply_buf_dma + sz;
3138
3139	/* reply free queue, 8 byte align */
3140	sz = mrioc->reply_free_qsz * 8;
3141	mrioc->reply_free_q_pool = dma_pool_create("reply_free_q pool",
3142	    &mrioc->pdev->dev, sz, 8, 0);
3143	if (!mrioc->reply_free_q_pool) {
3144		ioc_err(mrioc, "reply_free_q pool: dma_pool_create failed\n");
3145		goto out_failed;
3146	}
3147	mrioc->reply_free_q = dma_pool_zalloc(mrioc->reply_free_q_pool,
3148	    GFP_KERNEL, &mrioc->reply_free_q_dma);
3149	if (!mrioc->reply_free_q)
3150		goto out_failed;
3151
3152	/* sense buffer pool,  4 byte align */
3153	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3154	mrioc->sense_buf_pool = dma_pool_create("sense_buf pool",
3155	    &mrioc->pdev->dev, sz, 4, 0);
3156	if (!mrioc->sense_buf_pool) {
3157		ioc_err(mrioc, "sense_buf pool: dma_pool_create failed\n");
3158		goto out_failed;
3159	}
3160	mrioc->sense_buf = dma_pool_zalloc(mrioc->sense_buf_pool, GFP_KERNEL,
3161	    &mrioc->sense_buf_dma);
3162	if (!mrioc->sense_buf)
3163		goto out_failed;
3164
3165	/* sense buffer queue, 8 byte align */
3166	sz = mrioc->sense_buf_q_sz * 8;
3167	mrioc->sense_buf_q_pool = dma_pool_create("sense_buf_q pool",
3168	    &mrioc->pdev->dev, sz, 8, 0);
3169	if (!mrioc->sense_buf_q_pool) {
3170		ioc_err(mrioc, "sense_buf_q pool: dma_pool_create failed\n");
3171		goto out_failed;
3172	}
3173	mrioc->sense_buf_q = dma_pool_zalloc(mrioc->sense_buf_q_pool,
3174	    GFP_KERNEL, &mrioc->sense_buf_q_dma);
3175	if (!mrioc->sense_buf_q)
3176		goto out_failed;
3177
3178	return retval;
3179
3180out_failed:
3181	retval = -1;
3182	return retval;
3183}
3184
3185/**
3186 * mpimr_initialize_reply_sbuf_queues - initialize reply sense
3187 * buffers
3188 * @mrioc: Adapter instance reference
3189 *
3190 * Helper function to initialize reply and sense buffers along
3191 * with some debug prints.
3192 *
3193 * Return:  None.
3194 */
3195static void mpimr_initialize_reply_sbuf_queues(struct mpi3mr_ioc *mrioc)
3196{
3197	u32 sz, i;
3198	dma_addr_t phy_addr;
3199
3200	sz = mrioc->num_reply_bufs * mrioc->reply_sz;
3201	ioc_info(mrioc,
3202	    "reply buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
3203	    mrioc->reply_buf, mrioc->num_reply_bufs, mrioc->reply_sz,
3204	    (sz / 1024), (unsigned long long)mrioc->reply_buf_dma);
3205	sz = mrioc->reply_free_qsz * 8;
3206	ioc_info(mrioc,
3207	    "reply_free_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), reply_dma(0x%llx)\n",
3208	    mrioc->reply_free_q, mrioc->reply_free_qsz, 8, (sz / 1024),
3209	    (unsigned long long)mrioc->reply_free_q_dma);
3210	sz = mrioc->num_sense_bufs * MPI3MR_SENSE_BUF_SZ;
3211	ioc_info(mrioc,
3212	    "sense_buf pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
3213	    mrioc->sense_buf, mrioc->num_sense_bufs, MPI3MR_SENSE_BUF_SZ,
3214	    (sz / 1024), (unsigned long long)mrioc->sense_buf_dma);
3215	sz = mrioc->sense_buf_q_sz * 8;
3216	ioc_info(mrioc,
3217	    "sense_buf_q pool(0x%p): depth(%d), frame_size(%d), pool_size(%d kB), sense_dma(0x%llx)\n",
3218	    mrioc->sense_buf_q, mrioc->sense_buf_q_sz, 8, (sz / 1024),
3219	    (unsigned long long)mrioc->sense_buf_q_dma);
3220
3221	/* initialize Reply buffer Queue */
3222	for (i = 0, phy_addr = mrioc->reply_buf_dma;
3223	    i < mrioc->num_reply_bufs; i++, phy_addr += mrioc->reply_sz)
3224		mrioc->reply_free_q[i] = cpu_to_le64(phy_addr);
3225	mrioc->reply_free_q[i] = cpu_to_le64(0);
3226
3227	/* initialize Sense Buffer Queue */
3228	for (i = 0, phy_addr = mrioc->sense_buf_dma;
3229	    i < mrioc->num_sense_bufs; i++, phy_addr += MPI3MR_SENSE_BUF_SZ)
3230		mrioc->sense_buf_q[i] = cpu_to_le64(phy_addr);
3231	mrioc->sense_buf_q[i] = cpu_to_le64(0);
3232}
3233
3234/**
3235 * mpi3mr_issue_iocinit - Send IOC Init
3236 * @mrioc: Adapter instance reference
3237 *
3238 * Issue IOC Init MPI request through admin queue and wait for
3239 * the completion of it or time out.
3240 *
3241 * Return: 0 on success, non-zero on failures.
3242 */
3243static int mpi3mr_issue_iocinit(struct mpi3mr_ioc *mrioc)
3244{
3245	struct mpi3_ioc_init_request iocinit_req;
3246	struct mpi3_driver_info_layout *drv_info;
3247	dma_addr_t data_dma;
3248	u32 data_len = sizeof(*drv_info);
3249	int retval = 0;
3250	ktime_t current_time;
3251
3252	drv_info = dma_alloc_coherent(&mrioc->pdev->dev, data_len, &data_dma,
3253	    GFP_KERNEL);
3254	if (!drv_info) {
3255		retval = -1;
3256		goto out;
3257	}
3258	mpimr_initialize_reply_sbuf_queues(mrioc);
3259
3260	drv_info->information_length = cpu_to_le32(data_len);
3261	strscpy(drv_info->driver_signature, "Broadcom", sizeof(drv_info->driver_signature));
3262	strscpy(drv_info->os_name, utsname()->sysname, sizeof(drv_info->os_name));
3263	strscpy(drv_info->os_version, utsname()->release, sizeof(drv_info->os_version));
3264	strscpy(drv_info->driver_name, MPI3MR_DRIVER_NAME, sizeof(drv_info->driver_name));
3265	strscpy(drv_info->driver_version, MPI3MR_DRIVER_VERSION, sizeof(drv_info->driver_version));
3266	strscpy(drv_info->driver_release_date, MPI3MR_DRIVER_RELDATE,
3267	    sizeof(drv_info->driver_release_date));
3268	drv_info->driver_capabilities = 0;
3269	memcpy((u8 *)&mrioc->driver_info, (u8 *)drv_info,
3270	    sizeof(mrioc->driver_info));
3271
3272	memset(&iocinit_req, 0, sizeof(iocinit_req));
3273	mutex_lock(&mrioc->init_cmds.mutex);
3274	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3275		retval = -1;
3276		ioc_err(mrioc, "Issue IOCInit: Init command is in use\n");
3277		mutex_unlock(&mrioc->init_cmds.mutex);
3278		goto out;
3279	}
3280	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3281	mrioc->init_cmds.is_waiting = 1;
3282	mrioc->init_cmds.callback = NULL;
3283	iocinit_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3284	iocinit_req.function = MPI3_FUNCTION_IOC_INIT;
3285	iocinit_req.mpi_version.mpi3_version.dev = MPI3_VERSION_DEV;
3286	iocinit_req.mpi_version.mpi3_version.unit = MPI3_VERSION_UNIT;
3287	iocinit_req.mpi_version.mpi3_version.major = MPI3_VERSION_MAJOR;
3288	iocinit_req.mpi_version.mpi3_version.minor = MPI3_VERSION_MINOR;
3289	iocinit_req.who_init = MPI3_WHOINIT_HOST_DRIVER;
3290	iocinit_req.reply_free_queue_depth = cpu_to_le16(mrioc->reply_free_qsz);
3291	iocinit_req.reply_free_queue_address =
3292	    cpu_to_le64(mrioc->reply_free_q_dma);
3293	iocinit_req.sense_buffer_length = cpu_to_le16(MPI3MR_SENSE_BUF_SZ);
3294	iocinit_req.sense_buffer_free_queue_depth =
3295	    cpu_to_le16(mrioc->sense_buf_q_sz);
3296	iocinit_req.sense_buffer_free_queue_address =
3297	    cpu_to_le64(mrioc->sense_buf_q_dma);
3298	iocinit_req.driver_information_address = cpu_to_le64(data_dma);
3299
3300	current_time = ktime_get_real();
3301	iocinit_req.time_stamp = cpu_to_le64(ktime_to_ms(current_time));
3302
3303	iocinit_req.msg_flags |=
3304	    MPI3_IOCINIT_MSGFLAGS_SCSIIOSTATUSREPLY_SUPPORTED;
3305
3306	init_completion(&mrioc->init_cmds.done);
3307	retval = mpi3mr_admin_request_post(mrioc, &iocinit_req,
3308	    sizeof(iocinit_req), 1);
3309	if (retval) {
3310		ioc_err(mrioc, "Issue IOCInit: Admin Post failed\n");
3311		goto out_unlock;
3312	}
3313	wait_for_completion_timeout(&mrioc->init_cmds.done,
3314	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3315	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3316		mpi3mr_check_rh_fault_ioc(mrioc,
3317		    MPI3MR_RESET_FROM_IOCINIT_TIMEOUT);
3318		ioc_err(mrioc, "ioc_init timed out\n");
3319		retval = -1;
3320		goto out_unlock;
3321	}
3322	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3323	    != MPI3_IOCSTATUS_SUCCESS) {
3324		ioc_err(mrioc,
3325		    "Issue IOCInit: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3326		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3327		    mrioc->init_cmds.ioc_loginfo);
3328		retval = -1;
3329		goto out_unlock;
3330	}
3331
3332	mrioc->reply_free_queue_host_index = mrioc->num_reply_bufs;
3333	writel(mrioc->reply_free_queue_host_index,
3334	    &mrioc->sysif_regs->reply_free_host_index);
3335
3336	mrioc->sbq_host_index = mrioc->num_sense_bufs;
3337	writel(mrioc->sbq_host_index,
3338	    &mrioc->sysif_regs->sense_buffer_free_host_index);
3339out_unlock:
3340	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3341	mutex_unlock(&mrioc->init_cmds.mutex);
3342
3343out:
3344	if (drv_info)
3345		dma_free_coherent(&mrioc->pdev->dev, data_len, drv_info,
3346		    data_dma);
3347
3348	return retval;
3349}
3350
3351/**
3352 * mpi3mr_unmask_events - Unmask events in event mask bitmap
3353 * @mrioc: Adapter instance reference
3354 * @event: MPI event ID
3355 *
3356 * Un mask the specific event by resetting the event_mask
3357 * bitmap.
3358 *
3359 * Return: 0 on success, non-zero on failures.
3360 */
3361static void mpi3mr_unmask_events(struct mpi3mr_ioc *mrioc, u16 event)
3362{
3363	u32 desired_event;
3364	u8 word;
3365
3366	if (event >= 128)
3367		return;
3368
3369	desired_event = (1 << (event % 32));
3370	word = event / 32;
3371
3372	mrioc->event_masks[word] &= ~desired_event;
3373}
3374
3375/**
3376 * mpi3mr_issue_event_notification - Send event notification
3377 * @mrioc: Adapter instance reference
3378 *
3379 * Issue event notification MPI request through admin queue and
3380 * wait for the completion of it or time out.
3381 *
3382 * Return: 0 on success, non-zero on failures.
3383 */
3384static int mpi3mr_issue_event_notification(struct mpi3mr_ioc *mrioc)
3385{
3386	struct mpi3_event_notification_request evtnotify_req;
3387	int retval = 0;
3388	u8 i;
3389
3390	memset(&evtnotify_req, 0, sizeof(evtnotify_req));
3391	mutex_lock(&mrioc->init_cmds.mutex);
3392	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3393		retval = -1;
3394		ioc_err(mrioc, "Issue EvtNotify: Init command is in use\n");
3395		mutex_unlock(&mrioc->init_cmds.mutex);
3396		goto out;
3397	}
3398	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3399	mrioc->init_cmds.is_waiting = 1;
3400	mrioc->init_cmds.callback = NULL;
3401	evtnotify_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3402	evtnotify_req.function = MPI3_FUNCTION_EVENT_NOTIFICATION;
3403	for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3404		evtnotify_req.event_masks[i] =
3405		    cpu_to_le32(mrioc->event_masks[i]);
3406	init_completion(&mrioc->init_cmds.done);
3407	retval = mpi3mr_admin_request_post(mrioc, &evtnotify_req,
3408	    sizeof(evtnotify_req), 1);
3409	if (retval) {
3410		ioc_err(mrioc, "Issue EvtNotify: Admin Post failed\n");
3411		goto out_unlock;
3412	}
3413	wait_for_completion_timeout(&mrioc->init_cmds.done,
3414	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3415	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3416		ioc_err(mrioc, "event notification timed out\n");
3417		mpi3mr_check_rh_fault_ioc(mrioc,
3418		    MPI3MR_RESET_FROM_EVTNOTIFY_TIMEOUT);
3419		retval = -1;
3420		goto out_unlock;
3421	}
3422	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3423	    != MPI3_IOCSTATUS_SUCCESS) {
3424		ioc_err(mrioc,
3425		    "Issue EvtNotify: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3426		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3427		    mrioc->init_cmds.ioc_loginfo);
3428		retval = -1;
3429		goto out_unlock;
3430	}
3431
3432out_unlock:
3433	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3434	mutex_unlock(&mrioc->init_cmds.mutex);
3435out:
3436	return retval;
3437}
3438
3439/**
3440 * mpi3mr_process_event_ack - Process event acknowledgment
3441 * @mrioc: Adapter instance reference
3442 * @event: MPI3 event ID
3443 * @event_ctx: event context
3444 *
3445 * Send event acknowledgment through admin queue and wait for
3446 * it to complete.
3447 *
3448 * Return: 0 on success, non-zero on failures.
3449 */
3450int mpi3mr_process_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
3451	u32 event_ctx)
3452{
3453	struct mpi3_event_ack_request evtack_req;
3454	int retval = 0;
3455
3456	memset(&evtack_req, 0, sizeof(evtack_req));
3457	mutex_lock(&mrioc->init_cmds.mutex);
3458	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3459		retval = -1;
3460		ioc_err(mrioc, "Send EvtAck: Init command is in use\n");
3461		mutex_unlock(&mrioc->init_cmds.mutex);
3462		goto out;
3463	}
3464	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3465	mrioc->init_cmds.is_waiting = 1;
3466	mrioc->init_cmds.callback = NULL;
3467	evtack_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3468	evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
3469	evtack_req.event = event;
3470	evtack_req.event_context = cpu_to_le32(event_ctx);
3471
3472	init_completion(&mrioc->init_cmds.done);
3473	retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
3474	    sizeof(evtack_req), 1);
3475	if (retval) {
3476		ioc_err(mrioc, "Send EvtAck: Admin Post failed\n");
3477		goto out_unlock;
3478	}
3479	wait_for_completion_timeout(&mrioc->init_cmds.done,
3480	    (MPI3MR_INTADMCMD_TIMEOUT * HZ));
3481	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3482		ioc_err(mrioc, "Issue EvtNotify: command timed out\n");
3483		if (!(mrioc->init_cmds.state & MPI3MR_CMD_RESET))
3484			mpi3mr_check_rh_fault_ioc(mrioc,
3485			    MPI3MR_RESET_FROM_EVTACK_TIMEOUT);
3486		retval = -1;
3487		goto out_unlock;
3488	}
3489	if ((mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK)
3490	    != MPI3_IOCSTATUS_SUCCESS) {
3491		ioc_err(mrioc,
3492		    "Send EvtAck: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
3493		    (mrioc->init_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
3494		    mrioc->init_cmds.ioc_loginfo);
3495		retval = -1;
3496		goto out_unlock;
3497	}
3498
3499out_unlock:
3500	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3501	mutex_unlock(&mrioc->init_cmds.mutex);
3502out:
3503	return retval;
3504}
3505
3506/**
3507 * mpi3mr_alloc_chain_bufs - Allocate chain buffers
3508 * @mrioc: Adapter instance reference
3509 *
3510 * Allocate chain buffers and set a bitmap to indicate free
3511 * chain buffers. Chain buffers are used to pass the SGE
3512 * information along with MPI3 SCSI IO requests for host I/O.
3513 *
3514 * Return: 0 on success, non-zero on failure
3515 */
3516static int mpi3mr_alloc_chain_bufs(struct mpi3mr_ioc *mrioc)
3517{
3518	int retval = 0;
3519	u32 sz, i;
3520	u16 num_chains;
3521
3522	if (mrioc->chain_sgl_list)
3523		return retval;
3524
3525	num_chains = mrioc->max_host_ios / MPI3MR_CHAINBUF_FACTOR;
3526
3527	if (prot_mask & (SHOST_DIX_TYPE0_PROTECTION
3528	    | SHOST_DIX_TYPE1_PROTECTION
3529	    | SHOST_DIX_TYPE2_PROTECTION
3530	    | SHOST_DIX_TYPE3_PROTECTION))
3531		num_chains += (num_chains / MPI3MR_CHAINBUFDIX_FACTOR);
3532
3533	mrioc->chain_buf_count = num_chains;
3534	sz = sizeof(struct chain_element) * num_chains;
3535	mrioc->chain_sgl_list = kzalloc(sz, GFP_KERNEL);
3536	if (!mrioc->chain_sgl_list)
3537		goto out_failed;
3538
3539	if (mrioc->max_sgl_entries > (mrioc->facts.max_data_length /
3540		MPI3MR_PAGE_SIZE_4K))
3541		mrioc->max_sgl_entries = mrioc->facts.max_data_length /
3542			MPI3MR_PAGE_SIZE_4K;
3543	sz = mrioc->max_sgl_entries * sizeof(struct mpi3_sge_common);
3544	ioc_info(mrioc, "number of sgl entries=%d chain buffer size=%dKB\n",
3545			mrioc->max_sgl_entries, sz/1024);
3546
3547	mrioc->chain_buf_pool = dma_pool_create("chain_buf pool",
3548	    &mrioc->pdev->dev, sz, 16, 0);
3549	if (!mrioc->chain_buf_pool) {
3550		ioc_err(mrioc, "chain buf pool: dma_pool_create failed\n");
3551		goto out_failed;
3552	}
3553
3554	for (i = 0; i < num_chains; i++) {
3555		mrioc->chain_sgl_list[i].addr =
3556		    dma_pool_zalloc(mrioc->chain_buf_pool, GFP_KERNEL,
3557		    &mrioc->chain_sgl_list[i].dma_addr);
3558
3559		if (!mrioc->chain_sgl_list[i].addr)
3560			goto out_failed;
3561	}
3562	mrioc->chain_bitmap = bitmap_zalloc(num_chains, GFP_KERNEL);
 
 
 
3563	if (!mrioc->chain_bitmap)
3564		goto out_failed;
3565	return retval;
3566out_failed:
3567	retval = -1;
3568	return retval;
3569}
3570
3571/**
3572 * mpi3mr_port_enable_complete - Mark port enable complete
3573 * @mrioc: Adapter instance reference
3574 * @drv_cmd: Internal command tracker
3575 *
3576 * Call back for asynchronous port enable request sets the
3577 * driver command to indicate port enable request is complete.
3578 *
3579 * Return: Nothing
3580 */
3581static void mpi3mr_port_enable_complete(struct mpi3mr_ioc *mrioc,
3582	struct mpi3mr_drv_cmd *drv_cmd)
3583{
3584	drv_cmd->callback = NULL;
3585	mrioc->scan_started = 0;
3586	if (drv_cmd->state & MPI3MR_CMD_RESET)
3587		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
3588	else
3589		mrioc->scan_failed = drv_cmd->ioc_status;
3590	drv_cmd->state = MPI3MR_CMD_NOTUSED;
3591}
3592
3593/**
3594 * mpi3mr_issue_port_enable - Issue Port Enable
3595 * @mrioc: Adapter instance reference
3596 * @async: Flag to wait for completion or not
3597 *
3598 * Issue Port Enable MPI request through admin queue and if the
3599 * async flag is not set wait for the completion of the port
3600 * enable or time out.
3601 *
3602 * Return: 0 on success, non-zero on failures.
3603 */
3604int mpi3mr_issue_port_enable(struct mpi3mr_ioc *mrioc, u8 async)
3605{
3606	struct mpi3_port_enable_request pe_req;
3607	int retval = 0;
3608	u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
3609
3610	memset(&pe_req, 0, sizeof(pe_req));
3611	mutex_lock(&mrioc->init_cmds.mutex);
3612	if (mrioc->init_cmds.state & MPI3MR_CMD_PENDING) {
3613		retval = -1;
3614		ioc_err(mrioc, "Issue PortEnable: Init command is in use\n");
3615		mutex_unlock(&mrioc->init_cmds.mutex);
3616		goto out;
3617	}
3618	mrioc->init_cmds.state = MPI3MR_CMD_PENDING;
3619	if (async) {
3620		mrioc->init_cmds.is_waiting = 0;
3621		mrioc->init_cmds.callback = mpi3mr_port_enable_complete;
3622	} else {
3623		mrioc->init_cmds.is_waiting = 1;
3624		mrioc->init_cmds.callback = NULL;
3625		init_completion(&mrioc->init_cmds.done);
3626	}
3627	pe_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INITCMDS);
3628	pe_req.function = MPI3_FUNCTION_PORT_ENABLE;
3629
3630	retval = mpi3mr_admin_request_post(mrioc, &pe_req, sizeof(pe_req), 1);
3631	if (retval) {
3632		ioc_err(mrioc, "Issue PortEnable: Admin Post failed\n");
3633		goto out_unlock;
3634	}
3635	if (async) {
3636		mutex_unlock(&mrioc->init_cmds.mutex);
3637		goto out;
3638	}
3639
3640	wait_for_completion_timeout(&mrioc->init_cmds.done, (pe_timeout * HZ));
3641	if (!(mrioc->init_cmds.state & MPI3MR_CMD_COMPLETE)) {
3642		ioc_err(mrioc, "port enable timed out\n");
3643		retval = -1;
3644		mpi3mr_check_rh_fault_ioc(mrioc, MPI3MR_RESET_FROM_PE_TIMEOUT);
3645		goto out_unlock;
3646	}
3647	mpi3mr_port_enable_complete(mrioc, &mrioc->init_cmds);
3648
3649out_unlock:
3650	mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
3651	mutex_unlock(&mrioc->init_cmds.mutex);
3652out:
3653	return retval;
3654}
3655
3656/* Protocol type to name mapper structure */
3657static const struct {
3658	u8 protocol;
3659	char *name;
3660} mpi3mr_protocols[] = {
3661	{ MPI3_IOCFACTS_PROTOCOL_SCSI_INITIATOR, "Initiator" },
3662	{ MPI3_IOCFACTS_PROTOCOL_SCSI_TARGET, "Target" },
3663	{ MPI3_IOCFACTS_PROTOCOL_NVME, "NVMe attachment" },
3664};
3665
3666/* Capability to name mapper structure*/
3667static const struct {
3668	u32 capability;
3669	char *name;
3670} mpi3mr_capabilities[] = {
3671	{ MPI3_IOCFACTS_CAPABILITY_RAID_CAPABLE, "RAID" },
3672	{ MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED, "MultiPath" },
3673};
3674
3675/**
3676 * mpi3mr_print_ioc_info - Display controller information
3677 * @mrioc: Adapter instance reference
3678 *
3679 * Display controller personalit, capability, supported
3680 * protocols etc.
3681 *
3682 * Return: Nothing
3683 */
3684static void
3685mpi3mr_print_ioc_info(struct mpi3mr_ioc *mrioc)
3686{
3687	int i = 0, bytes_written = 0;
3688	char personality[16];
3689	char protocol[50] = {0};
3690	char capabilities[100] = {0};
3691	struct mpi3mr_compimg_ver *fwver = &mrioc->facts.fw_ver;
3692
3693	switch (mrioc->facts.personality) {
3694	case MPI3_IOCFACTS_FLAGS_PERSONALITY_EHBA:
3695		strncpy(personality, "Enhanced HBA", sizeof(personality));
3696		break;
3697	case MPI3_IOCFACTS_FLAGS_PERSONALITY_RAID_DDR:
3698		strncpy(personality, "RAID", sizeof(personality));
3699		break;
3700	default:
3701		strncpy(personality, "Unknown", sizeof(personality));
3702		break;
3703	}
3704
3705	ioc_info(mrioc, "Running in %s Personality", personality);
3706
3707	ioc_info(mrioc, "FW version(%d.%d.%d.%d.%d.%d)\n",
3708	    fwver->gen_major, fwver->gen_minor, fwver->ph_major,
3709	    fwver->ph_minor, fwver->cust_id, fwver->build_num);
3710
3711	for (i = 0; i < ARRAY_SIZE(mpi3mr_protocols); i++) {
3712		if (mrioc->facts.protocol_flags &
3713		    mpi3mr_protocols[i].protocol) {
3714			bytes_written += scnprintf(protocol + bytes_written,
3715				    sizeof(protocol) - bytes_written, "%s%s",
3716				    bytes_written ? "," : "",
3717				    mpi3mr_protocols[i].name);
3718		}
3719	}
3720
3721	bytes_written = 0;
3722	for (i = 0; i < ARRAY_SIZE(mpi3mr_capabilities); i++) {
3723		if (mrioc->facts.protocol_flags &
3724		    mpi3mr_capabilities[i].capability) {
3725			bytes_written += scnprintf(capabilities + bytes_written,
3726				    sizeof(capabilities) - bytes_written, "%s%s",
3727				    bytes_written ? "," : "",
3728				    mpi3mr_capabilities[i].name);
3729		}
3730	}
3731
3732	ioc_info(mrioc, "Protocol=(%s), Capabilities=(%s)\n",
3733		 protocol, capabilities);
3734}
3735
3736/**
3737 * mpi3mr_cleanup_resources - Free PCI resources
3738 * @mrioc: Adapter instance reference
3739 *
3740 * Unmap PCI device memory and disable PCI device.
3741 *
3742 * Return: 0 on success and non-zero on failure.
3743 */
3744void mpi3mr_cleanup_resources(struct mpi3mr_ioc *mrioc)
3745{
3746	struct pci_dev *pdev = mrioc->pdev;
3747
3748	mpi3mr_cleanup_isr(mrioc);
3749
3750	if (mrioc->sysif_regs) {
3751		iounmap((void __iomem *)mrioc->sysif_regs);
3752		mrioc->sysif_regs = NULL;
3753	}
3754
3755	if (pci_is_enabled(pdev)) {
3756		if (mrioc->bars)
3757			pci_release_selected_regions(pdev, mrioc->bars);
3758		pci_disable_device(pdev);
3759	}
3760}
3761
3762/**
3763 * mpi3mr_setup_resources - Enable PCI resources
3764 * @mrioc: Adapter instance reference
3765 *
3766 * Enable PCI device memory, MSI-x registers and set DMA mask.
3767 *
3768 * Return: 0 on success and non-zero on failure.
3769 */
3770int mpi3mr_setup_resources(struct mpi3mr_ioc *mrioc)
3771{
3772	struct pci_dev *pdev = mrioc->pdev;
3773	u32 memap_sz = 0;
3774	int i, retval = 0, capb = 0;
3775	u16 message_control;
3776	u64 dma_mask = mrioc->dma_mask ? mrioc->dma_mask :
3777	    ((sizeof(dma_addr_t) > 4) ? DMA_BIT_MASK(64) : DMA_BIT_MASK(32));
3778
3779	if (pci_enable_device_mem(pdev)) {
3780		ioc_err(mrioc, "pci_enable_device_mem: failed\n");
3781		retval = -ENODEV;
3782		goto out_failed;
3783	}
3784
3785	capb = pci_find_capability(pdev, PCI_CAP_ID_MSIX);
3786	if (!capb) {
3787		ioc_err(mrioc, "Unable to find MSI-X Capabilities\n");
3788		retval = -ENODEV;
3789		goto out_failed;
3790	}
3791	mrioc->bars = pci_select_bars(pdev, IORESOURCE_MEM);
3792
3793	if (pci_request_selected_regions(pdev, mrioc->bars,
3794	    mrioc->driver_name)) {
3795		ioc_err(mrioc, "pci_request_selected_regions: failed\n");
3796		retval = -ENODEV;
3797		goto out_failed;
3798	}
3799
3800	for (i = 0; (i < DEVICE_COUNT_RESOURCE); i++) {
3801		if (pci_resource_flags(pdev, i) & IORESOURCE_MEM) {
3802			mrioc->sysif_regs_phys = pci_resource_start(pdev, i);
3803			memap_sz = pci_resource_len(pdev, i);
3804			mrioc->sysif_regs =
3805			    ioremap(mrioc->sysif_regs_phys, memap_sz);
3806			break;
3807		}
3808	}
3809
3810	pci_set_master(pdev);
3811
3812	retval = dma_set_mask_and_coherent(&pdev->dev, dma_mask);
3813	if (retval) {
3814		if (dma_mask != DMA_BIT_MASK(32)) {
3815			ioc_warn(mrioc, "Setting 64 bit DMA mask failed\n");
3816			dma_mask = DMA_BIT_MASK(32);
3817			retval = dma_set_mask_and_coherent(&pdev->dev,
3818			    dma_mask);
3819		}
3820		if (retval) {
3821			mrioc->dma_mask = 0;
3822			ioc_err(mrioc, "Setting 32 bit DMA mask also failed\n");
3823			goto out_failed;
3824		}
3825	}
3826	mrioc->dma_mask = dma_mask;
3827
3828	if (!mrioc->sysif_regs) {
3829		ioc_err(mrioc,
3830		    "Unable to map adapter memory or resource not found\n");
3831		retval = -EINVAL;
3832		goto out_failed;
3833	}
3834
3835	pci_read_config_word(pdev, capb + 2, &message_control);
3836	mrioc->msix_count = (message_control & 0x3FF) + 1;
3837
3838	pci_save_state(pdev);
3839
3840	pci_set_drvdata(pdev, mrioc->shost);
3841
3842	mpi3mr_ioc_disable_intr(mrioc);
3843
3844	ioc_info(mrioc, "iomem(0x%016llx), mapped(0x%p), size(%d)\n",
3845	    (unsigned long long)mrioc->sysif_regs_phys,
3846	    mrioc->sysif_regs, memap_sz);
3847	ioc_info(mrioc, "Number of MSI-X vectors found in capabilities: (%d)\n",
3848	    mrioc->msix_count);
3849
3850	if (!reset_devices && poll_queues > 0)
3851		mrioc->requested_poll_qcount = min_t(int, poll_queues,
3852				mrioc->msix_count - 2);
3853	return retval;
3854
3855out_failed:
3856	mpi3mr_cleanup_resources(mrioc);
3857	return retval;
3858}
3859
3860/**
3861 * mpi3mr_enable_events - Enable required events
3862 * @mrioc: Adapter instance reference
3863 *
3864 * This routine unmasks the events required by the driver by
3865 * sennding appropriate event mask bitmapt through an event
3866 * notification request.
3867 *
3868 * Return: 0 on success and non-zero on failure.
3869 */
3870static int mpi3mr_enable_events(struct mpi3mr_ioc *mrioc)
3871{
3872	int retval = 0;
3873	u32  i;
3874
3875	for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
3876		mrioc->event_masks[i] = -1;
3877
3878	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_ADDED);
3879	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_INFO_CHANGED);
3880	mpi3mr_unmask_events(mrioc, MPI3_EVENT_DEVICE_STATUS_CHANGE);
3881	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE);
3882	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENCL_DEVICE_ADDED);
3883	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST);
3884	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DISCOVERY);
3885	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR);
3886	mpi3mr_unmask_events(mrioc, MPI3_EVENT_SAS_BROADCAST_PRIMITIVE);
3887	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST);
3888	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PCIE_ENUMERATION);
3889	mpi3mr_unmask_events(mrioc, MPI3_EVENT_PREPARE_FOR_RESET);
3890	mpi3mr_unmask_events(mrioc, MPI3_EVENT_CABLE_MGMT);
3891	mpi3mr_unmask_events(mrioc, MPI3_EVENT_ENERGY_PACK_CHANGE);
3892
3893	retval = mpi3mr_issue_event_notification(mrioc);
3894	if (retval)
3895		ioc_err(mrioc, "failed to issue event notification %d\n",
3896		    retval);
3897	return retval;
3898}
3899
3900/**
3901 * mpi3mr_init_ioc - Initialize the controller
3902 * @mrioc: Adapter instance reference
3903 *
3904 * This the controller initialization routine, executed either
3905 * after soft reset or from pci probe callback.
3906 * Setup the required resources, memory map the controller
3907 * registers, create admin and operational reply queue pairs,
3908 * allocate required memory for reply pool, sense buffer pool,
3909 * issue IOC init request to the firmware, unmask the events and
3910 * issue port enable to discover SAS/SATA/NVMe devies and RAID
3911 * volumes.
3912 *
3913 * Return: 0 on success and non-zero on failure.
3914 */
3915int mpi3mr_init_ioc(struct mpi3mr_ioc *mrioc)
3916{
3917	int retval = 0;
3918	u8 retry = 0;
3919	struct mpi3_ioc_facts_data facts_data;
3920	u32 sz;
3921
3922retry_init:
3923	retval = mpi3mr_bring_ioc_ready(mrioc);
3924	if (retval) {
3925		ioc_err(mrioc, "Failed to bring ioc ready: error %d\n",
3926		    retval);
3927		goto out_failed_noretry;
3928	}
3929
3930	retval = mpi3mr_setup_isr(mrioc, 1);
3931	if (retval) {
3932		ioc_err(mrioc, "Failed to setup ISR error %d\n",
3933		    retval);
3934		goto out_failed_noretry;
3935	}
3936
3937	retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
3938	if (retval) {
3939		ioc_err(mrioc, "Failed to Issue IOC Facts %d\n",
3940		    retval);
3941		goto out_failed;
3942	}
3943
3944	mrioc->max_host_ios = mrioc->facts.max_reqs - MPI3MR_INTERNAL_CMDS_RESVD;
3945	mrioc->shost->max_sectors = mrioc->facts.max_data_length / 512;
3946	mrioc->num_io_throttle_group = mrioc->facts.max_io_throttle_group;
3947	atomic_set(&mrioc->pend_large_data_sz, 0);
3948
3949	if (reset_devices)
3950		mrioc->max_host_ios = min_t(int, mrioc->max_host_ios,
3951		    MPI3MR_HOST_IOS_KDUMP);
3952
3953	if (!(mrioc->facts.ioc_capabilities &
3954	    MPI3_IOCFACTS_CAPABILITY_MULTIPATH_ENABLED)) {
3955		mrioc->sas_transport_enabled = 1;
3956		mrioc->scsi_device_channel = 1;
3957		mrioc->shost->max_channel = 1;
3958		mrioc->shost->transportt = mpi3mr_transport_template;
3959	}
3960
3961	mrioc->reply_sz = mrioc->facts.reply_sz;
3962
3963	retval = mpi3mr_check_reset_dma_mask(mrioc);
3964	if (retval) {
3965		ioc_err(mrioc, "Resetting dma mask failed %d\n",
3966		    retval);
3967		goto out_failed_noretry;
3968	}
3969
3970	mpi3mr_print_ioc_info(mrioc);
3971
3972	if (!mrioc->cfg_page) {
3973		dprint_init(mrioc, "allocating config page buffers\n");
3974		mrioc->cfg_page_sz = MPI3MR_DEFAULT_CFG_PAGE_SZ;
3975		mrioc->cfg_page = dma_alloc_coherent(&mrioc->pdev->dev,
3976		    mrioc->cfg_page_sz, &mrioc->cfg_page_dma, GFP_KERNEL);
3977		if (!mrioc->cfg_page) {
3978			retval = -1;
3979			goto out_failed_noretry;
3980		}
3981	}
3982
3983	dprint_init(mrioc, "allocating ioctl dma buffers\n");
3984	mpi3mr_alloc_ioctl_dma_memory(mrioc);
3985
3986	if (!mrioc->init_cmds.reply) {
3987		retval = mpi3mr_alloc_reply_sense_bufs(mrioc);
3988		if (retval) {
3989			ioc_err(mrioc,
3990			    "%s :Failed to allocated reply sense buffers %d\n",
3991			    __func__, retval);
3992			goto out_failed_noretry;
3993		}
3994	}
3995
3996	if (!mrioc->chain_sgl_list) {
3997		retval = mpi3mr_alloc_chain_bufs(mrioc);
3998		if (retval) {
3999			ioc_err(mrioc, "Failed to allocated chain buffers %d\n",
4000			    retval);
4001			goto out_failed_noretry;
4002		}
4003	}
4004
4005	retval = mpi3mr_issue_iocinit(mrioc);
4006	if (retval) {
4007		ioc_err(mrioc, "Failed to Issue IOC Init %d\n",
4008		    retval);
4009		goto out_failed;
4010	}
4011
4012	retval = mpi3mr_print_pkg_ver(mrioc);
4013	if (retval) {
4014		ioc_err(mrioc, "failed to get package version\n");
4015		goto out_failed;
4016	}
4017
4018	retval = mpi3mr_setup_isr(mrioc, 0);
4019	if (retval) {
4020		ioc_err(mrioc, "Failed to re-setup ISR, error %d\n",
4021		    retval);
4022		goto out_failed_noretry;
4023	}
4024
4025	retval = mpi3mr_create_op_queues(mrioc);
4026	if (retval) {
4027		ioc_err(mrioc, "Failed to create OpQueues error %d\n",
4028		    retval);
4029		goto out_failed;
4030	}
4031
4032	if (!mrioc->pel_seqnum_virt) {
4033		dprint_init(mrioc, "allocating memory for pel_seqnum_virt\n");
4034		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
4035		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
4036		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
4037		    GFP_KERNEL);
4038		if (!mrioc->pel_seqnum_virt) {
4039			retval = -ENOMEM;
4040			goto out_failed_noretry;
4041		}
4042	}
4043
4044	if (!mrioc->throttle_groups && mrioc->num_io_throttle_group) {
4045		dprint_init(mrioc, "allocating memory for throttle groups\n");
4046		sz = sizeof(struct mpi3mr_throttle_group_info);
4047		mrioc->throttle_groups = kcalloc(mrioc->num_io_throttle_group, sz, GFP_KERNEL);
4048		if (!mrioc->throttle_groups) {
4049			retval = -1;
4050			goto out_failed_noretry;
4051		}
4052	}
4053
4054	retval = mpi3mr_enable_events(mrioc);
4055	if (retval) {
4056		ioc_err(mrioc, "failed to enable events %d\n",
4057		    retval);
4058		goto out_failed;
4059	}
4060
4061	ioc_info(mrioc, "controller initialization completed successfully\n");
4062	return retval;
4063out_failed:
4064	if (retry < 2) {
4065		retry++;
4066		ioc_warn(mrioc, "retrying controller initialization, retry_count:%d\n",
4067		    retry);
4068		mpi3mr_memset_buffers(mrioc);
4069		goto retry_init;
4070	}
4071	retval = -1;
4072out_failed_noretry:
4073	ioc_err(mrioc, "controller initialization failed\n");
4074	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
4075	    MPI3MR_RESET_FROM_CTLR_CLEANUP);
4076	mrioc->unrecoverable = 1;
4077	return retval;
4078}
4079
4080/**
4081 * mpi3mr_reinit_ioc - Re-Initialize the controller
4082 * @mrioc: Adapter instance reference
4083 * @is_resume: Called from resume or reset path
4084 *
4085 * This the controller re-initialization routine, executed from
4086 * the soft reset handler or resume callback. Creates
4087 * operational reply queue pairs, allocate required memory for
4088 * reply pool, sense buffer pool, issue IOC init request to the
4089 * firmware, unmask the events and issue port enable to discover
4090 * SAS/SATA/NVMe devices and RAID volumes.
4091 *
4092 * Return: 0 on success and non-zero on failure.
4093 */
4094int mpi3mr_reinit_ioc(struct mpi3mr_ioc *mrioc, u8 is_resume)
4095{
4096	int retval = 0;
4097	u8 retry = 0;
4098	struct mpi3_ioc_facts_data facts_data;
4099	u32 pe_timeout, ioc_status;
4100
4101retry_init:
4102	pe_timeout =
4103	    (MPI3MR_PORTENABLE_TIMEOUT / MPI3MR_PORTENABLE_POLL_INTERVAL);
4104
4105	dprint_reset(mrioc, "bringing up the controller to ready state\n");
4106	retval = mpi3mr_bring_ioc_ready(mrioc);
4107	if (retval) {
4108		ioc_err(mrioc, "failed to bring to ready state\n");
4109		goto out_failed_noretry;
4110	}
4111
4112	if (is_resume) {
4113		dprint_reset(mrioc, "setting up single ISR\n");
4114		retval = mpi3mr_setup_isr(mrioc, 1);
4115		if (retval) {
4116			ioc_err(mrioc, "failed to setup ISR\n");
4117			goto out_failed_noretry;
4118		}
4119	} else
4120		mpi3mr_ioc_enable_intr(mrioc);
4121
4122	dprint_reset(mrioc, "getting ioc_facts\n");
4123	retval = mpi3mr_issue_iocfacts(mrioc, &facts_data);
4124	if (retval) {
4125		ioc_err(mrioc, "failed to get ioc_facts\n");
4126		goto out_failed;
4127	}
4128
4129	dprint_reset(mrioc, "validating ioc_facts\n");
4130	retval = mpi3mr_revalidate_factsdata(mrioc);
4131	if (retval) {
4132		ioc_err(mrioc, "failed to revalidate ioc_facts data\n");
4133		goto out_failed_noretry;
4134	}
4135
4136	mpi3mr_print_ioc_info(mrioc);
4137
4138	dprint_reset(mrioc, "sending ioc_init\n");
4139	retval = mpi3mr_issue_iocinit(mrioc);
4140	if (retval) {
4141		ioc_err(mrioc, "failed to send ioc_init\n");
4142		goto out_failed;
4143	}
4144
4145	dprint_reset(mrioc, "getting package version\n");
4146	retval = mpi3mr_print_pkg_ver(mrioc);
4147	if (retval) {
4148		ioc_err(mrioc, "failed to get package version\n");
4149		goto out_failed;
4150	}
4151
4152	if (is_resume) {
4153		dprint_reset(mrioc, "setting up multiple ISR\n");
4154		retval = mpi3mr_setup_isr(mrioc, 0);
4155		if (retval) {
4156			ioc_err(mrioc, "failed to re-setup ISR\n");
4157			goto out_failed_noretry;
4158		}
4159	}
4160
4161	dprint_reset(mrioc, "creating operational queue pairs\n");
4162	retval = mpi3mr_create_op_queues(mrioc);
4163	if (retval) {
4164		ioc_err(mrioc, "failed to create operational queue pairs\n");
4165		goto out_failed;
4166	}
4167
4168	if (!mrioc->pel_seqnum_virt) {
4169		dprint_reset(mrioc, "allocating memory for pel_seqnum_virt\n");
4170		mrioc->pel_seqnum_sz = sizeof(struct mpi3_pel_seq);
4171		mrioc->pel_seqnum_virt = dma_alloc_coherent(&mrioc->pdev->dev,
4172		    mrioc->pel_seqnum_sz, &mrioc->pel_seqnum_dma,
4173		    GFP_KERNEL);
4174		if (!mrioc->pel_seqnum_virt) {
4175			retval = -ENOMEM;
4176			goto out_failed_noretry;
4177		}
4178	}
4179
4180	if (mrioc->shost->nr_hw_queues > mrioc->num_op_reply_q) {
4181		ioc_err(mrioc,
4182		    "cannot create minimum number of operational queues expected:%d created:%d\n",
4183		    mrioc->shost->nr_hw_queues, mrioc->num_op_reply_q);
4184		retval = -1;
4185		goto out_failed_noretry;
4186	}
4187
4188	dprint_reset(mrioc, "enabling events\n");
4189	retval = mpi3mr_enable_events(mrioc);
4190	if (retval) {
4191		ioc_err(mrioc, "failed to enable events\n");
4192		goto out_failed;
4193	}
4194
4195	mrioc->device_refresh_on = 1;
4196	mpi3mr_add_event_wait_for_device_refresh(mrioc);
4197
4198	ioc_info(mrioc, "sending port enable\n");
4199	retval = mpi3mr_issue_port_enable(mrioc, 1);
4200	if (retval) {
4201		ioc_err(mrioc, "failed to issue port enable\n");
4202		goto out_failed;
4203	}
4204	do {
4205		ssleep(MPI3MR_PORTENABLE_POLL_INTERVAL);
4206		if (mrioc->init_cmds.state == MPI3MR_CMD_NOTUSED)
4207			break;
4208		if (!pci_device_is_present(mrioc->pdev))
4209			mrioc->unrecoverable = 1;
4210		if (mrioc->unrecoverable) {
4211			retval = -1;
4212			goto out_failed_noretry;
4213		}
4214		ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4215		if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
4216		    (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
4217			mpi3mr_print_fault_info(mrioc);
4218			mrioc->init_cmds.is_waiting = 0;
4219			mrioc->init_cmds.callback = NULL;
4220			mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4221			goto out_failed;
4222		}
4223	} while (--pe_timeout);
4224
4225	if (!pe_timeout) {
4226		ioc_err(mrioc, "port enable timed out\n");
4227		mpi3mr_check_rh_fault_ioc(mrioc,
4228		    MPI3MR_RESET_FROM_PE_TIMEOUT);
4229		mrioc->init_cmds.is_waiting = 0;
4230		mrioc->init_cmds.callback = NULL;
4231		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4232		goto out_failed;
4233	} else if (mrioc->scan_failed) {
4234		ioc_err(mrioc,
4235		    "port enable failed with status=0x%04x\n",
4236		    mrioc->scan_failed);
4237	} else
4238		ioc_info(mrioc, "port enable completed successfully\n");
4239
4240	ioc_info(mrioc, "controller %s completed successfully\n",
4241	    (is_resume)?"resume":"re-initialization");
4242	return retval;
4243out_failed:
4244	if (retry < 2) {
4245		retry++;
4246		ioc_warn(mrioc, "retrying controller %s, retry_count:%d\n",
4247		    (is_resume)?"resume":"re-initialization", retry);
4248		mpi3mr_memset_buffers(mrioc);
4249		goto retry_init;
4250	}
4251	retval = -1;
4252out_failed_noretry:
4253	ioc_err(mrioc, "controller %s is failed\n",
4254	    (is_resume)?"resume":"re-initialization");
4255	mpi3mr_issue_reset(mrioc, MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT,
4256	    MPI3MR_RESET_FROM_CTLR_CLEANUP);
4257	mrioc->unrecoverable = 1;
4258	return retval;
4259}
4260
4261/**
4262 * mpi3mr_memset_op_reply_q_buffers - memset the operational reply queue's
4263 *					segments
4264 * @mrioc: Adapter instance reference
4265 * @qidx: Operational reply queue index
4266 *
4267 * Return: Nothing.
4268 */
4269static void mpi3mr_memset_op_reply_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4270{
4271	struct op_reply_qinfo *op_reply_q = mrioc->op_reply_qinfo + qidx;
4272	struct segments *segments;
4273	int i, size;
4274
4275	if (!op_reply_q->q_segments)
4276		return;
4277
4278	size = op_reply_q->segment_qd * mrioc->op_reply_desc_sz;
4279	segments = op_reply_q->q_segments;
4280	for (i = 0; i < op_reply_q->num_segments; i++)
4281		memset(segments[i].segment, 0, size);
4282}
4283
4284/**
4285 * mpi3mr_memset_op_req_q_buffers - memset the operational request queue's
4286 *					segments
4287 * @mrioc: Adapter instance reference
4288 * @qidx: Operational request queue index
4289 *
4290 * Return: Nothing.
4291 */
4292static void mpi3mr_memset_op_req_q_buffers(struct mpi3mr_ioc *mrioc, u16 qidx)
4293{
4294	struct op_req_qinfo *op_req_q = mrioc->req_qinfo + qidx;
4295	struct segments *segments;
4296	int i, size;
4297
4298	if (!op_req_q->q_segments)
4299		return;
4300
4301	size = op_req_q->segment_qd * mrioc->facts.op_req_sz;
4302	segments = op_req_q->q_segments;
4303	for (i = 0; i < op_req_q->num_segments; i++)
4304		memset(segments[i].segment, 0, size);
4305}
4306
4307/**
4308 * mpi3mr_memset_buffers - memset memory for a controller
4309 * @mrioc: Adapter instance reference
4310 *
4311 * clear all the memory allocated for a controller, typically
4312 * called post reset to reuse the memory allocated during the
4313 * controller init.
4314 *
4315 * Return: Nothing.
4316 */
4317void mpi3mr_memset_buffers(struct mpi3mr_ioc *mrioc)
4318{
4319	u16 i;
4320	struct mpi3mr_throttle_group_info *tg;
4321
4322	mrioc->change_count = 0;
4323	mrioc->active_poll_qcount = 0;
4324	mrioc->default_qcount = 0;
4325	if (mrioc->admin_req_base)
4326		memset(mrioc->admin_req_base, 0, mrioc->admin_req_q_sz);
4327	if (mrioc->admin_reply_base)
4328		memset(mrioc->admin_reply_base, 0, mrioc->admin_reply_q_sz);
4329	atomic_set(&mrioc->admin_reply_q_in_use, 0);
4330
4331	if (mrioc->init_cmds.reply) {
4332		memset(mrioc->init_cmds.reply, 0, sizeof(*mrioc->init_cmds.reply));
4333		memset(mrioc->bsg_cmds.reply, 0,
4334		    sizeof(*mrioc->bsg_cmds.reply));
4335		memset(mrioc->host_tm_cmds.reply, 0,
4336		    sizeof(*mrioc->host_tm_cmds.reply));
4337		memset(mrioc->pel_cmds.reply, 0,
4338		    sizeof(*mrioc->pel_cmds.reply));
4339		memset(mrioc->pel_abort_cmd.reply, 0,
4340		    sizeof(*mrioc->pel_abort_cmd.reply));
4341		memset(mrioc->transport_cmds.reply, 0,
4342		    sizeof(*mrioc->transport_cmds.reply));
4343		for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
4344			memset(mrioc->dev_rmhs_cmds[i].reply, 0,
4345			    sizeof(*mrioc->dev_rmhs_cmds[i].reply));
4346		for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++)
4347			memset(mrioc->evtack_cmds[i].reply, 0,
4348			    sizeof(*mrioc->evtack_cmds[i].reply));
4349		bitmap_clear(mrioc->removepend_bitmap, 0,
4350			     mrioc->dev_handle_bitmap_bits);
4351		bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);
4352		bitmap_clear(mrioc->evtack_cmds_bitmap, 0,
4353			     MPI3MR_NUM_EVTACKCMD);
4354	}
4355
4356	for (i = 0; i < mrioc->num_queues; i++) {
4357		mrioc->op_reply_qinfo[i].qid = 0;
4358		mrioc->op_reply_qinfo[i].ci = 0;
4359		mrioc->op_reply_qinfo[i].num_replies = 0;
4360		mrioc->op_reply_qinfo[i].ephase = 0;
4361		atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
4362		atomic_set(&mrioc->op_reply_qinfo[i].in_use, 0);
4363		mpi3mr_memset_op_reply_q_buffers(mrioc, i);
4364
4365		mrioc->req_qinfo[i].ci = 0;
4366		mrioc->req_qinfo[i].pi = 0;
4367		mrioc->req_qinfo[i].num_requests = 0;
4368		mrioc->req_qinfo[i].qid = 0;
4369		mrioc->req_qinfo[i].reply_qid = 0;
4370		spin_lock_init(&mrioc->req_qinfo[i].q_lock);
4371		mpi3mr_memset_op_req_q_buffers(mrioc, i);
4372	}
4373
4374	atomic_set(&mrioc->pend_large_data_sz, 0);
4375	if (mrioc->throttle_groups) {
4376		tg = mrioc->throttle_groups;
4377		for (i = 0; i < mrioc->num_io_throttle_group; i++, tg++) {
4378			tg->id = 0;
4379			tg->fw_qd = 0;
4380			tg->modified_qd = 0;
4381			tg->io_divert = 0;
4382			tg->need_qd_reduction = 0;
4383			tg->high = 0;
4384			tg->low = 0;
4385			tg->qd_reduction = 0;
4386			atomic_set(&tg->pend_large_data_sz, 0);
4387		}
4388	}
4389}
4390
4391/**
4392 * mpi3mr_free_mem - Free memory allocated for a controller
4393 * @mrioc: Adapter instance reference
4394 *
4395 * Free all the memory allocated for a controller.
4396 *
4397 * Return: Nothing.
4398 */
4399void mpi3mr_free_mem(struct mpi3mr_ioc *mrioc)
4400{
4401	u16 i;
4402	struct mpi3mr_intr_info *intr_info;
4403
4404	mpi3mr_free_enclosure_list(mrioc);
4405	mpi3mr_free_ioctl_dma_memory(mrioc);
4406
4407	if (mrioc->sense_buf_pool) {
4408		if (mrioc->sense_buf)
4409			dma_pool_free(mrioc->sense_buf_pool, mrioc->sense_buf,
4410			    mrioc->sense_buf_dma);
4411		dma_pool_destroy(mrioc->sense_buf_pool);
4412		mrioc->sense_buf = NULL;
4413		mrioc->sense_buf_pool = NULL;
4414	}
4415	if (mrioc->sense_buf_q_pool) {
4416		if (mrioc->sense_buf_q)
4417			dma_pool_free(mrioc->sense_buf_q_pool,
4418			    mrioc->sense_buf_q, mrioc->sense_buf_q_dma);
4419		dma_pool_destroy(mrioc->sense_buf_q_pool);
4420		mrioc->sense_buf_q = NULL;
4421		mrioc->sense_buf_q_pool = NULL;
4422	}
4423
4424	if (mrioc->reply_buf_pool) {
4425		if (mrioc->reply_buf)
4426			dma_pool_free(mrioc->reply_buf_pool, mrioc->reply_buf,
4427			    mrioc->reply_buf_dma);
4428		dma_pool_destroy(mrioc->reply_buf_pool);
4429		mrioc->reply_buf = NULL;
4430		mrioc->reply_buf_pool = NULL;
4431	}
4432	if (mrioc->reply_free_q_pool) {
4433		if (mrioc->reply_free_q)
4434			dma_pool_free(mrioc->reply_free_q_pool,
4435			    mrioc->reply_free_q, mrioc->reply_free_q_dma);
4436		dma_pool_destroy(mrioc->reply_free_q_pool);
4437		mrioc->reply_free_q = NULL;
4438		mrioc->reply_free_q_pool = NULL;
4439	}
4440
4441	for (i = 0; i < mrioc->num_op_req_q; i++)
4442		mpi3mr_free_op_req_q_segments(mrioc, i);
4443
4444	for (i = 0; i < mrioc->num_op_reply_q; i++)
4445		mpi3mr_free_op_reply_q_segments(mrioc, i);
4446
4447	for (i = 0; i < mrioc->intr_info_count; i++) {
4448		intr_info = mrioc->intr_info + i;
4449		intr_info->op_reply_q = NULL;
4450	}
4451
4452	kfree(mrioc->req_qinfo);
4453	mrioc->req_qinfo = NULL;
4454	mrioc->num_op_req_q = 0;
4455
4456	kfree(mrioc->op_reply_qinfo);
4457	mrioc->op_reply_qinfo = NULL;
4458	mrioc->num_op_reply_q = 0;
4459
4460	kfree(mrioc->init_cmds.reply);
4461	mrioc->init_cmds.reply = NULL;
4462
4463	kfree(mrioc->bsg_cmds.reply);
4464	mrioc->bsg_cmds.reply = NULL;
4465
4466	kfree(mrioc->host_tm_cmds.reply);
4467	mrioc->host_tm_cmds.reply = NULL;
4468
4469	kfree(mrioc->pel_cmds.reply);
4470	mrioc->pel_cmds.reply = NULL;
4471
4472	kfree(mrioc->pel_abort_cmd.reply);
4473	mrioc->pel_abort_cmd.reply = NULL;
4474
4475	for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4476		kfree(mrioc->evtack_cmds[i].reply);
4477		mrioc->evtack_cmds[i].reply = NULL;
4478	}
4479
4480	bitmap_free(mrioc->removepend_bitmap);
4481	mrioc->removepend_bitmap = NULL;
4482
4483	bitmap_free(mrioc->devrem_bitmap);
4484	mrioc->devrem_bitmap = NULL;
4485
4486	bitmap_free(mrioc->evtack_cmds_bitmap);
4487	mrioc->evtack_cmds_bitmap = NULL;
4488
4489	bitmap_free(mrioc->chain_bitmap);
4490	mrioc->chain_bitmap = NULL;
4491
4492	kfree(mrioc->transport_cmds.reply);
4493	mrioc->transport_cmds.reply = NULL;
4494
4495	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4496		kfree(mrioc->dev_rmhs_cmds[i].reply);
4497		mrioc->dev_rmhs_cmds[i].reply = NULL;
4498	}
4499
4500	if (mrioc->chain_buf_pool) {
4501		for (i = 0; i < mrioc->chain_buf_count; i++) {
4502			if (mrioc->chain_sgl_list[i].addr) {
4503				dma_pool_free(mrioc->chain_buf_pool,
4504				    mrioc->chain_sgl_list[i].addr,
4505				    mrioc->chain_sgl_list[i].dma_addr);
4506				mrioc->chain_sgl_list[i].addr = NULL;
4507			}
4508		}
4509		dma_pool_destroy(mrioc->chain_buf_pool);
4510		mrioc->chain_buf_pool = NULL;
4511	}
4512
4513	kfree(mrioc->chain_sgl_list);
4514	mrioc->chain_sgl_list = NULL;
4515
4516	if (mrioc->admin_reply_base) {
4517		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_reply_q_sz,
4518		    mrioc->admin_reply_base, mrioc->admin_reply_dma);
4519		mrioc->admin_reply_base = NULL;
4520	}
4521	if (mrioc->admin_req_base) {
4522		dma_free_coherent(&mrioc->pdev->dev, mrioc->admin_req_q_sz,
4523		    mrioc->admin_req_base, mrioc->admin_req_dma);
4524		mrioc->admin_req_base = NULL;
4525	}
4526	if (mrioc->cfg_page) {
4527		dma_free_coherent(&mrioc->pdev->dev, mrioc->cfg_page_sz,
4528		    mrioc->cfg_page, mrioc->cfg_page_dma);
4529		mrioc->cfg_page = NULL;
4530	}
4531	if (mrioc->pel_seqnum_virt) {
4532		dma_free_coherent(&mrioc->pdev->dev, mrioc->pel_seqnum_sz,
4533		    mrioc->pel_seqnum_virt, mrioc->pel_seqnum_dma);
4534		mrioc->pel_seqnum_virt = NULL;
4535	}
4536
4537	kfree(mrioc->throttle_groups);
4538	mrioc->throttle_groups = NULL;
4539
4540	kfree(mrioc->logdata_buf);
4541	mrioc->logdata_buf = NULL;
4542
4543}
4544
4545/**
4546 * mpi3mr_issue_ioc_shutdown - shutdown controller
4547 * @mrioc: Adapter instance reference
4548 *
4549 * Send shutodwn notification to the controller and wait for the
4550 * shutdown_timeout for it to be completed.
4551 *
4552 * Return: Nothing.
4553 */
4554static void mpi3mr_issue_ioc_shutdown(struct mpi3mr_ioc *mrioc)
4555{
4556	u32 ioc_config, ioc_status;
4557	u8 retval = 1;
4558	u32 timeout = MPI3MR_DEFAULT_SHUTDOWN_TIME * 10;
4559
4560	ioc_info(mrioc, "Issuing shutdown Notification\n");
4561	if (mrioc->unrecoverable) {
4562		ioc_warn(mrioc,
4563		    "IOC is unrecoverable shutdown is not issued\n");
4564		return;
4565	}
4566	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4567	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4568	    == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS) {
4569		ioc_info(mrioc, "shutdown already in progress\n");
4570		return;
4571	}
4572
4573	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4574	ioc_config |= MPI3_SYSIF_IOC_CONFIG_SHUTDOWN_NORMAL;
4575	ioc_config |= MPI3_SYSIF_IOC_CONFIG_DEVICE_SHUTDOWN_SEND_REQ;
4576
4577	writel(ioc_config, &mrioc->sysif_regs->ioc_configuration);
4578
4579	if (mrioc->facts.shutdown_timeout)
4580		timeout = mrioc->facts.shutdown_timeout * 10;
4581
4582	do {
4583		ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4584		if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4585		    == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_COMPLETE) {
4586			retval = 0;
4587			break;
4588		}
4589		msleep(100);
4590	} while (--timeout);
4591
4592	ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4593	ioc_config = readl(&mrioc->sysif_regs->ioc_configuration);
4594
4595	if (retval) {
4596		if ((ioc_status & MPI3_SYSIF_IOC_STATUS_SHUTDOWN_MASK)
4597		    == MPI3_SYSIF_IOC_STATUS_SHUTDOWN_IN_PROGRESS)
4598			ioc_warn(mrioc,
4599			    "shutdown still in progress after timeout\n");
4600	}
4601
4602	ioc_info(mrioc,
4603	    "Base IOC Sts/Config after %s shutdown is (0x%x)/(0x%x)\n",
4604	    (!retval) ? "successful" : "failed", ioc_status,
4605	    ioc_config);
4606}
4607
4608/**
4609 * mpi3mr_cleanup_ioc - Cleanup controller
4610 * @mrioc: Adapter instance reference
4611 *
4612 * controller cleanup handler, Message unit reset or soft reset
4613 * and shutdown notification is issued to the controller.
4614 *
4615 * Return: Nothing.
4616 */
4617void mpi3mr_cleanup_ioc(struct mpi3mr_ioc *mrioc)
4618{
4619	enum mpi3mr_iocstate ioc_state;
4620
4621	dprint_exit(mrioc, "cleaning up the controller\n");
4622	mpi3mr_ioc_disable_intr(mrioc);
4623
4624	ioc_state = mpi3mr_get_iocstate(mrioc);
4625
4626	if ((!mrioc->unrecoverable) && (!mrioc->reset_in_progress) &&
4627	    (ioc_state == MRIOC_STATE_READY)) {
4628		if (mpi3mr_issue_and_process_mur(mrioc,
4629		    MPI3MR_RESET_FROM_CTLR_CLEANUP))
4630			mpi3mr_issue_reset(mrioc,
4631			    MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET,
4632			    MPI3MR_RESET_FROM_MUR_FAILURE);
4633		mpi3mr_issue_ioc_shutdown(mrioc);
4634	}
4635	dprint_exit(mrioc, "controller cleanup completed\n");
4636}
4637
4638/**
4639 * mpi3mr_drv_cmd_comp_reset - Flush a internal driver command
4640 * @mrioc: Adapter instance reference
4641 * @cmdptr: Internal command tracker
4642 *
4643 * Complete an internal driver commands with state indicating it
4644 * is completed due to reset.
4645 *
4646 * Return: Nothing.
4647 */
4648static inline void mpi3mr_drv_cmd_comp_reset(struct mpi3mr_ioc *mrioc,
4649	struct mpi3mr_drv_cmd *cmdptr)
4650{
4651	if (cmdptr->state & MPI3MR_CMD_PENDING) {
4652		cmdptr->state |= MPI3MR_CMD_RESET;
4653		cmdptr->state &= ~MPI3MR_CMD_PENDING;
4654		if (cmdptr->is_waiting) {
4655			complete(&cmdptr->done);
4656			cmdptr->is_waiting = 0;
4657		} else if (cmdptr->callback)
4658			cmdptr->callback(mrioc, cmdptr);
4659	}
4660}
4661
4662/**
4663 * mpi3mr_flush_drv_cmds - Flush internaldriver commands
4664 * @mrioc: Adapter instance reference
4665 *
4666 * Flush all internal driver commands post reset
4667 *
4668 * Return: Nothing.
4669 */
4670void mpi3mr_flush_drv_cmds(struct mpi3mr_ioc *mrioc)
4671{
4672	struct mpi3mr_drv_cmd *cmdptr;
4673	u8 i;
4674
4675	cmdptr = &mrioc->init_cmds;
4676	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4677
4678	cmdptr = &mrioc->cfg_cmds;
4679	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4680
4681	cmdptr = &mrioc->bsg_cmds;
4682	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4683	cmdptr = &mrioc->host_tm_cmds;
4684	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4685
4686	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++) {
4687		cmdptr = &mrioc->dev_rmhs_cmds[i];
4688		mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4689	}
4690
4691	for (i = 0; i < MPI3MR_NUM_EVTACKCMD; i++) {
4692		cmdptr = &mrioc->evtack_cmds[i];
4693		mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4694	}
4695
4696	cmdptr = &mrioc->pel_cmds;
4697	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4698
4699	cmdptr = &mrioc->pel_abort_cmd;
4700	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4701
4702	cmdptr = &mrioc->transport_cmds;
4703	mpi3mr_drv_cmd_comp_reset(mrioc, cmdptr);
4704}
4705
4706/**
4707 * mpi3mr_pel_wait_post - Issue PEL Wait
4708 * @mrioc: Adapter instance reference
4709 * @drv_cmd: Internal command tracker
4710 *
4711 * Issue PEL Wait MPI request through admin queue and return.
4712 *
4713 * Return: Nothing.
4714 */
4715static void mpi3mr_pel_wait_post(struct mpi3mr_ioc *mrioc,
4716	struct mpi3mr_drv_cmd *drv_cmd)
4717{
4718	struct mpi3_pel_req_action_wait pel_wait;
4719
4720	mrioc->pel_abort_requested = false;
4721
4722	memset(&pel_wait, 0, sizeof(pel_wait));
4723	drv_cmd->state = MPI3MR_CMD_PENDING;
4724	drv_cmd->is_waiting = 0;
4725	drv_cmd->callback = mpi3mr_pel_wait_complete;
4726	drv_cmd->ioc_status = 0;
4727	drv_cmd->ioc_loginfo = 0;
4728	pel_wait.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4729	pel_wait.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4730	pel_wait.action = MPI3_PEL_ACTION_WAIT;
4731	pel_wait.starting_sequence_number = cpu_to_le32(mrioc->pel_newest_seqnum);
4732	pel_wait.locale = cpu_to_le16(mrioc->pel_locale);
4733	pel_wait.class = cpu_to_le16(mrioc->pel_class);
4734	pel_wait.wait_time = MPI3_PEL_WAITTIME_INFINITE_WAIT;
4735	dprint_bsg_info(mrioc, "sending pel_wait seqnum(%d), class(%d), locale(0x%08x)\n",
4736	    mrioc->pel_newest_seqnum, mrioc->pel_class, mrioc->pel_locale);
4737
4738	if (mpi3mr_admin_request_post(mrioc, &pel_wait, sizeof(pel_wait), 0)) {
4739		dprint_bsg_err(mrioc,
4740			    "Issuing PELWait: Admin post failed\n");
4741		drv_cmd->state = MPI3MR_CMD_NOTUSED;
4742		drv_cmd->callback = NULL;
4743		drv_cmd->retry_count = 0;
4744		mrioc->pel_enabled = false;
4745	}
4746}
4747
4748/**
4749 * mpi3mr_pel_get_seqnum_post - Issue PEL Get Sequence number
4750 * @mrioc: Adapter instance reference
4751 * @drv_cmd: Internal command tracker
4752 *
4753 * Issue PEL get sequence number MPI request through admin queue
4754 * and return.
4755 *
4756 * Return: 0 on success, non-zero on failure.
4757 */
4758int mpi3mr_pel_get_seqnum_post(struct mpi3mr_ioc *mrioc,
4759	struct mpi3mr_drv_cmd *drv_cmd)
4760{
4761	struct mpi3_pel_req_action_get_sequence_numbers pel_getseq_req;
4762	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
4763	int retval = 0;
4764
4765	memset(&pel_getseq_req, 0, sizeof(pel_getseq_req));
4766	mrioc->pel_cmds.state = MPI3MR_CMD_PENDING;
4767	mrioc->pel_cmds.is_waiting = 0;
4768	mrioc->pel_cmds.ioc_status = 0;
4769	mrioc->pel_cmds.ioc_loginfo = 0;
4770	mrioc->pel_cmds.callback = mpi3mr_pel_get_seqnum_complete;
4771	pel_getseq_req.host_tag = cpu_to_le16(MPI3MR_HOSTTAG_PEL_WAIT);
4772	pel_getseq_req.function = MPI3_FUNCTION_PERSISTENT_EVENT_LOG;
4773	pel_getseq_req.action = MPI3_PEL_ACTION_GET_SEQNUM;
4774	mpi3mr_add_sg_single(&pel_getseq_req.sgl, sgl_flags,
4775	    mrioc->pel_seqnum_sz, mrioc->pel_seqnum_dma);
4776
4777	retval = mpi3mr_admin_request_post(mrioc, &pel_getseq_req,
4778			sizeof(pel_getseq_req), 0);
4779	if (retval) {
4780		if (drv_cmd) {
4781			drv_cmd->state = MPI3MR_CMD_NOTUSED;
4782			drv_cmd->callback = NULL;
4783			drv_cmd->retry_count = 0;
4784		}
4785		mrioc->pel_enabled = false;
4786	}
4787
4788	return retval;
4789}
4790
4791/**
4792 * mpi3mr_pel_wait_complete - PELWait Completion callback
4793 * @mrioc: Adapter instance reference
4794 * @drv_cmd: Internal command tracker
4795 *
4796 * This is a callback handler for the PELWait request and
4797 * firmware completes a PELWait request when it is aborted or a
4798 * new PEL entry is available. This sends AEN to the application
4799 * and if the PELwait completion is not due to PELAbort then
4800 * this will send a request for new PEL Sequence number
4801 *
4802 * Return: Nothing.
4803 */
4804static void mpi3mr_pel_wait_complete(struct mpi3mr_ioc *mrioc,
4805	struct mpi3mr_drv_cmd *drv_cmd)
4806{
4807	struct mpi3_pel_reply *pel_reply = NULL;
4808	u16 ioc_status, pe_log_status;
4809	bool do_retry = false;
4810
4811	if (drv_cmd->state & MPI3MR_CMD_RESET)
4812		goto cleanup_drv_cmd;
4813
4814	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4815	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4816		ioc_err(mrioc, "%s: Failed ioc_status(0x%04x) Loginfo(0x%08x)\n",
4817			__func__, ioc_status, drv_cmd->ioc_loginfo);
4818		dprint_bsg_err(mrioc,
4819		    "pel_wait: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4820		    ioc_status, drv_cmd->ioc_loginfo);
4821		do_retry = true;
4822	}
4823
4824	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4825		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4826
4827	if (!pel_reply) {
4828		dprint_bsg_err(mrioc,
4829		    "pel_wait: failed due to no reply\n");
4830		goto out_failed;
4831	}
4832
4833	pe_log_status = le16_to_cpu(pel_reply->pe_log_status);
4834	if ((pe_log_status != MPI3_PEL_STATUS_SUCCESS) &&
4835	    (pe_log_status != MPI3_PEL_STATUS_ABORTED)) {
4836		ioc_err(mrioc, "%s: Failed pe_log_status(0x%04x)\n",
4837			__func__, pe_log_status);
4838		dprint_bsg_err(mrioc,
4839		    "pel_wait: failed due to pel_log_status(0x%04x)\n",
4840		    pe_log_status);
4841		do_retry = true;
4842	}
4843
4844	if (do_retry) {
4845		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4846			drv_cmd->retry_count++;
4847			dprint_bsg_err(mrioc, "pel_wait: retrying(%d)\n",
4848			    drv_cmd->retry_count);
4849			mpi3mr_pel_wait_post(mrioc, drv_cmd);
4850			return;
4851		}
4852		dprint_bsg_err(mrioc,
4853		    "pel_wait: failed after all retries(%d)\n",
4854		    drv_cmd->retry_count);
4855		goto out_failed;
4856	}
4857	atomic64_inc(&event_counter);
4858	if (!mrioc->pel_abort_requested) {
4859		mrioc->pel_cmds.retry_count = 0;
4860		mpi3mr_pel_get_seqnum_post(mrioc, &mrioc->pel_cmds);
4861	}
4862
4863	return;
4864out_failed:
4865	mrioc->pel_enabled = false;
4866cleanup_drv_cmd:
4867	drv_cmd->state = MPI3MR_CMD_NOTUSED;
4868	drv_cmd->callback = NULL;
4869	drv_cmd->retry_count = 0;
4870}
4871
4872/**
4873 * mpi3mr_pel_get_seqnum_complete - PELGetSeqNum Completion callback
4874 * @mrioc: Adapter instance reference
4875 * @drv_cmd: Internal command tracker
4876 *
4877 * This is a callback handler for the PEL get sequence number
4878 * request and a new PEL wait request will be issued to the
4879 * firmware from this
4880 *
4881 * Return: Nothing.
4882 */
4883void mpi3mr_pel_get_seqnum_complete(struct mpi3mr_ioc *mrioc,
4884	struct mpi3mr_drv_cmd *drv_cmd)
4885{
4886	struct mpi3_pel_reply *pel_reply = NULL;
4887	struct mpi3_pel_seq *pel_seqnum_virt;
4888	u16 ioc_status;
4889	bool do_retry = false;
4890
4891	pel_seqnum_virt = (struct mpi3_pel_seq *)mrioc->pel_seqnum_virt;
4892
4893	if (drv_cmd->state & MPI3MR_CMD_RESET)
4894		goto cleanup_drv_cmd;
4895
4896	ioc_status = drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
4897	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
4898		dprint_bsg_err(mrioc,
4899		    "pel_get_seqnum: failed with ioc_status(0x%04x), log_info(0x%08x)\n",
4900		    ioc_status, drv_cmd->ioc_loginfo);
4901		do_retry = true;
4902	}
4903
4904	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
4905		pel_reply = (struct mpi3_pel_reply *)drv_cmd->reply;
4906	if (!pel_reply) {
4907		dprint_bsg_err(mrioc,
4908		    "pel_get_seqnum: failed due to no reply\n");
4909		goto out_failed;
4910	}
4911
4912	if (le16_to_cpu(pel_reply->pe_log_status) != MPI3_PEL_STATUS_SUCCESS) {
4913		dprint_bsg_err(mrioc,
4914		    "pel_get_seqnum: failed due to pel_log_status(0x%04x)\n",
4915		    le16_to_cpu(pel_reply->pe_log_status));
4916		do_retry = true;
4917	}
4918
4919	if (do_retry) {
4920		if (drv_cmd->retry_count < MPI3MR_PEL_RETRY_COUNT) {
4921			drv_cmd->retry_count++;
4922			dprint_bsg_err(mrioc,
4923			    "pel_get_seqnum: retrying(%d)\n",
4924			    drv_cmd->retry_count);
4925			mpi3mr_pel_get_seqnum_post(mrioc, drv_cmd);
4926			return;
4927		}
4928
4929		dprint_bsg_err(mrioc,
4930		    "pel_get_seqnum: failed after all retries(%d)\n",
4931		    drv_cmd->retry_count);
4932		goto out_failed;
4933	}
4934	mrioc->pel_newest_seqnum = le32_to_cpu(pel_seqnum_virt->newest) + 1;
4935	drv_cmd->retry_count = 0;
4936	mpi3mr_pel_wait_post(mrioc, drv_cmd);
4937
4938	return;
4939out_failed:
4940	mrioc->pel_enabled = false;
4941cleanup_drv_cmd:
4942	drv_cmd->state = MPI3MR_CMD_NOTUSED;
4943	drv_cmd->callback = NULL;
4944	drv_cmd->retry_count = 0;
4945}
4946
4947/**
4948 * mpi3mr_soft_reset_handler - Reset the controller
4949 * @mrioc: Adapter instance reference
4950 * @reset_reason: Reset reason code
4951 * @snapdump: Flag to generate snapdump in firmware or not
4952 *
4953 * This is an handler for recovering controller by issuing soft
4954 * reset are diag fault reset.  This is a blocking function and
4955 * when one reset is executed if any other resets they will be
4956 * blocked. All BSG requests will be blocked during the reset. If
4957 * controller reset is successful then the controller will be
4958 * reinitalized, otherwise the controller will be marked as not
4959 * recoverable
4960 *
4961 * In snapdump bit is set, the controller is issued with diag
4962 * fault reset so that the firmware can create a snap dump and
4963 * post that the firmware will result in F000 fault and the
4964 * driver will issue soft reset to recover from that.
4965 *
4966 * Return: 0 on success, non-zero on failure.
4967 */
4968int mpi3mr_soft_reset_handler(struct mpi3mr_ioc *mrioc,
4969	u32 reset_reason, u8 snapdump)
4970{
4971	int retval = 0, i;
4972	unsigned long flags;
4973	u32 host_diagnostic, timeout = MPI3_SYSIF_DIAG_SAVE_TIMEOUT * 10;
4974
4975	/* Block the reset handler until diag save in progress*/
4976	dprint_reset(mrioc,
4977	    "soft_reset_handler: check and block on diagsave_timeout(%d)\n",
4978	    mrioc->diagsave_timeout);
4979	while (mrioc->diagsave_timeout)
4980		ssleep(1);
4981	/*
4982	 * Block new resets until the currently executing one is finished and
4983	 * return the status of the existing reset for all blocked resets
4984	 */
4985	dprint_reset(mrioc, "soft_reset_handler: acquiring reset_mutex\n");
4986	if (!mutex_trylock(&mrioc->reset_mutex)) {
4987		ioc_info(mrioc,
4988		    "controller reset triggered by %s is blocked due to another reset in progress\n",
4989		    mpi3mr_reset_rc_name(reset_reason));
4990		do {
4991			ssleep(1);
4992		} while (mrioc->reset_in_progress == 1);
4993		ioc_info(mrioc,
4994		    "returning previous reset result(%d) for the reset triggered by %s\n",
4995		    mrioc->prev_reset_result,
4996		    mpi3mr_reset_rc_name(reset_reason));
4997		return mrioc->prev_reset_result;
4998	}
4999	ioc_info(mrioc, "controller reset is triggered by %s\n",
5000	    mpi3mr_reset_rc_name(reset_reason));
5001
5002	mrioc->device_refresh_on = 0;
5003	mrioc->reset_in_progress = 1;
5004	mrioc->stop_bsgs = 1;
5005	mrioc->prev_reset_result = -1;
5006
5007	if ((!snapdump) && (reset_reason != MPI3MR_RESET_FROM_FAULT_WATCH) &&
5008	    (reset_reason != MPI3MR_RESET_FROM_FIRMWARE) &&
5009	    (reset_reason != MPI3MR_RESET_FROM_CIACTIV_FAULT)) {
5010		for (i = 0; i < MPI3_EVENT_NOTIFY_EVENTMASK_WORDS; i++)
5011			mrioc->event_masks[i] = -1;
5012
5013		dprint_reset(mrioc, "soft_reset_handler: masking events\n");
5014		mpi3mr_issue_event_notification(mrioc);
5015	}
5016
5017	mpi3mr_wait_for_host_io(mrioc, MPI3MR_RESET_HOST_IOWAIT_TIMEOUT);
5018
5019	mpi3mr_ioc_disable_intr(mrioc);
5020
5021	if (snapdump) {
5022		mpi3mr_set_diagsave(mrioc);
5023		retval = mpi3mr_issue_reset(mrioc,
5024		    MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
5025		if (!retval) {
5026			do {
5027				host_diagnostic =
5028				    readl(&mrioc->sysif_regs->host_diagnostic);
5029				if (!(host_diagnostic &
5030				    MPI3_SYSIF_HOST_DIAG_SAVE_IN_PROGRESS))
5031					break;
5032				msleep(100);
5033			} while (--timeout);
5034		}
5035	}
5036
5037	retval = mpi3mr_issue_reset(mrioc,
5038	    MPI3_SYSIF_HOST_DIAG_RESET_ACTION_SOFT_RESET, reset_reason);
5039	if (retval) {
5040		ioc_err(mrioc, "Failed to issue soft reset to the ioc\n");
5041		goto out;
5042	}
5043	if (mrioc->num_io_throttle_group !=
5044	    mrioc->facts.max_io_throttle_group) {
5045		ioc_err(mrioc,
5046		    "max io throttle group doesn't match old(%d), new(%d)\n",
5047		    mrioc->num_io_throttle_group,
5048		    mrioc->facts.max_io_throttle_group);
5049		retval = -EPERM;
5050		goto out;
5051	}
5052
5053	mpi3mr_flush_delayed_cmd_lists(mrioc);
5054	mpi3mr_flush_drv_cmds(mrioc);
5055	bitmap_clear(mrioc->devrem_bitmap, 0, MPI3MR_NUM_DEVRMCMD);
5056	bitmap_clear(mrioc->removepend_bitmap, 0,
5057		     mrioc->dev_handle_bitmap_bits);
5058	bitmap_clear(mrioc->evtack_cmds_bitmap, 0, MPI3MR_NUM_EVTACKCMD);
5059	mpi3mr_flush_host_io(mrioc);
5060	mpi3mr_cleanup_fwevt_list(mrioc);
5061	mpi3mr_invalidate_devhandles(mrioc);
5062	mpi3mr_free_enclosure_list(mrioc);
5063
5064	if (mrioc->prepare_for_reset) {
5065		mrioc->prepare_for_reset = 0;
5066		mrioc->prepare_for_reset_timeout_counter = 0;
5067	}
5068	mpi3mr_memset_buffers(mrioc);
5069	retval = mpi3mr_reinit_ioc(mrioc, 0);
5070	if (retval) {
5071		pr_err(IOCNAME "reinit after soft reset failed: reason %d\n",
5072		    mrioc->name, reset_reason);
5073		goto out;
5074	}
5075	ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME);
5076
5077out:
5078	if (!retval) {
5079		mrioc->diagsave_timeout = 0;
5080		mrioc->reset_in_progress = 0;
5081		mrioc->pel_abort_requested = 0;
5082		if (mrioc->pel_enabled) {
5083			mrioc->pel_cmds.retry_count = 0;
5084			mpi3mr_pel_wait_post(mrioc, &mrioc->pel_cmds);
5085		}
5086
5087		mrioc->device_refresh_on = 0;
5088
5089		mrioc->ts_update_counter = 0;
5090		spin_lock_irqsave(&mrioc->watchdog_lock, flags);
5091		if (mrioc->watchdog_work_q)
5092			queue_delayed_work(mrioc->watchdog_work_q,
5093			    &mrioc->watchdog_work,
5094			    msecs_to_jiffies(MPI3MR_WATCHDOG_INTERVAL));
5095		spin_unlock_irqrestore(&mrioc->watchdog_lock, flags);
5096		mrioc->stop_bsgs = 0;
5097		if (mrioc->pel_enabled)
5098			atomic64_inc(&event_counter);
5099	} else {
5100		mpi3mr_issue_reset(mrioc,
5101		    MPI3_SYSIF_HOST_DIAG_RESET_ACTION_DIAG_FAULT, reset_reason);
5102		mrioc->device_refresh_on = 0;
5103		mrioc->unrecoverable = 1;
5104		mrioc->reset_in_progress = 0;
5105		retval = -1;
5106		mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
5107	}
5108	mrioc->prev_reset_result = retval;
5109	mutex_unlock(&mrioc->reset_mutex);
5110	ioc_info(mrioc, "controller reset is %s\n",
5111	    ((retval == 0) ? "successful" : "failed"));
5112	return retval;
5113}
5114
5115
5116/**
5117 * mpi3mr_free_config_dma_memory - free memory for config page
5118 * @mrioc: Adapter instance reference
5119 * @mem_desc: memory descriptor structure
5120 *
5121 * Check whether the size of the buffer specified by the memory
5122 * descriptor is greater than the default page size if so then
5123 * free the memory pointed by the descriptor.
5124 *
5125 * Return: Nothing.
5126 */
5127static void mpi3mr_free_config_dma_memory(struct mpi3mr_ioc *mrioc,
5128	struct dma_memory_desc *mem_desc)
5129{
5130	if ((mem_desc->size > mrioc->cfg_page_sz) && mem_desc->addr) {
5131		dma_free_coherent(&mrioc->pdev->dev, mem_desc->size,
5132		    mem_desc->addr, mem_desc->dma_addr);
5133		mem_desc->addr = NULL;
5134	}
5135}
5136
5137/**
5138 * mpi3mr_alloc_config_dma_memory - Alloc memory for config page
5139 * @mrioc: Adapter instance reference
5140 * @mem_desc: Memory descriptor to hold dma memory info
5141 *
5142 * This function allocates new dmaable memory or provides the
5143 * default config page dmaable memory based on the memory size
5144 * described by the descriptor.
5145 *
5146 * Return: 0 on success, non-zero on failure.
5147 */
5148static int mpi3mr_alloc_config_dma_memory(struct mpi3mr_ioc *mrioc,
5149	struct dma_memory_desc *mem_desc)
5150{
5151	if (mem_desc->size > mrioc->cfg_page_sz) {
5152		mem_desc->addr = dma_alloc_coherent(&mrioc->pdev->dev,
5153		    mem_desc->size, &mem_desc->dma_addr, GFP_KERNEL);
5154		if (!mem_desc->addr)
5155			return -ENOMEM;
5156	} else {
5157		mem_desc->addr = mrioc->cfg_page;
5158		mem_desc->dma_addr = mrioc->cfg_page_dma;
5159		memset(mem_desc->addr, 0, mrioc->cfg_page_sz);
5160	}
5161	return 0;
5162}
5163
5164/**
5165 * mpi3mr_post_cfg_req - Issue config requests and wait
5166 * @mrioc: Adapter instance reference
5167 * @cfg_req: Configuration request
5168 * @timeout: Timeout in seconds
5169 * @ioc_status: Pointer to return ioc status
5170 *
5171 * A generic function for posting MPI3 configuration request to
5172 * the firmware. This blocks for the completion of request for
5173 * timeout seconds and if the request times out this function
5174 * faults the controller with proper reason code.
5175 *
5176 * On successful completion of the request this function returns
5177 * appropriate ioc status from the firmware back to the caller.
5178 *
5179 * Return: 0 on success, non-zero on failure.
5180 */
5181static int mpi3mr_post_cfg_req(struct mpi3mr_ioc *mrioc,
5182	struct mpi3_config_request *cfg_req, int timeout, u16 *ioc_status)
5183{
5184	int retval = 0;
5185
5186	mutex_lock(&mrioc->cfg_cmds.mutex);
5187	if (mrioc->cfg_cmds.state & MPI3MR_CMD_PENDING) {
5188		retval = -1;
5189		ioc_err(mrioc, "sending config request failed due to command in use\n");
5190		mutex_unlock(&mrioc->cfg_cmds.mutex);
5191		goto out;
5192	}
5193	mrioc->cfg_cmds.state = MPI3MR_CMD_PENDING;
5194	mrioc->cfg_cmds.is_waiting = 1;
5195	mrioc->cfg_cmds.callback = NULL;
5196	mrioc->cfg_cmds.ioc_status = 0;
5197	mrioc->cfg_cmds.ioc_loginfo = 0;
5198
5199	cfg_req->host_tag = cpu_to_le16(MPI3MR_HOSTTAG_CFG_CMDS);
5200	cfg_req->function = MPI3_FUNCTION_CONFIG;
5201
5202	init_completion(&mrioc->cfg_cmds.done);
5203	dprint_cfg_info(mrioc, "posting config request\n");
5204	if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5205		dprint_dump(cfg_req, sizeof(struct mpi3_config_request),
5206		    "mpi3_cfg_req");
5207	retval = mpi3mr_admin_request_post(mrioc, cfg_req, sizeof(*cfg_req), 1);
5208	if (retval) {
5209		ioc_err(mrioc, "posting config request failed\n");
5210		goto out_unlock;
5211	}
5212	wait_for_completion_timeout(&mrioc->cfg_cmds.done, (timeout * HZ));
5213	if (!(mrioc->cfg_cmds.state & MPI3MR_CMD_COMPLETE)) {
5214		mpi3mr_check_rh_fault_ioc(mrioc,
5215		    MPI3MR_RESET_FROM_CFG_REQ_TIMEOUT);
5216		ioc_err(mrioc, "config request timed out\n");
5217		retval = -1;
5218		goto out_unlock;
5219	}
5220	*ioc_status = mrioc->cfg_cmds.ioc_status & MPI3_IOCSTATUS_STATUS_MASK;
5221	if ((*ioc_status) != MPI3_IOCSTATUS_SUCCESS)
5222		dprint_cfg_err(mrioc,
5223		    "cfg_page request returned with ioc_status(0x%04x), log_info(0x%08x)\n",
5224		    *ioc_status, mrioc->cfg_cmds.ioc_loginfo);
5225
5226out_unlock:
5227	mrioc->cfg_cmds.state = MPI3MR_CMD_NOTUSED;
5228	mutex_unlock(&mrioc->cfg_cmds.mutex);
5229
5230out:
5231	return retval;
5232}
5233
5234/**
5235 * mpi3mr_process_cfg_req - config page request processor
5236 * @mrioc: Adapter instance reference
5237 * @cfg_req: Configuration request
5238 * @cfg_hdr: Configuration page header
5239 * @timeout: Timeout in seconds
5240 * @ioc_status: Pointer to return ioc status
5241 * @cfg_buf: Memory pointer to copy config page or header
5242 * @cfg_buf_sz: Size of the memory to get config page or header
5243 *
5244 * This is handler for config page read, write and config page
5245 * header read operations.
5246 *
5247 * This function expects the cfg_req to be populated with page
5248 * type, page number, action for the header read and with page
5249 * address for all other operations.
5250 *
5251 * The cfg_hdr can be passed as null for reading required header
5252 * details for read/write pages the cfg_hdr should point valid
5253 * configuration page header.
5254 *
5255 * This allocates dmaable memory based on the size of the config
5256 * buffer and set the SGE of the cfg_req.
5257 *
5258 * For write actions, the config page data has to be passed in
5259 * the cfg_buf and size of the data has to be mentioned in the
5260 * cfg_buf_sz.
5261 *
5262 * For read/header actions, on successful completion of the
5263 * request with successful ioc_status the data will be copied
5264 * into the cfg_buf limited to a minimum of actual page size and
5265 * cfg_buf_sz
5266 *
5267 *
5268 * Return: 0 on success, non-zero on failure.
5269 */
5270static int mpi3mr_process_cfg_req(struct mpi3mr_ioc *mrioc,
5271	struct mpi3_config_request *cfg_req,
5272	struct mpi3_config_page_header *cfg_hdr, int timeout, u16 *ioc_status,
5273	void *cfg_buf, u32 cfg_buf_sz)
5274{
5275	struct dma_memory_desc mem_desc;
5276	int retval = -1;
5277	u8 invalid_action = 0;
5278	u8 sgl_flags = MPI3MR_SGEFLAGS_SYSTEM_SIMPLE_END_OF_LIST;
5279
5280	memset(&mem_desc, 0, sizeof(struct dma_memory_desc));
5281
5282	if (cfg_req->action == MPI3_CONFIG_ACTION_PAGE_HEADER)
5283		mem_desc.size = sizeof(struct mpi3_config_page_header);
5284	else {
5285		if (!cfg_hdr) {
5286			ioc_err(mrioc, "null config header passed for config action(%d), page_type(0x%02x), page_num(%d)\n",
5287			    cfg_req->action, cfg_req->page_type,
5288			    cfg_req->page_number);
5289			goto out;
5290		}
5291		switch (cfg_hdr->page_attribute & MPI3_CONFIG_PAGEATTR_MASK) {
5292		case MPI3_CONFIG_PAGEATTR_READ_ONLY:
5293			if (cfg_req->action
5294			    != MPI3_CONFIG_ACTION_READ_CURRENT)
5295				invalid_action = 1;
5296			break;
5297		case MPI3_CONFIG_PAGEATTR_CHANGEABLE:
5298			if ((cfg_req->action ==
5299			     MPI3_CONFIG_ACTION_READ_PERSISTENT) ||
5300			    (cfg_req->action ==
5301			     MPI3_CONFIG_ACTION_WRITE_PERSISTENT))
5302				invalid_action = 1;
5303			break;
5304		case MPI3_CONFIG_PAGEATTR_PERSISTENT:
5305		default:
5306			break;
5307		}
5308		if (invalid_action) {
5309			ioc_err(mrioc,
5310			    "config action(%d) is not allowed for page_type(0x%02x), page_num(%d) with page_attribute(0x%02x)\n",
5311			    cfg_req->action, cfg_req->page_type,
5312			    cfg_req->page_number, cfg_hdr->page_attribute);
5313			goto out;
5314		}
5315		mem_desc.size = le16_to_cpu(cfg_hdr->page_length) * 4;
5316		cfg_req->page_length = cfg_hdr->page_length;
5317		cfg_req->page_version = cfg_hdr->page_version;
5318	}
5319	if (mpi3mr_alloc_config_dma_memory(mrioc, &mem_desc))
5320		goto out;
5321
5322	mpi3mr_add_sg_single(&cfg_req->sgl, sgl_flags, mem_desc.size,
5323	    mem_desc.dma_addr);
5324
5325	if ((cfg_req->action == MPI3_CONFIG_ACTION_WRITE_PERSISTENT) ||
5326	    (cfg_req->action == MPI3_CONFIG_ACTION_WRITE_CURRENT)) {
5327		memcpy(mem_desc.addr, cfg_buf, min_t(u16, mem_desc.size,
5328		    cfg_buf_sz));
5329		dprint_cfg_info(mrioc, "config buffer to be written\n");
5330		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5331			dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf");
5332	}
5333
5334	if (mpi3mr_post_cfg_req(mrioc, cfg_req, timeout, ioc_status))
5335		goto out;
5336
5337	retval = 0;
5338	if ((*ioc_status == MPI3_IOCSTATUS_SUCCESS) &&
5339	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_PERSISTENT) &&
5340	    (cfg_req->action != MPI3_CONFIG_ACTION_WRITE_CURRENT)) {
5341		memcpy(cfg_buf, mem_desc.addr, min_t(u16, mem_desc.size,
5342		    cfg_buf_sz));
5343		dprint_cfg_info(mrioc, "config buffer read\n");
5344		if (mrioc->logging_level & MPI3_DEBUG_CFG_INFO)
5345			dprint_dump(mem_desc.addr, mem_desc.size, "cfg_buf");
5346	}
5347
5348out:
5349	mpi3mr_free_config_dma_memory(mrioc, &mem_desc);
5350	return retval;
5351}
5352
5353/**
5354 * mpi3mr_cfg_get_dev_pg0 - Read current device page0
5355 * @mrioc: Adapter instance reference
5356 * @ioc_status: Pointer to return ioc status
5357 * @dev_pg0: Pointer to return device page 0
5358 * @pg_sz: Size of the memory allocated to the page pointer
5359 * @form: The form to be used for addressing the page
5360 * @form_spec: Form specific information like device handle
5361 *
5362 * This is handler for config page read for a specific device
5363 * page0. The ioc_status has the controller returned ioc_status.
5364 * This routine doesn't check ioc_status to decide whether the
5365 * page read is success or not and it is the callers
5366 * responsibility.
5367 *
5368 * Return: 0 on success, non-zero on failure.
5369 */
5370int mpi3mr_cfg_get_dev_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5371	struct mpi3_device_page0 *dev_pg0, u16 pg_sz, u32 form, u32 form_spec)
5372{
5373	struct mpi3_config_page_header cfg_hdr;
5374	struct mpi3_config_request cfg_req;
5375	u32 page_address;
5376
5377	memset(dev_pg0, 0, pg_sz);
5378	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5379	memset(&cfg_req, 0, sizeof(cfg_req));
5380
5381	cfg_req.function = MPI3_FUNCTION_CONFIG;
5382	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5383	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DEVICE;
5384	cfg_req.page_number = 0;
5385	cfg_req.page_address = 0;
5386
5387	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5388	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5389		ioc_err(mrioc, "device page0 header read failed\n");
5390		goto out_failed;
5391	}
5392	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5393		ioc_err(mrioc, "device page0 header read failed with ioc_status(0x%04x)\n",
5394		    *ioc_status);
5395		goto out_failed;
5396	}
5397	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5398	page_address = ((form & MPI3_DEVICE_PGAD_FORM_MASK) |
5399	    (form_spec & MPI3_DEVICE_PGAD_HANDLE_MASK));
5400	cfg_req.page_address = cpu_to_le32(page_address);
5401	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5402	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, dev_pg0, pg_sz)) {
5403		ioc_err(mrioc, "device page0 read failed\n");
5404		goto out_failed;
5405	}
5406	return 0;
5407out_failed:
5408	return -1;
5409}
5410
5411
5412/**
5413 * mpi3mr_cfg_get_sas_phy_pg0 - Read current SAS Phy page0
5414 * @mrioc: Adapter instance reference
5415 * @ioc_status: Pointer to return ioc status
5416 * @phy_pg0: Pointer to return SAS Phy page 0
5417 * @pg_sz: Size of the memory allocated to the page pointer
5418 * @form: The form to be used for addressing the page
5419 * @form_spec: Form specific information like phy number
5420 *
5421 * This is handler for config page read for a specific SAS Phy
5422 * page0. The ioc_status has the controller returned ioc_status.
5423 * This routine doesn't check ioc_status to decide whether the
5424 * page read is success or not and it is the callers
5425 * responsibility.
5426 *
5427 * Return: 0 on success, non-zero on failure.
5428 */
5429int mpi3mr_cfg_get_sas_phy_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5430	struct mpi3_sas_phy_page0 *phy_pg0, u16 pg_sz, u32 form,
5431	u32 form_spec)
5432{
5433	struct mpi3_config_page_header cfg_hdr;
5434	struct mpi3_config_request cfg_req;
5435	u32 page_address;
5436
5437	memset(phy_pg0, 0, pg_sz);
5438	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5439	memset(&cfg_req, 0, sizeof(cfg_req));
5440
5441	cfg_req.function = MPI3_FUNCTION_CONFIG;
5442	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5443	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY;
5444	cfg_req.page_number = 0;
5445	cfg_req.page_address = 0;
5446
5447	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5448	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5449		ioc_err(mrioc, "sas phy page0 header read failed\n");
5450		goto out_failed;
5451	}
5452	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5453		ioc_err(mrioc, "sas phy page0 header read failed with ioc_status(0x%04x)\n",
5454		    *ioc_status);
5455		goto out_failed;
5456	}
5457	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5458	page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) |
5459	    (form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK));
5460	cfg_req.page_address = cpu_to_le32(page_address);
5461	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5462	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, phy_pg0, pg_sz)) {
5463		ioc_err(mrioc, "sas phy page0 read failed\n");
5464		goto out_failed;
5465	}
5466	return 0;
5467out_failed:
5468	return -1;
5469}
5470
5471/**
5472 * mpi3mr_cfg_get_sas_phy_pg1 - Read current SAS Phy page1
5473 * @mrioc: Adapter instance reference
5474 * @ioc_status: Pointer to return ioc status
5475 * @phy_pg1: Pointer to return SAS Phy page 1
5476 * @pg_sz: Size of the memory allocated to the page pointer
5477 * @form: The form to be used for addressing the page
5478 * @form_spec: Form specific information like phy number
5479 *
5480 * This is handler for config page read for a specific SAS Phy
5481 * page1. The ioc_status has the controller returned ioc_status.
5482 * This routine doesn't check ioc_status to decide whether the
5483 * page read is success or not and it is the callers
5484 * responsibility.
5485 *
5486 * Return: 0 on success, non-zero on failure.
5487 */
5488int mpi3mr_cfg_get_sas_phy_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5489	struct mpi3_sas_phy_page1 *phy_pg1, u16 pg_sz, u32 form,
5490	u32 form_spec)
5491{
5492	struct mpi3_config_page_header cfg_hdr;
5493	struct mpi3_config_request cfg_req;
5494	u32 page_address;
5495
5496	memset(phy_pg1, 0, pg_sz);
5497	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5498	memset(&cfg_req, 0, sizeof(cfg_req));
5499
5500	cfg_req.function = MPI3_FUNCTION_CONFIG;
5501	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5502	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_PHY;
5503	cfg_req.page_number = 1;
5504	cfg_req.page_address = 0;
5505
5506	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5507	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5508		ioc_err(mrioc, "sas phy page1 header read failed\n");
5509		goto out_failed;
5510	}
5511	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5512		ioc_err(mrioc, "sas phy page1 header read failed with ioc_status(0x%04x)\n",
5513		    *ioc_status);
5514		goto out_failed;
5515	}
5516	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5517	page_address = ((form & MPI3_SAS_PHY_PGAD_FORM_MASK) |
5518	    (form_spec & MPI3_SAS_PHY_PGAD_PHY_NUMBER_MASK));
5519	cfg_req.page_address = cpu_to_le32(page_address);
5520	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5521	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, phy_pg1, pg_sz)) {
5522		ioc_err(mrioc, "sas phy page1 read failed\n");
5523		goto out_failed;
5524	}
5525	return 0;
5526out_failed:
5527	return -1;
5528}
5529
5530
5531/**
5532 * mpi3mr_cfg_get_sas_exp_pg0 - Read current SAS Expander page0
5533 * @mrioc: Adapter instance reference
5534 * @ioc_status: Pointer to return ioc status
5535 * @exp_pg0: Pointer to return SAS Expander page 0
5536 * @pg_sz: Size of the memory allocated to the page pointer
5537 * @form: The form to be used for addressing the page
5538 * @form_spec: Form specific information like device handle
5539 *
5540 * This is handler for config page read for a specific SAS
5541 * Expander page0. The ioc_status has the controller returned
5542 * ioc_status. This routine doesn't check ioc_status to decide
5543 * whether the page read is success or not and it is the callers
5544 * responsibility.
5545 *
5546 * Return: 0 on success, non-zero on failure.
5547 */
5548int mpi3mr_cfg_get_sas_exp_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5549	struct mpi3_sas_expander_page0 *exp_pg0, u16 pg_sz, u32 form,
5550	u32 form_spec)
5551{
5552	struct mpi3_config_page_header cfg_hdr;
5553	struct mpi3_config_request cfg_req;
5554	u32 page_address;
5555
5556	memset(exp_pg0, 0, pg_sz);
5557	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5558	memset(&cfg_req, 0, sizeof(cfg_req));
5559
5560	cfg_req.function = MPI3_FUNCTION_CONFIG;
5561	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5562	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER;
5563	cfg_req.page_number = 0;
5564	cfg_req.page_address = 0;
5565
5566	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5567	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5568		ioc_err(mrioc, "expander page0 header read failed\n");
5569		goto out_failed;
5570	}
5571	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5572		ioc_err(mrioc, "expander page0 header read failed with ioc_status(0x%04x)\n",
5573		    *ioc_status);
5574		goto out_failed;
5575	}
5576	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5577	page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) |
5578	    (form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK |
5579	    MPI3_SAS_EXPAND_PGAD_HANDLE_MASK)));
5580	cfg_req.page_address = cpu_to_le32(page_address);
5581	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5582	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, exp_pg0, pg_sz)) {
5583		ioc_err(mrioc, "expander page0 read failed\n");
5584		goto out_failed;
5585	}
5586	return 0;
5587out_failed:
5588	return -1;
5589}
5590
5591/**
5592 * mpi3mr_cfg_get_sas_exp_pg1 - Read current SAS Expander page1
5593 * @mrioc: Adapter instance reference
5594 * @ioc_status: Pointer to return ioc status
5595 * @exp_pg1: Pointer to return SAS Expander page 1
5596 * @pg_sz: Size of the memory allocated to the page pointer
5597 * @form: The form to be used for addressing the page
5598 * @form_spec: Form specific information like phy number
5599 *
5600 * This is handler for config page read for a specific SAS
5601 * Expander page1. The ioc_status has the controller returned
5602 * ioc_status. This routine doesn't check ioc_status to decide
5603 * whether the page read is success or not and it is the callers
5604 * responsibility.
5605 *
5606 * Return: 0 on success, non-zero on failure.
5607 */
5608int mpi3mr_cfg_get_sas_exp_pg1(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5609	struct mpi3_sas_expander_page1 *exp_pg1, u16 pg_sz, u32 form,
5610	u32 form_spec)
5611{
5612	struct mpi3_config_page_header cfg_hdr;
5613	struct mpi3_config_request cfg_req;
5614	u32 page_address;
5615
5616	memset(exp_pg1, 0, pg_sz);
5617	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5618	memset(&cfg_req, 0, sizeof(cfg_req));
5619
5620	cfg_req.function = MPI3_FUNCTION_CONFIG;
5621	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5622	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_EXPANDER;
5623	cfg_req.page_number = 1;
5624	cfg_req.page_address = 0;
5625
5626	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5627	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5628		ioc_err(mrioc, "expander page1 header read failed\n");
5629		goto out_failed;
5630	}
5631	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5632		ioc_err(mrioc, "expander page1 header read failed with ioc_status(0x%04x)\n",
5633		    *ioc_status);
5634		goto out_failed;
5635	}
5636	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5637	page_address = ((form & MPI3_SAS_EXPAND_PGAD_FORM_MASK) |
5638	    (form_spec & (MPI3_SAS_EXPAND_PGAD_PHYNUM_MASK |
5639	    MPI3_SAS_EXPAND_PGAD_HANDLE_MASK)));
5640	cfg_req.page_address = cpu_to_le32(page_address);
5641	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5642	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, exp_pg1, pg_sz)) {
5643		ioc_err(mrioc, "expander page1 read failed\n");
5644		goto out_failed;
5645	}
5646	return 0;
5647out_failed:
5648	return -1;
5649}
5650
5651/**
5652 * mpi3mr_cfg_get_enclosure_pg0 - Read current Enclosure page0
5653 * @mrioc: Adapter instance reference
5654 * @ioc_status: Pointer to return ioc status
5655 * @encl_pg0: Pointer to return Enclosure page 0
5656 * @pg_sz: Size of the memory allocated to the page pointer
5657 * @form: The form to be used for addressing the page
5658 * @form_spec: Form specific information like device handle
5659 *
5660 * This is handler for config page read for a specific Enclosure
5661 * page0. The ioc_status has the controller returned ioc_status.
5662 * This routine doesn't check ioc_status to decide whether the
5663 * page read is success or not and it is the callers
5664 * responsibility.
5665 *
5666 * Return: 0 on success, non-zero on failure.
5667 */
5668int mpi3mr_cfg_get_enclosure_pg0(struct mpi3mr_ioc *mrioc, u16 *ioc_status,
5669	struct mpi3_enclosure_page0 *encl_pg0, u16 pg_sz, u32 form,
5670	u32 form_spec)
5671{
5672	struct mpi3_config_page_header cfg_hdr;
5673	struct mpi3_config_request cfg_req;
5674	u32 page_address;
5675
5676	memset(encl_pg0, 0, pg_sz);
5677	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5678	memset(&cfg_req, 0, sizeof(cfg_req));
5679
5680	cfg_req.function = MPI3_FUNCTION_CONFIG;
5681	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5682	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_ENCLOSURE;
5683	cfg_req.page_number = 0;
5684	cfg_req.page_address = 0;
5685
5686	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5687	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5688		ioc_err(mrioc, "enclosure page0 header read failed\n");
5689		goto out_failed;
5690	}
5691	if (*ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5692		ioc_err(mrioc, "enclosure page0 header read failed with ioc_status(0x%04x)\n",
5693		    *ioc_status);
5694		goto out_failed;
5695	}
5696	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5697	page_address = ((form & MPI3_ENCLOS_PGAD_FORM_MASK) |
5698	    (form_spec & MPI3_ENCLOS_PGAD_HANDLE_MASK));
5699	cfg_req.page_address = cpu_to_le32(page_address);
5700	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5701	    MPI3MR_INTADMCMD_TIMEOUT, ioc_status, encl_pg0, pg_sz)) {
5702		ioc_err(mrioc, "enclosure page0 read failed\n");
5703		goto out_failed;
5704	}
5705	return 0;
5706out_failed:
5707	return -1;
5708}
5709
5710
5711/**
5712 * mpi3mr_cfg_get_sas_io_unit_pg0 - Read current SASIOUnit page0
5713 * @mrioc: Adapter instance reference
5714 * @sas_io_unit_pg0: Pointer to return SAS IO Unit page 0
5715 * @pg_sz: Size of the memory allocated to the page pointer
5716 *
5717 * This is handler for config page read for the SAS IO Unit
5718 * page0. This routine checks ioc_status to decide whether the
5719 * page read is success or not.
5720 *
5721 * Return: 0 on success, non-zero on failure.
5722 */
5723int mpi3mr_cfg_get_sas_io_unit_pg0(struct mpi3mr_ioc *mrioc,
5724	struct mpi3_sas_io_unit_page0 *sas_io_unit_pg0, u16 pg_sz)
5725{
5726	struct mpi3_config_page_header cfg_hdr;
5727	struct mpi3_config_request cfg_req;
5728	u16 ioc_status = 0;
5729
5730	memset(sas_io_unit_pg0, 0, pg_sz);
5731	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5732	memset(&cfg_req, 0, sizeof(cfg_req));
5733
5734	cfg_req.function = MPI3_FUNCTION_CONFIG;
5735	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5736	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5737	cfg_req.page_number = 0;
5738	cfg_req.page_address = 0;
5739
5740	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5741	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5742		ioc_err(mrioc, "sas io unit page0 header read failed\n");
5743		goto out_failed;
5744	}
5745	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5746		ioc_err(mrioc, "sas io unit page0 header read failed with ioc_status(0x%04x)\n",
5747		    ioc_status);
5748		goto out_failed;
5749	}
5750	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5751
5752	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5753	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg0, pg_sz)) {
5754		ioc_err(mrioc, "sas io unit page0 read failed\n");
5755		goto out_failed;
5756	}
5757	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5758		ioc_err(mrioc, "sas io unit page0 read failed with ioc_status(0x%04x)\n",
5759		    ioc_status);
5760		goto out_failed;
5761	}
5762	return 0;
5763out_failed:
5764	return -1;
5765}
5766
5767/**
5768 * mpi3mr_cfg_get_sas_io_unit_pg1 - Read current SASIOUnit page1
5769 * @mrioc: Adapter instance reference
5770 * @sas_io_unit_pg1: Pointer to return SAS IO Unit page 1
5771 * @pg_sz: Size of the memory allocated to the page pointer
5772 *
5773 * This is handler for config page read for the SAS IO Unit
5774 * page1. This routine checks ioc_status to decide whether the
5775 * page read is success or not.
5776 *
5777 * Return: 0 on success, non-zero on failure.
5778 */
5779int mpi3mr_cfg_get_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5780	struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz)
5781{
5782	struct mpi3_config_page_header cfg_hdr;
5783	struct mpi3_config_request cfg_req;
5784	u16 ioc_status = 0;
5785
5786	memset(sas_io_unit_pg1, 0, pg_sz);
5787	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5788	memset(&cfg_req, 0, sizeof(cfg_req));
5789
5790	cfg_req.function = MPI3_FUNCTION_CONFIG;
5791	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5792	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5793	cfg_req.page_number = 1;
5794	cfg_req.page_address = 0;
5795
5796	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5797	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5798		ioc_err(mrioc, "sas io unit page1 header read failed\n");
5799		goto out_failed;
5800	}
5801	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5802		ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5803		    ioc_status);
5804		goto out_failed;
5805	}
5806	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5807
5808	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5809	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5810		ioc_err(mrioc, "sas io unit page1 read failed\n");
5811		goto out_failed;
5812	}
5813	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5814		ioc_err(mrioc, "sas io unit page1 read failed with ioc_status(0x%04x)\n",
5815		    ioc_status);
5816		goto out_failed;
5817	}
5818	return 0;
5819out_failed:
5820	return -1;
5821}
5822
5823/**
5824 * mpi3mr_cfg_set_sas_io_unit_pg1 - Write SASIOUnit page1
5825 * @mrioc: Adapter instance reference
5826 * @sas_io_unit_pg1: Pointer to the SAS IO Unit page 1 to write
5827 * @pg_sz: Size of the memory allocated to the page pointer
5828 *
5829 * This is handler for config page write for the SAS IO Unit
5830 * page1. This routine checks ioc_status to decide whether the
5831 * page read is success or not. This will modify both current
5832 * and persistent page.
5833 *
5834 * Return: 0 on success, non-zero on failure.
5835 */
5836int mpi3mr_cfg_set_sas_io_unit_pg1(struct mpi3mr_ioc *mrioc,
5837	struct mpi3_sas_io_unit_page1 *sas_io_unit_pg1, u16 pg_sz)
5838{
5839	struct mpi3_config_page_header cfg_hdr;
5840	struct mpi3_config_request cfg_req;
5841	u16 ioc_status = 0;
5842
5843	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5844	memset(&cfg_req, 0, sizeof(cfg_req));
5845
5846	cfg_req.function = MPI3_FUNCTION_CONFIG;
5847	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5848	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_SAS_IO_UNIT;
5849	cfg_req.page_number = 1;
5850	cfg_req.page_address = 0;
5851
5852	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5853	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5854		ioc_err(mrioc, "sas io unit page1 header read failed\n");
5855		goto out_failed;
5856	}
5857	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5858		ioc_err(mrioc, "sas io unit page1 header read failed with ioc_status(0x%04x)\n",
5859		    ioc_status);
5860		goto out_failed;
5861	}
5862	cfg_req.action = MPI3_CONFIG_ACTION_WRITE_CURRENT;
5863
5864	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5865	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5866		ioc_err(mrioc, "sas io unit page1 write current failed\n");
5867		goto out_failed;
5868	}
5869	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5870		ioc_err(mrioc, "sas io unit page1 write current failed with ioc_status(0x%04x)\n",
5871		    ioc_status);
5872		goto out_failed;
5873	}
5874
5875	cfg_req.action = MPI3_CONFIG_ACTION_WRITE_PERSISTENT;
5876
5877	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5878	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, sas_io_unit_pg1, pg_sz)) {
5879		ioc_err(mrioc, "sas io unit page1 write persistent failed\n");
5880		goto out_failed;
5881	}
5882	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5883		ioc_err(mrioc, "sas io unit page1 write persistent failed with ioc_status(0x%04x)\n",
5884		    ioc_status);
5885		goto out_failed;
5886	}
5887	return 0;
5888out_failed:
5889	return -1;
5890}
5891
5892/**
5893 * mpi3mr_cfg_get_driver_pg1 - Read current Driver page1
5894 * @mrioc: Adapter instance reference
5895 * @driver_pg1: Pointer to return Driver page 1
5896 * @pg_sz: Size of the memory allocated to the page pointer
5897 *
5898 * This is handler for config page read for the Driver page1.
5899 * This routine checks ioc_status to decide whether the page
5900 * read is success or not.
5901 *
5902 * Return: 0 on success, non-zero on failure.
5903 */
5904int mpi3mr_cfg_get_driver_pg1(struct mpi3mr_ioc *mrioc,
5905	struct mpi3_driver_page1 *driver_pg1, u16 pg_sz)
5906{
5907	struct mpi3_config_page_header cfg_hdr;
5908	struct mpi3_config_request cfg_req;
5909	u16 ioc_status = 0;
5910
5911	memset(driver_pg1, 0, pg_sz);
5912	memset(&cfg_hdr, 0, sizeof(cfg_hdr));
5913	memset(&cfg_req, 0, sizeof(cfg_req));
5914
5915	cfg_req.function = MPI3_FUNCTION_CONFIG;
5916	cfg_req.action = MPI3_CONFIG_ACTION_PAGE_HEADER;
5917	cfg_req.page_type = MPI3_CONFIG_PAGETYPE_DRIVER;
5918	cfg_req.page_number = 1;
5919	cfg_req.page_address = 0;
5920
5921	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, NULL,
5922	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, &cfg_hdr, sizeof(cfg_hdr))) {
5923		ioc_err(mrioc, "driver page1 header read failed\n");
5924		goto out_failed;
5925	}
5926	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5927		ioc_err(mrioc, "driver page1 header read failed with ioc_status(0x%04x)\n",
5928		    ioc_status);
5929		goto out_failed;
5930	}
5931	cfg_req.action = MPI3_CONFIG_ACTION_READ_CURRENT;
5932
5933	if (mpi3mr_process_cfg_req(mrioc, &cfg_req, &cfg_hdr,
5934	    MPI3MR_INTADMCMD_TIMEOUT, &ioc_status, driver_pg1, pg_sz)) {
5935		ioc_err(mrioc, "driver page1 read failed\n");
5936		goto out_failed;
5937	}
5938	if (ioc_status != MPI3_IOCSTATUS_SUCCESS) {
5939		ioc_err(mrioc, "driver page1 read failed with ioc_status(0x%04x)\n",
5940		    ioc_status);
5941		goto out_failed;
5942	}
5943	return 0;
5944out_failed:
5945	return -1;
5946}