1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Driver for Broadcom MPI3 Storage Controllers
   4 *
   5 * Copyright (C) 2017-2022 Broadcom Inc.
   6 *  (mailto: mpi3mr-linuxdrv.pdl@broadcom.com)
   7 *
   8 */
   9
  10#include "mpi3mr.h"
  11
  12/* global driver scop variables */
  13LIST_HEAD(mrioc_list);
  14DEFINE_SPINLOCK(mrioc_list_lock);
  15static int mrioc_ids;
  16static int warn_non_secure_ctlr;
  17atomic64_t event_counter;
  18
  19MODULE_AUTHOR(MPI3MR_DRIVER_AUTHOR);
  20MODULE_DESCRIPTION(MPI3MR_DRIVER_DESC);
  21MODULE_LICENSE(MPI3MR_DRIVER_LICENSE);
  22MODULE_VERSION(MPI3MR_DRIVER_VERSION);
  23
  24/* Module parameters*/
  25int prot_mask = -1;
  26module_param(prot_mask, int, 0);
  27MODULE_PARM_DESC(prot_mask, "Host protection capabilities mask, def=0x07");
  28
  29static int prot_guard_mask = 3;
  30module_param(prot_guard_mask, int, 0);
  31MODULE_PARM_DESC(prot_guard_mask, " Host protection guard mask, def=3");
  32static int logging_level;
  33module_param(logging_level, int, 0);
  34MODULE_PARM_DESC(logging_level,
  35	" bits for enabling additional logging info (default=0)");
  36
  37/* Forward declarations*/
  38static void mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
  39	struct mpi3mr_drv_cmd *cmdparam, u32 event_ctx);
  40
  41#define MPI3MR_DRIVER_EVENT_TG_QD_REDUCTION	(0xFFFF)
  42
  43#define MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH	(0xFFFE)
  44
  45/**
  46 * mpi3mr_host_tag_for_scmd - Get host tag for a scmd
  47 * @mrioc: Adapter instance reference
  48 * @scmd: SCSI command reference
  49 *
  50 * Calculate the host tag based on block tag for a given scmd.
  51 *
  52 * Return: Valid host tag or MPI3MR_HOSTTAG_INVALID.
  53 */
  54static u16 mpi3mr_host_tag_for_scmd(struct mpi3mr_ioc *mrioc,
  55	struct scsi_cmnd *scmd)
  56{
  57	struct scmd_priv *priv = NULL;
  58	u32 unique_tag;
  59	u16 host_tag, hw_queue;
  60
  61	unique_tag = blk_mq_unique_tag(scsi_cmd_to_rq(scmd));
  62
  63	hw_queue = blk_mq_unique_tag_to_hwq(unique_tag);
  64	if (hw_queue >= mrioc->num_op_reply_q)
  65		return MPI3MR_HOSTTAG_INVALID;
  66	host_tag = blk_mq_unique_tag_to_tag(unique_tag);
  67
  68	if (WARN_ON(host_tag >= mrioc->max_host_ios))
  69		return MPI3MR_HOSTTAG_INVALID;
  70
  71	priv = scsi_cmd_priv(scmd);
  72	/*host_tag 0 is invalid hence incrementing by 1*/
  73	priv->host_tag = host_tag + 1;
  74	priv->scmd = scmd;
  75	priv->in_lld_scope = 1;
  76	priv->req_q_idx = hw_queue;
  77	priv->meta_chain_idx = -1;
  78	priv->chain_idx = -1;
  79	priv->meta_sg_valid = 0;
  80	return priv->host_tag;
  81}
  82
  83/**
  84 * mpi3mr_scmd_from_host_tag - Get SCSI command from host tag
  85 * @mrioc: Adapter instance reference
  86 * @host_tag: Host tag
  87 * @qidx: Operational queue index
  88 *
  89 * Identify the block tag from the host tag and queue index and
  90 * retrieve associated scsi command using scsi_host_find_tag().
  91 *
  92 * Return: SCSI command reference or NULL.
  93 */
  94static struct scsi_cmnd *mpi3mr_scmd_from_host_tag(
  95	struct mpi3mr_ioc *mrioc, u16 host_tag, u16 qidx)
  96{
  97	struct scsi_cmnd *scmd = NULL;
  98	struct scmd_priv *priv = NULL;
  99	u32 unique_tag = host_tag - 1;
 100
 101	if (WARN_ON(host_tag > mrioc->max_host_ios))
 102		goto out;
 103
 104	unique_tag |= (qidx << BLK_MQ_UNIQUE_TAG_BITS);
 105
 106	scmd = scsi_host_find_tag(mrioc->shost, unique_tag);
 107	if (scmd) {
 108		priv = scsi_cmd_priv(scmd);
 109		if (!priv->in_lld_scope)
 110			scmd = NULL;
 111	}
 112out:
 113	return scmd;
 114}
 115
 116/**
 117 * mpi3mr_clear_scmd_priv - Cleanup SCSI command private date
 118 * @mrioc: Adapter instance reference
 119 * @scmd: SCSI command reference
 120 *
 121 * Invalidate the SCSI command private data to mark the command
 122 * is not in LLD scope anymore.
 123 *
 124 * Return: Nothing.
 125 */
 126static void mpi3mr_clear_scmd_priv(struct mpi3mr_ioc *mrioc,
 127	struct scsi_cmnd *scmd)
 128{
 129	struct scmd_priv *priv = NULL;
 130
 131	priv = scsi_cmd_priv(scmd);
 132
 133	if (WARN_ON(priv->in_lld_scope == 0))
 134		return;
 135	priv->host_tag = MPI3MR_HOSTTAG_INVALID;
 136	priv->req_q_idx = 0xFFFF;
 137	priv->scmd = NULL;
 138	priv->in_lld_scope = 0;
 139	priv->meta_sg_valid = 0;
 140	if (priv->chain_idx >= 0) {
 141		clear_bit(priv->chain_idx, mrioc->chain_bitmap);
 142		priv->chain_idx = -1;
 143	}
 144	if (priv->meta_chain_idx >= 0) {
 145		clear_bit(priv->meta_chain_idx, mrioc->chain_bitmap);
 146		priv->meta_chain_idx = -1;
 147	}
 148}
 149
 150static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_ioc *mrioc, u16 handle,
 151	struct mpi3mr_drv_cmd *cmdparam, u8 iou_rc);
 152static void mpi3mr_fwevt_worker(struct work_struct *work);
 153
 154/**
 155 * mpi3mr_fwevt_free - firmware event memory dealloctor
 156 * @r: k reference pointer of the firmware event
 157 *
 158 * Free firmware event memory when no reference.
 159 */
 160static void mpi3mr_fwevt_free(struct kref *r)
 161{
 162	kfree(container_of(r, struct mpi3mr_fwevt, ref_count));
 163}
 164
 165/**
 166 * mpi3mr_fwevt_get - k reference incrementor
 167 * @fwevt: Firmware event reference
 168 *
 169 * Increment firmware event reference count.
 170 */
 171static void mpi3mr_fwevt_get(struct mpi3mr_fwevt *fwevt)
 172{
 173	kref_get(&fwevt->ref_count);
 174}
 175
 176/**
 177 * mpi3mr_fwevt_put - k reference decrementor
 178 * @fwevt: Firmware event reference
 179 *
 180 * decrement firmware event reference count.
 181 */
 182static void mpi3mr_fwevt_put(struct mpi3mr_fwevt *fwevt)
 183{
 184	kref_put(&fwevt->ref_count, mpi3mr_fwevt_free);
 185}
 186
 187/**
 188 * mpi3mr_alloc_fwevt - Allocate firmware event
 189 * @len: length of firmware event data to allocate
 190 *
 191 * Allocate firmware event with required length and initialize
 192 * the reference counter.
 193 *
 194 * Return: firmware event reference.
 195 */
 196static struct mpi3mr_fwevt *mpi3mr_alloc_fwevt(int len)
 197{
 198	struct mpi3mr_fwevt *fwevt;
 199
 200	fwevt = kzalloc(sizeof(*fwevt) + len, GFP_ATOMIC);
 201	if (!fwevt)
 202		return NULL;
 203
 204	kref_init(&fwevt->ref_count);
 205	return fwevt;
 206}
 207
 208/**
 209 * mpi3mr_fwevt_add_to_list - Add firmware event to the list
 210 * @mrioc: Adapter instance reference
 211 * @fwevt: Firmware event reference
 212 *
 213 * Add the given firmware event to the firmware event list.
 214 *
 215 * Return: Nothing.
 216 */
 217static void mpi3mr_fwevt_add_to_list(struct mpi3mr_ioc *mrioc,
 218	struct mpi3mr_fwevt *fwevt)
 219{
 220	unsigned long flags;
 221
 222	if (!mrioc->fwevt_worker_thread)
 223		return;
 224
 225	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 226	/* get fwevt reference count while adding it to fwevt_list */
 227	mpi3mr_fwevt_get(fwevt);
 228	INIT_LIST_HEAD(&fwevt->list);
 229	list_add_tail(&fwevt->list, &mrioc->fwevt_list);
 230	INIT_WORK(&fwevt->work, mpi3mr_fwevt_worker);
 231	/* get fwevt reference count while enqueueing it to worker queue */
 232	mpi3mr_fwevt_get(fwevt);
 233	queue_work(mrioc->fwevt_worker_thread, &fwevt->work);
 234	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 235}
 236
 237/**
 238 * mpi3mr_fwevt_del_from_list - Delete firmware event from list
 239 * @mrioc: Adapter instance reference
 240 * @fwevt: Firmware event reference
 241 *
 242 * Delete the given firmware event from the firmware event list.
 243 *
 244 * Return: Nothing.
 245 */
 246static void mpi3mr_fwevt_del_from_list(struct mpi3mr_ioc *mrioc,
 247	struct mpi3mr_fwevt *fwevt)
 248{
 249	unsigned long flags;
 250
 251	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 252	if (!list_empty(&fwevt->list)) {
 253		list_del_init(&fwevt->list);
 254		/*
 255		 * Put fwevt reference count after
 256		 * removing it from fwevt_list
 257		 */
 258		mpi3mr_fwevt_put(fwevt);
 259	}
 260	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 261}
 262
 263/**
 264 * mpi3mr_dequeue_fwevt - Dequeue firmware event from the list
 265 * @mrioc: Adapter instance reference
 266 *
 267 * Dequeue a firmware event from the firmware event list.
 268 *
 269 * Return: firmware event.
 270 */
 271static struct mpi3mr_fwevt *mpi3mr_dequeue_fwevt(
 272	struct mpi3mr_ioc *mrioc)
 273{
 274	unsigned long flags;
 275	struct mpi3mr_fwevt *fwevt = NULL;
 276
 277	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
 278	if (!list_empty(&mrioc->fwevt_list)) {
 279		fwevt = list_first_entry(&mrioc->fwevt_list,
 280		    struct mpi3mr_fwevt, list);
 281		list_del_init(&fwevt->list);
 282		/*
 283		 * Put fwevt reference count after
 284		 * removing it from fwevt_list
 285		 */
 286		mpi3mr_fwevt_put(fwevt);
 287	}
 288	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
 289
 290	return fwevt;
 291}
 292
 293/**
 294 * mpi3mr_cancel_work - cancel firmware event
 295 * @fwevt: fwevt object which needs to be canceled
 296 *
 297 * Return: Nothing.
 298 */
 299static void mpi3mr_cancel_work(struct mpi3mr_fwevt *fwevt)
 300{
 301	/*
 302	 * Wait on the fwevt to complete. If this returns 1, then
 303	 * the event was never executed.
 304	 *
 305	 * If it did execute, we wait for it to finish, and the put will
 306	 * happen from mpi3mr_process_fwevt()
 307	 */
 308	if (cancel_work_sync(&fwevt->work)) {
 309		/*
 310		 * Put fwevt reference count after
 311		 * dequeuing it from worker queue
 312		 */
 313		mpi3mr_fwevt_put(fwevt);
 314		/*
 315		 * Put fwevt reference count to neutralize
 316		 * kref_init increment
 317		 */
 318		mpi3mr_fwevt_put(fwevt);
 319	}
 320}
 321
 322/**
 323 * mpi3mr_cleanup_fwevt_list - Cleanup firmware event list
 324 * @mrioc: Adapter instance reference
 325 *
 326 * Flush all pending firmware events from the firmware event
 327 * list.
 328 *
 329 * Return: Nothing.
 330 */
 331void mpi3mr_cleanup_fwevt_list(struct mpi3mr_ioc *mrioc)
 332{
 333	struct mpi3mr_fwevt *fwevt = NULL;
 334
 335	if ((list_empty(&mrioc->fwevt_list) && !mrioc->current_event) ||
 336	    !mrioc->fwevt_worker_thread)
 337		return;
 338
 339	while ((fwevt = mpi3mr_dequeue_fwevt(mrioc)))
 340		mpi3mr_cancel_work(fwevt);
 341
 342	if (mrioc->current_event) {
 343		fwevt = mrioc->current_event;
 344		/*
 345		 * Don't call cancel_work_sync() API for the
 346		 * fwevt work if the controller reset is
 347		 * get called as part of processing the
 348		 * same fwevt work (or) when worker thread is
 349		 * waiting for device add/remove APIs to complete.
 350		 * Otherwise we will see deadlock.
 351		 */
 352		if (current_work() == &fwevt->work || fwevt->pending_at_sml) {
 353			fwevt->discard = 1;
 354			return;
 355		}
 356
 357		mpi3mr_cancel_work(fwevt);
 358	}
 359}
 360
 361/**
 362 * mpi3mr_queue_qd_reduction_event - Queue TG QD reduction event
 363 * @mrioc: Adapter instance reference
 364 * @tg: Throttle group information pointer
 365 *
 366 * Accessor to queue on synthetically generated driver event to
 367 * the event worker thread, the driver event will be used to
 368 * reduce the QD of all VDs in the TG from the worker thread.
 369 *
 370 * Return: None.
 371 */
 372static void mpi3mr_queue_qd_reduction_event(struct mpi3mr_ioc *mrioc,
 373	struct mpi3mr_throttle_group_info *tg)
 374{
 375	struct mpi3mr_fwevt *fwevt;
 376	u16 sz = sizeof(struct mpi3mr_throttle_group_info *);
 377
 378	/*
 379	 * If the QD reduction event is already queued due to throttle and if
 380	 * the QD is not restored through device info change event
 381	 * then dont queue further reduction events
 382	 */
 383	if (tg->fw_qd != tg->modified_qd)
 384		return;
 385
 386	fwevt = mpi3mr_alloc_fwevt(sz);
 387	if (!fwevt) {
 388		ioc_warn(mrioc, "failed to queue TG QD reduction event\n");
 389		return;
 390	}
 391	*(struct mpi3mr_throttle_group_info **)fwevt->event_data = tg;
 392	fwevt->mrioc = mrioc;
 393	fwevt->event_id = MPI3MR_DRIVER_EVENT_TG_QD_REDUCTION;
 394	fwevt->send_ack = 0;
 395	fwevt->process_evt = 1;
 396	fwevt->evt_ctx = 0;
 397	fwevt->event_data_size = sz;
 398	tg->modified_qd = max_t(u16, (tg->fw_qd * tg->qd_reduction) / 10, 8);
 399
 400	dprint_event_bh(mrioc, "qd reduction event queued for tg_id(%d)\n",
 401	    tg->id);
 402	mpi3mr_fwevt_add_to_list(mrioc, fwevt);
 403}
 404
 405/**
 406 * mpi3mr_invalidate_devhandles -Invalidate device handles
 407 * @mrioc: Adapter instance reference
 408 *
 409 * Invalidate the device handles in the target device structures
 410 * . Called post reset prior to reinitializing the controller.
 411 *
 412 * Return: Nothing.
 413 */
 414void mpi3mr_invalidate_devhandles(struct mpi3mr_ioc *mrioc)
 415{
 416	struct mpi3mr_tgt_dev *tgtdev;
 417	struct mpi3mr_stgt_priv_data *tgt_priv;
 418
 419	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) {
 420		tgtdev->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
 421		if (tgtdev->starget && tgtdev->starget->hostdata) {
 422			tgt_priv = tgtdev->starget->hostdata;
 423			tgt_priv->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
 424			tgt_priv->io_throttle_enabled = 0;
 425			tgt_priv->io_divert = 0;
 426			tgt_priv->throttle_group = NULL;
 427			if (tgtdev->host_exposed)
 428				atomic_set(&tgt_priv->block_io, 1);
 429		}
 430	}
 431}
 432
 433/**
 434 * mpi3mr_print_scmd - print individual SCSI command
 435 * @rq: Block request
 436 * @data: Adapter instance reference
 437 *
 438 * Print the SCSI command details if it is in LLD scope.
 439 *
 440 * Return: true always.
 441 */
 442static bool mpi3mr_print_scmd(struct request *rq, void *data)
 443{
 444	struct mpi3mr_ioc *mrioc = (struct mpi3mr_ioc *)data;
 445	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
 446	struct scmd_priv *priv = NULL;
 447
 448	if (scmd) {
 449		priv = scsi_cmd_priv(scmd);
 450		if (!priv->in_lld_scope)
 451			goto out;
 452
 453		ioc_info(mrioc, "%s :Host Tag = %d, qid = %d\n",
 454		    __func__, priv->host_tag, priv->req_q_idx + 1);
 455		scsi_print_command(scmd);
 456	}
 457
 458out:
 459	return(true);
 460}
 461
 462/**
 463 * mpi3mr_flush_scmd - Flush individual SCSI command
 464 * @rq: Block request
 465 * @data: Adapter instance reference
 466 *
 467 * Return the SCSI command to the upper layers if it is in LLD
 468 * scope.
 469 *
 470 * Return: true always.
 471 */
 472
 473static bool mpi3mr_flush_scmd(struct request *rq, void *data)
 474{
 475	struct mpi3mr_ioc *mrioc = (struct mpi3mr_ioc *)data;
 476	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
 477	struct scmd_priv *priv = NULL;
 478
 479	if (scmd) {
 480		priv = scsi_cmd_priv(scmd);
 481		if (!priv->in_lld_scope)
 482			goto out;
 483
 484		if (priv->meta_sg_valid)
 485			dma_unmap_sg(&mrioc->pdev->dev, scsi_prot_sglist(scmd),
 486			    scsi_prot_sg_count(scmd), scmd->sc_data_direction);
 487		mpi3mr_clear_scmd_priv(mrioc, scmd);
 488		scsi_dma_unmap(scmd);
 489		scmd->result = DID_RESET << 16;
 490		scsi_print_command(scmd);
 491		scsi_done(scmd);
 492		mrioc->flush_io_count++;
 493	}
 494
 495out:
 496	return(true);
 497}
 498
 499/**
 500 * mpi3mr_count_dev_pending - Count commands pending for a lun
 501 * @rq: Block request
 502 * @data: SCSI device reference
 503 *
 504 * This is an iterator function called for each SCSI command in
 505 * a host and if the command is pending in the LLD for the
 506 * specific device(lun) then device specific pending I/O counter
 507 * is updated in the device structure.
 508 *
 509 * Return: true always.
 510 */
 511
 512static bool mpi3mr_count_dev_pending(struct request *rq, void *data)
 513{
 514	struct scsi_device *sdev = (struct scsi_device *)data;
 515	struct mpi3mr_sdev_priv_data *sdev_priv_data = sdev->hostdata;
 516	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
 517	struct scmd_priv *priv;
 518
 519	if (scmd) {
 520		priv = scsi_cmd_priv(scmd);
 521		if (!priv->in_lld_scope)
 522			goto out;
 523		if (scmd->device == sdev)
 524			sdev_priv_data->pend_count++;
 525	}
 526
 527out:
 528	return true;
 529}
 530
 531/**
 532 * mpi3mr_count_tgt_pending - Count commands pending for target
 533 * @rq: Block request
 534 * @data: SCSI target reference
 535 *
 536 * This is an iterator function called for each SCSI command in
 537 * a host and if the command is pending in the LLD for the
 538 * specific target then target specific pending I/O counter is
 539 * updated in the target structure.
 540 *
 541 * Return: true always.
 542 */
 543
 544static bool mpi3mr_count_tgt_pending(struct request *rq, void *data)
 545{
 546	struct scsi_target *starget = (struct scsi_target *)data;
 547	struct mpi3mr_stgt_priv_data *stgt_priv_data = starget->hostdata;
 548	struct scsi_cmnd *scmd = blk_mq_rq_to_pdu(rq);
 549	struct scmd_priv *priv;
 550
 551	if (scmd) {
 552		priv = scsi_cmd_priv(scmd);
 553		if (!priv->in_lld_scope)
 554			goto out;
 555		if (scmd->device && (scsi_target(scmd->device) == starget))
 556			stgt_priv_data->pend_count++;
 557	}
 558
 559out:
 560	return true;
 561}
 562
 563/**
 564 * mpi3mr_flush_host_io -  Flush host I/Os
 565 * @mrioc: Adapter instance reference
 566 *
 567 * Flush all of the pending I/Os by calling
 568 * blk_mq_tagset_busy_iter() for each possible tag. This is
 569 * executed post controller reset
 570 *
 571 * Return: Nothing.
 572 */
 573void mpi3mr_flush_host_io(struct mpi3mr_ioc *mrioc)
 574{
 575	struct Scsi_Host *shost = mrioc->shost;
 576
 577	mrioc->flush_io_count = 0;
 578	ioc_info(mrioc, "%s :Flushing Host I/O cmds post reset\n", __func__);
 579	blk_mq_tagset_busy_iter(&shost->tag_set,
 580	    mpi3mr_flush_scmd, (void *)mrioc);
 581	ioc_info(mrioc, "%s :Flushed %d Host I/O cmds\n", __func__,
 582	    mrioc->flush_io_count);
 583}
 584
 585/**
 586 * mpi3mr_flush_cmds_for_unrecovered_controller - Flush all pending cmds
 587 * @mrioc: Adapter instance reference
 588 *
 589 * This function waits for currently running IO poll threads to
 590 * exit and then flushes all host I/Os and any internal pending
 591 * cmds. This is executed after controller is marked as
 592 * unrecoverable.
 593 *
 594 * Return: Nothing.
 595 */
 596void mpi3mr_flush_cmds_for_unrecovered_controller(struct mpi3mr_ioc *mrioc)
 597{
 598	struct Scsi_Host *shost = mrioc->shost;
 599	int i;
 600
 601	if (!mrioc->unrecoverable)
 602		return;
 603
 604	if (mrioc->op_reply_qinfo) {
 605		for (i = 0; i < mrioc->num_queues; i++) {
 606			while (atomic_read(&mrioc->op_reply_qinfo[i].in_use))
 607				udelay(500);
 608			atomic_set(&mrioc->op_reply_qinfo[i].pend_ios, 0);
 609		}
 610	}
 611	mrioc->flush_io_count = 0;
 612	blk_mq_tagset_busy_iter(&shost->tag_set,
 613	    mpi3mr_flush_scmd, (void *)mrioc);
 614	mpi3mr_flush_delayed_cmd_lists(mrioc);
 615	mpi3mr_flush_drv_cmds(mrioc);
 616}
 617
 618/**
 619 * mpi3mr_alloc_tgtdev - target device allocator
 620 *
 621 * Allocate target device instance and initialize the reference
 622 * count
 623 *
 624 * Return: target device instance.
 625 */
 626static struct mpi3mr_tgt_dev *mpi3mr_alloc_tgtdev(void)
 627{
 628	struct mpi3mr_tgt_dev *tgtdev;
 629
 630	tgtdev = kzalloc(sizeof(*tgtdev), GFP_ATOMIC);
 631	if (!tgtdev)
 632		return NULL;
 633	kref_init(&tgtdev->ref_count);
 634	return tgtdev;
 635}
 636
 637/**
 638 * mpi3mr_tgtdev_add_to_list -Add tgtdevice to the list
 639 * @mrioc: Adapter instance reference
 640 * @tgtdev: Target device
 641 *
 642 * Add the target device to the target device list
 643 *
 644 * Return: Nothing.
 645 */
 646static void mpi3mr_tgtdev_add_to_list(struct mpi3mr_ioc *mrioc,
 647	struct mpi3mr_tgt_dev *tgtdev)
 648{
 649	unsigned long flags;
 650
 651	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
 652	mpi3mr_tgtdev_get(tgtdev);
 653	INIT_LIST_HEAD(&tgtdev->list);
 654	list_add_tail(&tgtdev->list, &mrioc->tgtdev_list);
 655	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 656}
 657
 658/**
 659 * mpi3mr_tgtdev_del_from_list -Delete tgtdevice from the list
 660 * @mrioc: Adapter instance reference
 661 * @tgtdev: Target device
 662 *
 663 * Remove the target device from the target device list
 664 *
 665 * Return: Nothing.
 666 */
 667static void mpi3mr_tgtdev_del_from_list(struct mpi3mr_ioc *mrioc,
 668	struct mpi3mr_tgt_dev *tgtdev)
 669{
 670	unsigned long flags;
 671
 672	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
 673	if (!list_empty(&tgtdev->list)) {
 674		list_del_init(&tgtdev->list);
 675		mpi3mr_tgtdev_put(tgtdev);
 676	}
 677	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 678}
 679
 680/**
 681 * __mpi3mr_get_tgtdev_by_handle -Get tgtdev from device handle
 682 * @mrioc: Adapter instance reference
 683 * @handle: Device handle
 684 *
 685 * Accessor to retrieve target device from the device handle.
 686 * Non Lock version
 687 *
 688 * Return: Target device reference.
 689 */
 690static struct mpi3mr_tgt_dev  *__mpi3mr_get_tgtdev_by_handle(
 691	struct mpi3mr_ioc *mrioc, u16 handle)
 692{
 693	struct mpi3mr_tgt_dev *tgtdev;
 694
 695	assert_spin_locked(&mrioc->tgtdev_lock);
 696	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list)
 697		if (tgtdev->dev_handle == handle)
 698			goto found_tgtdev;
 699	return NULL;
 700
 701found_tgtdev:
 702	mpi3mr_tgtdev_get(tgtdev);
 703	return tgtdev;
 704}
 705
 706/**
 707 * mpi3mr_get_tgtdev_by_handle -Get tgtdev from device handle
 708 * @mrioc: Adapter instance reference
 709 * @handle: Device handle
 710 *
 711 * Accessor to retrieve target device from the device handle.
 712 * Lock version
 713 *
 714 * Return: Target device reference.
 715 */
 716struct mpi3mr_tgt_dev *mpi3mr_get_tgtdev_by_handle(
 717	struct mpi3mr_ioc *mrioc, u16 handle)
 718{
 719	struct mpi3mr_tgt_dev *tgtdev;
 720	unsigned long flags;
 721
 722	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
 723	tgtdev = __mpi3mr_get_tgtdev_by_handle(mrioc, handle);
 724	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 725	return tgtdev;
 726}
 727
 728/**
 729 * __mpi3mr_get_tgtdev_by_perst_id -Get tgtdev from persist ID
 730 * @mrioc: Adapter instance reference
 731 * @persist_id: Persistent ID
 732 *
 733 * Accessor to retrieve target device from the Persistent ID.
 734 * Non Lock version
 735 *
 736 * Return: Target device reference.
 737 */
 738static struct mpi3mr_tgt_dev  *__mpi3mr_get_tgtdev_by_perst_id(
 739	struct mpi3mr_ioc *mrioc, u16 persist_id)
 740{
 741	struct mpi3mr_tgt_dev *tgtdev;
 742
 743	assert_spin_locked(&mrioc->tgtdev_lock);
 744	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list)
 745		if (tgtdev->perst_id == persist_id)
 746			goto found_tgtdev;
 747	return NULL;
 748
 749found_tgtdev:
 750	mpi3mr_tgtdev_get(tgtdev);
 751	return tgtdev;
 752}
 753
 754/**
 755 * mpi3mr_get_tgtdev_by_perst_id -Get tgtdev from persistent ID
 756 * @mrioc: Adapter instance reference
 757 * @persist_id: Persistent ID
 758 *
 759 * Accessor to retrieve target device from the Persistent ID.
 760 * Lock version
 761 *
 762 * Return: Target device reference.
 763 */
 764static struct mpi3mr_tgt_dev *mpi3mr_get_tgtdev_by_perst_id(
 765	struct mpi3mr_ioc *mrioc, u16 persist_id)
 766{
 767	struct mpi3mr_tgt_dev *tgtdev;
 768	unsigned long flags;
 769
 770	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
 771	tgtdev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, persist_id);
 772	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 773	return tgtdev;
 774}
 775
 776/**
 777 * __mpi3mr_get_tgtdev_from_tgtpriv -Get tgtdev from tgt private
 778 * @mrioc: Adapter instance reference
 779 * @tgt_priv: Target private data
 780 *
 781 * Accessor to return target device from the target private
 782 * data. Non Lock version
 783 *
 784 * Return: Target device reference.
 785 */
 786static struct mpi3mr_tgt_dev  *__mpi3mr_get_tgtdev_from_tgtpriv(
 787	struct mpi3mr_ioc *mrioc, struct mpi3mr_stgt_priv_data *tgt_priv)
 788{
 789	struct mpi3mr_tgt_dev *tgtdev;
 790
 791	assert_spin_locked(&mrioc->tgtdev_lock);
 792	tgtdev = tgt_priv->tgt_dev;
 793	if (tgtdev)
 794		mpi3mr_tgtdev_get(tgtdev);
 795	return tgtdev;
 796}
 797
 798/**
 799 * mpi3mr_set_io_divert_for_all_vd_in_tg -set divert for TG VDs
 800 * @mrioc: Adapter instance reference
 801 * @tg: Throttle group information pointer
 802 * @divert_value: 1 or 0
 803 *
 804 * Accessor to set io_divert flag for each device associated
 805 * with the given throttle group with the given value.
 806 *
 807 * Return: None.
 808 */
 809static void mpi3mr_set_io_divert_for_all_vd_in_tg(struct mpi3mr_ioc *mrioc,
 810	struct mpi3mr_throttle_group_info *tg, u8 divert_value)
 811{
 812	unsigned long flags;
 813	struct mpi3mr_tgt_dev *tgtdev;
 814	struct mpi3mr_stgt_priv_data *tgt_priv;
 815
 816	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
 817	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) {
 818		if (tgtdev->starget && tgtdev->starget->hostdata) {
 819			tgt_priv = tgtdev->starget->hostdata;
 820			if (tgt_priv->throttle_group == tg)
 821				tgt_priv->io_divert = divert_value;
 822		}
 823	}
 824	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
 825}
 826
 827/**
 828 * mpi3mr_print_device_event_notice - print notice related to post processing of
 829 *					device event after controller reset.
 830 *
 831 * @mrioc: Adapter instance reference
 832 * @device_add: true for device add event and false for device removal event
 833 *
 834 * Return: None.
 835 */
 836void mpi3mr_print_device_event_notice(struct mpi3mr_ioc *mrioc,
 837	bool device_add)
 838{
 839	ioc_notice(mrioc, "Device %s was in progress before the reset and\n",
 840	    (device_add ? "addition" : "removal"));
 841	ioc_notice(mrioc, "completed after reset, verify whether the exposed devices\n");
 842	ioc_notice(mrioc, "are matched with attached devices for correctness\n");
 843}
 844
 845/**
 846 * mpi3mr_remove_tgtdev_from_host - Remove dev from upper layers
 847 * @mrioc: Adapter instance reference
 848 * @tgtdev: Target device structure
 849 *
 850 * Checks whether the device is exposed to upper layers and if it
 851 * is then remove the device from upper layers by calling
 852 * scsi_remove_target().
 853 *
 854 * Return: 0 on success, non zero on failure.
 855 */
 856void mpi3mr_remove_tgtdev_from_host(struct mpi3mr_ioc *mrioc,
 857	struct mpi3mr_tgt_dev *tgtdev)
 858{
 859	struct mpi3mr_stgt_priv_data *tgt_priv;
 860
 861	ioc_info(mrioc, "%s :Removing handle(0x%04x), wwid(0x%016llx)\n",
 862	    __func__, tgtdev->dev_handle, (unsigned long long)tgtdev->wwid);
 863	if (tgtdev->starget && tgtdev->starget->hostdata) {
 864		tgt_priv = tgtdev->starget->hostdata;
 865		atomic_set(&tgt_priv->block_io, 0);
 866		tgt_priv->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
 867	}
 868
 869	if (!mrioc->sas_transport_enabled || (tgtdev->dev_type !=
 870	    MPI3_DEVICE_DEVFORM_SAS_SATA) || tgtdev->non_stl) {
 871		if (tgtdev->starget) {
 872			if (mrioc->current_event)
 873				mrioc->current_event->pending_at_sml = 1;
 874			scsi_remove_target(&tgtdev->starget->dev);
 875			tgtdev->host_exposed = 0;
 876			if (mrioc->current_event) {
 877				mrioc->current_event->pending_at_sml = 0;
 878				if (mrioc->current_event->discard) {
 879					mpi3mr_print_device_event_notice(mrioc,
 880					    false);
 881					return;
 882				}
 883			}
 884		}
 885	} else
 886		mpi3mr_remove_tgtdev_from_sas_transport(mrioc, tgtdev);
 887
 888	ioc_info(mrioc, "%s :Removed handle(0x%04x), wwid(0x%016llx)\n",
 889	    __func__, tgtdev->dev_handle, (unsigned long long)tgtdev->wwid);
 890}
 891
 892/**
 893 * mpi3mr_report_tgtdev_to_host - Expose device to upper layers
 894 * @mrioc: Adapter instance reference
 895 * @perst_id: Persistent ID of the device
 896 *
 897 * Checks whether the device can be exposed to upper layers and
 898 * if it is not then expose the device to upper layers by
 899 * calling scsi_scan_target().
 900 *
 901 * Return: 0 on success, non zero on failure.
 902 */
 903static int mpi3mr_report_tgtdev_to_host(struct mpi3mr_ioc *mrioc,
 904	u16 perst_id)
 905{
 906	int retval = 0;
 907	struct mpi3mr_tgt_dev *tgtdev;
 908
 909	if (mrioc->reset_in_progress)
 910		return -1;
 911
 912	tgtdev = mpi3mr_get_tgtdev_by_perst_id(mrioc, perst_id);
 913	if (!tgtdev) {
 914		retval = -1;
 915		goto out;
 916	}
 917	if (tgtdev->is_hidden || tgtdev->host_exposed) {
 918		retval = -1;
 919		goto out;
 920	}
 921	if (!mrioc->sas_transport_enabled || (tgtdev->dev_type !=
 922	    MPI3_DEVICE_DEVFORM_SAS_SATA) || tgtdev->non_stl){
 923		tgtdev->host_exposed = 1;
 924		if (mrioc->current_event)
 925			mrioc->current_event->pending_at_sml = 1;
 926		scsi_scan_target(&mrioc->shost->shost_gendev,
 927		    mrioc->scsi_device_channel, tgtdev->perst_id,
 928		    SCAN_WILD_CARD, SCSI_SCAN_INITIAL);
 929		if (!tgtdev->starget)
 930			tgtdev->host_exposed = 0;
 931		if (mrioc->current_event) {
 932			mrioc->current_event->pending_at_sml = 0;
 933			if (mrioc->current_event->discard) {
 934				mpi3mr_print_device_event_notice(mrioc, true);
 935				goto out;
 936			}
 937		}
 938	} else
 939		mpi3mr_report_tgtdev_to_sas_transport(mrioc, tgtdev);
 940out:
 941	if (tgtdev)
 942		mpi3mr_tgtdev_put(tgtdev);
 943
 944	return retval;
 945}
 946
 947/**
 948 * mpi3mr_change_queue_depth- Change QD callback handler
 949 * @sdev: SCSI device reference
 950 * @q_depth: Queue depth
 951 *
 952 * Validate and limit QD and call scsi_change_queue_depth.
 953 *
 954 * Return: return value of scsi_change_queue_depth
 955 */
 956static int mpi3mr_change_queue_depth(struct scsi_device *sdev,
 957	int q_depth)
 958{
 959	struct scsi_target *starget = scsi_target(sdev);
 960	struct Scsi_Host *shost = dev_to_shost(&starget->dev);
 961	int retval = 0;
 962
 963	if (!sdev->tagged_supported)
 964		q_depth = 1;
 965	if (q_depth > shost->can_queue)
 966		q_depth = shost->can_queue;
 967	else if (!q_depth)
 968		q_depth = MPI3MR_DEFAULT_SDEV_QD;
 969	retval = scsi_change_queue_depth(sdev, q_depth);
 970	sdev->max_queue_depth = sdev->queue_depth;
 971
 972	return retval;
 973}
 974
 975/**
 976 * mpi3mr_update_sdev - Update SCSI device information
 977 * @sdev: SCSI device reference
 978 * @data: target device reference
 979 *
 980 * This is an iterator function called for each SCSI device in a
 981 * target to update the target specific information into each
 982 * SCSI device.
 983 *
 984 * Return: Nothing.
 985 */
 986static void
 987mpi3mr_update_sdev(struct scsi_device *sdev, void *data)
 988{
 989	struct mpi3mr_tgt_dev *tgtdev;
 990
 991	tgtdev = (struct mpi3mr_tgt_dev *)data;
 992	if (!tgtdev)
 993		return;
 994
 995	mpi3mr_change_queue_depth(sdev, tgtdev->q_depth);
 996	switch (tgtdev->dev_type) {
 997	case MPI3_DEVICE_DEVFORM_PCIE:
 998		/*The block layer hw sector size = 512*/
 999		if ((tgtdev->dev_spec.pcie_inf.dev_info &
1000		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) ==
1001		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE) {
1002			blk_queue_max_hw_sectors(sdev->request_queue,
1003			    tgtdev->dev_spec.pcie_inf.mdts / 512);
1004			if (tgtdev->dev_spec.pcie_inf.pgsz == 0)
1005				blk_queue_virt_boundary(sdev->request_queue,
1006				    ((1 << MPI3MR_DEFAULT_PGSZEXP) - 1));
1007			else
1008				blk_queue_virt_boundary(sdev->request_queue,
1009				    ((1 << tgtdev->dev_spec.pcie_inf.pgsz) - 1));
1010		}
1011		break;
1012	default:
1013		break;
1014	}
1015}
1016
1017/**
1018 * mpi3mr_rfresh_tgtdevs - Refresh target device exposure
1019 * @mrioc: Adapter instance reference
1020 *
1021 * This is executed post controller reset to identify any
1022 * missing devices during reset and remove from the upper layers
1023 * or expose any newly detected device to the upper layers.
1024 *
1025 * Return: Nothing.
1026 */
1027
1028void mpi3mr_rfresh_tgtdevs(struct mpi3mr_ioc *mrioc)
1029{
1030	struct mpi3mr_tgt_dev *tgtdev, *tgtdev_next;
1031
1032	list_for_each_entry_safe(tgtdev, tgtdev_next, &mrioc->tgtdev_list,
1033	    list) {
1034		if (tgtdev->dev_handle == MPI3MR_INVALID_DEV_HANDLE) {
1035			dprint_reset(mrioc, "removing target device with perst_id(%d)\n",
1036			    tgtdev->perst_id);
1037			if (tgtdev->host_exposed)
1038				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1039			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
1040			mpi3mr_tgtdev_put(tgtdev);
1041		}
1042	}
1043
1044	tgtdev = NULL;
1045	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) {
1046		if ((tgtdev->dev_handle != MPI3MR_INVALID_DEV_HANDLE) &&
1047		    !tgtdev->is_hidden && !tgtdev->host_exposed)
1048			mpi3mr_report_tgtdev_to_host(mrioc, tgtdev->perst_id);
1049	}
1050}
1051
1052/**
1053 * mpi3mr_update_tgtdev - DevStatusChange evt bottomhalf
1054 * @mrioc: Adapter instance reference
1055 * @tgtdev: Target device internal structure
1056 * @dev_pg0: New device page0
1057 * @is_added: Flag to indicate the device is just added
1058 *
1059 * Update the information from the device page0 into the driver
1060 * cached target device structure.
1061 *
1062 * Return: Nothing.
1063 */
1064static void mpi3mr_update_tgtdev(struct mpi3mr_ioc *mrioc,
1065	struct mpi3mr_tgt_dev *tgtdev, struct mpi3_device_page0 *dev_pg0,
1066	bool is_added)
1067{
1068	u16 flags = 0;
1069	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
1070	struct mpi3mr_enclosure_node *enclosure_dev = NULL;
1071	u8 prot_mask = 0;
1072
1073	tgtdev->perst_id = le16_to_cpu(dev_pg0->persistent_id);
1074	tgtdev->dev_handle = le16_to_cpu(dev_pg0->dev_handle);
1075	tgtdev->dev_type = dev_pg0->device_form;
1076	tgtdev->io_unit_port = dev_pg0->io_unit_port;
1077	tgtdev->encl_handle = le16_to_cpu(dev_pg0->enclosure_handle);
1078	tgtdev->parent_handle = le16_to_cpu(dev_pg0->parent_dev_handle);
1079	tgtdev->slot = le16_to_cpu(dev_pg0->slot);
1080	tgtdev->q_depth = le16_to_cpu(dev_pg0->queue_depth);
1081	tgtdev->wwid = le64_to_cpu(dev_pg0->wwid);
1082	tgtdev->devpg0_flag = le16_to_cpu(dev_pg0->flags);
1083
1084	if (tgtdev->encl_handle)
1085		enclosure_dev = mpi3mr_enclosure_find_by_handle(mrioc,
1086		    tgtdev->encl_handle);
1087	if (enclosure_dev)
1088		tgtdev->enclosure_logical_id = le64_to_cpu(
1089		    enclosure_dev->pg0.enclosure_logical_id);
1090
1091	flags = tgtdev->devpg0_flag;
1092
1093	tgtdev->is_hidden = (flags & MPI3_DEVICE0_FLAGS_HIDDEN);
1094
1095	if (is_added == true)
1096		tgtdev->io_throttle_enabled =
1097		    (flags & MPI3_DEVICE0_FLAGS_IO_THROTTLING_REQUIRED) ? 1 : 0;
1098
1099
1100	if (tgtdev->starget && tgtdev->starget->hostdata) {
1101		scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
1102		    tgtdev->starget->hostdata;
1103		scsi_tgt_priv_data->perst_id = tgtdev->perst_id;
1104		scsi_tgt_priv_data->dev_handle = tgtdev->dev_handle;
1105		scsi_tgt_priv_data->dev_type = tgtdev->dev_type;
1106		scsi_tgt_priv_data->io_throttle_enabled =
1107		    tgtdev->io_throttle_enabled;
1108		if (is_added == true)
1109			atomic_set(&scsi_tgt_priv_data->block_io, 0);
1110	}
1111
1112	switch (dev_pg0->access_status) {
1113	case MPI3_DEVICE0_ASTATUS_NO_ERRORS:
1114	case MPI3_DEVICE0_ASTATUS_PREPARE:
1115	case MPI3_DEVICE0_ASTATUS_NEEDS_INITIALIZATION:
1116	case MPI3_DEVICE0_ASTATUS_DEVICE_MISSING_DELAY:
1117		break;
1118	default:
1119		tgtdev->is_hidden = 1;
1120		break;
1121	}
1122
1123	switch (tgtdev->dev_type) {
1124	case MPI3_DEVICE_DEVFORM_SAS_SATA:
1125	{
1126		struct mpi3_device0_sas_sata_format *sasinf =
1127		    &dev_pg0->device_specific.sas_sata_format;
1128		u16 dev_info = le16_to_cpu(sasinf->device_info);
1129
1130		tgtdev->dev_spec.sas_sata_inf.dev_info = dev_info;
1131		tgtdev->dev_spec.sas_sata_inf.sas_address =
1132		    le64_to_cpu(sasinf->sas_address);
1133		tgtdev->dev_spec.sas_sata_inf.phy_id = sasinf->phy_num;
1134		tgtdev->dev_spec.sas_sata_inf.attached_phy_id =
1135		    sasinf->attached_phy_identifier;
1136		if ((dev_info & MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_MASK) !=
1137		    MPI3_SAS_DEVICE_INFO_DEVICE_TYPE_END_DEVICE)
1138			tgtdev->is_hidden = 1;
1139		else if (!(dev_info & (MPI3_SAS_DEVICE_INFO_STP_SATA_TARGET |
1140		    MPI3_SAS_DEVICE_INFO_SSP_TARGET)))
1141			tgtdev->is_hidden = 1;
1142
1143		if (((tgtdev->devpg0_flag &
1144		    MPI3_DEVICE0_FLAGS_ATT_METHOD_DIR_ATTACHED)
1145		    && (tgtdev->devpg0_flag &
1146		    MPI3_DEVICE0_FLAGS_ATT_METHOD_VIRTUAL)) ||
1147		    (tgtdev->parent_handle == 0xFFFF))
1148			tgtdev->non_stl = 1;
1149		if (tgtdev->dev_spec.sas_sata_inf.hba_port)
1150			tgtdev->dev_spec.sas_sata_inf.hba_port->port_id =
1151			    dev_pg0->io_unit_port;
1152		break;
1153	}
1154	case MPI3_DEVICE_DEVFORM_PCIE:
1155	{
1156		struct mpi3_device0_pcie_format *pcieinf =
1157		    &dev_pg0->device_specific.pcie_format;
1158		u16 dev_info = le16_to_cpu(pcieinf->device_info);
1159
1160		tgtdev->dev_spec.pcie_inf.dev_info = dev_info;
1161		tgtdev->dev_spec.pcie_inf.capb =
1162		    le32_to_cpu(pcieinf->capabilities);
1163		tgtdev->dev_spec.pcie_inf.mdts = MPI3MR_DEFAULT_MDTS;
1164		/* 2^12 = 4096 */
1165		tgtdev->dev_spec.pcie_inf.pgsz = 12;
1166		if (dev_pg0->access_status == MPI3_DEVICE0_ASTATUS_NO_ERRORS) {
1167			tgtdev->dev_spec.pcie_inf.mdts =
1168			    le32_to_cpu(pcieinf->maximum_data_transfer_size);
1169			tgtdev->dev_spec.pcie_inf.pgsz = pcieinf->page_size;
1170			tgtdev->dev_spec.pcie_inf.reset_to =
1171			    max_t(u8, pcieinf->controller_reset_to,
1172			     MPI3MR_INTADMCMD_TIMEOUT);
1173			tgtdev->dev_spec.pcie_inf.abort_to =
1174			    max_t(u8, pcieinf->nvme_abort_to,
1175			    MPI3MR_INTADMCMD_TIMEOUT);
1176		}
1177		if (tgtdev->dev_spec.pcie_inf.mdts > (1024 * 1024))
1178			tgtdev->dev_spec.pcie_inf.mdts = (1024 * 1024);
1179		if (((dev_info & MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) !=
1180		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE) &&
1181		    ((dev_info & MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) !=
1182		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_SCSI_DEVICE))
1183			tgtdev->is_hidden = 1;
1184		tgtdev->non_stl = 1;
1185		if (!mrioc->shost)
1186			break;
1187		prot_mask = scsi_host_get_prot(mrioc->shost);
1188		if (prot_mask & SHOST_DIX_TYPE0_PROTECTION) {
1189			scsi_host_set_prot(mrioc->shost, prot_mask & 0x77);
1190			ioc_info(mrioc,
1191			    "%s : Disabling DIX0 prot capability\n", __func__);
1192			ioc_info(mrioc,
1193			    "because HBA does not support DIX0 operation on NVME drives\n");
1194		}
1195		break;
1196	}
1197	case MPI3_DEVICE_DEVFORM_VD:
1198	{
1199		struct mpi3_device0_vd_format *vdinf =
1200		    &dev_pg0->device_specific.vd_format;
1201		struct mpi3mr_throttle_group_info *tg = NULL;
1202		u16 vdinf_io_throttle_group =
1203		    le16_to_cpu(vdinf->io_throttle_group);
1204
1205		tgtdev->dev_spec.vd_inf.state = vdinf->vd_state;
1206		if (vdinf->vd_state == MPI3_DEVICE0_VD_STATE_OFFLINE)
1207			tgtdev->is_hidden = 1;
1208		tgtdev->non_stl = 1;
1209		tgtdev->dev_spec.vd_inf.tg_id = vdinf_io_throttle_group;
1210		tgtdev->dev_spec.vd_inf.tg_high =
1211		    le16_to_cpu(vdinf->io_throttle_group_high) * 2048;
1212		tgtdev->dev_spec.vd_inf.tg_low =
1213		    le16_to_cpu(vdinf->io_throttle_group_low) * 2048;
1214		if (vdinf_io_throttle_group < mrioc->num_io_throttle_group) {
1215			tg = mrioc->throttle_groups + vdinf_io_throttle_group;
1216			tg->id = vdinf_io_throttle_group;
1217			tg->high = tgtdev->dev_spec.vd_inf.tg_high;
1218			tg->low = tgtdev->dev_spec.vd_inf.tg_low;
1219			tg->qd_reduction =
1220			    tgtdev->dev_spec.vd_inf.tg_qd_reduction;
1221			if (is_added == true)
1222				tg->fw_qd = tgtdev->q_depth;
1223			tg->modified_qd = tgtdev->q_depth;
1224		}
1225		tgtdev->dev_spec.vd_inf.tg = tg;
1226		if (scsi_tgt_priv_data)
1227			scsi_tgt_priv_data->throttle_group = tg;
1228		break;
1229	}
1230	default:
1231		break;
1232	}
1233}
1234
1235/**
1236 * mpi3mr_devstatuschg_evt_bh - DevStatusChange evt bottomhalf
1237 * @mrioc: Adapter instance reference
1238 * @fwevt: Firmware event information.
1239 *
1240 * Process Device status Change event and based on device's new
1241 * information, either expose the device to the upper layers, or
1242 * remove the device from upper layers.
1243 *
1244 * Return: Nothing.
1245 */
1246static void mpi3mr_devstatuschg_evt_bh(struct mpi3mr_ioc *mrioc,
1247	struct mpi3mr_fwevt *fwevt)
1248{
1249	u16 dev_handle = 0;
1250	u8 uhide = 0, delete = 0, cleanup = 0;
1251	struct mpi3mr_tgt_dev *tgtdev = NULL;
1252	struct mpi3_event_data_device_status_change *evtdata =
1253	    (struct mpi3_event_data_device_status_change *)fwevt->event_data;
1254
1255	dev_handle = le16_to_cpu(evtdata->dev_handle);
1256	ioc_info(mrioc,
1257	    "%s :device status change: handle(0x%04x): reason code(0x%x)\n",
1258	    __func__, dev_handle, evtdata->reason_code);
1259	switch (evtdata->reason_code) {
1260	case MPI3_EVENT_DEV_STAT_RC_HIDDEN:
1261		delete = 1;
1262		break;
1263	case MPI3_EVENT_DEV_STAT_RC_NOT_HIDDEN:
1264		uhide = 1;
1265		break;
1266	case MPI3_EVENT_DEV_STAT_RC_VD_NOT_RESPONDING:
1267		delete = 1;
1268		cleanup = 1;
1269		break;
1270	default:
1271		ioc_info(mrioc, "%s :Unhandled reason code(0x%x)\n", __func__,
1272		    evtdata->reason_code);
1273		break;
1274	}
1275
1276	tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle);
1277	if (!tgtdev)
1278		goto out;
1279	if (uhide) {
1280		tgtdev->is_hidden = 0;
1281		if (!tgtdev->host_exposed)
1282			mpi3mr_report_tgtdev_to_host(mrioc, tgtdev->perst_id);
1283	}
1284	if (tgtdev->starget && tgtdev->starget->hostdata) {
1285		if (delete)
1286			mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1287	}
1288	if (cleanup) {
1289		mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
1290		mpi3mr_tgtdev_put(tgtdev);
1291	}
1292
1293out:
1294	if (tgtdev)
1295		mpi3mr_tgtdev_put(tgtdev);
1296}
1297
1298/**
1299 * mpi3mr_devinfochg_evt_bh - DeviceInfoChange evt bottomhalf
1300 * @mrioc: Adapter instance reference
1301 * @dev_pg0: New device page0
1302 *
1303 * Process Device Info Change event and based on device's new
1304 * information, either expose the device to the upper layers, or
1305 * remove the device from upper layers or update the details of
1306 * the device.
1307 *
1308 * Return: Nothing.
1309 */
1310static void mpi3mr_devinfochg_evt_bh(struct mpi3mr_ioc *mrioc,
1311	struct mpi3_device_page0 *dev_pg0)
1312{
1313	struct mpi3mr_tgt_dev *tgtdev = NULL;
1314	u16 dev_handle = 0, perst_id = 0;
1315
1316	perst_id = le16_to_cpu(dev_pg0->persistent_id);
1317	dev_handle = le16_to_cpu(dev_pg0->dev_handle);
1318	ioc_info(mrioc,
1319	    "%s :Device info change: handle(0x%04x): persist_id(0x%x)\n",
1320	    __func__, dev_handle, perst_id);
1321	tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle);
1322	if (!tgtdev)
1323		goto out;
1324	mpi3mr_update_tgtdev(mrioc, tgtdev, dev_pg0, false);
1325	if (!tgtdev->is_hidden && !tgtdev->host_exposed)
1326		mpi3mr_report_tgtdev_to_host(mrioc, perst_id);
1327	if (tgtdev->is_hidden && tgtdev->host_exposed)
1328		mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1329	if (!tgtdev->is_hidden && tgtdev->host_exposed && tgtdev->starget)
1330		starget_for_each_device(tgtdev->starget, (void *)tgtdev,
1331		    mpi3mr_update_sdev);
1332out:
1333	if (tgtdev)
1334		mpi3mr_tgtdev_put(tgtdev);
1335}
1336
1337/**
1338 * mpi3mr_free_enclosure_list - release enclosures
1339 * @mrioc: Adapter instance reference
1340 *
1341 * Free memory allocated during encloure add.
1342 *
1343 * Return nothing.
1344 */
1345void mpi3mr_free_enclosure_list(struct mpi3mr_ioc *mrioc)
1346{
1347	struct mpi3mr_enclosure_node *enclosure_dev, *enclosure_dev_next;
1348
1349	list_for_each_entry_safe(enclosure_dev,
1350	    enclosure_dev_next, &mrioc->enclosure_list, list) {
1351		list_del(&enclosure_dev->list);
1352		kfree(enclosure_dev);
1353	}
1354}
1355
1356/**
1357 * mpi3mr_enclosure_find_by_handle - enclosure search by handle
1358 * @mrioc: Adapter instance reference
1359 * @handle: Firmware device handle of the enclosure
1360 *
1361 * This searches for enclosure device based on handle, then returns the
1362 * enclosure object.
1363 *
1364 * Return: Enclosure object reference or NULL
1365 */
1366struct mpi3mr_enclosure_node *mpi3mr_enclosure_find_by_handle(
1367	struct mpi3mr_ioc *mrioc, u16 handle)
1368{
1369	struct mpi3mr_enclosure_node *enclosure_dev, *r = NULL;
1370
1371	list_for_each_entry(enclosure_dev, &mrioc->enclosure_list, list) {
1372		if (le16_to_cpu(enclosure_dev->pg0.enclosure_handle) != handle)
1373			continue;
1374		r = enclosure_dev;
1375		goto out;
1376	}
1377out:
1378	return r;
1379}
1380
1381/**
1382 * mpi3mr_encldev_add_chg_evt_debug - debug for enclosure event
1383 * @mrioc: Adapter instance reference
1384 * @encl_pg0: Enclosure page 0.
1385 * @is_added: Added event or not
1386 *
1387 * Return nothing.
1388 */
1389static void mpi3mr_encldev_add_chg_evt_debug(struct mpi3mr_ioc *mrioc,
1390	struct mpi3_enclosure_page0 *encl_pg0, u8 is_added)
1391{
1392	char *reason_str = NULL;
1393
1394	if (!(mrioc->logging_level & MPI3_DEBUG_EVENT_WORK_TASK))
1395		return;
1396
1397	if (is_added)
1398		reason_str = "enclosure added";
1399	else
1400		reason_str = "enclosure dev status changed";
1401
1402	ioc_info(mrioc,
1403	    "%s: handle(0x%04x), enclosure logical id(0x%016llx)\n",
1404	    reason_str, le16_to_cpu(encl_pg0->enclosure_handle),
1405	    (unsigned long long)le64_to_cpu(encl_pg0->enclosure_logical_id));
1406	ioc_info(mrioc,
1407	    "number of slots(%d), port(%d), flags(0x%04x), present(%d)\n",
1408	    le16_to_cpu(encl_pg0->num_slots), encl_pg0->io_unit_port,
1409	    le16_to_cpu(encl_pg0->flags),
1410	    ((le16_to_cpu(encl_pg0->flags) &
1411	      MPI3_ENCLS0_FLAGS_ENCL_DEV_PRESENT_MASK) >> 4));
1412}
1413
1414/**
1415 * mpi3mr_encldev_add_chg_evt_bh - Enclosure evt bottomhalf
1416 * @mrioc: Adapter instance reference
1417 * @fwevt: Firmware event reference
1418 *
1419 * Prints information about the Enclosure device status or
1420 * Enclosure add events if logging is enabled and add or remove
1421 * the enclosure from the controller's internal list of
1422 * enclosures.
1423 *
1424 * Return: Nothing.
1425 */
1426static void mpi3mr_encldev_add_chg_evt_bh(struct mpi3mr_ioc *mrioc,
1427	struct mpi3mr_fwevt *fwevt)
1428{
1429	struct mpi3mr_enclosure_node *enclosure_dev = NULL;
1430	struct mpi3_enclosure_page0 *encl_pg0;
1431	u16 encl_handle;
1432	u8 added, present;
1433
1434	encl_pg0 = (struct mpi3_enclosure_page0 *) fwevt->event_data;
1435	added = (fwevt->event_id == MPI3_EVENT_ENCL_DEVICE_ADDED) ? 1 : 0;
1436	mpi3mr_encldev_add_chg_evt_debug(mrioc, encl_pg0, added);
1437
1438
1439	encl_handle = le16_to_cpu(encl_pg0->enclosure_handle);
1440	present = ((le16_to_cpu(encl_pg0->flags) &
1441	      MPI3_ENCLS0_FLAGS_ENCL_DEV_PRESENT_MASK) >> 4);
1442
1443	if (encl_handle)
1444		enclosure_dev = mpi3mr_enclosure_find_by_handle(mrioc,
1445		    encl_handle);
1446	if (!enclosure_dev && present) {
1447		enclosure_dev =
1448			kzalloc(sizeof(struct mpi3mr_enclosure_node),
1449			    GFP_KERNEL);
1450		if (!enclosure_dev)
1451			return;
1452		list_add_tail(&enclosure_dev->list,
1453		    &mrioc->enclosure_list);
1454	}
1455	if (enclosure_dev) {
1456		if (!present) {
1457			list_del(&enclosure_dev->list);
1458			kfree(enclosure_dev);
1459		} else
1460			memcpy(&enclosure_dev->pg0, encl_pg0,
1461			    sizeof(enclosure_dev->pg0));
1462
1463	}
1464}
1465
1466/**
1467 * mpi3mr_sastopochg_evt_debug - SASTopoChange details
1468 * @mrioc: Adapter instance reference
1469 * @event_data: SAS topology change list event data
1470 *
1471 * Prints information about the SAS topology change event.
1472 *
1473 * Return: Nothing.
1474 */
1475static void
1476mpi3mr_sastopochg_evt_debug(struct mpi3mr_ioc *mrioc,
1477	struct mpi3_event_data_sas_topology_change_list *event_data)
1478{
1479	int i;
1480	u16 handle;
1481	u8 reason_code, phy_number;
1482	char *status_str = NULL;
1483	u8 link_rate, prev_link_rate;
1484
1485	switch (event_data->exp_status) {
1486	case MPI3_EVENT_SAS_TOPO_ES_NOT_RESPONDING:
1487		status_str = "remove";
1488		break;
1489	case MPI3_EVENT_SAS_TOPO_ES_RESPONDING:
1490		status_str =  "responding";
1491		break;
1492	case MPI3_EVENT_SAS_TOPO_ES_DELAY_NOT_RESPONDING:
1493		status_str = "remove delay";
1494		break;
1495	case MPI3_EVENT_SAS_TOPO_ES_NO_EXPANDER:
1496		status_str = "direct attached";
1497		break;
1498	default:
1499		status_str = "unknown status";
1500		break;
1501	}
1502	ioc_info(mrioc, "%s :sas topology change: (%s)\n",
1503	    __func__, status_str);
1504	ioc_info(mrioc,
1505	    "%s :\texpander_handle(0x%04x), port(%d), enclosure_handle(0x%04x) start_phy(%02d), num_entries(%d)\n",
1506	    __func__, le16_to_cpu(event_data->expander_dev_handle),
1507	    event_data->io_unit_port,
1508	    le16_to_cpu(event_data->enclosure_handle),
1509	    event_data->start_phy_num, event_data->num_entries);
1510	for (i = 0; i < event_data->num_entries; i++) {
1511		handle = le16_to_cpu(event_data->phy_entry[i].attached_dev_handle);
1512		if (!handle)
1513			continue;
1514		phy_number = event_data->start_phy_num + i;
1515		reason_code = event_data->phy_entry[i].status &
1516		    MPI3_EVENT_SAS_TOPO_PHY_RC_MASK;
1517		switch (reason_code) {
1518		case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
1519			status_str = "target remove";
1520			break;
1521		case MPI3_EVENT_SAS_TOPO_PHY_RC_DELAY_NOT_RESPONDING:
1522			status_str = "delay target remove";
1523			break;
1524		case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED:
1525			status_str = "link status change";
1526			break;
1527		case MPI3_EVENT_SAS_TOPO_PHY_RC_NO_CHANGE:
1528			status_str = "link status no change";
1529			break;
1530		case MPI3_EVENT_SAS_TOPO_PHY_RC_RESPONDING:
1531			status_str = "target responding";
1532			break;
1533		default:
1534			status_str = "unknown";
1535			break;
1536		}
1537		link_rate = event_data->phy_entry[i].link_rate >> 4;
1538		prev_link_rate = event_data->phy_entry[i].link_rate & 0xF;
1539		ioc_info(mrioc,
1540		    "%s :\tphy(%02d), attached_handle(0x%04x): %s: link rate: new(0x%02x), old(0x%02x)\n",
1541		    __func__, phy_number, handle, status_str, link_rate,
1542		    prev_link_rate);
1543	}
1544}
1545
1546/**
1547 * mpi3mr_sastopochg_evt_bh - SASTopologyChange evt bottomhalf
1548 * @mrioc: Adapter instance reference
1549 * @fwevt: Firmware event reference
1550 *
1551 * Prints information about the SAS topology change event and
1552 * for "not responding" event code, removes the device from the
1553 * upper layers.
1554 *
1555 * Return: Nothing.
1556 */
1557static void mpi3mr_sastopochg_evt_bh(struct mpi3mr_ioc *mrioc,
1558	struct mpi3mr_fwevt *fwevt)
1559{
1560	struct mpi3_event_data_sas_topology_change_list *event_data =
1561	    (struct mpi3_event_data_sas_topology_change_list *)fwevt->event_data;
1562	int i;
1563	u16 handle;
1564	u8 reason_code;
1565	u64 exp_sas_address = 0, parent_sas_address = 0;
1566	struct mpi3mr_hba_port *hba_port = NULL;
1567	struct mpi3mr_tgt_dev *tgtdev = NULL;
1568	struct mpi3mr_sas_node *sas_expander = NULL;
1569	unsigned long flags;
1570	u8 link_rate, prev_link_rate, parent_phy_number;
1571
1572	mpi3mr_sastopochg_evt_debug(mrioc, event_data);
1573	if (mrioc->sas_transport_enabled) {
1574		hba_port = mpi3mr_get_hba_port_by_id(mrioc,
1575		    event_data->io_unit_port);
1576		if (le16_to_cpu(event_data->expander_dev_handle)) {
1577			spin_lock_irqsave(&mrioc->sas_node_lock, flags);
1578			sas_expander = __mpi3mr_expander_find_by_handle(mrioc,
1579			    le16_to_cpu(event_data->expander_dev_handle));
1580			if (sas_expander) {
1581				exp_sas_address = sas_expander->sas_address;
1582				hba_port = sas_expander->hba_port;
1583			}
1584			spin_unlock_irqrestore(&mrioc->sas_node_lock, flags);
1585			parent_sas_address = exp_sas_address;
1586		} else
1587			parent_sas_address = mrioc->sas_hba.sas_address;
1588	}
1589
1590	for (i = 0; i < event_data->num_entries; i++) {
1591		if (fwevt->discard)
1592			return;
1593		handle = le16_to_cpu(event_data->phy_entry[i].attached_dev_handle);
1594		if (!handle)
1595			continue;
1596		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
1597		if (!tgtdev)
1598			continue;
1599
1600		reason_code = event_data->phy_entry[i].status &
1601		    MPI3_EVENT_SAS_TOPO_PHY_RC_MASK;
1602
1603		switch (reason_code) {
1604		case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
1605			if (tgtdev->host_exposed)
1606				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1607			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
1608			mpi3mr_tgtdev_put(tgtdev);
1609			break;
1610		case MPI3_EVENT_SAS_TOPO_PHY_RC_RESPONDING:
1611		case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED:
1612		case MPI3_EVENT_SAS_TOPO_PHY_RC_NO_CHANGE:
1613		{
1614			if (!mrioc->sas_transport_enabled || tgtdev->non_stl
1615			    || tgtdev->is_hidden)
1616				break;
1617			link_rate = event_data->phy_entry[i].link_rate >> 4;
1618			prev_link_rate = event_data->phy_entry[i].link_rate & 0xF;
1619			if (link_rate == prev_link_rate)
1620				break;
1621			if (!parent_sas_address)
1622				break;
1623			parent_phy_number = event_data->start_phy_num + i;
1624			mpi3mr_update_links(mrioc, parent_sas_address, handle,
1625			    parent_phy_number, link_rate, hba_port);
1626			break;
1627		}
1628		default:
1629			break;
1630		}
1631		if (tgtdev)
1632			mpi3mr_tgtdev_put(tgtdev);
1633	}
1634
1635	if (mrioc->sas_transport_enabled && (event_data->exp_status ==
1636	    MPI3_EVENT_SAS_TOPO_ES_NOT_RESPONDING)) {
1637		if (sas_expander)
1638			mpi3mr_expander_remove(mrioc, exp_sas_address,
1639			    hba_port);
1640	}
1641}
1642
1643/**
1644 * mpi3mr_pcietopochg_evt_debug - PCIeTopoChange details
1645 * @mrioc: Adapter instance reference
1646 * @event_data: PCIe topology change list event data
1647 *
1648 * Prints information about the PCIe topology change event.
1649 *
1650 * Return: Nothing.
1651 */
1652static void
1653mpi3mr_pcietopochg_evt_debug(struct mpi3mr_ioc *mrioc,
1654	struct mpi3_event_data_pcie_topology_change_list *event_data)
1655{
1656	int i;
1657	u16 handle;
1658	u16 reason_code;
1659	u8 port_number;
1660	char *status_str = NULL;
1661	u8 link_rate, prev_link_rate;
1662
1663	switch (event_data->switch_status) {
1664	case MPI3_EVENT_PCIE_TOPO_SS_NOT_RESPONDING:
1665		status_str = "remove";
1666		break;
1667	case MPI3_EVENT_PCIE_TOPO_SS_RESPONDING:
1668		status_str =  "responding";
1669		break;
1670	case MPI3_EVENT_PCIE_TOPO_SS_DELAY_NOT_RESPONDING:
1671		status_str = "remove delay";
1672		break;
1673	case MPI3_EVENT_PCIE_TOPO_SS_NO_PCIE_SWITCH:
1674		status_str = "direct attached";
1675		break;
1676	default:
1677		status_str = "unknown status";
1678		break;
1679	}
1680	ioc_info(mrioc, "%s :pcie topology change: (%s)\n",
1681	    __func__, status_str);
1682	ioc_info(mrioc,
1683	    "%s :\tswitch_handle(0x%04x), enclosure_handle(0x%04x) start_port(%02d), num_entries(%d)\n",
1684	    __func__, le16_to_cpu(event_data->switch_dev_handle),
1685	    le16_to_cpu(event_data->enclosure_handle),
1686	    event_data->start_port_num, event_data->num_entries);
1687	for (i = 0; i < event_data->num_entries; i++) {
1688		handle =
1689		    le16_to_cpu(event_data->port_entry[i].attached_dev_handle);
1690		if (!handle)
1691			continue;
1692		port_number = event_data->start_port_num + i;
1693		reason_code = event_data->port_entry[i].port_status;
1694		switch (reason_code) {
1695		case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
1696			status_str = "target remove";
1697			break;
1698		case MPI3_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
1699			status_str = "delay target remove";
1700			break;
1701		case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
1702			status_str = "link status change";
1703			break;
1704		case MPI3_EVENT_PCIE_TOPO_PS_NO_CHANGE:
1705			status_str = "link status no change";
1706			break;
1707		case MPI3_EVENT_PCIE_TOPO_PS_RESPONDING:
1708			status_str = "target responding";
1709			break;
1710		default:
1711			status_str = "unknown";
1712			break;
1713		}
1714		link_rate = event_data->port_entry[i].current_port_info &
1715		    MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK;
1716		prev_link_rate = event_data->port_entry[i].previous_port_info &
1717		    MPI3_EVENT_PCIE_TOPO_PI_RATE_MASK;
1718		ioc_info(mrioc,
1719		    "%s :\tport(%02d), attached_handle(0x%04x): %s: link rate: new(0x%02x), old(0x%02x)\n",
1720		    __func__, port_number, handle, status_str, link_rate,
1721		    prev_link_rate);
1722	}
1723}
1724
1725/**
1726 * mpi3mr_pcietopochg_evt_bh - PCIeTopologyChange evt bottomhalf
1727 * @mrioc: Adapter instance reference
1728 * @fwevt: Firmware event reference
1729 *
1730 * Prints information about the PCIe topology change event and
1731 * for "not responding" event code, removes the device from the
1732 * upper layers.
1733 *
1734 * Return: Nothing.
1735 */
1736static void mpi3mr_pcietopochg_evt_bh(struct mpi3mr_ioc *mrioc,
1737	struct mpi3mr_fwevt *fwevt)
1738{
1739	struct mpi3_event_data_pcie_topology_change_list *event_data =
1740	    (struct mpi3_event_data_pcie_topology_change_list *)fwevt->event_data;
1741	int i;
1742	u16 handle;
1743	u8 reason_code;
1744	struct mpi3mr_tgt_dev *tgtdev = NULL;
1745
1746	mpi3mr_pcietopochg_evt_debug(mrioc, event_data);
1747
1748	for (i = 0; i < event_data->num_entries; i++) {
1749		if (fwevt->discard)
1750			return;
1751		handle =
1752		    le16_to_cpu(event_data->port_entry[i].attached_dev_handle);
1753		if (!handle)
1754			continue;
1755		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
1756		if (!tgtdev)
1757			continue;
1758
1759		reason_code = event_data->port_entry[i].port_status;
1760
1761		switch (reason_code) {
1762		case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
1763			if (tgtdev->host_exposed)
1764				mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
1765			mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
1766			mpi3mr_tgtdev_put(tgtdev);
1767			break;
1768		default:
1769			break;
1770		}
1771		if (tgtdev)
1772			mpi3mr_tgtdev_put(tgtdev);
1773	}
1774}
1775
1776/**
1777 * mpi3mr_logdata_evt_bh -  Log data event bottomhalf
1778 * @mrioc: Adapter instance reference
1779 * @fwevt: Firmware event reference
1780 *
1781 * Extracts the event data and calls application interfacing
1782 * function to process the event further.
1783 *
1784 * Return: Nothing.
1785 */
1786static void mpi3mr_logdata_evt_bh(struct mpi3mr_ioc *mrioc,
1787	struct mpi3mr_fwevt *fwevt)
1788{
1789	mpi3mr_app_save_logdata(mrioc, fwevt->event_data,
1790	    fwevt->event_data_size);
1791}
1792
1793/**
1794 * mpi3mr_update_sdev_qd - Update SCSI device queue depath
1795 * @sdev: SCSI device reference
1796 * @data: Queue depth reference
1797 *
1798 * This is an iterator function called for each SCSI device in a
1799 * target to update the QD of each SCSI device.
1800 *
1801 * Return: Nothing.
1802 */
1803static void mpi3mr_update_sdev_qd(struct scsi_device *sdev, void *data)
1804{
1805	u16 *q_depth = (u16 *)data;
1806
1807	scsi_change_queue_depth(sdev, (int)*q_depth);
1808	sdev->max_queue_depth = sdev->queue_depth;
1809}
1810
1811/**
1812 * mpi3mr_set_qd_for_all_vd_in_tg -set QD for TG VDs
1813 * @mrioc: Adapter instance reference
1814 * @tg: Throttle group information pointer
1815 *
1816 * Accessor to reduce QD for each device associated with the
1817 * given throttle group.
1818 *
1819 * Return: None.
1820 */
1821static void mpi3mr_set_qd_for_all_vd_in_tg(struct mpi3mr_ioc *mrioc,
1822	struct mpi3mr_throttle_group_info *tg)
1823{
1824	unsigned long flags;
1825	struct mpi3mr_tgt_dev *tgtdev;
1826	struct mpi3mr_stgt_priv_data *tgt_priv;
1827
1828
1829	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
1830	list_for_each_entry(tgtdev, &mrioc->tgtdev_list, list) {
1831		if (tgtdev->starget && tgtdev->starget->hostdata) {
1832			tgt_priv = tgtdev->starget->hostdata;
1833			if (tgt_priv->throttle_group == tg) {
1834				dprint_event_bh(mrioc,
1835				    "updating qd due to throttling for persist_id(%d) original_qd(%d), reduced_qd (%d)\n",
1836				    tgt_priv->perst_id, tgtdev->q_depth,
1837				    tg->modified_qd);
1838				starget_for_each_device(tgtdev->starget,
1839				    (void *)&tg->modified_qd,
1840				    mpi3mr_update_sdev_qd);
1841			}
1842		}
1843	}
1844	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
1845}
1846
1847/**
1848 * mpi3mr_fwevt_bh - Firmware event bottomhalf handler
1849 * @mrioc: Adapter instance reference
1850 * @fwevt: Firmware event reference
1851 *
1852 * Identifies the firmware event and calls corresponding bottomg
1853 * half handler and sends event acknowledgment if required.
1854 *
1855 * Return: Nothing.
1856 */
1857static void mpi3mr_fwevt_bh(struct mpi3mr_ioc *mrioc,
1858	struct mpi3mr_fwevt *fwevt)
1859{
1860	struct mpi3_device_page0 *dev_pg0 = NULL;
1861	u16 perst_id, handle, dev_info;
1862	struct mpi3_device0_sas_sata_format *sasinf = NULL;
1863
1864	mpi3mr_fwevt_del_from_list(mrioc, fwevt);
1865	mrioc->current_event = fwevt;
1866
1867	if (mrioc->stop_drv_processing)
1868		goto out;
1869
1870	if (mrioc->unrecoverable) {
1871		dprint_event_bh(mrioc,
1872		    "ignoring event(0x%02x) in bottom half handler due to unrecoverable controller\n",
1873		    fwevt->event_id);
1874		goto out;
1875	}
1876
1877	if (!fwevt->process_evt)
1878		goto evt_ack;
1879
1880	switch (fwevt->event_id) {
1881	case MPI3_EVENT_DEVICE_ADDED:
1882	{
1883		dev_pg0 = (struct mpi3_device_page0 *)fwevt->event_data;
1884		perst_id = le16_to_cpu(dev_pg0->persistent_id);
1885		handle = le16_to_cpu(dev_pg0->dev_handle);
1886		if (perst_id != MPI3_DEVICE0_PERSISTENTID_INVALID)
1887			mpi3mr_report_tgtdev_to_host(mrioc, perst_id);
1888		else if (mrioc->sas_transport_enabled &&
1889		    (dev_pg0->device_form == MPI3_DEVICE_DEVFORM_SAS_SATA)) {
1890			sasinf = &dev_pg0->device_specific.sas_sata_format;
1891			dev_info = le16_to_cpu(sasinf->device_info);
1892			if (!mrioc->sas_hba.num_phys)
1893				mpi3mr_sas_host_add(mrioc);
1894			else
1895				mpi3mr_sas_host_refresh(mrioc);
1896
1897			if (mpi3mr_is_expander_device(dev_info))
1898				mpi3mr_expander_add(mrioc, handle);
1899		}
1900		break;
1901	}
1902	case MPI3_EVENT_DEVICE_INFO_CHANGED:
1903	{
1904		dev_pg0 = (struct mpi3_device_page0 *)fwevt->event_data;
1905		perst_id = le16_to_cpu(dev_pg0->persistent_id);
1906		if (perst_id != MPI3_DEVICE0_PERSISTENTID_INVALID)
1907			mpi3mr_devinfochg_evt_bh(mrioc, dev_pg0);
1908		break;
1909	}
1910	case MPI3_EVENT_DEVICE_STATUS_CHANGE:
1911	{
1912		mpi3mr_devstatuschg_evt_bh(mrioc, fwevt);
1913		break;
1914	}
1915	case MPI3_EVENT_ENCL_DEVICE_ADDED:
1916	case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
1917	{
1918		mpi3mr_encldev_add_chg_evt_bh(mrioc, fwevt);
1919		break;
1920	}
1921
1922	case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
1923	{
1924		mpi3mr_sastopochg_evt_bh(mrioc, fwevt);
1925		break;
1926	}
1927	case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
1928	{
1929		mpi3mr_pcietopochg_evt_bh(mrioc, fwevt);
1930		break;
1931	}
1932	case MPI3_EVENT_LOG_DATA:
1933	{
1934		mpi3mr_logdata_evt_bh(mrioc, fwevt);
1935		break;
1936	}
1937	case MPI3MR_DRIVER_EVENT_TG_QD_REDUCTION:
1938	{
1939		struct mpi3mr_throttle_group_info *tg;
1940
1941		tg = *(struct mpi3mr_throttle_group_info **)fwevt->event_data;
1942		dprint_event_bh(mrioc,
1943		    "qd reduction event processed for tg_id(%d) reduction_needed(%d)\n",
1944		    tg->id, tg->need_qd_reduction);
1945		if (tg->need_qd_reduction) {
1946			mpi3mr_set_qd_for_all_vd_in_tg(mrioc, tg);
1947			tg->need_qd_reduction = 0;
1948		}
1949		break;
1950	}
1951	case MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH:
1952	{
1953		while (mrioc->device_refresh_on)
1954			msleep(500);
1955
1956		dprint_event_bh(mrioc,
1957		    "scan for non responding and newly added devices after soft reset started\n");
1958		if (mrioc->sas_transport_enabled) {
1959			mpi3mr_refresh_sas_ports(mrioc);
1960			mpi3mr_refresh_expanders(mrioc);
1961		}
1962		mpi3mr_rfresh_tgtdevs(mrioc);
1963		ioc_info(mrioc,
1964		    "scan for non responding and newly added devices after soft reset completed\n");
1965		break;
1966	}
1967	default:
1968		break;
1969	}
1970
1971evt_ack:
1972	if (fwevt->send_ack)
1973		mpi3mr_process_event_ack(mrioc, fwevt->event_id,
1974		    fwevt->evt_ctx);
1975out:
1976	/* Put fwevt reference count to neutralize kref_init increment */
1977	mpi3mr_fwevt_put(fwevt);
1978	mrioc->current_event = NULL;
1979}
1980
1981/**
1982 * mpi3mr_fwevt_worker - Firmware event worker
1983 * @work: Work struct containing firmware event
1984 *
1985 * Extracts the firmware event and calls mpi3mr_fwevt_bh.
1986 *
1987 * Return: Nothing.
1988 */
1989static void mpi3mr_fwevt_worker(struct work_struct *work)
1990{
1991	struct mpi3mr_fwevt *fwevt = container_of(work, struct mpi3mr_fwevt,
1992	    work);
1993	mpi3mr_fwevt_bh(fwevt->mrioc, fwevt);
1994	/*
1995	 * Put fwevt reference count after
1996	 * dequeuing it from worker queue
1997	 */
1998	mpi3mr_fwevt_put(fwevt);
1999}
2000
2001/**
2002 * mpi3mr_create_tgtdev - Create and add a target device
2003 * @mrioc: Adapter instance reference
2004 * @dev_pg0: Device Page 0 data
2005 *
2006 * If the device specified by the device page 0 data is not
2007 * present in the driver's internal list, allocate the memory
2008 * for the device, populate the data and add to the list, else
2009 * update the device data.  The key is persistent ID.
2010 *
2011 * Return: 0 on success, -ENOMEM on memory allocation failure
2012 */
2013static int mpi3mr_create_tgtdev(struct mpi3mr_ioc *mrioc,
2014	struct mpi3_device_page0 *dev_pg0)
2015{
2016	int retval = 0;
2017	struct mpi3mr_tgt_dev *tgtdev = NULL;
2018	u16 perst_id = 0;
2019
2020	perst_id = le16_to_cpu(dev_pg0->persistent_id);
2021	if (perst_id == MPI3_DEVICE0_PERSISTENTID_INVALID)
2022		return retval;
2023
2024	tgtdev = mpi3mr_get_tgtdev_by_perst_id(mrioc, perst_id);
2025	if (tgtdev) {
2026		mpi3mr_update_tgtdev(mrioc, tgtdev, dev_pg0, true);
2027		mpi3mr_tgtdev_put(tgtdev);
2028	} else {
2029		tgtdev = mpi3mr_alloc_tgtdev();
2030		if (!tgtdev)
2031			return -ENOMEM;
2032		mpi3mr_update_tgtdev(mrioc, tgtdev, dev_pg0, true);
2033		mpi3mr_tgtdev_add_to_list(mrioc, tgtdev);
2034	}
2035
2036	return retval;
2037}
2038
2039/**
2040 * mpi3mr_flush_delayed_cmd_lists - Flush pending commands
2041 * @mrioc: Adapter instance reference
2042 *
2043 * Flush pending commands in the delayed lists due to a
2044 * controller reset or driver removal as a cleanup.
2045 *
2046 * Return: Nothing
2047 */
2048void mpi3mr_flush_delayed_cmd_lists(struct mpi3mr_ioc *mrioc)
2049{
2050	struct delayed_dev_rmhs_node *_rmhs_node;
2051	struct delayed_evt_ack_node *_evtack_node;
2052
2053	dprint_reset(mrioc, "flushing delayed dev_remove_hs commands\n");
2054	while (!list_empty(&mrioc->delayed_rmhs_list)) {
2055		_rmhs_node = list_entry(mrioc->delayed_rmhs_list.next,
2056		    struct delayed_dev_rmhs_node, list);
2057		list_del(&_rmhs_node->list);
2058		kfree(_rmhs_node);
2059	}
2060	dprint_reset(mrioc, "flushing delayed event ack commands\n");
2061	while (!list_empty(&mrioc->delayed_evtack_cmds_list)) {
2062		_evtack_node = list_entry(mrioc->delayed_evtack_cmds_list.next,
2063		    struct delayed_evt_ack_node, list);
2064		list_del(&_evtack_node->list);
2065		kfree(_evtack_node);
2066	}
2067}
2068
2069/**
2070 * mpi3mr_dev_rmhs_complete_iou - Device removal IOUC completion
2071 * @mrioc: Adapter instance reference
2072 * @drv_cmd: Internal command tracker
2073 *
2074 * Issues a target reset TM to the firmware from the device
2075 * removal TM pend list or retry the removal handshake sequence
2076 * based on the IOU control request IOC status.
2077 *
2078 * Return: Nothing
2079 */
2080static void mpi3mr_dev_rmhs_complete_iou(struct mpi3mr_ioc *mrioc,
2081	struct mpi3mr_drv_cmd *drv_cmd)
2082{
2083	u16 cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
2084	struct delayed_dev_rmhs_node *delayed_dev_rmhs = NULL;
2085
2086	if (drv_cmd->state & MPI3MR_CMD_RESET)
2087		goto clear_drv_cmd;
2088
2089	ioc_info(mrioc,
2090	    "%s :dev_rmhs_iouctrl_complete:handle(0x%04x), ioc_status(0x%04x), loginfo(0x%08x)\n",
2091	    __func__, drv_cmd->dev_handle, drv_cmd->ioc_status,
2092	    drv_cmd->ioc_loginfo);
2093	if (drv_cmd->ioc_status != MPI3_IOCSTATUS_SUCCESS) {
2094		if (drv_cmd->retry_count < MPI3MR_DEV_RMHS_RETRY_COUNT) {
2095			drv_cmd->retry_count++;
2096			ioc_info(mrioc,
2097			    "%s :dev_rmhs_iouctrl_complete: handle(0x%04x)retrying handshake retry=%d\n",
2098			    __func__, drv_cmd->dev_handle,
2099			    drv_cmd->retry_count);
2100			mpi3mr_dev_rmhs_send_tm(mrioc, drv_cmd->dev_handle,
2101			    drv_cmd, drv_cmd->iou_rc);
2102			return;
2103		}
2104		ioc_err(mrioc,
2105		    "%s :dev removal handshake failed after all retries: handle(0x%04x)\n",
2106		    __func__, drv_cmd->dev_handle);
2107	} else {
2108		ioc_info(mrioc,
2109		    "%s :dev removal handshake completed successfully: handle(0x%04x)\n",
2110		    __func__, drv_cmd->dev_handle);
2111		clear_bit(drv_cmd->dev_handle, mrioc->removepend_bitmap);
2112	}
2113
2114	if (!list_empty(&mrioc->delayed_rmhs_list)) {
2115		delayed_dev_rmhs = list_entry(mrioc->delayed_rmhs_list.next,
2116		    struct delayed_dev_rmhs_node, list);
2117		drv_cmd->dev_handle = delayed_dev_rmhs->handle;
2118		drv_cmd->retry_count = 0;
2119		drv_cmd->iou_rc = delayed_dev_rmhs->iou_rc;
2120		ioc_info(mrioc,
2121		    "%s :dev_rmhs_iouctrl_complete: processing delayed TM: handle(0x%04x)\n",
2122		    __func__, drv_cmd->dev_handle);
2123		mpi3mr_dev_rmhs_send_tm(mrioc, drv_cmd->dev_handle, drv_cmd,
2124		    drv_cmd->iou_rc);
2125		list_del(&delayed_dev_rmhs->list);
2126		kfree(delayed_dev_rmhs);
2127		return;
2128	}
2129
2130clear_drv_cmd:
2131	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2132	drv_cmd->callback = NULL;
2133	drv_cmd->retry_count = 0;
2134	drv_cmd->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
2135	clear_bit(cmd_idx, mrioc->devrem_bitmap);
2136}
2137
2138/**
2139 * mpi3mr_dev_rmhs_complete_tm - Device removal TM completion
2140 * @mrioc: Adapter instance reference
2141 * @drv_cmd: Internal command tracker
2142 *
2143 * Issues a target reset TM to the firmware from the device
2144 * removal TM pend list or issue IO unit control request as
2145 * part of device removal or hidden acknowledgment handshake.
2146 *
2147 * Return: Nothing
2148 */
2149static void mpi3mr_dev_rmhs_complete_tm(struct mpi3mr_ioc *mrioc,
2150	struct mpi3mr_drv_cmd *drv_cmd)
2151{
2152	struct mpi3_iounit_control_request iou_ctrl;
2153	u16 cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
2154	struct mpi3_scsi_task_mgmt_reply *tm_reply = NULL;
2155	int retval;
2156
2157	if (drv_cmd->state & MPI3MR_CMD_RESET)
2158		goto clear_drv_cmd;
2159
2160	if (drv_cmd->state & MPI3MR_CMD_REPLY_VALID)
2161		tm_reply = (struct mpi3_scsi_task_mgmt_reply *)drv_cmd->reply;
2162
2163	if (tm_reply)
2164		pr_info(IOCNAME
2165		    "dev_rmhs_tr_complete:handle(0x%04x), ioc_status(0x%04x), loginfo(0x%08x), term_count(%d)\n",
2166		    mrioc->name, drv_cmd->dev_handle, drv_cmd->ioc_status,
2167		    drv_cmd->ioc_loginfo,
2168		    le32_to_cpu(tm_reply->termination_count));
2169
2170	pr_info(IOCNAME "Issuing IOU CTL: handle(0x%04x) dev_rmhs idx(%d)\n",
2171	    mrioc->name, drv_cmd->dev_handle, cmd_idx);
2172
2173	memset(&iou_ctrl, 0, sizeof(iou_ctrl));
2174
2175	drv_cmd->state = MPI3MR_CMD_PENDING;
2176	drv_cmd->is_waiting = 0;
2177	drv_cmd->callback = mpi3mr_dev_rmhs_complete_iou;
2178	iou_ctrl.operation = drv_cmd->iou_rc;
2179	iou_ctrl.param16[0] = cpu_to_le16(drv_cmd->dev_handle);
2180	iou_ctrl.host_tag = cpu_to_le16(drv_cmd->host_tag);
2181	iou_ctrl.function = MPI3_FUNCTION_IO_UNIT_CONTROL;
2182
2183	retval = mpi3mr_admin_request_post(mrioc, &iou_ctrl, sizeof(iou_ctrl),
2184	    1);
2185	if (retval) {
2186		pr_err(IOCNAME "Issue DevRmHsTMIOUCTL: Admin post failed\n",
2187		    mrioc->name);
2188		goto clear_drv_cmd;
2189	}
2190
2191	return;
2192clear_drv_cmd:
2193	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2194	drv_cmd->callback = NULL;
2195	drv_cmd->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
2196	drv_cmd->retry_count = 0;
2197	clear_bit(cmd_idx, mrioc->devrem_bitmap);
2198}
2199
2200/**
2201 * mpi3mr_dev_rmhs_send_tm - Issue TM for device removal
2202 * @mrioc: Adapter instance reference
2203 * @handle: Device handle
2204 * @cmdparam: Internal command tracker
2205 * @iou_rc: IO unit reason code
2206 *
2207 * Issues a target reset TM to the firmware or add it to a pend
2208 * list as part of device removal or hidden acknowledgment
2209 * handshake.
2210 *
2211 * Return: Nothing
2212 */
2213static void mpi3mr_dev_rmhs_send_tm(struct mpi3mr_ioc *mrioc, u16 handle,
2214	struct mpi3mr_drv_cmd *cmdparam, u8 iou_rc)
2215{
2216	struct mpi3_scsi_task_mgmt_request tm_req;
2217	int retval = 0;
2218	u16 cmd_idx = MPI3MR_NUM_DEVRMCMD;
2219	u8 retrycount = 5;
2220	struct mpi3mr_drv_cmd *drv_cmd = cmdparam;
2221	struct delayed_dev_rmhs_node *delayed_dev_rmhs = NULL;
2222
2223	if (drv_cmd)
2224		goto issue_cmd;
2225	do {
2226		cmd_idx = find_first_zero_bit(mrioc->devrem_bitmap,
2227		    MPI3MR_NUM_DEVRMCMD);
2228		if (cmd_idx < MPI3MR_NUM_DEVRMCMD) {
2229			if (!test_and_set_bit(cmd_idx, mrioc->devrem_bitmap))
2230				break;
2231			cmd_idx = MPI3MR_NUM_DEVRMCMD;
2232		}
2233	} while (retrycount--);
2234
2235	if (cmd_idx >= MPI3MR_NUM_DEVRMCMD) {
2236		delayed_dev_rmhs = kzalloc(sizeof(*delayed_dev_rmhs),
2237		    GFP_ATOMIC);
2238		if (!delayed_dev_rmhs)
2239			return;
2240		INIT_LIST_HEAD(&delayed_dev_rmhs->list);
2241		delayed_dev_rmhs->handle = handle;
2242		delayed_dev_rmhs->iou_rc = iou_rc;
2243		list_add_tail(&delayed_dev_rmhs->list,
2244		    &mrioc->delayed_rmhs_list);
2245		ioc_info(mrioc, "%s :DevRmHs: tr:handle(0x%04x) is postponed\n",
2246		    __func__, handle);
2247		return;
2248	}
2249	drv_cmd = &mrioc->dev_rmhs_cmds[cmd_idx];
2250
2251issue_cmd:
2252	cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_DEVRMCMD_MIN;
2253	ioc_info(mrioc,
2254	    "%s :Issuing TR TM: for devhandle 0x%04x with dev_rmhs %d\n",
2255	    __func__, handle, cmd_idx);
2256
2257	memset(&tm_req, 0, sizeof(tm_req));
2258	if (drv_cmd->state & MPI3MR_CMD_PENDING) {
2259		ioc_err(mrioc, "%s :Issue TM: Command is in use\n", __func__);
2260		goto out;
2261	}
2262	drv_cmd->state = MPI3MR_CMD_PENDING;
2263	drv_cmd->is_waiting = 0;
2264	drv_cmd->callback = mpi3mr_dev_rmhs_complete_tm;
2265	drv_cmd->dev_handle = handle;
2266	drv_cmd->iou_rc = iou_rc;
2267	tm_req.dev_handle = cpu_to_le16(handle);
2268	tm_req.task_type = MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET;
2269	tm_req.host_tag = cpu_to_le16(drv_cmd->host_tag);
2270	tm_req.task_host_tag = cpu_to_le16(MPI3MR_HOSTTAG_INVALID);
2271	tm_req.function = MPI3_FUNCTION_SCSI_TASK_MGMT;
2272
2273	set_bit(handle, mrioc->removepend_bitmap);
2274	retval = mpi3mr_admin_request_post(mrioc, &tm_req, sizeof(tm_req), 1);
2275	if (retval) {
2276		ioc_err(mrioc, "%s :Issue DevRmHsTM: Admin Post failed\n",
2277		    __func__);
2278		goto out_failed;
2279	}
2280out:
2281	return;
2282out_failed:
2283	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2284	drv_cmd->callback = NULL;
2285	drv_cmd->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
2286	drv_cmd->retry_count = 0;
2287	clear_bit(cmd_idx, mrioc->devrem_bitmap);
2288}
2289
2290/**
2291 * mpi3mr_complete_evt_ack - event ack request completion
2292 * @mrioc: Adapter instance reference
2293 * @drv_cmd: Internal command tracker
2294 *
2295 * This is the completion handler for non blocking event
2296 * acknowledgment sent to the firmware and this will issue any
2297 * pending event acknowledgment request.
2298 *
2299 * Return: Nothing
2300 */
2301static void mpi3mr_complete_evt_ack(struct mpi3mr_ioc *mrioc,
2302	struct mpi3mr_drv_cmd *drv_cmd)
2303{
2304	u16 cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
2305	struct delayed_evt_ack_node *delayed_evtack = NULL;
2306
2307	if (drv_cmd->state & MPI3MR_CMD_RESET)
2308		goto clear_drv_cmd;
2309
2310	if (drv_cmd->ioc_status != MPI3_IOCSTATUS_SUCCESS) {
2311		dprint_event_th(mrioc,
2312		    "immediate event ack failed with ioc_status(0x%04x) log_info(0x%08x)\n",
2313		    (drv_cmd->ioc_status & MPI3_IOCSTATUS_STATUS_MASK),
2314		    drv_cmd->ioc_loginfo);
2315	}
2316
2317	if (!list_empty(&mrioc->delayed_evtack_cmds_list)) {
2318		delayed_evtack =
2319			list_entry(mrioc->delayed_evtack_cmds_list.next,
2320			    struct delayed_evt_ack_node, list);
2321		mpi3mr_send_event_ack(mrioc, delayed_evtack->event, drv_cmd,
2322		    delayed_evtack->event_ctx);
2323		list_del(&delayed_evtack->list);
2324		kfree(delayed_evtack);
2325		return;
2326	}
2327clear_drv_cmd:
2328	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2329	drv_cmd->callback = NULL;
2330	clear_bit(cmd_idx, mrioc->evtack_cmds_bitmap);
2331}
2332
2333/**
2334 * mpi3mr_send_event_ack - Issue event acknwoledgment request
2335 * @mrioc: Adapter instance reference
2336 * @event: MPI3 event id
2337 * @cmdparam: Internal command tracker
2338 * @event_ctx: event context
2339 *
2340 * Issues event acknowledgment request to the firmware if there
2341 * is a free command to send the event ack else it to a pend
2342 * list so that it will be processed on a completion of a prior
2343 * event acknowledgment .
2344 *
2345 * Return: Nothing
2346 */
2347static void mpi3mr_send_event_ack(struct mpi3mr_ioc *mrioc, u8 event,
2348	struct mpi3mr_drv_cmd *cmdparam, u32 event_ctx)
2349{
2350	struct mpi3_event_ack_request evtack_req;
2351	int retval = 0;
2352	u8 retrycount = 5;
2353	u16 cmd_idx = MPI3MR_NUM_EVTACKCMD;
2354	struct mpi3mr_drv_cmd *drv_cmd = cmdparam;
2355	struct delayed_evt_ack_node *delayed_evtack = NULL;
2356
2357	if (drv_cmd) {
2358		dprint_event_th(mrioc,
2359		    "sending delayed event ack in the top half for event(0x%02x), event_ctx(0x%08x)\n",
2360		    event, event_ctx);
2361		goto issue_cmd;
2362	}
2363	dprint_event_th(mrioc,
2364	    "sending event ack in the top half for event(0x%02x), event_ctx(0x%08x)\n",
2365	    event, event_ctx);
2366	do {
2367		cmd_idx = find_first_zero_bit(mrioc->evtack_cmds_bitmap,
2368		    MPI3MR_NUM_EVTACKCMD);
2369		if (cmd_idx < MPI3MR_NUM_EVTACKCMD) {
2370			if (!test_and_set_bit(cmd_idx,
2371			    mrioc->evtack_cmds_bitmap))
2372				break;
2373			cmd_idx = MPI3MR_NUM_EVTACKCMD;
2374		}
2375	} while (retrycount--);
2376
2377	if (cmd_idx >= MPI3MR_NUM_EVTACKCMD) {
2378		delayed_evtack = kzalloc(sizeof(*delayed_evtack),
2379		    GFP_ATOMIC);
2380		if (!delayed_evtack)
2381			return;
2382		INIT_LIST_HEAD(&delayed_evtack->list);
2383		delayed_evtack->event = event;
2384		delayed_evtack->event_ctx = event_ctx;
2385		list_add_tail(&delayed_evtack->list,
2386		    &mrioc->delayed_evtack_cmds_list);
2387		dprint_event_th(mrioc,
2388		    "event ack in the top half for event(0x%02x), event_ctx(0x%08x) is postponed\n",
2389		    event, event_ctx);
2390		return;
2391	}
2392	drv_cmd = &mrioc->evtack_cmds[cmd_idx];
2393
2394issue_cmd:
2395	cmd_idx = drv_cmd->host_tag - MPI3MR_HOSTTAG_EVTACKCMD_MIN;
2396
2397	memset(&evtack_req, 0, sizeof(evtack_req));
2398	if (drv_cmd->state & MPI3MR_CMD_PENDING) {
2399		dprint_event_th(mrioc,
2400		    "sending event ack failed due to command in use\n");
2401		goto out;
2402	}
2403	drv_cmd->state = MPI3MR_CMD_PENDING;
2404	drv_cmd->is_waiting = 0;
2405	drv_cmd->callback = mpi3mr_complete_evt_ack;
2406	evtack_req.host_tag = cpu_to_le16(drv_cmd->host_tag);
2407	evtack_req.function = MPI3_FUNCTION_EVENT_ACK;
2408	evtack_req.event = event;
2409	evtack_req.event_context = cpu_to_le32(event_ctx);
2410	retval = mpi3mr_admin_request_post(mrioc, &evtack_req,
2411	    sizeof(evtack_req), 1);
2412	if (retval) {
2413		dprint_event_th(mrioc,
2414		    "posting event ack request is failed\n");
2415		goto out_failed;
2416	}
2417
2418	dprint_event_th(mrioc,
2419	    "event ack in the top half for event(0x%02x), event_ctx(0x%08x) is posted\n",
2420	    event, event_ctx);
2421out:
2422	return;
2423out_failed:
2424	drv_cmd->state = MPI3MR_CMD_NOTUSED;
2425	drv_cmd->callback = NULL;
2426	clear_bit(cmd_idx, mrioc->evtack_cmds_bitmap);
2427}
2428
2429/**
2430 * mpi3mr_pcietopochg_evt_th - PCIETopologyChange evt tophalf
2431 * @mrioc: Adapter instance reference
2432 * @event_reply: event data
2433 *
2434 * Checks for the reason code and based on that either block I/O
2435 * to device, or unblock I/O to the device, or start the device
2436 * removal handshake with reason as remove with the firmware for
2437 * PCIe devices.
2438 *
2439 * Return: Nothing
2440 */
2441static void mpi3mr_pcietopochg_evt_th(struct mpi3mr_ioc *mrioc,
2442	struct mpi3_event_notification_reply *event_reply)
2443{
2444	struct mpi3_event_data_pcie_topology_change_list *topo_evt =
2445	    (struct mpi3_event_data_pcie_topology_change_list *)event_reply->event_data;
2446	int i;
2447	u16 handle;
2448	u8 reason_code;
2449	struct mpi3mr_tgt_dev *tgtdev = NULL;
2450	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
2451
2452	for (i = 0; i < topo_evt->num_entries; i++) {
2453		handle = le16_to_cpu(topo_evt->port_entry[i].attached_dev_handle);
2454		if (!handle)
2455			continue;
2456		reason_code = topo_evt->port_entry[i].port_status;
2457		scsi_tgt_priv_data =  NULL;
2458		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
2459		if (tgtdev && tgtdev->starget && tgtdev->starget->hostdata)
2460			scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
2461			    tgtdev->starget->hostdata;
2462		switch (reason_code) {
2463		case MPI3_EVENT_PCIE_TOPO_PS_NOT_RESPONDING:
2464			if (scsi_tgt_priv_data) {
2465				scsi_tgt_priv_data->dev_removed = 1;
2466				scsi_tgt_priv_data->dev_removedelay = 0;
2467				atomic_set(&scsi_tgt_priv_data->block_io, 0);
2468			}
2469			mpi3mr_dev_rmhs_send_tm(mrioc, handle, NULL,
2470			    MPI3_CTRL_OP_REMOVE_DEVICE);
2471			break;
2472		case MPI3_EVENT_PCIE_TOPO_PS_DELAY_NOT_RESPONDING:
2473			if (scsi_tgt_priv_data) {
2474				scsi_tgt_priv_data->dev_removedelay = 1;
2475				atomic_inc(&scsi_tgt_priv_data->block_io);
2476			}
2477			break;
2478		case MPI3_EVENT_PCIE_TOPO_PS_RESPONDING:
2479			if (scsi_tgt_priv_data &&
2480			    scsi_tgt_priv_data->dev_removedelay) {
2481				scsi_tgt_priv_data->dev_removedelay = 0;
2482				atomic_dec_if_positive
2483				    (&scsi_tgt_priv_data->block_io);
2484			}
2485			break;
2486		case MPI3_EVENT_PCIE_TOPO_PS_PORT_CHANGED:
2487		default:
2488			break;
2489		}
2490		if (tgtdev)
2491			mpi3mr_tgtdev_put(tgtdev);
2492	}
2493}
2494
2495/**
2496 * mpi3mr_sastopochg_evt_th - SASTopologyChange evt tophalf
2497 * @mrioc: Adapter instance reference
2498 * @event_reply: event data
2499 *
2500 * Checks for the reason code and based on that either block I/O
2501 * to device, or unblock I/O to the device, or start the device
2502 * removal handshake with reason as remove with the firmware for
2503 * SAS/SATA devices.
2504 *
2505 * Return: Nothing
2506 */
2507static void mpi3mr_sastopochg_evt_th(struct mpi3mr_ioc *mrioc,
2508	struct mpi3_event_notification_reply *event_reply)
2509{
2510	struct mpi3_event_data_sas_topology_change_list *topo_evt =
2511	    (struct mpi3_event_data_sas_topology_change_list *)event_reply->event_data;
2512	int i;
2513	u16 handle;
2514	u8 reason_code;
2515	struct mpi3mr_tgt_dev *tgtdev = NULL;
2516	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
2517
2518	for (i = 0; i < topo_evt->num_entries; i++) {
2519		handle = le16_to_cpu(topo_evt->phy_entry[i].attached_dev_handle);
2520		if (!handle)
2521			continue;
2522		reason_code = topo_evt->phy_entry[i].status &
2523		    MPI3_EVENT_SAS_TOPO_PHY_RC_MASK;
2524		scsi_tgt_priv_data =  NULL;
2525		tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
2526		if (tgtdev && tgtdev->starget && tgtdev->starget->hostdata)
2527			scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
2528			    tgtdev->starget->hostdata;
2529		switch (reason_code) {
2530		case MPI3_EVENT_SAS_TOPO_PHY_RC_TARG_NOT_RESPONDING:
2531			if (scsi_tgt_priv_data) {
2532				scsi_tgt_priv_data->dev_removed = 1;
2533				scsi_tgt_priv_data->dev_removedelay = 0;
2534				atomic_set(&scsi_tgt_priv_data->block_io, 0);
2535			}
2536			mpi3mr_dev_rmhs_send_tm(mrioc, handle, NULL,
2537			    MPI3_CTRL_OP_REMOVE_DEVICE);
2538			break;
2539		case MPI3_EVENT_SAS_TOPO_PHY_RC_DELAY_NOT_RESPONDING:
2540			if (scsi_tgt_priv_data) {
2541				scsi_tgt_priv_data->dev_removedelay = 1;
2542				atomic_inc(&scsi_tgt_priv_data->block_io);
2543			}
2544			break;
2545		case MPI3_EVENT_SAS_TOPO_PHY_RC_RESPONDING:
2546			if (scsi_tgt_priv_data &&
2547			    scsi_tgt_priv_data->dev_removedelay) {
2548				scsi_tgt_priv_data->dev_removedelay = 0;
2549				atomic_dec_if_positive
2550				    (&scsi_tgt_priv_data->block_io);
2551			}
2552			break;
2553		case MPI3_EVENT_SAS_TOPO_PHY_RC_PHY_CHANGED:
2554		default:
2555			break;
2556		}
2557		if (tgtdev)
2558			mpi3mr_tgtdev_put(tgtdev);
2559	}
2560}
2561
2562/**
2563 * mpi3mr_devstatuschg_evt_th - DeviceStatusChange evt tophalf
2564 * @mrioc: Adapter instance reference
2565 * @event_reply: event data
2566 *
2567 * Checks for the reason code and based on that either block I/O
2568 * to device, or unblock I/O to the device, or start the device
2569 * removal handshake with reason as remove/hide acknowledgment
2570 * with the firmware.
2571 *
2572 * Return: Nothing
2573 */
2574static void mpi3mr_devstatuschg_evt_th(struct mpi3mr_ioc *mrioc,
2575	struct mpi3_event_notification_reply *event_reply)
2576{
2577	u16 dev_handle = 0;
2578	u8 ublock = 0, block = 0, hide = 0, delete = 0, remove = 0;
2579	struct mpi3mr_tgt_dev *tgtdev = NULL;
2580	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
2581	struct mpi3_event_data_device_status_change *evtdata =
2582	    (struct mpi3_event_data_device_status_change *)event_reply->event_data;
2583
2584	if (mrioc->stop_drv_processing)
2585		goto out;
2586
2587	dev_handle = le16_to_cpu(evtdata->dev_handle);
2588
2589	switch (evtdata->reason_code) {
2590	case MPI3_EVENT_DEV_STAT_RC_INT_DEVICE_RESET_STRT:
2591	case MPI3_EVENT_DEV_STAT_RC_INT_IT_NEXUS_RESET_STRT:
2592		block = 1;
2593		break;
2594	case MPI3_EVENT_DEV_STAT_RC_HIDDEN:
2595		delete = 1;
2596		hide = 1;
2597		break;
2598	case MPI3_EVENT_DEV_STAT_RC_VD_NOT_RESPONDING:
2599		delete = 1;
2600		remove = 1;
2601		break;
2602	case MPI3_EVENT_DEV_STAT_RC_INT_DEVICE_RESET_CMP:
2603	case MPI3_EVENT_DEV_STAT_RC_INT_IT_NEXUS_RESET_CMP:
2604		ublock = 1;
2605		break;
2606	default:
2607		break;
2608	}
2609
2610	tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, dev_handle);
2611	if (!tgtdev)
2612		goto out;
2613	if (hide)
2614		tgtdev->is_hidden = hide;
2615	if (tgtdev->starget && tgtdev->starget->hostdata) {
2616		scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
2617		    tgtdev->starget->hostdata;
2618		if (block)
2619			atomic_inc(&scsi_tgt_priv_data->block_io);
2620		if (delete)
2621			scsi_tgt_priv_data->dev_removed = 1;
2622		if (ublock)
2623			atomic_dec_if_positive(&scsi_tgt_priv_data->block_io);
2624	}
2625	if (remove)
2626		mpi3mr_dev_rmhs_send_tm(mrioc, dev_handle, NULL,
2627		    MPI3_CTRL_OP_REMOVE_DEVICE);
2628	if (hide)
2629		mpi3mr_dev_rmhs_send_tm(mrioc, dev_handle, NULL,
2630		    MPI3_CTRL_OP_HIDDEN_ACK);
2631
2632out:
2633	if (tgtdev)
2634		mpi3mr_tgtdev_put(tgtdev);
2635}
2636
2637/**
2638 * mpi3mr_preparereset_evt_th - Prepare for reset event tophalf
2639 * @mrioc: Adapter instance reference
2640 * @event_reply: event data
2641 *
2642 * Blocks and unblocks host level I/O based on the reason code
2643 *
2644 * Return: Nothing
2645 */
2646static void mpi3mr_preparereset_evt_th(struct mpi3mr_ioc *mrioc,
2647	struct mpi3_event_notification_reply *event_reply)
2648{
2649	struct mpi3_event_data_prepare_for_reset *evtdata =
2650	    (struct mpi3_event_data_prepare_for_reset *)event_reply->event_data;
2651
2652	if (evtdata->reason_code == MPI3_EVENT_PREPARE_RESET_RC_START) {
2653		dprint_event_th(mrioc,
2654		    "prepare for reset event top half with rc=start\n");
2655		if (mrioc->prepare_for_reset)
2656			return;
2657		mrioc->prepare_for_reset = 1;
2658		mrioc->prepare_for_reset_timeout_counter = 0;
2659	} else if (evtdata->reason_code == MPI3_EVENT_PREPARE_RESET_RC_ABORT) {
2660		dprint_event_th(mrioc,
2661		    "prepare for reset top half with rc=abort\n");
2662		mrioc->prepare_for_reset = 0;
2663		mrioc->prepare_for_reset_timeout_counter = 0;
2664	}
2665	if ((event_reply->msg_flags & MPI3_EVENT_NOTIFY_MSGFLAGS_ACK_MASK)
2666	    == MPI3_EVENT_NOTIFY_MSGFLAGS_ACK_REQUIRED)
2667		mpi3mr_send_event_ack(mrioc, event_reply->event, NULL,
2668		    le32_to_cpu(event_reply->event_context));
2669}
2670
2671/**
2672 * mpi3mr_energypackchg_evt_th - Energy pack change evt tophalf
2673 * @mrioc: Adapter instance reference
2674 * @event_reply: event data
2675 *
2676 * Identifies the new shutdown timeout value and update.
2677 *
2678 * Return: Nothing
2679 */
2680static void mpi3mr_energypackchg_evt_th(struct mpi3mr_ioc *mrioc,
2681	struct mpi3_event_notification_reply *event_reply)
2682{
2683	struct mpi3_event_data_energy_pack_change *evtdata =
2684	    (struct mpi3_event_data_energy_pack_change *)event_reply->event_data;
2685	u16 shutdown_timeout = le16_to_cpu(evtdata->shutdown_timeout);
2686
2687	if (shutdown_timeout <= 0) {
2688		ioc_warn(mrioc,
2689		    "%s :Invalid Shutdown Timeout received = %d\n",
2690		    __func__, shutdown_timeout);
2691		return;
2692	}
2693
2694	ioc_info(mrioc,
2695	    "%s :Previous Shutdown Timeout Value = %d New Shutdown Timeout Value = %d\n",
2696	    __func__, mrioc->facts.shutdown_timeout, shutdown_timeout);
2697	mrioc->facts.shutdown_timeout = shutdown_timeout;
2698}
2699
2700/**
2701 * mpi3mr_cablemgmt_evt_th - Cable management event tophalf
2702 * @mrioc: Adapter instance reference
2703 * @event_reply: event data
2704 *
2705 * Displays Cable manegemt event details.
2706 *
2707 * Return: Nothing
2708 */
2709static void mpi3mr_cablemgmt_evt_th(struct mpi3mr_ioc *mrioc,
2710	struct mpi3_event_notification_reply *event_reply)
2711{
2712	struct mpi3_event_data_cable_management *evtdata =
2713	    (struct mpi3_event_data_cable_management *)event_reply->event_data;
2714
2715	switch (evtdata->status) {
2716	case MPI3_EVENT_CABLE_MGMT_STATUS_INSUFFICIENT_POWER:
2717	{
2718		ioc_info(mrioc, "An active cable with receptacle_id %d cannot be powered.\n"
2719		    "Devices connected to this cable are not detected.\n"
2720		    "This cable requires %d mW of power.\n",
2721		    evtdata->receptacle_id,
2722		    le32_to_cpu(evtdata->active_cable_power_requirement));
2723		break;
2724	}
2725	case MPI3_EVENT_CABLE_MGMT_STATUS_DEGRADED:
2726	{
2727		ioc_info(mrioc, "A cable with receptacle_id %d is not running at optimal speed\n",
2728		    evtdata->receptacle_id);
2729		break;
2730	}
2731	default:
2732		break;
2733	}
2734}
2735
2736/**
2737 * mpi3mr_add_event_wait_for_device_refresh - Add Wait for Device Refresh Event
2738 * @mrioc: Adapter instance reference
2739 *
2740 * Add driver specific event to make sure that the driver won't process the
2741 * events until all the devices are refreshed during soft reset.
2742 *
2743 * Return: Nothing
2744 */
2745void mpi3mr_add_event_wait_for_device_refresh(struct mpi3mr_ioc *mrioc)
2746{
2747	struct mpi3mr_fwevt *fwevt = NULL;
2748
2749	fwevt = mpi3mr_alloc_fwevt(0);
2750	if (!fwevt) {
2751		dprint_event_th(mrioc,
2752		    "failed to schedule bottom half handler for event(0x%02x)\n",
2753		    MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH);
2754		return;
2755	}
2756	fwevt->mrioc = mrioc;
2757	fwevt->event_id = MPI3_EVENT_WAIT_FOR_DEVICES_TO_REFRESH;
2758	fwevt->send_ack = 0;
2759	fwevt->process_evt = 1;
2760	fwevt->evt_ctx = 0;
2761	fwevt->event_data_size = 0;
2762	mpi3mr_fwevt_add_to_list(mrioc, fwevt);
2763}
2764
2765/**
2766 * mpi3mr_os_handle_events - Firmware event handler
2767 * @mrioc: Adapter instance reference
2768 * @event_reply: event data
2769 *
2770 * Identify whteher the event has to handled and acknowledged
2771 * and either process the event in the tophalf and/or schedule a
2772 * bottom half through mpi3mr_fwevt_worker.
2773 *
2774 * Return: Nothing
2775 */
2776void mpi3mr_os_handle_events(struct mpi3mr_ioc *mrioc,
2777	struct mpi3_event_notification_reply *event_reply)
2778{
2779	u16 evt_type, sz;
2780	struct mpi3mr_fwevt *fwevt = NULL;
2781	bool ack_req = 0, process_evt_bh = 0;
2782
2783	if (mrioc->stop_drv_processing)
2784		return;
2785
2786	if ((event_reply->msg_flags & MPI3_EVENT_NOTIFY_MSGFLAGS_ACK_MASK)
2787	    == MPI3_EVENT_NOTIFY_MSGFLAGS_ACK_REQUIRED)
2788		ack_req = 1;
2789
2790	evt_type = event_reply->event;
2791
2792	switch (evt_type) {
2793	case MPI3_EVENT_DEVICE_ADDED:
2794	{
2795		struct mpi3_device_page0 *dev_pg0 =
2796		    (struct mpi3_device_page0 *)event_reply->event_data;
2797		if (mpi3mr_create_tgtdev(mrioc, dev_pg0))
2798			ioc_err(mrioc,
2799			    "%s :Failed to add device in the device add event\n",
2800			    __func__);
2801		else
2802			process_evt_bh = 1;
2803		break;
2804	}
2805	case MPI3_EVENT_DEVICE_STATUS_CHANGE:
2806	{
2807		process_evt_bh = 1;
2808		mpi3mr_devstatuschg_evt_th(mrioc, event_reply);
2809		break;
2810	}
2811	case MPI3_EVENT_SAS_TOPOLOGY_CHANGE_LIST:
2812	{
2813		process_evt_bh = 1;
2814		mpi3mr_sastopochg_evt_th(mrioc, event_reply);
2815		break;
2816	}
2817	case MPI3_EVENT_PCIE_TOPOLOGY_CHANGE_LIST:
2818	{
2819		process_evt_bh = 1;
2820		mpi3mr_pcietopochg_evt_th(mrioc, event_reply);
2821		break;
2822	}
2823	case MPI3_EVENT_PREPARE_FOR_RESET:
2824	{
2825		mpi3mr_preparereset_evt_th(mrioc, event_reply);
2826		ack_req = 0;
2827		break;
2828	}
2829	case MPI3_EVENT_DEVICE_INFO_CHANGED:
2830	case MPI3_EVENT_LOG_DATA:
2831	case MPI3_EVENT_ENCL_DEVICE_STATUS_CHANGE:
2832	case MPI3_EVENT_ENCL_DEVICE_ADDED:
2833	{
2834		process_evt_bh = 1;
2835		break;
2836	}
2837	case MPI3_EVENT_ENERGY_PACK_CHANGE:
2838	{
2839		mpi3mr_energypackchg_evt_th(mrioc, event_reply);
2840		break;
2841	}
2842	case MPI3_EVENT_CABLE_MGMT:
2843	{
2844		mpi3mr_cablemgmt_evt_th(mrioc, event_reply);
2845		break;
2846	}
2847	case MPI3_EVENT_SAS_DISCOVERY:
2848	case MPI3_EVENT_SAS_DEVICE_DISCOVERY_ERROR:
2849	case MPI3_EVENT_SAS_BROADCAST_PRIMITIVE:
2850	case MPI3_EVENT_PCIE_ENUMERATION:
2851		break;
2852	default:
2853		ioc_info(mrioc, "%s :event 0x%02x is not handled\n",
2854		    __func__, evt_type);
2855		break;
2856	}
2857	if (process_evt_bh || ack_req) {
2858		sz = event_reply->event_data_length * 4;
2859		fwevt = mpi3mr_alloc_fwevt(sz);
2860		if (!fwevt) {
2861			ioc_info(mrioc, "%s :failure at %s:%d/%s()!\n",
2862			    __func__, __FILE__, __LINE__, __func__);
2863			return;
2864		}
2865
2866		memcpy(fwevt->event_data, event_reply->event_data, sz);
2867		fwevt->mrioc = mrioc;
2868		fwevt->event_id = evt_type;
2869		fwevt->send_ack = ack_req;
2870		fwevt->process_evt = process_evt_bh;
2871		fwevt->evt_ctx = le32_to_cpu(event_reply->event_context);
2872		mpi3mr_fwevt_add_to_list(mrioc, fwevt);
2873	}
2874}
2875
2876/**
2877 * mpi3mr_setup_eedp - Setup EEDP information in MPI3 SCSI IO
2878 * @mrioc: Adapter instance reference
2879 * @scmd: SCSI command reference
2880 * @scsiio_req: MPI3 SCSI IO request
2881 *
2882 * Identifies the protection information flags from the SCSI
2883 * command and set appropriate flags in the MPI3 SCSI IO
2884 * request.
2885 *
2886 * Return: Nothing
2887 */
2888static void mpi3mr_setup_eedp(struct mpi3mr_ioc *mrioc,
2889	struct scsi_cmnd *scmd, struct mpi3_scsi_io_request *scsiio_req)
2890{
2891	u16 eedp_flags = 0;
2892	unsigned char prot_op = scsi_get_prot_op(scmd);
2893
2894	switch (prot_op) {
2895	case SCSI_PROT_NORMAL:
2896		return;
2897	case SCSI_PROT_READ_STRIP:
2898		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK_REMOVE;
2899		break;
2900	case SCSI_PROT_WRITE_INSERT:
2901		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_INSERT;
2902		break;
2903	case SCSI_PROT_READ_INSERT:
2904		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_INSERT;
2905		scsiio_req->msg_flags |= MPI3_SCSIIO_MSGFLAGS_METASGL_VALID;
2906		break;
2907	case SCSI_PROT_WRITE_STRIP:
2908		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK_REMOVE;
2909		scsiio_req->msg_flags |= MPI3_SCSIIO_MSGFLAGS_METASGL_VALID;
2910		break;
2911	case SCSI_PROT_READ_PASS:
2912		eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK;
2913		scsiio_req->msg_flags |= MPI3_SCSIIO_MSGFLAGS_METASGL_VALID;
2914		break;
2915	case SCSI_PROT_WRITE_PASS:
2916		if (scmd->prot_flags & SCSI_PROT_IP_CHECKSUM) {
2917			eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK_REGEN;
2918			scsiio_req->sgl[0].eedp.application_tag_translation_mask =
2919			    0xffff;
2920		} else
2921			eedp_flags = MPI3_EEDPFLAGS_EEDP_OP_CHECK;
2922
2923		scsiio_req->msg_flags |= MPI3_SCSIIO_MSGFLAGS_METASGL_VALID;
2924		break;
2925	default:
2926		return;
2927	}
2928
2929	if (scmd->prot_flags & SCSI_PROT_GUARD_CHECK)
2930		eedp_flags |= MPI3_EEDPFLAGS_CHK_GUARD;
2931
2932	if (scmd->prot_flags & SCSI_PROT_IP_CHECKSUM)
2933		eedp_flags |= MPI3_EEDPFLAGS_HOST_GUARD_IP_CHKSUM;
2934
2935	if (scmd->prot_flags & SCSI_PROT_REF_CHECK) {
2936		eedp_flags |= MPI3_EEDPFLAGS_CHK_REF_TAG |
2937			MPI3_EEDPFLAGS_INCR_PRI_REF_TAG;
2938		scsiio_req->cdb.eedp32.primary_reference_tag =
2939			cpu_to_be32(scsi_prot_ref_tag(scmd));
2940	}
2941
2942	if (scmd->prot_flags & SCSI_PROT_REF_INCREMENT)
2943		eedp_flags |= MPI3_EEDPFLAGS_INCR_PRI_REF_TAG;
2944
2945	eedp_flags |= MPI3_EEDPFLAGS_ESC_MODE_APPTAG_DISABLE;
2946
2947	switch (scsi_prot_interval(scmd)) {
2948	case 512:
2949		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_512;
2950		break;
2951	case 520:
2952		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_520;
2953		break;
2954	case 4080:
2955		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4080;
2956		break;
2957	case 4088:
2958		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4088;
2959		break;
2960	case 4096:
2961		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4096;
2962		break;
2963	case 4104:
2964		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4104;
2965		break;
2966	case 4160:
2967		scsiio_req->sgl[0].eedp.user_data_size = MPI3_EEDP_UDS_4160;
2968		break;
2969	default:
2970		break;
2971	}
2972
2973	scsiio_req->sgl[0].eedp.eedp_flags = cpu_to_le16(eedp_flags);
2974	scsiio_req->sgl[0].eedp.flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_EXTENDED;
2975}
2976
2977/**
2978 * mpi3mr_build_sense_buffer - Map sense information
2979 * @desc: Sense type
2980 * @buf: Sense buffer to populate
2981 * @key: Sense key
2982 * @asc: Additional sense code
2983 * @ascq: Additional sense code qualifier
2984 *
2985 * Maps the given sense information into either descriptor or
2986 * fixed format sense data.
2987 *
2988 * Return: Nothing
2989 */
2990static inline void mpi3mr_build_sense_buffer(int desc, u8 *buf, u8 key,
2991	u8 asc, u8 ascq)
2992{
2993	if (desc) {
2994		buf[0] = 0x72;	/* descriptor, current */
2995		buf[1] = key;
2996		buf[2] = asc;
2997		buf[3] = ascq;
2998		buf[7] = 0;
2999	} else {
3000		buf[0] = 0x70;	/* fixed, current */
3001		buf[2] = key;
3002		buf[7] = 0xa;
3003		buf[12] = asc;
3004		buf[13] = ascq;
3005	}
3006}
3007
3008/**
3009 * mpi3mr_map_eedp_error - Map EEDP errors from IOC status
3010 * @scmd: SCSI command reference
3011 * @ioc_status: status of MPI3 request
3012 *
3013 * Maps the EEDP error status of the SCSI IO request to sense
3014 * data.
3015 *
3016 * Return: Nothing
3017 */
3018static void mpi3mr_map_eedp_error(struct scsi_cmnd *scmd,
3019	u16 ioc_status)
3020{
3021	u8 ascq = 0;
3022
3023	switch (ioc_status) {
3024	case MPI3_IOCSTATUS_EEDP_GUARD_ERROR:
3025		ascq = 0x01;
3026		break;
3027	case MPI3_IOCSTATUS_EEDP_APP_TAG_ERROR:
3028		ascq = 0x02;
3029		break;
3030	case MPI3_IOCSTATUS_EEDP_REF_TAG_ERROR:
3031		ascq = 0x03;
3032		break;
3033	default:
3034		ascq = 0x00;
3035		break;
3036	}
3037
3038	mpi3mr_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
3039	    0x10, ascq);
3040	scmd->result = (DID_ABORT << 16) | SAM_STAT_CHECK_CONDITION;
3041}
3042
3043/**
3044 * mpi3mr_process_op_reply_desc - reply descriptor handler
3045 * @mrioc: Adapter instance reference
3046 * @reply_desc: Operational reply descriptor
3047 * @reply_dma: place holder for reply DMA address
3048 * @qidx: Operational queue index
3049 *
3050 * Process the operational reply descriptor and identifies the
3051 * descriptor type. Based on the descriptor map the MPI3 request
3052 * status to a SCSI command status and calls scsi_done call
3053 * back.
3054 *
3055 * Return: Nothing
3056 */
3057void mpi3mr_process_op_reply_desc(struct mpi3mr_ioc *mrioc,
3058	struct mpi3_default_reply_descriptor *reply_desc, u64 *reply_dma, u16 qidx)
3059{
3060	u16 reply_desc_type, host_tag = 0;
3061	u16 ioc_status = MPI3_IOCSTATUS_SUCCESS;
3062	u32 ioc_loginfo = 0;
3063	struct mpi3_status_reply_descriptor *status_desc = NULL;
3064	struct mpi3_address_reply_descriptor *addr_desc = NULL;
3065	struct mpi3_success_reply_descriptor *success_desc = NULL;
3066	struct mpi3_scsi_io_reply *scsi_reply = NULL;
3067	struct scsi_cmnd *scmd = NULL;
3068	struct scmd_priv *priv = NULL;
3069	u8 *sense_buf = NULL;
3070	u8 scsi_state = 0, scsi_status = 0, sense_state = 0;
3071	u32 xfer_count = 0, sense_count = 0, resp_data = 0;
3072	u16 dev_handle = 0xFFFF;
3073	struct scsi_sense_hdr sshdr;
3074	struct mpi3mr_stgt_priv_data *stgt_priv_data = NULL;
3075	struct mpi3mr_sdev_priv_data *sdev_priv_data = NULL;
3076	u32 ioc_pend_data_len = 0, tg_pend_data_len = 0, data_len_blks = 0;
3077	struct mpi3mr_throttle_group_info *tg = NULL;
3078	u8 throttle_enabled_dev = 0;
3079
3080	*reply_dma = 0;
3081	reply_desc_type = le16_to_cpu(reply_desc->reply_flags) &
3082	    MPI3_REPLY_DESCRIPT_FLAGS_TYPE_MASK;
3083	switch (reply_desc_type) {
3084	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_STATUS:
3085		status_desc = (struct mpi3_status_reply_descriptor *)reply_desc;
3086		host_tag = le16_to_cpu(status_desc->host_tag);
3087		ioc_status = le16_to_cpu(status_desc->ioc_status);
3088		if (ioc_status &
3089		    MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
3090			ioc_loginfo = le32_to_cpu(status_desc->ioc_log_info);
3091		ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
3092		break;
3093	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_ADDRESS_REPLY:
3094		addr_desc = (struct mpi3_address_reply_descriptor *)reply_desc;
3095		*reply_dma = le64_to_cpu(addr_desc->reply_frame_address);
3096		scsi_reply = mpi3mr_get_reply_virt_addr(mrioc,
3097		    *reply_dma);
3098		if (!scsi_reply) {
3099			panic("%s: scsi_reply is NULL, this shouldn't happen\n",
3100			    mrioc->name);
3101			goto out;
3102		}
3103		host_tag = le16_to_cpu(scsi_reply->host_tag);
3104		ioc_status = le16_to_cpu(scsi_reply->ioc_status);
3105		scsi_status = scsi_reply->scsi_status;
3106		scsi_state = scsi_reply->scsi_state;
3107		dev_handle = le16_to_cpu(scsi_reply->dev_handle);
3108		sense_state = (scsi_state & MPI3_SCSI_STATE_SENSE_MASK);
3109		xfer_count = le32_to_cpu(scsi_reply->transfer_count);
3110		sense_count = le32_to_cpu(scsi_reply->sense_count);
3111		resp_data = le32_to_cpu(scsi_reply->response_data);
3112		sense_buf = mpi3mr_get_sensebuf_virt_addr(mrioc,
3113		    le64_to_cpu(scsi_reply->sense_data_buffer_address));
3114		if (ioc_status &
3115		    MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_LOGINFOAVAIL)
3116			ioc_loginfo = le32_to_cpu(scsi_reply->ioc_log_info);
3117		ioc_status &= MPI3_REPLY_DESCRIPT_STATUS_IOCSTATUS_STATUS_MASK;
3118		if (sense_state == MPI3_SCSI_STATE_SENSE_BUFF_Q_EMPTY)
3119			panic("%s: Ran out of sense buffers\n", mrioc->name);
3120		break;
3121	case MPI3_REPLY_DESCRIPT_FLAGS_TYPE_SUCCESS:
3122		success_desc = (struct mpi3_success_reply_descriptor *)reply_desc;
3123		host_tag = le16_to_cpu(success_desc->host_tag);
3124		break;
3125	default:
3126		break;
3127	}
3128	scmd = mpi3mr_scmd_from_host_tag(mrioc, host_tag, qidx);
3129	if (!scmd) {
3130		panic("%s: Cannot Identify scmd for host_tag 0x%x\n",
3131		    mrioc->name, host_tag);
3132		goto out;
3133	}
3134	priv = scsi_cmd_priv(scmd);
3135
3136	data_len_blks = scsi_bufflen(scmd) >> 9;
3137	sdev_priv_data = scmd->device->hostdata;
3138	if (sdev_priv_data) {
3139		stgt_priv_data = sdev_priv_data->tgt_priv_data;
3140		if (stgt_priv_data) {
3141			tg = stgt_priv_data->throttle_group;
3142			throttle_enabled_dev =
3143			    stgt_priv_data->io_throttle_enabled;
3144		}
3145	}
3146	if (unlikely((data_len_blks >= mrioc->io_throttle_data_length) &&
3147	    throttle_enabled_dev)) {
3148		ioc_pend_data_len = atomic_sub_return(data_len_blks,
3149		    &mrioc->pend_large_data_sz);
3150		if (tg) {
3151			tg_pend_data_len = atomic_sub_return(data_len_blks,
3152			    &tg->pend_large_data_sz);
3153			if (tg->io_divert  && ((ioc_pend_data_len <=
3154			    mrioc->io_throttle_low) &&
3155			    (tg_pend_data_len <= tg->low))) {
3156				tg->io_divert = 0;
3157				mpi3mr_set_io_divert_for_all_vd_in_tg(
3158				    mrioc, tg, 0);
3159			}
3160		} else {
3161			if (ioc_pend_data_len <= mrioc->io_throttle_low)
3162				stgt_priv_data->io_divert = 0;
3163		}
3164	} else if (unlikely((stgt_priv_data && stgt_priv_data->io_divert))) {
3165		ioc_pend_data_len = atomic_read(&mrioc->pend_large_data_sz);
3166		if (!tg) {
3167			if (ioc_pend_data_len <= mrioc->io_throttle_low)
3168				stgt_priv_data->io_divert = 0;
3169
3170		} else if (ioc_pend_data_len <= mrioc->io_throttle_low) {
3171			tg_pend_data_len = atomic_read(&tg->pend_large_data_sz);
3172			if (tg->io_divert  && (tg_pend_data_len <= tg->low)) {
3173				tg->io_divert = 0;
3174				mpi3mr_set_io_divert_for_all_vd_in_tg(
3175				    mrioc, tg, 0);
3176			}
3177		}
3178	}
3179
3180	if (success_desc) {
3181		scmd->result = DID_OK << 16;
3182		goto out_success;
3183	}
3184
3185	scsi_set_resid(scmd, scsi_bufflen(scmd) - xfer_count);
3186	if (ioc_status == MPI3_IOCSTATUS_SCSI_DATA_UNDERRUN &&
3187	    xfer_count == 0 && (scsi_status == MPI3_SCSI_STATUS_BUSY ||
3188	    scsi_status == MPI3_SCSI_STATUS_RESERVATION_CONFLICT ||
3189	    scsi_status == MPI3_SCSI_STATUS_TASK_SET_FULL))
3190		ioc_status = MPI3_IOCSTATUS_SUCCESS;
3191
3192	if ((sense_state == MPI3_SCSI_STATE_SENSE_VALID) && sense_count &&
3193	    sense_buf) {
3194		u32 sz = min_t(u32, SCSI_SENSE_BUFFERSIZE, sense_count);
3195
3196		memcpy(scmd->sense_buffer, sense_buf, sz);
3197	}
3198
3199	switch (ioc_status) {
3200	case MPI3_IOCSTATUS_BUSY:
3201	case MPI3_IOCSTATUS_INSUFFICIENT_RESOURCES:
3202		scmd->result = SAM_STAT_BUSY;
3203		break;
3204	case MPI3_IOCSTATUS_SCSI_DEVICE_NOT_THERE:
3205		scmd->result = DID_NO_CONNECT << 16;
3206		break;
3207	case MPI3_IOCSTATUS_SCSI_IOC_TERMINATED:
3208		scmd->result = DID_SOFT_ERROR << 16;
3209		break;
3210	case MPI3_IOCSTATUS_SCSI_TASK_TERMINATED:
3211	case MPI3_IOCSTATUS_SCSI_EXT_TERMINATED:
3212		scmd->result = DID_RESET << 16;
3213		break;
3214	case MPI3_IOCSTATUS_SCSI_RESIDUAL_MISMATCH:
3215		if ((xfer_count == 0) || (scmd->underflow > xfer_count))
3216			scmd->result = DID_SOFT_ERROR << 16;
3217		else
3218			scmd->result = (DID_OK << 16) | scsi_status;
3219		break;
3220	case MPI3_IOCSTATUS_SCSI_DATA_UNDERRUN:
3221		scmd->result = (DID_OK << 16) | scsi_status;
3222		if (sense_state == MPI3_SCSI_STATE_SENSE_VALID)
3223			break;
3224		if (xfer_count < scmd->underflow) {
3225			if (scsi_status == SAM_STAT_BUSY)
3226				scmd->result = SAM_STAT_BUSY;
3227			else
3228				scmd->result = DID_SOFT_ERROR << 16;
3229		} else if ((scsi_state & (MPI3_SCSI_STATE_NO_SCSI_STATUS)) ||
3230		    (sense_state != MPI3_SCSI_STATE_SENSE_NOT_AVAILABLE))
3231			scmd->result = DID_SOFT_ERROR << 16;
3232		else if (scsi_state & MPI3_SCSI_STATE_TERMINATED)
3233			scmd->result = DID_RESET << 16;
3234		break;
3235	case MPI3_IOCSTATUS_SCSI_DATA_OVERRUN:
3236		scsi_set_resid(scmd, 0);
3237		fallthrough;
3238	case MPI3_IOCSTATUS_SCSI_RECOVERED_ERROR:
3239	case MPI3_IOCSTATUS_SUCCESS:
3240		scmd->result = (DID_OK << 16) | scsi_status;
3241		if ((scsi_state & (MPI3_SCSI_STATE_NO_SCSI_STATUS)) ||
3242		    (sense_state == MPI3_SCSI_STATE_SENSE_FAILED) ||
3243			(sense_state == MPI3_SCSI_STATE_SENSE_BUFF_Q_EMPTY))
3244			scmd->result = DID_SOFT_ERROR << 16;
3245		else if (scsi_state & MPI3_SCSI_STATE_TERMINATED)
3246			scmd->result = DID_RESET << 16;
3247		break;
3248	case MPI3_IOCSTATUS_EEDP_GUARD_ERROR:
3249	case MPI3_IOCSTATUS_EEDP_REF_TAG_ERROR:
3250	case MPI3_IOCSTATUS_EEDP_APP_TAG_ERROR:
3251		mpi3mr_map_eedp_error(scmd, ioc_status);
3252		break;
3253	case MPI3_IOCSTATUS_SCSI_PROTOCOL_ERROR:
3254	case MPI3_IOCSTATUS_INVALID_FUNCTION:
3255	case MPI3_IOCSTATUS_INVALID_SGL:
3256	case MPI3_IOCSTATUS_INTERNAL_ERROR:
3257	case MPI3_IOCSTATUS_INVALID_FIELD:
3258	case MPI3_IOCSTATUS_INVALID_STATE:
3259	case MPI3_IOCSTATUS_SCSI_IO_DATA_ERROR:
3260	case MPI3_IOCSTATUS_SCSI_TASK_MGMT_FAILED:
3261	case MPI3_IOCSTATUS_INSUFFICIENT_POWER:
3262	default:
3263		scmd->result = DID_SOFT_ERROR << 16;
3264		break;
3265	}
3266
3267	if (scmd->result != (DID_OK << 16) && (scmd->cmnd[0] != ATA_12) &&
3268	    (scmd->cmnd[0] != ATA_16) &&
3269	    mrioc->logging_level & MPI3_DEBUG_SCSI_ERROR) {
3270		ioc_info(mrioc, "%s :scmd->result 0x%x\n", __func__,
3271		    scmd->result);
3272		scsi_print_command(scmd);
3273		ioc_info(mrioc,
3274		    "%s :Command issued to handle 0x%02x returned with error 0x%04x loginfo 0x%08x, qid %d\n",
3275		    __func__, dev_handle, ioc_status, ioc_loginfo,
3276		    priv->req_q_idx + 1);
3277		ioc_info(mrioc,
3278		    " host_tag %d scsi_state 0x%02x scsi_status 0x%02x, xfer_cnt %d resp_data 0x%x\n",
3279		    host_tag, scsi_state, scsi_status, xfer_count, resp_data);
3280		if (sense_buf) {
3281			scsi_normalize_sense(sense_buf, sense_count, &sshdr);
3282			ioc_info(mrioc,
3283			    "%s :sense_count 0x%x, sense_key 0x%x ASC 0x%x, ASCQ 0x%x\n",
3284			    __func__, sense_count, sshdr.sense_key,
3285			    sshdr.asc, sshdr.ascq);
3286		}
3287	}
3288out_success:
3289	if (priv->meta_sg_valid) {
3290		dma_unmap_sg(&mrioc->pdev->dev, scsi_prot_sglist(scmd),
3291		    scsi_prot_sg_count(scmd), scmd->sc_data_direction);
3292	}
3293	mpi3mr_clear_scmd_priv(mrioc, scmd);
3294	scsi_dma_unmap(scmd);
3295	scsi_done(scmd);
3296out:
3297	if (sense_buf)
3298		mpi3mr_repost_sense_buf(mrioc,
3299		    le64_to_cpu(scsi_reply->sense_data_buffer_address));
3300}
3301
3302/**
3303 * mpi3mr_get_chain_idx - get free chain buffer index
3304 * @mrioc: Adapter instance reference
3305 *
3306 * Try to get a free chain buffer index from the free pool.
3307 *
3308 * Return: -1 on failure or the free chain buffer index
3309 */
3310static int mpi3mr_get_chain_idx(struct mpi3mr_ioc *mrioc)
3311{
3312	u8 retry_count = 5;
3313	int cmd_idx = -1;
3314
3315	do {
3316		spin_lock(&mrioc->chain_buf_lock);
3317		cmd_idx = find_first_zero_bit(mrioc->chain_bitmap,
3318		    mrioc->chain_buf_count);
3319		if (cmd_idx < mrioc->chain_buf_count) {
3320			set_bit(cmd_idx, mrioc->chain_bitmap);
3321			spin_unlock(&mrioc->chain_buf_lock);
3322			break;
3323		}
3324		spin_unlock(&mrioc->chain_buf_lock);
3325		cmd_idx = -1;
3326	} while (retry_count--);
3327	return cmd_idx;
3328}
3329
3330/**
3331 * mpi3mr_prepare_sg_scmd - build scatter gather list
3332 * @mrioc: Adapter instance reference
3333 * @scmd: SCSI command reference
3334 * @scsiio_req: MPI3 SCSI IO request
3335 *
3336 * This function maps SCSI command's data and protection SGEs to
3337 * MPI request SGEs. If required additional 4K chain buffer is
3338 * used to send the SGEs.
3339 *
3340 * Return: 0 on success, -ENOMEM on dma_map_sg failure
3341 */
3342static int mpi3mr_prepare_sg_scmd(struct mpi3mr_ioc *mrioc,
3343	struct scsi_cmnd *scmd, struct mpi3_scsi_io_request *scsiio_req)
3344{
3345	dma_addr_t chain_dma;
3346	struct scatterlist *sg_scmd;
3347	void *sg_local, *chain;
3348	u32 chain_length;
3349	int sges_left, chain_idx;
3350	u32 sges_in_segment;
3351	u8 simple_sgl_flags;
3352	u8 simple_sgl_flags_last;
3353	u8 last_chain_sgl_flags;
3354	struct chain_element *chain_req;
3355	struct scmd_priv *priv = NULL;
3356	u32 meta_sg = le32_to_cpu(scsiio_req->flags) &
3357	    MPI3_SCSIIO_FLAGS_DMAOPERATION_HOST_PI;
3358
3359	priv = scsi_cmd_priv(scmd);
3360
3361	simple_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_SIMPLE |
3362	    MPI3_SGE_FLAGS_DLAS_SYSTEM;
3363	simple_sgl_flags_last = simple_sgl_flags |
3364	    MPI3_SGE_FLAGS_END_OF_LIST;
3365	last_chain_sgl_flags = MPI3_SGE_FLAGS_ELEMENT_TYPE_LAST_CHAIN |
3366	    MPI3_SGE_FLAGS_DLAS_SYSTEM;
3367
3368	if (meta_sg)
3369		sg_local = &scsiio_req->sgl[MPI3_SCSIIO_METASGL_INDEX];
3370	else
3371		sg_local = &scsiio_req->sgl;
3372
3373	if (!scsiio_req->data_length && !meta_sg) {
3374		mpi3mr_build_zero_len_sge(sg_local);
3375		return 0;
3376	}
3377
3378	if (meta_sg) {
3379		sg_scmd = scsi_prot_sglist(scmd);
3380		sges_left = dma_map_sg(&mrioc->pdev->dev,
3381		    scsi_prot_sglist(scmd),
3382		    scsi_prot_sg_count(scmd),
3383		    scmd->sc_data_direction);
3384		priv->meta_sg_valid = 1; /* To unmap meta sg DMA */
3385	} else {
3386		sg_scmd = scsi_sglist(scmd);
3387		sges_left = scsi_dma_map(scmd);
3388	}
3389
3390	if (sges_left < 0) {
3391		sdev_printk(KERN_ERR, scmd->device,
3392		    "scsi_dma_map failed: request for %d bytes!\n",
3393		    scsi_bufflen(scmd));
3394		return -ENOMEM;
3395	}
3396	if (sges_left > MPI3MR_SG_DEPTH) {
3397		sdev_printk(KERN_ERR, scmd->device,
3398		    "scsi_dma_map returned unsupported sge count %d!\n",
3399		    sges_left);
3400		return -ENOMEM;
3401	}
3402
3403	sges_in_segment = (mrioc->facts.op_req_sz -
3404	    offsetof(struct mpi3_scsi_io_request, sgl)) / sizeof(struct mpi3_sge_common);
3405
3406	if (scsiio_req->sgl[0].eedp.flags ==
3407	    MPI3_SGE_FLAGS_ELEMENT_TYPE_EXTENDED && !meta_sg) {
3408		sg_local += sizeof(struct mpi3_sge_common);
3409		sges_in_segment--;
3410		/* Reserve 1st segment (scsiio_req->sgl[0]) for eedp */
3411	}
3412
3413	if (scsiio_req->msg_flags ==
3414	    MPI3_SCSIIO_MSGFLAGS_METASGL_VALID && !meta_sg) {
3415		sges_in_segment--;
3416		/* Reserve last segment (scsiio_req->sgl[3]) for meta sg */
3417	}
3418
3419	if (meta_sg)
3420		sges_in_segment = 1;
3421
3422	if (sges_left <= sges_in_segment)
3423		goto fill_in_last_segment;
3424
3425	/* fill in main message segment when there is a chain following */
3426	while (sges_in_segment > 1) {
3427		mpi3mr_add_sg_single(sg_local, simple_sgl_flags,
3428		    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
3429		sg_scmd = sg_next(sg_scmd);
3430		sg_local += sizeof(struct mpi3_sge_common);
3431		sges_left--;
3432		sges_in_segment--;
3433	}
3434
3435	chain_idx = mpi3mr_get_chain_idx(mrioc);
3436	if (chain_idx < 0)
3437		return -1;
3438	chain_req = &mrioc->chain_sgl_list[chain_idx];
3439	if (meta_sg)
3440		priv->meta_chain_idx = chain_idx;
3441	else
3442		priv->chain_idx = chain_idx;
3443
3444	chain = chain_req->addr;
3445	chain_dma = chain_req->dma_addr;
3446	sges_in_segment = sges_left;
3447	chain_length = sges_in_segment * sizeof(struct mpi3_sge_common);
3448
3449	mpi3mr_add_sg_single(sg_local, last_chain_sgl_flags,
3450	    chain_length, chain_dma);
3451
3452	sg_local = chain;
3453
3454fill_in_last_segment:
3455	while (sges_left > 0) {
3456		if (sges_left == 1)
3457			mpi3mr_add_sg_single(sg_local,
3458			    simple_sgl_flags_last, sg_dma_len(sg_scmd),
3459			    sg_dma_address(sg_scmd));
3460		else
3461			mpi3mr_add_sg_single(sg_local, simple_sgl_flags,
3462			    sg_dma_len(sg_scmd), sg_dma_address(sg_scmd));
3463		sg_scmd = sg_next(sg_scmd);
3464		sg_local += sizeof(struct mpi3_sge_common);
3465		sges_left--;
3466	}
3467
3468	return 0;
3469}
3470
3471/**
3472 * mpi3mr_build_sg_scmd - build scatter gather list for SCSI IO
3473 * @mrioc: Adapter instance reference
3474 * @scmd: SCSI command reference
3475 * @scsiio_req: MPI3 SCSI IO request
3476 *
3477 * This function calls mpi3mr_prepare_sg_scmd for constructing
3478 * both data SGEs and protection information SGEs in the MPI
3479 * format from the SCSI Command as appropriate .
3480 *
3481 * Return: return value of mpi3mr_prepare_sg_scmd.
3482 */
3483static int mpi3mr_build_sg_scmd(struct mpi3mr_ioc *mrioc,
3484	struct scsi_cmnd *scmd, struct mpi3_scsi_io_request *scsiio_req)
3485{
3486	int ret;
3487
3488	ret = mpi3mr_prepare_sg_scmd(mrioc, scmd, scsiio_req);
3489	if (ret)
3490		return ret;
3491
3492	if (scsiio_req->msg_flags == MPI3_SCSIIO_MSGFLAGS_METASGL_VALID) {
3493		/* There is a valid meta sg */
3494		scsiio_req->flags |=
3495		    cpu_to_le32(MPI3_SCSIIO_FLAGS_DMAOPERATION_HOST_PI);
3496		ret = mpi3mr_prepare_sg_scmd(mrioc, scmd, scsiio_req);
3497	}
3498
3499	return ret;
3500}
3501
3502/**
3503 * mpi3mr_tm_response_name -  get TM response as a string
3504 * @resp_code: TM response code
3505 *
3506 * Convert known task management response code as a readable
3507 * string.
3508 *
3509 * Return: response code string.
3510 */
3511static const char *mpi3mr_tm_response_name(u8 resp_code)
3512{
3513	char *desc;
3514
3515	switch (resp_code) {
3516	case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE:
3517		desc = "task management request completed";
3518		break;
3519	case MPI3_SCSITASKMGMT_RSPCODE_INVALID_FRAME:
3520		desc = "invalid frame";
3521		break;
3522	case MPI3_SCSITASKMGMT_RSPCODE_TM_FUNCTION_NOT_SUPPORTED:
3523		desc = "task management request not supported";
3524		break;
3525	case MPI3_SCSITASKMGMT_RSPCODE_TM_FAILED:
3526		desc = "task management request failed";
3527		break;
3528	case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED:
3529		desc = "task management request succeeded";
3530		break;
3531	case MPI3_SCSITASKMGMT_RSPCODE_TM_INVALID_LUN:
3532		desc = "invalid LUN";
3533		break;
3534	case MPI3_SCSITASKMGMT_RSPCODE_TM_OVERLAPPED_TAG:
3535		desc = "overlapped tag attempted";
3536		break;
3537	case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC:
3538		desc = "task queued, however not sent to target";
3539		break;
3540	case MPI3_SCSITASKMGMT_RSPCODE_TM_NVME_DENIED:
3541		desc = "task management request denied by NVMe device";
3542		break;
3543	default:
3544		desc = "unknown";
3545		break;
3546	}
3547
3548	return desc;
3549}
3550
3551inline void mpi3mr_poll_pend_io_completions(struct mpi3mr_ioc *mrioc)
3552{
3553	int i;
3554	int num_of_reply_queues =
3555	    mrioc->num_op_reply_q + mrioc->op_reply_q_offset;
3556
3557	for (i = mrioc->op_reply_q_offset; i < num_of_reply_queues; i++)
3558		mpi3mr_process_op_reply_q(mrioc,
3559		    mrioc->intr_info[i].op_reply_q);
3560}
3561
3562/**
3563 * mpi3mr_issue_tm - Issue Task Management request
3564 * @mrioc: Adapter instance reference
3565 * @tm_type: Task Management type
3566 * @handle: Device handle
3567 * @lun: lun ID
3568 * @htag: Host tag of the TM request
3569 * @timeout: TM timeout value
3570 * @drv_cmd: Internal command tracker
3571 * @resp_code: Response code place holder
3572 * @scmd: SCSI command
3573 *
3574 * Issues a Task Management Request to the controller for a
3575 * specified target, lun and command and wait for its completion
3576 * and check TM response. Recover the TM if it timed out by
3577 * issuing controller reset.
3578 *
3579 * Return: 0 on success, non-zero on errors
3580 */
3581int mpi3mr_issue_tm(struct mpi3mr_ioc *mrioc, u8 tm_type,
3582	u16 handle, uint lun, u16 htag, ulong timeout,
3583	struct mpi3mr_drv_cmd *drv_cmd,
3584	u8 *resp_code, struct scsi_cmnd *scmd)
3585{
3586	struct mpi3_scsi_task_mgmt_request tm_req;
3587	struct mpi3_scsi_task_mgmt_reply *tm_reply = NULL;
3588	int retval = 0;
3589	struct mpi3mr_tgt_dev *tgtdev = NULL;
3590	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data = NULL;
3591	struct scmd_priv *cmd_priv = NULL;
3592	struct scsi_device *sdev = NULL;
3593	struct mpi3mr_sdev_priv_data *sdev_priv_data = NULL;
3594
3595	ioc_info(mrioc, "%s :Issue TM: TM type (0x%x) for devhandle 0x%04x\n",
3596	     __func__, tm_type, handle);
3597	if (mrioc->unrecoverable) {
3598		retval = -1;
3599		ioc_err(mrioc, "%s :Issue TM: Unrecoverable controller\n",
3600		    __func__);
3601		goto out;
3602	}
3603
3604	memset(&tm_req, 0, sizeof(tm_req));
3605	mutex_lock(&drv_cmd->mutex);
3606	if (drv_cmd->state & MPI3MR_CMD_PENDING) {
3607		retval = -1;
3608		ioc_err(mrioc, "%s :Issue TM: Command is in use\n", __func__);
3609		mutex_unlock(&drv_cmd->mutex);
3610		goto out;
3611	}
3612	if (mrioc->reset_in_progress) {
3613		retval = -1;
3614		ioc_err(mrioc, "%s :Issue TM: Reset in progress\n", __func__);
3615		mutex_unlock(&drv_cmd->mutex);
3616		goto out;
3617	}
3618
3619	drv_cmd->state = MPI3MR_CMD_PENDING;
3620	drv_cmd->is_waiting = 1;
3621	drv_cmd->callback = NULL;
3622	tm_req.dev_handle = cpu_to_le16(handle);
3623	tm_req.task_type = tm_type;
3624	tm_req.host_tag = cpu_to_le16(htag);
3625
3626	int_to_scsilun(lun, (struct scsi_lun *)tm_req.lun);
3627	tm_req.function = MPI3_FUNCTION_SCSI_TASK_MGMT;
3628
3629	tgtdev = mpi3mr_get_tgtdev_by_handle(mrioc, handle);
3630
3631	if (scmd) {
3632		sdev = scmd->device;
3633		sdev_priv_data = sdev->hostdata;
3634		scsi_tgt_priv_data = ((sdev_priv_data) ?
3635		    sdev_priv_data->tgt_priv_data : NULL);
3636	} else {
3637		if (tgtdev && tgtdev->starget && tgtdev->starget->hostdata)
3638			scsi_tgt_priv_data = (struct mpi3mr_stgt_priv_data *)
3639			    tgtdev->starget->hostdata;
3640	}
3641
3642	if (scsi_tgt_priv_data)
3643		atomic_inc(&scsi_tgt_priv_data->block_io);
3644
3645	if (tgtdev && (tgtdev->dev_type == MPI3_DEVICE_DEVFORM_PCIE)) {
3646		if (cmd_priv && tgtdev->dev_spec.pcie_inf.abort_to)
3647			timeout = tgtdev->dev_spec.pcie_inf.abort_to;
3648		else if (!cmd_priv && tgtdev->dev_spec.pcie_inf.reset_to)
3649			timeout = tgtdev->dev_spec.pcie_inf.reset_to;
3650	}
3651
3652	init_completion(&drv_cmd->done);
3653	retval = mpi3mr_admin_request_post(mrioc, &tm_req, sizeof(tm_req), 1);
3654	if (retval) {
3655		ioc_err(mrioc, "%s :Issue TM: Admin Post failed\n", __func__);
3656		goto out_unlock;
3657	}
3658	wait_for_completion_timeout(&drv_cmd->done, (timeout * HZ));
3659
3660	if (!(drv_cmd->state & MPI3MR_CMD_COMPLETE)) {
3661		drv_cmd->is_waiting = 0;
3662		retval = -1;
3663		if (!(drv_cmd->state & MPI3MR_CMD_RESET)) {
3664			dprint_tm(mrioc,
3665			    "task management request timed out after %ld seconds\n",
3666			    timeout);
3667			if (mrioc->logging_level & MPI3_DEBUG_TM)
3668				dprint_dump_req(&tm_req, sizeof(tm_req)/4);
3669			mpi3mr_soft_reset_handler(mrioc,
3670			    MPI3MR_RESET_FROM_TM_TIMEOUT, 1);
3671		}
3672		goto out_unlock;
3673	}
3674
3675	if (!(drv_cmd->state & MPI3MR_CMD_REPLY_VALID)) {
3676		dprint_tm(mrioc, "invalid task management reply message\n");
3677		retval = -1;
3678		goto out_unlock;
3679	}
3680
3681	tm_reply = (struct mpi3_scsi_task_mgmt_reply *)drv_cmd->reply;
3682
3683	switch (drv_cmd->ioc_status) {
3684	case MPI3_IOCSTATUS_SUCCESS:
3685		*resp_code = le32_to_cpu(tm_reply->response_data) &
3686			MPI3MR_RI_MASK_RESPCODE;
3687		break;
3688	case MPI3_IOCSTATUS_SCSI_IOC_TERMINATED:
3689		*resp_code = MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE;
3690		break;
3691	default:
3692		dprint_tm(mrioc,
3693		    "task management request to handle(0x%04x) is failed with ioc_status(0x%04x) log_info(0x%08x)\n",
3694		    handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo);
3695		retval = -1;
3696		goto out_unlock;
3697	}
3698
3699	switch (*resp_code) {
3700	case MPI3_SCSITASKMGMT_RSPCODE_TM_SUCCEEDED:
3701	case MPI3_SCSITASKMGMT_RSPCODE_TM_COMPLETE:
3702		break;
3703	case MPI3_SCSITASKMGMT_RSPCODE_IO_QUEUED_ON_IOC:
3704		if (tm_type != MPI3_SCSITASKMGMT_TASKTYPE_QUERY_TASK)
3705			retval = -1;
3706		break;
3707	default:
3708		retval = -1;
3709		break;
3710	}
3711
3712	dprint_tm(mrioc,
3713	    "task management request type(%d) completed for handle(0x%04x) with ioc_status(0x%04x), log_info(0x%08x), termination_count(%d), response:%s(0x%x)\n",
3714	    tm_type, handle, drv_cmd->ioc_status, drv_cmd->ioc_loginfo,
3715	    le32_to_cpu(tm_reply->termination_count),
3716	    mpi3mr_tm_response_name(*resp_code), *resp_code);
3717
3718	if (!retval) {
3719		mpi3mr_ioc_disable_intr(mrioc);
3720		mpi3mr_poll_pend_io_completions(mrioc);
3721		mpi3mr_ioc_enable_intr(mrioc);
3722		mpi3mr_poll_pend_io_completions(mrioc);
3723	}
3724	switch (tm_type) {
3725	case MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET:
3726		if (!scsi_tgt_priv_data)
3727			break;
3728		scsi_tgt_priv_data->pend_count = 0;
3729		blk_mq_tagset_busy_iter(&mrioc->shost->tag_set,
3730		    mpi3mr_count_tgt_pending,
3731		    (void *)scsi_tgt_priv_data->starget);
3732		break;
3733	case MPI3_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET:
3734		if (!sdev_priv_data)
3735			break;
3736		sdev_priv_data->pend_count = 0;
3737		blk_mq_tagset_busy_iter(&mrioc->shost->tag_set,
3738		    mpi3mr_count_dev_pending, (void *)sdev);
3739		break;
3740	default:
3741		break;
3742	}
3743
3744out_unlock:
3745	drv_cmd->state = MPI3MR_CMD_NOTUSED;
3746	mutex_unlock(&drv_cmd->mutex);
3747	if (scsi_tgt_priv_data)
3748		atomic_dec_if_positive(&scsi_tgt_priv_data->block_io);
3749	if (tgtdev)
3750		mpi3mr_tgtdev_put(tgtdev);
3751out:
3752	return retval;
3753}
3754
3755/**
3756 * mpi3mr_bios_param - BIOS param callback
3757 * @sdev: SCSI device reference
3758 * @bdev: Block device reference
3759 * @capacity: Capacity in logical sectors
3760 * @params: Parameter array
3761 *
3762 * Just the parameters with heads/secots/cylinders.
3763 *
3764 * Return: 0 always
3765 */
3766static int mpi3mr_bios_param(struct scsi_device *sdev,
3767	struct block_device *bdev, sector_t capacity, int params[])
3768{
3769	int heads;
3770	int sectors;
3771	sector_t cylinders;
3772	ulong dummy;
3773
3774	heads = 64;
3775	sectors = 32;
3776
3777	dummy = heads * sectors;
3778	cylinders = capacity;
3779	sector_div(cylinders, dummy);
3780
3781	if ((ulong)capacity >= 0x200000) {
3782		heads = 255;
3783		sectors = 63;
3784		dummy = heads * sectors;
3785		cylinders = capacity;
3786		sector_div(cylinders, dummy);
3787	}
3788
3789	params[0] = heads;
3790	params[1] = sectors;
3791	params[2] = cylinders;
3792	return 0;
3793}
3794
3795/**
3796 * mpi3mr_map_queues - Map queues callback handler
3797 * @shost: SCSI host reference
3798 *
3799 * Maps default and poll queues.
3800 *
3801 * Return: return zero.
3802 */
3803static void mpi3mr_map_queues(struct Scsi_Host *shost)
3804{
3805	struct mpi3mr_ioc *mrioc = shost_priv(shost);
3806	int i, qoff, offset;
3807	struct blk_mq_queue_map *map = NULL;
3808
3809	offset = mrioc->op_reply_q_offset;
3810
3811	for (i = 0, qoff = 0; i < HCTX_MAX_TYPES; i++) {
3812		map = &shost->tag_set.map[i];
3813
3814		map->nr_queues  = 0;
3815
3816		if (i == HCTX_TYPE_DEFAULT)
3817			map->nr_queues = mrioc->default_qcount;
3818		else if (i == HCTX_TYPE_POLL)
3819			map->nr_queues = mrioc->active_poll_qcount;
3820
3821		if (!map->nr_queues) {
3822			BUG_ON(i == HCTX_TYPE_DEFAULT);
3823			continue;
3824		}
3825
3826		/*
3827		 * The poll queue(s) doesn't have an IRQ (and hence IRQ
3828		 * affinity), so use the regular blk-mq cpu mapping
3829		 */
3830		map->queue_offset = qoff;
3831		if (i != HCTX_TYPE_POLL)
3832			blk_mq_pci_map_queues(map, mrioc->pdev, offset);
3833		else
3834			blk_mq_map_queues(map);
3835
3836		qoff += map->nr_queues;
3837		offset += map->nr_queues;
3838	}
3839}
3840
3841/**
3842 * mpi3mr_get_fw_pending_ios - Calculate pending I/O count
3843 * @mrioc: Adapter instance reference
3844 *
3845 * Calculate the pending I/Os for the controller and return.
3846 *
3847 * Return: Number of pending I/Os
3848 */
3849static inline int mpi3mr_get_fw_pending_ios(struct mpi3mr_ioc *mrioc)
3850{
3851	u16 i;
3852	uint pend_ios = 0;
3853
3854	for (i = 0; i < mrioc->num_op_reply_q; i++)
3855		pend_ios += atomic_read(&mrioc->op_reply_qinfo[i].pend_ios);
3856	return pend_ios;
3857}
3858
3859/**
3860 * mpi3mr_print_pending_host_io - print pending I/Os
3861 * @mrioc: Adapter instance reference
3862 *
3863 * Print number of pending I/Os and each I/O details prior to
3864 * reset for debug purpose.
3865 *
3866 * Return: Nothing
3867 */
3868static void mpi3mr_print_pending_host_io(struct mpi3mr_ioc *mrioc)
3869{
3870	struct Scsi_Host *shost = mrioc->shost;
3871
3872	ioc_info(mrioc, "%s :Pending commands prior to reset: %d\n",
3873	    __func__, mpi3mr_get_fw_pending_ios(mrioc));
3874	blk_mq_tagset_busy_iter(&shost->tag_set,
3875	    mpi3mr_print_scmd, (void *)mrioc);
3876}
3877
3878/**
3879 * mpi3mr_wait_for_host_io - block for I/Os to complete
3880 * @mrioc: Adapter instance reference
3881 * @timeout: time out in seconds
3882 * Waits for pending I/Os for the given adapter to complete or
3883 * to hit the timeout.
3884 *
3885 * Return: Nothing
3886 */
3887void mpi3mr_wait_for_host_io(struct mpi3mr_ioc *mrioc, u32 timeout)
3888{
3889	enum mpi3mr_iocstate iocstate;
3890	int i = 0;
3891
3892	iocstate = mpi3mr_get_iocstate(mrioc);
3893	if (iocstate != MRIOC_STATE_READY)
3894		return;
3895
3896	if (!mpi3mr_get_fw_pending_ios(mrioc))
3897		return;
3898	ioc_info(mrioc,
3899	    "%s :Waiting for %d seconds prior to reset for %d I/O\n",
3900	    __func__, timeout, mpi3mr_get_fw_pending_ios(mrioc));
3901
3902	for (i = 0; i < timeout; i++) {
3903		if (!mpi3mr_get_fw_pending_ios(mrioc))
3904			break;
3905		iocstate = mpi3mr_get_iocstate(mrioc);
3906		if (iocstate != MRIOC_STATE_READY)
3907			break;
3908		msleep(1000);
3909	}
3910
3911	ioc_info(mrioc, "%s :Pending I/Os after wait is: %d\n", __func__,
3912	    mpi3mr_get_fw_pending_ios(mrioc));
3913}
3914
3915/**
3916 * mpi3mr_eh_host_reset - Host reset error handling callback
3917 * @scmd: SCSI command reference
3918 *
3919 * Issue controller reset if the scmd is for a Physical Device,
3920 * if the scmd is for RAID volume, then wait for
3921 * MPI3MR_RAID_ERRREC_RESET_TIMEOUT and checke whether any
3922 * pending I/Os prior to issuing reset to the controller.
3923 *
3924 * Return: SUCCESS of successful reset else FAILED
3925 */
3926static int mpi3mr_eh_host_reset(struct scsi_cmnd *scmd)
3927{
3928	struct mpi3mr_ioc *mrioc = shost_priv(scmd->device->host);
3929	struct mpi3mr_stgt_priv_data *stgt_priv_data;
3930	struct mpi3mr_sdev_priv_data *sdev_priv_data;
3931	u8 dev_type = MPI3_DEVICE_DEVFORM_VD;
3932	int retval = FAILED, ret;
3933
3934	sdev_priv_data = scmd->device->hostdata;
3935	if (sdev_priv_data && sdev_priv_data->tgt_priv_data) {
3936		stgt_priv_data = sdev_priv_data->tgt_priv_data;
3937		dev_type = stgt_priv_data->dev_type;
3938	}
3939
3940	if (dev_type == MPI3_DEVICE_DEVFORM_VD) {
3941		mpi3mr_wait_for_host_io(mrioc,
3942		    MPI3MR_RAID_ERRREC_RESET_TIMEOUT);
3943		if (!mpi3mr_get_fw_pending_ios(mrioc)) {
3944			retval = SUCCESS;
3945			goto out;
3946		}
3947	}
3948
3949	mpi3mr_print_pending_host_io(mrioc);
3950	ret = mpi3mr_soft_reset_handler(mrioc,
3951	    MPI3MR_RESET_FROM_EH_HOS, 1);
3952	if (ret)
3953		goto out;
3954
3955	retval = SUCCESS;
3956out:
3957	sdev_printk(KERN_INFO, scmd->device,
3958	    "Host reset is %s for scmd(%p)\n",
3959	    ((retval == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
3960
3961	return retval;
3962}
3963
3964/**
3965 * mpi3mr_eh_target_reset - Target reset error handling callback
3966 * @scmd: SCSI command reference
3967 *
3968 * Issue Target reset Task Management and verify the scmd is
3969 * terminated successfully and return status accordingly.
3970 *
3971 * Return: SUCCESS of successful termination of the scmd else
3972 *         FAILED
3973 */
3974static int mpi3mr_eh_target_reset(struct scsi_cmnd *scmd)
3975{
3976	struct mpi3mr_ioc *mrioc = shost_priv(scmd->device->host);
3977	struct mpi3mr_stgt_priv_data *stgt_priv_data;
3978	struct mpi3mr_sdev_priv_data *sdev_priv_data;
3979	u16 dev_handle;
3980	u8 resp_code = 0;
3981	int retval = FAILED, ret = 0;
3982
3983	sdev_printk(KERN_INFO, scmd->device,
3984	    "Attempting Target Reset! scmd(%p)\n", scmd);
3985	scsi_print_command(scmd);
3986
3987	sdev_priv_data = scmd->device->hostdata;
3988	if (!sdev_priv_data || !sdev_priv_data->tgt_priv_data) {
3989		sdev_printk(KERN_INFO, scmd->device,
3990		    "SCSI device is not available\n");
3991		retval = SUCCESS;
3992		goto out;
3993	}
3994
3995	stgt_priv_data = sdev_priv_data->tgt_priv_data;
3996	dev_handle = stgt_priv_data->dev_handle;
3997	if (stgt_priv_data->dev_removed) {
3998		sdev_printk(KERN_INFO, scmd->device,
3999		    "%s:target(handle = 0x%04x) is removed, target reset is not issued\n",
4000		    mrioc->name, dev_handle);
4001		retval = FAILED;
4002		goto out;
4003	}
4004	sdev_printk(KERN_INFO, scmd->device,
4005	    "Target Reset is issued to handle(0x%04x)\n",
4006	    dev_handle);
4007
4008	ret = mpi3mr_issue_tm(mrioc,
4009	    MPI3_SCSITASKMGMT_TASKTYPE_TARGET_RESET, dev_handle,
4010	    sdev_priv_data->lun_id, MPI3MR_HOSTTAG_BLK_TMS,
4011	    MPI3MR_RESETTM_TIMEOUT, &mrioc->host_tm_cmds, &resp_code, scmd);
4012
4013	if (ret)
4014		goto out;
4015
4016	if (stgt_priv_data->pend_count) {
4017		sdev_printk(KERN_INFO, scmd->device,
4018		    "%s: target has %d pending commands, target reset is failed\n",
4019		    mrioc->name, stgt_priv_data->pend_count);
4020		goto out;
4021	}
4022
4023	retval = SUCCESS;
4024out:
4025	sdev_printk(KERN_INFO, scmd->device,
4026	    "%s: target reset is %s for scmd(%p)\n", mrioc->name,
4027	    ((retval == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4028
4029	return retval;
4030}
4031
4032/**
4033 * mpi3mr_eh_dev_reset- Device reset error handling callback
4034 * @scmd: SCSI command reference
4035 *
4036 * Issue lun reset Task Management and verify the scmd is
4037 * terminated successfully and return status accordingly.
4038 *
4039 * Return: SUCCESS of successful termination of the scmd else
4040 *         FAILED
4041 */
4042static int mpi3mr_eh_dev_reset(struct scsi_cmnd *scmd)
4043{
4044	struct mpi3mr_ioc *mrioc = shost_priv(scmd->device->host);
4045	struct mpi3mr_stgt_priv_data *stgt_priv_data;
4046	struct mpi3mr_sdev_priv_data *sdev_priv_data;
4047	u16 dev_handle;
4048	u8 resp_code = 0;
4049	int retval = FAILED, ret = 0;
4050
4051	sdev_printk(KERN_INFO, scmd->device,
4052	    "Attempting Device(lun) Reset! scmd(%p)\n", scmd);
4053	scsi_print_command(scmd);
4054
4055	sdev_priv_data = scmd->device->hostdata;
4056	if (!sdev_priv_data || !sdev_priv_data->tgt_priv_data) {
4057		sdev_printk(KERN_INFO, scmd->device,
4058		    "SCSI device is not available\n");
4059		retval = SUCCESS;
4060		goto out;
4061	}
4062
4063	stgt_priv_data = sdev_priv_data->tgt_priv_data;
4064	dev_handle = stgt_priv_data->dev_handle;
4065	if (stgt_priv_data->dev_removed) {
4066		sdev_printk(KERN_INFO, scmd->device,
4067		    "%s: device(handle = 0x%04x) is removed, device(LUN) reset is not issued\n",
4068		    mrioc->name, dev_handle);
4069		retval = FAILED;
4070		goto out;
4071	}
4072	sdev_printk(KERN_INFO, scmd->device,
4073	    "Device(lun) Reset is issued to handle(0x%04x)\n", dev_handle);
4074
4075	ret = mpi3mr_issue_tm(mrioc,
4076	    MPI3_SCSITASKMGMT_TASKTYPE_LOGICAL_UNIT_RESET, dev_handle,
4077	    sdev_priv_data->lun_id, MPI3MR_HOSTTAG_BLK_TMS,
4078	    MPI3MR_RESETTM_TIMEOUT, &mrioc->host_tm_cmds, &resp_code, scmd);
4079
4080	if (ret)
4081		goto out;
4082
4083	if (sdev_priv_data->pend_count) {
4084		sdev_printk(KERN_INFO, scmd->device,
4085		    "%s: device has %d pending commands, device(LUN) reset is failed\n",
4086		    mrioc->name, sdev_priv_data->pend_count);
4087		goto out;
4088	}
4089	retval = SUCCESS;
4090out:
4091	sdev_printk(KERN_INFO, scmd->device,
4092	    "%s: device(LUN) reset is %s for scmd(%p)\n", mrioc->name,
4093	    ((retval == SUCCESS) ? "SUCCESS" : "FAILED"), scmd);
4094
4095	return retval;
4096}
4097
4098/**
4099 * mpi3mr_scan_start - Scan start callback handler
4100 * @shost: SCSI host reference
4101 *
4102 * Issue port enable request asynchronously.
4103 *
4104 * Return: Nothing
4105 */
4106static void mpi3mr_scan_start(struct Scsi_Host *shost)
4107{
4108	struct mpi3mr_ioc *mrioc = shost_priv(shost);
4109
4110	mrioc->scan_started = 1;
4111	ioc_info(mrioc, "%s :Issuing Port Enable\n", __func__);
4112	if (mpi3mr_issue_port_enable(mrioc, 1)) {
4113		ioc_err(mrioc, "%s :Issuing port enable failed\n", __func__);
4114		mrioc->scan_started = 0;
4115		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
4116	}
4117}
4118
4119/**
4120 * mpi3mr_scan_finished - Scan finished callback handler
4121 * @shost: SCSI host reference
4122 * @time: Jiffies from the scan start
4123 *
4124 * Checks whether the port enable is completed or timedout or
4125 * failed and set the scan status accordingly after taking any
4126 * recovery if required.
4127 *
4128 * Return: 1 on scan finished or timed out, 0 for in progress
4129 */
4130static int mpi3mr_scan_finished(struct Scsi_Host *shost,
4131	unsigned long time)
4132{
4133	struct mpi3mr_ioc *mrioc = shost_priv(shost);
4134	u32 pe_timeout = MPI3MR_PORTENABLE_TIMEOUT;
4135	u32 ioc_status = readl(&mrioc->sysif_regs->ioc_status);
4136
4137	if ((ioc_status & MPI3_SYSIF_IOC_STATUS_RESET_HISTORY) ||
4138	    (ioc_status & MPI3_SYSIF_IOC_STATUS_FAULT)) {
4139		ioc_err(mrioc, "port enable failed due to fault or reset\n");
4140		mpi3mr_print_fault_info(mrioc);
4141		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
4142		mrioc->scan_started = 0;
4143		mrioc->init_cmds.is_waiting = 0;
4144		mrioc->init_cmds.callback = NULL;
4145		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4146	}
4147
4148	if (time >= (pe_timeout * HZ)) {
4149		ioc_err(mrioc, "port enable failed due to time out\n");
4150		mpi3mr_check_rh_fault_ioc(mrioc,
4151		    MPI3MR_RESET_FROM_PE_TIMEOUT);
4152		mrioc->scan_failed = MPI3_IOCSTATUS_INTERNAL_ERROR;
4153		mrioc->scan_started = 0;
4154		mrioc->init_cmds.is_waiting = 0;
4155		mrioc->init_cmds.callback = NULL;
4156		mrioc->init_cmds.state = MPI3MR_CMD_NOTUSED;
4157	}
4158
4159	if (mrioc->scan_started)
4160		return 0;
4161
4162	if (mrioc->scan_failed) {
4163		ioc_err(mrioc,
4164		    "port enable failed with status=0x%04x\n",
4165		    mrioc->scan_failed);
4166	} else
4167		ioc_info(mrioc, "port enable is successfully completed\n");
4168
4169	mpi3mr_start_watchdog(mrioc);
4170	mrioc->is_driver_loading = 0;
4171	mrioc->stop_bsgs = 0;
4172	return 1;
4173}
4174
4175/**
4176 * mpi3mr_slave_destroy - Slave destroy callback handler
4177 * @sdev: SCSI device reference
4178 *
4179 * Cleanup and free per device(lun) private data.
4180 *
4181 * Return: Nothing.
4182 */
4183static void mpi3mr_slave_destroy(struct scsi_device *sdev)
4184{
4185	struct Scsi_Host *shost;
4186	struct mpi3mr_ioc *mrioc;
4187	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data;
4188	struct mpi3mr_tgt_dev *tgt_dev = NULL;
4189	unsigned long flags;
4190	struct scsi_target *starget;
4191	struct sas_rphy *rphy = NULL;
4192
4193	if (!sdev->hostdata)
4194		return;
4195
4196	starget = scsi_target(sdev);
4197	shost = dev_to_shost(&starget->dev);
4198	mrioc = shost_priv(shost);
4199	scsi_tgt_priv_data = starget->hostdata;
4200
4201	scsi_tgt_priv_data->num_luns--;
4202
4203	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4204	if (starget->channel == mrioc->scsi_device_channel)
4205		tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id);
4206	else if (mrioc->sas_transport_enabled && !starget->channel) {
4207		rphy = dev_to_rphy(starget->dev.parent);
4208		tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc,
4209		    rphy->identify.sas_address, rphy);
4210	}
4211
4212	if (tgt_dev && (!scsi_tgt_priv_data->num_luns))
4213		tgt_dev->starget = NULL;
4214	if (tgt_dev)
4215		mpi3mr_tgtdev_put(tgt_dev);
4216	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4217
4218	kfree(sdev->hostdata);
4219	sdev->hostdata = NULL;
4220}
4221
4222/**
4223 * mpi3mr_target_destroy - Target destroy callback handler
4224 * @starget: SCSI target reference
4225 *
4226 * Cleanup and free per target private data.
4227 *
4228 * Return: Nothing.
4229 */
4230static void mpi3mr_target_destroy(struct scsi_target *starget)
4231{
4232	struct Scsi_Host *shost;
4233	struct mpi3mr_ioc *mrioc;
4234	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data;
4235	struct mpi3mr_tgt_dev *tgt_dev;
4236	unsigned long flags;
4237
4238	if (!starget->hostdata)
4239		return;
4240
4241	shost = dev_to_shost(&starget->dev);
4242	mrioc = shost_priv(shost);
4243	scsi_tgt_priv_data = starget->hostdata;
4244
4245	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4246	tgt_dev = __mpi3mr_get_tgtdev_from_tgtpriv(mrioc, scsi_tgt_priv_data);
4247	if (tgt_dev && (tgt_dev->starget == starget) &&
4248	    (tgt_dev->perst_id == starget->id))
4249		tgt_dev->starget = NULL;
4250	if (tgt_dev) {
4251		scsi_tgt_priv_data->tgt_dev = NULL;
4252		scsi_tgt_priv_data->perst_id = 0;
4253		mpi3mr_tgtdev_put(tgt_dev);
4254		mpi3mr_tgtdev_put(tgt_dev);
4255	}
4256	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4257
4258	kfree(starget->hostdata);
4259	starget->hostdata = NULL;
4260}
4261
4262/**
4263 * mpi3mr_slave_configure - Slave configure callback handler
4264 * @sdev: SCSI device reference
4265 *
4266 * Configure queue depth, max hardware sectors and virt boundary
4267 * as required
4268 *
4269 * Return: 0 always.
4270 */
4271static int mpi3mr_slave_configure(struct scsi_device *sdev)
4272{
4273	struct scsi_target *starget;
4274	struct Scsi_Host *shost;
4275	struct mpi3mr_ioc *mrioc;
4276	struct mpi3mr_tgt_dev *tgt_dev = NULL;
4277	unsigned long flags;
4278	int retval = 0;
4279	struct sas_rphy *rphy = NULL;
4280
4281	starget = scsi_target(sdev);
4282	shost = dev_to_shost(&starget->dev);
4283	mrioc = shost_priv(shost);
4284
4285	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4286	if (starget->channel == mrioc->scsi_device_channel)
4287		tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id);
4288	else if (mrioc->sas_transport_enabled && !starget->channel) {
4289		rphy = dev_to_rphy(starget->dev.parent);
4290		tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc,
4291		    rphy->identify.sas_address, rphy);
4292	}
4293	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4294	if (!tgt_dev)
4295		return -ENXIO;
4296
4297	mpi3mr_change_queue_depth(sdev, tgt_dev->q_depth);
4298
4299	sdev->eh_timeout = MPI3MR_EH_SCMD_TIMEOUT;
4300	blk_queue_rq_timeout(sdev->request_queue, MPI3MR_SCMD_TIMEOUT);
4301
4302	switch (tgt_dev->dev_type) {
4303	case MPI3_DEVICE_DEVFORM_PCIE:
4304		/*The block layer hw sector size = 512*/
4305		if ((tgt_dev->dev_spec.pcie_inf.dev_info &
4306		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_MASK) ==
4307		    MPI3_DEVICE0_PCIE_DEVICE_INFO_TYPE_NVME_DEVICE) {
4308			blk_queue_max_hw_sectors(sdev->request_queue,
4309			    tgt_dev->dev_spec.pcie_inf.mdts / 512);
4310			if (tgt_dev->dev_spec.pcie_inf.pgsz == 0)
4311				blk_queue_virt_boundary(sdev->request_queue,
4312				    ((1 << MPI3MR_DEFAULT_PGSZEXP) - 1));
4313			else
4314				blk_queue_virt_boundary(sdev->request_queue,
4315				    ((1 << tgt_dev->dev_spec.pcie_inf.pgsz) - 1));
4316		}
4317		break;
4318	default:
4319		break;
4320	}
4321
4322	mpi3mr_tgtdev_put(tgt_dev);
4323
4324	return retval;
4325}
4326
4327/**
4328 * mpi3mr_slave_alloc -Slave alloc callback handler
4329 * @sdev: SCSI device reference
4330 *
4331 * Allocate per device(lun) private data and initialize it.
4332 *
4333 * Return: 0 on success -ENOMEM on memory allocation failure.
4334 */
4335static int mpi3mr_slave_alloc(struct scsi_device *sdev)
4336{
4337	struct Scsi_Host *shost;
4338	struct mpi3mr_ioc *mrioc;
4339	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data;
4340	struct mpi3mr_tgt_dev *tgt_dev = NULL;
4341	struct mpi3mr_sdev_priv_data *scsi_dev_priv_data;
4342	unsigned long flags;
4343	struct scsi_target *starget;
4344	int retval = 0;
4345	struct sas_rphy *rphy = NULL;
4346
4347	starget = scsi_target(sdev);
4348	shost = dev_to_shost(&starget->dev);
4349	mrioc = shost_priv(shost);
4350	scsi_tgt_priv_data = starget->hostdata;
4351
4352	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4353
4354	if (starget->channel == mrioc->scsi_device_channel)
4355		tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id);
4356	else if (mrioc->sas_transport_enabled && !starget->channel) {
4357		rphy = dev_to_rphy(starget->dev.parent);
4358		tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc,
4359		    rphy->identify.sas_address, rphy);
4360	}
4361
4362	if (tgt_dev) {
4363		if (tgt_dev->starget == NULL)
4364			tgt_dev->starget = starget;
4365		mpi3mr_tgtdev_put(tgt_dev);
4366		retval = 0;
4367	} else {
4368		spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4369		return -ENXIO;
4370	}
4371
4372	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4373
4374	scsi_dev_priv_data = kzalloc(sizeof(*scsi_dev_priv_data), GFP_KERNEL);
4375	if (!scsi_dev_priv_data)
4376		return -ENOMEM;
4377
4378	scsi_dev_priv_data->lun_id = sdev->lun;
4379	scsi_dev_priv_data->tgt_priv_data = scsi_tgt_priv_data;
4380	sdev->hostdata = scsi_dev_priv_data;
4381
4382	scsi_tgt_priv_data->num_luns++;
4383
4384	return retval;
4385}
4386
4387/**
4388 * mpi3mr_target_alloc - Target alloc callback handler
4389 * @starget: SCSI target reference
4390 *
4391 * Allocate per target private data and initialize it.
4392 *
4393 * Return: 0 on success -ENOMEM on memory allocation failure.
4394 */
4395static int mpi3mr_target_alloc(struct scsi_target *starget)
4396{
4397	struct Scsi_Host *shost = dev_to_shost(&starget->dev);
4398	struct mpi3mr_ioc *mrioc = shost_priv(shost);
4399	struct mpi3mr_stgt_priv_data *scsi_tgt_priv_data;
4400	struct mpi3mr_tgt_dev *tgt_dev;
4401	unsigned long flags;
4402	int retval = 0;
4403	struct sas_rphy *rphy = NULL;
4404	bool update_stgt_priv_data = false;
4405
4406	scsi_tgt_priv_data = kzalloc(sizeof(*scsi_tgt_priv_data), GFP_KERNEL);
4407	if (!scsi_tgt_priv_data)
4408		return -ENOMEM;
4409
4410	starget->hostdata = scsi_tgt_priv_data;
4411
4412	spin_lock_irqsave(&mrioc->tgtdev_lock, flags);
4413
4414	if (starget->channel == mrioc->scsi_device_channel) {
4415		tgt_dev = __mpi3mr_get_tgtdev_by_perst_id(mrioc, starget->id);
4416		if (tgt_dev && !tgt_dev->is_hidden)
4417			update_stgt_priv_data = true;
4418		else
4419			retval = -ENXIO;
4420	} else if (mrioc->sas_transport_enabled && !starget->channel) {
4421		rphy = dev_to_rphy(starget->dev.parent);
4422		tgt_dev = __mpi3mr_get_tgtdev_by_addr_and_rphy(mrioc,
4423		    rphy->identify.sas_address, rphy);
4424		if (tgt_dev && !tgt_dev->is_hidden && !tgt_dev->non_stl &&
4425		    (tgt_dev->dev_type == MPI3_DEVICE_DEVFORM_SAS_SATA))
4426			update_stgt_priv_data = true;
4427		else
4428			retval = -ENXIO;
4429	}
4430
4431	if (update_stgt_priv_data) {
4432		scsi_tgt_priv_data->starget = starget;
4433		scsi_tgt_priv_data->dev_handle = tgt_dev->dev_handle;
4434		scsi_tgt_priv_data->perst_id = tgt_dev->perst_id;
4435		scsi_tgt_priv_data->dev_type = tgt_dev->dev_type;
4436		scsi_tgt_priv_data->tgt_dev = tgt_dev;
4437		tgt_dev->starget = starget;
4438		atomic_set(&scsi_tgt_priv_data->block_io, 0);
4439		retval = 0;
4440		scsi_tgt_priv_data->io_throttle_enabled =
4441		    tgt_dev->io_throttle_enabled;
4442		if (tgt_dev->dev_type == MPI3_DEVICE_DEVFORM_VD)
4443			scsi_tgt_priv_data->throttle_group =
4444			    tgt_dev->dev_spec.vd_inf.tg;
4445	}
4446	spin_unlock_irqrestore(&mrioc->tgtdev_lock, flags);
4447
4448	return retval;
4449}
4450
4451/**
4452 * mpi3mr_check_return_unmap - Whether an unmap is allowed
4453 * @mrioc: Adapter instance reference
4454 * @scmd: SCSI Command reference
4455 *
4456 * The controller hardware cannot handle certain unmap commands
4457 * for NVMe drives, this routine checks those and return true
4458 * and completes the SCSI command with proper status and sense
4459 * data.
4460 *
4461 * Return: TRUE for not  allowed unmap, FALSE otherwise.
4462 */
4463static bool mpi3mr_check_return_unmap(struct mpi3mr_ioc *mrioc,
4464	struct scsi_cmnd *scmd)
4465{
4466	unsigned char *buf;
4467	u16 param_len, desc_len, trunc_param_len;
4468
4469	trunc_param_len = param_len = get_unaligned_be16(scmd->cmnd + 7);
4470
4471	if (mrioc->pdev->revision) {
4472		if ((param_len > 24) && ((param_len - 8) & 0xF)) {
4473			trunc_param_len -= (param_len - 8) & 0xF;
4474			dprint_scsi_command(mrioc, scmd, MPI3_DEBUG_SCSI_ERROR);
4475			dprint_scsi_err(mrioc,
4476			    "truncating param_len from (%d) to (%d)\n",
4477			    param_len, trunc_param_len);
4478			put_unaligned_be16(trunc_param_len, scmd->cmnd + 7);
4479			dprint_scsi_command(mrioc, scmd, MPI3_DEBUG_SCSI_ERROR);
4480		}
4481		return false;
4482	}
4483
4484	if (!param_len) {
4485		ioc_warn(mrioc,
4486		    "%s: cdb received with zero parameter length\n",
4487		    __func__);
4488		scsi_print_command(scmd);
4489		scmd->result = DID_OK << 16;
4490		scsi_done(scmd);
4491		return true;
4492	}
4493
4494	if (param_len < 24) {
4495		ioc_warn(mrioc,
4496		    "%s: cdb received with invalid param_len: %d\n",
4497		    __func__, param_len);
4498		scsi_print_command(scmd);
4499		scmd->result = SAM_STAT_CHECK_CONDITION;
4500		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
4501		    0x1A, 0);
4502		scsi_done(scmd);
4503		return true;
4504	}
4505	if (param_len != scsi_bufflen(scmd)) {
4506		ioc_warn(mrioc,
4507		    "%s: cdb received with param_len: %d bufflen: %d\n",
4508		    __func__, param_len, scsi_bufflen(scmd));
4509		scsi_print_command(scmd);
4510		scmd->result = SAM_STAT_CHECK_CONDITION;
4511		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
4512		    0x1A, 0);
4513		scsi_done(scmd);
4514		return true;
4515	}
4516	buf = kzalloc(scsi_bufflen(scmd), GFP_ATOMIC);
4517	if (!buf) {
4518		scsi_print_command(scmd);
4519		scmd->result = SAM_STAT_CHECK_CONDITION;
4520		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
4521		    0x55, 0x03);
4522		scsi_done(scmd);
4523		return true;
4524	}
4525	scsi_sg_copy_to_buffer(scmd, buf, scsi_bufflen(scmd));
4526	desc_len = get_unaligned_be16(&buf[2]);
4527
4528	if (desc_len < 16) {
4529		ioc_warn(mrioc,
4530		    "%s: Invalid descriptor length in param list: %d\n",
4531		    __func__, desc_len);
4532		scsi_print_command(scmd);
4533		scmd->result = SAM_STAT_CHECK_CONDITION;
4534		scsi_build_sense_buffer(0, scmd->sense_buffer, ILLEGAL_REQUEST,
4535		    0x26, 0);
4536		scsi_done(scmd);
4537		kfree(buf);
4538		return true;
4539	}
4540
4541	if (param_len > (desc_len + 8)) {
4542		trunc_param_len = desc_len + 8;
4543		scsi_print_command(scmd);
4544		dprint_scsi_err(mrioc,
4545		    "truncating param_len(%d) to desc_len+8(%d)\n",
4546		    param_len, trunc_param_len);
4547		put_unaligned_be16(trunc_param_len, scmd->cmnd + 7);
4548		scsi_print_command(scmd);
4549	}
4550
4551	kfree(buf);
4552	return false;
4553}
4554
4555/**
4556 * mpi3mr_allow_scmd_to_fw - Command is allowed during shutdown
4557 * @scmd: SCSI Command reference
4558 *
4559 * Checks whether a cdb is allowed during shutdown or not.
4560 *
4561 * Return: TRUE for allowed commands, FALSE otherwise.
4562 */
4563
4564inline bool mpi3mr_allow_scmd_to_fw(struct scsi_cmnd *scmd)
4565{
4566	switch (scmd->cmnd[0]) {
4567	case SYNCHRONIZE_CACHE:
4568	case START_STOP:
4569		return true;
4570	default:
4571		return false;
4572	}
4573}
4574
4575/**
4576 * mpi3mr_qcmd - I/O request despatcher
4577 * @shost: SCSI Host reference
4578 * @scmd: SCSI Command reference
4579 *
4580 * Issues the SCSI Command as an MPI3 request.
4581 *
4582 * Return: 0 on successful queueing of the request or if the
4583 *         request is completed with failure.
4584 *         SCSI_MLQUEUE_DEVICE_BUSY when the device is busy.
4585 *         SCSI_MLQUEUE_HOST_BUSY when the host queue is full.
4586 */
4587static int mpi3mr_qcmd(struct Scsi_Host *shost,
4588	struct scsi_cmnd *scmd)
4589{
4590	struct mpi3mr_ioc *mrioc = shost_priv(shost);
4591	struct mpi3mr_stgt_priv_data *stgt_priv_data;
4592	struct mpi3mr_sdev_priv_data *sdev_priv_data;
4593	struct scmd_priv *scmd_priv_data = NULL;
4594	struct mpi3_scsi_io_request *scsiio_req = NULL;
4595	struct op_req_qinfo *op_req_q = NULL;
4596	int retval = 0;
4597	u16 dev_handle;
4598	u16 host_tag;
4599	u32 scsiio_flags = 0, data_len_blks = 0;
4600	struct request *rq = scsi_cmd_to_rq(scmd);
4601	int iprio_class;
4602	u8 is_pcie_dev = 0;
4603	u32 tracked_io_sz = 0;
4604	u32 ioc_pend_data_len = 0, tg_pend_data_len = 0;
4605	struct mpi3mr_throttle_group_info *tg = NULL;
4606
4607	if (mrioc->unrecoverable) {
4608		scmd->result = DID_ERROR << 16;
4609		scsi_done(scmd);
4610		goto out;
4611	}
4612
4613	sdev_priv_data = scmd->device->hostdata;
4614	if (!sdev_priv_data || !sdev_priv_data->tgt_priv_data) {
4615		scmd->result = DID_NO_CONNECT << 16;
4616		scsi_done(scmd);
4617		goto out;
4618	}
4619
4620	if (mrioc->stop_drv_processing &&
4621	    !(mpi3mr_allow_scmd_to_fw(scmd))) {
4622		scmd->result = DID_NO_CONNECT << 16;
4623		scsi_done(scmd);
4624		goto out;
4625	}
4626
4627	if (mrioc->reset_in_progress) {
4628		retval = SCSI_MLQUEUE_HOST_BUSY;
4629		goto out;
4630	}
4631
4632	stgt_priv_data = sdev_priv_data->tgt_priv_data;
4633
4634	if (atomic_read(&stgt_priv_data->block_io)) {
4635		if (mrioc->stop_drv_processing) {
4636			scmd->result = DID_NO_CONNECT << 16;
4637			scsi_done(scmd);
4638			goto out;
4639		}
4640		retval = SCSI_MLQUEUE_DEVICE_BUSY;
4641		goto out;
4642	}
4643
4644	dev_handle = stgt_priv_data->dev_handle;
4645	if (dev_handle == MPI3MR_INVALID_DEV_HANDLE) {
4646		scmd->result = DID_NO_CONNECT << 16;
4647		scsi_done(scmd);
4648		goto out;
4649	}
4650	if (stgt_priv_data->dev_removed) {
4651		scmd->result = DID_NO_CONNECT << 16;
4652		scsi_done(scmd);
4653		goto out;
4654	}
4655
4656	if (stgt_priv_data->dev_type == MPI3_DEVICE_DEVFORM_PCIE)
4657		is_pcie_dev = 1;
4658	if ((scmd->cmnd[0] == UNMAP) && is_pcie_dev &&
4659	    (mrioc->pdev->device == MPI3_MFGPAGE_DEVID_SAS4116) &&
4660	    mpi3mr_check_return_unmap(mrioc, scmd))
4661		goto out;
4662
4663	host_tag = mpi3mr_host_tag_for_scmd(mrioc, scmd);
4664	if (host_tag == MPI3MR_HOSTTAG_INVALID) {
4665		scmd->result = DID_ERROR << 16;
4666		scsi_done(scmd);
4667		goto out;
4668	}
4669
4670	if (scmd->sc_data_direction == DMA_FROM_DEVICE)
4671		scsiio_flags = MPI3_SCSIIO_FLAGS_DATADIRECTION_READ;
4672	else if (scmd->sc_data_direction == DMA_TO_DEVICE)
4673		scsiio_flags = MPI3_SCSIIO_FLAGS_DATADIRECTION_WRITE;
4674	else
4675		scsiio_flags = MPI3_SCSIIO_FLAGS_DATADIRECTION_NO_DATA_TRANSFER;
4676
4677	scsiio_flags |= MPI3_SCSIIO_FLAGS_TASKATTRIBUTE_SIMPLEQ;
4678
4679	if (sdev_priv_data->ncq_prio_enable) {
4680		iprio_class = IOPRIO_PRIO_CLASS(req_get_ioprio(rq));
4681		if (iprio_class == IOPRIO_CLASS_RT)
4682			scsiio_flags |= 1 << MPI3_SCSIIO_FLAGS_CMDPRI_SHIFT;
4683	}
4684
4685	if (scmd->cmd_len > 16)
4686		scsiio_flags |= MPI3_SCSIIO_FLAGS_CDB_GREATER_THAN_16;
4687
4688	scmd_priv_data = scsi_cmd_priv(scmd);
4689	memset(scmd_priv_data->mpi3mr_scsiio_req, 0, MPI3MR_ADMIN_REQ_FRAME_SZ);
4690	scsiio_req = (struct mpi3_scsi_io_request *)scmd_priv_data->mpi3mr_scsiio_req;
4691	scsiio_req->function = MPI3_FUNCTION_SCSI_IO;
4692	scsiio_req->host_tag = cpu_to_le16(host_tag);
4693
4694	mpi3mr_setup_eedp(mrioc, scmd, scsiio_req);
4695
4696	memcpy(scsiio_req->cdb.cdb32, scmd->cmnd, scmd->cmd_len);
4697	scsiio_req->data_length = cpu_to_le32(scsi_bufflen(scmd));
4698	scsiio_req->dev_handle = cpu_to_le16(dev_handle);
4699	scsiio_req->flags = cpu_to_le32(scsiio_flags);
4700	int_to_scsilun(sdev_priv_data->lun_id,
4701	    (struct scsi_lun *)scsiio_req->lun);
4702
4703	if (mpi3mr_build_sg_scmd(mrioc, scmd, scsiio_req)) {
4704		mpi3mr_clear_scmd_priv(mrioc, scmd);
4705		retval = SCSI_MLQUEUE_HOST_BUSY;
4706		goto out;
4707	}
4708	op_req_q = &mrioc->req_qinfo[scmd_priv_data->req_q_idx];
4709	data_len_blks = scsi_bufflen(scmd) >> 9;
4710	if ((data_len_blks >= mrioc->io_throttle_data_length) &&
4711	    stgt_priv_data->io_throttle_enabled) {
4712		tracked_io_sz = data_len_blks;
4713		tg = stgt_priv_data->throttle_group;
4714		if (tg) {
4715			ioc_pend_data_len = atomic_add_return(data_len_blks,
4716			    &mrioc->pend_large_data_sz);
4717			tg_pend_data_len = atomic_add_return(data_len_blks,
4718			    &tg->pend_large_data_sz);
4719			if (!tg->io_divert  && ((ioc_pend_data_len >=
4720			    mrioc->io_throttle_high) ||
4721			    (tg_pend_data_len >= tg->high))) {
4722				tg->io_divert = 1;
4723				tg->need_qd_reduction = 1;
4724				mpi3mr_set_io_divert_for_all_vd_in_tg(mrioc,
4725				    tg, 1);
4726				mpi3mr_queue_qd_reduction_event(mrioc, tg);
4727			}
4728		} else {
4729			ioc_pend_data_len = atomic_add_return(data_len_blks,
4730			    &mrioc->pend_large_data_sz);
4731			if (ioc_pend_data_len >= mrioc->io_throttle_high)
4732				stgt_priv_data->io_divert = 1;
4733		}
4734	}
4735
4736	if (stgt_priv_data->io_divert) {
4737		scsiio_req->msg_flags |=
4738		    MPI3_SCSIIO_MSGFLAGS_DIVERT_TO_FIRMWARE;
4739		scsiio_flags |= MPI3_SCSIIO_FLAGS_DIVERT_REASON_IO_THROTTLING;
4740	}
4741	scsiio_req->flags = cpu_to_le32(scsiio_flags);
4742
4743	if (mpi3mr_op_request_post(mrioc, op_req_q,
4744	    scmd_priv_data->mpi3mr_scsiio_req)) {
4745		mpi3mr_clear_scmd_priv(mrioc, scmd);
4746		retval = SCSI_MLQUEUE_HOST_BUSY;
4747		if (tracked_io_sz) {
4748			atomic_sub(tracked_io_sz, &mrioc->pend_large_data_sz);
4749			if (tg)
4750				atomic_sub(tracked_io_sz,
4751				    &tg->pend_large_data_sz);
4752		}
4753		goto out;
4754	}
4755
4756out:
4757	return retval;
4758}
4759
4760static struct scsi_host_template mpi3mr_driver_template = {
4761	.module				= THIS_MODULE,
4762	.name				= "MPI3 Storage Controller",
4763	.proc_name			= MPI3MR_DRIVER_NAME,
4764	.queuecommand			= mpi3mr_qcmd,
4765	.target_alloc			= mpi3mr_target_alloc,
4766	.slave_alloc			= mpi3mr_slave_alloc,
4767	.slave_configure		= mpi3mr_slave_configure,
4768	.target_destroy			= mpi3mr_target_destroy,
4769	.slave_destroy			= mpi3mr_slave_destroy,
4770	.scan_finished			= mpi3mr_scan_finished,
4771	.scan_start			= mpi3mr_scan_start,
4772	.change_queue_depth		= mpi3mr_change_queue_depth,
4773	.eh_device_reset_handler	= mpi3mr_eh_dev_reset,
4774	.eh_target_reset_handler	= mpi3mr_eh_target_reset,
4775	.eh_host_reset_handler		= mpi3mr_eh_host_reset,
4776	.bios_param			= mpi3mr_bios_param,
4777	.map_queues			= mpi3mr_map_queues,
4778	.mq_poll                        = mpi3mr_blk_mq_poll,
4779	.no_write_same			= 1,
4780	.can_queue			= 1,
4781	.this_id			= -1,
4782	.sg_tablesize			= MPI3MR_SG_DEPTH,
4783	/* max xfer supported is 1M (2K in 512 byte sized sectors)
4784	 */
4785	.max_sectors			= 2048,
4786	.cmd_per_lun			= MPI3MR_MAX_CMDS_LUN,
4787	.max_segment_size		= 0xffffffff,
4788	.track_queue_depth		= 1,
4789	.cmd_size			= sizeof(struct scmd_priv),
4790	.shost_groups			= mpi3mr_host_groups,
4791	.sdev_groups			= mpi3mr_dev_groups,
4792};
4793
4794/**
4795 * mpi3mr_init_drv_cmd - Initialize internal command tracker
4796 * @cmdptr: Internal command tracker
4797 * @host_tag: Host tag used for the specific command
4798 *
4799 * Initialize the internal command tracker structure with
4800 * specified host tag.
4801 *
4802 * Return: Nothing.
4803 */
4804static inline void mpi3mr_init_drv_cmd(struct mpi3mr_drv_cmd *cmdptr,
4805	u16 host_tag)
4806{
4807	mutex_init(&cmdptr->mutex);
4808	cmdptr->reply = NULL;
4809	cmdptr->state = MPI3MR_CMD_NOTUSED;
4810	cmdptr->dev_handle = MPI3MR_INVALID_DEV_HANDLE;
4811	cmdptr->host_tag = host_tag;
4812}
4813
4814/**
4815 * osintfc_mrioc_security_status -Check controller secure status
4816 * @pdev: PCI device instance
4817 *
4818 * Read the Device Serial Number capability from PCI config
4819 * space and decide whether the controller is secure or not.
4820 *
4821 * Return: 0 on success, non-zero on failure.
4822 */
4823static int
4824osintfc_mrioc_security_status(struct pci_dev *pdev)
4825{
4826	u32 cap_data;
4827	int base;
4828	u32 ctlr_status;
4829	u32 debug_status;
4830	int retval = 0;
4831
4832	base = pci_find_ext_capability(pdev, PCI_EXT_CAP_ID_DSN);
4833	if (!base) {
4834		dev_err(&pdev->dev,
4835		    "%s: PCI_EXT_CAP_ID_DSN is not supported\n", __func__);
4836		return -1;
4837	}
4838
4839	pci_read_config_dword(pdev, base + 4, &cap_data);
4840
4841	debug_status = cap_data & MPI3MR_CTLR_SECURE_DBG_STATUS_MASK;
4842	ctlr_status = cap_data & MPI3MR_CTLR_SECURITY_STATUS_MASK;
4843
4844	switch (ctlr_status) {
4845	case MPI3MR_INVALID_DEVICE:
4846		dev_err(&pdev->dev,
4847		    "%s: Non secure ctlr (Invalid) is detected: DID: 0x%x: SVID: 0x%x: SDID: 0x%x\n",
4848		    __func__, pdev->device, pdev->subsystem_vendor,
4849		    pdev->subsystem_device);
4850		retval = -1;
4851		break;
4852	case MPI3MR_CONFIG_SECURE_DEVICE:
4853		if (!debug_status)
4854			dev_info(&pdev->dev,
4855			    "%s: Config secure ctlr is detected\n",
4856			    __func__);
4857		break;
4858	case MPI3MR_HARD_SECURE_DEVICE:
4859		break;
4860	case MPI3MR_TAMPERED_DEVICE:
4861		dev_err(&pdev->dev,
4862		    "%s: Non secure ctlr (Tampered) is detected: DID: 0x%x: SVID: 0x%x: SDID: 0x%x\n",
4863		    __func__, pdev->device, pdev->subsystem_vendor,
4864		    pdev->subsystem_device);
4865		retval = -1;
4866		break;
4867	default:
4868		retval = -1;
4869			break;
4870	}
4871
4872	if (!retval && debug_status) {
4873		dev_err(&pdev->dev,
4874		    "%s: Non secure ctlr (Secure Dbg) is detected: DID: 0x%x: SVID: 0x%x: SDID: 0x%x\n",
4875		    __func__, pdev->device, pdev->subsystem_vendor,
4876		    pdev->subsystem_device);
4877		retval = -1;
4878	}
4879
4880	return retval;
4881}
4882
4883/**
4884 * mpi3mr_probe - PCI probe callback
4885 * @pdev: PCI device instance
4886 * @id: PCI device ID details
4887 *
4888 * controller initialization routine. Checks the security status
4889 * of the controller and if it is invalid or tampered return the
4890 * probe without initializing the controller. Otherwise,
4891 * allocate per adapter instance through shost_priv and
4892 * initialize controller specific data structures, initializae
4893 * the controller hardware, add shost to the SCSI subsystem.
4894 *
4895 * Return: 0 on success, non-zero on failure.
4896 */
4897
4898static int
4899mpi3mr_probe(struct pci_dev *pdev, const struct pci_device_id *id)
4900{
4901	struct mpi3mr_ioc *mrioc = NULL;
4902	struct Scsi_Host *shost = NULL;
4903	int retval = 0, i;
4904
4905	if (osintfc_mrioc_security_status(pdev)) {
4906		warn_non_secure_ctlr = 1;
4907		return 1; /* For Invalid and Tampered device */
4908	}
4909
4910	shost = scsi_host_alloc(&mpi3mr_driver_template,
4911	    sizeof(struct mpi3mr_ioc));
4912	if (!shost) {
4913		retval = -ENODEV;
4914		goto shost_failed;
4915	}
4916
4917	mrioc = shost_priv(shost);
4918	mrioc->id = mrioc_ids++;
4919	sprintf(mrioc->driver_name, "%s", MPI3MR_DRIVER_NAME);
4920	sprintf(mrioc->name, "%s%d", mrioc->driver_name, mrioc->id);
4921	INIT_LIST_HEAD(&mrioc->list);
4922	spin_lock(&mrioc_list_lock);
4923	list_add_tail(&mrioc->list, &mrioc_list);
4924	spin_unlock(&mrioc_list_lock);
4925
4926	spin_lock_init(&mrioc->admin_req_lock);
4927	spin_lock_init(&mrioc->reply_free_queue_lock);
4928	spin_lock_init(&mrioc->sbq_lock);
4929	spin_lock_init(&mrioc->fwevt_lock);
4930	spin_lock_init(&mrioc->tgtdev_lock);
4931	spin_lock_init(&mrioc->watchdog_lock);
4932	spin_lock_init(&mrioc->chain_buf_lock);
4933	spin_lock_init(&mrioc->sas_node_lock);
4934
4935	INIT_LIST_HEAD(&mrioc->fwevt_list);
4936	INIT_LIST_HEAD(&mrioc->tgtdev_list);
4937	INIT_LIST_HEAD(&mrioc->delayed_rmhs_list);
4938	INIT_LIST_HEAD(&mrioc->delayed_evtack_cmds_list);
4939	INIT_LIST_HEAD(&mrioc->sas_expander_list);
4940	INIT_LIST_HEAD(&mrioc->hba_port_table_list);
4941	INIT_LIST_HEAD(&mrioc->enclosure_list);
4942
4943	mutex_init(&mrioc->reset_mutex);
4944	mpi3mr_init_drv_cmd(&mrioc->init_cmds, MPI3MR_HOSTTAG_INITCMDS);
4945	mpi3mr_init_drv_cmd(&mrioc->host_tm_cmds, MPI3MR_HOSTTAG_BLK_TMS);
4946	mpi3mr_init_drv_cmd(&mrioc->bsg_cmds, MPI3MR_HOSTTAG_BSG_CMDS);
4947	mpi3mr_init_drv_cmd(&mrioc->cfg_cmds, MPI3MR_HOSTTAG_CFG_CMDS);
4948	mpi3mr_init_drv_cmd(&mrioc->transport_cmds,
4949	    MPI3MR_HOSTTAG_TRANSPORT_CMDS);
4950
4951	for (i = 0; i < MPI3MR_NUM_DEVRMCMD; i++)
4952		mpi3mr_init_drv_cmd(&mrioc->dev_rmhs_cmds[i],
4953		    MPI3MR_HOSTTAG_DEVRMCMD_MIN + i);
4954
4955	if (pdev->revision)
4956		mrioc->enable_segqueue = true;
4957
4958	init_waitqueue_head(&mrioc->reset_waitq);
4959	mrioc->logging_level = logging_level;
4960	mrioc->shost = shost;
4961	mrioc->pdev = pdev;
4962	mrioc->stop_bsgs = 1;
4963
4964	/* init shost parameters */
4965	shost->max_cmd_len = MPI3MR_MAX_CDB_LENGTH;
4966	shost->max_lun = -1;
4967	shost->unique_id = mrioc->id;
4968
4969	shost->max_channel = 0;
4970	shost->max_id = 0xFFFFFFFF;
4971
4972	shost->host_tagset = 1;
4973
4974	if (prot_mask >= 0)
4975		scsi_host_set_prot(shost, prot_mask);
4976	else {
4977		prot_mask = SHOST_DIF_TYPE1_PROTECTION
4978		    | SHOST_DIF_TYPE2_PROTECTION
4979		    | SHOST_DIF_TYPE3_PROTECTION;
4980		scsi_host_set_prot(shost, prot_mask);
4981	}
4982
4983	ioc_info(mrioc,
4984	    "%s :host protection capabilities enabled %s%s%s%s%s%s%s\n",
4985	    __func__,
4986	    (prot_mask & SHOST_DIF_TYPE1_PROTECTION) ? " DIF1" : "",
4987	    (prot_mask & SHOST_DIF_TYPE2_PROTECTION) ? " DIF2" : "",
4988	    (prot_mask & SHOST_DIF_TYPE3_PROTECTION) ? " DIF3" : "",
4989	    (prot_mask & SHOST_DIX_TYPE0_PROTECTION) ? " DIX0" : "",
4990	    (prot_mask & SHOST_DIX_TYPE1_PROTECTION) ? " DIX1" : "",
4991	    (prot_mask & SHOST_DIX_TYPE2_PROTECTION) ? " DIX2" : "",
4992	    (prot_mask & SHOST_DIX_TYPE3_PROTECTION) ? " DIX3" : "");
4993
4994	if (prot_guard_mask)
4995		scsi_host_set_guard(shost, (prot_guard_mask & 3));
4996	else
4997		scsi_host_set_guard(shost, SHOST_DIX_GUARD_CRC);
4998
4999	snprintf(mrioc->fwevt_worker_name, sizeof(mrioc->fwevt_worker_name),
5000	    "%s%d_fwevt_wrkr", mrioc->driver_name, mrioc->id);
5001	mrioc->fwevt_worker_thread = alloc_ordered_workqueue(
5002	    mrioc->fwevt_worker_name, 0);
5003	if (!mrioc->fwevt_worker_thread) {
5004		ioc_err(mrioc, "failure at %s:%d/%s()!\n",
5005		    __FILE__, __LINE__, __func__);
5006		retval = -ENODEV;
5007		goto fwevtthread_failed;
5008	}
5009
5010	mrioc->is_driver_loading = 1;
5011	mrioc->cpu_count = num_online_cpus();
5012	if (mpi3mr_setup_resources(mrioc)) {
5013		ioc_err(mrioc, "setup resources failed\n");
5014		retval = -ENODEV;
5015		goto resource_alloc_failed;
5016	}
5017	if (mpi3mr_init_ioc(mrioc)) {
5018		ioc_err(mrioc, "initializing IOC failed\n");
5019		retval = -ENODEV;
5020		goto init_ioc_failed;
5021	}
5022
5023	shost->nr_hw_queues = mrioc->num_op_reply_q;
5024	if (mrioc->active_poll_qcount)
5025		shost->nr_maps = 3;
5026
5027	shost->can_queue = mrioc->max_host_ios;
5028	shost->sg_tablesize = MPI3MR_SG_DEPTH;
5029	shost->max_id = mrioc->facts.max_perids + 1;
5030
5031	retval = scsi_add_host(shost, &pdev->dev);
5032	if (retval) {
5033		ioc_err(mrioc, "failure at %s:%d/%s()!\n",
5034		    __FILE__, __LINE__, __func__);
5035		goto addhost_failed;
5036	}
5037
5038	scsi_scan_host(shost);
5039	mpi3mr_bsg_init(mrioc);
5040	return retval;
5041
5042addhost_failed:
5043	mpi3mr_stop_watchdog(mrioc);
5044	mpi3mr_cleanup_ioc(mrioc);
5045init_ioc_failed:
5046	mpi3mr_free_mem(mrioc);
5047	mpi3mr_cleanup_resources(mrioc);
5048resource_alloc_failed:
5049	destroy_workqueue(mrioc->fwevt_worker_thread);
5050fwevtthread_failed:
5051	spin_lock(&mrioc_list_lock);
5052	list_del(&mrioc->list);
5053	spin_unlock(&mrioc_list_lock);
5054	scsi_host_put(shost);
5055shost_failed:
5056	return retval;
5057}
5058
5059/**
5060 * mpi3mr_remove - PCI remove callback
5061 * @pdev: PCI device instance
5062 *
5063 * Cleanup the IOC by issuing MUR and shutdown notification.
5064 * Free up all memory and resources associated with the
5065 * controllerand target devices, unregister the shost.
5066 *
5067 * Return: Nothing.
5068 */
5069static void mpi3mr_remove(struct pci_dev *pdev)
5070{
5071	struct Scsi_Host *shost = pci_get_drvdata(pdev);
5072	struct mpi3mr_ioc *mrioc;
5073	struct workqueue_struct	*wq;
5074	unsigned long flags;
5075	struct mpi3mr_tgt_dev *tgtdev, *tgtdev_next;
5076
5077	if (!shost)
5078		return;
5079
5080	mrioc = shost_priv(shost);
5081	while (mrioc->reset_in_progress || mrioc->is_driver_loading)
5082		ssleep(1);
5083
5084	if (!pci_device_is_present(mrioc->pdev)) {
5085		mrioc->unrecoverable = 1;
5086		mpi3mr_flush_cmds_for_unrecovered_controller(mrioc);
5087	}
5088
5089	mpi3mr_bsg_exit(mrioc);
5090	mrioc->stop_drv_processing = 1;
5091	mpi3mr_cleanup_fwevt_list(mrioc);
5092	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
5093	wq = mrioc->fwevt_worker_thread;
5094	mrioc->fwevt_worker_thread = NULL;
5095	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
5096	if (wq)
5097		destroy_workqueue(wq);
5098
5099	if (mrioc->sas_transport_enabled)
5100		sas_remove_host(shost);
5101	else
5102		scsi_remove_host(shost);
5103
5104	list_for_each_entry_safe(tgtdev, tgtdev_next, &mrioc->tgtdev_list,
5105	    list) {
5106		mpi3mr_remove_tgtdev_from_host(mrioc, tgtdev);
5107		mpi3mr_tgtdev_del_from_list(mrioc, tgtdev);
5108		mpi3mr_tgtdev_put(tgtdev);
5109	}
5110	mpi3mr_stop_watchdog(mrioc);
5111	mpi3mr_cleanup_ioc(mrioc);
5112	mpi3mr_free_mem(mrioc);
5113	mpi3mr_cleanup_resources(mrioc);
5114
5115	spin_lock(&mrioc_list_lock);
5116	list_del(&mrioc->list);
5117	spin_unlock(&mrioc_list_lock);
5118
5119	scsi_host_put(shost);
5120}
5121
5122/**
5123 * mpi3mr_shutdown - PCI shutdown callback
5124 * @pdev: PCI device instance
5125 *
5126 * Free up all memory and resources associated with the
5127 * controller
5128 *
5129 * Return: Nothing.
5130 */
5131static void mpi3mr_shutdown(struct pci_dev *pdev)
5132{
5133	struct Scsi_Host *shost = pci_get_drvdata(pdev);
5134	struct mpi3mr_ioc *mrioc;
5135	struct workqueue_struct	*wq;
5136	unsigned long flags;
5137
5138	if (!shost)
5139		return;
5140
5141	mrioc = shost_priv(shost);
5142	while (mrioc->reset_in_progress || mrioc->is_driver_loading)
5143		ssleep(1);
5144
5145	mrioc->stop_drv_processing = 1;
5146	mpi3mr_cleanup_fwevt_list(mrioc);
5147	spin_lock_irqsave(&mrioc->fwevt_lock, flags);
5148	wq = mrioc->fwevt_worker_thread;
5149	mrioc->fwevt_worker_thread = NULL;
5150	spin_unlock_irqrestore(&mrioc->fwevt_lock, flags);
5151	if (wq)
5152		destroy_workqueue(wq);
5153
5154	mpi3mr_stop_watchdog(mrioc);
5155	mpi3mr_cleanup_ioc(mrioc);
5156	mpi3mr_cleanup_resources(mrioc);
5157}
5158
5159/**
5160 * mpi3mr_suspend - PCI power management suspend callback
5161 * @dev: Device struct
5162 *
5163 * Change the power state to the given value and cleanup the IOC
5164 * by issuing MUR and shutdown notification
5165 *
5166 * Return: 0 always.
5167 */
5168static int __maybe_unused
5169mpi3mr_suspend(struct device *dev)
5170{
5171	struct pci_dev *pdev = to_pci_dev(dev);
5172	struct Scsi_Host *shost = pci_get_drvdata(pdev);
5173	struct mpi3mr_ioc *mrioc;
5174
5175	if (!shost)
5176		return 0;
5177
5178	mrioc = shost_priv(shost);
5179	while (mrioc->reset_in_progress || mrioc->is_driver_loading)
5180		ssleep(1);
5181	mrioc->stop_drv_processing = 1;
5182	mpi3mr_cleanup_fwevt_list(mrioc);
5183	scsi_block_requests(shost);
5184	mpi3mr_stop_watchdog(mrioc);
5185	mpi3mr_cleanup_ioc(mrioc);
5186
5187	ioc_info(mrioc, "pdev=0x%p, slot=%s, entering operating state\n",
5188	    pdev, pci_name(pdev));
5189	mpi3mr_cleanup_resources(mrioc);
5190
5191	return 0;
5192}
5193
5194/**
5195 * mpi3mr_resume - PCI power management resume callback
5196 * @dev: Device struct
5197 *
5198 * Restore the power state to D0 and reinitialize the controller
5199 * and resume I/O operations to the target devices
5200 *
5201 * Return: 0 on success, non-zero on failure
5202 */
5203static int __maybe_unused
5204mpi3mr_resume(struct device *dev)
5205{
5206	struct pci_dev *pdev = to_pci_dev(dev);
5207	struct Scsi_Host *shost = pci_get_drvdata(pdev);
5208	struct mpi3mr_ioc *mrioc;
5209	pci_power_t device_state = pdev->current_state;
5210	int r;
5211
5212	if (!shost)
5213		return 0;
5214
5215	mrioc = shost_priv(shost);
5216
5217	ioc_info(mrioc, "pdev=0x%p, slot=%s, previous operating state [D%d]\n",
5218	    pdev, pci_name(pdev), device_state);
5219	mrioc->pdev = pdev;
5220	mrioc->cpu_count = num_online_cpus();
5221	r = mpi3mr_setup_resources(mrioc);
5222	if (r) {
5223		ioc_info(mrioc, "%s: Setup resources failed[%d]\n",
5224		    __func__, r);
5225		return r;
5226	}
5227
5228	mrioc->stop_drv_processing = 0;
5229	mpi3mr_invalidate_devhandles(mrioc);
5230	mpi3mr_free_enclosure_list(mrioc);
5231	mpi3mr_memset_buffers(mrioc);
5232	r = mpi3mr_reinit_ioc(mrioc, 1);
5233	if (r) {
5234		ioc_err(mrioc, "resuming controller failed[%d]\n", r);
5235		return r;
5236	}
5237	ssleep(MPI3MR_RESET_TOPOLOGY_SETTLE_TIME);
5238	scsi_unblock_requests(shost);
5239	mrioc->device_refresh_on = 0;
5240	mpi3mr_start_watchdog(mrioc);
5241
5242	return 0;
5243}
5244
5245static const struct pci_device_id mpi3mr_pci_id_table[] = {
5246	{
5247		PCI_DEVICE_SUB(MPI3_MFGPAGE_VENDORID_BROADCOM,
5248		    MPI3_MFGPAGE_DEVID_SAS4116, PCI_ANY_ID, PCI_ANY_ID)
5249	},
5250	{ 0 }
5251};
5252MODULE_DEVICE_TABLE(pci, mpi3mr_pci_id_table);
5253
5254static SIMPLE_DEV_PM_OPS(mpi3mr_pm_ops, mpi3mr_suspend, mpi3mr_resume);
5255
5256static struct pci_driver mpi3mr_pci_driver = {
5257	.name = MPI3MR_DRIVER_NAME,
5258	.id_table = mpi3mr_pci_id_table,
5259	.probe = mpi3mr_probe,
5260	.remove = mpi3mr_remove,
5261	.shutdown = mpi3mr_shutdown,
5262	.driver.pm = &mpi3mr_pm_ops,
5263};
5264
5265static ssize_t event_counter_show(struct device_driver *dd, char *buf)
5266{
5267	return sprintf(buf, "%llu\n", atomic64_read(&event_counter));
5268}
5269static DRIVER_ATTR_RO(event_counter);
5270
5271static int __init mpi3mr_init(void)
5272{
5273	int ret_val;
5274
5275	pr_info("Loading %s version %s\n", MPI3MR_DRIVER_NAME,
5276	    MPI3MR_DRIVER_VERSION);
5277
5278	mpi3mr_transport_template =
5279	    sas_attach_transport(&mpi3mr_transport_functions);
5280	if (!mpi3mr_transport_template) {
5281		pr_err("%s failed to load due to sas transport attach failure\n",
5282		    MPI3MR_DRIVER_NAME);
5283		return -ENODEV;
5284	}
5285
5286	ret_val = pci_register_driver(&mpi3mr_pci_driver);
5287	if (ret_val) {
5288		pr_err("%s failed to load due to pci register driver failure\n",
5289		    MPI3MR_DRIVER_NAME);
5290		goto err_pci_reg_fail;
5291	}
5292
5293	ret_val = driver_create_file(&mpi3mr_pci_driver.driver,
5294				     &driver_attr_event_counter);
5295	if (ret_val)
5296		goto err_event_counter;
5297
5298	return ret_val;
5299
5300err_event_counter:
5301	pci_unregister_driver(&mpi3mr_pci_driver);
5302
5303err_pci_reg_fail:
5304	sas_release_transport(mpi3mr_transport_template);
5305	return ret_val;
5306}
5307
5308static void __exit mpi3mr_exit(void)
5309{
5310	if (warn_non_secure_ctlr)
5311		pr_warn(
5312		    "Unloading %s version %s while managing a non secure controller\n",
5313		    MPI3MR_DRIVER_NAME, MPI3MR_DRIVER_VERSION);
5314	else
5315		pr_info("Unloading %s version %s\n", MPI3MR_DRIVER_NAME,
5316		    MPI3MR_DRIVER_VERSION);
5317
5318	driver_remove_file(&mpi3mr_pci_driver.driver,
5319			   &driver_attr_event_counter);
5320	pci_unregister_driver(&mpi3mr_pci_driver);
5321	sas_release_transport(mpi3mr_transport_template);
5322}
5323
5324module_init(mpi3mr_init);
5325module_exit(mpi3mr_exit);