1/*
   2 * This file is part of the Chelsio FCoE driver for Linux.
   3 *
   4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
   5 *
   6 * This software is available to you under a choice of one of two
   7 * licenses.  You may choose to be licensed under the terms of the GNU
   8 * General Public License (GPL) Version 2, available from the file
   9 * COPYING in the main directory of this source tree, or the
  10 * OpenIB.org BSD license below:
  11 *
  12 *     Redistribution and use in source and binary forms, with or
  13 *     without modification, are permitted provided that the following
  14 *     conditions are met:
  15 *
  16 *      - Redistributions of source code must retain the above
  17 *        copyright notice, this list of conditions and the following
  18 *        disclaimer.
  19 *
  20 *      - Redistributions in binary form must reproduce the above
  21 *        copyright notice, this list of conditions and the following
  22 *        disclaimer in the documentation and/or other materials
  23 *        provided with the distribution.
  24 *
  25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  32 * SOFTWARE.
  33 */
  34
  35#include <linux/device.h>
  36#include <linux/delay.h>
  37#include <linux/ctype.h>
  38#include <linux/kernel.h>
  39#include <linux/slab.h>
  40#include <linux/string.h>
  41#include <linux/compiler.h>
  42#include <linux/export.h>
  43#include <linux/module.h>
  44#include <asm/unaligned.h>
  45#include <asm/page.h>
  46#include <scsi/scsi.h>
  47#include <scsi/scsi_device.h>
  48#include <scsi/scsi_transport_fc.h>
  49
  50#include "csio_hw.h"
  51#include "csio_lnode.h"
  52#include "csio_rnode.h"
  53#include "csio_scsi.h"
  54#include "csio_init.h"
  55
  56int csio_scsi_eqsize = 65536;
  57int csio_scsi_iqlen = 128;
  58int csio_scsi_ioreqs = 2048;
  59uint32_t csio_max_scan_tmo;
  60uint32_t csio_delta_scan_tmo = 5;
  61int csio_lun_qdepth = 32;
  62
  63static int csio_ddp_descs = 128;
  64
  65static int csio_do_abrt_cls(struct csio_hw *,
  66				      struct csio_ioreq *, bool);
  67
  68static void csio_scsis_uninit(struct csio_ioreq *, enum csio_scsi_ev);
  69static void csio_scsis_io_active(struct csio_ioreq *, enum csio_scsi_ev);
  70static void csio_scsis_tm_active(struct csio_ioreq *, enum csio_scsi_ev);
  71static void csio_scsis_aborting(struct csio_ioreq *, enum csio_scsi_ev);
  72static void csio_scsis_closing(struct csio_ioreq *, enum csio_scsi_ev);
  73static void csio_scsis_shost_cmpl_await(struct csio_ioreq *, enum csio_scsi_ev);
  74
  75/*
  76 * csio_scsi_match_io - Match an ioreq with the given SCSI level data.
  77 * @ioreq: The I/O request
  78 * @sld: Level information
  79 *
  80 * Should be called with lock held.
  81 *
  82 */
  83static bool
  84csio_scsi_match_io(struct csio_ioreq *ioreq, struct csio_scsi_level_data *sld)
  85{
  86	struct scsi_cmnd *scmnd = csio_scsi_cmnd(ioreq);
  87
  88	switch (sld->level) {
  89	case CSIO_LEV_LUN:
  90		if (scmnd == NULL)
  91			return false;
  92
  93		return ((ioreq->lnode == sld->lnode) &&
  94			(ioreq->rnode == sld->rnode) &&
  95			((uint64_t)scmnd->device->lun == sld->oslun));
  96
  97	case CSIO_LEV_RNODE:
  98		return ((ioreq->lnode == sld->lnode) &&
  99				(ioreq->rnode == sld->rnode));
 100	case CSIO_LEV_LNODE:
 101		return (ioreq->lnode == sld->lnode);
 102	case CSIO_LEV_ALL:
 103		return true;
 104	default:
 105		return false;
 106	}
 107}
 108
 109/*
 110 * csio_scsi_gather_active_ios - Gather active I/Os based on level
 111 * @scm: SCSI module
 112 * @sld: Level information
 113 * @dest: The queue where these I/Os have to be gathered.
 114 *
 115 * Should be called with lock held.
 116 */
 117static void
 118csio_scsi_gather_active_ios(struct csio_scsim *scm,
 119			    struct csio_scsi_level_data *sld,
 120			    struct list_head *dest)
 121{
 122	struct list_head *tmp, *next;
 123
 124	if (list_empty(&scm->active_q))
 125		return;
 126
 127	/* Just splice the entire active_q into dest */
 128	if (sld->level == CSIO_LEV_ALL) {
 129		list_splice_tail_init(&scm->active_q, dest);
 130		return;
 131	}
 132
 133	list_for_each_safe(tmp, next, &scm->active_q) {
 134		if (csio_scsi_match_io((struct csio_ioreq *)tmp, sld)) {
 135			list_del_init(tmp);
 136			list_add_tail(tmp, dest);
 137		}
 138	}
 139}
 140
 141static inline bool
 142csio_scsi_itnexus_loss_error(uint16_t error)
 143{
 144	switch (error) {
 145	case FW_ERR_LINK_DOWN:
 146	case FW_RDEV_NOT_READY:
 147	case FW_ERR_RDEV_LOST:
 148	case FW_ERR_RDEV_LOGO:
 149	case FW_ERR_RDEV_IMPL_LOGO:
 150		return true;
 151	}
 152	return false;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 153}
 154
 155/*
 156 * csio_scsi_fcp_cmnd - Frame the SCSI FCP command paylod.
 157 * @req: IO req structure.
 158 * @addr: DMA location to place the payload.
 159 *
 160 * This routine is shared between FCP_WRITE, FCP_READ and FCP_CMD requests.
 161 */
 162static inline void
 163csio_scsi_fcp_cmnd(struct csio_ioreq *req, void *addr)
 164{
 165	struct fcp_cmnd *fcp_cmnd = (struct fcp_cmnd *)addr;
 166	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
 167
 168	/* Check for Task Management */
 169	if (likely(csio_priv(scmnd)->fc_tm_flags == 0)) {
 170		int_to_scsilun(scmnd->device->lun, &fcp_cmnd->fc_lun);
 171		fcp_cmnd->fc_tm_flags = 0;
 172		fcp_cmnd->fc_cmdref = 0;
 
 173
 174		memcpy(fcp_cmnd->fc_cdb, scmnd->cmnd, 16);
 175		fcp_cmnd->fc_pri_ta = FCP_PTA_SIMPLE;
 
 176		fcp_cmnd->fc_dl = cpu_to_be32(scsi_bufflen(scmnd));
 177
 178		if (req->nsge)
 179			if (req->datadir == DMA_TO_DEVICE)
 180				fcp_cmnd->fc_flags = FCP_CFL_WRDATA;
 181			else
 182				fcp_cmnd->fc_flags = FCP_CFL_RDDATA;
 183		else
 184			fcp_cmnd->fc_flags = 0;
 185	} else {
 186		memset(fcp_cmnd, 0, sizeof(*fcp_cmnd));
 187		int_to_scsilun(scmnd->device->lun, &fcp_cmnd->fc_lun);
 188		fcp_cmnd->fc_tm_flags = csio_priv(scmnd)->fc_tm_flags;
 189	}
 190}
 191
 192/*
 193 * csio_scsi_init_cmd_wr - Initialize the SCSI CMD WR.
 194 * @req: IO req structure.
 195 * @addr: DMA location to place the payload.
 196 * @size: Size of WR (including FW WR + immed data + rsp SG entry
 197 *
 198 * Wrapper for populating fw_scsi_cmd_wr.
 199 */
 200static inline void
 201csio_scsi_init_cmd_wr(struct csio_ioreq *req, void *addr, uint32_t size)
 202{
 203	struct csio_hw *hw = req->lnode->hwp;
 204	struct csio_rnode *rn = req->rnode;
 205	struct fw_scsi_cmd_wr *wr = (struct fw_scsi_cmd_wr *)addr;
 206	struct csio_dma_buf *dma_buf;
 207	uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
 208
 209	wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_CMD_WR) |
 210					  FW_SCSI_CMD_WR_IMMDLEN(imm));
 211	wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
 212					    FW_WR_LEN16_V(
 213						DIV_ROUND_UP(size, 16)));
 214
 215	wr->cookie = (uintptr_t) req;
 216	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
 217	wr->tmo_val = (uint8_t) req->tmo;
 218	wr->r3 = 0;
 219	memset(&wr->r5, 0, 8);
 220
 221	/* Get RSP DMA buffer */
 222	dma_buf = &req->dma_buf;
 223
 224	/* Prepare RSP SGL */
 225	wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
 226	wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
 227
 228	wr->r6 = 0;
 229
 230	wr->u.fcoe.ctl_pri = 0;
 231	wr->u.fcoe.cp_en_class = 0;
 232	wr->u.fcoe.r4_lo[0] = 0;
 233	wr->u.fcoe.r4_lo[1] = 0;
 234
 235	/* Frame a FCP command */
 236	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)addr +
 237				    sizeof(struct fw_scsi_cmd_wr)));
 238}
 239
 240#define CSIO_SCSI_CMD_WR_SZ(_imm)					\
 241	(sizeof(struct fw_scsi_cmd_wr) +		/* WR size */	\
 242	 ALIGN((_imm), 16))				/* Immed data */
 243
 244#define CSIO_SCSI_CMD_WR_SZ_16(_imm)					\
 245			(ALIGN(CSIO_SCSI_CMD_WR_SZ((_imm)), 16))
 246
 247/*
 248 * csio_scsi_cmd - Create a SCSI CMD WR.
 249 * @req: IO req structure.
 250 *
 251 * Gets a WR slot in the ingress queue and initializes it with SCSI CMD WR.
 252 *
 253 */
 254static inline void
 255csio_scsi_cmd(struct csio_ioreq *req)
 256{
 257	struct csio_wr_pair wrp;
 258	struct csio_hw *hw = req->lnode->hwp;
 259	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
 260	uint32_t size = CSIO_SCSI_CMD_WR_SZ_16(scsim->proto_cmd_len);
 261
 262	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
 263	if (unlikely(req->drv_status != 0))
 264		return;
 265
 266	if (wrp.size1 >= size) {
 267		/* Initialize WR in one shot */
 268		csio_scsi_init_cmd_wr(req, wrp.addr1, size);
 269	} else {
 270		uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
 271
 272		/*
 273		 * Make a temporary copy of the WR and write back
 274		 * the copy into the WR pair.
 275		 */
 276		csio_scsi_init_cmd_wr(req, (void *)tmpwr, size);
 277		memcpy(wrp.addr1, tmpwr, wrp.size1);
 278		memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
 279	}
 280}
 281
 282/*
 283 * csio_scsi_init_ulptx_dsgl - Fill in a ULP_TX_SC_DSGL
 284 * @hw: HW module
 285 * @req: IO request
 286 * @sgl: ULP TX SGL pointer.
 287 *
 288 */
 289static inline void
 290csio_scsi_init_ultptx_dsgl(struct csio_hw *hw, struct csio_ioreq *req,
 291			   struct ulptx_sgl *sgl)
 292{
 293	struct ulptx_sge_pair *sge_pair = NULL;
 294	struct scatterlist *sgel;
 295	uint32_t i = 0;
 296	uint32_t xfer_len;
 297	struct list_head *tmp;
 298	struct csio_dma_buf *dma_buf;
 299	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
 300
 301	sgl->cmd_nsge = htonl(ULPTX_CMD_V(ULP_TX_SC_DSGL) | ULPTX_MORE_F |
 302				     ULPTX_NSGE_V(req->nsge));
 303	/* Now add the data SGLs */
 304	if (likely(!req->dcopy)) {
 305		scsi_for_each_sg(scmnd, sgel, req->nsge, i) {
 306			if (i == 0) {
 307				sgl->addr0 = cpu_to_be64(sg_dma_address(sgel));
 308				sgl->len0 = cpu_to_be32(sg_dma_len(sgel));
 309				sge_pair = (struct ulptx_sge_pair *)(sgl + 1);
 310				continue;
 311			}
 312			if ((i - 1) & 0x1) {
 313				sge_pair->addr[1] = cpu_to_be64(
 314							sg_dma_address(sgel));
 315				sge_pair->len[1] = cpu_to_be32(
 316							sg_dma_len(sgel));
 317				sge_pair++;
 318			} else {
 319				sge_pair->addr[0] = cpu_to_be64(
 320							sg_dma_address(sgel));
 321				sge_pair->len[0] = cpu_to_be32(
 322							sg_dma_len(sgel));
 323			}
 324		}
 325	} else {
 326		/* Program sg elements with driver's DDP buffer */
 327		xfer_len = scsi_bufflen(scmnd);
 328		list_for_each(tmp, &req->gen_list) {
 329			dma_buf = (struct csio_dma_buf *)tmp;
 330			if (i == 0) {
 331				sgl->addr0 = cpu_to_be64(dma_buf->paddr);
 332				sgl->len0 = cpu_to_be32(
 333						min(xfer_len, dma_buf->len));
 334				sge_pair = (struct ulptx_sge_pair *)(sgl + 1);
 335			} else if ((i - 1) & 0x1) {
 336				sge_pair->addr[1] = cpu_to_be64(dma_buf->paddr);
 337				sge_pair->len[1] = cpu_to_be32(
 338						min(xfer_len, dma_buf->len));
 339				sge_pair++;
 340			} else {
 341				sge_pair->addr[0] = cpu_to_be64(dma_buf->paddr);
 342				sge_pair->len[0] = cpu_to_be32(
 343						min(xfer_len, dma_buf->len));
 344			}
 345			xfer_len -= min(xfer_len, dma_buf->len);
 346			i++;
 347		}
 348	}
 349}
 350
 351/*
 352 * csio_scsi_init_read_wr - Initialize the READ SCSI WR.
 353 * @req: IO req structure.
 354 * @wrp: DMA location to place the payload.
 355 * @size: Size of WR (including FW WR + immed data + rsp SG entry + data SGL
 356 *
 357 * Wrapper for populating fw_scsi_read_wr.
 358 */
 359static inline void
 360csio_scsi_init_read_wr(struct csio_ioreq *req, void *wrp, uint32_t size)
 361{
 362	struct csio_hw *hw = req->lnode->hwp;
 363	struct csio_rnode *rn = req->rnode;
 364	struct fw_scsi_read_wr *wr = (struct fw_scsi_read_wr *)wrp;
 365	struct ulptx_sgl *sgl;
 366	struct csio_dma_buf *dma_buf;
 367	uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
 368	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
 369
 370	wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_READ_WR) |
 371				     FW_SCSI_READ_WR_IMMDLEN(imm));
 372	wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
 373				       FW_WR_LEN16_V(DIV_ROUND_UP(size, 16)));
 374	wr->cookie = (uintptr_t)req;
 375	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
 376	wr->tmo_val = (uint8_t)(req->tmo);
 377	wr->use_xfer_cnt = 1;
 378	wr->xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
 379	wr->ini_xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
 380	/* Get RSP DMA buffer */
 381	dma_buf = &req->dma_buf;
 382
 383	/* Prepare RSP SGL */
 384	wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
 385	wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
 386
 387	wr->r4 = 0;
 388
 389	wr->u.fcoe.ctl_pri = 0;
 390	wr->u.fcoe.cp_en_class = 0;
 391	wr->u.fcoe.r3_lo[0] = 0;
 392	wr->u.fcoe.r3_lo[1] = 0;
 393	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +
 394					sizeof(struct fw_scsi_read_wr)));
 395
 396	/* Move WR pointer past command and immediate data */
 397	sgl = (struct ulptx_sgl *)((uintptr_t)wrp +
 398			      sizeof(struct fw_scsi_read_wr) + ALIGN(imm, 16));
 399
 400	/* Fill in the DSGL */
 401	csio_scsi_init_ultptx_dsgl(hw, req, sgl);
 402}
 403
 404/*
 405 * csio_scsi_init_write_wr - Initialize the WRITE SCSI WR.
 406 * @req: IO req structure.
 407 * @wrp: DMA location to place the payload.
 408 * @size: Size of WR (including FW WR + immed data + rsp SG entry + data SGL
 409 *
 410 * Wrapper for populating fw_scsi_write_wr.
 411 */
 412static inline void
 413csio_scsi_init_write_wr(struct csio_ioreq *req, void *wrp, uint32_t size)
 414{
 415	struct csio_hw *hw = req->lnode->hwp;
 416	struct csio_rnode *rn = req->rnode;
 417	struct fw_scsi_write_wr *wr = (struct fw_scsi_write_wr *)wrp;
 418	struct ulptx_sgl *sgl;
 419	struct csio_dma_buf *dma_buf;
 420	uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
 421	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
 422
 423	wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_WRITE_WR) |
 424				     FW_SCSI_WRITE_WR_IMMDLEN(imm));
 425	wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
 426				       FW_WR_LEN16_V(DIV_ROUND_UP(size, 16)));
 427	wr->cookie = (uintptr_t)req;
 428	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
 429	wr->tmo_val = (uint8_t)(req->tmo);
 430	wr->use_xfer_cnt = 1;
 431	wr->xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
 432	wr->ini_xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
 433	/* Get RSP DMA buffer */
 434	dma_buf = &req->dma_buf;
 435
 436	/* Prepare RSP SGL */
 437	wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
 438	wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
 439
 440	wr->r4 = 0;
 441
 442	wr->u.fcoe.ctl_pri = 0;
 443	wr->u.fcoe.cp_en_class = 0;
 444	wr->u.fcoe.r3_lo[0] = 0;
 445	wr->u.fcoe.r3_lo[1] = 0;
 446	csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +
 447					sizeof(struct fw_scsi_write_wr)));
 448
 449	/* Move WR pointer past command and immediate data */
 450	sgl = (struct ulptx_sgl *)((uintptr_t)wrp +
 451			      sizeof(struct fw_scsi_write_wr) + ALIGN(imm, 16));
 452
 453	/* Fill in the DSGL */
 454	csio_scsi_init_ultptx_dsgl(hw, req, sgl);
 455}
 456
 457/* Calculate WR size needed for fw_scsi_read_wr/fw_scsi_write_wr */
 458#define CSIO_SCSI_DATA_WRSZ(req, oper, sz, imm)				       \
 459do {									       \
 460	(sz) = sizeof(struct fw_scsi_##oper##_wr) +	/* WR size */          \
 461	       ALIGN((imm), 16) +			/* Immed data */       \
 462	       sizeof(struct ulptx_sgl);		/* ulptx_sgl */	       \
 463									       \
 464	if (unlikely((req)->nsge > 1))				               \
 465		(sz) += (sizeof(struct ulptx_sge_pair) *		       \
 466				(ALIGN(((req)->nsge - 1), 2) / 2));            \
 467							/* Data SGE */	       \
 468} while (0)
 469
 470/*
 471 * csio_scsi_read - Create a SCSI READ WR.
 472 * @req: IO req structure.
 473 *
 474 * Gets a WR slot in the ingress queue and initializes it with
 475 * SCSI READ WR.
 476 *
 477 */
 478static inline void
 479csio_scsi_read(struct csio_ioreq *req)
 480{
 481	struct csio_wr_pair wrp;
 482	uint32_t size;
 483	struct csio_hw *hw = req->lnode->hwp;
 484	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
 485
 486	CSIO_SCSI_DATA_WRSZ(req, read, size, scsim->proto_cmd_len);
 487	size = ALIGN(size, 16);
 488
 489	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
 490	if (likely(req->drv_status == 0)) {
 491		if (likely(wrp.size1 >= size)) {
 492			/* Initialize WR in one shot */
 493			csio_scsi_init_read_wr(req, wrp.addr1, size);
 494		} else {
 495			uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
 496			/*
 497			 * Make a temporary copy of the WR and write back
 498			 * the copy into the WR pair.
 499			 */
 500			csio_scsi_init_read_wr(req, (void *)tmpwr, size);
 501			memcpy(wrp.addr1, tmpwr, wrp.size1);
 502			memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
 503		}
 504	}
 505}
 506
 507/*
 508 * csio_scsi_write - Create a SCSI WRITE WR.
 509 * @req: IO req structure.
 510 *
 511 * Gets a WR slot in the ingress queue and initializes it with
 512 * SCSI WRITE WR.
 513 *
 514 */
 515static inline void
 516csio_scsi_write(struct csio_ioreq *req)
 517{
 518	struct csio_wr_pair wrp;
 519	uint32_t size;
 520	struct csio_hw *hw = req->lnode->hwp;
 521	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
 522
 523	CSIO_SCSI_DATA_WRSZ(req, write, size, scsim->proto_cmd_len);
 524	size = ALIGN(size, 16);
 525
 526	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
 527	if (likely(req->drv_status == 0)) {
 528		if (likely(wrp.size1 >= size)) {
 529			/* Initialize WR in one shot */
 530			csio_scsi_init_write_wr(req, wrp.addr1, size);
 531		} else {
 532			uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
 533			/*
 534			 * Make a temporary copy of the WR and write back
 535			 * the copy into the WR pair.
 536			 */
 537			csio_scsi_init_write_wr(req, (void *)tmpwr, size);
 538			memcpy(wrp.addr1, tmpwr, wrp.size1);
 539			memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
 540		}
 541	}
 542}
 543
 544/*
 545 * csio_setup_ddp - Setup DDP buffers for Read request.
 546 * @req: IO req structure.
 547 *
 548 * Checks SGLs/Data buffers are virtually contiguous required for DDP.
 549 * If contiguous,driver posts SGLs in the WR otherwise post internal
 550 * buffers for such request for DDP.
 551 */
 552static inline void
 553csio_setup_ddp(struct csio_scsim *scsim, struct csio_ioreq *req)
 554{
 555#ifdef __CSIO_DEBUG__
 556	struct csio_hw *hw = req->lnode->hwp;
 557#endif
 558	struct scatterlist *sgel = NULL;
 559	struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
 560	uint64_t sg_addr = 0;
 561	uint32_t ddp_pagesz = 4096;
 562	uint32_t buf_off;
 563	struct csio_dma_buf *dma_buf = NULL;
 564	uint32_t alloc_len = 0;
 565	uint32_t xfer_len = 0;
 566	uint32_t sg_len = 0;
 567	uint32_t i;
 568
 569	scsi_for_each_sg(scmnd, sgel, req->nsge, i) {
 570		sg_addr = sg_dma_address(sgel);
 571		sg_len	= sg_dma_len(sgel);
 572
 573		buf_off = sg_addr & (ddp_pagesz - 1);
 574
 575		/* Except 1st buffer,all buffer addr have to be Page aligned */
 576		if (i != 0 && buf_off) {
 577			csio_dbg(hw, "SGL addr not DDP aligned (%llx:%d)\n",
 578				 sg_addr, sg_len);
 579			goto unaligned;
 580		}
 581
 582		/* Except last buffer,all buffer must end on page boundary */
 583		if ((i != (req->nsge - 1)) &&
 584			((buf_off + sg_len) & (ddp_pagesz - 1))) {
 585			csio_dbg(hw,
 586				 "SGL addr not ending on page boundary"
 587				 "(%llx:%d)\n", sg_addr, sg_len);
 588			goto unaligned;
 589		}
 590	}
 591
 592	/* SGL's are virtually contiguous. HW will DDP to SGLs */
 593	req->dcopy = 0;
 594	csio_scsi_read(req);
 595
 596	return;
 597
 598unaligned:
 599	CSIO_INC_STATS(scsim, n_unaligned);
 600	/*
 601	 * For unaligned SGLs, driver will allocate internal DDP buffer.
 602	 * Once command is completed data from DDP buffer copied to SGLs
 603	 */
 604	req->dcopy = 1;
 605
 606	/* Use gen_list to store the DDP buffers */
 607	INIT_LIST_HEAD(&req->gen_list);
 608	xfer_len = scsi_bufflen(scmnd);
 609
 610	i = 0;
 611	/* Allocate ddp buffers for this request */
 612	while (alloc_len < xfer_len) {
 613		dma_buf = csio_get_scsi_ddp(scsim);
 614		if (dma_buf == NULL || i > scsim->max_sge) {
 615			req->drv_status = -EBUSY;
 616			break;
 617		}
 618		alloc_len += dma_buf->len;
 619		/* Added to IO req */
 620		list_add_tail(&dma_buf->list, &req->gen_list);
 621		i++;
 622	}
 623
 624	if (!req->drv_status) {
 625		/* set number of ddp bufs used */
 626		req->nsge = i;
 627		csio_scsi_read(req);
 628		return;
 629	}
 630
 631	 /* release dma descs */
 632	if (i > 0)
 633		csio_put_scsi_ddp_list(scsim, &req->gen_list, i);
 634}
 635
 636/*
 637 * csio_scsi_init_abrt_cls_wr - Initialize an ABORT/CLOSE WR.
 638 * @req: IO req structure.
 639 * @addr: DMA location to place the payload.
 640 * @size: Size of WR
 641 * @abort: abort OR close
 642 *
 643 * Wrapper for populating fw_scsi_cmd_wr.
 644 */
 645static inline void
 646csio_scsi_init_abrt_cls_wr(struct csio_ioreq *req, void *addr, uint32_t size,
 647			   bool abort)
 648{
 649	struct csio_hw *hw = req->lnode->hwp;
 650	struct csio_rnode *rn = req->rnode;
 651	struct fw_scsi_abrt_cls_wr *wr = (struct fw_scsi_abrt_cls_wr *)addr;
 652
 653	wr->op_immdlen = cpu_to_be32(FW_WR_OP_V(FW_SCSI_ABRT_CLS_WR));
 654	wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID_V(rn->flowid) |
 655					    FW_WR_LEN16_V(
 656						DIV_ROUND_UP(size, 16)));
 657
 658	wr->cookie = (uintptr_t) req;
 659	wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
 660	wr->tmo_val = (uint8_t) req->tmo;
 661	/* 0 for CHK_ALL_IO tells FW to look up t_cookie */
 662	wr->sub_opcode_to_chk_all_io =
 663				(FW_SCSI_ABRT_CLS_WR_SUB_OPCODE(abort) |
 664				 FW_SCSI_ABRT_CLS_WR_CHK_ALL_IO(0));
 665	wr->r3[0] = 0;
 666	wr->r3[1] = 0;
 667	wr->r3[2] = 0;
 668	wr->r3[3] = 0;
 669	/* Since we re-use the same ioreq for abort as well */
 670	wr->t_cookie = (uintptr_t) req;
 671}
 672
 673static inline void
 674csio_scsi_abrt_cls(struct csio_ioreq *req, bool abort)
 675{
 676	struct csio_wr_pair wrp;
 677	struct csio_hw *hw = req->lnode->hwp;
 678	uint32_t size = ALIGN(sizeof(struct fw_scsi_abrt_cls_wr), 16);
 679
 680	req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
 681	if (req->drv_status != 0)
 682		return;
 683
 684	if (wrp.size1 >= size) {
 685		/* Initialize WR in one shot */
 686		csio_scsi_init_abrt_cls_wr(req, wrp.addr1, size, abort);
 687	} else {
 688		uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
 689		/*
 690		 * Make a temporary copy of the WR and write back
 691		 * the copy into the WR pair.
 692		 */
 693		csio_scsi_init_abrt_cls_wr(req, (void *)tmpwr, size, abort);
 694		memcpy(wrp.addr1, tmpwr, wrp.size1);
 695		memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
 696	}
 697}
 698
 699/*****************************************************************************/
 700/* START: SCSI SM                                                            */
 701/*****************************************************************************/
 702static void
 703csio_scsis_uninit(struct csio_ioreq *req, enum csio_scsi_ev evt)
 704{
 705	struct csio_hw *hw = req->lnode->hwp;
 706	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
 707
 708	switch (evt) {
 709	case CSIO_SCSIE_START_IO:
 710
 711		if (req->nsge) {
 712			if (req->datadir == DMA_TO_DEVICE) {
 713				req->dcopy = 0;
 714				csio_scsi_write(req);
 715			} else
 716				csio_setup_ddp(scsim, req);
 717		} else {
 718			csio_scsi_cmd(req);
 719		}
 720
 721		if (likely(req->drv_status == 0)) {
 722			/* change state and enqueue on active_q */
 723			csio_set_state(&req->sm, csio_scsis_io_active);
 724			list_add_tail(&req->sm.sm_list, &scsim->active_q);
 725			csio_wr_issue(hw, req->eq_idx, false);
 726			CSIO_INC_STATS(scsim, n_active);
 727
 728			return;
 729		}
 730		break;
 731
 732	case CSIO_SCSIE_START_TM:
 733		csio_scsi_cmd(req);
 734		if (req->drv_status == 0) {
 735			/*
 736			 * NOTE: We collect the affected I/Os prior to issuing
 737			 * LUN reset, and not after it. This is to prevent
 738			 * aborting I/Os that get issued after the LUN reset,
 739			 * but prior to LUN reset completion (in the event that
 740			 * the host stack has not blocked I/Os to a LUN that is
 741			 * being reset.
 742			 */
 743			csio_set_state(&req->sm, csio_scsis_tm_active);
 744			list_add_tail(&req->sm.sm_list, &scsim->active_q);
 745			csio_wr_issue(hw, req->eq_idx, false);
 746			CSIO_INC_STATS(scsim, n_tm_active);
 747		}
 748		return;
 749
 750	case CSIO_SCSIE_ABORT:
 751	case CSIO_SCSIE_CLOSE:
 752		/*
 753		 * NOTE:
 754		 * We could get here due to  :
 755		 * - a window in the cleanup path of the SCSI module
 756		 *   (csio_scsi_abort_io()). Please see NOTE in this function.
 757		 * - a window in the time we tried to issue an abort/close
 758		 *   of a request to FW, and the FW completed the request
 759		 *   itself.
 760		 *   Print a message for now, and return INVAL either way.
 761		 */
 762		req->drv_status = -EINVAL;
 763		csio_warn(hw, "Trying to abort/close completed IO:%p!\n", req);
 764		break;
 765
 766	default:
 767		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
 768		CSIO_DB_ASSERT(0);
 769	}
 770}
 771
 772static void
 773csio_scsis_io_active(struct csio_ioreq *req, enum csio_scsi_ev evt)
 774{
 775	struct csio_hw *hw = req->lnode->hwp;
 776	struct csio_scsim *scm = csio_hw_to_scsim(hw);
 777	struct csio_rnode *rn;
 778
 779	switch (evt) {
 780	case CSIO_SCSIE_COMPLETED:
 781		CSIO_DEC_STATS(scm, n_active);
 782		list_del_init(&req->sm.sm_list);
 783		csio_set_state(&req->sm, csio_scsis_uninit);
 784		/*
 785		 * In MSIX mode, with multiple queues, the SCSI compeltions
 786		 * could reach us sooner than the FW events sent to indicate
 787		 * I-T nexus loss (link down, remote device logo etc). We
 788		 * dont want to be returning such I/Os to the upper layer
 789		 * immediately, since we wouldnt have reported the I-T nexus
 790		 * loss itself. This forces us to serialize such completions
 791		 * with the reporting of the I-T nexus loss. Therefore, we
 792		 * internally queue up such up such completions in the rnode.
 793		 * The reporting of I-T nexus loss to the upper layer is then
 794		 * followed by the returning of I/Os in this internal queue.
 795		 * Having another state alongwith another queue helps us take
 796		 * actions for events such as ABORT received while we are
 797		 * in this rnode queue.
 798		 */
 799		if (unlikely(req->wr_status != FW_SUCCESS)) {
 800			rn = req->rnode;
 801			/*
 802			 * FW says remote device is lost, but rnode
 803			 * doesnt reflect it.
 804			 */
 805			if (csio_scsi_itnexus_loss_error(req->wr_status) &&
 806						csio_is_rnode_ready(rn)) {
 807				csio_set_state(&req->sm,
 808						csio_scsis_shost_cmpl_await);
 809				list_add_tail(&req->sm.sm_list,
 810					      &rn->host_cmpl_q);
 811			}
 812		}
 813
 814		break;
 815
 816	case CSIO_SCSIE_ABORT:
 817		csio_scsi_abrt_cls(req, SCSI_ABORT);
 818		if (req->drv_status == 0) {
 819			csio_wr_issue(hw, req->eq_idx, false);
 820			csio_set_state(&req->sm, csio_scsis_aborting);
 821		}
 822		break;
 823
 824	case CSIO_SCSIE_CLOSE:
 825		csio_scsi_abrt_cls(req, SCSI_CLOSE);
 826		if (req->drv_status == 0) {
 827			csio_wr_issue(hw, req->eq_idx, false);
 828			csio_set_state(&req->sm, csio_scsis_closing);
 829		}
 830		break;
 831
 832	case CSIO_SCSIE_DRVCLEANUP:
 833		req->wr_status = FW_HOSTERROR;
 834		CSIO_DEC_STATS(scm, n_active);
 835		csio_set_state(&req->sm, csio_scsis_uninit);
 836		break;
 837
 838	default:
 839		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
 840		CSIO_DB_ASSERT(0);
 841	}
 842}
 843
 844static void
 845csio_scsis_tm_active(struct csio_ioreq *req, enum csio_scsi_ev evt)
 846{
 847	struct csio_hw *hw = req->lnode->hwp;
 848	struct csio_scsim *scm = csio_hw_to_scsim(hw);
 849
 850	switch (evt) {
 851	case CSIO_SCSIE_COMPLETED:
 852		CSIO_DEC_STATS(scm, n_tm_active);
 853		list_del_init(&req->sm.sm_list);
 854		csio_set_state(&req->sm, csio_scsis_uninit);
 855
 856		break;
 857
 858	case CSIO_SCSIE_ABORT:
 859		csio_scsi_abrt_cls(req, SCSI_ABORT);
 860		if (req->drv_status == 0) {
 861			csio_wr_issue(hw, req->eq_idx, false);
 862			csio_set_state(&req->sm, csio_scsis_aborting);
 863		}
 864		break;
 865
 866
 867	case CSIO_SCSIE_CLOSE:
 868		csio_scsi_abrt_cls(req, SCSI_CLOSE);
 869		if (req->drv_status == 0) {
 870			csio_wr_issue(hw, req->eq_idx, false);
 871			csio_set_state(&req->sm, csio_scsis_closing);
 872		}
 873		break;
 874
 875	case CSIO_SCSIE_DRVCLEANUP:
 876		req->wr_status = FW_HOSTERROR;
 877		CSIO_DEC_STATS(scm, n_tm_active);
 878		csio_set_state(&req->sm, csio_scsis_uninit);
 879		break;
 880
 881	default:
 882		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
 883		CSIO_DB_ASSERT(0);
 884	}
 885}
 886
 887static void
 888csio_scsis_aborting(struct csio_ioreq *req, enum csio_scsi_ev evt)
 889{
 890	struct csio_hw *hw = req->lnode->hwp;
 891	struct csio_scsim *scm = csio_hw_to_scsim(hw);
 892
 893	switch (evt) {
 894	case CSIO_SCSIE_COMPLETED:
 895		csio_dbg(hw,
 896			 "ioreq %p recvd cmpltd (wr_status:%d) "
 897			 "in aborting st\n", req, req->wr_status);
 898		/*
 899		 * Use -ECANCELED to explicitly tell the ABORTED event that
 900		 * the original I/O was returned to driver by FW.
 901		 * We dont really care if the I/O was returned with success by
 902		 * FW (because the ABORT and completion of the I/O crossed each
 903		 * other), or any other return value. Once we are in aborting
 904		 * state, the success or failure of the I/O is unimportant to
 905		 * us.
 906		 */
 907		req->drv_status = -ECANCELED;
 908		break;
 909
 910	case CSIO_SCSIE_ABORT:
 911		CSIO_INC_STATS(scm, n_abrt_dups);
 912		break;
 913
 914	case CSIO_SCSIE_ABORTED:
 915
 916		csio_dbg(hw, "abort of %p return status:0x%x drv_status:%x\n",
 917			 req, req->wr_status, req->drv_status);
 918		/*
 919		 * Check if original I/O WR completed before the Abort
 920		 * completion.
 921		 */
 922		if (req->drv_status != -ECANCELED) {
 923			csio_warn(hw,
 924				  "Abort completed before original I/O,"
 925				   " req:%p\n", req);
 926			CSIO_DB_ASSERT(0);
 927		}
 928
 929		/*
 930		 * There are the following possible scenarios:
 931		 * 1. The abort completed successfully, FW returned FW_SUCCESS.
 932		 * 2. The completion of an I/O and the receipt of
 933		 *    abort for that I/O by the FW crossed each other.
 934		 *    The FW returned FW_EINVAL. The original I/O would have
 935		 *    returned with FW_SUCCESS or any other SCSI error.
 936		 * 3. The FW couldn't sent the abort out on the wire, as there
 937		 *    was an I-T nexus loss (link down, remote device logged
 938		 *    out etc). FW sent back an appropriate IT nexus loss status
 939		 *    for the abort.
 940		 * 4. FW sent an abort, but abort timed out (remote device
 941		 *    didnt respond). FW replied back with
 942		 *    FW_SCSI_ABORT_TIMEDOUT.
 943		 * 5. FW couldn't genuinely abort the request for some reason,
 944		 *    and sent us an error.
 945		 *
 946		 * The first 3 scenarios are treated as  succesful abort
 947		 * operations by the host, while the last 2 are failed attempts
 948		 * to abort. Manipulate the return value of the request
 949		 * appropriately, so that host can convey these results
 950		 * back to the upper layer.
 951		 */
 952		if ((req->wr_status == FW_SUCCESS) ||
 953		    (req->wr_status == FW_EINVAL) ||
 954		    csio_scsi_itnexus_loss_error(req->wr_status))
 955			req->wr_status = FW_SCSI_ABORT_REQUESTED;
 956
 957		CSIO_DEC_STATS(scm, n_active);
 958		list_del_init(&req->sm.sm_list);
 959		csio_set_state(&req->sm, csio_scsis_uninit);
 960		break;
 961
 962	case CSIO_SCSIE_DRVCLEANUP:
 963		req->wr_status = FW_HOSTERROR;
 964		CSIO_DEC_STATS(scm, n_active);
 965		csio_set_state(&req->sm, csio_scsis_uninit);
 966		break;
 967
 968	case CSIO_SCSIE_CLOSE:
 969		/*
 970		 * We can receive this event from the module
 971		 * cleanup paths, if the FW forgot to reply to the ABORT WR
 972		 * and left this ioreq in this state. For now, just ignore
 973		 * the event. The CLOSE event is sent to this state, as
 974		 * the LINK may have already gone down.
 975		 */
 976		break;
 977
 978	default:
 979		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
 980		CSIO_DB_ASSERT(0);
 981	}
 982}
 983
 984static void
 985csio_scsis_closing(struct csio_ioreq *req, enum csio_scsi_ev evt)
 986{
 987	struct csio_hw *hw = req->lnode->hwp;
 988	struct csio_scsim *scm = csio_hw_to_scsim(hw);
 989
 990	switch (evt) {
 991	case CSIO_SCSIE_COMPLETED:
 992		csio_dbg(hw,
 993			 "ioreq %p recvd cmpltd (wr_status:%d) "
 994			 "in closing st\n", req, req->wr_status);
 995		/*
 996		 * Use -ECANCELED to explicitly tell the CLOSED event that
 997		 * the original I/O was returned to driver by FW.
 998		 * We dont really care if the I/O was returned with success by
 999		 * FW (because the CLOSE and completion of the I/O crossed each
1000		 * other), or any other return value. Once we are in aborting
1001		 * state, the success or failure of the I/O is unimportant to
1002		 * us.
1003		 */
1004		req->drv_status = -ECANCELED;
1005		break;
1006
1007	case CSIO_SCSIE_CLOSED:
1008		/*
1009		 * Check if original I/O WR completed before the Close
1010		 * completion.
1011		 */
1012		if (req->drv_status != -ECANCELED) {
1013			csio_fatal(hw,
1014				   "Close completed before original I/O,"
1015				   " req:%p\n", req);
1016			CSIO_DB_ASSERT(0);
1017		}
1018
1019		/*
1020		 * Either close succeeded, or we issued close to FW at the
1021		 * same time FW compelted it to us. Either way, the I/O
1022		 * is closed.
1023		 */
1024		CSIO_DB_ASSERT((req->wr_status == FW_SUCCESS) ||
1025					(req->wr_status == FW_EINVAL));
1026		req->wr_status = FW_SCSI_CLOSE_REQUESTED;
1027
1028		CSIO_DEC_STATS(scm, n_active);
1029		list_del_init(&req->sm.sm_list);
1030		csio_set_state(&req->sm, csio_scsis_uninit);
1031		break;
1032
1033	case CSIO_SCSIE_CLOSE:
1034		break;
1035
1036	case CSIO_SCSIE_DRVCLEANUP:
1037		req->wr_status = FW_HOSTERROR;
1038		CSIO_DEC_STATS(scm, n_active);
1039		csio_set_state(&req->sm, csio_scsis_uninit);
1040		break;
1041
1042	default:
1043		csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
1044		CSIO_DB_ASSERT(0);
1045	}
1046}
1047
1048static void
1049csio_scsis_shost_cmpl_await(struct csio_ioreq *req, enum csio_scsi_ev evt)
1050{
1051	switch (evt) {
1052	case CSIO_SCSIE_ABORT:
1053	case CSIO_SCSIE_CLOSE:
1054		/*
1055		 * Just succeed the abort request, and hope that
1056		 * the remote device unregister path will cleanup
1057		 * this I/O to the upper layer within a sane
1058		 * amount of time.
1059		 */
1060		/*
1061		 * A close can come in during a LINK DOWN. The FW would have
1062		 * returned us the I/O back, but not the remote device lost
1063		 * FW event. In this interval, if the I/O times out at the upper
1064		 * layer, a close can come in. Take the same action as abort:
1065		 * return success, and hope that the remote device unregister
1066		 * path will cleanup this I/O. If the FW still doesnt send
1067		 * the msg, the close times out, and the upper layer resorts
1068		 * to the next level of error recovery.
1069		 */
1070		req->drv_status = 0;
1071		break;
1072	case CSIO_SCSIE_DRVCLEANUP:
1073		csio_set_state(&req->sm, csio_scsis_uninit);
1074		break;
1075	default:
1076		csio_dbg(req->lnode->hwp, "Unhandled event:%d sent to req:%p\n",
1077			 evt, req);
1078		CSIO_DB_ASSERT(0);
1079	}
1080}
1081
1082/*
1083 * csio_scsi_cmpl_handler - WR completion handler for SCSI.
1084 * @hw: HW module.
1085 * @wr: The completed WR from the ingress queue.
1086 * @len: Length of the WR.
1087 * @flb: Freelist buffer array.
1088 * @priv: Private object
1089 * @scsiwr: Pointer to SCSI WR.
1090 *
1091 * This is the WR completion handler called per completion from the
1092 * ISR. It is called with lock held. It walks past the RSS and CPL message
1093 * header where the actual WR is present.
1094 * It then gets the status, WR handle (ioreq pointer) and the len of
1095 * the WR, based on WR opcode. Only on a non-good status is the entire
1096 * WR copied into the WR cache (ioreq->fw_wr).
1097 * The ioreq corresponding to the WR is returned to the caller.
1098 * NOTE: The SCSI queue doesnt allocate a freelist today, hence
1099 * no freelist buffer is expected.
1100 */
1101struct csio_ioreq *
1102csio_scsi_cmpl_handler(struct csio_hw *hw, void *wr, uint32_t len,
1103		     struct csio_fl_dma_buf *flb, void *priv, uint8_t **scsiwr)
1104{
1105	struct csio_ioreq *ioreq = NULL;
1106	struct cpl_fw6_msg *cpl;
1107	uint8_t *tempwr;
1108	uint8_t	status;
1109	struct csio_scsim *scm = csio_hw_to_scsim(hw);
1110
1111	/* skip RSS header */
1112	cpl = (struct cpl_fw6_msg *)((uintptr_t)wr + sizeof(__be64));
1113
1114	if (unlikely(cpl->opcode != CPL_FW6_MSG)) {
1115		csio_warn(hw, "Error: Invalid CPL msg %x recvd on SCSI q\n",
1116			  cpl->opcode);
1117		CSIO_INC_STATS(scm, n_inval_cplop);
1118		return NULL;
1119	}
1120
1121	tempwr = (uint8_t *)(cpl->data);
1122	status = csio_wr_status(tempwr);
1123	*scsiwr = tempwr;
1124
1125	if (likely((*tempwr == FW_SCSI_READ_WR) ||
1126			(*tempwr == FW_SCSI_WRITE_WR) ||
1127			(*tempwr == FW_SCSI_CMD_WR))) {
1128		ioreq = (struct csio_ioreq *)((uintptr_t)
1129				 (((struct fw_scsi_read_wr *)tempwr)->cookie));
1130		CSIO_DB_ASSERT(virt_addr_valid(ioreq));
1131
1132		ioreq->wr_status = status;
1133
1134		return ioreq;
1135	}
1136
1137	if (*tempwr == FW_SCSI_ABRT_CLS_WR) {
1138		ioreq = (struct csio_ioreq *)((uintptr_t)
1139			 (((struct fw_scsi_abrt_cls_wr *)tempwr)->cookie));
1140		CSIO_DB_ASSERT(virt_addr_valid(ioreq));
1141
1142		ioreq->wr_status = status;
1143		return ioreq;
1144	}
1145
1146	csio_warn(hw, "WR with invalid opcode in SCSI IQ: %x\n", *tempwr);
1147	CSIO_INC_STATS(scm, n_inval_scsiop);
1148	return NULL;
1149}
1150
1151/*
1152 * csio_scsi_cleanup_io_q - Cleanup the given queue.
1153 * @scm: SCSI module.
1154 * @q: Queue to be cleaned up.
1155 *
1156 * Called with lock held. Has to exit with lock held.
1157 */
1158void
1159csio_scsi_cleanup_io_q(struct csio_scsim *scm, struct list_head *q)
1160{
1161	struct csio_hw *hw = scm->hw;
1162	struct csio_ioreq *ioreq;
1163	struct list_head *tmp, *next;
1164	struct scsi_cmnd *scmnd;
1165
1166	/* Call back the completion routines of the active_q */
1167	list_for_each_safe(tmp, next, q) {
1168		ioreq = (struct csio_ioreq *)tmp;
1169		csio_scsi_drvcleanup(ioreq);
1170		list_del_init(&ioreq->sm.sm_list);
1171		scmnd = csio_scsi_cmnd(ioreq);
1172		spin_unlock_irq(&hw->lock);
1173
1174		/*
1175		 * Upper layers may have cleared this command, hence this
1176		 * check to avoid accessing stale references.
1177		 */
1178		if (scmnd != NULL)
1179			ioreq->io_cbfn(hw, ioreq);
1180
1181		spin_lock_irq(&scm->freelist_lock);
1182		csio_put_scsi_ioreq(scm, ioreq);
1183		spin_unlock_irq(&scm->freelist_lock);
1184
1185		spin_lock_irq(&hw->lock);
1186	}
1187}
1188
1189#define CSIO_SCSI_ABORT_Q_POLL_MS		2000
1190
1191static void
1192csio_abrt_cls(struct csio_ioreq *ioreq, struct scsi_cmnd *scmnd)
1193{
1194	struct csio_lnode *ln = ioreq->lnode;
1195	struct csio_hw *hw = ln->hwp;
1196	int ready = 0;
1197	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1198	int rv;
1199
1200	if (csio_scsi_cmnd(ioreq) != scmnd) {
1201		CSIO_INC_STATS(scsim, n_abrt_race_comp);
1202		return;
1203	}
1204
1205	ready = csio_is_lnode_ready(ln);
1206
1207	rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
1208	if (rv != 0) {
1209		if (ready)
1210			CSIO_INC_STATS(scsim, n_abrt_busy_error);
1211		else
1212			CSIO_INC_STATS(scsim, n_cls_busy_error);
1213	}
1214}
1215
1216/*
1217 * csio_scsi_abort_io_q - Abort all I/Os on given queue
1218 * @scm: SCSI module.
1219 * @q: Queue to abort.
1220 * @tmo: Timeout in ms
1221 *
1222 * Attempt to abort all I/Os on given queue, and wait for a max
1223 * of tmo milliseconds for them to complete. Returns success
1224 * if all I/Os are aborted. Else returns -ETIMEDOUT.
1225 * Should be entered with lock held. Exits with lock held.
1226 * NOTE:
1227 * Lock has to be held across the loop that aborts I/Os, since dropping the lock
1228 * in between can cause the list to be corrupted. As a result, the caller
1229 * of this function has to ensure that the number of I/os to be aborted
1230 * is finite enough to not cause lock-held-for-too-long issues.
1231 */
1232static int
1233csio_scsi_abort_io_q(struct csio_scsim *scm, struct list_head *q, uint32_t tmo)
1234{
1235	struct csio_hw *hw = scm->hw;
1236	struct list_head *tmp, *next;
1237	int count = DIV_ROUND_UP(tmo, CSIO_SCSI_ABORT_Q_POLL_MS);
1238	struct scsi_cmnd *scmnd;
1239
1240	if (list_empty(q))
1241		return 0;
1242
1243	csio_dbg(hw, "Aborting SCSI I/Os\n");
1244
1245	/* Now abort/close I/Os in the queue passed */
1246	list_for_each_safe(tmp, next, q) {
1247		scmnd = csio_scsi_cmnd((struct csio_ioreq *)tmp);
1248		csio_abrt_cls((struct csio_ioreq *)tmp, scmnd);
1249	}
1250
1251	/* Wait till all active I/Os are completed/aborted/closed */
1252	while (!list_empty(q) && count--) {
1253		spin_unlock_irq(&hw->lock);
1254		msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1255		spin_lock_irq(&hw->lock);
1256	}
1257
1258	/* all aborts completed */
1259	if (list_empty(q))
1260		return 0;
1261
1262	return -ETIMEDOUT;
1263}
1264
1265/*
1266 * csio_scsim_cleanup_io - Cleanup all I/Os in SCSI module.
1267 * @scm: SCSI module.
1268 * @abort: abort required.
1269 * Called with lock held, should exit with lock held.
1270 * Can sleep when waiting for I/Os to complete.
1271 */
1272int
1273csio_scsim_cleanup_io(struct csio_scsim *scm, bool abort)
1274{
1275	struct csio_hw *hw = scm->hw;
1276	int rv = 0;
1277	int count = DIV_ROUND_UP(60 * 1000, CSIO_SCSI_ABORT_Q_POLL_MS);
1278
1279	/* No I/Os pending */
1280	if (list_empty(&scm->active_q))
1281		return 0;
1282
1283	/* Wait until all active I/Os are completed */
1284	while (!list_empty(&scm->active_q) && count--) {
1285		spin_unlock_irq(&hw->lock);
1286		msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1287		spin_lock_irq(&hw->lock);
1288	}
1289
1290	/* all I/Os completed */
1291	if (list_empty(&scm->active_q))
1292		return 0;
1293
1294	/* Else abort */
1295	if (abort) {
1296		rv = csio_scsi_abort_io_q(scm, &scm->active_q, 30000);
1297		if (rv == 0)
1298			return rv;
1299		csio_dbg(hw, "Some I/O aborts timed out, cleaning up..\n");
1300	}
1301
1302	csio_scsi_cleanup_io_q(scm, &scm->active_q);
1303
1304	CSIO_DB_ASSERT(list_empty(&scm->active_q));
1305
1306	return rv;
1307}
1308
1309/*
1310 * csio_scsim_cleanup_io_lnode - Cleanup all I/Os of given lnode.
1311 * @scm: SCSI module.
1312 * @lnode: lnode
1313 *
1314 * Called with lock held, should exit with lock held.
1315 * Can sleep (with dropped lock) when waiting for I/Os to complete.
1316 */
1317int
1318csio_scsim_cleanup_io_lnode(struct csio_scsim *scm, struct csio_lnode *ln)
1319{
1320	struct csio_hw *hw = scm->hw;
1321	struct csio_scsi_level_data sld;
1322	int rv;
1323	int count = DIV_ROUND_UP(60 * 1000, CSIO_SCSI_ABORT_Q_POLL_MS);
1324
1325	csio_dbg(hw, "Gathering all SCSI I/Os on lnode %p\n", ln);
1326
1327	sld.level = CSIO_LEV_LNODE;
1328	sld.lnode = ln;
1329	INIT_LIST_HEAD(&ln->cmpl_q);
1330	csio_scsi_gather_active_ios(scm, &sld, &ln->cmpl_q);
1331
1332	/* No I/Os pending on this lnode  */
1333	if (list_empty(&ln->cmpl_q))
1334		return 0;
1335
1336	/* Wait until all active I/Os on this lnode are completed */
1337	while (!list_empty(&ln->cmpl_q) && count--) {
1338		spin_unlock_irq(&hw->lock);
1339		msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1340		spin_lock_irq(&hw->lock);
1341	}
1342
1343	/* all I/Os completed */
1344	if (list_empty(&ln->cmpl_q))
1345		return 0;
1346
1347	csio_dbg(hw, "Some I/Os pending on ln:%p, aborting them..\n", ln);
1348
1349	/* I/Os are pending, abort them */
1350	rv = csio_scsi_abort_io_q(scm, &ln->cmpl_q, 30000);
1351	if (rv != 0) {
1352		csio_dbg(hw, "Some I/O aborts timed out, cleaning up..\n");
1353		csio_scsi_cleanup_io_q(scm, &ln->cmpl_q);
1354	}
1355
1356	CSIO_DB_ASSERT(list_empty(&ln->cmpl_q));
1357
1358	return rv;
1359}
1360
1361static ssize_t
1362csio_show_hw_state(struct device *dev,
1363		   struct device_attribute *attr, char *buf)
1364{
1365	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1366	struct csio_hw *hw = csio_lnode_to_hw(ln);
1367
1368	if (csio_is_hw_ready(hw))
1369		return sysfs_emit(buf, "ready\n");
1370
1371	return sysfs_emit(buf, "not ready\n");
1372}
1373
1374/* Device reset */
1375static ssize_t
1376csio_device_reset(struct device *dev,
1377		   struct device_attribute *attr, const char *buf, size_t count)
1378{
1379	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1380	struct csio_hw *hw = csio_lnode_to_hw(ln);
1381
1382	if (*buf != '1')
1383		return -EINVAL;
1384
1385	/* Delete NPIV lnodes */
1386	csio_lnodes_exit(hw, 1);
1387
1388	/* Block upper IOs */
1389	csio_lnodes_block_request(hw);
1390
1391	spin_lock_irq(&hw->lock);
1392	csio_hw_reset(hw);
1393	spin_unlock_irq(&hw->lock);
1394
1395	/* Unblock upper IOs */
1396	csio_lnodes_unblock_request(hw);
1397	return count;
1398}
1399
1400/* disable port */
1401static ssize_t
1402csio_disable_port(struct device *dev,
1403		   struct device_attribute *attr, const char *buf, size_t count)
1404{
1405	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1406	struct csio_hw *hw = csio_lnode_to_hw(ln);
1407	bool disable;
1408
1409	if (*buf == '1' || *buf == '0')
1410		disable = (*buf == '1') ? true : false;
1411	else
1412		return -EINVAL;
1413
1414	/* Block upper IOs */
1415	csio_lnodes_block_by_port(hw, ln->portid);
1416
1417	spin_lock_irq(&hw->lock);
1418	csio_disable_lnodes(hw, ln->portid, disable);
1419	spin_unlock_irq(&hw->lock);
1420
1421	/* Unblock upper IOs */
1422	csio_lnodes_unblock_by_port(hw, ln->portid);
1423	return count;
1424}
1425
1426/* Show debug level */
1427static ssize_t
1428csio_show_dbg_level(struct device *dev,
1429		   struct device_attribute *attr, char *buf)
1430{
1431	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1432
1433	return sysfs_emit(buf, "%x\n", ln->params.log_level);
1434}
1435
1436/* Store debug level */
1437static ssize_t
1438csio_store_dbg_level(struct device *dev,
1439		   struct device_attribute *attr, const char *buf, size_t count)
1440{
1441	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1442	struct csio_hw *hw = csio_lnode_to_hw(ln);
1443	uint32_t dbg_level = 0;
1444
1445	if (!isdigit(buf[0]))
1446		return -EINVAL;
1447
1448	if (sscanf(buf, "%i", &dbg_level))
1449		return -EINVAL;
1450
1451	ln->params.log_level = dbg_level;
1452	hw->params.log_level = dbg_level;
1453
1454	return 0;
1455}
1456
1457static DEVICE_ATTR(hw_state, S_IRUGO, csio_show_hw_state, NULL);
1458static DEVICE_ATTR(device_reset, S_IWUSR, NULL, csio_device_reset);
1459static DEVICE_ATTR(disable_port, S_IWUSR, NULL, csio_disable_port);
1460static DEVICE_ATTR(dbg_level, S_IRUGO | S_IWUSR, csio_show_dbg_level,
1461		  csio_store_dbg_level);
1462
1463static struct attribute *csio_fcoe_lport_attrs[] = {
1464	&dev_attr_hw_state.attr,
1465	&dev_attr_device_reset.attr,
1466	&dev_attr_disable_port.attr,
1467	&dev_attr_dbg_level.attr,
1468	NULL,
1469};
1470
1471ATTRIBUTE_GROUPS(csio_fcoe_lport);
1472
1473static ssize_t
1474csio_show_num_reg_rnodes(struct device *dev,
1475		     struct device_attribute *attr, char *buf)
1476{
1477	struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1478
1479	return sysfs_emit(buf, "%d\n", ln->num_reg_rnodes);
1480}
1481
1482static DEVICE_ATTR(num_reg_rnodes, S_IRUGO, csio_show_num_reg_rnodes, NULL);
1483
1484static struct attribute *csio_fcoe_vport_attrs[] = {
1485	&dev_attr_num_reg_rnodes.attr,
1486	&dev_attr_dbg_level.attr,
1487	NULL,
1488};
1489
1490ATTRIBUTE_GROUPS(csio_fcoe_vport);
1491
1492static inline uint32_t
1493csio_scsi_copy_to_sgl(struct csio_hw *hw, struct csio_ioreq *req)
1494{
1495	struct scsi_cmnd *scmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1496	struct scatterlist *sg;
1497	uint32_t bytes_left;
1498	uint32_t bytes_copy;
1499	uint32_t buf_off = 0;
1500	uint32_t start_off = 0;
1501	uint32_t sg_off = 0;
1502	void *sg_addr;
1503	void *buf_addr;
1504	struct csio_dma_buf *dma_buf;
1505
1506	bytes_left = scsi_bufflen(scmnd);
1507	sg = scsi_sglist(scmnd);
1508	dma_buf = (struct csio_dma_buf *)csio_list_next(&req->gen_list);
1509
1510	/* Copy data from driver buffer to SGs of SCSI CMD */
1511	while (bytes_left > 0 && sg && dma_buf) {
1512		if (buf_off >= dma_buf->len) {
1513			buf_off = 0;
1514			dma_buf = (struct csio_dma_buf *)
1515					csio_list_next(dma_buf);
1516			continue;
1517		}
1518
1519		if (start_off >= sg->length) {
1520			start_off -= sg->length;
1521			sg = sg_next(sg);
1522			continue;
1523		}
1524
1525		buf_addr = dma_buf->vaddr + buf_off;
1526		sg_off = sg->offset + start_off;
1527		bytes_copy = min((dma_buf->len - buf_off),
1528				sg->length - start_off);
1529		bytes_copy = min((uint32_t)(PAGE_SIZE - (sg_off & ~PAGE_MASK)),
1530				 bytes_copy);
1531
1532		sg_addr = kmap_atomic(sg_page(sg) + (sg_off >> PAGE_SHIFT));
1533		if (!sg_addr) {
1534			csio_err(hw, "failed to kmap sg:%p of ioreq:%p\n",
1535				sg, req);
1536			break;
1537		}
1538
1539		csio_dbg(hw, "copy_to_sgl:sg_addr %p sg_off %d buf %p len %d\n",
1540				sg_addr, sg_off, buf_addr, bytes_copy);
1541		memcpy(sg_addr + (sg_off & ~PAGE_MASK), buf_addr, bytes_copy);
1542		kunmap_atomic(sg_addr);
1543
1544		start_off +=  bytes_copy;
1545		buf_off += bytes_copy;
1546		bytes_left -= bytes_copy;
1547	}
1548
1549	if (bytes_left > 0)
1550		return DID_ERROR;
1551	else
1552		return DID_OK;
1553}
1554
1555/*
1556 * csio_scsi_err_handler - SCSI error handler.
1557 * @hw: HW module.
1558 * @req: IO request.
1559 *
1560 */
1561static inline void
1562csio_scsi_err_handler(struct csio_hw *hw, struct csio_ioreq *req)
1563{
1564	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1565	struct csio_scsim *scm = csio_hw_to_scsim(hw);
1566	struct fcp_resp_with_ext *fcp_resp;
1567	struct fcp_resp_rsp_info *rsp_info;
1568	struct csio_dma_buf *dma_buf;
1569	uint8_t flags, scsi_status = 0;
1570	uint32_t host_status = DID_OK;
1571	uint32_t rsp_len = 0, sns_len = 0;
1572	struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1573
1574
1575	switch (req->wr_status) {
1576	case FW_HOSTERROR:
1577		if (unlikely(!csio_is_hw_ready(hw)))
1578			return;
1579
1580		host_status = DID_ERROR;
1581		CSIO_INC_STATS(scm, n_hosterror);
1582
1583		break;
1584	case FW_SCSI_RSP_ERR:
1585		dma_buf = &req->dma_buf;
1586		fcp_resp = (struct fcp_resp_with_ext *)dma_buf->vaddr;
1587		rsp_info = (struct fcp_resp_rsp_info *)(fcp_resp + 1);
1588		flags = fcp_resp->resp.fr_flags;
1589		scsi_status = fcp_resp->resp.fr_status;
1590
1591		if (flags & FCP_RSP_LEN_VAL) {
1592			rsp_len = be32_to_cpu(fcp_resp->ext.fr_rsp_len);
1593			if ((rsp_len != 0 && rsp_len != 4 && rsp_len != 8) ||
1594				(rsp_info->rsp_code != FCP_TMF_CMPL)) {
1595				host_status = DID_ERROR;
1596				goto out;
1597			}
1598		}
1599
1600		if ((flags & FCP_SNS_LEN_VAL) && fcp_resp->ext.fr_sns_len) {
1601			sns_len = be32_to_cpu(fcp_resp->ext.fr_sns_len);
1602			if (sns_len > SCSI_SENSE_BUFFERSIZE)
1603				sns_len = SCSI_SENSE_BUFFERSIZE;
1604
1605			memcpy(cmnd->sense_buffer,
1606			       &rsp_info->_fr_resvd[0] + rsp_len, sns_len);
1607			CSIO_INC_STATS(scm, n_autosense);
1608		}
1609
1610		scsi_set_resid(cmnd, 0);
1611
1612		/* Under run */
1613		if (flags & FCP_RESID_UNDER) {
1614			scsi_set_resid(cmnd,
1615				       be32_to_cpu(fcp_resp->ext.fr_resid));
1616
1617			if (!(flags & FCP_SNS_LEN_VAL) &&
1618			    (scsi_status == SAM_STAT_GOOD) &&
1619			    ((scsi_bufflen(cmnd) - scsi_get_resid(cmnd))
1620							< cmnd->underflow))
1621				host_status = DID_ERROR;
1622		} else if (flags & FCP_RESID_OVER)
1623			host_status = DID_ERROR;
1624
1625		CSIO_INC_STATS(scm, n_rsperror);
1626		break;
1627
1628	case FW_SCSI_OVER_FLOW_ERR:
1629		csio_warn(hw,
1630			  "Over-flow error,cmnd:0x%x expected len:0x%x"
1631			  " resid:0x%x\n", cmnd->cmnd[0],
1632			  scsi_bufflen(cmnd), scsi_get_resid(cmnd));
1633		host_status = DID_ERROR;
1634		CSIO_INC_STATS(scm, n_ovflerror);
1635		break;
1636
1637	case FW_SCSI_UNDER_FLOW_ERR:
1638		csio_warn(hw,
1639			  "Under-flow error,cmnd:0x%x expected"
1640			  " len:0x%x resid:0x%x lun:0x%llx ssn:0x%x\n",
1641			  cmnd->cmnd[0], scsi_bufflen(cmnd),
1642			  scsi_get_resid(cmnd), cmnd->device->lun,
1643			  rn->flowid);
1644		host_status = DID_ERROR;
1645		CSIO_INC_STATS(scm, n_unflerror);
1646		break;
1647
1648	case FW_SCSI_ABORT_REQUESTED:
1649	case FW_SCSI_ABORTED:
1650	case FW_SCSI_CLOSE_REQUESTED:
1651		csio_dbg(hw, "Req %p cmd:%p op:%x %s\n", req, cmnd,
1652			     cmnd->cmnd[0],
1653			    (req->wr_status == FW_SCSI_CLOSE_REQUESTED) ?
1654			    "closed" : "aborted");
1655		/*
1656		 * csio_eh_abort_handler checks this value to
1657		 * succeed or fail the abort request.
1658		 */
1659		host_status = DID_REQUEUE;
1660		if (req->wr_status == FW_SCSI_CLOSE_REQUESTED)
1661			CSIO_INC_STATS(scm, n_closed);
1662		else
1663			CSIO_INC_STATS(scm, n_aborted);
1664		break;
1665
1666	case FW_SCSI_ABORT_TIMEDOUT:
1667		/* FW timed out the abort itself */
1668		csio_dbg(hw, "FW timed out abort req:%p cmnd:%p status:%x\n",
1669			 req, cmnd, req->wr_status);
1670		host_status = DID_ERROR;
1671		CSIO_INC_STATS(scm, n_abrt_timedout);
1672		break;
1673
1674	case FW_RDEV_NOT_READY:
1675		/*
1676		 * In firmware, a RDEV can get into this state
1677		 * temporarily, before moving into dissapeared/lost
1678		 * state. So, the driver should complete the request equivalent
1679		 * to device-disappeared!
1680		 */
1681		CSIO_INC_STATS(scm, n_rdev_nr_error);
1682		host_status = DID_ERROR;
1683		break;
1684
1685	case FW_ERR_RDEV_LOST:
1686		CSIO_INC_STATS(scm, n_rdev_lost_error);
1687		host_status = DID_ERROR;
1688		break;
1689
1690	case FW_ERR_RDEV_LOGO:
1691		CSIO_INC_STATS(scm, n_rdev_logo_error);
1692		host_status = DID_ERROR;
1693		break;
1694
1695	case FW_ERR_RDEV_IMPL_LOGO:
1696		host_status = DID_ERROR;
1697		break;
1698
1699	case FW_ERR_LINK_DOWN:
1700		CSIO_INC_STATS(scm, n_link_down_error);
1701		host_status = DID_ERROR;
1702		break;
1703
1704	case FW_FCOE_NO_XCHG:
1705		CSIO_INC_STATS(scm, n_no_xchg_error);
1706		host_status = DID_ERROR;
1707		break;
1708
1709	default:
1710		csio_err(hw, "Unknown SCSI FW WR status:%d req:%p cmnd:%p\n",
1711			    req->wr_status, req, cmnd);
1712		CSIO_DB_ASSERT(0);
1713
1714		CSIO_INC_STATS(scm, n_unknown_error);
1715		host_status = DID_ERROR;
1716		break;
1717	}
1718
1719out:
1720	if (req->nsge > 0) {
1721		scsi_dma_unmap(cmnd);
1722		if (req->dcopy && (host_status == DID_OK))
1723			host_status = csio_scsi_copy_to_sgl(hw, req);
1724	}
1725
1726	cmnd->result = (((host_status) << 16) | scsi_status);
1727	scsi_done(cmnd);
1728
1729	/* Wake up waiting threads */
1730	csio_scsi_cmnd(req) = NULL;
1731	complete(&req->cmplobj);
1732}
1733
1734/*
1735 * csio_scsi_cbfn - SCSI callback function.
1736 * @hw: HW module.
1737 * @req: IO request.
1738 *
1739 */
1740static void
1741csio_scsi_cbfn(struct csio_hw *hw, struct csio_ioreq *req)
1742{
1743	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1744	uint8_t scsi_status = SAM_STAT_GOOD;
1745	uint32_t host_status = DID_OK;
1746
1747	if (likely(req->wr_status == FW_SUCCESS)) {
1748		if (req->nsge > 0) {
1749			scsi_dma_unmap(cmnd);
1750			if (req->dcopy)
1751				host_status = csio_scsi_copy_to_sgl(hw, req);
1752		}
1753
1754		cmnd->result = (((host_status) << 16) | scsi_status);
1755		scsi_done(cmnd);
1756		csio_scsi_cmnd(req) = NULL;
1757		CSIO_INC_STATS(csio_hw_to_scsim(hw), n_tot_success);
1758	} else {
1759		/* Error handling */
1760		csio_scsi_err_handler(hw, req);
1761	}
1762}
1763
1764/**
1765 * csio_queuecommand - Entry point to kickstart an I/O request.
1766 * @host:	The scsi_host pointer.
1767 * @cmnd:	The I/O request from ML.
1768 *
1769 * This routine does the following:
1770 *	- Checks for HW and Rnode module readiness.
1771 *	- Gets a free ioreq structure (which is already initialized
1772 *	  to uninit during its allocation).
1773 *	- Maps SG elements.
1774 *	- Initializes ioreq members.
1775 *	- Kicks off the SCSI state machine for this IO.
1776 *	- Returns busy status on error.
1777 */
1778static int
1779csio_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmnd)
1780{
1781	struct csio_lnode *ln = shost_priv(host);
1782	struct csio_hw *hw = csio_lnode_to_hw(ln);
1783	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1784	struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1785	struct csio_ioreq *ioreq = NULL;
1786	unsigned long flags;
1787	int nsge = 0;
1788	int rv = SCSI_MLQUEUE_HOST_BUSY, nr;
1789	int retval;
 
1790	struct csio_scsi_qset *sqset;
1791	struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
1792
1793	sqset = &hw->sqset[ln->portid][blk_mq_rq_cpu(scsi_cmd_to_rq(cmnd))];
 
 
 
 
 
1794
1795	nr = fc_remote_port_chkready(rport);
1796	if (nr) {
1797		cmnd->result = nr;
1798		CSIO_INC_STATS(scsim, n_rn_nr_error);
1799		goto err_done;
1800	}
1801
1802	if (unlikely(!csio_is_hw_ready(hw))) {
1803		cmnd->result = (DID_REQUEUE << 16);
1804		CSIO_INC_STATS(scsim, n_hw_nr_error);
1805		goto err_done;
1806	}
1807
1808	/* Get req->nsge, if there are SG elements to be mapped  */
1809	nsge = scsi_dma_map(cmnd);
1810	if (unlikely(nsge < 0)) {
1811		CSIO_INC_STATS(scsim, n_dmamap_error);
1812		goto err;
1813	}
1814
1815	/* Do we support so many mappings? */
1816	if (unlikely(nsge > scsim->max_sge)) {
1817		csio_warn(hw,
1818			  "More SGEs than can be supported."
1819			  " SGEs: %d, Max SGEs: %d\n", nsge, scsim->max_sge);
1820		CSIO_INC_STATS(scsim, n_unsupp_sge_error);
1821		goto err_dma_unmap;
1822	}
1823
1824	/* Get a free ioreq structure - SM is already set to uninit */
1825	ioreq = csio_get_scsi_ioreq_lock(hw, scsim);
1826	if (!ioreq) {
1827		csio_err(hw, "Out of I/O request elements. Active #:%d\n",
1828			 scsim->stats.n_active);
1829		CSIO_INC_STATS(scsim, n_no_req_error);
1830		goto err_dma_unmap;
1831	}
1832
1833	ioreq->nsge		= nsge;
1834	ioreq->lnode		= ln;
1835	ioreq->rnode		= rn;
1836	ioreq->iq_idx		= sqset->iq_idx;
1837	ioreq->eq_idx		= sqset->eq_idx;
1838	ioreq->wr_status	= 0;
1839	ioreq->drv_status	= 0;
1840	csio_scsi_cmnd(ioreq)	= (void *)cmnd;
1841	ioreq->tmo		= 0;
1842	ioreq->datadir		= cmnd->sc_data_direction;
1843
1844	if (cmnd->sc_data_direction == DMA_TO_DEVICE) {
1845		CSIO_INC_STATS(ln, n_output_requests);
1846		ln->stats.n_output_bytes += scsi_bufflen(cmnd);
1847	} else if (cmnd->sc_data_direction == DMA_FROM_DEVICE) {
1848		CSIO_INC_STATS(ln, n_input_requests);
1849		ln->stats.n_input_bytes += scsi_bufflen(cmnd);
1850	} else
1851		CSIO_INC_STATS(ln, n_control_requests);
1852
1853	/* Set cbfn */
1854	ioreq->io_cbfn = csio_scsi_cbfn;
1855
1856	/* Needed during abort */
1857	cmnd->host_scribble = (unsigned char *)ioreq;
1858	csio_priv(cmnd)->fc_tm_flags = 0;
1859
1860	/* Kick off SCSI IO SM on the ioreq */
1861	spin_lock_irqsave(&hw->lock, flags);
1862	retval = csio_scsi_start_io(ioreq);
1863	spin_unlock_irqrestore(&hw->lock, flags);
1864
1865	if (retval != 0) {
1866		csio_err(hw, "ioreq: %p couldn't be started, status:%d\n",
1867			 ioreq, retval);
1868		CSIO_INC_STATS(scsim, n_busy_error);
1869		goto err_put_req;
1870	}
1871
1872	return 0;
1873
1874err_put_req:
1875	csio_put_scsi_ioreq_lock(hw, scsim, ioreq);
1876err_dma_unmap:
1877	if (nsge > 0)
1878		scsi_dma_unmap(cmnd);
1879err:
1880	return rv;
1881
1882err_done:
1883	scsi_done(cmnd);
1884	return 0;
1885}
1886
1887static int
1888csio_do_abrt_cls(struct csio_hw *hw, struct csio_ioreq *ioreq, bool abort)
1889{
1890	int rv;
1891	int cpu = smp_processor_id();
1892	struct csio_lnode *ln = ioreq->lnode;
1893	struct csio_scsi_qset *sqset = &hw->sqset[ln->portid][cpu];
1894
1895	ioreq->tmo = CSIO_SCSI_ABRT_TMO_MS;
1896	/*
1897	 * Use current processor queue for posting the abort/close, but retain
1898	 * the ingress queue ID of the original I/O being aborted/closed - we
1899	 * need the abort/close completion to be received on the same queue
1900	 * as the original I/O.
1901	 */
1902	ioreq->eq_idx = sqset->eq_idx;
1903
1904	if (abort == SCSI_ABORT)
1905		rv = csio_scsi_abort(ioreq);
1906	else
1907		rv = csio_scsi_close(ioreq);
1908
1909	return rv;
1910}
1911
1912static int
1913csio_eh_abort_handler(struct scsi_cmnd *cmnd)
1914{
1915	struct csio_ioreq *ioreq;
1916	struct csio_lnode *ln = shost_priv(cmnd->device->host);
1917	struct csio_hw *hw = csio_lnode_to_hw(ln);
1918	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1919	int ready = 0, ret;
1920	unsigned long tmo = 0;
1921	int rv;
1922	struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1923
1924	ret = fc_block_scsi_eh(cmnd);
1925	if (ret)
1926		return ret;
1927
1928	ioreq = (struct csio_ioreq *)cmnd->host_scribble;
1929	if (!ioreq)
1930		return SUCCESS;
1931
1932	if (!rn)
1933		return FAILED;
1934
1935	csio_dbg(hw,
1936		 "Request to abort ioreq:%p cmd:%p cdb:%08llx"
1937		 " ssni:0x%x lun:%llu iq:0x%x\n",
1938		ioreq, cmnd, *((uint64_t *)cmnd->cmnd), rn->flowid,
1939		cmnd->device->lun, csio_q_physiqid(hw, ioreq->iq_idx));
1940
1941	if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) != cmnd) {
1942		CSIO_INC_STATS(scsim, n_abrt_race_comp);
1943		return SUCCESS;
1944	}
1945
1946	ready = csio_is_lnode_ready(ln);
1947	tmo = CSIO_SCSI_ABRT_TMO_MS;
1948
1949	reinit_completion(&ioreq->cmplobj);
1950	spin_lock_irq(&hw->lock);
1951	rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
1952	spin_unlock_irq(&hw->lock);
1953
1954	if (rv != 0) {
1955		if (rv == -EINVAL) {
1956			/* Return success, if abort/close request issued on
1957			 * already completed IO
1958			 */
1959			return SUCCESS;
1960		}
1961		if (ready)
1962			CSIO_INC_STATS(scsim, n_abrt_busy_error);
1963		else
1964			CSIO_INC_STATS(scsim, n_cls_busy_error);
1965
1966		goto inval_scmnd;
1967	}
1968
 
 
1969	wait_for_completion_timeout(&ioreq->cmplobj, msecs_to_jiffies(tmo));
1970
1971	/* FW didnt respond to abort within our timeout */
1972	if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd) {
1973
1974		csio_err(hw, "Abort timed out -- req: %p\n", ioreq);
1975		CSIO_INC_STATS(scsim, n_abrt_timedout);
1976
1977inval_scmnd:
1978		if (ioreq->nsge > 0)
1979			scsi_dma_unmap(cmnd);
1980
1981		spin_lock_irq(&hw->lock);
1982		csio_scsi_cmnd(ioreq) = NULL;
1983		spin_unlock_irq(&hw->lock);
1984
1985		cmnd->result = (DID_ERROR << 16);
1986		scsi_done(cmnd);
1987
1988		return FAILED;
1989	}
1990
1991	/* FW successfully aborted the request */
1992	if (host_byte(cmnd->result) == DID_REQUEUE) {
1993		csio_info(hw,
1994			"Aborted SCSI command to (%d:%llu) tag %u\n",
1995			cmnd->device->id, cmnd->device->lun,
1996			scsi_cmd_to_rq(cmnd)->tag);
1997		return SUCCESS;
1998	} else {
1999		csio_info(hw,
2000			"Failed to abort SCSI command, (%d:%llu) tag %u\n",
2001			cmnd->device->id, cmnd->device->lun,
2002			scsi_cmd_to_rq(cmnd)->tag);
2003		return FAILED;
2004	}
2005}
2006
2007/*
2008 * csio_tm_cbfn - TM callback function.
2009 * @hw: HW module.
2010 * @req: IO request.
2011 *
2012 * Cache the result in 'cmnd', since ioreq will be freed soon
2013 * after we return from here, and the waiting thread shouldnt trust
2014 * the ioreq contents.
2015 */
2016static void
2017csio_tm_cbfn(struct csio_hw *hw, struct csio_ioreq *req)
2018{
2019	struct scsi_cmnd *cmnd  = (struct scsi_cmnd *)csio_scsi_cmnd(req);
2020	struct csio_dma_buf *dma_buf;
2021	uint8_t flags = 0;
2022	struct fcp_resp_with_ext *fcp_resp;
2023	struct fcp_resp_rsp_info *rsp_info;
2024
2025	csio_dbg(hw, "req: %p in csio_tm_cbfn status: %d\n",
2026		      req, req->wr_status);
2027
2028	/* Cache FW return status */
2029	csio_priv(cmnd)->wr_status = req->wr_status;
2030
2031	/* Special handling based on FCP response */
2032
2033	/*
2034	 * FW returns us this error, if flags were set. FCP4 says
2035	 * FCP_RSP_LEN_VAL in flags shall be set for TM completions.
2036	 * So if a target were to set this bit, we expect that the
2037	 * rsp_code is set to FCP_TMF_CMPL for a successful TM
2038	 * completion. Any other rsp_code means TM operation failed.
2039	 * If a target were to just ignore setting flags, we treat
2040	 * the TM operation as success, and FW returns FW_SUCCESS.
2041	 */
2042	if (req->wr_status == FW_SCSI_RSP_ERR) {
2043		dma_buf = &req->dma_buf;
2044		fcp_resp = (struct fcp_resp_with_ext *)dma_buf->vaddr;
2045		rsp_info = (struct fcp_resp_rsp_info *)(fcp_resp + 1);
2046
2047		flags = fcp_resp->resp.fr_flags;
2048
2049		/* Modify return status if flags indicate success */
2050		if (flags & FCP_RSP_LEN_VAL)
2051			if (rsp_info->rsp_code == FCP_TMF_CMPL)
2052				csio_priv(cmnd)->wr_status = FW_SUCCESS;
2053
2054		csio_dbg(hw, "TM FCP rsp code: %d\n", rsp_info->rsp_code);
2055	}
2056
2057	/* Wake up the TM handler thread */
2058	csio_scsi_cmnd(req) = NULL;
2059}
2060
2061static int
2062csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd)
2063{
2064	struct csio_lnode *ln = shost_priv(cmnd->device->host);
2065	struct csio_hw *hw = csio_lnode_to_hw(ln);
2066	struct csio_scsim *scsim = csio_hw_to_scsim(hw);
2067	struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
2068	struct csio_ioreq *ioreq = NULL;
2069	struct csio_scsi_qset *sqset;
2070	unsigned long flags;
2071	int retval;
2072	int count, ret;
2073	LIST_HEAD(local_q);
2074	struct csio_scsi_level_data sld;
2075
2076	if (!rn)
2077		goto fail;
2078
2079	csio_dbg(hw, "Request to reset LUN:%llu (ssni:0x%x tgtid:%d)\n",
2080		      cmnd->device->lun, rn->flowid, rn->scsi_id);
2081
2082	if (!csio_is_lnode_ready(ln)) {
2083		csio_err(hw,
2084			 "LUN reset cannot be issued on non-ready"
2085			 " local node vnpi:0x%x (LUN:%llu)\n",
2086			 ln->vnp_flowid, cmnd->device->lun);
2087		goto fail;
2088	}
2089
2090	/* Lnode is ready, now wait on rport node readiness */
2091	ret = fc_block_scsi_eh(cmnd);
2092	if (ret)
2093		return ret;
2094
2095	/*
2096	 * If we have blocked in the previous call, at this point, either the
2097	 * remote node has come back online, or device loss timer has fired
2098	 * and the remote node is destroyed. Allow the LUN reset only for
2099	 * the former case, since LUN reset is a TMF I/O on the wire, and we
2100	 * need a valid session to issue it.
2101	 */
2102	if (fc_remote_port_chkready(rn->rport)) {
2103		csio_err(hw,
2104			 "LUN reset cannot be issued on non-ready"
2105			 " remote node ssni:0x%x (LUN:%llu)\n",
2106			 rn->flowid, cmnd->device->lun);
2107		goto fail;
2108	}
2109
2110	/* Get a free ioreq structure - SM is already set to uninit */
2111	ioreq = csio_get_scsi_ioreq_lock(hw, scsim);
2112
2113	if (!ioreq) {
2114		csio_err(hw, "Out of IO request elements. Active # :%d\n",
2115			 scsim->stats.n_active);
2116		goto fail;
2117	}
2118
2119	sqset			= &hw->sqset[ln->portid][smp_processor_id()];
2120	ioreq->nsge		= 0;
2121	ioreq->lnode		= ln;
2122	ioreq->rnode		= rn;
2123	ioreq->iq_idx		= sqset->iq_idx;
2124	ioreq->eq_idx		= sqset->eq_idx;
2125
2126	csio_scsi_cmnd(ioreq)	= cmnd;
2127	cmnd->host_scribble	= (unsigned char *)ioreq;
2128	csio_priv(cmnd)->wr_status = 0;
2129
2130	csio_priv(cmnd)->fc_tm_flags = FCP_TMF_LUN_RESET;
2131	ioreq->tmo		= CSIO_SCSI_LUNRST_TMO_MS / 1000;
2132
2133	/*
2134	 * FW times the LUN reset for ioreq->tmo, so we got to wait a little
2135	 * longer (10s for now) than that to allow FW to return the timed
2136	 * out command.
2137	 */
2138	count = DIV_ROUND_UP((ioreq->tmo + 10) * 1000, CSIO_SCSI_TM_POLL_MS);
2139
2140	/* Set cbfn */
2141	ioreq->io_cbfn = csio_tm_cbfn;
2142
2143	/* Save of the ioreq info for later use */
2144	sld.level = CSIO_LEV_LUN;
2145	sld.lnode = ioreq->lnode;
2146	sld.rnode = ioreq->rnode;
2147	sld.oslun = cmnd->device->lun;
2148
2149	spin_lock_irqsave(&hw->lock, flags);
2150	/* Kick off TM SM on the ioreq */
2151	retval = csio_scsi_start_tm(ioreq);
2152	spin_unlock_irqrestore(&hw->lock, flags);
2153
2154	if (retval != 0) {
2155		csio_err(hw, "Failed to issue LUN reset, req:%p, status:%d\n",
2156			    ioreq, retval);
2157		goto fail_ret_ioreq;
2158	}
2159
2160	csio_dbg(hw, "Waiting max %d secs for LUN reset completion\n",
2161		    count * (CSIO_SCSI_TM_POLL_MS / 1000));
2162	/* Wait for completion */
2163	while ((((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd)
2164								&& count--)
2165		msleep(CSIO_SCSI_TM_POLL_MS);
2166
2167	/* LUN reset timed-out */
2168	if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd) {
2169		csio_err(hw, "LUN reset (%d:%llu) timed out\n",
2170			 cmnd->device->id, cmnd->device->lun);
2171
2172		spin_lock_irq(&hw->lock);
2173		csio_scsi_drvcleanup(ioreq);
2174		list_del_init(&ioreq->sm.sm_list);
2175		spin_unlock_irq(&hw->lock);
2176
2177		goto fail_ret_ioreq;
2178	}
2179
2180	/* LUN reset returned, check cached status */
2181	if (csio_priv(cmnd)->wr_status != FW_SUCCESS) {
2182		csio_err(hw, "LUN reset failed (%d:%llu), status: %d\n",
2183			 cmnd->device->id, cmnd->device->lun,
2184			 csio_priv(cmnd)->wr_status);
2185		goto fail;
2186	}
2187
2188	/* LUN reset succeeded, Start aborting affected I/Os */
2189	/*
2190	 * Since the host guarantees during LUN reset that there
2191	 * will not be any more I/Os to that LUN, until the LUN reset
2192	 * completes, we gather pending I/Os after the LUN reset.
2193	 */
2194	spin_lock_irq(&hw->lock);
2195	csio_scsi_gather_active_ios(scsim, &sld, &local_q);
2196
2197	retval = csio_scsi_abort_io_q(scsim, &local_q, 30000);
2198	spin_unlock_irq(&hw->lock);
2199
2200	/* Aborts may have timed out */
2201	if (retval != 0) {
2202		csio_err(hw,
2203			 "Attempt to abort I/Os during LUN reset of %llu"
2204			 " returned %d\n", cmnd->device->lun, retval);
2205		/* Return I/Os back to active_q */
2206		spin_lock_irq(&hw->lock);
2207		list_splice_tail_init(&local_q, &scsim->active_q);
2208		spin_unlock_irq(&hw->lock);
2209		goto fail;
2210	}
2211
2212	CSIO_INC_STATS(rn, n_lun_rst);
2213
2214	csio_info(hw, "LUN reset occurred (%d:%llu)\n",
2215		  cmnd->device->id, cmnd->device->lun);
2216
2217	return SUCCESS;
2218
2219fail_ret_ioreq:
2220	csio_put_scsi_ioreq_lock(hw, scsim, ioreq);
2221fail:
2222	CSIO_INC_STATS(rn, n_lun_rst_fail);
2223	return FAILED;
2224}
2225
2226static int
2227csio_slave_alloc(struct scsi_device *sdev)
2228{
2229	struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
2230
2231	if (!rport || fc_remote_port_chkready(rport))
2232		return -ENXIO;
2233
2234	sdev->hostdata = *((struct csio_lnode **)(rport->dd_data));
2235
2236	return 0;
2237}
2238
2239static int
2240csio_slave_configure(struct scsi_device *sdev)
2241{
2242	scsi_change_queue_depth(sdev, csio_lun_qdepth);
 
 
 
 
2243	return 0;
2244}
2245
2246static void
2247csio_slave_destroy(struct scsi_device *sdev)
2248{
2249	sdev->hostdata = NULL;
2250}
2251
2252static int
2253csio_scan_finished(struct Scsi_Host *shost, unsigned long time)
2254{
2255	struct csio_lnode *ln = shost_priv(shost);
2256	int rv = 1;
2257
2258	spin_lock_irq(shost->host_lock);
2259	if (!ln->hwp || csio_list_deleted(&ln->sm.sm_list))
2260		goto out;
2261
2262	rv = csio_scan_done(ln, jiffies, time, csio_max_scan_tmo * HZ,
2263			    csio_delta_scan_tmo * HZ);
2264out:
2265	spin_unlock_irq(shost->host_lock);
2266
2267	return rv;
2268}
2269
2270struct scsi_host_template csio_fcoe_shost_template = {
2271	.module			= THIS_MODULE,
2272	.name			= CSIO_DRV_DESC,
2273	.proc_name		= KBUILD_MODNAME,
2274	.queuecommand		= csio_queuecommand,
2275	.cmd_size		= sizeof(struct csio_cmd_priv),
2276	.eh_timed_out		= fc_eh_timed_out,
2277	.eh_abort_handler	= csio_eh_abort_handler,
2278	.eh_device_reset_handler = csio_eh_lun_reset_handler,
2279	.slave_alloc		= csio_slave_alloc,
2280	.slave_configure	= csio_slave_configure,
2281	.slave_destroy		= csio_slave_destroy,
2282	.scan_finished		= csio_scan_finished,
2283	.this_id		= -1,
2284	.sg_tablesize		= CSIO_SCSI_MAX_SGE,
2285	.cmd_per_lun		= CSIO_MAX_CMD_PER_LUN,
2286	.shost_groups		= csio_fcoe_lport_groups,
 
2287	.max_sectors		= CSIO_MAX_SECTOR_SIZE,
2288};
2289
2290struct scsi_host_template csio_fcoe_shost_vport_template = {
2291	.module			= THIS_MODULE,
2292	.name			= CSIO_DRV_DESC,
2293	.proc_name		= KBUILD_MODNAME,
2294	.queuecommand		= csio_queuecommand,
2295	.eh_timed_out		= fc_eh_timed_out,
2296	.eh_abort_handler	= csio_eh_abort_handler,
2297	.eh_device_reset_handler = csio_eh_lun_reset_handler,
2298	.slave_alloc		= csio_slave_alloc,
2299	.slave_configure	= csio_slave_configure,
2300	.slave_destroy		= csio_slave_destroy,
2301	.scan_finished		= csio_scan_finished,
2302	.this_id		= -1,
2303	.sg_tablesize		= CSIO_SCSI_MAX_SGE,
2304	.cmd_per_lun		= CSIO_MAX_CMD_PER_LUN,
2305	.shost_groups		= csio_fcoe_vport_groups,
 
2306	.max_sectors		= CSIO_MAX_SECTOR_SIZE,
2307};
2308
2309/*
2310 * csio_scsi_alloc_ddp_bufs - Allocate buffers for DDP of unaligned SGLs.
2311 * @scm: SCSI Module
2312 * @hw: HW device.
2313 * @buf_size: buffer size
2314 * @num_buf : Number of buffers.
2315 *
2316 * This routine allocates DMA buffers required for SCSI Data xfer, if
2317 * each SGL buffer for a SCSI Read request posted by SCSI midlayer are
2318 * not virtually contiguous.
2319 */
2320static int
2321csio_scsi_alloc_ddp_bufs(struct csio_scsim *scm, struct csio_hw *hw,
2322			 int buf_size, int num_buf)
2323{
2324	int n = 0;
2325	struct list_head *tmp;
2326	struct csio_dma_buf *ddp_desc = NULL;
2327	uint32_t unit_size = 0;
2328
2329	if (!num_buf)
2330		return 0;
2331
2332	if (!buf_size)
2333		return -EINVAL;
2334
2335	INIT_LIST_HEAD(&scm->ddp_freelist);
2336
2337	/* Align buf size to page size */
2338	buf_size = (buf_size + PAGE_SIZE - 1) & PAGE_MASK;
2339	/* Initialize dma descriptors */
2340	for (n = 0; n < num_buf; n++) {
2341		/* Set unit size to request size */
2342		unit_size = buf_size;
2343		ddp_desc = kzalloc(sizeof(struct csio_dma_buf), GFP_KERNEL);
2344		if (!ddp_desc) {
2345			csio_err(hw,
2346				 "Failed to allocate ddp descriptors,"
2347				 " Num allocated = %d.\n",
2348				 scm->stats.n_free_ddp);
2349			goto no_mem;
2350		}
2351
2352		/* Allocate Dma buffers for DDP */
2353		ddp_desc->vaddr = dma_alloc_coherent(&hw->pdev->dev, unit_size,
2354				&ddp_desc->paddr, GFP_KERNEL);
2355		if (!ddp_desc->vaddr) {
2356			csio_err(hw,
2357				 "SCSI response DMA buffer (ddp) allocation"
2358				 " failed!\n");
2359			kfree(ddp_desc);
2360			goto no_mem;
2361		}
2362
2363		ddp_desc->len = unit_size;
2364
2365		/* Added it to scsi ddp freelist */
2366		list_add_tail(&ddp_desc->list, &scm->ddp_freelist);
2367		CSIO_INC_STATS(scm, n_free_ddp);
2368	}
2369
2370	return 0;
2371no_mem:
2372	/* release dma descs back to freelist and free dma memory */
2373	list_for_each(tmp, &scm->ddp_freelist) {
2374		ddp_desc = (struct csio_dma_buf *) tmp;
2375		tmp = csio_list_prev(tmp);
2376		dma_free_coherent(&hw->pdev->dev, ddp_desc->len,
2377				  ddp_desc->vaddr, ddp_desc->paddr);
2378		list_del_init(&ddp_desc->list);
2379		kfree(ddp_desc);
2380	}
2381	scm->stats.n_free_ddp = 0;
2382
2383	return -ENOMEM;
2384}
2385
2386/*
2387 * csio_scsi_free_ddp_bufs - free DDP buffers of unaligned SGLs.
2388 * @scm: SCSI Module
2389 * @hw: HW device.
2390 *
2391 * This routine frees ddp buffers.
2392 */
2393static void
2394csio_scsi_free_ddp_bufs(struct csio_scsim *scm, struct csio_hw *hw)
2395{
2396	struct list_head *tmp;
2397	struct csio_dma_buf *ddp_desc;
2398
2399	/* release dma descs back to freelist and free dma memory */
2400	list_for_each(tmp, &scm->ddp_freelist) {
2401		ddp_desc = (struct csio_dma_buf *) tmp;
2402		tmp = csio_list_prev(tmp);
2403		dma_free_coherent(&hw->pdev->dev, ddp_desc->len,
2404				  ddp_desc->vaddr, ddp_desc->paddr);
2405		list_del_init(&ddp_desc->list);
2406		kfree(ddp_desc);
2407	}
2408	scm->stats.n_free_ddp = 0;
2409}
2410
2411/**
2412 * csio_scsim_init - Initialize SCSI Module
2413 * @scm:	SCSI Module
2414 * @hw:		HW module
2415 *
2416 */
2417int
2418csio_scsim_init(struct csio_scsim *scm, struct csio_hw *hw)
2419{
2420	int i;
2421	struct csio_ioreq *ioreq;
2422	struct csio_dma_buf *dma_buf;
2423
2424	INIT_LIST_HEAD(&scm->active_q);
2425	scm->hw = hw;
2426
2427	scm->proto_cmd_len = sizeof(struct fcp_cmnd);
2428	scm->proto_rsp_len = CSIO_SCSI_RSP_LEN;
2429	scm->max_sge = CSIO_SCSI_MAX_SGE;
2430
2431	spin_lock_init(&scm->freelist_lock);
2432
2433	/* Pre-allocate ioreqs and initialize them */
2434	INIT_LIST_HEAD(&scm->ioreq_freelist);
2435	for (i = 0; i < csio_scsi_ioreqs; i++) {
2436
2437		ioreq = kzalloc(sizeof(struct csio_ioreq), GFP_KERNEL);
2438		if (!ioreq) {
2439			csio_err(hw,
2440				 "I/O request element allocation failed, "
2441				 " Num allocated = %d.\n",
2442				 scm->stats.n_free_ioreq);
2443
2444			goto free_ioreq;
2445		}
2446
2447		/* Allocate Dma buffers for Response Payload */
2448		dma_buf = &ioreq->dma_buf;
2449		dma_buf->vaddr = dma_pool_alloc(hw->scsi_dma_pool, GFP_KERNEL,
2450						&dma_buf->paddr);
2451		if (!dma_buf->vaddr) {
2452			csio_err(hw,
2453				 "SCSI response DMA buffer allocation"
2454				 " failed!\n");
2455			kfree(ioreq);
2456			goto free_ioreq;
2457		}
2458
2459		dma_buf->len = scm->proto_rsp_len;
2460
2461		/* Set state to uninit */
2462		csio_init_state(&ioreq->sm, csio_scsis_uninit);
2463		INIT_LIST_HEAD(&ioreq->gen_list);
2464		init_completion(&ioreq->cmplobj);
2465
2466		list_add_tail(&ioreq->sm.sm_list, &scm->ioreq_freelist);
2467		CSIO_INC_STATS(scm, n_free_ioreq);
2468	}
2469
2470	if (csio_scsi_alloc_ddp_bufs(scm, hw, PAGE_SIZE, csio_ddp_descs))
2471		goto free_ioreq;
2472
2473	return 0;
2474
2475free_ioreq:
2476	/*
2477	 * Free up existing allocations, since an error
2478	 * from here means we are returning for good
2479	 */
2480	while (!list_empty(&scm->ioreq_freelist)) {
2481		struct csio_sm *tmp;
2482
2483		tmp = list_first_entry(&scm->ioreq_freelist,
2484				       struct csio_sm, sm_list);
2485		list_del_init(&tmp->sm_list);
2486		ioreq = (struct csio_ioreq *)tmp;
2487
2488		dma_buf = &ioreq->dma_buf;
2489		dma_pool_free(hw->scsi_dma_pool, dma_buf->vaddr,
2490			      dma_buf->paddr);
2491
2492		kfree(ioreq);
2493	}
2494
2495	scm->stats.n_free_ioreq = 0;
2496
2497	return -ENOMEM;
2498}
2499
2500/**
2501 * csio_scsim_exit: Uninitialize SCSI Module
2502 * @scm: SCSI Module
2503 *
2504 */
2505void
2506csio_scsim_exit(struct csio_scsim *scm)
2507{
2508	struct csio_ioreq *ioreq;
2509	struct csio_dma_buf *dma_buf;
2510
2511	while (!list_empty(&scm->ioreq_freelist)) {
2512		struct csio_sm *tmp;
2513
2514		tmp = list_first_entry(&scm->ioreq_freelist,
2515				       struct csio_sm, sm_list);
2516		list_del_init(&tmp->sm_list);
2517		ioreq = (struct csio_ioreq *)tmp;
2518
2519		dma_buf = &ioreq->dma_buf;
2520		dma_pool_free(scm->hw->scsi_dma_pool, dma_buf->vaddr,
2521			      dma_buf->paddr);
2522
2523		kfree(ioreq);
2524	}
2525
2526	scm->stats.n_free_ioreq = 0;
2527
2528	csio_scsi_free_ddp_bufs(scm, scm->hw);
2529}