Loading...
Note: File does not exist in v3.5.6.
1/*
2 * This file is part of the Chelsio FCoE driver for Linux.
3 *
4 * Copyright (c) 2008-2012 Chelsio Communications, Inc. All rights reserved.
5 *
6 * This software is available to you under a choice of one of two
7 * licenses. You may choose to be licensed under the terms of the GNU
8 * General Public License (GPL) Version 2, available from the file
9 * COPYING in the main directory of this source tree, or the
10 * OpenIB.org BSD license below:
11 *
12 * Redistribution and use in source and binary forms, with or
13 * without modification, are permitted provided that the following
14 * conditions are met:
15 *
16 * - Redistributions of source code must retain the above
17 * copyright notice, this list of conditions and the following
18 * disclaimer.
19 *
20 * - Redistributions in binary form must reproduce the above
21 * copyright notice, this list of conditions and the following
22 * disclaimer in the documentation and/or other materials
23 * provided with the distribution.
24 *
25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
32 * SOFTWARE.
33 */
34
35#include <linux/device.h>
36#include <linux/delay.h>
37#include <linux/ctype.h>
38#include <linux/kernel.h>
39#include <linux/slab.h>
40#include <linux/string.h>
41#include <linux/compiler.h>
42#include <linux/export.h>
43#include <linux/module.h>
44#include <asm/unaligned.h>
45#include <asm/page.h>
46#include <scsi/scsi.h>
47#include <scsi/scsi_device.h>
48#include <scsi/scsi_transport_fc.h>
49
50#include "csio_hw.h"
51#include "csio_lnode.h"
52#include "csio_rnode.h"
53#include "csio_scsi.h"
54#include "csio_init.h"
55
56int csio_scsi_eqsize = 65536;
57int csio_scsi_iqlen = 128;
58int csio_scsi_ioreqs = 2048;
59uint32_t csio_max_scan_tmo;
60uint32_t csio_delta_scan_tmo = 5;
61int csio_lun_qdepth = 32;
62
63static int csio_ddp_descs = 128;
64
65static int csio_do_abrt_cls(struct csio_hw *,
66 struct csio_ioreq *, bool);
67
68static void csio_scsis_uninit(struct csio_ioreq *, enum csio_scsi_ev);
69static void csio_scsis_io_active(struct csio_ioreq *, enum csio_scsi_ev);
70static void csio_scsis_tm_active(struct csio_ioreq *, enum csio_scsi_ev);
71static void csio_scsis_aborting(struct csio_ioreq *, enum csio_scsi_ev);
72static void csio_scsis_closing(struct csio_ioreq *, enum csio_scsi_ev);
73static void csio_scsis_shost_cmpl_await(struct csio_ioreq *, enum csio_scsi_ev);
74
75/*
76 * csio_scsi_match_io - Match an ioreq with the given SCSI level data.
77 * @ioreq: The I/O request
78 * @sld: Level information
79 *
80 * Should be called with lock held.
81 *
82 */
83static bool
84csio_scsi_match_io(struct csio_ioreq *ioreq, struct csio_scsi_level_data *sld)
85{
86 struct scsi_cmnd *scmnd = csio_scsi_cmnd(ioreq);
87
88 switch (sld->level) {
89 case CSIO_LEV_LUN:
90 if (scmnd == NULL)
91 return false;
92
93 return ((ioreq->lnode == sld->lnode) &&
94 (ioreq->rnode == sld->rnode) &&
95 ((uint64_t)scmnd->device->lun == sld->oslun));
96
97 case CSIO_LEV_RNODE:
98 return ((ioreq->lnode == sld->lnode) &&
99 (ioreq->rnode == sld->rnode));
100 case CSIO_LEV_LNODE:
101 return (ioreq->lnode == sld->lnode);
102 case CSIO_LEV_ALL:
103 return true;
104 default:
105 return false;
106 }
107}
108
109/*
110 * csio_scsi_gather_active_ios - Gather active I/Os based on level
111 * @scm: SCSI module
112 * @sld: Level information
113 * @dest: The queue where these I/Os have to be gathered.
114 *
115 * Should be called with lock held.
116 */
117static void
118csio_scsi_gather_active_ios(struct csio_scsim *scm,
119 struct csio_scsi_level_data *sld,
120 struct list_head *dest)
121{
122 struct list_head *tmp, *next;
123
124 if (list_empty(&scm->active_q))
125 return;
126
127 /* Just splice the entire active_q into dest */
128 if (sld->level == CSIO_LEV_ALL) {
129 list_splice_tail_init(&scm->active_q, dest);
130 return;
131 }
132
133 list_for_each_safe(tmp, next, &scm->active_q) {
134 if (csio_scsi_match_io((struct csio_ioreq *)tmp, sld)) {
135 list_del_init(tmp);
136 list_add_tail(tmp, dest);
137 }
138 }
139}
140
141static inline bool
142csio_scsi_itnexus_loss_error(uint16_t error)
143{
144 switch (error) {
145 case FW_ERR_LINK_DOWN:
146 case FW_RDEV_NOT_READY:
147 case FW_ERR_RDEV_LOST:
148 case FW_ERR_RDEV_LOGO:
149 case FW_ERR_RDEV_IMPL_LOGO:
150 return 1;
151 }
152 return 0;
153}
154
155static inline void
156csio_scsi_tag(struct scsi_cmnd *scmnd, uint8_t *tag, uint8_t hq,
157 uint8_t oq, uint8_t sq)
158{
159 char stag[2];
160
161 if (scsi_populate_tag_msg(scmnd, stag)) {
162 switch (stag[0]) {
163 case HEAD_OF_QUEUE_TAG:
164 *tag = hq;
165 break;
166 case ORDERED_QUEUE_TAG:
167 *tag = oq;
168 break;
169 default:
170 *tag = sq;
171 break;
172 }
173 } else
174 *tag = 0;
175}
176
177/*
178 * csio_scsi_fcp_cmnd - Frame the SCSI FCP command paylod.
179 * @req: IO req structure.
180 * @addr: DMA location to place the payload.
181 *
182 * This routine is shared between FCP_WRITE, FCP_READ and FCP_CMD requests.
183 */
184static inline void
185csio_scsi_fcp_cmnd(struct csio_ioreq *req, void *addr)
186{
187 struct fcp_cmnd *fcp_cmnd = (struct fcp_cmnd *)addr;
188 struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
189
190 /* Check for Task Management */
191 if (likely(scmnd->SCp.Message == 0)) {
192 int_to_scsilun(scmnd->device->lun, &fcp_cmnd->fc_lun);
193 fcp_cmnd->fc_tm_flags = 0;
194 fcp_cmnd->fc_cmdref = 0;
195 fcp_cmnd->fc_pri_ta = 0;
196
197 memcpy(fcp_cmnd->fc_cdb, scmnd->cmnd, 16);
198 csio_scsi_tag(scmnd, &fcp_cmnd->fc_pri_ta,
199 FCP_PTA_HEADQ, FCP_PTA_ORDERED, FCP_PTA_SIMPLE);
200 fcp_cmnd->fc_dl = cpu_to_be32(scsi_bufflen(scmnd));
201
202 if (req->nsge)
203 if (req->datadir == DMA_TO_DEVICE)
204 fcp_cmnd->fc_flags = FCP_CFL_WRDATA;
205 else
206 fcp_cmnd->fc_flags = FCP_CFL_RDDATA;
207 else
208 fcp_cmnd->fc_flags = 0;
209 } else {
210 memset(fcp_cmnd, 0, sizeof(*fcp_cmnd));
211 int_to_scsilun(scmnd->device->lun, &fcp_cmnd->fc_lun);
212 fcp_cmnd->fc_tm_flags = (uint8_t)scmnd->SCp.Message;
213 }
214}
215
216/*
217 * csio_scsi_init_cmd_wr - Initialize the SCSI CMD WR.
218 * @req: IO req structure.
219 * @addr: DMA location to place the payload.
220 * @size: Size of WR (including FW WR + immed data + rsp SG entry
221 *
222 * Wrapper for populating fw_scsi_cmd_wr.
223 */
224static inline void
225csio_scsi_init_cmd_wr(struct csio_ioreq *req, void *addr, uint32_t size)
226{
227 struct csio_hw *hw = req->lnode->hwp;
228 struct csio_rnode *rn = req->rnode;
229 struct fw_scsi_cmd_wr *wr = (struct fw_scsi_cmd_wr *)addr;
230 struct csio_dma_buf *dma_buf;
231 uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
232
233 wr->op_immdlen = cpu_to_be32(FW_WR_OP(FW_SCSI_CMD_WR) |
234 FW_SCSI_CMD_WR_IMMDLEN(imm));
235 wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID(rn->flowid) |
236 FW_WR_LEN16(
237 DIV_ROUND_UP(size, 16)));
238
239 wr->cookie = (uintptr_t) req;
240 wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
241 wr->tmo_val = (uint8_t) req->tmo;
242 wr->r3 = 0;
243 memset(&wr->r5, 0, 8);
244
245 /* Get RSP DMA buffer */
246 dma_buf = &req->dma_buf;
247
248 /* Prepare RSP SGL */
249 wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
250 wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
251
252 wr->r6 = 0;
253
254 wr->u.fcoe.ctl_pri = 0;
255 wr->u.fcoe.cp_en_class = 0;
256 wr->u.fcoe.r4_lo[0] = 0;
257 wr->u.fcoe.r4_lo[1] = 0;
258
259 /* Frame a FCP command */
260 csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)addr +
261 sizeof(struct fw_scsi_cmd_wr)));
262}
263
264#define CSIO_SCSI_CMD_WR_SZ(_imm) \
265 (sizeof(struct fw_scsi_cmd_wr) + /* WR size */ \
266 ALIGN((_imm), 16)) /* Immed data */
267
268#define CSIO_SCSI_CMD_WR_SZ_16(_imm) \
269 (ALIGN(CSIO_SCSI_CMD_WR_SZ((_imm)), 16))
270
271/*
272 * csio_scsi_cmd - Create a SCSI CMD WR.
273 * @req: IO req structure.
274 *
275 * Gets a WR slot in the ingress queue and initializes it with SCSI CMD WR.
276 *
277 */
278static inline void
279csio_scsi_cmd(struct csio_ioreq *req)
280{
281 struct csio_wr_pair wrp;
282 struct csio_hw *hw = req->lnode->hwp;
283 struct csio_scsim *scsim = csio_hw_to_scsim(hw);
284 uint32_t size = CSIO_SCSI_CMD_WR_SZ_16(scsim->proto_cmd_len);
285
286 req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
287 if (unlikely(req->drv_status != 0))
288 return;
289
290 if (wrp.size1 >= size) {
291 /* Initialize WR in one shot */
292 csio_scsi_init_cmd_wr(req, wrp.addr1, size);
293 } else {
294 uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
295
296 /*
297 * Make a temporary copy of the WR and write back
298 * the copy into the WR pair.
299 */
300 csio_scsi_init_cmd_wr(req, (void *)tmpwr, size);
301 memcpy(wrp.addr1, tmpwr, wrp.size1);
302 memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
303 }
304}
305
306/*
307 * csio_scsi_init_ulptx_dsgl - Fill in a ULP_TX_SC_DSGL
308 * @hw: HW module
309 * @req: IO request
310 * @sgl: ULP TX SGL pointer.
311 *
312 */
313static inline void
314csio_scsi_init_ultptx_dsgl(struct csio_hw *hw, struct csio_ioreq *req,
315 struct ulptx_sgl *sgl)
316{
317 struct ulptx_sge_pair *sge_pair = NULL;
318 struct scatterlist *sgel;
319 uint32_t i = 0;
320 uint32_t xfer_len;
321 struct list_head *tmp;
322 struct csio_dma_buf *dma_buf;
323 struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
324
325 sgl->cmd_nsge = htonl(ULPTX_CMD(ULP_TX_SC_DSGL) | ULPTX_MORE |
326 ULPTX_NSGE(req->nsge));
327 /* Now add the data SGLs */
328 if (likely(!req->dcopy)) {
329 scsi_for_each_sg(scmnd, sgel, req->nsge, i) {
330 if (i == 0) {
331 sgl->addr0 = cpu_to_be64(sg_dma_address(sgel));
332 sgl->len0 = cpu_to_be32(sg_dma_len(sgel));
333 sge_pair = (struct ulptx_sge_pair *)(sgl + 1);
334 continue;
335 }
336 if ((i - 1) & 0x1) {
337 sge_pair->addr[1] = cpu_to_be64(
338 sg_dma_address(sgel));
339 sge_pair->len[1] = cpu_to_be32(
340 sg_dma_len(sgel));
341 sge_pair++;
342 } else {
343 sge_pair->addr[0] = cpu_to_be64(
344 sg_dma_address(sgel));
345 sge_pair->len[0] = cpu_to_be32(
346 sg_dma_len(sgel));
347 }
348 }
349 } else {
350 /* Program sg elements with driver's DDP buffer */
351 xfer_len = scsi_bufflen(scmnd);
352 list_for_each(tmp, &req->gen_list) {
353 dma_buf = (struct csio_dma_buf *)tmp;
354 if (i == 0) {
355 sgl->addr0 = cpu_to_be64(dma_buf->paddr);
356 sgl->len0 = cpu_to_be32(
357 min(xfer_len, dma_buf->len));
358 sge_pair = (struct ulptx_sge_pair *)(sgl + 1);
359 } else if ((i - 1) & 0x1) {
360 sge_pair->addr[1] = cpu_to_be64(dma_buf->paddr);
361 sge_pair->len[1] = cpu_to_be32(
362 min(xfer_len, dma_buf->len));
363 sge_pair++;
364 } else {
365 sge_pair->addr[0] = cpu_to_be64(dma_buf->paddr);
366 sge_pair->len[0] = cpu_to_be32(
367 min(xfer_len, dma_buf->len));
368 }
369 xfer_len -= min(xfer_len, dma_buf->len);
370 i++;
371 }
372 }
373}
374
375/*
376 * csio_scsi_init_read_wr - Initialize the READ SCSI WR.
377 * @req: IO req structure.
378 * @wrp: DMA location to place the payload.
379 * @size: Size of WR (including FW WR + immed data + rsp SG entry + data SGL
380 *
381 * Wrapper for populating fw_scsi_read_wr.
382 */
383static inline void
384csio_scsi_init_read_wr(struct csio_ioreq *req, void *wrp, uint32_t size)
385{
386 struct csio_hw *hw = req->lnode->hwp;
387 struct csio_rnode *rn = req->rnode;
388 struct fw_scsi_read_wr *wr = (struct fw_scsi_read_wr *)wrp;
389 struct ulptx_sgl *sgl;
390 struct csio_dma_buf *dma_buf;
391 uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
392 struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
393
394 wr->op_immdlen = cpu_to_be32(FW_WR_OP(FW_SCSI_READ_WR) |
395 FW_SCSI_READ_WR_IMMDLEN(imm));
396 wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID(rn->flowid) |
397 FW_WR_LEN16(DIV_ROUND_UP(size, 16)));
398 wr->cookie = (uintptr_t)req;
399 wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
400 wr->tmo_val = (uint8_t)(req->tmo);
401 wr->use_xfer_cnt = 1;
402 wr->xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
403 wr->ini_xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
404 /* Get RSP DMA buffer */
405 dma_buf = &req->dma_buf;
406
407 /* Prepare RSP SGL */
408 wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
409 wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
410
411 wr->r4 = 0;
412
413 wr->u.fcoe.ctl_pri = 0;
414 wr->u.fcoe.cp_en_class = 0;
415 wr->u.fcoe.r3_lo[0] = 0;
416 wr->u.fcoe.r3_lo[1] = 0;
417 csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +
418 sizeof(struct fw_scsi_read_wr)));
419
420 /* Move WR pointer past command and immediate data */
421 sgl = (struct ulptx_sgl *)((uintptr_t)wrp +
422 sizeof(struct fw_scsi_read_wr) + ALIGN(imm, 16));
423
424 /* Fill in the DSGL */
425 csio_scsi_init_ultptx_dsgl(hw, req, sgl);
426}
427
428/*
429 * csio_scsi_init_write_wr - Initialize the WRITE SCSI WR.
430 * @req: IO req structure.
431 * @wrp: DMA location to place the payload.
432 * @size: Size of WR (including FW WR + immed data + rsp SG entry + data SGL
433 *
434 * Wrapper for populating fw_scsi_write_wr.
435 */
436static inline void
437csio_scsi_init_write_wr(struct csio_ioreq *req, void *wrp, uint32_t size)
438{
439 struct csio_hw *hw = req->lnode->hwp;
440 struct csio_rnode *rn = req->rnode;
441 struct fw_scsi_write_wr *wr = (struct fw_scsi_write_wr *)wrp;
442 struct ulptx_sgl *sgl;
443 struct csio_dma_buf *dma_buf;
444 uint8_t imm = csio_hw_to_scsim(hw)->proto_cmd_len;
445 struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
446
447 wr->op_immdlen = cpu_to_be32(FW_WR_OP(FW_SCSI_WRITE_WR) |
448 FW_SCSI_WRITE_WR_IMMDLEN(imm));
449 wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID(rn->flowid) |
450 FW_WR_LEN16(DIV_ROUND_UP(size, 16)));
451 wr->cookie = (uintptr_t)req;
452 wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
453 wr->tmo_val = (uint8_t)(req->tmo);
454 wr->use_xfer_cnt = 1;
455 wr->xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
456 wr->ini_xfer_cnt = cpu_to_be32(scsi_bufflen(scmnd));
457 /* Get RSP DMA buffer */
458 dma_buf = &req->dma_buf;
459
460 /* Prepare RSP SGL */
461 wr->rsp_dmalen = cpu_to_be32(dma_buf->len);
462 wr->rsp_dmaaddr = cpu_to_be64(dma_buf->paddr);
463
464 wr->r4 = 0;
465
466 wr->u.fcoe.ctl_pri = 0;
467 wr->u.fcoe.cp_en_class = 0;
468 wr->u.fcoe.r3_lo[0] = 0;
469 wr->u.fcoe.r3_lo[1] = 0;
470 csio_scsi_fcp_cmnd(req, (void *)((uintptr_t)wrp +
471 sizeof(struct fw_scsi_write_wr)));
472
473 /* Move WR pointer past command and immediate data */
474 sgl = (struct ulptx_sgl *)((uintptr_t)wrp +
475 sizeof(struct fw_scsi_write_wr) + ALIGN(imm, 16));
476
477 /* Fill in the DSGL */
478 csio_scsi_init_ultptx_dsgl(hw, req, sgl);
479}
480
481/* Calculate WR size needed for fw_scsi_read_wr/fw_scsi_write_wr */
482#define CSIO_SCSI_DATA_WRSZ(req, oper, sz, imm) \
483do { \
484 (sz) = sizeof(struct fw_scsi_##oper##_wr) + /* WR size */ \
485 ALIGN((imm), 16) + /* Immed data */ \
486 sizeof(struct ulptx_sgl); /* ulptx_sgl */ \
487 \
488 if (unlikely((req)->nsge > 1)) \
489 (sz) += (sizeof(struct ulptx_sge_pair) * \
490 (ALIGN(((req)->nsge - 1), 2) / 2)); \
491 /* Data SGE */ \
492} while (0)
493
494/*
495 * csio_scsi_read - Create a SCSI READ WR.
496 * @req: IO req structure.
497 *
498 * Gets a WR slot in the ingress queue and initializes it with
499 * SCSI READ WR.
500 *
501 */
502static inline void
503csio_scsi_read(struct csio_ioreq *req)
504{
505 struct csio_wr_pair wrp;
506 uint32_t size;
507 struct csio_hw *hw = req->lnode->hwp;
508 struct csio_scsim *scsim = csio_hw_to_scsim(hw);
509
510 CSIO_SCSI_DATA_WRSZ(req, read, size, scsim->proto_cmd_len);
511 size = ALIGN(size, 16);
512
513 req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
514 if (likely(req->drv_status == 0)) {
515 if (likely(wrp.size1 >= size)) {
516 /* Initialize WR in one shot */
517 csio_scsi_init_read_wr(req, wrp.addr1, size);
518 } else {
519 uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
520 /*
521 * Make a temporary copy of the WR and write back
522 * the copy into the WR pair.
523 */
524 csio_scsi_init_read_wr(req, (void *)tmpwr, size);
525 memcpy(wrp.addr1, tmpwr, wrp.size1);
526 memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
527 }
528 }
529}
530
531/*
532 * csio_scsi_write - Create a SCSI WRITE WR.
533 * @req: IO req structure.
534 *
535 * Gets a WR slot in the ingress queue and initializes it with
536 * SCSI WRITE WR.
537 *
538 */
539static inline void
540csio_scsi_write(struct csio_ioreq *req)
541{
542 struct csio_wr_pair wrp;
543 uint32_t size;
544 struct csio_hw *hw = req->lnode->hwp;
545 struct csio_scsim *scsim = csio_hw_to_scsim(hw);
546
547 CSIO_SCSI_DATA_WRSZ(req, write, size, scsim->proto_cmd_len);
548 size = ALIGN(size, 16);
549
550 req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
551 if (likely(req->drv_status == 0)) {
552 if (likely(wrp.size1 >= size)) {
553 /* Initialize WR in one shot */
554 csio_scsi_init_write_wr(req, wrp.addr1, size);
555 } else {
556 uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
557 /*
558 * Make a temporary copy of the WR and write back
559 * the copy into the WR pair.
560 */
561 csio_scsi_init_write_wr(req, (void *)tmpwr, size);
562 memcpy(wrp.addr1, tmpwr, wrp.size1);
563 memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
564 }
565 }
566}
567
568/*
569 * csio_setup_ddp - Setup DDP buffers for Read request.
570 * @req: IO req structure.
571 *
572 * Checks SGLs/Data buffers are virtually contiguous required for DDP.
573 * If contiguous,driver posts SGLs in the WR otherwise post internal
574 * buffers for such request for DDP.
575 */
576static inline void
577csio_setup_ddp(struct csio_scsim *scsim, struct csio_ioreq *req)
578{
579#ifdef __CSIO_DEBUG__
580 struct csio_hw *hw = req->lnode->hwp;
581#endif
582 struct scatterlist *sgel = NULL;
583 struct scsi_cmnd *scmnd = csio_scsi_cmnd(req);
584 uint64_t sg_addr = 0;
585 uint32_t ddp_pagesz = 4096;
586 uint32_t buf_off;
587 struct csio_dma_buf *dma_buf = NULL;
588 uint32_t alloc_len = 0;
589 uint32_t xfer_len = 0;
590 uint32_t sg_len = 0;
591 uint32_t i;
592
593 scsi_for_each_sg(scmnd, sgel, req->nsge, i) {
594 sg_addr = sg_dma_address(sgel);
595 sg_len = sg_dma_len(sgel);
596
597 buf_off = sg_addr & (ddp_pagesz - 1);
598
599 /* Except 1st buffer,all buffer addr have to be Page aligned */
600 if (i != 0 && buf_off) {
601 csio_dbg(hw, "SGL addr not DDP aligned (%llx:%d)\n",
602 sg_addr, sg_len);
603 goto unaligned;
604 }
605
606 /* Except last buffer,all buffer must end on page boundary */
607 if ((i != (req->nsge - 1)) &&
608 ((buf_off + sg_len) & (ddp_pagesz - 1))) {
609 csio_dbg(hw,
610 "SGL addr not ending on page boundary"
611 "(%llx:%d)\n", sg_addr, sg_len);
612 goto unaligned;
613 }
614 }
615
616 /* SGL's are virtually contiguous. HW will DDP to SGLs */
617 req->dcopy = 0;
618 csio_scsi_read(req);
619
620 return;
621
622unaligned:
623 CSIO_INC_STATS(scsim, n_unaligned);
624 /*
625 * For unaligned SGLs, driver will allocate internal DDP buffer.
626 * Once command is completed data from DDP buffer copied to SGLs
627 */
628 req->dcopy = 1;
629
630 /* Use gen_list to store the DDP buffers */
631 INIT_LIST_HEAD(&req->gen_list);
632 xfer_len = scsi_bufflen(scmnd);
633
634 i = 0;
635 /* Allocate ddp buffers for this request */
636 while (alloc_len < xfer_len) {
637 dma_buf = csio_get_scsi_ddp(scsim);
638 if (dma_buf == NULL || i > scsim->max_sge) {
639 req->drv_status = -EBUSY;
640 break;
641 }
642 alloc_len += dma_buf->len;
643 /* Added to IO req */
644 list_add_tail(&dma_buf->list, &req->gen_list);
645 i++;
646 }
647
648 if (!req->drv_status) {
649 /* set number of ddp bufs used */
650 req->nsge = i;
651 csio_scsi_read(req);
652 return;
653 }
654
655 /* release dma descs */
656 if (i > 0)
657 csio_put_scsi_ddp_list(scsim, &req->gen_list, i);
658}
659
660/*
661 * csio_scsi_init_abrt_cls_wr - Initialize an ABORT/CLOSE WR.
662 * @req: IO req structure.
663 * @addr: DMA location to place the payload.
664 * @size: Size of WR
665 * @abort: abort OR close
666 *
667 * Wrapper for populating fw_scsi_cmd_wr.
668 */
669static inline void
670csio_scsi_init_abrt_cls_wr(struct csio_ioreq *req, void *addr, uint32_t size,
671 bool abort)
672{
673 struct csio_hw *hw = req->lnode->hwp;
674 struct csio_rnode *rn = req->rnode;
675 struct fw_scsi_abrt_cls_wr *wr = (struct fw_scsi_abrt_cls_wr *)addr;
676
677 wr->op_immdlen = cpu_to_be32(FW_WR_OP(FW_SCSI_ABRT_CLS_WR));
678 wr->flowid_len16 = cpu_to_be32(FW_WR_FLOWID(rn->flowid) |
679 FW_WR_LEN16(
680 DIV_ROUND_UP(size, 16)));
681
682 wr->cookie = (uintptr_t) req;
683 wr->iqid = cpu_to_be16(csio_q_physiqid(hw, req->iq_idx));
684 wr->tmo_val = (uint8_t) req->tmo;
685 /* 0 for CHK_ALL_IO tells FW to look up t_cookie */
686 wr->sub_opcode_to_chk_all_io =
687 (FW_SCSI_ABRT_CLS_WR_SUB_OPCODE(abort) |
688 FW_SCSI_ABRT_CLS_WR_CHK_ALL_IO(0));
689 wr->r3[0] = 0;
690 wr->r3[1] = 0;
691 wr->r3[2] = 0;
692 wr->r3[3] = 0;
693 /* Since we re-use the same ioreq for abort as well */
694 wr->t_cookie = (uintptr_t) req;
695}
696
697static inline void
698csio_scsi_abrt_cls(struct csio_ioreq *req, bool abort)
699{
700 struct csio_wr_pair wrp;
701 struct csio_hw *hw = req->lnode->hwp;
702 uint32_t size = ALIGN(sizeof(struct fw_scsi_abrt_cls_wr), 16);
703
704 req->drv_status = csio_wr_get(hw, req->eq_idx, size, &wrp);
705 if (req->drv_status != 0)
706 return;
707
708 if (wrp.size1 >= size) {
709 /* Initialize WR in one shot */
710 csio_scsi_init_abrt_cls_wr(req, wrp.addr1, size, abort);
711 } else {
712 uint8_t *tmpwr = csio_q_eq_wrap(hw, req->eq_idx);
713 /*
714 * Make a temporary copy of the WR and write back
715 * the copy into the WR pair.
716 */
717 csio_scsi_init_abrt_cls_wr(req, (void *)tmpwr, size, abort);
718 memcpy(wrp.addr1, tmpwr, wrp.size1);
719 memcpy(wrp.addr2, tmpwr + wrp.size1, size - wrp.size1);
720 }
721}
722
723/*****************************************************************************/
724/* START: SCSI SM */
725/*****************************************************************************/
726static void
727csio_scsis_uninit(struct csio_ioreq *req, enum csio_scsi_ev evt)
728{
729 struct csio_hw *hw = req->lnode->hwp;
730 struct csio_scsim *scsim = csio_hw_to_scsim(hw);
731
732 switch (evt) {
733 case CSIO_SCSIE_START_IO:
734
735 if (req->nsge) {
736 if (req->datadir == DMA_TO_DEVICE) {
737 req->dcopy = 0;
738 csio_scsi_write(req);
739 } else
740 csio_setup_ddp(scsim, req);
741 } else {
742 csio_scsi_cmd(req);
743 }
744
745 if (likely(req->drv_status == 0)) {
746 /* change state and enqueue on active_q */
747 csio_set_state(&req->sm, csio_scsis_io_active);
748 list_add_tail(&req->sm.sm_list, &scsim->active_q);
749 csio_wr_issue(hw, req->eq_idx, false);
750 CSIO_INC_STATS(scsim, n_active);
751
752 return;
753 }
754 break;
755
756 case CSIO_SCSIE_START_TM:
757 csio_scsi_cmd(req);
758 if (req->drv_status == 0) {
759 /*
760 * NOTE: We collect the affected I/Os prior to issuing
761 * LUN reset, and not after it. This is to prevent
762 * aborting I/Os that get issued after the LUN reset,
763 * but prior to LUN reset completion (in the event that
764 * the host stack has not blocked I/Os to a LUN that is
765 * being reset.
766 */
767 csio_set_state(&req->sm, csio_scsis_tm_active);
768 list_add_tail(&req->sm.sm_list, &scsim->active_q);
769 csio_wr_issue(hw, req->eq_idx, false);
770 CSIO_INC_STATS(scsim, n_tm_active);
771 }
772 return;
773
774 case CSIO_SCSIE_ABORT:
775 case CSIO_SCSIE_CLOSE:
776 /*
777 * NOTE:
778 * We could get here due to :
779 * - a window in the cleanup path of the SCSI module
780 * (csio_scsi_abort_io()). Please see NOTE in this function.
781 * - a window in the time we tried to issue an abort/close
782 * of a request to FW, and the FW completed the request
783 * itself.
784 * Print a message for now, and return INVAL either way.
785 */
786 req->drv_status = -EINVAL;
787 csio_warn(hw, "Trying to abort/close completed IO:%p!\n", req);
788 break;
789
790 default:
791 csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
792 CSIO_DB_ASSERT(0);
793 }
794}
795
796static void
797csio_scsis_io_active(struct csio_ioreq *req, enum csio_scsi_ev evt)
798{
799 struct csio_hw *hw = req->lnode->hwp;
800 struct csio_scsim *scm = csio_hw_to_scsim(hw);
801 struct csio_rnode *rn;
802
803 switch (evt) {
804 case CSIO_SCSIE_COMPLETED:
805 CSIO_DEC_STATS(scm, n_active);
806 list_del_init(&req->sm.sm_list);
807 csio_set_state(&req->sm, csio_scsis_uninit);
808 /*
809 * In MSIX mode, with multiple queues, the SCSI compeltions
810 * could reach us sooner than the FW events sent to indicate
811 * I-T nexus loss (link down, remote device logo etc). We
812 * dont want to be returning such I/Os to the upper layer
813 * immediately, since we wouldnt have reported the I-T nexus
814 * loss itself. This forces us to serialize such completions
815 * with the reporting of the I-T nexus loss. Therefore, we
816 * internally queue up such up such completions in the rnode.
817 * The reporting of I-T nexus loss to the upper layer is then
818 * followed by the returning of I/Os in this internal queue.
819 * Having another state alongwith another queue helps us take
820 * actions for events such as ABORT received while we are
821 * in this rnode queue.
822 */
823 if (unlikely(req->wr_status != FW_SUCCESS)) {
824 rn = req->rnode;
825 /*
826 * FW says remote device is lost, but rnode
827 * doesnt reflect it.
828 */
829 if (csio_scsi_itnexus_loss_error(req->wr_status) &&
830 csio_is_rnode_ready(rn)) {
831 csio_set_state(&req->sm,
832 csio_scsis_shost_cmpl_await);
833 list_add_tail(&req->sm.sm_list,
834 &rn->host_cmpl_q);
835 }
836 }
837
838 break;
839
840 case CSIO_SCSIE_ABORT:
841 csio_scsi_abrt_cls(req, SCSI_ABORT);
842 if (req->drv_status == 0) {
843 csio_wr_issue(hw, req->eq_idx, false);
844 csio_set_state(&req->sm, csio_scsis_aborting);
845 }
846 break;
847
848 case CSIO_SCSIE_CLOSE:
849 csio_scsi_abrt_cls(req, SCSI_CLOSE);
850 if (req->drv_status == 0) {
851 csio_wr_issue(hw, req->eq_idx, false);
852 csio_set_state(&req->sm, csio_scsis_closing);
853 }
854 break;
855
856 case CSIO_SCSIE_DRVCLEANUP:
857 req->wr_status = FW_HOSTERROR;
858 CSIO_DEC_STATS(scm, n_active);
859 csio_set_state(&req->sm, csio_scsis_uninit);
860 break;
861
862 default:
863 csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
864 CSIO_DB_ASSERT(0);
865 }
866}
867
868static void
869csio_scsis_tm_active(struct csio_ioreq *req, enum csio_scsi_ev evt)
870{
871 struct csio_hw *hw = req->lnode->hwp;
872 struct csio_scsim *scm = csio_hw_to_scsim(hw);
873
874 switch (evt) {
875 case CSIO_SCSIE_COMPLETED:
876 CSIO_DEC_STATS(scm, n_tm_active);
877 list_del_init(&req->sm.sm_list);
878 csio_set_state(&req->sm, csio_scsis_uninit);
879
880 break;
881
882 case CSIO_SCSIE_ABORT:
883 csio_scsi_abrt_cls(req, SCSI_ABORT);
884 if (req->drv_status == 0) {
885 csio_wr_issue(hw, req->eq_idx, false);
886 csio_set_state(&req->sm, csio_scsis_aborting);
887 }
888 break;
889
890
891 case CSIO_SCSIE_CLOSE:
892 csio_scsi_abrt_cls(req, SCSI_CLOSE);
893 if (req->drv_status == 0) {
894 csio_wr_issue(hw, req->eq_idx, false);
895 csio_set_state(&req->sm, csio_scsis_closing);
896 }
897 break;
898
899 case CSIO_SCSIE_DRVCLEANUP:
900 req->wr_status = FW_HOSTERROR;
901 CSIO_DEC_STATS(scm, n_tm_active);
902 csio_set_state(&req->sm, csio_scsis_uninit);
903 break;
904
905 default:
906 csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
907 CSIO_DB_ASSERT(0);
908 }
909}
910
911static void
912csio_scsis_aborting(struct csio_ioreq *req, enum csio_scsi_ev evt)
913{
914 struct csio_hw *hw = req->lnode->hwp;
915 struct csio_scsim *scm = csio_hw_to_scsim(hw);
916
917 switch (evt) {
918 case CSIO_SCSIE_COMPLETED:
919 csio_dbg(hw,
920 "ioreq %p recvd cmpltd (wr_status:%d) "
921 "in aborting st\n", req, req->wr_status);
922 /*
923 * Use -ECANCELED to explicitly tell the ABORTED event that
924 * the original I/O was returned to driver by FW.
925 * We dont really care if the I/O was returned with success by
926 * FW (because the ABORT and completion of the I/O crossed each
927 * other), or any other return value. Once we are in aborting
928 * state, the success or failure of the I/O is unimportant to
929 * us.
930 */
931 req->drv_status = -ECANCELED;
932 break;
933
934 case CSIO_SCSIE_ABORT:
935 CSIO_INC_STATS(scm, n_abrt_dups);
936 break;
937
938 case CSIO_SCSIE_ABORTED:
939
940 csio_dbg(hw, "abort of %p return status:0x%x drv_status:%x\n",
941 req, req->wr_status, req->drv_status);
942 /*
943 * Check if original I/O WR completed before the Abort
944 * completion.
945 */
946 if (req->drv_status != -ECANCELED) {
947 csio_warn(hw,
948 "Abort completed before original I/O,"
949 " req:%p\n", req);
950 CSIO_DB_ASSERT(0);
951 }
952
953 /*
954 * There are the following possible scenarios:
955 * 1. The abort completed successfully, FW returned FW_SUCCESS.
956 * 2. The completion of an I/O and the receipt of
957 * abort for that I/O by the FW crossed each other.
958 * The FW returned FW_EINVAL. The original I/O would have
959 * returned with FW_SUCCESS or any other SCSI error.
960 * 3. The FW couldnt sent the abort out on the wire, as there
961 * was an I-T nexus loss (link down, remote device logged
962 * out etc). FW sent back an appropriate IT nexus loss status
963 * for the abort.
964 * 4. FW sent an abort, but abort timed out (remote device
965 * didnt respond). FW replied back with
966 * FW_SCSI_ABORT_TIMEDOUT.
967 * 5. FW couldnt genuinely abort the request for some reason,
968 * and sent us an error.
969 *
970 * The first 3 scenarios are treated as succesful abort
971 * operations by the host, while the last 2 are failed attempts
972 * to abort. Manipulate the return value of the request
973 * appropriately, so that host can convey these results
974 * back to the upper layer.
975 */
976 if ((req->wr_status == FW_SUCCESS) ||
977 (req->wr_status == FW_EINVAL) ||
978 csio_scsi_itnexus_loss_error(req->wr_status))
979 req->wr_status = FW_SCSI_ABORT_REQUESTED;
980
981 CSIO_DEC_STATS(scm, n_active);
982 list_del_init(&req->sm.sm_list);
983 csio_set_state(&req->sm, csio_scsis_uninit);
984 break;
985
986 case CSIO_SCSIE_DRVCLEANUP:
987 req->wr_status = FW_HOSTERROR;
988 CSIO_DEC_STATS(scm, n_active);
989 csio_set_state(&req->sm, csio_scsis_uninit);
990 break;
991
992 case CSIO_SCSIE_CLOSE:
993 /*
994 * We can receive this event from the module
995 * cleanup paths, if the FW forgot to reply to the ABORT WR
996 * and left this ioreq in this state. For now, just ignore
997 * the event. The CLOSE event is sent to this state, as
998 * the LINK may have already gone down.
999 */
1000 break;
1001
1002 default:
1003 csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
1004 CSIO_DB_ASSERT(0);
1005 }
1006}
1007
1008static void
1009csio_scsis_closing(struct csio_ioreq *req, enum csio_scsi_ev evt)
1010{
1011 struct csio_hw *hw = req->lnode->hwp;
1012 struct csio_scsim *scm = csio_hw_to_scsim(hw);
1013
1014 switch (evt) {
1015 case CSIO_SCSIE_COMPLETED:
1016 csio_dbg(hw,
1017 "ioreq %p recvd cmpltd (wr_status:%d) "
1018 "in closing st\n", req, req->wr_status);
1019 /*
1020 * Use -ECANCELED to explicitly tell the CLOSED event that
1021 * the original I/O was returned to driver by FW.
1022 * We dont really care if the I/O was returned with success by
1023 * FW (because the CLOSE and completion of the I/O crossed each
1024 * other), or any other return value. Once we are in aborting
1025 * state, the success or failure of the I/O is unimportant to
1026 * us.
1027 */
1028 req->drv_status = -ECANCELED;
1029 break;
1030
1031 case CSIO_SCSIE_CLOSED:
1032 /*
1033 * Check if original I/O WR completed before the Close
1034 * completion.
1035 */
1036 if (req->drv_status != -ECANCELED) {
1037 csio_fatal(hw,
1038 "Close completed before original I/O,"
1039 " req:%p\n", req);
1040 CSIO_DB_ASSERT(0);
1041 }
1042
1043 /*
1044 * Either close succeeded, or we issued close to FW at the
1045 * same time FW compelted it to us. Either way, the I/O
1046 * is closed.
1047 */
1048 CSIO_DB_ASSERT((req->wr_status == FW_SUCCESS) ||
1049 (req->wr_status == FW_EINVAL));
1050 req->wr_status = FW_SCSI_CLOSE_REQUESTED;
1051
1052 CSIO_DEC_STATS(scm, n_active);
1053 list_del_init(&req->sm.sm_list);
1054 csio_set_state(&req->sm, csio_scsis_uninit);
1055 break;
1056
1057 case CSIO_SCSIE_CLOSE:
1058 break;
1059
1060 case CSIO_SCSIE_DRVCLEANUP:
1061 req->wr_status = FW_HOSTERROR;
1062 CSIO_DEC_STATS(scm, n_active);
1063 csio_set_state(&req->sm, csio_scsis_uninit);
1064 break;
1065
1066 default:
1067 csio_dbg(hw, "Unhandled event:%d sent to req:%p\n", evt, req);
1068 CSIO_DB_ASSERT(0);
1069 }
1070}
1071
1072static void
1073csio_scsis_shost_cmpl_await(struct csio_ioreq *req, enum csio_scsi_ev evt)
1074{
1075 switch (evt) {
1076 case CSIO_SCSIE_ABORT:
1077 case CSIO_SCSIE_CLOSE:
1078 /*
1079 * Just succeed the abort request, and hope that
1080 * the remote device unregister path will cleanup
1081 * this I/O to the upper layer within a sane
1082 * amount of time.
1083 */
1084 /*
1085 * A close can come in during a LINK DOWN. The FW would have
1086 * returned us the I/O back, but not the remote device lost
1087 * FW event. In this interval, if the I/O times out at the upper
1088 * layer, a close can come in. Take the same action as abort:
1089 * return success, and hope that the remote device unregister
1090 * path will cleanup this I/O. If the FW still doesnt send
1091 * the msg, the close times out, and the upper layer resorts
1092 * to the next level of error recovery.
1093 */
1094 req->drv_status = 0;
1095 break;
1096 case CSIO_SCSIE_DRVCLEANUP:
1097 csio_set_state(&req->sm, csio_scsis_uninit);
1098 break;
1099 default:
1100 csio_dbg(req->lnode->hwp, "Unhandled event:%d sent to req:%p\n",
1101 evt, req);
1102 CSIO_DB_ASSERT(0);
1103 }
1104}
1105
1106/*
1107 * csio_scsi_cmpl_handler - WR completion handler for SCSI.
1108 * @hw: HW module.
1109 * @wr: The completed WR from the ingress queue.
1110 * @len: Length of the WR.
1111 * @flb: Freelist buffer array.
1112 * @priv: Private object
1113 * @scsiwr: Pointer to SCSI WR.
1114 *
1115 * This is the WR completion handler called per completion from the
1116 * ISR. It is called with lock held. It walks past the RSS and CPL message
1117 * header where the actual WR is present.
1118 * It then gets the status, WR handle (ioreq pointer) and the len of
1119 * the WR, based on WR opcode. Only on a non-good status is the entire
1120 * WR copied into the WR cache (ioreq->fw_wr).
1121 * The ioreq corresponding to the WR is returned to the caller.
1122 * NOTE: The SCSI queue doesnt allocate a freelist today, hence
1123 * no freelist buffer is expected.
1124 */
1125struct csio_ioreq *
1126csio_scsi_cmpl_handler(struct csio_hw *hw, void *wr, uint32_t len,
1127 struct csio_fl_dma_buf *flb, void *priv, uint8_t **scsiwr)
1128{
1129 struct csio_ioreq *ioreq = NULL;
1130 struct cpl_fw6_msg *cpl;
1131 uint8_t *tempwr;
1132 uint8_t status;
1133 struct csio_scsim *scm = csio_hw_to_scsim(hw);
1134
1135 /* skip RSS header */
1136 cpl = (struct cpl_fw6_msg *)((uintptr_t)wr + sizeof(__be64));
1137
1138 if (unlikely(cpl->opcode != CPL_FW6_MSG)) {
1139 csio_warn(hw, "Error: Invalid CPL msg %x recvd on SCSI q\n",
1140 cpl->opcode);
1141 CSIO_INC_STATS(scm, n_inval_cplop);
1142 return NULL;
1143 }
1144
1145 tempwr = (uint8_t *)(cpl->data);
1146 status = csio_wr_status(tempwr);
1147 *scsiwr = tempwr;
1148
1149 if (likely((*tempwr == FW_SCSI_READ_WR) ||
1150 (*tempwr == FW_SCSI_WRITE_WR) ||
1151 (*tempwr == FW_SCSI_CMD_WR))) {
1152 ioreq = (struct csio_ioreq *)((uintptr_t)
1153 (((struct fw_scsi_read_wr *)tempwr)->cookie));
1154 CSIO_DB_ASSERT(virt_addr_valid(ioreq));
1155
1156 ioreq->wr_status = status;
1157
1158 return ioreq;
1159 }
1160
1161 if (*tempwr == FW_SCSI_ABRT_CLS_WR) {
1162 ioreq = (struct csio_ioreq *)((uintptr_t)
1163 (((struct fw_scsi_abrt_cls_wr *)tempwr)->cookie));
1164 CSIO_DB_ASSERT(virt_addr_valid(ioreq));
1165
1166 ioreq->wr_status = status;
1167 return ioreq;
1168 }
1169
1170 csio_warn(hw, "WR with invalid opcode in SCSI IQ: %x\n", *tempwr);
1171 CSIO_INC_STATS(scm, n_inval_scsiop);
1172 return NULL;
1173}
1174
1175/*
1176 * csio_scsi_cleanup_io_q - Cleanup the given queue.
1177 * @scm: SCSI module.
1178 * @q: Queue to be cleaned up.
1179 *
1180 * Called with lock held. Has to exit with lock held.
1181 */
1182void
1183csio_scsi_cleanup_io_q(struct csio_scsim *scm, struct list_head *q)
1184{
1185 struct csio_hw *hw = scm->hw;
1186 struct csio_ioreq *ioreq;
1187 struct list_head *tmp, *next;
1188 struct scsi_cmnd *scmnd;
1189
1190 /* Call back the completion routines of the active_q */
1191 list_for_each_safe(tmp, next, q) {
1192 ioreq = (struct csio_ioreq *)tmp;
1193 csio_scsi_drvcleanup(ioreq);
1194 list_del_init(&ioreq->sm.sm_list);
1195 scmnd = csio_scsi_cmnd(ioreq);
1196 spin_unlock_irq(&hw->lock);
1197
1198 /*
1199 * Upper layers may have cleared this command, hence this
1200 * check to avoid accessing stale references.
1201 */
1202 if (scmnd != NULL)
1203 ioreq->io_cbfn(hw, ioreq);
1204
1205 spin_lock_irq(&scm->freelist_lock);
1206 csio_put_scsi_ioreq(scm, ioreq);
1207 spin_unlock_irq(&scm->freelist_lock);
1208
1209 spin_lock_irq(&hw->lock);
1210 }
1211}
1212
1213#define CSIO_SCSI_ABORT_Q_POLL_MS 2000
1214
1215static void
1216csio_abrt_cls(struct csio_ioreq *ioreq, struct scsi_cmnd *scmnd)
1217{
1218 struct csio_lnode *ln = ioreq->lnode;
1219 struct csio_hw *hw = ln->hwp;
1220 int ready = 0;
1221 struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1222 int rv;
1223
1224 if (csio_scsi_cmnd(ioreq) != scmnd) {
1225 CSIO_INC_STATS(scsim, n_abrt_race_comp);
1226 return;
1227 }
1228
1229 ready = csio_is_lnode_ready(ln);
1230
1231 rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
1232 if (rv != 0) {
1233 if (ready)
1234 CSIO_INC_STATS(scsim, n_abrt_busy_error);
1235 else
1236 CSIO_INC_STATS(scsim, n_cls_busy_error);
1237 }
1238}
1239
1240/*
1241 * csio_scsi_abort_io_q - Abort all I/Os on given queue
1242 * @scm: SCSI module.
1243 * @q: Queue to abort.
1244 * @tmo: Timeout in ms
1245 *
1246 * Attempt to abort all I/Os on given queue, and wait for a max
1247 * of tmo milliseconds for them to complete. Returns success
1248 * if all I/Os are aborted. Else returns -ETIMEDOUT.
1249 * Should be entered with lock held. Exits with lock held.
1250 * NOTE:
1251 * Lock has to be held across the loop that aborts I/Os, since dropping the lock
1252 * in between can cause the list to be corrupted. As a result, the caller
1253 * of this function has to ensure that the number of I/os to be aborted
1254 * is finite enough to not cause lock-held-for-too-long issues.
1255 */
1256static int
1257csio_scsi_abort_io_q(struct csio_scsim *scm, struct list_head *q, uint32_t tmo)
1258{
1259 struct csio_hw *hw = scm->hw;
1260 struct list_head *tmp, *next;
1261 int count = DIV_ROUND_UP(tmo, CSIO_SCSI_ABORT_Q_POLL_MS);
1262 struct scsi_cmnd *scmnd;
1263
1264 if (list_empty(q))
1265 return 0;
1266
1267 csio_dbg(hw, "Aborting SCSI I/Os\n");
1268
1269 /* Now abort/close I/Os in the queue passed */
1270 list_for_each_safe(tmp, next, q) {
1271 scmnd = csio_scsi_cmnd((struct csio_ioreq *)tmp);
1272 csio_abrt_cls((struct csio_ioreq *)tmp, scmnd);
1273 }
1274
1275 /* Wait till all active I/Os are completed/aborted/closed */
1276 while (!list_empty(q) && count--) {
1277 spin_unlock_irq(&hw->lock);
1278 msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1279 spin_lock_irq(&hw->lock);
1280 }
1281
1282 /* all aborts completed */
1283 if (list_empty(q))
1284 return 0;
1285
1286 return -ETIMEDOUT;
1287}
1288
1289/*
1290 * csio_scsim_cleanup_io - Cleanup all I/Os in SCSI module.
1291 * @scm: SCSI module.
1292 * @abort: abort required.
1293 * Called with lock held, should exit with lock held.
1294 * Can sleep when waiting for I/Os to complete.
1295 */
1296int
1297csio_scsim_cleanup_io(struct csio_scsim *scm, bool abort)
1298{
1299 struct csio_hw *hw = scm->hw;
1300 int rv = 0;
1301 int count = DIV_ROUND_UP(60 * 1000, CSIO_SCSI_ABORT_Q_POLL_MS);
1302
1303 /* No I/Os pending */
1304 if (list_empty(&scm->active_q))
1305 return 0;
1306
1307 /* Wait until all active I/Os are completed */
1308 while (!list_empty(&scm->active_q) && count--) {
1309 spin_unlock_irq(&hw->lock);
1310 msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1311 spin_lock_irq(&hw->lock);
1312 }
1313
1314 /* all I/Os completed */
1315 if (list_empty(&scm->active_q))
1316 return 0;
1317
1318 /* Else abort */
1319 if (abort) {
1320 rv = csio_scsi_abort_io_q(scm, &scm->active_q, 30000);
1321 if (rv == 0)
1322 return rv;
1323 csio_dbg(hw, "Some I/O aborts timed out, cleaning up..\n");
1324 }
1325
1326 csio_scsi_cleanup_io_q(scm, &scm->active_q);
1327
1328 CSIO_DB_ASSERT(list_empty(&scm->active_q));
1329
1330 return rv;
1331}
1332
1333/*
1334 * csio_scsim_cleanup_io_lnode - Cleanup all I/Os of given lnode.
1335 * @scm: SCSI module.
1336 * @lnode: lnode
1337 *
1338 * Called with lock held, should exit with lock held.
1339 * Can sleep (with dropped lock) when waiting for I/Os to complete.
1340 */
1341int
1342csio_scsim_cleanup_io_lnode(struct csio_scsim *scm, struct csio_lnode *ln)
1343{
1344 struct csio_hw *hw = scm->hw;
1345 struct csio_scsi_level_data sld;
1346 int rv;
1347 int count = DIV_ROUND_UP(60 * 1000, CSIO_SCSI_ABORT_Q_POLL_MS);
1348
1349 csio_dbg(hw, "Gathering all SCSI I/Os on lnode %p\n", ln);
1350
1351 sld.level = CSIO_LEV_LNODE;
1352 sld.lnode = ln;
1353 INIT_LIST_HEAD(&ln->cmpl_q);
1354 csio_scsi_gather_active_ios(scm, &sld, &ln->cmpl_q);
1355
1356 /* No I/Os pending on this lnode */
1357 if (list_empty(&ln->cmpl_q))
1358 return 0;
1359
1360 /* Wait until all active I/Os on this lnode are completed */
1361 while (!list_empty(&ln->cmpl_q) && count--) {
1362 spin_unlock_irq(&hw->lock);
1363 msleep(CSIO_SCSI_ABORT_Q_POLL_MS);
1364 spin_lock_irq(&hw->lock);
1365 }
1366
1367 /* all I/Os completed */
1368 if (list_empty(&ln->cmpl_q))
1369 return 0;
1370
1371 csio_dbg(hw, "Some I/Os pending on ln:%p, aborting them..\n", ln);
1372
1373 /* I/Os are pending, abort them */
1374 rv = csio_scsi_abort_io_q(scm, &ln->cmpl_q, 30000);
1375 if (rv != 0) {
1376 csio_dbg(hw, "Some I/O aborts timed out, cleaning up..\n");
1377 csio_scsi_cleanup_io_q(scm, &ln->cmpl_q);
1378 }
1379
1380 CSIO_DB_ASSERT(list_empty(&ln->cmpl_q));
1381
1382 return rv;
1383}
1384
1385static ssize_t
1386csio_show_hw_state(struct device *dev,
1387 struct device_attribute *attr, char *buf)
1388{
1389 struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1390 struct csio_hw *hw = csio_lnode_to_hw(ln);
1391
1392 if (csio_is_hw_ready(hw))
1393 return snprintf(buf, PAGE_SIZE, "ready\n");
1394 else
1395 return snprintf(buf, PAGE_SIZE, "not ready\n");
1396}
1397
1398/* Device reset */
1399static ssize_t
1400csio_device_reset(struct device *dev,
1401 struct device_attribute *attr, const char *buf, size_t count)
1402{
1403 struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1404 struct csio_hw *hw = csio_lnode_to_hw(ln);
1405
1406 if (*buf != '1')
1407 return -EINVAL;
1408
1409 /* Delete NPIV lnodes */
1410 csio_lnodes_exit(hw, 1);
1411
1412 /* Block upper IOs */
1413 csio_lnodes_block_request(hw);
1414
1415 spin_lock_irq(&hw->lock);
1416 csio_hw_reset(hw);
1417 spin_unlock_irq(&hw->lock);
1418
1419 /* Unblock upper IOs */
1420 csio_lnodes_unblock_request(hw);
1421 return count;
1422}
1423
1424/* disable port */
1425static ssize_t
1426csio_disable_port(struct device *dev,
1427 struct device_attribute *attr, const char *buf, size_t count)
1428{
1429 struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1430 struct csio_hw *hw = csio_lnode_to_hw(ln);
1431 bool disable;
1432
1433 if (*buf == '1' || *buf == '0')
1434 disable = (*buf == '1') ? true : false;
1435 else
1436 return -EINVAL;
1437
1438 /* Block upper IOs */
1439 csio_lnodes_block_by_port(hw, ln->portid);
1440
1441 spin_lock_irq(&hw->lock);
1442 csio_disable_lnodes(hw, ln->portid, disable);
1443 spin_unlock_irq(&hw->lock);
1444
1445 /* Unblock upper IOs */
1446 csio_lnodes_unblock_by_port(hw, ln->portid);
1447 return count;
1448}
1449
1450/* Show debug level */
1451static ssize_t
1452csio_show_dbg_level(struct device *dev,
1453 struct device_attribute *attr, char *buf)
1454{
1455 struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1456
1457 return snprintf(buf, PAGE_SIZE, "%x\n", ln->params.log_level);
1458}
1459
1460/* Store debug level */
1461static ssize_t
1462csio_store_dbg_level(struct device *dev,
1463 struct device_attribute *attr, const char *buf, size_t count)
1464{
1465 struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1466 struct csio_hw *hw = csio_lnode_to_hw(ln);
1467 uint32_t dbg_level = 0;
1468
1469 if (!isdigit(buf[0]))
1470 return -EINVAL;
1471
1472 if (sscanf(buf, "%i", &dbg_level))
1473 return -EINVAL;
1474
1475 ln->params.log_level = dbg_level;
1476 hw->params.log_level = dbg_level;
1477
1478 return 0;
1479}
1480
1481static DEVICE_ATTR(hw_state, S_IRUGO, csio_show_hw_state, NULL);
1482static DEVICE_ATTR(device_reset, S_IWUSR, NULL, csio_device_reset);
1483static DEVICE_ATTR(disable_port, S_IWUSR, NULL, csio_disable_port);
1484static DEVICE_ATTR(dbg_level, S_IRUGO | S_IWUSR, csio_show_dbg_level,
1485 csio_store_dbg_level);
1486
1487static struct device_attribute *csio_fcoe_lport_attrs[] = {
1488 &dev_attr_hw_state,
1489 &dev_attr_device_reset,
1490 &dev_attr_disable_port,
1491 &dev_attr_dbg_level,
1492 NULL,
1493};
1494
1495static ssize_t
1496csio_show_num_reg_rnodes(struct device *dev,
1497 struct device_attribute *attr, char *buf)
1498{
1499 struct csio_lnode *ln = shost_priv(class_to_shost(dev));
1500
1501 return snprintf(buf, PAGE_SIZE, "%d\n", ln->num_reg_rnodes);
1502}
1503
1504static DEVICE_ATTR(num_reg_rnodes, S_IRUGO, csio_show_num_reg_rnodes, NULL);
1505
1506static struct device_attribute *csio_fcoe_vport_attrs[] = {
1507 &dev_attr_num_reg_rnodes,
1508 &dev_attr_dbg_level,
1509 NULL,
1510};
1511
1512static inline uint32_t
1513csio_scsi_copy_to_sgl(struct csio_hw *hw, struct csio_ioreq *req)
1514{
1515 struct scsi_cmnd *scmnd = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1516 struct scatterlist *sg;
1517 uint32_t bytes_left;
1518 uint32_t bytes_copy;
1519 uint32_t buf_off = 0;
1520 uint32_t start_off = 0;
1521 uint32_t sg_off = 0;
1522 void *sg_addr;
1523 void *buf_addr;
1524 struct csio_dma_buf *dma_buf;
1525
1526 bytes_left = scsi_bufflen(scmnd);
1527 sg = scsi_sglist(scmnd);
1528 dma_buf = (struct csio_dma_buf *)csio_list_next(&req->gen_list);
1529
1530 /* Copy data from driver buffer to SGs of SCSI CMD */
1531 while (bytes_left > 0 && sg && dma_buf) {
1532 if (buf_off >= dma_buf->len) {
1533 buf_off = 0;
1534 dma_buf = (struct csio_dma_buf *)
1535 csio_list_next(dma_buf);
1536 continue;
1537 }
1538
1539 if (start_off >= sg->length) {
1540 start_off -= sg->length;
1541 sg = sg_next(sg);
1542 continue;
1543 }
1544
1545 buf_addr = dma_buf->vaddr + buf_off;
1546 sg_off = sg->offset + start_off;
1547 bytes_copy = min((dma_buf->len - buf_off),
1548 sg->length - start_off);
1549 bytes_copy = min((uint32_t)(PAGE_SIZE - (sg_off & ~PAGE_MASK)),
1550 bytes_copy);
1551
1552 sg_addr = kmap_atomic(sg_page(sg) + (sg_off >> PAGE_SHIFT));
1553 if (!sg_addr) {
1554 csio_err(hw, "failed to kmap sg:%p of ioreq:%p\n",
1555 sg, req);
1556 break;
1557 }
1558
1559 csio_dbg(hw, "copy_to_sgl:sg_addr %p sg_off %d buf %p len %d\n",
1560 sg_addr, sg_off, buf_addr, bytes_copy);
1561 memcpy(sg_addr + (sg_off & ~PAGE_MASK), buf_addr, bytes_copy);
1562 kunmap_atomic(sg_addr);
1563
1564 start_off += bytes_copy;
1565 buf_off += bytes_copy;
1566 bytes_left -= bytes_copy;
1567 }
1568
1569 if (bytes_left > 0)
1570 return DID_ERROR;
1571 else
1572 return DID_OK;
1573}
1574
1575/*
1576 * csio_scsi_err_handler - SCSI error handler.
1577 * @hw: HW module.
1578 * @req: IO request.
1579 *
1580 */
1581static inline void
1582csio_scsi_err_handler(struct csio_hw *hw, struct csio_ioreq *req)
1583{
1584 struct scsi_cmnd *cmnd = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1585 struct csio_scsim *scm = csio_hw_to_scsim(hw);
1586 struct fcp_resp_with_ext *fcp_resp;
1587 struct fcp_resp_rsp_info *rsp_info;
1588 struct csio_dma_buf *dma_buf;
1589 uint8_t flags, scsi_status = 0;
1590 uint32_t host_status = DID_OK;
1591 uint32_t rsp_len = 0, sns_len = 0;
1592 struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1593
1594
1595 switch (req->wr_status) {
1596 case FW_HOSTERROR:
1597 if (unlikely(!csio_is_hw_ready(hw)))
1598 return;
1599
1600 host_status = DID_ERROR;
1601 CSIO_INC_STATS(scm, n_hosterror);
1602
1603 break;
1604 case FW_SCSI_RSP_ERR:
1605 dma_buf = &req->dma_buf;
1606 fcp_resp = (struct fcp_resp_with_ext *)dma_buf->vaddr;
1607 rsp_info = (struct fcp_resp_rsp_info *)(fcp_resp + 1);
1608 flags = fcp_resp->resp.fr_flags;
1609 scsi_status = fcp_resp->resp.fr_status;
1610
1611 if (flags & FCP_RSP_LEN_VAL) {
1612 rsp_len = be32_to_cpu(fcp_resp->ext.fr_rsp_len);
1613 if ((rsp_len != 0 && rsp_len != 4 && rsp_len != 8) ||
1614 (rsp_info->rsp_code != FCP_TMF_CMPL)) {
1615 host_status = DID_ERROR;
1616 goto out;
1617 }
1618 }
1619
1620 if ((flags & FCP_SNS_LEN_VAL) && fcp_resp->ext.fr_sns_len) {
1621 sns_len = be32_to_cpu(fcp_resp->ext.fr_sns_len);
1622 if (sns_len > SCSI_SENSE_BUFFERSIZE)
1623 sns_len = SCSI_SENSE_BUFFERSIZE;
1624
1625 memcpy(cmnd->sense_buffer,
1626 &rsp_info->_fr_resvd[0] + rsp_len, sns_len);
1627 CSIO_INC_STATS(scm, n_autosense);
1628 }
1629
1630 scsi_set_resid(cmnd, 0);
1631
1632 /* Under run */
1633 if (flags & FCP_RESID_UNDER) {
1634 scsi_set_resid(cmnd,
1635 be32_to_cpu(fcp_resp->ext.fr_resid));
1636
1637 if (!(flags & FCP_SNS_LEN_VAL) &&
1638 (scsi_status == SAM_STAT_GOOD) &&
1639 ((scsi_bufflen(cmnd) - scsi_get_resid(cmnd))
1640 < cmnd->underflow))
1641 host_status = DID_ERROR;
1642 } else if (flags & FCP_RESID_OVER)
1643 host_status = DID_ERROR;
1644
1645 CSIO_INC_STATS(scm, n_rsperror);
1646 break;
1647
1648 case FW_SCSI_OVER_FLOW_ERR:
1649 csio_warn(hw,
1650 "Over-flow error,cmnd:0x%x expected len:0x%x"
1651 " resid:0x%x\n", cmnd->cmnd[0],
1652 scsi_bufflen(cmnd), scsi_get_resid(cmnd));
1653 host_status = DID_ERROR;
1654 CSIO_INC_STATS(scm, n_ovflerror);
1655 break;
1656
1657 case FW_SCSI_UNDER_FLOW_ERR:
1658 csio_warn(hw,
1659 "Under-flow error,cmnd:0x%x expected"
1660 " len:0x%x resid:0x%x lun:0x%x ssn:0x%x\n",
1661 cmnd->cmnd[0], scsi_bufflen(cmnd),
1662 scsi_get_resid(cmnd), cmnd->device->lun,
1663 rn->flowid);
1664 host_status = DID_ERROR;
1665 CSIO_INC_STATS(scm, n_unflerror);
1666 break;
1667
1668 case FW_SCSI_ABORT_REQUESTED:
1669 case FW_SCSI_ABORTED:
1670 case FW_SCSI_CLOSE_REQUESTED:
1671 csio_dbg(hw, "Req %p cmd:%p op:%x %s\n", req, cmnd,
1672 cmnd->cmnd[0],
1673 (req->wr_status == FW_SCSI_CLOSE_REQUESTED) ?
1674 "closed" : "aborted");
1675 /*
1676 * csio_eh_abort_handler checks this value to
1677 * succeed or fail the abort request.
1678 */
1679 host_status = DID_REQUEUE;
1680 if (req->wr_status == FW_SCSI_CLOSE_REQUESTED)
1681 CSIO_INC_STATS(scm, n_closed);
1682 else
1683 CSIO_INC_STATS(scm, n_aborted);
1684 break;
1685
1686 case FW_SCSI_ABORT_TIMEDOUT:
1687 /* FW timed out the abort itself */
1688 csio_dbg(hw, "FW timed out abort req:%p cmnd:%p status:%x\n",
1689 req, cmnd, req->wr_status);
1690 host_status = DID_ERROR;
1691 CSIO_INC_STATS(scm, n_abrt_timedout);
1692 break;
1693
1694 case FW_RDEV_NOT_READY:
1695 /*
1696 * In firmware, a RDEV can get into this state
1697 * temporarily, before moving into dissapeared/lost
1698 * state. So, the driver should complete the request equivalent
1699 * to device-disappeared!
1700 */
1701 CSIO_INC_STATS(scm, n_rdev_nr_error);
1702 host_status = DID_ERROR;
1703 break;
1704
1705 case FW_ERR_RDEV_LOST:
1706 CSIO_INC_STATS(scm, n_rdev_lost_error);
1707 host_status = DID_ERROR;
1708 break;
1709
1710 case FW_ERR_RDEV_LOGO:
1711 CSIO_INC_STATS(scm, n_rdev_logo_error);
1712 host_status = DID_ERROR;
1713 break;
1714
1715 case FW_ERR_RDEV_IMPL_LOGO:
1716 host_status = DID_ERROR;
1717 break;
1718
1719 case FW_ERR_LINK_DOWN:
1720 CSIO_INC_STATS(scm, n_link_down_error);
1721 host_status = DID_ERROR;
1722 break;
1723
1724 case FW_FCOE_NO_XCHG:
1725 CSIO_INC_STATS(scm, n_no_xchg_error);
1726 host_status = DID_ERROR;
1727 break;
1728
1729 default:
1730 csio_err(hw, "Unknown SCSI FW WR status:%d req:%p cmnd:%p\n",
1731 req->wr_status, req, cmnd);
1732 CSIO_DB_ASSERT(0);
1733
1734 CSIO_INC_STATS(scm, n_unknown_error);
1735 host_status = DID_ERROR;
1736 break;
1737 }
1738
1739out:
1740 if (req->nsge > 0)
1741 scsi_dma_unmap(cmnd);
1742
1743 cmnd->result = (((host_status) << 16) | scsi_status);
1744 cmnd->scsi_done(cmnd);
1745
1746 /* Wake up waiting threads */
1747 csio_scsi_cmnd(req) = NULL;
1748 complete_all(&req->cmplobj);
1749}
1750
1751/*
1752 * csio_scsi_cbfn - SCSI callback function.
1753 * @hw: HW module.
1754 * @req: IO request.
1755 *
1756 */
1757static void
1758csio_scsi_cbfn(struct csio_hw *hw, struct csio_ioreq *req)
1759{
1760 struct scsi_cmnd *cmnd = (struct scsi_cmnd *)csio_scsi_cmnd(req);
1761 uint8_t scsi_status = SAM_STAT_GOOD;
1762 uint32_t host_status = DID_OK;
1763
1764 if (likely(req->wr_status == FW_SUCCESS)) {
1765 if (req->nsge > 0) {
1766 scsi_dma_unmap(cmnd);
1767 if (req->dcopy)
1768 host_status = csio_scsi_copy_to_sgl(hw, req);
1769 }
1770
1771 cmnd->result = (((host_status) << 16) | scsi_status);
1772 cmnd->scsi_done(cmnd);
1773 csio_scsi_cmnd(req) = NULL;
1774 CSIO_INC_STATS(csio_hw_to_scsim(hw), n_tot_success);
1775 } else {
1776 /* Error handling */
1777 csio_scsi_err_handler(hw, req);
1778 }
1779}
1780
1781/**
1782 * csio_queuecommand - Entry point to kickstart an I/O request.
1783 * @host: The scsi_host pointer.
1784 * @cmnd: The I/O request from ML.
1785 *
1786 * This routine does the following:
1787 * - Checks for HW and Rnode module readiness.
1788 * - Gets a free ioreq structure (which is already initialized
1789 * to uninit during its allocation).
1790 * - Maps SG elements.
1791 * - Initializes ioreq members.
1792 * - Kicks off the SCSI state machine for this IO.
1793 * - Returns busy status on error.
1794 */
1795static int
1796csio_queuecommand(struct Scsi_Host *host, struct scsi_cmnd *cmnd)
1797{
1798 struct csio_lnode *ln = shost_priv(host);
1799 struct csio_hw *hw = csio_lnode_to_hw(ln);
1800 struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1801 struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1802 struct csio_ioreq *ioreq = NULL;
1803 unsigned long flags;
1804 int nsge = 0;
1805 int rv = SCSI_MLQUEUE_HOST_BUSY, nr;
1806 int retval;
1807 int cpu;
1808 struct csio_scsi_qset *sqset;
1809 struct fc_rport *rport = starget_to_rport(scsi_target(cmnd->device));
1810
1811 if (!blk_rq_cpu_valid(cmnd->request))
1812 cpu = smp_processor_id();
1813 else
1814 cpu = cmnd->request->cpu;
1815
1816 sqset = &hw->sqset[ln->portid][cpu];
1817
1818 nr = fc_remote_port_chkready(rport);
1819 if (nr) {
1820 cmnd->result = nr;
1821 CSIO_INC_STATS(scsim, n_rn_nr_error);
1822 goto err_done;
1823 }
1824
1825 if (unlikely(!csio_is_hw_ready(hw))) {
1826 cmnd->result = (DID_REQUEUE << 16);
1827 CSIO_INC_STATS(scsim, n_hw_nr_error);
1828 goto err_done;
1829 }
1830
1831 /* Get req->nsge, if there are SG elements to be mapped */
1832 nsge = scsi_dma_map(cmnd);
1833 if (unlikely(nsge < 0)) {
1834 CSIO_INC_STATS(scsim, n_dmamap_error);
1835 goto err;
1836 }
1837
1838 /* Do we support so many mappings? */
1839 if (unlikely(nsge > scsim->max_sge)) {
1840 csio_warn(hw,
1841 "More SGEs than can be supported."
1842 " SGEs: %d, Max SGEs: %d\n", nsge, scsim->max_sge);
1843 CSIO_INC_STATS(scsim, n_unsupp_sge_error);
1844 goto err_dma_unmap;
1845 }
1846
1847 /* Get a free ioreq structure - SM is already set to uninit */
1848 ioreq = csio_get_scsi_ioreq_lock(hw, scsim);
1849 if (!ioreq) {
1850 csio_err(hw, "Out of I/O request elements. Active #:%d\n",
1851 scsim->stats.n_active);
1852 CSIO_INC_STATS(scsim, n_no_req_error);
1853 goto err_dma_unmap;
1854 }
1855
1856 ioreq->nsge = nsge;
1857 ioreq->lnode = ln;
1858 ioreq->rnode = rn;
1859 ioreq->iq_idx = sqset->iq_idx;
1860 ioreq->eq_idx = sqset->eq_idx;
1861 ioreq->wr_status = 0;
1862 ioreq->drv_status = 0;
1863 csio_scsi_cmnd(ioreq) = (void *)cmnd;
1864 ioreq->tmo = 0;
1865 ioreq->datadir = cmnd->sc_data_direction;
1866
1867 if (cmnd->sc_data_direction == DMA_TO_DEVICE) {
1868 CSIO_INC_STATS(ln, n_output_requests);
1869 ln->stats.n_output_bytes += scsi_bufflen(cmnd);
1870 } else if (cmnd->sc_data_direction == DMA_FROM_DEVICE) {
1871 CSIO_INC_STATS(ln, n_input_requests);
1872 ln->stats.n_input_bytes += scsi_bufflen(cmnd);
1873 } else
1874 CSIO_INC_STATS(ln, n_control_requests);
1875
1876 /* Set cbfn */
1877 ioreq->io_cbfn = csio_scsi_cbfn;
1878
1879 /* Needed during abort */
1880 cmnd->host_scribble = (unsigned char *)ioreq;
1881 cmnd->SCp.Message = 0;
1882
1883 /* Kick off SCSI IO SM on the ioreq */
1884 spin_lock_irqsave(&hw->lock, flags);
1885 retval = csio_scsi_start_io(ioreq);
1886 spin_unlock_irqrestore(&hw->lock, flags);
1887
1888 if (retval != 0) {
1889 csio_err(hw, "ioreq: %p couldnt be started, status:%d\n",
1890 ioreq, retval);
1891 CSIO_INC_STATS(scsim, n_busy_error);
1892 goto err_put_req;
1893 }
1894
1895 return 0;
1896
1897err_put_req:
1898 csio_put_scsi_ioreq_lock(hw, scsim, ioreq);
1899err_dma_unmap:
1900 if (nsge > 0)
1901 scsi_dma_unmap(cmnd);
1902err:
1903 return rv;
1904
1905err_done:
1906 cmnd->scsi_done(cmnd);
1907 return 0;
1908}
1909
1910static int
1911csio_do_abrt_cls(struct csio_hw *hw, struct csio_ioreq *ioreq, bool abort)
1912{
1913 int rv;
1914 int cpu = smp_processor_id();
1915 struct csio_lnode *ln = ioreq->lnode;
1916 struct csio_scsi_qset *sqset = &hw->sqset[ln->portid][cpu];
1917
1918 ioreq->tmo = CSIO_SCSI_ABRT_TMO_MS;
1919 /*
1920 * Use current processor queue for posting the abort/close, but retain
1921 * the ingress queue ID of the original I/O being aborted/closed - we
1922 * need the abort/close completion to be received on the same queue
1923 * as the original I/O.
1924 */
1925 ioreq->eq_idx = sqset->eq_idx;
1926
1927 if (abort == SCSI_ABORT)
1928 rv = csio_scsi_abort(ioreq);
1929 else
1930 rv = csio_scsi_close(ioreq);
1931
1932 return rv;
1933}
1934
1935static int
1936csio_eh_abort_handler(struct scsi_cmnd *cmnd)
1937{
1938 struct csio_ioreq *ioreq;
1939 struct csio_lnode *ln = shost_priv(cmnd->device->host);
1940 struct csio_hw *hw = csio_lnode_to_hw(ln);
1941 struct csio_scsim *scsim = csio_hw_to_scsim(hw);
1942 int ready = 0, ret;
1943 unsigned long tmo = 0;
1944 int rv;
1945 struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
1946
1947 ret = fc_block_scsi_eh(cmnd);
1948 if (ret)
1949 return ret;
1950
1951 ioreq = (struct csio_ioreq *)cmnd->host_scribble;
1952 if (!ioreq)
1953 return SUCCESS;
1954
1955 if (!rn)
1956 return FAILED;
1957
1958 csio_dbg(hw,
1959 "Request to abort ioreq:%p cmd:%p cdb:%08llx"
1960 " ssni:0x%x lun:%d iq:0x%x\n",
1961 ioreq, cmnd, *((uint64_t *)cmnd->cmnd), rn->flowid,
1962 cmnd->device->lun, csio_q_physiqid(hw, ioreq->iq_idx));
1963
1964 if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) != cmnd) {
1965 CSIO_INC_STATS(scsim, n_abrt_race_comp);
1966 return SUCCESS;
1967 }
1968
1969 ready = csio_is_lnode_ready(ln);
1970 tmo = CSIO_SCSI_ABRT_TMO_MS;
1971
1972 spin_lock_irq(&hw->lock);
1973 rv = csio_do_abrt_cls(hw, ioreq, (ready ? SCSI_ABORT : SCSI_CLOSE));
1974 spin_unlock_irq(&hw->lock);
1975
1976 if (rv != 0) {
1977 if (rv == -EINVAL) {
1978 /* Return success, if abort/close request issued on
1979 * already completed IO
1980 */
1981 return SUCCESS;
1982 }
1983 if (ready)
1984 CSIO_INC_STATS(scsim, n_abrt_busy_error);
1985 else
1986 CSIO_INC_STATS(scsim, n_cls_busy_error);
1987
1988 goto inval_scmnd;
1989 }
1990
1991 /* Wait for completion */
1992 init_completion(&ioreq->cmplobj);
1993 wait_for_completion_timeout(&ioreq->cmplobj, msecs_to_jiffies(tmo));
1994
1995 /* FW didnt respond to abort within our timeout */
1996 if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd) {
1997
1998 csio_err(hw, "Abort timed out -- req: %p\n", ioreq);
1999 CSIO_INC_STATS(scsim, n_abrt_timedout);
2000
2001inval_scmnd:
2002 if (ioreq->nsge > 0)
2003 scsi_dma_unmap(cmnd);
2004
2005 spin_lock_irq(&hw->lock);
2006 csio_scsi_cmnd(ioreq) = NULL;
2007 spin_unlock_irq(&hw->lock);
2008
2009 cmnd->result = (DID_ERROR << 16);
2010 cmnd->scsi_done(cmnd);
2011
2012 return FAILED;
2013 }
2014
2015 /* FW successfully aborted the request */
2016 if (host_byte(cmnd->result) == DID_REQUEUE) {
2017 csio_info(hw,
2018 "Aborted SCSI command to (%d:%d) serial#:0x%lx\n",
2019 cmnd->device->id, cmnd->device->lun,
2020 cmnd->serial_number);
2021 return SUCCESS;
2022 } else {
2023 csio_info(hw,
2024 "Failed to abort SCSI command, (%d:%d) serial#:0x%lx\n",
2025 cmnd->device->id, cmnd->device->lun,
2026 cmnd->serial_number);
2027 return FAILED;
2028 }
2029}
2030
2031/*
2032 * csio_tm_cbfn - TM callback function.
2033 * @hw: HW module.
2034 * @req: IO request.
2035 *
2036 * Cache the result in 'cmnd', since ioreq will be freed soon
2037 * after we return from here, and the waiting thread shouldnt trust
2038 * the ioreq contents.
2039 */
2040static void
2041csio_tm_cbfn(struct csio_hw *hw, struct csio_ioreq *req)
2042{
2043 struct scsi_cmnd *cmnd = (struct scsi_cmnd *)csio_scsi_cmnd(req);
2044 struct csio_dma_buf *dma_buf;
2045 uint8_t flags = 0;
2046 struct fcp_resp_with_ext *fcp_resp;
2047 struct fcp_resp_rsp_info *rsp_info;
2048
2049 csio_dbg(hw, "req: %p in csio_tm_cbfn status: %d\n",
2050 req, req->wr_status);
2051
2052 /* Cache FW return status */
2053 cmnd->SCp.Status = req->wr_status;
2054
2055 /* Special handling based on FCP response */
2056
2057 /*
2058 * FW returns us this error, if flags were set. FCP4 says
2059 * FCP_RSP_LEN_VAL in flags shall be set for TM completions.
2060 * So if a target were to set this bit, we expect that the
2061 * rsp_code is set to FCP_TMF_CMPL for a successful TM
2062 * completion. Any other rsp_code means TM operation failed.
2063 * If a target were to just ignore setting flags, we treat
2064 * the TM operation as success, and FW returns FW_SUCCESS.
2065 */
2066 if (req->wr_status == FW_SCSI_RSP_ERR) {
2067 dma_buf = &req->dma_buf;
2068 fcp_resp = (struct fcp_resp_with_ext *)dma_buf->vaddr;
2069 rsp_info = (struct fcp_resp_rsp_info *)(fcp_resp + 1);
2070
2071 flags = fcp_resp->resp.fr_flags;
2072
2073 /* Modify return status if flags indicate success */
2074 if (flags & FCP_RSP_LEN_VAL)
2075 if (rsp_info->rsp_code == FCP_TMF_CMPL)
2076 cmnd->SCp.Status = FW_SUCCESS;
2077
2078 csio_dbg(hw, "TM FCP rsp code: %d\n", rsp_info->rsp_code);
2079 }
2080
2081 /* Wake up the TM handler thread */
2082 csio_scsi_cmnd(req) = NULL;
2083}
2084
2085static int
2086csio_eh_lun_reset_handler(struct scsi_cmnd *cmnd)
2087{
2088 struct csio_lnode *ln = shost_priv(cmnd->device->host);
2089 struct csio_hw *hw = csio_lnode_to_hw(ln);
2090 struct csio_scsim *scsim = csio_hw_to_scsim(hw);
2091 struct csio_rnode *rn = (struct csio_rnode *)(cmnd->device->hostdata);
2092 struct csio_ioreq *ioreq = NULL;
2093 struct csio_scsi_qset *sqset;
2094 unsigned long flags;
2095 int retval;
2096 int count, ret;
2097 LIST_HEAD(local_q);
2098 struct csio_scsi_level_data sld;
2099
2100 if (!rn)
2101 goto fail;
2102
2103 csio_dbg(hw, "Request to reset LUN:%d (ssni:0x%x tgtid:%d)\n",
2104 cmnd->device->lun, rn->flowid, rn->scsi_id);
2105
2106 if (!csio_is_lnode_ready(ln)) {
2107 csio_err(hw,
2108 "LUN reset cannot be issued on non-ready"
2109 " local node vnpi:0x%x (LUN:%d)\n",
2110 ln->vnp_flowid, cmnd->device->lun);
2111 goto fail;
2112 }
2113
2114 /* Lnode is ready, now wait on rport node readiness */
2115 ret = fc_block_scsi_eh(cmnd);
2116 if (ret)
2117 return ret;
2118
2119 /*
2120 * If we have blocked in the previous call, at this point, either the
2121 * remote node has come back online, or device loss timer has fired
2122 * and the remote node is destroyed. Allow the LUN reset only for
2123 * the former case, since LUN reset is a TMF I/O on the wire, and we
2124 * need a valid session to issue it.
2125 */
2126 if (fc_remote_port_chkready(rn->rport)) {
2127 csio_err(hw,
2128 "LUN reset cannot be issued on non-ready"
2129 " remote node ssni:0x%x (LUN:%d)\n",
2130 rn->flowid, cmnd->device->lun);
2131 goto fail;
2132 }
2133
2134 /* Get a free ioreq structure - SM is already set to uninit */
2135 ioreq = csio_get_scsi_ioreq_lock(hw, scsim);
2136
2137 if (!ioreq) {
2138 csio_err(hw, "Out of IO request elements. Active # :%d\n",
2139 scsim->stats.n_active);
2140 goto fail;
2141 }
2142
2143 sqset = &hw->sqset[ln->portid][smp_processor_id()];
2144 ioreq->nsge = 0;
2145 ioreq->lnode = ln;
2146 ioreq->rnode = rn;
2147 ioreq->iq_idx = sqset->iq_idx;
2148 ioreq->eq_idx = sqset->eq_idx;
2149
2150 csio_scsi_cmnd(ioreq) = cmnd;
2151 cmnd->host_scribble = (unsigned char *)ioreq;
2152 cmnd->SCp.Status = 0;
2153
2154 cmnd->SCp.Message = FCP_TMF_LUN_RESET;
2155 ioreq->tmo = CSIO_SCSI_LUNRST_TMO_MS / 1000;
2156
2157 /*
2158 * FW times the LUN reset for ioreq->tmo, so we got to wait a little
2159 * longer (10s for now) than that to allow FW to return the timed
2160 * out command.
2161 */
2162 count = DIV_ROUND_UP((ioreq->tmo + 10) * 1000, CSIO_SCSI_TM_POLL_MS);
2163
2164 /* Set cbfn */
2165 ioreq->io_cbfn = csio_tm_cbfn;
2166
2167 /* Save of the ioreq info for later use */
2168 sld.level = CSIO_LEV_LUN;
2169 sld.lnode = ioreq->lnode;
2170 sld.rnode = ioreq->rnode;
2171 sld.oslun = (uint64_t)cmnd->device->lun;
2172
2173 spin_lock_irqsave(&hw->lock, flags);
2174 /* Kick off TM SM on the ioreq */
2175 retval = csio_scsi_start_tm(ioreq);
2176 spin_unlock_irqrestore(&hw->lock, flags);
2177
2178 if (retval != 0) {
2179 csio_err(hw, "Failed to issue LUN reset, req:%p, status:%d\n",
2180 ioreq, retval);
2181 goto fail_ret_ioreq;
2182 }
2183
2184 csio_dbg(hw, "Waiting max %d secs for LUN reset completion\n",
2185 count * (CSIO_SCSI_TM_POLL_MS / 1000));
2186 /* Wait for completion */
2187 while ((((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd)
2188 && count--)
2189 msleep(CSIO_SCSI_TM_POLL_MS);
2190
2191 /* LUN reset timed-out */
2192 if (((struct scsi_cmnd *)csio_scsi_cmnd(ioreq)) == cmnd) {
2193 csio_err(hw, "LUN reset (%d:%d) timed out\n",
2194 cmnd->device->id, cmnd->device->lun);
2195
2196 spin_lock_irq(&hw->lock);
2197 csio_scsi_drvcleanup(ioreq);
2198 list_del_init(&ioreq->sm.sm_list);
2199 spin_unlock_irq(&hw->lock);
2200
2201 goto fail_ret_ioreq;
2202 }
2203
2204 /* LUN reset returned, check cached status */
2205 if (cmnd->SCp.Status != FW_SUCCESS) {
2206 csio_err(hw, "LUN reset failed (%d:%d), status: %d\n",
2207 cmnd->device->id, cmnd->device->lun, cmnd->SCp.Status);
2208 goto fail;
2209 }
2210
2211 /* LUN reset succeeded, Start aborting affected I/Os */
2212 /*
2213 * Since the host guarantees during LUN reset that there
2214 * will not be any more I/Os to that LUN, until the LUN reset
2215 * completes, we gather pending I/Os after the LUN reset.
2216 */
2217 spin_lock_irq(&hw->lock);
2218 csio_scsi_gather_active_ios(scsim, &sld, &local_q);
2219
2220 retval = csio_scsi_abort_io_q(scsim, &local_q, 30000);
2221 spin_unlock_irq(&hw->lock);
2222
2223 /* Aborts may have timed out */
2224 if (retval != 0) {
2225 csio_err(hw,
2226 "Attempt to abort I/Os during LUN reset of %d"
2227 " returned %d\n", cmnd->device->lun, retval);
2228 /* Return I/Os back to active_q */
2229 spin_lock_irq(&hw->lock);
2230 list_splice_tail_init(&local_q, &scsim->active_q);
2231 spin_unlock_irq(&hw->lock);
2232 goto fail;
2233 }
2234
2235 CSIO_INC_STATS(rn, n_lun_rst);
2236
2237 csio_info(hw, "LUN reset occurred (%d:%d)\n",
2238 cmnd->device->id, cmnd->device->lun);
2239
2240 return SUCCESS;
2241
2242fail_ret_ioreq:
2243 csio_put_scsi_ioreq_lock(hw, scsim, ioreq);
2244fail:
2245 CSIO_INC_STATS(rn, n_lun_rst_fail);
2246 return FAILED;
2247}
2248
2249static int
2250csio_slave_alloc(struct scsi_device *sdev)
2251{
2252 struct fc_rport *rport = starget_to_rport(scsi_target(sdev));
2253
2254 if (!rport || fc_remote_port_chkready(rport))
2255 return -ENXIO;
2256
2257 sdev->hostdata = *((struct csio_lnode **)(rport->dd_data));
2258
2259 return 0;
2260}
2261
2262static int
2263csio_slave_configure(struct scsi_device *sdev)
2264{
2265 if (sdev->tagged_supported)
2266 scsi_activate_tcq(sdev, csio_lun_qdepth);
2267 else
2268 scsi_deactivate_tcq(sdev, csio_lun_qdepth);
2269
2270 return 0;
2271}
2272
2273static void
2274csio_slave_destroy(struct scsi_device *sdev)
2275{
2276 sdev->hostdata = NULL;
2277}
2278
2279static int
2280csio_scan_finished(struct Scsi_Host *shost, unsigned long time)
2281{
2282 struct csio_lnode *ln = shost_priv(shost);
2283 int rv = 1;
2284
2285 spin_lock_irq(shost->host_lock);
2286 if (!ln->hwp || csio_list_deleted(&ln->sm.sm_list))
2287 goto out;
2288
2289 rv = csio_scan_done(ln, jiffies, time, csio_max_scan_tmo * HZ,
2290 csio_delta_scan_tmo * HZ);
2291out:
2292 spin_unlock_irq(shost->host_lock);
2293
2294 return rv;
2295}
2296
2297struct scsi_host_template csio_fcoe_shost_template = {
2298 .module = THIS_MODULE,
2299 .name = CSIO_DRV_DESC,
2300 .proc_name = KBUILD_MODNAME,
2301 .queuecommand = csio_queuecommand,
2302 .eh_abort_handler = csio_eh_abort_handler,
2303 .eh_device_reset_handler = csio_eh_lun_reset_handler,
2304 .slave_alloc = csio_slave_alloc,
2305 .slave_configure = csio_slave_configure,
2306 .slave_destroy = csio_slave_destroy,
2307 .scan_finished = csio_scan_finished,
2308 .this_id = -1,
2309 .sg_tablesize = CSIO_SCSI_MAX_SGE,
2310 .cmd_per_lun = CSIO_MAX_CMD_PER_LUN,
2311 .use_clustering = ENABLE_CLUSTERING,
2312 .shost_attrs = csio_fcoe_lport_attrs,
2313 .max_sectors = CSIO_MAX_SECTOR_SIZE,
2314};
2315
2316struct scsi_host_template csio_fcoe_shost_vport_template = {
2317 .module = THIS_MODULE,
2318 .name = CSIO_DRV_DESC,
2319 .proc_name = KBUILD_MODNAME,
2320 .queuecommand = csio_queuecommand,
2321 .eh_abort_handler = csio_eh_abort_handler,
2322 .eh_device_reset_handler = csio_eh_lun_reset_handler,
2323 .slave_alloc = csio_slave_alloc,
2324 .slave_configure = csio_slave_configure,
2325 .slave_destroy = csio_slave_destroy,
2326 .scan_finished = csio_scan_finished,
2327 .this_id = -1,
2328 .sg_tablesize = CSIO_SCSI_MAX_SGE,
2329 .cmd_per_lun = CSIO_MAX_CMD_PER_LUN,
2330 .use_clustering = ENABLE_CLUSTERING,
2331 .shost_attrs = csio_fcoe_vport_attrs,
2332 .max_sectors = CSIO_MAX_SECTOR_SIZE,
2333};
2334
2335/*
2336 * csio_scsi_alloc_ddp_bufs - Allocate buffers for DDP of unaligned SGLs.
2337 * @scm: SCSI Module
2338 * @hw: HW device.
2339 * @buf_size: buffer size
2340 * @num_buf : Number of buffers.
2341 *
2342 * This routine allocates DMA buffers required for SCSI Data xfer, if
2343 * each SGL buffer for a SCSI Read request posted by SCSI midlayer are
2344 * not virtually contiguous.
2345 */
2346static int
2347csio_scsi_alloc_ddp_bufs(struct csio_scsim *scm, struct csio_hw *hw,
2348 int buf_size, int num_buf)
2349{
2350 int n = 0;
2351 struct list_head *tmp;
2352 struct csio_dma_buf *ddp_desc = NULL;
2353 uint32_t unit_size = 0;
2354
2355 if (!num_buf)
2356 return 0;
2357
2358 if (!buf_size)
2359 return -EINVAL;
2360
2361 INIT_LIST_HEAD(&scm->ddp_freelist);
2362
2363 /* Align buf size to page size */
2364 buf_size = (buf_size + PAGE_SIZE - 1) & PAGE_MASK;
2365 /* Initialize dma descriptors */
2366 for (n = 0; n < num_buf; n++) {
2367 /* Set unit size to request size */
2368 unit_size = buf_size;
2369 ddp_desc = kzalloc(sizeof(struct csio_dma_buf), GFP_KERNEL);
2370 if (!ddp_desc) {
2371 csio_err(hw,
2372 "Failed to allocate ddp descriptors,"
2373 " Num allocated = %d.\n",
2374 scm->stats.n_free_ddp);
2375 goto no_mem;
2376 }
2377
2378 /* Allocate Dma buffers for DDP */
2379 ddp_desc->vaddr = pci_alloc_consistent(hw->pdev, unit_size,
2380 &ddp_desc->paddr);
2381 if (!ddp_desc->vaddr) {
2382 csio_err(hw,
2383 "SCSI response DMA buffer (ddp) allocation"
2384 " failed!\n");
2385 kfree(ddp_desc);
2386 goto no_mem;
2387 }
2388
2389 ddp_desc->len = unit_size;
2390
2391 /* Added it to scsi ddp freelist */
2392 list_add_tail(&ddp_desc->list, &scm->ddp_freelist);
2393 CSIO_INC_STATS(scm, n_free_ddp);
2394 }
2395
2396 return 0;
2397no_mem:
2398 /* release dma descs back to freelist and free dma memory */
2399 list_for_each(tmp, &scm->ddp_freelist) {
2400 ddp_desc = (struct csio_dma_buf *) tmp;
2401 tmp = csio_list_prev(tmp);
2402 pci_free_consistent(hw->pdev, ddp_desc->len, ddp_desc->vaddr,
2403 ddp_desc->paddr);
2404 list_del_init(&ddp_desc->list);
2405 kfree(ddp_desc);
2406 }
2407 scm->stats.n_free_ddp = 0;
2408
2409 return -ENOMEM;
2410}
2411
2412/*
2413 * csio_scsi_free_ddp_bufs - free DDP buffers of unaligned SGLs.
2414 * @scm: SCSI Module
2415 * @hw: HW device.
2416 *
2417 * This routine frees ddp buffers.
2418 */
2419static void
2420csio_scsi_free_ddp_bufs(struct csio_scsim *scm, struct csio_hw *hw)
2421{
2422 struct list_head *tmp;
2423 struct csio_dma_buf *ddp_desc;
2424
2425 /* release dma descs back to freelist and free dma memory */
2426 list_for_each(tmp, &scm->ddp_freelist) {
2427 ddp_desc = (struct csio_dma_buf *) tmp;
2428 tmp = csio_list_prev(tmp);
2429 pci_free_consistent(hw->pdev, ddp_desc->len, ddp_desc->vaddr,
2430 ddp_desc->paddr);
2431 list_del_init(&ddp_desc->list);
2432 kfree(ddp_desc);
2433 }
2434 scm->stats.n_free_ddp = 0;
2435}
2436
2437/**
2438 * csio_scsim_init - Initialize SCSI Module
2439 * @scm: SCSI Module
2440 * @hw: HW module
2441 *
2442 */
2443int
2444csio_scsim_init(struct csio_scsim *scm, struct csio_hw *hw)
2445{
2446 int i;
2447 struct csio_ioreq *ioreq;
2448 struct csio_dma_buf *dma_buf;
2449
2450 INIT_LIST_HEAD(&scm->active_q);
2451 scm->hw = hw;
2452
2453 scm->proto_cmd_len = sizeof(struct fcp_cmnd);
2454 scm->proto_rsp_len = CSIO_SCSI_RSP_LEN;
2455 scm->max_sge = CSIO_SCSI_MAX_SGE;
2456
2457 spin_lock_init(&scm->freelist_lock);
2458
2459 /* Pre-allocate ioreqs and initialize them */
2460 INIT_LIST_HEAD(&scm->ioreq_freelist);
2461 for (i = 0; i < csio_scsi_ioreqs; i++) {
2462
2463 ioreq = kzalloc(sizeof(struct csio_ioreq), GFP_KERNEL);
2464 if (!ioreq) {
2465 csio_err(hw,
2466 "I/O request element allocation failed, "
2467 " Num allocated = %d.\n",
2468 scm->stats.n_free_ioreq);
2469
2470 goto free_ioreq;
2471 }
2472
2473 /* Allocate Dma buffers for Response Payload */
2474 dma_buf = &ioreq->dma_buf;
2475 dma_buf->vaddr = pci_pool_alloc(hw->scsi_pci_pool, GFP_KERNEL,
2476 &dma_buf->paddr);
2477 if (!dma_buf->vaddr) {
2478 csio_err(hw,
2479 "SCSI response DMA buffer allocation"
2480 " failed!\n");
2481 kfree(ioreq);
2482 goto free_ioreq;
2483 }
2484
2485 dma_buf->len = scm->proto_rsp_len;
2486
2487 /* Set state to uninit */
2488 csio_init_state(&ioreq->sm, csio_scsis_uninit);
2489 INIT_LIST_HEAD(&ioreq->gen_list);
2490 init_completion(&ioreq->cmplobj);
2491
2492 list_add_tail(&ioreq->sm.sm_list, &scm->ioreq_freelist);
2493 CSIO_INC_STATS(scm, n_free_ioreq);
2494 }
2495
2496 if (csio_scsi_alloc_ddp_bufs(scm, hw, PAGE_SIZE, csio_ddp_descs))
2497 goto free_ioreq;
2498
2499 return 0;
2500
2501free_ioreq:
2502 /*
2503 * Free up existing allocations, since an error
2504 * from here means we are returning for good
2505 */
2506 while (!list_empty(&scm->ioreq_freelist)) {
2507 struct csio_sm *tmp;
2508
2509 tmp = list_first_entry(&scm->ioreq_freelist,
2510 struct csio_sm, sm_list);
2511 list_del_init(&tmp->sm_list);
2512 ioreq = (struct csio_ioreq *)tmp;
2513
2514 dma_buf = &ioreq->dma_buf;
2515 pci_pool_free(hw->scsi_pci_pool, dma_buf->vaddr,
2516 dma_buf->paddr);
2517
2518 kfree(ioreq);
2519 }
2520
2521 scm->stats.n_free_ioreq = 0;
2522
2523 return -ENOMEM;
2524}
2525
2526/**
2527 * csio_scsim_exit: Uninitialize SCSI Module
2528 * @scm: SCSI Module
2529 *
2530 */
2531void
2532csio_scsim_exit(struct csio_scsim *scm)
2533{
2534 struct csio_ioreq *ioreq;
2535 struct csio_dma_buf *dma_buf;
2536
2537 while (!list_empty(&scm->ioreq_freelist)) {
2538 struct csio_sm *tmp;
2539
2540 tmp = list_first_entry(&scm->ioreq_freelist,
2541 struct csio_sm, sm_list);
2542 list_del_init(&tmp->sm_list);
2543 ioreq = (struct csio_ioreq *)tmp;
2544
2545 dma_buf = &ioreq->dma_buf;
2546 pci_pool_free(scm->hw->scsi_pci_pool, dma_buf->vaddr,
2547 dma_buf->paddr);
2548
2549 kfree(ioreq);
2550 }
2551
2552 scm->stats.n_free_ioreq = 0;
2553
2554 csio_scsi_free_ddp_bufs(scm, scm->hw);
2555}