1/******************************************************************************
   2 * This file contains error recovery level zero functions used by
   3 * the iSCSI Target driver.
   4 *
   5 * \u00a9 Copyright 2007-2011 RisingTide Systems LLC.
   6 *
   7 * Licensed to the Linux Foundation under the General Public License (GPL) version 2.
   8 *
   9 * Author: Nicholas A. Bellinger <nab@linux-iscsi.org>
  10 *
  11 * This program is free software; you can redistribute it and/or modify
  12 * it under the terms of the GNU General Public License as published by
  13 * the Free Software Foundation; either version 2 of the License, or
  14 * (at your option) any later version.
  15 *
  16 * This program is distributed in the hope that it will be useful,
  17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  19 * GNU General Public License for more details.
  20 ******************************************************************************/
  21
  22#include <scsi/iscsi_proto.h>
  23#include <target/target_core_base.h>
  24#include <target/target_core_transport.h>
  25
  26#include "iscsi_target_core.h"
  27#include "iscsi_target_seq_pdu_list.h"
  28#include "iscsi_target_tq.h"
  29#include "iscsi_target_erl0.h"
  30#include "iscsi_target_erl1.h"
  31#include "iscsi_target_erl2.h"
  32#include "iscsi_target_util.h"
  33#include "iscsi_target.h"
  34
  35/*
  36 *	Used to set values in struct iscsi_cmd that iscsit_dataout_check_sequence()
  37 *	checks against to determine a PDU's Offset+Length is within the current
  38 *	DataOUT Sequence.  Used for DataSequenceInOrder=Yes only.
  39 */
  40void iscsit_set_dataout_sequence_values(
  41	struct iscsi_cmd *cmd)
  42{
  43	struct iscsi_conn *conn = cmd->conn;
  44	/*
  45	 * Still set seq_start_offset and seq_end_offset for Unsolicited
  46	 * DataOUT, even if DataSequenceInOrder=No.
  47	 */
  48	if (cmd->unsolicited_data) {
  49		cmd->seq_start_offset = cmd->write_data_done;
  50		cmd->seq_end_offset = (cmd->write_data_done +
  51			(cmd->data_length >
  52			 conn->sess->sess_ops->FirstBurstLength) ?
  53			conn->sess->sess_ops->FirstBurstLength : cmd->data_length);
  54		return;
  55	}
  56
  57	if (!conn->sess->sess_ops->DataSequenceInOrder)
  58		return;
  59
  60	if (!cmd->seq_start_offset && !cmd->seq_end_offset) {
  61		cmd->seq_start_offset = cmd->write_data_done;
  62		cmd->seq_end_offset = (cmd->data_length >
  63			conn->sess->sess_ops->MaxBurstLength) ?
  64			(cmd->write_data_done +
  65			conn->sess->sess_ops->MaxBurstLength) : cmd->data_length;
  66	} else {
  67		cmd->seq_start_offset = cmd->seq_end_offset;
  68		cmd->seq_end_offset = ((cmd->seq_end_offset +
  69			conn->sess->sess_ops->MaxBurstLength) >=
  70			cmd->data_length) ? cmd->data_length :
  71			(cmd->seq_end_offset +
  72			 conn->sess->sess_ops->MaxBurstLength);
  73	}
  74}
  75
  76static int iscsit_dataout_within_command_recovery_check(
  77	struct iscsi_cmd *cmd,
  78	unsigned char *buf)
  79{
  80	struct iscsi_conn *conn = cmd->conn;
  81	struct iscsi_data *hdr = (struct iscsi_data *) buf;
  82	u32 payload_length = ntoh24(hdr->dlength);
  83
  84	/*
  85	 * We do the within-command recovery checks here as it is
  86	 * the first function called in iscsi_check_pre_dataout().
  87	 * Basically, if we are in within-command recovery and
  88	 * the PDU does not contain the offset the sequence needs,
  89	 * dump the payload.
  90	 *
  91	 * This only applies to DataPDUInOrder=Yes, for
  92	 * DataPDUInOrder=No we only re-request the failed PDU
  93	 * and check that all PDUs in a sequence are received
  94	 * upon end of sequence.
  95	 */
  96	if (conn->sess->sess_ops->DataSequenceInOrder) {
  97		if ((cmd->cmd_flags & ICF_WITHIN_COMMAND_RECOVERY) &&
  98		    (cmd->write_data_done != hdr->offset))
  99			goto dump;
 100
 101		cmd->cmd_flags &= ~ICF_WITHIN_COMMAND_RECOVERY;
 102	} else {
 103		struct iscsi_seq *seq;
 104
 105		seq = iscsit_get_seq_holder(cmd, hdr->offset, payload_length);
 106		if (!seq)
 107			return DATAOUT_CANNOT_RECOVER;
 108		/*
 109		 * Set the struct iscsi_seq pointer to reuse later.
 110		 */
 111		cmd->seq_ptr = seq;
 112
 113		if (conn->sess->sess_ops->DataPDUInOrder) {
 114			if ((seq->status ==
 115			     DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY) &&
 116			   ((seq->offset != hdr->offset) ||
 117			    (seq->data_sn != hdr->datasn)))
 118				goto dump;
 119		} else {
 120			if ((seq->status ==
 121			     DATAOUT_SEQUENCE_WITHIN_COMMAND_RECOVERY) &&
 122			    (seq->data_sn != hdr->datasn))
 123				goto dump;
 124		}
 125
 126		if (seq->status == DATAOUT_SEQUENCE_COMPLETE)
 127			goto dump;
 128
 129		if (seq->status != DATAOUT_SEQUENCE_COMPLETE)
 130			seq->status = 0;
 131	}
 132
 133	return DATAOUT_NORMAL;
 134
 135dump:
 136	pr_err("Dumping DataOUT PDU Offset: %u Length: %d DataSN:"
 137		" 0x%08x\n", hdr->offset, payload_length, hdr->datasn);
 138	return iscsit_dump_data_payload(conn, payload_length, 1);
 139}
 140
 141static int iscsit_dataout_check_unsolicited_sequence(
 142	struct iscsi_cmd *cmd,
 143	unsigned char *buf)
 144{
 145	u32 first_burst_len;
 146	struct iscsi_conn *conn = cmd->conn;
 147	struct iscsi_data *hdr = (struct iscsi_data *) buf;
 148	u32 payload_length = ntoh24(hdr->dlength);
 149
 150
 151	if ((hdr->offset < cmd->seq_start_offset) ||
 152	   ((hdr->offset + payload_length) > cmd->seq_end_offset)) {
 153		pr_err("Command ITT: 0x%08x with Offset: %u,"
 154		" Length: %u outside of Unsolicited Sequence %u:%u while"
 155		" DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 156		hdr->offset, payload_length, cmd->seq_start_offset,
 157			cmd->seq_end_offset);
 158		return DATAOUT_CANNOT_RECOVER;
 159	}
 160
 161	first_burst_len = (cmd->first_burst_len + payload_length);
 162
 163	if (first_burst_len > conn->sess->sess_ops->FirstBurstLength) {
 164		pr_err("Total %u bytes exceeds FirstBurstLength: %u"
 165			" for this Unsolicited DataOut Burst.\n",
 166			first_burst_len, conn->sess->sess_ops->FirstBurstLength);
 167		transport_send_check_condition_and_sense(&cmd->se_cmd,
 168				TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 169		return DATAOUT_CANNOT_RECOVER;
 170	}
 171
 172	/*
 173	 * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 174	 * checks for the current Unsolicited DataOUT Sequence.
 175	 */
 176	if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 177		/*
 178		 * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 179		 * sequence checks are handled in
 180		 * iscsit_dataout_datapduinorder_no_fbit().
 181		 */
 182		if (!conn->sess->sess_ops->DataPDUInOrder)
 183			goto out;
 184
 185		if ((first_burst_len != cmd->data_length) &&
 186		    (first_burst_len != conn->sess->sess_ops->FirstBurstLength)) {
 187			pr_err("Unsolicited non-immediate data"
 188			" received %u does not equal FirstBurstLength: %u, and"
 189			" does not equal ExpXferLen %u.\n", first_burst_len,
 190				conn->sess->sess_ops->FirstBurstLength,
 191				cmd->data_length);
 192			transport_send_check_condition_and_sense(&cmd->se_cmd,
 193					TCM_INCORRECT_AMOUNT_OF_DATA, 0);
 194			return DATAOUT_CANNOT_RECOVER;
 195		}
 196	} else {
 197		if (first_burst_len == conn->sess->sess_ops->FirstBurstLength) {
 198			pr_err("Command ITT: 0x%08x reached"
 199			" FirstBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 200				" error.\n", cmd->init_task_tag,
 201				conn->sess->sess_ops->FirstBurstLength);
 202			return DATAOUT_CANNOT_RECOVER;
 203		}
 204		if (first_burst_len == cmd->data_length) {
 205			pr_err("Command ITT: 0x%08x reached"
 206			" ExpXferLen: %u, but ISCSI_FLAG_CMD_FINAL is not set. protocol"
 207			" error.\n", cmd->init_task_tag, cmd->data_length);
 208			return DATAOUT_CANNOT_RECOVER;
 209		}
 210	}
 211
 212out:
 213	return DATAOUT_NORMAL;
 214}
 215
 216static int iscsit_dataout_check_sequence(
 217	struct iscsi_cmd *cmd,
 218	unsigned char *buf)
 219{
 220	u32 next_burst_len;
 221	struct iscsi_conn *conn = cmd->conn;
 222	struct iscsi_seq *seq = NULL;
 223	struct iscsi_data *hdr = (struct iscsi_data *) buf;
 224	u32 payload_length = ntoh24(hdr->dlength);
 225
 226	/*
 227	 * For DataSequenceInOrder=Yes: Check that the offset and offset+length
 228	 * is within range as defined by iscsi_set_dataout_sequence_values().
 229	 *
 230	 * For DataSequenceInOrder=No: Check that an struct iscsi_seq exists for
 231	 * offset+length tuple.
 232	 */
 233	if (conn->sess->sess_ops->DataSequenceInOrder) {
 234		/*
 235		 * Due to possibility of recovery DataOUT sent by the initiator
 236		 * fullfilling an Recovery R2T, it's best to just dump the
 237		 * payload here, instead of erroring out.
 238		 */
 239		if ((hdr->offset < cmd->seq_start_offset) ||
 240		   ((hdr->offset + payload_length) > cmd->seq_end_offset)) {
 241			pr_err("Command ITT: 0x%08x with Offset: %u,"
 242			" Length: %u outside of Sequence %u:%u while"
 243			" DataSequenceInOrder=Yes.\n", cmd->init_task_tag,
 244			hdr->offset, payload_length, cmd->seq_start_offset,
 245				cmd->seq_end_offset);
 246
 247			if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 248				return DATAOUT_CANNOT_RECOVER;
 249			return DATAOUT_WITHIN_COMMAND_RECOVERY;
 250		}
 251
 252		next_burst_len = (cmd->next_burst_len + payload_length);
 253	} else {
 254		seq = iscsit_get_seq_holder(cmd, hdr->offset, payload_length);
 255		if (!seq)
 256			return DATAOUT_CANNOT_RECOVER;
 257		/*
 258		 * Set the struct iscsi_seq pointer to reuse later.
 259		 */
 260		cmd->seq_ptr = seq;
 261
 262		if (seq->status == DATAOUT_SEQUENCE_COMPLETE) {
 263			if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 264				return DATAOUT_CANNOT_RECOVER;
 265			return DATAOUT_WITHIN_COMMAND_RECOVERY;
 266		}
 267
 268		next_burst_len = (seq->next_burst_len + payload_length);
 269	}
 270
 271	if (next_burst_len > conn->sess->sess_ops->MaxBurstLength) {
 272		pr_err("Command ITT: 0x%08x, NextBurstLength: %u and"
 273			" Length: %u exceeds MaxBurstLength: %u. protocol"
 274			" error.\n", cmd->init_task_tag,
 275			(next_burst_len - payload_length),
 276			payload_length, conn->sess->sess_ops->MaxBurstLength);
 277		return DATAOUT_CANNOT_RECOVER;
 278	}
 279
 280	/*
 281	 * Perform various MaxBurstLength and ISCSI_FLAG_CMD_FINAL sanity
 282	 * checks for the current DataOUT Sequence.
 283	 */
 284	if (hdr->flags & ISCSI_FLAG_CMD_FINAL) {
 285		/*
 286		 * Ignore ISCSI_FLAG_CMD_FINAL checks while DataPDUInOrder=No, end of
 287		 * sequence checks are handled in
 288		 * iscsit_dataout_datapduinorder_no_fbit().
 289		 */
 290		if (!conn->sess->sess_ops->DataPDUInOrder)
 291			goto out;
 292
 293		if (conn->sess->sess_ops->DataSequenceInOrder) {
 294			if ((next_burst_len <
 295			     conn->sess->sess_ops->MaxBurstLength) &&
 296			   ((cmd->write_data_done + payload_length) <
 297			     cmd->data_length)) {
 298				pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 299				" before end of DataOUT sequence, protocol"
 300				" error.\n", cmd->init_task_tag);
 301				return DATAOUT_CANNOT_RECOVER;
 302			}
 303		} else {
 304			if (next_burst_len < seq->xfer_len) {
 305				pr_err("Command ITT: 0x%08x set ISCSI_FLAG_CMD_FINAL"
 306				" before end of DataOUT sequence, protocol"
 307				" error.\n", cmd->init_task_tag);
 308				return DATAOUT_CANNOT_RECOVER;
 309			}
 310		}
 311	} else {
 312		if (conn->sess->sess_ops->DataSequenceInOrder) {
 313			if (next_burst_len ==
 314					conn->sess->sess_ops->MaxBurstLength) {
 315				pr_err("Command ITT: 0x%08x reached"
 316				" MaxBurstLength: %u, but ISCSI_FLAG_CMD_FINAL is"
 317				" not set, protocol error.", cmd->init_task_tag,
 318					conn->sess->sess_ops->MaxBurstLength);
 319				return DATAOUT_CANNOT_RECOVER;
 320			}
 321			if ((cmd->write_data_done + payload_length) ==
 322					cmd->data_length) {
 323				pr_err("Command ITT: 0x%08x reached"
 324				" last DataOUT PDU in sequence but ISCSI_FLAG_"
 325				"CMD_FINAL is not set, protocol error.\n",
 326					cmd->init_task_tag);
 327				return DATAOUT_CANNOT_RECOVER;
 328			}
 329		} else {
 330			if (next_burst_len == seq->xfer_len) {
 331				pr_err("Command ITT: 0x%08x reached"
 332				" last DataOUT PDU in sequence but ISCSI_FLAG_"
 333				"CMD_FINAL is not set, protocol error.\n",
 334					cmd->init_task_tag);
 335				return DATAOUT_CANNOT_RECOVER;
 336			}
 337		}
 338	}
 339
 340out:
 341	return DATAOUT_NORMAL;
 342}
 343
 344static int iscsit_dataout_check_datasn(
 345	struct iscsi_cmd *cmd,
 346	unsigned char *buf)
 347{
 348	int dump = 0, recovery = 0;
 349	u32 data_sn = 0;
 350	struct iscsi_conn *conn = cmd->conn;
 351	struct iscsi_data *hdr = (struct iscsi_data *) buf;
 352	u32 payload_length = ntoh24(hdr->dlength);
 353
 354	/*
 355	 * Considering the target has no method of re-requesting DataOUT
 356	 * by DataSN, if we receieve a greater DataSN than expected we
 357	 * assume the functions for DataPDUInOrder=[Yes,No] below will
 358	 * handle it.
 359	 *
 360	 * If the DataSN is less than expected, dump the payload.
 361	 */
 362	if (conn->sess->sess_ops->DataSequenceInOrder)
 363		data_sn = cmd->data_sn;
 364	else {
 365		struct iscsi_seq *seq = cmd->seq_ptr;
 366		data_sn = seq->data_sn;
 367	}
 368
 369	if (hdr->datasn > data_sn) {
 370		pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 371			" higher than expected 0x%08x.\n", cmd->init_task_tag,
 372				hdr->datasn, data_sn);
 373		recovery = 1;
 374		goto recover;
 375	} else if (hdr->datasn < data_sn) {
 376		pr_err("Command ITT: 0x%08x, received DataSN: 0x%08x"
 377			" lower than expected 0x%08x, discarding payload.\n",
 378			cmd->init_task_tag, hdr->datasn, data_sn);
 379		dump = 1;
 380		goto dump;
 381	}
 382
 383	return DATAOUT_NORMAL;
 384
 385recover:
 386	if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 387		pr_err("Unable to perform within-command recovery"
 388				" while ERL=0.\n");
 389		return DATAOUT_CANNOT_RECOVER;
 390	}
 391dump:
 392	if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 393		return DATAOUT_CANNOT_RECOVER;
 394
 395	return (recovery || dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY :
 396				DATAOUT_NORMAL;
 397}
 398
 399static int iscsit_dataout_pre_datapduinorder_yes(
 400	struct iscsi_cmd *cmd,
 401	unsigned char *buf)
 402{
 403	int dump = 0, recovery = 0;
 404	struct iscsi_conn *conn = cmd->conn;
 405	struct iscsi_data *hdr = (struct iscsi_data *) buf;
 406	u32 payload_length = ntoh24(hdr->dlength);
 407
 408	/*
 409	 * For DataSequenceInOrder=Yes: If the offset is greater than the global
 410	 * DataPDUInOrder=Yes offset counter in struct iscsi_cmd a protcol error has
 411	 * occured and fail the connection.
 412	 *
 413	 * For DataSequenceInOrder=No: If the offset is greater than the per
 414	 * sequence DataPDUInOrder=Yes offset counter in struct iscsi_seq a protocol
 415	 * error has occured and fail the connection.
 416	 */
 417	if (conn->sess->sess_ops->DataSequenceInOrder) {
 418		if (hdr->offset != cmd->write_data_done) {
 419			pr_err("Command ITT: 0x%08x, received offset"
 420			" %u different than expected %u.\n", cmd->init_task_tag,
 421				hdr->offset, cmd->write_data_done);
 422			recovery = 1;
 423			goto recover;
 424		}
 425	} else {
 426		struct iscsi_seq *seq = cmd->seq_ptr;
 427
 428		if (hdr->offset > seq->offset) {
 429			pr_err("Command ITT: 0x%08x, received offset"
 430			" %u greater than expected %u.\n", cmd->init_task_tag,
 431				hdr->offset, seq->offset);
 432			recovery = 1;
 433			goto recover;
 434		} else if (hdr->offset < seq->offset) {
 435			pr_err("Command ITT: 0x%08x, received offset"
 436			" %u less than expected %u, discarding payload.\n",
 437				cmd->init_task_tag, hdr->offset, seq->offset);
 438			dump = 1;
 439			goto dump;
 440		}
 441	}
 442
 443	return DATAOUT_NORMAL;
 444
 445recover:
 446	if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 447		pr_err("Unable to perform within-command recovery"
 448				" while ERL=0.\n");
 449		return DATAOUT_CANNOT_RECOVER;
 450	}
 451dump:
 452	if (iscsit_dump_data_payload(conn, payload_length, 1) < 0)
 453		return DATAOUT_CANNOT_RECOVER;
 454
 455	return (recovery) ? iscsit_recover_dataout_sequence(cmd,
 456		hdr->offset, payload_length) :
 457	       (dump) ? DATAOUT_WITHIN_COMMAND_RECOVERY : DATAOUT_NORMAL;
 458}
 459
 460static int iscsit_dataout_pre_datapduinorder_no(
 461	struct iscsi_cmd *cmd,
 462	unsigned char *buf)
 463{
 464	struct iscsi_pdu *pdu;
 465	struct iscsi_data *hdr = (struct iscsi_data *) buf;
 466	u32 payload_length = ntoh24(hdr->dlength);
 467
 468	pdu = iscsit_get_pdu_holder(cmd, hdr->offset, payload_length);
 469	if (!pdu)
 470		return DATAOUT_CANNOT_RECOVER;
 471
 472	cmd->pdu_ptr = pdu;
 473
 474	switch (pdu->status) {
 475	case ISCSI_PDU_NOT_RECEIVED:
 476	case ISCSI_PDU_CRC_FAILED:
 477	case ISCSI_PDU_TIMED_OUT:
 478		break;
 479	case ISCSI_PDU_RECEIVED_OK:
 480		pr_err("Command ITT: 0x%08x received already gotten"
 481			" Offset: %u, Length: %u\n", cmd->init_task_tag,
 482				hdr->offset, payload_length);
 483		return iscsit_dump_data_payload(cmd->conn, payload_length, 1);
 484	default:
 485		return DATAOUT_CANNOT_RECOVER;
 486	}
 487
 488	return DATAOUT_NORMAL;
 489}
 490
 491static int iscsit_dataout_update_r2t(struct iscsi_cmd *cmd, u32 offset, u32 length)
 492{
 493	struct iscsi_r2t *r2t;
 494
 495	if (cmd->unsolicited_data)
 496		return 0;
 497
 498	r2t = iscsit_get_r2t_for_eos(cmd, offset, length);
 499	if (!r2t)
 500		return -1;
 501
 502	spin_lock_bh(&cmd->r2t_lock);
 503	r2t->seq_complete = 1;
 504	cmd->outstanding_r2ts--;
 505	spin_unlock_bh(&cmd->r2t_lock);
 506
 507	return 0;
 508}
 509
 510static int iscsit_dataout_update_datapduinorder_no(
 511	struct iscsi_cmd *cmd,
 512	u32 data_sn,
 513	int f_bit)
 514{
 515	int ret = 0;
 516	struct iscsi_pdu *pdu = cmd->pdu_ptr;
 517
 518	pdu->data_sn = data_sn;
 519
 520	switch (pdu->status) {
 521	case ISCSI_PDU_NOT_RECEIVED:
 522		pdu->status = ISCSI_PDU_RECEIVED_OK;
 523		break;
 524	case ISCSI_PDU_CRC_FAILED:
 525		pdu->status = ISCSI_PDU_RECEIVED_OK;
 526		break;
 527	case ISCSI_PDU_TIMED_OUT:
 528		pdu->status = ISCSI_PDU_RECEIVED_OK;
 529		break;
 530	default:
 531		return DATAOUT_CANNOT_RECOVER;
 532	}
 533
 534	if (f_bit) {
 535		ret = iscsit_dataout_datapduinorder_no_fbit(cmd, pdu);
 536		if (ret == DATAOUT_CANNOT_RECOVER)
 537			return ret;
 538	}
 539
 540	return DATAOUT_NORMAL;
 541}
 542
 543static int iscsit_dataout_post_crc_passed(
 544	struct iscsi_cmd *cmd,
 545	unsigned char *buf)
 546{
 547	int ret, send_r2t = 0;
 548	struct iscsi_conn *conn = cmd->conn;
 549	struct iscsi_seq *seq = NULL;
 550	struct iscsi_data *hdr = (struct iscsi_data *) buf;
 551	u32 payload_length = ntoh24(hdr->dlength);
 552
 553	if (cmd->unsolicited_data) {
 554		if ((cmd->first_burst_len + payload_length) ==
 555		     conn->sess->sess_ops->FirstBurstLength) {
 556			if (iscsit_dataout_update_r2t(cmd, hdr->offset,
 557					payload_length) < 0)
 558				return DATAOUT_CANNOT_RECOVER;
 559			send_r2t = 1;
 560		}
 561
 562		if (!conn->sess->sess_ops->DataPDUInOrder) {
 563			ret = iscsit_dataout_update_datapduinorder_no(cmd,
 564				hdr->datasn, (hdr->flags & ISCSI_FLAG_CMD_FINAL));
 565			if (ret == DATAOUT_CANNOT_RECOVER)
 566				return ret;
 567		}
 568
 569		cmd->first_burst_len += payload_length;
 570
 571		if (conn->sess->sess_ops->DataSequenceInOrder)
 572			cmd->data_sn++;
 573		else {
 574			seq = cmd->seq_ptr;
 575			seq->data_sn++;
 576			seq->offset += payload_length;
 577		}
 578
 579		if (send_r2t) {
 580			if (seq)
 581				seq->status = DATAOUT_SEQUENCE_COMPLETE;
 582			cmd->first_burst_len = 0;
 583			cmd->unsolicited_data = 0;
 584		}
 585	} else {
 586		if (conn->sess->sess_ops->DataSequenceInOrder) {
 587			if ((cmd->next_burst_len + payload_length) ==
 588			     conn->sess->sess_ops->MaxBurstLength) {
 589				if (iscsit_dataout_update_r2t(cmd, hdr->offset,
 590						payload_length) < 0)
 591					return DATAOUT_CANNOT_RECOVER;
 592				send_r2t = 1;
 593			}
 594
 595			if (!conn->sess->sess_ops->DataPDUInOrder) {
 596				ret = iscsit_dataout_update_datapduinorder_no(
 597						cmd, hdr->datasn,
 598						(hdr->flags & ISCSI_FLAG_CMD_FINAL));
 599				if (ret == DATAOUT_CANNOT_RECOVER)
 600					return ret;
 601			}
 602
 603			cmd->next_burst_len += payload_length;
 604			cmd->data_sn++;
 605
 606			if (send_r2t)
 607				cmd->next_burst_len = 0;
 608		} else {
 609			seq = cmd->seq_ptr;
 610
 611			if ((seq->next_burst_len + payload_length) ==
 612			     seq->xfer_len) {
 613				if (iscsit_dataout_update_r2t(cmd, hdr->offset,
 614						payload_length) < 0)
 615					return DATAOUT_CANNOT_RECOVER;
 616				send_r2t = 1;
 617			}
 618
 619			if (!conn->sess->sess_ops->DataPDUInOrder) {
 620				ret = iscsit_dataout_update_datapduinorder_no(
 621						cmd, hdr->datasn,
 622						(hdr->flags & ISCSI_FLAG_CMD_FINAL));
 623				if (ret == DATAOUT_CANNOT_RECOVER)
 624					return ret;
 625			}
 626
 627			seq->data_sn++;
 628			seq->offset += payload_length;
 629			seq->next_burst_len += payload_length;
 630
 631			if (send_r2t) {
 632				seq->next_burst_len = 0;
 633				seq->status = DATAOUT_SEQUENCE_COMPLETE;
 634			}
 635		}
 636	}
 637
 638	if (send_r2t && conn->sess->sess_ops->DataSequenceInOrder)
 639		cmd->data_sn = 0;
 640
 641	cmd->write_data_done += payload_length;
 642
 643	return (cmd->write_data_done == cmd->data_length) ?
 644		DATAOUT_SEND_TO_TRANSPORT : (send_r2t) ?
 645		DATAOUT_SEND_R2T : DATAOUT_NORMAL;
 
 
 
 646}
 647
 648static int iscsit_dataout_post_crc_failed(
 649	struct iscsi_cmd *cmd,
 650	unsigned char *buf)
 651{
 652	struct iscsi_conn *conn = cmd->conn;
 653	struct iscsi_pdu *pdu;
 654	struct iscsi_data *hdr = (struct iscsi_data *) buf;
 655	u32 payload_length = ntoh24(hdr->dlength);
 656
 657	if (conn->sess->sess_ops->DataPDUInOrder)
 658		goto recover;
 659	/*
 660	 * The rest of this function is only called when DataPDUInOrder=No.
 661	 */
 662	pdu = cmd->pdu_ptr;
 663
 664	switch (pdu->status) {
 665	case ISCSI_PDU_NOT_RECEIVED:
 666		pdu->status = ISCSI_PDU_CRC_FAILED;
 667		break;
 668	case ISCSI_PDU_CRC_FAILED:
 669		break;
 670	case ISCSI_PDU_TIMED_OUT:
 671		pdu->status = ISCSI_PDU_CRC_FAILED;
 672		break;
 673	default:
 674		return DATAOUT_CANNOT_RECOVER;
 675	}
 676
 677recover:
 678	return iscsit_recover_dataout_sequence(cmd, hdr->offset, payload_length);
 679}
 680
 681/*
 682 *	Called from iscsit_handle_data_out() before DataOUT Payload is received
 683 *	and CRC computed.
 684 */
 685extern int iscsit_check_pre_dataout(
 686	struct iscsi_cmd *cmd,
 687	unsigned char *buf)
 688{
 689	int ret;
 690	struct iscsi_conn *conn = cmd->conn;
 691
 692	ret = iscsit_dataout_within_command_recovery_check(cmd, buf);
 693	if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 694	    (ret == DATAOUT_CANNOT_RECOVER))
 695		return ret;
 696
 697	ret = iscsit_dataout_check_datasn(cmd, buf);
 698	if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 699	    (ret == DATAOUT_CANNOT_RECOVER))
 700		return ret;
 701
 702	if (cmd->unsolicited_data) {
 703		ret = iscsit_dataout_check_unsolicited_sequence(cmd, buf);
 704		if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 705		    (ret == DATAOUT_CANNOT_RECOVER))
 706			return ret;
 707	} else {
 708		ret = iscsit_dataout_check_sequence(cmd, buf);
 709		if ((ret == DATAOUT_WITHIN_COMMAND_RECOVERY) ||
 710		    (ret == DATAOUT_CANNOT_RECOVER))
 711			return ret;
 712	}
 713
 714	return (conn->sess->sess_ops->DataPDUInOrder) ?
 715		iscsit_dataout_pre_datapduinorder_yes(cmd, buf) :
 716		iscsit_dataout_pre_datapduinorder_no(cmd, buf);
 717}
 718
 719/*
 720 *	Called from iscsit_handle_data_out() after DataOUT Payload is received
 721 *	and CRC computed.
 722 */
 723int iscsit_check_post_dataout(
 724	struct iscsi_cmd *cmd,
 725	unsigned char *buf,
 726	u8 data_crc_failed)
 727{
 728	struct iscsi_conn *conn = cmd->conn;
 729
 730	cmd->dataout_timeout_retries = 0;
 731
 732	if (!data_crc_failed)
 733		return iscsit_dataout_post_crc_passed(cmd, buf);
 734	else {
 735		if (!conn->sess->sess_ops->ErrorRecoveryLevel) {
 736			pr_err("Unable to recover from DataOUT CRC"
 737				" failure while ERL=0, closing session.\n");
 738			iscsit_add_reject_from_cmd(ISCSI_REASON_DATA_DIGEST_ERROR,
 739					1, 0, buf, cmd);
 740			return DATAOUT_CANNOT_RECOVER;
 741		}
 742
 743		iscsit_add_reject_from_cmd(ISCSI_REASON_DATA_DIGEST_ERROR,
 744				0, 0, buf, cmd);
 745		return iscsit_dataout_post_crc_failed(cmd, buf);
 746	}
 747}
 748
 749static void iscsit_handle_time2retain_timeout(unsigned long data)
 750{
 751	struct iscsi_session *sess = (struct iscsi_session *) data;
 752	struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
 753	struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 754
 755	spin_lock_bh(&se_tpg->session_lock);
 756	if (sess->time2retain_timer_flags & ISCSI_TF_STOP) {
 757		spin_unlock_bh(&se_tpg->session_lock);
 758		return;
 759	}
 760	if (atomic_read(&sess->session_reinstatement)) {
 761		pr_err("Exiting Time2Retain handler because"
 762				" session_reinstatement=1\n");
 763		spin_unlock_bh(&se_tpg->session_lock);
 764		return;
 765	}
 766	sess->time2retain_timer_flags |= ISCSI_TF_EXPIRED;
 767
 768	pr_err("Time2Retain timer expired for SID: %u, cleaning up"
 769			" iSCSI session.\n", sess->sid);
 770	{
 771	struct iscsi_tiqn *tiqn = tpg->tpg_tiqn;
 772
 773	if (tiqn) {
 774		spin_lock(&tiqn->sess_err_stats.lock);
 775		strcpy(tiqn->sess_err_stats.last_sess_fail_rem_name,
 776			(void *)sess->sess_ops->InitiatorName);
 777		tiqn->sess_err_stats.last_sess_failure_type =
 778				ISCSI_SESS_ERR_CXN_TIMEOUT;
 779		tiqn->sess_err_stats.cxn_timeout_errors++;
 780		sess->conn_timeout_errors++;
 781		spin_unlock(&tiqn->sess_err_stats.lock);
 782	}
 783	}
 784
 785	spin_unlock_bh(&se_tpg->session_lock);
 786	iscsit_close_session(sess);
 787}
 788
 789extern void iscsit_start_time2retain_handler(struct iscsi_session *sess)
 790{
 791	int tpg_active;
 792	/*
 793	 * Only start Time2Retain timer when the assoicated TPG is still in
 794	 * an ACTIVE (eg: not disabled or shutdown) state.
 795	 */
 796	spin_lock(&ISCSI_TPG_S(sess)->tpg_state_lock);
 797	tpg_active = (ISCSI_TPG_S(sess)->tpg_state == TPG_STATE_ACTIVE);
 798	spin_unlock(&ISCSI_TPG_S(sess)->tpg_state_lock);
 799
 800	if (!tpg_active)
 801		return;
 802
 803	if (sess->time2retain_timer_flags & ISCSI_TF_RUNNING)
 804		return;
 805
 806	pr_debug("Starting Time2Retain timer for %u seconds on"
 807		" SID: %u\n", sess->sess_ops->DefaultTime2Retain, sess->sid);
 808
 809	init_timer(&sess->time2retain_timer);
 810	sess->time2retain_timer.expires =
 811		(get_jiffies_64() + sess->sess_ops->DefaultTime2Retain * HZ);
 812	sess->time2retain_timer.data = (unsigned long)sess;
 813	sess->time2retain_timer.function = iscsit_handle_time2retain_timeout;
 814	sess->time2retain_timer_flags &= ~ISCSI_TF_STOP;
 815	sess->time2retain_timer_flags |= ISCSI_TF_RUNNING;
 816	add_timer(&sess->time2retain_timer);
 817}
 818
 819/*
 820 *	Called with spin_lock_bh(&struct se_portal_group->session_lock) held
 821 */
 822extern int iscsit_stop_time2retain_timer(struct iscsi_session *sess)
 823{
 824	struct iscsi_portal_group *tpg = ISCSI_TPG_S(sess);
 825	struct se_portal_group *se_tpg = &tpg->tpg_se_tpg;
 826
 827	if (sess->time2retain_timer_flags & ISCSI_TF_EXPIRED)
 828		return -1;
 829
 830	if (!(sess->time2retain_timer_flags & ISCSI_TF_RUNNING))
 831		return 0;
 832
 833	sess->time2retain_timer_flags |= ISCSI_TF_STOP;
 834	spin_unlock_bh(&se_tpg->session_lock);
 835
 836	del_timer_sync(&sess->time2retain_timer);
 837
 838	spin_lock_bh(&se_tpg->session_lock);
 839	sess->time2retain_timer_flags &= ~ISCSI_TF_RUNNING;
 840	pr_debug("Stopped Time2Retain Timer for SID: %u\n",
 841			sess->sid);
 842	return 0;
 843}
 844
 845void iscsit_connection_reinstatement_rcfr(struct iscsi_conn *conn)
 846{
 847	spin_lock_bh(&conn->state_lock);
 848	if (atomic_read(&conn->connection_exit)) {
 849		spin_unlock_bh(&conn->state_lock);
 850		goto sleep;
 851	}
 852
 853	if (atomic_read(&conn->transport_failed)) {
 854		spin_unlock_bh(&conn->state_lock);
 855		goto sleep;
 856	}
 857	spin_unlock_bh(&conn->state_lock);
 858
 859	iscsi_thread_set_force_reinstatement(conn);
 860
 861sleep:
 862	wait_for_completion(&conn->conn_wait_rcfr_comp);
 863	complete(&conn->conn_post_wait_comp);
 864}
 865
 866void iscsit_cause_connection_reinstatement(struct iscsi_conn *conn, int sleep)
 867{
 868	spin_lock_bh(&conn->state_lock);
 869	if (atomic_read(&conn->connection_exit)) {
 870		spin_unlock_bh(&conn->state_lock);
 871		return;
 872	}
 873
 874	if (atomic_read(&conn->transport_failed)) {
 875		spin_unlock_bh(&conn->state_lock);
 876		return;
 877	}
 878
 879	if (atomic_read(&conn->connection_reinstatement)) {
 880		spin_unlock_bh(&conn->state_lock);
 881		return;
 882	}
 883
 884	if (iscsi_thread_set_force_reinstatement(conn) < 0) {
 885		spin_unlock_bh(&conn->state_lock);
 886		return;
 887	}
 888
 889	atomic_set(&conn->connection_reinstatement, 1);
 890	if (!sleep) {
 891		spin_unlock_bh(&conn->state_lock);
 892		return;
 893	}
 894
 895	atomic_set(&conn->sleep_on_conn_wait_comp, 1);
 896	spin_unlock_bh(&conn->state_lock);
 897
 898	wait_for_completion(&conn->conn_wait_comp);
 899	complete(&conn->conn_post_wait_comp);
 900}
 901
 902void iscsit_fall_back_to_erl0(struct iscsi_session *sess)
 903{
 904	pr_debug("Falling back to ErrorRecoveryLevel=0 for SID:"
 905			" %u\n", sess->sid);
 906
 907	atomic_set(&sess->session_fall_back_to_erl0, 1);
 908}
 909
 910static void iscsit_handle_connection_cleanup(struct iscsi_conn *conn)
 911{
 912	struct iscsi_session *sess = conn->sess;
 913
 914	if ((sess->sess_ops->ErrorRecoveryLevel == 2) &&
 915	    !atomic_read(&sess->session_reinstatement) &&
 916	    !atomic_read(&sess->session_fall_back_to_erl0))
 917		iscsit_connection_recovery_transport_reset(conn);
 918	else {
 919		pr_debug("Performing cleanup for failed iSCSI"
 920			" Connection ID: %hu from %s\n", conn->cid,
 921			sess->sess_ops->InitiatorName);
 922		iscsit_close_connection(conn);
 923	}
 924}
 925
 926extern void iscsit_take_action_for_connection_exit(struct iscsi_conn *conn)
 927{
 928	spin_lock_bh(&conn->state_lock);
 929	if (atomic_read(&conn->connection_exit)) {
 930		spin_unlock_bh(&conn->state_lock);
 931		return;
 932	}
 933	atomic_set(&conn->connection_exit, 1);
 934
 935	if (conn->conn_state == TARG_CONN_STATE_IN_LOGOUT) {
 936		spin_unlock_bh(&conn->state_lock);
 937		iscsit_close_connection(conn);
 938		return;
 939	}
 940
 941	if (conn->conn_state == TARG_CONN_STATE_CLEANUP_WAIT) {
 942		spin_unlock_bh(&conn->state_lock);
 943		return;
 944	}
 945
 946	pr_debug("Moving to TARG_CONN_STATE_CLEANUP_WAIT.\n");
 947	conn->conn_state = TARG_CONN_STATE_CLEANUP_WAIT;
 948	spin_unlock_bh(&conn->state_lock);
 949
 950	iscsit_handle_connection_cleanup(conn);
 951}
 952
 953/*
 954 *	This is the simple function that makes the magic of
 955 *	sync and steering happen in the follow paradoxical order:
 956 *
 957 *	0) Receive conn->of_marker (bytes left until next OFMarker)
 958 *	   bytes into an offload buffer.  When we pass the exact number
 959 *	   of bytes in conn->of_marker, iscsit_dump_data_payload() and hence
 960 *	   rx_data() will automatically receive the identical u32 marker
 961 *	   values and store it in conn->of_marker_offset;
 962 *	1) Now conn->of_marker_offset will contain the offset to the start
 963 *	   of the next iSCSI PDU.  Dump these remaining bytes into another
 964 *	   offload buffer.
 965 *	2) We are done!
 966 *	   Next byte in the TCP stream will contain the next iSCSI PDU!
 967 *	   Cool Huh?!
 968 */
 969int iscsit_recover_from_unknown_opcode(struct iscsi_conn *conn)
 970{
 971	/*
 972	 * Make sure the remaining bytes to next maker is a sane value.
 973	 */
 974	if (conn->of_marker > (conn->conn_ops->OFMarkInt * 4)) {
 975		pr_err("Remaining bytes to OFMarker: %u exceeds"
 976			" OFMarkInt bytes: %u.\n", conn->of_marker,
 977				conn->conn_ops->OFMarkInt * 4);
 978		return -1;
 979	}
 980
 981	pr_debug("Advancing %u bytes in TCP stream to get to the"
 982			" next OFMarker.\n", conn->of_marker);
 983
 984	if (iscsit_dump_data_payload(conn, conn->of_marker, 0) < 0)
 985		return -1;
 986
 987	/*
 988	 * Make sure the offset marker we retrived is a valid value.
 989	 */
 990	if (conn->of_marker_offset > (ISCSI_HDR_LEN + (ISCSI_CRC_LEN * 2) +
 991	    conn->conn_ops->MaxRecvDataSegmentLength)) {
 992		pr_err("OfMarker offset value: %u exceeds limit.\n",
 993			conn->of_marker_offset);
 994		return -1;
 995	}
 996
 997	pr_debug("Discarding %u bytes of TCP stream to get to the"
 998			" next iSCSI Opcode.\n", conn->of_marker_offset);
 999
1000	if (iscsit_dump_data_payload(conn, conn->of_marker_offset, 0) < 0)
1001		return -1;
1002
1003	return 0;
1004}