1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * iSCSI Initiator over TCP/IP Data-Path
   4 *
   5 * Copyright (C) 2004 Dmitry Yusupov
   6 * Copyright (C) 2004 Alex Aizman
   7 * Copyright (C) 2005 - 2006 Mike Christie
   8 * Copyright (C) 2006 Red Hat, Inc.  All rights reserved.
   9 * maintained by open-iscsi@googlegroups.com
  10 *
  11 * See the file COPYING included with this distribution for more details.
  12 *
  13 * Credits:
  14 *	Christoph Hellwig
  15 *	FUJITA Tomonori
  16 *	Arne Redlich
  17 *	Zhenyu Wang
  18 */
  19
  20#include <crypto/hash.h>
  21#include <linux/types.h>
  22#include <linux/inet.h>
  23#include <linux/slab.h>
  24#include <linux/sched/mm.h>
  25#include <linux/file.h>
  26#include <linux/blkdev.h>
  27#include <linux/delay.h>
  28#include <linux/kfifo.h>
  29#include <linux/scatterlist.h>
  30#include <linux/module.h>
  31#include <linux/backing-dev.h>
  32#include <net/tcp.h>
  33#include <scsi/scsi_cmnd.h>
  34#include <scsi/scsi_device.h>
  35#include <scsi/scsi_host.h>
  36#include <scsi/scsi.h>
  37#include <scsi/scsi_transport_iscsi.h>
  38#include <trace/events/iscsi.h>
  39#include <trace/events/sock.h>
  40
  41#include "iscsi_tcp.h"
  42
  43MODULE_AUTHOR("Mike Christie <michaelc@cs.wisc.edu>, "
  44	      "Dmitry Yusupov <dmitry_yus@yahoo.com>, "
  45	      "Alex Aizman <itn780@yahoo.com>");
  46MODULE_DESCRIPTION("iSCSI/TCP data-path");
  47MODULE_LICENSE("GPL");
  48
  49static struct scsi_transport_template *iscsi_sw_tcp_scsi_transport;
  50static const struct scsi_host_template iscsi_sw_tcp_sht;
  51static struct iscsi_transport iscsi_sw_tcp_transport;
  52
  53static unsigned int iscsi_max_lun = ~0;
  54module_param_named(max_lun, iscsi_max_lun, uint, S_IRUGO);
  55
  56static bool iscsi_recv_from_iscsi_q;
  57module_param_named(recv_from_iscsi_q, iscsi_recv_from_iscsi_q, bool, 0644);
  58MODULE_PARM_DESC(recv_from_iscsi_q, "Set to true to read iSCSI data/headers from the iscsi_q workqueue. The default is false which will perform reads from the network softirq context.");
  59
  60static int iscsi_sw_tcp_dbg;
  61module_param_named(debug_iscsi_tcp, iscsi_sw_tcp_dbg, int,
  62		   S_IRUGO | S_IWUSR);
  63MODULE_PARM_DESC(debug_iscsi_tcp, "Turn on debugging for iscsi_tcp module "
  64		 "Set to 1 to turn on, and zero to turn off. Default is off.");
  65
  66#define ISCSI_SW_TCP_DBG(_conn, dbg_fmt, arg...)		\
  67	do {							\
  68		if (iscsi_sw_tcp_dbg)				\
  69			iscsi_conn_printk(KERN_INFO, _conn,	\
  70					     "%s " dbg_fmt,	\
  71					     __func__, ##arg);	\
  72		iscsi_dbg_trace(trace_iscsi_dbg_sw_tcp,		\
  73				&(_conn)->cls_conn->dev,	\
  74				"%s " dbg_fmt, __func__, ##arg);\
  75	} while (0);
  76
  77
  78/**
  79 * iscsi_sw_tcp_recv - TCP receive in sendfile fashion
  80 * @rd_desc: read descriptor
  81 * @skb: socket buffer
  82 * @offset: offset in skb
  83 * @len: skb->len - offset
  84 */
  85static int iscsi_sw_tcp_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
  86			     unsigned int offset, size_t len)
  87{
  88	struct iscsi_conn *conn = rd_desc->arg.data;
  89	unsigned int consumed, total_consumed = 0;
  90	int status;
  91
  92	ISCSI_SW_TCP_DBG(conn, "in %d bytes\n", skb->len - offset);
  93
  94	do {
  95		status = 0;
  96		consumed = iscsi_tcp_recv_skb(conn, skb, offset, 0, &status);
  97		offset += consumed;
  98		total_consumed += consumed;
  99	} while (consumed != 0 && status != ISCSI_TCP_SKB_DONE);
 100
 101	ISCSI_SW_TCP_DBG(conn, "read %d bytes status %d\n",
 102			 skb->len - offset, status);
 103	return total_consumed;
 104}
 105
 106/**
 107 * iscsi_sw_sk_state_check - check socket state
 108 * @sk: socket
 109 *
 110 * If the socket is in CLOSE or CLOSE_WAIT we should
 111 * not close the connection if there is still some
 112 * data pending.
 113 *
 114 * Must be called with sk_callback_lock.
 115 */
 116static inline int iscsi_sw_sk_state_check(struct sock *sk)
 117{
 118	struct iscsi_conn *conn = sk->sk_user_data;
 119
 120	if ((sk->sk_state == TCP_CLOSE_WAIT || sk->sk_state == TCP_CLOSE) &&
 121	    (conn->session->state != ISCSI_STATE_LOGGING_OUT) &&
 122	    !atomic_read(&sk->sk_rmem_alloc)) {
 123		ISCSI_SW_TCP_DBG(conn, "TCP_CLOSE|TCP_CLOSE_WAIT\n");
 124		iscsi_conn_failure(conn, ISCSI_ERR_TCP_CONN_CLOSE);
 125		return -ECONNRESET;
 126	}
 127	return 0;
 128}
 129
 130static void iscsi_sw_tcp_recv_data(struct iscsi_conn *conn)
 131{
 132	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 133	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 134	struct sock *sk = tcp_sw_conn->sock->sk;
 135	read_descriptor_t rd_desc;
 136
 137	/*
 138	 * Use rd_desc to pass 'conn' to iscsi_tcp_recv.
 139	 * We set count to 1 because we want the network layer to
 140	 * hand us all the skbs that are available. iscsi_tcp_recv
 141	 * handled pdus that cross buffers or pdus that still need data.
 142	 */
 143	rd_desc.arg.data = conn;
 144	rd_desc.count = 1;
 145
 146	tcp_read_sock(sk, &rd_desc, iscsi_sw_tcp_recv);
 147
 148	/* If we had to (atomically) map a highmem page,
 149	 * unmap it now. */
 150	iscsi_tcp_segment_unmap(&tcp_conn->in.segment);
 151
 152	iscsi_sw_sk_state_check(sk);
 153}
 154
 155static void iscsi_sw_tcp_recv_data_work(struct work_struct *work)
 156{
 157	struct iscsi_conn *conn = container_of(work, struct iscsi_conn,
 158					       recvwork);
 159	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 160	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 161	struct sock *sk = tcp_sw_conn->sock->sk;
 162
 163	lock_sock(sk);
 164	iscsi_sw_tcp_recv_data(conn);
 165	release_sock(sk);
 166}
 167
 168static void iscsi_sw_tcp_data_ready(struct sock *sk)
 169{
 170	struct iscsi_sw_tcp_conn *tcp_sw_conn;
 171	struct iscsi_tcp_conn *tcp_conn;
 172	struct iscsi_conn *conn;
 173
 174	trace_sk_data_ready(sk);
 175
 176	read_lock_bh(&sk->sk_callback_lock);
 177	conn = sk->sk_user_data;
 178	if (!conn) {
 179		read_unlock_bh(&sk->sk_callback_lock);
 180		return;
 181	}
 182	tcp_conn = conn->dd_data;
 183	tcp_sw_conn = tcp_conn->dd_data;
 184
 185	if (tcp_sw_conn->queue_recv)
 186		iscsi_conn_queue_recv(conn);
 187	else
 188		iscsi_sw_tcp_recv_data(conn);
 189	read_unlock_bh(&sk->sk_callback_lock);
 190}
 191
 192static void iscsi_sw_tcp_state_change(struct sock *sk)
 193{
 194	struct iscsi_tcp_conn *tcp_conn;
 195	struct iscsi_sw_tcp_conn *tcp_sw_conn;
 196	struct iscsi_conn *conn;
 197	void (*old_state_change)(struct sock *);
 198
 199	read_lock_bh(&sk->sk_callback_lock);
 200	conn = sk->sk_user_data;
 201	if (!conn) {
 202		read_unlock_bh(&sk->sk_callback_lock);
 203		return;
 204	}
 205
 206	iscsi_sw_sk_state_check(sk);
 207
 208	tcp_conn = conn->dd_data;
 209	tcp_sw_conn = tcp_conn->dd_data;
 210	old_state_change = tcp_sw_conn->old_state_change;
 211
 212	read_unlock_bh(&sk->sk_callback_lock);
 213
 214	old_state_change(sk);
 215}
 216
 217/**
 218 * iscsi_sw_tcp_write_space - Called when more output buffer space is available
 219 * @sk: socket space is available for
 220 **/
 221static void iscsi_sw_tcp_write_space(struct sock *sk)
 222{
 223	struct iscsi_conn *conn;
 224	struct iscsi_tcp_conn *tcp_conn;
 225	struct iscsi_sw_tcp_conn *tcp_sw_conn;
 226	void (*old_write_space)(struct sock *);
 227
 228	read_lock_bh(&sk->sk_callback_lock);
 229	conn = sk->sk_user_data;
 230	if (!conn) {
 231		read_unlock_bh(&sk->sk_callback_lock);
 232		return;
 233	}
 234
 235	tcp_conn = conn->dd_data;
 236	tcp_sw_conn = tcp_conn->dd_data;
 237	old_write_space = tcp_sw_conn->old_write_space;
 238	read_unlock_bh(&sk->sk_callback_lock);
 239
 240	old_write_space(sk);
 241
 242	ISCSI_SW_TCP_DBG(conn, "iscsi_write_space\n");
 243	iscsi_conn_queue_xmit(conn);
 244}
 245
 246static void iscsi_sw_tcp_conn_set_callbacks(struct iscsi_conn *conn)
 247{
 248	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 249	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 250	struct sock *sk = tcp_sw_conn->sock->sk;
 251
 252	/* assign new callbacks */
 253	write_lock_bh(&sk->sk_callback_lock);
 254	sk->sk_user_data = conn;
 255	tcp_sw_conn->old_data_ready = sk->sk_data_ready;
 256	tcp_sw_conn->old_state_change = sk->sk_state_change;
 257	tcp_sw_conn->old_write_space = sk->sk_write_space;
 258	sk->sk_data_ready = iscsi_sw_tcp_data_ready;
 259	sk->sk_state_change = iscsi_sw_tcp_state_change;
 260	sk->sk_write_space = iscsi_sw_tcp_write_space;
 261	write_unlock_bh(&sk->sk_callback_lock);
 262}
 263
 264static void
 265iscsi_sw_tcp_conn_restore_callbacks(struct iscsi_conn *conn)
 266{
 267	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 268	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 269	struct sock *sk = tcp_sw_conn->sock->sk;
 270
 271	/* restore socket callbacks, see also: iscsi_conn_set_callbacks() */
 272	write_lock_bh(&sk->sk_callback_lock);
 273	sk->sk_user_data    = NULL;
 274	sk->sk_data_ready   = tcp_sw_conn->old_data_ready;
 275	sk->sk_state_change = tcp_sw_conn->old_state_change;
 276	sk->sk_write_space  = tcp_sw_conn->old_write_space;
 277	sk->sk_no_check_tx = 0;
 278	write_unlock_bh(&sk->sk_callback_lock);
 279}
 280
 281/**
 282 * iscsi_sw_tcp_xmit_segment - transmit segment
 283 * @tcp_conn: the iSCSI TCP connection
 284 * @segment: the buffer to transmnit
 285 *
 286 * This function transmits as much of the buffer as
 287 * the network layer will accept, and returns the number of
 288 * bytes transmitted.
 289 *
 290 * If CRC hashing is enabled, the function will compute the
 291 * hash as it goes. When the entire segment has been transmitted,
 292 * it will retrieve the hash value and send it as well.
 293 */
 294static int iscsi_sw_tcp_xmit_segment(struct iscsi_tcp_conn *tcp_conn,
 295				     struct iscsi_segment *segment)
 296{
 297	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 298	struct socket *sk = tcp_sw_conn->sock;
 299	unsigned int copied = 0;
 300	int r = 0;
 301
 302	while (!iscsi_tcp_segment_done(tcp_conn, segment, 0, r)) {
 303		struct scatterlist *sg;
 304		struct msghdr msg = {};
 305		struct bio_vec bv;
 306		unsigned int offset, copy;
 307
 308		r = 0;
 309		offset = segment->copied;
 310		copy = segment->size - offset;
 311
 312		if (segment->total_copied + segment->size < segment->total_size)
 313			msg.msg_flags |= MSG_MORE;
 314
 315		if (tcp_sw_conn->queue_recv)
 316			msg.msg_flags |= MSG_DONTWAIT;
 317
 318		if (!segment->data) {
 319			if (!tcp_conn->iscsi_conn->datadgst_en)
 320				msg.msg_flags |= MSG_SPLICE_PAGES;
 321			sg = segment->sg;
 322			offset += segment->sg_offset + sg->offset;
 323			bvec_set_page(&bv, sg_page(sg), copy, offset);
 324		} else {
 325			bvec_set_virt(&bv, segment->data + offset, copy);
 326		}
 327		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bv, 1, copy);
 328
 329		r = sock_sendmsg(sk, &msg);
 330		if (r < 0) {
 331			iscsi_tcp_segment_unmap(segment);
 332			return r;
 333		}
 334		copied += r;
 335	}
 336	return copied;
 337}
 338
 339/**
 340 * iscsi_sw_tcp_xmit - TCP transmit
 341 * @conn: iscsi connection
 342 **/
 343static int iscsi_sw_tcp_xmit(struct iscsi_conn *conn)
 344{
 345	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 346	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 347	struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
 348	unsigned int consumed = 0;
 349	int rc = 0;
 350
 351	while (1) {
 352		rc = iscsi_sw_tcp_xmit_segment(tcp_conn, segment);
 353		/*
 354		 * We may not have been able to send data because the conn
 355		 * is getting stopped. libiscsi will know so propagate err
 356		 * for it to do the right thing.
 357		 */
 358		if (rc == -EAGAIN)
 359			return rc;
 360		else if (rc < 0) {
 361			rc = ISCSI_ERR_XMIT_FAILED;
 362			goto error;
 363		} else if (rc == 0)
 364			break;
 365
 366		consumed += rc;
 367
 368		if (segment->total_copied >= segment->total_size) {
 369			if (segment->done != NULL) {
 370				rc = segment->done(tcp_conn, segment);
 371				if (rc != 0)
 372					goto error;
 373			}
 374		}
 375	}
 376
 377	ISCSI_SW_TCP_DBG(conn, "xmit %d bytes\n", consumed);
 378
 379	conn->txdata_octets += consumed;
 380	return consumed;
 381
 382error:
 383	/* Transmit error. We could initiate error recovery
 384	 * here. */
 385	ISCSI_SW_TCP_DBG(conn, "Error sending PDU, errno=%d\n", rc);
 386	iscsi_conn_failure(conn, rc);
 387	return -EIO;
 388}
 389
 390/**
 391 * iscsi_sw_tcp_xmit_qlen - return the number of bytes queued for xmit
 392 * @conn: iscsi connection
 393 */
 394static inline int iscsi_sw_tcp_xmit_qlen(struct iscsi_conn *conn)
 395{
 396	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 397	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 398	struct iscsi_segment *segment = &tcp_sw_conn->out.segment;
 399
 400	return segment->total_copied - segment->total_size;
 401}
 402
 403static int iscsi_sw_tcp_pdu_xmit(struct iscsi_task *task)
 404{
 405	struct iscsi_conn *conn = task->conn;
 406	unsigned int noreclaim_flag;
 407	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 408	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 409	int rc = 0;
 410
 411	if (!tcp_sw_conn->sock) {
 412		iscsi_conn_printk(KERN_ERR, conn,
 413				  "Transport not bound to socket!\n");
 414		return -EINVAL;
 415	}
 416
 417	noreclaim_flag = memalloc_noreclaim_save();
 418
 419	while (iscsi_sw_tcp_xmit_qlen(conn)) {
 420		rc = iscsi_sw_tcp_xmit(conn);
 421		if (rc == 0) {
 422			rc = -EAGAIN;
 423			break;
 424		}
 425		if (rc < 0)
 426			break;
 427		rc = 0;
 428	}
 429
 430	memalloc_noreclaim_restore(noreclaim_flag);
 431	return rc;
 432}
 433
 434/*
 435 * This is called when we're done sending the header.
 436 * Simply copy the data_segment to the send segment, and return.
 437 */
 438static int iscsi_sw_tcp_send_hdr_done(struct iscsi_tcp_conn *tcp_conn,
 439				      struct iscsi_segment *segment)
 440{
 441	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 442
 443	tcp_sw_conn->out.segment = tcp_sw_conn->out.data_segment;
 444	ISCSI_SW_TCP_DBG(tcp_conn->iscsi_conn,
 445			 "Header done. Next segment size %u total_size %u\n",
 446			 tcp_sw_conn->out.segment.size,
 447			 tcp_sw_conn->out.segment.total_size);
 448	return 0;
 449}
 450
 451static void iscsi_sw_tcp_send_hdr_prep(struct iscsi_conn *conn, void *hdr,
 452				       size_t hdrlen)
 453{
 454	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 455	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 456
 457	ISCSI_SW_TCP_DBG(conn, "%s\n", conn->hdrdgst_en ?
 458			 "digest enabled" : "digest disabled");
 459
 460	/* Clear the data segment - needs to be filled in by the
 461	 * caller using iscsi_tcp_send_data_prep() */
 462	memset(&tcp_sw_conn->out.data_segment, 0,
 463	       sizeof(struct iscsi_segment));
 464
 465	/* If header digest is enabled, compute the CRC and
 466	 * place the digest into the same buffer. We make
 467	 * sure that both iscsi_tcp_task and mtask have
 468	 * sufficient room.
 469	 */
 470	if (conn->hdrdgst_en) {
 471		iscsi_tcp_dgst_header(tcp_sw_conn->tx_hash, hdr, hdrlen,
 472				      hdr + hdrlen);
 473		hdrlen += ISCSI_DIGEST_SIZE;
 474	}
 475
 476	/* Remember header pointer for later, when we need
 477	 * to decide whether there's a payload to go along
 478	 * with the header. */
 479	tcp_sw_conn->out.hdr = hdr;
 480
 481	iscsi_segment_init_linear(&tcp_sw_conn->out.segment, hdr, hdrlen,
 482				  iscsi_sw_tcp_send_hdr_done, NULL);
 483}
 484
 485/*
 486 * Prepare the send buffer for the payload data.
 487 * Padding and checksumming will all be taken care
 488 * of by the iscsi_segment routines.
 489 */
 490static int
 491iscsi_sw_tcp_send_data_prep(struct iscsi_conn *conn, struct scatterlist *sg,
 492			    unsigned int count, unsigned int offset,
 493			    unsigned int len)
 494{
 495	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 496	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 497	struct ahash_request *tx_hash = NULL;
 498	unsigned int hdr_spec_len;
 499
 500	ISCSI_SW_TCP_DBG(conn, "offset=%d, datalen=%d %s\n", offset, len,
 501			 conn->datadgst_en ?
 502			 "digest enabled" : "digest disabled");
 503
 504	/* Make sure the datalen matches what the caller
 505	   said he would send. */
 506	hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
 507	WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
 508
 509	if (conn->datadgst_en)
 510		tx_hash = tcp_sw_conn->tx_hash;
 511
 512	return iscsi_segment_seek_sg(&tcp_sw_conn->out.data_segment,
 513				     sg, count, offset, len,
 514				     NULL, tx_hash);
 515}
 516
 517static void
 518iscsi_sw_tcp_send_linear_data_prep(struct iscsi_conn *conn, void *data,
 519				   size_t len)
 520{
 521	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 522	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 523	struct ahash_request *tx_hash = NULL;
 524	unsigned int hdr_spec_len;
 525
 526	ISCSI_SW_TCP_DBG(conn, "datalen=%zd %s\n", len, conn->datadgst_en ?
 527			 "digest enabled" : "digest disabled");
 528
 529	/* Make sure the datalen matches what the caller
 530	   said he would send. */
 531	hdr_spec_len = ntoh24(tcp_sw_conn->out.hdr->dlength);
 532	WARN_ON(iscsi_padded(len) != iscsi_padded(hdr_spec_len));
 533
 534	if (conn->datadgst_en)
 535		tx_hash = tcp_sw_conn->tx_hash;
 536
 537	iscsi_segment_init_linear(&tcp_sw_conn->out.data_segment,
 538				data, len, NULL, tx_hash);
 539}
 540
 541static int iscsi_sw_tcp_pdu_init(struct iscsi_task *task,
 542				 unsigned int offset, unsigned int count)
 543{
 544	struct iscsi_conn *conn = task->conn;
 545	int err = 0;
 546
 547	iscsi_sw_tcp_send_hdr_prep(conn, task->hdr, task->hdr_len);
 548
 549	if (!count)
 550		return 0;
 551
 552	if (!task->sc)
 553		iscsi_sw_tcp_send_linear_data_prep(conn, task->data, count);
 554	else {
 555		struct scsi_data_buffer *sdb = &task->sc->sdb;
 556
 557		err = iscsi_sw_tcp_send_data_prep(conn, sdb->table.sgl,
 558						  sdb->table.nents, offset,
 559						  count);
 560	}
 561
 562	if (err) {
 563		/* got invalid offset/len */
 564		return -EIO;
 565	}
 566	return 0;
 567}
 568
 569static int iscsi_sw_tcp_pdu_alloc(struct iscsi_task *task, uint8_t opcode)
 570{
 571	struct iscsi_tcp_task *tcp_task = task->dd_data;
 572
 573	task->hdr = task->dd_data + sizeof(*tcp_task);
 574	task->hdr_max = sizeof(struct iscsi_sw_tcp_hdrbuf) - ISCSI_DIGEST_SIZE;
 575	return 0;
 576}
 577
 578static struct iscsi_cls_conn *
 579iscsi_sw_tcp_conn_create(struct iscsi_cls_session *cls_session,
 580			 uint32_t conn_idx)
 581{
 582	struct iscsi_conn *conn;
 583	struct iscsi_cls_conn *cls_conn;
 584	struct iscsi_tcp_conn *tcp_conn;
 585	struct iscsi_sw_tcp_conn *tcp_sw_conn;
 586	struct crypto_ahash *tfm;
 587
 588	cls_conn = iscsi_tcp_conn_setup(cls_session, sizeof(*tcp_sw_conn),
 589					conn_idx);
 590	if (!cls_conn)
 591		return NULL;
 592	conn = cls_conn->dd_data;
 593	tcp_conn = conn->dd_data;
 594	tcp_sw_conn = tcp_conn->dd_data;
 595	INIT_WORK(&conn->recvwork, iscsi_sw_tcp_recv_data_work);
 596	tcp_sw_conn->queue_recv = iscsi_recv_from_iscsi_q;
 597
 598	mutex_init(&tcp_sw_conn->sock_lock);
 599
 600	tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
 601	if (IS_ERR(tfm))
 602		goto free_conn;
 603
 604	tcp_sw_conn->tx_hash = ahash_request_alloc(tfm, GFP_KERNEL);
 605	if (!tcp_sw_conn->tx_hash)
 606		goto free_tfm;
 607	ahash_request_set_callback(tcp_sw_conn->tx_hash, 0, NULL, NULL);
 608
 609	tcp_sw_conn->rx_hash = ahash_request_alloc(tfm, GFP_KERNEL);
 610	if (!tcp_sw_conn->rx_hash)
 611		goto free_tx_hash;
 612	ahash_request_set_callback(tcp_sw_conn->rx_hash, 0, NULL, NULL);
 613
 614	tcp_conn->rx_hash = tcp_sw_conn->rx_hash;
 615
 616	return cls_conn;
 617
 618free_tx_hash:
 619	ahash_request_free(tcp_sw_conn->tx_hash);
 620free_tfm:
 621	crypto_free_ahash(tfm);
 622free_conn:
 623	iscsi_conn_printk(KERN_ERR, conn,
 624			  "Could not create connection due to crc32c "
 625			  "loading error. Make sure the crc32c "
 626			  "module is built as a module or into the "
 627			  "kernel\n");
 628	iscsi_tcp_conn_teardown(cls_conn);
 629	return NULL;
 630}
 631
 632static void iscsi_sw_tcp_release_conn(struct iscsi_conn *conn)
 633{
 634	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 635	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 636	struct socket *sock = tcp_sw_conn->sock;
 637
 638	/*
 639	 * The iscsi transport class will make sure we are not called in
 640	 * parallel with start, stop, bind and destroys. However, this can be
 641	 * called twice if userspace does a stop then a destroy.
 642	 */
 643	if (!sock)
 644		return;
 645
 646	/*
 647	 * Make sure we start socket shutdown now in case userspace is up
 648	 * but delayed in releasing the socket.
 649	 */
 650	kernel_sock_shutdown(sock, SHUT_RDWR);
 651
 652	sock_hold(sock->sk);
 653	iscsi_sw_tcp_conn_restore_callbacks(conn);
 654	sock_put(sock->sk);
 655
 656	iscsi_suspend_rx(conn);
 657
 658	mutex_lock(&tcp_sw_conn->sock_lock);
 659	tcp_sw_conn->sock = NULL;
 660	mutex_unlock(&tcp_sw_conn->sock_lock);
 661	sockfd_put(sock);
 662}
 663
 664static void iscsi_sw_tcp_conn_destroy(struct iscsi_cls_conn *cls_conn)
 665{
 666	struct iscsi_conn *conn = cls_conn->dd_data;
 667	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 668	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 669
 670	iscsi_sw_tcp_release_conn(conn);
 671
 672	ahash_request_free(tcp_sw_conn->rx_hash);
 673	if (tcp_sw_conn->tx_hash) {
 674		struct crypto_ahash *tfm;
 675
 676		tfm = crypto_ahash_reqtfm(tcp_sw_conn->tx_hash);
 677		ahash_request_free(tcp_sw_conn->tx_hash);
 678		crypto_free_ahash(tfm);
 679	}
 680
 681	iscsi_tcp_conn_teardown(cls_conn);
 682}
 683
 684static void iscsi_sw_tcp_conn_stop(struct iscsi_cls_conn *cls_conn, int flag)
 685{
 686	struct iscsi_conn *conn = cls_conn->dd_data;
 687	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 688	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 689	struct socket *sock = tcp_sw_conn->sock;
 690
 691	/* userspace may have goofed up and not bound us */
 692	if (!sock)
 693		return;
 694
 695	sock->sk->sk_err = EIO;
 696	wake_up_interruptible(sk_sleep(sock->sk));
 697
 698	/* stop xmit side */
 699	iscsi_suspend_tx(conn);
 700
 701	/* stop recv side and release socket */
 702	iscsi_sw_tcp_release_conn(conn);
 703
 704	iscsi_conn_stop(cls_conn, flag);
 705}
 706
 707static int
 708iscsi_sw_tcp_conn_bind(struct iscsi_cls_session *cls_session,
 709		       struct iscsi_cls_conn *cls_conn, uint64_t transport_eph,
 710		       int is_leading)
 711{
 712	struct iscsi_conn *conn = cls_conn->dd_data;
 713	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 714	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 715	struct sock *sk;
 716	struct socket *sock;
 717	int err;
 718
 719	/* lookup for existing socket */
 720	sock = sockfd_lookup((int)transport_eph, &err);
 721	if (!sock) {
 722		iscsi_conn_printk(KERN_ERR, conn,
 723				  "sockfd_lookup failed %d\n", err);
 724		return -EEXIST;
 725	}
 726
 727	err = -EINVAL;
 728	if (!sk_is_tcp(sock->sk))
 729		goto free_socket;
 730
 731	err = iscsi_conn_bind(cls_session, cls_conn, is_leading);
 732	if (err)
 733		goto free_socket;
 734
 735	mutex_lock(&tcp_sw_conn->sock_lock);
 736	/* bind iSCSI connection and socket */
 737	tcp_sw_conn->sock = sock;
 738	mutex_unlock(&tcp_sw_conn->sock_lock);
 739
 740	/* setup Socket parameters */
 741	sk = sock->sk;
 742	sk->sk_reuse = SK_CAN_REUSE;
 743	sk->sk_sndtimeo = 15 * HZ; /* FIXME: make it configurable */
 744	sk->sk_allocation = GFP_ATOMIC;
 745	sk->sk_use_task_frag = false;
 746	sk_set_memalloc(sk);
 747	sock_no_linger(sk);
 748
 749	iscsi_sw_tcp_conn_set_callbacks(conn);
 750	/*
 751	 * set receive state machine into initial state
 752	 */
 753	iscsi_tcp_hdr_recv_prep(tcp_conn);
 754	return 0;
 755
 756free_socket:
 757	sockfd_put(sock);
 758	return err;
 759}
 760
 761static int iscsi_sw_tcp_conn_set_param(struct iscsi_cls_conn *cls_conn,
 762				       enum iscsi_param param, char *buf,
 763				       int buflen)
 764{
 765	struct iscsi_conn *conn = cls_conn->dd_data;
 766	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 767	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 768
 769	switch(param) {
 770	case ISCSI_PARAM_HDRDGST_EN:
 771		iscsi_set_param(cls_conn, param, buf, buflen);
 772		break;
 773	case ISCSI_PARAM_DATADGST_EN:
 774		mutex_lock(&tcp_sw_conn->sock_lock);
 775		if (!tcp_sw_conn->sock) {
 776			mutex_unlock(&tcp_sw_conn->sock_lock);
 777			return -ENOTCONN;
 778		}
 779		iscsi_set_param(cls_conn, param, buf, buflen);
 780		mutex_unlock(&tcp_sw_conn->sock_lock);
 781		break;
 782	case ISCSI_PARAM_MAX_R2T:
 783		return iscsi_tcp_set_max_r2t(conn, buf);
 784	default:
 785		return iscsi_set_param(cls_conn, param, buf, buflen);
 786	}
 787
 788	return 0;
 789}
 790
 791static int iscsi_sw_tcp_conn_get_param(struct iscsi_cls_conn *cls_conn,
 792				       enum iscsi_param param, char *buf)
 793{
 794	struct iscsi_conn *conn = cls_conn->dd_data;
 795	struct iscsi_sw_tcp_conn *tcp_sw_conn;
 796	struct iscsi_tcp_conn *tcp_conn;
 797	struct sockaddr_in6 addr;
 798	struct socket *sock;
 799	int rc;
 800
 801	switch(param) {
 802	case ISCSI_PARAM_CONN_PORT:
 803	case ISCSI_PARAM_CONN_ADDRESS:
 804	case ISCSI_PARAM_LOCAL_PORT:
 805		spin_lock_bh(&conn->session->frwd_lock);
 806		if (!conn->session->leadconn) {
 807			spin_unlock_bh(&conn->session->frwd_lock);
 808			return -ENOTCONN;
 809		}
 810		/*
 811		 * The conn has been setup and bound, so just grab a ref
 812		 * incase a destroy runs while we are in the net layer.
 813		 */
 814		iscsi_get_conn(conn->cls_conn);
 815		spin_unlock_bh(&conn->session->frwd_lock);
 816
 817		tcp_conn = conn->dd_data;
 818		tcp_sw_conn = tcp_conn->dd_data;
 819
 820		mutex_lock(&tcp_sw_conn->sock_lock);
 821		sock = tcp_sw_conn->sock;
 822		if (!sock) {
 823			rc = -ENOTCONN;
 824			goto sock_unlock;
 825		}
 826
 827		if (param == ISCSI_PARAM_LOCAL_PORT)
 828			rc = kernel_getsockname(sock,
 829						(struct sockaddr *)&addr);
 830		else
 831			rc = kernel_getpeername(sock,
 832						(struct sockaddr *)&addr);
 833sock_unlock:
 834		mutex_unlock(&tcp_sw_conn->sock_lock);
 835		iscsi_put_conn(conn->cls_conn);
 836		if (rc < 0)
 837			return rc;
 838
 839		return iscsi_conn_get_addr_param((struct sockaddr_storage *)
 840						 &addr, param, buf);
 841	default:
 842		return iscsi_conn_get_param(cls_conn, param, buf);
 843	}
 844
 845	return 0;
 846}
 847
 848static int iscsi_sw_tcp_host_get_param(struct Scsi_Host *shost,
 849				       enum iscsi_host_param param, char *buf)
 850{
 851	struct iscsi_sw_tcp_host *tcp_sw_host = iscsi_host_priv(shost);
 852	struct iscsi_session *session;
 853	struct iscsi_conn *conn;
 854	struct iscsi_tcp_conn *tcp_conn;
 855	struct iscsi_sw_tcp_conn *tcp_sw_conn;
 856	struct sockaddr_in6 addr;
 857	struct socket *sock;
 858	int rc;
 859
 860	switch (param) {
 861	case ISCSI_HOST_PARAM_IPADDRESS:
 862		session = tcp_sw_host->session;
 863		if (!session)
 864			return -ENOTCONN;
 865
 866		spin_lock_bh(&session->frwd_lock);
 867		conn = session->leadconn;
 868		if (!conn) {
 869			spin_unlock_bh(&session->frwd_lock);
 870			return -ENOTCONN;
 871		}
 872		tcp_conn = conn->dd_data;
 873		tcp_sw_conn = tcp_conn->dd_data;
 874		/*
 875		 * The conn has been setup and bound, so just grab a ref
 876		 * incase a destroy runs while we are in the net layer.
 877		 */
 878		iscsi_get_conn(conn->cls_conn);
 879		spin_unlock_bh(&session->frwd_lock);
 880
 881		mutex_lock(&tcp_sw_conn->sock_lock);
 882		sock = tcp_sw_conn->sock;
 883		if (!sock)
 884			rc = -ENOTCONN;
 885		else
 886			rc = kernel_getsockname(sock, (struct sockaddr *)&addr);
 887		mutex_unlock(&tcp_sw_conn->sock_lock);
 888		iscsi_put_conn(conn->cls_conn);
 889		if (rc < 0)
 890			return rc;
 891
 892		return iscsi_conn_get_addr_param((struct sockaddr_storage *)
 893						 &addr,
 894						 (enum iscsi_param)param, buf);
 895	default:
 896		return iscsi_host_get_param(shost, param, buf);
 897	}
 898
 899	return 0;
 900}
 901
 902static void
 903iscsi_sw_tcp_conn_get_stats(struct iscsi_cls_conn *cls_conn,
 904			    struct iscsi_stats *stats)
 905{
 906	struct iscsi_conn *conn = cls_conn->dd_data;
 907	struct iscsi_tcp_conn *tcp_conn = conn->dd_data;
 908	struct iscsi_sw_tcp_conn *tcp_sw_conn = tcp_conn->dd_data;
 909
 910	stats->custom_length = 3;
 911	strcpy(stats->custom[0].desc, "tx_sendpage_failures");
 912	stats->custom[0].value = tcp_sw_conn->sendpage_failures_cnt;
 913	strcpy(stats->custom[1].desc, "rx_discontiguous_hdr");
 914	stats->custom[1].value = tcp_sw_conn->discontiguous_hdr_cnt;
 915	strcpy(stats->custom[2].desc, "eh_abort_cnt");
 916	stats->custom[2].value = conn->eh_abort_cnt;
 917
 918	iscsi_tcp_conn_get_stats(cls_conn, stats);
 919}
 920
 921static struct iscsi_cls_session *
 922iscsi_sw_tcp_session_create(struct iscsi_endpoint *ep, uint16_t cmds_max,
 923			    uint16_t qdepth, uint32_t initial_cmdsn)
 924{
 925	struct iscsi_cls_session *cls_session;
 926	struct iscsi_session *session;
 927	struct iscsi_sw_tcp_host *tcp_sw_host;
 928	struct Scsi_Host *shost;
 929	int rc;
 930
 931	if (ep) {
 932		printk(KERN_ERR "iscsi_tcp: invalid ep %p.\n", ep);
 933		return NULL;
 934	}
 935
 936	shost = iscsi_host_alloc(&iscsi_sw_tcp_sht,
 937				 sizeof(struct iscsi_sw_tcp_host), 1);
 938	if (!shost)
 939		return NULL;
 940	shost->transportt = iscsi_sw_tcp_scsi_transport;
 941	shost->cmd_per_lun = qdepth;
 942	shost->max_lun = iscsi_max_lun;
 943	shost->max_id = 0;
 944	shost->max_channel = 0;
 945	shost->max_cmd_len = SCSI_MAX_VARLEN_CDB_SIZE;
 
 946
 947	rc = iscsi_host_get_max_scsi_cmds(shost, cmds_max);
 948	if (rc < 0)
 949		goto free_host;
 950	shost->can_queue = rc;
 951
 952	if (iscsi_host_add(shost, NULL))
 953		goto free_host;
 954
 955	cls_session = iscsi_session_setup(&iscsi_sw_tcp_transport, shost,
 956					  cmds_max, 0,
 957					  sizeof(struct iscsi_tcp_task) +
 958					  sizeof(struct iscsi_sw_tcp_hdrbuf),
 959					  initial_cmdsn, 0);
 960	if (!cls_session)
 961		goto remove_host;
 962	session = cls_session->dd_data;
 963
 964	if (iscsi_tcp_r2tpool_alloc(session))
 965		goto remove_session;
 966
 967	/* We are now fully setup so expose the session to sysfs. */
 968	tcp_sw_host = iscsi_host_priv(shost);
 969	tcp_sw_host->session = session;
 970	return cls_session;
 971
 972remove_session:
 973	iscsi_session_teardown(cls_session);
 974remove_host:
 975	iscsi_host_remove(shost, false);
 976free_host:
 977	iscsi_host_free(shost);
 978	return NULL;
 979}
 980
 981static void iscsi_sw_tcp_session_destroy(struct iscsi_cls_session *cls_session)
 982{
 983	struct Scsi_Host *shost = iscsi_session_to_shost(cls_session);
 984	struct iscsi_session *session = cls_session->dd_data;
 985
 986	if (WARN_ON_ONCE(session->leadconn))
 987		return;
 988
 989	iscsi_session_remove(cls_session);
 990	/*
 991	 * Our get_host_param needs to access the session, so remove the
 992	 * host from sysfs before freeing the session to make sure userspace
 993	 * is no longer accessing the callout.
 994	 */
 995	iscsi_host_remove(shost, false);
 996
 997	iscsi_tcp_r2tpool_free(cls_session->dd_data);
 998
 999	iscsi_session_free(cls_session);
1000	iscsi_host_free(shost);
1001}
1002
1003static umode_t iscsi_sw_tcp_attr_is_visible(int param_type, int param)
1004{
1005	switch (param_type) {
1006	case ISCSI_HOST_PARAM:
1007		switch (param) {
1008		case ISCSI_HOST_PARAM_NETDEV_NAME:
1009		case ISCSI_HOST_PARAM_HWADDRESS:
1010		case ISCSI_HOST_PARAM_IPADDRESS:
1011		case ISCSI_HOST_PARAM_INITIATOR_NAME:
1012			return S_IRUGO;
1013		default:
1014			return 0;
1015		}
1016	case ISCSI_PARAM:
1017		switch (param) {
1018		case ISCSI_PARAM_MAX_RECV_DLENGTH:
1019		case ISCSI_PARAM_MAX_XMIT_DLENGTH:
1020		case ISCSI_PARAM_HDRDGST_EN:
1021		case ISCSI_PARAM_DATADGST_EN:
1022		case ISCSI_PARAM_CONN_ADDRESS:
1023		case ISCSI_PARAM_CONN_PORT:
1024		case ISCSI_PARAM_LOCAL_PORT:
1025		case ISCSI_PARAM_EXP_STATSN:
1026		case ISCSI_PARAM_PERSISTENT_ADDRESS:
1027		case ISCSI_PARAM_PERSISTENT_PORT:
1028		case ISCSI_PARAM_PING_TMO:
1029		case ISCSI_PARAM_RECV_TMO:
1030		case ISCSI_PARAM_INITIAL_R2T_EN:
1031		case ISCSI_PARAM_MAX_R2T:
1032		case ISCSI_PARAM_IMM_DATA_EN:
1033		case ISCSI_PARAM_FIRST_BURST:
1034		case ISCSI_PARAM_MAX_BURST:
1035		case ISCSI_PARAM_PDU_INORDER_EN:
1036		case ISCSI_PARAM_DATASEQ_INORDER_EN:
1037		case ISCSI_PARAM_ERL:
1038		case ISCSI_PARAM_TARGET_NAME:
1039		case ISCSI_PARAM_TPGT:
1040		case ISCSI_PARAM_USERNAME:
1041		case ISCSI_PARAM_PASSWORD:
1042		case ISCSI_PARAM_USERNAME_IN:
1043		case ISCSI_PARAM_PASSWORD_IN:
1044		case ISCSI_PARAM_FAST_ABORT:
1045		case ISCSI_PARAM_ABORT_TMO:
1046		case ISCSI_PARAM_LU_RESET_TMO:
1047		case ISCSI_PARAM_TGT_RESET_TMO:
1048		case ISCSI_PARAM_IFACE_NAME:
1049		case ISCSI_PARAM_INITIATOR_NAME:
1050			return S_IRUGO;
1051		default:
1052			return 0;
1053		}
1054	}
1055
1056	return 0;
1057}
1058
1059static int iscsi_sw_tcp_slave_configure(struct scsi_device *sdev)
 
1060{
1061	struct iscsi_sw_tcp_host *tcp_sw_host = iscsi_host_priv(sdev->host);
1062	struct iscsi_session *session = tcp_sw_host->session;
1063	struct iscsi_conn *conn = session->leadconn;
1064
1065	if (conn->datadgst_en)
1066		blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES,
1067				   sdev->request_queue);
1068	blk_queue_dma_alignment(sdev->request_queue, 0);
1069	return 0;
1070}
1071
1072static const struct scsi_host_template iscsi_sw_tcp_sht = {
1073	.module			= THIS_MODULE,
1074	.name			= "iSCSI Initiator over TCP/IP",
1075	.queuecommand           = iscsi_queuecommand,
1076	.change_queue_depth	= scsi_change_queue_depth,
1077	.can_queue		= ISCSI_TOTAL_CMDS_MAX,
1078	.sg_tablesize		= 4096,
1079	.max_sectors		= 0xFFFF,
1080	.cmd_per_lun		= ISCSI_DEF_CMD_PER_LUN,
1081	.eh_timed_out		= iscsi_eh_cmd_timed_out,
1082	.eh_abort_handler       = iscsi_eh_abort,
1083	.eh_device_reset_handler= iscsi_eh_device_reset,
1084	.eh_target_reset_handler = iscsi_eh_recover_target,
1085	.dma_boundary		= PAGE_SIZE - 1,
1086	.slave_configure        = iscsi_sw_tcp_slave_configure,
1087	.proc_name		= "iscsi_tcp",
1088	.this_id		= -1,
1089	.track_queue_depth	= 1,
1090	.cmd_size		= sizeof(struct iscsi_cmd),
1091};
1092
1093static struct iscsi_transport iscsi_sw_tcp_transport = {
1094	.owner			= THIS_MODULE,
1095	.name			= "tcp",
1096	.caps			= CAP_RECOVERY_L0 | CAP_MULTI_R2T | CAP_HDRDGST
1097				  | CAP_DATADGST,
1098	/* session management */
1099	.create_session		= iscsi_sw_tcp_session_create,
1100	.destroy_session	= iscsi_sw_tcp_session_destroy,
1101	/* connection management */
1102	.create_conn		= iscsi_sw_tcp_conn_create,
1103	.bind_conn		= iscsi_sw_tcp_conn_bind,
1104	.destroy_conn		= iscsi_sw_tcp_conn_destroy,
1105	.attr_is_visible	= iscsi_sw_tcp_attr_is_visible,
1106	.set_param		= iscsi_sw_tcp_conn_set_param,
1107	.get_conn_param		= iscsi_sw_tcp_conn_get_param,
1108	.get_session_param	= iscsi_session_get_param,
1109	.start_conn		= iscsi_conn_start,
1110	.stop_conn		= iscsi_sw_tcp_conn_stop,
1111	/* iscsi host params */
1112	.get_host_param		= iscsi_sw_tcp_host_get_param,
1113	.set_host_param		= iscsi_host_set_param,
1114	/* IO */
1115	.send_pdu		= iscsi_conn_send_pdu,
1116	.get_stats		= iscsi_sw_tcp_conn_get_stats,
1117	/* iscsi task/cmd helpers */
1118	.init_task		= iscsi_tcp_task_init,
1119	.xmit_task		= iscsi_tcp_task_xmit,
1120	.cleanup_task		= iscsi_tcp_cleanup_task,
1121	/* low level pdu helpers */
1122	.xmit_pdu		= iscsi_sw_tcp_pdu_xmit,
1123	.init_pdu		= iscsi_sw_tcp_pdu_init,
1124	.alloc_pdu		= iscsi_sw_tcp_pdu_alloc,
1125	/* recovery */
1126	.session_recovery_timedout = iscsi_session_recovery_timedout,
1127};
1128
1129static int __init iscsi_sw_tcp_init(void)
1130{
1131	if (iscsi_max_lun < 1) {
1132		printk(KERN_ERR "iscsi_tcp: Invalid max_lun value of %u\n",
1133		       iscsi_max_lun);
1134		return -EINVAL;
1135	}
1136
1137	iscsi_sw_tcp_scsi_transport = iscsi_register_transport(
1138						&iscsi_sw_tcp_transport);
1139	if (!iscsi_sw_tcp_scsi_transport)
1140		return -ENODEV;
1141
1142	return 0;
1143}
1144
1145static void __exit iscsi_sw_tcp_exit(void)
1146{
1147	iscsi_unregister_transport(&iscsi_sw_tcp_transport);
1148}
1149
1150module_init(iscsi_sw_tcp_init);
1151module_exit(iscsi_sw_tcp_exit);