1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * NVMe over Fabrics TCP host.
   4 * Copyright (c) 2018 Lightbits Labs. All rights reserved.
   5 */
   6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
   7#include <linux/module.h>
   8#include <linux/init.h>
   9#include <linux/slab.h>
  10#include <linux/err.h>
  11#include <linux/key.h>
  12#include <linux/nvme-tcp.h>
  13#include <linux/nvme-keyring.h>
  14#include <net/sock.h>
  15#include <net/tcp.h>
  16#include <net/tls.h>
  17#include <net/tls_prot.h>
  18#include <net/handshake.h>
  19#include <linux/blk-mq.h>
  20#include <crypto/hash.h>
  21#include <net/busy_poll.h>
  22#include <trace/events/sock.h>
  23
  24#include "nvme.h"
  25#include "fabrics.h"
  26
  27struct nvme_tcp_queue;
  28
  29/* Define the socket priority to use for connections were it is desirable
  30 * that the NIC consider performing optimized packet processing or filtering.
  31 * A non-zero value being sufficient to indicate general consideration of any
  32 * possible optimization.  Making it a module param allows for alternative
  33 * values that may be unique for some NIC implementations.
  34 */
  35static int so_priority;
  36module_param(so_priority, int, 0644);
  37MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
  38
  39/*
  40 * TLS handshake timeout
  41 */
  42static int tls_handshake_timeout = 10;
  43#ifdef CONFIG_NVME_TCP_TLS
  44module_param(tls_handshake_timeout, int, 0644);
  45MODULE_PARM_DESC(tls_handshake_timeout,
  46		 "nvme TLS handshake timeout in seconds (default 10)");
  47#endif
  48
  49#ifdef CONFIG_DEBUG_LOCK_ALLOC
  50/* lockdep can detect a circular dependency of the form
  51 *   sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
  52 * because dependencies are tracked for both nvme-tcp and user contexts. Using
  53 * a separate class prevents lockdep from conflating nvme-tcp socket use with
  54 * user-space socket API use.
  55 */
  56static struct lock_class_key nvme_tcp_sk_key[2];
  57static struct lock_class_key nvme_tcp_slock_key[2];
  58
  59static void nvme_tcp_reclassify_socket(struct socket *sock)
  60{
  61	struct sock *sk = sock->sk;
  62
  63	if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
  64		return;
  65
  66	switch (sk->sk_family) {
  67	case AF_INET:
  68		sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
  69					      &nvme_tcp_slock_key[0],
  70					      "sk_lock-AF_INET-NVME",
  71					      &nvme_tcp_sk_key[0]);
  72		break;
  73	case AF_INET6:
  74		sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
  75					      &nvme_tcp_slock_key[1],
  76					      "sk_lock-AF_INET6-NVME",
  77					      &nvme_tcp_sk_key[1]);
  78		break;
  79	default:
  80		WARN_ON_ONCE(1);
  81	}
  82}
  83#else
  84static void nvme_tcp_reclassify_socket(struct socket *sock) { }
  85#endif
  86
  87enum nvme_tcp_send_state {
  88	NVME_TCP_SEND_CMD_PDU = 0,
  89	NVME_TCP_SEND_H2C_PDU,
  90	NVME_TCP_SEND_DATA,
  91	NVME_TCP_SEND_DDGST,
  92};
  93
  94struct nvme_tcp_request {
  95	struct nvme_request	req;
  96	void			*pdu;
  97	struct nvme_tcp_queue	*queue;
  98	u32			data_len;
  99	u32			pdu_len;
 100	u32			pdu_sent;
 101	u32			h2cdata_left;
 102	u32			h2cdata_offset;
 103	u16			ttag;
 104	__le16			status;
 105	struct list_head	entry;
 106	struct llist_node	lentry;
 107	__le32			ddgst;
 108
 109	struct bio		*curr_bio;
 110	struct iov_iter		iter;
 111
 112	/* send state */
 113	size_t			offset;
 114	size_t			data_sent;
 115	enum nvme_tcp_send_state state;
 116};
 117
 118enum nvme_tcp_queue_flags {
 119	NVME_TCP_Q_ALLOCATED	= 0,
 120	NVME_TCP_Q_LIVE		= 1,
 121	NVME_TCP_Q_POLLING	= 2,
 122};
 123
 124enum nvme_tcp_recv_state {
 125	NVME_TCP_RECV_PDU = 0,
 126	NVME_TCP_RECV_DATA,
 127	NVME_TCP_RECV_DDGST,
 128};
 129
 130struct nvme_tcp_ctrl;
 131struct nvme_tcp_queue {
 132	struct socket		*sock;
 133	struct work_struct	io_work;
 134	int			io_cpu;
 135
 136	struct mutex		queue_lock;
 137	struct mutex		send_mutex;
 138	struct llist_head	req_list;
 139	struct list_head	send_list;
 140
 141	/* recv state */
 142	void			*pdu;
 143	int			pdu_remaining;
 144	int			pdu_offset;
 145	size_t			data_remaining;
 146	size_t			ddgst_remaining;
 147	unsigned int		nr_cqe;
 148
 149	/* send state */
 150	struct nvme_tcp_request *request;
 151
 152	u32			maxh2cdata;
 153	size_t			cmnd_capsule_len;
 154	struct nvme_tcp_ctrl	*ctrl;
 155	unsigned long		flags;
 156	bool			rd_enabled;
 157
 158	bool			hdr_digest;
 159	bool			data_digest;
 160	struct ahash_request	*rcv_hash;
 161	struct ahash_request	*snd_hash;
 162	__le32			exp_ddgst;
 163	__le32			recv_ddgst;
 164	struct completion       tls_complete;
 165	int                     tls_err;
 166	struct page_frag_cache	pf_cache;
 167
 168	void (*state_change)(struct sock *);
 169	void (*data_ready)(struct sock *);
 170	void (*write_space)(struct sock *);
 171};
 172
 173struct nvme_tcp_ctrl {
 174	/* read only in the hot path */
 175	struct nvme_tcp_queue	*queues;
 176	struct blk_mq_tag_set	tag_set;
 177
 178	/* other member variables */
 179	struct list_head	list;
 180	struct blk_mq_tag_set	admin_tag_set;
 181	struct sockaddr_storage addr;
 182	struct sockaddr_storage src_addr;
 183	struct nvme_ctrl	ctrl;
 184
 185	struct work_struct	err_work;
 186	struct delayed_work	connect_work;
 187	struct nvme_tcp_request async_req;
 188	u32			io_queues[HCTX_MAX_TYPES];
 189};
 190
 191static LIST_HEAD(nvme_tcp_ctrl_list);
 192static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
 193static struct workqueue_struct *nvme_tcp_wq;
 194static const struct blk_mq_ops nvme_tcp_mq_ops;
 195static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
 196static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
 197
 198static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
 199{
 200	return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
 201}
 202
 203static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
 204{
 205	return queue - queue->ctrl->queues;
 206}
 207
 208static inline bool nvme_tcp_tls(struct nvme_ctrl *ctrl)
 209{
 210	if (!IS_ENABLED(CONFIG_NVME_TCP_TLS))
 211		return 0;
 212
 213	return ctrl->opts->tls;
 214}
 215
 216static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
 217{
 218	u32 queue_idx = nvme_tcp_queue_id(queue);
 219
 220	if (queue_idx == 0)
 221		return queue->ctrl->admin_tag_set.tags[queue_idx];
 222	return queue->ctrl->tag_set.tags[queue_idx - 1];
 223}
 224
 225static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
 226{
 227	return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 228}
 229
 230static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
 231{
 232	return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
 233}
 234
 235static inline void *nvme_tcp_req_cmd_pdu(struct nvme_tcp_request *req)
 236{
 237	return req->pdu;
 238}
 239
 240static inline void *nvme_tcp_req_data_pdu(struct nvme_tcp_request *req)
 241{
 242	/* use the pdu space in the back for the data pdu */
 243	return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) -
 244		sizeof(struct nvme_tcp_data_pdu);
 245}
 246
 247static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req)
 248{
 249	if (nvme_is_fabrics(req->req.cmd))
 250		return NVME_TCP_ADMIN_CCSZ;
 251	return req->queue->cmnd_capsule_len - sizeof(struct nvme_command);
 252}
 253
 254static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
 255{
 256	return req == &req->queue->ctrl->async_req;
 257}
 258
 259static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
 260{
 261	struct request *rq;
 262
 263	if (unlikely(nvme_tcp_async_req(req)))
 264		return false; /* async events don't have a request */
 265
 266	rq = blk_mq_rq_from_pdu(req);
 267
 268	return rq_data_dir(rq) == WRITE && req->data_len &&
 269		req->data_len <= nvme_tcp_inline_data_size(req);
 270}
 271
 272static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
 273{
 274	return req->iter.bvec->bv_page;
 275}
 276
 277static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
 278{
 279	return req->iter.bvec->bv_offset + req->iter.iov_offset;
 280}
 281
 282static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
 283{
 284	return min_t(size_t, iov_iter_single_seg_count(&req->iter),
 285			req->pdu_len - req->pdu_sent);
 286}
 287
 288static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
 289{
 290	return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
 291			req->pdu_len - req->pdu_sent : 0;
 292}
 293
 294static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
 295		int len)
 296{
 297	return nvme_tcp_pdu_data_left(req) <= len;
 298}
 299
 300static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
 301		unsigned int dir)
 302{
 303	struct request *rq = blk_mq_rq_from_pdu(req);
 304	struct bio_vec *vec;
 305	unsigned int size;
 306	int nr_bvec;
 307	size_t offset;
 308
 309	if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
 310		vec = &rq->special_vec;
 311		nr_bvec = 1;
 312		size = blk_rq_payload_bytes(rq);
 313		offset = 0;
 314	} else {
 315		struct bio *bio = req->curr_bio;
 316		struct bvec_iter bi;
 317		struct bio_vec bv;
 318
 319		vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
 320		nr_bvec = 0;
 321		bio_for_each_bvec(bv, bio, bi) {
 322			nr_bvec++;
 323		}
 324		size = bio->bi_iter.bi_size;
 325		offset = bio->bi_iter.bi_bvec_done;
 326	}
 327
 328	iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size);
 329	req->iter.iov_offset = offset;
 330}
 331
 332static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
 333		int len)
 334{
 335	req->data_sent += len;
 336	req->pdu_sent += len;
 337	iov_iter_advance(&req->iter, len);
 338	if (!iov_iter_count(&req->iter) &&
 339	    req->data_sent < req->data_len) {
 340		req->curr_bio = req->curr_bio->bi_next;
 341		nvme_tcp_init_iter(req, ITER_SOURCE);
 342	}
 343}
 344
 345static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
 346{
 347	int ret;
 348
 349	/* drain the send queue as much as we can... */
 350	do {
 351		ret = nvme_tcp_try_send(queue);
 352	} while (ret > 0);
 353}
 354
 355static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
 356{
 357	return !list_empty(&queue->send_list) ||
 358		!llist_empty(&queue->req_list);
 359}
 360
 361static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
 362		bool sync, bool last)
 363{
 364	struct nvme_tcp_queue *queue = req->queue;
 365	bool empty;
 366
 367	empty = llist_add(&req->lentry, &queue->req_list) &&
 368		list_empty(&queue->send_list) && !queue->request;
 369
 370	/*
 371	 * if we're the first on the send_list and we can try to send
 372	 * directly, otherwise queue io_work. Also, only do that if we
 373	 * are on the same cpu, so we don't introduce contention.
 374	 */
 375	if (queue->io_cpu == raw_smp_processor_id() &&
 376	    sync && empty && mutex_trylock(&queue->send_mutex)) {
 377		nvme_tcp_send_all(queue);
 378		mutex_unlock(&queue->send_mutex);
 379	}
 380
 381	if (last && nvme_tcp_queue_more(queue))
 382		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 383}
 384
 385static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
 386{
 387	struct nvme_tcp_request *req;
 388	struct llist_node *node;
 389
 390	for (node = llist_del_all(&queue->req_list); node; node = node->next) {
 391		req = llist_entry(node, struct nvme_tcp_request, lentry);
 392		list_add(&req->entry, &queue->send_list);
 393	}
 394}
 395
 396static inline struct nvme_tcp_request *
 397nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
 398{
 399	struct nvme_tcp_request *req;
 400
 401	req = list_first_entry_or_null(&queue->send_list,
 402			struct nvme_tcp_request, entry);
 403	if (!req) {
 404		nvme_tcp_process_req_list(queue);
 405		req = list_first_entry_or_null(&queue->send_list,
 406				struct nvme_tcp_request, entry);
 407		if (unlikely(!req))
 408			return NULL;
 409	}
 410
 411	list_del(&req->entry);
 412	return req;
 413}
 414
 415static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
 416		__le32 *dgst)
 417{
 418	ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
 419	crypto_ahash_final(hash);
 420}
 421
 422static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
 423		struct page *page, off_t off, size_t len)
 424{
 425	struct scatterlist sg;
 426
 427	sg_init_table(&sg, 1);
 428	sg_set_page(&sg, page, len, off);
 429	ahash_request_set_crypt(hash, &sg, NULL, len);
 430	crypto_ahash_update(hash);
 431}
 432
 433static inline void nvme_tcp_hdgst(struct ahash_request *hash,
 434		void *pdu, size_t len)
 435{
 436	struct scatterlist sg;
 437
 438	sg_init_one(&sg, pdu, len);
 439	ahash_request_set_crypt(hash, &sg, pdu + len, len);
 440	crypto_ahash_digest(hash);
 441}
 442
 443static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
 444		void *pdu, size_t pdu_len)
 445{
 446	struct nvme_tcp_hdr *hdr = pdu;
 447	__le32 recv_digest;
 448	__le32 exp_digest;
 449
 450	if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
 451		dev_err(queue->ctrl->ctrl.device,
 452			"queue %d: header digest flag is cleared\n",
 453			nvme_tcp_queue_id(queue));
 454		return -EPROTO;
 455	}
 456
 457	recv_digest = *(__le32 *)(pdu + hdr->hlen);
 458	nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
 459	exp_digest = *(__le32 *)(pdu + hdr->hlen);
 460	if (recv_digest != exp_digest) {
 461		dev_err(queue->ctrl->ctrl.device,
 462			"header digest error: recv %#x expected %#x\n",
 463			le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
 464		return -EIO;
 465	}
 466
 467	return 0;
 468}
 469
 470static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
 471{
 472	struct nvme_tcp_hdr *hdr = pdu;
 473	u8 digest_len = nvme_tcp_hdgst_len(queue);
 474	u32 len;
 475
 476	len = le32_to_cpu(hdr->plen) - hdr->hlen -
 477		((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
 478
 479	if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
 480		dev_err(queue->ctrl->ctrl.device,
 481			"queue %d: data digest flag is cleared\n",
 482		nvme_tcp_queue_id(queue));
 483		return -EPROTO;
 484	}
 485	crypto_ahash_init(queue->rcv_hash);
 486
 487	return 0;
 488}
 489
 490static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
 491		struct request *rq, unsigned int hctx_idx)
 492{
 493	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 494
 495	page_frag_free(req->pdu);
 496}
 497
 498static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
 499		struct request *rq, unsigned int hctx_idx,
 500		unsigned int numa_node)
 501{
 502	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data);
 503	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 504	struct nvme_tcp_cmd_pdu *pdu;
 505	int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
 506	struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
 507	u8 hdgst = nvme_tcp_hdgst_len(queue);
 508
 509	req->pdu = page_frag_alloc(&queue->pf_cache,
 510		sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
 511		GFP_KERNEL | __GFP_ZERO);
 512	if (!req->pdu)
 513		return -ENOMEM;
 514
 515	pdu = req->pdu;
 516	req->queue = queue;
 517	nvme_req(rq)->ctrl = &ctrl->ctrl;
 518	nvme_req(rq)->cmd = &pdu->cmd;
 519
 520	return 0;
 521}
 522
 523static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 524		unsigned int hctx_idx)
 525{
 526	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data);
 527	struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
 528
 529	hctx->driver_data = queue;
 530	return 0;
 531}
 532
 533static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
 534		unsigned int hctx_idx)
 535{
 536	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data);
 537	struct nvme_tcp_queue *queue = &ctrl->queues[0];
 538
 539	hctx->driver_data = queue;
 540	return 0;
 541}
 542
 543static enum nvme_tcp_recv_state
 544nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
 545{
 546	return  (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
 547		(queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
 548		NVME_TCP_RECV_DATA;
 549}
 550
 551static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
 552{
 553	queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
 554				nvme_tcp_hdgst_len(queue);
 555	queue->pdu_offset = 0;
 556	queue->data_remaining = -1;
 557	queue->ddgst_remaining = 0;
 558}
 559
 560static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
 561{
 562	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
 563		return;
 564
 565	dev_warn(ctrl->device, "starting error recovery\n");
 566	queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
 567}
 568
 569static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
 570		struct nvme_completion *cqe)
 571{
 572	struct nvme_tcp_request *req;
 573	struct request *rq;
 574
 575	rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);
 576	if (!rq) {
 577		dev_err(queue->ctrl->ctrl.device,
 578			"got bad cqe.command_id %#x on queue %d\n",
 579			cqe->command_id, nvme_tcp_queue_id(queue));
 580		nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 581		return -EINVAL;
 582	}
 583
 584	req = blk_mq_rq_to_pdu(rq);
 585	if (req->status == cpu_to_le16(NVME_SC_SUCCESS))
 586		req->status = cqe->status;
 587
 588	if (!nvme_try_complete_req(rq, req->status, cqe->result))
 589		nvme_complete_rq(rq);
 590	queue->nr_cqe++;
 591
 592	return 0;
 593}
 594
 595static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
 596		struct nvme_tcp_data_pdu *pdu)
 597{
 598	struct request *rq;
 599
 600	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
 601	if (!rq) {
 602		dev_err(queue->ctrl->ctrl.device,
 603			"got bad c2hdata.command_id %#x on queue %d\n",
 604			pdu->command_id, nvme_tcp_queue_id(queue));
 605		return -ENOENT;
 606	}
 607
 608	if (!blk_rq_payload_bytes(rq)) {
 609		dev_err(queue->ctrl->ctrl.device,
 610			"queue %d tag %#x unexpected data\n",
 611			nvme_tcp_queue_id(queue), rq->tag);
 612		return -EIO;
 613	}
 614
 615	queue->data_remaining = le32_to_cpu(pdu->data_length);
 616
 617	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
 618	    unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
 619		dev_err(queue->ctrl->ctrl.device,
 620			"queue %d tag %#x SUCCESS set but not last PDU\n",
 621			nvme_tcp_queue_id(queue), rq->tag);
 622		nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 623		return -EPROTO;
 624	}
 625
 626	return 0;
 627}
 628
 629static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
 630		struct nvme_tcp_rsp_pdu *pdu)
 631{
 632	struct nvme_completion *cqe = &pdu->cqe;
 633	int ret = 0;
 634
 635	/*
 636	 * AEN requests are special as they don't time out and can
 637	 * survive any kind of queue freeze and often don't respond to
 638	 * aborts.  We don't even bother to allocate a struct request
 639	 * for them but rather special case them here.
 640	 */
 641	if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
 642				     cqe->command_id)))
 643		nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
 644				&cqe->result);
 645	else
 646		ret = nvme_tcp_process_nvme_cqe(queue, cqe);
 647
 648	return ret;
 649}
 650
 651static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req)
 652{
 653	struct nvme_tcp_data_pdu *data = nvme_tcp_req_data_pdu(req);
 654	struct nvme_tcp_queue *queue = req->queue;
 655	struct request *rq = blk_mq_rq_from_pdu(req);
 656	u32 h2cdata_sent = req->pdu_len;
 657	u8 hdgst = nvme_tcp_hdgst_len(queue);
 658	u8 ddgst = nvme_tcp_ddgst_len(queue);
 659
 660	req->state = NVME_TCP_SEND_H2C_PDU;
 661	req->offset = 0;
 662	req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata);
 663	req->pdu_sent = 0;
 664	req->h2cdata_left -= req->pdu_len;
 665	req->h2cdata_offset += h2cdata_sent;
 666
 667	memset(data, 0, sizeof(*data));
 668	data->hdr.type = nvme_tcp_h2c_data;
 669	if (!req->h2cdata_left)
 670		data->hdr.flags = NVME_TCP_F_DATA_LAST;
 671	if (queue->hdr_digest)
 672		data->hdr.flags |= NVME_TCP_F_HDGST;
 673	if (queue->data_digest)
 674		data->hdr.flags |= NVME_TCP_F_DDGST;
 675	data->hdr.hlen = sizeof(*data);
 676	data->hdr.pdo = data->hdr.hlen + hdgst;
 677	data->hdr.plen =
 678		cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
 679	data->ttag = req->ttag;
 680	data->command_id = nvme_cid(rq);
 681	data->data_offset = cpu_to_le32(req->h2cdata_offset);
 682	data->data_length = cpu_to_le32(req->pdu_len);
 683}
 684
 685static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
 686		struct nvme_tcp_r2t_pdu *pdu)
 687{
 688	struct nvme_tcp_request *req;
 689	struct request *rq;
 690	u32 r2t_length = le32_to_cpu(pdu->r2t_length);
 691	u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);
 692
 693	rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
 694	if (!rq) {
 695		dev_err(queue->ctrl->ctrl.device,
 696			"got bad r2t.command_id %#x on queue %d\n",
 697			pdu->command_id, nvme_tcp_queue_id(queue));
 698		return -ENOENT;
 699	}
 700	req = blk_mq_rq_to_pdu(rq);
 701
 702	if (unlikely(!r2t_length)) {
 703		dev_err(queue->ctrl->ctrl.device,
 704			"req %d r2t len is %u, probably a bug...\n",
 705			rq->tag, r2t_length);
 706		return -EPROTO;
 707	}
 708
 709	if (unlikely(req->data_sent + r2t_length > req->data_len)) {
 710		dev_err(queue->ctrl->ctrl.device,
 711			"req %d r2t len %u exceeded data len %u (%zu sent)\n",
 712			rq->tag, r2t_length, req->data_len, req->data_sent);
 713		return -EPROTO;
 714	}
 715
 716	if (unlikely(r2t_offset < req->data_sent)) {
 717		dev_err(queue->ctrl->ctrl.device,
 718			"req %d unexpected r2t offset %u (expected %zu)\n",
 719			rq->tag, r2t_offset, req->data_sent);
 720		return -EPROTO;
 721	}
 722
 723	req->pdu_len = 0;
 724	req->h2cdata_left = r2t_length;
 725	req->h2cdata_offset = r2t_offset;
 726	req->ttag = pdu->ttag;
 727
 728	nvme_tcp_setup_h2c_data_pdu(req);
 729	nvme_tcp_queue_request(req, false, true);
 730
 731	return 0;
 732}
 733
 734static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 735		unsigned int *offset, size_t *len)
 736{
 737	struct nvme_tcp_hdr *hdr;
 738	char *pdu = queue->pdu;
 739	size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
 740	int ret;
 741
 742	ret = skb_copy_bits(skb, *offset,
 743		&pdu[queue->pdu_offset], rcv_len);
 744	if (unlikely(ret))
 745		return ret;
 746
 747	queue->pdu_remaining -= rcv_len;
 748	queue->pdu_offset += rcv_len;
 749	*offset += rcv_len;
 750	*len -= rcv_len;
 751	if (queue->pdu_remaining)
 752		return 0;
 753
 754	hdr = queue->pdu;
 755	if (queue->hdr_digest) {
 756		ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
 757		if (unlikely(ret))
 758			return ret;
 759	}
 760
 761
 762	if (queue->data_digest) {
 763		ret = nvme_tcp_check_ddgst(queue, queue->pdu);
 764		if (unlikely(ret))
 765			return ret;
 766	}
 767
 768	switch (hdr->type) {
 769	case nvme_tcp_c2h_data:
 770		return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
 771	case nvme_tcp_rsp:
 772		nvme_tcp_init_recv_ctx(queue);
 773		return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
 774	case nvme_tcp_r2t:
 775		nvme_tcp_init_recv_ctx(queue);
 776		return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
 777	default:
 778		dev_err(queue->ctrl->ctrl.device,
 779			"unsupported pdu type (%d)\n", hdr->type);
 780		return -EINVAL;
 781	}
 782}
 783
 784static inline void nvme_tcp_end_request(struct request *rq, u16 status)
 785{
 786	union nvme_result res = {};
 787
 788	if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
 789		nvme_complete_rq(rq);
 790}
 791
 792static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
 793			      unsigned int *offset, size_t *len)
 794{
 795	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 796	struct request *rq =
 797		nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
 798	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 799
 800	while (true) {
 801		int recv_len, ret;
 802
 803		recv_len = min_t(size_t, *len, queue->data_remaining);
 804		if (!recv_len)
 805			break;
 806
 807		if (!iov_iter_count(&req->iter)) {
 808			req->curr_bio = req->curr_bio->bi_next;
 809
 810			/*
 811			 * If we don`t have any bios it means that controller
 812			 * sent more data than we requested, hence error
 813			 */
 814			if (!req->curr_bio) {
 815				dev_err(queue->ctrl->ctrl.device,
 816					"queue %d no space in request %#x",
 817					nvme_tcp_queue_id(queue), rq->tag);
 818				nvme_tcp_init_recv_ctx(queue);
 819				return -EIO;
 820			}
 821			nvme_tcp_init_iter(req, ITER_DEST);
 822		}
 823
 824		/* we can read only from what is left in this bio */
 825		recv_len = min_t(size_t, recv_len,
 826				iov_iter_count(&req->iter));
 827
 828		if (queue->data_digest)
 829			ret = skb_copy_and_hash_datagram_iter(skb, *offset,
 830				&req->iter, recv_len, queue->rcv_hash);
 831		else
 832			ret = skb_copy_datagram_iter(skb, *offset,
 833					&req->iter, recv_len);
 834		if (ret) {
 835			dev_err(queue->ctrl->ctrl.device,
 836				"queue %d failed to copy request %#x data",
 837				nvme_tcp_queue_id(queue), rq->tag);
 838			return ret;
 839		}
 840
 841		*len -= recv_len;
 842		*offset += recv_len;
 843		queue->data_remaining -= recv_len;
 844	}
 845
 846	if (!queue->data_remaining) {
 847		if (queue->data_digest) {
 848			nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
 849			queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
 850		} else {
 851			if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 852				nvme_tcp_end_request(rq,
 853						le16_to_cpu(req->status));
 854				queue->nr_cqe++;
 855			}
 856			nvme_tcp_init_recv_ctx(queue);
 857		}
 858	}
 859
 860	return 0;
 861}
 862
 863static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
 864		struct sk_buff *skb, unsigned int *offset, size_t *len)
 865{
 866	struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
 867	char *ddgst = (char *)&queue->recv_ddgst;
 868	size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
 869	off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
 870	int ret;
 871
 872	ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
 873	if (unlikely(ret))
 874		return ret;
 875
 876	queue->ddgst_remaining -= recv_len;
 877	*offset += recv_len;
 878	*len -= recv_len;
 879	if (queue->ddgst_remaining)
 880		return 0;
 881
 882	if (queue->recv_ddgst != queue->exp_ddgst) {
 883		struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
 884					pdu->command_id);
 885		struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 886
 887		req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR);
 888
 889		dev_err(queue->ctrl->ctrl.device,
 890			"data digest error: recv %#x expected %#x\n",
 891			le32_to_cpu(queue->recv_ddgst),
 892			le32_to_cpu(queue->exp_ddgst));
 893	}
 894
 895	if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
 896		struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
 897					pdu->command_id);
 898		struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
 899
 900		nvme_tcp_end_request(rq, le16_to_cpu(req->status));
 901		queue->nr_cqe++;
 902	}
 903
 904	nvme_tcp_init_recv_ctx(queue);
 905	return 0;
 906}
 907
 908static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
 909			     unsigned int offset, size_t len)
 910{
 911	struct nvme_tcp_queue *queue = desc->arg.data;
 912	size_t consumed = len;
 913	int result;
 914
 915	if (unlikely(!queue->rd_enabled))
 916		return -EFAULT;
 917
 918	while (len) {
 919		switch (nvme_tcp_recv_state(queue)) {
 920		case NVME_TCP_RECV_PDU:
 921			result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
 922			break;
 923		case NVME_TCP_RECV_DATA:
 924			result = nvme_tcp_recv_data(queue, skb, &offset, &len);
 925			break;
 926		case NVME_TCP_RECV_DDGST:
 927			result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
 928			break;
 929		default:
 930			result = -EFAULT;
 931		}
 932		if (result) {
 933			dev_err(queue->ctrl->ctrl.device,
 934				"receive failed:  %d\n", result);
 935			queue->rd_enabled = false;
 936			nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 937			return result;
 938		}
 939	}
 940
 941	return consumed;
 942}
 943
 944static void nvme_tcp_data_ready(struct sock *sk)
 945{
 946	struct nvme_tcp_queue *queue;
 947
 948	trace_sk_data_ready(sk);
 949
 950	read_lock_bh(&sk->sk_callback_lock);
 951	queue = sk->sk_user_data;
 952	if (likely(queue && queue->rd_enabled) &&
 953	    !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
 954		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 955	read_unlock_bh(&sk->sk_callback_lock);
 956}
 957
 958static void nvme_tcp_write_space(struct sock *sk)
 959{
 960	struct nvme_tcp_queue *queue;
 961
 962	read_lock_bh(&sk->sk_callback_lock);
 963	queue = sk->sk_user_data;
 964	if (likely(queue && sk_stream_is_writeable(sk))) {
 965		clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
 966		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
 967	}
 968	read_unlock_bh(&sk->sk_callback_lock);
 969}
 970
 971static void nvme_tcp_state_change(struct sock *sk)
 972{
 973	struct nvme_tcp_queue *queue;
 974
 975	read_lock_bh(&sk->sk_callback_lock);
 976	queue = sk->sk_user_data;
 977	if (!queue)
 978		goto done;
 979
 980	switch (sk->sk_state) {
 981	case TCP_CLOSE:
 982	case TCP_CLOSE_WAIT:
 983	case TCP_LAST_ACK:
 984	case TCP_FIN_WAIT1:
 985	case TCP_FIN_WAIT2:
 986		nvme_tcp_error_recovery(&queue->ctrl->ctrl);
 987		break;
 988	default:
 989		dev_info(queue->ctrl->ctrl.device,
 990			"queue %d socket state %d\n",
 991			nvme_tcp_queue_id(queue), sk->sk_state);
 992	}
 993
 994	queue->state_change(sk);
 995done:
 996	read_unlock_bh(&sk->sk_callback_lock);
 997}
 998
 999static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
1000{
1001	queue->request = NULL;
1002}
1003
1004static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
1005{
1006	if (nvme_tcp_async_req(req)) {
1007		union nvme_result res = {};
1008
1009		nvme_complete_async_event(&req->queue->ctrl->ctrl,
1010				cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
1011	} else {
1012		nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
1013				NVME_SC_HOST_PATH_ERROR);
1014	}
1015}
1016
1017static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
1018{
1019	struct nvme_tcp_queue *queue = req->queue;
1020	int req_data_len = req->data_len;
1021	u32 h2cdata_left = req->h2cdata_left;
1022
1023	while (true) {
1024		struct bio_vec bvec;
1025		struct msghdr msg = {
1026			.msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES,
1027		};
1028		struct page *page = nvme_tcp_req_cur_page(req);
1029		size_t offset = nvme_tcp_req_cur_offset(req);
1030		size_t len = nvme_tcp_req_cur_length(req);
1031		bool last = nvme_tcp_pdu_last_send(req, len);
1032		int req_data_sent = req->data_sent;
1033		int ret;
1034
1035		if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
1036			msg.msg_flags |= MSG_EOR;
1037		else
1038			msg.msg_flags |= MSG_MORE;
1039
1040		if (!sendpage_ok(page))
1041			msg.msg_flags &= ~MSG_SPLICE_PAGES;
1042
1043		bvec_set_page(&bvec, page, len, offset);
1044		iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1045		ret = sock_sendmsg(queue->sock, &msg);
1046		if (ret <= 0)
1047			return ret;
1048
1049		if (queue->data_digest)
1050			nvme_tcp_ddgst_update(queue->snd_hash, page,
1051					offset, ret);
1052
1053		/*
1054		 * update the request iterator except for the last payload send
1055		 * in the request where we don't want to modify it as we may
1056		 * compete with the RX path completing the request.
1057		 */
1058		if (req_data_sent + ret < req_data_len)
1059			nvme_tcp_advance_req(req, ret);
1060
1061		/* fully successful last send in current PDU */
1062		if (last && ret == len) {
1063			if (queue->data_digest) {
1064				nvme_tcp_ddgst_final(queue->snd_hash,
1065					&req->ddgst);
1066				req->state = NVME_TCP_SEND_DDGST;
1067				req->offset = 0;
1068			} else {
1069				if (h2cdata_left)
1070					nvme_tcp_setup_h2c_data_pdu(req);
1071				else
1072					nvme_tcp_done_send_req(queue);
1073			}
1074			return 1;
1075		}
1076	}
1077	return -EAGAIN;
1078}
1079
1080static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
1081{
1082	struct nvme_tcp_queue *queue = req->queue;
1083	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
1084	struct bio_vec bvec;
1085	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, };
1086	bool inline_data = nvme_tcp_has_inline_data(req);
1087	u8 hdgst = nvme_tcp_hdgst_len(queue);
1088	int len = sizeof(*pdu) + hdgst - req->offset;
1089	int ret;
1090
1091	if (inline_data || nvme_tcp_queue_more(queue))
1092		msg.msg_flags |= MSG_MORE;
1093	else
1094		msg.msg_flags |= MSG_EOR;
1095
1096	if (queue->hdr_digest && !req->offset)
1097		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1098
1099	bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
1100	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1101	ret = sock_sendmsg(queue->sock, &msg);
1102	if (unlikely(ret <= 0))
1103		return ret;
1104
1105	len -= ret;
1106	if (!len) {
1107		if (inline_data) {
1108			req->state = NVME_TCP_SEND_DATA;
1109			if (queue->data_digest)
1110				crypto_ahash_init(queue->snd_hash);
1111		} else {
1112			nvme_tcp_done_send_req(queue);
1113		}
1114		return 1;
1115	}
1116	req->offset += ret;
1117
1118	return -EAGAIN;
1119}
1120
1121static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
1122{
1123	struct nvme_tcp_queue *queue = req->queue;
1124	struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req);
1125	struct bio_vec bvec;
1126	struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_MORE, };
1127	u8 hdgst = nvme_tcp_hdgst_len(queue);
1128	int len = sizeof(*pdu) - req->offset + hdgst;
1129	int ret;
1130
1131	if (queue->hdr_digest && !req->offset)
1132		nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1133
1134	if (!req->h2cdata_left)
1135		msg.msg_flags |= MSG_SPLICE_PAGES;
1136
1137	bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
1138	iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1139	ret = sock_sendmsg(queue->sock, &msg);
1140	if (unlikely(ret <= 0))
1141		return ret;
1142
1143	len -= ret;
1144	if (!len) {
1145		req->state = NVME_TCP_SEND_DATA;
1146		if (queue->data_digest)
1147			crypto_ahash_init(queue->snd_hash);
1148		return 1;
1149	}
1150	req->offset += ret;
1151
1152	return -EAGAIN;
1153}
1154
1155static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1156{
1157	struct nvme_tcp_queue *queue = req->queue;
1158	size_t offset = req->offset;
1159	u32 h2cdata_left = req->h2cdata_left;
1160	int ret;
1161	struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1162	struct kvec iov = {
1163		.iov_base = (u8 *)&req->ddgst + req->offset,
1164		.iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1165	};
1166
1167	if (nvme_tcp_queue_more(queue))
1168		msg.msg_flags |= MSG_MORE;
1169	else
1170		msg.msg_flags |= MSG_EOR;
1171
1172	ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1173	if (unlikely(ret <= 0))
1174		return ret;
1175
1176	if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
1177		if (h2cdata_left)
1178			nvme_tcp_setup_h2c_data_pdu(req);
1179		else
1180			nvme_tcp_done_send_req(queue);
1181		return 1;
1182	}
1183
1184	req->offset += ret;
1185	return -EAGAIN;
1186}
1187
1188static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1189{
1190	struct nvme_tcp_request *req;
1191	unsigned int noreclaim_flag;
1192	int ret = 1;
1193
1194	if (!queue->request) {
1195		queue->request = nvme_tcp_fetch_request(queue);
1196		if (!queue->request)
1197			return 0;
1198	}
1199	req = queue->request;
1200
1201	noreclaim_flag = memalloc_noreclaim_save();
1202	if (req->state == NVME_TCP_SEND_CMD_PDU) {
1203		ret = nvme_tcp_try_send_cmd_pdu(req);
1204		if (ret <= 0)
1205			goto done;
1206		if (!nvme_tcp_has_inline_data(req))
1207			goto out;
1208	}
1209
1210	if (req->state == NVME_TCP_SEND_H2C_PDU) {
1211		ret = nvme_tcp_try_send_data_pdu(req);
1212		if (ret <= 0)
1213			goto done;
1214	}
1215
1216	if (req->state == NVME_TCP_SEND_DATA) {
1217		ret = nvme_tcp_try_send_data(req);
1218		if (ret <= 0)
1219			goto done;
1220	}
1221
1222	if (req->state == NVME_TCP_SEND_DDGST)
1223		ret = nvme_tcp_try_send_ddgst(req);
1224done:
1225	if (ret == -EAGAIN) {
1226		ret = 0;
1227	} else if (ret < 0) {
1228		dev_err(queue->ctrl->ctrl.device,
1229			"failed to send request %d\n", ret);
1230		nvme_tcp_fail_request(queue->request);
1231		nvme_tcp_done_send_req(queue);
1232	}
1233out:
1234	memalloc_noreclaim_restore(noreclaim_flag);
1235	return ret;
1236}
1237
1238static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1239{
1240	struct socket *sock = queue->sock;
1241	struct sock *sk = sock->sk;
1242	read_descriptor_t rd_desc;
1243	int consumed;
1244
1245	rd_desc.arg.data = queue;
1246	rd_desc.count = 1;
1247	lock_sock(sk);
1248	queue->nr_cqe = 0;
1249	consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1250	release_sock(sk);
1251	return consumed;
1252}
1253
1254static void nvme_tcp_io_work(struct work_struct *w)
1255{
1256	struct nvme_tcp_queue *queue =
1257		container_of(w, struct nvme_tcp_queue, io_work);
1258	unsigned long deadline = jiffies + msecs_to_jiffies(1);
1259
1260	do {
1261		bool pending = false;
1262		int result;
1263
1264		if (mutex_trylock(&queue->send_mutex)) {
1265			result = nvme_tcp_try_send(queue);
1266			mutex_unlock(&queue->send_mutex);
1267			if (result > 0)
1268				pending = true;
1269			else if (unlikely(result < 0))
1270				break;
1271		}
1272
1273		result = nvme_tcp_try_recv(queue);
1274		if (result > 0)
1275			pending = true;
1276		else if (unlikely(result < 0))
1277			return;
1278
1279		if (!pending || !queue->rd_enabled)
1280			return;
1281
1282	} while (!time_after(jiffies, deadline)); /* quota is exhausted */
1283
1284	queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1285}
1286
1287static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1288{
1289	struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1290
1291	ahash_request_free(queue->rcv_hash);
1292	ahash_request_free(queue->snd_hash);
1293	crypto_free_ahash(tfm);
1294}
1295
1296static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1297{
1298	struct crypto_ahash *tfm;
1299
1300	tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1301	if (IS_ERR(tfm))
1302		return PTR_ERR(tfm);
1303
1304	queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1305	if (!queue->snd_hash)
1306		goto free_tfm;
1307	ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1308
1309	queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1310	if (!queue->rcv_hash)
1311		goto free_snd_hash;
1312	ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1313
1314	return 0;
1315free_snd_hash:
1316	ahash_request_free(queue->snd_hash);
1317free_tfm:
1318	crypto_free_ahash(tfm);
1319	return -ENOMEM;
1320}
1321
1322static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1323{
1324	struct nvme_tcp_request *async = &ctrl->async_req;
1325
1326	page_frag_free(async->pdu);
1327}
1328
1329static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1330{
1331	struct nvme_tcp_queue *queue = &ctrl->queues[0];
1332	struct nvme_tcp_request *async = &ctrl->async_req;
1333	u8 hdgst = nvme_tcp_hdgst_len(queue);
1334
1335	async->pdu = page_frag_alloc(&queue->pf_cache,
1336		sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1337		GFP_KERNEL | __GFP_ZERO);
1338	if (!async->pdu)
1339		return -ENOMEM;
1340
1341	async->queue = &ctrl->queues[0];
1342	return 0;
1343}
1344
1345static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1346{
1347	struct page *page;
1348	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1349	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1350	unsigned int noreclaim_flag;
1351
1352	if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1353		return;
1354
1355	if (queue->hdr_digest || queue->data_digest)
1356		nvme_tcp_free_crypto(queue);
1357
1358	if (queue->pf_cache.va) {
1359		page = virt_to_head_page(queue->pf_cache.va);
1360		__page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias);
1361		queue->pf_cache.va = NULL;
1362	}
1363
1364	noreclaim_flag = memalloc_noreclaim_save();
1365	/* ->sock will be released by fput() */
1366	fput(queue->sock->file);
1367	queue->sock = NULL;
1368	memalloc_noreclaim_restore(noreclaim_flag);
1369
1370	kfree(queue->pdu);
1371	mutex_destroy(&queue->send_mutex);
1372	mutex_destroy(&queue->queue_lock);
1373}
1374
1375static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1376{
1377	struct nvme_tcp_icreq_pdu *icreq;
1378	struct nvme_tcp_icresp_pdu *icresp;
1379	char cbuf[CMSG_LEN(sizeof(char))] = {};
1380	u8 ctype;
1381	struct msghdr msg = {};
1382	struct kvec iov;
1383	bool ctrl_hdgst, ctrl_ddgst;
1384	u32 maxh2cdata;
1385	int ret;
1386
1387	icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1388	if (!icreq)
1389		return -ENOMEM;
1390
1391	icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1392	if (!icresp) {
1393		ret = -ENOMEM;
1394		goto free_icreq;
1395	}
1396
1397	icreq->hdr.type = nvme_tcp_icreq;
1398	icreq->hdr.hlen = sizeof(*icreq);
1399	icreq->hdr.pdo = 0;
1400	icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1401	icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1402	icreq->maxr2t = 0; /* single inflight r2t supported */
1403	icreq->hpda = 0; /* no alignment constraint */
1404	if (queue->hdr_digest)
1405		icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1406	if (queue->data_digest)
1407		icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1408
1409	iov.iov_base = icreq;
1410	iov.iov_len = sizeof(*icreq);
1411	ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1412	if (ret < 0) {
1413		pr_warn("queue %d: failed to send icreq, error %d\n",
1414			nvme_tcp_queue_id(queue), ret);
1415		goto free_icresp;
1416	}
1417
1418	memset(&msg, 0, sizeof(msg));
1419	iov.iov_base = icresp;
1420	iov.iov_len = sizeof(*icresp);
1421	if (nvme_tcp_tls(&queue->ctrl->ctrl)) {
1422		msg.msg_control = cbuf;
1423		msg.msg_controllen = sizeof(cbuf);
1424	}
1425	ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1426			iov.iov_len, msg.msg_flags);
1427	if (ret < 0) {
1428		pr_warn("queue %d: failed to receive icresp, error %d\n",
1429			nvme_tcp_queue_id(queue), ret);
1430		goto free_icresp;
1431	}
1432	ret = -ENOTCONN;
1433	if (nvme_tcp_tls(&queue->ctrl->ctrl)) {
1434		ctype = tls_get_record_type(queue->sock->sk,
1435					    (struct cmsghdr *)cbuf);
1436		if (ctype != TLS_RECORD_TYPE_DATA) {
1437			pr_err("queue %d: unhandled TLS record %d\n",
1438			       nvme_tcp_queue_id(queue), ctype);
1439			goto free_icresp;
1440		}
1441	}
1442	ret = -EINVAL;
1443	if (icresp->hdr.type != nvme_tcp_icresp) {
1444		pr_err("queue %d: bad type returned %d\n",
1445			nvme_tcp_queue_id(queue), icresp->hdr.type);
1446		goto free_icresp;
1447	}
1448
1449	if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1450		pr_err("queue %d: bad pdu length returned %d\n",
1451			nvme_tcp_queue_id(queue), icresp->hdr.plen);
1452		goto free_icresp;
1453	}
1454
1455	if (icresp->pfv != NVME_TCP_PFV_1_0) {
1456		pr_err("queue %d: bad pfv returned %d\n",
1457			nvme_tcp_queue_id(queue), icresp->pfv);
1458		goto free_icresp;
1459	}
1460
1461	ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1462	if ((queue->data_digest && !ctrl_ddgst) ||
1463	    (!queue->data_digest && ctrl_ddgst)) {
1464		pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1465			nvme_tcp_queue_id(queue),
1466			queue->data_digest ? "enabled" : "disabled",
1467			ctrl_ddgst ? "enabled" : "disabled");
1468		goto free_icresp;
1469	}
1470
1471	ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1472	if ((queue->hdr_digest && !ctrl_hdgst) ||
1473	    (!queue->hdr_digest && ctrl_hdgst)) {
1474		pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1475			nvme_tcp_queue_id(queue),
1476			queue->hdr_digest ? "enabled" : "disabled",
1477			ctrl_hdgst ? "enabled" : "disabled");
1478		goto free_icresp;
1479	}
1480
1481	if (icresp->cpda != 0) {
1482		pr_err("queue %d: unsupported cpda returned %d\n",
1483			nvme_tcp_queue_id(queue), icresp->cpda);
1484		goto free_icresp;
1485	}
1486
1487	maxh2cdata = le32_to_cpu(icresp->maxdata);
1488	if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) {
1489		pr_err("queue %d: invalid maxh2cdata returned %u\n",
1490		       nvme_tcp_queue_id(queue), maxh2cdata);
1491		goto free_icresp;
1492	}
1493	queue->maxh2cdata = maxh2cdata;
1494
1495	ret = 0;
1496free_icresp:
1497	kfree(icresp);
1498free_icreq:
1499	kfree(icreq);
1500	return ret;
1501}
1502
1503static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1504{
1505	return nvme_tcp_queue_id(queue) == 0;
1506}
1507
1508static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1509{
1510	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1511	int qid = nvme_tcp_queue_id(queue);
1512
1513	return !nvme_tcp_admin_queue(queue) &&
1514		qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1515}
1516
1517static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1518{
1519	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1520	int qid = nvme_tcp_queue_id(queue);
1521
1522	return !nvme_tcp_admin_queue(queue) &&
1523		!nvme_tcp_default_queue(queue) &&
1524		qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1525			  ctrl->io_queues[HCTX_TYPE_READ];
1526}
1527
1528static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1529{
1530	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1531	int qid = nvme_tcp_queue_id(queue);
1532
1533	return !nvme_tcp_admin_queue(queue) &&
1534		!nvme_tcp_default_queue(queue) &&
1535		!nvme_tcp_read_queue(queue) &&
1536		qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1537			  ctrl->io_queues[HCTX_TYPE_READ] +
1538			  ctrl->io_queues[HCTX_TYPE_POLL];
1539}
1540
1541static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1542{
1543	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1544	int qid = nvme_tcp_queue_id(queue);
1545	int n = 0;
1546
1547	if (nvme_tcp_default_queue(queue))
1548		n = qid - 1;
1549	else if (nvme_tcp_read_queue(queue))
1550		n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1551	else if (nvme_tcp_poll_queue(queue))
1552		n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1553				ctrl->io_queues[HCTX_TYPE_READ] - 1;
1554	queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1555}
1556
1557static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid)
1558{
1559	struct nvme_tcp_queue *queue = data;
1560	struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1561	int qid = nvme_tcp_queue_id(queue);
1562	struct key *tls_key;
1563
1564	dev_dbg(ctrl->ctrl.device, "queue %d: TLS handshake done, key %x, status %d\n",
1565		qid, pskid, status);
1566
1567	if (status) {
1568		queue->tls_err = -status;
1569		goto out_complete;
1570	}
1571
1572	tls_key = key_lookup(pskid);
1573	if (IS_ERR(tls_key)) {
1574		dev_warn(ctrl->ctrl.device, "queue %d: Invalid key %x\n",
1575			 qid, pskid);
1576		queue->tls_err = -ENOKEY;
1577	} else {
1578		ctrl->ctrl.tls_key = tls_key;
1579		queue->tls_err = 0;
1580	}
1581
1582out_complete:
1583	complete(&queue->tls_complete);
1584}
1585
1586static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
1587			      struct nvme_tcp_queue *queue,
1588			      key_serial_t pskid)
1589{
1590	int qid = nvme_tcp_queue_id(queue);
1591	int ret;
1592	struct tls_handshake_args args;
1593	unsigned long tmo = tls_handshake_timeout * HZ;
1594	key_serial_t keyring = nvme_keyring_id();
1595
1596	dev_dbg(nctrl->device, "queue %d: start TLS with key %x\n",
1597		qid, pskid);
1598	memset(&args, 0, sizeof(args));
1599	args.ta_sock = queue->sock;
1600	args.ta_done = nvme_tcp_tls_done;
1601	args.ta_data = queue;
1602	args.ta_my_peerids[0] = pskid;
1603	args.ta_num_peerids = 1;
1604	if (nctrl->opts->keyring)
1605		keyring = key_serial(nctrl->opts->keyring);
1606	args.ta_keyring = keyring;
1607	args.ta_timeout_ms = tls_handshake_timeout * 1000;
1608	queue->tls_err = -EOPNOTSUPP;
1609	init_completion(&queue->tls_complete);
1610	ret = tls_client_hello_psk(&args, GFP_KERNEL);
1611	if (ret) {
1612		dev_err(nctrl->device, "queue %d: failed to start TLS: %d\n",
1613			qid, ret);
1614		return ret;
1615	}
1616	ret = wait_for_completion_interruptible_timeout(&queue->tls_complete, tmo);
1617	if (ret <= 0) {
1618		if (ret == 0)
1619			ret = -ETIMEDOUT;
1620
1621		dev_err(nctrl->device,
1622			"queue %d: TLS handshake failed, error %d\n",
1623			qid, ret);
1624		tls_handshake_cancel(queue->sock->sk);
1625	} else {
1626		dev_dbg(nctrl->device,
1627			"queue %d: TLS handshake complete, error %d\n",
1628			qid, queue->tls_err);
1629		ret = queue->tls_err;
1630	}
1631	return ret;
1632}
1633
1634static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
1635				key_serial_t pskid)
1636{
1637	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1638	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1639	int ret, rcv_pdu_size;
1640	struct file *sock_file;
1641
1642	mutex_init(&queue->queue_lock);
1643	queue->ctrl = ctrl;
1644	init_llist_head(&queue->req_list);
1645	INIT_LIST_HEAD(&queue->send_list);
1646	mutex_init(&queue->send_mutex);
1647	INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1648
1649	if (qid > 0)
1650		queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1651	else
1652		queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1653						NVME_TCP_ADMIN_CCSZ;
1654
1655	ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1656			IPPROTO_TCP, &queue->sock);
1657	if (ret) {
1658		dev_err(nctrl->device,
1659			"failed to create socket: %d\n", ret);
1660		goto err_destroy_mutex;
1661	}
1662
1663	sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL);
1664	if (IS_ERR(sock_file)) {
1665		ret = PTR_ERR(sock_file);
1666		goto err_destroy_mutex;
1667	}
1668	nvme_tcp_reclassify_socket(queue->sock);
1669
1670	/* Single syn retry */
1671	tcp_sock_set_syncnt(queue->sock->sk, 1);
1672
1673	/* Set TCP no delay */
1674	tcp_sock_set_nodelay(queue->sock->sk);
1675
1676	/*
1677	 * Cleanup whatever is sitting in the TCP transmit queue on socket
1678	 * close. This is done to prevent stale data from being sent should
1679	 * the network connection be restored before TCP times out.
1680	 */
1681	sock_no_linger(queue->sock->sk);
1682
1683	if (so_priority > 0)
1684		sock_set_priority(queue->sock->sk, so_priority);
1685
1686	/* Set socket type of service */
1687	if (nctrl->opts->tos >= 0)
1688		ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1689
1690	/* Set 10 seconds timeout for icresp recvmsg */
1691	queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1692
1693	queue->sock->sk->sk_allocation = GFP_ATOMIC;
1694	queue->sock->sk->sk_use_task_frag = false;
1695	nvme_tcp_set_queue_io_cpu(queue);
1696	queue->request = NULL;
1697	queue->data_remaining = 0;
1698	queue->ddgst_remaining = 0;
1699	queue->pdu_remaining = 0;
1700	queue->pdu_offset = 0;
1701	sk_set_memalloc(queue->sock->sk);
1702
1703	if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1704		ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1705			sizeof(ctrl->src_addr));
1706		if (ret) {
1707			dev_err(nctrl->device,
1708				"failed to bind queue %d socket %d\n",
1709				qid, ret);
1710			goto err_sock;
1711		}
1712	}
1713
1714	if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) {
1715		char *iface = nctrl->opts->host_iface;
1716		sockptr_t optval = KERNEL_SOCKPTR(iface);
1717
1718		ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE,
1719				      optval, strlen(iface));
1720		if (ret) {
1721			dev_err(nctrl->device,
1722			  "failed to bind to interface %s queue %d err %d\n",
1723			  iface, qid, ret);
1724			goto err_sock;
1725		}
1726	}
1727
1728	queue->hdr_digest = nctrl->opts->hdr_digest;
1729	queue->data_digest = nctrl->opts->data_digest;
1730	if (queue->hdr_digest || queue->data_digest) {
1731		ret = nvme_tcp_alloc_crypto(queue);
1732		if (ret) {
1733			dev_err(nctrl->device,
1734				"failed to allocate queue %d crypto\n", qid);
1735			goto err_sock;
1736		}
1737	}
1738
1739	rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1740			nvme_tcp_hdgst_len(queue);
1741	queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1742	if (!queue->pdu) {
1743		ret = -ENOMEM;
1744		goto err_crypto;
1745	}
1746
1747	dev_dbg(nctrl->device, "connecting queue %d\n",
1748			nvme_tcp_queue_id(queue));
1749
1750	ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1751		sizeof(ctrl->addr), 0);
1752	if (ret) {
1753		dev_err(nctrl->device,
1754			"failed to connect socket: %d\n", ret);
1755		goto err_rcv_pdu;
1756	}
1757
1758	/* If PSKs are configured try to start TLS */
1759	if (IS_ENABLED(CONFIG_NVME_TCP_TLS) && pskid) {
1760		ret = nvme_tcp_start_tls(nctrl, queue, pskid);
1761		if (ret)
1762			goto err_init_connect;
1763	}
1764
1765	ret = nvme_tcp_init_connection(queue);
1766	if (ret)
1767		goto err_init_connect;
1768
1769	set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1770
1771	return 0;
1772
1773err_init_connect:
1774	kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1775err_rcv_pdu:
1776	kfree(queue->pdu);
1777err_crypto:
1778	if (queue->hdr_digest || queue->data_digest)
1779		nvme_tcp_free_crypto(queue);
1780err_sock:
1781	/* ->sock will be released by fput() */
1782	fput(queue->sock->file);
1783	queue->sock = NULL;
1784err_destroy_mutex:
1785	mutex_destroy(&queue->send_mutex);
1786	mutex_destroy(&queue->queue_lock);
1787	return ret;
1788}
1789
1790static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue)
1791{
1792	struct socket *sock = queue->sock;
1793
1794	write_lock_bh(&sock->sk->sk_callback_lock);
1795	sock->sk->sk_user_data  = NULL;
1796	sock->sk->sk_data_ready = queue->data_ready;
1797	sock->sk->sk_state_change = queue->state_change;
1798	sock->sk->sk_write_space  = queue->write_space;
1799	write_unlock_bh(&sock->sk->sk_callback_lock);
1800}
1801
1802static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1803{
1804	kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1805	nvme_tcp_restore_sock_ops(queue);
1806	cancel_work_sync(&queue->io_work);
1807}
1808
1809static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1810{
1811	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1812	struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1813
1814	if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1815		return;
1816
1817	mutex_lock(&queue->queue_lock);
1818	if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1819		__nvme_tcp_stop_queue(queue);
1820	mutex_unlock(&queue->queue_lock);
1821}
1822
1823static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue)
1824{
1825	write_lock_bh(&queue->sock->sk->sk_callback_lock);
1826	queue->sock->sk->sk_user_data = queue;
1827	queue->state_change = queue->sock->sk->sk_state_change;
1828	queue->data_ready = queue->sock->sk->sk_data_ready;
1829	queue->write_space = queue->sock->sk->sk_write_space;
1830	queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1831	queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1832	queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1833#ifdef CONFIG_NET_RX_BUSY_POLL
1834	queue->sock->sk->sk_ll_usec = 1;
1835#endif
1836	write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1837}
1838
1839static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1840{
1841	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1842	struct nvme_tcp_queue *queue = &ctrl->queues[idx];
1843	int ret;
1844
1845	queue->rd_enabled = true;
1846	nvme_tcp_init_recv_ctx(queue);
1847	nvme_tcp_setup_sock_ops(queue);
1848
1849	if (idx)
1850		ret = nvmf_connect_io_queue(nctrl, idx);
1851	else
1852		ret = nvmf_connect_admin_queue(nctrl);
1853
1854	if (!ret) {
1855		set_bit(NVME_TCP_Q_LIVE, &queue->flags);
1856	} else {
1857		if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1858			__nvme_tcp_stop_queue(queue);
1859		dev_err(nctrl->device,
1860			"failed to connect queue: %d ret=%d\n", idx, ret);
1861	}
1862	return ret;
1863}
1864
1865static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1866{
1867	if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1868		cancel_work_sync(&ctrl->async_event_work);
1869		nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1870		to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1871	}
1872
1873	nvme_tcp_free_queue(ctrl, 0);
1874}
1875
1876static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1877{
1878	int i;
1879
1880	for (i = 1; i < ctrl->queue_count; i++)
1881		nvme_tcp_free_queue(ctrl, i);
1882}
1883
1884static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1885{
1886	int i;
1887
1888	for (i = 1; i < ctrl->queue_count; i++)
1889		nvme_tcp_stop_queue(ctrl, i);
1890}
1891
1892static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl,
1893				    int first, int last)
1894{
1895	int i, ret;
1896
1897	for (i = first; i < last; i++) {
1898		ret = nvme_tcp_start_queue(ctrl, i);
1899		if (ret)
1900			goto out_stop_queues;
1901	}
1902
1903	return 0;
1904
1905out_stop_queues:
1906	for (i--; i >= first; i--)
1907		nvme_tcp_stop_queue(ctrl, i);
1908	return ret;
1909}
1910
1911static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1912{
1913	int ret;
1914	key_serial_t pskid = 0;
1915
1916	if (nvme_tcp_tls(ctrl)) {
1917		if (ctrl->opts->tls_key)
1918			pskid = key_serial(ctrl->opts->tls_key);
1919		else
1920			pskid = nvme_tls_psk_default(ctrl->opts->keyring,
1921						      ctrl->opts->host->nqn,
1922						      ctrl->opts->subsysnqn);
1923		if (!pskid) {
1924			dev_err(ctrl->device, "no valid PSK found\n");
1925			return -ENOKEY;
1926		}
1927	}
1928
1929	ret = nvme_tcp_alloc_queue(ctrl, 0, pskid);
1930	if (ret)
1931		return ret;
1932
1933	ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1934	if (ret)
1935		goto out_free_queue;
1936
1937	return 0;
1938
1939out_free_queue:
1940	nvme_tcp_free_queue(ctrl, 0);
1941	return ret;
1942}
1943
1944static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1945{
1946	int i, ret;
1947
1948	if (nvme_tcp_tls(ctrl) && !ctrl->tls_key) {
1949		dev_err(ctrl->device, "no PSK negotiated\n");
1950		return -ENOKEY;
1951	}
1952	for (i = 1; i < ctrl->queue_count; i++) {
1953		ret = nvme_tcp_alloc_queue(ctrl, i,
1954				key_serial(ctrl->tls_key));
1955		if (ret)
1956			goto out_free_queues;
1957	}
1958
1959	return 0;
1960
1961out_free_queues:
1962	for (i--; i >= 1; i--)
1963		nvme_tcp_free_queue(ctrl, i);
1964
1965	return ret;
1966}
1967
1968static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1969{
1970	unsigned int nr_io_queues;
1971	int ret;
1972
1973	nr_io_queues = nvmf_nr_io_queues(ctrl->opts);
1974	ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1975	if (ret)
1976		return ret;
1977
1978	if (nr_io_queues == 0) {
1979		dev_err(ctrl->device,
1980			"unable to set any I/O queues\n");
1981		return -ENOMEM;
1982	}
1983
1984	ctrl->queue_count = nr_io_queues + 1;
1985	dev_info(ctrl->device,
1986		"creating %d I/O queues.\n", nr_io_queues);
1987
1988	nvmf_set_io_queues(ctrl->opts, nr_io_queues,
1989			   to_tcp_ctrl(ctrl)->io_queues);
1990	return __nvme_tcp_alloc_io_queues(ctrl);
1991}
1992
1993static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1994{
1995	nvme_tcp_stop_io_queues(ctrl);
1996	if (remove)
1997		nvme_remove_io_tag_set(ctrl);
1998	nvme_tcp_free_io_queues(ctrl);
1999}
2000
2001static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
2002{
2003	int ret, nr_queues;
2004
2005	ret = nvme_tcp_alloc_io_queues(ctrl);
2006	if (ret)
2007		return ret;
2008
2009	if (new) {
2010		ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set,
2011				&nvme_tcp_mq_ops,
2012				ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2,
2013				sizeof(struct nvme_tcp_request));
2014		if (ret)
2015			goto out_free_io_queues;
2016	}
2017
2018	/*
2019	 * Only start IO queues for which we have allocated the tagset
2020	 * and limitted it to the available queues. On reconnects, the
2021	 * queue number might have changed.
2022	 */
2023	nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count);
2024	ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues);
2025	if (ret)
2026		goto out_cleanup_connect_q;
2027
2028	if (!new) {
2029		nvme_start_freeze(ctrl);
2030		nvme_unquiesce_io_queues(ctrl);
2031		if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
2032			/*
2033			 * If we timed out waiting for freeze we are likely to
2034			 * be stuck.  Fail the controller initialization just
2035			 * to be safe.
2036			 */
2037			ret = -ENODEV;
2038			nvme_unfreeze(ctrl);
2039			goto out_wait_freeze_timed_out;
2040		}
2041		blk_mq_update_nr_hw_queues(ctrl->tagset,
2042			ctrl->queue_count - 1);
2043		nvme_unfreeze(ctrl);
2044	}
2045
2046	/*
2047	 * If the number of queues has increased (reconnect case)
2048	 * start all new queues now.
2049	 */
2050	ret = nvme_tcp_start_io_queues(ctrl, nr_queues,
2051				       ctrl->tagset->nr_hw_queues + 1);
2052	if (ret)
2053		goto out_wait_freeze_timed_out;
2054
2055	return 0;
2056
2057out_wait_freeze_timed_out:
2058	nvme_quiesce_io_queues(ctrl);
2059	nvme_sync_io_queues(ctrl);
2060	nvme_tcp_stop_io_queues(ctrl);
2061out_cleanup_connect_q:
2062	nvme_cancel_tagset(ctrl);
2063	if (new)
2064		nvme_remove_io_tag_set(ctrl);
2065out_free_io_queues:
2066	nvme_tcp_free_io_queues(ctrl);
2067	return ret;
2068}
2069
2070static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
2071{
2072	nvme_tcp_stop_queue(ctrl, 0);
2073	if (remove)
2074		nvme_remove_admin_tag_set(ctrl);
2075	nvme_tcp_free_admin_queue(ctrl);
2076}
2077
2078static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
2079{
2080	int error;
2081
2082	error = nvme_tcp_alloc_admin_queue(ctrl);
2083	if (error)
2084		return error;
2085
2086	if (new) {
2087		error = nvme_alloc_admin_tag_set(ctrl,
2088				&to_tcp_ctrl(ctrl)->admin_tag_set,
2089				&nvme_tcp_admin_mq_ops,
2090				sizeof(struct nvme_tcp_request));
2091		if (error)
2092			goto out_free_queue;
2093	}
2094
2095	error = nvme_tcp_start_queue(ctrl, 0);
2096	if (error)
2097		goto out_cleanup_tagset;
2098
2099	error = nvme_enable_ctrl(ctrl);
2100	if (error)
2101		goto out_stop_queue;
2102
2103	nvme_unquiesce_admin_queue(ctrl);
2104
2105	error = nvme_init_ctrl_finish(ctrl, false);
2106	if (error)
2107		goto out_quiesce_queue;
2108
2109	return 0;
2110
2111out_quiesce_queue:
2112	nvme_quiesce_admin_queue(ctrl);
2113	blk_sync_queue(ctrl->admin_q);
2114out_stop_queue:
2115	nvme_tcp_stop_queue(ctrl, 0);
2116	nvme_cancel_admin_tagset(ctrl);
2117out_cleanup_tagset:
2118	if (new)
2119		nvme_remove_admin_tag_set(ctrl);
2120out_free_queue:
2121	nvme_tcp_free_admin_queue(ctrl);
2122	return error;
2123}
2124
2125static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
2126		bool remove)
2127{
2128	nvme_quiesce_admin_queue(ctrl);
2129	blk_sync_queue(ctrl->admin_q);
2130	nvme_tcp_stop_queue(ctrl, 0);
2131	nvme_cancel_admin_tagset(ctrl);
2132	if (remove)
2133		nvme_unquiesce_admin_queue(ctrl);
2134	nvme_tcp_destroy_admin_queue(ctrl, remove);
2135}
2136
2137static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
2138		bool remove)
2139{
2140	if (ctrl->queue_count <= 1)
2141		return;
2142	nvme_quiesce_admin_queue(ctrl);
2143	nvme_quiesce_io_queues(ctrl);
2144	nvme_sync_io_queues(ctrl);
2145	nvme_tcp_stop_io_queues(ctrl);
2146	nvme_cancel_tagset(ctrl);
2147	if (remove)
2148		nvme_unquiesce_io_queues(ctrl);
2149	nvme_tcp_destroy_io_queues(ctrl, remove);
2150}
2151
2152static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
2153{
2154	enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2155
2156	/* If we are resetting/deleting then do nothing */
2157	if (state != NVME_CTRL_CONNECTING) {
2158		WARN_ON_ONCE(state == NVME_CTRL_NEW || state == NVME_CTRL_LIVE);
2159		return;
2160	}
2161
2162	if (nvmf_should_reconnect(ctrl)) {
2163		dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
2164			ctrl->opts->reconnect_delay);
2165		queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
2166				ctrl->opts->reconnect_delay * HZ);
2167	} else {
2168		dev_info(ctrl->device, "Removing controller...\n");
2169		nvme_delete_ctrl(ctrl);
2170	}
2171}
2172
2173static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
2174{
2175	struct nvmf_ctrl_options *opts = ctrl->opts;
2176	int ret;
2177
2178	ret = nvme_tcp_configure_admin_queue(ctrl, new);
2179	if (ret)
2180		return ret;
2181
2182	if (ctrl->icdoff) {
2183		ret = -EOPNOTSUPP;
2184		dev_err(ctrl->device, "icdoff is not supported!\n");
2185		goto destroy_admin;
2186	}
2187
2188	if (!nvme_ctrl_sgl_supported(ctrl)) {
2189		ret = -EOPNOTSUPP;
2190		dev_err(ctrl->device, "Mandatory sgls are not supported!\n");
2191		goto destroy_admin;
2192	}
2193
2194	if (opts->queue_size > ctrl->sqsize + 1)
2195		dev_warn(ctrl->device,
2196			"queue_size %zu > ctrl sqsize %u, clamping down\n",
2197			opts->queue_size, ctrl->sqsize + 1);
2198
2199	if (ctrl->sqsize + 1 > ctrl->maxcmd) {
2200		dev_warn(ctrl->device,
2201			"sqsize %u > ctrl maxcmd %u, clamping down\n",
2202			ctrl->sqsize + 1, ctrl->maxcmd);
2203		ctrl->sqsize = ctrl->maxcmd - 1;
2204	}
2205
2206	if (ctrl->queue_count > 1) {
2207		ret = nvme_tcp_configure_io_queues(ctrl, new);
2208		if (ret)
2209			goto destroy_admin;
2210	}
2211
2212	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
2213		/*
2214		 * state change failure is ok if we started ctrl delete,
2215		 * unless we're during creation of a new controller to
2216		 * avoid races with teardown flow.
2217		 */
2218		enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2219
2220		WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2221			     state != NVME_CTRL_DELETING_NOIO);
2222		WARN_ON_ONCE(new);
2223		ret = -EINVAL;
2224		goto destroy_io;
2225	}
2226
2227	nvme_start_ctrl(ctrl);
2228	return 0;
2229
2230destroy_io:
2231	if (ctrl->queue_count > 1) {
2232		nvme_quiesce_io_queues(ctrl);
2233		nvme_sync_io_queues(ctrl);
2234		nvme_tcp_stop_io_queues(ctrl);
2235		nvme_cancel_tagset(ctrl);
2236		nvme_tcp_destroy_io_queues(ctrl, new);
2237	}
2238destroy_admin:
2239	nvme_stop_keep_alive(ctrl);
2240	nvme_tcp_teardown_admin_queue(ctrl, false);
2241	return ret;
2242}
2243
2244static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2245{
2246	struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2247			struct nvme_tcp_ctrl, connect_work);
2248	struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2249
2250	++ctrl->nr_reconnects;
2251
2252	if (nvme_tcp_setup_ctrl(ctrl, false))
2253		goto requeue;
2254
2255	dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2256			ctrl->nr_reconnects);
2257
2258	ctrl->nr_reconnects = 0;
2259
2260	return;
2261
2262requeue:
2263	dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2264			ctrl->nr_reconnects);
2265	nvme_tcp_reconnect_or_remove(ctrl);
2266}
2267
2268static void nvme_tcp_error_recovery_work(struct work_struct *work)
2269{
2270	struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2271				struct nvme_tcp_ctrl, err_work);
2272	struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2273
2274	nvme_stop_keep_alive(ctrl);
2275	flush_work(&ctrl->async_event_work);
2276	nvme_tcp_teardown_io_queues(ctrl, false);
2277	/* unquiesce to fail fast pending requests */
2278	nvme_unquiesce_io_queues(ctrl);
2279	nvme_tcp_teardown_admin_queue(ctrl, false);
2280	nvme_unquiesce_admin_queue(ctrl);
2281	nvme_auth_stop(ctrl);
2282
2283	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2284		/* state change failure is ok if we started ctrl delete */
2285		enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2286
2287		WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2288			     state != NVME_CTRL_DELETING_NOIO);
2289		return;
2290	}
2291
2292	nvme_tcp_reconnect_or_remove(ctrl);
2293}
2294
2295static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2296{
2297	nvme_tcp_teardown_io_queues(ctrl, shutdown);
2298	nvme_quiesce_admin_queue(ctrl);
2299	nvme_disable_ctrl(ctrl, shutdown);
2300	nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2301}
2302
2303static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2304{
2305	nvme_tcp_teardown_ctrl(ctrl, true);
2306}
2307
2308static void nvme_reset_ctrl_work(struct work_struct *work)
2309{
2310	struct nvme_ctrl *ctrl =
2311		container_of(work, struct nvme_ctrl, reset_work);
2312
2313	nvme_stop_ctrl(ctrl);
2314	nvme_tcp_teardown_ctrl(ctrl, false);
2315
2316	if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2317		/* state change failure is ok if we started ctrl delete */
2318		enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2319
2320		WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2321			     state != NVME_CTRL_DELETING_NOIO);
2322		return;
2323	}
2324
2325	if (nvme_tcp_setup_ctrl(ctrl, false))
2326		goto out_fail;
2327
2328	return;
2329
2330out_fail:
2331	++ctrl->nr_reconnects;
2332	nvme_tcp_reconnect_or_remove(ctrl);
2333}
2334
2335static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
2336{
2337	flush_work(&to_tcp_ctrl(ctrl)->err_work);
2338	cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2339}
2340
2341static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2342{
2343	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2344
2345	if (list_empty(&ctrl->list))
2346		goto free_ctrl;
2347
2348	mutex_lock(&nvme_tcp_ctrl_mutex);
2349	list_del(&ctrl->list);
2350	mutex_unlock(&nvme_tcp_ctrl_mutex);
2351
2352	nvmf_free_options(nctrl->opts);
2353free_ctrl:
2354	kfree(ctrl->queues);
2355	kfree(ctrl);
2356}
2357
2358static void nvme_tcp_set_sg_null(struct nvme_command *c)
2359{
2360	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2361
2362	sg->addr = 0;
2363	sg->length = 0;
2364	sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2365			NVME_SGL_FMT_TRANSPORT_A;
2366}
2367
2368static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2369		struct nvme_command *c, u32 data_len)
2370{
2371	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2372
2373	sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2374	sg->length = cpu_to_le32(data_len);
2375	sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2376}
2377
2378static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2379		u32 data_len)
2380{
2381	struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2382
2383	sg->addr = 0;
2384	sg->length = cpu_to_le32(data_len);
2385	sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2386			NVME_SGL_FMT_TRANSPORT_A;
2387}
2388
2389static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2390{
2391	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2392	struct nvme_tcp_queue *queue = &ctrl->queues[0];
2393	struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2394	struct nvme_command *cmd = &pdu->cmd;
2395	u8 hdgst = nvme_tcp_hdgst_len(queue);
2396
2397	memset(pdu, 0, sizeof(*pdu));
2398	pdu->hdr.type = nvme_tcp_cmd;
2399	if (queue->hdr_digest)
2400		pdu->hdr.flags |= NVME_TCP_F_HDGST;
2401	pdu->hdr.hlen = sizeof(*pdu);
2402	pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2403
2404	cmd->common.opcode = nvme_admin_async_event;
2405	cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2406	cmd->common.flags |= NVME_CMD_SGL_METABUF;
2407	nvme_tcp_set_sg_null(cmd);
2408
2409	ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2410	ctrl->async_req.offset = 0;
2411	ctrl->async_req.curr_bio = NULL;
2412	ctrl->async_req.data_len = 0;
2413
2414	nvme_tcp_queue_request(&ctrl->async_req, true, true);
2415}
2416
2417static void nvme_tcp_complete_timed_out(struct request *rq)
2418{
2419	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2420	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2421
2422	nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2423	nvmf_complete_timed_out_request(rq);
2424}
2425
2426static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq)
2427{
2428	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2429	struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2430	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2431	struct nvme_command *cmd = &pdu->cmd;
2432	int qid = nvme_tcp_queue_id(req->queue);
2433
2434	dev_warn(ctrl->device,
2435		 "I/O tag %d (%04x) type %d opcode %#x (%s) QID %d timeout\n",
2436		 rq->tag, nvme_cid(rq), pdu->hdr.type, cmd->common.opcode,
2437		 nvme_fabrics_opcode_str(qid, cmd), qid);
2438
2439	if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) {
2440		/*
2441		 * If we are resetting, connecting or deleting we should
2442		 * complete immediately because we may block controller
2443		 * teardown or setup sequence
2444		 * - ctrl disable/shutdown fabrics requests
2445		 * - connect requests
2446		 * - initialization admin requests
2447		 * - I/O requests that entered after unquiescing and
2448		 *   the controller stopped responding
2449		 *
2450		 * All other requests should be cancelled by the error
2451		 * recovery work, so it's fine that we fail it here.
2452		 */
2453		nvme_tcp_complete_timed_out(rq);
2454		return BLK_EH_DONE;
2455	}
2456
2457	/*
2458	 * LIVE state should trigger the normal error recovery which will
2459	 * handle completing this request.
2460	 */
2461	nvme_tcp_error_recovery(ctrl);
2462	return BLK_EH_RESET_TIMER;
2463}
2464
2465static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2466			struct request *rq)
2467{
2468	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2469	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2470	struct nvme_command *c = &pdu->cmd;
2471
2472	c->common.flags |= NVME_CMD_SGL_METABUF;
2473
2474	if (!blk_rq_nr_phys_segments(rq))
2475		nvme_tcp_set_sg_null(c);
2476	else if (rq_data_dir(rq) == WRITE &&
2477	    req->data_len <= nvme_tcp_inline_data_size(req))
2478		nvme_tcp_set_sg_inline(queue, c, req->data_len);
2479	else
2480		nvme_tcp_set_sg_host_data(c, req->data_len);
2481
2482	return 0;
2483}
2484
2485static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2486		struct request *rq)
2487{
2488	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2489	struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2490	struct nvme_tcp_queue *queue = req->queue;
2491	u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2492	blk_status_t ret;
2493
2494	ret = nvme_setup_cmd(ns, rq);
2495	if (ret)
2496		return ret;
2497
2498	req->state = NVME_TCP_SEND_CMD_PDU;
2499	req->status = cpu_to_le16(NVME_SC_SUCCESS);
2500	req->offset = 0;
2501	req->data_sent = 0;
2502	req->pdu_len = 0;
2503	req->pdu_sent = 0;
2504	req->h2cdata_left = 0;
2505	req->data_len = blk_rq_nr_phys_segments(rq) ?
2506				blk_rq_payload_bytes(rq) : 0;
2507	req->curr_bio = rq->bio;
2508	if (req->curr_bio && req->data_len)
2509		nvme_tcp_init_iter(req, rq_data_dir(rq));
2510
2511	if (rq_data_dir(rq) == WRITE &&
2512	    req->data_len <= nvme_tcp_inline_data_size(req))
2513		req->pdu_len = req->data_len;
2514
2515	pdu->hdr.type = nvme_tcp_cmd;
2516	pdu->hdr.flags = 0;
2517	if (queue->hdr_digest)
2518		pdu->hdr.flags |= NVME_TCP_F_HDGST;
2519	if (queue->data_digest && req->pdu_len) {
2520		pdu->hdr.flags |= NVME_TCP_F_DDGST;
2521		ddgst = nvme_tcp_ddgst_len(queue);
2522	}
2523	pdu->hdr.hlen = sizeof(*pdu);
2524	pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2525	pdu->hdr.plen =
2526		cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2527
2528	ret = nvme_tcp_map_data(queue, rq);
2529	if (unlikely(ret)) {
2530		nvme_cleanup_cmd(rq);
2531		dev_err(queue->ctrl->ctrl.device,
2532			"Failed to map data (%d)\n", ret);
2533		return ret;
2534	}
2535
2536	return 0;
2537}
2538
2539static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2540{
2541	struct nvme_tcp_queue *queue = hctx->driver_data;
2542
2543	if (!llist_empty(&queue->req_list))
2544		queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2545}
2546
2547static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2548		const struct blk_mq_queue_data *bd)
2549{
2550	struct nvme_ns *ns = hctx->queue->queuedata;
2551	struct nvme_tcp_queue *queue = hctx->driver_data;
2552	struct request *rq = bd->rq;
2553	struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2554	bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2555	blk_status_t ret;
2556
2557	if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2558		return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2559
2560	ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2561	if (unlikely(ret))
2562		return ret;
2563
2564	nvme_start_request(rq);
2565
2566	nvme_tcp_queue_request(req, true, bd->last);
2567
2568	return BLK_STS_OK;
2569}
2570
2571static void nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2572{
2573	struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data);
2574
2575	nvmf_map_queues(set, &ctrl->ctrl, ctrl->io_queues);
2576}
2577
2578static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
2579{
2580	struct nvme_tcp_queue *queue = hctx->driver_data;
2581	struct sock *sk = queue->sock->sk;
2582
2583	if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2584		return 0;
2585
2586	set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2587	if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2588		sk_busy_loop(sk, true);
2589	nvme_tcp_try_recv(queue);
2590	clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2591	return queue->nr_cqe;
2592}
2593
2594static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
2595{
2596	struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0];
2597	struct sockaddr_storage src_addr;
2598	int ret, len;
2599
2600	len = nvmf_get_address(ctrl, buf, size);
2601
2602	mutex_lock(&queue->queue_lock);
2603
2604	if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2605		goto done;
2606	ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr);
2607	if (ret > 0) {
2608		if (len > 0)
2609			len--; /* strip trailing newline */
2610		len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n",
2611				(len) ? "," : "", &src_addr);
2612	}
2613done:
2614	mutex_unlock(&queue->queue_lock);
2615
2616	return len;
2617}
2618
2619static const struct blk_mq_ops nvme_tcp_mq_ops = {
2620	.queue_rq	= nvme_tcp_queue_rq,
2621	.commit_rqs	= nvme_tcp_commit_rqs,
2622	.complete	= nvme_complete_rq,
2623	.init_request	= nvme_tcp_init_request,
2624	.exit_request	= nvme_tcp_exit_request,
2625	.init_hctx	= nvme_tcp_init_hctx,
2626	.timeout	= nvme_tcp_timeout,
2627	.map_queues	= nvme_tcp_map_queues,
2628	.poll		= nvme_tcp_poll,
2629};
2630
2631static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2632	.queue_rq	= nvme_tcp_queue_rq,
2633	.complete	= nvme_complete_rq,
2634	.init_request	= nvme_tcp_init_request,
2635	.exit_request	= nvme_tcp_exit_request,
2636	.init_hctx	= nvme_tcp_init_admin_hctx,
2637	.timeout	= nvme_tcp_timeout,
2638};
2639
2640static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2641	.name			= "tcp",
2642	.module			= THIS_MODULE,
2643	.flags			= NVME_F_FABRICS | NVME_F_BLOCKING,
2644	.reg_read32		= nvmf_reg_read32,
2645	.reg_read64		= nvmf_reg_read64,
2646	.reg_write32		= nvmf_reg_write32,
2647	.free_ctrl		= nvme_tcp_free_ctrl,
2648	.submit_async_event	= nvme_tcp_submit_async_event,
2649	.delete_ctrl		= nvme_tcp_delete_ctrl,
2650	.get_address		= nvme_tcp_get_address,
2651	.stop_ctrl		= nvme_tcp_stop_ctrl,
2652};
2653
2654static bool
2655nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2656{
2657	struct nvme_tcp_ctrl *ctrl;
2658	bool found = false;
2659
2660	mutex_lock(&nvme_tcp_ctrl_mutex);
2661	list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2662		found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2663		if (found)
2664			break;
2665	}
2666	mutex_unlock(&nvme_tcp_ctrl_mutex);
2667
2668	return found;
2669}
2670
2671static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2672		struct nvmf_ctrl_options *opts)
2673{
2674	struct nvme_tcp_ctrl *ctrl;
2675	int ret;
2676
2677	ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2678	if (!ctrl)
2679		return ERR_PTR(-ENOMEM);
2680
2681	INIT_LIST_HEAD(&ctrl->list);
2682	ctrl->ctrl.opts = opts;
2683	ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2684				opts->nr_poll_queues + 1;
2685	ctrl->ctrl.sqsize = opts->queue_size - 1;
2686	ctrl->ctrl.kato = opts->kato;
2687
2688	INIT_DELAYED_WORK(&ctrl->connect_work,
2689			nvme_tcp_reconnect_ctrl_work);
2690	INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2691	INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2692
2693	if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2694		opts->trsvcid =
2695			kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2696		if (!opts->trsvcid) {
2697			ret = -ENOMEM;
2698			goto out_free_ctrl;
2699		}
2700		opts->mask |= NVMF_OPT_TRSVCID;
2701	}
2702
2703	ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2704			opts->traddr, opts->trsvcid, &ctrl->addr);
2705	if (ret) {
2706		pr_err("malformed address passed: %s:%s\n",
2707			opts->traddr, opts->trsvcid);
2708		goto out_free_ctrl;
2709	}
2710
2711	if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2712		ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2713			opts->host_traddr, NULL, &ctrl->src_addr);
2714		if (ret) {
2715			pr_err("malformed src address passed: %s\n",
2716			       opts->host_traddr);
2717			goto out_free_ctrl;
2718		}
2719	}
2720
2721	if (opts->mask & NVMF_OPT_HOST_IFACE) {
2722		if (!__dev_get_by_name(&init_net, opts->host_iface)) {
2723			pr_err("invalid interface passed: %s\n",
2724			       opts->host_iface);
2725			ret = -ENODEV;
2726			goto out_free_ctrl;
2727		}
2728	}
2729
2730	if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2731		ret = -EALREADY;
2732		goto out_free_ctrl;
2733	}
2734
2735	ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2736				GFP_KERNEL);
2737	if (!ctrl->queues) {
2738		ret = -ENOMEM;
2739		goto out_free_ctrl;
2740	}
2741
2742	ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2743	if (ret)
2744		goto out_kfree_queues;
2745
2746	if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2747		WARN_ON_ONCE(1);
2748		ret = -EINTR;
2749		goto out_uninit_ctrl;
2750	}
2751
2752	ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2753	if (ret)
2754		goto out_uninit_ctrl;
2755
2756	dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp, hostnqn: %s\n",
2757		nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr, opts->host->nqn);
2758
2759	mutex_lock(&nvme_tcp_ctrl_mutex);
2760	list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2761	mutex_unlock(&nvme_tcp_ctrl_mutex);
2762
2763	return &ctrl->ctrl;
2764
2765out_uninit_ctrl:
2766	nvme_uninit_ctrl(&ctrl->ctrl);
2767	nvme_put_ctrl(&ctrl->ctrl);
2768	if (ret > 0)
2769		ret = -EIO;
2770	return ERR_PTR(ret);
2771out_kfree_queues:
2772	kfree(ctrl->queues);
2773out_free_ctrl:
2774	kfree(ctrl);
2775	return ERR_PTR(ret);
2776}
2777
2778static struct nvmf_transport_ops nvme_tcp_transport = {
2779	.name		= "tcp",
2780	.module		= THIS_MODULE,
2781	.required_opts	= NVMF_OPT_TRADDR,
2782	.allowed_opts	= NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2783			  NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2784			  NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2785			  NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2786			  NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS |
2787			  NVMF_OPT_KEYRING | NVMF_OPT_TLS_KEY,
2788	.create_ctrl	= nvme_tcp_create_ctrl,
2789};
2790
2791static int __init nvme_tcp_init_module(void)
2792{
2793	BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8);
2794	BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72);
2795	BUILD_BUG_ON(sizeof(struct nvme_tcp_data_pdu) != 24);
2796	BUILD_BUG_ON(sizeof(struct nvme_tcp_rsp_pdu) != 24);
2797	BUILD_BUG_ON(sizeof(struct nvme_tcp_r2t_pdu) != 24);
2798	BUILD_BUG_ON(sizeof(struct nvme_tcp_icreq_pdu) != 128);
2799	BUILD_BUG_ON(sizeof(struct nvme_tcp_icresp_pdu) != 128);
2800	BUILD_BUG_ON(sizeof(struct nvme_tcp_term_pdu) != 24);
2801
2802	nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2803			WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2804	if (!nvme_tcp_wq)
2805		return -ENOMEM;
2806
2807	nvmf_register_transport(&nvme_tcp_transport);
2808	return 0;
2809}
2810
2811static void __exit nvme_tcp_cleanup_module(void)
2812{
2813	struct nvme_tcp_ctrl *ctrl;
2814
2815	nvmf_unregister_transport(&nvme_tcp_transport);
2816
2817	mutex_lock(&nvme_tcp_ctrl_mutex);
2818	list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2819		nvme_delete_ctrl(&ctrl->ctrl);
2820	mutex_unlock(&nvme_tcp_ctrl_mutex);
2821	flush_workqueue(nvme_delete_wq);
2822
2823	destroy_workqueue(nvme_tcp_wq);
2824}
2825
2826module_init(nvme_tcp_init_module);
2827module_exit(nvme_tcp_cleanup_module);
2828
2829MODULE_DESCRIPTION("NVMe host TCP transport driver");
2830MODULE_LICENSE("GPL v2");