Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * NVMe over Fabrics TCP host.
4 * Copyright (c) 2018 Lightbits Labs. All rights reserved.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7#include <linux/module.h>
8#include <linux/init.h>
9#include <linux/slab.h>
10#include <linux/err.h>
11#include <linux/key.h>
12#include <linux/nvme-tcp.h>
13#include <linux/nvme-keyring.h>
14#include <net/sock.h>
15#include <net/tcp.h>
16#include <net/tls.h>
17#include <net/tls_prot.h>
18#include <net/handshake.h>
19#include <linux/blk-mq.h>
20#include <crypto/hash.h>
21#include <net/busy_poll.h>
22#include <trace/events/sock.h>
23
24#include "nvme.h"
25#include "fabrics.h"
26
27struct nvme_tcp_queue;
28
29/* Define the socket priority to use for connections were it is desirable
30 * that the NIC consider performing optimized packet processing or filtering.
31 * A non-zero value being sufficient to indicate general consideration of any
32 * possible optimization. Making it a module param allows for alternative
33 * values that may be unique for some NIC implementations.
34 */
35static int so_priority;
36module_param(so_priority, int, 0644);
37MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
38
39/*
40 * TLS handshake timeout
41 */
42static int tls_handshake_timeout = 10;
43#ifdef CONFIG_NVME_TCP_TLS
44module_param(tls_handshake_timeout, int, 0644);
45MODULE_PARM_DESC(tls_handshake_timeout,
46 "nvme TLS handshake timeout in seconds (default 10)");
47#endif
48
49#ifdef CONFIG_DEBUG_LOCK_ALLOC
50/* lockdep can detect a circular dependency of the form
51 * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
52 * because dependencies are tracked for both nvme-tcp and user contexts. Using
53 * a separate class prevents lockdep from conflating nvme-tcp socket use with
54 * user-space socket API use.
55 */
56static struct lock_class_key nvme_tcp_sk_key[2];
57static struct lock_class_key nvme_tcp_slock_key[2];
58
59static void nvme_tcp_reclassify_socket(struct socket *sock)
60{
61 struct sock *sk = sock->sk;
62
63 if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
64 return;
65
66 switch (sk->sk_family) {
67 case AF_INET:
68 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
69 &nvme_tcp_slock_key[0],
70 "sk_lock-AF_INET-NVME",
71 &nvme_tcp_sk_key[0]);
72 break;
73 case AF_INET6:
74 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
75 &nvme_tcp_slock_key[1],
76 "sk_lock-AF_INET6-NVME",
77 &nvme_tcp_sk_key[1]);
78 break;
79 default:
80 WARN_ON_ONCE(1);
81 }
82}
83#else
84static void nvme_tcp_reclassify_socket(struct socket *sock) { }
85#endif
86
87enum nvme_tcp_send_state {
88 NVME_TCP_SEND_CMD_PDU = 0,
89 NVME_TCP_SEND_H2C_PDU,
90 NVME_TCP_SEND_DATA,
91 NVME_TCP_SEND_DDGST,
92};
93
94struct nvme_tcp_request {
95 struct nvme_request req;
96 void *pdu;
97 struct nvme_tcp_queue *queue;
98 u32 data_len;
99 u32 pdu_len;
100 u32 pdu_sent;
101 u32 h2cdata_left;
102 u32 h2cdata_offset;
103 u16 ttag;
104 __le16 status;
105 struct list_head entry;
106 struct llist_node lentry;
107 __le32 ddgst;
108
109 struct bio *curr_bio;
110 struct iov_iter iter;
111
112 /* send state */
113 size_t offset;
114 size_t data_sent;
115 enum nvme_tcp_send_state state;
116};
117
118enum nvme_tcp_queue_flags {
119 NVME_TCP_Q_ALLOCATED = 0,
120 NVME_TCP_Q_LIVE = 1,
121 NVME_TCP_Q_POLLING = 2,
122};
123
124enum nvme_tcp_recv_state {
125 NVME_TCP_RECV_PDU = 0,
126 NVME_TCP_RECV_DATA,
127 NVME_TCP_RECV_DDGST,
128};
129
130struct nvme_tcp_ctrl;
131struct nvme_tcp_queue {
132 struct socket *sock;
133 struct work_struct io_work;
134 int io_cpu;
135
136 struct mutex queue_lock;
137 struct mutex send_mutex;
138 struct llist_head req_list;
139 struct list_head send_list;
140
141 /* recv state */
142 void *pdu;
143 int pdu_remaining;
144 int pdu_offset;
145 size_t data_remaining;
146 size_t ddgst_remaining;
147 unsigned int nr_cqe;
148
149 /* send state */
150 struct nvme_tcp_request *request;
151
152 u32 maxh2cdata;
153 size_t cmnd_capsule_len;
154 struct nvme_tcp_ctrl *ctrl;
155 unsigned long flags;
156 bool rd_enabled;
157
158 bool hdr_digest;
159 bool data_digest;
160 struct ahash_request *rcv_hash;
161 struct ahash_request *snd_hash;
162 __le32 exp_ddgst;
163 __le32 recv_ddgst;
164 struct completion tls_complete;
165 int tls_err;
166 struct page_frag_cache pf_cache;
167
168 void (*state_change)(struct sock *);
169 void (*data_ready)(struct sock *);
170 void (*write_space)(struct sock *);
171};
172
173struct nvme_tcp_ctrl {
174 /* read only in the hot path */
175 struct nvme_tcp_queue *queues;
176 struct blk_mq_tag_set tag_set;
177
178 /* other member variables */
179 struct list_head list;
180 struct blk_mq_tag_set admin_tag_set;
181 struct sockaddr_storage addr;
182 struct sockaddr_storage src_addr;
183 struct nvme_ctrl ctrl;
184
185 struct work_struct err_work;
186 struct delayed_work connect_work;
187 struct nvme_tcp_request async_req;
188 u32 io_queues[HCTX_MAX_TYPES];
189};
190
191static LIST_HEAD(nvme_tcp_ctrl_list);
192static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
193static struct workqueue_struct *nvme_tcp_wq;
194static const struct blk_mq_ops nvme_tcp_mq_ops;
195static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
196static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
197
198static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
199{
200 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
201}
202
203static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
204{
205 return queue - queue->ctrl->queues;
206}
207
208static inline bool nvme_tcp_tls(struct nvme_ctrl *ctrl)
209{
210 if (!IS_ENABLED(CONFIG_NVME_TCP_TLS))
211 return 0;
212
213 return ctrl->opts->tls;
214}
215
216static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
217{
218 u32 queue_idx = nvme_tcp_queue_id(queue);
219
220 if (queue_idx == 0)
221 return queue->ctrl->admin_tag_set.tags[queue_idx];
222 return queue->ctrl->tag_set.tags[queue_idx - 1];
223}
224
225static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
226{
227 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
228}
229
230static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
231{
232 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
233}
234
235static inline void *nvme_tcp_req_cmd_pdu(struct nvme_tcp_request *req)
236{
237 return req->pdu;
238}
239
240static inline void *nvme_tcp_req_data_pdu(struct nvme_tcp_request *req)
241{
242 /* use the pdu space in the back for the data pdu */
243 return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) -
244 sizeof(struct nvme_tcp_data_pdu);
245}
246
247static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req)
248{
249 if (nvme_is_fabrics(req->req.cmd))
250 return NVME_TCP_ADMIN_CCSZ;
251 return req->queue->cmnd_capsule_len - sizeof(struct nvme_command);
252}
253
254static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
255{
256 return req == &req->queue->ctrl->async_req;
257}
258
259static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
260{
261 struct request *rq;
262
263 if (unlikely(nvme_tcp_async_req(req)))
264 return false; /* async events don't have a request */
265
266 rq = blk_mq_rq_from_pdu(req);
267
268 return rq_data_dir(rq) == WRITE && req->data_len &&
269 req->data_len <= nvme_tcp_inline_data_size(req);
270}
271
272static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
273{
274 return req->iter.bvec->bv_page;
275}
276
277static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
278{
279 return req->iter.bvec->bv_offset + req->iter.iov_offset;
280}
281
282static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
283{
284 return min_t(size_t, iov_iter_single_seg_count(&req->iter),
285 req->pdu_len - req->pdu_sent);
286}
287
288static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
289{
290 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
291 req->pdu_len - req->pdu_sent : 0;
292}
293
294static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
295 int len)
296{
297 return nvme_tcp_pdu_data_left(req) <= len;
298}
299
300static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
301 unsigned int dir)
302{
303 struct request *rq = blk_mq_rq_from_pdu(req);
304 struct bio_vec *vec;
305 unsigned int size;
306 int nr_bvec;
307 size_t offset;
308
309 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
310 vec = &rq->special_vec;
311 nr_bvec = 1;
312 size = blk_rq_payload_bytes(rq);
313 offset = 0;
314 } else {
315 struct bio *bio = req->curr_bio;
316 struct bvec_iter bi;
317 struct bio_vec bv;
318
319 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
320 nr_bvec = 0;
321 bio_for_each_bvec(bv, bio, bi) {
322 nr_bvec++;
323 }
324 size = bio->bi_iter.bi_size;
325 offset = bio->bi_iter.bi_bvec_done;
326 }
327
328 iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size);
329 req->iter.iov_offset = offset;
330}
331
332static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
333 int len)
334{
335 req->data_sent += len;
336 req->pdu_sent += len;
337 iov_iter_advance(&req->iter, len);
338 if (!iov_iter_count(&req->iter) &&
339 req->data_sent < req->data_len) {
340 req->curr_bio = req->curr_bio->bi_next;
341 nvme_tcp_init_iter(req, ITER_SOURCE);
342 }
343}
344
345static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
346{
347 int ret;
348
349 /* drain the send queue as much as we can... */
350 do {
351 ret = nvme_tcp_try_send(queue);
352 } while (ret > 0);
353}
354
355static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
356{
357 return !list_empty(&queue->send_list) ||
358 !llist_empty(&queue->req_list);
359}
360
361static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
362 bool sync, bool last)
363{
364 struct nvme_tcp_queue *queue = req->queue;
365 bool empty;
366
367 empty = llist_add(&req->lentry, &queue->req_list) &&
368 list_empty(&queue->send_list) && !queue->request;
369
370 /*
371 * if we're the first on the send_list and we can try to send
372 * directly, otherwise queue io_work. Also, only do that if we
373 * are on the same cpu, so we don't introduce contention.
374 */
375 if (queue->io_cpu == raw_smp_processor_id() &&
376 sync && empty && mutex_trylock(&queue->send_mutex)) {
377 nvme_tcp_send_all(queue);
378 mutex_unlock(&queue->send_mutex);
379 }
380
381 if (last && nvme_tcp_queue_more(queue))
382 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
383}
384
385static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
386{
387 struct nvme_tcp_request *req;
388 struct llist_node *node;
389
390 for (node = llist_del_all(&queue->req_list); node; node = node->next) {
391 req = llist_entry(node, struct nvme_tcp_request, lentry);
392 list_add(&req->entry, &queue->send_list);
393 }
394}
395
396static inline struct nvme_tcp_request *
397nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
398{
399 struct nvme_tcp_request *req;
400
401 req = list_first_entry_or_null(&queue->send_list,
402 struct nvme_tcp_request, entry);
403 if (!req) {
404 nvme_tcp_process_req_list(queue);
405 req = list_first_entry_or_null(&queue->send_list,
406 struct nvme_tcp_request, entry);
407 if (unlikely(!req))
408 return NULL;
409 }
410
411 list_del(&req->entry);
412 return req;
413}
414
415static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
416 __le32 *dgst)
417{
418 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
419 crypto_ahash_final(hash);
420}
421
422static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
423 struct page *page, off_t off, size_t len)
424{
425 struct scatterlist sg;
426
427 sg_init_table(&sg, 1);
428 sg_set_page(&sg, page, len, off);
429 ahash_request_set_crypt(hash, &sg, NULL, len);
430 crypto_ahash_update(hash);
431}
432
433static inline void nvme_tcp_hdgst(struct ahash_request *hash,
434 void *pdu, size_t len)
435{
436 struct scatterlist sg;
437
438 sg_init_one(&sg, pdu, len);
439 ahash_request_set_crypt(hash, &sg, pdu + len, len);
440 crypto_ahash_digest(hash);
441}
442
443static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
444 void *pdu, size_t pdu_len)
445{
446 struct nvme_tcp_hdr *hdr = pdu;
447 __le32 recv_digest;
448 __le32 exp_digest;
449
450 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
451 dev_err(queue->ctrl->ctrl.device,
452 "queue %d: header digest flag is cleared\n",
453 nvme_tcp_queue_id(queue));
454 return -EPROTO;
455 }
456
457 recv_digest = *(__le32 *)(pdu + hdr->hlen);
458 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
459 exp_digest = *(__le32 *)(pdu + hdr->hlen);
460 if (recv_digest != exp_digest) {
461 dev_err(queue->ctrl->ctrl.device,
462 "header digest error: recv %#x expected %#x\n",
463 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
464 return -EIO;
465 }
466
467 return 0;
468}
469
470static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
471{
472 struct nvme_tcp_hdr *hdr = pdu;
473 u8 digest_len = nvme_tcp_hdgst_len(queue);
474 u32 len;
475
476 len = le32_to_cpu(hdr->plen) - hdr->hlen -
477 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
478
479 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
480 dev_err(queue->ctrl->ctrl.device,
481 "queue %d: data digest flag is cleared\n",
482 nvme_tcp_queue_id(queue));
483 return -EPROTO;
484 }
485 crypto_ahash_init(queue->rcv_hash);
486
487 return 0;
488}
489
490static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
491 struct request *rq, unsigned int hctx_idx)
492{
493 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
494
495 page_frag_free(req->pdu);
496}
497
498static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
499 struct request *rq, unsigned int hctx_idx,
500 unsigned int numa_node)
501{
502 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data);
503 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
504 struct nvme_tcp_cmd_pdu *pdu;
505 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
506 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
507 u8 hdgst = nvme_tcp_hdgst_len(queue);
508
509 req->pdu = page_frag_alloc(&queue->pf_cache,
510 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
511 GFP_KERNEL | __GFP_ZERO);
512 if (!req->pdu)
513 return -ENOMEM;
514
515 pdu = req->pdu;
516 req->queue = queue;
517 nvme_req(rq)->ctrl = &ctrl->ctrl;
518 nvme_req(rq)->cmd = &pdu->cmd;
519
520 return 0;
521}
522
523static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
524 unsigned int hctx_idx)
525{
526 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data);
527 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
528
529 hctx->driver_data = queue;
530 return 0;
531}
532
533static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
534 unsigned int hctx_idx)
535{
536 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data);
537 struct nvme_tcp_queue *queue = &ctrl->queues[0];
538
539 hctx->driver_data = queue;
540 return 0;
541}
542
543static enum nvme_tcp_recv_state
544nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
545{
546 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
547 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
548 NVME_TCP_RECV_DATA;
549}
550
551static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
552{
553 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
554 nvme_tcp_hdgst_len(queue);
555 queue->pdu_offset = 0;
556 queue->data_remaining = -1;
557 queue->ddgst_remaining = 0;
558}
559
560static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
561{
562 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
563 return;
564
565 dev_warn(ctrl->device, "starting error recovery\n");
566 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
567}
568
569static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
570 struct nvme_completion *cqe)
571{
572 struct nvme_tcp_request *req;
573 struct request *rq;
574
575 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);
576 if (!rq) {
577 dev_err(queue->ctrl->ctrl.device,
578 "got bad cqe.command_id %#x on queue %d\n",
579 cqe->command_id, nvme_tcp_queue_id(queue));
580 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
581 return -EINVAL;
582 }
583
584 req = blk_mq_rq_to_pdu(rq);
585 if (req->status == cpu_to_le16(NVME_SC_SUCCESS))
586 req->status = cqe->status;
587
588 if (!nvme_try_complete_req(rq, req->status, cqe->result))
589 nvme_complete_rq(rq);
590 queue->nr_cqe++;
591
592 return 0;
593}
594
595static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
596 struct nvme_tcp_data_pdu *pdu)
597{
598 struct request *rq;
599
600 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
601 if (!rq) {
602 dev_err(queue->ctrl->ctrl.device,
603 "got bad c2hdata.command_id %#x on queue %d\n",
604 pdu->command_id, nvme_tcp_queue_id(queue));
605 return -ENOENT;
606 }
607
608 if (!blk_rq_payload_bytes(rq)) {
609 dev_err(queue->ctrl->ctrl.device,
610 "queue %d tag %#x unexpected data\n",
611 nvme_tcp_queue_id(queue), rq->tag);
612 return -EIO;
613 }
614
615 queue->data_remaining = le32_to_cpu(pdu->data_length);
616
617 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
618 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
619 dev_err(queue->ctrl->ctrl.device,
620 "queue %d tag %#x SUCCESS set but not last PDU\n",
621 nvme_tcp_queue_id(queue), rq->tag);
622 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
623 return -EPROTO;
624 }
625
626 return 0;
627}
628
629static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
630 struct nvme_tcp_rsp_pdu *pdu)
631{
632 struct nvme_completion *cqe = &pdu->cqe;
633 int ret = 0;
634
635 /*
636 * AEN requests are special as they don't time out and can
637 * survive any kind of queue freeze and often don't respond to
638 * aborts. We don't even bother to allocate a struct request
639 * for them but rather special case them here.
640 */
641 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
642 cqe->command_id)))
643 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
644 &cqe->result);
645 else
646 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
647
648 return ret;
649}
650
651static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req)
652{
653 struct nvme_tcp_data_pdu *data = nvme_tcp_req_data_pdu(req);
654 struct nvme_tcp_queue *queue = req->queue;
655 struct request *rq = blk_mq_rq_from_pdu(req);
656 u32 h2cdata_sent = req->pdu_len;
657 u8 hdgst = nvme_tcp_hdgst_len(queue);
658 u8 ddgst = nvme_tcp_ddgst_len(queue);
659
660 req->state = NVME_TCP_SEND_H2C_PDU;
661 req->offset = 0;
662 req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata);
663 req->pdu_sent = 0;
664 req->h2cdata_left -= req->pdu_len;
665 req->h2cdata_offset += h2cdata_sent;
666
667 memset(data, 0, sizeof(*data));
668 data->hdr.type = nvme_tcp_h2c_data;
669 if (!req->h2cdata_left)
670 data->hdr.flags = NVME_TCP_F_DATA_LAST;
671 if (queue->hdr_digest)
672 data->hdr.flags |= NVME_TCP_F_HDGST;
673 if (queue->data_digest)
674 data->hdr.flags |= NVME_TCP_F_DDGST;
675 data->hdr.hlen = sizeof(*data);
676 data->hdr.pdo = data->hdr.hlen + hdgst;
677 data->hdr.plen =
678 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
679 data->ttag = req->ttag;
680 data->command_id = nvme_cid(rq);
681 data->data_offset = cpu_to_le32(req->h2cdata_offset);
682 data->data_length = cpu_to_le32(req->pdu_len);
683}
684
685static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
686 struct nvme_tcp_r2t_pdu *pdu)
687{
688 struct nvme_tcp_request *req;
689 struct request *rq;
690 u32 r2t_length = le32_to_cpu(pdu->r2t_length);
691 u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);
692
693 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
694 if (!rq) {
695 dev_err(queue->ctrl->ctrl.device,
696 "got bad r2t.command_id %#x on queue %d\n",
697 pdu->command_id, nvme_tcp_queue_id(queue));
698 return -ENOENT;
699 }
700 req = blk_mq_rq_to_pdu(rq);
701
702 if (unlikely(!r2t_length)) {
703 dev_err(queue->ctrl->ctrl.device,
704 "req %d r2t len is %u, probably a bug...\n",
705 rq->tag, r2t_length);
706 return -EPROTO;
707 }
708
709 if (unlikely(req->data_sent + r2t_length > req->data_len)) {
710 dev_err(queue->ctrl->ctrl.device,
711 "req %d r2t len %u exceeded data len %u (%zu sent)\n",
712 rq->tag, r2t_length, req->data_len, req->data_sent);
713 return -EPROTO;
714 }
715
716 if (unlikely(r2t_offset < req->data_sent)) {
717 dev_err(queue->ctrl->ctrl.device,
718 "req %d unexpected r2t offset %u (expected %zu)\n",
719 rq->tag, r2t_offset, req->data_sent);
720 return -EPROTO;
721 }
722
723 req->pdu_len = 0;
724 req->h2cdata_left = r2t_length;
725 req->h2cdata_offset = r2t_offset;
726 req->ttag = pdu->ttag;
727
728 nvme_tcp_setup_h2c_data_pdu(req);
729 nvme_tcp_queue_request(req, false, true);
730
731 return 0;
732}
733
734static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
735 unsigned int *offset, size_t *len)
736{
737 struct nvme_tcp_hdr *hdr;
738 char *pdu = queue->pdu;
739 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
740 int ret;
741
742 ret = skb_copy_bits(skb, *offset,
743 &pdu[queue->pdu_offset], rcv_len);
744 if (unlikely(ret))
745 return ret;
746
747 queue->pdu_remaining -= rcv_len;
748 queue->pdu_offset += rcv_len;
749 *offset += rcv_len;
750 *len -= rcv_len;
751 if (queue->pdu_remaining)
752 return 0;
753
754 hdr = queue->pdu;
755 if (queue->hdr_digest) {
756 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
757 if (unlikely(ret))
758 return ret;
759 }
760
761
762 if (queue->data_digest) {
763 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
764 if (unlikely(ret))
765 return ret;
766 }
767
768 switch (hdr->type) {
769 case nvme_tcp_c2h_data:
770 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
771 case nvme_tcp_rsp:
772 nvme_tcp_init_recv_ctx(queue);
773 return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
774 case nvme_tcp_r2t:
775 nvme_tcp_init_recv_ctx(queue);
776 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
777 default:
778 dev_err(queue->ctrl->ctrl.device,
779 "unsupported pdu type (%d)\n", hdr->type);
780 return -EINVAL;
781 }
782}
783
784static inline void nvme_tcp_end_request(struct request *rq, u16 status)
785{
786 union nvme_result res = {};
787
788 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
789 nvme_complete_rq(rq);
790}
791
792static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
793 unsigned int *offset, size_t *len)
794{
795 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
796 struct request *rq =
797 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
798 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
799
800 while (true) {
801 int recv_len, ret;
802
803 recv_len = min_t(size_t, *len, queue->data_remaining);
804 if (!recv_len)
805 break;
806
807 if (!iov_iter_count(&req->iter)) {
808 req->curr_bio = req->curr_bio->bi_next;
809
810 /*
811 * If we don`t have any bios it means that controller
812 * sent more data than we requested, hence error
813 */
814 if (!req->curr_bio) {
815 dev_err(queue->ctrl->ctrl.device,
816 "queue %d no space in request %#x",
817 nvme_tcp_queue_id(queue), rq->tag);
818 nvme_tcp_init_recv_ctx(queue);
819 return -EIO;
820 }
821 nvme_tcp_init_iter(req, ITER_DEST);
822 }
823
824 /* we can read only from what is left in this bio */
825 recv_len = min_t(size_t, recv_len,
826 iov_iter_count(&req->iter));
827
828 if (queue->data_digest)
829 ret = skb_copy_and_hash_datagram_iter(skb, *offset,
830 &req->iter, recv_len, queue->rcv_hash);
831 else
832 ret = skb_copy_datagram_iter(skb, *offset,
833 &req->iter, recv_len);
834 if (ret) {
835 dev_err(queue->ctrl->ctrl.device,
836 "queue %d failed to copy request %#x data",
837 nvme_tcp_queue_id(queue), rq->tag);
838 return ret;
839 }
840
841 *len -= recv_len;
842 *offset += recv_len;
843 queue->data_remaining -= recv_len;
844 }
845
846 if (!queue->data_remaining) {
847 if (queue->data_digest) {
848 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
849 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
850 } else {
851 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
852 nvme_tcp_end_request(rq,
853 le16_to_cpu(req->status));
854 queue->nr_cqe++;
855 }
856 nvme_tcp_init_recv_ctx(queue);
857 }
858 }
859
860 return 0;
861}
862
863static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
864 struct sk_buff *skb, unsigned int *offset, size_t *len)
865{
866 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
867 char *ddgst = (char *)&queue->recv_ddgst;
868 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
869 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
870 int ret;
871
872 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
873 if (unlikely(ret))
874 return ret;
875
876 queue->ddgst_remaining -= recv_len;
877 *offset += recv_len;
878 *len -= recv_len;
879 if (queue->ddgst_remaining)
880 return 0;
881
882 if (queue->recv_ddgst != queue->exp_ddgst) {
883 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
884 pdu->command_id);
885 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
886
887 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR);
888
889 dev_err(queue->ctrl->ctrl.device,
890 "data digest error: recv %#x expected %#x\n",
891 le32_to_cpu(queue->recv_ddgst),
892 le32_to_cpu(queue->exp_ddgst));
893 }
894
895 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
896 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
897 pdu->command_id);
898 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
899
900 nvme_tcp_end_request(rq, le16_to_cpu(req->status));
901 queue->nr_cqe++;
902 }
903
904 nvme_tcp_init_recv_ctx(queue);
905 return 0;
906}
907
908static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
909 unsigned int offset, size_t len)
910{
911 struct nvme_tcp_queue *queue = desc->arg.data;
912 size_t consumed = len;
913 int result;
914
915 if (unlikely(!queue->rd_enabled))
916 return -EFAULT;
917
918 while (len) {
919 switch (nvme_tcp_recv_state(queue)) {
920 case NVME_TCP_RECV_PDU:
921 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
922 break;
923 case NVME_TCP_RECV_DATA:
924 result = nvme_tcp_recv_data(queue, skb, &offset, &len);
925 break;
926 case NVME_TCP_RECV_DDGST:
927 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
928 break;
929 default:
930 result = -EFAULT;
931 }
932 if (result) {
933 dev_err(queue->ctrl->ctrl.device,
934 "receive failed: %d\n", result);
935 queue->rd_enabled = false;
936 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
937 return result;
938 }
939 }
940
941 return consumed;
942}
943
944static void nvme_tcp_data_ready(struct sock *sk)
945{
946 struct nvme_tcp_queue *queue;
947
948 trace_sk_data_ready(sk);
949
950 read_lock_bh(&sk->sk_callback_lock);
951 queue = sk->sk_user_data;
952 if (likely(queue && queue->rd_enabled) &&
953 !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
954 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
955 read_unlock_bh(&sk->sk_callback_lock);
956}
957
958static void nvme_tcp_write_space(struct sock *sk)
959{
960 struct nvme_tcp_queue *queue;
961
962 read_lock_bh(&sk->sk_callback_lock);
963 queue = sk->sk_user_data;
964 if (likely(queue && sk_stream_is_writeable(sk))) {
965 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
966 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
967 }
968 read_unlock_bh(&sk->sk_callback_lock);
969}
970
971static void nvme_tcp_state_change(struct sock *sk)
972{
973 struct nvme_tcp_queue *queue;
974
975 read_lock_bh(&sk->sk_callback_lock);
976 queue = sk->sk_user_data;
977 if (!queue)
978 goto done;
979
980 switch (sk->sk_state) {
981 case TCP_CLOSE:
982 case TCP_CLOSE_WAIT:
983 case TCP_LAST_ACK:
984 case TCP_FIN_WAIT1:
985 case TCP_FIN_WAIT2:
986 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
987 break;
988 default:
989 dev_info(queue->ctrl->ctrl.device,
990 "queue %d socket state %d\n",
991 nvme_tcp_queue_id(queue), sk->sk_state);
992 }
993
994 queue->state_change(sk);
995done:
996 read_unlock_bh(&sk->sk_callback_lock);
997}
998
999static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
1000{
1001 queue->request = NULL;
1002}
1003
1004static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
1005{
1006 if (nvme_tcp_async_req(req)) {
1007 union nvme_result res = {};
1008
1009 nvme_complete_async_event(&req->queue->ctrl->ctrl,
1010 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
1011 } else {
1012 nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
1013 NVME_SC_HOST_PATH_ERROR);
1014 }
1015}
1016
1017static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
1018{
1019 struct nvme_tcp_queue *queue = req->queue;
1020 int req_data_len = req->data_len;
1021 u32 h2cdata_left = req->h2cdata_left;
1022
1023 while (true) {
1024 struct bio_vec bvec;
1025 struct msghdr msg = {
1026 .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES,
1027 };
1028 struct page *page = nvme_tcp_req_cur_page(req);
1029 size_t offset = nvme_tcp_req_cur_offset(req);
1030 size_t len = nvme_tcp_req_cur_length(req);
1031 bool last = nvme_tcp_pdu_last_send(req, len);
1032 int req_data_sent = req->data_sent;
1033 int ret;
1034
1035 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
1036 msg.msg_flags |= MSG_EOR;
1037 else
1038 msg.msg_flags |= MSG_MORE;
1039
1040 if (!sendpage_ok(page))
1041 msg.msg_flags &= ~MSG_SPLICE_PAGES;
1042
1043 bvec_set_page(&bvec, page, len, offset);
1044 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1045 ret = sock_sendmsg(queue->sock, &msg);
1046 if (ret <= 0)
1047 return ret;
1048
1049 if (queue->data_digest)
1050 nvme_tcp_ddgst_update(queue->snd_hash, page,
1051 offset, ret);
1052
1053 /*
1054 * update the request iterator except for the last payload send
1055 * in the request where we don't want to modify it as we may
1056 * compete with the RX path completing the request.
1057 */
1058 if (req_data_sent + ret < req_data_len)
1059 nvme_tcp_advance_req(req, ret);
1060
1061 /* fully successful last send in current PDU */
1062 if (last && ret == len) {
1063 if (queue->data_digest) {
1064 nvme_tcp_ddgst_final(queue->snd_hash,
1065 &req->ddgst);
1066 req->state = NVME_TCP_SEND_DDGST;
1067 req->offset = 0;
1068 } else {
1069 if (h2cdata_left)
1070 nvme_tcp_setup_h2c_data_pdu(req);
1071 else
1072 nvme_tcp_done_send_req(queue);
1073 }
1074 return 1;
1075 }
1076 }
1077 return -EAGAIN;
1078}
1079
1080static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
1081{
1082 struct nvme_tcp_queue *queue = req->queue;
1083 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
1084 struct bio_vec bvec;
1085 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, };
1086 bool inline_data = nvme_tcp_has_inline_data(req);
1087 u8 hdgst = nvme_tcp_hdgst_len(queue);
1088 int len = sizeof(*pdu) + hdgst - req->offset;
1089 int ret;
1090
1091 if (inline_data || nvme_tcp_queue_more(queue))
1092 msg.msg_flags |= MSG_MORE;
1093 else
1094 msg.msg_flags |= MSG_EOR;
1095
1096 if (queue->hdr_digest && !req->offset)
1097 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1098
1099 bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
1100 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1101 ret = sock_sendmsg(queue->sock, &msg);
1102 if (unlikely(ret <= 0))
1103 return ret;
1104
1105 len -= ret;
1106 if (!len) {
1107 if (inline_data) {
1108 req->state = NVME_TCP_SEND_DATA;
1109 if (queue->data_digest)
1110 crypto_ahash_init(queue->snd_hash);
1111 } else {
1112 nvme_tcp_done_send_req(queue);
1113 }
1114 return 1;
1115 }
1116 req->offset += ret;
1117
1118 return -EAGAIN;
1119}
1120
1121static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
1122{
1123 struct nvme_tcp_queue *queue = req->queue;
1124 struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req);
1125 struct bio_vec bvec;
1126 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_MORE, };
1127 u8 hdgst = nvme_tcp_hdgst_len(queue);
1128 int len = sizeof(*pdu) - req->offset + hdgst;
1129 int ret;
1130
1131 if (queue->hdr_digest && !req->offset)
1132 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1133
1134 if (!req->h2cdata_left)
1135 msg.msg_flags |= MSG_SPLICE_PAGES;
1136
1137 bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
1138 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1139 ret = sock_sendmsg(queue->sock, &msg);
1140 if (unlikely(ret <= 0))
1141 return ret;
1142
1143 len -= ret;
1144 if (!len) {
1145 req->state = NVME_TCP_SEND_DATA;
1146 if (queue->data_digest)
1147 crypto_ahash_init(queue->snd_hash);
1148 return 1;
1149 }
1150 req->offset += ret;
1151
1152 return -EAGAIN;
1153}
1154
1155static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1156{
1157 struct nvme_tcp_queue *queue = req->queue;
1158 size_t offset = req->offset;
1159 u32 h2cdata_left = req->h2cdata_left;
1160 int ret;
1161 struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1162 struct kvec iov = {
1163 .iov_base = (u8 *)&req->ddgst + req->offset,
1164 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1165 };
1166
1167 if (nvme_tcp_queue_more(queue))
1168 msg.msg_flags |= MSG_MORE;
1169 else
1170 msg.msg_flags |= MSG_EOR;
1171
1172 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1173 if (unlikely(ret <= 0))
1174 return ret;
1175
1176 if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
1177 if (h2cdata_left)
1178 nvme_tcp_setup_h2c_data_pdu(req);
1179 else
1180 nvme_tcp_done_send_req(queue);
1181 return 1;
1182 }
1183
1184 req->offset += ret;
1185 return -EAGAIN;
1186}
1187
1188static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1189{
1190 struct nvme_tcp_request *req;
1191 unsigned int noreclaim_flag;
1192 int ret = 1;
1193
1194 if (!queue->request) {
1195 queue->request = nvme_tcp_fetch_request(queue);
1196 if (!queue->request)
1197 return 0;
1198 }
1199 req = queue->request;
1200
1201 noreclaim_flag = memalloc_noreclaim_save();
1202 if (req->state == NVME_TCP_SEND_CMD_PDU) {
1203 ret = nvme_tcp_try_send_cmd_pdu(req);
1204 if (ret <= 0)
1205 goto done;
1206 if (!nvme_tcp_has_inline_data(req))
1207 goto out;
1208 }
1209
1210 if (req->state == NVME_TCP_SEND_H2C_PDU) {
1211 ret = nvme_tcp_try_send_data_pdu(req);
1212 if (ret <= 0)
1213 goto done;
1214 }
1215
1216 if (req->state == NVME_TCP_SEND_DATA) {
1217 ret = nvme_tcp_try_send_data(req);
1218 if (ret <= 0)
1219 goto done;
1220 }
1221
1222 if (req->state == NVME_TCP_SEND_DDGST)
1223 ret = nvme_tcp_try_send_ddgst(req);
1224done:
1225 if (ret == -EAGAIN) {
1226 ret = 0;
1227 } else if (ret < 0) {
1228 dev_err(queue->ctrl->ctrl.device,
1229 "failed to send request %d\n", ret);
1230 nvme_tcp_fail_request(queue->request);
1231 nvme_tcp_done_send_req(queue);
1232 }
1233out:
1234 memalloc_noreclaim_restore(noreclaim_flag);
1235 return ret;
1236}
1237
1238static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1239{
1240 struct socket *sock = queue->sock;
1241 struct sock *sk = sock->sk;
1242 read_descriptor_t rd_desc;
1243 int consumed;
1244
1245 rd_desc.arg.data = queue;
1246 rd_desc.count = 1;
1247 lock_sock(sk);
1248 queue->nr_cqe = 0;
1249 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1250 release_sock(sk);
1251 return consumed;
1252}
1253
1254static void nvme_tcp_io_work(struct work_struct *w)
1255{
1256 struct nvme_tcp_queue *queue =
1257 container_of(w, struct nvme_tcp_queue, io_work);
1258 unsigned long deadline = jiffies + msecs_to_jiffies(1);
1259
1260 do {
1261 bool pending = false;
1262 int result;
1263
1264 if (mutex_trylock(&queue->send_mutex)) {
1265 result = nvme_tcp_try_send(queue);
1266 mutex_unlock(&queue->send_mutex);
1267 if (result > 0)
1268 pending = true;
1269 else if (unlikely(result < 0))
1270 break;
1271 }
1272
1273 result = nvme_tcp_try_recv(queue);
1274 if (result > 0)
1275 pending = true;
1276 else if (unlikely(result < 0))
1277 return;
1278
1279 if (!pending || !queue->rd_enabled)
1280 return;
1281
1282 } while (!time_after(jiffies, deadline)); /* quota is exhausted */
1283
1284 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1285}
1286
1287static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1288{
1289 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1290
1291 ahash_request_free(queue->rcv_hash);
1292 ahash_request_free(queue->snd_hash);
1293 crypto_free_ahash(tfm);
1294}
1295
1296static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1297{
1298 struct crypto_ahash *tfm;
1299
1300 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1301 if (IS_ERR(tfm))
1302 return PTR_ERR(tfm);
1303
1304 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1305 if (!queue->snd_hash)
1306 goto free_tfm;
1307 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1308
1309 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1310 if (!queue->rcv_hash)
1311 goto free_snd_hash;
1312 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1313
1314 return 0;
1315free_snd_hash:
1316 ahash_request_free(queue->snd_hash);
1317free_tfm:
1318 crypto_free_ahash(tfm);
1319 return -ENOMEM;
1320}
1321
1322static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1323{
1324 struct nvme_tcp_request *async = &ctrl->async_req;
1325
1326 page_frag_free(async->pdu);
1327}
1328
1329static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1330{
1331 struct nvme_tcp_queue *queue = &ctrl->queues[0];
1332 struct nvme_tcp_request *async = &ctrl->async_req;
1333 u8 hdgst = nvme_tcp_hdgst_len(queue);
1334
1335 async->pdu = page_frag_alloc(&queue->pf_cache,
1336 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1337 GFP_KERNEL | __GFP_ZERO);
1338 if (!async->pdu)
1339 return -ENOMEM;
1340
1341 async->queue = &ctrl->queues[0];
1342 return 0;
1343}
1344
1345static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1346{
1347 struct page *page;
1348 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1349 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1350 unsigned int noreclaim_flag;
1351
1352 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1353 return;
1354
1355 if (queue->hdr_digest || queue->data_digest)
1356 nvme_tcp_free_crypto(queue);
1357
1358 if (queue->pf_cache.va) {
1359 page = virt_to_head_page(queue->pf_cache.va);
1360 __page_frag_cache_drain(page, queue->pf_cache.pagecnt_bias);
1361 queue->pf_cache.va = NULL;
1362 }
1363
1364 noreclaim_flag = memalloc_noreclaim_save();
1365 /* ->sock will be released by fput() */
1366 fput(queue->sock->file);
1367 queue->sock = NULL;
1368 memalloc_noreclaim_restore(noreclaim_flag);
1369
1370 kfree(queue->pdu);
1371 mutex_destroy(&queue->send_mutex);
1372 mutex_destroy(&queue->queue_lock);
1373}
1374
1375static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1376{
1377 struct nvme_tcp_icreq_pdu *icreq;
1378 struct nvme_tcp_icresp_pdu *icresp;
1379 char cbuf[CMSG_LEN(sizeof(char))] = {};
1380 u8 ctype;
1381 struct msghdr msg = {};
1382 struct kvec iov;
1383 bool ctrl_hdgst, ctrl_ddgst;
1384 u32 maxh2cdata;
1385 int ret;
1386
1387 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1388 if (!icreq)
1389 return -ENOMEM;
1390
1391 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1392 if (!icresp) {
1393 ret = -ENOMEM;
1394 goto free_icreq;
1395 }
1396
1397 icreq->hdr.type = nvme_tcp_icreq;
1398 icreq->hdr.hlen = sizeof(*icreq);
1399 icreq->hdr.pdo = 0;
1400 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1401 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1402 icreq->maxr2t = 0; /* single inflight r2t supported */
1403 icreq->hpda = 0; /* no alignment constraint */
1404 if (queue->hdr_digest)
1405 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1406 if (queue->data_digest)
1407 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1408
1409 iov.iov_base = icreq;
1410 iov.iov_len = sizeof(*icreq);
1411 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1412 if (ret < 0) {
1413 pr_warn("queue %d: failed to send icreq, error %d\n",
1414 nvme_tcp_queue_id(queue), ret);
1415 goto free_icresp;
1416 }
1417
1418 memset(&msg, 0, sizeof(msg));
1419 iov.iov_base = icresp;
1420 iov.iov_len = sizeof(*icresp);
1421 if (nvme_tcp_tls(&queue->ctrl->ctrl)) {
1422 msg.msg_control = cbuf;
1423 msg.msg_controllen = sizeof(cbuf);
1424 }
1425 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1426 iov.iov_len, msg.msg_flags);
1427 if (ret < 0) {
1428 pr_warn("queue %d: failed to receive icresp, error %d\n",
1429 nvme_tcp_queue_id(queue), ret);
1430 goto free_icresp;
1431 }
1432 ret = -ENOTCONN;
1433 if (nvme_tcp_tls(&queue->ctrl->ctrl)) {
1434 ctype = tls_get_record_type(queue->sock->sk,
1435 (struct cmsghdr *)cbuf);
1436 if (ctype != TLS_RECORD_TYPE_DATA) {
1437 pr_err("queue %d: unhandled TLS record %d\n",
1438 nvme_tcp_queue_id(queue), ctype);
1439 goto free_icresp;
1440 }
1441 }
1442 ret = -EINVAL;
1443 if (icresp->hdr.type != nvme_tcp_icresp) {
1444 pr_err("queue %d: bad type returned %d\n",
1445 nvme_tcp_queue_id(queue), icresp->hdr.type);
1446 goto free_icresp;
1447 }
1448
1449 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1450 pr_err("queue %d: bad pdu length returned %d\n",
1451 nvme_tcp_queue_id(queue), icresp->hdr.plen);
1452 goto free_icresp;
1453 }
1454
1455 if (icresp->pfv != NVME_TCP_PFV_1_0) {
1456 pr_err("queue %d: bad pfv returned %d\n",
1457 nvme_tcp_queue_id(queue), icresp->pfv);
1458 goto free_icresp;
1459 }
1460
1461 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1462 if ((queue->data_digest && !ctrl_ddgst) ||
1463 (!queue->data_digest && ctrl_ddgst)) {
1464 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1465 nvme_tcp_queue_id(queue),
1466 queue->data_digest ? "enabled" : "disabled",
1467 ctrl_ddgst ? "enabled" : "disabled");
1468 goto free_icresp;
1469 }
1470
1471 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1472 if ((queue->hdr_digest && !ctrl_hdgst) ||
1473 (!queue->hdr_digest && ctrl_hdgst)) {
1474 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1475 nvme_tcp_queue_id(queue),
1476 queue->hdr_digest ? "enabled" : "disabled",
1477 ctrl_hdgst ? "enabled" : "disabled");
1478 goto free_icresp;
1479 }
1480
1481 if (icresp->cpda != 0) {
1482 pr_err("queue %d: unsupported cpda returned %d\n",
1483 nvme_tcp_queue_id(queue), icresp->cpda);
1484 goto free_icresp;
1485 }
1486
1487 maxh2cdata = le32_to_cpu(icresp->maxdata);
1488 if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) {
1489 pr_err("queue %d: invalid maxh2cdata returned %u\n",
1490 nvme_tcp_queue_id(queue), maxh2cdata);
1491 goto free_icresp;
1492 }
1493 queue->maxh2cdata = maxh2cdata;
1494
1495 ret = 0;
1496free_icresp:
1497 kfree(icresp);
1498free_icreq:
1499 kfree(icreq);
1500 return ret;
1501}
1502
1503static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1504{
1505 return nvme_tcp_queue_id(queue) == 0;
1506}
1507
1508static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1509{
1510 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1511 int qid = nvme_tcp_queue_id(queue);
1512
1513 return !nvme_tcp_admin_queue(queue) &&
1514 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1515}
1516
1517static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1518{
1519 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1520 int qid = nvme_tcp_queue_id(queue);
1521
1522 return !nvme_tcp_admin_queue(queue) &&
1523 !nvme_tcp_default_queue(queue) &&
1524 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1525 ctrl->io_queues[HCTX_TYPE_READ];
1526}
1527
1528static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1529{
1530 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1531 int qid = nvme_tcp_queue_id(queue);
1532
1533 return !nvme_tcp_admin_queue(queue) &&
1534 !nvme_tcp_default_queue(queue) &&
1535 !nvme_tcp_read_queue(queue) &&
1536 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1537 ctrl->io_queues[HCTX_TYPE_READ] +
1538 ctrl->io_queues[HCTX_TYPE_POLL];
1539}
1540
1541static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1542{
1543 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1544 int qid = nvme_tcp_queue_id(queue);
1545 int n = 0;
1546
1547 if (nvme_tcp_default_queue(queue))
1548 n = qid - 1;
1549 else if (nvme_tcp_read_queue(queue))
1550 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] - 1;
1551 else if (nvme_tcp_poll_queue(queue))
1552 n = qid - ctrl->io_queues[HCTX_TYPE_DEFAULT] -
1553 ctrl->io_queues[HCTX_TYPE_READ] - 1;
1554 queue->io_cpu = cpumask_next_wrap(n - 1, cpu_online_mask, -1, false);
1555}
1556
1557static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid)
1558{
1559 struct nvme_tcp_queue *queue = data;
1560 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1561 int qid = nvme_tcp_queue_id(queue);
1562 struct key *tls_key;
1563
1564 dev_dbg(ctrl->ctrl.device, "queue %d: TLS handshake done, key %x, status %d\n",
1565 qid, pskid, status);
1566
1567 if (status) {
1568 queue->tls_err = -status;
1569 goto out_complete;
1570 }
1571
1572 tls_key = key_lookup(pskid);
1573 if (IS_ERR(tls_key)) {
1574 dev_warn(ctrl->ctrl.device, "queue %d: Invalid key %x\n",
1575 qid, pskid);
1576 queue->tls_err = -ENOKEY;
1577 } else {
1578 ctrl->ctrl.tls_key = tls_key;
1579 queue->tls_err = 0;
1580 }
1581
1582out_complete:
1583 complete(&queue->tls_complete);
1584}
1585
1586static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
1587 struct nvme_tcp_queue *queue,
1588 key_serial_t pskid)
1589{
1590 int qid = nvme_tcp_queue_id(queue);
1591 int ret;
1592 struct tls_handshake_args args;
1593 unsigned long tmo = tls_handshake_timeout * HZ;
1594 key_serial_t keyring = nvme_keyring_id();
1595
1596 dev_dbg(nctrl->device, "queue %d: start TLS with key %x\n",
1597 qid, pskid);
1598 memset(&args, 0, sizeof(args));
1599 args.ta_sock = queue->sock;
1600 args.ta_done = nvme_tcp_tls_done;
1601 args.ta_data = queue;
1602 args.ta_my_peerids[0] = pskid;
1603 args.ta_num_peerids = 1;
1604 if (nctrl->opts->keyring)
1605 keyring = key_serial(nctrl->opts->keyring);
1606 args.ta_keyring = keyring;
1607 args.ta_timeout_ms = tls_handshake_timeout * 1000;
1608 queue->tls_err = -EOPNOTSUPP;
1609 init_completion(&queue->tls_complete);
1610 ret = tls_client_hello_psk(&args, GFP_KERNEL);
1611 if (ret) {
1612 dev_err(nctrl->device, "queue %d: failed to start TLS: %d\n",
1613 qid, ret);
1614 return ret;
1615 }
1616 ret = wait_for_completion_interruptible_timeout(&queue->tls_complete, tmo);
1617 if (ret <= 0) {
1618 if (ret == 0)
1619 ret = -ETIMEDOUT;
1620
1621 dev_err(nctrl->device,
1622 "queue %d: TLS handshake failed, error %d\n",
1623 qid, ret);
1624 tls_handshake_cancel(queue->sock->sk);
1625 } else {
1626 dev_dbg(nctrl->device,
1627 "queue %d: TLS handshake complete, error %d\n",
1628 qid, queue->tls_err);
1629 ret = queue->tls_err;
1630 }
1631 return ret;
1632}
1633
1634static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
1635 key_serial_t pskid)
1636{
1637 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1638 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1639 int ret, rcv_pdu_size;
1640 struct file *sock_file;
1641
1642 mutex_init(&queue->queue_lock);
1643 queue->ctrl = ctrl;
1644 init_llist_head(&queue->req_list);
1645 INIT_LIST_HEAD(&queue->send_list);
1646 mutex_init(&queue->send_mutex);
1647 INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1648
1649 if (qid > 0)
1650 queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1651 else
1652 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1653 NVME_TCP_ADMIN_CCSZ;
1654
1655 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1656 IPPROTO_TCP, &queue->sock);
1657 if (ret) {
1658 dev_err(nctrl->device,
1659 "failed to create socket: %d\n", ret);
1660 goto err_destroy_mutex;
1661 }
1662
1663 sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL);
1664 if (IS_ERR(sock_file)) {
1665 ret = PTR_ERR(sock_file);
1666 goto err_destroy_mutex;
1667 }
1668 nvme_tcp_reclassify_socket(queue->sock);
1669
1670 /* Single syn retry */
1671 tcp_sock_set_syncnt(queue->sock->sk, 1);
1672
1673 /* Set TCP no delay */
1674 tcp_sock_set_nodelay(queue->sock->sk);
1675
1676 /*
1677 * Cleanup whatever is sitting in the TCP transmit queue on socket
1678 * close. This is done to prevent stale data from being sent should
1679 * the network connection be restored before TCP times out.
1680 */
1681 sock_no_linger(queue->sock->sk);
1682
1683 if (so_priority > 0)
1684 sock_set_priority(queue->sock->sk, so_priority);
1685
1686 /* Set socket type of service */
1687 if (nctrl->opts->tos >= 0)
1688 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1689
1690 /* Set 10 seconds timeout for icresp recvmsg */
1691 queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1692
1693 queue->sock->sk->sk_allocation = GFP_ATOMIC;
1694 queue->sock->sk->sk_use_task_frag = false;
1695 nvme_tcp_set_queue_io_cpu(queue);
1696 queue->request = NULL;
1697 queue->data_remaining = 0;
1698 queue->ddgst_remaining = 0;
1699 queue->pdu_remaining = 0;
1700 queue->pdu_offset = 0;
1701 sk_set_memalloc(queue->sock->sk);
1702
1703 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1704 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1705 sizeof(ctrl->src_addr));
1706 if (ret) {
1707 dev_err(nctrl->device,
1708 "failed to bind queue %d socket %d\n",
1709 qid, ret);
1710 goto err_sock;
1711 }
1712 }
1713
1714 if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) {
1715 char *iface = nctrl->opts->host_iface;
1716 sockptr_t optval = KERNEL_SOCKPTR(iface);
1717
1718 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE,
1719 optval, strlen(iface));
1720 if (ret) {
1721 dev_err(nctrl->device,
1722 "failed to bind to interface %s queue %d err %d\n",
1723 iface, qid, ret);
1724 goto err_sock;
1725 }
1726 }
1727
1728 queue->hdr_digest = nctrl->opts->hdr_digest;
1729 queue->data_digest = nctrl->opts->data_digest;
1730 if (queue->hdr_digest || queue->data_digest) {
1731 ret = nvme_tcp_alloc_crypto(queue);
1732 if (ret) {
1733 dev_err(nctrl->device,
1734 "failed to allocate queue %d crypto\n", qid);
1735 goto err_sock;
1736 }
1737 }
1738
1739 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1740 nvme_tcp_hdgst_len(queue);
1741 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1742 if (!queue->pdu) {
1743 ret = -ENOMEM;
1744 goto err_crypto;
1745 }
1746
1747 dev_dbg(nctrl->device, "connecting queue %d\n",
1748 nvme_tcp_queue_id(queue));
1749
1750 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1751 sizeof(ctrl->addr), 0);
1752 if (ret) {
1753 dev_err(nctrl->device,
1754 "failed to connect socket: %d\n", ret);
1755 goto err_rcv_pdu;
1756 }
1757
1758 /* If PSKs are configured try to start TLS */
1759 if (IS_ENABLED(CONFIG_NVME_TCP_TLS) && pskid) {
1760 ret = nvme_tcp_start_tls(nctrl, queue, pskid);
1761 if (ret)
1762 goto err_init_connect;
1763 }
1764
1765 ret = nvme_tcp_init_connection(queue);
1766 if (ret)
1767 goto err_init_connect;
1768
1769 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1770
1771 return 0;
1772
1773err_init_connect:
1774 kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1775err_rcv_pdu:
1776 kfree(queue->pdu);
1777err_crypto:
1778 if (queue->hdr_digest || queue->data_digest)
1779 nvme_tcp_free_crypto(queue);
1780err_sock:
1781 /* ->sock will be released by fput() */
1782 fput(queue->sock->file);
1783 queue->sock = NULL;
1784err_destroy_mutex:
1785 mutex_destroy(&queue->send_mutex);
1786 mutex_destroy(&queue->queue_lock);
1787 return ret;
1788}
1789
1790static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue)
1791{
1792 struct socket *sock = queue->sock;
1793
1794 write_lock_bh(&sock->sk->sk_callback_lock);
1795 sock->sk->sk_user_data = NULL;
1796 sock->sk->sk_data_ready = queue->data_ready;
1797 sock->sk->sk_state_change = queue->state_change;
1798 sock->sk->sk_write_space = queue->write_space;
1799 write_unlock_bh(&sock->sk->sk_callback_lock);
1800}
1801
1802static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1803{
1804 kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1805 nvme_tcp_restore_sock_ops(queue);
1806 cancel_work_sync(&queue->io_work);
1807}
1808
1809static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1810{
1811 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1812 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1813
1814 if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1815 return;
1816
1817 mutex_lock(&queue->queue_lock);
1818 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1819 __nvme_tcp_stop_queue(queue);
1820 mutex_unlock(&queue->queue_lock);
1821}
1822
1823static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue)
1824{
1825 write_lock_bh(&queue->sock->sk->sk_callback_lock);
1826 queue->sock->sk->sk_user_data = queue;
1827 queue->state_change = queue->sock->sk->sk_state_change;
1828 queue->data_ready = queue->sock->sk->sk_data_ready;
1829 queue->write_space = queue->sock->sk->sk_write_space;
1830 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1831 queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1832 queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1833#ifdef CONFIG_NET_RX_BUSY_POLL
1834 queue->sock->sk->sk_ll_usec = 1;
1835#endif
1836 write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1837}
1838
1839static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1840{
1841 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1842 struct nvme_tcp_queue *queue = &ctrl->queues[idx];
1843 int ret;
1844
1845 queue->rd_enabled = true;
1846 nvme_tcp_init_recv_ctx(queue);
1847 nvme_tcp_setup_sock_ops(queue);
1848
1849 if (idx)
1850 ret = nvmf_connect_io_queue(nctrl, idx);
1851 else
1852 ret = nvmf_connect_admin_queue(nctrl);
1853
1854 if (!ret) {
1855 set_bit(NVME_TCP_Q_LIVE, &queue->flags);
1856 } else {
1857 if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1858 __nvme_tcp_stop_queue(queue);
1859 dev_err(nctrl->device,
1860 "failed to connect queue: %d ret=%d\n", idx, ret);
1861 }
1862 return ret;
1863}
1864
1865static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
1866{
1867 if (to_tcp_ctrl(ctrl)->async_req.pdu) {
1868 cancel_work_sync(&ctrl->async_event_work);
1869 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
1870 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
1871 }
1872
1873 nvme_tcp_free_queue(ctrl, 0);
1874}
1875
1876static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
1877{
1878 int i;
1879
1880 for (i = 1; i < ctrl->queue_count; i++)
1881 nvme_tcp_free_queue(ctrl, i);
1882}
1883
1884static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
1885{
1886 int i;
1887
1888 for (i = 1; i < ctrl->queue_count; i++)
1889 nvme_tcp_stop_queue(ctrl, i);
1890}
1891
1892static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl,
1893 int first, int last)
1894{
1895 int i, ret;
1896
1897 for (i = first; i < last; i++) {
1898 ret = nvme_tcp_start_queue(ctrl, i);
1899 if (ret)
1900 goto out_stop_queues;
1901 }
1902
1903 return 0;
1904
1905out_stop_queues:
1906 for (i--; i >= first; i--)
1907 nvme_tcp_stop_queue(ctrl, i);
1908 return ret;
1909}
1910
1911static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
1912{
1913 int ret;
1914 key_serial_t pskid = 0;
1915
1916 if (nvme_tcp_tls(ctrl)) {
1917 if (ctrl->opts->tls_key)
1918 pskid = key_serial(ctrl->opts->tls_key);
1919 else
1920 pskid = nvme_tls_psk_default(ctrl->opts->keyring,
1921 ctrl->opts->host->nqn,
1922 ctrl->opts->subsysnqn);
1923 if (!pskid) {
1924 dev_err(ctrl->device, "no valid PSK found\n");
1925 return -ENOKEY;
1926 }
1927 }
1928
1929 ret = nvme_tcp_alloc_queue(ctrl, 0, pskid);
1930 if (ret)
1931 return ret;
1932
1933 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
1934 if (ret)
1935 goto out_free_queue;
1936
1937 return 0;
1938
1939out_free_queue:
1940 nvme_tcp_free_queue(ctrl, 0);
1941 return ret;
1942}
1943
1944static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1945{
1946 int i, ret;
1947
1948 if (nvme_tcp_tls(ctrl) && !ctrl->tls_key) {
1949 dev_err(ctrl->device, "no PSK negotiated\n");
1950 return -ENOKEY;
1951 }
1952 for (i = 1; i < ctrl->queue_count; i++) {
1953 ret = nvme_tcp_alloc_queue(ctrl, i,
1954 key_serial(ctrl->tls_key));
1955 if (ret)
1956 goto out_free_queues;
1957 }
1958
1959 return 0;
1960
1961out_free_queues:
1962 for (i--; i >= 1; i--)
1963 nvme_tcp_free_queue(ctrl, i);
1964
1965 return ret;
1966}
1967
1968static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
1969{
1970 unsigned int nr_io_queues;
1971 int ret;
1972
1973 nr_io_queues = nvmf_nr_io_queues(ctrl->opts);
1974 ret = nvme_set_queue_count(ctrl, &nr_io_queues);
1975 if (ret)
1976 return ret;
1977
1978 if (nr_io_queues == 0) {
1979 dev_err(ctrl->device,
1980 "unable to set any I/O queues\n");
1981 return -ENOMEM;
1982 }
1983
1984 ctrl->queue_count = nr_io_queues + 1;
1985 dev_info(ctrl->device,
1986 "creating %d I/O queues.\n", nr_io_queues);
1987
1988 nvmf_set_io_queues(ctrl->opts, nr_io_queues,
1989 to_tcp_ctrl(ctrl)->io_queues);
1990 return __nvme_tcp_alloc_io_queues(ctrl);
1991}
1992
1993static void nvme_tcp_destroy_io_queues(struct nvme_ctrl *ctrl, bool remove)
1994{
1995 nvme_tcp_stop_io_queues(ctrl);
1996 if (remove)
1997 nvme_remove_io_tag_set(ctrl);
1998 nvme_tcp_free_io_queues(ctrl);
1999}
2000
2001static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
2002{
2003 int ret, nr_queues;
2004
2005 ret = nvme_tcp_alloc_io_queues(ctrl);
2006 if (ret)
2007 return ret;
2008
2009 if (new) {
2010 ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set,
2011 &nvme_tcp_mq_ops,
2012 ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2,
2013 sizeof(struct nvme_tcp_request));
2014 if (ret)
2015 goto out_free_io_queues;
2016 }
2017
2018 /*
2019 * Only start IO queues for which we have allocated the tagset
2020 * and limitted it to the available queues. On reconnects, the
2021 * queue number might have changed.
2022 */
2023 nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count);
2024 ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues);
2025 if (ret)
2026 goto out_cleanup_connect_q;
2027
2028 if (!new) {
2029 nvme_start_freeze(ctrl);
2030 nvme_unquiesce_io_queues(ctrl);
2031 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
2032 /*
2033 * If we timed out waiting for freeze we are likely to
2034 * be stuck. Fail the controller initialization just
2035 * to be safe.
2036 */
2037 ret = -ENODEV;
2038 nvme_unfreeze(ctrl);
2039 goto out_wait_freeze_timed_out;
2040 }
2041 blk_mq_update_nr_hw_queues(ctrl->tagset,
2042 ctrl->queue_count - 1);
2043 nvme_unfreeze(ctrl);
2044 }
2045
2046 /*
2047 * If the number of queues has increased (reconnect case)
2048 * start all new queues now.
2049 */
2050 ret = nvme_tcp_start_io_queues(ctrl, nr_queues,
2051 ctrl->tagset->nr_hw_queues + 1);
2052 if (ret)
2053 goto out_wait_freeze_timed_out;
2054
2055 return 0;
2056
2057out_wait_freeze_timed_out:
2058 nvme_quiesce_io_queues(ctrl);
2059 nvme_sync_io_queues(ctrl);
2060 nvme_tcp_stop_io_queues(ctrl);
2061out_cleanup_connect_q:
2062 nvme_cancel_tagset(ctrl);
2063 if (new)
2064 nvme_remove_io_tag_set(ctrl);
2065out_free_io_queues:
2066 nvme_tcp_free_io_queues(ctrl);
2067 return ret;
2068}
2069
2070static void nvme_tcp_destroy_admin_queue(struct nvme_ctrl *ctrl, bool remove)
2071{
2072 nvme_tcp_stop_queue(ctrl, 0);
2073 if (remove)
2074 nvme_remove_admin_tag_set(ctrl);
2075 nvme_tcp_free_admin_queue(ctrl);
2076}
2077
2078static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
2079{
2080 int error;
2081
2082 error = nvme_tcp_alloc_admin_queue(ctrl);
2083 if (error)
2084 return error;
2085
2086 if (new) {
2087 error = nvme_alloc_admin_tag_set(ctrl,
2088 &to_tcp_ctrl(ctrl)->admin_tag_set,
2089 &nvme_tcp_admin_mq_ops,
2090 sizeof(struct nvme_tcp_request));
2091 if (error)
2092 goto out_free_queue;
2093 }
2094
2095 error = nvme_tcp_start_queue(ctrl, 0);
2096 if (error)
2097 goto out_cleanup_tagset;
2098
2099 error = nvme_enable_ctrl(ctrl);
2100 if (error)
2101 goto out_stop_queue;
2102
2103 nvme_unquiesce_admin_queue(ctrl);
2104
2105 error = nvme_init_ctrl_finish(ctrl, false);
2106 if (error)
2107 goto out_quiesce_queue;
2108
2109 return 0;
2110
2111out_quiesce_queue:
2112 nvme_quiesce_admin_queue(ctrl);
2113 blk_sync_queue(ctrl->admin_q);
2114out_stop_queue:
2115 nvme_tcp_stop_queue(ctrl, 0);
2116 nvme_cancel_admin_tagset(ctrl);
2117out_cleanup_tagset:
2118 if (new)
2119 nvme_remove_admin_tag_set(ctrl);
2120out_free_queue:
2121 nvme_tcp_free_admin_queue(ctrl);
2122 return error;
2123}
2124
2125static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
2126 bool remove)
2127{
2128 nvme_quiesce_admin_queue(ctrl);
2129 blk_sync_queue(ctrl->admin_q);
2130 nvme_tcp_stop_queue(ctrl, 0);
2131 nvme_cancel_admin_tagset(ctrl);
2132 if (remove)
2133 nvme_unquiesce_admin_queue(ctrl);
2134 nvme_tcp_destroy_admin_queue(ctrl, remove);
2135}
2136
2137static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
2138 bool remove)
2139{
2140 if (ctrl->queue_count <= 1)
2141 return;
2142 nvme_quiesce_admin_queue(ctrl);
2143 nvme_quiesce_io_queues(ctrl);
2144 nvme_sync_io_queues(ctrl);
2145 nvme_tcp_stop_io_queues(ctrl);
2146 nvme_cancel_tagset(ctrl);
2147 if (remove)
2148 nvme_unquiesce_io_queues(ctrl);
2149 nvme_tcp_destroy_io_queues(ctrl, remove);
2150}
2151
2152static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl)
2153{
2154 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2155
2156 /* If we are resetting/deleting then do nothing */
2157 if (state != NVME_CTRL_CONNECTING) {
2158 WARN_ON_ONCE(state == NVME_CTRL_NEW || state == NVME_CTRL_LIVE);
2159 return;
2160 }
2161
2162 if (nvmf_should_reconnect(ctrl)) {
2163 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
2164 ctrl->opts->reconnect_delay);
2165 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
2166 ctrl->opts->reconnect_delay * HZ);
2167 } else {
2168 dev_info(ctrl->device, "Removing controller...\n");
2169 nvme_delete_ctrl(ctrl);
2170 }
2171}
2172
2173static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
2174{
2175 struct nvmf_ctrl_options *opts = ctrl->opts;
2176 int ret;
2177
2178 ret = nvme_tcp_configure_admin_queue(ctrl, new);
2179 if (ret)
2180 return ret;
2181
2182 if (ctrl->icdoff) {
2183 ret = -EOPNOTSUPP;
2184 dev_err(ctrl->device, "icdoff is not supported!\n");
2185 goto destroy_admin;
2186 }
2187
2188 if (!nvme_ctrl_sgl_supported(ctrl)) {
2189 ret = -EOPNOTSUPP;
2190 dev_err(ctrl->device, "Mandatory sgls are not supported!\n");
2191 goto destroy_admin;
2192 }
2193
2194 if (opts->queue_size > ctrl->sqsize + 1)
2195 dev_warn(ctrl->device,
2196 "queue_size %zu > ctrl sqsize %u, clamping down\n",
2197 opts->queue_size, ctrl->sqsize + 1);
2198
2199 if (ctrl->sqsize + 1 > ctrl->maxcmd) {
2200 dev_warn(ctrl->device,
2201 "sqsize %u > ctrl maxcmd %u, clamping down\n",
2202 ctrl->sqsize + 1, ctrl->maxcmd);
2203 ctrl->sqsize = ctrl->maxcmd - 1;
2204 }
2205
2206 if (ctrl->queue_count > 1) {
2207 ret = nvme_tcp_configure_io_queues(ctrl, new);
2208 if (ret)
2209 goto destroy_admin;
2210 }
2211
2212 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
2213 /*
2214 * state change failure is ok if we started ctrl delete,
2215 * unless we're during creation of a new controller to
2216 * avoid races with teardown flow.
2217 */
2218 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2219
2220 WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2221 state != NVME_CTRL_DELETING_NOIO);
2222 WARN_ON_ONCE(new);
2223 ret = -EINVAL;
2224 goto destroy_io;
2225 }
2226
2227 nvme_start_ctrl(ctrl);
2228 return 0;
2229
2230destroy_io:
2231 if (ctrl->queue_count > 1) {
2232 nvme_quiesce_io_queues(ctrl);
2233 nvme_sync_io_queues(ctrl);
2234 nvme_tcp_stop_io_queues(ctrl);
2235 nvme_cancel_tagset(ctrl);
2236 nvme_tcp_destroy_io_queues(ctrl, new);
2237 }
2238destroy_admin:
2239 nvme_stop_keep_alive(ctrl);
2240 nvme_tcp_teardown_admin_queue(ctrl, false);
2241 return ret;
2242}
2243
2244static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2245{
2246 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2247 struct nvme_tcp_ctrl, connect_work);
2248 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2249
2250 ++ctrl->nr_reconnects;
2251
2252 if (nvme_tcp_setup_ctrl(ctrl, false))
2253 goto requeue;
2254
2255 dev_info(ctrl->device, "Successfully reconnected (%d attempt)\n",
2256 ctrl->nr_reconnects);
2257
2258 ctrl->nr_reconnects = 0;
2259
2260 return;
2261
2262requeue:
2263 dev_info(ctrl->device, "Failed reconnect attempt %d\n",
2264 ctrl->nr_reconnects);
2265 nvme_tcp_reconnect_or_remove(ctrl);
2266}
2267
2268static void nvme_tcp_error_recovery_work(struct work_struct *work)
2269{
2270 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2271 struct nvme_tcp_ctrl, err_work);
2272 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2273
2274 nvme_stop_keep_alive(ctrl);
2275 flush_work(&ctrl->async_event_work);
2276 nvme_tcp_teardown_io_queues(ctrl, false);
2277 /* unquiesce to fail fast pending requests */
2278 nvme_unquiesce_io_queues(ctrl);
2279 nvme_tcp_teardown_admin_queue(ctrl, false);
2280 nvme_unquiesce_admin_queue(ctrl);
2281 nvme_auth_stop(ctrl);
2282
2283 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2284 /* state change failure is ok if we started ctrl delete */
2285 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2286
2287 WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2288 state != NVME_CTRL_DELETING_NOIO);
2289 return;
2290 }
2291
2292 nvme_tcp_reconnect_or_remove(ctrl);
2293}
2294
2295static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2296{
2297 nvme_tcp_teardown_io_queues(ctrl, shutdown);
2298 nvme_quiesce_admin_queue(ctrl);
2299 nvme_disable_ctrl(ctrl, shutdown);
2300 nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2301}
2302
2303static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2304{
2305 nvme_tcp_teardown_ctrl(ctrl, true);
2306}
2307
2308static void nvme_reset_ctrl_work(struct work_struct *work)
2309{
2310 struct nvme_ctrl *ctrl =
2311 container_of(work, struct nvme_ctrl, reset_work);
2312
2313 nvme_stop_ctrl(ctrl);
2314 nvme_tcp_teardown_ctrl(ctrl, false);
2315
2316 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2317 /* state change failure is ok if we started ctrl delete */
2318 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2319
2320 WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2321 state != NVME_CTRL_DELETING_NOIO);
2322 return;
2323 }
2324
2325 if (nvme_tcp_setup_ctrl(ctrl, false))
2326 goto out_fail;
2327
2328 return;
2329
2330out_fail:
2331 ++ctrl->nr_reconnects;
2332 nvme_tcp_reconnect_or_remove(ctrl);
2333}
2334
2335static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
2336{
2337 flush_work(&to_tcp_ctrl(ctrl)->err_work);
2338 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2339}
2340
2341static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2342{
2343 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2344
2345 if (list_empty(&ctrl->list))
2346 goto free_ctrl;
2347
2348 mutex_lock(&nvme_tcp_ctrl_mutex);
2349 list_del(&ctrl->list);
2350 mutex_unlock(&nvme_tcp_ctrl_mutex);
2351
2352 nvmf_free_options(nctrl->opts);
2353free_ctrl:
2354 kfree(ctrl->queues);
2355 kfree(ctrl);
2356}
2357
2358static void nvme_tcp_set_sg_null(struct nvme_command *c)
2359{
2360 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2361
2362 sg->addr = 0;
2363 sg->length = 0;
2364 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2365 NVME_SGL_FMT_TRANSPORT_A;
2366}
2367
2368static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2369 struct nvme_command *c, u32 data_len)
2370{
2371 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2372
2373 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2374 sg->length = cpu_to_le32(data_len);
2375 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2376}
2377
2378static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2379 u32 data_len)
2380{
2381 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2382
2383 sg->addr = 0;
2384 sg->length = cpu_to_le32(data_len);
2385 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2386 NVME_SGL_FMT_TRANSPORT_A;
2387}
2388
2389static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2390{
2391 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2392 struct nvme_tcp_queue *queue = &ctrl->queues[0];
2393 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2394 struct nvme_command *cmd = &pdu->cmd;
2395 u8 hdgst = nvme_tcp_hdgst_len(queue);
2396
2397 memset(pdu, 0, sizeof(*pdu));
2398 pdu->hdr.type = nvme_tcp_cmd;
2399 if (queue->hdr_digest)
2400 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2401 pdu->hdr.hlen = sizeof(*pdu);
2402 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2403
2404 cmd->common.opcode = nvme_admin_async_event;
2405 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2406 cmd->common.flags |= NVME_CMD_SGL_METABUF;
2407 nvme_tcp_set_sg_null(cmd);
2408
2409 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2410 ctrl->async_req.offset = 0;
2411 ctrl->async_req.curr_bio = NULL;
2412 ctrl->async_req.data_len = 0;
2413
2414 nvme_tcp_queue_request(&ctrl->async_req, true, true);
2415}
2416
2417static void nvme_tcp_complete_timed_out(struct request *rq)
2418{
2419 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2420 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2421
2422 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2423 nvmf_complete_timed_out_request(rq);
2424}
2425
2426static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq)
2427{
2428 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2429 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2430 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2431 struct nvme_command *cmd = &pdu->cmd;
2432 int qid = nvme_tcp_queue_id(req->queue);
2433
2434 dev_warn(ctrl->device,
2435 "I/O tag %d (%04x) type %d opcode %#x (%s) QID %d timeout\n",
2436 rq->tag, nvme_cid(rq), pdu->hdr.type, cmd->common.opcode,
2437 nvme_fabrics_opcode_str(qid, cmd), qid);
2438
2439 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) {
2440 /*
2441 * If we are resetting, connecting or deleting we should
2442 * complete immediately because we may block controller
2443 * teardown or setup sequence
2444 * - ctrl disable/shutdown fabrics requests
2445 * - connect requests
2446 * - initialization admin requests
2447 * - I/O requests that entered after unquiescing and
2448 * the controller stopped responding
2449 *
2450 * All other requests should be cancelled by the error
2451 * recovery work, so it's fine that we fail it here.
2452 */
2453 nvme_tcp_complete_timed_out(rq);
2454 return BLK_EH_DONE;
2455 }
2456
2457 /*
2458 * LIVE state should trigger the normal error recovery which will
2459 * handle completing this request.
2460 */
2461 nvme_tcp_error_recovery(ctrl);
2462 return BLK_EH_RESET_TIMER;
2463}
2464
2465static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2466 struct request *rq)
2467{
2468 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2469 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2470 struct nvme_command *c = &pdu->cmd;
2471
2472 c->common.flags |= NVME_CMD_SGL_METABUF;
2473
2474 if (!blk_rq_nr_phys_segments(rq))
2475 nvme_tcp_set_sg_null(c);
2476 else if (rq_data_dir(rq) == WRITE &&
2477 req->data_len <= nvme_tcp_inline_data_size(req))
2478 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2479 else
2480 nvme_tcp_set_sg_host_data(c, req->data_len);
2481
2482 return 0;
2483}
2484
2485static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2486 struct request *rq)
2487{
2488 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2489 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2490 struct nvme_tcp_queue *queue = req->queue;
2491 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2492 blk_status_t ret;
2493
2494 ret = nvme_setup_cmd(ns, rq);
2495 if (ret)
2496 return ret;
2497
2498 req->state = NVME_TCP_SEND_CMD_PDU;
2499 req->status = cpu_to_le16(NVME_SC_SUCCESS);
2500 req->offset = 0;
2501 req->data_sent = 0;
2502 req->pdu_len = 0;
2503 req->pdu_sent = 0;
2504 req->h2cdata_left = 0;
2505 req->data_len = blk_rq_nr_phys_segments(rq) ?
2506 blk_rq_payload_bytes(rq) : 0;
2507 req->curr_bio = rq->bio;
2508 if (req->curr_bio && req->data_len)
2509 nvme_tcp_init_iter(req, rq_data_dir(rq));
2510
2511 if (rq_data_dir(rq) == WRITE &&
2512 req->data_len <= nvme_tcp_inline_data_size(req))
2513 req->pdu_len = req->data_len;
2514
2515 pdu->hdr.type = nvme_tcp_cmd;
2516 pdu->hdr.flags = 0;
2517 if (queue->hdr_digest)
2518 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2519 if (queue->data_digest && req->pdu_len) {
2520 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2521 ddgst = nvme_tcp_ddgst_len(queue);
2522 }
2523 pdu->hdr.hlen = sizeof(*pdu);
2524 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2525 pdu->hdr.plen =
2526 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2527
2528 ret = nvme_tcp_map_data(queue, rq);
2529 if (unlikely(ret)) {
2530 nvme_cleanup_cmd(rq);
2531 dev_err(queue->ctrl->ctrl.device,
2532 "Failed to map data (%d)\n", ret);
2533 return ret;
2534 }
2535
2536 return 0;
2537}
2538
2539static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2540{
2541 struct nvme_tcp_queue *queue = hctx->driver_data;
2542
2543 if (!llist_empty(&queue->req_list))
2544 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2545}
2546
2547static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2548 const struct blk_mq_queue_data *bd)
2549{
2550 struct nvme_ns *ns = hctx->queue->queuedata;
2551 struct nvme_tcp_queue *queue = hctx->driver_data;
2552 struct request *rq = bd->rq;
2553 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2554 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2555 blk_status_t ret;
2556
2557 if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2558 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2559
2560 ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2561 if (unlikely(ret))
2562 return ret;
2563
2564 nvme_start_request(rq);
2565
2566 nvme_tcp_queue_request(req, true, bd->last);
2567
2568 return BLK_STS_OK;
2569}
2570
2571static void nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2572{
2573 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data);
2574
2575 nvmf_map_queues(set, &ctrl->ctrl, ctrl->io_queues);
2576}
2577
2578static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
2579{
2580 struct nvme_tcp_queue *queue = hctx->driver_data;
2581 struct sock *sk = queue->sock->sk;
2582
2583 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2584 return 0;
2585
2586 set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2587 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2588 sk_busy_loop(sk, true);
2589 nvme_tcp_try_recv(queue);
2590 clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2591 return queue->nr_cqe;
2592}
2593
2594static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
2595{
2596 struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0];
2597 struct sockaddr_storage src_addr;
2598 int ret, len;
2599
2600 len = nvmf_get_address(ctrl, buf, size);
2601
2602 mutex_lock(&queue->queue_lock);
2603
2604 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2605 goto done;
2606 ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr);
2607 if (ret > 0) {
2608 if (len > 0)
2609 len--; /* strip trailing newline */
2610 len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n",
2611 (len) ? "," : "", &src_addr);
2612 }
2613done:
2614 mutex_unlock(&queue->queue_lock);
2615
2616 return len;
2617}
2618
2619static const struct blk_mq_ops nvme_tcp_mq_ops = {
2620 .queue_rq = nvme_tcp_queue_rq,
2621 .commit_rqs = nvme_tcp_commit_rqs,
2622 .complete = nvme_complete_rq,
2623 .init_request = nvme_tcp_init_request,
2624 .exit_request = nvme_tcp_exit_request,
2625 .init_hctx = nvme_tcp_init_hctx,
2626 .timeout = nvme_tcp_timeout,
2627 .map_queues = nvme_tcp_map_queues,
2628 .poll = nvme_tcp_poll,
2629};
2630
2631static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2632 .queue_rq = nvme_tcp_queue_rq,
2633 .complete = nvme_complete_rq,
2634 .init_request = nvme_tcp_init_request,
2635 .exit_request = nvme_tcp_exit_request,
2636 .init_hctx = nvme_tcp_init_admin_hctx,
2637 .timeout = nvme_tcp_timeout,
2638};
2639
2640static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2641 .name = "tcp",
2642 .module = THIS_MODULE,
2643 .flags = NVME_F_FABRICS | NVME_F_BLOCKING,
2644 .reg_read32 = nvmf_reg_read32,
2645 .reg_read64 = nvmf_reg_read64,
2646 .reg_write32 = nvmf_reg_write32,
2647 .free_ctrl = nvme_tcp_free_ctrl,
2648 .submit_async_event = nvme_tcp_submit_async_event,
2649 .delete_ctrl = nvme_tcp_delete_ctrl,
2650 .get_address = nvme_tcp_get_address,
2651 .stop_ctrl = nvme_tcp_stop_ctrl,
2652};
2653
2654static bool
2655nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2656{
2657 struct nvme_tcp_ctrl *ctrl;
2658 bool found = false;
2659
2660 mutex_lock(&nvme_tcp_ctrl_mutex);
2661 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2662 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2663 if (found)
2664 break;
2665 }
2666 mutex_unlock(&nvme_tcp_ctrl_mutex);
2667
2668 return found;
2669}
2670
2671static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2672 struct nvmf_ctrl_options *opts)
2673{
2674 struct nvme_tcp_ctrl *ctrl;
2675 int ret;
2676
2677 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2678 if (!ctrl)
2679 return ERR_PTR(-ENOMEM);
2680
2681 INIT_LIST_HEAD(&ctrl->list);
2682 ctrl->ctrl.opts = opts;
2683 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2684 opts->nr_poll_queues + 1;
2685 ctrl->ctrl.sqsize = opts->queue_size - 1;
2686 ctrl->ctrl.kato = opts->kato;
2687
2688 INIT_DELAYED_WORK(&ctrl->connect_work,
2689 nvme_tcp_reconnect_ctrl_work);
2690 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2691 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2692
2693 if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2694 opts->trsvcid =
2695 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2696 if (!opts->trsvcid) {
2697 ret = -ENOMEM;
2698 goto out_free_ctrl;
2699 }
2700 opts->mask |= NVMF_OPT_TRSVCID;
2701 }
2702
2703 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2704 opts->traddr, opts->trsvcid, &ctrl->addr);
2705 if (ret) {
2706 pr_err("malformed address passed: %s:%s\n",
2707 opts->traddr, opts->trsvcid);
2708 goto out_free_ctrl;
2709 }
2710
2711 if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2712 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2713 opts->host_traddr, NULL, &ctrl->src_addr);
2714 if (ret) {
2715 pr_err("malformed src address passed: %s\n",
2716 opts->host_traddr);
2717 goto out_free_ctrl;
2718 }
2719 }
2720
2721 if (opts->mask & NVMF_OPT_HOST_IFACE) {
2722 if (!__dev_get_by_name(&init_net, opts->host_iface)) {
2723 pr_err("invalid interface passed: %s\n",
2724 opts->host_iface);
2725 ret = -ENODEV;
2726 goto out_free_ctrl;
2727 }
2728 }
2729
2730 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2731 ret = -EALREADY;
2732 goto out_free_ctrl;
2733 }
2734
2735 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2736 GFP_KERNEL);
2737 if (!ctrl->queues) {
2738 ret = -ENOMEM;
2739 goto out_free_ctrl;
2740 }
2741
2742 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2743 if (ret)
2744 goto out_kfree_queues;
2745
2746 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2747 WARN_ON_ONCE(1);
2748 ret = -EINTR;
2749 goto out_uninit_ctrl;
2750 }
2751
2752 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2753 if (ret)
2754 goto out_uninit_ctrl;
2755
2756 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp, hostnqn: %s\n",
2757 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr, opts->host->nqn);
2758
2759 mutex_lock(&nvme_tcp_ctrl_mutex);
2760 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2761 mutex_unlock(&nvme_tcp_ctrl_mutex);
2762
2763 return &ctrl->ctrl;
2764
2765out_uninit_ctrl:
2766 nvme_uninit_ctrl(&ctrl->ctrl);
2767 nvme_put_ctrl(&ctrl->ctrl);
2768 if (ret > 0)
2769 ret = -EIO;
2770 return ERR_PTR(ret);
2771out_kfree_queues:
2772 kfree(ctrl->queues);
2773out_free_ctrl:
2774 kfree(ctrl);
2775 return ERR_PTR(ret);
2776}
2777
2778static struct nvmf_transport_ops nvme_tcp_transport = {
2779 .name = "tcp",
2780 .module = THIS_MODULE,
2781 .required_opts = NVMF_OPT_TRADDR,
2782 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2783 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2784 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2785 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2786 NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS |
2787 NVMF_OPT_KEYRING | NVMF_OPT_TLS_KEY,
2788 .create_ctrl = nvme_tcp_create_ctrl,
2789};
2790
2791static int __init nvme_tcp_init_module(void)
2792{
2793 BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8);
2794 BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72);
2795 BUILD_BUG_ON(sizeof(struct nvme_tcp_data_pdu) != 24);
2796 BUILD_BUG_ON(sizeof(struct nvme_tcp_rsp_pdu) != 24);
2797 BUILD_BUG_ON(sizeof(struct nvme_tcp_r2t_pdu) != 24);
2798 BUILD_BUG_ON(sizeof(struct nvme_tcp_icreq_pdu) != 128);
2799 BUILD_BUG_ON(sizeof(struct nvme_tcp_icresp_pdu) != 128);
2800 BUILD_BUG_ON(sizeof(struct nvme_tcp_term_pdu) != 24);
2801
2802 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq",
2803 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
2804 if (!nvme_tcp_wq)
2805 return -ENOMEM;
2806
2807 nvmf_register_transport(&nvme_tcp_transport);
2808 return 0;
2809}
2810
2811static void __exit nvme_tcp_cleanup_module(void)
2812{
2813 struct nvme_tcp_ctrl *ctrl;
2814
2815 nvmf_unregister_transport(&nvme_tcp_transport);
2816
2817 mutex_lock(&nvme_tcp_ctrl_mutex);
2818 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2819 nvme_delete_ctrl(&ctrl->ctrl);
2820 mutex_unlock(&nvme_tcp_ctrl_mutex);
2821 flush_workqueue(nvme_delete_wq);
2822
2823 destroy_workqueue(nvme_tcp_wq);
2824}
2825
2826module_init(nvme_tcp_init_module);
2827module_exit(nvme_tcp_cleanup_module);
2828
2829MODULE_DESCRIPTION("NVMe host TCP transport driver");
2830MODULE_LICENSE("GPL v2");
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * NVMe over Fabrics TCP host.
4 * Copyright (c) 2018 Lightbits Labs. All rights reserved.
5 */
6#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
7#include <linux/module.h>
8#include <linux/init.h>
9#include <linux/slab.h>
10#include <linux/err.h>
11#include <linux/key.h>
12#include <linux/nvme-tcp.h>
13#include <linux/nvme-keyring.h>
14#include <net/sock.h>
15#include <net/tcp.h>
16#include <net/tls.h>
17#include <net/tls_prot.h>
18#include <net/handshake.h>
19#include <linux/blk-mq.h>
20#include <crypto/hash.h>
21#include <net/busy_poll.h>
22#include <trace/events/sock.h>
23
24#include "nvme.h"
25#include "fabrics.h"
26
27struct nvme_tcp_queue;
28
29/* Define the socket priority to use for connections were it is desirable
30 * that the NIC consider performing optimized packet processing or filtering.
31 * A non-zero value being sufficient to indicate general consideration of any
32 * possible optimization. Making it a module param allows for alternative
33 * values that may be unique for some NIC implementations.
34 */
35static int so_priority;
36module_param(so_priority, int, 0644);
37MODULE_PARM_DESC(so_priority, "nvme tcp socket optimize priority");
38
39/*
40 * Use the unbound workqueue for nvme_tcp_wq, then we can set the cpu affinity
41 * from sysfs.
42 */
43static bool wq_unbound;
44module_param(wq_unbound, bool, 0644);
45MODULE_PARM_DESC(wq_unbound, "Use unbound workqueue for nvme-tcp IO context (default false)");
46
47/*
48 * TLS handshake timeout
49 */
50static int tls_handshake_timeout = 10;
51#ifdef CONFIG_NVME_TCP_TLS
52module_param(tls_handshake_timeout, int, 0644);
53MODULE_PARM_DESC(tls_handshake_timeout,
54 "nvme TLS handshake timeout in seconds (default 10)");
55#endif
56
57static atomic_t nvme_tcp_cpu_queues[NR_CPUS];
58
59#ifdef CONFIG_DEBUG_LOCK_ALLOC
60/* lockdep can detect a circular dependency of the form
61 * sk_lock -> mmap_lock (page fault) -> fs locks -> sk_lock
62 * because dependencies are tracked for both nvme-tcp and user contexts. Using
63 * a separate class prevents lockdep from conflating nvme-tcp socket use with
64 * user-space socket API use.
65 */
66static struct lock_class_key nvme_tcp_sk_key[2];
67static struct lock_class_key nvme_tcp_slock_key[2];
68
69static void nvme_tcp_reclassify_socket(struct socket *sock)
70{
71 struct sock *sk = sock->sk;
72
73 if (WARN_ON_ONCE(!sock_allow_reclassification(sk)))
74 return;
75
76 switch (sk->sk_family) {
77 case AF_INET:
78 sock_lock_init_class_and_name(sk, "slock-AF_INET-NVME",
79 &nvme_tcp_slock_key[0],
80 "sk_lock-AF_INET-NVME",
81 &nvme_tcp_sk_key[0]);
82 break;
83 case AF_INET6:
84 sock_lock_init_class_and_name(sk, "slock-AF_INET6-NVME",
85 &nvme_tcp_slock_key[1],
86 "sk_lock-AF_INET6-NVME",
87 &nvme_tcp_sk_key[1]);
88 break;
89 default:
90 WARN_ON_ONCE(1);
91 }
92}
93#else
94static void nvme_tcp_reclassify_socket(struct socket *sock) { }
95#endif
96
97enum nvme_tcp_send_state {
98 NVME_TCP_SEND_CMD_PDU = 0,
99 NVME_TCP_SEND_H2C_PDU,
100 NVME_TCP_SEND_DATA,
101 NVME_TCP_SEND_DDGST,
102};
103
104struct nvme_tcp_request {
105 struct nvme_request req;
106 void *pdu;
107 struct nvme_tcp_queue *queue;
108 u32 data_len;
109 u32 pdu_len;
110 u32 pdu_sent;
111 u32 h2cdata_left;
112 u32 h2cdata_offset;
113 u16 ttag;
114 __le16 status;
115 struct list_head entry;
116 struct llist_node lentry;
117 __le32 ddgst;
118
119 struct bio *curr_bio;
120 struct iov_iter iter;
121
122 /* send state */
123 size_t offset;
124 size_t data_sent;
125 enum nvme_tcp_send_state state;
126};
127
128enum nvme_tcp_queue_flags {
129 NVME_TCP_Q_ALLOCATED = 0,
130 NVME_TCP_Q_LIVE = 1,
131 NVME_TCP_Q_POLLING = 2,
132 NVME_TCP_Q_IO_CPU_SET = 3,
133};
134
135enum nvme_tcp_recv_state {
136 NVME_TCP_RECV_PDU = 0,
137 NVME_TCP_RECV_DATA,
138 NVME_TCP_RECV_DDGST,
139};
140
141struct nvme_tcp_ctrl;
142struct nvme_tcp_queue {
143 struct socket *sock;
144 struct work_struct io_work;
145 int io_cpu;
146
147 struct mutex queue_lock;
148 struct mutex send_mutex;
149 struct llist_head req_list;
150 struct list_head send_list;
151
152 /* recv state */
153 void *pdu;
154 int pdu_remaining;
155 int pdu_offset;
156 size_t data_remaining;
157 size_t ddgst_remaining;
158 unsigned int nr_cqe;
159
160 /* send state */
161 struct nvme_tcp_request *request;
162
163 u32 maxh2cdata;
164 size_t cmnd_capsule_len;
165 struct nvme_tcp_ctrl *ctrl;
166 unsigned long flags;
167 bool rd_enabled;
168
169 bool hdr_digest;
170 bool data_digest;
171 bool tls_enabled;
172 struct ahash_request *rcv_hash;
173 struct ahash_request *snd_hash;
174 __le32 exp_ddgst;
175 __le32 recv_ddgst;
176 struct completion tls_complete;
177 int tls_err;
178 struct page_frag_cache pf_cache;
179
180 void (*state_change)(struct sock *);
181 void (*data_ready)(struct sock *);
182 void (*write_space)(struct sock *);
183};
184
185struct nvme_tcp_ctrl {
186 /* read only in the hot path */
187 struct nvme_tcp_queue *queues;
188 struct blk_mq_tag_set tag_set;
189
190 /* other member variables */
191 struct list_head list;
192 struct blk_mq_tag_set admin_tag_set;
193 struct sockaddr_storage addr;
194 struct sockaddr_storage src_addr;
195 struct nvme_ctrl ctrl;
196
197 struct work_struct err_work;
198 struct delayed_work connect_work;
199 struct nvme_tcp_request async_req;
200 u32 io_queues[HCTX_MAX_TYPES];
201};
202
203static LIST_HEAD(nvme_tcp_ctrl_list);
204static DEFINE_MUTEX(nvme_tcp_ctrl_mutex);
205static struct workqueue_struct *nvme_tcp_wq;
206static const struct blk_mq_ops nvme_tcp_mq_ops;
207static const struct blk_mq_ops nvme_tcp_admin_mq_ops;
208static int nvme_tcp_try_send(struct nvme_tcp_queue *queue);
209
210static inline struct nvme_tcp_ctrl *to_tcp_ctrl(struct nvme_ctrl *ctrl)
211{
212 return container_of(ctrl, struct nvme_tcp_ctrl, ctrl);
213}
214
215static inline int nvme_tcp_queue_id(struct nvme_tcp_queue *queue)
216{
217 return queue - queue->ctrl->queues;
218}
219
220static inline bool nvme_tcp_recv_pdu_supported(enum nvme_tcp_pdu_type type)
221{
222 switch (type) {
223 case nvme_tcp_c2h_term:
224 case nvme_tcp_c2h_data:
225 case nvme_tcp_r2t:
226 case nvme_tcp_rsp:
227 return true;
228 default:
229 return false;
230 }
231}
232
233/*
234 * Check if the queue is TLS encrypted
235 */
236static inline bool nvme_tcp_queue_tls(struct nvme_tcp_queue *queue)
237{
238 if (!IS_ENABLED(CONFIG_NVME_TCP_TLS))
239 return 0;
240
241 return queue->tls_enabled;
242}
243
244/*
245 * Check if TLS is configured for the controller.
246 */
247static inline bool nvme_tcp_tls_configured(struct nvme_ctrl *ctrl)
248{
249 if (!IS_ENABLED(CONFIG_NVME_TCP_TLS))
250 return 0;
251
252 return ctrl->opts->tls;
253}
254
255static inline struct blk_mq_tags *nvme_tcp_tagset(struct nvme_tcp_queue *queue)
256{
257 u32 queue_idx = nvme_tcp_queue_id(queue);
258
259 if (queue_idx == 0)
260 return queue->ctrl->admin_tag_set.tags[queue_idx];
261 return queue->ctrl->tag_set.tags[queue_idx - 1];
262}
263
264static inline u8 nvme_tcp_hdgst_len(struct nvme_tcp_queue *queue)
265{
266 return queue->hdr_digest ? NVME_TCP_DIGEST_LENGTH : 0;
267}
268
269static inline u8 nvme_tcp_ddgst_len(struct nvme_tcp_queue *queue)
270{
271 return queue->data_digest ? NVME_TCP_DIGEST_LENGTH : 0;
272}
273
274static inline void *nvme_tcp_req_cmd_pdu(struct nvme_tcp_request *req)
275{
276 return req->pdu;
277}
278
279static inline void *nvme_tcp_req_data_pdu(struct nvme_tcp_request *req)
280{
281 /* use the pdu space in the back for the data pdu */
282 return req->pdu + sizeof(struct nvme_tcp_cmd_pdu) -
283 sizeof(struct nvme_tcp_data_pdu);
284}
285
286static inline size_t nvme_tcp_inline_data_size(struct nvme_tcp_request *req)
287{
288 if (nvme_is_fabrics(req->req.cmd))
289 return NVME_TCP_ADMIN_CCSZ;
290 return req->queue->cmnd_capsule_len - sizeof(struct nvme_command);
291}
292
293static inline bool nvme_tcp_async_req(struct nvme_tcp_request *req)
294{
295 return req == &req->queue->ctrl->async_req;
296}
297
298static inline bool nvme_tcp_has_inline_data(struct nvme_tcp_request *req)
299{
300 struct request *rq;
301
302 if (unlikely(nvme_tcp_async_req(req)))
303 return false; /* async events don't have a request */
304
305 rq = blk_mq_rq_from_pdu(req);
306
307 return rq_data_dir(rq) == WRITE && req->data_len &&
308 req->data_len <= nvme_tcp_inline_data_size(req);
309}
310
311static inline struct page *nvme_tcp_req_cur_page(struct nvme_tcp_request *req)
312{
313 return req->iter.bvec->bv_page;
314}
315
316static inline size_t nvme_tcp_req_cur_offset(struct nvme_tcp_request *req)
317{
318 return req->iter.bvec->bv_offset + req->iter.iov_offset;
319}
320
321static inline size_t nvme_tcp_req_cur_length(struct nvme_tcp_request *req)
322{
323 return min_t(size_t, iov_iter_single_seg_count(&req->iter),
324 req->pdu_len - req->pdu_sent);
325}
326
327static inline size_t nvme_tcp_pdu_data_left(struct nvme_tcp_request *req)
328{
329 return rq_data_dir(blk_mq_rq_from_pdu(req)) == WRITE ?
330 req->pdu_len - req->pdu_sent : 0;
331}
332
333static inline size_t nvme_tcp_pdu_last_send(struct nvme_tcp_request *req,
334 int len)
335{
336 return nvme_tcp_pdu_data_left(req) <= len;
337}
338
339static void nvme_tcp_init_iter(struct nvme_tcp_request *req,
340 unsigned int dir)
341{
342 struct request *rq = blk_mq_rq_from_pdu(req);
343 struct bio_vec *vec;
344 unsigned int size;
345 int nr_bvec;
346 size_t offset;
347
348 if (rq->rq_flags & RQF_SPECIAL_PAYLOAD) {
349 vec = &rq->special_vec;
350 nr_bvec = 1;
351 size = blk_rq_payload_bytes(rq);
352 offset = 0;
353 } else {
354 struct bio *bio = req->curr_bio;
355 struct bvec_iter bi;
356 struct bio_vec bv;
357
358 vec = __bvec_iter_bvec(bio->bi_io_vec, bio->bi_iter);
359 nr_bvec = 0;
360 bio_for_each_bvec(bv, bio, bi) {
361 nr_bvec++;
362 }
363 size = bio->bi_iter.bi_size;
364 offset = bio->bi_iter.bi_bvec_done;
365 }
366
367 iov_iter_bvec(&req->iter, dir, vec, nr_bvec, size);
368 req->iter.iov_offset = offset;
369}
370
371static inline void nvme_tcp_advance_req(struct nvme_tcp_request *req,
372 int len)
373{
374 req->data_sent += len;
375 req->pdu_sent += len;
376 iov_iter_advance(&req->iter, len);
377 if (!iov_iter_count(&req->iter) &&
378 req->data_sent < req->data_len) {
379 req->curr_bio = req->curr_bio->bi_next;
380 nvme_tcp_init_iter(req, ITER_SOURCE);
381 }
382}
383
384static inline void nvme_tcp_send_all(struct nvme_tcp_queue *queue)
385{
386 int ret;
387
388 /* drain the send queue as much as we can... */
389 do {
390 ret = nvme_tcp_try_send(queue);
391 } while (ret > 0);
392}
393
394static inline bool nvme_tcp_queue_has_pending(struct nvme_tcp_queue *queue)
395{
396 return !list_empty(&queue->send_list) ||
397 !llist_empty(&queue->req_list);
398}
399
400static inline bool nvme_tcp_queue_more(struct nvme_tcp_queue *queue)
401{
402 return !nvme_tcp_queue_tls(queue) &&
403 nvme_tcp_queue_has_pending(queue);
404}
405
406static inline void nvme_tcp_queue_request(struct nvme_tcp_request *req,
407 bool sync, bool last)
408{
409 struct nvme_tcp_queue *queue = req->queue;
410 bool empty;
411
412 empty = llist_add(&req->lentry, &queue->req_list) &&
413 list_empty(&queue->send_list) && !queue->request;
414
415 /*
416 * if we're the first on the send_list and we can try to send
417 * directly, otherwise queue io_work. Also, only do that if we
418 * are on the same cpu, so we don't introduce contention.
419 */
420 if (queue->io_cpu == raw_smp_processor_id() &&
421 sync && empty && mutex_trylock(&queue->send_mutex)) {
422 nvme_tcp_send_all(queue);
423 mutex_unlock(&queue->send_mutex);
424 }
425
426 if (last && nvme_tcp_queue_has_pending(queue))
427 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
428}
429
430static void nvme_tcp_process_req_list(struct nvme_tcp_queue *queue)
431{
432 struct nvme_tcp_request *req;
433 struct llist_node *node;
434
435 for (node = llist_del_all(&queue->req_list); node; node = node->next) {
436 req = llist_entry(node, struct nvme_tcp_request, lentry);
437 list_add(&req->entry, &queue->send_list);
438 }
439}
440
441static inline struct nvme_tcp_request *
442nvme_tcp_fetch_request(struct nvme_tcp_queue *queue)
443{
444 struct nvme_tcp_request *req;
445
446 req = list_first_entry_or_null(&queue->send_list,
447 struct nvme_tcp_request, entry);
448 if (!req) {
449 nvme_tcp_process_req_list(queue);
450 req = list_first_entry_or_null(&queue->send_list,
451 struct nvme_tcp_request, entry);
452 if (unlikely(!req))
453 return NULL;
454 }
455
456 list_del(&req->entry);
457 return req;
458}
459
460static inline void nvme_tcp_ddgst_final(struct ahash_request *hash,
461 __le32 *dgst)
462{
463 ahash_request_set_crypt(hash, NULL, (u8 *)dgst, 0);
464 crypto_ahash_final(hash);
465}
466
467static inline void nvme_tcp_ddgst_update(struct ahash_request *hash,
468 struct page *page, off_t off, size_t len)
469{
470 struct scatterlist sg;
471
472 sg_init_table(&sg, 1);
473 sg_set_page(&sg, page, len, off);
474 ahash_request_set_crypt(hash, &sg, NULL, len);
475 crypto_ahash_update(hash);
476}
477
478static inline void nvme_tcp_hdgst(struct ahash_request *hash,
479 void *pdu, size_t len)
480{
481 struct scatterlist sg;
482
483 sg_init_one(&sg, pdu, len);
484 ahash_request_set_crypt(hash, &sg, pdu + len, len);
485 crypto_ahash_digest(hash);
486}
487
488static int nvme_tcp_verify_hdgst(struct nvme_tcp_queue *queue,
489 void *pdu, size_t pdu_len)
490{
491 struct nvme_tcp_hdr *hdr = pdu;
492 __le32 recv_digest;
493 __le32 exp_digest;
494
495 if (unlikely(!(hdr->flags & NVME_TCP_F_HDGST))) {
496 dev_err(queue->ctrl->ctrl.device,
497 "queue %d: header digest flag is cleared\n",
498 nvme_tcp_queue_id(queue));
499 return -EPROTO;
500 }
501
502 recv_digest = *(__le32 *)(pdu + hdr->hlen);
503 nvme_tcp_hdgst(queue->rcv_hash, pdu, pdu_len);
504 exp_digest = *(__le32 *)(pdu + hdr->hlen);
505 if (recv_digest != exp_digest) {
506 dev_err(queue->ctrl->ctrl.device,
507 "header digest error: recv %#x expected %#x\n",
508 le32_to_cpu(recv_digest), le32_to_cpu(exp_digest));
509 return -EIO;
510 }
511
512 return 0;
513}
514
515static int nvme_tcp_check_ddgst(struct nvme_tcp_queue *queue, void *pdu)
516{
517 struct nvme_tcp_hdr *hdr = pdu;
518 u8 digest_len = nvme_tcp_hdgst_len(queue);
519 u32 len;
520
521 len = le32_to_cpu(hdr->plen) - hdr->hlen -
522 ((hdr->flags & NVME_TCP_F_HDGST) ? digest_len : 0);
523
524 if (unlikely(len && !(hdr->flags & NVME_TCP_F_DDGST))) {
525 dev_err(queue->ctrl->ctrl.device,
526 "queue %d: data digest flag is cleared\n",
527 nvme_tcp_queue_id(queue));
528 return -EPROTO;
529 }
530 crypto_ahash_init(queue->rcv_hash);
531
532 return 0;
533}
534
535static void nvme_tcp_exit_request(struct blk_mq_tag_set *set,
536 struct request *rq, unsigned int hctx_idx)
537{
538 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
539
540 page_frag_free(req->pdu);
541}
542
543static int nvme_tcp_init_request(struct blk_mq_tag_set *set,
544 struct request *rq, unsigned int hctx_idx,
545 unsigned int numa_node)
546{
547 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data);
548 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
549 struct nvme_tcp_cmd_pdu *pdu;
550 int queue_idx = (set == &ctrl->tag_set) ? hctx_idx + 1 : 0;
551 struct nvme_tcp_queue *queue = &ctrl->queues[queue_idx];
552 u8 hdgst = nvme_tcp_hdgst_len(queue);
553
554 req->pdu = page_frag_alloc(&queue->pf_cache,
555 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
556 GFP_KERNEL | __GFP_ZERO);
557 if (!req->pdu)
558 return -ENOMEM;
559
560 pdu = req->pdu;
561 req->queue = queue;
562 nvme_req(rq)->ctrl = &ctrl->ctrl;
563 nvme_req(rq)->cmd = &pdu->cmd;
564
565 return 0;
566}
567
568static int nvme_tcp_init_hctx(struct blk_mq_hw_ctx *hctx, void *data,
569 unsigned int hctx_idx)
570{
571 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data);
572 struct nvme_tcp_queue *queue = &ctrl->queues[hctx_idx + 1];
573
574 hctx->driver_data = queue;
575 return 0;
576}
577
578static int nvme_tcp_init_admin_hctx(struct blk_mq_hw_ctx *hctx, void *data,
579 unsigned int hctx_idx)
580{
581 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(data);
582 struct nvme_tcp_queue *queue = &ctrl->queues[0];
583
584 hctx->driver_data = queue;
585 return 0;
586}
587
588static enum nvme_tcp_recv_state
589nvme_tcp_recv_state(struct nvme_tcp_queue *queue)
590{
591 return (queue->pdu_remaining) ? NVME_TCP_RECV_PDU :
592 (queue->ddgst_remaining) ? NVME_TCP_RECV_DDGST :
593 NVME_TCP_RECV_DATA;
594}
595
596static void nvme_tcp_init_recv_ctx(struct nvme_tcp_queue *queue)
597{
598 queue->pdu_remaining = sizeof(struct nvme_tcp_rsp_pdu) +
599 nvme_tcp_hdgst_len(queue);
600 queue->pdu_offset = 0;
601 queue->data_remaining = -1;
602 queue->ddgst_remaining = 0;
603}
604
605static void nvme_tcp_error_recovery(struct nvme_ctrl *ctrl)
606{
607 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_RESETTING))
608 return;
609
610 dev_warn(ctrl->device, "starting error recovery\n");
611 queue_work(nvme_reset_wq, &to_tcp_ctrl(ctrl)->err_work);
612}
613
614static int nvme_tcp_process_nvme_cqe(struct nvme_tcp_queue *queue,
615 struct nvme_completion *cqe)
616{
617 struct nvme_tcp_request *req;
618 struct request *rq;
619
620 rq = nvme_find_rq(nvme_tcp_tagset(queue), cqe->command_id);
621 if (!rq) {
622 dev_err(queue->ctrl->ctrl.device,
623 "got bad cqe.command_id %#x on queue %d\n",
624 cqe->command_id, nvme_tcp_queue_id(queue));
625 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
626 return -EINVAL;
627 }
628
629 req = blk_mq_rq_to_pdu(rq);
630 if (req->status == cpu_to_le16(NVME_SC_SUCCESS))
631 req->status = cqe->status;
632
633 if (!nvme_try_complete_req(rq, req->status, cqe->result))
634 nvme_complete_rq(rq);
635 queue->nr_cqe++;
636
637 return 0;
638}
639
640static int nvme_tcp_handle_c2h_data(struct nvme_tcp_queue *queue,
641 struct nvme_tcp_data_pdu *pdu)
642{
643 struct request *rq;
644
645 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
646 if (!rq) {
647 dev_err(queue->ctrl->ctrl.device,
648 "got bad c2hdata.command_id %#x on queue %d\n",
649 pdu->command_id, nvme_tcp_queue_id(queue));
650 return -ENOENT;
651 }
652
653 if (!blk_rq_payload_bytes(rq)) {
654 dev_err(queue->ctrl->ctrl.device,
655 "queue %d tag %#x unexpected data\n",
656 nvme_tcp_queue_id(queue), rq->tag);
657 return -EIO;
658 }
659
660 queue->data_remaining = le32_to_cpu(pdu->data_length);
661
662 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS &&
663 unlikely(!(pdu->hdr.flags & NVME_TCP_F_DATA_LAST))) {
664 dev_err(queue->ctrl->ctrl.device,
665 "queue %d tag %#x SUCCESS set but not last PDU\n",
666 nvme_tcp_queue_id(queue), rq->tag);
667 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
668 return -EPROTO;
669 }
670
671 return 0;
672}
673
674static int nvme_tcp_handle_comp(struct nvme_tcp_queue *queue,
675 struct nvme_tcp_rsp_pdu *pdu)
676{
677 struct nvme_completion *cqe = &pdu->cqe;
678 int ret = 0;
679
680 /*
681 * AEN requests are special as they don't time out and can
682 * survive any kind of queue freeze and often don't respond to
683 * aborts. We don't even bother to allocate a struct request
684 * for them but rather special case them here.
685 */
686 if (unlikely(nvme_is_aen_req(nvme_tcp_queue_id(queue),
687 cqe->command_id)))
688 nvme_complete_async_event(&queue->ctrl->ctrl, cqe->status,
689 &cqe->result);
690 else
691 ret = nvme_tcp_process_nvme_cqe(queue, cqe);
692
693 return ret;
694}
695
696static void nvme_tcp_setup_h2c_data_pdu(struct nvme_tcp_request *req)
697{
698 struct nvme_tcp_data_pdu *data = nvme_tcp_req_data_pdu(req);
699 struct nvme_tcp_queue *queue = req->queue;
700 struct request *rq = blk_mq_rq_from_pdu(req);
701 u32 h2cdata_sent = req->pdu_len;
702 u8 hdgst = nvme_tcp_hdgst_len(queue);
703 u8 ddgst = nvme_tcp_ddgst_len(queue);
704
705 req->state = NVME_TCP_SEND_H2C_PDU;
706 req->offset = 0;
707 req->pdu_len = min(req->h2cdata_left, queue->maxh2cdata);
708 req->pdu_sent = 0;
709 req->h2cdata_left -= req->pdu_len;
710 req->h2cdata_offset += h2cdata_sent;
711
712 memset(data, 0, sizeof(*data));
713 data->hdr.type = nvme_tcp_h2c_data;
714 if (!req->h2cdata_left)
715 data->hdr.flags = NVME_TCP_F_DATA_LAST;
716 if (queue->hdr_digest)
717 data->hdr.flags |= NVME_TCP_F_HDGST;
718 if (queue->data_digest)
719 data->hdr.flags |= NVME_TCP_F_DDGST;
720 data->hdr.hlen = sizeof(*data);
721 data->hdr.pdo = data->hdr.hlen + hdgst;
722 data->hdr.plen =
723 cpu_to_le32(data->hdr.hlen + hdgst + req->pdu_len + ddgst);
724 data->ttag = req->ttag;
725 data->command_id = nvme_cid(rq);
726 data->data_offset = cpu_to_le32(req->h2cdata_offset);
727 data->data_length = cpu_to_le32(req->pdu_len);
728}
729
730static int nvme_tcp_handle_r2t(struct nvme_tcp_queue *queue,
731 struct nvme_tcp_r2t_pdu *pdu)
732{
733 struct nvme_tcp_request *req;
734 struct request *rq;
735 u32 r2t_length = le32_to_cpu(pdu->r2t_length);
736 u32 r2t_offset = le32_to_cpu(pdu->r2t_offset);
737
738 rq = nvme_find_rq(nvme_tcp_tagset(queue), pdu->command_id);
739 if (!rq) {
740 dev_err(queue->ctrl->ctrl.device,
741 "got bad r2t.command_id %#x on queue %d\n",
742 pdu->command_id, nvme_tcp_queue_id(queue));
743 return -ENOENT;
744 }
745 req = blk_mq_rq_to_pdu(rq);
746
747 if (unlikely(!r2t_length)) {
748 dev_err(queue->ctrl->ctrl.device,
749 "req %d r2t len is %u, probably a bug...\n",
750 rq->tag, r2t_length);
751 return -EPROTO;
752 }
753
754 if (unlikely(req->data_sent + r2t_length > req->data_len)) {
755 dev_err(queue->ctrl->ctrl.device,
756 "req %d r2t len %u exceeded data len %u (%zu sent)\n",
757 rq->tag, r2t_length, req->data_len, req->data_sent);
758 return -EPROTO;
759 }
760
761 if (unlikely(r2t_offset < req->data_sent)) {
762 dev_err(queue->ctrl->ctrl.device,
763 "req %d unexpected r2t offset %u (expected %zu)\n",
764 rq->tag, r2t_offset, req->data_sent);
765 return -EPROTO;
766 }
767
768 req->pdu_len = 0;
769 req->h2cdata_left = r2t_length;
770 req->h2cdata_offset = r2t_offset;
771 req->ttag = pdu->ttag;
772
773 nvme_tcp_setup_h2c_data_pdu(req);
774 nvme_tcp_queue_request(req, false, true);
775
776 return 0;
777}
778
779static void nvme_tcp_handle_c2h_term(struct nvme_tcp_queue *queue,
780 struct nvme_tcp_term_pdu *pdu)
781{
782 u16 fes;
783 const char *msg;
784 u32 plen = le32_to_cpu(pdu->hdr.plen);
785
786 static const char * const msg_table[] = {
787 [NVME_TCP_FES_INVALID_PDU_HDR] = "Invalid PDU Header Field",
788 [NVME_TCP_FES_PDU_SEQ_ERR] = "PDU Sequence Error",
789 [NVME_TCP_FES_HDR_DIGEST_ERR] = "Header Digest Error",
790 [NVME_TCP_FES_DATA_OUT_OF_RANGE] = "Data Transfer Out Of Range",
791 [NVME_TCP_FES_DATA_LIMIT_EXCEEDED] = "Data Transfer Limit Exceeded",
792 [NVME_TCP_FES_UNSUPPORTED_PARAM] = "Unsupported Parameter",
793 };
794
795 if (plen < NVME_TCP_MIN_C2HTERM_PLEN ||
796 plen > NVME_TCP_MAX_C2HTERM_PLEN) {
797 dev_err(queue->ctrl->ctrl.device,
798 "Received a malformed C2HTermReq PDU (plen = %u)\n",
799 plen);
800 return;
801 }
802
803 fes = le16_to_cpu(pdu->fes);
804 if (fes && fes < ARRAY_SIZE(msg_table))
805 msg = msg_table[fes];
806 else
807 msg = "Unknown";
808
809 dev_err(queue->ctrl->ctrl.device,
810 "Received C2HTermReq (FES = %s)\n", msg);
811}
812
813static int nvme_tcp_recv_pdu(struct nvme_tcp_queue *queue, struct sk_buff *skb,
814 unsigned int *offset, size_t *len)
815{
816 struct nvme_tcp_hdr *hdr;
817 char *pdu = queue->pdu;
818 size_t rcv_len = min_t(size_t, *len, queue->pdu_remaining);
819 int ret;
820
821 ret = skb_copy_bits(skb, *offset,
822 &pdu[queue->pdu_offset], rcv_len);
823 if (unlikely(ret))
824 return ret;
825
826 queue->pdu_remaining -= rcv_len;
827 queue->pdu_offset += rcv_len;
828 *offset += rcv_len;
829 *len -= rcv_len;
830 if (queue->pdu_remaining)
831 return 0;
832
833 hdr = queue->pdu;
834 if (unlikely(hdr->hlen != sizeof(struct nvme_tcp_rsp_pdu))) {
835 if (!nvme_tcp_recv_pdu_supported(hdr->type))
836 goto unsupported_pdu;
837
838 dev_err(queue->ctrl->ctrl.device,
839 "pdu type %d has unexpected header length (%d)\n",
840 hdr->type, hdr->hlen);
841 return -EPROTO;
842 }
843
844 if (unlikely(hdr->type == nvme_tcp_c2h_term)) {
845 /*
846 * C2HTermReq never includes Header or Data digests.
847 * Skip the checks.
848 */
849 nvme_tcp_handle_c2h_term(queue, (void *)queue->pdu);
850 return -EINVAL;
851 }
852
853 if (queue->hdr_digest) {
854 ret = nvme_tcp_verify_hdgst(queue, queue->pdu, hdr->hlen);
855 if (unlikely(ret))
856 return ret;
857 }
858
859
860 if (queue->data_digest) {
861 ret = nvme_tcp_check_ddgst(queue, queue->pdu);
862 if (unlikely(ret))
863 return ret;
864 }
865
866 switch (hdr->type) {
867 case nvme_tcp_c2h_data:
868 return nvme_tcp_handle_c2h_data(queue, (void *)queue->pdu);
869 case nvme_tcp_rsp:
870 nvme_tcp_init_recv_ctx(queue);
871 return nvme_tcp_handle_comp(queue, (void *)queue->pdu);
872 case nvme_tcp_r2t:
873 nvme_tcp_init_recv_ctx(queue);
874 return nvme_tcp_handle_r2t(queue, (void *)queue->pdu);
875 default:
876 goto unsupported_pdu;
877 }
878
879unsupported_pdu:
880 dev_err(queue->ctrl->ctrl.device,
881 "unsupported pdu type (%d)\n", hdr->type);
882 return -EINVAL;
883}
884
885static inline void nvme_tcp_end_request(struct request *rq, u16 status)
886{
887 union nvme_result res = {};
888
889 if (!nvme_try_complete_req(rq, cpu_to_le16(status << 1), res))
890 nvme_complete_rq(rq);
891}
892
893static int nvme_tcp_recv_data(struct nvme_tcp_queue *queue, struct sk_buff *skb,
894 unsigned int *offset, size_t *len)
895{
896 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
897 struct request *rq =
898 nvme_cid_to_rq(nvme_tcp_tagset(queue), pdu->command_id);
899 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
900
901 while (true) {
902 int recv_len, ret;
903
904 recv_len = min_t(size_t, *len, queue->data_remaining);
905 if (!recv_len)
906 break;
907
908 if (!iov_iter_count(&req->iter)) {
909 req->curr_bio = req->curr_bio->bi_next;
910
911 /*
912 * If we don`t have any bios it means that controller
913 * sent more data than we requested, hence error
914 */
915 if (!req->curr_bio) {
916 dev_err(queue->ctrl->ctrl.device,
917 "queue %d no space in request %#x",
918 nvme_tcp_queue_id(queue), rq->tag);
919 nvme_tcp_init_recv_ctx(queue);
920 return -EIO;
921 }
922 nvme_tcp_init_iter(req, ITER_DEST);
923 }
924
925 /* we can read only from what is left in this bio */
926 recv_len = min_t(size_t, recv_len,
927 iov_iter_count(&req->iter));
928
929 if (queue->data_digest)
930 ret = skb_copy_and_hash_datagram_iter(skb, *offset,
931 &req->iter, recv_len, queue->rcv_hash);
932 else
933 ret = skb_copy_datagram_iter(skb, *offset,
934 &req->iter, recv_len);
935 if (ret) {
936 dev_err(queue->ctrl->ctrl.device,
937 "queue %d failed to copy request %#x data",
938 nvme_tcp_queue_id(queue), rq->tag);
939 return ret;
940 }
941
942 *len -= recv_len;
943 *offset += recv_len;
944 queue->data_remaining -= recv_len;
945 }
946
947 if (!queue->data_remaining) {
948 if (queue->data_digest) {
949 nvme_tcp_ddgst_final(queue->rcv_hash, &queue->exp_ddgst);
950 queue->ddgst_remaining = NVME_TCP_DIGEST_LENGTH;
951 } else {
952 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
953 nvme_tcp_end_request(rq,
954 le16_to_cpu(req->status));
955 queue->nr_cqe++;
956 }
957 nvme_tcp_init_recv_ctx(queue);
958 }
959 }
960
961 return 0;
962}
963
964static int nvme_tcp_recv_ddgst(struct nvme_tcp_queue *queue,
965 struct sk_buff *skb, unsigned int *offset, size_t *len)
966{
967 struct nvme_tcp_data_pdu *pdu = (void *)queue->pdu;
968 char *ddgst = (char *)&queue->recv_ddgst;
969 size_t recv_len = min_t(size_t, *len, queue->ddgst_remaining);
970 off_t off = NVME_TCP_DIGEST_LENGTH - queue->ddgst_remaining;
971 int ret;
972
973 ret = skb_copy_bits(skb, *offset, &ddgst[off], recv_len);
974 if (unlikely(ret))
975 return ret;
976
977 queue->ddgst_remaining -= recv_len;
978 *offset += recv_len;
979 *len -= recv_len;
980 if (queue->ddgst_remaining)
981 return 0;
982
983 if (queue->recv_ddgst != queue->exp_ddgst) {
984 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
985 pdu->command_id);
986 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
987
988 req->status = cpu_to_le16(NVME_SC_DATA_XFER_ERROR);
989
990 dev_err(queue->ctrl->ctrl.device,
991 "data digest error: recv %#x expected %#x\n",
992 le32_to_cpu(queue->recv_ddgst),
993 le32_to_cpu(queue->exp_ddgst));
994 }
995
996 if (pdu->hdr.flags & NVME_TCP_F_DATA_SUCCESS) {
997 struct request *rq = nvme_cid_to_rq(nvme_tcp_tagset(queue),
998 pdu->command_id);
999 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
1000
1001 nvme_tcp_end_request(rq, le16_to_cpu(req->status));
1002 queue->nr_cqe++;
1003 }
1004
1005 nvme_tcp_init_recv_ctx(queue);
1006 return 0;
1007}
1008
1009static int nvme_tcp_recv_skb(read_descriptor_t *desc, struct sk_buff *skb,
1010 unsigned int offset, size_t len)
1011{
1012 struct nvme_tcp_queue *queue = desc->arg.data;
1013 size_t consumed = len;
1014 int result;
1015
1016 if (unlikely(!queue->rd_enabled))
1017 return -EFAULT;
1018
1019 while (len) {
1020 switch (nvme_tcp_recv_state(queue)) {
1021 case NVME_TCP_RECV_PDU:
1022 result = nvme_tcp_recv_pdu(queue, skb, &offset, &len);
1023 break;
1024 case NVME_TCP_RECV_DATA:
1025 result = nvme_tcp_recv_data(queue, skb, &offset, &len);
1026 break;
1027 case NVME_TCP_RECV_DDGST:
1028 result = nvme_tcp_recv_ddgst(queue, skb, &offset, &len);
1029 break;
1030 default:
1031 result = -EFAULT;
1032 }
1033 if (result) {
1034 dev_err(queue->ctrl->ctrl.device,
1035 "receive failed: %d\n", result);
1036 queue->rd_enabled = false;
1037 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
1038 return result;
1039 }
1040 }
1041
1042 return consumed;
1043}
1044
1045static void nvme_tcp_data_ready(struct sock *sk)
1046{
1047 struct nvme_tcp_queue *queue;
1048
1049 trace_sk_data_ready(sk);
1050
1051 read_lock_bh(&sk->sk_callback_lock);
1052 queue = sk->sk_user_data;
1053 if (likely(queue && queue->rd_enabled) &&
1054 !test_bit(NVME_TCP_Q_POLLING, &queue->flags))
1055 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1056 read_unlock_bh(&sk->sk_callback_lock);
1057}
1058
1059static void nvme_tcp_write_space(struct sock *sk)
1060{
1061 struct nvme_tcp_queue *queue;
1062
1063 read_lock_bh(&sk->sk_callback_lock);
1064 queue = sk->sk_user_data;
1065 if (likely(queue && sk_stream_is_writeable(sk))) {
1066 clear_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1067 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1068 }
1069 read_unlock_bh(&sk->sk_callback_lock);
1070}
1071
1072static void nvme_tcp_state_change(struct sock *sk)
1073{
1074 struct nvme_tcp_queue *queue;
1075
1076 read_lock_bh(&sk->sk_callback_lock);
1077 queue = sk->sk_user_data;
1078 if (!queue)
1079 goto done;
1080
1081 switch (sk->sk_state) {
1082 case TCP_CLOSE:
1083 case TCP_CLOSE_WAIT:
1084 case TCP_LAST_ACK:
1085 case TCP_FIN_WAIT1:
1086 case TCP_FIN_WAIT2:
1087 nvme_tcp_error_recovery(&queue->ctrl->ctrl);
1088 break;
1089 default:
1090 dev_info(queue->ctrl->ctrl.device,
1091 "queue %d socket state %d\n",
1092 nvme_tcp_queue_id(queue), sk->sk_state);
1093 }
1094
1095 queue->state_change(sk);
1096done:
1097 read_unlock_bh(&sk->sk_callback_lock);
1098}
1099
1100static inline void nvme_tcp_done_send_req(struct nvme_tcp_queue *queue)
1101{
1102 queue->request = NULL;
1103}
1104
1105static void nvme_tcp_fail_request(struct nvme_tcp_request *req)
1106{
1107 if (nvme_tcp_async_req(req)) {
1108 union nvme_result res = {};
1109
1110 nvme_complete_async_event(&req->queue->ctrl->ctrl,
1111 cpu_to_le16(NVME_SC_HOST_PATH_ERROR), &res);
1112 } else {
1113 nvme_tcp_end_request(blk_mq_rq_from_pdu(req),
1114 NVME_SC_HOST_PATH_ERROR);
1115 }
1116}
1117
1118static int nvme_tcp_try_send_data(struct nvme_tcp_request *req)
1119{
1120 struct nvme_tcp_queue *queue = req->queue;
1121 int req_data_len = req->data_len;
1122 u32 h2cdata_left = req->h2cdata_left;
1123
1124 while (true) {
1125 struct bio_vec bvec;
1126 struct msghdr msg = {
1127 .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES,
1128 };
1129 struct page *page = nvme_tcp_req_cur_page(req);
1130 size_t offset = nvme_tcp_req_cur_offset(req);
1131 size_t len = nvme_tcp_req_cur_length(req);
1132 bool last = nvme_tcp_pdu_last_send(req, len);
1133 int req_data_sent = req->data_sent;
1134 int ret;
1135
1136 if (last && !queue->data_digest && !nvme_tcp_queue_more(queue))
1137 msg.msg_flags |= MSG_EOR;
1138 else
1139 msg.msg_flags |= MSG_MORE;
1140
1141 if (!sendpages_ok(page, len, offset))
1142 msg.msg_flags &= ~MSG_SPLICE_PAGES;
1143
1144 bvec_set_page(&bvec, page, len, offset);
1145 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1146 ret = sock_sendmsg(queue->sock, &msg);
1147 if (ret <= 0)
1148 return ret;
1149
1150 if (queue->data_digest)
1151 nvme_tcp_ddgst_update(queue->snd_hash, page,
1152 offset, ret);
1153
1154 /*
1155 * update the request iterator except for the last payload send
1156 * in the request where we don't want to modify it as we may
1157 * compete with the RX path completing the request.
1158 */
1159 if (req_data_sent + ret < req_data_len)
1160 nvme_tcp_advance_req(req, ret);
1161
1162 /* fully successful last send in current PDU */
1163 if (last && ret == len) {
1164 if (queue->data_digest) {
1165 nvme_tcp_ddgst_final(queue->snd_hash,
1166 &req->ddgst);
1167 req->state = NVME_TCP_SEND_DDGST;
1168 req->offset = 0;
1169 } else {
1170 if (h2cdata_left)
1171 nvme_tcp_setup_h2c_data_pdu(req);
1172 else
1173 nvme_tcp_done_send_req(queue);
1174 }
1175 return 1;
1176 }
1177 }
1178 return -EAGAIN;
1179}
1180
1181static int nvme_tcp_try_send_cmd_pdu(struct nvme_tcp_request *req)
1182{
1183 struct nvme_tcp_queue *queue = req->queue;
1184 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
1185 struct bio_vec bvec;
1186 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_SPLICE_PAGES, };
1187 bool inline_data = nvme_tcp_has_inline_data(req);
1188 u8 hdgst = nvme_tcp_hdgst_len(queue);
1189 int len = sizeof(*pdu) + hdgst - req->offset;
1190 int ret;
1191
1192 if (inline_data || nvme_tcp_queue_more(queue))
1193 msg.msg_flags |= MSG_MORE;
1194 else
1195 msg.msg_flags |= MSG_EOR;
1196
1197 if (queue->hdr_digest && !req->offset)
1198 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1199
1200 bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
1201 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1202 ret = sock_sendmsg(queue->sock, &msg);
1203 if (unlikely(ret <= 0))
1204 return ret;
1205
1206 len -= ret;
1207 if (!len) {
1208 if (inline_data) {
1209 req->state = NVME_TCP_SEND_DATA;
1210 if (queue->data_digest)
1211 crypto_ahash_init(queue->snd_hash);
1212 } else {
1213 nvme_tcp_done_send_req(queue);
1214 }
1215 return 1;
1216 }
1217 req->offset += ret;
1218
1219 return -EAGAIN;
1220}
1221
1222static int nvme_tcp_try_send_data_pdu(struct nvme_tcp_request *req)
1223{
1224 struct nvme_tcp_queue *queue = req->queue;
1225 struct nvme_tcp_data_pdu *pdu = nvme_tcp_req_data_pdu(req);
1226 struct bio_vec bvec;
1227 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_MORE, };
1228 u8 hdgst = nvme_tcp_hdgst_len(queue);
1229 int len = sizeof(*pdu) - req->offset + hdgst;
1230 int ret;
1231
1232 if (queue->hdr_digest && !req->offset)
1233 nvme_tcp_hdgst(queue->snd_hash, pdu, sizeof(*pdu));
1234
1235 if (!req->h2cdata_left)
1236 msg.msg_flags |= MSG_SPLICE_PAGES;
1237
1238 bvec_set_virt(&bvec, (void *)pdu + req->offset, len);
1239 iov_iter_bvec(&msg.msg_iter, ITER_SOURCE, &bvec, 1, len);
1240 ret = sock_sendmsg(queue->sock, &msg);
1241 if (unlikely(ret <= 0))
1242 return ret;
1243
1244 len -= ret;
1245 if (!len) {
1246 req->state = NVME_TCP_SEND_DATA;
1247 if (queue->data_digest)
1248 crypto_ahash_init(queue->snd_hash);
1249 return 1;
1250 }
1251 req->offset += ret;
1252
1253 return -EAGAIN;
1254}
1255
1256static int nvme_tcp_try_send_ddgst(struct nvme_tcp_request *req)
1257{
1258 struct nvme_tcp_queue *queue = req->queue;
1259 size_t offset = req->offset;
1260 u32 h2cdata_left = req->h2cdata_left;
1261 int ret;
1262 struct msghdr msg = { .msg_flags = MSG_DONTWAIT };
1263 struct kvec iov = {
1264 .iov_base = (u8 *)&req->ddgst + req->offset,
1265 .iov_len = NVME_TCP_DIGEST_LENGTH - req->offset
1266 };
1267
1268 if (nvme_tcp_queue_more(queue))
1269 msg.msg_flags |= MSG_MORE;
1270 else
1271 msg.msg_flags |= MSG_EOR;
1272
1273 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1274 if (unlikely(ret <= 0))
1275 return ret;
1276
1277 if (offset + ret == NVME_TCP_DIGEST_LENGTH) {
1278 if (h2cdata_left)
1279 nvme_tcp_setup_h2c_data_pdu(req);
1280 else
1281 nvme_tcp_done_send_req(queue);
1282 return 1;
1283 }
1284
1285 req->offset += ret;
1286 return -EAGAIN;
1287}
1288
1289static int nvme_tcp_try_send(struct nvme_tcp_queue *queue)
1290{
1291 struct nvme_tcp_request *req;
1292 unsigned int noreclaim_flag;
1293 int ret = 1;
1294
1295 if (!queue->request) {
1296 queue->request = nvme_tcp_fetch_request(queue);
1297 if (!queue->request)
1298 return 0;
1299 }
1300 req = queue->request;
1301
1302 noreclaim_flag = memalloc_noreclaim_save();
1303 if (req->state == NVME_TCP_SEND_CMD_PDU) {
1304 ret = nvme_tcp_try_send_cmd_pdu(req);
1305 if (ret <= 0)
1306 goto done;
1307 if (!nvme_tcp_has_inline_data(req))
1308 goto out;
1309 }
1310
1311 if (req->state == NVME_TCP_SEND_H2C_PDU) {
1312 ret = nvme_tcp_try_send_data_pdu(req);
1313 if (ret <= 0)
1314 goto done;
1315 }
1316
1317 if (req->state == NVME_TCP_SEND_DATA) {
1318 ret = nvme_tcp_try_send_data(req);
1319 if (ret <= 0)
1320 goto done;
1321 }
1322
1323 if (req->state == NVME_TCP_SEND_DDGST)
1324 ret = nvme_tcp_try_send_ddgst(req);
1325done:
1326 if (ret == -EAGAIN) {
1327 ret = 0;
1328 } else if (ret < 0) {
1329 dev_err(queue->ctrl->ctrl.device,
1330 "failed to send request %d\n", ret);
1331 nvme_tcp_fail_request(queue->request);
1332 nvme_tcp_done_send_req(queue);
1333 }
1334out:
1335 memalloc_noreclaim_restore(noreclaim_flag);
1336 return ret;
1337}
1338
1339static int nvme_tcp_try_recv(struct nvme_tcp_queue *queue)
1340{
1341 struct socket *sock = queue->sock;
1342 struct sock *sk = sock->sk;
1343 read_descriptor_t rd_desc;
1344 int consumed;
1345
1346 rd_desc.arg.data = queue;
1347 rd_desc.count = 1;
1348 lock_sock(sk);
1349 queue->nr_cqe = 0;
1350 consumed = sock->ops->read_sock(sk, &rd_desc, nvme_tcp_recv_skb);
1351 release_sock(sk);
1352 return consumed;
1353}
1354
1355static void nvme_tcp_io_work(struct work_struct *w)
1356{
1357 struct nvme_tcp_queue *queue =
1358 container_of(w, struct nvme_tcp_queue, io_work);
1359 unsigned long deadline = jiffies + msecs_to_jiffies(1);
1360
1361 do {
1362 bool pending = false;
1363 int result;
1364
1365 if (mutex_trylock(&queue->send_mutex)) {
1366 result = nvme_tcp_try_send(queue);
1367 mutex_unlock(&queue->send_mutex);
1368 if (result > 0)
1369 pending = true;
1370 else if (unlikely(result < 0))
1371 break;
1372 }
1373
1374 result = nvme_tcp_try_recv(queue);
1375 if (result > 0)
1376 pending = true;
1377 else if (unlikely(result < 0))
1378 return;
1379
1380 if (!pending || !queue->rd_enabled)
1381 return;
1382
1383 } while (!time_after(jiffies, deadline)); /* quota is exhausted */
1384
1385 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
1386}
1387
1388static void nvme_tcp_free_crypto(struct nvme_tcp_queue *queue)
1389{
1390 struct crypto_ahash *tfm = crypto_ahash_reqtfm(queue->rcv_hash);
1391
1392 ahash_request_free(queue->rcv_hash);
1393 ahash_request_free(queue->snd_hash);
1394 crypto_free_ahash(tfm);
1395}
1396
1397static int nvme_tcp_alloc_crypto(struct nvme_tcp_queue *queue)
1398{
1399 struct crypto_ahash *tfm;
1400
1401 tfm = crypto_alloc_ahash("crc32c", 0, CRYPTO_ALG_ASYNC);
1402 if (IS_ERR(tfm))
1403 return PTR_ERR(tfm);
1404
1405 queue->snd_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1406 if (!queue->snd_hash)
1407 goto free_tfm;
1408 ahash_request_set_callback(queue->snd_hash, 0, NULL, NULL);
1409
1410 queue->rcv_hash = ahash_request_alloc(tfm, GFP_KERNEL);
1411 if (!queue->rcv_hash)
1412 goto free_snd_hash;
1413 ahash_request_set_callback(queue->rcv_hash, 0, NULL, NULL);
1414
1415 return 0;
1416free_snd_hash:
1417 ahash_request_free(queue->snd_hash);
1418free_tfm:
1419 crypto_free_ahash(tfm);
1420 return -ENOMEM;
1421}
1422
1423static void nvme_tcp_free_async_req(struct nvme_tcp_ctrl *ctrl)
1424{
1425 struct nvme_tcp_request *async = &ctrl->async_req;
1426
1427 page_frag_free(async->pdu);
1428}
1429
1430static int nvme_tcp_alloc_async_req(struct nvme_tcp_ctrl *ctrl)
1431{
1432 struct nvme_tcp_queue *queue = &ctrl->queues[0];
1433 struct nvme_tcp_request *async = &ctrl->async_req;
1434 u8 hdgst = nvme_tcp_hdgst_len(queue);
1435
1436 async->pdu = page_frag_alloc(&queue->pf_cache,
1437 sizeof(struct nvme_tcp_cmd_pdu) + hdgst,
1438 GFP_KERNEL | __GFP_ZERO);
1439 if (!async->pdu)
1440 return -ENOMEM;
1441
1442 async->queue = &ctrl->queues[0];
1443 return 0;
1444}
1445
1446static void nvme_tcp_free_queue(struct nvme_ctrl *nctrl, int qid)
1447{
1448 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1449 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1450 unsigned int noreclaim_flag;
1451
1452 if (!test_and_clear_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1453 return;
1454
1455 if (queue->hdr_digest || queue->data_digest)
1456 nvme_tcp_free_crypto(queue);
1457
1458 page_frag_cache_drain(&queue->pf_cache);
1459
1460 noreclaim_flag = memalloc_noreclaim_save();
1461 /* ->sock will be released by fput() */
1462 fput(queue->sock->file);
1463 queue->sock = NULL;
1464 memalloc_noreclaim_restore(noreclaim_flag);
1465
1466 kfree(queue->pdu);
1467 mutex_destroy(&queue->send_mutex);
1468 mutex_destroy(&queue->queue_lock);
1469}
1470
1471static int nvme_tcp_init_connection(struct nvme_tcp_queue *queue)
1472{
1473 struct nvme_tcp_icreq_pdu *icreq;
1474 struct nvme_tcp_icresp_pdu *icresp;
1475 char cbuf[CMSG_LEN(sizeof(char))] = {};
1476 u8 ctype;
1477 struct msghdr msg = {};
1478 struct kvec iov;
1479 bool ctrl_hdgst, ctrl_ddgst;
1480 u32 maxh2cdata;
1481 int ret;
1482
1483 icreq = kzalloc(sizeof(*icreq), GFP_KERNEL);
1484 if (!icreq)
1485 return -ENOMEM;
1486
1487 icresp = kzalloc(sizeof(*icresp), GFP_KERNEL);
1488 if (!icresp) {
1489 ret = -ENOMEM;
1490 goto free_icreq;
1491 }
1492
1493 icreq->hdr.type = nvme_tcp_icreq;
1494 icreq->hdr.hlen = sizeof(*icreq);
1495 icreq->hdr.pdo = 0;
1496 icreq->hdr.plen = cpu_to_le32(icreq->hdr.hlen);
1497 icreq->pfv = cpu_to_le16(NVME_TCP_PFV_1_0);
1498 icreq->maxr2t = 0; /* single inflight r2t supported */
1499 icreq->hpda = 0; /* no alignment constraint */
1500 if (queue->hdr_digest)
1501 icreq->digest |= NVME_TCP_HDR_DIGEST_ENABLE;
1502 if (queue->data_digest)
1503 icreq->digest |= NVME_TCP_DATA_DIGEST_ENABLE;
1504
1505 iov.iov_base = icreq;
1506 iov.iov_len = sizeof(*icreq);
1507 ret = kernel_sendmsg(queue->sock, &msg, &iov, 1, iov.iov_len);
1508 if (ret < 0) {
1509 pr_warn("queue %d: failed to send icreq, error %d\n",
1510 nvme_tcp_queue_id(queue), ret);
1511 goto free_icresp;
1512 }
1513
1514 memset(&msg, 0, sizeof(msg));
1515 iov.iov_base = icresp;
1516 iov.iov_len = sizeof(*icresp);
1517 if (nvme_tcp_queue_tls(queue)) {
1518 msg.msg_control = cbuf;
1519 msg.msg_controllen = sizeof(cbuf);
1520 }
1521 msg.msg_flags = MSG_WAITALL;
1522 ret = kernel_recvmsg(queue->sock, &msg, &iov, 1,
1523 iov.iov_len, msg.msg_flags);
1524 if (ret >= 0 && ret < sizeof(*icresp))
1525 ret = -ECONNRESET;
1526 if (ret < 0) {
1527 pr_warn("queue %d: failed to receive icresp, error %d\n",
1528 nvme_tcp_queue_id(queue), ret);
1529 goto free_icresp;
1530 }
1531 ret = -ENOTCONN;
1532 if (nvme_tcp_queue_tls(queue)) {
1533 ctype = tls_get_record_type(queue->sock->sk,
1534 (struct cmsghdr *)cbuf);
1535 if (ctype != TLS_RECORD_TYPE_DATA) {
1536 pr_err("queue %d: unhandled TLS record %d\n",
1537 nvme_tcp_queue_id(queue), ctype);
1538 goto free_icresp;
1539 }
1540 }
1541 ret = -EINVAL;
1542 if (icresp->hdr.type != nvme_tcp_icresp) {
1543 pr_err("queue %d: bad type returned %d\n",
1544 nvme_tcp_queue_id(queue), icresp->hdr.type);
1545 goto free_icresp;
1546 }
1547
1548 if (le32_to_cpu(icresp->hdr.plen) != sizeof(*icresp)) {
1549 pr_err("queue %d: bad pdu length returned %d\n",
1550 nvme_tcp_queue_id(queue), icresp->hdr.plen);
1551 goto free_icresp;
1552 }
1553
1554 if (icresp->pfv != NVME_TCP_PFV_1_0) {
1555 pr_err("queue %d: bad pfv returned %d\n",
1556 nvme_tcp_queue_id(queue), icresp->pfv);
1557 goto free_icresp;
1558 }
1559
1560 ctrl_ddgst = !!(icresp->digest & NVME_TCP_DATA_DIGEST_ENABLE);
1561 if ((queue->data_digest && !ctrl_ddgst) ||
1562 (!queue->data_digest && ctrl_ddgst)) {
1563 pr_err("queue %d: data digest mismatch host: %s ctrl: %s\n",
1564 nvme_tcp_queue_id(queue),
1565 queue->data_digest ? "enabled" : "disabled",
1566 ctrl_ddgst ? "enabled" : "disabled");
1567 goto free_icresp;
1568 }
1569
1570 ctrl_hdgst = !!(icresp->digest & NVME_TCP_HDR_DIGEST_ENABLE);
1571 if ((queue->hdr_digest && !ctrl_hdgst) ||
1572 (!queue->hdr_digest && ctrl_hdgst)) {
1573 pr_err("queue %d: header digest mismatch host: %s ctrl: %s\n",
1574 nvme_tcp_queue_id(queue),
1575 queue->hdr_digest ? "enabled" : "disabled",
1576 ctrl_hdgst ? "enabled" : "disabled");
1577 goto free_icresp;
1578 }
1579
1580 if (icresp->cpda != 0) {
1581 pr_err("queue %d: unsupported cpda returned %d\n",
1582 nvme_tcp_queue_id(queue), icresp->cpda);
1583 goto free_icresp;
1584 }
1585
1586 maxh2cdata = le32_to_cpu(icresp->maxdata);
1587 if ((maxh2cdata % 4) || (maxh2cdata < NVME_TCP_MIN_MAXH2CDATA)) {
1588 pr_err("queue %d: invalid maxh2cdata returned %u\n",
1589 nvme_tcp_queue_id(queue), maxh2cdata);
1590 goto free_icresp;
1591 }
1592 queue->maxh2cdata = maxh2cdata;
1593
1594 ret = 0;
1595free_icresp:
1596 kfree(icresp);
1597free_icreq:
1598 kfree(icreq);
1599 return ret;
1600}
1601
1602static bool nvme_tcp_admin_queue(struct nvme_tcp_queue *queue)
1603{
1604 return nvme_tcp_queue_id(queue) == 0;
1605}
1606
1607static bool nvme_tcp_default_queue(struct nvme_tcp_queue *queue)
1608{
1609 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1610 int qid = nvme_tcp_queue_id(queue);
1611
1612 return !nvme_tcp_admin_queue(queue) &&
1613 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT];
1614}
1615
1616static bool nvme_tcp_read_queue(struct nvme_tcp_queue *queue)
1617{
1618 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1619 int qid = nvme_tcp_queue_id(queue);
1620
1621 return !nvme_tcp_admin_queue(queue) &&
1622 !nvme_tcp_default_queue(queue) &&
1623 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1624 ctrl->io_queues[HCTX_TYPE_READ];
1625}
1626
1627static bool nvme_tcp_poll_queue(struct nvme_tcp_queue *queue)
1628{
1629 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1630 int qid = nvme_tcp_queue_id(queue);
1631
1632 return !nvme_tcp_admin_queue(queue) &&
1633 !nvme_tcp_default_queue(queue) &&
1634 !nvme_tcp_read_queue(queue) &&
1635 qid < 1 + ctrl->io_queues[HCTX_TYPE_DEFAULT] +
1636 ctrl->io_queues[HCTX_TYPE_READ] +
1637 ctrl->io_queues[HCTX_TYPE_POLL];
1638}
1639
1640/*
1641 * Track the number of queues assigned to each cpu using a global per-cpu
1642 * counter and select the least used cpu from the mq_map. Our goal is to spread
1643 * different controllers I/O threads across different cpu cores.
1644 *
1645 * Note that the accounting is not 100% perfect, but we don't need to be, we're
1646 * simply putting our best effort to select the best candidate cpu core that we
1647 * find at any given point.
1648 */
1649static void nvme_tcp_set_queue_io_cpu(struct nvme_tcp_queue *queue)
1650{
1651 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1652 struct blk_mq_tag_set *set = &ctrl->tag_set;
1653 int qid = nvme_tcp_queue_id(queue) - 1;
1654 unsigned int *mq_map = NULL;
1655 int cpu, min_queues = INT_MAX, io_cpu;
1656
1657 if (wq_unbound)
1658 goto out;
1659
1660 if (nvme_tcp_default_queue(queue))
1661 mq_map = set->map[HCTX_TYPE_DEFAULT].mq_map;
1662 else if (nvme_tcp_read_queue(queue))
1663 mq_map = set->map[HCTX_TYPE_READ].mq_map;
1664 else if (nvme_tcp_poll_queue(queue))
1665 mq_map = set->map[HCTX_TYPE_POLL].mq_map;
1666
1667 if (WARN_ON(!mq_map))
1668 goto out;
1669
1670 /* Search for the least used cpu from the mq_map */
1671 io_cpu = WORK_CPU_UNBOUND;
1672 for_each_online_cpu(cpu) {
1673 int num_queues = atomic_read(&nvme_tcp_cpu_queues[cpu]);
1674
1675 if (mq_map[cpu] != qid)
1676 continue;
1677 if (num_queues < min_queues) {
1678 io_cpu = cpu;
1679 min_queues = num_queues;
1680 }
1681 }
1682 if (io_cpu != WORK_CPU_UNBOUND) {
1683 queue->io_cpu = io_cpu;
1684 atomic_inc(&nvme_tcp_cpu_queues[io_cpu]);
1685 set_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags);
1686 }
1687out:
1688 dev_dbg(ctrl->ctrl.device, "queue %d: using cpu %d\n",
1689 qid, queue->io_cpu);
1690}
1691
1692static void nvme_tcp_tls_done(void *data, int status, key_serial_t pskid)
1693{
1694 struct nvme_tcp_queue *queue = data;
1695 struct nvme_tcp_ctrl *ctrl = queue->ctrl;
1696 int qid = nvme_tcp_queue_id(queue);
1697 struct key *tls_key;
1698
1699 dev_dbg(ctrl->ctrl.device, "queue %d: TLS handshake done, key %x, status %d\n",
1700 qid, pskid, status);
1701
1702 if (status) {
1703 queue->tls_err = -status;
1704 goto out_complete;
1705 }
1706
1707 tls_key = nvme_tls_key_lookup(pskid);
1708 if (IS_ERR(tls_key)) {
1709 dev_warn(ctrl->ctrl.device, "queue %d: Invalid key %x\n",
1710 qid, pskid);
1711 queue->tls_err = -ENOKEY;
1712 } else {
1713 queue->tls_enabled = true;
1714 if (qid == 0)
1715 ctrl->ctrl.tls_pskid = key_serial(tls_key);
1716 key_put(tls_key);
1717 queue->tls_err = 0;
1718 }
1719
1720out_complete:
1721 complete(&queue->tls_complete);
1722}
1723
1724static int nvme_tcp_start_tls(struct nvme_ctrl *nctrl,
1725 struct nvme_tcp_queue *queue,
1726 key_serial_t pskid)
1727{
1728 int qid = nvme_tcp_queue_id(queue);
1729 int ret;
1730 struct tls_handshake_args args;
1731 unsigned long tmo = tls_handshake_timeout * HZ;
1732 key_serial_t keyring = nvme_keyring_id();
1733
1734 dev_dbg(nctrl->device, "queue %d: start TLS with key %x\n",
1735 qid, pskid);
1736 memset(&args, 0, sizeof(args));
1737 args.ta_sock = queue->sock;
1738 args.ta_done = nvme_tcp_tls_done;
1739 args.ta_data = queue;
1740 args.ta_my_peerids[0] = pskid;
1741 args.ta_num_peerids = 1;
1742 if (nctrl->opts->keyring)
1743 keyring = key_serial(nctrl->opts->keyring);
1744 args.ta_keyring = keyring;
1745 args.ta_timeout_ms = tls_handshake_timeout * 1000;
1746 queue->tls_err = -EOPNOTSUPP;
1747 init_completion(&queue->tls_complete);
1748 ret = tls_client_hello_psk(&args, GFP_KERNEL);
1749 if (ret) {
1750 dev_err(nctrl->device, "queue %d: failed to start TLS: %d\n",
1751 qid, ret);
1752 return ret;
1753 }
1754 ret = wait_for_completion_interruptible_timeout(&queue->tls_complete, tmo);
1755 if (ret <= 0) {
1756 if (ret == 0)
1757 ret = -ETIMEDOUT;
1758
1759 dev_err(nctrl->device,
1760 "queue %d: TLS handshake failed, error %d\n",
1761 qid, ret);
1762 tls_handshake_cancel(queue->sock->sk);
1763 } else {
1764 dev_dbg(nctrl->device,
1765 "queue %d: TLS handshake complete, error %d\n",
1766 qid, queue->tls_err);
1767 ret = queue->tls_err;
1768 }
1769 return ret;
1770}
1771
1772static int nvme_tcp_alloc_queue(struct nvme_ctrl *nctrl, int qid,
1773 key_serial_t pskid)
1774{
1775 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1776 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1777 int ret, rcv_pdu_size;
1778 struct file *sock_file;
1779
1780 mutex_init(&queue->queue_lock);
1781 queue->ctrl = ctrl;
1782 init_llist_head(&queue->req_list);
1783 INIT_LIST_HEAD(&queue->send_list);
1784 mutex_init(&queue->send_mutex);
1785 INIT_WORK(&queue->io_work, nvme_tcp_io_work);
1786
1787 if (qid > 0)
1788 queue->cmnd_capsule_len = nctrl->ioccsz * 16;
1789 else
1790 queue->cmnd_capsule_len = sizeof(struct nvme_command) +
1791 NVME_TCP_ADMIN_CCSZ;
1792
1793 ret = sock_create(ctrl->addr.ss_family, SOCK_STREAM,
1794 IPPROTO_TCP, &queue->sock);
1795 if (ret) {
1796 dev_err(nctrl->device,
1797 "failed to create socket: %d\n", ret);
1798 goto err_destroy_mutex;
1799 }
1800
1801 sock_file = sock_alloc_file(queue->sock, O_CLOEXEC, NULL);
1802 if (IS_ERR(sock_file)) {
1803 ret = PTR_ERR(sock_file);
1804 goto err_destroy_mutex;
1805 }
1806 nvme_tcp_reclassify_socket(queue->sock);
1807
1808 /* Single syn retry */
1809 tcp_sock_set_syncnt(queue->sock->sk, 1);
1810
1811 /* Set TCP no delay */
1812 tcp_sock_set_nodelay(queue->sock->sk);
1813
1814 /*
1815 * Cleanup whatever is sitting in the TCP transmit queue on socket
1816 * close. This is done to prevent stale data from being sent should
1817 * the network connection be restored before TCP times out.
1818 */
1819 sock_no_linger(queue->sock->sk);
1820
1821 if (so_priority > 0)
1822 sock_set_priority(queue->sock->sk, so_priority);
1823
1824 /* Set socket type of service */
1825 if (nctrl->opts->tos >= 0)
1826 ip_sock_set_tos(queue->sock->sk, nctrl->opts->tos);
1827
1828 /* Set 10 seconds timeout for icresp recvmsg */
1829 queue->sock->sk->sk_rcvtimeo = 10 * HZ;
1830
1831 queue->sock->sk->sk_allocation = GFP_ATOMIC;
1832 queue->sock->sk->sk_use_task_frag = false;
1833 queue->io_cpu = WORK_CPU_UNBOUND;
1834 queue->request = NULL;
1835 queue->data_remaining = 0;
1836 queue->ddgst_remaining = 0;
1837 queue->pdu_remaining = 0;
1838 queue->pdu_offset = 0;
1839 sk_set_memalloc(queue->sock->sk);
1840
1841 if (nctrl->opts->mask & NVMF_OPT_HOST_TRADDR) {
1842 ret = kernel_bind(queue->sock, (struct sockaddr *)&ctrl->src_addr,
1843 sizeof(ctrl->src_addr));
1844 if (ret) {
1845 dev_err(nctrl->device,
1846 "failed to bind queue %d socket %d\n",
1847 qid, ret);
1848 goto err_sock;
1849 }
1850 }
1851
1852 if (nctrl->opts->mask & NVMF_OPT_HOST_IFACE) {
1853 char *iface = nctrl->opts->host_iface;
1854 sockptr_t optval = KERNEL_SOCKPTR(iface);
1855
1856 ret = sock_setsockopt(queue->sock, SOL_SOCKET, SO_BINDTODEVICE,
1857 optval, strlen(iface));
1858 if (ret) {
1859 dev_err(nctrl->device,
1860 "failed to bind to interface %s queue %d err %d\n",
1861 iface, qid, ret);
1862 goto err_sock;
1863 }
1864 }
1865
1866 queue->hdr_digest = nctrl->opts->hdr_digest;
1867 queue->data_digest = nctrl->opts->data_digest;
1868 if (queue->hdr_digest || queue->data_digest) {
1869 ret = nvme_tcp_alloc_crypto(queue);
1870 if (ret) {
1871 dev_err(nctrl->device,
1872 "failed to allocate queue %d crypto\n", qid);
1873 goto err_sock;
1874 }
1875 }
1876
1877 rcv_pdu_size = sizeof(struct nvme_tcp_rsp_pdu) +
1878 nvme_tcp_hdgst_len(queue);
1879 queue->pdu = kmalloc(rcv_pdu_size, GFP_KERNEL);
1880 if (!queue->pdu) {
1881 ret = -ENOMEM;
1882 goto err_crypto;
1883 }
1884
1885 dev_dbg(nctrl->device, "connecting queue %d\n",
1886 nvme_tcp_queue_id(queue));
1887
1888 ret = kernel_connect(queue->sock, (struct sockaddr *)&ctrl->addr,
1889 sizeof(ctrl->addr), 0);
1890 if (ret) {
1891 dev_err(nctrl->device,
1892 "failed to connect socket: %d\n", ret);
1893 goto err_rcv_pdu;
1894 }
1895
1896 /* If PSKs are configured try to start TLS */
1897 if (nvme_tcp_tls_configured(nctrl) && pskid) {
1898 ret = nvme_tcp_start_tls(nctrl, queue, pskid);
1899 if (ret)
1900 goto err_init_connect;
1901 }
1902
1903 ret = nvme_tcp_init_connection(queue);
1904 if (ret)
1905 goto err_init_connect;
1906
1907 set_bit(NVME_TCP_Q_ALLOCATED, &queue->flags);
1908
1909 return 0;
1910
1911err_init_connect:
1912 kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1913err_rcv_pdu:
1914 kfree(queue->pdu);
1915err_crypto:
1916 if (queue->hdr_digest || queue->data_digest)
1917 nvme_tcp_free_crypto(queue);
1918err_sock:
1919 /* ->sock will be released by fput() */
1920 fput(queue->sock->file);
1921 queue->sock = NULL;
1922err_destroy_mutex:
1923 mutex_destroy(&queue->send_mutex);
1924 mutex_destroy(&queue->queue_lock);
1925 return ret;
1926}
1927
1928static void nvme_tcp_restore_sock_ops(struct nvme_tcp_queue *queue)
1929{
1930 struct socket *sock = queue->sock;
1931
1932 write_lock_bh(&sock->sk->sk_callback_lock);
1933 sock->sk->sk_user_data = NULL;
1934 sock->sk->sk_data_ready = queue->data_ready;
1935 sock->sk->sk_state_change = queue->state_change;
1936 sock->sk->sk_write_space = queue->write_space;
1937 write_unlock_bh(&sock->sk->sk_callback_lock);
1938}
1939
1940static void __nvme_tcp_stop_queue(struct nvme_tcp_queue *queue)
1941{
1942 kernel_sock_shutdown(queue->sock, SHUT_RDWR);
1943 nvme_tcp_restore_sock_ops(queue);
1944 cancel_work_sync(&queue->io_work);
1945}
1946
1947static void nvme_tcp_stop_queue(struct nvme_ctrl *nctrl, int qid)
1948{
1949 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1950 struct nvme_tcp_queue *queue = &ctrl->queues[qid];
1951
1952 if (!test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
1953 return;
1954
1955 if (test_and_clear_bit(NVME_TCP_Q_IO_CPU_SET, &queue->flags))
1956 atomic_dec(&nvme_tcp_cpu_queues[queue->io_cpu]);
1957
1958 mutex_lock(&queue->queue_lock);
1959 if (test_and_clear_bit(NVME_TCP_Q_LIVE, &queue->flags))
1960 __nvme_tcp_stop_queue(queue);
1961 /* Stopping the queue will disable TLS */
1962 queue->tls_enabled = false;
1963 mutex_unlock(&queue->queue_lock);
1964}
1965
1966static void nvme_tcp_setup_sock_ops(struct nvme_tcp_queue *queue)
1967{
1968 write_lock_bh(&queue->sock->sk->sk_callback_lock);
1969 queue->sock->sk->sk_user_data = queue;
1970 queue->state_change = queue->sock->sk->sk_state_change;
1971 queue->data_ready = queue->sock->sk->sk_data_ready;
1972 queue->write_space = queue->sock->sk->sk_write_space;
1973 queue->sock->sk->sk_data_ready = nvme_tcp_data_ready;
1974 queue->sock->sk->sk_state_change = nvme_tcp_state_change;
1975 queue->sock->sk->sk_write_space = nvme_tcp_write_space;
1976#ifdef CONFIG_NET_RX_BUSY_POLL
1977 queue->sock->sk->sk_ll_usec = 1;
1978#endif
1979 write_unlock_bh(&queue->sock->sk->sk_callback_lock);
1980}
1981
1982static int nvme_tcp_start_queue(struct nvme_ctrl *nctrl, int idx)
1983{
1984 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
1985 struct nvme_tcp_queue *queue = &ctrl->queues[idx];
1986 int ret;
1987
1988 queue->rd_enabled = true;
1989 nvme_tcp_init_recv_ctx(queue);
1990 nvme_tcp_setup_sock_ops(queue);
1991
1992 if (idx) {
1993 nvme_tcp_set_queue_io_cpu(queue);
1994 ret = nvmf_connect_io_queue(nctrl, idx);
1995 } else
1996 ret = nvmf_connect_admin_queue(nctrl);
1997
1998 if (!ret) {
1999 set_bit(NVME_TCP_Q_LIVE, &queue->flags);
2000 } else {
2001 if (test_bit(NVME_TCP_Q_ALLOCATED, &queue->flags))
2002 __nvme_tcp_stop_queue(queue);
2003 dev_err(nctrl->device,
2004 "failed to connect queue: %d ret=%d\n", idx, ret);
2005 }
2006 return ret;
2007}
2008
2009static void nvme_tcp_free_admin_queue(struct nvme_ctrl *ctrl)
2010{
2011 if (to_tcp_ctrl(ctrl)->async_req.pdu) {
2012 cancel_work_sync(&ctrl->async_event_work);
2013 nvme_tcp_free_async_req(to_tcp_ctrl(ctrl));
2014 to_tcp_ctrl(ctrl)->async_req.pdu = NULL;
2015 }
2016
2017 nvme_tcp_free_queue(ctrl, 0);
2018}
2019
2020static void nvme_tcp_free_io_queues(struct nvme_ctrl *ctrl)
2021{
2022 int i;
2023
2024 for (i = 1; i < ctrl->queue_count; i++)
2025 nvme_tcp_free_queue(ctrl, i);
2026}
2027
2028static void nvme_tcp_stop_io_queues(struct nvme_ctrl *ctrl)
2029{
2030 int i;
2031
2032 for (i = 1; i < ctrl->queue_count; i++)
2033 nvme_tcp_stop_queue(ctrl, i);
2034}
2035
2036static int nvme_tcp_start_io_queues(struct nvme_ctrl *ctrl,
2037 int first, int last)
2038{
2039 int i, ret;
2040
2041 for (i = first; i < last; i++) {
2042 ret = nvme_tcp_start_queue(ctrl, i);
2043 if (ret)
2044 goto out_stop_queues;
2045 }
2046
2047 return 0;
2048
2049out_stop_queues:
2050 for (i--; i >= first; i--)
2051 nvme_tcp_stop_queue(ctrl, i);
2052 return ret;
2053}
2054
2055static int nvme_tcp_alloc_admin_queue(struct nvme_ctrl *ctrl)
2056{
2057 int ret;
2058 key_serial_t pskid = 0;
2059
2060 if (nvme_tcp_tls_configured(ctrl)) {
2061 if (ctrl->opts->tls_key)
2062 pskid = key_serial(ctrl->opts->tls_key);
2063 else {
2064 pskid = nvme_tls_psk_default(ctrl->opts->keyring,
2065 ctrl->opts->host->nqn,
2066 ctrl->opts->subsysnqn);
2067 if (!pskid) {
2068 dev_err(ctrl->device, "no valid PSK found\n");
2069 return -ENOKEY;
2070 }
2071 }
2072 }
2073
2074 ret = nvme_tcp_alloc_queue(ctrl, 0, pskid);
2075 if (ret)
2076 return ret;
2077
2078 ret = nvme_tcp_alloc_async_req(to_tcp_ctrl(ctrl));
2079 if (ret)
2080 goto out_free_queue;
2081
2082 return 0;
2083
2084out_free_queue:
2085 nvme_tcp_free_queue(ctrl, 0);
2086 return ret;
2087}
2088
2089static int __nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
2090{
2091 int i, ret;
2092
2093 if (nvme_tcp_tls_configured(ctrl) && !ctrl->tls_pskid) {
2094 dev_err(ctrl->device, "no PSK negotiated\n");
2095 return -ENOKEY;
2096 }
2097
2098 for (i = 1; i < ctrl->queue_count; i++) {
2099 ret = nvme_tcp_alloc_queue(ctrl, i,
2100 ctrl->tls_pskid);
2101 if (ret)
2102 goto out_free_queues;
2103 }
2104
2105 return 0;
2106
2107out_free_queues:
2108 for (i--; i >= 1; i--)
2109 nvme_tcp_free_queue(ctrl, i);
2110
2111 return ret;
2112}
2113
2114static int nvme_tcp_alloc_io_queues(struct nvme_ctrl *ctrl)
2115{
2116 unsigned int nr_io_queues;
2117 int ret;
2118
2119 nr_io_queues = nvmf_nr_io_queues(ctrl->opts);
2120 ret = nvme_set_queue_count(ctrl, &nr_io_queues);
2121 if (ret)
2122 return ret;
2123
2124 if (nr_io_queues == 0) {
2125 dev_err(ctrl->device,
2126 "unable to set any I/O queues\n");
2127 return -ENOMEM;
2128 }
2129
2130 ctrl->queue_count = nr_io_queues + 1;
2131 dev_info(ctrl->device,
2132 "creating %d I/O queues.\n", nr_io_queues);
2133
2134 nvmf_set_io_queues(ctrl->opts, nr_io_queues,
2135 to_tcp_ctrl(ctrl)->io_queues);
2136 return __nvme_tcp_alloc_io_queues(ctrl);
2137}
2138
2139static int nvme_tcp_configure_io_queues(struct nvme_ctrl *ctrl, bool new)
2140{
2141 int ret, nr_queues;
2142
2143 ret = nvme_tcp_alloc_io_queues(ctrl);
2144 if (ret)
2145 return ret;
2146
2147 if (new) {
2148 ret = nvme_alloc_io_tag_set(ctrl, &to_tcp_ctrl(ctrl)->tag_set,
2149 &nvme_tcp_mq_ops,
2150 ctrl->opts->nr_poll_queues ? HCTX_MAX_TYPES : 2,
2151 sizeof(struct nvme_tcp_request));
2152 if (ret)
2153 goto out_free_io_queues;
2154 }
2155
2156 /*
2157 * Only start IO queues for which we have allocated the tagset
2158 * and limitted it to the available queues. On reconnects, the
2159 * queue number might have changed.
2160 */
2161 nr_queues = min(ctrl->tagset->nr_hw_queues + 1, ctrl->queue_count);
2162 ret = nvme_tcp_start_io_queues(ctrl, 1, nr_queues);
2163 if (ret)
2164 goto out_cleanup_connect_q;
2165
2166 if (!new) {
2167 nvme_start_freeze(ctrl);
2168 nvme_unquiesce_io_queues(ctrl);
2169 if (!nvme_wait_freeze_timeout(ctrl, NVME_IO_TIMEOUT)) {
2170 /*
2171 * If we timed out waiting for freeze we are likely to
2172 * be stuck. Fail the controller initialization just
2173 * to be safe.
2174 */
2175 ret = -ENODEV;
2176 nvme_unfreeze(ctrl);
2177 goto out_wait_freeze_timed_out;
2178 }
2179 blk_mq_update_nr_hw_queues(ctrl->tagset,
2180 ctrl->queue_count - 1);
2181 nvme_unfreeze(ctrl);
2182 }
2183
2184 /*
2185 * If the number of queues has increased (reconnect case)
2186 * start all new queues now.
2187 */
2188 ret = nvme_tcp_start_io_queues(ctrl, nr_queues,
2189 ctrl->tagset->nr_hw_queues + 1);
2190 if (ret)
2191 goto out_wait_freeze_timed_out;
2192
2193 return 0;
2194
2195out_wait_freeze_timed_out:
2196 nvme_quiesce_io_queues(ctrl);
2197 nvme_sync_io_queues(ctrl);
2198 nvme_tcp_stop_io_queues(ctrl);
2199out_cleanup_connect_q:
2200 nvme_cancel_tagset(ctrl);
2201 if (new)
2202 nvme_remove_io_tag_set(ctrl);
2203out_free_io_queues:
2204 nvme_tcp_free_io_queues(ctrl);
2205 return ret;
2206}
2207
2208static int nvme_tcp_configure_admin_queue(struct nvme_ctrl *ctrl, bool new)
2209{
2210 int error;
2211
2212 error = nvme_tcp_alloc_admin_queue(ctrl);
2213 if (error)
2214 return error;
2215
2216 if (new) {
2217 error = nvme_alloc_admin_tag_set(ctrl,
2218 &to_tcp_ctrl(ctrl)->admin_tag_set,
2219 &nvme_tcp_admin_mq_ops,
2220 sizeof(struct nvme_tcp_request));
2221 if (error)
2222 goto out_free_queue;
2223 }
2224
2225 error = nvme_tcp_start_queue(ctrl, 0);
2226 if (error)
2227 goto out_cleanup_tagset;
2228
2229 error = nvme_enable_ctrl(ctrl);
2230 if (error)
2231 goto out_stop_queue;
2232
2233 nvme_unquiesce_admin_queue(ctrl);
2234
2235 error = nvme_init_ctrl_finish(ctrl, false);
2236 if (error)
2237 goto out_quiesce_queue;
2238
2239 return 0;
2240
2241out_quiesce_queue:
2242 nvme_quiesce_admin_queue(ctrl);
2243 blk_sync_queue(ctrl->admin_q);
2244out_stop_queue:
2245 nvme_tcp_stop_queue(ctrl, 0);
2246 nvme_cancel_admin_tagset(ctrl);
2247out_cleanup_tagset:
2248 if (new)
2249 nvme_remove_admin_tag_set(ctrl);
2250out_free_queue:
2251 nvme_tcp_free_admin_queue(ctrl);
2252 return error;
2253}
2254
2255static void nvme_tcp_teardown_admin_queue(struct nvme_ctrl *ctrl,
2256 bool remove)
2257{
2258 nvme_quiesce_admin_queue(ctrl);
2259 blk_sync_queue(ctrl->admin_q);
2260 nvme_tcp_stop_queue(ctrl, 0);
2261 nvme_cancel_admin_tagset(ctrl);
2262 if (remove) {
2263 nvme_unquiesce_admin_queue(ctrl);
2264 nvme_remove_admin_tag_set(ctrl);
2265 }
2266 nvme_tcp_free_admin_queue(ctrl);
2267 if (ctrl->tls_pskid) {
2268 dev_dbg(ctrl->device, "Wipe negotiated TLS_PSK %08x\n",
2269 ctrl->tls_pskid);
2270 ctrl->tls_pskid = 0;
2271 }
2272}
2273
2274static void nvme_tcp_teardown_io_queues(struct nvme_ctrl *ctrl,
2275 bool remove)
2276{
2277 if (ctrl->queue_count <= 1)
2278 return;
2279 nvme_quiesce_io_queues(ctrl);
2280 nvme_sync_io_queues(ctrl);
2281 nvme_tcp_stop_io_queues(ctrl);
2282 nvme_cancel_tagset(ctrl);
2283 if (remove) {
2284 nvme_unquiesce_io_queues(ctrl);
2285 nvme_remove_io_tag_set(ctrl);
2286 }
2287 nvme_tcp_free_io_queues(ctrl);
2288}
2289
2290static void nvme_tcp_reconnect_or_remove(struct nvme_ctrl *ctrl,
2291 int status)
2292{
2293 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2294
2295 /* If we are resetting/deleting then do nothing */
2296 if (state != NVME_CTRL_CONNECTING) {
2297 WARN_ON_ONCE(state == NVME_CTRL_NEW || state == NVME_CTRL_LIVE);
2298 return;
2299 }
2300
2301 if (nvmf_should_reconnect(ctrl, status)) {
2302 dev_info(ctrl->device, "Reconnecting in %d seconds...\n",
2303 ctrl->opts->reconnect_delay);
2304 queue_delayed_work(nvme_wq, &to_tcp_ctrl(ctrl)->connect_work,
2305 ctrl->opts->reconnect_delay * HZ);
2306 } else {
2307 dev_info(ctrl->device, "Removing controller (%d)...\n",
2308 status);
2309 nvme_delete_ctrl(ctrl);
2310 }
2311}
2312
2313static int nvme_tcp_setup_ctrl(struct nvme_ctrl *ctrl, bool new)
2314{
2315 struct nvmf_ctrl_options *opts = ctrl->opts;
2316 int ret;
2317
2318 ret = nvme_tcp_configure_admin_queue(ctrl, new);
2319 if (ret)
2320 return ret;
2321
2322 if (ctrl->icdoff) {
2323 ret = -EOPNOTSUPP;
2324 dev_err(ctrl->device, "icdoff is not supported!\n");
2325 goto destroy_admin;
2326 }
2327
2328 if (!nvme_ctrl_sgl_supported(ctrl)) {
2329 ret = -EOPNOTSUPP;
2330 dev_err(ctrl->device, "Mandatory sgls are not supported!\n");
2331 goto destroy_admin;
2332 }
2333
2334 if (opts->queue_size > ctrl->sqsize + 1)
2335 dev_warn(ctrl->device,
2336 "queue_size %zu > ctrl sqsize %u, clamping down\n",
2337 opts->queue_size, ctrl->sqsize + 1);
2338
2339 if (ctrl->sqsize + 1 > ctrl->maxcmd) {
2340 dev_warn(ctrl->device,
2341 "sqsize %u > ctrl maxcmd %u, clamping down\n",
2342 ctrl->sqsize + 1, ctrl->maxcmd);
2343 ctrl->sqsize = ctrl->maxcmd - 1;
2344 }
2345
2346 if (ctrl->queue_count > 1) {
2347 ret = nvme_tcp_configure_io_queues(ctrl, new);
2348 if (ret)
2349 goto destroy_admin;
2350 }
2351
2352 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_LIVE)) {
2353 /*
2354 * state change failure is ok if we started ctrl delete,
2355 * unless we're during creation of a new controller to
2356 * avoid races with teardown flow.
2357 */
2358 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2359
2360 WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2361 state != NVME_CTRL_DELETING_NOIO);
2362 WARN_ON_ONCE(new);
2363 ret = -EINVAL;
2364 goto destroy_io;
2365 }
2366
2367 nvme_start_ctrl(ctrl);
2368 return 0;
2369
2370destroy_io:
2371 if (ctrl->queue_count > 1) {
2372 nvme_quiesce_io_queues(ctrl);
2373 nvme_sync_io_queues(ctrl);
2374 nvme_tcp_stop_io_queues(ctrl);
2375 nvme_cancel_tagset(ctrl);
2376 if (new)
2377 nvme_remove_io_tag_set(ctrl);
2378 nvme_tcp_free_io_queues(ctrl);
2379 }
2380destroy_admin:
2381 nvme_stop_keep_alive(ctrl);
2382 nvme_tcp_teardown_admin_queue(ctrl, new);
2383 return ret;
2384}
2385
2386static void nvme_tcp_reconnect_ctrl_work(struct work_struct *work)
2387{
2388 struct nvme_tcp_ctrl *tcp_ctrl = container_of(to_delayed_work(work),
2389 struct nvme_tcp_ctrl, connect_work);
2390 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2391 int ret;
2392
2393 ++ctrl->nr_reconnects;
2394
2395 ret = nvme_tcp_setup_ctrl(ctrl, false);
2396 if (ret)
2397 goto requeue;
2398
2399 dev_info(ctrl->device, "Successfully reconnected (attempt %d/%d)\n",
2400 ctrl->nr_reconnects, ctrl->opts->max_reconnects);
2401
2402 ctrl->nr_reconnects = 0;
2403
2404 return;
2405
2406requeue:
2407 dev_info(ctrl->device, "Failed reconnect attempt %d/%d\n",
2408 ctrl->nr_reconnects, ctrl->opts->max_reconnects);
2409 nvme_tcp_reconnect_or_remove(ctrl, ret);
2410}
2411
2412static void nvme_tcp_error_recovery_work(struct work_struct *work)
2413{
2414 struct nvme_tcp_ctrl *tcp_ctrl = container_of(work,
2415 struct nvme_tcp_ctrl, err_work);
2416 struct nvme_ctrl *ctrl = &tcp_ctrl->ctrl;
2417
2418 nvme_stop_keep_alive(ctrl);
2419 flush_work(&ctrl->async_event_work);
2420 nvme_tcp_teardown_io_queues(ctrl, false);
2421 /* unquiesce to fail fast pending requests */
2422 nvme_unquiesce_io_queues(ctrl);
2423 nvme_tcp_teardown_admin_queue(ctrl, false);
2424 nvme_unquiesce_admin_queue(ctrl);
2425 nvme_auth_stop(ctrl);
2426
2427 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2428 /* state change failure is ok if we started ctrl delete */
2429 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2430
2431 WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2432 state != NVME_CTRL_DELETING_NOIO);
2433 return;
2434 }
2435
2436 nvme_tcp_reconnect_or_remove(ctrl, 0);
2437}
2438
2439static void nvme_tcp_teardown_ctrl(struct nvme_ctrl *ctrl, bool shutdown)
2440{
2441 nvme_tcp_teardown_io_queues(ctrl, shutdown);
2442 nvme_quiesce_admin_queue(ctrl);
2443 nvme_disable_ctrl(ctrl, shutdown);
2444 nvme_tcp_teardown_admin_queue(ctrl, shutdown);
2445}
2446
2447static void nvme_tcp_delete_ctrl(struct nvme_ctrl *ctrl)
2448{
2449 nvme_tcp_teardown_ctrl(ctrl, true);
2450}
2451
2452static void nvme_reset_ctrl_work(struct work_struct *work)
2453{
2454 struct nvme_ctrl *ctrl =
2455 container_of(work, struct nvme_ctrl, reset_work);
2456 int ret;
2457
2458 nvme_stop_ctrl(ctrl);
2459 nvme_tcp_teardown_ctrl(ctrl, false);
2460
2461 if (!nvme_change_ctrl_state(ctrl, NVME_CTRL_CONNECTING)) {
2462 /* state change failure is ok if we started ctrl delete */
2463 enum nvme_ctrl_state state = nvme_ctrl_state(ctrl);
2464
2465 WARN_ON_ONCE(state != NVME_CTRL_DELETING &&
2466 state != NVME_CTRL_DELETING_NOIO);
2467 return;
2468 }
2469
2470 ret = nvme_tcp_setup_ctrl(ctrl, false);
2471 if (ret)
2472 goto out_fail;
2473
2474 return;
2475
2476out_fail:
2477 ++ctrl->nr_reconnects;
2478 nvme_tcp_reconnect_or_remove(ctrl, ret);
2479}
2480
2481static void nvme_tcp_stop_ctrl(struct nvme_ctrl *ctrl)
2482{
2483 flush_work(&to_tcp_ctrl(ctrl)->err_work);
2484 cancel_delayed_work_sync(&to_tcp_ctrl(ctrl)->connect_work);
2485}
2486
2487static void nvme_tcp_free_ctrl(struct nvme_ctrl *nctrl)
2488{
2489 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(nctrl);
2490
2491 if (list_empty(&ctrl->list))
2492 goto free_ctrl;
2493
2494 mutex_lock(&nvme_tcp_ctrl_mutex);
2495 list_del(&ctrl->list);
2496 mutex_unlock(&nvme_tcp_ctrl_mutex);
2497
2498 nvmf_free_options(nctrl->opts);
2499free_ctrl:
2500 kfree(ctrl->queues);
2501 kfree(ctrl);
2502}
2503
2504static void nvme_tcp_set_sg_null(struct nvme_command *c)
2505{
2506 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2507
2508 sg->addr = 0;
2509 sg->length = 0;
2510 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2511 NVME_SGL_FMT_TRANSPORT_A;
2512}
2513
2514static void nvme_tcp_set_sg_inline(struct nvme_tcp_queue *queue,
2515 struct nvme_command *c, u32 data_len)
2516{
2517 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2518
2519 sg->addr = cpu_to_le64(queue->ctrl->ctrl.icdoff);
2520 sg->length = cpu_to_le32(data_len);
2521 sg->type = (NVME_SGL_FMT_DATA_DESC << 4) | NVME_SGL_FMT_OFFSET;
2522}
2523
2524static void nvme_tcp_set_sg_host_data(struct nvme_command *c,
2525 u32 data_len)
2526{
2527 struct nvme_sgl_desc *sg = &c->common.dptr.sgl;
2528
2529 sg->addr = 0;
2530 sg->length = cpu_to_le32(data_len);
2531 sg->type = (NVME_TRANSPORT_SGL_DATA_DESC << 4) |
2532 NVME_SGL_FMT_TRANSPORT_A;
2533}
2534
2535static void nvme_tcp_submit_async_event(struct nvme_ctrl *arg)
2536{
2537 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(arg);
2538 struct nvme_tcp_queue *queue = &ctrl->queues[0];
2539 struct nvme_tcp_cmd_pdu *pdu = ctrl->async_req.pdu;
2540 struct nvme_command *cmd = &pdu->cmd;
2541 u8 hdgst = nvme_tcp_hdgst_len(queue);
2542
2543 memset(pdu, 0, sizeof(*pdu));
2544 pdu->hdr.type = nvme_tcp_cmd;
2545 if (queue->hdr_digest)
2546 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2547 pdu->hdr.hlen = sizeof(*pdu);
2548 pdu->hdr.plen = cpu_to_le32(pdu->hdr.hlen + hdgst);
2549
2550 cmd->common.opcode = nvme_admin_async_event;
2551 cmd->common.command_id = NVME_AQ_BLK_MQ_DEPTH;
2552 cmd->common.flags |= NVME_CMD_SGL_METABUF;
2553 nvme_tcp_set_sg_null(cmd);
2554
2555 ctrl->async_req.state = NVME_TCP_SEND_CMD_PDU;
2556 ctrl->async_req.offset = 0;
2557 ctrl->async_req.curr_bio = NULL;
2558 ctrl->async_req.data_len = 0;
2559
2560 nvme_tcp_queue_request(&ctrl->async_req, true, true);
2561}
2562
2563static void nvme_tcp_complete_timed_out(struct request *rq)
2564{
2565 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2566 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2567
2568 nvme_tcp_stop_queue(ctrl, nvme_tcp_queue_id(req->queue));
2569 nvmf_complete_timed_out_request(rq);
2570}
2571
2572static enum blk_eh_timer_return nvme_tcp_timeout(struct request *rq)
2573{
2574 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2575 struct nvme_ctrl *ctrl = &req->queue->ctrl->ctrl;
2576 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2577 struct nvme_command *cmd = &pdu->cmd;
2578 int qid = nvme_tcp_queue_id(req->queue);
2579
2580 dev_warn(ctrl->device,
2581 "I/O tag %d (%04x) type %d opcode %#x (%s) QID %d timeout\n",
2582 rq->tag, nvme_cid(rq), pdu->hdr.type, cmd->common.opcode,
2583 nvme_fabrics_opcode_str(qid, cmd), qid);
2584
2585 if (nvme_ctrl_state(ctrl) != NVME_CTRL_LIVE) {
2586 /*
2587 * If we are resetting, connecting or deleting we should
2588 * complete immediately because we may block controller
2589 * teardown or setup sequence
2590 * - ctrl disable/shutdown fabrics requests
2591 * - connect requests
2592 * - initialization admin requests
2593 * - I/O requests that entered after unquiescing and
2594 * the controller stopped responding
2595 *
2596 * All other requests should be cancelled by the error
2597 * recovery work, so it's fine that we fail it here.
2598 */
2599 nvme_tcp_complete_timed_out(rq);
2600 return BLK_EH_DONE;
2601 }
2602
2603 /*
2604 * LIVE state should trigger the normal error recovery which will
2605 * handle completing this request.
2606 */
2607 nvme_tcp_error_recovery(ctrl);
2608 return BLK_EH_RESET_TIMER;
2609}
2610
2611static blk_status_t nvme_tcp_map_data(struct nvme_tcp_queue *queue,
2612 struct request *rq)
2613{
2614 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2615 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2616 struct nvme_command *c = &pdu->cmd;
2617
2618 c->common.flags |= NVME_CMD_SGL_METABUF;
2619
2620 if (!blk_rq_nr_phys_segments(rq))
2621 nvme_tcp_set_sg_null(c);
2622 else if (rq_data_dir(rq) == WRITE &&
2623 req->data_len <= nvme_tcp_inline_data_size(req))
2624 nvme_tcp_set_sg_inline(queue, c, req->data_len);
2625 else
2626 nvme_tcp_set_sg_host_data(c, req->data_len);
2627
2628 return 0;
2629}
2630
2631static blk_status_t nvme_tcp_setup_cmd_pdu(struct nvme_ns *ns,
2632 struct request *rq)
2633{
2634 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2635 struct nvme_tcp_cmd_pdu *pdu = nvme_tcp_req_cmd_pdu(req);
2636 struct nvme_tcp_queue *queue = req->queue;
2637 u8 hdgst = nvme_tcp_hdgst_len(queue), ddgst = 0;
2638 blk_status_t ret;
2639
2640 ret = nvme_setup_cmd(ns, rq);
2641 if (ret)
2642 return ret;
2643
2644 req->state = NVME_TCP_SEND_CMD_PDU;
2645 req->status = cpu_to_le16(NVME_SC_SUCCESS);
2646 req->offset = 0;
2647 req->data_sent = 0;
2648 req->pdu_len = 0;
2649 req->pdu_sent = 0;
2650 req->h2cdata_left = 0;
2651 req->data_len = blk_rq_nr_phys_segments(rq) ?
2652 blk_rq_payload_bytes(rq) : 0;
2653 req->curr_bio = rq->bio;
2654 if (req->curr_bio && req->data_len)
2655 nvme_tcp_init_iter(req, rq_data_dir(rq));
2656
2657 if (rq_data_dir(rq) == WRITE &&
2658 req->data_len <= nvme_tcp_inline_data_size(req))
2659 req->pdu_len = req->data_len;
2660
2661 pdu->hdr.type = nvme_tcp_cmd;
2662 pdu->hdr.flags = 0;
2663 if (queue->hdr_digest)
2664 pdu->hdr.flags |= NVME_TCP_F_HDGST;
2665 if (queue->data_digest && req->pdu_len) {
2666 pdu->hdr.flags |= NVME_TCP_F_DDGST;
2667 ddgst = nvme_tcp_ddgst_len(queue);
2668 }
2669 pdu->hdr.hlen = sizeof(*pdu);
2670 pdu->hdr.pdo = req->pdu_len ? pdu->hdr.hlen + hdgst : 0;
2671 pdu->hdr.plen =
2672 cpu_to_le32(pdu->hdr.hlen + hdgst + req->pdu_len + ddgst);
2673
2674 ret = nvme_tcp_map_data(queue, rq);
2675 if (unlikely(ret)) {
2676 nvme_cleanup_cmd(rq);
2677 dev_err(queue->ctrl->ctrl.device,
2678 "Failed to map data (%d)\n", ret);
2679 return ret;
2680 }
2681
2682 return 0;
2683}
2684
2685static void nvme_tcp_commit_rqs(struct blk_mq_hw_ctx *hctx)
2686{
2687 struct nvme_tcp_queue *queue = hctx->driver_data;
2688
2689 if (!llist_empty(&queue->req_list))
2690 queue_work_on(queue->io_cpu, nvme_tcp_wq, &queue->io_work);
2691}
2692
2693static blk_status_t nvme_tcp_queue_rq(struct blk_mq_hw_ctx *hctx,
2694 const struct blk_mq_queue_data *bd)
2695{
2696 struct nvme_ns *ns = hctx->queue->queuedata;
2697 struct nvme_tcp_queue *queue = hctx->driver_data;
2698 struct request *rq = bd->rq;
2699 struct nvme_tcp_request *req = blk_mq_rq_to_pdu(rq);
2700 bool queue_ready = test_bit(NVME_TCP_Q_LIVE, &queue->flags);
2701 blk_status_t ret;
2702
2703 if (!nvme_check_ready(&queue->ctrl->ctrl, rq, queue_ready))
2704 return nvme_fail_nonready_command(&queue->ctrl->ctrl, rq);
2705
2706 ret = nvme_tcp_setup_cmd_pdu(ns, rq);
2707 if (unlikely(ret))
2708 return ret;
2709
2710 nvme_start_request(rq);
2711
2712 nvme_tcp_queue_request(req, true, bd->last);
2713
2714 return BLK_STS_OK;
2715}
2716
2717static void nvme_tcp_map_queues(struct blk_mq_tag_set *set)
2718{
2719 struct nvme_tcp_ctrl *ctrl = to_tcp_ctrl(set->driver_data);
2720
2721 nvmf_map_queues(set, &ctrl->ctrl, ctrl->io_queues);
2722}
2723
2724static int nvme_tcp_poll(struct blk_mq_hw_ctx *hctx, struct io_comp_batch *iob)
2725{
2726 struct nvme_tcp_queue *queue = hctx->driver_data;
2727 struct sock *sk = queue->sock->sk;
2728
2729 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2730 return 0;
2731
2732 set_bit(NVME_TCP_Q_POLLING, &queue->flags);
2733 if (sk_can_busy_loop(sk) && skb_queue_empty_lockless(&sk->sk_receive_queue))
2734 sk_busy_loop(sk, true);
2735 nvme_tcp_try_recv(queue);
2736 clear_bit(NVME_TCP_Q_POLLING, &queue->flags);
2737 return queue->nr_cqe;
2738}
2739
2740static int nvme_tcp_get_address(struct nvme_ctrl *ctrl, char *buf, int size)
2741{
2742 struct nvme_tcp_queue *queue = &to_tcp_ctrl(ctrl)->queues[0];
2743 struct sockaddr_storage src_addr;
2744 int ret, len;
2745
2746 len = nvmf_get_address(ctrl, buf, size);
2747
2748 if (!test_bit(NVME_TCP_Q_LIVE, &queue->flags))
2749 return len;
2750
2751 mutex_lock(&queue->queue_lock);
2752
2753 ret = kernel_getsockname(queue->sock, (struct sockaddr *)&src_addr);
2754 if (ret > 0) {
2755 if (len > 0)
2756 len--; /* strip trailing newline */
2757 len += scnprintf(buf + len, size - len, "%ssrc_addr=%pISc\n",
2758 (len) ? "," : "", &src_addr);
2759 }
2760
2761 mutex_unlock(&queue->queue_lock);
2762
2763 return len;
2764}
2765
2766static const struct blk_mq_ops nvme_tcp_mq_ops = {
2767 .queue_rq = nvme_tcp_queue_rq,
2768 .commit_rqs = nvme_tcp_commit_rqs,
2769 .complete = nvme_complete_rq,
2770 .init_request = nvme_tcp_init_request,
2771 .exit_request = nvme_tcp_exit_request,
2772 .init_hctx = nvme_tcp_init_hctx,
2773 .timeout = nvme_tcp_timeout,
2774 .map_queues = nvme_tcp_map_queues,
2775 .poll = nvme_tcp_poll,
2776};
2777
2778static const struct blk_mq_ops nvme_tcp_admin_mq_ops = {
2779 .queue_rq = nvme_tcp_queue_rq,
2780 .complete = nvme_complete_rq,
2781 .init_request = nvme_tcp_init_request,
2782 .exit_request = nvme_tcp_exit_request,
2783 .init_hctx = nvme_tcp_init_admin_hctx,
2784 .timeout = nvme_tcp_timeout,
2785};
2786
2787static const struct nvme_ctrl_ops nvme_tcp_ctrl_ops = {
2788 .name = "tcp",
2789 .module = THIS_MODULE,
2790 .flags = NVME_F_FABRICS | NVME_F_BLOCKING,
2791 .reg_read32 = nvmf_reg_read32,
2792 .reg_read64 = nvmf_reg_read64,
2793 .reg_write32 = nvmf_reg_write32,
2794 .subsystem_reset = nvmf_subsystem_reset,
2795 .free_ctrl = nvme_tcp_free_ctrl,
2796 .submit_async_event = nvme_tcp_submit_async_event,
2797 .delete_ctrl = nvme_tcp_delete_ctrl,
2798 .get_address = nvme_tcp_get_address,
2799 .stop_ctrl = nvme_tcp_stop_ctrl,
2800};
2801
2802static bool
2803nvme_tcp_existing_controller(struct nvmf_ctrl_options *opts)
2804{
2805 struct nvme_tcp_ctrl *ctrl;
2806 bool found = false;
2807
2808 mutex_lock(&nvme_tcp_ctrl_mutex);
2809 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list) {
2810 found = nvmf_ip_options_match(&ctrl->ctrl, opts);
2811 if (found)
2812 break;
2813 }
2814 mutex_unlock(&nvme_tcp_ctrl_mutex);
2815
2816 return found;
2817}
2818
2819static struct nvme_tcp_ctrl *nvme_tcp_alloc_ctrl(struct device *dev,
2820 struct nvmf_ctrl_options *opts)
2821{
2822 struct nvme_tcp_ctrl *ctrl;
2823 int ret;
2824
2825 ctrl = kzalloc(sizeof(*ctrl), GFP_KERNEL);
2826 if (!ctrl)
2827 return ERR_PTR(-ENOMEM);
2828
2829 INIT_LIST_HEAD(&ctrl->list);
2830 ctrl->ctrl.opts = opts;
2831 ctrl->ctrl.queue_count = opts->nr_io_queues + opts->nr_write_queues +
2832 opts->nr_poll_queues + 1;
2833 ctrl->ctrl.sqsize = opts->queue_size - 1;
2834 ctrl->ctrl.kato = opts->kato;
2835
2836 INIT_DELAYED_WORK(&ctrl->connect_work,
2837 nvme_tcp_reconnect_ctrl_work);
2838 INIT_WORK(&ctrl->err_work, nvme_tcp_error_recovery_work);
2839 INIT_WORK(&ctrl->ctrl.reset_work, nvme_reset_ctrl_work);
2840
2841 if (!(opts->mask & NVMF_OPT_TRSVCID)) {
2842 opts->trsvcid =
2843 kstrdup(__stringify(NVME_TCP_DISC_PORT), GFP_KERNEL);
2844 if (!opts->trsvcid) {
2845 ret = -ENOMEM;
2846 goto out_free_ctrl;
2847 }
2848 opts->mask |= NVMF_OPT_TRSVCID;
2849 }
2850
2851 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2852 opts->traddr, opts->trsvcid, &ctrl->addr);
2853 if (ret) {
2854 pr_err("malformed address passed: %s:%s\n",
2855 opts->traddr, opts->trsvcid);
2856 goto out_free_ctrl;
2857 }
2858
2859 if (opts->mask & NVMF_OPT_HOST_TRADDR) {
2860 ret = inet_pton_with_scope(&init_net, AF_UNSPEC,
2861 opts->host_traddr, NULL, &ctrl->src_addr);
2862 if (ret) {
2863 pr_err("malformed src address passed: %s\n",
2864 opts->host_traddr);
2865 goto out_free_ctrl;
2866 }
2867 }
2868
2869 if (opts->mask & NVMF_OPT_HOST_IFACE) {
2870 if (!__dev_get_by_name(&init_net, opts->host_iface)) {
2871 pr_err("invalid interface passed: %s\n",
2872 opts->host_iface);
2873 ret = -ENODEV;
2874 goto out_free_ctrl;
2875 }
2876 }
2877
2878 if (!opts->duplicate_connect && nvme_tcp_existing_controller(opts)) {
2879 ret = -EALREADY;
2880 goto out_free_ctrl;
2881 }
2882
2883 ctrl->queues = kcalloc(ctrl->ctrl.queue_count, sizeof(*ctrl->queues),
2884 GFP_KERNEL);
2885 if (!ctrl->queues) {
2886 ret = -ENOMEM;
2887 goto out_free_ctrl;
2888 }
2889
2890 ret = nvme_init_ctrl(&ctrl->ctrl, dev, &nvme_tcp_ctrl_ops, 0);
2891 if (ret)
2892 goto out_kfree_queues;
2893
2894 return ctrl;
2895out_kfree_queues:
2896 kfree(ctrl->queues);
2897out_free_ctrl:
2898 kfree(ctrl);
2899 return ERR_PTR(ret);
2900}
2901
2902static struct nvme_ctrl *nvme_tcp_create_ctrl(struct device *dev,
2903 struct nvmf_ctrl_options *opts)
2904{
2905 struct nvme_tcp_ctrl *ctrl;
2906 int ret;
2907
2908 ctrl = nvme_tcp_alloc_ctrl(dev, opts);
2909 if (IS_ERR(ctrl))
2910 return ERR_CAST(ctrl);
2911
2912 ret = nvme_add_ctrl(&ctrl->ctrl);
2913 if (ret)
2914 goto out_put_ctrl;
2915
2916 if (!nvme_change_ctrl_state(&ctrl->ctrl, NVME_CTRL_CONNECTING)) {
2917 WARN_ON_ONCE(1);
2918 ret = -EINTR;
2919 goto out_uninit_ctrl;
2920 }
2921
2922 ret = nvme_tcp_setup_ctrl(&ctrl->ctrl, true);
2923 if (ret)
2924 goto out_uninit_ctrl;
2925
2926 dev_info(ctrl->ctrl.device, "new ctrl: NQN \"%s\", addr %pISp, hostnqn: %s\n",
2927 nvmf_ctrl_subsysnqn(&ctrl->ctrl), &ctrl->addr, opts->host->nqn);
2928
2929 mutex_lock(&nvme_tcp_ctrl_mutex);
2930 list_add_tail(&ctrl->list, &nvme_tcp_ctrl_list);
2931 mutex_unlock(&nvme_tcp_ctrl_mutex);
2932
2933 return &ctrl->ctrl;
2934
2935out_uninit_ctrl:
2936 nvme_uninit_ctrl(&ctrl->ctrl);
2937out_put_ctrl:
2938 nvme_put_ctrl(&ctrl->ctrl);
2939 if (ret > 0)
2940 ret = -EIO;
2941 return ERR_PTR(ret);
2942}
2943
2944static struct nvmf_transport_ops nvme_tcp_transport = {
2945 .name = "tcp",
2946 .module = THIS_MODULE,
2947 .required_opts = NVMF_OPT_TRADDR,
2948 .allowed_opts = NVMF_OPT_TRSVCID | NVMF_OPT_RECONNECT_DELAY |
2949 NVMF_OPT_HOST_TRADDR | NVMF_OPT_CTRL_LOSS_TMO |
2950 NVMF_OPT_HDR_DIGEST | NVMF_OPT_DATA_DIGEST |
2951 NVMF_OPT_NR_WRITE_QUEUES | NVMF_OPT_NR_POLL_QUEUES |
2952 NVMF_OPT_TOS | NVMF_OPT_HOST_IFACE | NVMF_OPT_TLS |
2953 NVMF_OPT_KEYRING | NVMF_OPT_TLS_KEY,
2954 .create_ctrl = nvme_tcp_create_ctrl,
2955};
2956
2957static int __init nvme_tcp_init_module(void)
2958{
2959 unsigned int wq_flags = WQ_MEM_RECLAIM | WQ_HIGHPRI | WQ_SYSFS;
2960 int cpu;
2961
2962 BUILD_BUG_ON(sizeof(struct nvme_tcp_hdr) != 8);
2963 BUILD_BUG_ON(sizeof(struct nvme_tcp_cmd_pdu) != 72);
2964 BUILD_BUG_ON(sizeof(struct nvme_tcp_data_pdu) != 24);
2965 BUILD_BUG_ON(sizeof(struct nvme_tcp_rsp_pdu) != 24);
2966 BUILD_BUG_ON(sizeof(struct nvme_tcp_r2t_pdu) != 24);
2967 BUILD_BUG_ON(sizeof(struct nvme_tcp_icreq_pdu) != 128);
2968 BUILD_BUG_ON(sizeof(struct nvme_tcp_icresp_pdu) != 128);
2969 BUILD_BUG_ON(sizeof(struct nvme_tcp_term_pdu) != 24);
2970
2971 if (wq_unbound)
2972 wq_flags |= WQ_UNBOUND;
2973
2974 nvme_tcp_wq = alloc_workqueue("nvme_tcp_wq", wq_flags, 0);
2975 if (!nvme_tcp_wq)
2976 return -ENOMEM;
2977
2978 for_each_possible_cpu(cpu)
2979 atomic_set(&nvme_tcp_cpu_queues[cpu], 0);
2980
2981 nvmf_register_transport(&nvme_tcp_transport);
2982 return 0;
2983}
2984
2985static void __exit nvme_tcp_cleanup_module(void)
2986{
2987 struct nvme_tcp_ctrl *ctrl;
2988
2989 nvmf_unregister_transport(&nvme_tcp_transport);
2990
2991 mutex_lock(&nvme_tcp_ctrl_mutex);
2992 list_for_each_entry(ctrl, &nvme_tcp_ctrl_list, list)
2993 nvme_delete_ctrl(&ctrl->ctrl);
2994 mutex_unlock(&nvme_tcp_ctrl_mutex);
2995 flush_workqueue(nvme_delete_wq);
2996
2997 destroy_workqueue(nvme_tcp_wq);
2998}
2999
3000module_init(nvme_tcp_init_module);
3001module_exit(nvme_tcp_cleanup_module);
3002
3003MODULE_DESCRIPTION("NVMe host TCP transport driver");
3004MODULE_LICENSE("GPL v2");