Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0 or BSD-3-Clause
2
3/* Authors: Bernard Metzler <bmt@zurich.ibm.com> */
4/* Copyright (c) 2008-2019, IBM Corporation */
5
6#include <linux/errno.h>
7#include <linux/types.h>
8#include <linux/net.h>
9#include <linux/scatterlist.h>
10#include <linux/llist.h>
11#include <asm/barrier.h>
12#include <net/tcp.h>
13
14#include "siw.h"
15#include "siw_verbs.h"
16#include "siw_mem.h"
17
18static char siw_qp_state_to_string[SIW_QP_STATE_COUNT][sizeof "TERMINATE"] = {
19 [SIW_QP_STATE_IDLE] = "IDLE",
20 [SIW_QP_STATE_RTR] = "RTR",
21 [SIW_QP_STATE_RTS] = "RTS",
22 [SIW_QP_STATE_CLOSING] = "CLOSING",
23 [SIW_QP_STATE_TERMINATE] = "TERMINATE",
24 [SIW_QP_STATE_ERROR] = "ERROR"
25};
26
27/*
28 * iWARP (RDMAP, DDP and MPA) parameters as well as Softiwarp settings on a
29 * per-RDMAP message basis. Please keep order of initializer. All MPA len
30 * is initialized to minimum packet size.
31 */
32struct iwarp_msg_info iwarp_pktinfo[RDMAP_TERMINATE + 1] = {
33 { /* RDMAP_RDMA_WRITE */
34 .hdr_len = sizeof(struct iwarp_rdma_write),
35 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_write) - 2),
36 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
37 cpu_to_be16(DDP_VERSION << 8) |
38 cpu_to_be16(RDMAP_VERSION << 6) |
39 cpu_to_be16(RDMAP_RDMA_WRITE),
40 .rx_data = siw_proc_write },
41 { /* RDMAP_RDMA_READ_REQ */
42 .hdr_len = sizeof(struct iwarp_rdma_rreq),
43 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rreq) - 2),
44 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
45 cpu_to_be16(RDMAP_VERSION << 6) |
46 cpu_to_be16(RDMAP_RDMA_READ_REQ),
47 .rx_data = siw_proc_rreq },
48 { /* RDMAP_RDMA_READ_RESP */
49 .hdr_len = sizeof(struct iwarp_rdma_rresp),
50 .ctrl.mpa_len = htons(sizeof(struct iwarp_rdma_rresp) - 2),
51 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_TAGGED | DDP_FLAG_LAST |
52 cpu_to_be16(DDP_VERSION << 8) |
53 cpu_to_be16(RDMAP_VERSION << 6) |
54 cpu_to_be16(RDMAP_RDMA_READ_RESP),
55 .rx_data = siw_proc_rresp },
56 { /* RDMAP_SEND */
57 .hdr_len = sizeof(struct iwarp_send),
58 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
59 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
60 cpu_to_be16(RDMAP_VERSION << 6) |
61 cpu_to_be16(RDMAP_SEND),
62 .rx_data = siw_proc_send },
63 { /* RDMAP_SEND_INVAL */
64 .hdr_len = sizeof(struct iwarp_send_inv),
65 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
66 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
67 cpu_to_be16(RDMAP_VERSION << 6) |
68 cpu_to_be16(RDMAP_SEND_INVAL),
69 .rx_data = siw_proc_send },
70 { /* RDMAP_SEND_SE */
71 .hdr_len = sizeof(struct iwarp_send),
72 .ctrl.mpa_len = htons(sizeof(struct iwarp_send) - 2),
73 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
74 cpu_to_be16(RDMAP_VERSION << 6) |
75 cpu_to_be16(RDMAP_SEND_SE),
76 .rx_data = siw_proc_send },
77 { /* RDMAP_SEND_SE_INVAL */
78 .hdr_len = sizeof(struct iwarp_send_inv),
79 .ctrl.mpa_len = htons(sizeof(struct iwarp_send_inv) - 2),
80 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
81 cpu_to_be16(RDMAP_VERSION << 6) |
82 cpu_to_be16(RDMAP_SEND_SE_INVAL),
83 .rx_data = siw_proc_send },
84 { /* RDMAP_TERMINATE */
85 .hdr_len = sizeof(struct iwarp_terminate),
86 .ctrl.mpa_len = htons(sizeof(struct iwarp_terminate) - 2),
87 .ctrl.ddp_rdmap_ctrl = DDP_FLAG_LAST | cpu_to_be16(DDP_VERSION << 8) |
88 cpu_to_be16(RDMAP_VERSION << 6) |
89 cpu_to_be16(RDMAP_TERMINATE),
90 .rx_data = siw_proc_terminate }
91};
92
93void siw_qp_llp_data_ready(struct sock *sk)
94{
95 struct siw_qp *qp;
96
97 read_lock(&sk->sk_callback_lock);
98
99 if (unlikely(!sk->sk_user_data || !sk_to_qp(sk)))
100 goto done;
101
102 qp = sk_to_qp(sk);
103
104 if (likely(!qp->rx_stream.rx_suspend &&
105 down_read_trylock(&qp->state_lock))) {
106 read_descriptor_t rd_desc = { .arg.data = qp, .count = 1 };
107
108 if (likely(qp->attrs.state == SIW_QP_STATE_RTS))
109 /*
110 * Implements data receive operation during
111 * socket callback. TCP gracefully catches
112 * the case where there is nothing to receive
113 * (not calling siw_tcp_rx_data() then).
114 */
115 tcp_read_sock(sk, &rd_desc, siw_tcp_rx_data);
116
117 up_read(&qp->state_lock);
118 } else {
119 siw_dbg_qp(qp, "unable to process RX, suspend: %d\n",
120 qp->rx_stream.rx_suspend);
121 }
122done:
123 read_unlock(&sk->sk_callback_lock);
124}
125
126void siw_qp_llp_close(struct siw_qp *qp)
127{
128 siw_dbg_qp(qp, "enter llp close, state = %s\n",
129 siw_qp_state_to_string[qp->attrs.state]);
130
131 down_write(&qp->state_lock);
132
133 qp->rx_stream.rx_suspend = 1;
134 qp->tx_ctx.tx_suspend = 1;
135 qp->attrs.sk = NULL;
136
137 switch (qp->attrs.state) {
138 case SIW_QP_STATE_RTS:
139 case SIW_QP_STATE_RTR:
140 case SIW_QP_STATE_IDLE:
141 case SIW_QP_STATE_TERMINATE:
142 qp->attrs.state = SIW_QP_STATE_ERROR;
143 break;
144 /*
145 * SIW_QP_STATE_CLOSING:
146 *
147 * This is a forced close. shall the QP be moved to
148 * ERROR or IDLE ?
149 */
150 case SIW_QP_STATE_CLOSING:
151 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE)
152 qp->attrs.state = SIW_QP_STATE_ERROR;
153 else
154 qp->attrs.state = SIW_QP_STATE_IDLE;
155 break;
156
157 default:
158 siw_dbg_qp(qp, "llp close: no state transition needed: %s\n",
159 siw_qp_state_to_string[qp->attrs.state]);
160 break;
161 }
162 siw_sq_flush(qp);
163 siw_rq_flush(qp);
164
165 /*
166 * Dereference closing CEP
167 */
168 if (qp->cep) {
169 siw_cep_put(qp->cep);
170 qp->cep = NULL;
171 }
172
173 up_write(&qp->state_lock);
174
175 siw_dbg_qp(qp, "llp close exit: state %s\n",
176 siw_qp_state_to_string[qp->attrs.state]);
177}
178
179/*
180 * socket callback routine informing about newly available send space.
181 * Function schedules SQ work for processing SQ items.
182 */
183void siw_qp_llp_write_space(struct sock *sk)
184{
185 struct siw_cep *cep;
186
187 read_lock(&sk->sk_callback_lock);
188
189 cep = sk_to_cep(sk);
190 if (cep) {
191 cep->sk_write_space(sk);
192
193 if (!test_bit(SOCK_NOSPACE, &sk->sk_socket->flags))
194 (void)siw_sq_start(cep->qp);
195 }
196
197 read_unlock(&sk->sk_callback_lock);
198}
199
200static int siw_qp_readq_init(struct siw_qp *qp, int irq_size, int orq_size)
201{
202 if (irq_size) {
203 irq_size = roundup_pow_of_two(irq_size);
204 qp->irq = vzalloc(irq_size * sizeof(struct siw_sqe));
205 if (!qp->irq) {
206 qp->attrs.irq_size = 0;
207 return -ENOMEM;
208 }
209 }
210 if (orq_size) {
211 orq_size = roundup_pow_of_two(orq_size);
212 qp->orq = vzalloc(orq_size * sizeof(struct siw_sqe));
213 if (!qp->orq) {
214 qp->attrs.orq_size = 0;
215 qp->attrs.irq_size = 0;
216 vfree(qp->irq);
217 return -ENOMEM;
218 }
219 }
220 qp->attrs.irq_size = irq_size;
221 qp->attrs.orq_size = orq_size;
222 siw_dbg_qp(qp, "ORD %d, IRD %d\n", orq_size, irq_size);
223 return 0;
224}
225
226static int siw_qp_enable_crc(struct siw_qp *qp)
227{
228 struct siw_rx_stream *c_rx = &qp->rx_stream;
229 struct siw_iwarp_tx *c_tx = &qp->tx_ctx;
230 int size;
231
232 if (siw_crypto_shash == NULL)
233 return -ENOENT;
234
235 size = crypto_shash_descsize(siw_crypto_shash) +
236 sizeof(struct shash_desc);
237
238 c_tx->mpa_crc_hd = kzalloc(size, GFP_KERNEL);
239 c_rx->mpa_crc_hd = kzalloc(size, GFP_KERNEL);
240 if (!c_tx->mpa_crc_hd || !c_rx->mpa_crc_hd) {
241 kfree(c_tx->mpa_crc_hd);
242 kfree(c_rx->mpa_crc_hd);
243 c_tx->mpa_crc_hd = NULL;
244 c_rx->mpa_crc_hd = NULL;
245 return -ENOMEM;
246 }
247 c_tx->mpa_crc_hd->tfm = siw_crypto_shash;
248 c_rx->mpa_crc_hd->tfm = siw_crypto_shash;
249
250 return 0;
251}
252
253/*
254 * Send a non signalled READ or WRITE to peer side as negotiated
255 * with MPAv2 P2P setup protocol. The work request is only created
256 * as a current active WR and does not consume Send Queue space.
257 *
258 * Caller must hold QP state lock.
259 */
260int siw_qp_mpa_rts(struct siw_qp *qp, enum mpa_v2_ctrl ctrl)
261{
262 struct siw_wqe *wqe = tx_wqe(qp);
263 unsigned long flags;
264 int rv = 0;
265
266 spin_lock_irqsave(&qp->sq_lock, flags);
267
268 if (unlikely(wqe->wr_status != SIW_WR_IDLE)) {
269 spin_unlock_irqrestore(&qp->sq_lock, flags);
270 return -EIO;
271 }
272 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
273
274 wqe->wr_status = SIW_WR_QUEUED;
275 wqe->sqe.flags = 0;
276 wqe->sqe.num_sge = 1;
277 wqe->sqe.sge[0].length = 0;
278 wqe->sqe.sge[0].laddr = 0;
279 wqe->sqe.sge[0].lkey = 0;
280 /*
281 * While it must not be checked for inbound zero length
282 * READ/WRITE, some HW may treat STag 0 special.
283 */
284 wqe->sqe.rkey = 1;
285 wqe->sqe.raddr = 0;
286 wqe->processed = 0;
287
288 if (ctrl & MPA_V2_RDMA_WRITE_RTR)
289 wqe->sqe.opcode = SIW_OP_WRITE;
290 else if (ctrl & MPA_V2_RDMA_READ_RTR) {
291 struct siw_sqe *rreq = NULL;
292
293 wqe->sqe.opcode = SIW_OP_READ;
294
295 spin_lock(&qp->orq_lock);
296
297 if (qp->attrs.orq_size)
298 rreq = orq_get_free(qp);
299 if (rreq) {
300 siw_read_to_orq(rreq, &wqe->sqe);
301 qp->orq_put++;
302 } else
303 rv = -EIO;
304
305 spin_unlock(&qp->orq_lock);
306 } else
307 rv = -EINVAL;
308
309 if (rv)
310 wqe->wr_status = SIW_WR_IDLE;
311
312 spin_unlock_irqrestore(&qp->sq_lock, flags);
313
314 if (!rv)
315 rv = siw_sq_start(qp);
316
317 return rv;
318}
319
320/*
321 * Map memory access error to DDP tagged error
322 */
323enum ddp_ecode siw_tagged_error(enum siw_access_state state)
324{
325 switch (state) {
326 case E_STAG_INVALID:
327 return DDP_ECODE_T_INVALID_STAG;
328 case E_BASE_BOUNDS:
329 return DDP_ECODE_T_BASE_BOUNDS;
330 case E_PD_MISMATCH:
331 return DDP_ECODE_T_STAG_NOT_ASSOC;
332 case E_ACCESS_PERM:
333 /*
334 * RFC 5041 (DDP) lacks an ecode for insufficient access
335 * permissions. 'Invalid STag' seem to be the closest
336 * match though.
337 */
338 return DDP_ECODE_T_INVALID_STAG;
339 default:
340 WARN_ON(1);
341 return DDP_ECODE_T_INVALID_STAG;
342 }
343}
344
345/*
346 * Map memory access error to RDMAP protection error
347 */
348enum rdmap_ecode siw_rdmap_error(enum siw_access_state state)
349{
350 switch (state) {
351 case E_STAG_INVALID:
352 return RDMAP_ECODE_INVALID_STAG;
353 case E_BASE_BOUNDS:
354 return RDMAP_ECODE_BASE_BOUNDS;
355 case E_PD_MISMATCH:
356 return RDMAP_ECODE_STAG_NOT_ASSOC;
357 case E_ACCESS_PERM:
358 return RDMAP_ECODE_ACCESS_RIGHTS;
359 default:
360 return RDMAP_ECODE_UNSPECIFIED;
361 }
362}
363
364void siw_init_terminate(struct siw_qp *qp, enum term_elayer layer, u8 etype,
365 u8 ecode, int in_tx)
366{
367 if (!qp->term_info.valid) {
368 memset(&qp->term_info, 0, sizeof(qp->term_info));
369 qp->term_info.layer = layer;
370 qp->term_info.etype = etype;
371 qp->term_info.ecode = ecode;
372 qp->term_info.in_tx = in_tx;
373 qp->term_info.valid = 1;
374 }
375 siw_dbg_qp(qp, "init TERM: layer %d, type %d, code %d, in tx %s\n",
376 layer, etype, ecode, in_tx ? "yes" : "no");
377}
378
379/*
380 * Send a TERMINATE message, as defined in RFC's 5040/5041/5044/6581.
381 * Sending TERMINATE messages is best effort - such messages
382 * can only be send if the QP is still connected and it does
383 * not have another outbound message in-progress, i.e. the
384 * TERMINATE message must not interfer with an incomplete current
385 * transmit operation.
386 */
387void siw_send_terminate(struct siw_qp *qp)
388{
389 struct kvec iov[3];
390 struct msghdr msg = { .msg_flags = MSG_DONTWAIT | MSG_EOR };
391 struct iwarp_terminate *term = NULL;
392 union iwarp_hdr *err_hdr = NULL;
393 struct socket *s = qp->attrs.sk;
394 struct siw_rx_stream *srx = &qp->rx_stream;
395 union iwarp_hdr *rx_hdr = &srx->hdr;
396 u32 crc = 0;
397 int num_frags, len_terminate, rv;
398
399 if (!qp->term_info.valid)
400 return;
401
402 qp->term_info.valid = 0;
403
404 if (tx_wqe(qp)->wr_status == SIW_WR_INPROGRESS) {
405 siw_dbg_qp(qp, "cannot send TERMINATE: op %d in progress\n",
406 tx_type(tx_wqe(qp)));
407 return;
408 }
409 if (!s && qp->cep)
410 /* QP not yet in RTS. Take socket from connection end point */
411 s = qp->cep->sock;
412
413 if (!s) {
414 siw_dbg_qp(qp, "cannot send TERMINATE: not connected\n");
415 return;
416 }
417
418 term = kzalloc(sizeof(*term), GFP_KERNEL);
419 if (!term)
420 return;
421
422 term->ddp_qn = cpu_to_be32(RDMAP_UNTAGGED_QN_TERMINATE);
423 term->ddp_mo = 0;
424 term->ddp_msn = cpu_to_be32(1);
425
426 iov[0].iov_base = term;
427 iov[0].iov_len = sizeof(*term);
428
429 if ((qp->term_info.layer == TERM_ERROR_LAYER_DDP) ||
430 ((qp->term_info.layer == TERM_ERROR_LAYER_RDMAP) &&
431 (qp->term_info.etype != RDMAP_ETYPE_CATASTROPHIC))) {
432 err_hdr = kzalloc(sizeof(*err_hdr), GFP_KERNEL);
433 if (!err_hdr) {
434 kfree(term);
435 return;
436 }
437 }
438 memcpy(&term->ctrl, &iwarp_pktinfo[RDMAP_TERMINATE].ctrl,
439 sizeof(struct iwarp_ctrl));
440
441 __rdmap_term_set_layer(term, qp->term_info.layer);
442 __rdmap_term_set_etype(term, qp->term_info.etype);
443 __rdmap_term_set_ecode(term, qp->term_info.ecode);
444
445 switch (qp->term_info.layer) {
446 case TERM_ERROR_LAYER_RDMAP:
447 if (qp->term_info.etype == RDMAP_ETYPE_CATASTROPHIC)
448 /* No additional DDP/RDMAP header to be included */
449 break;
450
451 if (qp->term_info.etype == RDMAP_ETYPE_REMOTE_PROTECTION) {
452 /*
453 * Complete RDMAP frame will get attached, and
454 * DDP segment length is valid
455 */
456 term->flag_m = 1;
457 term->flag_d = 1;
458 term->flag_r = 1;
459
460 if (qp->term_info.in_tx) {
461 struct iwarp_rdma_rreq *rreq;
462 struct siw_wqe *wqe = tx_wqe(qp);
463
464 /* Inbound RREQ error, detected during
465 * RRESP creation. Take state from
466 * current TX work queue element to
467 * reconstruct peers RREQ.
468 */
469 rreq = (struct iwarp_rdma_rreq *)err_hdr;
470
471 memcpy(&rreq->ctrl,
472 &iwarp_pktinfo[RDMAP_RDMA_READ_REQ].ctrl,
473 sizeof(struct iwarp_ctrl));
474
475 rreq->rsvd = 0;
476 rreq->ddp_qn =
477 htonl(RDMAP_UNTAGGED_QN_RDMA_READ);
478
479 /* Provide RREQ's MSN as kept aside */
480 rreq->ddp_msn = htonl(wqe->sqe.sge[0].length);
481
482 rreq->ddp_mo = htonl(wqe->processed);
483 rreq->sink_stag = htonl(wqe->sqe.rkey);
484 rreq->sink_to = cpu_to_be64(wqe->sqe.raddr);
485 rreq->read_size = htonl(wqe->sqe.sge[0].length);
486 rreq->source_stag = htonl(wqe->sqe.sge[0].lkey);
487 rreq->source_to =
488 cpu_to_be64(wqe->sqe.sge[0].laddr);
489
490 iov[1].iov_base = rreq;
491 iov[1].iov_len = sizeof(*rreq);
492
493 rx_hdr = (union iwarp_hdr *)rreq;
494 } else {
495 /* Take RDMAP/DDP information from
496 * current (failed) inbound frame.
497 */
498 iov[1].iov_base = rx_hdr;
499
500 if (__rdmap_get_opcode(&rx_hdr->ctrl) ==
501 RDMAP_RDMA_READ_REQ)
502 iov[1].iov_len =
503 sizeof(struct iwarp_rdma_rreq);
504 else /* SEND type */
505 iov[1].iov_len =
506 sizeof(struct iwarp_send);
507 }
508 } else {
509 /* Do not report DDP hdr information if packet
510 * layout is unknown
511 */
512 if ((qp->term_info.ecode == RDMAP_ECODE_VERSION) ||
513 (qp->term_info.ecode == RDMAP_ECODE_OPCODE))
514 break;
515
516 iov[1].iov_base = rx_hdr;
517
518 /* Only DDP frame will get attached */
519 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
520 iov[1].iov_len =
521 sizeof(struct iwarp_rdma_write);
522 else
523 iov[1].iov_len = sizeof(struct iwarp_send);
524
525 term->flag_m = 1;
526 term->flag_d = 1;
527 }
528 term->ctrl.mpa_len = cpu_to_be16(iov[1].iov_len);
529 break;
530
531 case TERM_ERROR_LAYER_DDP:
532 /* Report error encountered while DDP processing.
533 * This can only happen as a result of inbound
534 * DDP processing
535 */
536
537 /* Do not report DDP hdr information if packet
538 * layout is unknown
539 */
540 if (((qp->term_info.etype == DDP_ETYPE_TAGGED_BUF) &&
541 (qp->term_info.ecode == DDP_ECODE_T_VERSION)) ||
542 ((qp->term_info.etype == DDP_ETYPE_UNTAGGED_BUF) &&
543 (qp->term_info.ecode == DDP_ECODE_UT_VERSION)))
544 break;
545
546 iov[1].iov_base = rx_hdr;
547
548 if (rx_hdr->ctrl.ddp_rdmap_ctrl & DDP_FLAG_TAGGED)
549 iov[1].iov_len = sizeof(struct iwarp_ctrl_tagged);
550 else
551 iov[1].iov_len = sizeof(struct iwarp_ctrl_untagged);
552
553 term->flag_m = 1;
554 term->flag_d = 1;
555 break;
556
557 default:
558 break;
559 }
560 if (term->flag_m || term->flag_d || term->flag_r) {
561 iov[2].iov_base = &crc;
562 iov[2].iov_len = sizeof(crc);
563 len_terminate = sizeof(*term) + iov[1].iov_len + MPA_CRC_SIZE;
564 num_frags = 3;
565 } else {
566 iov[1].iov_base = &crc;
567 iov[1].iov_len = sizeof(crc);
568 len_terminate = sizeof(*term) + MPA_CRC_SIZE;
569 num_frags = 2;
570 }
571
572 /* Adjust DDP Segment Length parameter, if valid */
573 if (term->flag_m) {
574 u32 real_ddp_len = be16_to_cpu(rx_hdr->ctrl.mpa_len);
575 enum rdma_opcode op = __rdmap_get_opcode(&rx_hdr->ctrl);
576
577 real_ddp_len -= iwarp_pktinfo[op].hdr_len - MPA_HDR_SIZE;
578 rx_hdr->ctrl.mpa_len = cpu_to_be16(real_ddp_len);
579 }
580
581 term->ctrl.mpa_len =
582 cpu_to_be16(len_terminate - (MPA_HDR_SIZE + MPA_CRC_SIZE));
583 if (qp->tx_ctx.mpa_crc_hd) {
584 crypto_shash_init(qp->tx_ctx.mpa_crc_hd);
585 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,
586 (u8 *)iov[0].iov_base,
587 iov[0].iov_len))
588 goto out;
589
590 if (num_frags == 3) {
591 if (crypto_shash_update(qp->tx_ctx.mpa_crc_hd,
592 (u8 *)iov[1].iov_base,
593 iov[1].iov_len))
594 goto out;
595 }
596 crypto_shash_final(qp->tx_ctx.mpa_crc_hd, (u8 *)&crc);
597 }
598
599 rv = kernel_sendmsg(s, &msg, iov, num_frags, len_terminate);
600 siw_dbg_qp(qp, "sent TERM: %s, layer %d, type %d, code %d (%d bytes)\n",
601 rv == len_terminate ? "success" : "failure",
602 __rdmap_term_layer(term), __rdmap_term_etype(term),
603 __rdmap_term_ecode(term), rv);
604out:
605 kfree(term);
606 kfree(err_hdr);
607}
608
609/*
610 * Handle all attrs other than state
611 */
612static void siw_qp_modify_nonstate(struct siw_qp *qp,
613 struct siw_qp_attrs *attrs,
614 enum siw_qp_attr_mask mask)
615{
616 if (mask & SIW_QP_ATTR_ACCESS_FLAGS) {
617 if (attrs->flags & SIW_RDMA_BIND_ENABLED)
618 qp->attrs.flags |= SIW_RDMA_BIND_ENABLED;
619 else
620 qp->attrs.flags &= ~SIW_RDMA_BIND_ENABLED;
621
622 if (attrs->flags & SIW_RDMA_WRITE_ENABLED)
623 qp->attrs.flags |= SIW_RDMA_WRITE_ENABLED;
624 else
625 qp->attrs.flags &= ~SIW_RDMA_WRITE_ENABLED;
626
627 if (attrs->flags & SIW_RDMA_READ_ENABLED)
628 qp->attrs.flags |= SIW_RDMA_READ_ENABLED;
629 else
630 qp->attrs.flags &= ~SIW_RDMA_READ_ENABLED;
631 }
632}
633
634static int siw_qp_nextstate_from_idle(struct siw_qp *qp,
635 struct siw_qp_attrs *attrs,
636 enum siw_qp_attr_mask mask)
637{
638 int rv = 0;
639
640 switch (attrs->state) {
641 case SIW_QP_STATE_RTS:
642 if (attrs->flags & SIW_MPA_CRC) {
643 rv = siw_qp_enable_crc(qp);
644 if (rv)
645 break;
646 }
647 if (!(mask & SIW_QP_ATTR_LLP_HANDLE)) {
648 siw_dbg_qp(qp, "no socket\n");
649 rv = -EINVAL;
650 break;
651 }
652 if (!(mask & SIW_QP_ATTR_MPA)) {
653 siw_dbg_qp(qp, "no MPA\n");
654 rv = -EINVAL;
655 break;
656 }
657 /*
658 * Initialize iWARP TX state
659 */
660 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 0;
661 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 0;
662 qp->tx_ctx.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 0;
663
664 /*
665 * Initialize iWARP RX state
666 */
667 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_SEND] = 1;
668 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_RDMA_READ] = 1;
669 qp->rx_stream.ddp_msn[RDMAP_UNTAGGED_QN_TERMINATE] = 1;
670
671 /*
672 * init IRD free queue, caller has already checked
673 * limits.
674 */
675 rv = siw_qp_readq_init(qp, attrs->irq_size,
676 attrs->orq_size);
677 if (rv)
678 break;
679
680 qp->attrs.sk = attrs->sk;
681 qp->attrs.state = SIW_QP_STATE_RTS;
682
683 siw_dbg_qp(qp, "enter RTS: crc=%s, ord=%u, ird=%u\n",
684 attrs->flags & SIW_MPA_CRC ? "y" : "n",
685 qp->attrs.orq_size, qp->attrs.irq_size);
686 break;
687
688 case SIW_QP_STATE_ERROR:
689 siw_rq_flush(qp);
690 qp->attrs.state = SIW_QP_STATE_ERROR;
691 if (qp->cep) {
692 siw_cep_put(qp->cep);
693 qp->cep = NULL;
694 }
695 break;
696
697 default:
698 break;
699 }
700 return rv;
701}
702
703static int siw_qp_nextstate_from_rts(struct siw_qp *qp,
704 struct siw_qp_attrs *attrs)
705{
706 int drop_conn = 0;
707
708 switch (attrs->state) {
709 case SIW_QP_STATE_CLOSING:
710 /*
711 * Verbs: move to IDLE if SQ and ORQ are empty.
712 * Move to ERROR otherwise. But first of all we must
713 * close the connection. So we keep CLOSING or ERROR
714 * as a transient state, schedule connection drop work
715 * and wait for the socket state change upcall to
716 * come back closed.
717 */
718 if (tx_wqe(qp)->wr_status == SIW_WR_IDLE) {
719 qp->attrs.state = SIW_QP_STATE_CLOSING;
720 } else {
721 qp->attrs.state = SIW_QP_STATE_ERROR;
722 siw_sq_flush(qp);
723 }
724 siw_rq_flush(qp);
725
726 drop_conn = 1;
727 break;
728
729 case SIW_QP_STATE_TERMINATE:
730 qp->attrs.state = SIW_QP_STATE_TERMINATE;
731
732 siw_init_terminate(qp, TERM_ERROR_LAYER_RDMAP,
733 RDMAP_ETYPE_CATASTROPHIC,
734 RDMAP_ECODE_UNSPECIFIED, 1);
735 drop_conn = 1;
736 break;
737
738 case SIW_QP_STATE_ERROR:
739 /*
740 * This is an emergency close.
741 *
742 * Any in progress transmit operation will get
743 * cancelled.
744 * This will likely result in a protocol failure,
745 * if a TX operation is in transit. The caller
746 * could unconditional wait to give the current
747 * operation a chance to complete.
748 * Esp., how to handle the non-empty IRQ case?
749 * The peer was asking for data transfer at a valid
750 * point in time.
751 */
752 siw_sq_flush(qp);
753 siw_rq_flush(qp);
754 qp->attrs.state = SIW_QP_STATE_ERROR;
755 drop_conn = 1;
756 break;
757
758 default:
759 break;
760 }
761 return drop_conn;
762}
763
764static void siw_qp_nextstate_from_term(struct siw_qp *qp,
765 struct siw_qp_attrs *attrs)
766{
767 switch (attrs->state) {
768 case SIW_QP_STATE_ERROR:
769 siw_rq_flush(qp);
770 qp->attrs.state = SIW_QP_STATE_ERROR;
771
772 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
773 siw_sq_flush(qp);
774 break;
775
776 default:
777 break;
778 }
779}
780
781static int siw_qp_nextstate_from_close(struct siw_qp *qp,
782 struct siw_qp_attrs *attrs)
783{
784 int rv = 0;
785
786 switch (attrs->state) {
787 case SIW_QP_STATE_IDLE:
788 WARN_ON(tx_wqe(qp)->wr_status != SIW_WR_IDLE);
789 qp->attrs.state = SIW_QP_STATE_IDLE;
790 break;
791
792 case SIW_QP_STATE_CLOSING:
793 /*
794 * The LLP may already moved the QP to closing
795 * due to graceful peer close init
796 */
797 break;
798
799 case SIW_QP_STATE_ERROR:
800 /*
801 * QP was moved to CLOSING by LLP event
802 * not yet seen by user.
803 */
804 qp->attrs.state = SIW_QP_STATE_ERROR;
805
806 if (tx_wqe(qp)->wr_status != SIW_WR_IDLE)
807 siw_sq_flush(qp);
808
809 siw_rq_flush(qp);
810 break;
811
812 default:
813 siw_dbg_qp(qp, "state transition undefined: %s => %s\n",
814 siw_qp_state_to_string[qp->attrs.state],
815 siw_qp_state_to_string[attrs->state]);
816
817 rv = -ECONNABORTED;
818 }
819 return rv;
820}
821
822/*
823 * Caller must hold qp->state_lock
824 */
825int siw_qp_modify(struct siw_qp *qp, struct siw_qp_attrs *attrs,
826 enum siw_qp_attr_mask mask)
827{
828 int drop_conn = 0, rv = 0;
829
830 if (!mask)
831 return 0;
832
833 siw_dbg_qp(qp, "state: %s => %s\n",
834 siw_qp_state_to_string[qp->attrs.state],
835 siw_qp_state_to_string[attrs->state]);
836
837 if (mask != SIW_QP_ATTR_STATE)
838 siw_qp_modify_nonstate(qp, attrs, mask);
839
840 if (!(mask & SIW_QP_ATTR_STATE))
841 return 0;
842
843 switch (qp->attrs.state) {
844 case SIW_QP_STATE_IDLE:
845 case SIW_QP_STATE_RTR:
846 rv = siw_qp_nextstate_from_idle(qp, attrs, mask);
847 break;
848
849 case SIW_QP_STATE_RTS:
850 drop_conn = siw_qp_nextstate_from_rts(qp, attrs);
851 break;
852
853 case SIW_QP_STATE_TERMINATE:
854 siw_qp_nextstate_from_term(qp, attrs);
855 break;
856
857 case SIW_QP_STATE_CLOSING:
858 siw_qp_nextstate_from_close(qp, attrs);
859 break;
860 default:
861 break;
862 }
863 if (drop_conn)
864 siw_qp_cm_drop(qp, 0);
865
866 return rv;
867}
868
869void siw_read_to_orq(struct siw_sqe *rreq, struct siw_sqe *sqe)
870{
871 rreq->id = sqe->id;
872 rreq->opcode = sqe->opcode;
873 rreq->sge[0].laddr = sqe->sge[0].laddr;
874 rreq->sge[0].length = sqe->sge[0].length;
875 rreq->sge[0].lkey = sqe->sge[0].lkey;
876 rreq->sge[1].lkey = sqe->sge[1].lkey;
877 rreq->flags = sqe->flags | SIW_WQE_VALID;
878 rreq->num_sge = 1;
879}
880
881static int siw_activate_tx_from_sq(struct siw_qp *qp)
882{
883 struct siw_sqe *sqe;
884 struct siw_wqe *wqe = tx_wqe(qp);
885 int rv = 1;
886
887 sqe = sq_get_next(qp);
888 if (!sqe)
889 return 0;
890
891 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
892 wqe->wr_status = SIW_WR_QUEUED;
893
894 /* First copy SQE to kernel private memory */
895 memcpy(&wqe->sqe, sqe, sizeof(*sqe));
896
897 if (wqe->sqe.opcode >= SIW_NUM_OPCODES) {
898 rv = -EINVAL;
899 goto out;
900 }
901 if (wqe->sqe.flags & SIW_WQE_INLINE) {
902 if (wqe->sqe.opcode != SIW_OP_SEND &&
903 wqe->sqe.opcode != SIW_OP_WRITE) {
904 rv = -EINVAL;
905 goto out;
906 }
907 if (wqe->sqe.sge[0].length > SIW_MAX_INLINE) {
908 rv = -EINVAL;
909 goto out;
910 }
911 wqe->sqe.sge[0].laddr = (uintptr_t)&wqe->sqe.sge[1];
912 wqe->sqe.sge[0].lkey = 0;
913 wqe->sqe.num_sge = 1;
914 }
915 if (wqe->sqe.flags & SIW_WQE_READ_FENCE) {
916 /* A READ cannot be fenced */
917 if (unlikely(wqe->sqe.opcode == SIW_OP_READ ||
918 wqe->sqe.opcode ==
919 SIW_OP_READ_LOCAL_INV)) {
920 siw_dbg_qp(qp, "cannot fence read\n");
921 rv = -EINVAL;
922 goto out;
923 }
924 spin_lock(&qp->orq_lock);
925
926 if (qp->attrs.orq_size && !siw_orq_empty(qp)) {
927 qp->tx_ctx.orq_fence = 1;
928 rv = 0;
929 }
930 spin_unlock(&qp->orq_lock);
931
932 } else if (wqe->sqe.opcode == SIW_OP_READ ||
933 wqe->sqe.opcode == SIW_OP_READ_LOCAL_INV) {
934 struct siw_sqe *rreq;
935
936 if (unlikely(!qp->attrs.orq_size)) {
937 /* We negotiated not to send READ req's */
938 rv = -EINVAL;
939 goto out;
940 }
941 wqe->sqe.num_sge = 1;
942
943 spin_lock(&qp->orq_lock);
944
945 rreq = orq_get_free(qp);
946 if (rreq) {
947 /*
948 * Make an immediate copy in ORQ to be ready
949 * to process loopback READ reply
950 */
951 siw_read_to_orq(rreq, &wqe->sqe);
952 qp->orq_put++;
953 } else {
954 qp->tx_ctx.orq_fence = 1;
955 rv = 0;
956 }
957 spin_unlock(&qp->orq_lock);
958 }
959
960 /* Clear SQE, can be re-used by application */
961 smp_store_mb(sqe->flags, 0);
962 qp->sq_get++;
963out:
964 if (unlikely(rv < 0)) {
965 siw_dbg_qp(qp, "error %d\n", rv);
966 wqe->wr_status = SIW_WR_IDLE;
967 }
968 return rv;
969}
970
971/*
972 * Must be called with SQ locked.
973 * To avoid complete SQ starvation by constant inbound READ requests,
974 * the active IRQ will not be served after qp->irq_burst, if the
975 * SQ has pending work.
976 */
977int siw_activate_tx(struct siw_qp *qp)
978{
979 struct siw_sqe *irqe;
980 struct siw_wqe *wqe = tx_wqe(qp);
981
982 if (!qp->attrs.irq_size)
983 return siw_activate_tx_from_sq(qp);
984
985 irqe = &qp->irq[qp->irq_get % qp->attrs.irq_size];
986
987 if (!(irqe->flags & SIW_WQE_VALID))
988 return siw_activate_tx_from_sq(qp);
989
990 /*
991 * Avoid local WQE processing starvation in case
992 * of constant inbound READ request stream
993 */
994 if (sq_get_next(qp) && ++qp->irq_burst >= SIW_IRQ_MAXBURST_SQ_ACTIVE) {
995 qp->irq_burst = 0;
996 return siw_activate_tx_from_sq(qp);
997 }
998 memset(wqe->mem, 0, sizeof(*wqe->mem) * SIW_MAX_SGE);
999 wqe->wr_status = SIW_WR_QUEUED;
1000
1001 /* start READ RESPONSE */
1002 wqe->sqe.opcode = SIW_OP_READ_RESPONSE;
1003 wqe->sqe.flags = 0;
1004 if (irqe->num_sge) {
1005 wqe->sqe.num_sge = 1;
1006 wqe->sqe.sge[0].length = irqe->sge[0].length;
1007 wqe->sqe.sge[0].laddr = irqe->sge[0].laddr;
1008 wqe->sqe.sge[0].lkey = irqe->sge[0].lkey;
1009 } else {
1010 wqe->sqe.num_sge = 0;
1011 }
1012
1013 /* Retain original RREQ's message sequence number for
1014 * potential error reporting cases.
1015 */
1016 wqe->sqe.sge[1].length = irqe->sge[1].length;
1017
1018 wqe->sqe.rkey = irqe->rkey;
1019 wqe->sqe.raddr = irqe->raddr;
1020
1021 wqe->processed = 0;
1022 qp->irq_get++;
1023
1024 /* mark current IRQ entry free */
1025 smp_store_mb(irqe->flags, 0);
1026
1027 return 1;
1028}
1029
1030/*
1031 * Check if current CQ state qualifies for calling CQ completion
1032 * handler. Must be called with CQ lock held.
1033 */
1034static bool siw_cq_notify_now(struct siw_cq *cq, u32 flags)
1035{
1036 u32 cq_notify;
1037
1038 if (!cq->base_cq.comp_handler)
1039 return false;
1040
1041 /* Read application shared notification state */
1042 cq_notify = READ_ONCE(cq->notify->flags);
1043
1044 if ((cq_notify & SIW_NOTIFY_NEXT_COMPLETION) ||
1045 ((cq_notify & SIW_NOTIFY_SOLICITED) &&
1046 (flags & SIW_WQE_SOLICITED))) {
1047 /*
1048 * CQ notification is one-shot: Since the
1049 * current CQE causes user notification,
1050 * the CQ gets dis-aremd and must be re-aremd
1051 * by the user for a new notification.
1052 */
1053 WRITE_ONCE(cq->notify->flags, SIW_NOTIFY_NOT);
1054
1055 return true;
1056 }
1057 return false;
1058}
1059
1060int siw_sqe_complete(struct siw_qp *qp, struct siw_sqe *sqe, u32 bytes,
1061 enum siw_wc_status status)
1062{
1063 struct siw_cq *cq = qp->scq;
1064 int rv = 0;
1065
1066 if (cq) {
1067 u32 sqe_flags = sqe->flags;
1068 struct siw_cqe *cqe;
1069 u32 idx;
1070 unsigned long flags;
1071
1072 spin_lock_irqsave(&cq->lock, flags);
1073
1074 idx = cq->cq_put % cq->num_cqe;
1075 cqe = &cq->queue[idx];
1076
1077 if (!READ_ONCE(cqe->flags)) {
1078 bool notify;
1079
1080 cqe->id = sqe->id;
1081 cqe->opcode = sqe->opcode;
1082 cqe->status = status;
1083 cqe->imm_data = 0;
1084 cqe->bytes = bytes;
1085
1086 if (rdma_is_kernel_res(&cq->base_cq.res))
1087 cqe->base_qp = &qp->base_qp;
1088 else
1089 cqe->qp_id = qp_id(qp);
1090
1091 /* mark CQE valid for application */
1092 WRITE_ONCE(cqe->flags, SIW_WQE_VALID);
1093 /* recycle SQE */
1094 smp_store_mb(sqe->flags, 0);
1095
1096 cq->cq_put++;
1097 notify = siw_cq_notify_now(cq, sqe_flags);
1098
1099 spin_unlock_irqrestore(&cq->lock, flags);
1100
1101 if (notify) {
1102 siw_dbg_cq(cq, "Call completion handler\n");
1103 cq->base_cq.comp_handler(&cq->base_cq,
1104 cq->base_cq.cq_context);
1105 }
1106 } else {
1107 spin_unlock_irqrestore(&cq->lock, flags);
1108 rv = -ENOMEM;
1109 siw_cq_event(cq, IB_EVENT_CQ_ERR);
1110 }
1111 } else {
1112 /* recycle SQE */
1113 smp_store_mb(sqe->flags, 0);
1114 }
1115 return rv;
1116}
1117
1118int siw_rqe_complete(struct siw_qp *qp, struct siw_rqe *rqe, u32 bytes,
1119 u32 inval_stag, enum siw_wc_status status)
1120{
1121 struct siw_cq *cq = qp->rcq;
1122 int rv = 0;
1123
1124 if (cq) {
1125 struct siw_cqe *cqe;
1126 u32 idx;
1127 unsigned long flags;
1128
1129 spin_lock_irqsave(&cq->lock, flags);
1130
1131 idx = cq->cq_put % cq->num_cqe;
1132 cqe = &cq->queue[idx];
1133
1134 if (!READ_ONCE(cqe->flags)) {
1135 bool notify;
1136 u8 cqe_flags = SIW_WQE_VALID;
1137
1138 cqe->id = rqe->id;
1139 cqe->opcode = SIW_OP_RECEIVE;
1140 cqe->status = status;
1141 cqe->imm_data = 0;
1142 cqe->bytes = bytes;
1143
1144 if (rdma_is_kernel_res(&cq->base_cq.res)) {
1145 cqe->base_qp = &qp->base_qp;
1146 if (inval_stag) {
1147 cqe_flags |= SIW_WQE_REM_INVAL;
1148 cqe->inval_stag = inval_stag;
1149 }
1150 } else {
1151 cqe->qp_id = qp_id(qp);
1152 }
1153 /* mark CQE valid for application */
1154 WRITE_ONCE(cqe->flags, cqe_flags);
1155 /* recycle RQE */
1156 smp_store_mb(rqe->flags, 0);
1157
1158 cq->cq_put++;
1159 notify = siw_cq_notify_now(cq, SIW_WQE_SIGNALLED);
1160
1161 spin_unlock_irqrestore(&cq->lock, flags);
1162
1163 if (notify) {
1164 siw_dbg_cq(cq, "Call completion handler\n");
1165 cq->base_cq.comp_handler(&cq->base_cq,
1166 cq->base_cq.cq_context);
1167 }
1168 } else {
1169 spin_unlock_irqrestore(&cq->lock, flags);
1170 rv = -ENOMEM;
1171 siw_cq_event(cq, IB_EVENT_CQ_ERR);
1172 }
1173 } else {
1174 /* recycle RQE */
1175 smp_store_mb(rqe->flags, 0);
1176 }
1177 return rv;
1178}
1179
1180/*
1181 * siw_sq_flush()
1182 *
1183 * Flush SQ and ORRQ entries to CQ.
1184 *
1185 * Must be called with QP state write lock held.
1186 * Therefore, SQ and ORQ lock must not be taken.
1187 */
1188void siw_sq_flush(struct siw_qp *qp)
1189{
1190 struct siw_sqe *sqe;
1191 struct siw_wqe *wqe = tx_wqe(qp);
1192 int async_event = 0;
1193
1194 /*
1195 * Start with completing any work currently on the ORQ
1196 */
1197 while (qp->attrs.orq_size) {
1198 sqe = &qp->orq[qp->orq_get % qp->attrs.orq_size];
1199 if (!READ_ONCE(sqe->flags))
1200 break;
1201
1202 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1203 break;
1204
1205 WRITE_ONCE(sqe->flags, 0);
1206 qp->orq_get++;
1207 }
1208 /*
1209 * Flush an in-progress WQE if present
1210 */
1211 if (wqe->wr_status != SIW_WR_IDLE) {
1212 siw_dbg_qp(qp, "flush current SQE, type %d, status %d\n",
1213 tx_type(wqe), wqe->wr_status);
1214
1215 siw_wqe_put_mem(wqe, tx_type(wqe));
1216
1217 if (tx_type(wqe) != SIW_OP_READ_RESPONSE &&
1218 ((tx_type(wqe) != SIW_OP_READ &&
1219 tx_type(wqe) != SIW_OP_READ_LOCAL_INV) ||
1220 wqe->wr_status == SIW_WR_QUEUED))
1221 /*
1222 * An in-progress Read Request is already in
1223 * the ORQ
1224 */
1225 siw_sqe_complete(qp, &wqe->sqe, wqe->bytes,
1226 SIW_WC_WR_FLUSH_ERR);
1227
1228 wqe->wr_status = SIW_WR_IDLE;
1229 }
1230 /*
1231 * Flush the Send Queue
1232 */
1233 while (qp->attrs.sq_size) {
1234 sqe = &qp->sendq[qp->sq_get % qp->attrs.sq_size];
1235 if (!READ_ONCE(sqe->flags))
1236 break;
1237
1238 async_event = 1;
1239 if (siw_sqe_complete(qp, sqe, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1240 /*
1241 * Shall IB_EVENT_SQ_DRAINED be supressed if work
1242 * completion fails?
1243 */
1244 break;
1245
1246 WRITE_ONCE(sqe->flags, 0);
1247 qp->sq_get++;
1248 }
1249 if (async_event)
1250 siw_qp_event(qp, IB_EVENT_SQ_DRAINED);
1251}
1252
1253/*
1254 * siw_rq_flush()
1255 *
1256 * Flush recv queue entries to CQ. Also
1257 * takes care of pending active tagged and untagged
1258 * inbound transfers, which have target memory
1259 * referenced.
1260 *
1261 * Must be called with QP state write lock held.
1262 * Therefore, RQ lock must not be taken.
1263 */
1264void siw_rq_flush(struct siw_qp *qp)
1265{
1266 struct siw_wqe *wqe = &qp->rx_untagged.wqe_active;
1267
1268 /*
1269 * Flush an in-progress untagged operation if present
1270 */
1271 if (wqe->wr_status != SIW_WR_IDLE) {
1272 siw_dbg_qp(qp, "flush current rqe, type %d, status %d\n",
1273 rx_type(wqe), wqe->wr_status);
1274
1275 siw_wqe_put_mem(wqe, rx_type(wqe));
1276
1277 if (rx_type(wqe) == SIW_OP_RECEIVE) {
1278 siw_rqe_complete(qp, &wqe->rqe, wqe->bytes,
1279 0, SIW_WC_WR_FLUSH_ERR);
1280 } else if (rx_type(wqe) != SIW_OP_READ &&
1281 rx_type(wqe) != SIW_OP_READ_RESPONSE &&
1282 rx_type(wqe) != SIW_OP_WRITE) {
1283 siw_sqe_complete(qp, &wqe->sqe, 0, SIW_WC_WR_FLUSH_ERR);
1284 }
1285 wqe->wr_status = SIW_WR_IDLE;
1286 }
1287 wqe = &qp->rx_tagged.wqe_active;
1288
1289 if (wqe->wr_status != SIW_WR_IDLE) {
1290 siw_wqe_put_mem(wqe, rx_type(wqe));
1291 wqe->wr_status = SIW_WR_IDLE;
1292 }
1293 /*
1294 * Flush the Receive Queue
1295 */
1296 while (qp->attrs.rq_size) {
1297 struct siw_rqe *rqe =
1298 &qp->recvq[qp->rq_get % qp->attrs.rq_size];
1299
1300 if (!READ_ONCE(rqe->flags))
1301 break;
1302
1303 if (siw_rqe_complete(qp, rqe, 0, 0, SIW_WC_WR_FLUSH_ERR) != 0)
1304 break;
1305
1306 WRITE_ONCE(rqe->flags, 0);
1307 qp->rq_get++;
1308 }
1309}
1310
1311int siw_qp_add(struct siw_device *sdev, struct siw_qp *qp)
1312{
1313 int rv = xa_alloc(&sdev->qp_xa, &qp->base_qp.qp_num, qp, xa_limit_32b,
1314 GFP_KERNEL);
1315
1316 if (!rv) {
1317 kref_init(&qp->ref);
1318 qp->sdev = sdev;
1319 siw_dbg_qp(qp, "new QP\n");
1320 }
1321 return rv;
1322}
1323
1324void siw_free_qp(struct kref *ref)
1325{
1326 struct siw_qp *found, *qp = container_of(ref, struct siw_qp, ref);
1327 struct siw_device *sdev = qp->sdev;
1328 unsigned long flags;
1329
1330 if (qp->cep)
1331 siw_cep_put(qp->cep);
1332
1333 found = xa_erase(&sdev->qp_xa, qp_id(qp));
1334 WARN_ON(found != qp);
1335 spin_lock_irqsave(&sdev->lock, flags);
1336 list_del(&qp->devq);
1337 spin_unlock_irqrestore(&sdev->lock, flags);
1338
1339 vfree(qp->sendq);
1340 vfree(qp->recvq);
1341 vfree(qp->irq);
1342 vfree(qp->orq);
1343
1344 siw_put_tx_cpu(qp->tx_cpu);
1345 complete(&qp->qp_free);
1346 atomic_dec(&sdev->num_qp);
1347}