Loading...
Note: File does not exist in v5.9.
1/*
2 * Copyright (c) 2004 Topspin Communications. All rights reserved.
3 * Copyright (c) 2005 Cisco Systems. All rights reserved.
4 * Copyright (c) 2005 Mellanox Technologies. All rights reserved.
5 * Copyright (c) 2004 Voltaire, Inc. All rights reserved.
6 * Copyright (c) 2005 Open Grid Computing, Inc. All rights reserved.
7 *
8 * This software is available to you under a choice of one of two
9 * licenses. You may choose to be licensed under the terms of the GNU
10 * General Public License (GPL) Version 2, available from the file
11 * COPYING in the main directory of this source tree, or the
12 * OpenIB.org BSD license below:
13 *
14 * Redistribution and use in source and binary forms, with or
15 * without modification, are permitted provided that the following
16 * conditions are met:
17 *
18 * - Redistributions of source code must retain the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer.
21 *
22 * - Redistributions in binary form must reproduce the above
23 * copyright notice, this list of conditions and the following
24 * disclaimer in the documentation and/or other materials
25 * provided with the distribution.
26 *
27 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
28 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
29 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
30 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
31 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
32 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
33 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
34 * SOFTWARE.
35 *
36 */
37
38#include <linux/delay.h>
39#include <linux/gfp.h>
40
41#include "c2.h"
42#include "c2_vq.h"
43#include "c2_status.h"
44
45#define C2_MAX_ORD_PER_QP 128
46#define C2_MAX_IRD_PER_QP 128
47
48#define C2_HINT_MAKE(q_index, hint_count) (((q_index) << 16) | hint_count)
49#define C2_HINT_GET_INDEX(hint) (((hint) & 0x7FFF0000) >> 16)
50#define C2_HINT_GET_COUNT(hint) ((hint) & 0x0000FFFF)
51
52#define NO_SUPPORT -1
53static const u8 c2_opcode[] = {
54 [IB_WR_SEND] = C2_WR_TYPE_SEND,
55 [IB_WR_SEND_WITH_IMM] = NO_SUPPORT,
56 [IB_WR_RDMA_WRITE] = C2_WR_TYPE_RDMA_WRITE,
57 [IB_WR_RDMA_WRITE_WITH_IMM] = NO_SUPPORT,
58 [IB_WR_RDMA_READ] = C2_WR_TYPE_RDMA_READ,
59 [IB_WR_ATOMIC_CMP_AND_SWP] = NO_SUPPORT,
60 [IB_WR_ATOMIC_FETCH_AND_ADD] = NO_SUPPORT,
61};
62
63static int to_c2_state(enum ib_qp_state ib_state)
64{
65 switch (ib_state) {
66 case IB_QPS_RESET:
67 return C2_QP_STATE_IDLE;
68 case IB_QPS_RTS:
69 return C2_QP_STATE_RTS;
70 case IB_QPS_SQD:
71 return C2_QP_STATE_CLOSING;
72 case IB_QPS_SQE:
73 return C2_QP_STATE_CLOSING;
74 case IB_QPS_ERR:
75 return C2_QP_STATE_ERROR;
76 default:
77 return -1;
78 }
79}
80
81static int to_ib_state(enum c2_qp_state c2_state)
82{
83 switch (c2_state) {
84 case C2_QP_STATE_IDLE:
85 return IB_QPS_RESET;
86 case C2_QP_STATE_CONNECTING:
87 return IB_QPS_RTR;
88 case C2_QP_STATE_RTS:
89 return IB_QPS_RTS;
90 case C2_QP_STATE_CLOSING:
91 return IB_QPS_SQD;
92 case C2_QP_STATE_ERROR:
93 return IB_QPS_ERR;
94 case C2_QP_STATE_TERMINATE:
95 return IB_QPS_SQE;
96 default:
97 return -1;
98 }
99}
100
101static const char *to_ib_state_str(int ib_state)
102{
103 static const char *state_str[] = {
104 "IB_QPS_RESET",
105 "IB_QPS_INIT",
106 "IB_QPS_RTR",
107 "IB_QPS_RTS",
108 "IB_QPS_SQD",
109 "IB_QPS_SQE",
110 "IB_QPS_ERR"
111 };
112 if (ib_state < IB_QPS_RESET ||
113 ib_state > IB_QPS_ERR)
114 return "<invalid IB QP state>";
115
116 ib_state -= IB_QPS_RESET;
117 return state_str[ib_state];
118}
119
120void c2_set_qp_state(struct c2_qp *qp, int c2_state)
121{
122 int new_state = to_ib_state(c2_state);
123
124 pr_debug("%s: qp[%p] state modify %s --> %s\n",
125 __func__,
126 qp,
127 to_ib_state_str(qp->state),
128 to_ib_state_str(new_state));
129 qp->state = new_state;
130}
131
132#define C2_QP_NO_ATTR_CHANGE 0xFFFFFFFF
133
134int c2_qp_modify(struct c2_dev *c2dev, struct c2_qp *qp,
135 struct ib_qp_attr *attr, int attr_mask)
136{
137 struct c2wr_qp_modify_req wr;
138 struct c2wr_qp_modify_rep *reply;
139 struct c2_vq_req *vq_req;
140 unsigned long flags;
141 u8 next_state;
142 int err;
143
144 pr_debug("%s:%d qp=%p, %s --> %s\n",
145 __func__, __LINE__,
146 qp,
147 to_ib_state_str(qp->state),
148 to_ib_state_str(attr->qp_state));
149
150 vq_req = vq_req_alloc(c2dev);
151 if (!vq_req)
152 return -ENOMEM;
153
154 c2_wr_set_id(&wr, CCWR_QP_MODIFY);
155 wr.hdr.context = (unsigned long) vq_req;
156 wr.rnic_handle = c2dev->adapter_handle;
157 wr.qp_handle = qp->adapter_handle;
158 wr.ord = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
159 wr.ird = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
160 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
161 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
162
163 if (attr_mask & IB_QP_STATE) {
164 /* Ensure the state is valid */
165 if (attr->qp_state < 0 || attr->qp_state > IB_QPS_ERR) {
166 err = -EINVAL;
167 goto bail0;
168 }
169
170 wr.next_qp_state = cpu_to_be32(to_c2_state(attr->qp_state));
171
172 if (attr->qp_state == IB_QPS_ERR) {
173 spin_lock_irqsave(&qp->lock, flags);
174 if (qp->cm_id && qp->state == IB_QPS_RTS) {
175 pr_debug("Generating CLOSE event for QP-->ERR, "
176 "qp=%p, cm_id=%p\n",qp,qp->cm_id);
177 /* Generate an CLOSE event */
178 vq_req->cm_id = qp->cm_id;
179 vq_req->event = IW_CM_EVENT_CLOSE;
180 }
181 spin_unlock_irqrestore(&qp->lock, flags);
182 }
183 next_state = attr->qp_state;
184
185 } else if (attr_mask & IB_QP_CUR_STATE) {
186
187 if (attr->cur_qp_state != IB_QPS_RTR &&
188 attr->cur_qp_state != IB_QPS_RTS &&
189 attr->cur_qp_state != IB_QPS_SQD &&
190 attr->cur_qp_state != IB_QPS_SQE) {
191 err = -EINVAL;
192 goto bail0;
193 } else
194 wr.next_qp_state =
195 cpu_to_be32(to_c2_state(attr->cur_qp_state));
196
197 next_state = attr->cur_qp_state;
198
199 } else {
200 err = 0;
201 goto bail0;
202 }
203
204 /* reference the request struct */
205 vq_req_get(c2dev, vq_req);
206
207 err = vq_send_wr(c2dev, (union c2wr *) & wr);
208 if (err) {
209 vq_req_put(c2dev, vq_req);
210 goto bail0;
211 }
212
213 err = vq_wait_for_reply(c2dev, vq_req);
214 if (err)
215 goto bail0;
216
217 reply = (struct c2wr_qp_modify_rep *) (unsigned long) vq_req->reply_msg;
218 if (!reply) {
219 err = -ENOMEM;
220 goto bail0;
221 }
222
223 err = c2_errno(reply);
224 if (!err)
225 qp->state = next_state;
226#ifdef DEBUG
227 else
228 pr_debug("%s: c2_errno=%d\n", __func__, err);
229#endif
230 /*
231 * If we're going to error and generating the event here, then
232 * we need to remove the reference because there will be no
233 * close event generated by the adapter
234 */
235 spin_lock_irqsave(&qp->lock, flags);
236 if (vq_req->event==IW_CM_EVENT_CLOSE && qp->cm_id) {
237 qp->cm_id->rem_ref(qp->cm_id);
238 qp->cm_id = NULL;
239 }
240 spin_unlock_irqrestore(&qp->lock, flags);
241
242 vq_repbuf_free(c2dev, reply);
243 bail0:
244 vq_req_free(c2dev, vq_req);
245
246 pr_debug("%s:%d qp=%p, cur_state=%s\n",
247 __func__, __LINE__,
248 qp,
249 to_ib_state_str(qp->state));
250 return err;
251}
252
253int c2_qp_set_read_limits(struct c2_dev *c2dev, struct c2_qp *qp,
254 int ord, int ird)
255{
256 struct c2wr_qp_modify_req wr;
257 struct c2wr_qp_modify_rep *reply;
258 struct c2_vq_req *vq_req;
259 int err;
260
261 vq_req = vq_req_alloc(c2dev);
262 if (!vq_req)
263 return -ENOMEM;
264
265 c2_wr_set_id(&wr, CCWR_QP_MODIFY);
266 wr.hdr.context = (unsigned long) vq_req;
267 wr.rnic_handle = c2dev->adapter_handle;
268 wr.qp_handle = qp->adapter_handle;
269 wr.ord = cpu_to_be32(ord);
270 wr.ird = cpu_to_be32(ird);
271 wr.sq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
272 wr.rq_depth = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
273 wr.next_qp_state = cpu_to_be32(C2_QP_NO_ATTR_CHANGE);
274
275 /* reference the request struct */
276 vq_req_get(c2dev, vq_req);
277
278 err = vq_send_wr(c2dev, (union c2wr *) & wr);
279 if (err) {
280 vq_req_put(c2dev, vq_req);
281 goto bail0;
282 }
283
284 err = vq_wait_for_reply(c2dev, vq_req);
285 if (err)
286 goto bail0;
287
288 reply = (struct c2wr_qp_modify_rep *) (unsigned long)
289 vq_req->reply_msg;
290 if (!reply) {
291 err = -ENOMEM;
292 goto bail0;
293 }
294
295 err = c2_errno(reply);
296 vq_repbuf_free(c2dev, reply);
297 bail0:
298 vq_req_free(c2dev, vq_req);
299 return err;
300}
301
302static int destroy_qp(struct c2_dev *c2dev, struct c2_qp *qp)
303{
304 struct c2_vq_req *vq_req;
305 struct c2wr_qp_destroy_req wr;
306 struct c2wr_qp_destroy_rep *reply;
307 unsigned long flags;
308 int err;
309
310 /*
311 * Allocate a verb request message
312 */
313 vq_req = vq_req_alloc(c2dev);
314 if (!vq_req) {
315 return -ENOMEM;
316 }
317
318 /*
319 * Initialize the WR
320 */
321 c2_wr_set_id(&wr, CCWR_QP_DESTROY);
322 wr.hdr.context = (unsigned long) vq_req;
323 wr.rnic_handle = c2dev->adapter_handle;
324 wr.qp_handle = qp->adapter_handle;
325
326 /*
327 * reference the request struct. dereferenced in the int handler.
328 */
329 vq_req_get(c2dev, vq_req);
330
331 spin_lock_irqsave(&qp->lock, flags);
332 if (qp->cm_id && qp->state == IB_QPS_RTS) {
333 pr_debug("destroy_qp: generating CLOSE event for QP-->ERR, "
334 "qp=%p, cm_id=%p\n",qp,qp->cm_id);
335 /* Generate an CLOSE event */
336 vq_req->qp = qp;
337 vq_req->cm_id = qp->cm_id;
338 vq_req->event = IW_CM_EVENT_CLOSE;
339 }
340 spin_unlock_irqrestore(&qp->lock, flags);
341
342 /*
343 * Send WR to adapter
344 */
345 err = vq_send_wr(c2dev, (union c2wr *) & wr);
346 if (err) {
347 vq_req_put(c2dev, vq_req);
348 goto bail0;
349 }
350
351 /*
352 * Wait for reply from adapter
353 */
354 err = vq_wait_for_reply(c2dev, vq_req);
355 if (err) {
356 goto bail0;
357 }
358
359 /*
360 * Process reply
361 */
362 reply = (struct c2wr_qp_destroy_rep *) (unsigned long) (vq_req->reply_msg);
363 if (!reply) {
364 err = -ENOMEM;
365 goto bail0;
366 }
367
368 spin_lock_irqsave(&qp->lock, flags);
369 if (qp->cm_id) {
370 qp->cm_id->rem_ref(qp->cm_id);
371 qp->cm_id = NULL;
372 }
373 spin_unlock_irqrestore(&qp->lock, flags);
374
375 vq_repbuf_free(c2dev, reply);
376 bail0:
377 vq_req_free(c2dev, vq_req);
378 return err;
379}
380
381static int c2_alloc_qpn(struct c2_dev *c2dev, struct c2_qp *qp)
382{
383 int ret;
384
385 do {
386 spin_lock_irq(&c2dev->qp_table.lock);
387 ret = idr_get_new_above(&c2dev->qp_table.idr, qp,
388 c2dev->qp_table.last++, &qp->qpn);
389 spin_unlock_irq(&c2dev->qp_table.lock);
390 } while ((ret == -EAGAIN) &&
391 idr_pre_get(&c2dev->qp_table.idr, GFP_KERNEL));
392 return ret;
393}
394
395static void c2_free_qpn(struct c2_dev *c2dev, int qpn)
396{
397 spin_lock_irq(&c2dev->qp_table.lock);
398 idr_remove(&c2dev->qp_table.idr, qpn);
399 spin_unlock_irq(&c2dev->qp_table.lock);
400}
401
402struct c2_qp *c2_find_qpn(struct c2_dev *c2dev, int qpn)
403{
404 unsigned long flags;
405 struct c2_qp *qp;
406
407 spin_lock_irqsave(&c2dev->qp_table.lock, flags);
408 qp = idr_find(&c2dev->qp_table.idr, qpn);
409 spin_unlock_irqrestore(&c2dev->qp_table.lock, flags);
410 return qp;
411}
412
413int c2_alloc_qp(struct c2_dev *c2dev,
414 struct c2_pd *pd,
415 struct ib_qp_init_attr *qp_attrs, struct c2_qp *qp)
416{
417 struct c2wr_qp_create_req wr;
418 struct c2wr_qp_create_rep *reply;
419 struct c2_vq_req *vq_req;
420 struct c2_cq *send_cq = to_c2cq(qp_attrs->send_cq);
421 struct c2_cq *recv_cq = to_c2cq(qp_attrs->recv_cq);
422 unsigned long peer_pa;
423 u32 q_size, msg_size, mmap_size;
424 void __iomem *mmap;
425 int err;
426
427 err = c2_alloc_qpn(c2dev, qp);
428 if (err)
429 return err;
430 qp->ibqp.qp_num = qp->qpn;
431 qp->ibqp.qp_type = IB_QPT_RC;
432
433 /* Allocate the SQ and RQ shared pointers */
434 qp->sq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
435 &qp->sq_mq.shared_dma, GFP_KERNEL);
436 if (!qp->sq_mq.shared) {
437 err = -ENOMEM;
438 goto bail0;
439 }
440
441 qp->rq_mq.shared = c2_alloc_mqsp(c2dev, c2dev->kern_mqsp_pool,
442 &qp->rq_mq.shared_dma, GFP_KERNEL);
443 if (!qp->rq_mq.shared) {
444 err = -ENOMEM;
445 goto bail1;
446 }
447
448 /* Allocate the verbs request */
449 vq_req = vq_req_alloc(c2dev);
450 if (vq_req == NULL) {
451 err = -ENOMEM;
452 goto bail2;
453 }
454
455 /* Initialize the work request */
456 memset(&wr, 0, sizeof(wr));
457 c2_wr_set_id(&wr, CCWR_QP_CREATE);
458 wr.hdr.context = (unsigned long) vq_req;
459 wr.rnic_handle = c2dev->adapter_handle;
460 wr.sq_cq_handle = send_cq->adapter_handle;
461 wr.rq_cq_handle = recv_cq->adapter_handle;
462 wr.sq_depth = cpu_to_be32(qp_attrs->cap.max_send_wr + 1);
463 wr.rq_depth = cpu_to_be32(qp_attrs->cap.max_recv_wr + 1);
464 wr.srq_handle = 0;
465 wr.flags = cpu_to_be32(QP_RDMA_READ | QP_RDMA_WRITE | QP_MW_BIND |
466 QP_ZERO_STAG | QP_RDMA_READ_RESPONSE);
467 wr.send_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
468 wr.recv_sgl_depth = cpu_to_be32(qp_attrs->cap.max_recv_sge);
469 wr.rdma_write_sgl_depth = cpu_to_be32(qp_attrs->cap.max_send_sge);
470 wr.shared_sq_ht = cpu_to_be64(qp->sq_mq.shared_dma);
471 wr.shared_rq_ht = cpu_to_be64(qp->rq_mq.shared_dma);
472 wr.ord = cpu_to_be32(C2_MAX_ORD_PER_QP);
473 wr.ird = cpu_to_be32(C2_MAX_IRD_PER_QP);
474 wr.pd_id = pd->pd_id;
475 wr.user_context = (unsigned long) qp;
476
477 vq_req_get(c2dev, vq_req);
478
479 /* Send the WR to the adapter */
480 err = vq_send_wr(c2dev, (union c2wr *) & wr);
481 if (err) {
482 vq_req_put(c2dev, vq_req);
483 goto bail3;
484 }
485
486 /* Wait for the verb reply */
487 err = vq_wait_for_reply(c2dev, vq_req);
488 if (err) {
489 goto bail3;
490 }
491
492 /* Process the reply */
493 reply = (struct c2wr_qp_create_rep *) (unsigned long) (vq_req->reply_msg);
494 if (!reply) {
495 err = -ENOMEM;
496 goto bail3;
497 }
498
499 if ((err = c2_wr_get_result(reply)) != 0) {
500 goto bail4;
501 }
502
503 /* Fill in the kernel QP struct */
504 atomic_set(&qp->refcount, 1);
505 qp->adapter_handle = reply->qp_handle;
506 qp->state = IB_QPS_RESET;
507 qp->send_sgl_depth = qp_attrs->cap.max_send_sge;
508 qp->rdma_write_sgl_depth = qp_attrs->cap.max_send_sge;
509 qp->recv_sgl_depth = qp_attrs->cap.max_recv_sge;
510 init_waitqueue_head(&qp->wait);
511
512 /* Initialize the SQ MQ */
513 q_size = be32_to_cpu(reply->sq_depth);
514 msg_size = be32_to_cpu(reply->sq_msg_size);
515 peer_pa = c2dev->pa + be32_to_cpu(reply->sq_mq_start);
516 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
517 mmap = ioremap_nocache(peer_pa, mmap_size);
518 if (!mmap) {
519 err = -ENOMEM;
520 goto bail5;
521 }
522
523 c2_mq_req_init(&qp->sq_mq,
524 be32_to_cpu(reply->sq_mq_index),
525 q_size,
526 msg_size,
527 mmap + sizeof(struct c2_mq_shared), /* pool start */
528 mmap, /* peer */
529 C2_MQ_ADAPTER_TARGET);
530
531 /* Initialize the RQ mq */
532 q_size = be32_to_cpu(reply->rq_depth);
533 msg_size = be32_to_cpu(reply->rq_msg_size);
534 peer_pa = c2dev->pa + be32_to_cpu(reply->rq_mq_start);
535 mmap_size = PAGE_ALIGN(sizeof(struct c2_mq_shared) + msg_size * q_size);
536 mmap = ioremap_nocache(peer_pa, mmap_size);
537 if (!mmap) {
538 err = -ENOMEM;
539 goto bail6;
540 }
541
542 c2_mq_req_init(&qp->rq_mq,
543 be32_to_cpu(reply->rq_mq_index),
544 q_size,
545 msg_size,
546 mmap + sizeof(struct c2_mq_shared), /* pool start */
547 mmap, /* peer */
548 C2_MQ_ADAPTER_TARGET);
549
550 vq_repbuf_free(c2dev, reply);
551 vq_req_free(c2dev, vq_req);
552
553 return 0;
554
555 bail6:
556 iounmap(qp->sq_mq.peer);
557 bail5:
558 destroy_qp(c2dev, qp);
559 bail4:
560 vq_repbuf_free(c2dev, reply);
561 bail3:
562 vq_req_free(c2dev, vq_req);
563 bail2:
564 c2_free_mqsp(qp->rq_mq.shared);
565 bail1:
566 c2_free_mqsp(qp->sq_mq.shared);
567 bail0:
568 c2_free_qpn(c2dev, qp->qpn);
569 return err;
570}
571
572static inline void c2_lock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
573{
574 if (send_cq == recv_cq)
575 spin_lock_irq(&send_cq->lock);
576 else if (send_cq > recv_cq) {
577 spin_lock_irq(&send_cq->lock);
578 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
579 } else {
580 spin_lock_irq(&recv_cq->lock);
581 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
582 }
583}
584
585static inline void c2_unlock_cqs(struct c2_cq *send_cq, struct c2_cq *recv_cq)
586{
587 if (send_cq == recv_cq)
588 spin_unlock_irq(&send_cq->lock);
589 else if (send_cq > recv_cq) {
590 spin_unlock(&recv_cq->lock);
591 spin_unlock_irq(&send_cq->lock);
592 } else {
593 spin_unlock(&send_cq->lock);
594 spin_unlock_irq(&recv_cq->lock);
595 }
596}
597
598void c2_free_qp(struct c2_dev *c2dev, struct c2_qp *qp)
599{
600 struct c2_cq *send_cq;
601 struct c2_cq *recv_cq;
602
603 send_cq = to_c2cq(qp->ibqp.send_cq);
604 recv_cq = to_c2cq(qp->ibqp.recv_cq);
605
606 /*
607 * Lock CQs here, so that CQ polling code can do QP lookup
608 * without taking a lock.
609 */
610 c2_lock_cqs(send_cq, recv_cq);
611 c2_free_qpn(c2dev, qp->qpn);
612 c2_unlock_cqs(send_cq, recv_cq);
613
614 /*
615 * Destroy qp in the rnic...
616 */
617 destroy_qp(c2dev, qp);
618
619 /*
620 * Mark any unreaped CQEs as null and void.
621 */
622 c2_cq_clean(c2dev, qp, send_cq->cqn);
623 if (send_cq != recv_cq)
624 c2_cq_clean(c2dev, qp, recv_cq->cqn);
625 /*
626 * Unmap the MQs and return the shared pointers
627 * to the message pool.
628 */
629 iounmap(qp->sq_mq.peer);
630 iounmap(qp->rq_mq.peer);
631 c2_free_mqsp(qp->sq_mq.shared);
632 c2_free_mqsp(qp->rq_mq.shared);
633
634 atomic_dec(&qp->refcount);
635 wait_event(qp->wait, !atomic_read(&qp->refcount));
636}
637
638/*
639 * Function: move_sgl
640 *
641 * Description:
642 * Move an SGL from the user's work request struct into a CCIL Work Request
643 * message, swapping to WR byte order and ensure the total length doesn't
644 * overflow.
645 *
646 * IN:
647 * dst - ptr to CCIL Work Request message SGL memory.
648 * src - ptr to the consumers SGL memory.
649 *
650 * OUT: none
651 *
652 * Return:
653 * CCIL status codes.
654 */
655static int
656move_sgl(struct c2_data_addr * dst, struct ib_sge *src, int count, u32 * p_len,
657 u8 * actual_count)
658{
659 u32 tot = 0; /* running total */
660 u8 acount = 0; /* running total non-0 len sge's */
661
662 while (count > 0) {
663 /*
664 * If the addition of this SGE causes the
665 * total SGL length to exceed 2^32-1, then
666 * fail-n-bail.
667 *
668 * If the current total plus the next element length
669 * wraps, then it will go negative and be less than the
670 * current total...
671 */
672 if ((tot + src->length) < tot) {
673 return -EINVAL;
674 }
675 /*
676 * Bug: 1456 (as well as 1498 & 1643)
677 * Skip over any sge's supplied with len=0
678 */
679 if (src->length) {
680 tot += src->length;
681 dst->stag = cpu_to_be32(src->lkey);
682 dst->to = cpu_to_be64(src->addr);
683 dst->length = cpu_to_be32(src->length);
684 dst++;
685 acount++;
686 }
687 src++;
688 count--;
689 }
690
691 if (acount == 0) {
692 /*
693 * Bug: 1476 (as well as 1498, 1456 and 1643)
694 * Setup the SGL in the WR to make it easier for the RNIC.
695 * This way, the FW doesn't have to deal with special cases.
696 * Setting length=0 should be sufficient.
697 */
698 dst->stag = 0;
699 dst->to = 0;
700 dst->length = 0;
701 }
702
703 *p_len = tot;
704 *actual_count = acount;
705 return 0;
706}
707
708/*
709 * Function: c2_activity (private function)
710 *
711 * Description:
712 * Post an mq index to the host->adapter activity fifo.
713 *
714 * IN:
715 * c2dev - ptr to c2dev structure
716 * mq_index - mq index to post
717 * shared - value most recently written to shared
718 *
719 * OUT:
720 *
721 * Return:
722 * none
723 */
724static inline void c2_activity(struct c2_dev *c2dev, u32 mq_index, u16 shared)
725{
726 /*
727 * First read the register to see if the FIFO is full, and if so,
728 * spin until it's not. This isn't perfect -- there is no
729 * synchronization among the clients of the register, but in
730 * practice it prevents multiple CPU from hammering the bus
731 * with PCI RETRY. Note that when this does happen, the card
732 * cannot get on the bus and the card and system hang in a
733 * deadlock -- thus the need for this code. [TOT]
734 */
735 while (readl(c2dev->regs + PCI_BAR0_ADAPTER_HINT) & 0x80000000)
736 udelay(10);
737
738 __raw_writel(C2_HINT_MAKE(mq_index, shared),
739 c2dev->regs + PCI_BAR0_ADAPTER_HINT);
740}
741
742/*
743 * Function: qp_wr_post
744 *
745 * Description:
746 * This in-line function allocates a MQ msg, then moves the host-copy of
747 * the completed WR into msg. Then it posts the message.
748 *
749 * IN:
750 * q - ptr to user MQ.
751 * wr - ptr to host-copy of the WR.
752 * qp - ptr to user qp
753 * size - Number of bytes to post. Assumed to be divisible by 4.
754 *
755 * OUT: none
756 *
757 * Return:
758 * CCIL status codes.
759 */
760static int qp_wr_post(struct c2_mq *q, union c2wr * wr, struct c2_qp *qp, u32 size)
761{
762 union c2wr *msg;
763
764 msg = c2_mq_alloc(q);
765 if (msg == NULL) {
766 return -EINVAL;
767 }
768#ifdef CCMSGMAGIC
769 ((c2wr_hdr_t *) wr)->magic = cpu_to_be32(CCWR_MAGIC);
770#endif
771
772 /*
773 * Since all header fields in the WR are the same as the
774 * CQE, set the following so the adapter need not.
775 */
776 c2_wr_set_result(wr, CCERR_PENDING);
777
778 /*
779 * Copy the wr down to the adapter
780 */
781 memcpy((void *) msg, (void *) wr, size);
782
783 c2_mq_produce(q);
784 return 0;
785}
786
787
788int c2_post_send(struct ib_qp *ibqp, struct ib_send_wr *ib_wr,
789 struct ib_send_wr **bad_wr)
790{
791 struct c2_dev *c2dev = to_c2dev(ibqp->device);
792 struct c2_qp *qp = to_c2qp(ibqp);
793 union c2wr wr;
794 unsigned long lock_flags;
795 int err = 0;
796
797 u32 flags;
798 u32 tot_len;
799 u8 actual_sge_count;
800 u32 msg_size;
801
802 if (qp->state > IB_QPS_RTS) {
803 err = -EINVAL;
804 goto out;
805 }
806
807 while (ib_wr) {
808
809 flags = 0;
810 wr.sqwr.sq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
811 if (ib_wr->send_flags & IB_SEND_SIGNALED) {
812 flags |= SQ_SIGNALED;
813 }
814
815 switch (ib_wr->opcode) {
816 case IB_WR_SEND:
817 case IB_WR_SEND_WITH_INV:
818 if (ib_wr->opcode == IB_WR_SEND) {
819 if (ib_wr->send_flags & IB_SEND_SOLICITED)
820 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE);
821 else
822 c2_wr_set_id(&wr, C2_WR_TYPE_SEND);
823 wr.sqwr.send.remote_stag = 0;
824 } else {
825 if (ib_wr->send_flags & IB_SEND_SOLICITED)
826 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_SE_INV);
827 else
828 c2_wr_set_id(&wr, C2_WR_TYPE_SEND_INV);
829 wr.sqwr.send.remote_stag =
830 cpu_to_be32(ib_wr->ex.invalidate_rkey);
831 }
832
833 msg_size = sizeof(struct c2wr_send_req) +
834 sizeof(struct c2_data_addr) * ib_wr->num_sge;
835 if (ib_wr->num_sge > qp->send_sgl_depth) {
836 err = -EINVAL;
837 break;
838 }
839 if (ib_wr->send_flags & IB_SEND_FENCE) {
840 flags |= SQ_READ_FENCE;
841 }
842 err = move_sgl((struct c2_data_addr *) & (wr.sqwr.send.data),
843 ib_wr->sg_list,
844 ib_wr->num_sge,
845 &tot_len, &actual_sge_count);
846 wr.sqwr.send.sge_len = cpu_to_be32(tot_len);
847 c2_wr_set_sge_count(&wr, actual_sge_count);
848 break;
849 case IB_WR_RDMA_WRITE:
850 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_WRITE);
851 msg_size = sizeof(struct c2wr_rdma_write_req) +
852 (sizeof(struct c2_data_addr) * ib_wr->num_sge);
853 if (ib_wr->num_sge > qp->rdma_write_sgl_depth) {
854 err = -EINVAL;
855 break;
856 }
857 if (ib_wr->send_flags & IB_SEND_FENCE) {
858 flags |= SQ_READ_FENCE;
859 }
860 wr.sqwr.rdma_write.remote_stag =
861 cpu_to_be32(ib_wr->wr.rdma.rkey);
862 wr.sqwr.rdma_write.remote_to =
863 cpu_to_be64(ib_wr->wr.rdma.remote_addr);
864 err = move_sgl((struct c2_data_addr *)
865 & (wr.sqwr.rdma_write.data),
866 ib_wr->sg_list,
867 ib_wr->num_sge,
868 &tot_len, &actual_sge_count);
869 wr.sqwr.rdma_write.sge_len = cpu_to_be32(tot_len);
870 c2_wr_set_sge_count(&wr, actual_sge_count);
871 break;
872 case IB_WR_RDMA_READ:
873 c2_wr_set_id(&wr, C2_WR_TYPE_RDMA_READ);
874 msg_size = sizeof(struct c2wr_rdma_read_req);
875
876 /* IWarp only suppots 1 sge for RDMA reads */
877 if (ib_wr->num_sge > 1) {
878 err = -EINVAL;
879 break;
880 }
881
882 /*
883 * Move the local and remote stag/to/len into the WR.
884 */
885 wr.sqwr.rdma_read.local_stag =
886 cpu_to_be32(ib_wr->sg_list->lkey);
887 wr.sqwr.rdma_read.local_to =
888 cpu_to_be64(ib_wr->sg_list->addr);
889 wr.sqwr.rdma_read.remote_stag =
890 cpu_to_be32(ib_wr->wr.rdma.rkey);
891 wr.sqwr.rdma_read.remote_to =
892 cpu_to_be64(ib_wr->wr.rdma.remote_addr);
893 wr.sqwr.rdma_read.length =
894 cpu_to_be32(ib_wr->sg_list->length);
895 break;
896 default:
897 /* error */
898 msg_size = 0;
899 err = -EINVAL;
900 break;
901 }
902
903 /*
904 * If we had an error on the last wr build, then
905 * break out. Possible errors include bogus WR
906 * type, and a bogus SGL length...
907 */
908 if (err) {
909 break;
910 }
911
912 /*
913 * Store flags
914 */
915 c2_wr_set_flags(&wr, flags);
916
917 /*
918 * Post the puppy!
919 */
920 spin_lock_irqsave(&qp->lock, lock_flags);
921 err = qp_wr_post(&qp->sq_mq, &wr, qp, msg_size);
922 if (err) {
923 spin_unlock_irqrestore(&qp->lock, lock_flags);
924 break;
925 }
926
927 /*
928 * Enqueue mq index to activity FIFO.
929 */
930 c2_activity(c2dev, qp->sq_mq.index, qp->sq_mq.hint_count);
931 spin_unlock_irqrestore(&qp->lock, lock_flags);
932
933 ib_wr = ib_wr->next;
934 }
935
936out:
937 if (err)
938 *bad_wr = ib_wr;
939 return err;
940}
941
942int c2_post_receive(struct ib_qp *ibqp, struct ib_recv_wr *ib_wr,
943 struct ib_recv_wr **bad_wr)
944{
945 struct c2_dev *c2dev = to_c2dev(ibqp->device);
946 struct c2_qp *qp = to_c2qp(ibqp);
947 union c2wr wr;
948 unsigned long lock_flags;
949 int err = 0;
950
951 if (qp->state > IB_QPS_RTS) {
952 err = -EINVAL;
953 goto out;
954 }
955
956 /*
957 * Try and post each work request
958 */
959 while (ib_wr) {
960 u32 tot_len;
961 u8 actual_sge_count;
962
963 if (ib_wr->num_sge > qp->recv_sgl_depth) {
964 err = -EINVAL;
965 break;
966 }
967
968 /*
969 * Create local host-copy of the WR
970 */
971 wr.rqwr.rq_hdr.user_hdr.hdr.context = ib_wr->wr_id;
972 c2_wr_set_id(&wr, CCWR_RECV);
973 c2_wr_set_flags(&wr, 0);
974
975 /* sge_count is limited to eight bits. */
976 BUG_ON(ib_wr->num_sge >= 256);
977 err = move_sgl((struct c2_data_addr *) & (wr.rqwr.data),
978 ib_wr->sg_list,
979 ib_wr->num_sge, &tot_len, &actual_sge_count);
980 c2_wr_set_sge_count(&wr, actual_sge_count);
981
982 /*
983 * If we had an error on the last wr build, then
984 * break out. Possible errors include bogus WR
985 * type, and a bogus SGL length...
986 */
987 if (err) {
988 break;
989 }
990
991 spin_lock_irqsave(&qp->lock, lock_flags);
992 err = qp_wr_post(&qp->rq_mq, &wr, qp, qp->rq_mq.msg_size);
993 if (err) {
994 spin_unlock_irqrestore(&qp->lock, lock_flags);
995 break;
996 }
997
998 /*
999 * Enqueue mq index to activity FIFO
1000 */
1001 c2_activity(c2dev, qp->rq_mq.index, qp->rq_mq.hint_count);
1002 spin_unlock_irqrestore(&qp->lock, lock_flags);
1003
1004 ib_wr = ib_wr->next;
1005 }
1006
1007out:
1008 if (err)
1009 *bad_wr = ib_wr;
1010 return err;
1011}
1012
1013void __devinit c2_init_qp_table(struct c2_dev *c2dev)
1014{
1015 spin_lock_init(&c2dev->qp_table.lock);
1016 idr_init(&c2dev->qp_table.idr);
1017}
1018
1019void __devexit c2_cleanup_qp_table(struct c2_dev *c2dev)
1020{
1021 idr_destroy(&c2dev->qp_table.idr);
1022}