Loading...
1/*
2 * Copyright (c) 2006, 2007, 2008 QLogic Corporation. All rights reserved.
3 * Copyright (c) 2005, 2006 PathScale, Inc. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include <linux/err.h>
35#include <linux/sched.h>
36#include <linux/slab.h>
37#include <linux/vmalloc.h>
38
39#include "ipath_verbs.h"
40#include "ipath_kernel.h"
41
42#define BITS_PER_PAGE (PAGE_SIZE*BITS_PER_BYTE)
43#define BITS_PER_PAGE_MASK (BITS_PER_PAGE-1)
44#define mk_qpn(qpt, map, off) (((map) - (qpt)->map) * BITS_PER_PAGE + \
45 (off))
46#define find_next_offset(map, off) find_next_zero_bit((map)->page, \
47 BITS_PER_PAGE, off)
48
49/*
50 * Convert the AETH credit code into the number of credits.
51 */
52static u32 credit_table[31] = {
53 0, /* 0 */
54 1, /* 1 */
55 2, /* 2 */
56 3, /* 3 */
57 4, /* 4 */
58 6, /* 5 */
59 8, /* 6 */
60 12, /* 7 */
61 16, /* 8 */
62 24, /* 9 */
63 32, /* A */
64 48, /* B */
65 64, /* C */
66 96, /* D */
67 128, /* E */
68 192, /* F */
69 256, /* 10 */
70 384, /* 11 */
71 512, /* 12 */
72 768, /* 13 */
73 1024, /* 14 */
74 1536, /* 15 */
75 2048, /* 16 */
76 3072, /* 17 */
77 4096, /* 18 */
78 6144, /* 19 */
79 8192, /* 1A */
80 12288, /* 1B */
81 16384, /* 1C */
82 24576, /* 1D */
83 32768 /* 1E */
84};
85
86
87static void get_map_page(struct ipath_qp_table *qpt, struct qpn_map *map)
88{
89 unsigned long page = get_zeroed_page(GFP_KERNEL);
90 unsigned long flags;
91
92 /*
93 * Free the page if someone raced with us installing it.
94 */
95
96 spin_lock_irqsave(&qpt->lock, flags);
97 if (map->page)
98 free_page(page);
99 else
100 map->page = (void *)page;
101 spin_unlock_irqrestore(&qpt->lock, flags);
102}
103
104
105static int alloc_qpn(struct ipath_qp_table *qpt, enum ib_qp_type type)
106{
107 u32 i, offset, max_scan, qpn;
108 struct qpn_map *map;
109 u32 ret = -1;
110
111 if (type == IB_QPT_SMI)
112 ret = 0;
113 else if (type == IB_QPT_GSI)
114 ret = 1;
115
116 if (ret != -1) {
117 map = &qpt->map[0];
118 if (unlikely(!map->page)) {
119 get_map_page(qpt, map);
120 if (unlikely(!map->page)) {
121 ret = -ENOMEM;
122 goto bail;
123 }
124 }
125 if (!test_and_set_bit(ret, map->page))
126 atomic_dec(&map->n_free);
127 else
128 ret = -EBUSY;
129 goto bail;
130 }
131
132 qpn = qpt->last + 1;
133 if (qpn >= QPN_MAX)
134 qpn = 2;
135 offset = qpn & BITS_PER_PAGE_MASK;
136 map = &qpt->map[qpn / BITS_PER_PAGE];
137 max_scan = qpt->nmaps - !offset;
138 for (i = 0;;) {
139 if (unlikely(!map->page)) {
140 get_map_page(qpt, map);
141 if (unlikely(!map->page))
142 break;
143 }
144 if (likely(atomic_read(&map->n_free))) {
145 do {
146 if (!test_and_set_bit(offset, map->page)) {
147 atomic_dec(&map->n_free);
148 qpt->last = qpn;
149 ret = qpn;
150 goto bail;
151 }
152 offset = find_next_offset(map, offset);
153 qpn = mk_qpn(qpt, map, offset);
154 /*
155 * This test differs from alloc_pidmap().
156 * If find_next_offset() does find a zero
157 * bit, we don't need to check for QPN
158 * wrapping around past our starting QPN.
159 * We just need to be sure we don't loop
160 * forever.
161 */
162 } while (offset < BITS_PER_PAGE && qpn < QPN_MAX);
163 }
164 /*
165 * In order to keep the number of pages allocated to a
166 * minimum, we scan the all existing pages before increasing
167 * the size of the bitmap table.
168 */
169 if (++i > max_scan) {
170 if (qpt->nmaps == QPNMAP_ENTRIES)
171 break;
172 map = &qpt->map[qpt->nmaps++];
173 offset = 0;
174 } else if (map < &qpt->map[qpt->nmaps]) {
175 ++map;
176 offset = 0;
177 } else {
178 map = &qpt->map[0];
179 offset = 2;
180 }
181 qpn = mk_qpn(qpt, map, offset);
182 }
183
184 ret = -ENOMEM;
185
186bail:
187 return ret;
188}
189
190static void free_qpn(struct ipath_qp_table *qpt, u32 qpn)
191{
192 struct qpn_map *map;
193
194 map = qpt->map + qpn / BITS_PER_PAGE;
195 if (map->page)
196 clear_bit(qpn & BITS_PER_PAGE_MASK, map->page);
197 atomic_inc(&map->n_free);
198}
199
200/**
201 * ipath_alloc_qpn - allocate a QP number
202 * @qpt: the QP table
203 * @qp: the QP
204 * @type: the QP type (IB_QPT_SMI and IB_QPT_GSI are special)
205 *
206 * Allocate the next available QPN and put the QP into the hash table.
207 * The hash table holds a reference to the QP.
208 */
209static int ipath_alloc_qpn(struct ipath_qp_table *qpt, struct ipath_qp *qp,
210 enum ib_qp_type type)
211{
212 unsigned long flags;
213 int ret;
214
215 ret = alloc_qpn(qpt, type);
216 if (ret < 0)
217 goto bail;
218 qp->ibqp.qp_num = ret;
219
220 /* Add the QP to the hash table. */
221 spin_lock_irqsave(&qpt->lock, flags);
222
223 ret %= qpt->max;
224 qp->next = qpt->table[ret];
225 qpt->table[ret] = qp;
226 atomic_inc(&qp->refcount);
227
228 spin_unlock_irqrestore(&qpt->lock, flags);
229 ret = 0;
230
231bail:
232 return ret;
233}
234
235/**
236 * ipath_free_qp - remove a QP from the QP table
237 * @qpt: the QP table
238 * @qp: the QP to remove
239 *
240 * Remove the QP from the table so it can't be found asynchronously by
241 * the receive interrupt routine.
242 */
243static void ipath_free_qp(struct ipath_qp_table *qpt, struct ipath_qp *qp)
244{
245 struct ipath_qp *q, **qpp;
246 unsigned long flags;
247
248 spin_lock_irqsave(&qpt->lock, flags);
249
250 /* Remove QP from the hash table. */
251 qpp = &qpt->table[qp->ibqp.qp_num % qpt->max];
252 for (; (q = *qpp) != NULL; qpp = &q->next) {
253 if (q == qp) {
254 *qpp = qp->next;
255 qp->next = NULL;
256 atomic_dec(&qp->refcount);
257 break;
258 }
259 }
260
261 spin_unlock_irqrestore(&qpt->lock, flags);
262}
263
264/**
265 * ipath_free_all_qps - check for QPs still in use
266 * @qpt: the QP table to empty
267 *
268 * There should not be any QPs still in use.
269 * Free memory for table.
270 */
271unsigned ipath_free_all_qps(struct ipath_qp_table *qpt)
272{
273 unsigned long flags;
274 struct ipath_qp *qp;
275 u32 n, qp_inuse = 0;
276
277 spin_lock_irqsave(&qpt->lock, flags);
278 for (n = 0; n < qpt->max; n++) {
279 qp = qpt->table[n];
280 qpt->table[n] = NULL;
281
282 for (; qp; qp = qp->next)
283 qp_inuse++;
284 }
285 spin_unlock_irqrestore(&qpt->lock, flags);
286
287 for (n = 0; n < ARRAY_SIZE(qpt->map); n++)
288 if (qpt->map[n].page)
289 free_page((unsigned long) qpt->map[n].page);
290 return qp_inuse;
291}
292
293/**
294 * ipath_lookup_qpn - return the QP with the given QPN
295 * @qpt: the QP table
296 * @qpn: the QP number to look up
297 *
298 * The caller is responsible for decrementing the QP reference count
299 * when done.
300 */
301struct ipath_qp *ipath_lookup_qpn(struct ipath_qp_table *qpt, u32 qpn)
302{
303 unsigned long flags;
304 struct ipath_qp *qp;
305
306 spin_lock_irqsave(&qpt->lock, flags);
307
308 for (qp = qpt->table[qpn % qpt->max]; qp; qp = qp->next) {
309 if (qp->ibqp.qp_num == qpn) {
310 atomic_inc(&qp->refcount);
311 break;
312 }
313 }
314
315 spin_unlock_irqrestore(&qpt->lock, flags);
316 return qp;
317}
318
319/**
320 * ipath_reset_qp - initialize the QP state to the reset state
321 * @qp: the QP to reset
322 * @type: the QP type
323 */
324static void ipath_reset_qp(struct ipath_qp *qp, enum ib_qp_type type)
325{
326 qp->remote_qpn = 0;
327 qp->qkey = 0;
328 qp->qp_access_flags = 0;
329 atomic_set(&qp->s_dma_busy, 0);
330 qp->s_flags &= IPATH_S_SIGNAL_REQ_WR;
331 qp->s_hdrwords = 0;
332 qp->s_wqe = NULL;
333 qp->s_pkt_delay = 0;
334 qp->s_draining = 0;
335 qp->s_psn = 0;
336 qp->r_psn = 0;
337 qp->r_msn = 0;
338 if (type == IB_QPT_RC) {
339 qp->s_state = IB_OPCODE_RC_SEND_LAST;
340 qp->r_state = IB_OPCODE_RC_SEND_LAST;
341 } else {
342 qp->s_state = IB_OPCODE_UC_SEND_LAST;
343 qp->r_state = IB_OPCODE_UC_SEND_LAST;
344 }
345 qp->s_ack_state = IB_OPCODE_RC_ACKNOWLEDGE;
346 qp->r_nak_state = 0;
347 qp->r_aflags = 0;
348 qp->r_flags = 0;
349 qp->s_rnr_timeout = 0;
350 qp->s_head = 0;
351 qp->s_tail = 0;
352 qp->s_cur = 0;
353 qp->s_last = 0;
354 qp->s_ssn = 1;
355 qp->s_lsn = 0;
356 memset(qp->s_ack_queue, 0, sizeof(qp->s_ack_queue));
357 qp->r_head_ack_queue = 0;
358 qp->s_tail_ack_queue = 0;
359 qp->s_num_rd_atomic = 0;
360 if (qp->r_rq.wq) {
361 qp->r_rq.wq->head = 0;
362 qp->r_rq.wq->tail = 0;
363 }
364}
365
366/**
367 * ipath_error_qp - put a QP into the error state
368 * @qp: the QP to put into the error state
369 * @err: the receive completion error to signal if a RWQE is active
370 *
371 * Flushes both send and receive work queues.
372 * Returns true if last WQE event should be generated.
373 * The QP s_lock should be held and interrupts disabled.
374 * If we are already in error state, just return.
375 */
376
377int ipath_error_qp(struct ipath_qp *qp, enum ib_wc_status err)
378{
379 struct ipath_ibdev *dev = to_idev(qp->ibqp.device);
380 struct ib_wc wc;
381 int ret = 0;
382
383 if (qp->state == IB_QPS_ERR)
384 goto bail;
385
386 qp->state = IB_QPS_ERR;
387
388 spin_lock(&dev->pending_lock);
389 if (!list_empty(&qp->timerwait))
390 list_del_init(&qp->timerwait);
391 if (!list_empty(&qp->piowait))
392 list_del_init(&qp->piowait);
393 spin_unlock(&dev->pending_lock);
394
395 /* Schedule the sending tasklet to drain the send work queue. */
396 if (qp->s_last != qp->s_head)
397 ipath_schedule_send(qp);
398
399 memset(&wc, 0, sizeof(wc));
400 wc.qp = &qp->ibqp;
401 wc.opcode = IB_WC_RECV;
402
403 if (test_and_clear_bit(IPATH_R_WRID_VALID, &qp->r_aflags)) {
404 wc.wr_id = qp->r_wr_id;
405 wc.status = err;
406 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
407 }
408 wc.status = IB_WC_WR_FLUSH_ERR;
409
410 if (qp->r_rq.wq) {
411 struct ipath_rwq *wq;
412 u32 head;
413 u32 tail;
414
415 spin_lock(&qp->r_rq.lock);
416
417 /* sanity check pointers before trusting them */
418 wq = qp->r_rq.wq;
419 head = wq->head;
420 if (head >= qp->r_rq.size)
421 head = 0;
422 tail = wq->tail;
423 if (tail >= qp->r_rq.size)
424 tail = 0;
425 while (tail != head) {
426 wc.wr_id = get_rwqe_ptr(&qp->r_rq, tail)->wr_id;
427 if (++tail >= qp->r_rq.size)
428 tail = 0;
429 ipath_cq_enter(to_icq(qp->ibqp.recv_cq), &wc, 1);
430 }
431 wq->tail = tail;
432
433 spin_unlock(&qp->r_rq.lock);
434 } else if (qp->ibqp.event_handler)
435 ret = 1;
436
437bail:
438 return ret;
439}
440
441/**
442 * ipath_modify_qp - modify the attributes of a queue pair
443 * @ibqp: the queue pair who's attributes we're modifying
444 * @attr: the new attributes
445 * @attr_mask: the mask of attributes to modify
446 * @udata: user data for ipathverbs.so
447 *
448 * Returns 0 on success, otherwise returns an errno.
449 */
450int ipath_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
451 int attr_mask, struct ib_udata *udata)
452{
453 struct ipath_ibdev *dev = to_idev(ibqp->device);
454 struct ipath_qp *qp = to_iqp(ibqp);
455 enum ib_qp_state cur_state, new_state;
456 int lastwqe = 0;
457 int ret;
458
459 spin_lock_irq(&qp->s_lock);
460
461 cur_state = attr_mask & IB_QP_CUR_STATE ?
462 attr->cur_qp_state : qp->state;
463 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
464
465 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
466 attr_mask))
467 goto inval;
468
469 if (attr_mask & IB_QP_AV) {
470 if (attr->ah_attr.dlid == 0 ||
471 attr->ah_attr.dlid >= IPATH_MULTICAST_LID_BASE)
472 goto inval;
473
474 if ((attr->ah_attr.ah_flags & IB_AH_GRH) &&
475 (attr->ah_attr.grh.sgid_index > 1))
476 goto inval;
477 }
478
479 if (attr_mask & IB_QP_PKEY_INDEX)
480 if (attr->pkey_index >= ipath_get_npkeys(dev->dd))
481 goto inval;
482
483 if (attr_mask & IB_QP_MIN_RNR_TIMER)
484 if (attr->min_rnr_timer > 31)
485 goto inval;
486
487 if (attr_mask & IB_QP_PORT)
488 if (attr->port_num == 0 ||
489 attr->port_num > ibqp->device->phys_port_cnt)
490 goto inval;
491
492 /*
493 * don't allow invalid Path MTU values or greater than 2048
494 * unless we are configured for a 4KB MTU
495 */
496 if ((attr_mask & IB_QP_PATH_MTU) &&
497 (ib_mtu_enum_to_int(attr->path_mtu) == -1 ||
498 (attr->path_mtu > IB_MTU_2048 && !ipath_mtu4096)))
499 goto inval;
500
501 if (attr_mask & IB_QP_PATH_MIG_STATE)
502 if (attr->path_mig_state != IB_MIG_MIGRATED &&
503 attr->path_mig_state != IB_MIG_REARM)
504 goto inval;
505
506 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
507 if (attr->max_dest_rd_atomic > IPATH_MAX_RDMA_ATOMIC)
508 goto inval;
509
510 switch (new_state) {
511 case IB_QPS_RESET:
512 if (qp->state != IB_QPS_RESET) {
513 qp->state = IB_QPS_RESET;
514 spin_lock(&dev->pending_lock);
515 if (!list_empty(&qp->timerwait))
516 list_del_init(&qp->timerwait);
517 if (!list_empty(&qp->piowait))
518 list_del_init(&qp->piowait);
519 spin_unlock(&dev->pending_lock);
520 qp->s_flags &= ~IPATH_S_ANY_WAIT;
521 spin_unlock_irq(&qp->s_lock);
522 /* Stop the sending tasklet */
523 tasklet_kill(&qp->s_task);
524 wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy));
525 spin_lock_irq(&qp->s_lock);
526 }
527 ipath_reset_qp(qp, ibqp->qp_type);
528 break;
529
530 case IB_QPS_SQD:
531 qp->s_draining = qp->s_last != qp->s_cur;
532 qp->state = new_state;
533 break;
534
535 case IB_QPS_SQE:
536 if (qp->ibqp.qp_type == IB_QPT_RC)
537 goto inval;
538 qp->state = new_state;
539 break;
540
541 case IB_QPS_ERR:
542 lastwqe = ipath_error_qp(qp, IB_WC_WR_FLUSH_ERR);
543 break;
544
545 default:
546 qp->state = new_state;
547 break;
548 }
549
550 if (attr_mask & IB_QP_PKEY_INDEX)
551 qp->s_pkey_index = attr->pkey_index;
552
553 if (attr_mask & IB_QP_DEST_QPN)
554 qp->remote_qpn = attr->dest_qp_num;
555
556 if (attr_mask & IB_QP_SQ_PSN) {
557 qp->s_psn = qp->s_next_psn = attr->sq_psn;
558 qp->s_last_psn = qp->s_next_psn - 1;
559 }
560
561 if (attr_mask & IB_QP_RQ_PSN)
562 qp->r_psn = attr->rq_psn;
563
564 if (attr_mask & IB_QP_ACCESS_FLAGS)
565 qp->qp_access_flags = attr->qp_access_flags;
566
567 if (attr_mask & IB_QP_AV) {
568 qp->remote_ah_attr = attr->ah_attr;
569 qp->s_dmult = ipath_ib_rate_to_mult(attr->ah_attr.static_rate);
570 }
571
572 if (attr_mask & IB_QP_PATH_MTU)
573 qp->path_mtu = attr->path_mtu;
574
575 if (attr_mask & IB_QP_RETRY_CNT)
576 qp->s_retry = qp->s_retry_cnt = attr->retry_cnt;
577
578 if (attr_mask & IB_QP_RNR_RETRY) {
579 qp->s_rnr_retry = attr->rnr_retry;
580 if (qp->s_rnr_retry > 7)
581 qp->s_rnr_retry = 7;
582 qp->s_rnr_retry_cnt = qp->s_rnr_retry;
583 }
584
585 if (attr_mask & IB_QP_MIN_RNR_TIMER)
586 qp->r_min_rnr_timer = attr->min_rnr_timer;
587
588 if (attr_mask & IB_QP_TIMEOUT)
589 qp->timeout = attr->timeout;
590
591 if (attr_mask & IB_QP_QKEY)
592 qp->qkey = attr->qkey;
593
594 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
595 qp->r_max_rd_atomic = attr->max_dest_rd_atomic;
596
597 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC)
598 qp->s_max_rd_atomic = attr->max_rd_atomic;
599
600 spin_unlock_irq(&qp->s_lock);
601
602 if (lastwqe) {
603 struct ib_event ev;
604
605 ev.device = qp->ibqp.device;
606 ev.element.qp = &qp->ibqp;
607 ev.event = IB_EVENT_QP_LAST_WQE_REACHED;
608 qp->ibqp.event_handler(&ev, qp->ibqp.qp_context);
609 }
610 ret = 0;
611 goto bail;
612
613inval:
614 spin_unlock_irq(&qp->s_lock);
615 ret = -EINVAL;
616
617bail:
618 return ret;
619}
620
621int ipath_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
622 int attr_mask, struct ib_qp_init_attr *init_attr)
623{
624 struct ipath_qp *qp = to_iqp(ibqp);
625
626 attr->qp_state = qp->state;
627 attr->cur_qp_state = attr->qp_state;
628 attr->path_mtu = qp->path_mtu;
629 attr->path_mig_state = 0;
630 attr->qkey = qp->qkey;
631 attr->rq_psn = qp->r_psn;
632 attr->sq_psn = qp->s_next_psn;
633 attr->dest_qp_num = qp->remote_qpn;
634 attr->qp_access_flags = qp->qp_access_flags;
635 attr->cap.max_send_wr = qp->s_size - 1;
636 attr->cap.max_recv_wr = qp->ibqp.srq ? 0 : qp->r_rq.size - 1;
637 attr->cap.max_send_sge = qp->s_max_sge;
638 attr->cap.max_recv_sge = qp->r_rq.max_sge;
639 attr->cap.max_inline_data = 0;
640 attr->ah_attr = qp->remote_ah_attr;
641 memset(&attr->alt_ah_attr, 0, sizeof(attr->alt_ah_attr));
642 attr->pkey_index = qp->s_pkey_index;
643 attr->alt_pkey_index = 0;
644 attr->en_sqd_async_notify = 0;
645 attr->sq_draining = qp->s_draining;
646 attr->max_rd_atomic = qp->s_max_rd_atomic;
647 attr->max_dest_rd_atomic = qp->r_max_rd_atomic;
648 attr->min_rnr_timer = qp->r_min_rnr_timer;
649 attr->port_num = 1;
650 attr->timeout = qp->timeout;
651 attr->retry_cnt = qp->s_retry_cnt;
652 attr->rnr_retry = qp->s_rnr_retry_cnt;
653 attr->alt_port_num = 0;
654 attr->alt_timeout = 0;
655
656 init_attr->event_handler = qp->ibqp.event_handler;
657 init_attr->qp_context = qp->ibqp.qp_context;
658 init_attr->send_cq = qp->ibqp.send_cq;
659 init_attr->recv_cq = qp->ibqp.recv_cq;
660 init_attr->srq = qp->ibqp.srq;
661 init_attr->cap = attr->cap;
662 if (qp->s_flags & IPATH_S_SIGNAL_REQ_WR)
663 init_attr->sq_sig_type = IB_SIGNAL_REQ_WR;
664 else
665 init_attr->sq_sig_type = IB_SIGNAL_ALL_WR;
666 init_attr->qp_type = qp->ibqp.qp_type;
667 init_attr->port_num = 1;
668 return 0;
669}
670
671/**
672 * ipath_compute_aeth - compute the AETH (syndrome + MSN)
673 * @qp: the queue pair to compute the AETH for
674 *
675 * Returns the AETH.
676 */
677__be32 ipath_compute_aeth(struct ipath_qp *qp)
678{
679 u32 aeth = qp->r_msn & IPATH_MSN_MASK;
680
681 if (qp->ibqp.srq) {
682 /*
683 * Shared receive queues don't generate credits.
684 * Set the credit field to the invalid value.
685 */
686 aeth |= IPATH_AETH_CREDIT_INVAL << IPATH_AETH_CREDIT_SHIFT;
687 } else {
688 u32 min, max, x;
689 u32 credits;
690 struct ipath_rwq *wq = qp->r_rq.wq;
691 u32 head;
692 u32 tail;
693
694 /* sanity check pointers before trusting them */
695 head = wq->head;
696 if (head >= qp->r_rq.size)
697 head = 0;
698 tail = wq->tail;
699 if (tail >= qp->r_rq.size)
700 tail = 0;
701 /*
702 * Compute the number of credits available (RWQEs).
703 * XXX Not holding the r_rq.lock here so there is a small
704 * chance that the pair of reads are not atomic.
705 */
706 credits = head - tail;
707 if ((int)credits < 0)
708 credits += qp->r_rq.size;
709 /*
710 * Binary search the credit table to find the code to
711 * use.
712 */
713 min = 0;
714 max = 31;
715 for (;;) {
716 x = (min + max) / 2;
717 if (credit_table[x] == credits)
718 break;
719 if (credit_table[x] > credits)
720 max = x;
721 else if (min == x)
722 break;
723 else
724 min = x;
725 }
726 aeth |= x << IPATH_AETH_CREDIT_SHIFT;
727 }
728 return cpu_to_be32(aeth);
729}
730
731/**
732 * ipath_create_qp - create a queue pair for a device
733 * @ibpd: the protection domain who's device we create the queue pair for
734 * @init_attr: the attributes of the queue pair
735 * @udata: unused by InfiniPath
736 *
737 * Returns the queue pair on success, otherwise returns an errno.
738 *
739 * Called by the ib_create_qp() core verbs function.
740 */
741struct ib_qp *ipath_create_qp(struct ib_pd *ibpd,
742 struct ib_qp_init_attr *init_attr,
743 struct ib_udata *udata)
744{
745 struct ipath_qp *qp;
746 int err;
747 struct ipath_swqe *swq = NULL;
748 struct ipath_ibdev *dev;
749 size_t sz;
750 size_t sg_list_sz;
751 struct ib_qp *ret;
752
753 if (init_attr->create_flags) {
754 ret = ERR_PTR(-EINVAL);
755 goto bail;
756 }
757
758 if (init_attr->cap.max_send_sge > ib_ipath_max_sges ||
759 init_attr->cap.max_send_wr > ib_ipath_max_qp_wrs) {
760 ret = ERR_PTR(-EINVAL);
761 goto bail;
762 }
763
764 /* Check receive queue parameters if no SRQ is specified. */
765 if (!init_attr->srq) {
766 if (init_attr->cap.max_recv_sge > ib_ipath_max_sges ||
767 init_attr->cap.max_recv_wr > ib_ipath_max_qp_wrs) {
768 ret = ERR_PTR(-EINVAL);
769 goto bail;
770 }
771 if (init_attr->cap.max_send_sge +
772 init_attr->cap.max_send_wr +
773 init_attr->cap.max_recv_sge +
774 init_attr->cap.max_recv_wr == 0) {
775 ret = ERR_PTR(-EINVAL);
776 goto bail;
777 }
778 }
779
780 switch (init_attr->qp_type) {
781 case IB_QPT_UC:
782 case IB_QPT_RC:
783 case IB_QPT_UD:
784 case IB_QPT_SMI:
785 case IB_QPT_GSI:
786 sz = sizeof(struct ipath_sge) *
787 init_attr->cap.max_send_sge +
788 sizeof(struct ipath_swqe);
789 swq = vmalloc((init_attr->cap.max_send_wr + 1) * sz);
790 if (swq == NULL) {
791 ret = ERR_PTR(-ENOMEM);
792 goto bail;
793 }
794 sz = sizeof(*qp);
795 sg_list_sz = 0;
796 if (init_attr->srq) {
797 struct ipath_srq *srq = to_isrq(init_attr->srq);
798
799 if (srq->rq.max_sge > 1)
800 sg_list_sz = sizeof(*qp->r_sg_list) *
801 (srq->rq.max_sge - 1);
802 } else if (init_attr->cap.max_recv_sge > 1)
803 sg_list_sz = sizeof(*qp->r_sg_list) *
804 (init_attr->cap.max_recv_sge - 1);
805 qp = kmalloc(sz + sg_list_sz, GFP_KERNEL);
806 if (!qp) {
807 ret = ERR_PTR(-ENOMEM);
808 goto bail_swq;
809 }
810 if (sg_list_sz && (init_attr->qp_type == IB_QPT_UD ||
811 init_attr->qp_type == IB_QPT_SMI ||
812 init_attr->qp_type == IB_QPT_GSI)) {
813 qp->r_ud_sg_list = kmalloc(sg_list_sz, GFP_KERNEL);
814 if (!qp->r_ud_sg_list) {
815 ret = ERR_PTR(-ENOMEM);
816 goto bail_qp;
817 }
818 } else
819 qp->r_ud_sg_list = NULL;
820 if (init_attr->srq) {
821 sz = 0;
822 qp->r_rq.size = 0;
823 qp->r_rq.max_sge = 0;
824 qp->r_rq.wq = NULL;
825 init_attr->cap.max_recv_wr = 0;
826 init_attr->cap.max_recv_sge = 0;
827 } else {
828 qp->r_rq.size = init_attr->cap.max_recv_wr + 1;
829 qp->r_rq.max_sge = init_attr->cap.max_recv_sge;
830 sz = (sizeof(struct ib_sge) * qp->r_rq.max_sge) +
831 sizeof(struct ipath_rwqe);
832 qp->r_rq.wq = vmalloc_user(sizeof(struct ipath_rwq) +
833 qp->r_rq.size * sz);
834 if (!qp->r_rq.wq) {
835 ret = ERR_PTR(-ENOMEM);
836 goto bail_sg_list;
837 }
838 }
839
840 /*
841 * ib_create_qp() will initialize qp->ibqp
842 * except for qp->ibqp.qp_num.
843 */
844 spin_lock_init(&qp->s_lock);
845 spin_lock_init(&qp->r_rq.lock);
846 atomic_set(&qp->refcount, 0);
847 init_waitqueue_head(&qp->wait);
848 init_waitqueue_head(&qp->wait_dma);
849 tasklet_init(&qp->s_task, ipath_do_send, (unsigned long)qp);
850 INIT_LIST_HEAD(&qp->piowait);
851 INIT_LIST_HEAD(&qp->timerwait);
852 qp->state = IB_QPS_RESET;
853 qp->s_wq = swq;
854 qp->s_size = init_attr->cap.max_send_wr + 1;
855 qp->s_max_sge = init_attr->cap.max_send_sge;
856 if (init_attr->sq_sig_type == IB_SIGNAL_REQ_WR)
857 qp->s_flags = IPATH_S_SIGNAL_REQ_WR;
858 else
859 qp->s_flags = 0;
860 dev = to_idev(ibpd->device);
861 err = ipath_alloc_qpn(&dev->qp_table, qp,
862 init_attr->qp_type);
863 if (err) {
864 ret = ERR_PTR(err);
865 vfree(qp->r_rq.wq);
866 goto bail_sg_list;
867 }
868 qp->ip = NULL;
869 qp->s_tx = NULL;
870 ipath_reset_qp(qp, init_attr->qp_type);
871 break;
872
873 default:
874 /* Don't support raw QPs */
875 ret = ERR_PTR(-ENOSYS);
876 goto bail;
877 }
878
879 init_attr->cap.max_inline_data = 0;
880
881 /*
882 * Return the address of the RWQ as the offset to mmap.
883 * See ipath_mmap() for details.
884 */
885 if (udata && udata->outlen >= sizeof(__u64)) {
886 if (!qp->r_rq.wq) {
887 __u64 offset = 0;
888
889 err = ib_copy_to_udata(udata, &offset,
890 sizeof(offset));
891 if (err) {
892 ret = ERR_PTR(err);
893 goto bail_ip;
894 }
895 } else {
896 u32 s = sizeof(struct ipath_rwq) +
897 qp->r_rq.size * sz;
898
899 qp->ip =
900 ipath_create_mmap_info(dev, s,
901 ibpd->uobject->context,
902 qp->r_rq.wq);
903 if (!qp->ip) {
904 ret = ERR_PTR(-ENOMEM);
905 goto bail_ip;
906 }
907
908 err = ib_copy_to_udata(udata, &(qp->ip->offset),
909 sizeof(qp->ip->offset));
910 if (err) {
911 ret = ERR_PTR(err);
912 goto bail_ip;
913 }
914 }
915 }
916
917 spin_lock(&dev->n_qps_lock);
918 if (dev->n_qps_allocated == ib_ipath_max_qps) {
919 spin_unlock(&dev->n_qps_lock);
920 ret = ERR_PTR(-ENOMEM);
921 goto bail_ip;
922 }
923
924 dev->n_qps_allocated++;
925 spin_unlock(&dev->n_qps_lock);
926
927 if (qp->ip) {
928 spin_lock_irq(&dev->pending_lock);
929 list_add(&qp->ip->pending_mmaps, &dev->pending_mmaps);
930 spin_unlock_irq(&dev->pending_lock);
931 }
932
933 ret = &qp->ibqp;
934 goto bail;
935
936bail_ip:
937 if (qp->ip)
938 kref_put(&qp->ip->ref, ipath_release_mmap_info);
939 else
940 vfree(qp->r_rq.wq);
941 ipath_free_qp(&dev->qp_table, qp);
942 free_qpn(&dev->qp_table, qp->ibqp.qp_num);
943bail_sg_list:
944 kfree(qp->r_ud_sg_list);
945bail_qp:
946 kfree(qp);
947bail_swq:
948 vfree(swq);
949bail:
950 return ret;
951}
952
953/**
954 * ipath_destroy_qp - destroy a queue pair
955 * @ibqp: the queue pair to destroy
956 *
957 * Returns 0 on success.
958 *
959 * Note that this can be called while the QP is actively sending or
960 * receiving!
961 */
962int ipath_destroy_qp(struct ib_qp *ibqp)
963{
964 struct ipath_qp *qp = to_iqp(ibqp);
965 struct ipath_ibdev *dev = to_idev(ibqp->device);
966
967 /* Make sure HW and driver activity is stopped. */
968 spin_lock_irq(&qp->s_lock);
969 if (qp->state != IB_QPS_RESET) {
970 qp->state = IB_QPS_RESET;
971 spin_lock(&dev->pending_lock);
972 if (!list_empty(&qp->timerwait))
973 list_del_init(&qp->timerwait);
974 if (!list_empty(&qp->piowait))
975 list_del_init(&qp->piowait);
976 spin_unlock(&dev->pending_lock);
977 qp->s_flags &= ~IPATH_S_ANY_WAIT;
978 spin_unlock_irq(&qp->s_lock);
979 /* Stop the sending tasklet */
980 tasklet_kill(&qp->s_task);
981 wait_event(qp->wait_dma, !atomic_read(&qp->s_dma_busy));
982 } else
983 spin_unlock_irq(&qp->s_lock);
984
985 ipath_free_qp(&dev->qp_table, qp);
986
987 if (qp->s_tx) {
988 atomic_dec(&qp->refcount);
989 if (qp->s_tx->txreq.flags & IPATH_SDMA_TXREQ_F_FREEBUF)
990 kfree(qp->s_tx->txreq.map_addr);
991 spin_lock_irq(&dev->pending_lock);
992 list_add(&qp->s_tx->txreq.list, &dev->txreq_free);
993 spin_unlock_irq(&dev->pending_lock);
994 qp->s_tx = NULL;
995 }
996
997 wait_event(qp->wait, !atomic_read(&qp->refcount));
998
999 /* all user's cleaned up, mark it available */
1000 free_qpn(&dev->qp_table, qp->ibqp.qp_num);
1001 spin_lock(&dev->n_qps_lock);
1002 dev->n_qps_allocated--;
1003 spin_unlock(&dev->n_qps_lock);
1004
1005 if (qp->ip)
1006 kref_put(&qp->ip->ref, ipath_release_mmap_info);
1007 else
1008 vfree(qp->r_rq.wq);
1009 kfree(qp->r_ud_sg_list);
1010 vfree(qp->s_wq);
1011 kfree(qp);
1012 return 0;
1013}
1014
1015/**
1016 * ipath_init_qp_table - initialize the QP table for a device
1017 * @idev: the device who's QP table we're initializing
1018 * @size: the size of the QP table
1019 *
1020 * Returns 0 on success, otherwise returns an errno.
1021 */
1022int ipath_init_qp_table(struct ipath_ibdev *idev, int size)
1023{
1024 int i;
1025 int ret;
1026
1027 idev->qp_table.last = 1; /* QPN 0 and 1 are special. */
1028 idev->qp_table.max = size;
1029 idev->qp_table.nmaps = 1;
1030 idev->qp_table.table = kzalloc(size * sizeof(*idev->qp_table.table),
1031 GFP_KERNEL);
1032 if (idev->qp_table.table == NULL) {
1033 ret = -ENOMEM;
1034 goto bail;
1035 }
1036
1037 for (i = 0; i < ARRAY_SIZE(idev->qp_table.map); i++) {
1038 atomic_set(&idev->qp_table.map[i].n_free, BITS_PER_PAGE);
1039 idev->qp_table.map[i].page = NULL;
1040 }
1041
1042 ret = 0;
1043
1044bail:
1045 return ret;
1046}
1047
1048/**
1049 * ipath_get_credit - flush the send work queue of a QP
1050 * @qp: the qp who's send work queue to flush
1051 * @aeth: the Acknowledge Extended Transport Header
1052 *
1053 * The QP s_lock should be held.
1054 */
1055void ipath_get_credit(struct ipath_qp *qp, u32 aeth)
1056{
1057 u32 credit = (aeth >> IPATH_AETH_CREDIT_SHIFT) & IPATH_AETH_CREDIT_MASK;
1058
1059 /*
1060 * If the credit is invalid, we can send
1061 * as many packets as we like. Otherwise, we have to
1062 * honor the credit field.
1063 */
1064 if (credit == IPATH_AETH_CREDIT_INVAL)
1065 qp->s_lsn = (u32) -1;
1066 else if (qp->s_lsn != (u32) -1) {
1067 /* Compute new LSN (i.e., MSN + credit) */
1068 credit = (aeth + credit_table[credit]) & IPATH_MSN_MASK;
1069 if (ipath_cmp24(credit, qp->s_lsn) > 0)
1070 qp->s_lsn = credit;
1071 }
1072
1073 /* Restart sending if it was blocked due to lack of credits. */
1074 if ((qp->s_flags & IPATH_S_WAIT_SSN_CREDIT) &&
1075 qp->s_cur != qp->s_head &&
1076 (qp->s_lsn == (u32) -1 ||
1077 ipath_cmp24(get_swqe_ptr(qp, qp->s_cur)->ssn,
1078 qp->s_lsn + 1) <= 0))
1079 ipath_schedule_send(qp);
1080}