Loading...
1/*
2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include <linux/pci.h>
35#include <rdma/ib_addr.h>
36#include <rdma/ib_umem.h>
37#include <rdma/uverbs_ioctl.h>
38#include "hns_roce_common.h"
39#include "hns_roce_device.h"
40#include "hns_roce_hem.h"
41
42static void flush_work_handle(struct work_struct *work)
43{
44 struct hns_roce_work *flush_work = container_of(work,
45 struct hns_roce_work, work);
46 struct hns_roce_qp *hr_qp = container_of(flush_work,
47 struct hns_roce_qp, flush_work);
48 struct device *dev = flush_work->hr_dev->dev;
49 struct ib_qp_attr attr;
50 int attr_mask;
51 int ret;
52
53 attr_mask = IB_QP_STATE;
54 attr.qp_state = IB_QPS_ERR;
55
56 if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) {
57 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL);
58 if (ret)
59 dev_err(dev, "modify QP to error state failed(%d) during CQE flush\n",
60 ret);
61 }
62
63 /*
64 * make sure we signal QP destroy leg that flush QP was completed
65 * so that it can safely proceed ahead now and destroy QP
66 */
67 if (refcount_dec_and_test(&hr_qp->refcount))
68 complete(&hr_qp->free);
69}
70
71void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
72{
73 struct hns_roce_work *flush_work = &hr_qp->flush_work;
74
75 flush_work->hr_dev = hr_dev;
76 INIT_WORK(&flush_work->work, flush_work_handle);
77 refcount_inc(&hr_qp->refcount);
78 queue_work(hr_dev->irq_workq, &flush_work->work);
79}
80
81void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp)
82{
83 /*
84 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state
85 * gets into errored mode. Hence, as a workaround to this
86 * hardware limitation, driver needs to assist in flushing. But
87 * the flushing operation uses mailbox to convey the QP state to
88 * the hardware and which can sleep due to the mutex protection
89 * around the mailbox calls. Hence, use the deferred flush for
90 * now.
91 */
92 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
93 init_flush_work(dev, qp);
94}
95
96void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
97{
98 struct device *dev = hr_dev->dev;
99 struct hns_roce_qp *qp;
100
101 xa_lock(&hr_dev->qp_table_xa);
102 qp = __hns_roce_qp_lookup(hr_dev, qpn);
103 if (qp)
104 refcount_inc(&qp->refcount);
105 xa_unlock(&hr_dev->qp_table_xa);
106
107 if (!qp) {
108 dev_warn(dev, "async event for bogus QP %08x\n", qpn);
109 return;
110 }
111
112 if (event_type == HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR ||
113 event_type == HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR ||
114 event_type == HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR ||
115 event_type == HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION ||
116 event_type == HNS_ROCE_EVENT_TYPE_INVALID_XRCETH) {
117 qp->state = IB_QPS_ERR;
118
119 flush_cqe(hr_dev, qp);
120 }
121
122 qp->event(qp, (enum hns_roce_event)event_type);
123
124 if (refcount_dec_and_test(&qp->refcount))
125 complete(&qp->free);
126}
127
128static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
129 enum hns_roce_event type)
130{
131 struct ib_qp *ibqp = &hr_qp->ibqp;
132 struct ib_event event;
133
134 if (ibqp->event_handler) {
135 event.device = ibqp->device;
136 event.element.qp = ibqp;
137 switch (type) {
138 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
139 event.event = IB_EVENT_PATH_MIG;
140 break;
141 case HNS_ROCE_EVENT_TYPE_COMM_EST:
142 event.event = IB_EVENT_COMM_EST;
143 break;
144 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
145 event.event = IB_EVENT_SQ_DRAINED;
146 break;
147 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
148 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
149 break;
150 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
151 event.event = IB_EVENT_QP_FATAL;
152 break;
153 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
154 event.event = IB_EVENT_PATH_MIG_ERR;
155 break;
156 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
157 event.event = IB_EVENT_QP_REQ_ERR;
158 break;
159 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
160 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
161 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
162 event.event = IB_EVENT_QP_ACCESS_ERR;
163 break;
164 default:
165 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n",
166 type, hr_qp->qpn);
167 return;
168 }
169 ibqp->event_handler(&event, ibqp->qp_context);
170 }
171}
172
173static u8 get_affinity_cq_bank(u8 qp_bank)
174{
175 return (qp_bank >> 1) & CQ_BANKID_MASK;
176}
177
178static u8 get_least_load_bankid_for_qp(struct ib_qp_init_attr *init_attr,
179 struct hns_roce_bank *bank)
180{
181#define INVALID_LOAD_QPNUM 0xFFFFFFFF
182 struct ib_cq *scq = init_attr->send_cq;
183 u32 least_load = INVALID_LOAD_QPNUM;
184 unsigned long cqn = 0;
185 u8 bankid = 0;
186 u32 bankcnt;
187 u8 i;
188
189 if (scq)
190 cqn = to_hr_cq(scq)->cqn;
191
192 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) {
193 if (scq && (get_affinity_cq_bank(i) != (cqn & CQ_BANKID_MASK)))
194 continue;
195
196 bankcnt = bank[i].inuse;
197 if (bankcnt < least_load) {
198 least_load = bankcnt;
199 bankid = i;
200 }
201 }
202
203 return bankid;
204}
205
206static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid,
207 unsigned long *qpn)
208{
209 int id;
210
211 id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL);
212 if (id < 0) {
213 id = ida_alloc_range(&bank->ida, bank->min, bank->max,
214 GFP_KERNEL);
215 if (id < 0)
216 return id;
217 }
218
219 /* the QPN should keep increasing until the max value is reached. */
220 bank->next = (id + 1) > bank->max ? bank->min : id + 1;
221
222 /* the lower 3 bits is bankid */
223 *qpn = (id << 3) | bankid;
224
225 return 0;
226}
227static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
228 struct ib_qp_init_attr *init_attr)
229{
230 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
231 unsigned long num = 0;
232 u8 bankid;
233 int ret;
234
235 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
236 num = 1;
237 } else {
238 mutex_lock(&qp_table->bank_mutex);
239 bankid = get_least_load_bankid_for_qp(init_attr, qp_table->bank);
240
241 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid,
242 &num);
243 if (ret) {
244 ibdev_err(&hr_dev->ib_dev,
245 "failed to alloc QPN, ret = %d\n", ret);
246 mutex_unlock(&qp_table->bank_mutex);
247 return ret;
248 }
249
250 qp_table->bank[bankid].inuse++;
251 mutex_unlock(&qp_table->bank_mutex);
252 }
253
254 hr_qp->qpn = num;
255
256 return 0;
257}
258
259static void add_qp_to_list(struct hns_roce_dev *hr_dev,
260 struct hns_roce_qp *hr_qp,
261 struct ib_cq *send_cq, struct ib_cq *recv_cq)
262{
263 struct hns_roce_cq *hr_send_cq, *hr_recv_cq;
264 unsigned long flags;
265
266 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL;
267 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL;
268
269 spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
270 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq);
271
272 list_add_tail(&hr_qp->node, &hr_dev->qp_list);
273 if (hr_send_cq)
274 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list);
275 if (hr_recv_cq)
276 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list);
277
278 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq);
279 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
280}
281
282static int hns_roce_qp_store(struct hns_roce_dev *hr_dev,
283 struct hns_roce_qp *hr_qp,
284 struct ib_qp_init_attr *init_attr)
285{
286 struct xarray *xa = &hr_dev->qp_table_xa;
287 int ret;
288
289 if (!hr_qp->qpn)
290 return -EINVAL;
291
292 ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL));
293 if (ret)
294 dev_err(hr_dev->dev, "failed to xa store for QPC\n");
295 else
296 /* add QP to device's QP list for softwc */
297 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq,
298 init_attr->recv_cq);
299
300 return ret;
301}
302
303static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
304{
305 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
306 struct device *dev = hr_dev->dev;
307 int ret;
308
309 if (!hr_qp->qpn)
310 return -EINVAL;
311
312 /* Alloc memory for QPC */
313 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
314 if (ret) {
315 dev_err(dev, "failed to get QPC table\n");
316 goto err_out;
317 }
318
319 /* Alloc memory for IRRL */
320 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
321 if (ret) {
322 dev_err(dev, "failed to get IRRL table\n");
323 goto err_put_qp;
324 }
325
326 if (hr_dev->caps.trrl_entry_sz) {
327 /* Alloc memory for TRRL */
328 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table,
329 hr_qp->qpn);
330 if (ret) {
331 dev_err(dev, "failed to get TRRL table\n");
332 goto err_put_irrl;
333 }
334 }
335
336 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
337 /* Alloc memory for SCC CTX */
338 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table,
339 hr_qp->qpn);
340 if (ret) {
341 dev_err(dev, "failed to get SCC CTX table\n");
342 goto err_put_trrl;
343 }
344 }
345
346 return 0;
347
348err_put_trrl:
349 if (hr_dev->caps.trrl_entry_sz)
350 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
351
352err_put_irrl:
353 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
354
355err_put_qp:
356 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
357
358err_out:
359 return ret;
360}
361
362static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp)
363{
364 rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry);
365}
366
367void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
368{
369 struct xarray *xa = &hr_dev->qp_table_xa;
370 unsigned long flags;
371
372 list_del(&hr_qp->node);
373
374 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT)
375 list_del(&hr_qp->sq_node);
376
377 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI &&
378 hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT)
379 list_del(&hr_qp->rq_node);
380
381 xa_lock_irqsave(xa, flags);
382 __xa_erase(xa, hr_qp->qpn);
383 xa_unlock_irqrestore(xa, flags);
384}
385
386static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
387{
388 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
389
390 if (hr_dev->caps.trrl_entry_sz)
391 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
392 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
393}
394
395static inline u8 get_qp_bankid(unsigned long qpn)
396{
397 /* The lower 3 bits of QPN are used to hash to different banks */
398 return (u8)(qpn & GENMASK(2, 0));
399}
400
401static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
402{
403 u8 bankid;
404
405 if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
406 return;
407
408 if (hr_qp->qpn < hr_dev->caps.reserved_qps)
409 return;
410
411 bankid = get_qp_bankid(hr_qp->qpn);
412
413 ida_free(&hr_dev->qp_table.bank[bankid].ida, hr_qp->qpn >> 3);
414
415 mutex_lock(&hr_dev->qp_table.bank_mutex);
416 hr_dev->qp_table.bank[bankid].inuse--;
417 mutex_unlock(&hr_dev->qp_table.bank_mutex);
418}
419
420static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp,
421 bool user)
422{
423 u32 max_sge = dev->caps.max_rq_sg;
424
425 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
426 return max_sge;
427
428 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will
429 * calculate number of max_sge with reserved SGEs when allocating wqe
430 * buf, so there is no need to do this again in kernel. But the number
431 * may exceed the capacity of SGEs recorded in the firmware, so the
432 * kernel driver should just adapt the value accordingly.
433 */
434 if (user)
435 max_sge = roundup_pow_of_two(max_sge + 1);
436 else
437 hr_qp->rq.rsv_sge = 1;
438
439 return max_sge;
440}
441
442static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap,
443 struct hns_roce_qp *hr_qp, int has_rq, bool user)
444{
445 u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user);
446 u32 cnt;
447
448 /* If srq exist, set zero for relative number of rq */
449 if (!has_rq) {
450 hr_qp->rq.wqe_cnt = 0;
451 hr_qp->rq.max_gs = 0;
452 cap->max_recv_wr = 0;
453 cap->max_recv_sge = 0;
454
455 return 0;
456 }
457
458 /* Check the validity of QP support capacity */
459 if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes ||
460 cap->max_recv_sge > max_sge) {
461 ibdev_err(&hr_dev->ib_dev,
462 "RQ config error, depth = %u, sge = %u\n",
463 cap->max_recv_wr, cap->max_recv_sge);
464 return -EINVAL;
465 }
466
467 cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes));
468 if (cnt > hr_dev->caps.max_wqes) {
469 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n",
470 cap->max_recv_wr);
471 return -EINVAL;
472 }
473
474 hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) +
475 hr_qp->rq.rsv_sge);
476
477 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz *
478 hr_qp->rq.max_gs);
479
480 hr_qp->rq.wqe_cnt = cnt;
481
482 cap->max_recv_wr = cnt;
483 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
484
485 return 0;
486}
487
488static u32 get_max_inline_data(struct hns_roce_dev *hr_dev,
489 struct ib_qp_cap *cap)
490{
491 if (cap->max_inline_data) {
492 cap->max_inline_data = roundup_pow_of_two(cap->max_inline_data);
493 return min(cap->max_inline_data,
494 hr_dev->caps.max_sq_inline);
495 }
496
497 return 0;
498}
499
500static void update_inline_data(struct hns_roce_qp *hr_qp,
501 struct ib_qp_cap *cap)
502{
503 u32 sge_num = hr_qp->sq.ext_sge_cnt;
504
505 if (hr_qp->config & HNS_ROCE_EXSGE_FLAGS) {
506 if (!(hr_qp->ibqp.qp_type == IB_QPT_GSI ||
507 hr_qp->ibqp.qp_type == IB_QPT_UD))
508 sge_num = max((u32)HNS_ROCE_SGE_IN_WQE, sge_num);
509
510 cap->max_inline_data = max(cap->max_inline_data,
511 sge_num * HNS_ROCE_SGE_SIZE);
512 }
513
514 hr_qp->max_inline_data = cap->max_inline_data;
515}
516
517static u32 get_sge_num_from_max_send_sge(bool is_ud_or_gsi,
518 u32 max_send_sge)
519{
520 unsigned int std_sge_num;
521 unsigned int min_sge;
522
523 std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;
524 min_sge = is_ud_or_gsi ? 1 : 0;
525 return max_send_sge > std_sge_num ? (max_send_sge - std_sge_num) :
526 min_sge;
527}
528
529static unsigned int get_sge_num_from_max_inl_data(bool is_ud_or_gsi,
530 u32 max_inline_data)
531{
532 unsigned int inline_sge;
533
534 inline_sge = roundup_pow_of_two(max_inline_data) / HNS_ROCE_SGE_SIZE;
535
536 /*
537 * if max_inline_data less than
538 * HNS_ROCE_SGE_IN_WQE * HNS_ROCE_SGE_SIZE,
539 * In addition to ud's mode, no need to extend sge.
540 */
541 if (!is_ud_or_gsi && inline_sge <= HNS_ROCE_SGE_IN_WQE)
542 inline_sge = 0;
543
544 return inline_sge;
545}
546
547static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
548 struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap)
549{
550 bool is_ud_or_gsi = (hr_qp->ibqp.qp_type == IB_QPT_GSI ||
551 hr_qp->ibqp.qp_type == IB_QPT_UD);
552 unsigned int std_sge_num;
553 u32 inline_ext_sge = 0;
554 u32 ext_wqe_sge_cnt;
555 u32 total_sge_cnt;
556
557 cap->max_inline_data = get_max_inline_data(hr_dev, cap);
558
559 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
560 std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;
561 ext_wqe_sge_cnt = get_sge_num_from_max_send_sge(is_ud_or_gsi,
562 cap->max_send_sge);
563
564 if (hr_qp->config & HNS_ROCE_EXSGE_FLAGS) {
565 inline_ext_sge = max(ext_wqe_sge_cnt,
566 get_sge_num_from_max_inl_data(is_ud_or_gsi,
567 cap->max_inline_data));
568 hr_qp->sq.ext_sge_cnt = inline_ext_sge ?
569 roundup_pow_of_two(inline_ext_sge) : 0;
570
571 hr_qp->sq.max_gs = max(1U, (hr_qp->sq.ext_sge_cnt + std_sge_num));
572 hr_qp->sq.max_gs = min(hr_qp->sq.max_gs, hr_dev->caps.max_sq_sg);
573
574 ext_wqe_sge_cnt = hr_qp->sq.ext_sge_cnt;
575 } else {
576 hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
577 hr_qp->sq.max_gs = min(hr_qp->sq.max_gs, hr_dev->caps.max_sq_sg);
578 hr_qp->sq.ext_sge_cnt = hr_qp->sq.max_gs;
579 }
580
581 /* If the number of extended sge is not zero, they MUST use the
582 * space of HNS_HW_PAGE_SIZE at least.
583 */
584 if (ext_wqe_sge_cnt) {
585 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * ext_wqe_sge_cnt);
586 hr_qp->sge.sge_cnt = max(total_sge_cnt,
587 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);
588 }
589
590 update_inline_data(hr_qp, cap);
591}
592
593static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
594 struct ib_qp_cap *cap,
595 struct hns_roce_ib_create_qp *ucmd)
596{
597 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz);
598 u8 max_sq_stride = ilog2(roundup_sq_stride);
599
600 /* Sanity check SQ size before proceeding */
601 if (ucmd->log_sq_stride > max_sq_stride ||
602 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
603 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n");
604 return -EINVAL;
605 }
606
607 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) {
608 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n",
609 cap->max_send_sge);
610 return -EINVAL;
611 }
612
613 return 0;
614}
615
616static int set_user_sq_size(struct hns_roce_dev *hr_dev,
617 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp,
618 struct hns_roce_ib_create_qp *ucmd)
619{
620 struct ib_device *ibdev = &hr_dev->ib_dev;
621 u32 cnt = 0;
622 int ret;
623
624 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) ||
625 cnt > hr_dev->caps.max_wqes)
626 return -EINVAL;
627
628 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);
629 if (ret) {
630 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n",
631 ret);
632 return ret;
633 }
634
635 set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
636
637 hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
638 hr_qp->sq.wqe_cnt = cnt;
639 cap->max_send_sge = hr_qp->sq.max_gs;
640
641 return 0;
642}
643
644static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
645 struct hns_roce_qp *hr_qp,
646 struct hns_roce_buf_attr *buf_attr)
647{
648 int buf_size;
649 int idx = 0;
650
651 hr_qp->buff_size = 0;
652
653 /* SQ WQE */
654 hr_qp->sq.offset = 0;
655 buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt,
656 hr_qp->sq.wqe_shift);
657 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
658 buf_attr->region[idx].size = buf_size;
659 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num;
660 idx++;
661 hr_qp->buff_size += buf_size;
662 }
663
664 /* extend SGE WQE in SQ */
665 hr_qp->sge.offset = hr_qp->buff_size;
666 buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt,
667 hr_qp->sge.sge_shift);
668 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
669 buf_attr->region[idx].size = buf_size;
670 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num;
671 idx++;
672 hr_qp->buff_size += buf_size;
673 }
674
675 /* RQ WQE */
676 hr_qp->rq.offset = hr_qp->buff_size;
677 buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt,
678 hr_qp->rq.wqe_shift);
679 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
680 buf_attr->region[idx].size = buf_size;
681 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num;
682 idx++;
683 hr_qp->buff_size += buf_size;
684 }
685
686 if (hr_qp->buff_size < 1)
687 return -EINVAL;
688
689 buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
690 buf_attr->region_count = idx;
691
692 return 0;
693}
694
695static int set_kernel_sq_size(struct hns_roce_dev *hr_dev,
696 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp)
697{
698 struct ib_device *ibdev = &hr_dev->ib_dev;
699 u32 cnt;
700
701 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes ||
702 cap->max_send_sge > hr_dev->caps.max_sq_sg) {
703 ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n");
704 return -EINVAL;
705 }
706
707 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes));
708 if (cnt > hr_dev->caps.max_wqes) {
709 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n",
710 cnt);
711 return -EINVAL;
712 }
713
714 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
715 hr_qp->sq.wqe_cnt = cnt;
716
717 set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
718
719 /* sync the parameters of kernel QP to user's configuration */
720 cap->max_send_wr = cnt;
721 cap->max_send_sge = hr_qp->sq.max_gs;
722
723 return 0;
724}
725
726static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr)
727{
728 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr)
729 return 0;
730
731 return 1;
732}
733
734static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)
735{
736 if (attr->qp_type == IB_QPT_XRC_INI ||
737 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
738 !attr->cap.max_recv_wr)
739 return 0;
740
741 return 1;
742}
743
744static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
745 struct ib_qp_init_attr *init_attr,
746 struct ib_udata *udata, unsigned long addr)
747{
748 struct ib_device *ibdev = &hr_dev->ib_dev;
749 struct hns_roce_buf_attr buf_attr = {};
750 int ret;
751
752 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
753 if (ret) {
754 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
755 goto err_inline;
756 }
757 ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
758 PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
759 udata, addr);
760 if (ret) {
761 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
762 goto err_inline;
763 }
764
765 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE)
766 hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE;
767
768 return 0;
769
770err_inline:
771
772 return ret;
773}
774
775static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
776{
777 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
778}
779
780static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,
781 struct ib_qp_init_attr *init_attr,
782 struct ib_udata *udata,
783 struct hns_roce_ib_create_qp_resp *resp,
784 struct hns_roce_ib_create_qp *ucmd)
785{
786 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
787 udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
788 hns_roce_qp_has_sq(init_attr) &&
789 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr));
790}
791
792static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev,
793 struct ib_qp_init_attr *init_attr,
794 struct ib_udata *udata,
795 struct hns_roce_ib_create_qp_resp *resp)
796{
797 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
798 udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
799 hns_roce_qp_has_rq(init_attr));
800}
801
802static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev,
803 struct ib_qp_init_attr *init_attr)
804{
805 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
806 hns_roce_qp_has_rq(init_attr));
807}
808
809static int qp_mmap_entry(struct hns_roce_qp *hr_qp,
810 struct hns_roce_dev *hr_dev,
811 struct ib_udata *udata,
812 struct hns_roce_ib_create_qp_resp *resp)
813{
814 struct hns_roce_ucontext *uctx =
815 rdma_udata_to_drv_context(udata,
816 struct hns_roce_ucontext, ibucontext);
817 struct rdma_user_mmap_entry *rdma_entry;
818 u64 address;
819
820 address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE;
821
822 hr_qp->dwqe_mmap_entry =
823 hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address,
824 HNS_ROCE_DWQE_SIZE,
825 HNS_ROCE_MMAP_TYPE_DWQE);
826
827 if (!hr_qp->dwqe_mmap_entry) {
828 ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n");
829 return -ENOMEM;
830 }
831
832 rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry;
833 resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry);
834
835 return 0;
836}
837
838static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,
839 struct hns_roce_qp *hr_qp,
840 struct ib_qp_init_attr *init_attr,
841 struct ib_udata *udata,
842 struct hns_roce_ib_create_qp *ucmd,
843 struct hns_roce_ib_create_qp_resp *resp)
844{
845 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,
846 struct hns_roce_ucontext, ibucontext);
847 struct ib_device *ibdev = &hr_dev->ib_dev;
848 int ret;
849
850 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) {
851 ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, &hr_qp->sdb);
852 if (ret) {
853 ibdev_err(ibdev,
854 "failed to map user SQ doorbell, ret = %d.\n",
855 ret);
856 goto err_out;
857 }
858 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
859 }
860
861 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) {
862 ret = hns_roce_db_map_user(uctx, ucmd->db_addr, &hr_qp->rdb);
863 if (ret) {
864 ibdev_err(ibdev,
865 "failed to map user RQ doorbell, ret = %d.\n",
866 ret);
867 goto err_sdb;
868 }
869 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
870 }
871
872 return 0;
873
874err_sdb:
875 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
876 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
877err_out:
878 return ret;
879}
880
881static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev,
882 struct hns_roce_qp *hr_qp,
883 struct ib_qp_init_attr *init_attr)
884{
885 struct ib_device *ibdev = &hr_dev->ib_dev;
886 int ret;
887
888 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
889 hr_qp->sq.db_reg = hr_dev->mem_base +
890 HNS_ROCE_DWQE_SIZE * hr_qp->qpn;
891 else
892 hr_qp->sq.db_reg = hr_dev->reg_base + hr_dev->sdb_offset +
893 DB_REG_OFFSET * hr_dev->priv_uar.index;
894
895 hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset +
896 DB_REG_OFFSET * hr_dev->priv_uar.index;
897
898 if (kernel_qp_has_rdb(hr_dev, init_attr)) {
899 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0);
900 if (ret) {
901 ibdev_err(ibdev,
902 "failed to alloc kernel RQ doorbell, ret = %d.\n",
903 ret);
904 return ret;
905 }
906 *hr_qp->rdb.db_record = 0;
907 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
908 }
909
910 return 0;
911}
912
913static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
914 struct ib_qp_init_attr *init_attr,
915 struct ib_udata *udata,
916 struct hns_roce_ib_create_qp *ucmd,
917 struct hns_roce_ib_create_qp_resp *resp)
918{
919 int ret;
920
921 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE)
922 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;
923
924 if (udata) {
925 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) {
926 ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp);
927 if (ret)
928 return ret;
929 }
930
931 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd,
932 resp);
933 if (ret)
934 goto err_remove_qp;
935 } else {
936 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr);
937 if (ret)
938 return ret;
939 }
940
941 return 0;
942
943err_remove_qp:
944 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
945 qp_user_mmap_entry_remove(hr_qp);
946
947 return ret;
948}
949
950static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
951 struct ib_udata *udata)
952{
953 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
954 udata, struct hns_roce_ucontext, ibucontext);
955
956 if (udata) {
957 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
958 hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
959 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
960 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
961 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
962 qp_user_mmap_entry_remove(hr_qp);
963 } else {
964 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
965 hns_roce_free_db(hr_dev, &hr_qp->rdb);
966 }
967}
968
969static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev,
970 struct hns_roce_qp *hr_qp)
971{
972 struct ib_device *ibdev = &hr_dev->ib_dev;
973 u64 *sq_wrid = NULL;
974 u64 *rq_wrid = NULL;
975 int ret;
976
977 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL);
978 if (ZERO_OR_NULL_PTR(sq_wrid)) {
979 ibdev_err(ibdev, "failed to alloc SQ wrid.\n");
980 return -ENOMEM;
981 }
982
983 if (hr_qp->rq.wqe_cnt) {
984 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL);
985 if (ZERO_OR_NULL_PTR(rq_wrid)) {
986 ibdev_err(ibdev, "failed to alloc RQ wrid.\n");
987 ret = -ENOMEM;
988 goto err_sq;
989 }
990 }
991
992 hr_qp->sq.wrid = sq_wrid;
993 hr_qp->rq.wrid = rq_wrid;
994 return 0;
995err_sq:
996 kfree(sq_wrid);
997
998 return ret;
999}
1000
1001static void free_kernel_wrid(struct hns_roce_qp *hr_qp)
1002{
1003 kfree(hr_qp->rq.wrid);
1004 kfree(hr_qp->sq.wrid);
1005}
1006
1007static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
1008 struct ib_qp_init_attr *init_attr,
1009 struct ib_udata *udata,
1010 struct hns_roce_ib_create_qp *ucmd)
1011{
1012 struct ib_device *ibdev = &hr_dev->ib_dev;
1013 struct hns_roce_ucontext *uctx;
1014 int ret;
1015
1016 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1017 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
1018 else
1019 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
1020
1021 ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp,
1022 hns_roce_qp_has_rq(init_attr), !!udata);
1023 if (ret) {
1024 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n",
1025 ret);
1026 return ret;
1027 }
1028
1029 if (udata) {
1030 ret = ib_copy_from_udata(ucmd, udata,
1031 min(udata->inlen, sizeof(*ucmd)));
1032 if (ret) {
1033 ibdev_err(ibdev,
1034 "failed to copy QP ucmd, ret = %d\n", ret);
1035 return ret;
1036 }
1037
1038 uctx = rdma_udata_to_drv_context(udata, struct hns_roce_ucontext,
1039 ibucontext);
1040 hr_qp->config = uctx->config;
1041 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);
1042 if (ret)
1043 ibdev_err(ibdev,
1044 "failed to set user SQ size, ret = %d.\n",
1045 ret);
1046 } else {
1047 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
1048 hr_qp->config = HNS_ROCE_EXSGE_FLAGS;
1049 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);
1050 if (ret)
1051 ibdev_err(ibdev,
1052 "failed to set kernel SQ size, ret = %d.\n",
1053 ret);
1054 }
1055
1056 return ret;
1057}
1058
1059static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
1060 struct ib_pd *ib_pd,
1061 struct ib_qp_init_attr *init_attr,
1062 struct ib_udata *udata,
1063 struct hns_roce_qp *hr_qp)
1064{
1065 struct hns_roce_ib_create_qp_resp resp = {};
1066 struct ib_device *ibdev = &hr_dev->ib_dev;
1067 struct hns_roce_ib_create_qp ucmd = {};
1068 int ret;
1069
1070 mutex_init(&hr_qp->mutex);
1071 spin_lock_init(&hr_qp->sq.lock);
1072 spin_lock_init(&hr_qp->rq.lock);
1073
1074 hr_qp->state = IB_QPS_RESET;
1075 hr_qp->flush_flag = 0;
1076
1077 if (init_attr->create_flags)
1078 return -EOPNOTSUPP;
1079
1080 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd);
1081 if (ret) {
1082 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret);
1083 return ret;
1084 }
1085
1086 if (!udata) {
1087 ret = alloc_kernel_wrid(hr_dev, hr_qp);
1088 if (ret) {
1089 ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n",
1090 ret);
1091 return ret;
1092 }
1093 }
1094
1095 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);
1096 if (ret) {
1097 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret);
1098 goto err_buf;
1099 }
1100
1101 ret = alloc_qpn(hr_dev, hr_qp, init_attr);
1102 if (ret) {
1103 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret);
1104 goto err_qpn;
1105 }
1106
1107 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp);
1108 if (ret) {
1109 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n",
1110 ret);
1111 goto err_db;
1112 }
1113
1114 ret = alloc_qpc(hr_dev, hr_qp);
1115 if (ret) {
1116 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n",
1117 ret);
1118 goto err_qpc;
1119 }
1120
1121 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr);
1122 if (ret) {
1123 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret);
1124 goto err_store;
1125 }
1126
1127 if (udata) {
1128 resp.cap_flags = hr_qp->en_flags;
1129 ret = ib_copy_to_udata(udata, &resp,
1130 min(udata->outlen, sizeof(resp)));
1131 if (ret) {
1132 ibdev_err(ibdev, "copy qp resp failed!\n");
1133 goto err_store;
1134 }
1135 }
1136
1137 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
1138 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp);
1139 if (ret)
1140 goto err_flow_ctrl;
1141 }
1142
1143 hr_qp->ibqp.qp_num = hr_qp->qpn;
1144 hr_qp->event = hns_roce_ib_qp_event;
1145 refcount_set(&hr_qp->refcount, 1);
1146 init_completion(&hr_qp->free);
1147
1148 return 0;
1149
1150err_flow_ctrl:
1151 hns_roce_qp_remove(hr_dev, hr_qp);
1152err_store:
1153 free_qpc(hr_dev, hr_qp);
1154err_qpc:
1155 free_qp_db(hr_dev, hr_qp, udata);
1156err_db:
1157 free_qpn(hr_dev, hr_qp);
1158err_qpn:
1159 free_qp_buf(hr_dev, hr_qp);
1160err_buf:
1161 free_kernel_wrid(hr_qp);
1162 return ret;
1163}
1164
1165void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
1166 struct ib_udata *udata)
1167{
1168 if (refcount_dec_and_test(&hr_qp->refcount))
1169 complete(&hr_qp->free);
1170 wait_for_completion(&hr_qp->free);
1171
1172 free_qpc(hr_dev, hr_qp);
1173 free_qpn(hr_dev, hr_qp);
1174 free_qp_buf(hr_dev, hr_qp);
1175 free_kernel_wrid(hr_qp);
1176 free_qp_db(hr_dev, hr_qp, udata);
1177}
1178
1179static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type,
1180 bool is_user)
1181{
1182 switch (type) {
1183 case IB_QPT_XRC_INI:
1184 case IB_QPT_XRC_TGT:
1185 if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC))
1186 goto out;
1187 break;
1188 case IB_QPT_UD:
1189 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 &&
1190 is_user)
1191 goto out;
1192 break;
1193 case IB_QPT_RC:
1194 case IB_QPT_GSI:
1195 break;
1196 default:
1197 goto out;
1198 }
1199
1200 return 0;
1201
1202out:
1203 ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type);
1204
1205 return -EOPNOTSUPP;
1206}
1207
1208int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr,
1209 struct ib_udata *udata)
1210{
1211 struct ib_device *ibdev = qp->device;
1212 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
1213 struct hns_roce_qp *hr_qp = to_hr_qp(qp);
1214 struct ib_pd *pd = qp->pd;
1215 int ret;
1216
1217 ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata);
1218 if (ret)
1219 goto err_out;
1220
1221 if (init_attr->qp_type == IB_QPT_XRC_TGT)
1222 hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn;
1223
1224 if (init_attr->qp_type == IB_QPT_GSI) {
1225 hr_qp->port = init_attr->port_num - 1;
1226 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
1227 }
1228
1229 ret = hns_roce_create_qp_common(hr_dev, pd, init_attr, udata, hr_qp);
1230 if (ret)
1231 ibdev_err(ibdev, "create QP type 0x%x failed(%d)\n",
1232 init_attr->qp_type, ret);
1233
1234err_out:
1235 if (ret)
1236 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_QP_CREATE_ERR_CNT]);
1237
1238 return ret;
1239}
1240
1241int to_hr_qp_type(int qp_type)
1242{
1243 switch (qp_type) {
1244 case IB_QPT_RC:
1245 return SERV_TYPE_RC;
1246 case IB_QPT_UD:
1247 case IB_QPT_GSI:
1248 return SERV_TYPE_UD;
1249 case IB_QPT_XRC_INI:
1250 case IB_QPT_XRC_TGT:
1251 return SERV_TYPE_XRC;
1252 default:
1253 return -1;
1254 }
1255}
1256
1257static int check_mtu_validate(struct hns_roce_dev *hr_dev,
1258 struct hns_roce_qp *hr_qp,
1259 struct ib_qp_attr *attr, int attr_mask)
1260{
1261 enum ib_mtu active_mtu;
1262 int p;
1263
1264 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1265 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu);
1266
1267 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 &&
1268 attr->path_mtu > hr_dev->caps.max_mtu) ||
1269 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) {
1270 ibdev_err(&hr_dev->ib_dev,
1271 "attr path_mtu(%d)invalid while modify qp",
1272 attr->path_mtu);
1273 return -EINVAL;
1274 }
1275
1276 return 0;
1277}
1278
1279static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1280 int attr_mask)
1281{
1282 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1283 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1284 int p;
1285
1286 if ((attr_mask & IB_QP_PORT) &&
1287 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
1288 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n",
1289 attr->port_num);
1290 return -EINVAL;
1291 }
1292
1293 if (attr_mask & IB_QP_PKEY_INDEX) {
1294 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1295 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
1296 ibdev_err(&hr_dev->ib_dev,
1297 "invalid attr, pkey_index = %u.\n",
1298 attr->pkey_index);
1299 return -EINVAL;
1300 }
1301 }
1302
1303 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1304 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
1305 ibdev_err(&hr_dev->ib_dev,
1306 "invalid attr, max_rd_atomic = %u.\n",
1307 attr->max_rd_atomic);
1308 return -EINVAL;
1309 }
1310
1311 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1312 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
1313 ibdev_err(&hr_dev->ib_dev,
1314 "invalid attr, max_dest_rd_atomic = %u.\n",
1315 attr->max_dest_rd_atomic);
1316 return -EINVAL;
1317 }
1318
1319 if (attr_mask & IB_QP_PATH_MTU)
1320 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask);
1321
1322 return 0;
1323}
1324
1325int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1326 int attr_mask, struct ib_udata *udata)
1327{
1328 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1329 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1330 enum ib_qp_state cur_state, new_state;
1331 int ret = -EINVAL;
1332
1333 mutex_lock(&hr_qp->mutex);
1334
1335 if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state)
1336 goto out;
1337
1338 cur_state = hr_qp->state;
1339 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1340
1341 if (ibqp->uobject &&
1342 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) {
1343 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) {
1344 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
1345
1346 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
1347 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
1348 } else {
1349 ibdev_warn(&hr_dev->ib_dev,
1350 "flush cqe is not supported in userspace!\n");
1351 goto out;
1352 }
1353 }
1354
1355 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
1356 attr_mask)) {
1357 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n");
1358 goto out;
1359 }
1360
1361 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask);
1362 if (ret)
1363 goto out;
1364
1365 if (cur_state == new_state && cur_state == IB_QPS_RESET)
1366 goto out;
1367
1368 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1369 new_state, udata);
1370
1371out:
1372 mutex_unlock(&hr_qp->mutex);
1373 if (ret)
1374 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_QP_MODIFY_ERR_CNT]);
1375
1376 return ret;
1377}
1378
1379void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
1380 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1381{
1382 if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1383 __acquire(&send_cq->lock);
1384 __acquire(&recv_cq->lock);
1385 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1386 spin_lock_irq(&send_cq->lock);
1387 __acquire(&recv_cq->lock);
1388 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1389 spin_lock_irq(&recv_cq->lock);
1390 __acquire(&send_cq->lock);
1391 } else if (send_cq == recv_cq) {
1392 spin_lock_irq(&send_cq->lock);
1393 __acquire(&recv_cq->lock);
1394 } else if (send_cq->cqn < recv_cq->cqn) {
1395 spin_lock_irq(&send_cq->lock);
1396 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1397 } else {
1398 spin_lock_irq(&recv_cq->lock);
1399 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1400 }
1401}
1402
1403void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1404 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
1405 __releases(&recv_cq->lock)
1406{
1407 if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1408 __release(&recv_cq->lock);
1409 __release(&send_cq->lock);
1410 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1411 __release(&recv_cq->lock);
1412 spin_unlock(&send_cq->lock);
1413 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1414 __release(&send_cq->lock);
1415 spin_unlock(&recv_cq->lock);
1416 } else if (send_cq == recv_cq) {
1417 __release(&recv_cq->lock);
1418 spin_unlock_irq(&send_cq->lock);
1419 } else if (send_cq->cqn < recv_cq->cqn) {
1420 spin_unlock(&recv_cq->lock);
1421 spin_unlock_irq(&send_cq->lock);
1422 } else {
1423 spin_unlock(&send_cq->lock);
1424 spin_unlock_irq(&recv_cq->lock);
1425 }
1426}
1427
1428static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset)
1429{
1430 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);
1431}
1432
1433void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
1434{
1435 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
1436}
1437
1438void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
1439{
1440 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
1441}
1442
1443void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n)
1444{
1445 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift));
1446}
1447
1448bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,
1449 struct ib_cq *ib_cq)
1450{
1451 struct hns_roce_cq *hr_cq;
1452 u32 cur;
1453
1454 cur = hr_wq->head - hr_wq->tail;
1455 if (likely(cur + nreq < hr_wq->wqe_cnt))
1456 return false;
1457
1458 hr_cq = to_hr_cq(ib_cq);
1459 spin_lock(&hr_cq->lock);
1460 cur = hr_wq->head - hr_wq->tail;
1461 spin_unlock(&hr_cq->lock);
1462
1463 return cur + nreq >= hr_wq->wqe_cnt;
1464}
1465
1466int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
1467{
1468 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
1469 unsigned int reserved_from_bot;
1470 unsigned int i;
1471
1472 qp_table->idx_table.spare_idx = kcalloc(hr_dev->caps.num_qps,
1473 sizeof(u32), GFP_KERNEL);
1474 if (!qp_table->idx_table.spare_idx)
1475 return -ENOMEM;
1476
1477 mutex_init(&qp_table->scc_mutex);
1478 mutex_init(&qp_table->bank_mutex);
1479 xa_init(&hr_dev->qp_table_xa);
1480
1481 reserved_from_bot = hr_dev->caps.reserved_qps;
1482
1483 for (i = 0; i < reserved_from_bot; i++) {
1484 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++;
1485 hr_dev->qp_table.bank[get_qp_bankid(i)].min++;
1486 }
1487
1488 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) {
1489 ida_init(&hr_dev->qp_table.bank[i].ida);
1490 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps /
1491 HNS_ROCE_QP_BANK_NUM - 1;
1492 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min;
1493 }
1494
1495 return 0;
1496}
1497
1498void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
1499{
1500 int i;
1501
1502 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++)
1503 ida_destroy(&hr_dev->qp_table.bank[i].ida);
1504 kfree(hr_dev->qp_table.idx_table.spare_idx);
1505}
1/*
2 * Copyright (c) 2016 Hisilicon Limited.
3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
4 *
5 * This software is available to you under a choice of one of two
6 * licenses. You may choose to be licensed under the terms of the GNU
7 * General Public License (GPL) Version 2, available from the file
8 * COPYING in the main directory of this source tree, or the
9 * OpenIB.org BSD license below:
10 *
11 * Redistribution and use in source and binary forms, with or
12 * without modification, are permitted provided that the following
13 * conditions are met:
14 *
15 * - Redistributions of source code must retain the above
16 * copyright notice, this list of conditions and the following
17 * disclaimer.
18 *
19 * - Redistributions in binary form must reproduce the above
20 * copyright notice, this list of conditions and the following
21 * disclaimer in the documentation and/or other materials
22 * provided with the distribution.
23 *
24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
31 * SOFTWARE.
32 */
33
34#include <linux/pci.h>
35#include <rdma/ib_addr.h>
36#include <rdma/ib_umem.h>
37#include <rdma/uverbs_ioctl.h>
38#include "hns_roce_common.h"
39#include "hns_roce_device.h"
40#include "hns_roce_hem.h"
41
42static struct hns_roce_qp *hns_roce_qp_lookup(struct hns_roce_dev *hr_dev,
43 u32 qpn)
44{
45 struct device *dev = hr_dev->dev;
46 struct hns_roce_qp *qp;
47 unsigned long flags;
48
49 xa_lock_irqsave(&hr_dev->qp_table_xa, flags);
50 qp = __hns_roce_qp_lookup(hr_dev, qpn);
51 if (qp)
52 refcount_inc(&qp->refcount);
53 xa_unlock_irqrestore(&hr_dev->qp_table_xa, flags);
54
55 if (!qp)
56 dev_warn(dev, "async event for bogus QP %08x\n", qpn);
57
58 return qp;
59}
60
61static void flush_work_handle(struct work_struct *work)
62{
63 struct hns_roce_work *flush_work = container_of(work,
64 struct hns_roce_work, work);
65 struct hns_roce_qp *hr_qp = container_of(flush_work,
66 struct hns_roce_qp, flush_work);
67 struct device *dev = flush_work->hr_dev->dev;
68 struct ib_qp_attr attr;
69 int attr_mask;
70 int ret;
71
72 attr_mask = IB_QP_STATE;
73 attr.qp_state = IB_QPS_ERR;
74
75 if (test_and_clear_bit(HNS_ROCE_FLUSH_FLAG, &hr_qp->flush_flag)) {
76 ret = hns_roce_modify_qp(&hr_qp->ibqp, &attr, attr_mask, NULL);
77 if (ret)
78 dev_err(dev, "modify QP to error state failed(%d) during CQE flush\n",
79 ret);
80 }
81
82 /*
83 * make sure we signal QP destroy leg that flush QP was completed
84 * so that it can safely proceed ahead now and destroy QP
85 */
86 if (refcount_dec_and_test(&hr_qp->refcount))
87 complete(&hr_qp->free);
88}
89
90void init_flush_work(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
91{
92 struct hns_roce_work *flush_work = &hr_qp->flush_work;
93 unsigned long flags;
94
95 spin_lock_irqsave(&hr_qp->flush_lock, flags);
96 /* Exit directly after destroy_qp() */
97 if (test_bit(HNS_ROCE_STOP_FLUSH_FLAG, &hr_qp->flush_flag)) {
98 spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
99 return;
100 }
101
102 refcount_inc(&hr_qp->refcount);
103 queue_work(hr_dev->irq_workq, &flush_work->work);
104 spin_unlock_irqrestore(&hr_qp->flush_lock, flags);
105}
106
107void flush_cqe(struct hns_roce_dev *dev, struct hns_roce_qp *qp)
108{
109 /*
110 * Hip08 hardware cannot flush the WQEs in SQ/RQ if the QP state
111 * gets into errored mode. Hence, as a workaround to this
112 * hardware limitation, driver needs to assist in flushing. But
113 * the flushing operation uses mailbox to convey the QP state to
114 * the hardware and which can sleep due to the mutex protection
115 * around the mailbox calls. Hence, use the deferred flush for
116 * now.
117 */
118 if (!test_and_set_bit(HNS_ROCE_FLUSH_FLAG, &qp->flush_flag))
119 init_flush_work(dev, qp);
120}
121
122void hns_roce_qp_event(struct hns_roce_dev *hr_dev, u32 qpn, int event_type)
123{
124 struct hns_roce_qp *qp;
125
126 qp = hns_roce_qp_lookup(hr_dev, qpn);
127 if (!qp)
128 return;
129
130 qp->event(qp, (enum hns_roce_event)event_type);
131
132 if (refcount_dec_and_test(&qp->refcount))
133 complete(&qp->free);
134}
135
136void hns_roce_flush_cqe(struct hns_roce_dev *hr_dev, u32 qpn)
137{
138 struct hns_roce_qp *qp;
139
140 qp = hns_roce_qp_lookup(hr_dev, qpn);
141 if (!qp)
142 return;
143
144 qp->state = IB_QPS_ERR;
145 flush_cqe(hr_dev, qp);
146
147 if (refcount_dec_and_test(&qp->refcount))
148 complete(&qp->free);
149}
150
151static void hns_roce_ib_qp_event(struct hns_roce_qp *hr_qp,
152 enum hns_roce_event type)
153{
154 struct ib_qp *ibqp = &hr_qp->ibqp;
155 struct ib_event event;
156
157 if (ibqp->event_handler) {
158 event.device = ibqp->device;
159 event.element.qp = ibqp;
160 switch (type) {
161 case HNS_ROCE_EVENT_TYPE_PATH_MIG:
162 event.event = IB_EVENT_PATH_MIG;
163 break;
164 case HNS_ROCE_EVENT_TYPE_COMM_EST:
165 event.event = IB_EVENT_COMM_EST;
166 break;
167 case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
168 event.event = IB_EVENT_SQ_DRAINED;
169 break;
170 case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
171 event.event = IB_EVENT_QP_LAST_WQE_REACHED;
172 break;
173 case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
174 event.event = IB_EVENT_QP_FATAL;
175 break;
176 case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
177 event.event = IB_EVENT_PATH_MIG_ERR;
178 break;
179 case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
180 event.event = IB_EVENT_QP_REQ_ERR;
181 break;
182 case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
183 case HNS_ROCE_EVENT_TYPE_XRCD_VIOLATION:
184 case HNS_ROCE_EVENT_TYPE_INVALID_XRCETH:
185 event.event = IB_EVENT_QP_ACCESS_ERR;
186 break;
187 default:
188 dev_dbg(ibqp->device->dev.parent, "roce_ib: Unexpected event type %d on QP %06lx\n",
189 type, hr_qp->qpn);
190 return;
191 }
192 ibqp->event_handler(&event, ibqp->qp_context);
193 }
194}
195
196static u8 get_affinity_cq_bank(u8 qp_bank)
197{
198 return (qp_bank >> 1) & CQ_BANKID_MASK;
199}
200
201static u8 get_least_load_bankid_for_qp(struct ib_qp_init_attr *init_attr,
202 struct hns_roce_bank *bank)
203{
204#define INVALID_LOAD_QPNUM 0xFFFFFFFF
205 struct ib_cq *scq = init_attr->send_cq;
206 u32 least_load = INVALID_LOAD_QPNUM;
207 unsigned long cqn = 0;
208 u8 bankid = 0;
209 u32 bankcnt;
210 u8 i;
211
212 if (scq)
213 cqn = to_hr_cq(scq)->cqn;
214
215 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) {
216 if (scq && (get_affinity_cq_bank(i) != (cqn & CQ_BANKID_MASK)))
217 continue;
218
219 bankcnt = bank[i].inuse;
220 if (bankcnt < least_load) {
221 least_load = bankcnt;
222 bankid = i;
223 }
224 }
225
226 return bankid;
227}
228
229static int alloc_qpn_with_bankid(struct hns_roce_bank *bank, u8 bankid,
230 unsigned long *qpn)
231{
232 int id;
233
234 id = ida_alloc_range(&bank->ida, bank->next, bank->max, GFP_KERNEL);
235 if (id < 0) {
236 id = ida_alloc_range(&bank->ida, bank->min, bank->max,
237 GFP_KERNEL);
238 if (id < 0)
239 return id;
240 }
241
242 /* the QPN should keep increasing until the max value is reached. */
243 bank->next = (id + 1) > bank->max ? bank->min : id + 1;
244
245 /* the lower 3 bits is bankid */
246 *qpn = (id << 3) | bankid;
247
248 return 0;
249}
250static int alloc_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
251 struct ib_qp_init_attr *init_attr)
252{
253 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
254 unsigned long num = 0;
255 u8 bankid;
256 int ret;
257
258 if (hr_qp->ibqp.qp_type == IB_QPT_GSI) {
259 num = 1;
260 } else {
261 mutex_lock(&qp_table->bank_mutex);
262 bankid = get_least_load_bankid_for_qp(init_attr, qp_table->bank);
263
264 ret = alloc_qpn_with_bankid(&qp_table->bank[bankid], bankid,
265 &num);
266 if (ret) {
267 ibdev_err(&hr_dev->ib_dev,
268 "failed to alloc QPN, ret = %d\n", ret);
269 mutex_unlock(&qp_table->bank_mutex);
270 return ret;
271 }
272
273 qp_table->bank[bankid].inuse++;
274 mutex_unlock(&qp_table->bank_mutex);
275 }
276
277 hr_qp->qpn = num;
278
279 return 0;
280}
281
282static void add_qp_to_list(struct hns_roce_dev *hr_dev,
283 struct hns_roce_qp *hr_qp,
284 struct ib_cq *send_cq, struct ib_cq *recv_cq)
285{
286 struct hns_roce_cq *hr_send_cq, *hr_recv_cq;
287 unsigned long flags;
288
289 hr_send_cq = send_cq ? to_hr_cq(send_cq) : NULL;
290 hr_recv_cq = recv_cq ? to_hr_cq(recv_cq) : NULL;
291
292 spin_lock_irqsave(&hr_dev->qp_list_lock, flags);
293 hns_roce_lock_cqs(hr_send_cq, hr_recv_cq);
294
295 list_add_tail(&hr_qp->node, &hr_dev->qp_list);
296 if (hr_send_cq)
297 list_add_tail(&hr_qp->sq_node, &hr_send_cq->sq_list);
298 if (hr_recv_cq)
299 list_add_tail(&hr_qp->rq_node, &hr_recv_cq->rq_list);
300
301 hns_roce_unlock_cqs(hr_send_cq, hr_recv_cq);
302 spin_unlock_irqrestore(&hr_dev->qp_list_lock, flags);
303}
304
305static int hns_roce_qp_store(struct hns_roce_dev *hr_dev,
306 struct hns_roce_qp *hr_qp,
307 struct ib_qp_init_attr *init_attr)
308{
309 struct xarray *xa = &hr_dev->qp_table_xa;
310 int ret;
311
312 if (!hr_qp->qpn)
313 return -EINVAL;
314
315 ret = xa_err(xa_store_irq(xa, hr_qp->qpn, hr_qp, GFP_KERNEL));
316 if (ret)
317 dev_err(hr_dev->dev, "failed to xa store for QPC\n");
318 else
319 /* add QP to device's QP list for softwc */
320 add_qp_to_list(hr_dev, hr_qp, init_attr->send_cq,
321 init_attr->recv_cq);
322
323 return ret;
324}
325
326static int alloc_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
327{
328 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
329 struct device *dev = hr_dev->dev;
330 int ret;
331
332 if (!hr_qp->qpn)
333 return -EINVAL;
334
335 /* Alloc memory for QPC */
336 ret = hns_roce_table_get(hr_dev, &qp_table->qp_table, hr_qp->qpn);
337 if (ret) {
338 dev_err(dev, "failed to get QPC table\n");
339 goto err_out;
340 }
341
342 /* Alloc memory for IRRL */
343 ret = hns_roce_table_get(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
344 if (ret) {
345 dev_err(dev, "failed to get IRRL table\n");
346 goto err_put_qp;
347 }
348
349 if (hr_dev->caps.trrl_entry_sz) {
350 /* Alloc memory for TRRL */
351 ret = hns_roce_table_get(hr_dev, &qp_table->trrl_table,
352 hr_qp->qpn);
353 if (ret) {
354 dev_err(dev, "failed to get TRRL table\n");
355 goto err_put_irrl;
356 }
357 }
358
359 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
360 /* Alloc memory for SCC CTX */
361 ret = hns_roce_table_get(hr_dev, &qp_table->sccc_table,
362 hr_qp->qpn);
363 if (ret) {
364 dev_err(dev, "failed to get SCC CTX table\n");
365 goto err_put_trrl;
366 }
367 }
368
369 return 0;
370
371err_put_trrl:
372 if (hr_dev->caps.trrl_entry_sz)
373 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
374
375err_put_irrl:
376 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
377
378err_put_qp:
379 hns_roce_table_put(hr_dev, &qp_table->qp_table, hr_qp->qpn);
380
381err_out:
382 return ret;
383}
384
385static void qp_user_mmap_entry_remove(struct hns_roce_qp *hr_qp)
386{
387 rdma_user_mmap_entry_remove(&hr_qp->dwqe_mmap_entry->rdma_entry);
388}
389
390void hns_roce_qp_remove(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
391{
392 struct xarray *xa = &hr_dev->qp_table_xa;
393 unsigned long flags;
394
395 list_del(&hr_qp->node);
396
397 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT)
398 list_del(&hr_qp->sq_node);
399
400 if (hr_qp->ibqp.qp_type != IB_QPT_XRC_INI &&
401 hr_qp->ibqp.qp_type != IB_QPT_XRC_TGT)
402 list_del(&hr_qp->rq_node);
403
404 xa_lock_irqsave(xa, flags);
405 __xa_erase(xa, hr_qp->qpn);
406 xa_unlock_irqrestore(xa, flags);
407}
408
409static void free_qpc(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
410{
411 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
412
413 if (hr_dev->caps.trrl_entry_sz)
414 hns_roce_table_put(hr_dev, &qp_table->trrl_table, hr_qp->qpn);
415 hns_roce_table_put(hr_dev, &qp_table->irrl_table, hr_qp->qpn);
416}
417
418static inline u8 get_qp_bankid(unsigned long qpn)
419{
420 /* The lower 3 bits of QPN are used to hash to different banks */
421 return (u8)(qpn & GENMASK(2, 0));
422}
423
424static void free_qpn(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
425{
426 u8 bankid;
427
428 if (hr_qp->ibqp.qp_type == IB_QPT_GSI)
429 return;
430
431 if (hr_qp->qpn < hr_dev->caps.reserved_qps)
432 return;
433
434 bankid = get_qp_bankid(hr_qp->qpn);
435
436 ida_free(&hr_dev->qp_table.bank[bankid].ida,
437 hr_qp->qpn / HNS_ROCE_QP_BANK_NUM);
438
439 mutex_lock(&hr_dev->qp_table.bank_mutex);
440 hr_dev->qp_table.bank[bankid].inuse--;
441 mutex_unlock(&hr_dev->qp_table.bank_mutex);
442}
443
444static u32 proc_rq_sge(struct hns_roce_dev *dev, struct hns_roce_qp *hr_qp,
445 bool user)
446{
447 u32 max_sge = dev->caps.max_rq_sg;
448
449 if (dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
450 return max_sge;
451
452 /* Reserve SGEs only for HIP08 in kernel; The userspace driver will
453 * calculate number of max_sge with reserved SGEs when allocating wqe
454 * buf, so there is no need to do this again in kernel. But the number
455 * may exceed the capacity of SGEs recorded in the firmware, so the
456 * kernel driver should just adapt the value accordingly.
457 */
458 if (user)
459 max_sge = roundup_pow_of_two(max_sge + 1);
460 else
461 hr_qp->rq.rsv_sge = 1;
462
463 return max_sge;
464}
465
466static int set_rq_size(struct hns_roce_dev *hr_dev, struct ib_qp_cap *cap,
467 struct hns_roce_qp *hr_qp, int has_rq, bool user)
468{
469 u32 max_sge = proc_rq_sge(hr_dev, hr_qp, user);
470 u32 cnt;
471
472 /* If srq exist, set zero for relative number of rq */
473 if (!has_rq) {
474 hr_qp->rq.wqe_cnt = 0;
475 hr_qp->rq.max_gs = 0;
476 cap->max_recv_wr = 0;
477 cap->max_recv_sge = 0;
478
479 return 0;
480 }
481
482 /* Check the validity of QP support capacity */
483 if (!cap->max_recv_wr || cap->max_recv_wr > hr_dev->caps.max_wqes ||
484 cap->max_recv_sge > max_sge) {
485 ibdev_err(&hr_dev->ib_dev,
486 "RQ config error, depth = %u, sge = %u\n",
487 cap->max_recv_wr, cap->max_recv_sge);
488 return -EINVAL;
489 }
490
491 cnt = roundup_pow_of_two(max(cap->max_recv_wr, hr_dev->caps.min_wqes));
492 if (cnt > hr_dev->caps.max_wqes) {
493 ibdev_err(&hr_dev->ib_dev, "rq depth %u too large\n",
494 cap->max_recv_wr);
495 return -EINVAL;
496 }
497
498 hr_qp->rq.max_gs = roundup_pow_of_two(max(1U, cap->max_recv_sge) +
499 hr_qp->rq.rsv_sge);
500
501 hr_qp->rq.wqe_shift = ilog2(hr_dev->caps.max_rq_desc_sz *
502 hr_qp->rq.max_gs);
503
504 hr_qp->rq.wqe_cnt = cnt;
505
506 cap->max_recv_wr = cnt;
507 cap->max_recv_sge = hr_qp->rq.max_gs - hr_qp->rq.rsv_sge;
508
509 return 0;
510}
511
512static u32 get_max_inline_data(struct hns_roce_dev *hr_dev,
513 struct ib_qp_cap *cap)
514{
515 if (cap->max_inline_data) {
516 cap->max_inline_data = roundup_pow_of_two(cap->max_inline_data);
517 return min(cap->max_inline_data,
518 hr_dev->caps.max_sq_inline);
519 }
520
521 return 0;
522}
523
524static void update_inline_data(struct hns_roce_qp *hr_qp,
525 struct ib_qp_cap *cap)
526{
527 u32 sge_num = hr_qp->sq.ext_sge_cnt;
528
529 if (hr_qp->config & HNS_ROCE_EXSGE_FLAGS) {
530 if (!(hr_qp->ibqp.qp_type == IB_QPT_GSI ||
531 hr_qp->ibqp.qp_type == IB_QPT_UD))
532 sge_num = max((u32)HNS_ROCE_SGE_IN_WQE, sge_num);
533
534 cap->max_inline_data = max(cap->max_inline_data,
535 sge_num * HNS_ROCE_SGE_SIZE);
536 }
537
538 hr_qp->max_inline_data = cap->max_inline_data;
539}
540
541static u32 get_sge_num_from_max_send_sge(bool is_ud_or_gsi,
542 u32 max_send_sge)
543{
544 unsigned int std_sge_num;
545 unsigned int min_sge;
546
547 std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;
548 min_sge = is_ud_or_gsi ? 1 : 0;
549 return max_send_sge > std_sge_num ? (max_send_sge - std_sge_num) :
550 min_sge;
551}
552
553static unsigned int get_sge_num_from_max_inl_data(bool is_ud_or_gsi,
554 u32 max_inline_data)
555{
556 unsigned int inline_sge;
557
558 if (!max_inline_data)
559 return 0;
560
561 /*
562 * if max_inline_data less than
563 * HNS_ROCE_SGE_IN_WQE * HNS_ROCE_SGE_SIZE,
564 * In addition to ud's mode, no need to extend sge.
565 */
566 inline_sge = roundup_pow_of_two(max_inline_data) / HNS_ROCE_SGE_SIZE;
567 if (!is_ud_or_gsi && inline_sge <= HNS_ROCE_SGE_IN_WQE)
568 inline_sge = 0;
569
570 return inline_sge;
571}
572
573static void set_ext_sge_param(struct hns_roce_dev *hr_dev, u32 sq_wqe_cnt,
574 struct hns_roce_qp *hr_qp, struct ib_qp_cap *cap)
575{
576 bool is_ud_or_gsi = (hr_qp->ibqp.qp_type == IB_QPT_GSI ||
577 hr_qp->ibqp.qp_type == IB_QPT_UD);
578 unsigned int std_sge_num;
579 u32 inline_ext_sge = 0;
580 u32 ext_wqe_sge_cnt;
581 u32 total_sge_cnt;
582
583 cap->max_inline_data = get_max_inline_data(hr_dev, cap);
584
585 hr_qp->sge.sge_shift = HNS_ROCE_SGE_SHIFT;
586 std_sge_num = is_ud_or_gsi ? 0 : HNS_ROCE_SGE_IN_WQE;
587 ext_wqe_sge_cnt = get_sge_num_from_max_send_sge(is_ud_or_gsi,
588 cap->max_send_sge);
589
590 if (hr_qp->config & HNS_ROCE_EXSGE_FLAGS) {
591 inline_ext_sge = max(ext_wqe_sge_cnt,
592 get_sge_num_from_max_inl_data(is_ud_or_gsi,
593 cap->max_inline_data));
594 hr_qp->sq.ext_sge_cnt = inline_ext_sge ?
595 roundup_pow_of_two(inline_ext_sge) : 0;
596
597 hr_qp->sq.max_gs = max(1U, (hr_qp->sq.ext_sge_cnt + std_sge_num));
598 hr_qp->sq.max_gs = min(hr_qp->sq.max_gs, hr_dev->caps.max_sq_sg);
599
600 ext_wqe_sge_cnt = hr_qp->sq.ext_sge_cnt;
601 } else {
602 hr_qp->sq.max_gs = max(1U, cap->max_send_sge);
603 hr_qp->sq.max_gs = min(hr_qp->sq.max_gs, hr_dev->caps.max_sq_sg);
604 hr_qp->sq.ext_sge_cnt = hr_qp->sq.max_gs;
605 }
606
607 /* If the number of extended sge is not zero, they MUST use the
608 * space of HNS_HW_PAGE_SIZE at least.
609 */
610 if (ext_wqe_sge_cnt) {
611 total_sge_cnt = roundup_pow_of_two(sq_wqe_cnt * ext_wqe_sge_cnt);
612 hr_qp->sge.sge_cnt = max(total_sge_cnt,
613 (u32)HNS_HW_PAGE_SIZE / HNS_ROCE_SGE_SIZE);
614 }
615
616 update_inline_data(hr_qp, cap);
617}
618
619static int check_sq_size_with_integrity(struct hns_roce_dev *hr_dev,
620 struct ib_qp_cap *cap,
621 struct hns_roce_ib_create_qp *ucmd)
622{
623 u32 roundup_sq_stride = roundup_pow_of_two(hr_dev->caps.max_sq_desc_sz);
624 u8 max_sq_stride = ilog2(roundup_sq_stride);
625
626 /* Sanity check SQ size before proceeding */
627 if (ucmd->log_sq_stride > max_sq_stride ||
628 ucmd->log_sq_stride < HNS_ROCE_IB_MIN_SQ_STRIDE) {
629 ibdev_err(&hr_dev->ib_dev, "failed to check SQ stride size.\n");
630 return -EINVAL;
631 }
632
633 if (cap->max_send_sge > hr_dev->caps.max_sq_sg) {
634 ibdev_err(&hr_dev->ib_dev, "failed to check SQ SGE size %u.\n",
635 cap->max_send_sge);
636 return -EINVAL;
637 }
638
639 return 0;
640}
641
642static int set_user_sq_size(struct hns_roce_dev *hr_dev,
643 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp,
644 struct hns_roce_ib_create_qp *ucmd)
645{
646 struct ib_device *ibdev = &hr_dev->ib_dev;
647 u32 cnt = 0;
648 int ret;
649
650 if (check_shl_overflow(1, ucmd->log_sq_bb_count, &cnt) ||
651 cnt > hr_dev->caps.max_wqes)
652 return -EINVAL;
653
654 ret = check_sq_size_with_integrity(hr_dev, cap, ucmd);
655 if (ret) {
656 ibdev_err(ibdev, "failed to check user SQ size, ret = %d.\n",
657 ret);
658 return ret;
659 }
660
661 set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
662
663 hr_qp->sq.wqe_shift = ucmd->log_sq_stride;
664 hr_qp->sq.wqe_cnt = cnt;
665 cap->max_send_sge = hr_qp->sq.max_gs;
666
667 return 0;
668}
669
670static int set_wqe_buf_attr(struct hns_roce_dev *hr_dev,
671 struct hns_roce_qp *hr_qp,
672 struct hns_roce_buf_attr *buf_attr)
673{
674 int buf_size;
675 int idx = 0;
676
677 hr_qp->buff_size = 0;
678
679 /* SQ WQE */
680 hr_qp->sq.offset = 0;
681 buf_size = to_hr_hem_entries_size(hr_qp->sq.wqe_cnt,
682 hr_qp->sq.wqe_shift);
683 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
684 buf_attr->region[idx].size = buf_size;
685 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sq_hop_num;
686 idx++;
687 hr_qp->buff_size += buf_size;
688 }
689
690 /* extend SGE WQE in SQ */
691 hr_qp->sge.offset = hr_qp->buff_size;
692 buf_size = to_hr_hem_entries_size(hr_qp->sge.sge_cnt,
693 hr_qp->sge.sge_shift);
694 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
695 buf_attr->region[idx].size = buf_size;
696 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_sge_hop_num;
697 idx++;
698 hr_qp->buff_size += buf_size;
699 }
700
701 /* RQ WQE */
702 hr_qp->rq.offset = hr_qp->buff_size;
703 buf_size = to_hr_hem_entries_size(hr_qp->rq.wqe_cnt,
704 hr_qp->rq.wqe_shift);
705 if (buf_size > 0 && idx < ARRAY_SIZE(buf_attr->region)) {
706 buf_attr->region[idx].size = buf_size;
707 buf_attr->region[idx].hopnum = hr_dev->caps.wqe_rq_hop_num;
708 idx++;
709 hr_qp->buff_size += buf_size;
710 }
711
712 if (hr_qp->buff_size < 1)
713 return -EINVAL;
714
715 buf_attr->page_shift = HNS_HW_PAGE_SHIFT + hr_dev->caps.mtt_buf_pg_sz;
716 buf_attr->region_count = idx;
717
718 return 0;
719}
720
721static int set_kernel_sq_size(struct hns_roce_dev *hr_dev,
722 struct ib_qp_cap *cap, struct hns_roce_qp *hr_qp)
723{
724 struct ib_device *ibdev = &hr_dev->ib_dev;
725 u32 cnt;
726
727 if (!cap->max_send_wr || cap->max_send_wr > hr_dev->caps.max_wqes ||
728 cap->max_send_sge > hr_dev->caps.max_sq_sg) {
729 ibdev_err(ibdev, "failed to check SQ WR or SGE num.\n");
730 return -EINVAL;
731 }
732
733 cnt = roundup_pow_of_two(max(cap->max_send_wr, hr_dev->caps.min_wqes));
734 if (cnt > hr_dev->caps.max_wqes) {
735 ibdev_err(ibdev, "failed to check WQE num, WQE num = %u.\n",
736 cnt);
737 return -EINVAL;
738 }
739
740 hr_qp->sq.wqe_shift = ilog2(hr_dev->caps.max_sq_desc_sz);
741 hr_qp->sq.wqe_cnt = cnt;
742
743 set_ext_sge_param(hr_dev, cnt, hr_qp, cap);
744
745 /* sync the parameters of kernel QP to user's configuration */
746 cap->max_send_wr = cnt;
747 cap->max_send_sge = hr_qp->sq.max_gs;
748
749 return 0;
750}
751
752static int hns_roce_qp_has_sq(struct ib_qp_init_attr *attr)
753{
754 if (attr->qp_type == IB_QPT_XRC_TGT || !attr->cap.max_send_wr)
755 return 0;
756
757 return 1;
758}
759
760static int hns_roce_qp_has_rq(struct ib_qp_init_attr *attr)
761{
762 if (attr->qp_type == IB_QPT_XRC_INI ||
763 attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
764 !attr->cap.max_recv_wr)
765 return 0;
766
767 return 1;
768}
769
770static int alloc_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
771 struct ib_qp_init_attr *init_attr,
772 struct ib_udata *udata, unsigned long addr)
773{
774 struct ib_device *ibdev = &hr_dev->ib_dev;
775 struct hns_roce_buf_attr buf_attr = {};
776 int ret;
777
778 ret = set_wqe_buf_attr(hr_dev, hr_qp, &buf_attr);
779 if (ret) {
780 ibdev_err(ibdev, "failed to split WQE buf, ret = %d.\n", ret);
781 goto err_inline;
782 }
783 ret = hns_roce_mtr_create(hr_dev, &hr_qp->mtr, &buf_attr,
784 PAGE_SHIFT + hr_dev->caps.mtt_ba_pg_sz,
785 udata, addr);
786 if (ret) {
787 ibdev_err(ibdev, "failed to create WQE mtr, ret = %d.\n", ret);
788 goto err_inline;
789 }
790
791 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_DIRECT_WQE)
792 hr_qp->en_flags |= HNS_ROCE_QP_CAP_DIRECT_WQE;
793
794 return 0;
795
796err_inline:
797
798 return ret;
799}
800
801static void free_qp_buf(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp)
802{
803 hns_roce_mtr_destroy(hr_dev, &hr_qp->mtr);
804}
805
806static inline bool user_qp_has_sdb(struct hns_roce_dev *hr_dev,
807 struct ib_qp_init_attr *init_attr,
808 struct ib_udata *udata,
809 struct hns_roce_ib_create_qp_resp *resp,
810 struct hns_roce_ib_create_qp *ucmd)
811{
812 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
813 udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
814 hns_roce_qp_has_sq(init_attr) &&
815 udata->inlen >= offsetofend(typeof(*ucmd), sdb_addr));
816}
817
818static inline bool user_qp_has_rdb(struct hns_roce_dev *hr_dev,
819 struct ib_qp_init_attr *init_attr,
820 struct ib_udata *udata,
821 struct hns_roce_ib_create_qp_resp *resp)
822{
823 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
824 udata->outlen >= offsetofend(typeof(*resp), cap_flags) &&
825 hns_roce_qp_has_rq(init_attr));
826}
827
828static inline bool kernel_qp_has_rdb(struct hns_roce_dev *hr_dev,
829 struct ib_qp_init_attr *init_attr)
830{
831 return ((hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_RECORD_DB) &&
832 hns_roce_qp_has_rq(init_attr));
833}
834
835static int qp_mmap_entry(struct hns_roce_qp *hr_qp,
836 struct hns_roce_dev *hr_dev,
837 struct ib_udata *udata,
838 struct hns_roce_ib_create_qp_resp *resp)
839{
840 struct hns_roce_ucontext *uctx =
841 rdma_udata_to_drv_context(udata,
842 struct hns_roce_ucontext, ibucontext);
843 struct rdma_user_mmap_entry *rdma_entry;
844 u64 address;
845
846 address = hr_dev->dwqe_page + hr_qp->qpn * HNS_ROCE_DWQE_SIZE;
847
848 hr_qp->dwqe_mmap_entry =
849 hns_roce_user_mmap_entry_insert(&uctx->ibucontext, address,
850 HNS_ROCE_DWQE_SIZE,
851 HNS_ROCE_MMAP_TYPE_DWQE);
852
853 if (!hr_qp->dwqe_mmap_entry) {
854 ibdev_err(&hr_dev->ib_dev, "failed to get dwqe mmap entry.\n");
855 return -ENOMEM;
856 }
857
858 rdma_entry = &hr_qp->dwqe_mmap_entry->rdma_entry;
859 resp->dwqe_mmap_key = rdma_user_mmap_get_offset(rdma_entry);
860
861 return 0;
862}
863
864static int alloc_user_qp_db(struct hns_roce_dev *hr_dev,
865 struct hns_roce_qp *hr_qp,
866 struct ib_qp_init_attr *init_attr,
867 struct ib_udata *udata,
868 struct hns_roce_ib_create_qp *ucmd,
869 struct hns_roce_ib_create_qp_resp *resp)
870{
871 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(udata,
872 struct hns_roce_ucontext, ibucontext);
873 struct ib_device *ibdev = &hr_dev->ib_dev;
874 int ret;
875
876 if (user_qp_has_sdb(hr_dev, init_attr, udata, resp, ucmd)) {
877 ret = hns_roce_db_map_user(uctx, ucmd->sdb_addr, &hr_qp->sdb);
878 if (ret) {
879 ibdev_err(ibdev,
880 "failed to map user SQ doorbell, ret = %d.\n",
881 ret);
882 goto err_out;
883 }
884 hr_qp->en_flags |= HNS_ROCE_QP_CAP_SQ_RECORD_DB;
885 }
886
887 if (user_qp_has_rdb(hr_dev, init_attr, udata, resp)) {
888 ret = hns_roce_db_map_user(uctx, ucmd->db_addr, &hr_qp->rdb);
889 if (ret) {
890 ibdev_err(ibdev,
891 "failed to map user RQ doorbell, ret = %d.\n",
892 ret);
893 goto err_sdb;
894 }
895 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
896 }
897
898 return 0;
899
900err_sdb:
901 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
902 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
903err_out:
904 return ret;
905}
906
907static int alloc_kernel_qp_db(struct hns_roce_dev *hr_dev,
908 struct hns_roce_qp *hr_qp,
909 struct ib_qp_init_attr *init_attr)
910{
911 struct ib_device *ibdev = &hr_dev->ib_dev;
912 int ret;
913
914 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
915 hr_qp->sq.db_reg = hr_dev->mem_base +
916 HNS_ROCE_DWQE_SIZE * hr_qp->qpn;
917 else
918 hr_qp->sq.db_reg = hr_dev->reg_base + hr_dev->sdb_offset +
919 DB_REG_OFFSET * hr_dev->priv_uar.index;
920
921 hr_qp->rq.db_reg = hr_dev->reg_base + hr_dev->odb_offset +
922 DB_REG_OFFSET * hr_dev->priv_uar.index;
923
924 if (kernel_qp_has_rdb(hr_dev, init_attr)) {
925 ret = hns_roce_alloc_db(hr_dev, &hr_qp->rdb, 0);
926 if (ret) {
927 ibdev_err(ibdev,
928 "failed to alloc kernel RQ doorbell, ret = %d.\n",
929 ret);
930 return ret;
931 }
932 *hr_qp->rdb.db_record = 0;
933 hr_qp->en_flags |= HNS_ROCE_QP_CAP_RQ_RECORD_DB;
934 }
935
936 return 0;
937}
938
939static int alloc_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
940 struct ib_qp_init_attr *init_attr,
941 struct ib_udata *udata,
942 struct hns_roce_ib_create_qp *ucmd,
943 struct hns_roce_ib_create_qp_resp *resp)
944{
945 int ret;
946
947 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_SDI_MODE)
948 hr_qp->en_flags |= HNS_ROCE_QP_CAP_OWNER_DB;
949
950 if (udata) {
951 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE) {
952 ret = qp_mmap_entry(hr_qp, hr_dev, udata, resp);
953 if (ret)
954 return ret;
955 }
956
957 ret = alloc_user_qp_db(hr_dev, hr_qp, init_attr, udata, ucmd,
958 resp);
959 if (ret)
960 goto err_remove_qp;
961 } else {
962 ret = alloc_kernel_qp_db(hr_dev, hr_qp, init_attr);
963 if (ret)
964 return ret;
965 }
966
967 return 0;
968
969err_remove_qp:
970 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
971 qp_user_mmap_entry_remove(hr_qp);
972
973 return ret;
974}
975
976static void free_qp_db(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
977 struct ib_udata *udata)
978{
979 struct hns_roce_ucontext *uctx = rdma_udata_to_drv_context(
980 udata, struct hns_roce_ucontext, ibucontext);
981
982 if (udata) {
983 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
984 hns_roce_db_unmap_user(uctx, &hr_qp->rdb);
985 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB)
986 hns_roce_db_unmap_user(uctx, &hr_qp->sdb);
987 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_DIRECT_WQE)
988 qp_user_mmap_entry_remove(hr_qp);
989 } else {
990 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
991 hns_roce_free_db(hr_dev, &hr_qp->rdb);
992 }
993}
994
995static int alloc_kernel_wrid(struct hns_roce_dev *hr_dev,
996 struct hns_roce_qp *hr_qp)
997{
998 struct ib_device *ibdev = &hr_dev->ib_dev;
999 u64 *sq_wrid = NULL;
1000 u64 *rq_wrid = NULL;
1001 int ret;
1002
1003 sq_wrid = kcalloc(hr_qp->sq.wqe_cnt, sizeof(u64), GFP_KERNEL);
1004 if (ZERO_OR_NULL_PTR(sq_wrid)) {
1005 ibdev_err(ibdev, "failed to alloc SQ wrid.\n");
1006 return -ENOMEM;
1007 }
1008
1009 if (hr_qp->rq.wqe_cnt) {
1010 rq_wrid = kcalloc(hr_qp->rq.wqe_cnt, sizeof(u64), GFP_KERNEL);
1011 if (ZERO_OR_NULL_PTR(rq_wrid)) {
1012 ibdev_err(ibdev, "failed to alloc RQ wrid.\n");
1013 ret = -ENOMEM;
1014 goto err_sq;
1015 }
1016 }
1017
1018 hr_qp->sq.wrid = sq_wrid;
1019 hr_qp->rq.wrid = rq_wrid;
1020 return 0;
1021err_sq:
1022 kfree(sq_wrid);
1023
1024 return ret;
1025}
1026
1027static void free_kernel_wrid(struct hns_roce_qp *hr_qp)
1028{
1029 kfree(hr_qp->rq.wrid);
1030 kfree(hr_qp->sq.wrid);
1031}
1032
1033static void default_congest_type(struct hns_roce_dev *hr_dev,
1034 struct hns_roce_qp *hr_qp)
1035{
1036 if (hr_qp->ibqp.qp_type == IB_QPT_UD ||
1037 hr_qp->ibqp.qp_type == IB_QPT_GSI)
1038 hr_qp->cong_type = CONG_TYPE_DCQCN;
1039 else
1040 hr_qp->cong_type = hr_dev->caps.default_cong_type;
1041}
1042
1043static int set_congest_type(struct hns_roce_qp *hr_qp,
1044 struct hns_roce_ib_create_qp *ucmd)
1045{
1046 struct hns_roce_dev *hr_dev = to_hr_dev(hr_qp->ibqp.device);
1047
1048 switch (ucmd->cong_type_flags) {
1049 case HNS_ROCE_CREATE_QP_FLAGS_DCQCN:
1050 hr_qp->cong_type = CONG_TYPE_DCQCN;
1051 break;
1052 case HNS_ROCE_CREATE_QP_FLAGS_LDCP:
1053 hr_qp->cong_type = CONG_TYPE_LDCP;
1054 break;
1055 case HNS_ROCE_CREATE_QP_FLAGS_HC3:
1056 hr_qp->cong_type = CONG_TYPE_HC3;
1057 break;
1058 case HNS_ROCE_CREATE_QP_FLAGS_DIP:
1059 hr_qp->cong_type = CONG_TYPE_DIP;
1060 break;
1061 default:
1062 return -EINVAL;
1063 }
1064
1065 if (!test_bit(hr_qp->cong_type, (unsigned long *)&hr_dev->caps.cong_cap))
1066 return -EOPNOTSUPP;
1067
1068 if (hr_qp->ibqp.qp_type == IB_QPT_UD &&
1069 hr_qp->cong_type != CONG_TYPE_DCQCN)
1070 return -EOPNOTSUPP;
1071
1072 return 0;
1073}
1074
1075static int set_congest_param(struct hns_roce_dev *hr_dev,
1076 struct hns_roce_qp *hr_qp,
1077 struct hns_roce_ib_create_qp *ucmd)
1078{
1079 if (ucmd->comp_mask & HNS_ROCE_CREATE_QP_MASK_CONGEST_TYPE)
1080 return set_congest_type(hr_qp, ucmd);
1081
1082 default_congest_type(hr_dev, hr_qp);
1083
1084 return 0;
1085}
1086
1087static int set_qp_param(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
1088 struct ib_qp_init_attr *init_attr,
1089 struct ib_udata *udata,
1090 struct hns_roce_ib_create_qp *ucmd)
1091{
1092 struct ib_device *ibdev = &hr_dev->ib_dev;
1093 struct hns_roce_ucontext *uctx;
1094 int ret;
1095
1096 if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
1097 hr_qp->sq_signal_bits = IB_SIGNAL_ALL_WR;
1098 else
1099 hr_qp->sq_signal_bits = IB_SIGNAL_REQ_WR;
1100
1101 ret = set_rq_size(hr_dev, &init_attr->cap, hr_qp,
1102 hns_roce_qp_has_rq(init_attr), !!udata);
1103 if (ret) {
1104 ibdev_err(ibdev, "failed to set user RQ size, ret = %d.\n",
1105 ret);
1106 return ret;
1107 }
1108
1109 if (udata) {
1110 ret = ib_copy_from_udata(ucmd, udata,
1111 min(udata->inlen, sizeof(*ucmd)));
1112 if (ret) {
1113 ibdev_err(ibdev,
1114 "failed to copy QP ucmd, ret = %d\n", ret);
1115 return ret;
1116 }
1117
1118 uctx = rdma_udata_to_drv_context(udata, struct hns_roce_ucontext,
1119 ibucontext);
1120 hr_qp->config = uctx->config;
1121 ret = set_user_sq_size(hr_dev, &init_attr->cap, hr_qp, ucmd);
1122 if (ret)
1123 ibdev_err(ibdev,
1124 "failed to set user SQ size, ret = %d.\n",
1125 ret);
1126
1127 ret = set_congest_param(hr_dev, hr_qp, ucmd);
1128 if (ret)
1129 return ret;
1130 } else {
1131 if (hr_dev->pci_dev->revision >= PCI_REVISION_ID_HIP09)
1132 hr_qp->config = HNS_ROCE_EXSGE_FLAGS;
1133 ret = set_kernel_sq_size(hr_dev, &init_attr->cap, hr_qp);
1134 if (ret)
1135 ibdev_err(ibdev,
1136 "failed to set kernel SQ size, ret = %d.\n",
1137 ret);
1138
1139 default_congest_type(hr_dev, hr_qp);
1140 }
1141
1142 return ret;
1143}
1144
1145static int hns_roce_create_qp_common(struct hns_roce_dev *hr_dev,
1146 struct ib_qp_init_attr *init_attr,
1147 struct ib_udata *udata,
1148 struct hns_roce_qp *hr_qp)
1149{
1150 struct hns_roce_work *flush_work = &hr_qp->flush_work;
1151 struct hns_roce_ib_create_qp_resp resp = {};
1152 struct ib_device *ibdev = &hr_dev->ib_dev;
1153 struct hns_roce_ib_create_qp ucmd = {};
1154 int ret;
1155
1156 mutex_init(&hr_qp->mutex);
1157 spin_lock_init(&hr_qp->sq.lock);
1158 spin_lock_init(&hr_qp->rq.lock);
1159 spin_lock_init(&hr_qp->flush_lock);
1160
1161 hr_qp->state = IB_QPS_RESET;
1162 hr_qp->flush_flag = 0;
1163 flush_work->hr_dev = hr_dev;
1164 INIT_WORK(&flush_work->work, flush_work_handle);
1165
1166 if (init_attr->create_flags)
1167 return -EOPNOTSUPP;
1168
1169 ret = set_qp_param(hr_dev, hr_qp, init_attr, udata, &ucmd);
1170 if (ret) {
1171 ibdev_err(ibdev, "failed to set QP param, ret = %d.\n", ret);
1172 goto err_out;
1173 }
1174
1175 if (!udata) {
1176 ret = alloc_kernel_wrid(hr_dev, hr_qp);
1177 if (ret) {
1178 ibdev_err(ibdev, "failed to alloc wrid, ret = %d.\n",
1179 ret);
1180 goto err_out;
1181 }
1182 }
1183
1184 ret = alloc_qp_buf(hr_dev, hr_qp, init_attr, udata, ucmd.buf_addr);
1185 if (ret) {
1186 ibdev_err(ibdev, "failed to alloc QP buffer, ret = %d.\n", ret);
1187 goto err_buf;
1188 }
1189
1190 ret = alloc_qpn(hr_dev, hr_qp, init_attr);
1191 if (ret) {
1192 ibdev_err(ibdev, "failed to alloc QPN, ret = %d.\n", ret);
1193 goto err_qpn;
1194 }
1195
1196 ret = alloc_qp_db(hr_dev, hr_qp, init_attr, udata, &ucmd, &resp);
1197 if (ret) {
1198 ibdev_err(ibdev, "failed to alloc QP doorbell, ret = %d.\n",
1199 ret);
1200 goto err_db;
1201 }
1202
1203 ret = alloc_qpc(hr_dev, hr_qp);
1204 if (ret) {
1205 ibdev_err(ibdev, "failed to alloc QP context, ret = %d.\n",
1206 ret);
1207 goto err_qpc;
1208 }
1209
1210 ret = hns_roce_qp_store(hr_dev, hr_qp, init_attr);
1211 if (ret) {
1212 ibdev_err(ibdev, "failed to store QP, ret = %d.\n", ret);
1213 goto err_store;
1214 }
1215
1216 if (udata) {
1217 resp.cap_flags = hr_qp->en_flags;
1218 ret = ib_copy_to_udata(udata, &resp,
1219 min(udata->outlen, sizeof(resp)));
1220 if (ret) {
1221 ibdev_err(ibdev, "copy qp resp failed!\n");
1222 goto err_store;
1223 }
1224 }
1225
1226 if (hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_QP_FLOW_CTRL) {
1227 ret = hr_dev->hw->qp_flow_control_init(hr_dev, hr_qp);
1228 if (ret)
1229 goto err_flow_ctrl;
1230 }
1231
1232 hr_qp->ibqp.qp_num = hr_qp->qpn;
1233 hr_qp->event = hns_roce_ib_qp_event;
1234 refcount_set(&hr_qp->refcount, 1);
1235 init_completion(&hr_qp->free);
1236
1237 return 0;
1238
1239err_flow_ctrl:
1240 hns_roce_qp_remove(hr_dev, hr_qp);
1241err_store:
1242 free_qpc(hr_dev, hr_qp);
1243err_qpc:
1244 free_qp_db(hr_dev, hr_qp, udata);
1245err_db:
1246 free_qpn(hr_dev, hr_qp);
1247err_qpn:
1248 free_qp_buf(hr_dev, hr_qp);
1249err_buf:
1250 free_kernel_wrid(hr_qp);
1251err_out:
1252 mutex_destroy(&hr_qp->mutex);
1253 return ret;
1254}
1255
1256void hns_roce_qp_destroy(struct hns_roce_dev *hr_dev, struct hns_roce_qp *hr_qp,
1257 struct ib_udata *udata)
1258{
1259 if (refcount_dec_and_test(&hr_qp->refcount))
1260 complete(&hr_qp->free);
1261 wait_for_completion(&hr_qp->free);
1262
1263 free_qpc(hr_dev, hr_qp);
1264 free_qpn(hr_dev, hr_qp);
1265 free_qp_buf(hr_dev, hr_qp);
1266 free_kernel_wrid(hr_qp);
1267 free_qp_db(hr_dev, hr_qp, udata);
1268 mutex_destroy(&hr_qp->mutex);
1269}
1270
1271static int check_qp_type(struct hns_roce_dev *hr_dev, enum ib_qp_type type,
1272 bool is_user)
1273{
1274 switch (type) {
1275 case IB_QPT_XRC_INI:
1276 case IB_QPT_XRC_TGT:
1277 if (!(hr_dev->caps.flags & HNS_ROCE_CAP_FLAG_XRC))
1278 goto out;
1279 break;
1280 case IB_QPT_UD:
1281 if (hr_dev->pci_dev->revision == PCI_REVISION_ID_HIP08 &&
1282 is_user)
1283 goto out;
1284 break;
1285 case IB_QPT_RC:
1286 case IB_QPT_GSI:
1287 break;
1288 default:
1289 goto out;
1290 }
1291
1292 return 0;
1293
1294out:
1295 ibdev_err(&hr_dev->ib_dev, "not support QP type %d\n", type);
1296
1297 return -EOPNOTSUPP;
1298}
1299
1300int hns_roce_create_qp(struct ib_qp *qp, struct ib_qp_init_attr *init_attr,
1301 struct ib_udata *udata)
1302{
1303 struct ib_device *ibdev = qp->device;
1304 struct hns_roce_dev *hr_dev = to_hr_dev(ibdev);
1305 struct hns_roce_qp *hr_qp = to_hr_qp(qp);
1306 int ret;
1307
1308 ret = check_qp_type(hr_dev, init_attr->qp_type, !!udata);
1309 if (ret)
1310 goto err_out;
1311
1312 if (init_attr->qp_type == IB_QPT_XRC_TGT)
1313 hr_qp->xrcdn = to_hr_xrcd(init_attr->xrcd)->xrcdn;
1314
1315 if (init_attr->qp_type == IB_QPT_GSI) {
1316 hr_qp->port = init_attr->port_num - 1;
1317 hr_qp->phy_port = hr_dev->iboe.phy_port[hr_qp->port];
1318 }
1319
1320 ret = hns_roce_create_qp_common(hr_dev, init_attr, udata, hr_qp);
1321 if (ret)
1322 ibdev_err(ibdev, "create QP type 0x%x failed(%d)\n",
1323 init_attr->qp_type, ret);
1324
1325err_out:
1326 if (ret)
1327 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_QP_CREATE_ERR_CNT]);
1328
1329 return ret;
1330}
1331
1332int to_hr_qp_type(int qp_type)
1333{
1334 switch (qp_type) {
1335 case IB_QPT_RC:
1336 return SERV_TYPE_RC;
1337 case IB_QPT_UD:
1338 case IB_QPT_GSI:
1339 return SERV_TYPE_UD;
1340 case IB_QPT_XRC_INI:
1341 case IB_QPT_XRC_TGT:
1342 return SERV_TYPE_XRC;
1343 default:
1344 return -1;
1345 }
1346}
1347
1348static int check_mtu_validate(struct hns_roce_dev *hr_dev,
1349 struct hns_roce_qp *hr_qp,
1350 struct ib_qp_attr *attr, int attr_mask)
1351{
1352 enum ib_mtu active_mtu;
1353 int p;
1354
1355 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1356 active_mtu = iboe_get_mtu(hr_dev->iboe.netdevs[p]->mtu);
1357
1358 if ((hr_dev->caps.max_mtu >= IB_MTU_2048 &&
1359 attr->path_mtu > hr_dev->caps.max_mtu) ||
1360 attr->path_mtu < IB_MTU_256 || attr->path_mtu > active_mtu) {
1361 ibdev_err(&hr_dev->ib_dev,
1362 "attr path_mtu(%d)invalid while modify qp",
1363 attr->path_mtu);
1364 return -EINVAL;
1365 }
1366
1367 return 0;
1368}
1369
1370static int hns_roce_check_qp_attr(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1371 int attr_mask)
1372{
1373 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1374 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1375 int p;
1376
1377 if ((attr_mask & IB_QP_PORT) &&
1378 (attr->port_num == 0 || attr->port_num > hr_dev->caps.num_ports)) {
1379 ibdev_err(&hr_dev->ib_dev, "invalid attr, port_num = %u.\n",
1380 attr->port_num);
1381 return -EINVAL;
1382 }
1383
1384 if (attr_mask & IB_QP_PKEY_INDEX) {
1385 p = attr_mask & IB_QP_PORT ? (attr->port_num - 1) : hr_qp->port;
1386 if (attr->pkey_index >= hr_dev->caps.pkey_table_len[p]) {
1387 ibdev_err(&hr_dev->ib_dev,
1388 "invalid attr, pkey_index = %u.\n",
1389 attr->pkey_index);
1390 return -EINVAL;
1391 }
1392 }
1393
1394 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1395 attr->max_rd_atomic > hr_dev->caps.max_qp_init_rdma) {
1396 ibdev_err(&hr_dev->ib_dev,
1397 "invalid attr, max_rd_atomic = %u.\n",
1398 attr->max_rd_atomic);
1399 return -EINVAL;
1400 }
1401
1402 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1403 attr->max_dest_rd_atomic > hr_dev->caps.max_qp_dest_rdma) {
1404 ibdev_err(&hr_dev->ib_dev,
1405 "invalid attr, max_dest_rd_atomic = %u.\n",
1406 attr->max_dest_rd_atomic);
1407 return -EINVAL;
1408 }
1409
1410 if (attr_mask & IB_QP_PATH_MTU)
1411 return check_mtu_validate(hr_dev, hr_qp, attr, attr_mask);
1412
1413 return 0;
1414}
1415
1416int hns_roce_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1417 int attr_mask, struct ib_udata *udata)
1418{
1419 struct hns_roce_dev *hr_dev = to_hr_dev(ibqp->device);
1420 struct hns_roce_ib_modify_qp_resp resp = {};
1421 struct hns_roce_qp *hr_qp = to_hr_qp(ibqp);
1422 enum ib_qp_state cur_state, new_state;
1423 int ret = -EINVAL;
1424
1425 mutex_lock(&hr_qp->mutex);
1426
1427 if (attr_mask & IB_QP_CUR_STATE && attr->cur_qp_state != hr_qp->state)
1428 goto out;
1429
1430 cur_state = hr_qp->state;
1431 new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1432
1433 if (ibqp->uobject &&
1434 (attr_mask & IB_QP_STATE) && new_state == IB_QPS_ERR) {
1435 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_SQ_RECORD_DB) {
1436 hr_qp->sq.head = *(int *)(hr_qp->sdb.virt_addr);
1437
1438 if (hr_qp->en_flags & HNS_ROCE_QP_CAP_RQ_RECORD_DB)
1439 hr_qp->rq.head = *(int *)(hr_qp->rdb.virt_addr);
1440 } else {
1441 ibdev_warn(&hr_dev->ib_dev,
1442 "flush cqe is not supported in userspace!\n");
1443 goto out;
1444 }
1445 }
1446
1447 if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type,
1448 attr_mask)) {
1449 ibdev_err(&hr_dev->ib_dev, "ib_modify_qp_is_ok failed\n");
1450 goto out;
1451 }
1452
1453 ret = hns_roce_check_qp_attr(ibqp, attr, attr_mask);
1454 if (ret)
1455 goto out;
1456
1457 if (cur_state == new_state && cur_state == IB_QPS_RESET)
1458 goto out;
1459
1460 ret = hr_dev->hw->modify_qp(ibqp, attr, attr_mask, cur_state,
1461 new_state, udata);
1462 if (ret)
1463 goto out;
1464
1465 if (udata && udata->outlen) {
1466 resp.tc_mode = hr_qp->tc_mode;
1467 resp.priority = hr_qp->sl;
1468 ret = ib_copy_to_udata(udata, &resp,
1469 min(udata->outlen, sizeof(resp)));
1470 if (ret)
1471 ibdev_err_ratelimited(&hr_dev->ib_dev,
1472 "failed to copy modify qp resp.\n");
1473 }
1474
1475out:
1476 mutex_unlock(&hr_qp->mutex);
1477 if (ret)
1478 atomic64_inc(&hr_dev->dfx_cnt[HNS_ROCE_DFX_QP_MODIFY_ERR_CNT]);
1479
1480 return ret;
1481}
1482
1483void hns_roce_lock_cqs(struct hns_roce_cq *send_cq, struct hns_roce_cq *recv_cq)
1484 __acquires(&send_cq->lock) __acquires(&recv_cq->lock)
1485{
1486 if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1487 __acquire(&send_cq->lock);
1488 __acquire(&recv_cq->lock);
1489 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1490 spin_lock(&send_cq->lock);
1491 __acquire(&recv_cq->lock);
1492 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1493 spin_lock(&recv_cq->lock);
1494 __acquire(&send_cq->lock);
1495 } else if (send_cq == recv_cq) {
1496 spin_lock(&send_cq->lock);
1497 __acquire(&recv_cq->lock);
1498 } else if (send_cq->cqn < recv_cq->cqn) {
1499 spin_lock(&send_cq->lock);
1500 spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
1501 } else {
1502 spin_lock(&recv_cq->lock);
1503 spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
1504 }
1505}
1506
1507void hns_roce_unlock_cqs(struct hns_roce_cq *send_cq,
1508 struct hns_roce_cq *recv_cq) __releases(&send_cq->lock)
1509 __releases(&recv_cq->lock)
1510{
1511 if (unlikely(send_cq == NULL && recv_cq == NULL)) {
1512 __release(&recv_cq->lock);
1513 __release(&send_cq->lock);
1514 } else if (unlikely(send_cq != NULL && recv_cq == NULL)) {
1515 __release(&recv_cq->lock);
1516 spin_unlock(&send_cq->lock);
1517 } else if (unlikely(send_cq == NULL && recv_cq != NULL)) {
1518 __release(&send_cq->lock);
1519 spin_unlock(&recv_cq->lock);
1520 } else if (send_cq == recv_cq) {
1521 __release(&recv_cq->lock);
1522 spin_unlock(&send_cq->lock);
1523 } else if (send_cq->cqn < recv_cq->cqn) {
1524 spin_unlock(&recv_cq->lock);
1525 spin_unlock(&send_cq->lock);
1526 } else {
1527 spin_unlock(&send_cq->lock);
1528 spin_unlock(&recv_cq->lock);
1529 }
1530}
1531
1532static inline void *get_wqe(struct hns_roce_qp *hr_qp, u32 offset)
1533{
1534 return hns_roce_buf_offset(hr_qp->mtr.kmem, offset);
1535}
1536
1537void *hns_roce_get_recv_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
1538{
1539 return get_wqe(hr_qp, hr_qp->rq.offset + (n << hr_qp->rq.wqe_shift));
1540}
1541
1542void *hns_roce_get_send_wqe(struct hns_roce_qp *hr_qp, unsigned int n)
1543{
1544 return get_wqe(hr_qp, hr_qp->sq.offset + (n << hr_qp->sq.wqe_shift));
1545}
1546
1547void *hns_roce_get_extend_sge(struct hns_roce_qp *hr_qp, unsigned int n)
1548{
1549 return get_wqe(hr_qp, hr_qp->sge.offset + (n << hr_qp->sge.sge_shift));
1550}
1551
1552bool hns_roce_wq_overflow(struct hns_roce_wq *hr_wq, u32 nreq,
1553 struct ib_cq *ib_cq)
1554{
1555 struct hns_roce_cq *hr_cq;
1556 u32 cur;
1557
1558 cur = hr_wq->head - hr_wq->tail;
1559 if (likely(cur + nreq < hr_wq->wqe_cnt))
1560 return false;
1561
1562 hr_cq = to_hr_cq(ib_cq);
1563 spin_lock(&hr_cq->lock);
1564 cur = hr_wq->head - hr_wq->tail;
1565 spin_unlock(&hr_cq->lock);
1566
1567 return cur + nreq >= hr_wq->wqe_cnt;
1568}
1569
1570int hns_roce_init_qp_table(struct hns_roce_dev *hr_dev)
1571{
1572 struct hns_roce_qp_table *qp_table = &hr_dev->qp_table;
1573 unsigned int reserved_from_bot;
1574 unsigned int i;
1575
1576 mutex_init(&qp_table->scc_mutex);
1577 mutex_init(&qp_table->bank_mutex);
1578 xa_init(&hr_dev->qp_table_xa);
1579 xa_init(&qp_table->dip_xa);
1580
1581 reserved_from_bot = hr_dev->caps.reserved_qps;
1582
1583 for (i = 0; i < reserved_from_bot; i++) {
1584 hr_dev->qp_table.bank[get_qp_bankid(i)].inuse++;
1585 hr_dev->qp_table.bank[get_qp_bankid(i)].min++;
1586 }
1587
1588 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++) {
1589 ida_init(&hr_dev->qp_table.bank[i].ida);
1590 hr_dev->qp_table.bank[i].max = hr_dev->caps.num_qps /
1591 HNS_ROCE_QP_BANK_NUM - 1;
1592 hr_dev->qp_table.bank[i].next = hr_dev->qp_table.bank[i].min;
1593 }
1594
1595 return 0;
1596}
1597
1598void hns_roce_cleanup_qp_table(struct hns_roce_dev *hr_dev)
1599{
1600 int i;
1601
1602 for (i = 0; i < HNS_ROCE_QP_BANK_NUM; i++)
1603 ida_destroy(&hr_dev->qp_table.bank[i].ida);
1604 xa_destroy(&hr_dev->qp_table.dip_xa);
1605 mutex_destroy(&hr_dev->qp_table.bank_mutex);
1606 mutex_destroy(&hr_dev->qp_table.scc_mutex);
1607}