Loading...
1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
45#include <rdma/uverbs_ioctl.h>
46
47#include <linux/qed/common_hsi.h>
48#include "qedr_hsi_rdma.h"
49#include <linux/qed/qed_if.h>
50#include "qedr.h"
51#include "verbs.h"
52#include <rdma/qedr-abi.h>
53#include "qedr_roce_cm.h"
54#include "qedr_iw_cm.h"
55
56#define QEDR_SRQ_WQE_ELEM_SIZE sizeof(union rdma_srq_elm)
57#define RDMA_MAX_SGE_PER_SRQ (4)
58#define RDMA_MAX_SRQ_WQE_SIZE (RDMA_MAX_SGE_PER_SRQ + 1)
59
60#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
61
62enum {
63 QEDR_USER_MMAP_IO_WC = 0,
64 QEDR_USER_MMAP_PHYS_PAGE,
65};
66
67static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
68 size_t len)
69{
70 size_t min_len = min_t(size_t, len, udata->outlen);
71
72 return ib_copy_to_udata(udata, src, min_len);
73}
74
75int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
76{
77 if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
78 return -EINVAL;
79
80 *pkey = QEDR_ROCE_PKEY_DEFAULT;
81 return 0;
82}
83
84int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
85 int index, union ib_gid *sgid)
86{
87 struct qedr_dev *dev = get_qedr_dev(ibdev);
88
89 memset(sgid->raw, 0, sizeof(sgid->raw));
90 ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
91
92 DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
93 sgid->global.interface_id, sgid->global.subnet_prefix);
94
95 return 0;
96}
97
98int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
99{
100 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
101 struct qedr_device_attr *qattr = &dev->attr;
102 struct qedr_srq *srq = get_qedr_srq(ibsrq);
103
104 srq_attr->srq_limit = srq->srq_limit;
105 srq_attr->max_wr = qattr->max_srq_wr;
106 srq_attr->max_sge = qattr->max_sge;
107
108 return 0;
109}
110
111int qedr_query_device(struct ib_device *ibdev,
112 struct ib_device_attr *attr, struct ib_udata *udata)
113{
114 struct qedr_dev *dev = get_qedr_dev(ibdev);
115 struct qedr_device_attr *qattr = &dev->attr;
116
117 if (!dev->rdma_ctx) {
118 DP_ERR(dev,
119 "qedr_query_device called with invalid params rdma_ctx=%p\n",
120 dev->rdma_ctx);
121 return -EINVAL;
122 }
123
124 memset(attr, 0, sizeof(*attr));
125
126 attr->fw_ver = qattr->fw_ver;
127 attr->sys_image_guid = qattr->sys_image_guid;
128 attr->max_mr_size = qattr->max_mr_size;
129 attr->page_size_cap = qattr->page_size_caps;
130 attr->vendor_id = qattr->vendor_id;
131 attr->vendor_part_id = qattr->vendor_part_id;
132 attr->hw_ver = qattr->hw_ver;
133 attr->max_qp = qattr->max_qp;
134 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
135 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
136 IB_DEVICE_RC_RNR_NAK_GEN |
137 IB_DEVICE_MEM_MGT_EXTENSIONS;
138 attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
139
140 if (!rdma_protocol_iwarp(&dev->ibdev, 1))
141 attr->device_cap_flags |= IB_DEVICE_XRC;
142 attr->max_send_sge = qattr->max_sge;
143 attr->max_recv_sge = qattr->max_sge;
144 attr->max_sge_rd = qattr->max_sge;
145 attr->max_cq = qattr->max_cq;
146 attr->max_cqe = qattr->max_cqe;
147 attr->max_mr = qattr->max_mr;
148 attr->max_mw = qattr->max_mw;
149 attr->max_pd = qattr->max_pd;
150 attr->atomic_cap = dev->atomic_cap;
151 attr->max_qp_init_rd_atom =
152 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
153 attr->max_qp_rd_atom =
154 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
155 attr->max_qp_init_rd_atom);
156
157 attr->max_srq = qattr->max_srq;
158 attr->max_srq_sge = qattr->max_srq_sge;
159 attr->max_srq_wr = qattr->max_srq_wr;
160
161 attr->local_ca_ack_delay = qattr->dev_ack_delay;
162 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
163 attr->max_pkeys = qattr->max_pkey;
164 attr->max_ah = qattr->max_ah;
165
166 return 0;
167}
168
169static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
170 u8 *ib_width)
171{
172 switch (speed) {
173 case 1000:
174 *ib_speed = IB_SPEED_SDR;
175 *ib_width = IB_WIDTH_1X;
176 break;
177 case 10000:
178 *ib_speed = IB_SPEED_QDR;
179 *ib_width = IB_WIDTH_1X;
180 break;
181
182 case 20000:
183 *ib_speed = IB_SPEED_DDR;
184 *ib_width = IB_WIDTH_4X;
185 break;
186
187 case 25000:
188 *ib_speed = IB_SPEED_EDR;
189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 40000:
193 *ib_speed = IB_SPEED_QDR;
194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 50000:
198 *ib_speed = IB_SPEED_HDR;
199 *ib_width = IB_WIDTH_1X;
200 break;
201
202 case 100000:
203 *ib_speed = IB_SPEED_EDR;
204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 default:
208 /* Unsupported */
209 *ib_speed = IB_SPEED_SDR;
210 *ib_width = IB_WIDTH_1X;
211 }
212}
213
214int qedr_query_port(struct ib_device *ibdev, u32 port,
215 struct ib_port_attr *attr)
216{
217 struct qedr_dev *dev;
218 struct qed_rdma_port *rdma_port;
219
220 dev = get_qedr_dev(ibdev);
221
222 if (!dev->rdma_ctx) {
223 DP_ERR(dev, "rdma_ctx is NULL\n");
224 return -EINVAL;
225 }
226
227 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
228
229 /* *attr being zeroed by the caller, avoid zeroing it here */
230 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
231 attr->state = IB_PORT_ACTIVE;
232 attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
233 } else {
234 attr->state = IB_PORT_DOWN;
235 attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
236 }
237 attr->max_mtu = IB_MTU_4096;
238 attr->lid = 0;
239 attr->lmc = 0;
240 attr->sm_lid = 0;
241 attr->sm_sl = 0;
242 attr->ip_gids = true;
243 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
244 attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
245 attr->gid_tbl_len = 1;
246 } else {
247 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
248 attr->gid_tbl_len = QEDR_MAX_SGID;
249 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
250 }
251 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
252 attr->qkey_viol_cntr = 0;
253 get_link_speed_and_width(rdma_port->link_speed,
254 &attr->active_speed, &attr->active_width);
255 attr->max_msg_sz = rdma_port->max_msg_size;
256 attr->max_vl_num = 4;
257
258 return 0;
259}
260
261int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
262{
263 struct ib_device *ibdev = uctx->device;
264 int rc;
265 struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
266 struct qedr_alloc_ucontext_resp uresp = {};
267 struct qedr_alloc_ucontext_req ureq = {};
268 struct qedr_dev *dev = get_qedr_dev(ibdev);
269 struct qed_rdma_add_user_out_params oparams;
270 struct qedr_user_mmap_entry *entry;
271
272 if (!udata)
273 return -EFAULT;
274
275 if (udata->inlen) {
276 rc = ib_copy_from_udata(&ureq, udata,
277 min(sizeof(ureq), udata->inlen));
278 if (rc) {
279 DP_ERR(dev, "Problem copying data from user space\n");
280 return -EFAULT;
281 }
282 ctx->edpm_mode = !!(ureq.context_flags &
283 QEDR_ALLOC_UCTX_EDPM_MODE);
284 ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
285 }
286
287 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
288 if (rc) {
289 DP_ERR(dev,
290 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
291 rc);
292 return rc;
293 }
294
295 ctx->dpi = oparams.dpi;
296 ctx->dpi_addr = oparams.dpi_addr;
297 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
298 ctx->dpi_size = oparams.dpi_size;
299 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
300 if (!entry) {
301 rc = -ENOMEM;
302 goto err;
303 }
304
305 entry->io_address = ctx->dpi_phys_addr;
306 entry->length = ctx->dpi_size;
307 entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
308 entry->dpi = ctx->dpi;
309 entry->dev = dev;
310 rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
311 ctx->dpi_size);
312 if (rc) {
313 kfree(entry);
314 goto err;
315 }
316 ctx->db_mmap_entry = &entry->rdma_entry;
317
318 if (!dev->user_dpm_enabled)
319 uresp.dpm_flags = 0;
320 else if (rdma_protocol_iwarp(&dev->ibdev, 1))
321 uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
322 else
323 uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
324 QEDR_DPM_TYPE_ROCE_LEGACY |
325 QEDR_DPM_TYPE_ROCE_EDPM_MODE;
326
327 if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
328 uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
329 uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
330 uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
331 uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
332 }
333
334 uresp.wids_enabled = 1;
335 uresp.wid_count = oparams.wid_count;
336 uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
337 uresp.db_size = ctx->dpi_size;
338 uresp.max_send_wr = dev->attr.max_sqe;
339 uresp.max_recv_wr = dev->attr.max_rqe;
340 uresp.max_srq_wr = dev->attr.max_srq_wr;
341 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
342 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
343 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
344 uresp.max_cqes = QEDR_MAX_CQES;
345
346 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
347 if (rc)
348 goto err;
349
350 ctx->dev = dev;
351
352 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
353 &ctx->ibucontext);
354 return 0;
355
356err:
357 if (!ctx->db_mmap_entry)
358 dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
359 else
360 rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
361
362 return rc;
363}
364
365void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
366{
367 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
368
369 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
370 uctx);
371
372 rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
373}
374
375void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
376{
377 struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
378 struct qedr_dev *dev = entry->dev;
379
380 if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
381 free_page((unsigned long)entry->address);
382 else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
383 dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
384
385 kfree(entry);
386}
387
388int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
389{
390 struct ib_device *dev = ucontext->device;
391 size_t length = vma->vm_end - vma->vm_start;
392 struct rdma_user_mmap_entry *rdma_entry;
393 struct qedr_user_mmap_entry *entry;
394 int rc = 0;
395 u64 pfn;
396
397 ibdev_dbg(dev,
398 "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
399 vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
400
401 rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
402 if (!rdma_entry) {
403 ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
404 vma->vm_pgoff);
405 return -EINVAL;
406 }
407 entry = get_qedr_mmap_entry(rdma_entry);
408 ibdev_dbg(dev,
409 "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
410 entry->io_address, length, entry->mmap_flag);
411
412 switch (entry->mmap_flag) {
413 case QEDR_USER_MMAP_IO_WC:
414 pfn = entry->io_address >> PAGE_SHIFT;
415 rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
416 pgprot_writecombine(vma->vm_page_prot),
417 rdma_entry);
418 break;
419 case QEDR_USER_MMAP_PHYS_PAGE:
420 rc = vm_insert_page(vma, vma->vm_start,
421 virt_to_page(entry->address));
422 break;
423 default:
424 rc = -EINVAL;
425 }
426
427 if (rc)
428 ibdev_dbg(dev,
429 "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
430 entry->io_address, length, entry->mmap_flag, rc);
431
432 rdma_user_mmap_entry_put(rdma_entry);
433 return rc;
434}
435
436int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
437{
438 struct ib_device *ibdev = ibpd->device;
439 struct qedr_dev *dev = get_qedr_dev(ibdev);
440 struct qedr_pd *pd = get_qedr_pd(ibpd);
441 u16 pd_id;
442 int rc;
443
444 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
445 udata ? "User Lib" : "Kernel");
446
447 if (!dev->rdma_ctx) {
448 DP_ERR(dev, "invalid RDMA context\n");
449 return -EINVAL;
450 }
451
452 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
453 if (rc)
454 return rc;
455
456 pd->pd_id = pd_id;
457
458 if (udata) {
459 struct qedr_alloc_pd_uresp uresp = {
460 .pd_id = pd_id,
461 };
462 struct qedr_ucontext *context = rdma_udata_to_drv_context(
463 udata, struct qedr_ucontext, ibucontext);
464
465 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
466 if (rc) {
467 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
468 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
469 return rc;
470 }
471
472 pd->uctx = context;
473 pd->uctx->pd = pd;
474 }
475
476 return 0;
477}
478
479int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
480{
481 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
482 struct qedr_pd *pd = get_qedr_pd(ibpd);
483
484 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
485 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
486 return 0;
487}
488
489
490int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
491{
492 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
493 struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
494
495 return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
496}
497
498int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
499{
500 struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
501 u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
502
503 dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
504 return 0;
505}
506static void qedr_free_pbl(struct qedr_dev *dev,
507 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
508{
509 struct pci_dev *pdev = dev->pdev;
510 int i;
511
512 for (i = 0; i < pbl_info->num_pbls; i++) {
513 if (!pbl[i].va)
514 continue;
515 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
516 pbl[i].va, pbl[i].pa);
517 }
518
519 kfree(pbl);
520}
521
522#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
523#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
524
525#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
526#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
527#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
528
529static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
530 struct qedr_pbl_info *pbl_info,
531 gfp_t flags)
532{
533 struct pci_dev *pdev = dev->pdev;
534 struct qedr_pbl *pbl_table;
535 dma_addr_t *pbl_main_tbl;
536 dma_addr_t pa;
537 void *va;
538 int i;
539
540 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
541 if (!pbl_table)
542 return ERR_PTR(-ENOMEM);
543
544 for (i = 0; i < pbl_info->num_pbls; i++) {
545 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
546 flags);
547 if (!va)
548 goto err;
549
550 pbl_table[i].va = va;
551 pbl_table[i].pa = pa;
552 }
553
554 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
555 * the first one with physical pointers to all of the rest
556 */
557 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
558 for (i = 0; i < pbl_info->num_pbls - 1; i++)
559 pbl_main_tbl[i] = pbl_table[i + 1].pa;
560
561 return pbl_table;
562
563err:
564 for (i--; i >= 0; i--)
565 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
566 pbl_table[i].va, pbl_table[i].pa);
567
568 qedr_free_pbl(dev, pbl_info, pbl_table);
569
570 return ERR_PTR(-ENOMEM);
571}
572
573static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
574 struct qedr_pbl_info *pbl_info,
575 u32 num_pbes, int two_layer_capable)
576{
577 u32 pbl_capacity;
578 u32 pbl_size;
579 u32 num_pbls;
580
581 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
582 if (num_pbes > MAX_PBES_TWO_LAYER) {
583 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
584 num_pbes);
585 return -EINVAL;
586 }
587
588 /* calculate required pbl page size */
589 pbl_size = MIN_FW_PBL_PAGE_SIZE;
590 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
591 NUM_PBES_ON_PAGE(pbl_size);
592
593 while (pbl_capacity < num_pbes) {
594 pbl_size *= 2;
595 pbl_capacity = pbl_size / sizeof(u64);
596 pbl_capacity = pbl_capacity * pbl_capacity;
597 }
598
599 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
600 num_pbls++; /* One for the layer0 ( points to the pbls) */
601 pbl_info->two_layered = true;
602 } else {
603 /* One layered PBL */
604 num_pbls = 1;
605 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
606 roundup_pow_of_two((num_pbes * sizeof(u64))));
607 pbl_info->two_layered = false;
608 }
609
610 pbl_info->num_pbls = num_pbls;
611 pbl_info->pbl_size = pbl_size;
612 pbl_info->num_pbes = num_pbes;
613
614 DP_DEBUG(dev, QEDR_MSG_MR,
615 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
616 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
617
618 return 0;
619}
620
621static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
622 struct qedr_pbl *pbl,
623 struct qedr_pbl_info *pbl_info, u32 pg_shift)
624{
625 int pbe_cnt, total_num_pbes = 0;
626 struct qedr_pbl *pbl_tbl;
627 struct ib_block_iter biter;
628 struct regpair *pbe;
629
630 if (!pbl_info->num_pbes)
631 return;
632
633 /* If we have a two layered pbl, the first pbl points to the rest
634 * of the pbls and the first entry lays on the second pbl in the table
635 */
636 if (pbl_info->two_layered)
637 pbl_tbl = &pbl[1];
638 else
639 pbl_tbl = pbl;
640
641 pbe = (struct regpair *)pbl_tbl->va;
642 if (!pbe) {
643 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
644 return;
645 }
646
647 pbe_cnt = 0;
648
649 rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
650 u64 pg_addr = rdma_block_iter_dma_address(&biter);
651
652 pbe->lo = cpu_to_le32(pg_addr);
653 pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
654
655 pbe_cnt++;
656 total_num_pbes++;
657 pbe++;
658
659 if (total_num_pbes == pbl_info->num_pbes)
660 return;
661
662 /* If the given pbl is full storing the pbes, move to next pbl.
663 */
664 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
665 pbl_tbl++;
666 pbe = (struct regpair *)pbl_tbl->va;
667 pbe_cnt = 0;
668 }
669 }
670}
671
672static int qedr_db_recovery_add(struct qedr_dev *dev,
673 void __iomem *db_addr,
674 void *db_data,
675 enum qed_db_rec_width db_width,
676 enum qed_db_rec_space db_space)
677{
678 if (!db_data) {
679 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
680 return 0;
681 }
682
683 return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
684 db_width, db_space);
685}
686
687static void qedr_db_recovery_del(struct qedr_dev *dev,
688 void __iomem *db_addr,
689 void *db_data)
690{
691 if (!db_data) {
692 DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
693 return;
694 }
695
696 /* Ignore return code as there is not much we can do about it. Error
697 * log will be printed inside.
698 */
699 dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
700}
701
702static int qedr_copy_cq_uresp(struct qedr_dev *dev,
703 struct qedr_cq *cq, struct ib_udata *udata,
704 u32 db_offset)
705{
706 struct qedr_create_cq_uresp uresp;
707 int rc;
708
709 memset(&uresp, 0, sizeof(uresp));
710
711 uresp.db_offset = db_offset;
712 uresp.icid = cq->icid;
713 if (cq->q.db_mmap_entry)
714 uresp.db_rec_addr =
715 rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
716
717 rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
718 if (rc)
719 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
720
721 return rc;
722}
723
724static void consume_cqe(struct qedr_cq *cq)
725{
726 if (cq->latest_cqe == cq->toggle_cqe)
727 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
728
729 cq->latest_cqe = qed_chain_consume(&cq->pbl);
730}
731
732static inline int qedr_align_cq_entries(int entries)
733{
734 u64 size, aligned_size;
735
736 /* We allocate an extra entry that we don't report to the FW. */
737 size = (entries + 1) * QEDR_CQE_SIZE;
738 aligned_size = ALIGN(size, PAGE_SIZE);
739
740 return aligned_size / QEDR_CQE_SIZE;
741}
742
743static int qedr_init_user_db_rec(struct ib_udata *udata,
744 struct qedr_dev *dev, struct qedr_userq *q,
745 bool requires_db_rec)
746{
747 struct qedr_ucontext *uctx =
748 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
749 ibucontext);
750 struct qedr_user_mmap_entry *entry;
751 int rc;
752
753 /* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
754 if (requires_db_rec == 0 || !uctx->db_rec)
755 return 0;
756
757 /* Allocate a page for doorbell recovery, add to mmap */
758 q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
759 if (!q->db_rec_data) {
760 DP_ERR(dev, "get_zeroed_page failed\n");
761 return -ENOMEM;
762 }
763
764 entry = kzalloc(sizeof(*entry), GFP_KERNEL);
765 if (!entry)
766 goto err_free_db_data;
767
768 entry->address = q->db_rec_data;
769 entry->length = PAGE_SIZE;
770 entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
771 rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
772 &entry->rdma_entry,
773 PAGE_SIZE);
774 if (rc)
775 goto err_free_entry;
776
777 q->db_mmap_entry = &entry->rdma_entry;
778
779 return 0;
780
781err_free_entry:
782 kfree(entry);
783
784err_free_db_data:
785 free_page((unsigned long)q->db_rec_data);
786 q->db_rec_data = NULL;
787 return -ENOMEM;
788}
789
790static inline int qedr_init_user_queue(struct ib_udata *udata,
791 struct qedr_dev *dev,
792 struct qedr_userq *q, u64 buf_addr,
793 size_t buf_len, bool requires_db_rec,
794 int access,
795 int alloc_and_init)
796{
797 u32 fw_pages;
798 int rc;
799
800 q->buf_addr = buf_addr;
801 q->buf_len = buf_len;
802 q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
803 if (IS_ERR(q->umem)) {
804 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
805 PTR_ERR(q->umem));
806 return PTR_ERR(q->umem);
807 }
808
809 fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
810 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
811 if (rc)
812 goto err0;
813
814 if (alloc_and_init) {
815 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
816 if (IS_ERR(q->pbl_tbl)) {
817 rc = PTR_ERR(q->pbl_tbl);
818 goto err0;
819 }
820 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
821 FW_PAGE_SHIFT);
822 } else {
823 q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
824 if (!q->pbl_tbl) {
825 rc = -ENOMEM;
826 goto err0;
827 }
828 }
829
830 /* mmap the user address used to store doorbell data for recovery */
831 return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
832
833err0:
834 ib_umem_release(q->umem);
835 q->umem = NULL;
836
837 return rc;
838}
839
840static inline void qedr_init_cq_params(struct qedr_cq *cq,
841 struct qedr_ucontext *ctx,
842 struct qedr_dev *dev, int vector,
843 int chain_entries, int page_cnt,
844 u64 pbl_ptr,
845 struct qed_rdma_create_cq_in_params
846 *params)
847{
848 memset(params, 0, sizeof(*params));
849 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
850 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
851 params->cnq_id = vector;
852 params->cq_size = chain_entries - 1;
853 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
854 params->pbl_num_pages = page_cnt;
855 params->pbl_ptr = pbl_ptr;
856 params->pbl_two_level = 0;
857}
858
859static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
860{
861 cq->db.data.agg_flags = flags;
862 cq->db.data.value = cpu_to_le32(cons);
863 writeq(cq->db.raw, cq->db_addr);
864}
865
866int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
867{
868 struct qedr_cq *cq = get_qedr_cq(ibcq);
869 unsigned long sflags;
870 struct qedr_dev *dev;
871
872 dev = get_qedr_dev(ibcq->device);
873
874 if (cq->destroyed) {
875 DP_ERR(dev,
876 "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
877 cq, cq->icid);
878 return -EINVAL;
879 }
880
881
882 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
883 return 0;
884
885 spin_lock_irqsave(&cq->cq_lock, sflags);
886
887 cq->arm_flags = 0;
888
889 if (flags & IB_CQ_SOLICITED)
890 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
891
892 if (flags & IB_CQ_NEXT_COMP)
893 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
894
895 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
896
897 spin_unlock_irqrestore(&cq->cq_lock, sflags);
898
899 return 0;
900}
901
902int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
903 struct ib_udata *udata)
904{
905 struct ib_device *ibdev = ibcq->device;
906 struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
907 udata, struct qedr_ucontext, ibucontext);
908 struct qed_rdma_destroy_cq_out_params destroy_oparams;
909 struct qed_rdma_destroy_cq_in_params destroy_iparams;
910 struct qed_chain_init_params chain_params = {
911 .mode = QED_CHAIN_MODE_PBL,
912 .intended_use = QED_CHAIN_USE_TO_CONSUME,
913 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
914 .elem_size = sizeof(union rdma_cqe),
915 };
916 struct qedr_dev *dev = get_qedr_dev(ibdev);
917 struct qed_rdma_create_cq_in_params params;
918 struct qedr_create_cq_ureq ureq = {};
919 int vector = attr->comp_vector;
920 int entries = attr->cqe;
921 struct qedr_cq *cq = get_qedr_cq(ibcq);
922 int chain_entries;
923 u32 db_offset;
924 int page_cnt;
925 u64 pbl_ptr;
926 u16 icid;
927 int rc;
928
929 DP_DEBUG(dev, QEDR_MSG_INIT,
930 "create_cq: called from %s. entries=%d, vector=%d\n",
931 udata ? "User Lib" : "Kernel", entries, vector);
932
933 if (attr->flags)
934 return -EOPNOTSUPP;
935
936 if (entries > QEDR_MAX_CQES) {
937 DP_ERR(dev,
938 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
939 entries, QEDR_MAX_CQES);
940 return -EINVAL;
941 }
942
943 chain_entries = qedr_align_cq_entries(entries);
944 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
945 chain_params.num_elems = chain_entries;
946
947 /* calc db offset. user will add DPI base, kernel will add db addr */
948 db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
949
950 if (udata) {
951 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
952 udata->inlen))) {
953 DP_ERR(dev,
954 "create cq: problem copying data from user space\n");
955 goto err0;
956 }
957
958 if (!ureq.len) {
959 DP_ERR(dev,
960 "create cq: cannot create a cq with 0 entries\n");
961 goto err0;
962 }
963
964 cq->cq_type = QEDR_CQ_TYPE_USER;
965
966 rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
967 ureq.len, true, IB_ACCESS_LOCAL_WRITE,
968 1);
969 if (rc)
970 goto err0;
971
972 pbl_ptr = cq->q.pbl_tbl->pa;
973 page_cnt = cq->q.pbl_info.num_pbes;
974
975 cq->ibcq.cqe = chain_entries;
976 cq->q.db_addr = ctx->dpi_addr + db_offset;
977 } else {
978 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
979
980 rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
981 &chain_params);
982 if (rc)
983 goto err0;
984
985 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
986 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
987 cq->ibcq.cqe = cq->pbl.capacity;
988 }
989
990 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
991 pbl_ptr, ¶ms);
992
993 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
994 if (rc)
995 goto err1;
996
997 cq->icid = icid;
998 cq->sig = QEDR_CQ_MAGIC_NUMBER;
999 spin_lock_init(&cq->cq_lock);
1000
1001 if (udata) {
1002 rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1003 if (rc)
1004 goto err2;
1005
1006 rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1007 &cq->q.db_rec_data->db_data,
1008 DB_REC_WIDTH_64B,
1009 DB_REC_USER);
1010 if (rc)
1011 goto err2;
1012
1013 } else {
1014 /* Generate doorbell address. */
1015 cq->db.data.icid = cq->icid;
1016 cq->db_addr = dev->db_addr + db_offset;
1017 cq->db.data.params = DB_AGG_CMD_MAX <<
1018 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1019
1020 /* point to the very last element, passing it we will toggle */
1021 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1022 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1023 cq->latest_cqe = NULL;
1024 consume_cqe(cq);
1025 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1026
1027 rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1028 DB_REC_WIDTH_64B, DB_REC_KERNEL);
1029 if (rc)
1030 goto err2;
1031 }
1032
1033 DP_DEBUG(dev, QEDR_MSG_CQ,
1034 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1035 cq->icid, cq, params.cq_size);
1036
1037 return 0;
1038
1039err2:
1040 destroy_iparams.icid = cq->icid;
1041 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1042 &destroy_oparams);
1043err1:
1044 if (udata) {
1045 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1046 ib_umem_release(cq->q.umem);
1047 if (cq->q.db_mmap_entry)
1048 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1049 } else {
1050 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1051 }
1052err0:
1053 return -EINVAL;
1054}
1055
1056#define QEDR_DESTROY_CQ_MAX_ITERATIONS (10)
1057#define QEDR_DESTROY_CQ_ITER_DURATION (10)
1058
1059int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1060{
1061 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1062 struct qed_rdma_destroy_cq_out_params oparams;
1063 struct qed_rdma_destroy_cq_in_params iparams;
1064 struct qedr_cq *cq = get_qedr_cq(ibcq);
1065 int iter;
1066
1067 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1068
1069 cq->destroyed = 1;
1070
1071 /* GSIs CQs are handled by driver, so they don't exist in the FW */
1072 if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1073 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1074 return 0;
1075 }
1076
1077 iparams.icid = cq->icid;
1078 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1079 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1080
1081 if (udata) {
1082 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1083 ib_umem_release(cq->q.umem);
1084
1085 if (cq->q.db_rec_data) {
1086 qedr_db_recovery_del(dev, cq->q.db_addr,
1087 &cq->q.db_rec_data->db_data);
1088 rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1089 }
1090 } else {
1091 qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1092 }
1093
1094 /* We don't want the IRQ handler to handle a non-existing CQ so we
1095 * wait until all CNQ interrupts, if any, are received. This will always
1096 * happen and will always happen very fast. If not, then a serious error
1097 * has occured. That is why we can use a long delay.
1098 * We spin for a short time so we don’t lose time on context switching
1099 * in case all the completions are handled in that span. Otherwise
1100 * we sleep for a while and check again. Since the CNQ may be
1101 * associated with (only) the current CPU we use msleep to allow the
1102 * current CPU to be freed.
1103 * The CNQ notification is increased in qedr_irq_handler().
1104 */
1105 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1106 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1107 udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1108 iter--;
1109 }
1110
1111 iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1112 while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1113 msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1114 iter--;
1115 }
1116
1117 /* Note that we don't need to have explicit code to wait for the
1118 * completion of the event handler because it is invoked from the EQ.
1119 * Since the destroy CQ ramrod has also been received on the EQ we can
1120 * be certain that there's no event handler in process.
1121 */
1122 return 0;
1123}
1124
1125static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1126 struct ib_qp_attr *attr,
1127 int attr_mask,
1128 struct qed_rdma_modify_qp_in_params
1129 *qp_params)
1130{
1131 const struct ib_gid_attr *gid_attr;
1132 enum rdma_network_type nw_type;
1133 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1134 u32 ipv4_addr;
1135 int ret;
1136 int i;
1137
1138 gid_attr = grh->sgid_attr;
1139 ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1140 if (ret)
1141 return ret;
1142
1143 nw_type = rdma_gid_attr_network_type(gid_attr);
1144 switch (nw_type) {
1145 case RDMA_NETWORK_IPV6:
1146 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1147 sizeof(qp_params->sgid));
1148 memcpy(&qp_params->dgid.bytes[0],
1149 &grh->dgid,
1150 sizeof(qp_params->dgid));
1151 qp_params->roce_mode = ROCE_V2_IPV6;
1152 SET_FIELD(qp_params->modify_flags,
1153 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154 break;
1155 case RDMA_NETWORK_ROCE_V1:
1156 memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1157 sizeof(qp_params->sgid));
1158 memcpy(&qp_params->dgid.bytes[0],
1159 &grh->dgid,
1160 sizeof(qp_params->dgid));
1161 qp_params->roce_mode = ROCE_V1;
1162 break;
1163 case RDMA_NETWORK_IPV4:
1164 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1165 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1166 ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1167 qp_params->sgid.ipv4_addr = ipv4_addr;
1168 ipv4_addr =
1169 qedr_get_ipv4_from_gid(grh->dgid.raw);
1170 qp_params->dgid.ipv4_addr = ipv4_addr;
1171 SET_FIELD(qp_params->modify_flags,
1172 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1173 qp_params->roce_mode = ROCE_V2_IPV4;
1174 break;
1175 default:
1176 return -EINVAL;
1177 }
1178
1179 for (i = 0; i < 4; i++) {
1180 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1181 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1182 }
1183
1184 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1185 qp_params->vlan_id = 0;
1186
1187 return 0;
1188}
1189
1190static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1191 struct ib_qp_init_attr *attrs,
1192 struct ib_udata *udata)
1193{
1194 struct qedr_device_attr *qattr = &dev->attr;
1195
1196 /* QP0... attrs->qp_type == IB_QPT_GSI */
1197 if (attrs->qp_type != IB_QPT_RC &&
1198 attrs->qp_type != IB_QPT_GSI &&
1199 attrs->qp_type != IB_QPT_XRC_INI &&
1200 attrs->qp_type != IB_QPT_XRC_TGT) {
1201 DP_DEBUG(dev, QEDR_MSG_QP,
1202 "create qp: unsupported qp type=0x%x requested\n",
1203 attrs->qp_type);
1204 return -EOPNOTSUPP;
1205 }
1206
1207 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1208 DP_ERR(dev,
1209 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1210 attrs->cap.max_send_wr, qattr->max_sqe);
1211 return -EINVAL;
1212 }
1213
1214 if (attrs->cap.max_inline_data > qattr->max_inline) {
1215 DP_ERR(dev,
1216 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1217 attrs->cap.max_inline_data, qattr->max_inline);
1218 return -EINVAL;
1219 }
1220
1221 if (attrs->cap.max_send_sge > qattr->max_sge) {
1222 DP_ERR(dev,
1223 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1224 attrs->cap.max_send_sge, qattr->max_sge);
1225 return -EINVAL;
1226 }
1227
1228 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1229 DP_ERR(dev,
1230 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1231 attrs->cap.max_recv_sge, qattr->max_sge);
1232 return -EINVAL;
1233 }
1234
1235 /* verify consumer QPs are not trying to use GSI QP's CQ.
1236 * TGT QP isn't associated with RQ/SQ
1237 */
1238 if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1239 (attrs->qp_type != IB_QPT_XRC_TGT) &&
1240 (attrs->qp_type != IB_QPT_XRC_INI)) {
1241 struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1242 struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1243
1244 if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1245 (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1246 DP_ERR(dev,
1247 "create qp: consumer QP cannot use GSI CQs.\n");
1248 return -EINVAL;
1249 }
1250 }
1251
1252 return 0;
1253}
1254
1255static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1256 struct qedr_srq *srq, struct ib_udata *udata)
1257{
1258 struct qedr_create_srq_uresp uresp = {};
1259 int rc;
1260
1261 uresp.srq_id = srq->srq_id;
1262
1263 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1264 if (rc)
1265 DP_ERR(dev, "create srq: problem copying data to user space\n");
1266
1267 return rc;
1268}
1269
1270static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1271 struct qedr_create_qp_uresp *uresp,
1272 struct qedr_qp *qp)
1273{
1274 /* iWARP requires two doorbells per RQ. */
1275 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1276 uresp->rq_db_offset =
1277 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1278 uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1279 } else {
1280 uresp->rq_db_offset =
1281 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1282 }
1283
1284 uresp->rq_icid = qp->icid;
1285 if (qp->urq.db_mmap_entry)
1286 uresp->rq_db_rec_addr =
1287 rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1288}
1289
1290static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1291 struct qedr_create_qp_uresp *uresp,
1292 struct qedr_qp *qp)
1293{
1294 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1295
1296 /* iWARP uses the same cid for rq and sq */
1297 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1298 uresp->sq_icid = qp->icid;
1299 else
1300 uresp->sq_icid = qp->icid + 1;
1301
1302 if (qp->usq.db_mmap_entry)
1303 uresp->sq_db_rec_addr =
1304 rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1305}
1306
1307static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1308 struct qedr_qp *qp, struct ib_udata *udata,
1309 struct qedr_create_qp_uresp *uresp)
1310{
1311 int rc;
1312
1313 memset(uresp, 0, sizeof(*uresp));
1314
1315 if (qedr_qp_has_sq(qp))
1316 qedr_copy_sq_uresp(dev, uresp, qp);
1317
1318 if (qedr_qp_has_rq(qp))
1319 qedr_copy_rq_uresp(dev, uresp, qp);
1320
1321 uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1322 uresp->qp_id = qp->qp_id;
1323
1324 rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1325 if (rc)
1326 DP_ERR(dev,
1327 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1328 qp->icid);
1329
1330 return rc;
1331}
1332
1333static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1334{
1335 qed_chain_reset(&qph->pbl);
1336 qph->prod = 0;
1337 qph->cons = 0;
1338 qph->wqe_cons = 0;
1339 qph->db_data.data.value = cpu_to_le16(0);
1340}
1341
1342static void qedr_set_common_qp_params(struct qedr_dev *dev,
1343 struct qedr_qp *qp,
1344 struct qedr_pd *pd,
1345 struct ib_qp_init_attr *attrs)
1346{
1347 spin_lock_init(&qp->q_lock);
1348 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1349 kref_init(&qp->refcnt);
1350 init_completion(&qp->iwarp_cm_comp);
1351 init_completion(&qp->qp_rel_comp);
1352 }
1353
1354 qp->pd = pd;
1355 qp->qp_type = attrs->qp_type;
1356 qp->max_inline_data = attrs->cap.max_inline_data;
1357 qp->state = QED_ROCE_QP_STATE_RESET;
1358
1359 qp->prev_wqe_size = 0;
1360
1361 qp->signaled = attrs->sq_sig_type == IB_SIGNAL_ALL_WR;
1362 qp->dev = dev;
1363 if (qedr_qp_has_sq(qp)) {
1364 qedr_reset_qp_hwq_info(&qp->sq);
1365 qp->sq.max_sges = attrs->cap.max_send_sge;
1366 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1367 DP_DEBUG(dev, QEDR_MSG_QP,
1368 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1369 qp->sq.max_sges, qp->sq_cq->icid);
1370 }
1371
1372 if (attrs->srq)
1373 qp->srq = get_qedr_srq(attrs->srq);
1374
1375 if (qedr_qp_has_rq(qp)) {
1376 qedr_reset_qp_hwq_info(&qp->rq);
1377 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1378 qp->rq.max_sges = attrs->cap.max_recv_sge;
1379 DP_DEBUG(dev, QEDR_MSG_QP,
1380 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1381 qp->rq.max_sges, qp->rq_cq->icid);
1382 }
1383
1384 DP_DEBUG(dev, QEDR_MSG_QP,
1385 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1386 pd->pd_id, qp->qp_type, qp->max_inline_data,
1387 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1388 DP_DEBUG(dev, QEDR_MSG_QP,
1389 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1390 qp->sq.max_sges, qp->sq_cq->icid);
1391}
1392
1393static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1394{
1395 int rc = 0;
1396
1397 if (qedr_qp_has_sq(qp)) {
1398 qp->sq.db = dev->db_addr +
1399 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1400 qp->sq.db_data.data.icid = qp->icid + 1;
1401 rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1402 DB_REC_WIDTH_32B, DB_REC_KERNEL);
1403 if (rc)
1404 return rc;
1405 }
1406
1407 if (qedr_qp_has_rq(qp)) {
1408 qp->rq.db = dev->db_addr +
1409 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1410 qp->rq.db_data.data.icid = qp->icid;
1411 rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1412 DB_REC_WIDTH_32B, DB_REC_KERNEL);
1413 if (rc && qedr_qp_has_sq(qp))
1414 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1415 }
1416
1417 return rc;
1418}
1419
1420static int qedr_check_srq_params(struct qedr_dev *dev,
1421 struct ib_srq_init_attr *attrs,
1422 struct ib_udata *udata)
1423{
1424 struct qedr_device_attr *qattr = &dev->attr;
1425
1426 if (attrs->attr.max_wr > qattr->max_srq_wr) {
1427 DP_ERR(dev,
1428 "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1429 attrs->attr.max_wr, qattr->max_srq_wr);
1430 return -EINVAL;
1431 }
1432
1433 if (attrs->attr.max_sge > qattr->max_sge) {
1434 DP_ERR(dev,
1435 "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1436 attrs->attr.max_sge, qattr->max_sge);
1437 }
1438
1439 if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1440 DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1441 return -EINVAL;
1442 }
1443
1444 return 0;
1445}
1446
1447static void qedr_free_srq_user_params(struct qedr_srq *srq)
1448{
1449 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1450 ib_umem_release(srq->usrq.umem);
1451 ib_umem_release(srq->prod_umem);
1452}
1453
1454static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1455{
1456 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1457 struct qedr_dev *dev = srq->dev;
1458
1459 dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1460
1461 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1462 hw_srq->virt_prod_pair_addr,
1463 hw_srq->phy_prod_pair_addr);
1464}
1465
1466static int qedr_init_srq_user_params(struct ib_udata *udata,
1467 struct qedr_srq *srq,
1468 struct qedr_create_srq_ureq *ureq,
1469 int access)
1470{
1471 struct scatterlist *sg;
1472 int rc;
1473
1474 rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1475 ureq->srq_len, false, access, 1);
1476 if (rc)
1477 return rc;
1478
1479 srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1480 sizeof(struct rdma_srq_producers), access);
1481 if (IS_ERR(srq->prod_umem)) {
1482 qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1483 ib_umem_release(srq->usrq.umem);
1484 DP_ERR(srq->dev,
1485 "create srq: failed ib_umem_get for producer, got %ld\n",
1486 PTR_ERR(srq->prod_umem));
1487 return PTR_ERR(srq->prod_umem);
1488 }
1489
1490 sg = srq->prod_umem->sgt_append.sgt.sgl;
1491 srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1492
1493 return 0;
1494}
1495
1496static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1497 struct qedr_dev *dev,
1498 struct ib_srq_init_attr *init_attr)
1499{
1500 struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1501 struct qed_chain_init_params params = {
1502 .mode = QED_CHAIN_MODE_PBL,
1503 .intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1504 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
1505 .elem_size = QEDR_SRQ_WQE_ELEM_SIZE,
1506 };
1507 dma_addr_t phy_prod_pair_addr;
1508 u32 num_elems;
1509 void *va;
1510 int rc;
1511
1512 va = dma_alloc_coherent(&dev->pdev->dev,
1513 sizeof(struct rdma_srq_producers),
1514 &phy_prod_pair_addr, GFP_KERNEL);
1515 if (!va) {
1516 DP_ERR(dev,
1517 "create srq: failed to allocate dma memory for producer\n");
1518 return -ENOMEM;
1519 }
1520
1521 hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1522 hw_srq->virt_prod_pair_addr = va;
1523
1524 num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1525 params.num_elems = num_elems;
1526
1527 rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, ¶ms);
1528 if (rc)
1529 goto err0;
1530
1531 hw_srq->num_elems = num_elems;
1532
1533 return 0;
1534
1535err0:
1536 dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1537 va, phy_prod_pair_addr);
1538 return rc;
1539}
1540
1541int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1542 struct ib_udata *udata)
1543{
1544 struct qed_rdma_destroy_srq_in_params destroy_in_params;
1545 struct qed_rdma_create_srq_in_params in_params = {};
1546 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1547 struct qed_rdma_create_srq_out_params out_params;
1548 struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1549 struct qedr_create_srq_ureq ureq = {};
1550 u64 pbl_base_addr, phy_prod_pair_addr;
1551 struct qedr_srq_hwq_info *hw_srq;
1552 u32 page_cnt, page_size;
1553 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1554 int rc = 0;
1555
1556 DP_DEBUG(dev, QEDR_MSG_QP,
1557 "create SRQ called from %s (pd %p)\n",
1558 (udata) ? "User lib" : "kernel", pd);
1559
1560 if (init_attr->srq_type != IB_SRQT_BASIC &&
1561 init_attr->srq_type != IB_SRQT_XRC)
1562 return -EOPNOTSUPP;
1563
1564 rc = qedr_check_srq_params(dev, init_attr, udata);
1565 if (rc)
1566 return -EINVAL;
1567
1568 srq->dev = dev;
1569 srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1570 hw_srq = &srq->hw_srq;
1571 spin_lock_init(&srq->lock);
1572
1573 hw_srq->max_wr = init_attr->attr.max_wr;
1574 hw_srq->max_sges = init_attr->attr.max_sge;
1575
1576 if (udata) {
1577 if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1578 udata->inlen))) {
1579 DP_ERR(dev,
1580 "create srq: problem copying data from user space\n");
1581 goto err0;
1582 }
1583
1584 rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1585 if (rc)
1586 goto err0;
1587
1588 page_cnt = srq->usrq.pbl_info.num_pbes;
1589 pbl_base_addr = srq->usrq.pbl_tbl->pa;
1590 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1591 page_size = PAGE_SIZE;
1592 } else {
1593 struct qed_chain *pbl;
1594
1595 rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1596 if (rc)
1597 goto err0;
1598
1599 pbl = &hw_srq->pbl;
1600 page_cnt = qed_chain_get_page_cnt(pbl);
1601 pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1602 phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1603 page_size = QED_CHAIN_PAGE_SIZE;
1604 }
1605
1606 in_params.pd_id = pd->pd_id;
1607 in_params.pbl_base_addr = pbl_base_addr;
1608 in_params.prod_pair_addr = phy_prod_pair_addr;
1609 in_params.num_pages = page_cnt;
1610 in_params.page_size = page_size;
1611 if (srq->is_xrc) {
1612 struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1613 struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1614
1615 in_params.is_xrc = 1;
1616 in_params.xrcd_id = xrcd->xrcd_id;
1617 in_params.cq_cid = cq->icid;
1618 }
1619
1620 rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1621 if (rc)
1622 goto err1;
1623
1624 srq->srq_id = out_params.srq_id;
1625
1626 if (udata) {
1627 rc = qedr_copy_srq_uresp(dev, srq, udata);
1628 if (rc)
1629 goto err2;
1630 }
1631
1632 rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1633 if (rc)
1634 goto err2;
1635
1636 DP_DEBUG(dev, QEDR_MSG_SRQ,
1637 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1638 return 0;
1639
1640err2:
1641 destroy_in_params.srq_id = srq->srq_id;
1642
1643 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1644err1:
1645 if (udata)
1646 qedr_free_srq_user_params(srq);
1647 else
1648 qedr_free_srq_kernel_params(srq);
1649err0:
1650 return -EFAULT;
1651}
1652
1653int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1654{
1655 struct qed_rdma_destroy_srq_in_params in_params = {};
1656 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1657 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1658
1659 xa_erase_irq(&dev->srqs, srq->srq_id);
1660 in_params.srq_id = srq->srq_id;
1661 in_params.is_xrc = srq->is_xrc;
1662 dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1663
1664 if (ibsrq->uobject)
1665 qedr_free_srq_user_params(srq);
1666 else
1667 qedr_free_srq_kernel_params(srq);
1668
1669 DP_DEBUG(dev, QEDR_MSG_SRQ,
1670 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1671 srq->srq_id);
1672 return 0;
1673}
1674
1675int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1676 enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1677{
1678 struct qed_rdma_modify_srq_in_params in_params = {};
1679 struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1680 struct qedr_srq *srq = get_qedr_srq(ibsrq);
1681 int rc;
1682
1683 if (attr_mask & IB_SRQ_MAX_WR) {
1684 DP_ERR(dev,
1685 "modify srq: invalid attribute mask=0x%x specified for %p\n",
1686 attr_mask, srq);
1687 return -EINVAL;
1688 }
1689
1690 if (attr_mask & IB_SRQ_LIMIT) {
1691 if (attr->srq_limit >= srq->hw_srq.max_wr) {
1692 DP_ERR(dev,
1693 "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1694 attr->srq_limit, srq->hw_srq.max_wr);
1695 return -EINVAL;
1696 }
1697
1698 in_params.srq_id = srq->srq_id;
1699 in_params.wqe_limit = attr->srq_limit;
1700 rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1701 if (rc)
1702 return rc;
1703 }
1704
1705 srq->srq_limit = attr->srq_limit;
1706
1707 DP_DEBUG(dev, QEDR_MSG_SRQ,
1708 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1709
1710 return 0;
1711}
1712
1713static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1714{
1715 switch (ib_qp_type) {
1716 case IB_QPT_RC:
1717 return QED_RDMA_QP_TYPE_RC;
1718 case IB_QPT_XRC_INI:
1719 return QED_RDMA_QP_TYPE_XRC_INI;
1720 case IB_QPT_XRC_TGT:
1721 return QED_RDMA_QP_TYPE_XRC_TGT;
1722 default:
1723 return QED_RDMA_QP_TYPE_INVAL;
1724 }
1725}
1726
1727static inline void
1728qedr_init_common_qp_in_params(struct qedr_dev *dev,
1729 struct qedr_pd *pd,
1730 struct qedr_qp *qp,
1731 struct ib_qp_init_attr *attrs,
1732 bool fmr_and_reserved_lkey,
1733 struct qed_rdma_create_qp_in_params *params)
1734{
1735 /* QP handle to be written in an async event */
1736 params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1737 params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1738
1739 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1740 params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1741 params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1742 params->stats_queue = 0;
1743
1744 if (pd) {
1745 params->pd = pd->pd_id;
1746 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1747 }
1748
1749 if (qedr_qp_has_sq(qp))
1750 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1751
1752 if (qedr_qp_has_rq(qp))
1753 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1754
1755 if (qedr_qp_has_srq(qp)) {
1756 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1757 params->srq_id = qp->srq->srq_id;
1758 params->use_srq = true;
1759 } else {
1760 params->srq_id = 0;
1761 params->use_srq = false;
1762 }
1763}
1764
1765static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1766{
1767 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1768 "qp=%p. "
1769 "sq_addr=0x%llx, "
1770 "sq_len=%zd, "
1771 "rq_addr=0x%llx, "
1772 "rq_len=%zd"
1773 "\n",
1774 qp,
1775 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1776 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1777 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1778 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1779}
1780
1781static inline void
1782qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1783 struct qedr_qp *qp,
1784 struct qed_rdma_create_qp_out_params *out_params)
1785{
1786 qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1787 qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1788
1789 qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1790 &qp->usq.pbl_info, FW_PAGE_SHIFT);
1791 if (!qp->srq) {
1792 qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1793 qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1794 }
1795
1796 qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1797 &qp->urq.pbl_info, FW_PAGE_SHIFT);
1798}
1799
1800static void qedr_cleanup_user(struct qedr_dev *dev,
1801 struct qedr_ucontext *ctx,
1802 struct qedr_qp *qp)
1803{
1804 if (qedr_qp_has_sq(qp)) {
1805 ib_umem_release(qp->usq.umem);
1806 qp->usq.umem = NULL;
1807 }
1808
1809 if (qedr_qp_has_rq(qp)) {
1810 ib_umem_release(qp->urq.umem);
1811 qp->urq.umem = NULL;
1812 }
1813
1814 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1815 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1816 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1817 } else {
1818 kfree(qp->usq.pbl_tbl);
1819 kfree(qp->urq.pbl_tbl);
1820 }
1821
1822 if (qp->usq.db_rec_data) {
1823 qedr_db_recovery_del(dev, qp->usq.db_addr,
1824 &qp->usq.db_rec_data->db_data);
1825 rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1826 }
1827
1828 if (qp->urq.db_rec_data) {
1829 qedr_db_recovery_del(dev, qp->urq.db_addr,
1830 &qp->urq.db_rec_data->db_data);
1831 rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1832 }
1833
1834 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1835 qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1836 &qp->urq.db_rec_db2_data);
1837}
1838
1839static int qedr_create_user_qp(struct qedr_dev *dev,
1840 struct qedr_qp *qp,
1841 struct ib_pd *ibpd,
1842 struct ib_udata *udata,
1843 struct ib_qp_init_attr *attrs)
1844{
1845 struct qed_rdma_create_qp_in_params in_params;
1846 struct qed_rdma_create_qp_out_params out_params;
1847 struct qedr_create_qp_uresp uresp = {};
1848 struct qedr_create_qp_ureq ureq = {};
1849 int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1850 struct qedr_ucontext *ctx = NULL;
1851 struct qedr_pd *pd = NULL;
1852 int rc = 0;
1853
1854 qp->create_type = QEDR_QP_CREATE_USER;
1855
1856 if (ibpd) {
1857 pd = get_qedr_pd(ibpd);
1858 ctx = pd->uctx;
1859 }
1860
1861 if (udata) {
1862 rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1863 udata->inlen));
1864 if (rc) {
1865 DP_ERR(dev, "Problem copying data from user space\n");
1866 return rc;
1867 }
1868 }
1869
1870 if (qedr_qp_has_sq(qp)) {
1871 /* SQ - read access only (0) */
1872 rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1873 ureq.sq_len, true, 0, alloc_and_init);
1874 if (rc)
1875 return rc;
1876 }
1877
1878 if (qedr_qp_has_rq(qp)) {
1879 /* RQ - read access only (0) */
1880 rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1881 ureq.rq_len, true, 0, alloc_and_init);
1882 if (rc) {
1883 ib_umem_release(qp->usq.umem);
1884 qp->usq.umem = NULL;
1885 if (rdma_protocol_roce(&dev->ibdev, 1)) {
1886 qedr_free_pbl(dev, &qp->usq.pbl_info,
1887 qp->usq.pbl_tbl);
1888 } else {
1889 kfree(qp->usq.pbl_tbl);
1890 }
1891 return rc;
1892 }
1893 }
1894
1895 memset(&in_params, 0, sizeof(in_params));
1896 qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1897 in_params.qp_handle_lo = ureq.qp_handle_lo;
1898 in_params.qp_handle_hi = ureq.qp_handle_hi;
1899
1900 if (qp->qp_type == IB_QPT_XRC_TGT) {
1901 struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1902
1903 in_params.xrcd_id = xrcd->xrcd_id;
1904 in_params.qp_handle_lo = qp->qp_id;
1905 in_params.use_srq = 1;
1906 }
1907
1908 if (qedr_qp_has_sq(qp)) {
1909 in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1910 in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1911 }
1912
1913 if (qedr_qp_has_rq(qp)) {
1914 in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1915 in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1916 }
1917
1918 if (ctx)
1919 SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1920
1921 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1922 &in_params, &out_params);
1923
1924 if (!qp->qed_qp) {
1925 rc = -ENOMEM;
1926 goto err1;
1927 }
1928
1929 if (rdma_protocol_iwarp(&dev->ibdev, 1))
1930 qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1931
1932 qp->qp_id = out_params.qp_id;
1933 qp->icid = out_params.icid;
1934
1935 if (udata) {
1936 rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1937 if (rc)
1938 goto err;
1939 }
1940
1941 /* db offset was calculated in copy_qp_uresp, now set in the user q */
1942 if (qedr_qp_has_sq(qp)) {
1943 qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1944 qp->sq.max_wr = attrs->cap.max_send_wr;
1945 rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1946 &qp->usq.db_rec_data->db_data,
1947 DB_REC_WIDTH_32B,
1948 DB_REC_USER);
1949 if (rc)
1950 goto err;
1951 }
1952
1953 if (qedr_qp_has_rq(qp)) {
1954 qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1955 qp->rq.max_wr = attrs->cap.max_recv_wr;
1956 rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1957 &qp->urq.db_rec_data->db_data,
1958 DB_REC_WIDTH_32B,
1959 DB_REC_USER);
1960 if (rc)
1961 goto err;
1962 }
1963
1964 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1965 qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1966
1967 /* calculate the db_rec_db2 data since it is constant so no
1968 * need to reflect from user
1969 */
1970 qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1971 qp->urq.db_rec_db2_data.data.value =
1972 cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1973
1974 rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1975 &qp->urq.db_rec_db2_data,
1976 DB_REC_WIDTH_32B,
1977 DB_REC_USER);
1978 if (rc)
1979 goto err;
1980 }
1981 qedr_qp_user_print(dev, qp);
1982 return rc;
1983err:
1984 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1985 if (rc)
1986 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1987
1988err1:
1989 qedr_cleanup_user(dev, ctx, qp);
1990 return rc;
1991}
1992
1993static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1994{
1995 int rc;
1996
1997 qp->sq.db = dev->db_addr +
1998 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1999 qp->sq.db_data.data.icid = qp->icid;
2000
2001 rc = qedr_db_recovery_add(dev, qp->sq.db,
2002 &qp->sq.db_data,
2003 DB_REC_WIDTH_32B,
2004 DB_REC_KERNEL);
2005 if (rc)
2006 return rc;
2007
2008 qp->rq.db = dev->db_addr +
2009 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2010 qp->rq.db_data.data.icid = qp->icid;
2011 qp->rq.iwarp_db2 = dev->db_addr +
2012 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2013 qp->rq.iwarp_db2_data.data.icid = qp->icid;
2014 qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2015
2016 rc = qedr_db_recovery_add(dev, qp->rq.db,
2017 &qp->rq.db_data,
2018 DB_REC_WIDTH_32B,
2019 DB_REC_KERNEL);
2020 if (rc)
2021 return rc;
2022
2023 rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2024 &qp->rq.iwarp_db2_data,
2025 DB_REC_WIDTH_32B,
2026 DB_REC_KERNEL);
2027 return rc;
2028}
2029
2030static int
2031qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2032 struct qedr_qp *qp,
2033 struct qed_rdma_create_qp_in_params *in_params,
2034 u32 n_sq_elems, u32 n_rq_elems)
2035{
2036 struct qed_rdma_create_qp_out_params out_params;
2037 struct qed_chain_init_params params = {
2038 .mode = QED_CHAIN_MODE_PBL,
2039 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
2040 };
2041 int rc;
2042
2043 params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2044 params.num_elems = n_sq_elems;
2045 params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2046
2047 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms);
2048 if (rc)
2049 return rc;
2050
2051 in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2052 in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2053
2054 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2055 params.num_elems = n_rq_elems;
2056 params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2057
2058 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms);
2059 if (rc)
2060 return rc;
2061
2062 in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2063 in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2064
2065 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2066 in_params, &out_params);
2067
2068 if (!qp->qed_qp)
2069 return -EINVAL;
2070
2071 qp->qp_id = out_params.qp_id;
2072 qp->icid = out_params.icid;
2073
2074 return qedr_set_roce_db_info(dev, qp);
2075}
2076
2077static int
2078qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2079 struct qedr_qp *qp,
2080 struct qed_rdma_create_qp_in_params *in_params,
2081 u32 n_sq_elems, u32 n_rq_elems)
2082{
2083 struct qed_rdma_create_qp_out_params out_params;
2084 struct qed_chain_init_params params = {
2085 .mode = QED_CHAIN_MODE_PBL,
2086 .cnt_type = QED_CHAIN_CNT_TYPE_U32,
2087 };
2088 int rc;
2089
2090 in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2091 QEDR_SQE_ELEMENT_SIZE,
2092 QED_CHAIN_PAGE_SIZE,
2093 QED_CHAIN_MODE_PBL);
2094 in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2095 QEDR_RQE_ELEMENT_SIZE,
2096 QED_CHAIN_PAGE_SIZE,
2097 QED_CHAIN_MODE_PBL);
2098
2099 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2100 in_params, &out_params);
2101
2102 if (!qp->qed_qp)
2103 return -EINVAL;
2104
2105 /* Now we allocate the chain */
2106
2107 params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2108 params.num_elems = n_sq_elems;
2109 params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2110 params.ext_pbl_virt = out_params.sq_pbl_virt;
2111 params.ext_pbl_phys = out_params.sq_pbl_phys;
2112
2113 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, ¶ms);
2114 if (rc)
2115 goto err;
2116
2117 params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2118 params.num_elems = n_rq_elems;
2119 params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2120 params.ext_pbl_virt = out_params.rq_pbl_virt;
2121 params.ext_pbl_phys = out_params.rq_pbl_phys;
2122
2123 rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, ¶ms);
2124 if (rc)
2125 goto err;
2126
2127 qp->qp_id = out_params.qp_id;
2128 qp->icid = out_params.icid;
2129
2130 return qedr_set_iwarp_db_info(dev, qp);
2131
2132err:
2133 dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2134
2135 return rc;
2136}
2137
2138static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2139{
2140 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2141 kfree(qp->wqe_wr_id);
2142
2143 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2144 kfree(qp->rqe_wr_id);
2145
2146 /* GSI qp is not registered to db mechanism so no need to delete */
2147 if (qp->qp_type == IB_QPT_GSI)
2148 return;
2149
2150 qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2151
2152 if (!qp->srq) {
2153 qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2154
2155 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2156 qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2157 &qp->rq.iwarp_db2_data);
2158 }
2159}
2160
2161static int qedr_create_kernel_qp(struct qedr_dev *dev,
2162 struct qedr_qp *qp,
2163 struct ib_pd *ibpd,
2164 struct ib_qp_init_attr *attrs)
2165{
2166 struct qed_rdma_create_qp_in_params in_params;
2167 struct qedr_pd *pd = get_qedr_pd(ibpd);
2168 int rc = -EINVAL;
2169 u32 n_rq_elems;
2170 u32 n_sq_elems;
2171 u32 n_sq_entries;
2172
2173 memset(&in_params, 0, sizeof(in_params));
2174 qp->create_type = QEDR_QP_CREATE_KERNEL;
2175
2176 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2177 * the ring. The ring should allow at least a single WR, even if the
2178 * user requested none, due to allocation issues.
2179 * We should add an extra WR since the prod and cons indices of
2180 * wqe_wr_id are managed in such a way that the WQ is considered full
2181 * when (prod+1)%max_wr==cons. We currently don't do that because we
2182 * double the number of entries due an iSER issue that pushes far more
2183 * WRs than indicated. If we decline its ib_post_send() then we get
2184 * error prints in the dmesg we'd like to avoid.
2185 */
2186 qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2187 dev->attr.max_sqe);
2188
2189 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2190 GFP_KERNEL);
2191 if (!qp->wqe_wr_id) {
2192 DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2193 return -ENOMEM;
2194 }
2195
2196 /* QP handle to be written in CQE */
2197 in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2198 in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2199
2200 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2201 * the ring. There ring should allow at least a single WR, even if the
2202 * user requested none, due to allocation issues.
2203 */
2204 qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2205
2206 /* Allocate driver internal RQ array */
2207 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2208 GFP_KERNEL);
2209 if (!qp->rqe_wr_id) {
2210 DP_ERR(dev,
2211 "create qp: failed RQ shadow memory allocation\n");
2212 kfree(qp->wqe_wr_id);
2213 return -ENOMEM;
2214 }
2215
2216 qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2217
2218 n_sq_entries = attrs->cap.max_send_wr;
2219 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2220 n_sq_entries = max_t(u32, n_sq_entries, 1);
2221 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2222
2223 n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2224
2225 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2226 rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2227 n_sq_elems, n_rq_elems);
2228 else
2229 rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2230 n_sq_elems, n_rq_elems);
2231 if (rc)
2232 qedr_cleanup_kernel(dev, qp);
2233
2234 return rc;
2235}
2236
2237static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2238 struct ib_udata *udata)
2239{
2240 struct qedr_ucontext *ctx =
2241 rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2242 ibucontext);
2243 int rc;
2244
2245 if (qp->qp_type != IB_QPT_GSI) {
2246 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2247 if (rc)
2248 return rc;
2249 }
2250
2251 if (qp->create_type == QEDR_QP_CREATE_USER)
2252 qedr_cleanup_user(dev, ctx, qp);
2253 else
2254 qedr_cleanup_kernel(dev, qp);
2255
2256 return 0;
2257}
2258
2259int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2260 struct ib_udata *udata)
2261{
2262 struct qedr_xrcd *xrcd = NULL;
2263 struct ib_pd *ibpd = ibqp->pd;
2264 struct qedr_pd *pd = get_qedr_pd(ibpd);
2265 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2266 struct qedr_qp *qp = get_qedr_qp(ibqp);
2267 int rc = 0;
2268
2269 if (attrs->create_flags)
2270 return -EOPNOTSUPP;
2271
2272 if (attrs->qp_type == IB_QPT_XRC_TGT)
2273 xrcd = get_qedr_xrcd(attrs->xrcd);
2274 else
2275 pd = get_qedr_pd(ibpd);
2276
2277 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2278 udata ? "user library" : "kernel", pd);
2279
2280 rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2281 if (rc)
2282 return rc;
2283
2284 DP_DEBUG(dev, QEDR_MSG_QP,
2285 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2286 udata ? "user library" : "kernel", attrs->event_handler, pd,
2287 get_qedr_cq(attrs->send_cq),
2288 get_qedr_cq(attrs->send_cq)->icid,
2289 get_qedr_cq(attrs->recv_cq),
2290 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2291
2292 qedr_set_common_qp_params(dev, qp, pd, attrs);
2293
2294 if (attrs->qp_type == IB_QPT_GSI)
2295 return qedr_create_gsi_qp(dev, attrs, qp);
2296
2297 if (udata || xrcd)
2298 rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2299 else
2300 rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2301
2302 if (rc)
2303 return rc;
2304
2305 qp->ibqp.qp_num = qp->qp_id;
2306
2307 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2308 rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2309 if (rc)
2310 goto out_free_qp_resources;
2311 }
2312
2313 return 0;
2314
2315out_free_qp_resources:
2316 qedr_free_qp_resources(dev, qp, udata);
2317 return -EFAULT;
2318}
2319
2320static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2321{
2322 switch (qp_state) {
2323 case QED_ROCE_QP_STATE_RESET:
2324 return IB_QPS_RESET;
2325 case QED_ROCE_QP_STATE_INIT:
2326 return IB_QPS_INIT;
2327 case QED_ROCE_QP_STATE_RTR:
2328 return IB_QPS_RTR;
2329 case QED_ROCE_QP_STATE_RTS:
2330 return IB_QPS_RTS;
2331 case QED_ROCE_QP_STATE_SQD:
2332 return IB_QPS_SQD;
2333 case QED_ROCE_QP_STATE_ERR:
2334 return IB_QPS_ERR;
2335 case QED_ROCE_QP_STATE_SQE:
2336 return IB_QPS_SQE;
2337 }
2338 return IB_QPS_ERR;
2339}
2340
2341static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2342 enum ib_qp_state qp_state)
2343{
2344 switch (qp_state) {
2345 case IB_QPS_RESET:
2346 return QED_ROCE_QP_STATE_RESET;
2347 case IB_QPS_INIT:
2348 return QED_ROCE_QP_STATE_INIT;
2349 case IB_QPS_RTR:
2350 return QED_ROCE_QP_STATE_RTR;
2351 case IB_QPS_RTS:
2352 return QED_ROCE_QP_STATE_RTS;
2353 case IB_QPS_SQD:
2354 return QED_ROCE_QP_STATE_SQD;
2355 case IB_QPS_ERR:
2356 return QED_ROCE_QP_STATE_ERR;
2357 default:
2358 return QED_ROCE_QP_STATE_ERR;
2359 }
2360}
2361
2362static int qedr_update_qp_state(struct qedr_dev *dev,
2363 struct qedr_qp *qp,
2364 enum qed_roce_qp_state cur_state,
2365 enum qed_roce_qp_state new_state)
2366{
2367 int status = 0;
2368
2369 if (new_state == cur_state)
2370 return 0;
2371
2372 switch (cur_state) {
2373 case QED_ROCE_QP_STATE_RESET:
2374 switch (new_state) {
2375 case QED_ROCE_QP_STATE_INIT:
2376 break;
2377 default:
2378 status = -EINVAL;
2379 break;
2380 }
2381 break;
2382 case QED_ROCE_QP_STATE_INIT:
2383 switch (new_state) {
2384 case QED_ROCE_QP_STATE_RTR:
2385 /* Update doorbell (in case post_recv was
2386 * done before move to RTR)
2387 */
2388
2389 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2390 writel(qp->rq.db_data.raw, qp->rq.db);
2391 }
2392 break;
2393 case QED_ROCE_QP_STATE_ERR:
2394 break;
2395 default:
2396 /* Invalid state change. */
2397 status = -EINVAL;
2398 break;
2399 }
2400 break;
2401 case QED_ROCE_QP_STATE_RTR:
2402 /* RTR->XXX */
2403 switch (new_state) {
2404 case QED_ROCE_QP_STATE_RTS:
2405 break;
2406 case QED_ROCE_QP_STATE_ERR:
2407 break;
2408 default:
2409 /* Invalid state change. */
2410 status = -EINVAL;
2411 break;
2412 }
2413 break;
2414 case QED_ROCE_QP_STATE_RTS:
2415 /* RTS->XXX */
2416 switch (new_state) {
2417 case QED_ROCE_QP_STATE_SQD:
2418 break;
2419 case QED_ROCE_QP_STATE_ERR:
2420 break;
2421 default:
2422 /* Invalid state change. */
2423 status = -EINVAL;
2424 break;
2425 }
2426 break;
2427 case QED_ROCE_QP_STATE_SQD:
2428 /* SQD->XXX */
2429 switch (new_state) {
2430 case QED_ROCE_QP_STATE_RTS:
2431 case QED_ROCE_QP_STATE_ERR:
2432 break;
2433 default:
2434 /* Invalid state change. */
2435 status = -EINVAL;
2436 break;
2437 }
2438 break;
2439 case QED_ROCE_QP_STATE_ERR:
2440 /* ERR->XXX */
2441 switch (new_state) {
2442 case QED_ROCE_QP_STATE_RESET:
2443 if ((qp->rq.prod != qp->rq.cons) ||
2444 (qp->sq.prod != qp->sq.cons)) {
2445 DP_NOTICE(dev,
2446 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2447 qp->rq.prod, qp->rq.cons, qp->sq.prod,
2448 qp->sq.cons);
2449 status = -EINVAL;
2450 }
2451 break;
2452 default:
2453 status = -EINVAL;
2454 break;
2455 }
2456 break;
2457 default:
2458 status = -EINVAL;
2459 break;
2460 }
2461
2462 return status;
2463}
2464
2465int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2466 int attr_mask, struct ib_udata *udata)
2467{
2468 struct qedr_qp *qp = get_qedr_qp(ibqp);
2469 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2470 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2471 const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2472 enum ib_qp_state old_qp_state, new_qp_state;
2473 enum qed_roce_qp_state cur_state;
2474 int rc = 0;
2475
2476 DP_DEBUG(dev, QEDR_MSG_QP,
2477 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2478 attr->qp_state);
2479
2480 if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2481 return -EOPNOTSUPP;
2482
2483 old_qp_state = qedr_get_ibqp_state(qp->state);
2484 if (attr_mask & IB_QP_STATE)
2485 new_qp_state = attr->qp_state;
2486 else
2487 new_qp_state = old_qp_state;
2488
2489 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2490 if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2491 ibqp->qp_type, attr_mask)) {
2492 DP_ERR(dev,
2493 "modify qp: invalid attribute mask=0x%x specified for\n"
2494 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2495 attr_mask, qp->qp_id, ibqp->qp_type,
2496 old_qp_state, new_qp_state);
2497 rc = -EINVAL;
2498 goto err;
2499 }
2500 }
2501
2502 /* Translate the masks... */
2503 if (attr_mask & IB_QP_STATE) {
2504 SET_FIELD(qp_params.modify_flags,
2505 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2506 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2507 }
2508
2509 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2510 qp_params.sqd_async = true;
2511
2512 if (attr_mask & IB_QP_PKEY_INDEX) {
2513 SET_FIELD(qp_params.modify_flags,
2514 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2515 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2516 rc = -EINVAL;
2517 goto err;
2518 }
2519
2520 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2521 }
2522
2523 if (attr_mask & IB_QP_QKEY)
2524 qp->qkey = attr->qkey;
2525
2526 if (attr_mask & IB_QP_ACCESS_FLAGS) {
2527 SET_FIELD(qp_params.modify_flags,
2528 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2529 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2530 IB_ACCESS_REMOTE_READ;
2531 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2532 IB_ACCESS_REMOTE_WRITE;
2533 qp_params.incoming_atomic_en = attr->qp_access_flags &
2534 IB_ACCESS_REMOTE_ATOMIC;
2535 }
2536
2537 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2538 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2539 return -EINVAL;
2540
2541 if (attr_mask & IB_QP_PATH_MTU) {
2542 if (attr->path_mtu < IB_MTU_256 ||
2543 attr->path_mtu > IB_MTU_4096) {
2544 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2545 rc = -EINVAL;
2546 goto err;
2547 }
2548 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2549 ib_mtu_enum_to_int(iboe_get_mtu
2550 (dev->ndev->mtu)));
2551 }
2552
2553 if (!qp->mtu) {
2554 qp->mtu =
2555 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2556 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2557 }
2558
2559 SET_FIELD(qp_params.modify_flags,
2560 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2561
2562 qp_params.traffic_class_tos = grh->traffic_class;
2563 qp_params.flow_label = grh->flow_label;
2564 qp_params.hop_limit_ttl = grh->hop_limit;
2565
2566 qp->sgid_idx = grh->sgid_index;
2567
2568 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2569 if (rc) {
2570 DP_ERR(dev,
2571 "modify qp: problems with GID index %d (rc=%d)\n",
2572 grh->sgid_index, rc);
2573 return rc;
2574 }
2575
2576 rc = qedr_get_dmac(dev, &attr->ah_attr,
2577 qp_params.remote_mac_addr);
2578 if (rc)
2579 return rc;
2580
2581 qp_params.use_local_mac = true;
2582 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2583
2584 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2585 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2586 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2587 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2588 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2589 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2590 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2591 qp_params.remote_mac_addr);
2592
2593 qp_params.mtu = qp->mtu;
2594 qp_params.lb_indication = false;
2595 }
2596
2597 if (!qp_params.mtu) {
2598 /* Stay with current MTU */
2599 if (qp->mtu)
2600 qp_params.mtu = qp->mtu;
2601 else
2602 qp_params.mtu =
2603 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2604 }
2605
2606 if (attr_mask & IB_QP_TIMEOUT) {
2607 SET_FIELD(qp_params.modify_flags,
2608 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2609
2610 /* The received timeout value is an exponent used like this:
2611 * "12.7.34 LOCAL ACK TIMEOUT
2612 * Value representing the transport (ACK) timeout for use by
2613 * the remote, expressed as: 4.096 * 2^timeout [usec]"
2614 * The FW expects timeout in msec so we need to divide the usec
2615 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2616 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2617 * The value of zero means infinite so we use a 'max_t' to make
2618 * sure that sub 1 msec values will be configured as 1 msec.
2619 */
2620 if (attr->timeout)
2621 qp_params.ack_timeout =
2622 1 << max_t(int, attr->timeout - 8, 0);
2623 else
2624 qp_params.ack_timeout = 0;
2625
2626 qp->timeout = attr->timeout;
2627 }
2628
2629 if (attr_mask & IB_QP_RETRY_CNT) {
2630 SET_FIELD(qp_params.modify_flags,
2631 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2632 qp_params.retry_cnt = attr->retry_cnt;
2633 }
2634
2635 if (attr_mask & IB_QP_RNR_RETRY) {
2636 SET_FIELD(qp_params.modify_flags,
2637 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2638 qp_params.rnr_retry_cnt = attr->rnr_retry;
2639 }
2640
2641 if (attr_mask & IB_QP_RQ_PSN) {
2642 SET_FIELD(qp_params.modify_flags,
2643 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2644 qp_params.rq_psn = attr->rq_psn;
2645 qp->rq_psn = attr->rq_psn;
2646 }
2647
2648 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2649 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2650 rc = -EINVAL;
2651 DP_ERR(dev,
2652 "unsupported max_rd_atomic=%d, supported=%d\n",
2653 attr->max_rd_atomic,
2654 dev->attr.max_qp_req_rd_atomic_resc);
2655 goto err;
2656 }
2657
2658 SET_FIELD(qp_params.modify_flags,
2659 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2660 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2661 }
2662
2663 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2664 SET_FIELD(qp_params.modify_flags,
2665 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2666 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2667 }
2668
2669 if (attr_mask & IB_QP_SQ_PSN) {
2670 SET_FIELD(qp_params.modify_flags,
2671 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2672 qp_params.sq_psn = attr->sq_psn;
2673 qp->sq_psn = attr->sq_psn;
2674 }
2675
2676 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2677 if (attr->max_dest_rd_atomic >
2678 dev->attr.max_qp_resp_rd_atomic_resc) {
2679 DP_ERR(dev,
2680 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2681 attr->max_dest_rd_atomic,
2682 dev->attr.max_qp_resp_rd_atomic_resc);
2683
2684 rc = -EINVAL;
2685 goto err;
2686 }
2687
2688 SET_FIELD(qp_params.modify_flags,
2689 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2690 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2691 }
2692
2693 if (attr_mask & IB_QP_DEST_QPN) {
2694 SET_FIELD(qp_params.modify_flags,
2695 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2696
2697 qp_params.dest_qp = attr->dest_qp_num;
2698 qp->dest_qp_num = attr->dest_qp_num;
2699 }
2700
2701 cur_state = qp->state;
2702
2703 /* Update the QP state before the actual ramrod to prevent a race with
2704 * fast path. Modifying the QP state to error will cause the device to
2705 * flush the CQEs and while polling the flushed CQEs will considered as
2706 * a potential issue if the QP isn't in error state.
2707 */
2708 if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2709 !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2710 qp->state = QED_ROCE_QP_STATE_ERR;
2711
2712 if (qp->qp_type != IB_QPT_GSI)
2713 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2714 qp->qed_qp, &qp_params);
2715
2716 if (attr_mask & IB_QP_STATE) {
2717 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2718 rc = qedr_update_qp_state(dev, qp, cur_state,
2719 qp_params.new_state);
2720 qp->state = qp_params.new_state;
2721 }
2722
2723err:
2724 return rc;
2725}
2726
2727static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2728{
2729 int ib_qp_acc_flags = 0;
2730
2731 if (params->incoming_rdma_write_en)
2732 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2733 if (params->incoming_rdma_read_en)
2734 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2735 if (params->incoming_atomic_en)
2736 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2737 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2738 return ib_qp_acc_flags;
2739}
2740
2741int qedr_query_qp(struct ib_qp *ibqp,
2742 struct ib_qp_attr *qp_attr,
2743 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2744{
2745 struct qed_rdma_query_qp_out_params params;
2746 struct qedr_qp *qp = get_qedr_qp(ibqp);
2747 struct qedr_dev *dev = qp->dev;
2748 int rc = 0;
2749
2750 memset(¶ms, 0, sizeof(params));
2751 memset(qp_attr, 0, sizeof(*qp_attr));
2752 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2753
2754 if (qp->qp_type != IB_QPT_GSI) {
2755 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2756 if (rc)
2757 goto err;
2758 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2759 } else {
2760 qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2761 }
2762
2763 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2764 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2765 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2766 qp_attr->rq_psn = params.rq_psn;
2767 qp_attr->sq_psn = params.sq_psn;
2768 qp_attr->dest_qp_num = params.dest_qp;
2769
2770 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2771
2772 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2773 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2774 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2775 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2776 qp_attr->cap.max_inline_data = dev->attr.max_inline;
2777 qp_init_attr->cap = qp_attr->cap;
2778
2779 qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2780 rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2781 params.flow_label, qp->sgid_idx,
2782 params.hop_limit_ttl, params.traffic_class_tos);
2783 rdma_ah_set_dgid_raw(&qp_attr->ah_attr, ¶ms.dgid.bytes[0]);
2784 rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2785 rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2786 qp_attr->timeout = qp->timeout;
2787 qp_attr->rnr_retry = params.rnr_retry;
2788 qp_attr->retry_cnt = params.retry_cnt;
2789 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2790 qp_attr->pkey_index = params.pkey_index;
2791 qp_attr->port_num = 1;
2792 rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2793 rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2794 qp_attr->alt_pkey_index = 0;
2795 qp_attr->alt_port_num = 0;
2796 qp_attr->alt_timeout = 0;
2797 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2798
2799 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2800 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2801 qp_attr->max_rd_atomic = params.max_rd_atomic;
2802 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2803
2804 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2805 qp_attr->cap.max_inline_data);
2806
2807err:
2808 return rc;
2809}
2810
2811int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2812{
2813 struct qedr_qp *qp = get_qedr_qp(ibqp);
2814 struct qedr_dev *dev = qp->dev;
2815 struct ib_qp_attr attr;
2816 int attr_mask = 0;
2817
2818 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2819 qp, qp->qp_type);
2820
2821 if (rdma_protocol_roce(&dev->ibdev, 1)) {
2822 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2823 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2824 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2825
2826 attr.qp_state = IB_QPS_ERR;
2827 attr_mask |= IB_QP_STATE;
2828
2829 /* Change the QP state to ERROR */
2830 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2831 }
2832 } else {
2833 /* If connection establishment started the WAIT_FOR_CONNECT
2834 * bit will be on and we need to Wait for the establishment
2835 * to complete before destroying the qp.
2836 */
2837 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2838 &qp->iwarp_cm_flags))
2839 wait_for_completion(&qp->iwarp_cm_comp);
2840
2841 /* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2842 * bit will be on, and we need to wait for the disconnect to
2843 * complete before continuing. We can use the same completion,
2844 * iwarp_cm_comp, since this is the only place that waits for
2845 * this completion and it is sequential. In addition,
2846 * disconnect can't occur before the connection is fully
2847 * established, therefore if WAIT_FOR_DISCONNECT is on it
2848 * means WAIT_FOR_CONNECT is also on and the completion for
2849 * CONNECT already occurred.
2850 */
2851 if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2852 &qp->iwarp_cm_flags))
2853 wait_for_completion(&qp->iwarp_cm_comp);
2854 }
2855
2856 if (qp->qp_type == IB_QPT_GSI)
2857 qedr_destroy_gsi_qp(dev);
2858
2859 /* We need to remove the entry from the xarray before we release the
2860 * qp_id to avoid a race of the qp_id being reallocated and failing
2861 * on xa_insert
2862 */
2863 if (rdma_protocol_iwarp(&dev->ibdev, 1))
2864 xa_erase(&dev->qps, qp->qp_id);
2865
2866 qedr_free_qp_resources(dev, qp, udata);
2867
2868 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2869 qedr_iw_qp_rem_ref(&qp->ibqp);
2870 wait_for_completion(&qp->qp_rel_comp);
2871 }
2872
2873 return 0;
2874}
2875
2876int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2877 struct ib_udata *udata)
2878{
2879 struct qedr_ah *ah = get_qedr_ah(ibah);
2880
2881 rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2882
2883 return 0;
2884}
2885
2886int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2887{
2888 struct qedr_ah *ah = get_qedr_ah(ibah);
2889
2890 rdma_destroy_ah_attr(&ah->attr);
2891 return 0;
2892}
2893
2894static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2895{
2896 struct qedr_pbl *pbl, *tmp;
2897
2898 if (info->pbl_table)
2899 list_add_tail(&info->pbl_table->list_entry,
2900 &info->free_pbl_list);
2901
2902 if (!list_empty(&info->inuse_pbl_list))
2903 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2904
2905 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2906 list_del(&pbl->list_entry);
2907 qedr_free_pbl(dev, &info->pbl_info, pbl);
2908 }
2909}
2910
2911static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2912 size_t page_list_len, bool two_layered)
2913{
2914 struct qedr_pbl *tmp;
2915 int rc;
2916
2917 INIT_LIST_HEAD(&info->free_pbl_list);
2918 INIT_LIST_HEAD(&info->inuse_pbl_list);
2919
2920 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2921 page_list_len, two_layered);
2922 if (rc)
2923 goto done;
2924
2925 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2926 if (IS_ERR(info->pbl_table)) {
2927 rc = PTR_ERR(info->pbl_table);
2928 goto done;
2929 }
2930
2931 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2932 &info->pbl_table->pa);
2933
2934 /* in usual case we use 2 PBLs, so we add one to free
2935 * list and allocating another one
2936 */
2937 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2938 if (IS_ERR(tmp)) {
2939 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2940 goto done;
2941 }
2942
2943 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2944
2945 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2946
2947done:
2948 if (rc)
2949 free_mr_info(dev, info);
2950
2951 return rc;
2952}
2953
2954struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2955 u64 usr_addr, int acc, struct ib_udata *udata)
2956{
2957 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2958 struct qedr_mr *mr;
2959 struct qedr_pd *pd;
2960 int rc = -ENOMEM;
2961
2962 pd = get_qedr_pd(ibpd);
2963 DP_DEBUG(dev, QEDR_MSG_MR,
2964 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2965 pd->pd_id, start, len, usr_addr, acc);
2966
2967 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2968 return ERR_PTR(-EINVAL);
2969
2970 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2971 if (!mr)
2972 return ERR_PTR(rc);
2973
2974 mr->type = QEDR_MR_USER;
2975
2976 mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2977 if (IS_ERR(mr->umem)) {
2978 rc = -EFAULT;
2979 goto err0;
2980 }
2981
2982 rc = init_mr_info(dev, &mr->info,
2983 ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2984 if (rc)
2985 goto err1;
2986
2987 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2988 &mr->info.pbl_info, PAGE_SHIFT);
2989
2990 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2991 if (rc) {
2992 if (rc == -EINVAL)
2993 DP_ERR(dev, "Out of MR resources\n");
2994 else
2995 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
2996
2997 goto err1;
2998 }
2999
3000 /* Index only, 18 bit long, lkey = itid << 8 | key */
3001 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3002 mr->hw_mr.key = 0;
3003 mr->hw_mr.pd = pd->pd_id;
3004 mr->hw_mr.local_read = 1;
3005 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3006 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3007 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3008 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3009 mr->hw_mr.mw_bind = false;
3010 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3011 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3012 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3013 mr->hw_mr.page_size_log = PAGE_SHIFT;
3014 mr->hw_mr.length = len;
3015 mr->hw_mr.vaddr = usr_addr;
3016 mr->hw_mr.phy_mr = false;
3017 mr->hw_mr.dma_mr = false;
3018
3019 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3020 if (rc) {
3021 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3022 goto err2;
3023 }
3024
3025 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3026 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3027 mr->hw_mr.remote_atomic)
3028 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3029
3030 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3031 mr->ibmr.lkey);
3032 return &mr->ibmr;
3033
3034err2:
3035 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3036err1:
3037 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3038err0:
3039 kfree(mr);
3040 return ERR_PTR(rc);
3041}
3042
3043int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3044{
3045 struct qedr_mr *mr = get_qedr_mr(ib_mr);
3046 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3047 int rc = 0;
3048
3049 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3050 if (rc)
3051 return rc;
3052
3053 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3054
3055 if (mr->type != QEDR_MR_DMA)
3056 free_mr_info(dev, &mr->info);
3057
3058 /* it could be user registered memory. */
3059 ib_umem_release(mr->umem);
3060
3061 kfree(mr);
3062
3063 return rc;
3064}
3065
3066static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3067 int max_page_list_len)
3068{
3069 struct qedr_pd *pd = get_qedr_pd(ibpd);
3070 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3071 struct qedr_mr *mr;
3072 int rc = -ENOMEM;
3073
3074 DP_DEBUG(dev, QEDR_MSG_MR,
3075 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3076 max_page_list_len);
3077
3078 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3079 if (!mr)
3080 return ERR_PTR(rc);
3081
3082 mr->dev = dev;
3083 mr->type = QEDR_MR_FRMR;
3084
3085 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3086 if (rc)
3087 goto err0;
3088
3089 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3090 if (rc) {
3091 if (rc == -EINVAL)
3092 DP_ERR(dev, "Out of MR resources\n");
3093 else
3094 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3095
3096 goto err1;
3097 }
3098
3099 /* Index only, 18 bit long, lkey = itid << 8 | key */
3100 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3101 mr->hw_mr.key = 0;
3102 mr->hw_mr.pd = pd->pd_id;
3103 mr->hw_mr.local_read = 1;
3104 mr->hw_mr.local_write = 0;
3105 mr->hw_mr.remote_read = 0;
3106 mr->hw_mr.remote_write = 0;
3107 mr->hw_mr.remote_atomic = 0;
3108 mr->hw_mr.mw_bind = false;
3109 mr->hw_mr.pbl_ptr = 0;
3110 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3111 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3112 mr->hw_mr.length = 0;
3113 mr->hw_mr.vaddr = 0;
3114 mr->hw_mr.phy_mr = true;
3115 mr->hw_mr.dma_mr = false;
3116
3117 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3118 if (rc) {
3119 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3120 goto err2;
3121 }
3122
3123 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3124 mr->ibmr.rkey = mr->ibmr.lkey;
3125
3126 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3127 return mr;
3128
3129err2:
3130 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3131err1:
3132 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3133err0:
3134 kfree(mr);
3135 return ERR_PTR(rc);
3136}
3137
3138struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3139 u32 max_num_sg)
3140{
3141 struct qedr_mr *mr;
3142
3143 if (mr_type != IB_MR_TYPE_MEM_REG)
3144 return ERR_PTR(-EINVAL);
3145
3146 mr = __qedr_alloc_mr(ibpd, max_num_sg);
3147
3148 if (IS_ERR(mr))
3149 return ERR_PTR(-EINVAL);
3150
3151 return &mr->ibmr;
3152}
3153
3154static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3155{
3156 struct qedr_mr *mr = get_qedr_mr(ibmr);
3157 struct qedr_pbl *pbl_table;
3158 struct regpair *pbe;
3159 u32 pbes_in_page;
3160
3161 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3162 DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3163 return -ENOMEM;
3164 }
3165
3166 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3167 mr->npages, addr);
3168
3169 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3170 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3171 pbe = (struct regpair *)pbl_table->va;
3172 pbe += mr->npages % pbes_in_page;
3173 pbe->lo = cpu_to_le32((u32)addr);
3174 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3175
3176 mr->npages++;
3177
3178 return 0;
3179}
3180
3181static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3182{
3183 int work = info->completed - info->completed_handled - 1;
3184
3185 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3186 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3187 struct qedr_pbl *pbl;
3188
3189 /* Free all the page list that are possible to be freed
3190 * (all the ones that were invalidated), under the assumption
3191 * that if an FMR was completed successfully that means that
3192 * if there was an invalidate operation before it also ended
3193 */
3194 pbl = list_first_entry(&info->inuse_pbl_list,
3195 struct qedr_pbl, list_entry);
3196 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3197 info->completed_handled++;
3198 }
3199}
3200
3201int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3202 int sg_nents, unsigned int *sg_offset)
3203{
3204 struct qedr_mr *mr = get_qedr_mr(ibmr);
3205
3206 mr->npages = 0;
3207
3208 handle_completed_mrs(mr->dev, &mr->info);
3209 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3210}
3211
3212struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3213{
3214 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3215 struct qedr_pd *pd = get_qedr_pd(ibpd);
3216 struct qedr_mr *mr;
3217 int rc;
3218
3219 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3220 if (!mr)
3221 return ERR_PTR(-ENOMEM);
3222
3223 mr->type = QEDR_MR_DMA;
3224
3225 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3226 if (rc) {
3227 if (rc == -EINVAL)
3228 DP_ERR(dev, "Out of MR resources\n");
3229 else
3230 DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3231
3232 goto err1;
3233 }
3234
3235 /* index only, 18 bit long, lkey = itid << 8 | key */
3236 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3237 mr->hw_mr.pd = pd->pd_id;
3238 mr->hw_mr.local_read = 1;
3239 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3240 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3241 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3242 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3243 mr->hw_mr.dma_mr = true;
3244
3245 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3246 if (rc) {
3247 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3248 goto err2;
3249 }
3250
3251 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3252 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3253 mr->hw_mr.remote_atomic)
3254 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3255
3256 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3257 return &mr->ibmr;
3258
3259err2:
3260 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3261err1:
3262 kfree(mr);
3263 return ERR_PTR(rc);
3264}
3265
3266static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3267{
3268 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3269}
3270
3271static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3272{
3273 int i, len = 0;
3274
3275 for (i = 0; i < num_sge; i++)
3276 len += sg_list[i].length;
3277
3278 return len;
3279}
3280
3281static void swap_wqe_data64(u64 *p)
3282{
3283 int i;
3284
3285 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3286 *p = cpu_to_be64(cpu_to_le64(*p));
3287}
3288
3289static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3290 struct qedr_qp *qp, u8 *wqe_size,
3291 const struct ib_send_wr *wr,
3292 const struct ib_send_wr **bad_wr,
3293 u8 *bits, u8 bit)
3294{
3295 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3296 char *seg_prt, *wqe;
3297 int i, seg_siz;
3298
3299 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3300 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3301 *bad_wr = wr;
3302 return 0;
3303 }
3304
3305 if (!data_size)
3306 return data_size;
3307
3308 *bits |= bit;
3309
3310 seg_prt = NULL;
3311 wqe = NULL;
3312 seg_siz = 0;
3313
3314 /* Copy data inline */
3315 for (i = 0; i < wr->num_sge; i++) {
3316 u32 len = wr->sg_list[i].length;
3317 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3318
3319 while (len > 0) {
3320 u32 cur;
3321
3322 /* New segment required */
3323 if (!seg_siz) {
3324 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3325 seg_prt = wqe;
3326 seg_siz = sizeof(struct rdma_sq_common_wqe);
3327 (*wqe_size)++;
3328 }
3329
3330 /* Calculate currently allowed length */
3331 cur = min_t(u32, len, seg_siz);
3332 memcpy(seg_prt, src, cur);
3333
3334 /* Update segment variables */
3335 seg_prt += cur;
3336 seg_siz -= cur;
3337
3338 /* Update sge variables */
3339 src += cur;
3340 len -= cur;
3341
3342 /* Swap fully-completed segments */
3343 if (!seg_siz)
3344 swap_wqe_data64((u64 *)wqe);
3345 }
3346 }
3347
3348 /* swap last not completed segment */
3349 if (seg_siz)
3350 swap_wqe_data64((u64 *)wqe);
3351
3352 return data_size;
3353}
3354
3355#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
3356 do { \
3357 DMA_REGPAIR_LE(sge->addr, vaddr); \
3358 (sge)->length = cpu_to_le32(vlength); \
3359 (sge)->flags = cpu_to_le32(vflags); \
3360 } while (0)
3361
3362#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
3363 do { \
3364 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
3365 (hdr)->num_sges = num_sge; \
3366 } while (0)
3367
3368#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
3369 do { \
3370 DMA_REGPAIR_LE(sge->addr, vaddr); \
3371 (sge)->length = cpu_to_le32(vlength); \
3372 (sge)->l_key = cpu_to_le32(vlkey); \
3373 } while (0)
3374
3375static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3376 const struct ib_send_wr *wr)
3377{
3378 u32 data_size = 0;
3379 int i;
3380
3381 for (i = 0; i < wr->num_sge; i++) {
3382 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3383
3384 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3385 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3386 sge->length = cpu_to_le32(wr->sg_list[i].length);
3387 data_size += wr->sg_list[i].length;
3388 }
3389
3390 if (wqe_size)
3391 *wqe_size += wr->num_sge;
3392
3393 return data_size;
3394}
3395
3396static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3397 struct qedr_qp *qp,
3398 struct rdma_sq_rdma_wqe_1st *rwqe,
3399 struct rdma_sq_rdma_wqe_2nd *rwqe2,
3400 const struct ib_send_wr *wr,
3401 const struct ib_send_wr **bad_wr)
3402{
3403 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3404 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3405
3406 if (wr->send_flags & IB_SEND_INLINE &&
3407 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3408 wr->opcode == IB_WR_RDMA_WRITE)) {
3409 u8 flags = 0;
3410
3411 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3412 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3413 bad_wr, &rwqe->flags, flags);
3414 }
3415
3416 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3417}
3418
3419static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3420 struct qedr_qp *qp,
3421 struct rdma_sq_send_wqe_1st *swqe,
3422 struct rdma_sq_send_wqe_2st *swqe2,
3423 const struct ib_send_wr *wr,
3424 const struct ib_send_wr **bad_wr)
3425{
3426 memset(swqe2, 0, sizeof(*swqe2));
3427 if (wr->send_flags & IB_SEND_INLINE) {
3428 u8 flags = 0;
3429
3430 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3431 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3432 bad_wr, &swqe->flags, flags);
3433 }
3434
3435 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3436}
3437
3438static int qedr_prepare_reg(struct qedr_qp *qp,
3439 struct rdma_sq_fmr_wqe_1st *fwqe1,
3440 const struct ib_reg_wr *wr)
3441{
3442 struct qedr_mr *mr = get_qedr_mr(wr->mr);
3443 struct rdma_sq_fmr_wqe_2nd *fwqe2;
3444
3445 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3446 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3447 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3448 fwqe1->l_key = wr->key;
3449
3450 fwqe2->access_ctrl = 0;
3451
3452 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3453 !!(wr->access & IB_ACCESS_REMOTE_READ));
3454 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3455 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3456 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3457 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3458 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3459 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3460 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3461 fwqe2->fmr_ctrl = 0;
3462
3463 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3464 ilog2(mr->ibmr.page_size) - 12);
3465
3466 fwqe2->length_hi = 0;
3467 fwqe2->length_lo = mr->ibmr.length;
3468 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3469 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3470
3471 qp->wqe_wr_id[qp->sq.prod].mr = mr;
3472
3473 return 0;
3474}
3475
3476static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3477{
3478 switch (opcode) {
3479 case IB_WR_RDMA_WRITE:
3480 case IB_WR_RDMA_WRITE_WITH_IMM:
3481 return IB_WC_RDMA_WRITE;
3482 case IB_WR_SEND_WITH_IMM:
3483 case IB_WR_SEND:
3484 case IB_WR_SEND_WITH_INV:
3485 return IB_WC_SEND;
3486 case IB_WR_RDMA_READ:
3487 case IB_WR_RDMA_READ_WITH_INV:
3488 return IB_WC_RDMA_READ;
3489 case IB_WR_ATOMIC_CMP_AND_SWP:
3490 return IB_WC_COMP_SWAP;
3491 case IB_WR_ATOMIC_FETCH_AND_ADD:
3492 return IB_WC_FETCH_ADD;
3493 case IB_WR_REG_MR:
3494 return IB_WC_REG_MR;
3495 case IB_WR_LOCAL_INV:
3496 return IB_WC_LOCAL_INV;
3497 default:
3498 return IB_WC_SEND;
3499 }
3500}
3501
3502static inline bool qedr_can_post_send(struct qedr_qp *qp,
3503 const struct ib_send_wr *wr)
3504{
3505 int wq_is_full, err_wr, pbl_is_full;
3506 struct qedr_dev *dev = qp->dev;
3507
3508 /* prevent SQ overflow and/or processing of a bad WR */
3509 err_wr = wr->num_sge > qp->sq.max_sges;
3510 wq_is_full = qedr_wq_is_full(&qp->sq);
3511 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3512 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3513 if (wq_is_full || err_wr || pbl_is_full) {
3514 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3515 DP_ERR(dev,
3516 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3517 qp);
3518 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3519 }
3520
3521 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3522 DP_ERR(dev,
3523 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3524 qp);
3525 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3526 }
3527
3528 if (pbl_is_full &&
3529 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3530 DP_ERR(dev,
3531 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3532 qp);
3533 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3534 }
3535 return false;
3536 }
3537 return true;
3538}
3539
3540static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3541 const struct ib_send_wr **bad_wr)
3542{
3543 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3544 struct qedr_qp *qp = get_qedr_qp(ibqp);
3545 struct rdma_sq_atomic_wqe_1st *awqe1;
3546 struct rdma_sq_atomic_wqe_2nd *awqe2;
3547 struct rdma_sq_atomic_wqe_3rd *awqe3;
3548 struct rdma_sq_send_wqe_2st *swqe2;
3549 struct rdma_sq_local_inv_wqe *iwqe;
3550 struct rdma_sq_rdma_wqe_2nd *rwqe2;
3551 struct rdma_sq_send_wqe_1st *swqe;
3552 struct rdma_sq_rdma_wqe_1st *rwqe;
3553 struct rdma_sq_fmr_wqe_1st *fwqe1;
3554 struct rdma_sq_common_wqe *wqe;
3555 u32 length;
3556 int rc = 0;
3557 bool comp;
3558
3559 if (!qedr_can_post_send(qp, wr)) {
3560 *bad_wr = wr;
3561 return -ENOMEM;
3562 }
3563
3564 wqe = qed_chain_produce(&qp->sq.pbl);
3565 qp->wqe_wr_id[qp->sq.prod].signaled =
3566 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3567
3568 wqe->flags = 0;
3569 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3570 !!(wr->send_flags & IB_SEND_SOLICITED));
3571 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3572 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3573 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3574 !!(wr->send_flags & IB_SEND_FENCE));
3575 wqe->prev_wqe_size = qp->prev_wqe_size;
3576
3577 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3578
3579 switch (wr->opcode) {
3580 case IB_WR_SEND_WITH_IMM:
3581 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3582 rc = -EINVAL;
3583 *bad_wr = wr;
3584 break;
3585 }
3586 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3587 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3588 swqe->wqe_size = 2;
3589 swqe2 = qed_chain_produce(&qp->sq.pbl);
3590
3591 swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3592 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3593 wr, bad_wr);
3594 swqe->length = cpu_to_le32(length);
3595 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3596 qp->prev_wqe_size = swqe->wqe_size;
3597 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3598 break;
3599 case IB_WR_SEND:
3600 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3601 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3602
3603 swqe->wqe_size = 2;
3604 swqe2 = qed_chain_produce(&qp->sq.pbl);
3605 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3606 wr, bad_wr);
3607 swqe->length = cpu_to_le32(length);
3608 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3609 qp->prev_wqe_size = swqe->wqe_size;
3610 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3611 break;
3612 case IB_WR_SEND_WITH_INV:
3613 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3614 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3615 swqe2 = qed_chain_produce(&qp->sq.pbl);
3616 swqe->wqe_size = 2;
3617 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3618 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3619 wr, bad_wr);
3620 swqe->length = cpu_to_le32(length);
3621 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3622 qp->prev_wqe_size = swqe->wqe_size;
3623 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3624 break;
3625
3626 case IB_WR_RDMA_WRITE_WITH_IMM:
3627 if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3628 rc = -EINVAL;
3629 *bad_wr = wr;
3630 break;
3631 }
3632 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3633 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3634
3635 rwqe->wqe_size = 2;
3636 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3637 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3638 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3639 wr, bad_wr);
3640 rwqe->length = cpu_to_le32(length);
3641 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3642 qp->prev_wqe_size = rwqe->wqe_size;
3643 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3644 break;
3645 case IB_WR_RDMA_WRITE:
3646 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3647 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3648
3649 rwqe->wqe_size = 2;
3650 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3651 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3652 wr, bad_wr);
3653 rwqe->length = cpu_to_le32(length);
3654 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3655 qp->prev_wqe_size = rwqe->wqe_size;
3656 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3657 break;
3658 case IB_WR_RDMA_READ_WITH_INV:
3659 SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3660 fallthrough; /* same is identical to RDMA READ */
3661
3662 case IB_WR_RDMA_READ:
3663 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3664 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3665
3666 rwqe->wqe_size = 2;
3667 rwqe2 = qed_chain_produce(&qp->sq.pbl);
3668 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3669 wr, bad_wr);
3670 rwqe->length = cpu_to_le32(length);
3671 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3672 qp->prev_wqe_size = rwqe->wqe_size;
3673 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3674 break;
3675
3676 case IB_WR_ATOMIC_CMP_AND_SWP:
3677 case IB_WR_ATOMIC_FETCH_AND_ADD:
3678 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3679 awqe1->wqe_size = 4;
3680
3681 awqe2 = qed_chain_produce(&qp->sq.pbl);
3682 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3683 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3684
3685 awqe3 = qed_chain_produce(&qp->sq.pbl);
3686
3687 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3688 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3689 DMA_REGPAIR_LE(awqe3->swap_data,
3690 atomic_wr(wr)->compare_add);
3691 } else {
3692 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3693 DMA_REGPAIR_LE(awqe3->swap_data,
3694 atomic_wr(wr)->swap);
3695 DMA_REGPAIR_LE(awqe3->cmp_data,
3696 atomic_wr(wr)->compare_add);
3697 }
3698
3699 qedr_prepare_sq_sges(qp, NULL, wr);
3700
3701 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3702 qp->prev_wqe_size = awqe1->wqe_size;
3703 break;
3704
3705 case IB_WR_LOCAL_INV:
3706 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3707 iwqe->wqe_size = 1;
3708
3709 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3710 iwqe->inv_l_key = wr->ex.invalidate_rkey;
3711 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3712 qp->prev_wqe_size = iwqe->wqe_size;
3713 break;
3714 case IB_WR_REG_MR:
3715 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3716 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3717 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3718 fwqe1->wqe_size = 2;
3719
3720 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3721 if (rc) {
3722 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3723 *bad_wr = wr;
3724 break;
3725 }
3726
3727 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3728 qp->prev_wqe_size = fwqe1->wqe_size;
3729 break;
3730 default:
3731 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3732 rc = -EINVAL;
3733 *bad_wr = wr;
3734 break;
3735 }
3736
3737 if (*bad_wr) {
3738 u16 value;
3739
3740 /* Restore prod to its position before
3741 * this WR was processed
3742 */
3743 value = le16_to_cpu(qp->sq.db_data.data.value);
3744 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3745
3746 /* Restore prev_wqe_size */
3747 qp->prev_wqe_size = wqe->prev_wqe_size;
3748 rc = -EINVAL;
3749 DP_ERR(dev, "POST SEND FAILED\n");
3750 }
3751
3752 return rc;
3753}
3754
3755int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3756 const struct ib_send_wr **bad_wr)
3757{
3758 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3759 struct qedr_qp *qp = get_qedr_qp(ibqp);
3760 unsigned long flags;
3761 int rc = 0;
3762
3763 *bad_wr = NULL;
3764
3765 if (qp->qp_type == IB_QPT_GSI)
3766 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3767
3768 spin_lock_irqsave(&qp->q_lock, flags);
3769
3770 if (rdma_protocol_roce(&dev->ibdev, 1)) {
3771 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3772 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3773 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3774 spin_unlock_irqrestore(&qp->q_lock, flags);
3775 *bad_wr = wr;
3776 DP_DEBUG(dev, QEDR_MSG_CQ,
3777 "QP in wrong state! QP icid=0x%x state %d\n",
3778 qp->icid, qp->state);
3779 return -EINVAL;
3780 }
3781 }
3782
3783 while (wr) {
3784 rc = __qedr_post_send(ibqp, wr, bad_wr);
3785 if (rc)
3786 break;
3787
3788 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3789
3790 qedr_inc_sw_prod(&qp->sq);
3791
3792 qp->sq.db_data.data.value++;
3793
3794 wr = wr->next;
3795 }
3796
3797 /* Trigger doorbell
3798 * If there was a failure in the first WR then it will be triggered in
3799 * vane. However this is not harmful (as long as the producer value is
3800 * unchanged). For performance reasons we avoid checking for this
3801 * redundant doorbell.
3802 *
3803 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3804 * soon as we give the doorbell, we could get a completion
3805 * for this wr, therefore we need to make sure that the
3806 * memory is updated before giving the doorbell.
3807 * During qedr_poll_cq, rmb is called before accessing the
3808 * cqe. This covers for the smp_rmb as well.
3809 */
3810 smp_wmb();
3811 writel(qp->sq.db_data.raw, qp->sq.db);
3812
3813 spin_unlock_irqrestore(&qp->q_lock, flags);
3814
3815 return rc;
3816}
3817
3818static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3819{
3820 u32 used;
3821
3822 /* Calculate number of elements used based on producer
3823 * count and consumer count and subtract it from max
3824 * work request supported so that we get elements left.
3825 */
3826 used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3827
3828 return hw_srq->max_wr - used;
3829}
3830
3831int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3832 const struct ib_recv_wr **bad_wr)
3833{
3834 struct qedr_srq *srq = get_qedr_srq(ibsrq);
3835 struct qedr_srq_hwq_info *hw_srq;
3836 struct qedr_dev *dev = srq->dev;
3837 struct qed_chain *pbl;
3838 unsigned long flags;
3839 int status = 0;
3840 u32 num_sge;
3841
3842 spin_lock_irqsave(&srq->lock, flags);
3843
3844 hw_srq = &srq->hw_srq;
3845 pbl = &srq->hw_srq.pbl;
3846 while (wr) {
3847 struct rdma_srq_wqe_header *hdr;
3848 int i;
3849
3850 if (!qedr_srq_elem_left(hw_srq) ||
3851 wr->num_sge > srq->hw_srq.max_sges) {
3852 DP_ERR(dev, "Can't post WR (%d,%d) || (%d > %d)\n",
3853 hw_srq->wr_prod_cnt,
3854 atomic_read(&hw_srq->wr_cons_cnt),
3855 wr->num_sge, srq->hw_srq.max_sges);
3856 status = -ENOMEM;
3857 *bad_wr = wr;
3858 break;
3859 }
3860
3861 hdr = qed_chain_produce(pbl);
3862 num_sge = wr->num_sge;
3863 /* Set number of sge and work request id in header */
3864 SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3865
3866 srq->hw_srq.wr_prod_cnt++;
3867 hw_srq->wqe_prod++;
3868 hw_srq->sge_prod++;
3869
3870 DP_DEBUG(dev, QEDR_MSG_SRQ,
3871 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3872 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3873
3874 for (i = 0; i < wr->num_sge; i++) {
3875 struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3876
3877 /* Set SGE length, lkey and address */
3878 SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3879 wr->sg_list[i].length, wr->sg_list[i].lkey);
3880
3881 DP_DEBUG(dev, QEDR_MSG_SRQ,
3882 "[%d]: len %d key %x addr %x:%x\n",
3883 i, srq_sge->length, srq_sge->l_key,
3884 srq_sge->addr.hi, srq_sge->addr.lo);
3885 hw_srq->sge_prod++;
3886 }
3887
3888 /* Update WQE and SGE information before
3889 * updating producer.
3890 */
3891 dma_wmb();
3892
3893 /* SRQ producer is 8 bytes. Need to update SGE producer index
3894 * in first 4 bytes and need to update WQE producer in
3895 * next 4 bytes.
3896 */
3897 srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3898 /* Make sure sge producer is updated first */
3899 dma_wmb();
3900 srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3901
3902 wr = wr->next;
3903 }
3904
3905 DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3906 qed_chain_get_elem_left(pbl));
3907 spin_unlock_irqrestore(&srq->lock, flags);
3908
3909 return status;
3910}
3911
3912int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3913 const struct ib_recv_wr **bad_wr)
3914{
3915 struct qedr_qp *qp = get_qedr_qp(ibqp);
3916 struct qedr_dev *dev = qp->dev;
3917 unsigned long flags;
3918 int status = 0;
3919
3920 if (qp->qp_type == IB_QPT_GSI)
3921 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3922
3923 spin_lock_irqsave(&qp->q_lock, flags);
3924
3925 while (wr) {
3926 int i;
3927
3928 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3929 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3930 wr->num_sge > qp->rq.max_sges) {
3931 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3932 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3933 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3934 qp->rq.max_sges);
3935 status = -ENOMEM;
3936 *bad_wr = wr;
3937 break;
3938 }
3939 for (i = 0; i < wr->num_sge; i++) {
3940 u32 flags = 0;
3941 struct rdma_rq_sge *rqe =
3942 qed_chain_produce(&qp->rq.pbl);
3943
3944 /* First one must include the number
3945 * of SGE in the list
3946 */
3947 if (!i)
3948 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3949 wr->num_sge);
3950
3951 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3952 wr->sg_list[i].lkey);
3953
3954 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3955 wr->sg_list[i].length, flags);
3956 }
3957
3958 /* Special case of no sges. FW requires between 1-4 sges...
3959 * in this case we need to post 1 sge with length zero. this is
3960 * because rdma write with immediate consumes an RQ.
3961 */
3962 if (!wr->num_sge) {
3963 u32 flags = 0;
3964 struct rdma_rq_sge *rqe =
3965 qed_chain_produce(&qp->rq.pbl);
3966
3967 /* First one must include the number
3968 * of SGE in the list
3969 */
3970 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3971 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3972
3973 RQ_SGE_SET(rqe, 0, 0, flags);
3974 i = 1;
3975 }
3976
3977 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3978 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3979
3980 qedr_inc_sw_prod(&qp->rq);
3981
3982 /* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3983 * soon as we give the doorbell, we could get a completion
3984 * for this wr, therefore we need to make sure that the
3985 * memory is update before giving the doorbell.
3986 * During qedr_poll_cq, rmb is called before accessing the
3987 * cqe. This covers for the smp_rmb as well.
3988 */
3989 smp_wmb();
3990
3991 qp->rq.db_data.data.value++;
3992
3993 writel(qp->rq.db_data.raw, qp->rq.db);
3994
3995 if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3996 writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3997 }
3998
3999 wr = wr->next;
4000 }
4001
4002 spin_unlock_irqrestore(&qp->q_lock, flags);
4003
4004 return status;
4005}
4006
4007static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
4008{
4009 struct rdma_cqe_requester *resp_cqe = &cqe->req;
4010
4011 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4012 cq->pbl_toggle;
4013}
4014
4015static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4016{
4017 struct rdma_cqe_requester *resp_cqe = &cqe->req;
4018 struct qedr_qp *qp;
4019
4020 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4021 resp_cqe->qp_handle.lo,
4022 u64);
4023 return qp;
4024}
4025
4026static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4027{
4028 struct rdma_cqe_requester *resp_cqe = &cqe->req;
4029
4030 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4031}
4032
4033/* Return latest CQE (needs processing) */
4034static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4035{
4036 return cq->latest_cqe;
4037}
4038
4039/* In fmr we need to increase the number of fmr completed counter for the fmr
4040 * algorithm determining whether we can free a pbl or not.
4041 * we need to perform this whether the work request was signaled or not. for
4042 * this purpose we call this function from the condition that checks if a wr
4043 * should be skipped, to make sure we don't miss it ( possibly this fmr
4044 * operation was not signalted)
4045 */
4046static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4047{
4048 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4049 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4050}
4051
4052static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4053 struct qedr_cq *cq, int num_entries,
4054 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4055 int force)
4056{
4057 u16 cnt = 0;
4058
4059 while (num_entries && qp->sq.wqe_cons != hw_cons) {
4060 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4061 qedr_chk_if_fmr(qp);
4062 /* skip WC */
4063 goto next_cqe;
4064 }
4065
4066 /* fill WC */
4067 wc->status = status;
4068 wc->vendor_err = 0;
4069 wc->wc_flags = 0;
4070 wc->src_qp = qp->id;
4071 wc->qp = &qp->ibqp;
4072
4073 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4074 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4075
4076 switch (wc->opcode) {
4077 case IB_WC_RDMA_WRITE:
4078 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4079 break;
4080 case IB_WC_COMP_SWAP:
4081 case IB_WC_FETCH_ADD:
4082 wc->byte_len = 8;
4083 break;
4084 case IB_WC_REG_MR:
4085 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4086 break;
4087 case IB_WC_RDMA_READ:
4088 case IB_WC_SEND:
4089 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4090 break;
4091 default:
4092 break;
4093 }
4094
4095 num_entries--;
4096 wc++;
4097 cnt++;
4098next_cqe:
4099 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4100 qed_chain_consume(&qp->sq.pbl);
4101 qedr_inc_sw_cons(&qp->sq);
4102 }
4103
4104 return cnt;
4105}
4106
4107static int qedr_poll_cq_req(struct qedr_dev *dev,
4108 struct qedr_qp *qp, struct qedr_cq *cq,
4109 int num_entries, struct ib_wc *wc,
4110 struct rdma_cqe_requester *req)
4111{
4112 int cnt = 0;
4113
4114 switch (req->status) {
4115 case RDMA_CQE_REQ_STS_OK:
4116 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4117 IB_WC_SUCCESS, 0);
4118 break;
4119 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4120 if (qp->state != QED_ROCE_QP_STATE_ERR)
4121 DP_DEBUG(dev, QEDR_MSG_CQ,
4122 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4123 cq->icid, qp->icid);
4124 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4125 IB_WC_WR_FLUSH_ERR, 1);
4126 break;
4127 default:
4128 /* process all WQE before the cosumer */
4129 qp->state = QED_ROCE_QP_STATE_ERR;
4130 cnt = process_req(dev, qp, cq, num_entries, wc,
4131 req->sq_cons - 1, IB_WC_SUCCESS, 0);
4132 wc += cnt;
4133 /* if we have extra WC fill it with actual error info */
4134 if (cnt < num_entries) {
4135 enum ib_wc_status wc_status;
4136
4137 switch (req->status) {
4138 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4139 DP_ERR(dev,
4140 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4141 cq->icid, qp->icid);
4142 wc_status = IB_WC_BAD_RESP_ERR;
4143 break;
4144 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4145 DP_ERR(dev,
4146 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4147 cq->icid, qp->icid);
4148 wc_status = IB_WC_LOC_LEN_ERR;
4149 break;
4150 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4151 DP_ERR(dev,
4152 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4153 cq->icid, qp->icid);
4154 wc_status = IB_WC_LOC_QP_OP_ERR;
4155 break;
4156 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4157 DP_ERR(dev,
4158 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4159 cq->icid, qp->icid);
4160 wc_status = IB_WC_LOC_PROT_ERR;
4161 break;
4162 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4163 DP_ERR(dev,
4164 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4165 cq->icid, qp->icid);
4166 wc_status = IB_WC_MW_BIND_ERR;
4167 break;
4168 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4169 DP_ERR(dev,
4170 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4171 cq->icid, qp->icid);
4172 wc_status = IB_WC_REM_INV_REQ_ERR;
4173 break;
4174 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4175 DP_ERR(dev,
4176 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4177 cq->icid, qp->icid);
4178 wc_status = IB_WC_REM_ACCESS_ERR;
4179 break;
4180 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4181 DP_ERR(dev,
4182 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4183 cq->icid, qp->icid);
4184 wc_status = IB_WC_REM_OP_ERR;
4185 break;
4186 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4187 DP_ERR(dev,
4188 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4189 cq->icid, qp->icid);
4190 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4191 break;
4192 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4193 DP_ERR(dev,
4194 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4195 cq->icid, qp->icid);
4196 wc_status = IB_WC_RETRY_EXC_ERR;
4197 break;
4198 default:
4199 DP_ERR(dev,
4200 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4201 cq->icid, qp->icid);
4202 wc_status = IB_WC_GENERAL_ERR;
4203 }
4204 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4205 wc_status, 1);
4206 }
4207 }
4208
4209 return cnt;
4210}
4211
4212static inline int qedr_cqe_resp_status_to_ib(u8 status)
4213{
4214 switch (status) {
4215 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4216 return IB_WC_LOC_ACCESS_ERR;
4217 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4218 return IB_WC_LOC_LEN_ERR;
4219 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4220 return IB_WC_LOC_QP_OP_ERR;
4221 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4222 return IB_WC_LOC_PROT_ERR;
4223 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4224 return IB_WC_MW_BIND_ERR;
4225 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4226 return IB_WC_REM_INV_RD_REQ_ERR;
4227 case RDMA_CQE_RESP_STS_OK:
4228 return IB_WC_SUCCESS;
4229 default:
4230 return IB_WC_GENERAL_ERR;
4231 }
4232}
4233
4234static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4235 struct ib_wc *wc)
4236{
4237 wc->status = IB_WC_SUCCESS;
4238 wc->byte_len = le32_to_cpu(resp->length);
4239
4240 if (resp->flags & QEDR_RESP_IMM) {
4241 wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4242 wc->wc_flags |= IB_WC_WITH_IMM;
4243
4244 if (resp->flags & QEDR_RESP_RDMA)
4245 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4246
4247 if (resp->flags & QEDR_RESP_INV)
4248 return -EINVAL;
4249
4250 } else if (resp->flags & QEDR_RESP_INV) {
4251 wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4252 wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4253
4254 if (resp->flags & QEDR_RESP_RDMA)
4255 return -EINVAL;
4256
4257 } else if (resp->flags & QEDR_RESP_RDMA) {
4258 return -EINVAL;
4259 }
4260
4261 return 0;
4262}
4263
4264static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4265 struct qedr_cq *cq, struct ib_wc *wc,
4266 struct rdma_cqe_responder *resp, u64 wr_id)
4267{
4268 /* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4269 wc->opcode = IB_WC_RECV;
4270 wc->wc_flags = 0;
4271
4272 if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4273 if (qedr_set_ok_cqe_resp_wc(resp, wc))
4274 DP_ERR(dev,
4275 "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4276 cq, cq->icid, resp->flags);
4277
4278 } else {
4279 wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4280 if (wc->status == IB_WC_GENERAL_ERR)
4281 DP_ERR(dev,
4282 "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4283 cq, cq->icid, resp->status);
4284 }
4285
4286 /* Fill the rest of the WC */
4287 wc->vendor_err = 0;
4288 wc->src_qp = qp->id;
4289 wc->qp = &qp->ibqp;
4290 wc->wr_id = wr_id;
4291}
4292
4293static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4294 struct qedr_cq *cq, struct ib_wc *wc,
4295 struct rdma_cqe_responder *resp)
4296{
4297 struct qedr_srq *srq = qp->srq;
4298 u64 wr_id;
4299
4300 wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4301 le32_to_cpu(resp->srq_wr_id.lo), u64);
4302
4303 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4304 wc->status = IB_WC_WR_FLUSH_ERR;
4305 wc->vendor_err = 0;
4306 wc->wr_id = wr_id;
4307 wc->byte_len = 0;
4308 wc->src_qp = qp->id;
4309 wc->qp = &qp->ibqp;
4310 wc->wr_id = wr_id;
4311 } else {
4312 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4313 }
4314 atomic_inc(&srq->hw_srq.wr_cons_cnt);
4315
4316 return 1;
4317}
4318static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4319 struct qedr_cq *cq, struct ib_wc *wc,
4320 struct rdma_cqe_responder *resp)
4321{
4322 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4323
4324 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
4325
4326 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4327 qed_chain_consume(&qp->rq.pbl);
4328 qedr_inc_sw_cons(&qp->rq);
4329
4330 return 1;
4331}
4332
4333static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4334 int num_entries, struct ib_wc *wc, u16 hw_cons)
4335{
4336 u16 cnt = 0;
4337
4338 while (num_entries && qp->rq.wqe_cons != hw_cons) {
4339 /* fill WC */
4340 wc->status = IB_WC_WR_FLUSH_ERR;
4341 wc->vendor_err = 0;
4342 wc->wc_flags = 0;
4343 wc->src_qp = qp->id;
4344 wc->byte_len = 0;
4345 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4346 wc->qp = &qp->ibqp;
4347 num_entries--;
4348 wc++;
4349 cnt++;
4350 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4351 qed_chain_consume(&qp->rq.pbl);
4352 qedr_inc_sw_cons(&qp->rq);
4353 }
4354
4355 return cnt;
4356}
4357
4358static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4359 struct rdma_cqe_responder *resp, int *update)
4360{
4361 if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4362 consume_cqe(cq);
4363 *update |= 1;
4364 }
4365}
4366
4367static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4368 struct qedr_cq *cq, int num_entries,
4369 struct ib_wc *wc,
4370 struct rdma_cqe_responder *resp)
4371{
4372 int cnt;
4373
4374 cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4375 consume_cqe(cq);
4376
4377 return cnt;
4378}
4379
4380static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4381 struct qedr_cq *cq, int num_entries,
4382 struct ib_wc *wc, struct rdma_cqe_responder *resp,
4383 int *update)
4384{
4385 int cnt;
4386
4387 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4388 cnt = process_resp_flush(qp, cq, num_entries, wc,
4389 resp->rq_cons_or_srq_id);
4390 try_consume_resp_cqe(cq, qp, resp, update);
4391 } else {
4392 cnt = process_resp_one(dev, qp, cq, wc, resp);
4393 consume_cqe(cq);
4394 *update |= 1;
4395 }
4396
4397 return cnt;
4398}
4399
4400static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4401 struct rdma_cqe_requester *req, int *update)
4402{
4403 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4404 consume_cqe(cq);
4405 *update |= 1;
4406 }
4407}
4408
4409int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4410{
4411 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4412 struct qedr_cq *cq = get_qedr_cq(ibcq);
4413 union rdma_cqe *cqe;
4414 u32 old_cons, new_cons;
4415 unsigned long flags;
4416 int update = 0;
4417 int done = 0;
4418
4419 if (cq->destroyed) {
4420 DP_ERR(dev,
4421 "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4422 cq, cq->icid);
4423 return 0;
4424 }
4425
4426 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4427 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4428
4429 spin_lock_irqsave(&cq->cq_lock, flags);
4430 cqe = cq->latest_cqe;
4431 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4432 while (num_entries && is_valid_cqe(cq, cqe)) {
4433 struct qedr_qp *qp;
4434 int cnt = 0;
4435
4436 /* prevent speculative reads of any field of CQE */
4437 rmb();
4438
4439 qp = cqe_get_qp(cqe);
4440 if (!qp) {
4441 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4442 break;
4443 }
4444
4445 wc->qp = &qp->ibqp;
4446
4447 switch (cqe_get_type(cqe)) {
4448 case RDMA_CQE_TYPE_REQUESTER:
4449 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4450 &cqe->req);
4451 try_consume_req_cqe(cq, qp, &cqe->req, &update);
4452 break;
4453 case RDMA_CQE_TYPE_RESPONDER_RQ:
4454 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4455 &cqe->resp, &update);
4456 break;
4457 case RDMA_CQE_TYPE_RESPONDER_SRQ:
4458 cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4459 wc, &cqe->resp);
4460 update = 1;
4461 break;
4462 case RDMA_CQE_TYPE_INVALID:
4463 default:
4464 DP_ERR(dev, "Error: invalid CQE type = %d\n",
4465 cqe_get_type(cqe));
4466 }
4467 num_entries -= cnt;
4468 wc += cnt;
4469 done += cnt;
4470
4471 cqe = get_cqe(cq);
4472 }
4473 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4474
4475 cq->cq_cons += new_cons - old_cons;
4476
4477 if (update)
4478 /* doorbell notifies abount latest VALID entry,
4479 * but chain already point to the next INVALID one
4480 */
4481 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4482
4483 spin_unlock_irqrestore(&cq->cq_lock, flags);
4484 return done;
4485}
4486
4487int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4488 u32 port_num, const struct ib_wc *in_wc,
4489 const struct ib_grh *in_grh, const struct ib_mad *in,
4490 struct ib_mad *out_mad, size_t *out_mad_size,
4491 u16 *out_mad_pkey_index)
4492{
4493 return IB_MAD_RESULT_SUCCESS;
4494}
1/* QLogic qedr NIC Driver
2 * Copyright (c) 2015-2016 QLogic Corporation
3 *
4 * This software is available to you under a choice of one of two
5 * licenses. You may choose to be licensed under the terms of the GNU
6 * General Public License (GPL) Version 2, available from the file
7 * COPYING in the main directory of this source tree, or the
8 * OpenIB.org BSD license below:
9 *
10 * Redistribution and use in source and binary forms, with or
11 * without modification, are permitted provided that the following
12 * conditions are met:
13 *
14 * - Redistributions of source code must retain the above
15 * copyright notice, this list of conditions and the following
16 * disclaimer.
17 *
18 * - Redistributions in binary form must reproduce the above
19 * copyright notice, this list of conditions and the following
20 * disclaimer in the documentation and /or other materials
21 * provided with the distribution.
22 *
23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
30 * SOFTWARE.
31 */
32#include <linux/dma-mapping.h>
33#include <linux/crc32.h>
34#include <net/ip.h>
35#include <net/ipv6.h>
36#include <net/udp.h>
37#include <linux/iommu.h>
38
39#include <rdma/ib_verbs.h>
40#include <rdma/ib_user_verbs.h>
41#include <rdma/iw_cm.h>
42#include <rdma/ib_umem.h>
43#include <rdma/ib_addr.h>
44#include <rdma/ib_cache.h>
45
46#include "qedr_hsi.h"
47#include <linux/qed/qed_if.h>
48#include "qedr.h"
49#include "verbs.h"
50#include <rdma/qedr-abi.h>
51#include "qedr_cm.h"
52
53#define DB_ADDR_SHIFT(addr) ((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
54
55int qedr_query_pkey(struct ib_device *ibdev, u8 port, u16 index, u16 *pkey)
56{
57 if (index > QEDR_ROCE_PKEY_TABLE_LEN)
58 return -EINVAL;
59
60 *pkey = QEDR_ROCE_PKEY_DEFAULT;
61 return 0;
62}
63
64int qedr_query_gid(struct ib_device *ibdev, u8 port, int index,
65 union ib_gid *sgid)
66{
67 struct qedr_dev *dev = get_qedr_dev(ibdev);
68 int rc = 0;
69
70 if (!rdma_cap_roce_gid_table(ibdev, port))
71 return -ENODEV;
72
73 rc = ib_get_cached_gid(ibdev, port, index, sgid, NULL);
74 if (rc == -EAGAIN) {
75 memcpy(sgid, &zgid, sizeof(*sgid));
76 return 0;
77 }
78
79 DP_DEBUG(dev, QEDR_MSG_INIT, "query gid: index=%d %llx:%llx\n", index,
80 sgid->global.interface_id, sgid->global.subnet_prefix);
81
82 return rc;
83}
84
85int qedr_add_gid(struct ib_device *device, u8 port_num,
86 unsigned int index, const union ib_gid *gid,
87 const struct ib_gid_attr *attr, void **context)
88{
89 if (!rdma_cap_roce_gid_table(device, port_num))
90 return -EINVAL;
91
92 if (port_num > QEDR_MAX_PORT)
93 return -EINVAL;
94
95 if (!context)
96 return -EINVAL;
97
98 return 0;
99}
100
101int qedr_del_gid(struct ib_device *device, u8 port_num,
102 unsigned int index, void **context)
103{
104 if (!rdma_cap_roce_gid_table(device, port_num))
105 return -EINVAL;
106
107 if (port_num > QEDR_MAX_PORT)
108 return -EINVAL;
109
110 if (!context)
111 return -EINVAL;
112
113 return 0;
114}
115
116int qedr_query_device(struct ib_device *ibdev,
117 struct ib_device_attr *attr, struct ib_udata *udata)
118{
119 struct qedr_dev *dev = get_qedr_dev(ibdev);
120 struct qedr_device_attr *qattr = &dev->attr;
121
122 if (!dev->rdma_ctx) {
123 DP_ERR(dev,
124 "qedr_query_device called with invalid params rdma_ctx=%p\n",
125 dev->rdma_ctx);
126 return -EINVAL;
127 }
128
129 memset(attr, 0, sizeof(*attr));
130
131 attr->fw_ver = qattr->fw_ver;
132 attr->sys_image_guid = qattr->sys_image_guid;
133 attr->max_mr_size = qattr->max_mr_size;
134 attr->page_size_cap = qattr->page_size_caps;
135 attr->vendor_id = qattr->vendor_id;
136 attr->vendor_part_id = qattr->vendor_part_id;
137 attr->hw_ver = qattr->hw_ver;
138 attr->max_qp = qattr->max_qp;
139 attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
140 attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
141 IB_DEVICE_RC_RNR_NAK_GEN |
142 IB_DEVICE_LOCAL_DMA_LKEY | IB_DEVICE_MEM_MGT_EXTENSIONS;
143
144 attr->max_sge = qattr->max_sge;
145 attr->max_sge_rd = qattr->max_sge;
146 attr->max_cq = qattr->max_cq;
147 attr->max_cqe = qattr->max_cqe;
148 attr->max_mr = qattr->max_mr;
149 attr->max_mw = qattr->max_mw;
150 attr->max_pd = qattr->max_pd;
151 attr->atomic_cap = dev->atomic_cap;
152 attr->max_fmr = qattr->max_fmr;
153 attr->max_map_per_fmr = 16;
154 attr->max_qp_init_rd_atom =
155 1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
156 attr->max_qp_rd_atom =
157 min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
158 attr->max_qp_init_rd_atom);
159
160 attr->max_srq = qattr->max_srq;
161 attr->max_srq_sge = qattr->max_srq_sge;
162 attr->max_srq_wr = qattr->max_srq_wr;
163
164 attr->local_ca_ack_delay = qattr->dev_ack_delay;
165 attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
166 attr->max_pkeys = QEDR_ROCE_PKEY_MAX;
167 attr->max_ah = qattr->max_ah;
168
169 return 0;
170}
171
172#define QEDR_SPEED_SDR (1)
173#define QEDR_SPEED_DDR (2)
174#define QEDR_SPEED_QDR (4)
175#define QEDR_SPEED_FDR10 (8)
176#define QEDR_SPEED_FDR (16)
177#define QEDR_SPEED_EDR (32)
178
179static inline void get_link_speed_and_width(int speed, u8 *ib_speed,
180 u8 *ib_width)
181{
182 switch (speed) {
183 case 1000:
184 *ib_speed = QEDR_SPEED_SDR;
185 *ib_width = IB_WIDTH_1X;
186 break;
187 case 10000:
188 *ib_speed = QEDR_SPEED_QDR;
189 *ib_width = IB_WIDTH_1X;
190 break;
191
192 case 20000:
193 *ib_speed = QEDR_SPEED_DDR;
194 *ib_width = IB_WIDTH_4X;
195 break;
196
197 case 25000:
198 *ib_speed = QEDR_SPEED_EDR;
199 *ib_width = IB_WIDTH_1X;
200 break;
201
202 case 40000:
203 *ib_speed = QEDR_SPEED_QDR;
204 *ib_width = IB_WIDTH_4X;
205 break;
206
207 case 50000:
208 *ib_speed = QEDR_SPEED_QDR;
209 *ib_width = IB_WIDTH_4X;
210 break;
211
212 case 100000:
213 *ib_speed = QEDR_SPEED_EDR;
214 *ib_width = IB_WIDTH_4X;
215 break;
216
217 default:
218 /* Unsupported */
219 *ib_speed = QEDR_SPEED_SDR;
220 *ib_width = IB_WIDTH_1X;
221 }
222}
223
224int qedr_query_port(struct ib_device *ibdev, u8 port, struct ib_port_attr *attr)
225{
226 struct qedr_dev *dev;
227 struct qed_rdma_port *rdma_port;
228
229 dev = get_qedr_dev(ibdev);
230 if (port > 1) {
231 DP_ERR(dev, "invalid_port=0x%x\n", port);
232 return -EINVAL;
233 }
234
235 if (!dev->rdma_ctx) {
236 DP_ERR(dev, "rdma_ctx is NULL\n");
237 return -EINVAL;
238 }
239
240 rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
241 memset(attr, 0, sizeof(*attr));
242
243 if (rdma_port->port_state == QED_RDMA_PORT_UP) {
244 attr->state = IB_PORT_ACTIVE;
245 attr->phys_state = 5;
246 } else {
247 attr->state = IB_PORT_DOWN;
248 attr->phys_state = 3;
249 }
250 attr->max_mtu = IB_MTU_4096;
251 attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
252 attr->lid = 0;
253 attr->lmc = 0;
254 attr->sm_lid = 0;
255 attr->sm_sl = 0;
256 attr->port_cap_flags = IB_PORT_IP_BASED_GIDS;
257 attr->gid_tbl_len = QEDR_MAX_SGID;
258 attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
259 attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
260 attr->qkey_viol_cntr = 0;
261 get_link_speed_and_width(rdma_port->link_speed,
262 &attr->active_speed, &attr->active_width);
263 attr->max_msg_sz = rdma_port->max_msg_size;
264 attr->max_vl_num = 4;
265
266 return 0;
267}
268
269int qedr_modify_port(struct ib_device *ibdev, u8 port, int mask,
270 struct ib_port_modify *props)
271{
272 struct qedr_dev *dev;
273
274 dev = get_qedr_dev(ibdev);
275 if (port > 1) {
276 DP_ERR(dev, "invalid_port=0x%x\n", port);
277 return -EINVAL;
278 }
279
280 return 0;
281}
282
283static int qedr_add_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
284 unsigned long len)
285{
286 struct qedr_mm *mm;
287
288 mm = kzalloc(sizeof(*mm), GFP_KERNEL);
289 if (!mm)
290 return -ENOMEM;
291
292 mm->key.phy_addr = phy_addr;
293 /* This function might be called with a length which is not a multiple
294 * of PAGE_SIZE, while the mapping is PAGE_SIZE grained and the kernel
295 * forces this granularity by increasing the requested size if needed.
296 * When qedr_mmap is called, it will search the list with the updated
297 * length as a key. To prevent search failures, the length is rounded up
298 * in advance to PAGE_SIZE.
299 */
300 mm->key.len = roundup(len, PAGE_SIZE);
301 INIT_LIST_HEAD(&mm->entry);
302
303 mutex_lock(&uctx->mm_list_lock);
304 list_add(&mm->entry, &uctx->mm_head);
305 mutex_unlock(&uctx->mm_list_lock);
306
307 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
308 "added (addr=0x%llx,len=0x%lx) for ctx=%p\n",
309 (unsigned long long)mm->key.phy_addr,
310 (unsigned long)mm->key.len, uctx);
311
312 return 0;
313}
314
315static bool qedr_search_mmap(struct qedr_ucontext *uctx, u64 phy_addr,
316 unsigned long len)
317{
318 bool found = false;
319 struct qedr_mm *mm;
320
321 mutex_lock(&uctx->mm_list_lock);
322 list_for_each_entry(mm, &uctx->mm_head, entry) {
323 if (len != mm->key.len || phy_addr != mm->key.phy_addr)
324 continue;
325
326 found = true;
327 break;
328 }
329 mutex_unlock(&uctx->mm_list_lock);
330 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
331 "searched for (addr=0x%llx,len=0x%lx) for ctx=%p, result=%d\n",
332 mm->key.phy_addr, mm->key.len, uctx, found);
333
334 return found;
335}
336
337struct ib_ucontext *qedr_alloc_ucontext(struct ib_device *ibdev,
338 struct ib_udata *udata)
339{
340 int rc;
341 struct qedr_ucontext *ctx;
342 struct qedr_alloc_ucontext_resp uresp;
343 struct qedr_dev *dev = get_qedr_dev(ibdev);
344 struct qed_rdma_add_user_out_params oparams;
345
346 if (!udata)
347 return ERR_PTR(-EFAULT);
348
349 ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
350 if (!ctx)
351 return ERR_PTR(-ENOMEM);
352
353 rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
354 if (rc) {
355 DP_ERR(dev,
356 "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
357 rc);
358 goto err;
359 }
360
361 ctx->dpi = oparams.dpi;
362 ctx->dpi_addr = oparams.dpi_addr;
363 ctx->dpi_phys_addr = oparams.dpi_phys_addr;
364 ctx->dpi_size = oparams.dpi_size;
365 INIT_LIST_HEAD(&ctx->mm_head);
366 mutex_init(&ctx->mm_list_lock);
367
368 memset(&uresp, 0, sizeof(uresp));
369
370 uresp.db_pa = ctx->dpi_phys_addr;
371 uresp.db_size = ctx->dpi_size;
372 uresp.max_send_wr = dev->attr.max_sqe;
373 uresp.max_recv_wr = dev->attr.max_rqe;
374 uresp.max_srq_wr = dev->attr.max_srq_wr;
375 uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
376 uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
377 uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
378 uresp.max_cqes = QEDR_MAX_CQES;
379
380 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
381 if (rc)
382 goto err;
383
384 ctx->dev = dev;
385
386 rc = qedr_add_mmap(ctx, ctx->dpi_phys_addr, ctx->dpi_size);
387 if (rc)
388 goto err;
389
390 DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
391 &ctx->ibucontext);
392 return &ctx->ibucontext;
393
394err:
395 kfree(ctx);
396 return ERR_PTR(rc);
397}
398
399int qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
400{
401 struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
402 struct qedr_mm *mm, *tmp;
403 int status = 0;
404
405 DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
406 uctx);
407 uctx->dev->ops->rdma_remove_user(uctx->dev->rdma_ctx, uctx->dpi);
408
409 list_for_each_entry_safe(mm, tmp, &uctx->mm_head, entry) {
410 DP_DEBUG(uctx->dev, QEDR_MSG_MISC,
411 "deleted (addr=0x%llx,len=0x%lx) for ctx=%p\n",
412 mm->key.phy_addr, mm->key.len, uctx);
413 list_del(&mm->entry);
414 kfree(mm);
415 }
416
417 kfree(uctx);
418 return status;
419}
420
421int qedr_mmap(struct ib_ucontext *context, struct vm_area_struct *vma)
422{
423 struct qedr_ucontext *ucontext = get_qedr_ucontext(context);
424 struct qedr_dev *dev = get_qedr_dev(context->device);
425 unsigned long vm_page = vma->vm_pgoff << PAGE_SHIFT;
426 u64 unmapped_db = dev->db_phys_addr;
427 unsigned long len = (vma->vm_end - vma->vm_start);
428 int rc = 0;
429 bool found;
430
431 DP_DEBUG(dev, QEDR_MSG_INIT,
432 "qedr_mmap called vm_page=0x%lx vm_pgoff=0x%lx unmapped_db=0x%llx db_size=%x, len=%lx\n",
433 vm_page, vma->vm_pgoff, unmapped_db, dev->db_size, len);
434 if (vma->vm_start & (PAGE_SIZE - 1)) {
435 DP_ERR(dev, "Vma_start not page aligned = %ld\n",
436 vma->vm_start);
437 return -EINVAL;
438 }
439
440 found = qedr_search_mmap(ucontext, vm_page, len);
441 if (!found) {
442 DP_ERR(dev, "Vma_pgoff not found in mapped array = %ld\n",
443 vma->vm_pgoff);
444 return -EINVAL;
445 }
446
447 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
448
449 if ((vm_page >= unmapped_db) && (vm_page <= (unmapped_db +
450 dev->db_size))) {
451 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping doorbell bar\n");
452 if (vma->vm_flags & VM_READ) {
453 DP_ERR(dev, "Trying to map doorbell bar for read\n");
454 return -EPERM;
455 }
456
457 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
458
459 rc = io_remap_pfn_range(vma, vma->vm_start, vma->vm_pgoff,
460 PAGE_SIZE, vma->vm_page_prot);
461 } else {
462 DP_DEBUG(dev, QEDR_MSG_INIT, "Mapping chains\n");
463 rc = remap_pfn_range(vma, vma->vm_start,
464 vma->vm_pgoff, len, vma->vm_page_prot);
465 }
466 DP_DEBUG(dev, QEDR_MSG_INIT, "qedr_mmap return code: %d\n", rc);
467 return rc;
468}
469
470struct ib_pd *qedr_alloc_pd(struct ib_device *ibdev,
471 struct ib_ucontext *context, struct ib_udata *udata)
472{
473 struct qedr_dev *dev = get_qedr_dev(ibdev);
474 struct qedr_pd *pd;
475 u16 pd_id;
476 int rc;
477
478 DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
479 (udata && context) ? "User Lib" : "Kernel");
480
481 if (!dev->rdma_ctx) {
482 DP_ERR(dev, "invlaid RDMA context\n");
483 return ERR_PTR(-EINVAL);
484 }
485
486 pd = kzalloc(sizeof(*pd), GFP_KERNEL);
487 if (!pd)
488 return ERR_PTR(-ENOMEM);
489
490 rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
491 if (rc)
492 goto err;
493
494 pd->pd_id = pd_id;
495
496 if (udata && context) {
497 struct qedr_alloc_pd_uresp uresp;
498
499 uresp.pd_id = pd_id;
500
501 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
502 if (rc) {
503 DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
504 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
505 goto err;
506 }
507
508 pd->uctx = get_qedr_ucontext(context);
509 pd->uctx->pd = pd;
510 }
511
512 return &pd->ibpd;
513
514err:
515 kfree(pd);
516 return ERR_PTR(rc);
517}
518
519int qedr_dealloc_pd(struct ib_pd *ibpd)
520{
521 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
522 struct qedr_pd *pd = get_qedr_pd(ibpd);
523
524 if (!pd) {
525 pr_err("Invalid PD received in dealloc_pd\n");
526 return -EINVAL;
527 }
528
529 DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
530 dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
531
532 kfree(pd);
533
534 return 0;
535}
536
537static void qedr_free_pbl(struct qedr_dev *dev,
538 struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
539{
540 struct pci_dev *pdev = dev->pdev;
541 int i;
542
543 for (i = 0; i < pbl_info->num_pbls; i++) {
544 if (!pbl[i].va)
545 continue;
546 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
547 pbl[i].va, pbl[i].pa);
548 }
549
550 kfree(pbl);
551}
552
553#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
554#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
555
556#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
557#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
558#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
559
560static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
561 struct qedr_pbl_info *pbl_info,
562 gfp_t flags)
563{
564 struct pci_dev *pdev = dev->pdev;
565 struct qedr_pbl *pbl_table;
566 dma_addr_t *pbl_main_tbl;
567 dma_addr_t pa;
568 void *va;
569 int i;
570
571 pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
572 if (!pbl_table)
573 return ERR_PTR(-ENOMEM);
574
575 for (i = 0; i < pbl_info->num_pbls; i++) {
576 va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size,
577 &pa, flags);
578 if (!va)
579 goto err;
580
581 memset(va, 0, pbl_info->pbl_size);
582 pbl_table[i].va = va;
583 pbl_table[i].pa = pa;
584 }
585
586 /* Two-Layer PBLs, if we have more than one pbl we need to initialize
587 * the first one with physical pointers to all of the rest
588 */
589 pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
590 for (i = 0; i < pbl_info->num_pbls - 1; i++)
591 pbl_main_tbl[i] = pbl_table[i + 1].pa;
592
593 return pbl_table;
594
595err:
596 for (i--; i >= 0; i--)
597 dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
598 pbl_table[i].va, pbl_table[i].pa);
599
600 qedr_free_pbl(dev, pbl_info, pbl_table);
601
602 return ERR_PTR(-ENOMEM);
603}
604
605static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
606 struct qedr_pbl_info *pbl_info,
607 u32 num_pbes, int two_layer_capable)
608{
609 u32 pbl_capacity;
610 u32 pbl_size;
611 u32 num_pbls;
612
613 if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
614 if (num_pbes > MAX_PBES_TWO_LAYER) {
615 DP_ERR(dev, "prepare pbl table: too many pages %d\n",
616 num_pbes);
617 return -EINVAL;
618 }
619
620 /* calculate required pbl page size */
621 pbl_size = MIN_FW_PBL_PAGE_SIZE;
622 pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
623 NUM_PBES_ON_PAGE(pbl_size);
624
625 while (pbl_capacity < num_pbes) {
626 pbl_size *= 2;
627 pbl_capacity = pbl_size / sizeof(u64);
628 pbl_capacity = pbl_capacity * pbl_capacity;
629 }
630
631 num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
632 num_pbls++; /* One for the layer0 ( points to the pbls) */
633 pbl_info->two_layered = true;
634 } else {
635 /* One layered PBL */
636 num_pbls = 1;
637 pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
638 roundup_pow_of_two((num_pbes * sizeof(u64))));
639 pbl_info->two_layered = false;
640 }
641
642 pbl_info->num_pbls = num_pbls;
643 pbl_info->pbl_size = pbl_size;
644 pbl_info->num_pbes = num_pbes;
645
646 DP_DEBUG(dev, QEDR_MSG_MR,
647 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
648 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
649
650 return 0;
651}
652
653static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
654 struct qedr_pbl *pbl,
655 struct qedr_pbl_info *pbl_info)
656{
657 int shift, pg_cnt, pages, pbe_cnt, total_num_pbes = 0;
658 struct qedr_pbl *pbl_tbl;
659 struct scatterlist *sg;
660 struct regpair *pbe;
661 int entry;
662 u32 addr;
663
664 if (!pbl_info->num_pbes)
665 return;
666
667 /* If we have a two layered pbl, the first pbl points to the rest
668 * of the pbls and the first entry lays on the second pbl in the table
669 */
670 if (pbl_info->two_layered)
671 pbl_tbl = &pbl[1];
672 else
673 pbl_tbl = pbl;
674
675 pbe = (struct regpair *)pbl_tbl->va;
676 if (!pbe) {
677 DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
678 return;
679 }
680
681 pbe_cnt = 0;
682
683 shift = ilog2(umem->page_size);
684
685 for_each_sg(umem->sg_head.sgl, sg, umem->nmap, entry) {
686 pages = sg_dma_len(sg) >> shift;
687 for (pg_cnt = 0; pg_cnt < pages; pg_cnt++) {
688 /* store the page address in pbe */
689 pbe->lo = cpu_to_le32(sg_dma_address(sg) +
690 umem->page_size * pg_cnt);
691 addr = upper_32_bits(sg_dma_address(sg) +
692 umem->page_size * pg_cnt);
693 pbe->hi = cpu_to_le32(addr);
694 pbe_cnt++;
695 total_num_pbes++;
696 pbe++;
697
698 if (total_num_pbes == pbl_info->num_pbes)
699 return;
700
701 /* If the given pbl is full storing the pbes,
702 * move to next pbl.
703 */
704 if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
705 pbl_tbl++;
706 pbe = (struct regpair *)pbl_tbl->va;
707 pbe_cnt = 0;
708 }
709 }
710 }
711}
712
713static int qedr_copy_cq_uresp(struct qedr_dev *dev,
714 struct qedr_cq *cq, struct ib_udata *udata)
715{
716 struct qedr_create_cq_uresp uresp;
717 int rc;
718
719 memset(&uresp, 0, sizeof(uresp));
720
721 uresp.db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
722 uresp.icid = cq->icid;
723
724 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
725 if (rc)
726 DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
727
728 return rc;
729}
730
731static void consume_cqe(struct qedr_cq *cq)
732{
733 if (cq->latest_cqe == cq->toggle_cqe)
734 cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
735
736 cq->latest_cqe = qed_chain_consume(&cq->pbl);
737}
738
739static inline int qedr_align_cq_entries(int entries)
740{
741 u64 size, aligned_size;
742
743 /* We allocate an extra entry that we don't report to the FW. */
744 size = (entries + 1) * QEDR_CQE_SIZE;
745 aligned_size = ALIGN(size, PAGE_SIZE);
746
747 return aligned_size / QEDR_CQE_SIZE;
748}
749
750static inline int qedr_init_user_queue(struct ib_ucontext *ib_ctx,
751 struct qedr_dev *dev,
752 struct qedr_userq *q,
753 u64 buf_addr, size_t buf_len,
754 int access, int dmasync)
755{
756 int page_cnt;
757 int rc;
758
759 q->buf_addr = buf_addr;
760 q->buf_len = buf_len;
761 q->umem = ib_umem_get(ib_ctx, q->buf_addr, q->buf_len, access, dmasync);
762 if (IS_ERR(q->umem)) {
763 DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
764 PTR_ERR(q->umem));
765 return PTR_ERR(q->umem);
766 }
767
768 page_cnt = ib_umem_page_count(q->umem);
769 rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, page_cnt, 0);
770 if (rc)
771 goto err0;
772
773 q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
774 if (IS_ERR_OR_NULL(q->pbl_tbl))
775 goto err0;
776
777 qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info);
778
779 return 0;
780
781err0:
782 ib_umem_release(q->umem);
783
784 return rc;
785}
786
787static inline void qedr_init_cq_params(struct qedr_cq *cq,
788 struct qedr_ucontext *ctx,
789 struct qedr_dev *dev, int vector,
790 int chain_entries, int page_cnt,
791 u64 pbl_ptr,
792 struct qed_rdma_create_cq_in_params
793 *params)
794{
795 memset(params, 0, sizeof(*params));
796 params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
797 params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
798 params->cnq_id = vector;
799 params->cq_size = chain_entries - 1;
800 params->dpi = (ctx) ? ctx->dpi : dev->dpi;
801 params->pbl_num_pages = page_cnt;
802 params->pbl_ptr = pbl_ptr;
803 params->pbl_two_level = 0;
804}
805
806static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
807{
808 /* Flush data before signalling doorbell */
809 wmb();
810 cq->db.data.agg_flags = flags;
811 cq->db.data.value = cpu_to_le32(cons);
812 writeq(cq->db.raw, cq->db_addr);
813
814 /* Make sure write would stick */
815 mmiowb();
816}
817
818int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
819{
820 struct qedr_cq *cq = get_qedr_cq(ibcq);
821 unsigned long sflags;
822
823 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
824 return 0;
825
826 spin_lock_irqsave(&cq->cq_lock, sflags);
827
828 cq->arm_flags = 0;
829
830 if (flags & IB_CQ_SOLICITED)
831 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
832
833 if (flags & IB_CQ_NEXT_COMP)
834 cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
835
836 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
837
838 spin_unlock_irqrestore(&cq->cq_lock, sflags);
839
840 return 0;
841}
842
843struct ib_cq *qedr_create_cq(struct ib_device *ibdev,
844 const struct ib_cq_init_attr *attr,
845 struct ib_ucontext *ib_ctx, struct ib_udata *udata)
846{
847 struct qedr_ucontext *ctx = get_qedr_ucontext(ib_ctx);
848 struct qed_rdma_destroy_cq_out_params destroy_oparams;
849 struct qed_rdma_destroy_cq_in_params destroy_iparams;
850 struct qedr_dev *dev = get_qedr_dev(ibdev);
851 struct qed_rdma_create_cq_in_params params;
852 struct qedr_create_cq_ureq ureq;
853 int vector = attr->comp_vector;
854 int entries = attr->cqe;
855 struct qedr_cq *cq;
856 int chain_entries;
857 int page_cnt;
858 u64 pbl_ptr;
859 u16 icid;
860 int rc;
861
862 DP_DEBUG(dev, QEDR_MSG_INIT,
863 "create_cq: called from %s. entries=%d, vector=%d\n",
864 udata ? "User Lib" : "Kernel", entries, vector);
865
866 if (entries > QEDR_MAX_CQES) {
867 DP_ERR(dev,
868 "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
869 entries, QEDR_MAX_CQES);
870 return ERR_PTR(-EINVAL);
871 }
872
873 chain_entries = qedr_align_cq_entries(entries);
874 chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
875
876 cq = kzalloc(sizeof(*cq), GFP_KERNEL);
877 if (!cq)
878 return ERR_PTR(-ENOMEM);
879
880 if (udata) {
881 memset(&ureq, 0, sizeof(ureq));
882 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
883 DP_ERR(dev,
884 "create cq: problem copying data from user space\n");
885 goto err0;
886 }
887
888 if (!ureq.len) {
889 DP_ERR(dev,
890 "create cq: cannot create a cq with 0 entries\n");
891 goto err0;
892 }
893
894 cq->cq_type = QEDR_CQ_TYPE_USER;
895
896 rc = qedr_init_user_queue(ib_ctx, dev, &cq->q, ureq.addr,
897 ureq.len, IB_ACCESS_LOCAL_WRITE, 1);
898 if (rc)
899 goto err0;
900
901 pbl_ptr = cq->q.pbl_tbl->pa;
902 page_cnt = cq->q.pbl_info.num_pbes;
903
904 cq->ibcq.cqe = chain_entries;
905 } else {
906 cq->cq_type = QEDR_CQ_TYPE_KERNEL;
907
908 rc = dev->ops->common->chain_alloc(dev->cdev,
909 QED_CHAIN_USE_TO_CONSUME,
910 QED_CHAIN_MODE_PBL,
911 QED_CHAIN_CNT_TYPE_U32,
912 chain_entries,
913 sizeof(union rdma_cqe),
914 &cq->pbl);
915 if (rc)
916 goto err1;
917
918 page_cnt = qed_chain_get_page_cnt(&cq->pbl);
919 pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
920 cq->ibcq.cqe = cq->pbl.capacity;
921 }
922
923 qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
924 pbl_ptr, ¶ms);
925
926 rc = dev->ops->rdma_create_cq(dev->rdma_ctx, ¶ms, &icid);
927 if (rc)
928 goto err2;
929
930 cq->icid = icid;
931 cq->sig = QEDR_CQ_MAGIC_NUMBER;
932 spin_lock_init(&cq->cq_lock);
933
934 if (ib_ctx) {
935 rc = qedr_copy_cq_uresp(dev, cq, udata);
936 if (rc)
937 goto err3;
938 } else {
939 /* Generate doorbell address. */
940 cq->db_addr = dev->db_addr +
941 DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
942 cq->db.data.icid = cq->icid;
943 cq->db.data.params = DB_AGG_CMD_SET <<
944 RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
945
946 /* point to the very last element, passing it we will toggle */
947 cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
948 cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
949 cq->latest_cqe = NULL;
950 consume_cqe(cq);
951 cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
952 }
953
954 DP_DEBUG(dev, QEDR_MSG_CQ,
955 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
956 cq->icid, cq, params.cq_size);
957
958 return &cq->ibcq;
959
960err3:
961 destroy_iparams.icid = cq->icid;
962 dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
963 &destroy_oparams);
964err2:
965 if (udata)
966 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
967 else
968 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
969err1:
970 if (udata)
971 ib_umem_release(cq->q.umem);
972err0:
973 kfree(cq);
974 return ERR_PTR(-EINVAL);
975}
976
977int qedr_resize_cq(struct ib_cq *ibcq, int new_cnt, struct ib_udata *udata)
978{
979 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
980 struct qedr_cq *cq = get_qedr_cq(ibcq);
981
982 DP_ERR(dev, "cq %p RESIZE NOT SUPPORTED\n", cq);
983
984 return 0;
985}
986
987int qedr_destroy_cq(struct ib_cq *ibcq)
988{
989 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
990 struct qed_rdma_destroy_cq_out_params oparams;
991 struct qed_rdma_destroy_cq_in_params iparams;
992 struct qedr_cq *cq = get_qedr_cq(ibcq);
993
994 DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq: cq_id %d", cq->icid);
995
996 /* GSIs CQs are handled by driver, so they don't exist in the FW */
997 if (cq->cq_type != QEDR_CQ_TYPE_GSI) {
998 int rc;
999
1000 iparams.icid = cq->icid;
1001 rc = dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams,
1002 &oparams);
1003 if (rc)
1004 return rc;
1005 dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1006 }
1007
1008 if (ibcq->uobject && ibcq->uobject->context) {
1009 qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1010 ib_umem_release(cq->q.umem);
1011 }
1012
1013 kfree(cq);
1014
1015 return 0;
1016}
1017
1018static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1019 struct ib_qp_attr *attr,
1020 int attr_mask,
1021 struct qed_rdma_modify_qp_in_params
1022 *qp_params)
1023{
1024 enum rdma_network_type nw_type;
1025 struct ib_gid_attr gid_attr;
1026 union ib_gid gid;
1027 u32 ipv4_addr;
1028 int rc = 0;
1029 int i;
1030
1031 rc = ib_get_cached_gid(ibqp->device, attr->ah_attr.port_num,
1032 attr->ah_attr.grh.sgid_index, &gid, &gid_attr);
1033 if (rc)
1034 return rc;
1035
1036 if (!memcmp(&gid, &zgid, sizeof(gid)))
1037 return -ENOENT;
1038
1039 if (gid_attr.ndev) {
1040 qp_params->vlan_id = rdma_vlan_dev_vlan_id(gid_attr.ndev);
1041
1042 dev_put(gid_attr.ndev);
1043 nw_type = ib_gid_to_network_type(gid_attr.gid_type, &gid);
1044 switch (nw_type) {
1045 case RDMA_NETWORK_IPV6:
1046 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1047 sizeof(qp_params->sgid));
1048 memcpy(&qp_params->dgid.bytes[0],
1049 &attr->ah_attr.grh.dgid,
1050 sizeof(qp_params->dgid));
1051 qp_params->roce_mode = ROCE_V2_IPV6;
1052 SET_FIELD(qp_params->modify_flags,
1053 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1054 break;
1055 case RDMA_NETWORK_IB:
1056 memcpy(&qp_params->sgid.bytes[0], &gid.raw[0],
1057 sizeof(qp_params->sgid));
1058 memcpy(&qp_params->dgid.bytes[0],
1059 &attr->ah_attr.grh.dgid,
1060 sizeof(qp_params->dgid));
1061 qp_params->roce_mode = ROCE_V1;
1062 break;
1063 case RDMA_NETWORK_IPV4:
1064 memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1065 memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1066 ipv4_addr = qedr_get_ipv4_from_gid(gid.raw);
1067 qp_params->sgid.ipv4_addr = ipv4_addr;
1068 ipv4_addr =
1069 qedr_get_ipv4_from_gid(attr->ah_attr.grh.dgid.raw);
1070 qp_params->dgid.ipv4_addr = ipv4_addr;
1071 SET_FIELD(qp_params->modify_flags,
1072 QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1073 qp_params->roce_mode = ROCE_V2_IPV4;
1074 break;
1075 }
1076 }
1077
1078 for (i = 0; i < 4; i++) {
1079 qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1080 qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1081 }
1082
1083 if (qp_params->vlan_id >= VLAN_CFI_MASK)
1084 qp_params->vlan_id = 0;
1085
1086 return 0;
1087}
1088
1089static void qedr_cleanup_user_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1090{
1091 qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1092 ib_umem_release(qp->usq.umem);
1093}
1094
1095static void qedr_cleanup_user_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1096{
1097 qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1098 ib_umem_release(qp->urq.umem);
1099}
1100
1101static void qedr_cleanup_kernel_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1102{
1103 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1104 kfree(qp->wqe_wr_id);
1105}
1106
1107static void qedr_cleanup_kernel_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1108{
1109 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1110 kfree(qp->rqe_wr_id);
1111}
1112
1113static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1114 struct ib_qp_init_attr *attrs)
1115{
1116 struct qedr_device_attr *qattr = &dev->attr;
1117
1118 /* QP0... attrs->qp_type == IB_QPT_GSI */
1119 if (attrs->qp_type != IB_QPT_RC && attrs->qp_type != IB_QPT_GSI) {
1120 DP_DEBUG(dev, QEDR_MSG_QP,
1121 "create qp: unsupported qp type=0x%x requested\n",
1122 attrs->qp_type);
1123 return -EINVAL;
1124 }
1125
1126 if (attrs->cap.max_send_wr > qattr->max_sqe) {
1127 DP_ERR(dev,
1128 "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1129 attrs->cap.max_send_wr, qattr->max_sqe);
1130 return -EINVAL;
1131 }
1132
1133 if (attrs->cap.max_inline_data > qattr->max_inline) {
1134 DP_ERR(dev,
1135 "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1136 attrs->cap.max_inline_data, qattr->max_inline);
1137 return -EINVAL;
1138 }
1139
1140 if (attrs->cap.max_send_sge > qattr->max_sge) {
1141 DP_ERR(dev,
1142 "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1143 attrs->cap.max_send_sge, qattr->max_sge);
1144 return -EINVAL;
1145 }
1146
1147 if (attrs->cap.max_recv_sge > qattr->max_sge) {
1148 DP_ERR(dev,
1149 "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1150 attrs->cap.max_recv_sge, qattr->max_sge);
1151 return -EINVAL;
1152 }
1153
1154 /* Unprivileged user space cannot create special QP */
1155 if (ibpd->uobject && attrs->qp_type == IB_QPT_GSI) {
1156 DP_ERR(dev,
1157 "create qp: userspace can't create special QPs of type=0x%x\n",
1158 attrs->qp_type);
1159 return -EINVAL;
1160 }
1161
1162 return 0;
1163}
1164
1165static void qedr_copy_rq_uresp(struct qedr_create_qp_uresp *uresp,
1166 struct qedr_qp *qp)
1167{
1168 uresp->rq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1169 uresp->rq_icid = qp->icid;
1170}
1171
1172static void qedr_copy_sq_uresp(struct qedr_create_qp_uresp *uresp,
1173 struct qedr_qp *qp)
1174{
1175 uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1176 uresp->sq_icid = qp->icid + 1;
1177}
1178
1179static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1180 struct qedr_qp *qp, struct ib_udata *udata)
1181{
1182 struct qedr_create_qp_uresp uresp;
1183 int rc;
1184
1185 memset(&uresp, 0, sizeof(uresp));
1186 qedr_copy_sq_uresp(&uresp, qp);
1187 qedr_copy_rq_uresp(&uresp, qp);
1188
1189 uresp.atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1190 uresp.qp_id = qp->qp_id;
1191
1192 rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1193 if (rc)
1194 DP_ERR(dev,
1195 "create qp: failed a copy to user space with qp icid=0x%x.\n",
1196 qp->icid);
1197
1198 return rc;
1199}
1200
1201static void qedr_set_qp_init_params(struct qedr_dev *dev,
1202 struct qedr_qp *qp,
1203 struct qedr_pd *pd,
1204 struct ib_qp_init_attr *attrs)
1205{
1206 qp->pd = pd;
1207
1208 spin_lock_init(&qp->q_lock);
1209
1210 qp->qp_type = attrs->qp_type;
1211 qp->max_inline_data = attrs->cap.max_inline_data;
1212 qp->sq.max_sges = attrs->cap.max_send_sge;
1213 qp->state = QED_ROCE_QP_STATE_RESET;
1214 qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1215 qp->sq_cq = get_qedr_cq(attrs->send_cq);
1216 qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1217 qp->dev = dev;
1218
1219 DP_DEBUG(dev, QEDR_MSG_QP,
1220 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1221 pd->pd_id, qp->qp_type, qp->max_inline_data,
1222 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1223 DP_DEBUG(dev, QEDR_MSG_QP,
1224 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1225 qp->sq.max_sges, qp->sq_cq->icid);
1226 qp->rq.max_sges = attrs->cap.max_recv_sge;
1227 DP_DEBUG(dev, QEDR_MSG_QP,
1228 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1229 qp->rq.max_sges, qp->rq_cq->icid);
1230}
1231
1232static inline void
1233qedr_init_qp_user_params(struct qed_rdma_create_qp_in_params *params,
1234 struct qedr_create_qp_ureq *ureq)
1235{
1236 /* QP handle to be written in CQE */
1237 params->qp_handle_lo = ureq->qp_handle_lo;
1238 params->qp_handle_hi = ureq->qp_handle_hi;
1239}
1240
1241static inline void
1242qedr_init_qp_kernel_doorbell_sq(struct qedr_dev *dev, struct qedr_qp *qp)
1243{
1244 qp->sq.db = dev->db_addr +
1245 DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1246 qp->sq.db_data.data.icid = qp->icid + 1;
1247}
1248
1249static inline void
1250qedr_init_qp_kernel_doorbell_rq(struct qedr_dev *dev, struct qedr_qp *qp)
1251{
1252 qp->rq.db = dev->db_addr +
1253 DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1254 qp->rq.db_data.data.icid = qp->icid;
1255}
1256
1257static inline int
1258qedr_init_qp_kernel_params_rq(struct qedr_dev *dev,
1259 struct qedr_qp *qp, struct ib_qp_init_attr *attrs)
1260{
1261 /* Allocate driver internal RQ array */
1262 qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
1263 GFP_KERNEL);
1264 if (!qp->rqe_wr_id)
1265 return -ENOMEM;
1266
1267 DP_DEBUG(dev, QEDR_MSG_QP, "RQ max_wr set to %d.\n", qp->rq.max_wr);
1268
1269 return 0;
1270}
1271
1272static inline int
1273qedr_init_qp_kernel_params_sq(struct qedr_dev *dev,
1274 struct qedr_qp *qp,
1275 struct ib_qp_init_attr *attrs,
1276 struct qed_rdma_create_qp_in_params *params)
1277{
1278 u32 temp_max_wr;
1279
1280 /* Allocate driver internal SQ array */
1281 temp_max_wr = attrs->cap.max_send_wr * dev->wq_multiplier;
1282 temp_max_wr = min_t(u32, temp_max_wr, dev->attr.max_sqe);
1283
1284 /* temp_max_wr < attr->max_sqe < u16 so the casting is safe */
1285 qp->sq.max_wr = (u16)temp_max_wr;
1286 qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
1287 GFP_KERNEL);
1288 if (!qp->wqe_wr_id)
1289 return -ENOMEM;
1290
1291 DP_DEBUG(dev, QEDR_MSG_QP, "SQ max_wr set to %d.\n", qp->sq.max_wr);
1292
1293 /* QP handle to be written in CQE */
1294 params->qp_handle_lo = lower_32_bits((uintptr_t)qp);
1295 params->qp_handle_hi = upper_32_bits((uintptr_t)qp);
1296
1297 return 0;
1298}
1299
1300static inline int qedr_init_qp_kernel_sq(struct qedr_dev *dev,
1301 struct qedr_qp *qp,
1302 struct ib_qp_init_attr *attrs)
1303{
1304 u32 n_sq_elems, n_sq_entries;
1305 int rc;
1306
1307 /* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
1308 * the ring. The ring should allow at least a single WR, even if the
1309 * user requested none, due to allocation issues.
1310 */
1311 n_sq_entries = attrs->cap.max_send_wr;
1312 n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
1313 n_sq_entries = max_t(u32, n_sq_entries, 1);
1314 n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
1315 rc = dev->ops->common->chain_alloc(dev->cdev,
1316 QED_CHAIN_USE_TO_PRODUCE,
1317 QED_CHAIN_MODE_PBL,
1318 QED_CHAIN_CNT_TYPE_U32,
1319 n_sq_elems,
1320 QEDR_SQE_ELEMENT_SIZE,
1321 &qp->sq.pbl);
1322 if (rc) {
1323 DP_ERR(dev, "failed to allocate QP %p SQ\n", qp);
1324 return rc;
1325 }
1326
1327 DP_DEBUG(dev, QEDR_MSG_SQ,
1328 "SQ Pbl base addr = %llx max_send_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1329 qed_chain_get_pbl_phys(&qp->sq.pbl), attrs->cap.max_send_wr,
1330 n_sq_entries, qed_chain_get_capacity(&qp->sq.pbl), rc);
1331 return 0;
1332}
1333
1334static inline int qedr_init_qp_kernel_rq(struct qedr_dev *dev,
1335 struct qedr_qp *qp,
1336 struct ib_qp_init_attr *attrs)
1337{
1338 u32 n_rq_elems, n_rq_entries;
1339 int rc;
1340
1341 /* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
1342 * the ring. There ring should allow at least a single WR, even if the
1343 * user requested none, due to allocation issues.
1344 */
1345 n_rq_entries = max_t(u32, attrs->cap.max_recv_wr, 1);
1346 n_rq_elems = n_rq_entries * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
1347 rc = dev->ops->common->chain_alloc(dev->cdev,
1348 QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1349 QED_CHAIN_MODE_PBL,
1350 QED_CHAIN_CNT_TYPE_U32,
1351 n_rq_elems,
1352 QEDR_RQE_ELEMENT_SIZE,
1353 &qp->rq.pbl);
1354
1355 if (rc) {
1356 DP_ERR(dev, "failed to allocate memory for QP %p RQ\n", qp);
1357 return -ENOMEM;
1358 }
1359
1360 DP_DEBUG(dev, QEDR_MSG_RQ,
1361 "RQ Pbl base addr = %llx max_recv_wr=%d max_wr=%d capacity=%d, rc=%d\n",
1362 qed_chain_get_pbl_phys(&qp->rq.pbl), attrs->cap.max_recv_wr,
1363 n_rq_entries, qed_chain_get_capacity(&qp->rq.pbl), rc);
1364
1365 /* n_rq_entries < u16 so the casting is safe */
1366 qp->rq.max_wr = (u16)n_rq_entries;
1367
1368 return 0;
1369}
1370
1371static inline void
1372qedr_init_qp_in_params_sq(struct qedr_dev *dev,
1373 struct qedr_pd *pd,
1374 struct qedr_qp *qp,
1375 struct ib_qp_init_attr *attrs,
1376 struct ib_udata *udata,
1377 struct qed_rdma_create_qp_in_params *params)
1378{
1379 /* QP handle to be written in an async event */
1380 params->qp_handle_async_lo = lower_32_bits((uintptr_t)qp);
1381 params->qp_handle_async_hi = upper_32_bits((uintptr_t)qp);
1382
1383 params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1384 params->fmr_and_reserved_lkey = !udata;
1385 params->pd = pd->pd_id;
1386 params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1387 params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1388 params->max_sq_sges = 0;
1389 params->stats_queue = 0;
1390
1391 if (udata) {
1392 params->sq_num_pages = qp->usq.pbl_info.num_pbes;
1393 params->sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1394 } else {
1395 params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
1396 params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
1397 }
1398}
1399
1400static inline void
1401qedr_init_qp_in_params_rq(struct qedr_qp *qp,
1402 struct ib_qp_init_attr *attrs,
1403 struct ib_udata *udata,
1404 struct qed_rdma_create_qp_in_params *params)
1405{
1406 params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1407 params->srq_id = 0;
1408 params->use_srq = false;
1409
1410 if (udata) {
1411 params->rq_num_pages = qp->urq.pbl_info.num_pbes;
1412 params->rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1413 } else {
1414 params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
1415 params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
1416 }
1417}
1418
1419static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1420{
1421 DP_DEBUG(dev, QEDR_MSG_QP,
1422 "create qp: successfully created user QP. qp=%p, sq_addr=0x%llx, sq_len=%zd, rq_addr=0x%llx, rq_len=%zd\n",
1423 qp, qp->usq.buf_addr, qp->usq.buf_len, qp->urq.buf_addr,
1424 qp->urq.buf_len);
1425}
1426
1427static inline int qedr_init_user_qp(struct ib_ucontext *ib_ctx,
1428 struct qedr_dev *dev,
1429 struct qedr_qp *qp,
1430 struct qedr_create_qp_ureq *ureq)
1431{
1432 int rc;
1433
1434 /* SQ - read access only (0), dma sync not required (0) */
1435 rc = qedr_init_user_queue(ib_ctx, dev, &qp->usq, ureq->sq_addr,
1436 ureq->sq_len, 0, 0);
1437 if (rc)
1438 return rc;
1439
1440 /* RQ - read access only (0), dma sync not required (0) */
1441 rc = qedr_init_user_queue(ib_ctx, dev, &qp->urq, ureq->rq_addr,
1442 ureq->rq_len, 0, 0);
1443
1444 if (rc)
1445 qedr_cleanup_user_sq(dev, qp);
1446 return rc;
1447}
1448
1449static inline int
1450qedr_init_kernel_qp(struct qedr_dev *dev,
1451 struct qedr_qp *qp,
1452 struct ib_qp_init_attr *attrs,
1453 struct qed_rdma_create_qp_in_params *params)
1454{
1455 int rc;
1456
1457 rc = qedr_init_qp_kernel_sq(dev, qp, attrs);
1458 if (rc) {
1459 DP_ERR(dev, "failed to init kernel QP %p SQ\n", qp);
1460 return rc;
1461 }
1462
1463 rc = qedr_init_qp_kernel_params_sq(dev, qp, attrs, params);
1464 if (rc) {
1465 dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
1466 DP_ERR(dev, "failed to init kernel QP %p SQ params\n", qp);
1467 return rc;
1468 }
1469
1470 rc = qedr_init_qp_kernel_rq(dev, qp, attrs);
1471 if (rc) {
1472 qedr_cleanup_kernel_sq(dev, qp);
1473 DP_ERR(dev, "failed to init kernel QP %p RQ\n", qp);
1474 return rc;
1475 }
1476
1477 rc = qedr_init_qp_kernel_params_rq(dev, qp, attrs);
1478 if (rc) {
1479 DP_ERR(dev, "failed to init kernel QP %p RQ params\n", qp);
1480 qedr_cleanup_kernel_sq(dev, qp);
1481 dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
1482 return rc;
1483 }
1484
1485 return rc;
1486}
1487
1488struct ib_qp *qedr_create_qp(struct ib_pd *ibpd,
1489 struct ib_qp_init_attr *attrs,
1490 struct ib_udata *udata)
1491{
1492 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
1493 struct qed_rdma_create_qp_out_params out_params;
1494 struct qed_rdma_create_qp_in_params in_params;
1495 struct qedr_pd *pd = get_qedr_pd(ibpd);
1496 struct ib_ucontext *ib_ctx = NULL;
1497 struct qedr_ucontext *ctx = NULL;
1498 struct qedr_create_qp_ureq ureq;
1499 struct qedr_qp *qp;
1500 struct ib_qp *ibqp;
1501 int rc = 0;
1502
1503 DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
1504 udata ? "user library" : "kernel", pd);
1505
1506 rc = qedr_check_qp_attrs(ibpd, dev, attrs);
1507 if (rc)
1508 return ERR_PTR(rc);
1509
1510 if (attrs->srq)
1511 return ERR_PTR(-EINVAL);
1512
1513 qp = kzalloc(sizeof(*qp), GFP_KERNEL);
1514 if (!qp)
1515 return ERR_PTR(-ENOMEM);
1516
1517 DP_DEBUG(dev, QEDR_MSG_QP,
1518 "create qp: sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
1519 get_qedr_cq(attrs->send_cq),
1520 get_qedr_cq(attrs->send_cq)->icid,
1521 get_qedr_cq(attrs->recv_cq),
1522 get_qedr_cq(attrs->recv_cq)->icid);
1523
1524 qedr_set_qp_init_params(dev, qp, pd, attrs);
1525
1526 if (attrs->qp_type == IB_QPT_GSI) {
1527 if (udata) {
1528 DP_ERR(dev,
1529 "create qp: unexpected udata when creating GSI QP\n");
1530 goto err0;
1531 }
1532 ibqp = qedr_create_gsi_qp(dev, attrs, qp);
1533 if (IS_ERR(ibqp))
1534 kfree(qp);
1535 return ibqp;
1536 }
1537
1538 memset(&in_params, 0, sizeof(in_params));
1539
1540 if (udata) {
1541 if (!(udata && ibpd->uobject && ibpd->uobject->context))
1542 goto err0;
1543
1544 ib_ctx = ibpd->uobject->context;
1545 ctx = get_qedr_ucontext(ib_ctx);
1546
1547 memset(&ureq, 0, sizeof(ureq));
1548 if (ib_copy_from_udata(&ureq, udata, sizeof(ureq))) {
1549 DP_ERR(dev,
1550 "create qp: problem copying data from user space\n");
1551 goto err0;
1552 }
1553
1554 rc = qedr_init_user_qp(ib_ctx, dev, qp, &ureq);
1555 if (rc)
1556 goto err0;
1557
1558 qedr_init_qp_user_params(&in_params, &ureq);
1559 } else {
1560 rc = qedr_init_kernel_qp(dev, qp, attrs, &in_params);
1561 if (rc)
1562 goto err0;
1563 }
1564
1565 qedr_init_qp_in_params_sq(dev, pd, qp, attrs, udata, &in_params);
1566 qedr_init_qp_in_params_rq(qp, attrs, udata, &in_params);
1567
1568 qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1569 &in_params, &out_params);
1570
1571 if (!qp->qed_qp)
1572 goto err1;
1573
1574 qp->qp_id = out_params.qp_id;
1575 qp->icid = out_params.icid;
1576 qp->ibqp.qp_num = qp->qp_id;
1577
1578 if (udata) {
1579 rc = qedr_copy_qp_uresp(dev, qp, udata);
1580 if (rc)
1581 goto err2;
1582
1583 qedr_qp_user_print(dev, qp);
1584 } else {
1585 qedr_init_qp_kernel_doorbell_sq(dev, qp);
1586 qedr_init_qp_kernel_doorbell_rq(dev, qp);
1587 }
1588
1589 DP_DEBUG(dev, QEDR_MSG_QP, "created %s space QP %p\n",
1590 udata ? "user" : "kernel", qp);
1591
1592 return &qp->ibqp;
1593
1594err2:
1595 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1596 if (rc)
1597 DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1598err1:
1599 if (udata) {
1600 qedr_cleanup_user_sq(dev, qp);
1601 qedr_cleanup_user_rq(dev, qp);
1602 } else {
1603 qedr_cleanup_kernel_sq(dev, qp);
1604 qedr_cleanup_kernel_rq(dev, qp);
1605 }
1606
1607err0:
1608 kfree(qp);
1609
1610 return ERR_PTR(-EFAULT);
1611}
1612
1613static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
1614{
1615 switch (qp_state) {
1616 case QED_ROCE_QP_STATE_RESET:
1617 return IB_QPS_RESET;
1618 case QED_ROCE_QP_STATE_INIT:
1619 return IB_QPS_INIT;
1620 case QED_ROCE_QP_STATE_RTR:
1621 return IB_QPS_RTR;
1622 case QED_ROCE_QP_STATE_RTS:
1623 return IB_QPS_RTS;
1624 case QED_ROCE_QP_STATE_SQD:
1625 return IB_QPS_SQD;
1626 case QED_ROCE_QP_STATE_ERR:
1627 return IB_QPS_ERR;
1628 case QED_ROCE_QP_STATE_SQE:
1629 return IB_QPS_SQE;
1630 }
1631 return IB_QPS_ERR;
1632}
1633
1634static enum qed_roce_qp_state qedr_get_state_from_ibqp(
1635 enum ib_qp_state qp_state)
1636{
1637 switch (qp_state) {
1638 case IB_QPS_RESET:
1639 return QED_ROCE_QP_STATE_RESET;
1640 case IB_QPS_INIT:
1641 return QED_ROCE_QP_STATE_INIT;
1642 case IB_QPS_RTR:
1643 return QED_ROCE_QP_STATE_RTR;
1644 case IB_QPS_RTS:
1645 return QED_ROCE_QP_STATE_RTS;
1646 case IB_QPS_SQD:
1647 return QED_ROCE_QP_STATE_SQD;
1648 case IB_QPS_ERR:
1649 return QED_ROCE_QP_STATE_ERR;
1650 default:
1651 return QED_ROCE_QP_STATE_ERR;
1652 }
1653}
1654
1655static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1656{
1657 qed_chain_reset(&qph->pbl);
1658 qph->prod = 0;
1659 qph->cons = 0;
1660 qph->wqe_cons = 0;
1661 qph->db_data.data.value = cpu_to_le16(0);
1662}
1663
1664static int qedr_update_qp_state(struct qedr_dev *dev,
1665 struct qedr_qp *qp,
1666 enum qed_roce_qp_state new_state)
1667{
1668 int status = 0;
1669
1670 if (new_state == qp->state)
1671 return 0;
1672
1673 switch (qp->state) {
1674 case QED_ROCE_QP_STATE_RESET:
1675 switch (new_state) {
1676 case QED_ROCE_QP_STATE_INIT:
1677 qp->prev_wqe_size = 0;
1678 qedr_reset_qp_hwq_info(&qp->sq);
1679 qedr_reset_qp_hwq_info(&qp->rq);
1680 break;
1681 default:
1682 status = -EINVAL;
1683 break;
1684 };
1685 break;
1686 case QED_ROCE_QP_STATE_INIT:
1687 switch (new_state) {
1688 case QED_ROCE_QP_STATE_RTR:
1689 /* Update doorbell (in case post_recv was
1690 * done before move to RTR)
1691 */
1692 wmb();
1693 writel(qp->rq.db_data.raw, qp->rq.db);
1694 /* Make sure write takes effect */
1695 mmiowb();
1696 break;
1697 case QED_ROCE_QP_STATE_ERR:
1698 break;
1699 default:
1700 /* Invalid state change. */
1701 status = -EINVAL;
1702 break;
1703 };
1704 break;
1705 case QED_ROCE_QP_STATE_RTR:
1706 /* RTR->XXX */
1707 switch (new_state) {
1708 case QED_ROCE_QP_STATE_RTS:
1709 break;
1710 case QED_ROCE_QP_STATE_ERR:
1711 break;
1712 default:
1713 /* Invalid state change. */
1714 status = -EINVAL;
1715 break;
1716 };
1717 break;
1718 case QED_ROCE_QP_STATE_RTS:
1719 /* RTS->XXX */
1720 switch (new_state) {
1721 case QED_ROCE_QP_STATE_SQD:
1722 break;
1723 case QED_ROCE_QP_STATE_ERR:
1724 break;
1725 default:
1726 /* Invalid state change. */
1727 status = -EINVAL;
1728 break;
1729 };
1730 break;
1731 case QED_ROCE_QP_STATE_SQD:
1732 /* SQD->XXX */
1733 switch (new_state) {
1734 case QED_ROCE_QP_STATE_RTS:
1735 case QED_ROCE_QP_STATE_ERR:
1736 break;
1737 default:
1738 /* Invalid state change. */
1739 status = -EINVAL;
1740 break;
1741 };
1742 break;
1743 case QED_ROCE_QP_STATE_ERR:
1744 /* ERR->XXX */
1745 switch (new_state) {
1746 case QED_ROCE_QP_STATE_RESET:
1747 if ((qp->rq.prod != qp->rq.cons) ||
1748 (qp->sq.prod != qp->sq.cons)) {
1749 DP_NOTICE(dev,
1750 "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
1751 qp->rq.prod, qp->rq.cons, qp->sq.prod,
1752 qp->sq.cons);
1753 status = -EINVAL;
1754 }
1755 break;
1756 default:
1757 status = -EINVAL;
1758 break;
1759 };
1760 break;
1761 default:
1762 status = -EINVAL;
1763 break;
1764 };
1765
1766 return status;
1767}
1768
1769int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1770 int attr_mask, struct ib_udata *udata)
1771{
1772 struct qedr_qp *qp = get_qedr_qp(ibqp);
1773 struct qed_rdma_modify_qp_in_params qp_params = { 0 };
1774 struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
1775 enum ib_qp_state old_qp_state, new_qp_state;
1776 int rc = 0;
1777
1778 DP_DEBUG(dev, QEDR_MSG_QP,
1779 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
1780 attr->qp_state);
1781
1782 old_qp_state = qedr_get_ibqp_state(qp->state);
1783 if (attr_mask & IB_QP_STATE)
1784 new_qp_state = attr->qp_state;
1785 else
1786 new_qp_state = old_qp_state;
1787
1788 if (!ib_modify_qp_is_ok
1789 (old_qp_state, new_qp_state, ibqp->qp_type, attr_mask,
1790 IB_LINK_LAYER_ETHERNET)) {
1791 DP_ERR(dev,
1792 "modify qp: invalid attribute mask=0x%x specified for\n"
1793 "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
1794 attr_mask, qp->qp_id, ibqp->qp_type, old_qp_state,
1795 new_qp_state);
1796 rc = -EINVAL;
1797 goto err;
1798 }
1799
1800 /* Translate the masks... */
1801 if (attr_mask & IB_QP_STATE) {
1802 SET_FIELD(qp_params.modify_flags,
1803 QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
1804 qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
1805 }
1806
1807 if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
1808 qp_params.sqd_async = true;
1809
1810 if (attr_mask & IB_QP_PKEY_INDEX) {
1811 SET_FIELD(qp_params.modify_flags,
1812 QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
1813 if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
1814 rc = -EINVAL;
1815 goto err;
1816 }
1817
1818 qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
1819 }
1820
1821 if (attr_mask & IB_QP_QKEY)
1822 qp->qkey = attr->qkey;
1823
1824 if (attr_mask & IB_QP_ACCESS_FLAGS) {
1825 SET_FIELD(qp_params.modify_flags,
1826 QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
1827 qp_params.incoming_rdma_read_en = attr->qp_access_flags &
1828 IB_ACCESS_REMOTE_READ;
1829 qp_params.incoming_rdma_write_en = attr->qp_access_flags &
1830 IB_ACCESS_REMOTE_WRITE;
1831 qp_params.incoming_atomic_en = attr->qp_access_flags &
1832 IB_ACCESS_REMOTE_ATOMIC;
1833 }
1834
1835 if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
1836 if (attr_mask & IB_QP_PATH_MTU) {
1837 if (attr->path_mtu < IB_MTU_256 ||
1838 attr->path_mtu > IB_MTU_4096) {
1839 pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
1840 rc = -EINVAL;
1841 goto err;
1842 }
1843 qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
1844 ib_mtu_enum_to_int(iboe_get_mtu
1845 (dev->ndev->mtu)));
1846 }
1847
1848 if (!qp->mtu) {
1849 qp->mtu =
1850 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1851 pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
1852 }
1853
1854 SET_FIELD(qp_params.modify_flags,
1855 QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
1856
1857 qp_params.traffic_class_tos = attr->ah_attr.grh.traffic_class;
1858 qp_params.flow_label = attr->ah_attr.grh.flow_label;
1859 qp_params.hop_limit_ttl = attr->ah_attr.grh.hop_limit;
1860
1861 qp->sgid_idx = attr->ah_attr.grh.sgid_index;
1862
1863 rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
1864 if (rc) {
1865 DP_ERR(dev,
1866 "modify qp: problems with GID index %d (rc=%d)\n",
1867 attr->ah_attr.grh.sgid_index, rc);
1868 return rc;
1869 }
1870
1871 rc = qedr_get_dmac(dev, &attr->ah_attr,
1872 qp_params.remote_mac_addr);
1873 if (rc)
1874 return rc;
1875
1876 qp_params.use_local_mac = true;
1877 ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
1878
1879 DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
1880 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
1881 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
1882 DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
1883 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
1884 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
1885 DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
1886 qp_params.remote_mac_addr);
1887
1888 qp_params.mtu = qp->mtu;
1889 qp_params.lb_indication = false;
1890 }
1891
1892 if (!qp_params.mtu) {
1893 /* Stay with current MTU */
1894 if (qp->mtu)
1895 qp_params.mtu = qp->mtu;
1896 else
1897 qp_params.mtu =
1898 ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
1899 }
1900
1901 if (attr_mask & IB_QP_TIMEOUT) {
1902 SET_FIELD(qp_params.modify_flags,
1903 QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
1904
1905 qp_params.ack_timeout = attr->timeout;
1906 if (attr->timeout) {
1907 u32 temp;
1908
1909 temp = 4096 * (1UL << attr->timeout) / 1000 / 1000;
1910 /* FW requires [msec] */
1911 qp_params.ack_timeout = temp;
1912 } else {
1913 /* Infinite */
1914 qp_params.ack_timeout = 0;
1915 }
1916 }
1917 if (attr_mask & IB_QP_RETRY_CNT) {
1918 SET_FIELD(qp_params.modify_flags,
1919 QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
1920 qp_params.retry_cnt = attr->retry_cnt;
1921 }
1922
1923 if (attr_mask & IB_QP_RNR_RETRY) {
1924 SET_FIELD(qp_params.modify_flags,
1925 QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
1926 qp_params.rnr_retry_cnt = attr->rnr_retry;
1927 }
1928
1929 if (attr_mask & IB_QP_RQ_PSN) {
1930 SET_FIELD(qp_params.modify_flags,
1931 QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
1932 qp_params.rq_psn = attr->rq_psn;
1933 qp->rq_psn = attr->rq_psn;
1934 }
1935
1936 if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1937 if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
1938 rc = -EINVAL;
1939 DP_ERR(dev,
1940 "unsupported max_rd_atomic=%d, supported=%d\n",
1941 attr->max_rd_atomic,
1942 dev->attr.max_qp_req_rd_atomic_resc);
1943 goto err;
1944 }
1945
1946 SET_FIELD(qp_params.modify_flags,
1947 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
1948 qp_params.max_rd_atomic_req = attr->max_rd_atomic;
1949 }
1950
1951 if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1952 SET_FIELD(qp_params.modify_flags,
1953 QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
1954 qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
1955 }
1956
1957 if (attr_mask & IB_QP_SQ_PSN) {
1958 SET_FIELD(qp_params.modify_flags,
1959 QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
1960 qp_params.sq_psn = attr->sq_psn;
1961 qp->sq_psn = attr->sq_psn;
1962 }
1963
1964 if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1965 if (attr->max_dest_rd_atomic >
1966 dev->attr.max_qp_resp_rd_atomic_resc) {
1967 DP_ERR(dev,
1968 "unsupported max_dest_rd_atomic=%d, supported=%d\n",
1969 attr->max_dest_rd_atomic,
1970 dev->attr.max_qp_resp_rd_atomic_resc);
1971
1972 rc = -EINVAL;
1973 goto err;
1974 }
1975
1976 SET_FIELD(qp_params.modify_flags,
1977 QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
1978 qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
1979 }
1980
1981 if (attr_mask & IB_QP_DEST_QPN) {
1982 SET_FIELD(qp_params.modify_flags,
1983 QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
1984
1985 qp_params.dest_qp = attr->dest_qp_num;
1986 qp->dest_qp_num = attr->dest_qp_num;
1987 }
1988
1989 if (qp->qp_type != IB_QPT_GSI)
1990 rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
1991 qp->qed_qp, &qp_params);
1992
1993 if (attr_mask & IB_QP_STATE) {
1994 if ((qp->qp_type != IB_QPT_GSI) && (!udata))
1995 rc = qedr_update_qp_state(dev, qp, qp_params.new_state);
1996 qp->state = qp_params.new_state;
1997 }
1998
1999err:
2000 return rc;
2001}
2002
2003static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2004{
2005 int ib_qp_acc_flags = 0;
2006
2007 if (params->incoming_rdma_write_en)
2008 ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2009 if (params->incoming_rdma_read_en)
2010 ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2011 if (params->incoming_atomic_en)
2012 ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2013 ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2014 return ib_qp_acc_flags;
2015}
2016
2017int qedr_query_qp(struct ib_qp *ibqp,
2018 struct ib_qp_attr *qp_attr,
2019 int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2020{
2021 struct qed_rdma_query_qp_out_params params;
2022 struct qedr_qp *qp = get_qedr_qp(ibqp);
2023 struct qedr_dev *dev = qp->dev;
2024 int rc = 0;
2025
2026 memset(¶ms, 0, sizeof(params));
2027
2028 rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, ¶ms);
2029 if (rc)
2030 goto err;
2031
2032 memset(qp_attr, 0, sizeof(*qp_attr));
2033 memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2034
2035 qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2036 qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2037 qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2038 qp_attr->path_mig_state = IB_MIG_MIGRATED;
2039 qp_attr->rq_psn = params.rq_psn;
2040 qp_attr->sq_psn = params.sq_psn;
2041 qp_attr->dest_qp_num = params.dest_qp;
2042
2043 qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(¶ms);
2044
2045 qp_attr->cap.max_send_wr = qp->sq.max_wr;
2046 qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2047 qp_attr->cap.max_send_sge = qp->sq.max_sges;
2048 qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2049 qp_attr->cap.max_inline_data = ROCE_REQ_MAX_INLINE_DATA_SIZE;
2050 qp_init_attr->cap = qp_attr->cap;
2051
2052 memcpy(&qp_attr->ah_attr.grh.dgid.raw[0], ¶ms.dgid.bytes[0],
2053 sizeof(qp_attr->ah_attr.grh.dgid.raw));
2054
2055 qp_attr->ah_attr.grh.flow_label = params.flow_label;
2056 qp_attr->ah_attr.grh.sgid_index = qp->sgid_idx;
2057 qp_attr->ah_attr.grh.hop_limit = params.hop_limit_ttl;
2058 qp_attr->ah_attr.grh.traffic_class = params.traffic_class_tos;
2059
2060 qp_attr->ah_attr.ah_flags = IB_AH_GRH;
2061 qp_attr->ah_attr.port_num = 1;
2062 qp_attr->ah_attr.sl = 0;
2063 qp_attr->timeout = params.timeout;
2064 qp_attr->rnr_retry = params.rnr_retry;
2065 qp_attr->retry_cnt = params.retry_cnt;
2066 qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2067 qp_attr->pkey_index = params.pkey_index;
2068 qp_attr->port_num = 1;
2069 qp_attr->ah_attr.src_path_bits = 0;
2070 qp_attr->ah_attr.static_rate = 0;
2071 qp_attr->alt_pkey_index = 0;
2072 qp_attr->alt_port_num = 0;
2073 qp_attr->alt_timeout = 0;
2074 memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2075
2076 qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2077 qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2078 qp_attr->max_rd_atomic = params.max_rd_atomic;
2079 qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2080
2081 DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2082 qp_attr->cap.max_inline_data);
2083
2084err:
2085 return rc;
2086}
2087
2088int qedr_destroy_qp(struct ib_qp *ibqp)
2089{
2090 struct qedr_qp *qp = get_qedr_qp(ibqp);
2091 struct qedr_dev *dev = qp->dev;
2092 struct ib_qp_attr attr;
2093 int attr_mask = 0;
2094 int rc = 0;
2095
2096 DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2097 qp, qp->qp_type);
2098
2099 if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2100 (qp->state != QED_ROCE_QP_STATE_ERR) &&
2101 (qp->state != QED_ROCE_QP_STATE_INIT)) {
2102
2103 attr.qp_state = IB_QPS_ERR;
2104 attr_mask |= IB_QP_STATE;
2105
2106 /* Change the QP state to ERROR */
2107 qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2108 }
2109
2110 if (qp->qp_type != IB_QPT_GSI) {
2111 rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2112 if (rc)
2113 return rc;
2114 } else {
2115 qedr_destroy_gsi_qp(dev);
2116 }
2117
2118 if (ibqp->uobject && ibqp->uobject->context) {
2119 qedr_cleanup_user_sq(dev, qp);
2120 qedr_cleanup_user_rq(dev, qp);
2121 } else {
2122 qedr_cleanup_kernel_sq(dev, qp);
2123 qedr_cleanup_kernel_rq(dev, qp);
2124 }
2125
2126 kfree(qp);
2127
2128 return rc;
2129}
2130
2131struct ib_ah *qedr_create_ah(struct ib_pd *ibpd, struct ib_ah_attr *attr,
2132 struct ib_udata *udata)
2133{
2134 struct qedr_ah *ah;
2135
2136 ah = kzalloc(sizeof(*ah), GFP_ATOMIC);
2137 if (!ah)
2138 return ERR_PTR(-ENOMEM);
2139
2140 ah->attr = *attr;
2141
2142 return &ah->ibah;
2143}
2144
2145int qedr_destroy_ah(struct ib_ah *ibah)
2146{
2147 struct qedr_ah *ah = get_qedr_ah(ibah);
2148
2149 kfree(ah);
2150 return 0;
2151}
2152
2153static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2154{
2155 struct qedr_pbl *pbl, *tmp;
2156
2157 if (info->pbl_table)
2158 list_add_tail(&info->pbl_table->list_entry,
2159 &info->free_pbl_list);
2160
2161 if (!list_empty(&info->inuse_pbl_list))
2162 list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2163
2164 list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2165 list_del(&pbl->list_entry);
2166 qedr_free_pbl(dev, &info->pbl_info, pbl);
2167 }
2168}
2169
2170static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2171 size_t page_list_len, bool two_layered)
2172{
2173 struct qedr_pbl *tmp;
2174 int rc;
2175
2176 INIT_LIST_HEAD(&info->free_pbl_list);
2177 INIT_LIST_HEAD(&info->inuse_pbl_list);
2178
2179 rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2180 page_list_len, two_layered);
2181 if (rc)
2182 goto done;
2183
2184 info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2185 if (!info->pbl_table) {
2186 rc = -ENOMEM;
2187 goto done;
2188 }
2189
2190 DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2191 &info->pbl_table->pa);
2192
2193 /* in usual case we use 2 PBLs, so we add one to free
2194 * list and allocating another one
2195 */
2196 tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2197 if (!tmp) {
2198 DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2199 goto done;
2200 }
2201
2202 list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2203
2204 DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2205
2206done:
2207 if (rc)
2208 free_mr_info(dev, info);
2209
2210 return rc;
2211}
2212
2213struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2214 u64 usr_addr, int acc, struct ib_udata *udata)
2215{
2216 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2217 struct qedr_mr *mr;
2218 struct qedr_pd *pd;
2219 int rc = -ENOMEM;
2220
2221 pd = get_qedr_pd(ibpd);
2222 DP_DEBUG(dev, QEDR_MSG_MR,
2223 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2224 pd->pd_id, start, len, usr_addr, acc);
2225
2226 if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2227 return ERR_PTR(-EINVAL);
2228
2229 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2230 if (!mr)
2231 return ERR_PTR(rc);
2232
2233 mr->type = QEDR_MR_USER;
2234
2235 mr->umem = ib_umem_get(ibpd->uobject->context, start, len, acc, 0);
2236 if (IS_ERR(mr->umem)) {
2237 rc = -EFAULT;
2238 goto err0;
2239 }
2240
2241 rc = init_mr_info(dev, &mr->info, ib_umem_page_count(mr->umem), 1);
2242 if (rc)
2243 goto err1;
2244
2245 qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2246 &mr->info.pbl_info);
2247
2248 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2249 if (rc) {
2250 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2251 goto err1;
2252 }
2253
2254 /* Index only, 18 bit long, lkey = itid << 8 | key */
2255 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2256 mr->hw_mr.key = 0;
2257 mr->hw_mr.pd = pd->pd_id;
2258 mr->hw_mr.local_read = 1;
2259 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2260 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2261 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2262 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2263 mr->hw_mr.mw_bind = false;
2264 mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
2265 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2266 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2267 mr->hw_mr.page_size_log = ilog2(mr->umem->page_size);
2268 mr->hw_mr.fbo = ib_umem_offset(mr->umem);
2269 mr->hw_mr.length = len;
2270 mr->hw_mr.vaddr = usr_addr;
2271 mr->hw_mr.zbva = false;
2272 mr->hw_mr.phy_mr = false;
2273 mr->hw_mr.dma_mr = false;
2274
2275 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2276 if (rc) {
2277 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2278 goto err2;
2279 }
2280
2281 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2282 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2283 mr->hw_mr.remote_atomic)
2284 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2285
2286 DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
2287 mr->ibmr.lkey);
2288 return &mr->ibmr;
2289
2290err2:
2291 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2292err1:
2293 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2294err0:
2295 kfree(mr);
2296 return ERR_PTR(rc);
2297}
2298
2299int qedr_dereg_mr(struct ib_mr *ib_mr)
2300{
2301 struct qedr_mr *mr = get_qedr_mr(ib_mr);
2302 struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
2303 int rc = 0;
2304
2305 rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
2306 if (rc)
2307 return rc;
2308
2309 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2310
2311 if ((mr->type != QEDR_MR_DMA) && (mr->type != QEDR_MR_FRMR))
2312 qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
2313
2314 /* it could be user registered memory. */
2315 if (mr->umem)
2316 ib_umem_release(mr->umem);
2317
2318 kfree(mr);
2319
2320 return rc;
2321}
2322
2323static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
2324 int max_page_list_len)
2325{
2326 struct qedr_pd *pd = get_qedr_pd(ibpd);
2327 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2328 struct qedr_mr *mr;
2329 int rc = -ENOMEM;
2330
2331 DP_DEBUG(dev, QEDR_MSG_MR,
2332 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
2333 max_page_list_len);
2334
2335 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2336 if (!mr)
2337 return ERR_PTR(rc);
2338
2339 mr->dev = dev;
2340 mr->type = QEDR_MR_FRMR;
2341
2342 rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
2343 if (rc)
2344 goto err0;
2345
2346 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2347 if (rc) {
2348 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2349 goto err0;
2350 }
2351
2352 /* Index only, 18 bit long, lkey = itid << 8 | key */
2353 mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
2354 mr->hw_mr.key = 0;
2355 mr->hw_mr.pd = pd->pd_id;
2356 mr->hw_mr.local_read = 1;
2357 mr->hw_mr.local_write = 0;
2358 mr->hw_mr.remote_read = 0;
2359 mr->hw_mr.remote_write = 0;
2360 mr->hw_mr.remote_atomic = 0;
2361 mr->hw_mr.mw_bind = false;
2362 mr->hw_mr.pbl_ptr = 0;
2363 mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
2364 mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
2365 mr->hw_mr.fbo = 0;
2366 mr->hw_mr.length = 0;
2367 mr->hw_mr.vaddr = 0;
2368 mr->hw_mr.zbva = false;
2369 mr->hw_mr.phy_mr = true;
2370 mr->hw_mr.dma_mr = false;
2371
2372 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2373 if (rc) {
2374 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2375 goto err1;
2376 }
2377
2378 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2379 mr->ibmr.rkey = mr->ibmr.lkey;
2380
2381 DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
2382 return mr;
2383
2384err1:
2385 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2386err0:
2387 kfree(mr);
2388 return ERR_PTR(rc);
2389}
2390
2391struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd,
2392 enum ib_mr_type mr_type, u32 max_num_sg)
2393{
2394 struct qedr_dev *dev;
2395 struct qedr_mr *mr;
2396
2397 if (mr_type != IB_MR_TYPE_MEM_REG)
2398 return ERR_PTR(-EINVAL);
2399
2400 mr = __qedr_alloc_mr(ibpd, max_num_sg);
2401
2402 if (IS_ERR(mr))
2403 return ERR_PTR(-EINVAL);
2404
2405 dev = mr->dev;
2406
2407 return &mr->ibmr;
2408}
2409
2410static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
2411{
2412 struct qedr_mr *mr = get_qedr_mr(ibmr);
2413 struct qedr_pbl *pbl_table;
2414 struct regpair *pbe;
2415 u32 pbes_in_page;
2416
2417 if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
2418 DP_ERR(mr->dev, "qedr_set_page failes when %d\n", mr->npages);
2419 return -ENOMEM;
2420 }
2421
2422 DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
2423 mr->npages, addr);
2424
2425 pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
2426 pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
2427 pbe = (struct regpair *)pbl_table->va;
2428 pbe += mr->npages % pbes_in_page;
2429 pbe->lo = cpu_to_le32((u32)addr);
2430 pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
2431
2432 mr->npages++;
2433
2434 return 0;
2435}
2436
2437static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
2438{
2439 int work = info->completed - info->completed_handled - 1;
2440
2441 DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
2442 while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
2443 struct qedr_pbl *pbl;
2444
2445 /* Free all the page list that are possible to be freed
2446 * (all the ones that were invalidated), under the assumption
2447 * that if an FMR was completed successfully that means that
2448 * if there was an invalidate operation before it also ended
2449 */
2450 pbl = list_first_entry(&info->inuse_pbl_list,
2451 struct qedr_pbl, list_entry);
2452 list_move_tail(&pbl->list_entry, &info->free_pbl_list);
2453 info->completed_handled++;
2454 }
2455}
2456
2457int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
2458 int sg_nents, unsigned int *sg_offset)
2459{
2460 struct qedr_mr *mr = get_qedr_mr(ibmr);
2461
2462 mr->npages = 0;
2463
2464 handle_completed_mrs(mr->dev, &mr->info);
2465 return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
2466}
2467
2468struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
2469{
2470 struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2471 struct qedr_pd *pd = get_qedr_pd(ibpd);
2472 struct qedr_mr *mr;
2473 int rc;
2474
2475 mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2476 if (!mr)
2477 return ERR_PTR(-ENOMEM);
2478
2479 mr->type = QEDR_MR_DMA;
2480
2481 rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2482 if (rc) {
2483 DP_ERR(dev, "roce alloc tid returned an error %d\n", rc);
2484 goto err1;
2485 }
2486
2487 /* index only, 18 bit long, lkey = itid << 8 | key */
2488 mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2489 mr->hw_mr.pd = pd->pd_id;
2490 mr->hw_mr.local_read = 1;
2491 mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2492 mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2493 mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2494 mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
2495 mr->hw_mr.dma_mr = true;
2496
2497 rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
2498 if (rc) {
2499 DP_ERR(dev, "roce register tid returned an error %d\n", rc);
2500 goto err2;
2501 }
2502
2503 mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2504 if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
2505 mr->hw_mr.remote_atomic)
2506 mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
2507
2508 DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
2509 return &mr->ibmr;
2510
2511err2:
2512 dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
2513err1:
2514 kfree(mr);
2515 return ERR_PTR(rc);
2516}
2517
2518static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
2519{
2520 return (((wq->prod + 1) % wq->max_wr) == wq->cons);
2521}
2522
2523static int sge_data_len(struct ib_sge *sg_list, int num_sge)
2524{
2525 int i, len = 0;
2526
2527 for (i = 0; i < num_sge; i++)
2528 len += sg_list[i].length;
2529
2530 return len;
2531}
2532
2533static void swap_wqe_data64(u64 *p)
2534{
2535 int i;
2536
2537 for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
2538 *p = cpu_to_be64(cpu_to_le64(*p));
2539}
2540
2541static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
2542 struct qedr_qp *qp, u8 *wqe_size,
2543 struct ib_send_wr *wr,
2544 struct ib_send_wr **bad_wr, u8 *bits,
2545 u8 bit)
2546{
2547 u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
2548 char *seg_prt, *wqe;
2549 int i, seg_siz;
2550
2551 if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
2552 DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
2553 *bad_wr = wr;
2554 return 0;
2555 }
2556
2557 if (!data_size)
2558 return data_size;
2559
2560 *bits |= bit;
2561
2562 seg_prt = NULL;
2563 wqe = NULL;
2564 seg_siz = 0;
2565
2566 /* Copy data inline */
2567 for (i = 0; i < wr->num_sge; i++) {
2568 u32 len = wr->sg_list[i].length;
2569 void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
2570
2571 while (len > 0) {
2572 u32 cur;
2573
2574 /* New segment required */
2575 if (!seg_siz) {
2576 wqe = (char *)qed_chain_produce(&qp->sq.pbl);
2577 seg_prt = wqe;
2578 seg_siz = sizeof(struct rdma_sq_common_wqe);
2579 (*wqe_size)++;
2580 }
2581
2582 /* Calculate currently allowed length */
2583 cur = min_t(u32, len, seg_siz);
2584 memcpy(seg_prt, src, cur);
2585
2586 /* Update segment variables */
2587 seg_prt += cur;
2588 seg_siz -= cur;
2589
2590 /* Update sge variables */
2591 src += cur;
2592 len -= cur;
2593
2594 /* Swap fully-completed segments */
2595 if (!seg_siz)
2596 swap_wqe_data64((u64 *)wqe);
2597 }
2598 }
2599
2600 /* swap last not completed segment */
2601 if (seg_siz)
2602 swap_wqe_data64((u64 *)wqe);
2603
2604 return data_size;
2605}
2606
2607#define RQ_SGE_SET(sge, vaddr, vlength, vflags) \
2608 do { \
2609 DMA_REGPAIR_LE(sge->addr, vaddr); \
2610 (sge)->length = cpu_to_le32(vlength); \
2611 (sge)->flags = cpu_to_le32(vflags); \
2612 } while (0)
2613
2614#define SRQ_HDR_SET(hdr, vwr_id, num_sge) \
2615 do { \
2616 DMA_REGPAIR_LE(hdr->wr_id, vwr_id); \
2617 (hdr)->num_sges = num_sge; \
2618 } while (0)
2619
2620#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey) \
2621 do { \
2622 DMA_REGPAIR_LE(sge->addr, vaddr); \
2623 (sge)->length = cpu_to_le32(vlength); \
2624 (sge)->l_key = cpu_to_le32(vlkey); \
2625 } while (0)
2626
2627static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
2628 struct ib_send_wr *wr)
2629{
2630 u32 data_size = 0;
2631 int i;
2632
2633 for (i = 0; i < wr->num_sge; i++) {
2634 struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
2635
2636 DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
2637 sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
2638 sge->length = cpu_to_le32(wr->sg_list[i].length);
2639 data_size += wr->sg_list[i].length;
2640 }
2641
2642 if (wqe_size)
2643 *wqe_size += wr->num_sge;
2644
2645 return data_size;
2646}
2647
2648static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
2649 struct qedr_qp *qp,
2650 struct rdma_sq_rdma_wqe_1st *rwqe,
2651 struct rdma_sq_rdma_wqe_2nd *rwqe2,
2652 struct ib_send_wr *wr,
2653 struct ib_send_wr **bad_wr)
2654{
2655 rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
2656 DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
2657
2658 if (wr->send_flags & IB_SEND_INLINE &&
2659 (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
2660 wr->opcode == IB_WR_RDMA_WRITE)) {
2661 u8 flags = 0;
2662
2663 SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
2664 return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
2665 bad_wr, &rwqe->flags, flags);
2666 }
2667
2668 return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
2669}
2670
2671static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
2672 struct qedr_qp *qp,
2673 struct rdma_sq_send_wqe_1st *swqe,
2674 struct rdma_sq_send_wqe_2st *swqe2,
2675 struct ib_send_wr *wr,
2676 struct ib_send_wr **bad_wr)
2677{
2678 memset(swqe2, 0, sizeof(*swqe2));
2679 if (wr->send_flags & IB_SEND_INLINE) {
2680 u8 flags = 0;
2681
2682 SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
2683 return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
2684 bad_wr, &swqe->flags, flags);
2685 }
2686
2687 return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
2688}
2689
2690static int qedr_prepare_reg(struct qedr_qp *qp,
2691 struct rdma_sq_fmr_wqe_1st *fwqe1,
2692 struct ib_reg_wr *wr)
2693{
2694 struct qedr_mr *mr = get_qedr_mr(wr->mr);
2695 struct rdma_sq_fmr_wqe_2nd *fwqe2;
2696
2697 fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
2698 fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
2699 fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
2700 fwqe1->l_key = wr->key;
2701
2702 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
2703 !!(wr->access & IB_ACCESS_REMOTE_READ));
2704 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
2705 !!(wr->access & IB_ACCESS_REMOTE_WRITE));
2706 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
2707 !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
2708 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
2709 SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
2710 !!(wr->access & IB_ACCESS_LOCAL_WRITE));
2711 fwqe2->fmr_ctrl = 0;
2712
2713 SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
2714 ilog2(mr->ibmr.page_size) - 12);
2715
2716 fwqe2->length_hi = 0;
2717 fwqe2->length_lo = mr->ibmr.length;
2718 fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
2719 fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
2720
2721 qp->wqe_wr_id[qp->sq.prod].mr = mr;
2722
2723 return 0;
2724}
2725
2726static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
2727{
2728 switch (opcode) {
2729 case IB_WR_RDMA_WRITE:
2730 case IB_WR_RDMA_WRITE_WITH_IMM:
2731 return IB_WC_RDMA_WRITE;
2732 case IB_WR_SEND_WITH_IMM:
2733 case IB_WR_SEND:
2734 case IB_WR_SEND_WITH_INV:
2735 return IB_WC_SEND;
2736 case IB_WR_RDMA_READ:
2737 return IB_WC_RDMA_READ;
2738 case IB_WR_ATOMIC_CMP_AND_SWP:
2739 return IB_WC_COMP_SWAP;
2740 case IB_WR_ATOMIC_FETCH_AND_ADD:
2741 return IB_WC_FETCH_ADD;
2742 case IB_WR_REG_MR:
2743 return IB_WC_REG_MR;
2744 case IB_WR_LOCAL_INV:
2745 return IB_WC_LOCAL_INV;
2746 default:
2747 return IB_WC_SEND;
2748 }
2749}
2750
2751static inline bool qedr_can_post_send(struct qedr_qp *qp, struct ib_send_wr *wr)
2752{
2753 int wq_is_full, err_wr, pbl_is_full;
2754 struct qedr_dev *dev = qp->dev;
2755
2756 /* prevent SQ overflow and/or processing of a bad WR */
2757 err_wr = wr->num_sge > qp->sq.max_sges;
2758 wq_is_full = qedr_wq_is_full(&qp->sq);
2759 pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
2760 QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2761 if (wq_is_full || err_wr || pbl_is_full) {
2762 if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
2763 DP_ERR(dev,
2764 "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
2765 qp);
2766 qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
2767 }
2768
2769 if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
2770 DP_ERR(dev,
2771 "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
2772 qp);
2773 qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
2774 }
2775
2776 if (pbl_is_full &&
2777 !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
2778 DP_ERR(dev,
2779 "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
2780 qp);
2781 qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
2782 }
2783 return false;
2784 }
2785 return true;
2786}
2787
2788static int __qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2789 struct ib_send_wr **bad_wr)
2790{
2791 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2792 struct qedr_qp *qp = get_qedr_qp(ibqp);
2793 struct rdma_sq_atomic_wqe_1st *awqe1;
2794 struct rdma_sq_atomic_wqe_2nd *awqe2;
2795 struct rdma_sq_atomic_wqe_3rd *awqe3;
2796 struct rdma_sq_send_wqe_2st *swqe2;
2797 struct rdma_sq_local_inv_wqe *iwqe;
2798 struct rdma_sq_rdma_wqe_2nd *rwqe2;
2799 struct rdma_sq_send_wqe_1st *swqe;
2800 struct rdma_sq_rdma_wqe_1st *rwqe;
2801 struct rdma_sq_fmr_wqe_1st *fwqe1;
2802 struct rdma_sq_common_wqe *wqe;
2803 u32 length;
2804 int rc = 0;
2805 bool comp;
2806
2807 if (!qedr_can_post_send(qp, wr)) {
2808 *bad_wr = wr;
2809 return -ENOMEM;
2810 }
2811
2812 wqe = qed_chain_produce(&qp->sq.pbl);
2813 qp->wqe_wr_id[qp->sq.prod].signaled =
2814 !!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
2815
2816 wqe->flags = 0;
2817 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
2818 !!(wr->send_flags & IB_SEND_SOLICITED));
2819 comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
2820 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
2821 SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
2822 !!(wr->send_flags & IB_SEND_FENCE));
2823 wqe->prev_wqe_size = qp->prev_wqe_size;
2824
2825 qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
2826
2827 switch (wr->opcode) {
2828 case IB_WR_SEND_WITH_IMM:
2829 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
2830 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2831 swqe->wqe_size = 2;
2832 swqe2 = qed_chain_produce(&qp->sq.pbl);
2833
2834 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.imm_data);
2835 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2836 wr, bad_wr);
2837 swqe->length = cpu_to_le32(length);
2838 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2839 qp->prev_wqe_size = swqe->wqe_size;
2840 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2841 break;
2842 case IB_WR_SEND:
2843 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
2844 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2845
2846 swqe->wqe_size = 2;
2847 swqe2 = qed_chain_produce(&qp->sq.pbl);
2848 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2849 wr, bad_wr);
2850 swqe->length = cpu_to_le32(length);
2851 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2852 qp->prev_wqe_size = swqe->wqe_size;
2853 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2854 break;
2855 case IB_WR_SEND_WITH_INV:
2856 wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
2857 swqe = (struct rdma_sq_send_wqe_1st *)wqe;
2858 swqe2 = qed_chain_produce(&qp->sq.pbl);
2859 swqe->wqe_size = 2;
2860 swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
2861 length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
2862 wr, bad_wr);
2863 swqe->length = cpu_to_le32(length);
2864 qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
2865 qp->prev_wqe_size = swqe->wqe_size;
2866 qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
2867 break;
2868
2869 case IB_WR_RDMA_WRITE_WITH_IMM:
2870 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
2871 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2872
2873 rwqe->wqe_size = 2;
2874 rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
2875 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2876 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2877 wr, bad_wr);
2878 rwqe->length = cpu_to_le32(length);
2879 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2880 qp->prev_wqe_size = rwqe->wqe_size;
2881 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2882 break;
2883 case IB_WR_RDMA_WRITE:
2884 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
2885 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2886
2887 rwqe->wqe_size = 2;
2888 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2889 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2890 wr, bad_wr);
2891 rwqe->length = cpu_to_le32(length);
2892 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2893 qp->prev_wqe_size = rwqe->wqe_size;
2894 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2895 break;
2896 case IB_WR_RDMA_READ_WITH_INV:
2897 DP_ERR(dev,
2898 "RDMA READ WITH INVALIDATE not supported\n");
2899 *bad_wr = wr;
2900 rc = -EINVAL;
2901 break;
2902
2903 case IB_WR_RDMA_READ:
2904 wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
2905 rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
2906
2907 rwqe->wqe_size = 2;
2908 rwqe2 = qed_chain_produce(&qp->sq.pbl);
2909 length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
2910 wr, bad_wr);
2911 rwqe->length = cpu_to_le32(length);
2912 qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
2913 qp->prev_wqe_size = rwqe->wqe_size;
2914 qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
2915 break;
2916
2917 case IB_WR_ATOMIC_CMP_AND_SWP:
2918 case IB_WR_ATOMIC_FETCH_AND_ADD:
2919 awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
2920 awqe1->wqe_size = 4;
2921
2922 awqe2 = qed_chain_produce(&qp->sq.pbl);
2923 DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
2924 awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
2925
2926 awqe3 = qed_chain_produce(&qp->sq.pbl);
2927
2928 if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
2929 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
2930 DMA_REGPAIR_LE(awqe3->swap_data,
2931 atomic_wr(wr)->compare_add);
2932 } else {
2933 wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
2934 DMA_REGPAIR_LE(awqe3->swap_data,
2935 atomic_wr(wr)->swap);
2936 DMA_REGPAIR_LE(awqe3->cmp_data,
2937 atomic_wr(wr)->compare_add);
2938 }
2939
2940 qedr_prepare_sq_sges(qp, NULL, wr);
2941
2942 qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
2943 qp->prev_wqe_size = awqe1->wqe_size;
2944 break;
2945
2946 case IB_WR_LOCAL_INV:
2947 iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
2948 iwqe->wqe_size = 1;
2949
2950 iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
2951 iwqe->inv_l_key = wr->ex.invalidate_rkey;
2952 qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
2953 qp->prev_wqe_size = iwqe->wqe_size;
2954 break;
2955 case IB_WR_REG_MR:
2956 DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
2957 wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
2958 fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
2959 fwqe1->wqe_size = 2;
2960
2961 rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
2962 if (rc) {
2963 DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
2964 *bad_wr = wr;
2965 break;
2966 }
2967
2968 qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
2969 qp->prev_wqe_size = fwqe1->wqe_size;
2970 break;
2971 default:
2972 DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
2973 rc = -EINVAL;
2974 *bad_wr = wr;
2975 break;
2976 }
2977
2978 if (*bad_wr) {
2979 u16 value;
2980
2981 /* Restore prod to its position before
2982 * this WR was processed
2983 */
2984 value = le16_to_cpu(qp->sq.db_data.data.value);
2985 qed_chain_set_prod(&qp->sq.pbl, value, wqe);
2986
2987 /* Restore prev_wqe_size */
2988 qp->prev_wqe_size = wqe->prev_wqe_size;
2989 rc = -EINVAL;
2990 DP_ERR(dev, "POST SEND FAILED\n");
2991 }
2992
2993 return rc;
2994}
2995
2996int qedr_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
2997 struct ib_send_wr **bad_wr)
2998{
2999 struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3000 struct qedr_qp *qp = get_qedr_qp(ibqp);
3001 unsigned long flags;
3002 int rc = 0;
3003
3004 *bad_wr = NULL;
3005
3006 if (qp->qp_type == IB_QPT_GSI)
3007 return qedr_gsi_post_send(ibqp, wr, bad_wr);
3008
3009 spin_lock_irqsave(&qp->q_lock, flags);
3010
3011 if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3012 (qp->state != QED_ROCE_QP_STATE_ERR) &&
3013 (qp->state != QED_ROCE_QP_STATE_SQD)) {
3014 spin_unlock_irqrestore(&qp->q_lock, flags);
3015 *bad_wr = wr;
3016 DP_DEBUG(dev, QEDR_MSG_CQ,
3017 "QP in wrong state! QP icid=0x%x state %d\n",
3018 qp->icid, qp->state);
3019 return -EINVAL;
3020 }
3021
3022 while (wr) {
3023 rc = __qedr_post_send(ibqp, wr, bad_wr);
3024 if (rc)
3025 break;
3026
3027 qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3028
3029 qedr_inc_sw_prod(&qp->sq);
3030
3031 qp->sq.db_data.data.value++;
3032
3033 wr = wr->next;
3034 }
3035
3036 /* Trigger doorbell
3037 * If there was a failure in the first WR then it will be triggered in
3038 * vane. However this is not harmful (as long as the producer value is
3039 * unchanged). For performance reasons we avoid checking for this
3040 * redundant doorbell.
3041 */
3042 wmb();
3043 writel(qp->sq.db_data.raw, qp->sq.db);
3044
3045 /* Make sure write sticks */
3046 mmiowb();
3047
3048 spin_unlock_irqrestore(&qp->q_lock, flags);
3049
3050 return rc;
3051}
3052
3053int qedr_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
3054 struct ib_recv_wr **bad_wr)
3055{
3056 struct qedr_qp *qp = get_qedr_qp(ibqp);
3057 struct qedr_dev *dev = qp->dev;
3058 unsigned long flags;
3059 int status = 0;
3060
3061 if (qp->qp_type == IB_QPT_GSI)
3062 return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3063
3064 spin_lock_irqsave(&qp->q_lock, flags);
3065
3066 if (qp->state == QED_ROCE_QP_STATE_RESET) {
3067 spin_unlock_irqrestore(&qp->q_lock, flags);
3068 *bad_wr = wr;
3069 return -EINVAL;
3070 }
3071
3072 while (wr) {
3073 int i;
3074
3075 if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3076 QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3077 wr->num_sge > qp->rq.max_sges) {
3078 DP_ERR(dev, "Can't post WR (%d < %d) || (%d > %d)\n",
3079 qed_chain_get_elem_left_u32(&qp->rq.pbl),
3080 QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3081 qp->rq.max_sges);
3082 status = -ENOMEM;
3083 *bad_wr = wr;
3084 break;
3085 }
3086 for (i = 0; i < wr->num_sge; i++) {
3087 u32 flags = 0;
3088 struct rdma_rq_sge *rqe =
3089 qed_chain_produce(&qp->rq.pbl);
3090
3091 /* First one must include the number
3092 * of SGE in the list
3093 */
3094 if (!i)
3095 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3096 wr->num_sge);
3097
3098 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY,
3099 wr->sg_list[i].lkey);
3100
3101 RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3102 wr->sg_list[i].length, flags);
3103 }
3104
3105 /* Special case of no sges. FW requires between 1-4 sges...
3106 * in this case we need to post 1 sge with length zero. this is
3107 * because rdma write with immediate consumes an RQ.
3108 */
3109 if (!wr->num_sge) {
3110 u32 flags = 0;
3111 struct rdma_rq_sge *rqe =
3112 qed_chain_produce(&qp->rq.pbl);
3113
3114 /* First one must include the number
3115 * of SGE in the list
3116 */
3117 SET_FIELD(flags, RDMA_RQ_SGE_L_KEY, 0);
3118 SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3119
3120 RQ_SGE_SET(rqe, 0, 0, flags);
3121 i = 1;
3122 }
3123
3124 qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3125 qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3126
3127 qedr_inc_sw_prod(&qp->rq);
3128
3129 /* Flush all the writes before signalling doorbell */
3130 wmb();
3131
3132 qp->rq.db_data.data.value++;
3133
3134 writel(qp->rq.db_data.raw, qp->rq.db);
3135
3136 /* Make sure write sticks */
3137 mmiowb();
3138
3139 wr = wr->next;
3140 }
3141
3142 spin_unlock_irqrestore(&qp->q_lock, flags);
3143
3144 return status;
3145}
3146
3147static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3148{
3149 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3150
3151 return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
3152 cq->pbl_toggle;
3153}
3154
3155static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
3156{
3157 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3158 struct qedr_qp *qp;
3159
3160 qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
3161 resp_cqe->qp_handle.lo,
3162 u64);
3163 return qp;
3164}
3165
3166static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
3167{
3168 struct rdma_cqe_requester *resp_cqe = &cqe->req;
3169
3170 return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
3171}
3172
3173/* Return latest CQE (needs processing) */
3174static union rdma_cqe *get_cqe(struct qedr_cq *cq)
3175{
3176 return cq->latest_cqe;
3177}
3178
3179/* In fmr we need to increase the number of fmr completed counter for the fmr
3180 * algorithm determining whether we can free a pbl or not.
3181 * we need to perform this whether the work request was signaled or not. for
3182 * this purpose we call this function from the condition that checks if a wr
3183 * should be skipped, to make sure we don't miss it ( possibly this fmr
3184 * operation was not signalted)
3185 */
3186static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
3187{
3188 if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
3189 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3190}
3191
3192static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
3193 struct qedr_cq *cq, int num_entries,
3194 struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
3195 int force)
3196{
3197 u16 cnt = 0;
3198
3199 while (num_entries && qp->sq.wqe_cons != hw_cons) {
3200 if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
3201 qedr_chk_if_fmr(qp);
3202 /* skip WC */
3203 goto next_cqe;
3204 }
3205
3206 /* fill WC */
3207 wc->status = status;
3208 wc->vendor_err = 0;
3209 wc->wc_flags = 0;
3210 wc->src_qp = qp->id;
3211 wc->qp = &qp->ibqp;
3212
3213 wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
3214 wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
3215
3216 switch (wc->opcode) {
3217 case IB_WC_RDMA_WRITE:
3218 wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
3219 break;
3220 case IB_WC_COMP_SWAP:
3221 case IB_WC_FETCH_ADD:
3222 wc->byte_len = 8;
3223 break;
3224 case IB_WC_REG_MR:
3225 qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
3226 break;
3227 default:
3228 break;
3229 }
3230
3231 num_entries--;
3232 wc++;
3233 cnt++;
3234next_cqe:
3235 while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
3236 qed_chain_consume(&qp->sq.pbl);
3237 qedr_inc_sw_cons(&qp->sq);
3238 }
3239
3240 return cnt;
3241}
3242
3243static int qedr_poll_cq_req(struct qedr_dev *dev,
3244 struct qedr_qp *qp, struct qedr_cq *cq,
3245 int num_entries, struct ib_wc *wc,
3246 struct rdma_cqe_requester *req)
3247{
3248 int cnt = 0;
3249
3250 switch (req->status) {
3251 case RDMA_CQE_REQ_STS_OK:
3252 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3253 IB_WC_SUCCESS, 0);
3254 break;
3255 case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
3256 if (qp->state != QED_ROCE_QP_STATE_ERR)
3257 DP_ERR(dev,
3258 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3259 cq->icid, qp->icid);
3260 cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
3261 IB_WC_WR_FLUSH_ERR, 1);
3262 break;
3263 default:
3264 /* process all WQE before the cosumer */
3265 qp->state = QED_ROCE_QP_STATE_ERR;
3266 cnt = process_req(dev, qp, cq, num_entries, wc,
3267 req->sq_cons - 1, IB_WC_SUCCESS, 0);
3268 wc += cnt;
3269 /* if we have extra WC fill it with actual error info */
3270 if (cnt < num_entries) {
3271 enum ib_wc_status wc_status;
3272
3273 switch (req->status) {
3274 case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
3275 DP_ERR(dev,
3276 "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3277 cq->icid, qp->icid);
3278 wc_status = IB_WC_BAD_RESP_ERR;
3279 break;
3280 case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
3281 DP_ERR(dev,
3282 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3283 cq->icid, qp->icid);
3284 wc_status = IB_WC_LOC_LEN_ERR;
3285 break;
3286 case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
3287 DP_ERR(dev,
3288 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3289 cq->icid, qp->icid);
3290 wc_status = IB_WC_LOC_QP_OP_ERR;
3291 break;
3292 case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
3293 DP_ERR(dev,
3294 "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3295 cq->icid, qp->icid);
3296 wc_status = IB_WC_LOC_PROT_ERR;
3297 break;
3298 case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
3299 DP_ERR(dev,
3300 "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3301 cq->icid, qp->icid);
3302 wc_status = IB_WC_MW_BIND_ERR;
3303 break;
3304 case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
3305 DP_ERR(dev,
3306 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3307 cq->icid, qp->icid);
3308 wc_status = IB_WC_REM_INV_REQ_ERR;
3309 break;
3310 case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
3311 DP_ERR(dev,
3312 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3313 cq->icid, qp->icid);
3314 wc_status = IB_WC_REM_ACCESS_ERR;
3315 break;
3316 case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
3317 DP_ERR(dev,
3318 "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3319 cq->icid, qp->icid);
3320 wc_status = IB_WC_REM_OP_ERR;
3321 break;
3322 case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
3323 DP_ERR(dev,
3324 "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3325 cq->icid, qp->icid);
3326 wc_status = IB_WC_RNR_RETRY_EXC_ERR;
3327 break;
3328 case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
3329 DP_ERR(dev,
3330 "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3331 cq->icid, qp->icid);
3332 wc_status = IB_WC_RETRY_EXC_ERR;
3333 break;
3334 default:
3335 DP_ERR(dev,
3336 "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
3337 cq->icid, qp->icid);
3338 wc_status = IB_WC_GENERAL_ERR;
3339 }
3340 cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
3341 wc_status, 1);
3342 }
3343 }
3344
3345 return cnt;
3346}
3347
3348static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3349 struct qedr_cq *cq, struct ib_wc *wc,
3350 struct rdma_cqe_responder *resp, u64 wr_id)
3351{
3352 enum ib_wc_status wc_status = IB_WC_SUCCESS;
3353 u8 flags;
3354
3355 wc->opcode = IB_WC_RECV;
3356 wc->wc_flags = 0;
3357
3358 switch (resp->status) {
3359 case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
3360 wc_status = IB_WC_LOC_ACCESS_ERR;
3361 break;
3362 case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
3363 wc_status = IB_WC_LOC_LEN_ERR;
3364 break;
3365 case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
3366 wc_status = IB_WC_LOC_QP_OP_ERR;
3367 break;
3368 case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
3369 wc_status = IB_WC_LOC_PROT_ERR;
3370 break;
3371 case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
3372 wc_status = IB_WC_MW_BIND_ERR;
3373 break;
3374 case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
3375 wc_status = IB_WC_REM_INV_RD_REQ_ERR;
3376 break;
3377 case RDMA_CQE_RESP_STS_OK:
3378 wc_status = IB_WC_SUCCESS;
3379 wc->byte_len = le32_to_cpu(resp->length);
3380
3381 flags = resp->flags & QEDR_RESP_RDMA_IMM;
3382
3383 if (flags == QEDR_RESP_RDMA_IMM)
3384 wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
3385
3386 if (flags == QEDR_RESP_RDMA_IMM || flags == QEDR_RESP_IMM) {
3387 wc->ex.imm_data =
3388 le32_to_cpu(resp->imm_data_or_inv_r_Key);
3389 wc->wc_flags |= IB_WC_WITH_IMM;
3390 }
3391 break;
3392 default:
3393 wc->status = IB_WC_GENERAL_ERR;
3394 DP_ERR(dev, "Invalid CQE status detected\n");
3395 }
3396
3397 /* fill WC */
3398 wc->status = wc_status;
3399 wc->vendor_err = 0;
3400 wc->src_qp = qp->id;
3401 wc->qp = &qp->ibqp;
3402 wc->wr_id = wr_id;
3403}
3404
3405static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
3406 struct qedr_cq *cq, struct ib_wc *wc,
3407 struct rdma_cqe_responder *resp)
3408{
3409 u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3410
3411 __process_resp_one(dev, qp, cq, wc, resp, wr_id);
3412
3413 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3414 qed_chain_consume(&qp->rq.pbl);
3415 qedr_inc_sw_cons(&qp->rq);
3416
3417 return 1;
3418}
3419
3420static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
3421 int num_entries, struct ib_wc *wc, u16 hw_cons)
3422{
3423 u16 cnt = 0;
3424
3425 while (num_entries && qp->rq.wqe_cons != hw_cons) {
3426 /* fill WC */
3427 wc->status = IB_WC_WR_FLUSH_ERR;
3428 wc->vendor_err = 0;
3429 wc->wc_flags = 0;
3430 wc->src_qp = qp->id;
3431 wc->byte_len = 0;
3432 wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
3433 wc->qp = &qp->ibqp;
3434 num_entries--;
3435 wc++;
3436 cnt++;
3437 while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
3438 qed_chain_consume(&qp->rq.pbl);
3439 qedr_inc_sw_cons(&qp->rq);
3440 }
3441
3442 return cnt;
3443}
3444
3445static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3446 struct rdma_cqe_responder *resp, int *update)
3447{
3448 if (le16_to_cpu(resp->rq_cons) == qp->rq.wqe_cons) {
3449 consume_cqe(cq);
3450 *update |= 1;
3451 }
3452}
3453
3454static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
3455 struct qedr_cq *cq, int num_entries,
3456 struct ib_wc *wc, struct rdma_cqe_responder *resp,
3457 int *update)
3458{
3459 int cnt;
3460
3461 if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
3462 cnt = process_resp_flush(qp, cq, num_entries, wc,
3463 resp->rq_cons);
3464 try_consume_resp_cqe(cq, qp, resp, update);
3465 } else {
3466 cnt = process_resp_one(dev, qp, cq, wc, resp);
3467 consume_cqe(cq);
3468 *update |= 1;
3469 }
3470
3471 return cnt;
3472}
3473
3474static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
3475 struct rdma_cqe_requester *req, int *update)
3476{
3477 if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
3478 consume_cqe(cq);
3479 *update |= 1;
3480 }
3481}
3482
3483int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
3484{
3485 struct qedr_dev *dev = get_qedr_dev(ibcq->device);
3486 struct qedr_cq *cq = get_qedr_cq(ibcq);
3487 union rdma_cqe *cqe = cq->latest_cqe;
3488 u32 old_cons, new_cons;
3489 unsigned long flags;
3490 int update = 0;
3491 int done = 0;
3492
3493 if (cq->cq_type == QEDR_CQ_TYPE_GSI)
3494 return qedr_gsi_poll_cq(ibcq, num_entries, wc);
3495
3496 spin_lock_irqsave(&cq->cq_lock, flags);
3497 old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3498 while (num_entries && is_valid_cqe(cq, cqe)) {
3499 struct qedr_qp *qp;
3500 int cnt = 0;
3501
3502 /* prevent speculative reads of any field of CQE */
3503 rmb();
3504
3505 qp = cqe_get_qp(cqe);
3506 if (!qp) {
3507 WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
3508 break;
3509 }
3510
3511 wc->qp = &qp->ibqp;
3512
3513 switch (cqe_get_type(cqe)) {
3514 case RDMA_CQE_TYPE_REQUESTER:
3515 cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
3516 &cqe->req);
3517 try_consume_req_cqe(cq, qp, &cqe->req, &update);
3518 break;
3519 case RDMA_CQE_TYPE_RESPONDER_RQ:
3520 cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
3521 &cqe->resp, &update);
3522 break;
3523 case RDMA_CQE_TYPE_INVALID:
3524 default:
3525 DP_ERR(dev, "Error: invalid CQE type = %d\n",
3526 cqe_get_type(cqe));
3527 }
3528 num_entries -= cnt;
3529 wc += cnt;
3530 done += cnt;
3531
3532 cqe = get_cqe(cq);
3533 }
3534 new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
3535
3536 cq->cq_cons += new_cons - old_cons;
3537
3538 if (update)
3539 /* doorbell notifies abount latest VALID entry,
3540 * but chain already point to the next INVALID one
3541 */
3542 doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
3543
3544 spin_unlock_irqrestore(&cq->cq_lock, flags);
3545 return done;
3546}
3547
3548int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
3549 u8 port_num,
3550 const struct ib_wc *in_wc,
3551 const struct ib_grh *in_grh,
3552 const struct ib_mad_hdr *mad_hdr,
3553 size_t in_mad_size, struct ib_mad_hdr *out_mad,
3554 size_t *out_mad_size, u16 *out_mad_pkey_index)
3555{
3556 struct qedr_dev *dev = get_qedr_dev(ibdev);
3557
3558 DP_DEBUG(dev, QEDR_MSG_GSI,
3559 "QEDR_PROCESS_MAD in_mad %x %x %x %x %x %x %x %x\n",
3560 mad_hdr->attr_id, mad_hdr->base_version, mad_hdr->attr_mod,
3561 mad_hdr->class_specific, mad_hdr->class_version,
3562 mad_hdr->method, mad_hdr->mgmt_class, mad_hdr->status);
3563 return IB_MAD_RESULT_SUCCESS;
3564}
3565
3566int qedr_port_immutable(struct ib_device *ibdev, u8 port_num,
3567 struct ib_port_immutable *immutable)
3568{
3569 struct ib_port_attr attr;
3570 int err;
3571
3572 err = qedr_query_port(ibdev, port_num, &attr);
3573 if (err)
3574 return err;
3575
3576 immutable->pkey_tbl_len = attr.pkey_tbl_len;
3577 immutable->gid_tbl_len = attr.gid_tbl_len;
3578 immutable->core_cap_flags = RDMA_CORE_PORT_IBA_ROCE |
3579 RDMA_CORE_PORT_IBA_ROCE_UDP_ENCAP;
3580 immutable->max_mad_size = IB_MGMT_MAD_SIZE;
3581
3582 return 0;
3583}