1/* QLogic qedr NIC Driver
   2 * Copyright (c) 2015-2016  QLogic Corporation
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and /or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32#include <linux/dma-mapping.h>
  33#include <linux/crc32.h>
  34#include <net/ip.h>
  35#include <net/ipv6.h>
  36#include <net/udp.h>
  37#include <linux/iommu.h>
  38
  39#include <rdma/ib_verbs.h>
  40#include <rdma/ib_user_verbs.h>
  41#include <rdma/iw_cm.h>
  42#include <rdma/ib_umem.h>
  43#include <rdma/ib_addr.h>
  44#include <rdma/ib_cache.h>
  45#include <rdma/uverbs_ioctl.h>
  46
  47#include <linux/qed/common_hsi.h>
  48#include "qedr_hsi_rdma.h"
  49#include <linux/qed/qed_if.h>
  50#include "qedr.h"
  51#include "verbs.h"
  52#include <rdma/qedr-abi.h>
  53#include "qedr_roce_cm.h"
  54#include "qedr_iw_cm.h"
  55
  56#define QEDR_SRQ_WQE_ELEM_SIZE	sizeof(union rdma_srq_elm)
  57#define	RDMA_MAX_SGE_PER_SRQ	(4)
  58#define RDMA_MAX_SRQ_WQE_SIZE	(RDMA_MAX_SGE_PER_SRQ + 1)
  59
  60#define DB_ADDR_SHIFT(addr)		((addr) << DB_PWM_ADDR_OFFSET_SHIFT)
  61
  62enum {
  63	QEDR_USER_MMAP_IO_WC = 0,
  64	QEDR_USER_MMAP_PHYS_PAGE,
  65};
  66
  67static inline int qedr_ib_copy_to_udata(struct ib_udata *udata, void *src,
  68					size_t len)
  69{
  70	size_t min_len = min_t(size_t, len, udata->outlen);
  71
  72	return ib_copy_to_udata(udata, src, min_len);
  73}
  74
  75int qedr_query_pkey(struct ib_device *ibdev, u32 port, u16 index, u16 *pkey)
  76{
  77	if (index >= QEDR_ROCE_PKEY_TABLE_LEN)
  78		return -EINVAL;
  79
  80	*pkey = QEDR_ROCE_PKEY_DEFAULT;
  81	return 0;
  82}
  83
  84int qedr_iw_query_gid(struct ib_device *ibdev, u32 port,
  85		      int index, union ib_gid *sgid)
  86{
  87	struct qedr_dev *dev = get_qedr_dev(ibdev);
  88
  89	memset(sgid->raw, 0, sizeof(sgid->raw));
  90	ether_addr_copy(sgid->raw, dev->ndev->dev_addr);
  91
  92	DP_DEBUG(dev, QEDR_MSG_INIT, "QUERY sgid[%d]=%llx:%llx\n", index,
  93		 sgid->global.interface_id, sgid->global.subnet_prefix);
  94
  95	return 0;
  96}
  97
  98int qedr_query_srq(struct ib_srq *ibsrq, struct ib_srq_attr *srq_attr)
  99{
 100	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
 101	struct qedr_device_attr *qattr = &dev->attr;
 102	struct qedr_srq *srq = get_qedr_srq(ibsrq);
 103
 104	srq_attr->srq_limit = srq->srq_limit;
 105	srq_attr->max_wr = qattr->max_srq_wr;
 106	srq_attr->max_sge = qattr->max_sge;
 107
 108	return 0;
 109}
 110
 111int qedr_query_device(struct ib_device *ibdev,
 112		      struct ib_device_attr *attr, struct ib_udata *udata)
 113{
 114	struct qedr_dev *dev = get_qedr_dev(ibdev);
 115	struct qedr_device_attr *qattr = &dev->attr;
 116
 117	if (!dev->rdma_ctx) {
 118		DP_ERR(dev,
 119		       "qedr_query_device called with invalid params rdma_ctx=%p\n",
 120		       dev->rdma_ctx);
 121		return -EINVAL;
 122	}
 123
 124	memset(attr, 0, sizeof(*attr));
 125
 126	attr->fw_ver = qattr->fw_ver;
 127	attr->sys_image_guid = qattr->sys_image_guid;
 128	attr->max_mr_size = qattr->max_mr_size;
 129	attr->page_size_cap = qattr->page_size_caps;
 130	attr->vendor_id = qattr->vendor_id;
 131	attr->vendor_part_id = qattr->vendor_part_id;
 132	attr->hw_ver = qattr->hw_ver;
 133	attr->max_qp = qattr->max_qp;
 134	attr->max_qp_wr = max_t(u32, qattr->max_sqe, qattr->max_rqe);
 135	attr->device_cap_flags = IB_DEVICE_CURR_QP_STATE_MOD |
 136	    IB_DEVICE_RC_RNR_NAK_GEN |
 137	    IB_DEVICE_MEM_MGT_EXTENSIONS;
 138	attr->kernel_cap_flags = IBK_LOCAL_DMA_LKEY;
 139
 140	if (!rdma_protocol_iwarp(&dev->ibdev, 1))
 141		attr->device_cap_flags |= IB_DEVICE_XRC;
 142	attr->max_send_sge = qattr->max_sge;
 143	attr->max_recv_sge = qattr->max_sge;
 144	attr->max_sge_rd = qattr->max_sge;
 145	attr->max_cq = qattr->max_cq;
 146	attr->max_cqe = qattr->max_cqe;
 147	attr->max_mr = qattr->max_mr;
 148	attr->max_mw = qattr->max_mw;
 149	attr->max_pd = qattr->max_pd;
 150	attr->atomic_cap = dev->atomic_cap;
 151	attr->max_qp_init_rd_atom =
 152	    1 << (fls(qattr->max_qp_req_rd_atomic_resc) - 1);
 153	attr->max_qp_rd_atom =
 154	    min(1 << (fls(qattr->max_qp_resp_rd_atomic_resc) - 1),
 155		attr->max_qp_init_rd_atom);
 156
 157	attr->max_srq = qattr->max_srq;
 158	attr->max_srq_sge = qattr->max_srq_sge;
 159	attr->max_srq_wr = qattr->max_srq_wr;
 160
 161	attr->local_ca_ack_delay = qattr->dev_ack_delay;
 162	attr->max_fast_reg_page_list_len = qattr->max_mr / 8;
 163	attr->max_pkeys = qattr->max_pkey;
 164	attr->max_ah = qattr->max_ah;
 165
 166	return 0;
 167}
 168
 169static inline void get_link_speed_and_width(int speed, u16 *ib_speed,
 170					    u8 *ib_width)
 171{
 172	switch (speed) {
 173	case 1000:
 174		*ib_speed = IB_SPEED_SDR;
 175		*ib_width = IB_WIDTH_1X;
 176		break;
 177	case 10000:
 178		*ib_speed = IB_SPEED_QDR;
 179		*ib_width = IB_WIDTH_1X;
 180		break;
 181
 182	case 20000:
 183		*ib_speed = IB_SPEED_DDR;
 184		*ib_width = IB_WIDTH_4X;
 185		break;
 186
 187	case 25000:
 188		*ib_speed = IB_SPEED_EDR;
 189		*ib_width = IB_WIDTH_1X;
 190		break;
 191
 192	case 40000:
 193		*ib_speed = IB_SPEED_QDR;
 194		*ib_width = IB_WIDTH_4X;
 195		break;
 196
 197	case 50000:
 198		*ib_speed = IB_SPEED_HDR;
 199		*ib_width = IB_WIDTH_1X;
 200		break;
 201
 202	case 100000:
 203		*ib_speed = IB_SPEED_EDR;
 204		*ib_width = IB_WIDTH_4X;
 205		break;
 206
 207	default:
 208		/* Unsupported */
 209		*ib_speed = IB_SPEED_SDR;
 210		*ib_width = IB_WIDTH_1X;
 211	}
 212}
 213
 214int qedr_query_port(struct ib_device *ibdev, u32 port,
 215		    struct ib_port_attr *attr)
 216{
 217	struct qedr_dev *dev;
 218	struct qed_rdma_port *rdma_port;
 219
 220	dev = get_qedr_dev(ibdev);
 221
 222	if (!dev->rdma_ctx) {
 223		DP_ERR(dev, "rdma_ctx is NULL\n");
 224		return -EINVAL;
 225	}
 226
 227	rdma_port = dev->ops->rdma_query_port(dev->rdma_ctx);
 228
 229	/* *attr being zeroed by the caller, avoid zeroing it here */
 230	if (rdma_port->port_state == QED_RDMA_PORT_UP) {
 231		attr->state = IB_PORT_ACTIVE;
 232		attr->phys_state = IB_PORT_PHYS_STATE_LINK_UP;
 233	} else {
 234		attr->state = IB_PORT_DOWN;
 235		attr->phys_state = IB_PORT_PHYS_STATE_DISABLED;
 236	}
 237	attr->max_mtu = IB_MTU_4096;
 238	attr->lid = 0;
 239	attr->lmc = 0;
 240	attr->sm_lid = 0;
 241	attr->sm_sl = 0;
 242	attr->ip_gids = true;
 243	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
 244		attr->active_mtu = iboe_get_mtu(dev->iwarp_max_mtu);
 245		attr->gid_tbl_len = 1;
 246	} else {
 247		attr->active_mtu = iboe_get_mtu(dev->ndev->mtu);
 248		attr->gid_tbl_len = QEDR_MAX_SGID;
 249		attr->pkey_tbl_len = QEDR_ROCE_PKEY_TABLE_LEN;
 250	}
 251	attr->bad_pkey_cntr = rdma_port->pkey_bad_counter;
 252	attr->qkey_viol_cntr = 0;
 253	get_link_speed_and_width(rdma_port->link_speed,
 254				 &attr->active_speed, &attr->active_width);
 255	attr->max_msg_sz = rdma_port->max_msg_size;
 256	attr->max_vl_num = 4;
 257
 258	return 0;
 259}
 260
 261int qedr_alloc_ucontext(struct ib_ucontext *uctx, struct ib_udata *udata)
 262{
 263	struct ib_device *ibdev = uctx->device;
 264	int rc;
 265	struct qedr_ucontext *ctx = get_qedr_ucontext(uctx);
 266	struct qedr_alloc_ucontext_resp uresp = {};
 267	struct qedr_alloc_ucontext_req ureq = {};
 268	struct qedr_dev *dev = get_qedr_dev(ibdev);
 269	struct qed_rdma_add_user_out_params oparams;
 270	struct qedr_user_mmap_entry *entry;
 271
 272	if (!udata)
 273		return -EFAULT;
 274
 275	if (udata->inlen) {
 276		rc = ib_copy_from_udata(&ureq, udata,
 277					min(sizeof(ureq), udata->inlen));
 278		if (rc) {
 279			DP_ERR(dev, "Problem copying data from user space\n");
 280			return -EFAULT;
 281		}
 282		ctx->edpm_mode = !!(ureq.context_flags &
 283				    QEDR_ALLOC_UCTX_EDPM_MODE);
 284		ctx->db_rec = !!(ureq.context_flags & QEDR_ALLOC_UCTX_DB_REC);
 285	}
 286
 287	rc = dev->ops->rdma_add_user(dev->rdma_ctx, &oparams);
 288	if (rc) {
 289		DP_ERR(dev,
 290		       "failed to allocate a DPI for a new RoCE application, rc=%d. To overcome this consider to increase the number of DPIs, increase the doorbell BAR size or just close unnecessary RoCE applications. In order to increase the number of DPIs consult the qedr readme\n",
 291		       rc);
 292		return rc;
 293	}
 294
 295	ctx->dpi = oparams.dpi;
 296	ctx->dpi_addr = oparams.dpi_addr;
 297	ctx->dpi_phys_addr = oparams.dpi_phys_addr;
 298	ctx->dpi_size = oparams.dpi_size;
 299	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 300	if (!entry) {
 301		rc = -ENOMEM;
 302		goto err;
 303	}
 304
 305	entry->io_address = ctx->dpi_phys_addr;
 306	entry->length = ctx->dpi_size;
 307	entry->mmap_flag = QEDR_USER_MMAP_IO_WC;
 308	entry->dpi = ctx->dpi;
 309	entry->dev = dev;
 310	rc = rdma_user_mmap_entry_insert(uctx, &entry->rdma_entry,
 311					 ctx->dpi_size);
 312	if (rc) {
 313		kfree(entry);
 314		goto err;
 315	}
 316	ctx->db_mmap_entry = &entry->rdma_entry;
 317
 318	if (!dev->user_dpm_enabled)
 319		uresp.dpm_flags = 0;
 320	else if (rdma_protocol_iwarp(&dev->ibdev, 1))
 321		uresp.dpm_flags = QEDR_DPM_TYPE_IWARP_LEGACY;
 322	else
 323		uresp.dpm_flags = QEDR_DPM_TYPE_ROCE_ENHANCED |
 324				  QEDR_DPM_TYPE_ROCE_LEGACY |
 325				  QEDR_DPM_TYPE_ROCE_EDPM_MODE;
 326
 327	if (ureq.context_flags & QEDR_SUPPORT_DPM_SIZES) {
 328		uresp.dpm_flags |= QEDR_DPM_SIZES_SET;
 329		uresp.ldpm_limit_size = QEDR_LDPM_MAX_SIZE;
 330		uresp.edpm_trans_size = QEDR_EDPM_TRANS_SIZE;
 331		uresp.edpm_limit_size = QEDR_EDPM_MAX_SIZE;
 332	}
 333
 334	uresp.wids_enabled = 1;
 335	uresp.wid_count = oparams.wid_count;
 336	uresp.db_pa = rdma_user_mmap_get_offset(ctx->db_mmap_entry);
 337	uresp.db_size = ctx->dpi_size;
 338	uresp.max_send_wr = dev->attr.max_sqe;
 339	uresp.max_recv_wr = dev->attr.max_rqe;
 340	uresp.max_srq_wr = dev->attr.max_srq_wr;
 341	uresp.sges_per_send_wr = QEDR_MAX_SQE_ELEMENTS_PER_SQE;
 342	uresp.sges_per_recv_wr = QEDR_MAX_RQE_ELEMENTS_PER_RQE;
 343	uresp.sges_per_srq_wr = dev->attr.max_srq_sge;
 344	uresp.max_cqes = QEDR_MAX_CQES;
 345
 346	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 347	if (rc)
 348		goto err;
 349
 350	ctx->dev = dev;
 351
 352	DP_DEBUG(dev, QEDR_MSG_INIT, "Allocating user context %p\n",
 353		 &ctx->ibucontext);
 354	return 0;
 355
 356err:
 357	if (!ctx->db_mmap_entry)
 358		dev->ops->rdma_remove_user(dev->rdma_ctx, ctx->dpi);
 359	else
 360		rdma_user_mmap_entry_remove(ctx->db_mmap_entry);
 361
 362	return rc;
 363}
 364
 365void qedr_dealloc_ucontext(struct ib_ucontext *ibctx)
 366{
 367	struct qedr_ucontext *uctx = get_qedr_ucontext(ibctx);
 368
 369	DP_DEBUG(uctx->dev, QEDR_MSG_INIT, "Deallocating user context %p\n",
 370		 uctx);
 371
 372	rdma_user_mmap_entry_remove(uctx->db_mmap_entry);
 373}
 374
 375void qedr_mmap_free(struct rdma_user_mmap_entry *rdma_entry)
 376{
 377	struct qedr_user_mmap_entry *entry = get_qedr_mmap_entry(rdma_entry);
 378	struct qedr_dev *dev = entry->dev;
 379
 380	if (entry->mmap_flag == QEDR_USER_MMAP_PHYS_PAGE)
 381		free_page((unsigned long)entry->address);
 382	else if (entry->mmap_flag == QEDR_USER_MMAP_IO_WC)
 383		dev->ops->rdma_remove_user(dev->rdma_ctx, entry->dpi);
 384
 385	kfree(entry);
 386}
 387
 388int qedr_mmap(struct ib_ucontext *ucontext, struct vm_area_struct *vma)
 389{
 390	struct ib_device *dev = ucontext->device;
 391	size_t length = vma->vm_end - vma->vm_start;
 392	struct rdma_user_mmap_entry *rdma_entry;
 393	struct qedr_user_mmap_entry *entry;
 394	int rc = 0;
 395	u64 pfn;
 396
 397	ibdev_dbg(dev,
 398		  "start %#lx, end %#lx, length = %#zx, pgoff = %#lx\n",
 399		  vma->vm_start, vma->vm_end, length, vma->vm_pgoff);
 400
 401	rdma_entry = rdma_user_mmap_entry_get(ucontext, vma);
 402	if (!rdma_entry) {
 403		ibdev_dbg(dev, "pgoff[%#lx] does not have valid entry\n",
 404			  vma->vm_pgoff);
 405		return -EINVAL;
 406	}
 407	entry = get_qedr_mmap_entry(rdma_entry);
 408	ibdev_dbg(dev,
 409		  "Mapping address[%#llx], length[%#zx], mmap_flag[%d]\n",
 410		  entry->io_address, length, entry->mmap_flag);
 411
 412	switch (entry->mmap_flag) {
 413	case QEDR_USER_MMAP_IO_WC:
 414		pfn = entry->io_address >> PAGE_SHIFT;
 415		rc = rdma_user_mmap_io(ucontext, vma, pfn, length,
 416				       pgprot_writecombine(vma->vm_page_prot),
 417				       rdma_entry);
 418		break;
 419	case QEDR_USER_MMAP_PHYS_PAGE:
 420		rc = vm_insert_page(vma, vma->vm_start,
 421				    virt_to_page(entry->address));
 422		break;
 423	default:
 424		rc = -EINVAL;
 425	}
 426
 427	if (rc)
 428		ibdev_dbg(dev,
 429			  "Couldn't mmap address[%#llx] length[%#zx] mmap_flag[%d] err[%d]\n",
 430			  entry->io_address, length, entry->mmap_flag, rc);
 431
 432	rdma_user_mmap_entry_put(rdma_entry);
 433	return rc;
 434}
 435
 436int qedr_alloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 437{
 438	struct ib_device *ibdev = ibpd->device;
 439	struct qedr_dev *dev = get_qedr_dev(ibdev);
 440	struct qedr_pd *pd = get_qedr_pd(ibpd);
 441	u16 pd_id;
 442	int rc;
 443
 444	DP_DEBUG(dev, QEDR_MSG_INIT, "Function called from: %s\n",
 445		 udata ? "User Lib" : "Kernel");
 446
 447	if (!dev->rdma_ctx) {
 448		DP_ERR(dev, "invalid RDMA context\n");
 449		return -EINVAL;
 450	}
 451
 452	rc = dev->ops->rdma_alloc_pd(dev->rdma_ctx, &pd_id);
 453	if (rc)
 454		return rc;
 455
 456	pd->pd_id = pd_id;
 457
 458	if (udata) {
 459		struct qedr_alloc_pd_uresp uresp = {
 460			.pd_id = pd_id,
 461		};
 462		struct qedr_ucontext *context = rdma_udata_to_drv_context(
 463			udata, struct qedr_ucontext, ibucontext);
 464
 465		rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 466		if (rc) {
 467			DP_ERR(dev, "copy error pd_id=0x%x.\n", pd_id);
 468			dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd_id);
 469			return rc;
 470		}
 471
 472		pd->uctx = context;
 473		pd->uctx->pd = pd;
 474	}
 475
 476	return 0;
 477}
 478
 479int qedr_dealloc_pd(struct ib_pd *ibpd, struct ib_udata *udata)
 480{
 481	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
 482	struct qedr_pd *pd = get_qedr_pd(ibpd);
 483
 484	DP_DEBUG(dev, QEDR_MSG_INIT, "Deallocating PD %d\n", pd->pd_id);
 485	dev->ops->rdma_dealloc_pd(dev->rdma_ctx, pd->pd_id);
 486	return 0;
 487}
 488
 489
 490int qedr_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 491{
 492	struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 493	struct qedr_xrcd *xrcd = get_qedr_xrcd(ibxrcd);
 494
 495	return dev->ops->rdma_alloc_xrcd(dev->rdma_ctx, &xrcd->xrcd_id);
 496}
 497
 498int qedr_dealloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
 499{
 500	struct qedr_dev *dev = get_qedr_dev(ibxrcd->device);
 501	u16 xrcd_id = get_qedr_xrcd(ibxrcd)->xrcd_id;
 502
 503	dev->ops->rdma_dealloc_xrcd(dev->rdma_ctx, xrcd_id);
 504	return 0;
 505}
 506static void qedr_free_pbl(struct qedr_dev *dev,
 507			  struct qedr_pbl_info *pbl_info, struct qedr_pbl *pbl)
 508{
 509	struct pci_dev *pdev = dev->pdev;
 510	int i;
 511
 512	for (i = 0; i < pbl_info->num_pbls; i++) {
 513		if (!pbl[i].va)
 514			continue;
 515		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 516				  pbl[i].va, pbl[i].pa);
 517	}
 518
 519	kfree(pbl);
 520}
 521
 522#define MIN_FW_PBL_PAGE_SIZE (4 * 1024)
 523#define MAX_FW_PBL_PAGE_SIZE (64 * 1024)
 524
 525#define NUM_PBES_ON_PAGE(_page_size) (_page_size / sizeof(u64))
 526#define MAX_PBES_ON_PAGE NUM_PBES_ON_PAGE(MAX_FW_PBL_PAGE_SIZE)
 527#define MAX_PBES_TWO_LAYER (MAX_PBES_ON_PAGE * MAX_PBES_ON_PAGE)
 528
 529static struct qedr_pbl *qedr_alloc_pbl_tbl(struct qedr_dev *dev,
 530					   struct qedr_pbl_info *pbl_info,
 531					   gfp_t flags)
 532{
 533	struct pci_dev *pdev = dev->pdev;
 534	struct qedr_pbl *pbl_table;
 535	dma_addr_t *pbl_main_tbl;
 536	dma_addr_t pa;
 537	void *va;
 538	int i;
 539
 540	pbl_table = kcalloc(pbl_info->num_pbls, sizeof(*pbl_table), flags);
 541	if (!pbl_table)
 542		return ERR_PTR(-ENOMEM);
 543
 544	for (i = 0; i < pbl_info->num_pbls; i++) {
 545		va = dma_alloc_coherent(&pdev->dev, pbl_info->pbl_size, &pa,
 546					flags);
 547		if (!va)
 548			goto err;
 549
 550		pbl_table[i].va = va;
 551		pbl_table[i].pa = pa;
 552	}
 553
 554	/* Two-Layer PBLs, if we have more than one pbl we need to initialize
 555	 * the first one with physical pointers to all of the rest
 556	 */
 557	pbl_main_tbl = (dma_addr_t *)pbl_table[0].va;
 558	for (i = 0; i < pbl_info->num_pbls - 1; i++)
 559		pbl_main_tbl[i] = pbl_table[i + 1].pa;
 560
 561	return pbl_table;
 562
 563err:
 564	for (i--; i >= 0; i--)
 565		dma_free_coherent(&pdev->dev, pbl_info->pbl_size,
 566				  pbl_table[i].va, pbl_table[i].pa);
 567
 568	qedr_free_pbl(dev, pbl_info, pbl_table);
 569
 570	return ERR_PTR(-ENOMEM);
 571}
 572
 573static int qedr_prepare_pbl_tbl(struct qedr_dev *dev,
 574				struct qedr_pbl_info *pbl_info,
 575				u32 num_pbes, int two_layer_capable)
 576{
 577	u32 pbl_capacity;
 578	u32 pbl_size;
 579	u32 num_pbls;
 580
 581	if ((num_pbes > MAX_PBES_ON_PAGE) && two_layer_capable) {
 582		if (num_pbes > MAX_PBES_TWO_LAYER) {
 583			DP_ERR(dev, "prepare pbl table: too many pages %d\n",
 584			       num_pbes);
 585			return -EINVAL;
 586		}
 587
 588		/* calculate required pbl page size */
 589		pbl_size = MIN_FW_PBL_PAGE_SIZE;
 590		pbl_capacity = NUM_PBES_ON_PAGE(pbl_size) *
 591			       NUM_PBES_ON_PAGE(pbl_size);
 592
 593		while (pbl_capacity < num_pbes) {
 594			pbl_size *= 2;
 595			pbl_capacity = pbl_size / sizeof(u64);
 596			pbl_capacity = pbl_capacity * pbl_capacity;
 597		}
 598
 599		num_pbls = DIV_ROUND_UP(num_pbes, NUM_PBES_ON_PAGE(pbl_size));
 600		num_pbls++;	/* One for the layer0 ( points to the pbls) */
 601		pbl_info->two_layered = true;
 602	} else {
 603		/* One layered PBL */
 604		num_pbls = 1;
 605		pbl_size = max_t(u32, MIN_FW_PBL_PAGE_SIZE,
 606				 roundup_pow_of_two((num_pbes * sizeof(u64))));
 607		pbl_info->two_layered = false;
 608	}
 609
 610	pbl_info->num_pbls = num_pbls;
 611	pbl_info->pbl_size = pbl_size;
 612	pbl_info->num_pbes = num_pbes;
 613
 614	DP_DEBUG(dev, QEDR_MSG_MR,
 615		 "prepare pbl table: num_pbes=%d, num_pbls=%d, pbl_size=%d\n",
 616		 pbl_info->num_pbes, pbl_info->num_pbls, pbl_info->pbl_size);
 617
 618	return 0;
 619}
 620
 621static void qedr_populate_pbls(struct qedr_dev *dev, struct ib_umem *umem,
 622			       struct qedr_pbl *pbl,
 623			       struct qedr_pbl_info *pbl_info, u32 pg_shift)
 624{
 625	int pbe_cnt, total_num_pbes = 0;
 626	struct qedr_pbl *pbl_tbl;
 627	struct ib_block_iter biter;
 628	struct regpair *pbe;
 629
 630	if (!pbl_info->num_pbes)
 631		return;
 632
 633	/* If we have a two layered pbl, the first pbl points to the rest
 634	 * of the pbls and the first entry lays on the second pbl in the table
 635	 */
 636	if (pbl_info->two_layered)
 637		pbl_tbl = &pbl[1];
 638	else
 639		pbl_tbl = pbl;
 640
 641	pbe = (struct regpair *)pbl_tbl->va;
 642	if (!pbe) {
 643		DP_ERR(dev, "cannot populate PBL due to a NULL PBE\n");
 644		return;
 645	}
 646
 647	pbe_cnt = 0;
 648
 649	rdma_umem_for_each_dma_block (umem, &biter, BIT(pg_shift)) {
 650		u64 pg_addr = rdma_block_iter_dma_address(&biter);
 651
 652		pbe->lo = cpu_to_le32(pg_addr);
 653		pbe->hi = cpu_to_le32(upper_32_bits(pg_addr));
 654
 655		pbe_cnt++;
 656		total_num_pbes++;
 657		pbe++;
 658
 659		if (total_num_pbes == pbl_info->num_pbes)
 660			return;
 661
 662		/* If the given pbl is full storing the pbes, move to next pbl.
 663		 */
 664		if (pbe_cnt == (pbl_info->pbl_size / sizeof(u64))) {
 665			pbl_tbl++;
 666			pbe = (struct regpair *)pbl_tbl->va;
 667			pbe_cnt = 0;
 668		}
 669	}
 670}
 671
 672static int qedr_db_recovery_add(struct qedr_dev *dev,
 673				void __iomem *db_addr,
 674				void *db_data,
 675				enum qed_db_rec_width db_width,
 676				enum qed_db_rec_space db_space)
 677{
 678	if (!db_data) {
 679		DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 680		return 0;
 681	}
 682
 683	return dev->ops->common->db_recovery_add(dev->cdev, db_addr, db_data,
 684						 db_width, db_space);
 685}
 686
 687static void qedr_db_recovery_del(struct qedr_dev *dev,
 688				 void __iomem *db_addr,
 689				 void *db_data)
 690{
 691	if (!db_data) {
 692		DP_DEBUG(dev, QEDR_MSG_INIT, "avoiding db rec since old lib\n");
 693		return;
 694	}
 695
 696	/* Ignore return code as there is not much we can do about it. Error
 697	 * log will be printed inside.
 698	 */
 699	dev->ops->common->db_recovery_del(dev->cdev, db_addr, db_data);
 700}
 701
 702static int qedr_copy_cq_uresp(struct qedr_dev *dev,
 703			      struct qedr_cq *cq, struct ib_udata *udata,
 704			      u32 db_offset)
 705{
 706	struct qedr_create_cq_uresp uresp;
 707	int rc;
 708
 709	memset(&uresp, 0, sizeof(uresp));
 710
 711	uresp.db_offset = db_offset;
 712	uresp.icid = cq->icid;
 713	if (cq->q.db_mmap_entry)
 714		uresp.db_rec_addr =
 715			rdma_user_mmap_get_offset(cq->q.db_mmap_entry);
 716
 717	rc = qedr_ib_copy_to_udata(udata, &uresp, sizeof(uresp));
 718	if (rc)
 719		DP_ERR(dev, "copy error cqid=0x%x.\n", cq->icid);
 720
 721	return rc;
 722}
 723
 724static void consume_cqe(struct qedr_cq *cq)
 725{
 726	if (cq->latest_cqe == cq->toggle_cqe)
 727		cq->pbl_toggle ^= RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
 728
 729	cq->latest_cqe = qed_chain_consume(&cq->pbl);
 730}
 731
 732static inline int qedr_align_cq_entries(int entries)
 733{
 734	u64 size, aligned_size;
 735
 736	/* We allocate an extra entry that we don't report to the FW. */
 737	size = (entries + 1) * QEDR_CQE_SIZE;
 738	aligned_size = ALIGN(size, PAGE_SIZE);
 739
 740	return aligned_size / QEDR_CQE_SIZE;
 741}
 742
 743static int qedr_init_user_db_rec(struct ib_udata *udata,
 744				 struct qedr_dev *dev, struct qedr_userq *q,
 745				 bool requires_db_rec)
 746{
 747	struct qedr_ucontext *uctx =
 748		rdma_udata_to_drv_context(udata, struct qedr_ucontext,
 749					  ibucontext);
 750	struct qedr_user_mmap_entry *entry;
 751	int rc;
 752
 753	/* Aborting for non doorbell userqueue (SRQ) or non-supporting lib */
 754	if (requires_db_rec == 0 || !uctx->db_rec)
 755		return 0;
 756
 757	/* Allocate a page for doorbell recovery, add to mmap */
 758	q->db_rec_data = (void *)get_zeroed_page(GFP_USER);
 759	if (!q->db_rec_data) {
 760		DP_ERR(dev, "get_zeroed_page failed\n");
 761		return -ENOMEM;
 762	}
 763
 764	entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 765	if (!entry)
 766		goto err_free_db_data;
 767
 768	entry->address = q->db_rec_data;
 769	entry->length = PAGE_SIZE;
 770	entry->mmap_flag = QEDR_USER_MMAP_PHYS_PAGE;
 771	rc = rdma_user_mmap_entry_insert(&uctx->ibucontext,
 772					 &entry->rdma_entry,
 773					 PAGE_SIZE);
 774	if (rc)
 775		goto err_free_entry;
 776
 777	q->db_mmap_entry = &entry->rdma_entry;
 778
 779	return 0;
 780
 781err_free_entry:
 782	kfree(entry);
 783
 784err_free_db_data:
 785	free_page((unsigned long)q->db_rec_data);
 786	q->db_rec_data = NULL;
 787	return -ENOMEM;
 788}
 789
 790static inline int qedr_init_user_queue(struct ib_udata *udata,
 791				       struct qedr_dev *dev,
 792				       struct qedr_userq *q, u64 buf_addr,
 793				       size_t buf_len, bool requires_db_rec,
 794				       int access,
 795				       int alloc_and_init)
 796{
 797	u32 fw_pages;
 798	int rc;
 799
 800	q->buf_addr = buf_addr;
 801	q->buf_len = buf_len;
 802	q->umem = ib_umem_get(&dev->ibdev, q->buf_addr, q->buf_len, access);
 803	if (IS_ERR(q->umem)) {
 804		DP_ERR(dev, "create user queue: failed ib_umem_get, got %ld\n",
 805		       PTR_ERR(q->umem));
 806		return PTR_ERR(q->umem);
 807	}
 808
 809	fw_pages = ib_umem_num_dma_blocks(q->umem, 1 << FW_PAGE_SHIFT);
 810	rc = qedr_prepare_pbl_tbl(dev, &q->pbl_info, fw_pages, 0);
 811	if (rc)
 812		goto err0;
 813
 814	if (alloc_and_init) {
 815		q->pbl_tbl = qedr_alloc_pbl_tbl(dev, &q->pbl_info, GFP_KERNEL);
 816		if (IS_ERR(q->pbl_tbl)) {
 817			rc = PTR_ERR(q->pbl_tbl);
 818			goto err0;
 819		}
 820		qedr_populate_pbls(dev, q->umem, q->pbl_tbl, &q->pbl_info,
 821				   FW_PAGE_SHIFT);
 822	} else {
 823		q->pbl_tbl = kzalloc(sizeof(*q->pbl_tbl), GFP_KERNEL);
 824		if (!q->pbl_tbl) {
 825			rc = -ENOMEM;
 826			goto err0;
 827		}
 828	}
 829
 830	/* mmap the user address used to store doorbell data for recovery */
 831	return qedr_init_user_db_rec(udata, dev, q, requires_db_rec);
 832
 833err0:
 834	ib_umem_release(q->umem);
 835	q->umem = NULL;
 836
 837	return rc;
 838}
 839
 840static inline void qedr_init_cq_params(struct qedr_cq *cq,
 841				       struct qedr_ucontext *ctx,
 842				       struct qedr_dev *dev, int vector,
 843				       int chain_entries, int page_cnt,
 844				       u64 pbl_ptr,
 845				       struct qed_rdma_create_cq_in_params
 846				       *params)
 847{
 848	memset(params, 0, sizeof(*params));
 849	params->cq_handle_hi = upper_32_bits((uintptr_t)cq);
 850	params->cq_handle_lo = lower_32_bits((uintptr_t)cq);
 851	params->cnq_id = vector;
 852	params->cq_size = chain_entries - 1;
 853	params->dpi = (ctx) ? ctx->dpi : dev->dpi;
 854	params->pbl_num_pages = page_cnt;
 855	params->pbl_ptr = pbl_ptr;
 856	params->pbl_two_level = 0;
 857}
 858
 859static void doorbell_cq(struct qedr_cq *cq, u32 cons, u8 flags)
 860{
 861	cq->db.data.agg_flags = flags;
 862	cq->db.data.value = cpu_to_le32(cons);
 863	writeq(cq->db.raw, cq->db_addr);
 864}
 865
 866int qedr_arm_cq(struct ib_cq *ibcq, enum ib_cq_notify_flags flags)
 867{
 868	struct qedr_cq *cq = get_qedr_cq(ibcq);
 869	unsigned long sflags;
 870	struct qedr_dev *dev;
 871
 872	dev = get_qedr_dev(ibcq->device);
 873
 874	if (cq->destroyed) {
 875		DP_ERR(dev,
 876		       "warning: arm was invoked after destroy for cq %p (icid=%d)\n",
 877		       cq, cq->icid);
 878		return -EINVAL;
 879	}
 880
 881
 882	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
 883		return 0;
 884
 885	spin_lock_irqsave(&cq->cq_lock, sflags);
 886
 887	cq->arm_flags = 0;
 888
 889	if (flags & IB_CQ_SOLICITED)
 890		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_SE_CF_CMD;
 891
 892	if (flags & IB_CQ_NEXT_COMP)
 893		cq->arm_flags |= DQ_UCM_ROCE_CQ_ARM_CF_CMD;
 894
 895	doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
 896
 897	spin_unlock_irqrestore(&cq->cq_lock, sflags);
 898
 899	return 0;
 900}
 901
 902int qedr_create_cq(struct ib_cq *ibcq, const struct ib_cq_init_attr *attr,
 903		   struct ib_udata *udata)
 904{
 905	struct ib_device *ibdev = ibcq->device;
 906	struct qedr_ucontext *ctx = rdma_udata_to_drv_context(
 907		udata, struct qedr_ucontext, ibucontext);
 908	struct qed_rdma_destroy_cq_out_params destroy_oparams;
 909	struct qed_rdma_destroy_cq_in_params destroy_iparams;
 910	struct qed_chain_init_params chain_params = {
 911		.mode		= QED_CHAIN_MODE_PBL,
 912		.intended_use	= QED_CHAIN_USE_TO_CONSUME,
 913		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
 914		.elem_size	= sizeof(union rdma_cqe),
 915	};
 916	struct qedr_dev *dev = get_qedr_dev(ibdev);
 917	struct qed_rdma_create_cq_in_params params;
 918	struct qedr_create_cq_ureq ureq = {};
 919	int vector = attr->comp_vector;
 920	int entries = attr->cqe;
 921	struct qedr_cq *cq = get_qedr_cq(ibcq);
 922	int chain_entries;
 923	u32 db_offset;
 924	int page_cnt;
 925	u64 pbl_ptr;
 926	u16 icid;
 927	int rc;
 928
 929	DP_DEBUG(dev, QEDR_MSG_INIT,
 930		 "create_cq: called from %s. entries=%d, vector=%d\n",
 931		 udata ? "User Lib" : "Kernel", entries, vector);
 932
 933	if (attr->flags)
 934		return -EOPNOTSUPP;
 935
 936	if (entries > QEDR_MAX_CQES) {
 937		DP_ERR(dev,
 938		       "create cq: the number of entries %d is too high. Must be equal or below %d.\n",
 939		       entries, QEDR_MAX_CQES);
 940		return -EINVAL;
 941	}
 942
 943	chain_entries = qedr_align_cq_entries(entries);
 944	chain_entries = min_t(int, chain_entries, QEDR_MAX_CQES);
 945	chain_params.num_elems = chain_entries;
 946
 947	/* calc db offset. user will add DPI base, kernel will add db addr */
 948	db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_UCM_RDMA_CQ_CONS_32BIT);
 949
 950	if (udata) {
 951		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
 952							 udata->inlen))) {
 953			DP_ERR(dev,
 954			       "create cq: problem copying data from user space\n");
 955			goto err0;
 956		}
 957
 958		if (!ureq.len) {
 959			DP_ERR(dev,
 960			       "create cq: cannot create a cq with 0 entries\n");
 961			goto err0;
 962		}
 963
 964		cq->cq_type = QEDR_CQ_TYPE_USER;
 965
 966		rc = qedr_init_user_queue(udata, dev, &cq->q, ureq.addr,
 967					  ureq.len, true, IB_ACCESS_LOCAL_WRITE,
 968					  1);
 969		if (rc)
 970			goto err0;
 971
 972		pbl_ptr = cq->q.pbl_tbl->pa;
 973		page_cnt = cq->q.pbl_info.num_pbes;
 974
 975		cq->ibcq.cqe = chain_entries;
 976		cq->q.db_addr = ctx->dpi_addr + db_offset;
 977	} else {
 978		cq->cq_type = QEDR_CQ_TYPE_KERNEL;
 979
 980		rc = dev->ops->common->chain_alloc(dev->cdev, &cq->pbl,
 981						   &chain_params);
 982		if (rc)
 983			goto err0;
 984
 985		page_cnt = qed_chain_get_page_cnt(&cq->pbl);
 986		pbl_ptr = qed_chain_get_pbl_phys(&cq->pbl);
 987		cq->ibcq.cqe = cq->pbl.capacity;
 988	}
 989
 990	qedr_init_cq_params(cq, ctx, dev, vector, chain_entries, page_cnt,
 991			    pbl_ptr, &params);
 992
 993	rc = dev->ops->rdma_create_cq(dev->rdma_ctx, &params, &icid);
 994	if (rc)
 995		goto err1;
 996
 997	cq->icid = icid;
 998	cq->sig = QEDR_CQ_MAGIC_NUMBER;
 999	spin_lock_init(&cq->cq_lock);
1000
1001	if (udata) {
1002		rc = qedr_copy_cq_uresp(dev, cq, udata, db_offset);
1003		if (rc)
1004			goto err2;
1005
1006		rc = qedr_db_recovery_add(dev, cq->q.db_addr,
1007					  &cq->q.db_rec_data->db_data,
1008					  DB_REC_WIDTH_64B,
1009					  DB_REC_USER);
1010		if (rc)
1011			goto err2;
1012
1013	} else {
1014		/* Generate doorbell address. */
1015		cq->db.data.icid = cq->icid;
1016		cq->db_addr = dev->db_addr + db_offset;
1017		cq->db.data.params = DB_AGG_CMD_MAX <<
1018		    RDMA_PWM_VAL32_DATA_AGG_CMD_SHIFT;
1019
1020		/* point to the very last element, passing it we will toggle */
1021		cq->toggle_cqe = qed_chain_get_last_elem(&cq->pbl);
1022		cq->pbl_toggle = RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK;
1023		cq->latest_cqe = NULL;
1024		consume_cqe(cq);
1025		cq->cq_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
1026
1027		rc = qedr_db_recovery_add(dev, cq->db_addr, &cq->db.data,
1028					  DB_REC_WIDTH_64B, DB_REC_KERNEL);
1029		if (rc)
1030			goto err2;
1031	}
1032
1033	DP_DEBUG(dev, QEDR_MSG_CQ,
1034		 "create cq: icid=0x%0x, addr=%p, size(entries)=0x%0x\n",
1035		 cq->icid, cq, params.cq_size);
1036
1037	return 0;
1038
1039err2:
1040	destroy_iparams.icid = cq->icid;
1041	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &destroy_iparams,
1042				  &destroy_oparams);
1043err1:
1044	if (udata) {
1045		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1046		ib_umem_release(cq->q.umem);
1047		if (cq->q.db_mmap_entry)
1048			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1049	} else {
1050		dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1051	}
1052err0:
1053	return -EINVAL;
1054}
1055
1056#define QEDR_DESTROY_CQ_MAX_ITERATIONS		(10)
1057#define QEDR_DESTROY_CQ_ITER_DURATION		(10)
1058
1059int qedr_destroy_cq(struct ib_cq *ibcq, struct ib_udata *udata)
1060{
1061	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
1062	struct qed_rdma_destroy_cq_out_params oparams;
1063	struct qed_rdma_destroy_cq_in_params iparams;
1064	struct qedr_cq *cq = get_qedr_cq(ibcq);
1065	int iter;
1066
1067	DP_DEBUG(dev, QEDR_MSG_CQ, "destroy cq %p (icid=%d)\n", cq, cq->icid);
1068
1069	cq->destroyed = 1;
1070
1071	/* GSIs CQs are handled by driver, so they don't exist in the FW */
1072	if (cq->cq_type == QEDR_CQ_TYPE_GSI) {
1073		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1074		return 0;
1075	}
1076
1077	iparams.icid = cq->icid;
1078	dev->ops->rdma_destroy_cq(dev->rdma_ctx, &iparams, &oparams);
1079	dev->ops->common->chain_free(dev->cdev, &cq->pbl);
1080
1081	if (udata) {
1082		qedr_free_pbl(dev, &cq->q.pbl_info, cq->q.pbl_tbl);
1083		ib_umem_release(cq->q.umem);
1084
1085		if (cq->q.db_rec_data) {
1086			qedr_db_recovery_del(dev, cq->q.db_addr,
1087					     &cq->q.db_rec_data->db_data);
1088			rdma_user_mmap_entry_remove(cq->q.db_mmap_entry);
1089		}
1090	} else {
1091		qedr_db_recovery_del(dev, cq->db_addr, &cq->db.data);
1092	}
1093
1094	/* We don't want the IRQ handler to handle a non-existing CQ so we
1095	 * wait until all CNQ interrupts, if any, are received. This will always
1096	 * happen and will always happen very fast. If not, then a serious error
1097	 * has occured. That is why we can use a long delay.
1098	 * We spin for a short time so we don’t lose time on context switching
1099	 * in case all the completions are handled in that span. Otherwise
1100	 * we sleep for a while and check again. Since the CNQ may be
1101	 * associated with (only) the current CPU we use msleep to allow the
1102	 * current CPU to be freed.
1103	 * The CNQ notification is increased in qedr_irq_handler().
1104	 */
1105	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1106	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1107		udelay(QEDR_DESTROY_CQ_ITER_DURATION);
1108		iter--;
1109	}
1110
1111	iter = QEDR_DESTROY_CQ_MAX_ITERATIONS;
1112	while (oparams.num_cq_notif != READ_ONCE(cq->cnq_notif) && iter) {
1113		msleep(QEDR_DESTROY_CQ_ITER_DURATION);
1114		iter--;
1115	}
1116
1117	/* Note that we don't need to have explicit code to wait for the
1118	 * completion of the event handler because it is invoked from the EQ.
1119	 * Since the destroy CQ ramrod has also been received on the EQ we can
1120	 * be certain that there's no event handler in process.
1121	 */
1122	return 0;
1123}
1124
1125static inline int get_gid_info_from_table(struct ib_qp *ibqp,
1126					  struct ib_qp_attr *attr,
1127					  int attr_mask,
1128					  struct qed_rdma_modify_qp_in_params
1129					  *qp_params)
1130{
1131	const struct ib_gid_attr *gid_attr;
1132	enum rdma_network_type nw_type;
1133	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
1134	u32 ipv4_addr;
1135	int ret;
1136	int i;
1137
1138	gid_attr = grh->sgid_attr;
1139	ret = rdma_read_gid_l2_fields(gid_attr, &qp_params->vlan_id, NULL);
1140	if (ret)
1141		return ret;
1142
1143	nw_type = rdma_gid_attr_network_type(gid_attr);
1144	switch (nw_type) {
1145	case RDMA_NETWORK_IPV6:
1146		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1147		       sizeof(qp_params->sgid));
1148		memcpy(&qp_params->dgid.bytes[0],
1149		       &grh->dgid,
1150		       sizeof(qp_params->dgid));
1151		qp_params->roce_mode = ROCE_V2_IPV6;
1152		SET_FIELD(qp_params->modify_flags,
1153			  QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1154		break;
1155	case RDMA_NETWORK_ROCE_V1:
1156		memcpy(&qp_params->sgid.bytes[0], &gid_attr->gid.raw[0],
1157		       sizeof(qp_params->sgid));
1158		memcpy(&qp_params->dgid.bytes[0],
1159		       &grh->dgid,
1160		       sizeof(qp_params->dgid));
1161		qp_params->roce_mode = ROCE_V1;
1162		break;
1163	case RDMA_NETWORK_IPV4:
1164		memset(&qp_params->sgid, 0, sizeof(qp_params->sgid));
1165		memset(&qp_params->dgid, 0, sizeof(qp_params->dgid));
1166		ipv4_addr = qedr_get_ipv4_from_gid(gid_attr->gid.raw);
1167		qp_params->sgid.ipv4_addr = ipv4_addr;
1168		ipv4_addr =
1169		    qedr_get_ipv4_from_gid(grh->dgid.raw);
1170		qp_params->dgid.ipv4_addr = ipv4_addr;
1171		SET_FIELD(qp_params->modify_flags,
1172			  QED_ROCE_MODIFY_QP_VALID_ROCE_MODE, 1);
1173		qp_params->roce_mode = ROCE_V2_IPV4;
1174		break;
1175	default:
1176		return -EINVAL;
1177	}
1178
1179	for (i = 0; i < 4; i++) {
1180		qp_params->sgid.dwords[i] = ntohl(qp_params->sgid.dwords[i]);
1181		qp_params->dgid.dwords[i] = ntohl(qp_params->dgid.dwords[i]);
1182	}
1183
1184	if (qp_params->vlan_id >= VLAN_CFI_MASK)
1185		qp_params->vlan_id = 0;
1186
1187	return 0;
1188}
1189
1190static int qedr_check_qp_attrs(struct ib_pd *ibpd, struct qedr_dev *dev,
1191			       struct ib_qp_init_attr *attrs,
1192			       struct ib_udata *udata)
1193{
1194	struct qedr_device_attr *qattr = &dev->attr;
1195
1196	/* QP0... attrs->qp_type == IB_QPT_GSI */
1197	if (attrs->qp_type != IB_QPT_RC &&
1198	    attrs->qp_type != IB_QPT_GSI &&
1199	    attrs->qp_type != IB_QPT_XRC_INI &&
1200	    attrs->qp_type != IB_QPT_XRC_TGT) {
1201		DP_DEBUG(dev, QEDR_MSG_QP,
1202			 "create qp: unsupported qp type=0x%x requested\n",
1203			 attrs->qp_type);
1204		return -EOPNOTSUPP;
1205	}
1206
1207	if (attrs->cap.max_send_wr > qattr->max_sqe) {
1208		DP_ERR(dev,
1209		       "create qp: cannot create a SQ with %d elements (max_send_wr=0x%x)\n",
1210		       attrs->cap.max_send_wr, qattr->max_sqe);
1211		return -EINVAL;
1212	}
1213
1214	if (attrs->cap.max_inline_data > qattr->max_inline) {
1215		DP_ERR(dev,
1216		       "create qp: unsupported inline data size=0x%x requested (max_inline=0x%x)\n",
1217		       attrs->cap.max_inline_data, qattr->max_inline);
1218		return -EINVAL;
1219	}
1220
1221	if (attrs->cap.max_send_sge > qattr->max_sge) {
1222		DP_ERR(dev,
1223		       "create qp: unsupported send_sge=0x%x requested (max_send_sge=0x%x)\n",
1224		       attrs->cap.max_send_sge, qattr->max_sge);
1225		return -EINVAL;
1226	}
1227
1228	if (attrs->cap.max_recv_sge > qattr->max_sge) {
1229		DP_ERR(dev,
1230		       "create qp: unsupported recv_sge=0x%x requested (max_recv_sge=0x%x)\n",
1231		       attrs->cap.max_recv_sge, qattr->max_sge);
1232		return -EINVAL;
1233	}
1234
1235	/* verify consumer QPs are not trying to use GSI QP's CQ.
1236	 * TGT QP isn't associated with RQ/SQ
1237	 */
1238	if ((attrs->qp_type != IB_QPT_GSI) && (dev->gsi_qp_created) &&
1239	    (attrs->qp_type != IB_QPT_XRC_TGT) &&
1240	    (attrs->qp_type != IB_QPT_XRC_INI)) {
1241		struct qedr_cq *send_cq = get_qedr_cq(attrs->send_cq);
1242		struct qedr_cq *recv_cq = get_qedr_cq(attrs->recv_cq);
1243
1244		if ((send_cq->cq_type == QEDR_CQ_TYPE_GSI) ||
1245		    (recv_cq->cq_type == QEDR_CQ_TYPE_GSI)) {
1246			DP_ERR(dev,
1247			       "create qp: consumer QP cannot use GSI CQs.\n");
1248			return -EINVAL;
1249		}
1250	}
1251
1252	return 0;
1253}
1254
1255static int qedr_copy_srq_uresp(struct qedr_dev *dev,
1256			       struct qedr_srq *srq, struct ib_udata *udata)
1257{
1258	struct qedr_create_srq_uresp uresp = {};
1259	int rc;
1260
1261	uresp.srq_id = srq->srq_id;
1262
1263	rc = ib_copy_to_udata(udata, &uresp, sizeof(uresp));
1264	if (rc)
1265		DP_ERR(dev, "create srq: problem copying data to user space\n");
1266
1267	return rc;
1268}
1269
1270static void qedr_copy_rq_uresp(struct qedr_dev *dev,
1271			       struct qedr_create_qp_uresp *uresp,
1272			       struct qedr_qp *qp)
1273{
1274	/* iWARP requires two doorbells per RQ. */
1275	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1276		uresp->rq_db_offset =
1277		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
1278		uresp->rq_db2_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
1279	} else {
1280		uresp->rq_db_offset =
1281		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1282	}
1283
1284	uresp->rq_icid = qp->icid;
1285	if (qp->urq.db_mmap_entry)
1286		uresp->rq_db_rec_addr =
1287			rdma_user_mmap_get_offset(qp->urq.db_mmap_entry);
1288}
1289
1290static void qedr_copy_sq_uresp(struct qedr_dev *dev,
1291			       struct qedr_create_qp_uresp *uresp,
1292			       struct qedr_qp *qp)
1293{
1294	uresp->sq_db_offset = DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1295
1296	/* iWARP uses the same cid for rq and sq */
1297	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1298		uresp->sq_icid = qp->icid;
1299	else
1300		uresp->sq_icid = qp->icid + 1;
1301
1302	if (qp->usq.db_mmap_entry)
1303		uresp->sq_db_rec_addr =
1304			rdma_user_mmap_get_offset(qp->usq.db_mmap_entry);
1305}
1306
1307static int qedr_copy_qp_uresp(struct qedr_dev *dev,
1308			      struct qedr_qp *qp, struct ib_udata *udata,
1309			      struct qedr_create_qp_uresp *uresp)
1310{
1311	int rc;
1312
1313	memset(uresp, 0, sizeof(*uresp));
1314
1315	if (qedr_qp_has_sq(qp))
1316		qedr_copy_sq_uresp(dev, uresp, qp);
1317
1318	if (qedr_qp_has_rq(qp))
1319		qedr_copy_rq_uresp(dev, uresp, qp);
1320
1321	uresp->atomic_supported = dev->atomic_cap != IB_ATOMIC_NONE;
1322	uresp->qp_id = qp->qp_id;
1323
1324	rc = qedr_ib_copy_to_udata(udata, uresp, sizeof(*uresp));
1325	if (rc)
1326		DP_ERR(dev,
1327		       "create qp: failed a copy to user space with qp icid=0x%x.\n",
1328		       qp->icid);
1329
1330	return rc;
1331}
1332
1333static void qedr_reset_qp_hwq_info(struct qedr_qp_hwq_info *qph)
1334{
1335	qed_chain_reset(&qph->pbl);
1336	qph->prod = 0;
1337	qph->cons = 0;
1338	qph->wqe_cons = 0;
1339	qph->db_data.data.value = cpu_to_le16(0);
1340}
1341
1342static void qedr_set_common_qp_params(struct qedr_dev *dev,
1343				      struct qedr_qp *qp,
1344				      struct qedr_pd *pd,
1345				      struct ib_qp_init_attr *attrs)
1346{
1347	spin_lock_init(&qp->q_lock);
1348	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1349		kref_init(&qp->refcnt);
1350		init_completion(&qp->iwarp_cm_comp);
1351		init_completion(&qp->qp_rel_comp);
1352	}
1353
1354	qp->pd = pd;
1355	qp->qp_type = attrs->qp_type;
1356	qp->max_inline_data = attrs->cap.max_inline_data;
1357	qp->state = QED_ROCE_QP_STATE_RESET;
1358
1359	qp->prev_wqe_size = 0;
1360
1361	qp->signaled = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR) ? true : false;
1362	qp->dev = dev;
1363	if (qedr_qp_has_sq(qp)) {
1364		qedr_reset_qp_hwq_info(&qp->sq);
1365		qp->sq.max_sges = attrs->cap.max_send_sge;
1366		qp->sq_cq = get_qedr_cq(attrs->send_cq);
1367		DP_DEBUG(dev, QEDR_MSG_QP,
1368			 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1369			 qp->sq.max_sges, qp->sq_cq->icid);
1370	}
1371
1372	if (attrs->srq)
1373		qp->srq = get_qedr_srq(attrs->srq);
1374
1375	if (qedr_qp_has_rq(qp)) {
1376		qedr_reset_qp_hwq_info(&qp->rq);
1377		qp->rq_cq = get_qedr_cq(attrs->recv_cq);
1378		qp->rq.max_sges = attrs->cap.max_recv_sge;
1379		DP_DEBUG(dev, QEDR_MSG_QP,
1380			 "RQ params:\trq_max_sges = %d, rq_cq_id = %d\n",
1381			 qp->rq.max_sges, qp->rq_cq->icid);
1382	}
1383
1384	DP_DEBUG(dev, QEDR_MSG_QP,
1385		 "QP params:\tpd = %d, qp_type = %d, max_inline_data = %d, state = %d, signaled = %d, use_srq=%d\n",
1386		 pd->pd_id, qp->qp_type, qp->max_inline_data,
1387		 qp->state, qp->signaled, (attrs->srq) ? 1 : 0);
1388	DP_DEBUG(dev, QEDR_MSG_QP,
1389		 "SQ params:\tsq_max_sges = %d, sq_cq_id = %d\n",
1390		 qp->sq.max_sges, qp->sq_cq->icid);
1391}
1392
1393static int qedr_set_roce_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1394{
1395	int rc = 0;
1396
1397	if (qedr_qp_has_sq(qp)) {
1398		qp->sq.db = dev->db_addr +
1399			    DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1400		qp->sq.db_data.data.icid = qp->icid + 1;
1401		rc = qedr_db_recovery_add(dev, qp->sq.db, &qp->sq.db_data,
1402					  DB_REC_WIDTH_32B, DB_REC_KERNEL);
1403		if (rc)
1404			return rc;
1405	}
1406
1407	if (qedr_qp_has_rq(qp)) {
1408		qp->rq.db = dev->db_addr +
1409			    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_ROCE_RQ_PROD);
1410		qp->rq.db_data.data.icid = qp->icid;
1411		rc = qedr_db_recovery_add(dev, qp->rq.db, &qp->rq.db_data,
1412					  DB_REC_WIDTH_32B, DB_REC_KERNEL);
1413		if (rc && qedr_qp_has_sq(qp))
1414			qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
1415	}
1416
1417	return rc;
1418}
1419
1420static int qedr_check_srq_params(struct qedr_dev *dev,
1421				 struct ib_srq_init_attr *attrs,
1422				 struct ib_udata *udata)
1423{
1424	struct qedr_device_attr *qattr = &dev->attr;
1425
1426	if (attrs->attr.max_wr > qattr->max_srq_wr) {
1427		DP_ERR(dev,
1428		       "create srq: unsupported srq_wr=0x%x requested (max_srq_wr=0x%x)\n",
1429		       attrs->attr.max_wr, qattr->max_srq_wr);
1430		return -EINVAL;
1431	}
1432
1433	if (attrs->attr.max_sge > qattr->max_sge) {
1434		DP_ERR(dev,
1435		       "create srq: unsupported sge=0x%x requested (max_srq_sge=0x%x)\n",
1436		       attrs->attr.max_sge, qattr->max_sge);
1437	}
1438
1439	if (!udata && attrs->srq_type == IB_SRQT_XRC) {
1440		DP_ERR(dev, "XRC SRQs are not supported in kernel-space\n");
1441		return -EINVAL;
1442	}
1443
1444	return 0;
1445}
1446
1447static void qedr_free_srq_user_params(struct qedr_srq *srq)
1448{
1449	qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1450	ib_umem_release(srq->usrq.umem);
1451	ib_umem_release(srq->prod_umem);
1452}
1453
1454static void qedr_free_srq_kernel_params(struct qedr_srq *srq)
1455{
1456	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1457	struct qedr_dev *dev = srq->dev;
1458
1459	dev->ops->common->chain_free(dev->cdev, &hw_srq->pbl);
1460
1461	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1462			  hw_srq->virt_prod_pair_addr,
1463			  hw_srq->phy_prod_pair_addr);
1464}
1465
1466static int qedr_init_srq_user_params(struct ib_udata *udata,
1467				     struct qedr_srq *srq,
1468				     struct qedr_create_srq_ureq *ureq,
1469				     int access)
1470{
1471	struct scatterlist *sg;
1472	int rc;
1473
1474	rc = qedr_init_user_queue(udata, srq->dev, &srq->usrq, ureq->srq_addr,
1475				  ureq->srq_len, false, access, 1);
1476	if (rc)
1477		return rc;
1478
1479	srq->prod_umem = ib_umem_get(srq->ibsrq.device, ureq->prod_pair_addr,
1480				     sizeof(struct rdma_srq_producers), access);
1481	if (IS_ERR(srq->prod_umem)) {
1482		qedr_free_pbl(srq->dev, &srq->usrq.pbl_info, srq->usrq.pbl_tbl);
1483		ib_umem_release(srq->usrq.umem);
1484		DP_ERR(srq->dev,
1485		       "create srq: failed ib_umem_get for producer, got %ld\n",
1486		       PTR_ERR(srq->prod_umem));
1487		return PTR_ERR(srq->prod_umem);
1488	}
1489
1490	sg = srq->prod_umem->sgt_append.sgt.sgl;
1491	srq->hw_srq.phy_prod_pair_addr = sg_dma_address(sg);
1492
1493	return 0;
1494}
1495
1496static int qedr_alloc_srq_kernel_params(struct qedr_srq *srq,
1497					struct qedr_dev *dev,
1498					struct ib_srq_init_attr *init_attr)
1499{
1500	struct qedr_srq_hwq_info *hw_srq = &srq->hw_srq;
1501	struct qed_chain_init_params params = {
1502		.mode		= QED_CHAIN_MODE_PBL,
1503		.intended_use	= QED_CHAIN_USE_TO_CONSUME_PRODUCE,
1504		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
1505		.elem_size	= QEDR_SRQ_WQE_ELEM_SIZE,
1506	};
1507	dma_addr_t phy_prod_pair_addr;
1508	u32 num_elems;
1509	void *va;
1510	int rc;
1511
1512	va = dma_alloc_coherent(&dev->pdev->dev,
1513				sizeof(struct rdma_srq_producers),
1514				&phy_prod_pair_addr, GFP_KERNEL);
1515	if (!va) {
1516		DP_ERR(dev,
1517		       "create srq: failed to allocate dma memory for producer\n");
1518		return -ENOMEM;
1519	}
1520
1521	hw_srq->phy_prod_pair_addr = phy_prod_pair_addr;
1522	hw_srq->virt_prod_pair_addr = va;
1523
1524	num_elems = init_attr->attr.max_wr * RDMA_MAX_SRQ_WQE_SIZE;
1525	params.num_elems = num_elems;
1526
1527	rc = dev->ops->common->chain_alloc(dev->cdev, &hw_srq->pbl, &params);
1528	if (rc)
1529		goto err0;
1530
1531	hw_srq->num_elems = num_elems;
1532
1533	return 0;
1534
1535err0:
1536	dma_free_coherent(&dev->pdev->dev, sizeof(struct rdma_srq_producers),
1537			  va, phy_prod_pair_addr);
1538	return rc;
1539}
1540
1541int qedr_create_srq(struct ib_srq *ibsrq, struct ib_srq_init_attr *init_attr,
1542		    struct ib_udata *udata)
1543{
1544	struct qed_rdma_destroy_srq_in_params destroy_in_params;
1545	struct qed_rdma_create_srq_in_params in_params = {};
1546	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1547	struct qed_rdma_create_srq_out_params out_params;
1548	struct qedr_pd *pd = get_qedr_pd(ibsrq->pd);
1549	struct qedr_create_srq_ureq ureq = {};
1550	u64 pbl_base_addr, phy_prod_pair_addr;
1551	struct qedr_srq_hwq_info *hw_srq;
1552	u32 page_cnt, page_size;
1553	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1554	int rc = 0;
1555
1556	DP_DEBUG(dev, QEDR_MSG_QP,
1557		 "create SRQ called from %s (pd %p)\n",
1558		 (udata) ? "User lib" : "kernel", pd);
1559
1560	if (init_attr->srq_type != IB_SRQT_BASIC &&
1561	    init_attr->srq_type != IB_SRQT_XRC)
1562		return -EOPNOTSUPP;
1563
1564	rc = qedr_check_srq_params(dev, init_attr, udata);
1565	if (rc)
1566		return -EINVAL;
1567
1568	srq->dev = dev;
1569	srq->is_xrc = (init_attr->srq_type == IB_SRQT_XRC);
1570	hw_srq = &srq->hw_srq;
1571	spin_lock_init(&srq->lock);
1572
1573	hw_srq->max_wr = init_attr->attr.max_wr;
1574	hw_srq->max_sges = init_attr->attr.max_sge;
1575
1576	if (udata) {
1577		if (ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1578							 udata->inlen))) {
1579			DP_ERR(dev,
1580			       "create srq: problem copying data from user space\n");
1581			goto err0;
1582		}
1583
1584		rc = qedr_init_srq_user_params(udata, srq, &ureq, 0);
1585		if (rc)
1586			goto err0;
1587
1588		page_cnt = srq->usrq.pbl_info.num_pbes;
1589		pbl_base_addr = srq->usrq.pbl_tbl->pa;
1590		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1591		page_size = PAGE_SIZE;
1592	} else {
1593		struct qed_chain *pbl;
1594
1595		rc = qedr_alloc_srq_kernel_params(srq, dev, init_attr);
1596		if (rc)
1597			goto err0;
1598
1599		pbl = &hw_srq->pbl;
1600		page_cnt = qed_chain_get_page_cnt(pbl);
1601		pbl_base_addr = qed_chain_get_pbl_phys(pbl);
1602		phy_prod_pair_addr = hw_srq->phy_prod_pair_addr;
1603		page_size = QED_CHAIN_PAGE_SIZE;
1604	}
1605
1606	in_params.pd_id = pd->pd_id;
1607	in_params.pbl_base_addr = pbl_base_addr;
1608	in_params.prod_pair_addr = phy_prod_pair_addr;
1609	in_params.num_pages = page_cnt;
1610	in_params.page_size = page_size;
1611	if (srq->is_xrc) {
1612		struct qedr_xrcd *xrcd = get_qedr_xrcd(init_attr->ext.xrc.xrcd);
1613		struct qedr_cq *cq = get_qedr_cq(init_attr->ext.cq);
1614
1615		in_params.is_xrc = 1;
1616		in_params.xrcd_id = xrcd->xrcd_id;
1617		in_params.cq_cid = cq->icid;
1618	}
1619
1620	rc = dev->ops->rdma_create_srq(dev->rdma_ctx, &in_params, &out_params);
1621	if (rc)
1622		goto err1;
1623
1624	srq->srq_id = out_params.srq_id;
1625
1626	if (udata) {
1627		rc = qedr_copy_srq_uresp(dev, srq, udata);
1628		if (rc)
1629			goto err2;
1630	}
1631
1632	rc = xa_insert_irq(&dev->srqs, srq->srq_id, srq, GFP_KERNEL);
1633	if (rc)
1634		goto err2;
1635
1636	DP_DEBUG(dev, QEDR_MSG_SRQ,
1637		 "create srq: created srq with srq_id=0x%0x\n", srq->srq_id);
1638	return 0;
1639
1640err2:
1641	destroy_in_params.srq_id = srq->srq_id;
1642
1643	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &destroy_in_params);
1644err1:
1645	if (udata)
1646		qedr_free_srq_user_params(srq);
1647	else
1648		qedr_free_srq_kernel_params(srq);
1649err0:
1650	return -EFAULT;
1651}
1652
1653int qedr_destroy_srq(struct ib_srq *ibsrq, struct ib_udata *udata)
1654{
1655	struct qed_rdma_destroy_srq_in_params in_params = {};
1656	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1657	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1658
1659	xa_erase_irq(&dev->srqs, srq->srq_id);
1660	in_params.srq_id = srq->srq_id;
1661	in_params.is_xrc = srq->is_xrc;
1662	dev->ops->rdma_destroy_srq(dev->rdma_ctx, &in_params);
1663
1664	if (ibsrq->uobject)
1665		qedr_free_srq_user_params(srq);
1666	else
1667		qedr_free_srq_kernel_params(srq);
1668
1669	DP_DEBUG(dev, QEDR_MSG_SRQ,
1670		 "destroy srq: destroyed srq with srq_id=0x%0x\n",
1671		 srq->srq_id);
1672	return 0;
1673}
1674
1675int qedr_modify_srq(struct ib_srq *ibsrq, struct ib_srq_attr *attr,
1676		    enum ib_srq_attr_mask attr_mask, struct ib_udata *udata)
1677{
1678	struct qed_rdma_modify_srq_in_params in_params = {};
1679	struct qedr_dev *dev = get_qedr_dev(ibsrq->device);
1680	struct qedr_srq *srq = get_qedr_srq(ibsrq);
1681	int rc;
1682
1683	if (attr_mask & IB_SRQ_MAX_WR) {
1684		DP_ERR(dev,
1685		       "modify srq: invalid attribute mask=0x%x specified for %p\n",
1686		       attr_mask, srq);
1687		return -EINVAL;
1688	}
1689
1690	if (attr_mask & IB_SRQ_LIMIT) {
1691		if (attr->srq_limit >= srq->hw_srq.max_wr) {
1692			DP_ERR(dev,
1693			       "modify srq: invalid srq_limit=0x%x (max_srq_limit=0x%x)\n",
1694			       attr->srq_limit, srq->hw_srq.max_wr);
1695			return -EINVAL;
1696		}
1697
1698		in_params.srq_id = srq->srq_id;
1699		in_params.wqe_limit = attr->srq_limit;
1700		rc = dev->ops->rdma_modify_srq(dev->rdma_ctx, &in_params);
1701		if (rc)
1702			return rc;
1703	}
1704
1705	srq->srq_limit = attr->srq_limit;
1706
1707	DP_DEBUG(dev, QEDR_MSG_SRQ,
1708		 "modify srq: modified srq with srq_id=0x%0x\n", srq->srq_id);
1709
1710	return 0;
1711}
1712
1713static enum qed_rdma_qp_type qedr_ib_to_qed_qp_type(enum ib_qp_type ib_qp_type)
1714{
1715	switch (ib_qp_type) {
1716	case IB_QPT_RC:
1717		return QED_RDMA_QP_TYPE_RC;
1718	case IB_QPT_XRC_INI:
1719		return QED_RDMA_QP_TYPE_XRC_INI;
1720	case IB_QPT_XRC_TGT:
1721		return QED_RDMA_QP_TYPE_XRC_TGT;
1722	default:
1723		return QED_RDMA_QP_TYPE_INVAL;
1724	}
1725}
1726
1727static inline void
1728qedr_init_common_qp_in_params(struct qedr_dev *dev,
1729			      struct qedr_pd *pd,
1730			      struct qedr_qp *qp,
1731			      struct ib_qp_init_attr *attrs,
1732			      bool fmr_and_reserved_lkey,
1733			      struct qed_rdma_create_qp_in_params *params)
1734{
1735	/* QP handle to be written in an async event */
1736	params->qp_handle_async_lo = lower_32_bits((uintptr_t) qp);
1737	params->qp_handle_async_hi = upper_32_bits((uintptr_t) qp);
1738
1739	params->signal_all = (attrs->sq_sig_type == IB_SIGNAL_ALL_WR);
1740	params->fmr_and_reserved_lkey = fmr_and_reserved_lkey;
1741	params->qp_type = qedr_ib_to_qed_qp_type(attrs->qp_type);
1742	params->stats_queue = 0;
1743
1744	if (pd) {
1745		params->pd = pd->pd_id;
1746		params->dpi = pd->uctx ? pd->uctx->dpi : dev->dpi;
1747	}
1748
1749	if (qedr_qp_has_sq(qp))
1750		params->sq_cq_id = get_qedr_cq(attrs->send_cq)->icid;
1751
1752	if (qedr_qp_has_rq(qp))
1753		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1754
1755	if (qedr_qp_has_srq(qp)) {
1756		params->rq_cq_id = get_qedr_cq(attrs->recv_cq)->icid;
1757		params->srq_id = qp->srq->srq_id;
1758		params->use_srq = true;
1759	} else {
1760		params->srq_id = 0;
1761		params->use_srq = false;
1762	}
1763}
1764
1765static inline void qedr_qp_user_print(struct qedr_dev *dev, struct qedr_qp *qp)
1766{
1767	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: successfully created user QP. "
1768		 "qp=%p. "
1769		 "sq_addr=0x%llx, "
1770		 "sq_len=%zd, "
1771		 "rq_addr=0x%llx, "
1772		 "rq_len=%zd"
1773		 "\n",
1774		 qp,
1775		 qedr_qp_has_sq(qp) ? qp->usq.buf_addr : 0x0,
1776		 qedr_qp_has_sq(qp) ? qp->usq.buf_len : 0,
1777		 qedr_qp_has_rq(qp) ? qp->urq.buf_addr : 0x0,
1778		 qedr_qp_has_sq(qp) ? qp->urq.buf_len : 0);
1779}
1780
1781static inline void
1782qedr_iwarp_populate_user_qp(struct qedr_dev *dev,
1783			    struct qedr_qp *qp,
1784			    struct qed_rdma_create_qp_out_params *out_params)
1785{
1786	qp->usq.pbl_tbl->va = out_params->sq_pbl_virt;
1787	qp->usq.pbl_tbl->pa = out_params->sq_pbl_phys;
1788
1789	qedr_populate_pbls(dev, qp->usq.umem, qp->usq.pbl_tbl,
1790			   &qp->usq.pbl_info, FW_PAGE_SHIFT);
1791	if (!qp->srq) {
1792		qp->urq.pbl_tbl->va = out_params->rq_pbl_virt;
1793		qp->urq.pbl_tbl->pa = out_params->rq_pbl_phys;
1794	}
1795
1796	qedr_populate_pbls(dev, qp->urq.umem, qp->urq.pbl_tbl,
1797			   &qp->urq.pbl_info, FW_PAGE_SHIFT);
1798}
1799
1800static void qedr_cleanup_user(struct qedr_dev *dev,
1801			      struct qedr_ucontext *ctx,
1802			      struct qedr_qp *qp)
1803{
1804	if (qedr_qp_has_sq(qp)) {
1805		ib_umem_release(qp->usq.umem);
1806		qp->usq.umem = NULL;
1807	}
1808
1809	if (qedr_qp_has_rq(qp)) {
1810		ib_umem_release(qp->urq.umem);
1811		qp->urq.umem = NULL;
1812	}
1813
1814	if (rdma_protocol_roce(&dev->ibdev, 1)) {
1815		qedr_free_pbl(dev, &qp->usq.pbl_info, qp->usq.pbl_tbl);
1816		qedr_free_pbl(dev, &qp->urq.pbl_info, qp->urq.pbl_tbl);
1817	} else {
1818		kfree(qp->usq.pbl_tbl);
1819		kfree(qp->urq.pbl_tbl);
1820	}
1821
1822	if (qp->usq.db_rec_data) {
1823		qedr_db_recovery_del(dev, qp->usq.db_addr,
1824				     &qp->usq.db_rec_data->db_data);
1825		rdma_user_mmap_entry_remove(qp->usq.db_mmap_entry);
1826	}
1827
1828	if (qp->urq.db_rec_data) {
1829		qedr_db_recovery_del(dev, qp->urq.db_addr,
1830				     &qp->urq.db_rec_data->db_data);
1831		rdma_user_mmap_entry_remove(qp->urq.db_mmap_entry);
1832	}
1833
1834	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1835		qedr_db_recovery_del(dev, qp->urq.db_rec_db2_addr,
1836				     &qp->urq.db_rec_db2_data);
1837}
1838
1839static int qedr_create_user_qp(struct qedr_dev *dev,
1840			       struct qedr_qp *qp,
1841			       struct ib_pd *ibpd,
1842			       struct ib_udata *udata,
1843			       struct ib_qp_init_attr *attrs)
1844{
1845	struct qed_rdma_create_qp_in_params in_params;
1846	struct qed_rdma_create_qp_out_params out_params;
1847	struct qedr_create_qp_uresp uresp = {};
1848	struct qedr_create_qp_ureq ureq = {};
1849	int alloc_and_init = rdma_protocol_roce(&dev->ibdev, 1);
1850	struct qedr_ucontext *ctx = NULL;
1851	struct qedr_pd *pd = NULL;
1852	int rc = 0;
1853
1854	qp->create_type = QEDR_QP_CREATE_USER;
1855
1856	if (ibpd) {
1857		pd = get_qedr_pd(ibpd);
1858		ctx = pd->uctx;
1859	}
1860
1861	if (udata) {
1862		rc = ib_copy_from_udata(&ureq, udata, min(sizeof(ureq),
1863					udata->inlen));
1864		if (rc) {
1865			DP_ERR(dev, "Problem copying data from user space\n");
1866			return rc;
1867		}
1868	}
1869
1870	if (qedr_qp_has_sq(qp)) {
1871		/* SQ - read access only (0) */
1872		rc = qedr_init_user_queue(udata, dev, &qp->usq, ureq.sq_addr,
1873					  ureq.sq_len, true, 0, alloc_and_init);
1874		if (rc)
1875			return rc;
1876	}
1877
1878	if (qedr_qp_has_rq(qp)) {
1879		/* RQ - read access only (0) */
1880		rc = qedr_init_user_queue(udata, dev, &qp->urq, ureq.rq_addr,
1881					  ureq.rq_len, true, 0, alloc_and_init);
1882		if (rc)
1883			return rc;
1884	}
1885
1886	memset(&in_params, 0, sizeof(in_params));
1887	qedr_init_common_qp_in_params(dev, pd, qp, attrs, false, &in_params);
1888	in_params.qp_handle_lo = ureq.qp_handle_lo;
1889	in_params.qp_handle_hi = ureq.qp_handle_hi;
1890
1891	if (qp->qp_type == IB_QPT_XRC_TGT) {
1892		struct qedr_xrcd *xrcd = get_qedr_xrcd(attrs->xrcd);
1893
1894		in_params.xrcd_id = xrcd->xrcd_id;
1895		in_params.qp_handle_lo = qp->qp_id;
1896		in_params.use_srq = 1;
1897	}
1898
1899	if (qedr_qp_has_sq(qp)) {
1900		in_params.sq_num_pages = qp->usq.pbl_info.num_pbes;
1901		in_params.sq_pbl_ptr = qp->usq.pbl_tbl->pa;
1902	}
1903
1904	if (qedr_qp_has_rq(qp)) {
1905		in_params.rq_num_pages = qp->urq.pbl_info.num_pbes;
1906		in_params.rq_pbl_ptr = qp->urq.pbl_tbl->pa;
1907	}
1908
1909	if (ctx)
1910		SET_FIELD(in_params.flags, QED_ROCE_EDPM_MODE, ctx->edpm_mode);
1911
1912	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
1913					      &in_params, &out_params);
1914
1915	if (!qp->qed_qp) {
1916		rc = -ENOMEM;
1917		goto err1;
1918	}
1919
1920	if (rdma_protocol_iwarp(&dev->ibdev, 1))
1921		qedr_iwarp_populate_user_qp(dev, qp, &out_params);
1922
1923	qp->qp_id = out_params.qp_id;
1924	qp->icid = out_params.icid;
1925
1926	if (udata) {
1927		rc = qedr_copy_qp_uresp(dev, qp, udata, &uresp);
1928		if (rc)
1929			goto err;
1930	}
1931
1932	/* db offset was calculated in copy_qp_uresp, now set in the user q */
1933	if (qedr_qp_has_sq(qp)) {
1934		qp->usq.db_addr = ctx->dpi_addr + uresp.sq_db_offset;
1935		qp->sq.max_wr = attrs->cap.max_send_wr;
1936		rc = qedr_db_recovery_add(dev, qp->usq.db_addr,
1937					  &qp->usq.db_rec_data->db_data,
1938					  DB_REC_WIDTH_32B,
1939					  DB_REC_USER);
1940		if (rc)
1941			goto err;
1942	}
1943
1944	if (qedr_qp_has_rq(qp)) {
1945		qp->urq.db_addr = ctx->dpi_addr + uresp.rq_db_offset;
1946		qp->rq.max_wr = attrs->cap.max_recv_wr;
1947		rc = qedr_db_recovery_add(dev, qp->urq.db_addr,
1948					  &qp->urq.db_rec_data->db_data,
1949					  DB_REC_WIDTH_32B,
1950					  DB_REC_USER);
1951		if (rc)
1952			goto err;
1953	}
1954
1955	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
1956		qp->urq.db_rec_db2_addr = ctx->dpi_addr + uresp.rq_db2_offset;
1957
1958		/* calculate the db_rec_db2 data since it is constant so no
1959		 * need to reflect from user
1960		 */
1961		qp->urq.db_rec_db2_data.data.icid = cpu_to_le16(qp->icid);
1962		qp->urq.db_rec_db2_data.data.value =
1963			cpu_to_le16(DQ_TCM_IWARP_POST_RQ_CF_CMD);
1964
1965		rc = qedr_db_recovery_add(dev, qp->urq.db_rec_db2_addr,
1966					  &qp->urq.db_rec_db2_data,
1967					  DB_REC_WIDTH_32B,
1968					  DB_REC_USER);
1969		if (rc)
1970			goto err;
1971	}
1972	qedr_qp_user_print(dev, qp);
1973	return rc;
1974err:
1975	rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
1976	if (rc)
1977		DP_ERR(dev, "create qp: fatal fault. rc=%d", rc);
1978
1979err1:
1980	qedr_cleanup_user(dev, ctx, qp);
1981	return rc;
1982}
1983
1984static int qedr_set_iwarp_db_info(struct qedr_dev *dev, struct qedr_qp *qp)
1985{
1986	int rc;
1987
1988	qp->sq.db = dev->db_addr +
1989	    DB_ADDR_SHIFT(DQ_PWM_OFFSET_XCM_RDMA_SQ_PROD);
1990	qp->sq.db_data.data.icid = qp->icid;
1991
1992	rc = qedr_db_recovery_add(dev, qp->sq.db,
1993				  &qp->sq.db_data,
1994				  DB_REC_WIDTH_32B,
1995				  DB_REC_KERNEL);
1996	if (rc)
1997		return rc;
1998
1999	qp->rq.db = dev->db_addr +
2000		    DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_IWARP_RQ_PROD);
2001	qp->rq.db_data.data.icid = qp->icid;
2002	qp->rq.iwarp_db2 = dev->db_addr +
2003			   DB_ADDR_SHIFT(DQ_PWM_OFFSET_TCM_FLAGS);
2004	qp->rq.iwarp_db2_data.data.icid = qp->icid;
2005	qp->rq.iwarp_db2_data.data.value = DQ_TCM_IWARP_POST_RQ_CF_CMD;
2006
2007	rc = qedr_db_recovery_add(dev, qp->rq.db,
2008				  &qp->rq.db_data,
2009				  DB_REC_WIDTH_32B,
2010				  DB_REC_KERNEL);
2011	if (rc)
2012		return rc;
2013
2014	rc = qedr_db_recovery_add(dev, qp->rq.iwarp_db2,
2015				  &qp->rq.iwarp_db2_data,
2016				  DB_REC_WIDTH_32B,
2017				  DB_REC_KERNEL);
2018	return rc;
2019}
2020
2021static int
2022qedr_roce_create_kernel_qp(struct qedr_dev *dev,
2023			   struct qedr_qp *qp,
2024			   struct qed_rdma_create_qp_in_params *in_params,
2025			   u32 n_sq_elems, u32 n_rq_elems)
2026{
2027	struct qed_rdma_create_qp_out_params out_params;
2028	struct qed_chain_init_params params = {
2029		.mode		= QED_CHAIN_MODE_PBL,
2030		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
2031	};
2032	int rc;
2033
2034	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2035	params.num_elems = n_sq_elems;
2036	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2037
2038	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2039	if (rc)
2040		return rc;
2041
2042	in_params->sq_num_pages = qed_chain_get_page_cnt(&qp->sq.pbl);
2043	in_params->sq_pbl_ptr = qed_chain_get_pbl_phys(&qp->sq.pbl);
2044
2045	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2046	params.num_elems = n_rq_elems;
2047	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2048
2049	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2050	if (rc)
2051		return rc;
2052
2053	in_params->rq_num_pages = qed_chain_get_page_cnt(&qp->rq.pbl);
2054	in_params->rq_pbl_ptr = qed_chain_get_pbl_phys(&qp->rq.pbl);
2055
2056	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2057					      in_params, &out_params);
2058
2059	if (!qp->qed_qp)
2060		return -EINVAL;
2061
2062	qp->qp_id = out_params.qp_id;
2063	qp->icid = out_params.icid;
2064
2065	return qedr_set_roce_db_info(dev, qp);
2066}
2067
2068static int
2069qedr_iwarp_create_kernel_qp(struct qedr_dev *dev,
2070			    struct qedr_qp *qp,
2071			    struct qed_rdma_create_qp_in_params *in_params,
2072			    u32 n_sq_elems, u32 n_rq_elems)
2073{
2074	struct qed_rdma_create_qp_out_params out_params;
2075	struct qed_chain_init_params params = {
2076		.mode		= QED_CHAIN_MODE_PBL,
2077		.cnt_type	= QED_CHAIN_CNT_TYPE_U32,
2078	};
2079	int rc;
2080
2081	in_params->sq_num_pages = QED_CHAIN_PAGE_CNT(n_sq_elems,
2082						     QEDR_SQE_ELEMENT_SIZE,
2083						     QED_CHAIN_PAGE_SIZE,
2084						     QED_CHAIN_MODE_PBL);
2085	in_params->rq_num_pages = QED_CHAIN_PAGE_CNT(n_rq_elems,
2086						     QEDR_RQE_ELEMENT_SIZE,
2087						     QED_CHAIN_PAGE_SIZE,
2088						     QED_CHAIN_MODE_PBL);
2089
2090	qp->qed_qp = dev->ops->rdma_create_qp(dev->rdma_ctx,
2091					      in_params, &out_params);
2092
2093	if (!qp->qed_qp)
2094		return -EINVAL;
2095
2096	/* Now we allocate the chain */
2097
2098	params.intended_use = QED_CHAIN_USE_TO_PRODUCE;
2099	params.num_elems = n_sq_elems;
2100	params.elem_size = QEDR_SQE_ELEMENT_SIZE;
2101	params.ext_pbl_virt = out_params.sq_pbl_virt;
2102	params.ext_pbl_phys = out_params.sq_pbl_phys;
2103
2104	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->sq.pbl, &params);
2105	if (rc)
2106		goto err;
2107
2108	params.intended_use = QED_CHAIN_USE_TO_CONSUME_PRODUCE;
2109	params.num_elems = n_rq_elems;
2110	params.elem_size = QEDR_RQE_ELEMENT_SIZE;
2111	params.ext_pbl_virt = out_params.rq_pbl_virt;
2112	params.ext_pbl_phys = out_params.rq_pbl_phys;
2113
2114	rc = dev->ops->common->chain_alloc(dev->cdev, &qp->rq.pbl, &params);
2115	if (rc)
2116		goto err;
2117
2118	qp->qp_id = out_params.qp_id;
2119	qp->icid = out_params.icid;
2120
2121	return qedr_set_iwarp_db_info(dev, qp);
2122
2123err:
2124	dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2125
2126	return rc;
2127}
2128
2129static void qedr_cleanup_kernel(struct qedr_dev *dev, struct qedr_qp *qp)
2130{
2131	dev->ops->common->chain_free(dev->cdev, &qp->sq.pbl);
2132	kfree(qp->wqe_wr_id);
2133
2134	dev->ops->common->chain_free(dev->cdev, &qp->rq.pbl);
2135	kfree(qp->rqe_wr_id);
2136
2137	/* GSI qp is not registered to db mechanism so no need to delete */
2138	if (qp->qp_type == IB_QPT_GSI)
2139		return;
2140
2141	qedr_db_recovery_del(dev, qp->sq.db, &qp->sq.db_data);
2142
2143	if (!qp->srq) {
2144		qedr_db_recovery_del(dev, qp->rq.db, &qp->rq.db_data);
2145
2146		if (rdma_protocol_iwarp(&dev->ibdev, 1))
2147			qedr_db_recovery_del(dev, qp->rq.iwarp_db2,
2148					     &qp->rq.iwarp_db2_data);
2149	}
2150}
2151
2152static int qedr_create_kernel_qp(struct qedr_dev *dev,
2153				 struct qedr_qp *qp,
2154				 struct ib_pd *ibpd,
2155				 struct ib_qp_init_attr *attrs)
2156{
2157	struct qed_rdma_create_qp_in_params in_params;
2158	struct qedr_pd *pd = get_qedr_pd(ibpd);
2159	int rc = -EINVAL;
2160	u32 n_rq_elems;
2161	u32 n_sq_elems;
2162	u32 n_sq_entries;
2163
2164	memset(&in_params, 0, sizeof(in_params));
2165	qp->create_type = QEDR_QP_CREATE_KERNEL;
2166
2167	/* A single work request may take up to QEDR_MAX_SQ_WQE_SIZE elements in
2168	 * the ring. The ring should allow at least a single WR, even if the
2169	 * user requested none, due to allocation issues.
2170	 * We should add an extra WR since the prod and cons indices of
2171	 * wqe_wr_id are managed in such a way that the WQ is considered full
2172	 * when (prod+1)%max_wr==cons. We currently don't do that because we
2173	 * double the number of entries due an iSER issue that pushes far more
2174	 * WRs than indicated. If we decline its ib_post_send() then we get
2175	 * error prints in the dmesg we'd like to avoid.
2176	 */
2177	qp->sq.max_wr = min_t(u32, attrs->cap.max_send_wr * dev->wq_multiplier,
2178			      dev->attr.max_sqe);
2179
2180	qp->wqe_wr_id = kcalloc(qp->sq.max_wr, sizeof(*qp->wqe_wr_id),
2181				GFP_KERNEL);
2182	if (!qp->wqe_wr_id) {
2183		DP_ERR(dev, "create qp: failed SQ shadow memory allocation\n");
2184		return -ENOMEM;
2185	}
2186
2187	/* QP handle to be written in CQE */
2188	in_params.qp_handle_lo = lower_32_bits((uintptr_t) qp);
2189	in_params.qp_handle_hi = upper_32_bits((uintptr_t) qp);
2190
2191	/* A single work request may take up to QEDR_MAX_RQ_WQE_SIZE elements in
2192	 * the ring. There ring should allow at least a single WR, even if the
2193	 * user requested none, due to allocation issues.
2194	 */
2195	qp->rq.max_wr = (u16) max_t(u32, attrs->cap.max_recv_wr, 1);
2196
2197	/* Allocate driver internal RQ array */
2198	qp->rqe_wr_id = kcalloc(qp->rq.max_wr, sizeof(*qp->rqe_wr_id),
2199				GFP_KERNEL);
2200	if (!qp->rqe_wr_id) {
2201		DP_ERR(dev,
2202		       "create qp: failed RQ shadow memory allocation\n");
2203		kfree(qp->wqe_wr_id);
2204		return -ENOMEM;
2205	}
2206
2207	qedr_init_common_qp_in_params(dev, pd, qp, attrs, true, &in_params);
2208
2209	n_sq_entries = attrs->cap.max_send_wr;
2210	n_sq_entries = min_t(u32, n_sq_entries, dev->attr.max_sqe);
2211	n_sq_entries = max_t(u32, n_sq_entries, 1);
2212	n_sq_elems = n_sq_entries * QEDR_MAX_SQE_ELEMENTS_PER_SQE;
2213
2214	n_rq_elems = qp->rq.max_wr * QEDR_MAX_RQE_ELEMENTS_PER_RQE;
2215
2216	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2217		rc = qedr_iwarp_create_kernel_qp(dev, qp, &in_params,
2218						 n_sq_elems, n_rq_elems);
2219	else
2220		rc = qedr_roce_create_kernel_qp(dev, qp, &in_params,
2221						n_sq_elems, n_rq_elems);
2222	if (rc)
2223		qedr_cleanup_kernel(dev, qp);
2224
2225	return rc;
2226}
2227
2228static int qedr_free_qp_resources(struct qedr_dev *dev, struct qedr_qp *qp,
2229				  struct ib_udata *udata)
2230{
2231	struct qedr_ucontext *ctx =
2232		rdma_udata_to_drv_context(udata, struct qedr_ucontext,
2233					  ibucontext);
2234	int rc;
2235
2236	if (qp->qp_type != IB_QPT_GSI) {
2237		rc = dev->ops->rdma_destroy_qp(dev->rdma_ctx, qp->qed_qp);
2238		if (rc)
2239			return rc;
2240	}
2241
2242	if (qp->create_type == QEDR_QP_CREATE_USER)
2243		qedr_cleanup_user(dev, ctx, qp);
2244	else
2245		qedr_cleanup_kernel(dev, qp);
2246
2247	return 0;
2248}
2249
2250int qedr_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attrs,
2251		   struct ib_udata *udata)
2252{
2253	struct qedr_xrcd *xrcd = NULL;
2254	struct ib_pd *ibpd = ibqp->pd;
2255	struct qedr_pd *pd = get_qedr_pd(ibpd);
2256	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
2257	struct qedr_qp *qp = get_qedr_qp(ibqp);
2258	int rc = 0;
2259
2260	if (attrs->create_flags)
2261		return -EOPNOTSUPP;
2262
2263	if (attrs->qp_type == IB_QPT_XRC_TGT)
2264		xrcd = get_qedr_xrcd(attrs->xrcd);
2265	else
2266		pd = get_qedr_pd(ibpd);
2267
2268	DP_DEBUG(dev, QEDR_MSG_QP, "create qp: called from %s, pd=%p\n",
2269		 udata ? "user library" : "kernel", pd);
2270
2271	rc = qedr_check_qp_attrs(ibpd, dev, attrs, udata);
2272	if (rc)
2273		return rc;
2274
2275	DP_DEBUG(dev, QEDR_MSG_QP,
2276		 "create qp: called from %s, event_handler=%p, eepd=%p sq_cq=%p, sq_icid=%d, rq_cq=%p, rq_icid=%d\n",
2277		 udata ? "user library" : "kernel", attrs->event_handler, pd,
2278		 get_qedr_cq(attrs->send_cq),
2279		 get_qedr_cq(attrs->send_cq)->icid,
2280		 get_qedr_cq(attrs->recv_cq),
2281		 attrs->recv_cq ? get_qedr_cq(attrs->recv_cq)->icid : 0);
2282
2283	qedr_set_common_qp_params(dev, qp, pd, attrs);
2284
2285	if (attrs->qp_type == IB_QPT_GSI)
2286		return qedr_create_gsi_qp(dev, attrs, qp);
2287
2288	if (udata || xrcd)
2289		rc = qedr_create_user_qp(dev, qp, ibpd, udata, attrs);
2290	else
2291		rc = qedr_create_kernel_qp(dev, qp, ibpd, attrs);
2292
2293	if (rc)
2294		return rc;
2295
2296	qp->ibqp.qp_num = qp->qp_id;
2297
2298	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2299		rc = xa_insert(&dev->qps, qp->qp_id, qp, GFP_KERNEL);
2300		if (rc)
2301			goto out_free_qp_resources;
2302	}
2303
2304	return 0;
2305
2306out_free_qp_resources:
2307	qedr_free_qp_resources(dev, qp, udata);
2308	return -EFAULT;
2309}
2310
2311static enum ib_qp_state qedr_get_ibqp_state(enum qed_roce_qp_state qp_state)
2312{
2313	switch (qp_state) {
2314	case QED_ROCE_QP_STATE_RESET:
2315		return IB_QPS_RESET;
2316	case QED_ROCE_QP_STATE_INIT:
2317		return IB_QPS_INIT;
2318	case QED_ROCE_QP_STATE_RTR:
2319		return IB_QPS_RTR;
2320	case QED_ROCE_QP_STATE_RTS:
2321		return IB_QPS_RTS;
2322	case QED_ROCE_QP_STATE_SQD:
2323		return IB_QPS_SQD;
2324	case QED_ROCE_QP_STATE_ERR:
2325		return IB_QPS_ERR;
2326	case QED_ROCE_QP_STATE_SQE:
2327		return IB_QPS_SQE;
2328	}
2329	return IB_QPS_ERR;
2330}
2331
2332static enum qed_roce_qp_state qedr_get_state_from_ibqp(
2333					enum ib_qp_state qp_state)
2334{
2335	switch (qp_state) {
2336	case IB_QPS_RESET:
2337		return QED_ROCE_QP_STATE_RESET;
2338	case IB_QPS_INIT:
2339		return QED_ROCE_QP_STATE_INIT;
2340	case IB_QPS_RTR:
2341		return QED_ROCE_QP_STATE_RTR;
2342	case IB_QPS_RTS:
2343		return QED_ROCE_QP_STATE_RTS;
2344	case IB_QPS_SQD:
2345		return QED_ROCE_QP_STATE_SQD;
2346	case IB_QPS_ERR:
2347		return QED_ROCE_QP_STATE_ERR;
2348	default:
2349		return QED_ROCE_QP_STATE_ERR;
2350	}
2351}
2352
2353static int qedr_update_qp_state(struct qedr_dev *dev,
2354				struct qedr_qp *qp,
2355				enum qed_roce_qp_state cur_state,
2356				enum qed_roce_qp_state new_state)
2357{
2358	int status = 0;
2359
2360	if (new_state == cur_state)
2361		return 0;
2362
2363	switch (cur_state) {
2364	case QED_ROCE_QP_STATE_RESET:
2365		switch (new_state) {
2366		case QED_ROCE_QP_STATE_INIT:
2367			break;
2368		default:
2369			status = -EINVAL;
2370			break;
2371		}
2372		break;
2373	case QED_ROCE_QP_STATE_INIT:
2374		switch (new_state) {
2375		case QED_ROCE_QP_STATE_RTR:
2376			/* Update doorbell (in case post_recv was
2377			 * done before move to RTR)
2378			 */
2379
2380			if (rdma_protocol_roce(&dev->ibdev, 1)) {
2381				writel(qp->rq.db_data.raw, qp->rq.db);
2382			}
2383			break;
2384		case QED_ROCE_QP_STATE_ERR:
2385			break;
2386		default:
2387			/* Invalid state change. */
2388			status = -EINVAL;
2389			break;
2390		}
2391		break;
2392	case QED_ROCE_QP_STATE_RTR:
2393		/* RTR->XXX */
2394		switch (new_state) {
2395		case QED_ROCE_QP_STATE_RTS:
2396			break;
2397		case QED_ROCE_QP_STATE_ERR:
2398			break;
2399		default:
2400			/* Invalid state change. */
2401			status = -EINVAL;
2402			break;
2403		}
2404		break;
2405	case QED_ROCE_QP_STATE_RTS:
2406		/* RTS->XXX */
2407		switch (new_state) {
2408		case QED_ROCE_QP_STATE_SQD:
2409			break;
2410		case QED_ROCE_QP_STATE_ERR:
2411			break;
2412		default:
2413			/* Invalid state change. */
2414			status = -EINVAL;
2415			break;
2416		}
2417		break;
2418	case QED_ROCE_QP_STATE_SQD:
2419		/* SQD->XXX */
2420		switch (new_state) {
2421		case QED_ROCE_QP_STATE_RTS:
2422		case QED_ROCE_QP_STATE_ERR:
2423			break;
2424		default:
2425			/* Invalid state change. */
2426			status = -EINVAL;
2427			break;
2428		}
2429		break;
2430	case QED_ROCE_QP_STATE_ERR:
2431		/* ERR->XXX */
2432		switch (new_state) {
2433		case QED_ROCE_QP_STATE_RESET:
2434			if ((qp->rq.prod != qp->rq.cons) ||
2435			    (qp->sq.prod != qp->sq.cons)) {
2436				DP_NOTICE(dev,
2437					  "Error->Reset with rq/sq not empty rq.prod=%x rq.cons=%x sq.prod=%x sq.cons=%x\n",
2438					  qp->rq.prod, qp->rq.cons, qp->sq.prod,
2439					  qp->sq.cons);
2440				status = -EINVAL;
2441			}
2442			break;
2443		default:
2444			status = -EINVAL;
2445			break;
2446		}
2447		break;
2448	default:
2449		status = -EINVAL;
2450		break;
2451	}
2452
2453	return status;
2454}
2455
2456int qedr_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
2457		   int attr_mask, struct ib_udata *udata)
2458{
2459	struct qedr_qp *qp = get_qedr_qp(ibqp);
2460	struct qed_rdma_modify_qp_in_params qp_params = { 0 };
2461	struct qedr_dev *dev = get_qedr_dev(&qp->dev->ibdev);
2462	const struct ib_global_route *grh = rdma_ah_read_grh(&attr->ah_attr);
2463	enum ib_qp_state old_qp_state, new_qp_state;
2464	enum qed_roce_qp_state cur_state;
2465	int rc = 0;
2466
2467	DP_DEBUG(dev, QEDR_MSG_QP,
2468		 "modify qp: qp %p attr_mask=0x%x, state=%d", qp, attr_mask,
2469		 attr->qp_state);
2470
2471	if (attr_mask & ~IB_QP_ATTR_STANDARD_BITS)
2472		return -EOPNOTSUPP;
2473
2474	old_qp_state = qedr_get_ibqp_state(qp->state);
2475	if (attr_mask & IB_QP_STATE)
2476		new_qp_state = attr->qp_state;
2477	else
2478		new_qp_state = old_qp_state;
2479
2480	if (rdma_protocol_roce(&dev->ibdev, 1)) {
2481		if (!ib_modify_qp_is_ok(old_qp_state, new_qp_state,
2482					ibqp->qp_type, attr_mask)) {
2483			DP_ERR(dev,
2484			       "modify qp: invalid attribute mask=0x%x specified for\n"
2485			       "qpn=0x%x of type=0x%x old_qp_state=0x%x, new_qp_state=0x%x\n",
2486			       attr_mask, qp->qp_id, ibqp->qp_type,
2487			       old_qp_state, new_qp_state);
2488			rc = -EINVAL;
2489			goto err;
2490		}
2491	}
2492
2493	/* Translate the masks... */
2494	if (attr_mask & IB_QP_STATE) {
2495		SET_FIELD(qp_params.modify_flags,
2496			  QED_RDMA_MODIFY_QP_VALID_NEW_STATE, 1);
2497		qp_params.new_state = qedr_get_state_from_ibqp(attr->qp_state);
2498	}
2499
2500	if (attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY)
2501		qp_params.sqd_async = true;
2502
2503	if (attr_mask & IB_QP_PKEY_INDEX) {
2504		SET_FIELD(qp_params.modify_flags,
2505			  QED_ROCE_MODIFY_QP_VALID_PKEY, 1);
2506		if (attr->pkey_index >= QEDR_ROCE_PKEY_TABLE_LEN) {
2507			rc = -EINVAL;
2508			goto err;
2509		}
2510
2511		qp_params.pkey = QEDR_ROCE_PKEY_DEFAULT;
2512	}
2513
2514	if (attr_mask & IB_QP_QKEY)
2515		qp->qkey = attr->qkey;
2516
2517	if (attr_mask & IB_QP_ACCESS_FLAGS) {
2518		SET_FIELD(qp_params.modify_flags,
2519			  QED_RDMA_MODIFY_QP_VALID_RDMA_OPS_EN, 1);
2520		qp_params.incoming_rdma_read_en = attr->qp_access_flags &
2521						  IB_ACCESS_REMOTE_READ;
2522		qp_params.incoming_rdma_write_en = attr->qp_access_flags &
2523						   IB_ACCESS_REMOTE_WRITE;
2524		qp_params.incoming_atomic_en = attr->qp_access_flags &
2525					       IB_ACCESS_REMOTE_ATOMIC;
2526	}
2527
2528	if (attr_mask & (IB_QP_AV | IB_QP_PATH_MTU)) {
2529		if (rdma_protocol_iwarp(&dev->ibdev, 1))
2530			return -EINVAL;
2531
2532		if (attr_mask & IB_QP_PATH_MTU) {
2533			if (attr->path_mtu < IB_MTU_256 ||
2534			    attr->path_mtu > IB_MTU_4096) {
2535				pr_err("error: Only MTU sizes of 256, 512, 1024, 2048 and 4096 are supported by RoCE\n");
2536				rc = -EINVAL;
2537				goto err;
2538			}
2539			qp->mtu = min(ib_mtu_enum_to_int(attr->path_mtu),
2540				      ib_mtu_enum_to_int(iboe_get_mtu
2541							 (dev->ndev->mtu)));
2542		}
2543
2544		if (!qp->mtu) {
2545			qp->mtu =
2546			ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2547			pr_err("Fixing zeroed MTU to qp->mtu = %d\n", qp->mtu);
2548		}
2549
2550		SET_FIELD(qp_params.modify_flags,
2551			  QED_ROCE_MODIFY_QP_VALID_ADDRESS_VECTOR, 1);
2552
2553		qp_params.traffic_class_tos = grh->traffic_class;
2554		qp_params.flow_label = grh->flow_label;
2555		qp_params.hop_limit_ttl = grh->hop_limit;
2556
2557		qp->sgid_idx = grh->sgid_index;
2558
2559		rc = get_gid_info_from_table(ibqp, attr, attr_mask, &qp_params);
2560		if (rc) {
2561			DP_ERR(dev,
2562			       "modify qp: problems with GID index %d (rc=%d)\n",
2563			       grh->sgid_index, rc);
2564			return rc;
2565		}
2566
2567		rc = qedr_get_dmac(dev, &attr->ah_attr,
2568				   qp_params.remote_mac_addr);
2569		if (rc)
2570			return rc;
2571
2572		qp_params.use_local_mac = true;
2573		ether_addr_copy(qp_params.local_mac_addr, dev->ndev->dev_addr);
2574
2575		DP_DEBUG(dev, QEDR_MSG_QP, "dgid=%x:%x:%x:%x\n",
2576			 qp_params.dgid.dwords[0], qp_params.dgid.dwords[1],
2577			 qp_params.dgid.dwords[2], qp_params.dgid.dwords[3]);
2578		DP_DEBUG(dev, QEDR_MSG_QP, "sgid=%x:%x:%x:%x\n",
2579			 qp_params.sgid.dwords[0], qp_params.sgid.dwords[1],
2580			 qp_params.sgid.dwords[2], qp_params.sgid.dwords[3]);
2581		DP_DEBUG(dev, QEDR_MSG_QP, "remote_mac=[%pM]\n",
2582			 qp_params.remote_mac_addr);
2583
2584		qp_params.mtu = qp->mtu;
2585		qp_params.lb_indication = false;
2586	}
2587
2588	if (!qp_params.mtu) {
2589		/* Stay with current MTU */
2590		if (qp->mtu)
2591			qp_params.mtu = qp->mtu;
2592		else
2593			qp_params.mtu =
2594			    ib_mtu_enum_to_int(iboe_get_mtu(dev->ndev->mtu));
2595	}
2596
2597	if (attr_mask & IB_QP_TIMEOUT) {
2598		SET_FIELD(qp_params.modify_flags,
2599			  QED_ROCE_MODIFY_QP_VALID_ACK_TIMEOUT, 1);
2600
2601		/* The received timeout value is an exponent used like this:
2602		 *    "12.7.34 LOCAL ACK TIMEOUT
2603		 *    Value representing the transport (ACK) timeout for use by
2604		 *    the remote, expressed as: 4.096 * 2^timeout [usec]"
2605		 * The FW expects timeout in msec so we need to divide the usec
2606		 * result by 1000. We'll approximate 1000~2^10, and 4.096 ~ 2^2,
2607		 * so we get: 2^2 * 2^timeout / 2^10 = 2^(timeout - 8).
2608		 * The value of zero means infinite so we use a 'max_t' to make
2609		 * sure that sub 1 msec values will be configured as 1 msec.
2610		 */
2611		if (attr->timeout)
2612			qp_params.ack_timeout =
2613					1 << max_t(int, attr->timeout - 8, 0);
2614		else
2615			qp_params.ack_timeout = 0;
2616
2617		qp->timeout = attr->timeout;
2618	}
2619
2620	if (attr_mask & IB_QP_RETRY_CNT) {
2621		SET_FIELD(qp_params.modify_flags,
2622			  QED_ROCE_MODIFY_QP_VALID_RETRY_CNT, 1);
2623		qp_params.retry_cnt = attr->retry_cnt;
2624	}
2625
2626	if (attr_mask & IB_QP_RNR_RETRY) {
2627		SET_FIELD(qp_params.modify_flags,
2628			  QED_ROCE_MODIFY_QP_VALID_RNR_RETRY_CNT, 1);
2629		qp_params.rnr_retry_cnt = attr->rnr_retry;
2630	}
2631
2632	if (attr_mask & IB_QP_RQ_PSN) {
2633		SET_FIELD(qp_params.modify_flags,
2634			  QED_ROCE_MODIFY_QP_VALID_RQ_PSN, 1);
2635		qp_params.rq_psn = attr->rq_psn;
2636		qp->rq_psn = attr->rq_psn;
2637	}
2638
2639	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
2640		if (attr->max_rd_atomic > dev->attr.max_qp_req_rd_atomic_resc) {
2641			rc = -EINVAL;
2642			DP_ERR(dev,
2643			       "unsupported max_rd_atomic=%d, supported=%d\n",
2644			       attr->max_rd_atomic,
2645			       dev->attr.max_qp_req_rd_atomic_resc);
2646			goto err;
2647		}
2648
2649		SET_FIELD(qp_params.modify_flags,
2650			  QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_REQ, 1);
2651		qp_params.max_rd_atomic_req = attr->max_rd_atomic;
2652	}
2653
2654	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
2655		SET_FIELD(qp_params.modify_flags,
2656			  QED_ROCE_MODIFY_QP_VALID_MIN_RNR_NAK_TIMER, 1);
2657		qp_params.min_rnr_nak_timer = attr->min_rnr_timer;
2658	}
2659
2660	if (attr_mask & IB_QP_SQ_PSN) {
2661		SET_FIELD(qp_params.modify_flags,
2662			  QED_ROCE_MODIFY_QP_VALID_SQ_PSN, 1);
2663		qp_params.sq_psn = attr->sq_psn;
2664		qp->sq_psn = attr->sq_psn;
2665	}
2666
2667	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
2668		if (attr->max_dest_rd_atomic >
2669		    dev->attr.max_qp_resp_rd_atomic_resc) {
2670			DP_ERR(dev,
2671			       "unsupported max_dest_rd_atomic=%d, supported=%d\n",
2672			       attr->max_dest_rd_atomic,
2673			       dev->attr.max_qp_resp_rd_atomic_resc);
2674
2675			rc = -EINVAL;
2676			goto err;
2677		}
2678
2679		SET_FIELD(qp_params.modify_flags,
2680			  QED_RDMA_MODIFY_QP_VALID_MAX_RD_ATOMIC_RESP, 1);
2681		qp_params.max_rd_atomic_resp = attr->max_dest_rd_atomic;
2682	}
2683
2684	if (attr_mask & IB_QP_DEST_QPN) {
2685		SET_FIELD(qp_params.modify_flags,
2686			  QED_ROCE_MODIFY_QP_VALID_DEST_QP, 1);
2687
2688		qp_params.dest_qp = attr->dest_qp_num;
2689		qp->dest_qp_num = attr->dest_qp_num;
2690	}
2691
2692	cur_state = qp->state;
2693
2694	/* Update the QP state before the actual ramrod to prevent a race with
2695	 * fast path. Modifying the QP state to error will cause the device to
2696	 * flush the CQEs and while polling the flushed CQEs will considered as
2697	 * a potential issue if the QP isn't in error state.
2698	 */
2699	if ((attr_mask & IB_QP_STATE) && qp->qp_type != IB_QPT_GSI &&
2700	    !udata && qp_params.new_state == QED_ROCE_QP_STATE_ERR)
2701		qp->state = QED_ROCE_QP_STATE_ERR;
2702
2703	if (qp->qp_type != IB_QPT_GSI)
2704		rc = dev->ops->rdma_modify_qp(dev->rdma_ctx,
2705					      qp->qed_qp, &qp_params);
2706
2707	if (attr_mask & IB_QP_STATE) {
2708		if ((qp->qp_type != IB_QPT_GSI) && (!udata))
2709			rc = qedr_update_qp_state(dev, qp, cur_state,
2710						  qp_params.new_state);
2711		qp->state = qp_params.new_state;
2712	}
2713
2714err:
2715	return rc;
2716}
2717
2718static int qedr_to_ib_qp_acc_flags(struct qed_rdma_query_qp_out_params *params)
2719{
2720	int ib_qp_acc_flags = 0;
2721
2722	if (params->incoming_rdma_write_en)
2723		ib_qp_acc_flags |= IB_ACCESS_REMOTE_WRITE;
2724	if (params->incoming_rdma_read_en)
2725		ib_qp_acc_flags |= IB_ACCESS_REMOTE_READ;
2726	if (params->incoming_atomic_en)
2727		ib_qp_acc_flags |= IB_ACCESS_REMOTE_ATOMIC;
2728	ib_qp_acc_flags |= IB_ACCESS_LOCAL_WRITE;
2729	return ib_qp_acc_flags;
2730}
2731
2732int qedr_query_qp(struct ib_qp *ibqp,
2733		  struct ib_qp_attr *qp_attr,
2734		  int attr_mask, struct ib_qp_init_attr *qp_init_attr)
2735{
2736	struct qed_rdma_query_qp_out_params params;
2737	struct qedr_qp *qp = get_qedr_qp(ibqp);
2738	struct qedr_dev *dev = qp->dev;
2739	int rc = 0;
2740
2741	memset(&params, 0, sizeof(params));
2742	memset(qp_attr, 0, sizeof(*qp_attr));
2743	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
2744
2745	if (qp->qp_type != IB_QPT_GSI) {
2746		rc = dev->ops->rdma_query_qp(dev->rdma_ctx, qp->qed_qp, &params);
2747		if (rc)
2748			goto err;
2749		qp_attr->qp_state = qedr_get_ibqp_state(params.state);
2750	} else {
2751		qp_attr->qp_state = qedr_get_ibqp_state(QED_ROCE_QP_STATE_RTS);
2752	}
2753
2754	qp_attr->cur_qp_state = qedr_get_ibqp_state(params.state);
2755	qp_attr->path_mtu = ib_mtu_int_to_enum(params.mtu);
2756	qp_attr->path_mig_state = IB_MIG_MIGRATED;
2757	qp_attr->rq_psn = params.rq_psn;
2758	qp_attr->sq_psn = params.sq_psn;
2759	qp_attr->dest_qp_num = params.dest_qp;
2760
2761	qp_attr->qp_access_flags = qedr_to_ib_qp_acc_flags(&params);
2762
2763	qp_attr->cap.max_send_wr = qp->sq.max_wr;
2764	qp_attr->cap.max_recv_wr = qp->rq.max_wr;
2765	qp_attr->cap.max_send_sge = qp->sq.max_sges;
2766	qp_attr->cap.max_recv_sge = qp->rq.max_sges;
2767	qp_attr->cap.max_inline_data = dev->attr.max_inline;
2768	qp_init_attr->cap = qp_attr->cap;
2769
2770	qp_attr->ah_attr.type = RDMA_AH_ATTR_TYPE_ROCE;
2771	rdma_ah_set_grh(&qp_attr->ah_attr, NULL,
2772			params.flow_label, qp->sgid_idx,
2773			params.hop_limit_ttl, params.traffic_class_tos);
2774	rdma_ah_set_dgid_raw(&qp_attr->ah_attr, &params.dgid.bytes[0]);
2775	rdma_ah_set_port_num(&qp_attr->ah_attr, 1);
2776	rdma_ah_set_sl(&qp_attr->ah_attr, 0);
2777	qp_attr->timeout = qp->timeout;
2778	qp_attr->rnr_retry = params.rnr_retry;
2779	qp_attr->retry_cnt = params.retry_cnt;
2780	qp_attr->min_rnr_timer = params.min_rnr_nak_timer;
2781	qp_attr->pkey_index = params.pkey_index;
2782	qp_attr->port_num = 1;
2783	rdma_ah_set_path_bits(&qp_attr->ah_attr, 0);
2784	rdma_ah_set_static_rate(&qp_attr->ah_attr, 0);
2785	qp_attr->alt_pkey_index = 0;
2786	qp_attr->alt_port_num = 0;
2787	qp_attr->alt_timeout = 0;
2788	memset(&qp_attr->alt_ah_attr, 0, sizeof(qp_attr->alt_ah_attr));
2789
2790	qp_attr->sq_draining = (params.state == QED_ROCE_QP_STATE_SQD) ? 1 : 0;
2791	qp_attr->max_dest_rd_atomic = params.max_dest_rd_atomic;
2792	qp_attr->max_rd_atomic = params.max_rd_atomic;
2793	qp_attr->en_sqd_async_notify = (params.sqd_async) ? 1 : 0;
2794
2795	DP_DEBUG(dev, QEDR_MSG_QP, "QEDR_QUERY_QP: max_inline_data=%d\n",
2796		 qp_attr->cap.max_inline_data);
2797
2798err:
2799	return rc;
2800}
2801
2802int qedr_destroy_qp(struct ib_qp *ibqp, struct ib_udata *udata)
2803{
2804	struct qedr_qp *qp = get_qedr_qp(ibqp);
2805	struct qedr_dev *dev = qp->dev;
2806	struct ib_qp_attr attr;
2807	int attr_mask = 0;
2808
2809	DP_DEBUG(dev, QEDR_MSG_QP, "destroy qp: destroying %p, qp type=%d\n",
2810		 qp, qp->qp_type);
2811
2812	if (rdma_protocol_roce(&dev->ibdev, 1)) {
2813		if ((qp->state != QED_ROCE_QP_STATE_RESET) &&
2814		    (qp->state != QED_ROCE_QP_STATE_ERR) &&
2815		    (qp->state != QED_ROCE_QP_STATE_INIT)) {
2816
2817			attr.qp_state = IB_QPS_ERR;
2818			attr_mask |= IB_QP_STATE;
2819
2820			/* Change the QP state to ERROR */
2821			qedr_modify_qp(ibqp, &attr, attr_mask, NULL);
2822		}
2823	} else {
2824		/* If connection establishment started the WAIT_FOR_CONNECT
2825		 * bit will be on and we need to Wait for the establishment
2826		 * to complete before destroying the qp.
2827		 */
2828		if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_CONNECT,
2829				     &qp->iwarp_cm_flags))
2830			wait_for_completion(&qp->iwarp_cm_comp);
2831
2832		/* If graceful disconnect started, the WAIT_FOR_DISCONNECT
2833		 * bit will be on, and we need to wait for the disconnect to
2834		 * complete before continuing. We can use the same completion,
2835		 * iwarp_cm_comp, since this is the only place that waits for
2836		 * this completion and it is sequential. In addition,
2837		 * disconnect can't occur before the connection is fully
2838		 * established, therefore if WAIT_FOR_DISCONNECT is on it
2839		 * means WAIT_FOR_CONNECT is also on and the completion for
2840		 * CONNECT already occurred.
2841		 */
2842		if (test_and_set_bit(QEDR_IWARP_CM_WAIT_FOR_DISCONNECT,
2843				     &qp->iwarp_cm_flags))
2844			wait_for_completion(&qp->iwarp_cm_comp);
2845	}
2846
2847	if (qp->qp_type == IB_QPT_GSI)
2848		qedr_destroy_gsi_qp(dev);
2849
2850	/* We need to remove the entry from the xarray before we release the
2851	 * qp_id to avoid a race of the qp_id being reallocated and failing
2852	 * on xa_insert
2853	 */
2854	if (rdma_protocol_iwarp(&dev->ibdev, 1))
2855		xa_erase(&dev->qps, qp->qp_id);
2856
2857	qedr_free_qp_resources(dev, qp, udata);
2858
2859	if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
2860		qedr_iw_qp_rem_ref(&qp->ibqp);
2861		wait_for_completion(&qp->qp_rel_comp);
2862	}
2863
2864	return 0;
2865}
2866
2867int qedr_create_ah(struct ib_ah *ibah, struct rdma_ah_init_attr *init_attr,
2868		   struct ib_udata *udata)
2869{
2870	struct qedr_ah *ah = get_qedr_ah(ibah);
2871
2872	rdma_copy_ah_attr(&ah->attr, init_attr->ah_attr);
2873
2874	return 0;
2875}
2876
2877int qedr_destroy_ah(struct ib_ah *ibah, u32 flags)
2878{
2879	struct qedr_ah *ah = get_qedr_ah(ibah);
2880
2881	rdma_destroy_ah_attr(&ah->attr);
2882	return 0;
2883}
2884
2885static void free_mr_info(struct qedr_dev *dev, struct mr_info *info)
2886{
2887	struct qedr_pbl *pbl, *tmp;
2888
2889	if (info->pbl_table)
2890		list_add_tail(&info->pbl_table->list_entry,
2891			      &info->free_pbl_list);
2892
2893	if (!list_empty(&info->inuse_pbl_list))
2894		list_splice(&info->inuse_pbl_list, &info->free_pbl_list);
2895
2896	list_for_each_entry_safe(pbl, tmp, &info->free_pbl_list, list_entry) {
2897		list_del(&pbl->list_entry);
2898		qedr_free_pbl(dev, &info->pbl_info, pbl);
2899	}
2900}
2901
2902static int init_mr_info(struct qedr_dev *dev, struct mr_info *info,
2903			size_t page_list_len, bool two_layered)
2904{
2905	struct qedr_pbl *tmp;
2906	int rc;
2907
2908	INIT_LIST_HEAD(&info->free_pbl_list);
2909	INIT_LIST_HEAD(&info->inuse_pbl_list);
2910
2911	rc = qedr_prepare_pbl_tbl(dev, &info->pbl_info,
2912				  page_list_len, two_layered);
2913	if (rc)
2914		goto done;
2915
2916	info->pbl_table = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2917	if (IS_ERR(info->pbl_table)) {
2918		rc = PTR_ERR(info->pbl_table);
2919		goto done;
2920	}
2921
2922	DP_DEBUG(dev, QEDR_MSG_MR, "pbl_table_pa = %pa\n",
2923		 &info->pbl_table->pa);
2924
2925	/* in usual case we use 2 PBLs, so we add one to free
2926	 * list and allocating another one
2927	 */
2928	tmp = qedr_alloc_pbl_tbl(dev, &info->pbl_info, GFP_KERNEL);
2929	if (IS_ERR(tmp)) {
2930		DP_DEBUG(dev, QEDR_MSG_MR, "Extra PBL is not allocated\n");
2931		goto done;
2932	}
2933
2934	list_add_tail(&tmp->list_entry, &info->free_pbl_list);
2935
2936	DP_DEBUG(dev, QEDR_MSG_MR, "extra pbl_table_pa = %pa\n", &tmp->pa);
2937
2938done:
2939	if (rc)
2940		free_mr_info(dev, info);
2941
2942	return rc;
2943}
2944
2945struct ib_mr *qedr_reg_user_mr(struct ib_pd *ibpd, u64 start, u64 len,
2946			       u64 usr_addr, int acc, struct ib_udata *udata)
2947{
2948	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
2949	struct qedr_mr *mr;
2950	struct qedr_pd *pd;
2951	int rc = -ENOMEM;
2952
2953	pd = get_qedr_pd(ibpd);
2954	DP_DEBUG(dev, QEDR_MSG_MR,
2955		 "qedr_register user mr pd = %d start = %lld, len = %lld, usr_addr = %lld, acc = %d\n",
2956		 pd->pd_id, start, len, usr_addr, acc);
2957
2958	if (acc & IB_ACCESS_REMOTE_WRITE && !(acc & IB_ACCESS_LOCAL_WRITE))
2959		return ERR_PTR(-EINVAL);
2960
2961	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
2962	if (!mr)
2963		return ERR_PTR(rc);
2964
2965	mr->type = QEDR_MR_USER;
2966
2967	mr->umem = ib_umem_get(ibpd->device, start, len, acc);
2968	if (IS_ERR(mr->umem)) {
2969		rc = -EFAULT;
2970		goto err0;
2971	}
2972
2973	rc = init_mr_info(dev, &mr->info,
2974			  ib_umem_num_dma_blocks(mr->umem, PAGE_SIZE), 1);
2975	if (rc)
2976		goto err1;
2977
2978	qedr_populate_pbls(dev, mr->umem, mr->info.pbl_table,
2979			   &mr->info.pbl_info, PAGE_SHIFT);
2980
2981	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
2982	if (rc) {
2983		if (rc == -EINVAL)
2984			DP_ERR(dev, "Out of MR resources\n");
2985		else
2986			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
2987
2988		goto err1;
2989	}
2990
2991	/* Index only, 18 bit long, lkey = itid << 8 | key */
2992	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
2993	mr->hw_mr.key = 0;
2994	mr->hw_mr.pd = pd->pd_id;
2995	mr->hw_mr.local_read = 1;
2996	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
2997	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
2998	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
2999	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3000	mr->hw_mr.mw_bind = false;
3001	mr->hw_mr.pbl_ptr = mr->info.pbl_table[0].pa;
3002	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3003	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3004	mr->hw_mr.page_size_log = PAGE_SHIFT;
3005	mr->hw_mr.length = len;
3006	mr->hw_mr.vaddr = usr_addr;
3007	mr->hw_mr.phy_mr = false;
3008	mr->hw_mr.dma_mr = false;
3009
3010	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3011	if (rc) {
3012		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3013		goto err2;
3014	}
3015
3016	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3017	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3018	    mr->hw_mr.remote_atomic)
3019		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3020
3021	DP_DEBUG(dev, QEDR_MSG_MR, "register user mr lkey: %x\n",
3022		 mr->ibmr.lkey);
3023	return &mr->ibmr;
3024
3025err2:
3026	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3027err1:
3028	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3029err0:
3030	kfree(mr);
3031	return ERR_PTR(rc);
3032}
3033
3034int qedr_dereg_mr(struct ib_mr *ib_mr, struct ib_udata *udata)
3035{
3036	struct qedr_mr *mr = get_qedr_mr(ib_mr);
3037	struct qedr_dev *dev = get_qedr_dev(ib_mr->device);
3038	int rc = 0;
3039
3040	rc = dev->ops->rdma_deregister_tid(dev->rdma_ctx, mr->hw_mr.itid);
3041	if (rc)
3042		return rc;
3043
3044	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3045
3046	if (mr->type != QEDR_MR_DMA)
3047		free_mr_info(dev, &mr->info);
3048
3049	/* it could be user registered memory. */
3050	ib_umem_release(mr->umem);
3051
3052	kfree(mr);
3053
3054	return rc;
3055}
3056
3057static struct qedr_mr *__qedr_alloc_mr(struct ib_pd *ibpd,
3058				       int max_page_list_len)
3059{
3060	struct qedr_pd *pd = get_qedr_pd(ibpd);
3061	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3062	struct qedr_mr *mr;
3063	int rc = -ENOMEM;
3064
3065	DP_DEBUG(dev, QEDR_MSG_MR,
3066		 "qedr_alloc_frmr pd = %d max_page_list_len= %d\n", pd->pd_id,
3067		 max_page_list_len);
3068
3069	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3070	if (!mr)
3071		return ERR_PTR(rc);
3072
3073	mr->dev = dev;
3074	mr->type = QEDR_MR_FRMR;
3075
3076	rc = init_mr_info(dev, &mr->info, max_page_list_len, 1);
3077	if (rc)
3078		goto err0;
3079
3080	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3081	if (rc) {
3082		if (rc == -EINVAL)
3083			DP_ERR(dev, "Out of MR resources\n");
3084		else
3085			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3086
3087		goto err1;
3088	}
3089
3090	/* Index only, 18 bit long, lkey = itid << 8 | key */
3091	mr->hw_mr.tid_type = QED_RDMA_TID_FMR;
3092	mr->hw_mr.key = 0;
3093	mr->hw_mr.pd = pd->pd_id;
3094	mr->hw_mr.local_read = 1;
3095	mr->hw_mr.local_write = 0;
3096	mr->hw_mr.remote_read = 0;
3097	mr->hw_mr.remote_write = 0;
3098	mr->hw_mr.remote_atomic = 0;
3099	mr->hw_mr.mw_bind = false;
3100	mr->hw_mr.pbl_ptr = 0;
3101	mr->hw_mr.pbl_two_level = mr->info.pbl_info.two_layered;
3102	mr->hw_mr.pbl_page_size_log = ilog2(mr->info.pbl_info.pbl_size);
3103	mr->hw_mr.length = 0;
3104	mr->hw_mr.vaddr = 0;
3105	mr->hw_mr.phy_mr = true;
3106	mr->hw_mr.dma_mr = false;
3107
3108	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3109	if (rc) {
3110		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3111		goto err2;
3112	}
3113
3114	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3115	mr->ibmr.rkey = mr->ibmr.lkey;
3116
3117	DP_DEBUG(dev, QEDR_MSG_MR, "alloc frmr: %x\n", mr->ibmr.lkey);
3118	return mr;
3119
3120err2:
3121	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3122err1:
3123	qedr_free_pbl(dev, &mr->info.pbl_info, mr->info.pbl_table);
3124err0:
3125	kfree(mr);
3126	return ERR_PTR(rc);
3127}
3128
3129struct ib_mr *qedr_alloc_mr(struct ib_pd *ibpd, enum ib_mr_type mr_type,
3130			    u32 max_num_sg)
3131{
3132	struct qedr_mr *mr;
3133
3134	if (mr_type != IB_MR_TYPE_MEM_REG)
3135		return ERR_PTR(-EINVAL);
3136
3137	mr = __qedr_alloc_mr(ibpd, max_num_sg);
3138
3139	if (IS_ERR(mr))
3140		return ERR_PTR(-EINVAL);
3141
3142	return &mr->ibmr;
3143}
3144
3145static int qedr_set_page(struct ib_mr *ibmr, u64 addr)
3146{
3147	struct qedr_mr *mr = get_qedr_mr(ibmr);
3148	struct qedr_pbl *pbl_table;
3149	struct regpair *pbe;
3150	u32 pbes_in_page;
3151
3152	if (unlikely(mr->npages == mr->info.pbl_info.num_pbes)) {
3153		DP_ERR(mr->dev, "qedr_set_page fails when %d\n", mr->npages);
3154		return -ENOMEM;
3155	}
3156
3157	DP_DEBUG(mr->dev, QEDR_MSG_MR, "qedr_set_page pages[%d] = 0x%llx\n",
3158		 mr->npages, addr);
3159
3160	pbes_in_page = mr->info.pbl_info.pbl_size / sizeof(u64);
3161	pbl_table = mr->info.pbl_table + (mr->npages / pbes_in_page);
3162	pbe = (struct regpair *)pbl_table->va;
3163	pbe +=  mr->npages % pbes_in_page;
3164	pbe->lo = cpu_to_le32((u32)addr);
3165	pbe->hi = cpu_to_le32((u32)upper_32_bits(addr));
3166
3167	mr->npages++;
3168
3169	return 0;
3170}
3171
3172static void handle_completed_mrs(struct qedr_dev *dev, struct mr_info *info)
3173{
3174	int work = info->completed - info->completed_handled - 1;
3175
3176	DP_DEBUG(dev, QEDR_MSG_MR, "Special FMR work = %d\n", work);
3177	while (work-- > 0 && !list_empty(&info->inuse_pbl_list)) {
3178		struct qedr_pbl *pbl;
3179
3180		/* Free all the page list that are possible to be freed
3181		 * (all the ones that were invalidated), under the assumption
3182		 * that if an FMR was completed successfully that means that
3183		 * if there was an invalidate operation before it also ended
3184		 */
3185		pbl = list_first_entry(&info->inuse_pbl_list,
3186				       struct qedr_pbl, list_entry);
3187		list_move_tail(&pbl->list_entry, &info->free_pbl_list);
3188		info->completed_handled++;
3189	}
3190}
3191
3192int qedr_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg,
3193		   int sg_nents, unsigned int *sg_offset)
3194{
3195	struct qedr_mr *mr = get_qedr_mr(ibmr);
3196
3197	mr->npages = 0;
3198
3199	handle_completed_mrs(mr->dev, &mr->info);
3200	return ib_sg_to_pages(ibmr, sg, sg_nents, NULL, qedr_set_page);
3201}
3202
3203struct ib_mr *qedr_get_dma_mr(struct ib_pd *ibpd, int acc)
3204{
3205	struct qedr_dev *dev = get_qedr_dev(ibpd->device);
3206	struct qedr_pd *pd = get_qedr_pd(ibpd);
3207	struct qedr_mr *mr;
3208	int rc;
3209
3210	mr = kzalloc(sizeof(*mr), GFP_KERNEL);
3211	if (!mr)
3212		return ERR_PTR(-ENOMEM);
3213
3214	mr->type = QEDR_MR_DMA;
3215
3216	rc = dev->ops->rdma_alloc_tid(dev->rdma_ctx, &mr->hw_mr.itid);
3217	if (rc) {
3218		if (rc == -EINVAL)
3219			DP_ERR(dev, "Out of MR resources\n");
3220		else
3221			DP_ERR(dev, "roce alloc tid returned error %d\n", rc);
3222
3223		goto err1;
3224	}
3225
3226	/* index only, 18 bit long, lkey = itid << 8 | key */
3227	mr->hw_mr.tid_type = QED_RDMA_TID_REGISTERED_MR;
3228	mr->hw_mr.pd = pd->pd_id;
3229	mr->hw_mr.local_read = 1;
3230	mr->hw_mr.local_write = (acc & IB_ACCESS_LOCAL_WRITE) ? 1 : 0;
3231	mr->hw_mr.remote_read = (acc & IB_ACCESS_REMOTE_READ) ? 1 : 0;
3232	mr->hw_mr.remote_write = (acc & IB_ACCESS_REMOTE_WRITE) ? 1 : 0;
3233	mr->hw_mr.remote_atomic = (acc & IB_ACCESS_REMOTE_ATOMIC) ? 1 : 0;
3234	mr->hw_mr.dma_mr = true;
3235
3236	rc = dev->ops->rdma_register_tid(dev->rdma_ctx, &mr->hw_mr);
3237	if (rc) {
3238		DP_ERR(dev, "roce register tid returned an error %d\n", rc);
3239		goto err2;
3240	}
3241
3242	mr->ibmr.lkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3243	if (mr->hw_mr.remote_write || mr->hw_mr.remote_read ||
3244	    mr->hw_mr.remote_atomic)
3245		mr->ibmr.rkey = mr->hw_mr.itid << 8 | mr->hw_mr.key;
3246
3247	DP_DEBUG(dev, QEDR_MSG_MR, "get dma mr: lkey = %x\n", mr->ibmr.lkey);
3248	return &mr->ibmr;
3249
3250err2:
3251	dev->ops->rdma_free_tid(dev->rdma_ctx, mr->hw_mr.itid);
3252err1:
3253	kfree(mr);
3254	return ERR_PTR(rc);
3255}
3256
3257static inline int qedr_wq_is_full(struct qedr_qp_hwq_info *wq)
3258{
3259	return (((wq->prod + 1) % wq->max_wr) == wq->cons);
3260}
3261
3262static int sge_data_len(struct ib_sge *sg_list, int num_sge)
3263{
3264	int i, len = 0;
3265
3266	for (i = 0; i < num_sge; i++)
3267		len += sg_list[i].length;
3268
3269	return len;
3270}
3271
3272static void swap_wqe_data64(u64 *p)
3273{
3274	int i;
3275
3276	for (i = 0; i < QEDR_SQE_ELEMENT_SIZE / sizeof(u64); i++, p++)
3277		*p = cpu_to_be64(cpu_to_le64(*p));
3278}
3279
3280static u32 qedr_prepare_sq_inline_data(struct qedr_dev *dev,
3281				       struct qedr_qp *qp, u8 *wqe_size,
3282				       const struct ib_send_wr *wr,
3283				       const struct ib_send_wr **bad_wr,
3284				       u8 *bits, u8 bit)
3285{
3286	u32 data_size = sge_data_len(wr->sg_list, wr->num_sge);
3287	char *seg_prt, *wqe;
3288	int i, seg_siz;
3289
3290	if (data_size > ROCE_REQ_MAX_INLINE_DATA_SIZE) {
3291		DP_ERR(dev, "Too much inline data in WR: %d\n", data_size);
3292		*bad_wr = wr;
3293		return 0;
3294	}
3295
3296	if (!data_size)
3297		return data_size;
3298
3299	*bits |= bit;
3300
3301	seg_prt = NULL;
3302	wqe = NULL;
3303	seg_siz = 0;
3304
3305	/* Copy data inline */
3306	for (i = 0; i < wr->num_sge; i++) {
3307		u32 len = wr->sg_list[i].length;
3308		void *src = (void *)(uintptr_t)wr->sg_list[i].addr;
3309
3310		while (len > 0) {
3311			u32 cur;
3312
3313			/* New segment required */
3314			if (!seg_siz) {
3315				wqe = (char *)qed_chain_produce(&qp->sq.pbl);
3316				seg_prt = wqe;
3317				seg_siz = sizeof(struct rdma_sq_common_wqe);
3318				(*wqe_size)++;
3319			}
3320
3321			/* Calculate currently allowed length */
3322			cur = min_t(u32, len, seg_siz);
3323			memcpy(seg_prt, src, cur);
3324
3325			/* Update segment variables */
3326			seg_prt += cur;
3327			seg_siz -= cur;
3328
3329			/* Update sge variables */
3330			src += cur;
3331			len -= cur;
3332
3333			/* Swap fully-completed segments */
3334			if (!seg_siz)
3335				swap_wqe_data64((u64 *)wqe);
3336		}
3337	}
3338
3339	/* swap last not completed segment */
3340	if (seg_siz)
3341		swap_wqe_data64((u64 *)wqe);
3342
3343	return data_size;
3344}
3345
3346#define RQ_SGE_SET(sge, vaddr, vlength, vflags)			\
3347	do {							\
3348		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3349		(sge)->length = cpu_to_le32(vlength);		\
3350		(sge)->flags = cpu_to_le32(vflags);		\
3351	} while (0)
3352
3353#define SRQ_HDR_SET(hdr, vwr_id, num_sge)			\
3354	do {							\
3355		DMA_REGPAIR_LE(hdr->wr_id, vwr_id);		\
3356		(hdr)->num_sges = num_sge;			\
3357	} while (0)
3358
3359#define SRQ_SGE_SET(sge, vaddr, vlength, vlkey)			\
3360	do {							\
3361		DMA_REGPAIR_LE(sge->addr, vaddr);		\
3362		(sge)->length = cpu_to_le32(vlength);		\
3363		(sge)->l_key = cpu_to_le32(vlkey);		\
3364	} while (0)
3365
3366static u32 qedr_prepare_sq_sges(struct qedr_qp *qp, u8 *wqe_size,
3367				const struct ib_send_wr *wr)
3368{
3369	u32 data_size = 0;
3370	int i;
3371
3372	for (i = 0; i < wr->num_sge; i++) {
3373		struct rdma_sq_sge *sge = qed_chain_produce(&qp->sq.pbl);
3374
3375		DMA_REGPAIR_LE(sge->addr, wr->sg_list[i].addr);
3376		sge->l_key = cpu_to_le32(wr->sg_list[i].lkey);
3377		sge->length = cpu_to_le32(wr->sg_list[i].length);
3378		data_size += wr->sg_list[i].length;
3379	}
3380
3381	if (wqe_size)
3382		*wqe_size += wr->num_sge;
3383
3384	return data_size;
3385}
3386
3387static u32 qedr_prepare_sq_rdma_data(struct qedr_dev *dev,
3388				     struct qedr_qp *qp,
3389				     struct rdma_sq_rdma_wqe_1st *rwqe,
3390				     struct rdma_sq_rdma_wqe_2nd *rwqe2,
3391				     const struct ib_send_wr *wr,
3392				     const struct ib_send_wr **bad_wr)
3393{
3394	rwqe2->r_key = cpu_to_le32(rdma_wr(wr)->rkey);
3395	DMA_REGPAIR_LE(rwqe2->remote_va, rdma_wr(wr)->remote_addr);
3396
3397	if (wr->send_flags & IB_SEND_INLINE &&
3398	    (wr->opcode == IB_WR_RDMA_WRITE_WITH_IMM ||
3399	     wr->opcode == IB_WR_RDMA_WRITE)) {
3400		u8 flags = 0;
3401
3402		SET_FIELD2(flags, RDMA_SQ_RDMA_WQE_1ST_INLINE_FLG, 1);
3403		return qedr_prepare_sq_inline_data(dev, qp, &rwqe->wqe_size, wr,
3404						   bad_wr, &rwqe->flags, flags);
3405	}
3406
3407	return qedr_prepare_sq_sges(qp, &rwqe->wqe_size, wr);
3408}
3409
3410static u32 qedr_prepare_sq_send_data(struct qedr_dev *dev,
3411				     struct qedr_qp *qp,
3412				     struct rdma_sq_send_wqe_1st *swqe,
3413				     struct rdma_sq_send_wqe_2st *swqe2,
3414				     const struct ib_send_wr *wr,
3415				     const struct ib_send_wr **bad_wr)
3416{
3417	memset(swqe2, 0, sizeof(*swqe2));
3418	if (wr->send_flags & IB_SEND_INLINE) {
3419		u8 flags = 0;
3420
3421		SET_FIELD2(flags, RDMA_SQ_SEND_WQE_INLINE_FLG, 1);
3422		return qedr_prepare_sq_inline_data(dev, qp, &swqe->wqe_size, wr,
3423						   bad_wr, &swqe->flags, flags);
3424	}
3425
3426	return qedr_prepare_sq_sges(qp, &swqe->wqe_size, wr);
3427}
3428
3429static int qedr_prepare_reg(struct qedr_qp *qp,
3430			    struct rdma_sq_fmr_wqe_1st *fwqe1,
3431			    const struct ib_reg_wr *wr)
3432{
3433	struct qedr_mr *mr = get_qedr_mr(wr->mr);
3434	struct rdma_sq_fmr_wqe_2nd *fwqe2;
3435
3436	fwqe2 = (struct rdma_sq_fmr_wqe_2nd *)qed_chain_produce(&qp->sq.pbl);
3437	fwqe1->addr.hi = upper_32_bits(mr->ibmr.iova);
3438	fwqe1->addr.lo = lower_32_bits(mr->ibmr.iova);
3439	fwqe1->l_key = wr->key;
3440
3441	fwqe2->access_ctrl = 0;
3442
3443	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_READ,
3444		   !!(wr->access & IB_ACCESS_REMOTE_READ));
3445	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_REMOTE_WRITE,
3446		   !!(wr->access & IB_ACCESS_REMOTE_WRITE));
3447	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_ENABLE_ATOMIC,
3448		   !!(wr->access & IB_ACCESS_REMOTE_ATOMIC));
3449	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_READ, 1);
3450	SET_FIELD2(fwqe2->access_ctrl, RDMA_SQ_FMR_WQE_2ND_LOCAL_WRITE,
3451		   !!(wr->access & IB_ACCESS_LOCAL_WRITE));
3452	fwqe2->fmr_ctrl = 0;
3453
3454	SET_FIELD2(fwqe2->fmr_ctrl, RDMA_SQ_FMR_WQE_2ND_PAGE_SIZE_LOG,
3455		   ilog2(mr->ibmr.page_size) - 12);
3456
3457	fwqe2->length_hi = 0;
3458	fwqe2->length_lo = mr->ibmr.length;
3459	fwqe2->pbl_addr.hi = upper_32_bits(mr->info.pbl_table->pa);
3460	fwqe2->pbl_addr.lo = lower_32_bits(mr->info.pbl_table->pa);
3461
3462	qp->wqe_wr_id[qp->sq.prod].mr = mr;
3463
3464	return 0;
3465}
3466
3467static enum ib_wc_opcode qedr_ib_to_wc_opcode(enum ib_wr_opcode opcode)
3468{
3469	switch (opcode) {
3470	case IB_WR_RDMA_WRITE:
3471	case IB_WR_RDMA_WRITE_WITH_IMM:
3472		return IB_WC_RDMA_WRITE;
3473	case IB_WR_SEND_WITH_IMM:
3474	case IB_WR_SEND:
3475	case IB_WR_SEND_WITH_INV:
3476		return IB_WC_SEND;
3477	case IB_WR_RDMA_READ:
3478	case IB_WR_RDMA_READ_WITH_INV:
3479		return IB_WC_RDMA_READ;
3480	case IB_WR_ATOMIC_CMP_AND_SWP:
3481		return IB_WC_COMP_SWAP;
3482	case IB_WR_ATOMIC_FETCH_AND_ADD:
3483		return IB_WC_FETCH_ADD;
3484	case IB_WR_REG_MR:
3485		return IB_WC_REG_MR;
3486	case IB_WR_LOCAL_INV:
3487		return IB_WC_LOCAL_INV;
3488	default:
3489		return IB_WC_SEND;
3490	}
3491}
3492
3493static inline bool qedr_can_post_send(struct qedr_qp *qp,
3494				      const struct ib_send_wr *wr)
3495{
3496	int wq_is_full, err_wr, pbl_is_full;
3497	struct qedr_dev *dev = qp->dev;
3498
3499	/* prevent SQ overflow and/or processing of a bad WR */
3500	err_wr = wr->num_sge > qp->sq.max_sges;
3501	wq_is_full = qedr_wq_is_full(&qp->sq);
3502	pbl_is_full = qed_chain_get_elem_left_u32(&qp->sq.pbl) <
3503		      QEDR_MAX_SQE_ELEMENTS_PER_SQE;
3504	if (wq_is_full || err_wr || pbl_is_full) {
3505		if (wq_is_full && !(qp->err_bitmap & QEDR_QP_ERR_SQ_FULL)) {
3506			DP_ERR(dev,
3507			       "error: WQ is full. Post send on QP %p failed (this error appears only once)\n",
3508			       qp);
3509			qp->err_bitmap |= QEDR_QP_ERR_SQ_FULL;
3510		}
3511
3512		if (err_wr && !(qp->err_bitmap & QEDR_QP_ERR_BAD_SR)) {
3513			DP_ERR(dev,
3514			       "error: WR is bad. Post send on QP %p failed (this error appears only once)\n",
3515			       qp);
3516			qp->err_bitmap |= QEDR_QP_ERR_BAD_SR;
3517		}
3518
3519		if (pbl_is_full &&
3520		    !(qp->err_bitmap & QEDR_QP_ERR_SQ_PBL_FULL)) {
3521			DP_ERR(dev,
3522			       "error: WQ PBL is full. Post send on QP %p failed (this error appears only once)\n",
3523			       qp);
3524			qp->err_bitmap |= QEDR_QP_ERR_SQ_PBL_FULL;
3525		}
3526		return false;
3527	}
3528	return true;
3529}
3530
3531static int __qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3532			    const struct ib_send_wr **bad_wr)
3533{
3534	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3535	struct qedr_qp *qp = get_qedr_qp(ibqp);
3536	struct rdma_sq_atomic_wqe_1st *awqe1;
3537	struct rdma_sq_atomic_wqe_2nd *awqe2;
3538	struct rdma_sq_atomic_wqe_3rd *awqe3;
3539	struct rdma_sq_send_wqe_2st *swqe2;
3540	struct rdma_sq_local_inv_wqe *iwqe;
3541	struct rdma_sq_rdma_wqe_2nd *rwqe2;
3542	struct rdma_sq_send_wqe_1st *swqe;
3543	struct rdma_sq_rdma_wqe_1st *rwqe;
3544	struct rdma_sq_fmr_wqe_1st *fwqe1;
3545	struct rdma_sq_common_wqe *wqe;
3546	u32 length;
3547	int rc = 0;
3548	bool comp;
3549
3550	if (!qedr_can_post_send(qp, wr)) {
3551		*bad_wr = wr;
3552		return -ENOMEM;
3553	}
3554
3555	wqe = qed_chain_produce(&qp->sq.pbl);
3556	qp->wqe_wr_id[qp->sq.prod].signaled =
3557		!!(wr->send_flags & IB_SEND_SIGNALED) || qp->signaled;
3558
3559	wqe->flags = 0;
3560	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_SE_FLG,
3561		   !!(wr->send_flags & IB_SEND_SOLICITED));
3562	comp = (!!(wr->send_flags & IB_SEND_SIGNALED)) || qp->signaled;
3563	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_COMP_FLG, comp);
3564	SET_FIELD2(wqe->flags, RDMA_SQ_SEND_WQE_RD_FENCE_FLG,
3565		   !!(wr->send_flags & IB_SEND_FENCE));
3566	wqe->prev_wqe_size = qp->prev_wqe_size;
3567
3568	qp->wqe_wr_id[qp->sq.prod].opcode = qedr_ib_to_wc_opcode(wr->opcode);
3569
3570	switch (wr->opcode) {
3571	case IB_WR_SEND_WITH_IMM:
3572		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3573			rc = -EINVAL;
3574			*bad_wr = wr;
3575			break;
3576		}
3577		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_IMM;
3578		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3579		swqe->wqe_size = 2;
3580		swqe2 = qed_chain_produce(&qp->sq.pbl);
3581
3582		swqe->inv_key_or_imm_data = cpu_to_le32(be32_to_cpu(wr->ex.imm_data));
3583		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3584						   wr, bad_wr);
3585		swqe->length = cpu_to_le32(length);
3586		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3587		qp->prev_wqe_size = swqe->wqe_size;
3588		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3589		break;
3590	case IB_WR_SEND:
3591		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND;
3592		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3593
3594		swqe->wqe_size = 2;
3595		swqe2 = qed_chain_produce(&qp->sq.pbl);
3596		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3597						   wr, bad_wr);
3598		swqe->length = cpu_to_le32(length);
3599		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3600		qp->prev_wqe_size = swqe->wqe_size;
3601		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3602		break;
3603	case IB_WR_SEND_WITH_INV:
3604		wqe->req_type = RDMA_SQ_REQ_TYPE_SEND_WITH_INVALIDATE;
3605		swqe = (struct rdma_sq_send_wqe_1st *)wqe;
3606		swqe2 = qed_chain_produce(&qp->sq.pbl);
3607		swqe->wqe_size = 2;
3608		swqe->inv_key_or_imm_data = cpu_to_le32(wr->ex.invalidate_rkey);
3609		length = qedr_prepare_sq_send_data(dev, qp, swqe, swqe2,
3610						   wr, bad_wr);
3611		swqe->length = cpu_to_le32(length);
3612		qp->wqe_wr_id[qp->sq.prod].wqe_size = swqe->wqe_size;
3613		qp->prev_wqe_size = swqe->wqe_size;
3614		qp->wqe_wr_id[qp->sq.prod].bytes_len = swqe->length;
3615		break;
3616
3617	case IB_WR_RDMA_WRITE_WITH_IMM:
3618		if (unlikely(rdma_protocol_iwarp(&dev->ibdev, 1))) {
3619			rc = -EINVAL;
3620			*bad_wr = wr;
3621			break;
3622		}
3623		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR_WITH_IMM;
3624		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3625
3626		rwqe->wqe_size = 2;
3627		rwqe->imm_data = htonl(cpu_to_le32(wr->ex.imm_data));
3628		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3629		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3630						   wr, bad_wr);
3631		rwqe->length = cpu_to_le32(length);
3632		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3633		qp->prev_wqe_size = rwqe->wqe_size;
3634		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3635		break;
3636	case IB_WR_RDMA_WRITE:
3637		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_WR;
3638		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3639
3640		rwqe->wqe_size = 2;
3641		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3642		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3643						   wr, bad_wr);
3644		rwqe->length = cpu_to_le32(length);
3645		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3646		qp->prev_wqe_size = rwqe->wqe_size;
3647		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3648		break;
3649	case IB_WR_RDMA_READ_WITH_INV:
3650		SET_FIELD2(wqe->flags, RDMA_SQ_RDMA_WQE_1ST_READ_INV_FLG, 1);
3651		fallthrough;	/* same is identical to RDMA READ */
3652
3653	case IB_WR_RDMA_READ:
3654		wqe->req_type = RDMA_SQ_REQ_TYPE_RDMA_RD;
3655		rwqe = (struct rdma_sq_rdma_wqe_1st *)wqe;
3656
3657		rwqe->wqe_size = 2;
3658		rwqe2 = qed_chain_produce(&qp->sq.pbl);
3659		length = qedr_prepare_sq_rdma_data(dev, qp, rwqe, rwqe2,
3660						   wr, bad_wr);
3661		rwqe->length = cpu_to_le32(length);
3662		qp->wqe_wr_id[qp->sq.prod].wqe_size = rwqe->wqe_size;
3663		qp->prev_wqe_size = rwqe->wqe_size;
3664		qp->wqe_wr_id[qp->sq.prod].bytes_len = rwqe->length;
3665		break;
3666
3667	case IB_WR_ATOMIC_CMP_AND_SWP:
3668	case IB_WR_ATOMIC_FETCH_AND_ADD:
3669		awqe1 = (struct rdma_sq_atomic_wqe_1st *)wqe;
3670		awqe1->wqe_size = 4;
3671
3672		awqe2 = qed_chain_produce(&qp->sq.pbl);
3673		DMA_REGPAIR_LE(awqe2->remote_va, atomic_wr(wr)->remote_addr);
3674		awqe2->r_key = cpu_to_le32(atomic_wr(wr)->rkey);
3675
3676		awqe3 = qed_chain_produce(&qp->sq.pbl);
3677
3678		if (wr->opcode == IB_WR_ATOMIC_FETCH_AND_ADD) {
3679			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_ADD;
3680			DMA_REGPAIR_LE(awqe3->swap_data,
3681				       atomic_wr(wr)->compare_add);
3682		} else {
3683			wqe->req_type = RDMA_SQ_REQ_TYPE_ATOMIC_CMP_AND_SWAP;
3684			DMA_REGPAIR_LE(awqe3->swap_data,
3685				       atomic_wr(wr)->swap);
3686			DMA_REGPAIR_LE(awqe3->cmp_data,
3687				       atomic_wr(wr)->compare_add);
3688		}
3689
3690		qedr_prepare_sq_sges(qp, NULL, wr);
3691
3692		qp->wqe_wr_id[qp->sq.prod].wqe_size = awqe1->wqe_size;
3693		qp->prev_wqe_size = awqe1->wqe_size;
3694		break;
3695
3696	case IB_WR_LOCAL_INV:
3697		iwqe = (struct rdma_sq_local_inv_wqe *)wqe;
3698		iwqe->wqe_size = 1;
3699
3700		iwqe->req_type = RDMA_SQ_REQ_TYPE_LOCAL_INVALIDATE;
3701		iwqe->inv_l_key = wr->ex.invalidate_rkey;
3702		qp->wqe_wr_id[qp->sq.prod].wqe_size = iwqe->wqe_size;
3703		qp->prev_wqe_size = iwqe->wqe_size;
3704		break;
3705	case IB_WR_REG_MR:
3706		DP_DEBUG(dev, QEDR_MSG_CQ, "REG_MR\n");
3707		wqe->req_type = RDMA_SQ_REQ_TYPE_FAST_MR;
3708		fwqe1 = (struct rdma_sq_fmr_wqe_1st *)wqe;
3709		fwqe1->wqe_size = 2;
3710
3711		rc = qedr_prepare_reg(qp, fwqe1, reg_wr(wr));
3712		if (rc) {
3713			DP_ERR(dev, "IB_REG_MR failed rc=%d\n", rc);
3714			*bad_wr = wr;
3715			break;
3716		}
3717
3718		qp->wqe_wr_id[qp->sq.prod].wqe_size = fwqe1->wqe_size;
3719		qp->prev_wqe_size = fwqe1->wqe_size;
3720		break;
3721	default:
3722		DP_ERR(dev, "invalid opcode 0x%x!\n", wr->opcode);
3723		rc = -EINVAL;
3724		*bad_wr = wr;
3725		break;
3726	}
3727
3728	if (*bad_wr) {
3729		u16 value;
3730
3731		/* Restore prod to its position before
3732		 * this WR was processed
3733		 */
3734		value = le16_to_cpu(qp->sq.db_data.data.value);
3735		qed_chain_set_prod(&qp->sq.pbl, value, wqe);
3736
3737		/* Restore prev_wqe_size */
3738		qp->prev_wqe_size = wqe->prev_wqe_size;
3739		rc = -EINVAL;
3740		DP_ERR(dev, "POST SEND FAILED\n");
3741	}
3742
3743	return rc;
3744}
3745
3746int qedr_post_send(struct ib_qp *ibqp, const struct ib_send_wr *wr,
3747		   const struct ib_send_wr **bad_wr)
3748{
3749	struct qedr_dev *dev = get_qedr_dev(ibqp->device);
3750	struct qedr_qp *qp = get_qedr_qp(ibqp);
3751	unsigned long flags;
3752	int rc = 0;
3753
3754	*bad_wr = NULL;
3755
3756	if (qp->qp_type == IB_QPT_GSI)
3757		return qedr_gsi_post_send(ibqp, wr, bad_wr);
3758
3759	spin_lock_irqsave(&qp->q_lock, flags);
3760
3761	if (rdma_protocol_roce(&dev->ibdev, 1)) {
3762		if ((qp->state != QED_ROCE_QP_STATE_RTS) &&
3763		    (qp->state != QED_ROCE_QP_STATE_ERR) &&
3764		    (qp->state != QED_ROCE_QP_STATE_SQD)) {
3765			spin_unlock_irqrestore(&qp->q_lock, flags);
3766			*bad_wr = wr;
3767			DP_DEBUG(dev, QEDR_MSG_CQ,
3768				 "QP in wrong state! QP icid=0x%x state %d\n",
3769				 qp->icid, qp->state);
3770			return -EINVAL;
3771		}
3772	}
3773
3774	while (wr) {
3775		rc = __qedr_post_send(ibqp, wr, bad_wr);
3776		if (rc)
3777			break;
3778
3779		qp->wqe_wr_id[qp->sq.prod].wr_id = wr->wr_id;
3780
3781		qedr_inc_sw_prod(&qp->sq);
3782
3783		qp->sq.db_data.data.value++;
3784
3785		wr = wr->next;
3786	}
3787
3788	/* Trigger doorbell
3789	 * If there was a failure in the first WR then it will be triggered in
3790	 * vane. However this is not harmful (as long as the producer value is
3791	 * unchanged). For performance reasons we avoid checking for this
3792	 * redundant doorbell.
3793	 *
3794	 * qp->wqe_wr_id is accessed during qedr_poll_cq, as
3795	 * soon as we give the doorbell, we could get a completion
3796	 * for this wr, therefore we need to make sure that the
3797	 * memory is updated before giving the doorbell.
3798	 * During qedr_poll_cq, rmb is called before accessing the
3799	 * cqe. This covers for the smp_rmb as well.
3800	 */
3801	smp_wmb();
3802	writel(qp->sq.db_data.raw, qp->sq.db);
3803
3804	spin_unlock_irqrestore(&qp->q_lock, flags);
3805
3806	return rc;
3807}
3808
3809static u32 qedr_srq_elem_left(struct qedr_srq_hwq_info *hw_srq)
3810{
3811	u32 used;
3812
3813	/* Calculate number of elements used based on producer
3814	 * count and consumer count and subtract it from max
3815	 * work request supported so that we get elements left.
3816	 */
3817	used = hw_srq->wr_prod_cnt - (u32)atomic_read(&hw_srq->wr_cons_cnt);
3818
3819	return hw_srq->max_wr - used;
3820}
3821
3822int qedr_post_srq_recv(struct ib_srq *ibsrq, const struct ib_recv_wr *wr,
3823		       const struct ib_recv_wr **bad_wr)
3824{
3825	struct qedr_srq *srq = get_qedr_srq(ibsrq);
3826	struct qedr_srq_hwq_info *hw_srq;
3827	struct qedr_dev *dev = srq->dev;
3828	struct qed_chain *pbl;
3829	unsigned long flags;
3830	int status = 0;
3831	u32 num_sge;
3832
3833	spin_lock_irqsave(&srq->lock, flags);
3834
3835	hw_srq = &srq->hw_srq;
3836	pbl = &srq->hw_srq.pbl;
3837	while (wr) {
3838		struct rdma_srq_wqe_header *hdr;
3839		int i;
3840
3841		if (!qedr_srq_elem_left(hw_srq) ||
3842		    wr->num_sge > srq->hw_srq.max_sges) {
3843			DP_ERR(dev, "Can't post WR  (%d,%d) || (%d > %d)\n",
3844			       hw_srq->wr_prod_cnt,
3845			       atomic_read(&hw_srq->wr_cons_cnt),
3846			       wr->num_sge, srq->hw_srq.max_sges);
3847			status = -ENOMEM;
3848			*bad_wr = wr;
3849			break;
3850		}
3851
3852		hdr = qed_chain_produce(pbl);
3853		num_sge = wr->num_sge;
3854		/* Set number of sge and work request id in header */
3855		SRQ_HDR_SET(hdr, wr->wr_id, num_sge);
3856
3857		srq->hw_srq.wr_prod_cnt++;
3858		hw_srq->wqe_prod++;
3859		hw_srq->sge_prod++;
3860
3861		DP_DEBUG(dev, QEDR_MSG_SRQ,
3862			 "SRQ WR: SGEs: %d with wr_id[%d] = %llx\n",
3863			 wr->num_sge, hw_srq->wqe_prod, wr->wr_id);
3864
3865		for (i = 0; i < wr->num_sge; i++) {
3866			struct rdma_srq_sge *srq_sge = qed_chain_produce(pbl);
3867
3868			/* Set SGE length, lkey and address */
3869			SRQ_SGE_SET(srq_sge, wr->sg_list[i].addr,
3870				    wr->sg_list[i].length, wr->sg_list[i].lkey);
3871
3872			DP_DEBUG(dev, QEDR_MSG_SRQ,
3873				 "[%d]: len %d key %x addr %x:%x\n",
3874				 i, srq_sge->length, srq_sge->l_key,
3875				 srq_sge->addr.hi, srq_sge->addr.lo);
3876			hw_srq->sge_prod++;
3877		}
3878
3879		/* Update WQE and SGE information before
3880		 * updating producer.
3881		 */
3882		dma_wmb();
3883
3884		/* SRQ producer is 8 bytes. Need to update SGE producer index
3885		 * in first 4 bytes and need to update WQE producer in
3886		 * next 4 bytes.
3887		 */
3888		srq->hw_srq.virt_prod_pair_addr->sge_prod = cpu_to_le32(hw_srq->sge_prod);
3889		/* Make sure sge producer is updated first */
3890		dma_wmb();
3891		srq->hw_srq.virt_prod_pair_addr->wqe_prod = cpu_to_le32(hw_srq->wqe_prod);
3892
3893		wr = wr->next;
3894	}
3895
3896	DP_DEBUG(dev, QEDR_MSG_SRQ, "POST: Elements in S-RQ: %d\n",
3897		 qed_chain_get_elem_left(pbl));
3898	spin_unlock_irqrestore(&srq->lock, flags);
3899
3900	return status;
3901}
3902
3903int qedr_post_recv(struct ib_qp *ibqp, const struct ib_recv_wr *wr,
3904		   const struct ib_recv_wr **bad_wr)
3905{
3906	struct qedr_qp *qp = get_qedr_qp(ibqp);
3907	struct qedr_dev *dev = qp->dev;
3908	unsigned long flags;
3909	int status = 0;
3910
3911	if (qp->qp_type == IB_QPT_GSI)
3912		return qedr_gsi_post_recv(ibqp, wr, bad_wr);
3913
3914	spin_lock_irqsave(&qp->q_lock, flags);
3915
3916	while (wr) {
3917		int i;
3918
3919		if (qed_chain_get_elem_left_u32(&qp->rq.pbl) <
3920		    QEDR_MAX_RQE_ELEMENTS_PER_RQE ||
3921		    wr->num_sge > qp->rq.max_sges) {
3922			DP_ERR(dev, "Can't post WR  (%d < %d) || (%d > %d)\n",
3923			       qed_chain_get_elem_left_u32(&qp->rq.pbl),
3924			       QEDR_MAX_RQE_ELEMENTS_PER_RQE, wr->num_sge,
3925			       qp->rq.max_sges);
3926			status = -ENOMEM;
3927			*bad_wr = wr;
3928			break;
3929		}
3930		for (i = 0; i < wr->num_sge; i++) {
3931			u32 flags = 0;
3932			struct rdma_rq_sge *rqe =
3933			    qed_chain_produce(&qp->rq.pbl);
3934
3935			/* First one must include the number
3936			 * of SGE in the list
3937			 */
3938			if (!i)
3939				SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES,
3940					  wr->num_sge);
3941
3942			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO,
3943				  wr->sg_list[i].lkey);
3944
3945			RQ_SGE_SET(rqe, wr->sg_list[i].addr,
3946				   wr->sg_list[i].length, flags);
3947		}
3948
3949		/* Special case of no sges. FW requires between 1-4 sges...
3950		 * in this case we need to post 1 sge with length zero. this is
3951		 * because rdma write with immediate consumes an RQ.
3952		 */
3953		if (!wr->num_sge) {
3954			u32 flags = 0;
3955			struct rdma_rq_sge *rqe =
3956			    qed_chain_produce(&qp->rq.pbl);
3957
3958			/* First one must include the number
3959			 * of SGE in the list
3960			 */
3961			SET_FIELD(flags, RDMA_RQ_SGE_L_KEY_LO, 0);
3962			SET_FIELD(flags, RDMA_RQ_SGE_NUM_SGES, 1);
3963
3964			RQ_SGE_SET(rqe, 0, 0, flags);
3965			i = 1;
3966		}
3967
3968		qp->rqe_wr_id[qp->rq.prod].wr_id = wr->wr_id;
3969		qp->rqe_wr_id[qp->rq.prod].wqe_size = i;
3970
3971		qedr_inc_sw_prod(&qp->rq);
3972
3973		/* qp->rqe_wr_id is accessed during qedr_poll_cq, as
3974		 * soon as we give the doorbell, we could get a completion
3975		 * for this wr, therefore we need to make sure that the
3976		 * memory is update before giving the doorbell.
3977		 * During qedr_poll_cq, rmb is called before accessing the
3978		 * cqe. This covers for the smp_rmb as well.
3979		 */
3980		smp_wmb();
3981
3982		qp->rq.db_data.data.value++;
3983
3984		writel(qp->rq.db_data.raw, qp->rq.db);
3985
3986		if (rdma_protocol_iwarp(&dev->ibdev, 1)) {
3987			writel(qp->rq.iwarp_db2_data.raw, qp->rq.iwarp_db2);
3988		}
3989
3990		wr = wr->next;
3991	}
3992
3993	spin_unlock_irqrestore(&qp->q_lock, flags);
3994
3995	return status;
3996}
3997
3998static int is_valid_cqe(struct qedr_cq *cq, union rdma_cqe *cqe)
3999{
4000	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4001
4002	return (resp_cqe->flags & RDMA_CQE_REQUESTER_TOGGLE_BIT_MASK) ==
4003		cq->pbl_toggle;
4004}
4005
4006static struct qedr_qp *cqe_get_qp(union rdma_cqe *cqe)
4007{
4008	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4009	struct qedr_qp *qp;
4010
4011	qp = (struct qedr_qp *)(uintptr_t)HILO_GEN(resp_cqe->qp_handle.hi,
4012						   resp_cqe->qp_handle.lo,
4013						   u64);
4014	return qp;
4015}
4016
4017static enum rdma_cqe_type cqe_get_type(union rdma_cqe *cqe)
4018{
4019	struct rdma_cqe_requester *resp_cqe = &cqe->req;
4020
4021	return GET_FIELD(resp_cqe->flags, RDMA_CQE_REQUESTER_TYPE);
4022}
4023
4024/* Return latest CQE (needs processing) */
4025static union rdma_cqe *get_cqe(struct qedr_cq *cq)
4026{
4027	return cq->latest_cqe;
4028}
4029
4030/* In fmr we need to increase the number of fmr completed counter for the fmr
4031 * algorithm determining whether we can free a pbl or not.
4032 * we need to perform this whether the work request was signaled or not. for
4033 * this purpose we call this function from the condition that checks if a wr
4034 * should be skipped, to make sure we don't miss it ( possibly this fmr
4035 * operation was not signalted)
4036 */
4037static inline void qedr_chk_if_fmr(struct qedr_qp *qp)
4038{
4039	if (qp->wqe_wr_id[qp->sq.cons].opcode == IB_WC_REG_MR)
4040		qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4041}
4042
4043static int process_req(struct qedr_dev *dev, struct qedr_qp *qp,
4044		       struct qedr_cq *cq, int num_entries,
4045		       struct ib_wc *wc, u16 hw_cons, enum ib_wc_status status,
4046		       int force)
4047{
4048	u16 cnt = 0;
4049
4050	while (num_entries && qp->sq.wqe_cons != hw_cons) {
4051		if (!qp->wqe_wr_id[qp->sq.cons].signaled && !force) {
4052			qedr_chk_if_fmr(qp);
4053			/* skip WC */
4054			goto next_cqe;
4055		}
4056
4057		/* fill WC */
4058		wc->status = status;
4059		wc->vendor_err = 0;
4060		wc->wc_flags = 0;
4061		wc->src_qp = qp->id;
4062		wc->qp = &qp->ibqp;
4063
4064		wc->wr_id = qp->wqe_wr_id[qp->sq.cons].wr_id;
4065		wc->opcode = qp->wqe_wr_id[qp->sq.cons].opcode;
4066
4067		switch (wc->opcode) {
4068		case IB_WC_RDMA_WRITE:
4069			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4070			break;
4071		case IB_WC_COMP_SWAP:
4072		case IB_WC_FETCH_ADD:
4073			wc->byte_len = 8;
4074			break;
4075		case IB_WC_REG_MR:
4076			qp->wqe_wr_id[qp->sq.cons].mr->info.completed++;
4077			break;
4078		case IB_WC_RDMA_READ:
4079		case IB_WC_SEND:
4080			wc->byte_len = qp->wqe_wr_id[qp->sq.cons].bytes_len;
4081			break;
4082		default:
4083			break;
4084		}
4085
4086		num_entries--;
4087		wc++;
4088		cnt++;
4089next_cqe:
4090		while (qp->wqe_wr_id[qp->sq.cons].wqe_size--)
4091			qed_chain_consume(&qp->sq.pbl);
4092		qedr_inc_sw_cons(&qp->sq);
4093	}
4094
4095	return cnt;
4096}
4097
4098static int qedr_poll_cq_req(struct qedr_dev *dev,
4099			    struct qedr_qp *qp, struct qedr_cq *cq,
4100			    int num_entries, struct ib_wc *wc,
4101			    struct rdma_cqe_requester *req)
4102{
4103	int cnt = 0;
4104
4105	switch (req->status) {
4106	case RDMA_CQE_REQ_STS_OK:
4107		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4108				  IB_WC_SUCCESS, 0);
4109		break;
4110	case RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR:
4111		if (qp->state != QED_ROCE_QP_STATE_ERR)
4112			DP_DEBUG(dev, QEDR_MSG_CQ,
4113				 "Error: POLL CQ with RDMA_CQE_REQ_STS_WORK_REQUEST_FLUSHED_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4114				 cq->icid, qp->icid);
4115		cnt = process_req(dev, qp, cq, num_entries, wc, req->sq_cons,
4116				  IB_WC_WR_FLUSH_ERR, 1);
4117		break;
4118	default:
4119		/* process all WQE before the cosumer */
4120		qp->state = QED_ROCE_QP_STATE_ERR;
4121		cnt = process_req(dev, qp, cq, num_entries, wc,
4122				  req->sq_cons - 1, IB_WC_SUCCESS, 0);
4123		wc += cnt;
4124		/* if we have extra WC fill it with actual error info */
4125		if (cnt < num_entries) {
4126			enum ib_wc_status wc_status;
4127
4128			switch (req->status) {
4129			case RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR:
4130				DP_ERR(dev,
4131				       "Error: POLL CQ with RDMA_CQE_REQ_STS_BAD_RESPONSE_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4132				       cq->icid, qp->icid);
4133				wc_status = IB_WC_BAD_RESP_ERR;
4134				break;
4135			case RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR:
4136				DP_ERR(dev,
4137				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_LENGTH_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4138				       cq->icid, qp->icid);
4139				wc_status = IB_WC_LOC_LEN_ERR;
4140				break;
4141			case RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR:
4142				DP_ERR(dev,
4143				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_QP_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4144				       cq->icid, qp->icid);
4145				wc_status = IB_WC_LOC_QP_OP_ERR;
4146				break;
4147			case RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR:
4148				DP_ERR(dev,
4149				       "Error: POLL CQ with RDMA_CQE_REQ_STS_LOCAL_PROTECTION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4150				       cq->icid, qp->icid);
4151				wc_status = IB_WC_LOC_PROT_ERR;
4152				break;
4153			case RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR:
4154				DP_ERR(dev,
4155				       "Error: POLL CQ with RDMA_CQE_REQ_STS_MEMORY_MGT_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4156				       cq->icid, qp->icid);
4157				wc_status = IB_WC_MW_BIND_ERR;
4158				break;
4159			case RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR:
4160				DP_ERR(dev,
4161				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_INVALID_REQUEST_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4162				       cq->icid, qp->icid);
4163				wc_status = IB_WC_REM_INV_REQ_ERR;
4164				break;
4165			case RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR:
4166				DP_ERR(dev,
4167				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_ACCESS_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4168				       cq->icid, qp->icid);
4169				wc_status = IB_WC_REM_ACCESS_ERR;
4170				break;
4171			case RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR:
4172				DP_ERR(dev,
4173				       "Error: POLL CQ with RDMA_CQE_REQ_STS_REMOTE_OPERATION_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4174				       cq->icid, qp->icid);
4175				wc_status = IB_WC_REM_OP_ERR;
4176				break;
4177			case RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR:
4178				DP_ERR(dev,
4179				       "Error: POLL CQ with RDMA_CQE_REQ_STS_RNR_NAK_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4180				       cq->icid, qp->icid);
4181				wc_status = IB_WC_RNR_RETRY_EXC_ERR;
4182				break;
4183			case RDMA_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR:
4184				DP_ERR(dev,
4185				       "Error: POLL CQ with ROCE_CQE_REQ_STS_TRANSPORT_RETRY_CNT_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4186				       cq->icid, qp->icid);
4187				wc_status = IB_WC_RETRY_EXC_ERR;
4188				break;
4189			default:
4190				DP_ERR(dev,
4191				       "Error: POLL CQ with IB_WC_GENERAL_ERR. CQ icid=0x%x, QP icid=0x%x\n",
4192				       cq->icid, qp->icid);
4193				wc_status = IB_WC_GENERAL_ERR;
4194			}
4195			cnt += process_req(dev, qp, cq, 1, wc, req->sq_cons,
4196					   wc_status, 1);
4197		}
4198	}
4199
4200	return cnt;
4201}
4202
4203static inline int qedr_cqe_resp_status_to_ib(u8 status)
4204{
4205	switch (status) {
4206	case RDMA_CQE_RESP_STS_LOCAL_ACCESS_ERR:
4207		return IB_WC_LOC_ACCESS_ERR;
4208	case RDMA_CQE_RESP_STS_LOCAL_LENGTH_ERR:
4209		return IB_WC_LOC_LEN_ERR;
4210	case RDMA_CQE_RESP_STS_LOCAL_QP_OPERATION_ERR:
4211		return IB_WC_LOC_QP_OP_ERR;
4212	case RDMA_CQE_RESP_STS_LOCAL_PROTECTION_ERR:
4213		return IB_WC_LOC_PROT_ERR;
4214	case RDMA_CQE_RESP_STS_MEMORY_MGT_OPERATION_ERR:
4215		return IB_WC_MW_BIND_ERR;
4216	case RDMA_CQE_RESP_STS_REMOTE_INVALID_REQUEST_ERR:
4217		return IB_WC_REM_INV_RD_REQ_ERR;
4218	case RDMA_CQE_RESP_STS_OK:
4219		return IB_WC_SUCCESS;
4220	default:
4221		return IB_WC_GENERAL_ERR;
4222	}
4223}
4224
4225static inline int qedr_set_ok_cqe_resp_wc(struct rdma_cqe_responder *resp,
4226					  struct ib_wc *wc)
4227{
4228	wc->status = IB_WC_SUCCESS;
4229	wc->byte_len = le32_to_cpu(resp->length);
4230
4231	if (resp->flags & QEDR_RESP_IMM) {
4232		wc->ex.imm_data = cpu_to_be32(le32_to_cpu(resp->imm_data_or_inv_r_Key));
4233		wc->wc_flags |= IB_WC_WITH_IMM;
4234
4235		if (resp->flags & QEDR_RESP_RDMA)
4236			wc->opcode = IB_WC_RECV_RDMA_WITH_IMM;
4237
4238		if (resp->flags & QEDR_RESP_INV)
4239			return -EINVAL;
4240
4241	} else if (resp->flags & QEDR_RESP_INV) {
4242		wc->ex.imm_data = le32_to_cpu(resp->imm_data_or_inv_r_Key);
4243		wc->wc_flags |= IB_WC_WITH_INVALIDATE;
4244
4245		if (resp->flags & QEDR_RESP_RDMA)
4246			return -EINVAL;
4247
4248	} else if (resp->flags & QEDR_RESP_RDMA) {
4249		return -EINVAL;
4250	}
4251
4252	return 0;
4253}
4254
4255static void __process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4256			       struct qedr_cq *cq, struct ib_wc *wc,
4257			       struct rdma_cqe_responder *resp, u64 wr_id)
4258{
4259	/* Must fill fields before qedr_set_ok_cqe_resp_wc() */
4260	wc->opcode = IB_WC_RECV;
4261	wc->wc_flags = 0;
4262
4263	if (likely(resp->status == RDMA_CQE_RESP_STS_OK)) {
4264		if (qedr_set_ok_cqe_resp_wc(resp, wc))
4265			DP_ERR(dev,
4266			       "CQ %p (icid=%d) has invalid CQE responder flags=0x%x\n",
4267			       cq, cq->icid, resp->flags);
4268
4269	} else {
4270		wc->status = qedr_cqe_resp_status_to_ib(resp->status);
4271		if (wc->status == IB_WC_GENERAL_ERR)
4272			DP_ERR(dev,
4273			       "CQ %p (icid=%d) contains an invalid CQE status %d\n",
4274			       cq, cq->icid, resp->status);
4275	}
4276
4277	/* Fill the rest of the WC */
4278	wc->vendor_err = 0;
4279	wc->src_qp = qp->id;
4280	wc->qp = &qp->ibqp;
4281	wc->wr_id = wr_id;
4282}
4283
4284static int process_resp_one_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4285				struct qedr_cq *cq, struct ib_wc *wc,
4286				struct rdma_cqe_responder *resp)
4287{
4288	struct qedr_srq *srq = qp->srq;
4289	u64 wr_id;
4290
4291	wr_id = HILO_GEN(le32_to_cpu(resp->srq_wr_id.hi),
4292			 le32_to_cpu(resp->srq_wr_id.lo), u64);
4293
4294	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4295		wc->status = IB_WC_WR_FLUSH_ERR;
4296		wc->vendor_err = 0;
4297		wc->wr_id = wr_id;
4298		wc->byte_len = 0;
4299		wc->src_qp = qp->id;
4300		wc->qp = &qp->ibqp;
4301		wc->wr_id = wr_id;
4302	} else {
4303		__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4304	}
4305	atomic_inc(&srq->hw_srq.wr_cons_cnt);
4306
4307	return 1;
4308}
4309static int process_resp_one(struct qedr_dev *dev, struct qedr_qp *qp,
4310			    struct qedr_cq *cq, struct ib_wc *wc,
4311			    struct rdma_cqe_responder *resp)
4312{
4313	u64 wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4314
4315	__process_resp_one(dev, qp, cq, wc, resp, wr_id);
4316
4317	while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4318		qed_chain_consume(&qp->rq.pbl);
4319	qedr_inc_sw_cons(&qp->rq);
4320
4321	return 1;
4322}
4323
4324static int process_resp_flush(struct qedr_qp *qp, struct qedr_cq *cq,
4325			      int num_entries, struct ib_wc *wc, u16 hw_cons)
4326{
4327	u16 cnt = 0;
4328
4329	while (num_entries && qp->rq.wqe_cons != hw_cons) {
4330		/* fill WC */
4331		wc->status = IB_WC_WR_FLUSH_ERR;
4332		wc->vendor_err = 0;
4333		wc->wc_flags = 0;
4334		wc->src_qp = qp->id;
4335		wc->byte_len = 0;
4336		wc->wr_id = qp->rqe_wr_id[qp->rq.cons].wr_id;
4337		wc->qp = &qp->ibqp;
4338		num_entries--;
4339		wc++;
4340		cnt++;
4341		while (qp->rqe_wr_id[qp->rq.cons].wqe_size--)
4342			qed_chain_consume(&qp->rq.pbl);
4343		qedr_inc_sw_cons(&qp->rq);
4344	}
4345
4346	return cnt;
4347}
4348
4349static void try_consume_resp_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4350				 struct rdma_cqe_responder *resp, int *update)
4351{
4352	if (le16_to_cpu(resp->rq_cons_or_srq_id) == qp->rq.wqe_cons) {
4353		consume_cqe(cq);
4354		*update |= 1;
4355	}
4356}
4357
4358static int qedr_poll_cq_resp_srq(struct qedr_dev *dev, struct qedr_qp *qp,
4359				 struct qedr_cq *cq, int num_entries,
4360				 struct ib_wc *wc,
4361				 struct rdma_cqe_responder *resp)
4362{
4363	int cnt;
4364
4365	cnt = process_resp_one_srq(dev, qp, cq, wc, resp);
4366	consume_cqe(cq);
4367
4368	return cnt;
4369}
4370
4371static int qedr_poll_cq_resp(struct qedr_dev *dev, struct qedr_qp *qp,
4372			     struct qedr_cq *cq, int num_entries,
4373			     struct ib_wc *wc, struct rdma_cqe_responder *resp,
4374			     int *update)
4375{
4376	int cnt;
4377
4378	if (resp->status == RDMA_CQE_RESP_STS_WORK_REQUEST_FLUSHED_ERR) {
4379		cnt = process_resp_flush(qp, cq, num_entries, wc,
4380					 resp->rq_cons_or_srq_id);
4381		try_consume_resp_cqe(cq, qp, resp, update);
4382	} else {
4383		cnt = process_resp_one(dev, qp, cq, wc, resp);
4384		consume_cqe(cq);
4385		*update |= 1;
4386	}
4387
4388	return cnt;
4389}
4390
4391static void try_consume_req_cqe(struct qedr_cq *cq, struct qedr_qp *qp,
4392				struct rdma_cqe_requester *req, int *update)
4393{
4394	if (le16_to_cpu(req->sq_cons) == qp->sq.wqe_cons) {
4395		consume_cqe(cq);
4396		*update |= 1;
4397	}
4398}
4399
4400int qedr_poll_cq(struct ib_cq *ibcq, int num_entries, struct ib_wc *wc)
4401{
4402	struct qedr_dev *dev = get_qedr_dev(ibcq->device);
4403	struct qedr_cq *cq = get_qedr_cq(ibcq);
4404	union rdma_cqe *cqe;
4405	u32 old_cons, new_cons;
4406	unsigned long flags;
4407	int update = 0;
4408	int done = 0;
4409
4410	if (cq->destroyed) {
4411		DP_ERR(dev,
4412		       "warning: poll was invoked after destroy for cq %p (icid=%d)\n",
4413		       cq, cq->icid);
4414		return 0;
4415	}
4416
4417	if (cq->cq_type == QEDR_CQ_TYPE_GSI)
4418		return qedr_gsi_poll_cq(ibcq, num_entries, wc);
4419
4420	spin_lock_irqsave(&cq->cq_lock, flags);
4421	cqe = cq->latest_cqe;
4422	old_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4423	while (num_entries && is_valid_cqe(cq, cqe)) {
4424		struct qedr_qp *qp;
4425		int cnt = 0;
4426
4427		/* prevent speculative reads of any field of CQE */
4428		rmb();
4429
4430		qp = cqe_get_qp(cqe);
4431		if (!qp) {
4432			WARN(1, "Error: CQE QP pointer is NULL. CQE=%p\n", cqe);
4433			break;
4434		}
4435
4436		wc->qp = &qp->ibqp;
4437
4438		switch (cqe_get_type(cqe)) {
4439		case RDMA_CQE_TYPE_REQUESTER:
4440			cnt = qedr_poll_cq_req(dev, qp, cq, num_entries, wc,
4441					       &cqe->req);
4442			try_consume_req_cqe(cq, qp, &cqe->req, &update);
4443			break;
4444		case RDMA_CQE_TYPE_RESPONDER_RQ:
4445			cnt = qedr_poll_cq_resp(dev, qp, cq, num_entries, wc,
4446						&cqe->resp, &update);
4447			break;
4448		case RDMA_CQE_TYPE_RESPONDER_SRQ:
4449			cnt = qedr_poll_cq_resp_srq(dev, qp, cq, num_entries,
4450						    wc, &cqe->resp);
4451			update = 1;
4452			break;
4453		case RDMA_CQE_TYPE_INVALID:
4454		default:
4455			DP_ERR(dev, "Error: invalid CQE type = %d\n",
4456			       cqe_get_type(cqe));
4457		}
4458		num_entries -= cnt;
4459		wc += cnt;
4460		done += cnt;
4461
4462		cqe = get_cqe(cq);
4463	}
4464	new_cons = qed_chain_get_cons_idx_u32(&cq->pbl);
4465
4466	cq->cq_cons += new_cons - old_cons;
4467
4468	if (update)
4469		/* doorbell notifies abount latest VALID entry,
4470		 * but chain already point to the next INVALID one
4471		 */
4472		doorbell_cq(cq, cq->cq_cons - 1, cq->arm_flags);
4473
4474	spin_unlock_irqrestore(&cq->cq_lock, flags);
4475	return done;
4476}
4477
4478int qedr_process_mad(struct ib_device *ibdev, int process_mad_flags,
4479		     u32 port_num, const struct ib_wc *in_wc,
4480		     const struct ib_grh *in_grh, const struct ib_mad *in,
4481		     struct ib_mad *out_mad, size_t *out_mad_size,
4482		     u16 *out_mad_pkey_index)
4483{
4484	return IB_MAD_RESULT_SUCCESS;
4485}