1/*
   2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3 *
   4 * This software is available to you under a choice of one of two
   5 * licenses.  You may choose to be licensed under the terms of the GNU
   6 * General Public License (GPL) Version 2, available from the file
   7 * COPYING in the main directory of this source tree, or the
   8 * OpenIB.org BSD license below:
   9 *
  10 *     Redistribution and use in source and binary forms, with or
  11 *     without modification, are permitted provided that the following
  12 *     conditions are met:
  13 *
  14 *      - Redistributions of source code must retain the above
  15 *        copyright notice, this list of conditions and the following
  16 *        disclaimer.
  17 *
  18 *      - Redistributions in binary form must reproduce the above
  19 *        copyright notice, this list of conditions and the following
  20 *        disclaimer in the documentation and/or other materials
  21 *        provided with the distribution.
  22 *
  23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30 * SOFTWARE.
  31 */
  32
  33#include <linux/etherdevice.h>
  34#include <rdma/ib_umem.h>
  35#include <rdma/ib_cache.h>
  36#include <rdma/ib_user_verbs.h>
  37#include <rdma/rdma_counter.h>
  38#include <linux/mlx5/fs.h>
  39#include "mlx5_ib.h"
  40#include "ib_rep.h"
  41#include "counters.h"
  42#include "cmd.h"
  43#include "umr.h"
  44#include "qp.h"
  45#include "wr.h"
  46
  47enum {
  48	MLX5_IB_ACK_REQ_FREQ	= 8,
  49};
  50
  51enum {
  52	MLX5_IB_DEFAULT_SCHED_QUEUE	= 0x83,
  53	MLX5_IB_DEFAULT_QP0_SCHED_QUEUE	= 0x3f,
  54	MLX5_IB_LINK_TYPE_IB		= 0,
  55	MLX5_IB_LINK_TYPE_ETH		= 1
  56};
  57
  58enum raw_qp_set_mask_map {
  59	MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID		= 1UL << 0,
  60	MLX5_RAW_QP_RATE_LIMIT			= 1UL << 1,
  61};
  62
  63enum {
  64	MLX5_QP_RM_GO_BACK_N			= 0x1,
  65};
  66
  67struct mlx5_modify_raw_qp_param {
  68	u16 operation;
  69
  70	u32 set_mask; /* raw_qp_set_mask_map */
  71
  72	struct mlx5_rate_limit rl;
  73
  74	u8 rq_q_ctr_id;
  75	u32 port;
  76};
  77
  78struct mlx5_ib_qp_event_work {
  79	struct work_struct work;
  80	struct mlx5_core_qp *qp;
  81	int type;
  82};
  83
  84static struct workqueue_struct *mlx5_ib_qp_event_wq;
  85
  86static void get_cqs(enum ib_qp_type qp_type,
  87		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
  88		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq);
  89
  90static int is_qp0(enum ib_qp_type qp_type)
  91{
  92	return qp_type == IB_QPT_SMI;
  93}
  94
  95static int is_sqp(enum ib_qp_type qp_type)
  96{
  97	return is_qp0(qp_type) || is_qp1(qp_type);
  98}
  99
 100/**
 101 * mlx5_ib_read_user_wqe_common() - Copy a WQE (or part of) from user WQ
 102 * to kernel buffer
 103 *
 104 * @umem: User space memory where the WQ is
 105 * @buffer: buffer to copy to
 106 * @buflen: buffer length
 107 * @wqe_index: index of WQE to copy from
 108 * @wq_offset: offset to start of WQ
 109 * @wq_wqe_cnt: number of WQEs in WQ
 110 * @wq_wqe_shift: log2 of WQE size
 111 * @bcnt: number of bytes to copy
 112 * @bytes_copied: number of bytes to copy (return value)
 113 *
 114 * Copies from start of WQE bcnt or less bytes.
 115 * Does not gurantee to copy the entire WQE.
 116 *
 117 * Return: zero on success, or an error code.
 118 */
 119static int mlx5_ib_read_user_wqe_common(struct ib_umem *umem, void *buffer,
 120					size_t buflen, int wqe_index,
 121					int wq_offset, int wq_wqe_cnt,
 122					int wq_wqe_shift, int bcnt,
 123					size_t *bytes_copied)
 124{
 125	size_t offset = wq_offset + ((wqe_index % wq_wqe_cnt) << wq_wqe_shift);
 126	size_t wq_end = wq_offset + (wq_wqe_cnt << wq_wqe_shift);
 127	size_t copy_length;
 128	int ret;
 129
 130	/* don't copy more than requested, more than buffer length or
 131	 * beyond WQ end
 132	 */
 133	copy_length = min_t(u32, buflen, wq_end - offset);
 134	copy_length = min_t(u32, copy_length, bcnt);
 135
 136	ret = ib_umem_copy_from(buffer, umem, offset, copy_length);
 137	if (ret)
 138		return ret;
 139
 140	if (!ret && bytes_copied)
 141		*bytes_copied = copy_length;
 142
 143	return 0;
 144}
 145
 146static int mlx5_ib_read_kernel_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
 147				      void *buffer, size_t buflen, size_t *bc)
 148{
 149	struct mlx5_wqe_ctrl_seg *ctrl;
 150	size_t bytes_copied = 0;
 151	size_t wqe_length;
 152	void *p;
 153	int ds;
 154
 155	wqe_index = wqe_index & qp->sq.fbc.sz_m1;
 156
 157	/* read the control segment first */
 158	p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
 159	ctrl = p;
 160	ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
 161	wqe_length = ds * MLX5_WQE_DS_UNITS;
 162
 163	/* read rest of WQE if it spreads over more than one stride */
 164	while (bytes_copied < wqe_length) {
 165		size_t copy_length =
 166			min_t(size_t, buflen - bytes_copied, MLX5_SEND_WQE_BB);
 167
 168		if (!copy_length)
 169			break;
 170
 171		memcpy(buffer + bytes_copied, p, copy_length);
 172		bytes_copied += copy_length;
 173
 174		wqe_index = (wqe_index + 1) & qp->sq.fbc.sz_m1;
 175		p = mlx5_frag_buf_get_wqe(&qp->sq.fbc, wqe_index);
 176	}
 177	*bc = bytes_copied;
 178	return 0;
 179}
 180
 181static int mlx5_ib_read_user_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index,
 182				    void *buffer, size_t buflen, size_t *bc)
 183{
 184	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 185	struct ib_umem *umem = base->ubuffer.umem;
 186	struct mlx5_ib_wq *wq = &qp->sq;
 187	struct mlx5_wqe_ctrl_seg *ctrl;
 188	size_t bytes_copied;
 189	size_t bytes_copied2;
 190	size_t wqe_length;
 191	int ret;
 192	int ds;
 193
 194	/* at first read as much as possible */
 195	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
 196					   wq->offset, wq->wqe_cnt,
 197					   wq->wqe_shift, buflen,
 198					   &bytes_copied);
 199	if (ret)
 200		return ret;
 201
 202	/* we need at least control segment size to proceed */
 203	if (bytes_copied < sizeof(*ctrl))
 204		return -EINVAL;
 205
 206	ctrl = buffer;
 207	ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
 208	wqe_length = ds * MLX5_WQE_DS_UNITS;
 209
 210	/* if we copied enough then we are done */
 211	if (bytes_copied >= wqe_length) {
 212		*bc = bytes_copied;
 213		return 0;
 214	}
 215
 216	/* otherwise this a wrapped around wqe
 217	 * so read the remaining bytes starting
 218	 * from  wqe_index 0
 219	 */
 220	ret = mlx5_ib_read_user_wqe_common(umem, buffer + bytes_copied,
 221					   buflen - bytes_copied, 0, wq->offset,
 222					   wq->wqe_cnt, wq->wqe_shift,
 223					   wqe_length - bytes_copied,
 224					   &bytes_copied2);
 225
 226	if (ret)
 227		return ret;
 228	*bc = bytes_copied + bytes_copied2;
 229	return 0;
 230}
 231
 232int mlx5_ib_read_wqe_sq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
 233			size_t buflen, size_t *bc)
 234{
 235	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 236	struct ib_umem *umem = base->ubuffer.umem;
 237
 238	if (buflen < sizeof(struct mlx5_wqe_ctrl_seg))
 239		return -EINVAL;
 240
 241	if (!umem)
 242		return mlx5_ib_read_kernel_wqe_sq(qp, wqe_index, buffer,
 243						  buflen, bc);
 244
 245	return mlx5_ib_read_user_wqe_sq(qp, wqe_index, buffer, buflen, bc);
 246}
 247
 248static int mlx5_ib_read_user_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index,
 249				    void *buffer, size_t buflen, size_t *bc)
 250{
 251	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 252	struct ib_umem *umem = base->ubuffer.umem;
 253	struct mlx5_ib_wq *wq = &qp->rq;
 254	size_t bytes_copied;
 255	int ret;
 256
 257	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index,
 258					   wq->offset, wq->wqe_cnt,
 259					   wq->wqe_shift, buflen,
 260					   &bytes_copied);
 261
 262	if (ret)
 263		return ret;
 264	*bc = bytes_copied;
 265	return 0;
 266}
 267
 268int mlx5_ib_read_wqe_rq(struct mlx5_ib_qp *qp, int wqe_index, void *buffer,
 269			size_t buflen, size_t *bc)
 270{
 271	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
 272	struct ib_umem *umem = base->ubuffer.umem;
 273	struct mlx5_ib_wq *wq = &qp->rq;
 274	size_t wqe_size = 1 << wq->wqe_shift;
 275
 276	if (buflen < wqe_size)
 277		return -EINVAL;
 278
 279	if (!umem)
 280		return -EOPNOTSUPP;
 281
 282	return mlx5_ib_read_user_wqe_rq(qp, wqe_index, buffer, buflen, bc);
 283}
 284
 285static int mlx5_ib_read_user_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index,
 286				     void *buffer, size_t buflen, size_t *bc)
 287{
 288	struct ib_umem *umem = srq->umem;
 289	size_t bytes_copied;
 290	int ret;
 291
 292	ret = mlx5_ib_read_user_wqe_common(umem, buffer, buflen, wqe_index, 0,
 293					   srq->msrq.max, srq->msrq.wqe_shift,
 294					   buflen, &bytes_copied);
 295
 296	if (ret)
 297		return ret;
 298	*bc = bytes_copied;
 299	return 0;
 300}
 301
 302int mlx5_ib_read_wqe_srq(struct mlx5_ib_srq *srq, int wqe_index, void *buffer,
 303			 size_t buflen, size_t *bc)
 304{
 305	struct ib_umem *umem = srq->umem;
 306	size_t wqe_size = 1 << srq->msrq.wqe_shift;
 307
 308	if (buflen < wqe_size)
 309		return -EINVAL;
 310
 311	if (!umem)
 312		return -EOPNOTSUPP;
 313
 314	return mlx5_ib_read_user_wqe_srq(srq, wqe_index, buffer, buflen, bc);
 315}
 316
 317static void mlx5_ib_qp_err_syndrome(struct ib_qp *ibqp)
 318{
 319	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
 320	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
 321	struct mlx5_ib_qp *qp = to_mqp(ibqp);
 322	void *pas_ext_union, *err_syn;
 323	u32 *outb;
 324	int err;
 325
 326	if (!MLX5_CAP_GEN(dev->mdev, qpc_extension) ||
 327	    !MLX5_CAP_GEN(dev->mdev, qp_error_syndrome))
 328		return;
 329
 330	outb = kzalloc(outlen, GFP_KERNEL);
 331	if (!outb)
 332		return;
 333
 334	err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
 335				 true);
 336	if (err)
 337		goto out;
 338
 339	pas_ext_union =
 340		MLX5_ADDR_OF(query_qp_out, outb, qp_pas_or_qpc_ext_and_pas);
 341	err_syn = MLX5_ADDR_OF(qpc_extension_and_pas_list_in, pas_ext_union,
 342			       qpc_data_extension.error_syndrome);
 343
 344	pr_err("%s/%d: QP %d error: %s (0x%x 0x%x 0x%x)\n",
 345	       ibqp->device->name, ibqp->port, ibqp->qp_num,
 346	       ib_wc_status_msg(
 347		       MLX5_GET(cqe_error_syndrome, err_syn, syndrome)),
 348	       MLX5_GET(cqe_error_syndrome, err_syn, vendor_error_syndrome),
 349	       MLX5_GET(cqe_error_syndrome, err_syn, hw_syndrome_type),
 350	       MLX5_GET(cqe_error_syndrome, err_syn, hw_error_syndrome));
 351out:
 352	kfree(outb);
 353}
 354
 355static void mlx5_ib_handle_qp_event(struct work_struct *_work)
 356{
 357	struct mlx5_ib_qp_event_work *qpe_work =
 358		container_of(_work, struct mlx5_ib_qp_event_work, work);
 359	struct ib_qp *ibqp = &to_mibqp(qpe_work->qp)->ibqp;
 360	struct ib_event event = {};
 361
 362	event.device = ibqp->device;
 363	event.element.qp = ibqp;
 364	switch (qpe_work->type) {
 365	case MLX5_EVENT_TYPE_PATH_MIG:
 366		event.event = IB_EVENT_PATH_MIG;
 367		break;
 368	case MLX5_EVENT_TYPE_COMM_EST:
 369		event.event = IB_EVENT_COMM_EST;
 370		break;
 371	case MLX5_EVENT_TYPE_SQ_DRAINED:
 372		event.event = IB_EVENT_SQ_DRAINED;
 373		break;
 374	case MLX5_EVENT_TYPE_SRQ_LAST_WQE:
 375		event.event = IB_EVENT_QP_LAST_WQE_REACHED;
 376		break;
 377	case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
 378		event.event = IB_EVENT_QP_FATAL;
 379		break;
 380	case MLX5_EVENT_TYPE_PATH_MIG_FAILED:
 381		event.event = IB_EVENT_PATH_MIG_ERR;
 382		break;
 383	case MLX5_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 384		event.event = IB_EVENT_QP_REQ_ERR;
 385		break;
 386	case MLX5_EVENT_TYPE_WQ_ACCESS_ERROR:
 387		event.event = IB_EVENT_QP_ACCESS_ERR;
 388		break;
 389	default:
 390		pr_warn("mlx5_ib: Unexpected event type %d on QP %06x\n",
 391			qpe_work->type, qpe_work->qp->qpn);
 392		goto out;
 393	}
 394
 395	if ((event.event == IB_EVENT_QP_FATAL) ||
 396	    (event.event == IB_EVENT_QP_ACCESS_ERR))
 397		mlx5_ib_qp_err_syndrome(ibqp);
 398
 399	ibqp->event_handler(&event, ibqp->qp_context);
 400
 401out:
 402	mlx5_core_res_put(&qpe_work->qp->common);
 403	kfree(qpe_work);
 404}
 405
 406static void mlx5_ib_qp_event(struct mlx5_core_qp *qp, int type)
 407{
 408	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
 409	struct mlx5_ib_qp_event_work *qpe_work;
 410
 411	if (type == MLX5_EVENT_TYPE_PATH_MIG) {
 412		/* This event is only valid for trans_qps */
 413		to_mibqp(qp)->port = to_mibqp(qp)->trans_qp.alt_port;
 414	}
 415
 416	if (!ibqp->event_handler)
 417		goto out_no_handler;
 418
 419	qpe_work = kzalloc(sizeof(*qpe_work), GFP_ATOMIC);
 420	if (!qpe_work)
 421		goto out_no_handler;
 422
 423	qpe_work->qp = qp;
 424	qpe_work->type = type;
 425	INIT_WORK(&qpe_work->work, mlx5_ib_handle_qp_event);
 426	queue_work(mlx5_ib_qp_event_wq, &qpe_work->work);
 427	return;
 428
 429out_no_handler:
 430	mlx5_core_res_put(&qp->common);
 431}
 432
 433static int set_rq_size(struct mlx5_ib_dev *dev, struct ib_qp_cap *cap,
 434		       int has_rq, struct mlx5_ib_qp *qp, struct mlx5_ib_create_qp *ucmd)
 435{
 436	int wqe_size;
 437	int wq_size;
 438
 439	/* Sanity check RQ size before proceeding */
 440	if (cap->max_recv_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz)))
 441		return -EINVAL;
 442
 443	if (!has_rq) {
 444		qp->rq.max_gs = 0;
 445		qp->rq.wqe_cnt = 0;
 446		qp->rq.wqe_shift = 0;
 447		cap->max_recv_wr = 0;
 448		cap->max_recv_sge = 0;
 449	} else {
 450		int wq_sig = !!(qp->flags_en & MLX5_QP_FLAG_SIGNATURE);
 451
 452		if (ucmd) {
 453			qp->rq.wqe_cnt = ucmd->rq_wqe_count;
 454			if (ucmd->rq_wqe_shift > BITS_PER_BYTE * sizeof(ucmd->rq_wqe_shift))
 455				return -EINVAL;
 456			qp->rq.wqe_shift = ucmd->rq_wqe_shift;
 457			if ((1 << qp->rq.wqe_shift) /
 458				    sizeof(struct mlx5_wqe_data_seg) <
 459			    wq_sig)
 460				return -EINVAL;
 461			qp->rq.max_gs =
 462				(1 << qp->rq.wqe_shift) /
 463					sizeof(struct mlx5_wqe_data_seg) -
 464				wq_sig;
 465			qp->rq.max_post = qp->rq.wqe_cnt;
 466		} else {
 467			wqe_size =
 468				wq_sig ? sizeof(struct mlx5_wqe_signature_seg) :
 469					 0;
 470			wqe_size += cap->max_recv_sge * sizeof(struct mlx5_wqe_data_seg);
 471			wqe_size = roundup_pow_of_two(wqe_size);
 472			wq_size = roundup_pow_of_two(cap->max_recv_wr) * wqe_size;
 473			wq_size = max_t(int, wq_size, MLX5_SEND_WQE_BB);
 474			qp->rq.wqe_cnt = wq_size / wqe_size;
 475			if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_rq)) {
 476				mlx5_ib_dbg(dev, "wqe_size %d, max %d\n",
 477					    wqe_size,
 478					    MLX5_CAP_GEN(dev->mdev,
 479							 max_wqe_sz_rq));
 480				return -EINVAL;
 481			}
 482			qp->rq.wqe_shift = ilog2(wqe_size);
 483			qp->rq.max_gs =
 484				(1 << qp->rq.wqe_shift) /
 485					sizeof(struct mlx5_wqe_data_seg) -
 486				wq_sig;
 487			qp->rq.max_post = qp->rq.wqe_cnt;
 488		}
 489	}
 490
 491	return 0;
 492}
 493
 494static int sq_overhead(struct ib_qp_init_attr *attr)
 495{
 496	int size = 0;
 497
 498	switch (attr->qp_type) {
 499	case IB_QPT_XRC_INI:
 500		size += sizeof(struct mlx5_wqe_xrc_seg);
 501		fallthrough;
 502	case IB_QPT_RC:
 503		size += sizeof(struct mlx5_wqe_ctrl_seg) +
 504			max(sizeof(struct mlx5_wqe_atomic_seg) +
 505			    sizeof(struct mlx5_wqe_raddr_seg),
 506			    sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 507			    sizeof(struct mlx5_mkey_seg) +
 508			    MLX5_IB_SQ_UMR_INLINE_THRESHOLD /
 509			    MLX5_IB_UMR_OCTOWORD);
 510		break;
 511
 512	case IB_QPT_XRC_TGT:
 513		return 0;
 514
 515	case IB_QPT_UC:
 516		size += sizeof(struct mlx5_wqe_ctrl_seg) +
 517			max(sizeof(struct mlx5_wqe_raddr_seg),
 518			    sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 519			    sizeof(struct mlx5_mkey_seg));
 520		break;
 521
 522	case IB_QPT_UD:
 523		if (attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
 524			size += sizeof(struct mlx5_wqe_eth_pad) +
 525				sizeof(struct mlx5_wqe_eth_seg);
 526		fallthrough;
 527	case IB_QPT_SMI:
 528	case MLX5_IB_QPT_HW_GSI:
 529		size += sizeof(struct mlx5_wqe_ctrl_seg) +
 530			sizeof(struct mlx5_wqe_datagram_seg);
 531		break;
 532
 533	case MLX5_IB_QPT_REG_UMR:
 534		size += sizeof(struct mlx5_wqe_ctrl_seg) +
 535			sizeof(struct mlx5_wqe_umr_ctrl_seg) +
 536			sizeof(struct mlx5_mkey_seg);
 537		break;
 538
 539	default:
 540		return -EINVAL;
 541	}
 542
 543	return size;
 544}
 545
 546static int calc_send_wqe(struct ib_qp_init_attr *attr)
 547{
 548	int inl_size = 0;
 549	int size;
 550
 551	size = sq_overhead(attr);
 552	if (size < 0)
 553		return size;
 554
 555	if (attr->cap.max_inline_data) {
 556		inl_size = size + sizeof(struct mlx5_wqe_inline_seg) +
 557			attr->cap.max_inline_data;
 558	}
 559
 560	size += attr->cap.max_send_sge * sizeof(struct mlx5_wqe_data_seg);
 561	if (attr->create_flags & IB_QP_CREATE_INTEGRITY_EN &&
 562	    ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB) < MLX5_SIG_WQE_SIZE)
 563		return MLX5_SIG_WQE_SIZE;
 564	else
 565		return ALIGN(max_t(int, inl_size, size), MLX5_SEND_WQE_BB);
 566}
 567
 568static int get_send_sge(struct ib_qp_init_attr *attr, int wqe_size)
 569{
 570	int max_sge;
 571
 572	if (attr->qp_type == IB_QPT_RC)
 573		max_sge = (min_t(int, wqe_size, 512) -
 574			   sizeof(struct mlx5_wqe_ctrl_seg) -
 575			   sizeof(struct mlx5_wqe_raddr_seg)) /
 576			sizeof(struct mlx5_wqe_data_seg);
 577	else if (attr->qp_type == IB_QPT_XRC_INI)
 578		max_sge = (min_t(int, wqe_size, 512) -
 579			   sizeof(struct mlx5_wqe_ctrl_seg) -
 580			   sizeof(struct mlx5_wqe_xrc_seg) -
 581			   sizeof(struct mlx5_wqe_raddr_seg)) /
 582			sizeof(struct mlx5_wqe_data_seg);
 583	else
 584		max_sge = (wqe_size - sq_overhead(attr)) /
 585			sizeof(struct mlx5_wqe_data_seg);
 586
 587	return min_t(int, max_sge, wqe_size - sq_overhead(attr) /
 588		     sizeof(struct mlx5_wqe_data_seg));
 589}
 590
 591static int calc_sq_size(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
 592			struct mlx5_ib_qp *qp)
 593{
 594	int wqe_size;
 595	int wq_size;
 596
 597	if (!attr->cap.max_send_wr)
 598		return 0;
 599
 600	wqe_size = calc_send_wqe(attr);
 601	mlx5_ib_dbg(dev, "wqe_size %d\n", wqe_size);
 602	if (wqe_size < 0)
 603		return wqe_size;
 604
 605	if (wqe_size > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
 606		mlx5_ib_dbg(dev, "wqe_size(%d) > max_sq_desc_sz(%d)\n",
 607			    wqe_size, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
 608		return -EINVAL;
 609	}
 610
 611	qp->max_inline_data = wqe_size - sq_overhead(attr) -
 612			      sizeof(struct mlx5_wqe_inline_seg);
 613	attr->cap.max_inline_data = qp->max_inline_data;
 614
 615	wq_size = roundup_pow_of_two(attr->cap.max_send_wr * wqe_size);
 616	qp->sq.wqe_cnt = wq_size / MLX5_SEND_WQE_BB;
 617	if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
 618		mlx5_ib_dbg(dev, "send queue size (%d * %d / %d -> %d) exceeds limits(%d)\n",
 619			    attr->cap.max_send_wr, wqe_size, MLX5_SEND_WQE_BB,
 620			    qp->sq.wqe_cnt,
 621			    1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
 622		return -ENOMEM;
 623	}
 624	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
 625	qp->sq.max_gs = get_send_sge(attr, wqe_size);
 626	if (qp->sq.max_gs < attr->cap.max_send_sge)
 627		return -ENOMEM;
 628
 629	attr->cap.max_send_sge = qp->sq.max_gs;
 630	qp->sq.max_post = wq_size / wqe_size;
 631	attr->cap.max_send_wr = qp->sq.max_post;
 632
 633	return wq_size;
 634}
 635
 636static int set_user_buf_size(struct mlx5_ib_dev *dev,
 637			    struct mlx5_ib_qp *qp,
 638			    struct mlx5_ib_create_qp *ucmd,
 639			    struct mlx5_ib_qp_base *base,
 640			    struct ib_qp_init_attr *attr)
 641{
 642	int desc_sz = 1 << qp->sq.wqe_shift;
 643
 644	if (desc_sz > MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq)) {
 645		mlx5_ib_warn(dev, "desc_sz %d, max_sq_desc_sz %d\n",
 646			     desc_sz, MLX5_CAP_GEN(dev->mdev, max_wqe_sz_sq));
 647		return -EINVAL;
 648	}
 649
 650	if (ucmd->sq_wqe_count && !is_power_of_2(ucmd->sq_wqe_count)) {
 651		mlx5_ib_warn(dev, "sq_wqe_count %d is not a power of two\n",
 652			     ucmd->sq_wqe_count);
 653		return -EINVAL;
 654	}
 655
 656	qp->sq.wqe_cnt = ucmd->sq_wqe_count;
 657
 658	if (qp->sq.wqe_cnt > (1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz))) {
 659		mlx5_ib_warn(dev, "wqe_cnt %d, max_wqes %d\n",
 660			     qp->sq.wqe_cnt,
 661			     1 << MLX5_CAP_GEN(dev->mdev, log_max_qp_sz));
 662		return -EINVAL;
 663	}
 664
 665	if (attr->qp_type == IB_QPT_RAW_PACKET ||
 666	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
 667		base->ubuffer.buf_size = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 668		qp->raw_packet_qp.sq.ubuffer.buf_size = qp->sq.wqe_cnt << 6;
 669	} else {
 670		base->ubuffer.buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
 671					 (qp->sq.wqe_cnt << 6);
 672	}
 673
 674	return 0;
 675}
 676
 677static int qp_has_rq(struct ib_qp_init_attr *attr)
 678{
 679	if (attr->qp_type == IB_QPT_XRC_INI ||
 680	    attr->qp_type == IB_QPT_XRC_TGT || attr->srq ||
 681	    attr->qp_type == MLX5_IB_QPT_REG_UMR ||
 682	    !attr->cap.max_recv_wr)
 683		return 0;
 684
 685	return 1;
 686}
 687
 688enum {
 689	/* this is the first blue flame register in the array of bfregs assigned
 690	 * to a processes. Since we do not use it for blue flame but rather
 691	 * regular 64 bit doorbells, we do not need a lock for maintaiing
 692	 * "odd/even" order
 693	 */
 694	NUM_NON_BLUE_FLAME_BFREGS = 1,
 695};
 696
 697static int max_bfregs(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi)
 698{
 699	return get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
 700	       bfregi->num_static_sys_pages * MLX5_NON_FP_BFREGS_PER_UAR;
 701}
 702
 703static int num_med_bfreg(struct mlx5_ib_dev *dev,
 704			 struct mlx5_bfreg_info *bfregi)
 705{
 706	int n;
 707
 708	n = max_bfregs(dev, bfregi) - bfregi->num_low_latency_bfregs -
 709	    NUM_NON_BLUE_FLAME_BFREGS;
 710
 711	return n >= 0 ? n : 0;
 712}
 713
 714static int first_med_bfreg(struct mlx5_ib_dev *dev,
 715			   struct mlx5_bfreg_info *bfregi)
 716{
 717	return num_med_bfreg(dev, bfregi) ? 1 : -ENOMEM;
 718}
 719
 720static int first_hi_bfreg(struct mlx5_ib_dev *dev,
 721			  struct mlx5_bfreg_info *bfregi)
 722{
 723	int med;
 724
 725	med = num_med_bfreg(dev, bfregi);
 726	return ++med;
 727}
 728
 729static int alloc_high_class_bfreg(struct mlx5_ib_dev *dev,
 730				  struct mlx5_bfreg_info *bfregi)
 731{
 732	int i;
 733
 734	for (i = first_hi_bfreg(dev, bfregi); i < max_bfregs(dev, bfregi); i++) {
 735		if (!bfregi->count[i]) {
 736			bfregi->count[i]++;
 737			return i;
 738		}
 739	}
 740
 741	return -ENOMEM;
 742}
 743
 744static int alloc_med_class_bfreg(struct mlx5_ib_dev *dev,
 745				 struct mlx5_bfreg_info *bfregi)
 746{
 747	int minidx = first_med_bfreg(dev, bfregi);
 748	int i;
 749
 750	if (minidx < 0)
 751		return minidx;
 752
 753	for (i = minidx; i < first_hi_bfreg(dev, bfregi); i++) {
 754		if (bfregi->count[i] < bfregi->count[minidx])
 755			minidx = i;
 756		if (!bfregi->count[minidx])
 757			break;
 758	}
 759
 760	bfregi->count[minidx]++;
 761	return minidx;
 762}
 763
 764static int alloc_bfreg(struct mlx5_ib_dev *dev,
 765		       struct mlx5_bfreg_info *bfregi)
 766{
 767	int bfregn = -ENOMEM;
 768
 769	if (bfregi->lib_uar_dyn)
 770		return -EINVAL;
 771
 772	mutex_lock(&bfregi->lock);
 773	if (bfregi->ver >= 2) {
 774		bfregn = alloc_high_class_bfreg(dev, bfregi);
 775		if (bfregn < 0)
 776			bfregn = alloc_med_class_bfreg(dev, bfregi);
 777	}
 778
 779	if (bfregn < 0) {
 780		BUILD_BUG_ON(NUM_NON_BLUE_FLAME_BFREGS != 1);
 781		bfregn = 0;
 782		bfregi->count[bfregn]++;
 783	}
 784	mutex_unlock(&bfregi->lock);
 785
 786	return bfregn;
 787}
 788
 789void mlx5_ib_free_bfreg(struct mlx5_ib_dev *dev, struct mlx5_bfreg_info *bfregi, int bfregn)
 790{
 791	mutex_lock(&bfregi->lock);
 792	bfregi->count[bfregn]--;
 793	mutex_unlock(&bfregi->lock);
 794}
 795
 796static enum mlx5_qp_state to_mlx5_state(enum ib_qp_state state)
 797{
 798	switch (state) {
 799	case IB_QPS_RESET:	return MLX5_QP_STATE_RST;
 800	case IB_QPS_INIT:	return MLX5_QP_STATE_INIT;
 801	case IB_QPS_RTR:	return MLX5_QP_STATE_RTR;
 802	case IB_QPS_RTS:	return MLX5_QP_STATE_RTS;
 803	case IB_QPS_SQD:	return MLX5_QP_STATE_SQD;
 804	case IB_QPS_SQE:	return MLX5_QP_STATE_SQER;
 805	case IB_QPS_ERR:	return MLX5_QP_STATE_ERR;
 806	default:		return -1;
 807	}
 808}
 809
 810static int to_mlx5_st(enum ib_qp_type type)
 811{
 812	switch (type) {
 813	case IB_QPT_RC:			return MLX5_QP_ST_RC;
 814	case IB_QPT_UC:			return MLX5_QP_ST_UC;
 815	case IB_QPT_UD:			return MLX5_QP_ST_UD;
 816	case MLX5_IB_QPT_REG_UMR:	return MLX5_QP_ST_REG_UMR;
 817	case IB_QPT_XRC_INI:
 818	case IB_QPT_XRC_TGT:		return MLX5_QP_ST_XRC;
 819	case IB_QPT_SMI:		return MLX5_QP_ST_QP0;
 820	case MLX5_IB_QPT_HW_GSI:	return MLX5_QP_ST_QP1;
 821	case MLX5_IB_QPT_DCI:		return MLX5_QP_ST_DCI;
 822	case IB_QPT_RAW_PACKET:		return MLX5_QP_ST_RAW_ETHERTYPE;
 823	default:		return -EINVAL;
 824	}
 825}
 826
 827static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq,
 828			     struct mlx5_ib_cq *recv_cq);
 829static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq,
 830			       struct mlx5_ib_cq *recv_cq);
 831
 832int bfregn_to_uar_index(struct mlx5_ib_dev *dev,
 833			struct mlx5_bfreg_info *bfregi, u32 bfregn,
 834			bool dyn_bfreg)
 835{
 836	unsigned int bfregs_per_sys_page;
 837	u32 index_of_sys_page;
 838	u32 offset;
 839
 840	if (bfregi->lib_uar_dyn)
 841		return -EINVAL;
 842
 843	bfregs_per_sys_page = get_uars_per_sys_page(dev, bfregi->lib_uar_4k) *
 844				MLX5_NON_FP_BFREGS_PER_UAR;
 845	index_of_sys_page = bfregn / bfregs_per_sys_page;
 846
 847	if (dyn_bfreg) {
 848		index_of_sys_page += bfregi->num_static_sys_pages;
 849
 850		if (index_of_sys_page >= bfregi->num_sys_pages)
 851			return -EINVAL;
 852
 853		if (bfregn > bfregi->num_dyn_bfregs ||
 854		    bfregi->sys_pages[index_of_sys_page] == MLX5_IB_INVALID_UAR_INDEX) {
 855			mlx5_ib_dbg(dev, "Invalid dynamic uar index\n");
 856			return -EINVAL;
 857		}
 858	}
 859
 860	offset = bfregn % bfregs_per_sys_page / MLX5_NON_FP_BFREGS_PER_UAR;
 861	return bfregi->sys_pages[index_of_sys_page] + offset;
 862}
 863
 864static void destroy_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 865			    struct mlx5_ib_rwq *rwq, struct ib_udata *udata)
 866{
 867	struct mlx5_ib_ucontext *context =
 868		rdma_udata_to_drv_context(
 869			udata,
 870			struct mlx5_ib_ucontext,
 871			ibucontext);
 872
 873	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_DELAY_DROP)
 874		atomic_dec(&dev->delay_drop.rqs_cnt);
 875
 876	mlx5_ib_db_unmap_user(context, &rwq->db);
 877	ib_umem_release(rwq->umem);
 878}
 879
 880static int create_user_rq(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 881			  struct ib_udata *udata, struct mlx5_ib_rwq *rwq,
 882			  struct mlx5_ib_create_wq *ucmd)
 883{
 884	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
 885		udata, struct mlx5_ib_ucontext, ibucontext);
 886	unsigned long page_size = 0;
 887	u32 offset = 0;
 888	int err;
 889
 890	if (!ucmd->buf_addr)
 891		return -EINVAL;
 892
 893	rwq->umem = ib_umem_get(&dev->ib_dev, ucmd->buf_addr, rwq->buf_size, 0);
 894	if (IS_ERR(rwq->umem)) {
 895		mlx5_ib_dbg(dev, "umem_get failed\n");
 896		err = PTR_ERR(rwq->umem);
 897		return err;
 898	}
 899
 900	page_size = mlx5_umem_find_best_quantized_pgoff(
 901		rwq->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
 902		page_offset, 64, &rwq->rq_page_offset);
 903	if (!page_size) {
 904		mlx5_ib_warn(dev, "bad offset\n");
 905		err = -EINVAL;
 906		goto err_umem;
 907	}
 908
 909	rwq->rq_num_pas = ib_umem_num_dma_blocks(rwq->umem, page_size);
 910	rwq->page_shift = order_base_2(page_size);
 911	rwq->log_page_size =  rwq->page_shift - MLX5_ADAPTER_PAGE_SHIFT;
 912	rwq->wq_sig = !!(ucmd->flags & MLX5_WQ_FLAG_SIGNATURE);
 913
 914	mlx5_ib_dbg(
 915		dev,
 916		"addr 0x%llx, size %zd, npages %zu, page_size %ld, ncont %d, offset %d\n",
 917		(unsigned long long)ucmd->buf_addr, rwq->buf_size,
 918		ib_umem_num_pages(rwq->umem), page_size, rwq->rq_num_pas,
 919		offset);
 920
 921	err = mlx5_ib_db_map_user(ucontext, ucmd->db_addr, &rwq->db);
 922	if (err) {
 923		mlx5_ib_dbg(dev, "map failed\n");
 924		goto err_umem;
 925	}
 926
 927	return 0;
 928
 929err_umem:
 930	ib_umem_release(rwq->umem);
 931	return err;
 932}
 933
 934static int adjust_bfregn(struct mlx5_ib_dev *dev,
 935			 struct mlx5_bfreg_info *bfregi, int bfregn)
 936{
 937	return bfregn / MLX5_NON_FP_BFREGS_PER_UAR * MLX5_BFREGS_PER_UAR +
 938				bfregn % MLX5_NON_FP_BFREGS_PER_UAR;
 939}
 940
 941static int _create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
 942			   struct mlx5_ib_qp *qp, struct ib_udata *udata,
 943			   struct ib_qp_init_attr *attr, u32 **in,
 944			   struct mlx5_ib_create_qp_resp *resp, int *inlen,
 945			   struct mlx5_ib_qp_base *base,
 946			   struct mlx5_ib_create_qp *ucmd)
 947{
 948	struct mlx5_ib_ucontext *context;
 949	struct mlx5_ib_ubuffer *ubuffer = &base->ubuffer;
 950	unsigned int page_offset_quantized = 0;
 951	unsigned long page_size = 0;
 952	int uar_index = 0;
 953	int bfregn;
 954	int ncont = 0;
 955	__be64 *pas;
 956	void *qpc;
 957	int err;
 958	u16 uid;
 959	u32 uar_flags;
 960
 961	context = rdma_udata_to_drv_context(udata, struct mlx5_ib_ucontext,
 962					    ibucontext);
 963	uar_flags = qp->flags_en &
 964		    (MLX5_QP_FLAG_UAR_PAGE_INDEX | MLX5_QP_FLAG_BFREG_INDEX);
 965	switch (uar_flags) {
 966	case MLX5_QP_FLAG_UAR_PAGE_INDEX:
 967		uar_index = ucmd->bfreg_index;
 968		bfregn = MLX5_IB_INVALID_BFREG;
 969		break;
 970	case MLX5_QP_FLAG_BFREG_INDEX:
 971		uar_index = bfregn_to_uar_index(dev, &context->bfregi,
 972						ucmd->bfreg_index, true);
 973		if (uar_index < 0)
 974			return uar_index;
 975		bfregn = MLX5_IB_INVALID_BFREG;
 976		break;
 977	case 0:
 978		if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
 979			return -EINVAL;
 980		bfregn = alloc_bfreg(dev, &context->bfregi);
 981		if (bfregn < 0)
 982			return bfregn;
 983		break;
 984	default:
 985		return -EINVAL;
 986	}
 987
 988	mlx5_ib_dbg(dev, "bfregn 0x%x, uar_index 0x%x\n", bfregn, uar_index);
 989	if (bfregn != MLX5_IB_INVALID_BFREG)
 990		uar_index = bfregn_to_uar_index(dev, &context->bfregi, bfregn,
 991						false);
 992
 993	qp->rq.offset = 0;
 994	qp->sq.wqe_shift = ilog2(MLX5_SEND_WQE_BB);
 995	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 996
 997	err = set_user_buf_size(dev, qp, ucmd, base, attr);
 998	if (err)
 999		goto err_bfreg;
1000
1001	if (ucmd->buf_addr && ubuffer->buf_size) {
1002		ubuffer->buf_addr = ucmd->buf_addr;
1003		ubuffer->umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr,
1004					    ubuffer->buf_size, 0);
1005		if (IS_ERR(ubuffer->umem)) {
1006			err = PTR_ERR(ubuffer->umem);
1007			goto err_bfreg;
1008		}
1009		page_size = mlx5_umem_find_best_quantized_pgoff(
1010			ubuffer->umem, qpc, log_page_size,
1011			MLX5_ADAPTER_PAGE_SHIFT, page_offset, 64,
1012			&page_offset_quantized);
1013		if (!page_size) {
1014			err = -EINVAL;
1015			goto err_umem;
1016		}
1017		ncont = ib_umem_num_dma_blocks(ubuffer->umem, page_size);
1018	} else {
1019		ubuffer->umem = NULL;
1020	}
1021
1022	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1023		 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * ncont;
1024	*in = kvzalloc(*inlen, GFP_KERNEL);
1025	if (!*in) {
1026		err = -ENOMEM;
1027		goto err_umem;
1028	}
1029
1030	uid = (attr->qp_type != IB_QPT_XRC_INI) ? to_mpd(pd)->uid : 0;
1031	MLX5_SET(create_qp_in, *in, uid, uid);
1032	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
1033	pas = (__be64 *)MLX5_ADDR_OF(create_qp_in, *in, pas);
1034	if (ubuffer->umem) {
1035		mlx5_ib_populate_pas(ubuffer->umem, page_size, pas, 0);
1036		MLX5_SET(qpc, qpc, log_page_size,
1037			 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1038		MLX5_SET(qpc, qpc, page_offset, page_offset_quantized);
1039	}
1040	MLX5_SET(qpc, qpc, uar_page, uar_index);
1041	if (bfregn != MLX5_IB_INVALID_BFREG)
1042		resp->bfreg_index = adjust_bfregn(dev, &context->bfregi, bfregn);
1043	else
1044		resp->bfreg_index = MLX5_IB_INVALID_BFREG;
1045	qp->bfregn = bfregn;
1046
1047	err = mlx5_ib_db_map_user(context, ucmd->db_addr, &qp->db);
1048	if (err) {
1049		mlx5_ib_dbg(dev, "map failed\n");
1050		goto err_free;
1051	}
1052
1053	return 0;
1054
1055err_free:
1056	kvfree(*in);
1057
1058err_umem:
1059	ib_umem_release(ubuffer->umem);
1060
1061err_bfreg:
1062	if (bfregn != MLX5_IB_INVALID_BFREG)
1063		mlx5_ib_free_bfreg(dev, &context->bfregi, bfregn);
1064	return err;
1065}
1066
1067static void destroy_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1068		       struct mlx5_ib_qp_base *base, struct ib_udata *udata)
1069{
1070	struct mlx5_ib_ucontext *context = rdma_udata_to_drv_context(
1071		udata, struct mlx5_ib_ucontext, ibucontext);
1072
1073	if (udata) {
1074		/* User QP */
1075		mlx5_ib_db_unmap_user(context, &qp->db);
1076		ib_umem_release(base->ubuffer.umem);
1077
1078		/*
1079		 * Free only the BFREGs which are handled by the kernel.
1080		 * BFREGs of UARs allocated dynamically are handled by user.
1081		 */
1082		if (qp->bfregn != MLX5_IB_INVALID_BFREG)
1083			mlx5_ib_free_bfreg(dev, &context->bfregi, qp->bfregn);
1084		return;
1085	}
1086
1087	/* Kernel QP */
1088	kvfree(qp->sq.wqe_head);
1089	kvfree(qp->sq.w_list);
1090	kvfree(qp->sq.wrid);
1091	kvfree(qp->sq.wr_data);
1092	kvfree(qp->rq.wrid);
1093	if (qp->db.db)
1094		mlx5_db_free(dev->mdev, &qp->db);
1095	if (qp->buf.frags)
1096		mlx5_frag_buf_free(dev->mdev, &qp->buf);
1097}
1098
1099static int _create_kernel_qp(struct mlx5_ib_dev *dev,
1100			     struct ib_qp_init_attr *init_attr,
1101			     struct mlx5_ib_qp *qp, u32 **in, int *inlen,
1102			     struct mlx5_ib_qp_base *base)
1103{
1104	int uar_index;
1105	void *qpc;
1106	int err;
1107
1108	if (init_attr->qp_type == MLX5_IB_QPT_REG_UMR)
1109		qp->bf.bfreg = &dev->fp_bfreg;
1110	else if (qp->flags & MLX5_IB_QP_CREATE_WC_TEST)
1111		qp->bf.bfreg = &dev->wc_bfreg;
1112	else
1113		qp->bf.bfreg = &dev->bfreg;
1114
1115	/* We need to divide by two since each register is comprised of
1116	 * two buffers of identical size, namely odd and even
1117	 */
1118	qp->bf.buf_size = (1 << MLX5_CAP_GEN(dev->mdev, log_bf_reg_size)) / 2;
1119	uar_index = qp->bf.bfreg->index;
1120
1121	err = calc_sq_size(dev, init_attr, qp);
1122	if (err < 0) {
1123		mlx5_ib_dbg(dev, "err %d\n", err);
1124		return err;
1125	}
1126
1127	qp->rq.offset = 0;
1128	qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
1129	base->ubuffer.buf_size = err + (qp->rq.wqe_cnt << qp->rq.wqe_shift);
1130
1131	err = mlx5_frag_buf_alloc_node(dev->mdev, base->ubuffer.buf_size,
1132				       &qp->buf, dev->mdev->priv.numa_node);
1133	if (err) {
1134		mlx5_ib_dbg(dev, "err %d\n", err);
1135		return err;
1136	}
1137
1138	if (qp->rq.wqe_cnt)
1139		mlx5_init_fbc(qp->buf.frags, qp->rq.wqe_shift,
1140			      ilog2(qp->rq.wqe_cnt), &qp->rq.fbc);
1141
1142	if (qp->sq.wqe_cnt) {
1143		int sq_strides_offset = (qp->sq.offset  & (PAGE_SIZE - 1)) /
1144					MLX5_SEND_WQE_BB;
1145		mlx5_init_fbc_offset(qp->buf.frags +
1146				     (qp->sq.offset / PAGE_SIZE),
1147				     ilog2(MLX5_SEND_WQE_BB),
1148				     ilog2(qp->sq.wqe_cnt),
1149				     sq_strides_offset, &qp->sq.fbc);
1150
1151		qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
1152	}
1153
1154	*inlen = MLX5_ST_SZ_BYTES(create_qp_in) +
1155		 MLX5_FLD_SZ_BYTES(create_qp_in, pas[0]) * qp->buf.npages;
1156	*in = kvzalloc(*inlen, GFP_KERNEL);
1157	if (!*in) {
1158		err = -ENOMEM;
1159		goto err_buf;
1160	}
1161
1162	qpc = MLX5_ADDR_OF(create_qp_in, *in, qpc);
1163	MLX5_SET(qpc, qpc, uar_page, uar_index);
1164	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));
1165	MLX5_SET(qpc, qpc, log_page_size, qp->buf.page_shift - MLX5_ADAPTER_PAGE_SHIFT);
1166
1167	/* Set "fast registration enabled" for all kernel QPs */
1168	MLX5_SET(qpc, qpc, fre, 1);
1169	MLX5_SET(qpc, qpc, rlky, 1);
1170
1171	if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
1172		MLX5_SET(qpc, qpc, deth_sqpn, 1);
1173
1174	mlx5_fill_page_frag_array(&qp->buf,
1175				  (__be64 *)MLX5_ADDR_OF(create_qp_in,
1176							 *in, pas));
1177
1178	err = mlx5_db_alloc(dev->mdev, &qp->db);
1179	if (err) {
1180		mlx5_ib_dbg(dev, "err %d\n", err);
1181		goto err_free;
1182	}
1183
1184	qp->sq.wrid = kvmalloc_array(qp->sq.wqe_cnt,
1185				     sizeof(*qp->sq.wrid), GFP_KERNEL);
1186	qp->sq.wr_data = kvmalloc_array(qp->sq.wqe_cnt,
1187					sizeof(*qp->sq.wr_data), GFP_KERNEL);
1188	qp->rq.wrid = kvmalloc_array(qp->rq.wqe_cnt,
1189				     sizeof(*qp->rq.wrid), GFP_KERNEL);
1190	qp->sq.w_list = kvmalloc_array(qp->sq.wqe_cnt,
1191				       sizeof(*qp->sq.w_list), GFP_KERNEL);
1192	qp->sq.wqe_head = kvmalloc_array(qp->sq.wqe_cnt,
1193					 sizeof(*qp->sq.wqe_head), GFP_KERNEL);
1194
1195	if (!qp->sq.wrid || !qp->sq.wr_data || !qp->rq.wrid ||
1196	    !qp->sq.w_list || !qp->sq.wqe_head) {
1197		err = -ENOMEM;
1198		goto err_wrid;
1199	}
1200
1201	return 0;
1202
1203err_wrid:
1204	kvfree(qp->sq.wqe_head);
1205	kvfree(qp->sq.w_list);
1206	kvfree(qp->sq.wrid);
1207	kvfree(qp->sq.wr_data);
1208	kvfree(qp->rq.wrid);
1209	mlx5_db_free(dev->mdev, &qp->db);
1210
1211err_free:
1212	kvfree(*in);
1213
1214err_buf:
1215	mlx5_frag_buf_free(dev->mdev, &qp->buf);
1216	return err;
1217}
1218
1219static u32 get_rx_type(struct mlx5_ib_qp *qp, struct ib_qp_init_attr *attr)
1220{
1221	if (attr->srq || (qp->type == IB_QPT_XRC_TGT) ||
1222	    (qp->type == MLX5_IB_QPT_DCI) || (qp->type == IB_QPT_XRC_INI))
1223		return MLX5_SRQ_RQ;
1224	else if (!qp->has_rq)
1225		return MLX5_ZERO_LEN_RQ;
1226
1227	return MLX5_NON_ZERO_RQ;
1228}
1229
1230static int create_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1231				    struct mlx5_ib_qp *qp,
1232				    struct mlx5_ib_sq *sq, u32 tdn,
1233				    struct ib_pd *pd)
1234{
1235	u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {};
1236	void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx);
1237
1238	MLX5_SET(create_tis_in, in, uid, to_mpd(pd)->uid);
1239	MLX5_SET(tisc, tisc, transport_domain, tdn);
1240	if (!mlx5_ib_lag_should_assign_affinity(dev) &&
1241	    mlx5_lag_is_lacp_owner(dev->mdev))
1242		MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1);
1243	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
1244		MLX5_SET(tisc, tisc, underlay_qpn, qp->underlay_qpn);
1245
1246	return mlx5_core_create_tis(dev->mdev, in, &sq->tisn);
1247}
1248
1249static void destroy_raw_packet_qp_tis(struct mlx5_ib_dev *dev,
1250				      struct mlx5_ib_sq *sq, struct ib_pd *pd)
1251{
1252	mlx5_cmd_destroy_tis(dev->mdev, sq->tisn, to_mpd(pd)->uid);
1253}
1254
1255static void destroy_flow_rule_vport_sq(struct mlx5_ib_sq *sq)
1256{
1257	if (sq->flow_rule)
1258		mlx5_del_flow_rules(sq->flow_rule);
1259	sq->flow_rule = NULL;
1260}
1261
1262static bool fr_supported(int ts_cap)
1263{
1264	return ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING ||
1265	       ts_cap == MLX5_TIMESTAMP_FORMAT_CAP_FREE_RUNNING_AND_REAL_TIME;
1266}
1267
1268static int get_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *cq,
1269			 bool fr_sup, bool rt_sup)
1270{
1271	if (cq->private_flags & MLX5_IB_CQ_PR_FLAGS_REAL_TIME_TS) {
1272		if (!rt_sup) {
1273			mlx5_ib_dbg(dev,
1274				    "Real time TS format is not supported\n");
1275			return -EOPNOTSUPP;
1276		}
1277		return MLX5_TIMESTAMP_FORMAT_REAL_TIME;
1278	}
1279	if (cq->create_flags & IB_UVERBS_CQ_FLAGS_TIMESTAMP_COMPLETION) {
1280		if (!fr_sup) {
1281			mlx5_ib_dbg(dev,
1282				    "Free running TS format is not supported\n");
1283			return -EOPNOTSUPP;
1284		}
1285		return MLX5_TIMESTAMP_FORMAT_FREE_RUNNING;
1286	}
1287	return fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :
1288			MLX5_TIMESTAMP_FORMAT_DEFAULT;
1289}
1290
1291static int get_rq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *recv_cq)
1292{
1293	u8 ts_cap = MLX5_CAP_GEN(dev->mdev, rq_ts_format);
1294
1295	return get_ts_format(dev, recv_cq, fr_supported(ts_cap),
1296			     rt_supported(ts_cap));
1297}
1298
1299static int get_sq_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq)
1300{
1301	u8 ts_cap = MLX5_CAP_GEN(dev->mdev, sq_ts_format);
1302
1303	return get_ts_format(dev, send_cq, fr_supported(ts_cap),
1304			     rt_supported(ts_cap));
1305}
1306
1307static int get_qp_ts_format(struct mlx5_ib_dev *dev, struct mlx5_ib_cq *send_cq,
1308			    struct mlx5_ib_cq *recv_cq)
1309{
1310	u8 ts_cap = MLX5_CAP_ROCE(dev->mdev, qp_ts_format);
1311	bool fr_sup = fr_supported(ts_cap);
1312	bool rt_sup = rt_supported(ts_cap);
1313	u8 default_ts = fr_sup ? MLX5_TIMESTAMP_FORMAT_FREE_RUNNING :
1314				 MLX5_TIMESTAMP_FORMAT_DEFAULT;
1315	int send_ts_format =
1316		send_cq ? get_ts_format(dev, send_cq, fr_sup, rt_sup) :
1317			  default_ts;
1318	int recv_ts_format =
1319		recv_cq ? get_ts_format(dev, recv_cq, fr_sup, rt_sup) :
1320			  default_ts;
1321
1322	if (send_ts_format < 0 || recv_ts_format < 0)
1323		return -EOPNOTSUPP;
1324
1325	if (send_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &&
1326	    recv_ts_format != MLX5_TIMESTAMP_FORMAT_DEFAULT &&
1327	    send_ts_format != recv_ts_format) {
1328		mlx5_ib_dbg(
1329			dev,
1330			"The send ts_format does not match the receive ts_format\n");
1331		return -EOPNOTSUPP;
1332	}
1333
1334	return send_ts_format == default_ts ? recv_ts_format : send_ts_format;
1335}
1336
1337static int create_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1338				   struct ib_udata *udata,
1339				   struct mlx5_ib_sq *sq, void *qpin,
1340				   struct ib_pd *pd, struct mlx5_ib_cq *cq)
1341{
1342	struct mlx5_ib_ubuffer *ubuffer = &sq->ubuffer;
1343	__be64 *pas;
1344	void *in;
1345	void *sqc;
1346	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1347	void *wq;
1348	int inlen;
1349	int err;
1350	unsigned int page_offset_quantized;
1351	unsigned long page_size;
1352	int ts_format;
1353
1354	ts_format = get_sq_ts_format(dev, cq);
1355	if (ts_format < 0)
1356		return ts_format;
1357
1358	sq->ubuffer.umem = ib_umem_get(&dev->ib_dev, ubuffer->buf_addr,
1359				       ubuffer->buf_size, 0);
1360	if (IS_ERR(sq->ubuffer.umem))
1361		return PTR_ERR(sq->ubuffer.umem);
1362	page_size = mlx5_umem_find_best_quantized_pgoff(
1363		ubuffer->umem, wq, log_wq_pg_sz, MLX5_ADAPTER_PAGE_SHIFT,
1364		page_offset, 64, &page_offset_quantized);
1365	if (!page_size) {
1366		err = -EINVAL;
1367		goto err_umem;
1368	}
1369
1370	inlen = MLX5_ST_SZ_BYTES(create_sq_in) +
1371		sizeof(u64) *
1372			ib_umem_num_dma_blocks(sq->ubuffer.umem, page_size);
1373	in = kvzalloc(inlen, GFP_KERNEL);
1374	if (!in) {
1375		err = -ENOMEM;
1376		goto err_umem;
1377	}
1378
1379	MLX5_SET(create_sq_in, in, uid, to_mpd(pd)->uid);
1380	sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
1381	MLX5_SET(sqc, sqc, flush_in_error_en, 1);
1382	if (MLX5_CAP_ETH(dev->mdev, multi_pkt_send_wqe))
1383		MLX5_SET(sqc, sqc, allow_multi_pkt_send_wqe, 1);
1384	MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST);
1385	MLX5_SET(sqc, sqc, ts_format, ts_format);
1386	MLX5_SET(sqc, sqc, user_index, MLX5_GET(qpc, qpc, user_index));
1387	MLX5_SET(sqc, sqc, cqn, MLX5_GET(qpc, qpc, cqn_snd));
1388	MLX5_SET(sqc, sqc, tis_lst_sz, 1);
1389	MLX5_SET(sqc, sqc, tis_num_0, sq->tisn);
1390	if (MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
1391	    MLX5_CAP_ETH(dev->mdev, swp))
1392		MLX5_SET(sqc, sqc, allow_swp, 1);
1393
1394	wq = MLX5_ADDR_OF(sqc, sqc, wq);
1395	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1396	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1397	MLX5_SET(wq, wq, uar_page, MLX5_GET(qpc, qpc, uar_page));
1398	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1399	MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB));
1400	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_sq_size));
1401	MLX5_SET(wq, wq, log_wq_pg_sz,
1402		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1403	MLX5_SET(wq, wq, page_offset, page_offset_quantized);
1404
1405	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1406	mlx5_ib_populate_pas(sq->ubuffer.umem, page_size, pas, 0);
1407
1408	err = mlx5_core_create_sq_tracked(dev, in, inlen, &sq->base.mqp);
1409
1410	kvfree(in);
1411
1412	if (err)
1413		goto err_umem;
1414
1415	return 0;
1416
1417err_umem:
1418	ib_umem_release(sq->ubuffer.umem);
1419	sq->ubuffer.umem = NULL;
1420
1421	return err;
1422}
1423
1424static void destroy_raw_packet_qp_sq(struct mlx5_ib_dev *dev,
1425				     struct mlx5_ib_sq *sq)
1426{
1427	destroy_flow_rule_vport_sq(sq);
1428	mlx5_core_destroy_sq_tracked(dev, &sq->base.mqp);
1429	ib_umem_release(sq->ubuffer.umem);
1430}
1431
1432static int create_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1433				   struct mlx5_ib_rq *rq, void *qpin,
1434				   struct ib_pd *pd, struct mlx5_ib_cq *cq)
1435{
1436	struct mlx5_ib_qp *mqp = rq->base.container_mibqp;
1437	__be64 *pas;
1438	void *in;
1439	void *rqc;
1440	void *wq;
1441	void *qpc = MLX5_ADDR_OF(create_qp_in, qpin, qpc);
1442	struct ib_umem *umem = rq->base.ubuffer.umem;
1443	unsigned int page_offset_quantized;
1444	unsigned long page_size = 0;
1445	int ts_format;
1446	size_t inlen;
1447	int err;
1448
1449	ts_format = get_rq_ts_format(dev, cq);
1450	if (ts_format < 0)
1451		return ts_format;
1452
1453	page_size = mlx5_umem_find_best_quantized_pgoff(umem, wq, log_wq_pg_sz,
1454							MLX5_ADAPTER_PAGE_SHIFT,
1455							page_offset, 64,
1456							&page_offset_quantized);
1457	if (!page_size)
1458		return -EINVAL;
1459
1460	inlen = MLX5_ST_SZ_BYTES(create_rq_in) +
1461		sizeof(u64) * ib_umem_num_dma_blocks(umem, page_size);
1462	in = kvzalloc(inlen, GFP_KERNEL);
1463	if (!in)
1464		return -ENOMEM;
1465
1466	MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
1467	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
1468	if (!(rq->flags & MLX5_IB_RQ_CVLAN_STRIPPING))
1469		MLX5_SET(rqc, rqc, vsd, 1);
1470	MLX5_SET(rqc, rqc, mem_rq_type, MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
1471	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST);
1472	MLX5_SET(rqc, rqc, ts_format, ts_format);
1473	MLX5_SET(rqc, rqc, flush_in_error_en, 1);
1474	MLX5_SET(rqc, rqc, user_index, MLX5_GET(qpc, qpc, user_index));
1475	MLX5_SET(rqc, rqc, cqn, MLX5_GET(qpc, qpc, cqn_rcv));
1476
1477	if (mqp->flags & IB_QP_CREATE_SCATTER_FCS)
1478		MLX5_SET(rqc, rqc, scatter_fcs, 1);
1479
1480	wq = MLX5_ADDR_OF(rqc, rqc, wq);
1481	MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC);
1482	if (rq->flags & MLX5_IB_RQ_PCI_WRITE_END_PADDING)
1483		MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
1484	MLX5_SET(wq, wq, page_offset, page_offset_quantized);
1485	MLX5_SET(wq, wq, pd, MLX5_GET(qpc, qpc, pd));
1486	MLX5_SET64(wq, wq, dbr_addr, MLX5_GET64(qpc, qpc, dbr_addr));
1487	MLX5_SET(wq, wq, log_wq_stride, MLX5_GET(qpc, qpc, log_rq_stride) + 4);
1488	MLX5_SET(wq, wq, log_wq_pg_sz,
1489		 order_base_2(page_size) - MLX5_ADAPTER_PAGE_SHIFT);
1490	MLX5_SET(wq, wq, log_wq_sz, MLX5_GET(qpc, qpc, log_rq_size));
1491
1492	pas = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
1493	mlx5_ib_populate_pas(umem, page_size, pas, 0);
1494
1495	err = mlx5_core_create_rq_tracked(dev, in, inlen, &rq->base.mqp);
1496
1497	kvfree(in);
1498
1499	return err;
1500}
1501
1502static void destroy_raw_packet_qp_rq(struct mlx5_ib_dev *dev,
1503				     struct mlx5_ib_rq *rq)
1504{
1505	mlx5_core_destroy_rq_tracked(dev, &rq->base.mqp);
1506}
1507
1508static void destroy_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1509				      struct mlx5_ib_rq *rq,
1510				      u32 qp_flags_en,
1511				      struct ib_pd *pd)
1512{
1513	if (qp_flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1514			   MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1515		mlx5_ib_disable_lb(dev, false, true);
1516	mlx5_cmd_destroy_tir(dev->mdev, rq->tirn, to_mpd(pd)->uid);
1517}
1518
1519static int create_raw_packet_qp_tir(struct mlx5_ib_dev *dev,
1520				    struct mlx5_ib_rq *rq, u32 tdn,
1521				    u32 *qp_flags_en, struct ib_pd *pd,
1522				    u32 *out)
1523{
1524	u8 lb_flag = 0;
1525	u32 *in;
1526	void *tirc;
1527	int inlen;
1528	int err;
1529
1530	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1531	in = kvzalloc(inlen, GFP_KERNEL);
1532	if (!in)
1533		return -ENOMEM;
1534
1535	MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
1536	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1537	MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_DIRECT);
1538	MLX5_SET(tirc, tirc, inline_rqn, rq->base.mqp.qpn);
1539	MLX5_SET(tirc, tirc, transport_domain, tdn);
1540	if (*qp_flags_en & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1541		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1542
1543	if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1544		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1545
1546	if (*qp_flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
1547		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1548
1549	if (dev->is_rep) {
1550		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1551		*qp_flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
1552	}
1553
1554	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
1555	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1556	err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1557	rq->tirn = MLX5_GET(create_tir_out, out, tirn);
1558	if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1559		err = mlx5_ib_enable_lb(dev, false, true);
1560
1561		if (err)
1562			destroy_raw_packet_qp_tir(dev, rq, 0, pd);
1563	}
1564	kvfree(in);
1565
1566	return err;
1567}
1568
1569static int create_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1570				u32 *in, size_t inlen, struct ib_pd *pd,
1571				struct ib_udata *udata,
1572				struct mlx5_ib_create_qp_resp *resp,
1573				struct ib_qp_init_attr *init_attr)
1574{
1575	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1576	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1577	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1578	struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1579		udata, struct mlx5_ib_ucontext, ibucontext);
1580	int err;
1581	u32 tdn = mucontext->tdn;
1582	u16 uid = to_mpd(pd)->uid;
1583	u32 out[MLX5_ST_SZ_DW(create_tir_out)] = {};
1584
1585	if (!qp->sq.wqe_cnt && !qp->rq.wqe_cnt)
1586		return -EINVAL;
1587	if (qp->sq.wqe_cnt) {
1588		err = create_raw_packet_qp_tis(dev, qp, sq, tdn, pd);
1589		if (err)
1590			return err;
1591
1592		err = create_raw_packet_qp_sq(dev, udata, sq, in, pd,
1593					      to_mcq(init_attr->send_cq));
1594		if (err)
1595			goto err_destroy_tis;
1596
1597		if (uid) {
1598			resp->tisn = sq->tisn;
1599			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TISN;
1600			resp->sqn = sq->base.mqp.qpn;
1601			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_SQN;
1602		}
1603
1604		sq->base.container_mibqp = qp;
1605		sq->base.mqp.event = mlx5_ib_qp_event;
1606	}
1607
1608	if (qp->rq.wqe_cnt) {
1609		rq->base.container_mibqp = qp;
1610
1611		if (qp->flags & IB_QP_CREATE_CVLAN_STRIPPING)
1612			rq->flags |= MLX5_IB_RQ_CVLAN_STRIPPING;
1613		if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING)
1614			rq->flags |= MLX5_IB_RQ_PCI_WRITE_END_PADDING;
1615		err = create_raw_packet_qp_rq(dev, rq, in, pd,
1616					      to_mcq(init_attr->recv_cq));
1617		if (err)
1618			goto err_destroy_sq;
1619
1620		err = create_raw_packet_qp_tir(dev, rq, tdn, &qp->flags_en, pd,
1621					       out);
1622		if (err)
1623			goto err_destroy_rq;
1624
1625		if (uid) {
1626			resp->rqn = rq->base.mqp.qpn;
1627			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_RQN;
1628			resp->tirn = rq->tirn;
1629			resp->comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1630			if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
1631			    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
1632				resp->tir_icm_addr = MLX5_GET(
1633					create_tir_out, out, icm_address_31_0);
1634				resp->tir_icm_addr |=
1635					(u64)MLX5_GET(create_tir_out, out,
1636						      icm_address_39_32)
1637					<< 32;
1638				resp->tir_icm_addr |=
1639					(u64)MLX5_GET(create_tir_out, out,
1640						      icm_address_63_40)
1641					<< 40;
1642				resp->comp_mask |=
1643					MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1644			}
1645		}
1646	}
1647
1648	qp->trans_qp.base.mqp.qpn = qp->sq.wqe_cnt ? sq->base.mqp.qpn :
1649						     rq->base.mqp.qpn;
1650	return 0;
1651
1652err_destroy_rq:
1653	destroy_raw_packet_qp_rq(dev, rq);
1654err_destroy_sq:
1655	if (!qp->sq.wqe_cnt)
1656		return err;
1657	destroy_raw_packet_qp_sq(dev, sq);
1658err_destroy_tis:
1659	destroy_raw_packet_qp_tis(dev, sq, pd);
1660
1661	return err;
1662}
1663
1664static void destroy_raw_packet_qp(struct mlx5_ib_dev *dev,
1665				  struct mlx5_ib_qp *qp)
1666{
1667	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
1668	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1669	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1670
1671	if (qp->rq.wqe_cnt) {
1672		destroy_raw_packet_qp_tir(dev, rq, qp->flags_en, qp->ibqp.pd);
1673		destroy_raw_packet_qp_rq(dev, rq);
1674	}
1675
1676	if (qp->sq.wqe_cnt) {
1677		destroy_raw_packet_qp_sq(dev, sq);
1678		destroy_raw_packet_qp_tis(dev, sq, qp->ibqp.pd);
1679	}
1680}
1681
1682static void raw_packet_qp_copy_info(struct mlx5_ib_qp *qp,
1683				    struct mlx5_ib_raw_packet_qp *raw_packet_qp)
1684{
1685	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
1686	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
1687
1688	sq->sq = &qp->sq;
1689	rq->rq = &qp->rq;
1690	sq->doorbell = &qp->db;
1691	rq->doorbell = &qp->db;
1692}
1693
1694static void destroy_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp)
1695{
1696	if (qp->flags_en & (MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
1697			    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC))
1698		mlx5_ib_disable_lb(dev, false, true);
1699	mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
1700			     to_mpd(qp->ibqp.pd)->uid);
1701}
1702
1703struct mlx5_create_qp_params {
1704	struct ib_udata *udata;
1705	size_t inlen;
1706	size_t outlen;
1707	size_t ucmd_size;
1708	void *ucmd;
1709	u8 is_rss_raw : 1;
1710	struct ib_qp_init_attr *attr;
1711	u32 uidx;
1712	struct mlx5_ib_create_qp_resp resp;
1713};
1714
1715static int create_rss_raw_qp_tir(struct mlx5_ib_dev *dev, struct ib_pd *pd,
1716				 struct mlx5_ib_qp *qp,
1717				 struct mlx5_create_qp_params *params)
1718{
1719	struct ib_qp_init_attr *init_attr = params->attr;
1720	struct mlx5_ib_create_qp_rss *ucmd = params->ucmd;
1721	struct ib_udata *udata = params->udata;
1722	struct mlx5_ib_ucontext *mucontext = rdma_udata_to_drv_context(
1723		udata, struct mlx5_ib_ucontext, ibucontext);
1724	int inlen;
1725	int outlen;
1726	int err;
1727	u32 *in;
1728	u32 *out;
1729	void *tirc;
1730	void *hfso;
1731	u32 selected_fields = 0;
1732	u32 outer_l4;
1733	u32 tdn = mucontext->tdn;
1734	u8 lb_flag = 0;
1735
1736	if (ucmd->comp_mask) {
1737		mlx5_ib_dbg(dev, "invalid comp mask\n");
1738		return -EOPNOTSUPP;
1739	}
1740
1741	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER &&
1742	    !(ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)) {
1743		mlx5_ib_dbg(dev, "Tunnel offloads must be set for inner RSS\n");
1744		return -EOPNOTSUPP;
1745	}
1746
1747	if (dev->is_rep)
1748		qp->flags_en |= MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC;
1749
1750	if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC)
1751		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_UNICAST;
1752
1753	if (qp->flags_en & MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC)
1754		lb_flag |= MLX5_TIRC_SELF_LB_BLOCK_BLOCK_MULTICAST;
1755
1756	inlen = MLX5_ST_SZ_BYTES(create_tir_in);
1757	outlen = MLX5_ST_SZ_BYTES(create_tir_out);
1758	in = kvzalloc(inlen + outlen, GFP_KERNEL);
1759	if (!in)
1760		return -ENOMEM;
1761
1762	out = in + MLX5_ST_SZ_DW(create_tir_in);
1763	MLX5_SET(create_tir_in, in, uid, to_mpd(pd)->uid);
1764	tirc = MLX5_ADDR_OF(create_tir_in, in, ctx);
1765	MLX5_SET(tirc, tirc, disp_type,
1766		 MLX5_TIRC_DISP_TYPE_INDIRECT);
1767	MLX5_SET(tirc, tirc, indirect_table,
1768		 init_attr->rwq_ind_tbl->ind_tbl_num);
1769	MLX5_SET(tirc, tirc, transport_domain, tdn);
1770
1771	hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1772
1773	if (ucmd->flags & MLX5_QP_FLAG_TUNNEL_OFFLOADS)
1774		MLX5_SET(tirc, tirc, tunneled_offload_en, 1);
1775
1776	MLX5_SET(tirc, tirc, self_lb_block, lb_flag);
1777
1778	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_INNER)
1779		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner);
1780	else
1781		hfso = MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer);
1782
1783	switch (ucmd->rx_hash_function) {
1784	case MLX5_RX_HASH_FUNC_TOEPLITZ:
1785	{
1786		void *rss_key = MLX5_ADDR_OF(tirc, tirc, rx_hash_toeplitz_key);
1787		size_t len = MLX5_FLD_SZ_BYTES(tirc, rx_hash_toeplitz_key);
1788
1789		if (len != ucmd->rx_key_len) {
1790			err = -EINVAL;
1791			goto err;
1792		}
1793
1794		MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_TOEPLITZ);
1795		memcpy(rss_key, ucmd->rx_hash_key, len);
1796		break;
1797	}
1798	default:
1799		err = -EOPNOTSUPP;
1800		goto err;
1801	}
1802
1803	if (!ucmd->rx_hash_fields_mask) {
1804		/* special case when this TIR serves as steering entry without hashing */
1805		if (!init_attr->rwq_ind_tbl->log_ind_tbl_size)
1806			goto create_tir;
1807		err = -EINVAL;
1808		goto err;
1809	}
1810
1811	if (((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1812	     (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4)) &&
1813	     ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1814	     (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))) {
1815		err = -EINVAL;
1816		goto err;
1817	}
1818
1819	/* If none of IPV4 & IPV6 SRC/DST was set - this bit field is ignored */
1820	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1821	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4))
1822		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1823			 MLX5_L3_PROT_TYPE_IPV4);
1824	else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6) ||
1825		 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1826		MLX5_SET(rx_hash_field_select, hfso, l3_prot_type,
1827			 MLX5_L3_PROT_TYPE_IPV6);
1828
1829	outer_l4 = ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1830		    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1831			   << 0 |
1832		   ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1833		    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1834			   << 1 |
1835		   (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI) << 2;
1836
1837	/* Check that only one l4 protocol is set */
1838	if (outer_l4 & (outer_l4 - 1)) {
1839		err = -EINVAL;
1840		goto err;
1841	}
1842
1843	/* If none of TCP & UDP SRC/DST was set - this bit field is ignored */
1844	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1845	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP))
1846		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1847			 MLX5_L4_PROT_TYPE_TCP);
1848	else if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP) ||
1849		 (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1850		MLX5_SET(rx_hash_field_select, hfso, l4_prot_type,
1851			 MLX5_L4_PROT_TYPE_UDP);
1852
1853	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV4) ||
1854	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_IPV6))
1855		selected_fields |= MLX5_HASH_FIELD_SEL_SRC_IP;
1856
1857	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV4) ||
1858	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_IPV6))
1859		selected_fields |= MLX5_HASH_FIELD_SEL_DST_IP;
1860
1861	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_TCP) ||
1862	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_SRC_PORT_UDP))
1863		selected_fields |= MLX5_HASH_FIELD_SEL_L4_SPORT;
1864
1865	if ((ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_TCP) ||
1866	    (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_DST_PORT_UDP))
1867		selected_fields |= MLX5_HASH_FIELD_SEL_L4_DPORT;
1868
1869	if (ucmd->rx_hash_fields_mask & MLX5_RX_HASH_IPSEC_SPI)
1870		selected_fields |= MLX5_HASH_FIELD_SEL_IPSEC_SPI;
1871
1872	MLX5_SET(rx_hash_field_select, hfso, selected_fields, selected_fields);
1873
1874create_tir:
1875	MLX5_SET(create_tir_in, in, opcode, MLX5_CMD_OP_CREATE_TIR);
1876	err = mlx5_cmd_exec_inout(dev->mdev, create_tir, in, out);
1877
1878	qp->rss_qp.tirn = MLX5_GET(create_tir_out, out, tirn);
1879	if (!err && MLX5_GET(tirc, tirc, self_lb_block)) {
1880		err = mlx5_ib_enable_lb(dev, false, true);
1881
1882		if (err)
1883			mlx5_cmd_destroy_tir(dev->mdev, qp->rss_qp.tirn,
1884					     to_mpd(pd)->uid);
1885	}
1886
1887	if (err)
1888		goto err;
1889
1890	if (mucontext->devx_uid) {
1891		params->resp.comp_mask |= MLX5_IB_CREATE_QP_RESP_MASK_TIRN;
1892		params->resp.tirn = qp->rss_qp.tirn;
1893		if (MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner) ||
1894		    MLX5_CAP_FLOWTABLE_NIC_RX(dev->mdev, sw_owner_v2)) {
1895			params->resp.tir_icm_addr =
1896				MLX5_GET(create_tir_out, out, icm_address_31_0);
1897			params->resp.tir_icm_addr |=
1898				(u64)MLX5_GET(create_tir_out, out,
1899					      icm_address_39_32)
1900				<< 32;
1901			params->resp.tir_icm_addr |=
1902				(u64)MLX5_GET(create_tir_out, out,
1903					      icm_address_63_40)
1904				<< 40;
1905			params->resp.comp_mask |=
1906				MLX5_IB_CREATE_QP_RESP_MASK_TIR_ICM_ADDR;
1907		}
1908	}
1909
1910	kvfree(in);
1911	/* qpn is reserved for that QP */
1912	qp->trans_qp.base.mqp.qpn = 0;
1913	qp->is_rss = true;
1914	return 0;
1915
1916err:
1917	kvfree(in);
1918	return err;
1919}
1920
1921static void configure_requester_scat_cqe(struct mlx5_ib_dev *dev,
1922					 struct mlx5_ib_qp *qp,
1923					 struct ib_qp_init_attr *init_attr,
1924					 void *qpc)
1925{
1926	int scqe_sz;
1927	bool allow_scat_cqe = false;
1928
1929	allow_scat_cqe = qp->flags_en & MLX5_QP_FLAG_ALLOW_SCATTER_CQE;
1930
1931	if (!allow_scat_cqe && init_attr->sq_sig_type != IB_SIGNAL_ALL_WR)
1932		return;
1933
1934	scqe_sz = mlx5_ib_get_cqe_size(init_attr->send_cq);
1935	if (scqe_sz == 128) {
1936		MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA64_CQE);
1937		return;
1938	}
1939
1940	if (init_attr->qp_type != MLX5_IB_QPT_DCI ||
1941	    MLX5_CAP_GEN(dev->mdev, dc_req_scat_data_cqe))
1942		MLX5_SET(qpc, qpc, cs_req, MLX5_REQ_SCAT_DATA32_CQE);
1943}
1944
1945static int atomic_size_to_mode(int size_mask)
1946{
1947	/* driver does not support atomic_size > 256B
1948	 * and does not know how to translate bigger sizes
1949	 */
1950	int supported_size_mask = size_mask & 0x1ff;
1951	int log_max_size;
1952
1953	if (!supported_size_mask)
1954		return -EOPNOTSUPP;
1955
1956	log_max_size = __fls(supported_size_mask);
1957
1958	if (log_max_size > 3)
1959		return log_max_size;
1960
1961	return MLX5_ATOMIC_MODE_8B;
1962}
1963
1964static int get_atomic_mode(struct mlx5_ib_dev *dev,
1965			   enum ib_qp_type qp_type)
1966{
1967	u8 atomic_operations = MLX5_CAP_ATOMIC(dev->mdev, atomic_operations);
1968	u8 atomic = MLX5_CAP_GEN(dev->mdev, atomic);
1969	int atomic_mode = -EOPNOTSUPP;
1970	int atomic_size_mask;
1971
1972	if (!atomic)
1973		return -EOPNOTSUPP;
1974
1975	if (qp_type == MLX5_IB_QPT_DCT)
1976		atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_dc);
1977	else
1978		atomic_size_mask = MLX5_CAP_ATOMIC(dev->mdev, atomic_size_qp);
1979
1980	if ((atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_CMP_SWAP) ||
1981	    (atomic_operations & MLX5_ATOMIC_OPS_EXTENDED_FETCH_ADD))
1982		atomic_mode = atomic_size_to_mode(atomic_size_mask);
1983
1984	if (atomic_mode <= 0 &&
1985	    (atomic_operations & MLX5_ATOMIC_OPS_CMP_SWAP &&
1986	     atomic_operations & MLX5_ATOMIC_OPS_FETCH_ADD))
1987		atomic_mode = MLX5_ATOMIC_MODE_IB_COMP;
1988
1989	return atomic_mode;
1990}
1991
1992static int create_xrc_tgt_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
1993			     struct mlx5_create_qp_params *params)
1994{
1995	struct ib_qp_init_attr *attr = params->attr;
1996	u32 uidx = params->uidx;
1997	struct mlx5_ib_resources *devr = &dev->devr;
1998	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
1999	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2000	struct mlx5_core_dev *mdev = dev->mdev;
2001	struct mlx5_ib_qp_base *base;
2002	unsigned long flags;
2003	void *qpc;
2004	u32 *in;
2005	int err;
2006
2007	if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2008		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2009
2010	in = kvzalloc(inlen, GFP_KERNEL);
2011	if (!in)
2012		return -ENOMEM;
2013
2014	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2015
2016	MLX5_SET(qpc, qpc, st, MLX5_QP_ST_XRC);
2017	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2018	MLX5_SET(qpc, qpc, pd, to_mpd(devr->p0)->pdn);
2019
2020	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2021		MLX5_SET(qpc, qpc, block_lb_mc, 1);
2022	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2023		MLX5_SET(qpc, qpc, cd_master, 1);
2024	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2025		MLX5_SET(qpc, qpc, cd_slave_send, 1);
2026	if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
2027		MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2028
2029	MLX5_SET(qpc, qpc, ts_format, mlx5_get_qp_default_ts(dev->mdev));
2030	MLX5_SET(qpc, qpc, rq_type, MLX5_SRQ_RQ);
2031	MLX5_SET(qpc, qpc, no_sq, 1);
2032	MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2033	MLX5_SET(qpc, qpc, cqn_snd, to_mcq(devr->c0)->mcq.cqn);
2034	MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
2035	MLX5_SET(qpc, qpc, xrcd, to_mxrcd(attr->xrcd)->xrcdn);
2036	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2037
2038	/* 0xffffff means we ask to work with cqe version 0 */
2039	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2040		MLX5_SET(qpc, qpc, user_index, uidx);
2041
2042	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
2043		MLX5_SET(qpc, qpc, end_padding_mode,
2044			 MLX5_WQ_END_PAD_MODE_ALIGN);
2045		/* Special case to clean flag */
2046		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2047	}
2048
2049	base = &qp->trans_qp.base;
2050	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2051	kvfree(in);
2052	if (err)
2053		return err;
2054
2055	base->container_mibqp = qp;
2056	base->mqp.event = mlx5_ib_qp_event;
2057	if (MLX5_CAP_GEN(mdev, ece_support))
2058		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2059
2060	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2061	list_add_tail(&qp->qps_list, &dev->qp_list);
2062	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2063
2064	qp->trans_qp.xrcdn = to_mxrcd(attr->xrcd)->xrcdn;
2065	return 0;
2066}
2067
2068static int create_dci(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2069		      struct mlx5_ib_qp *qp,
2070		      struct mlx5_create_qp_params *params)
2071{
2072	struct ib_qp_init_attr *init_attr = params->attr;
2073	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2074	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2075	struct ib_udata *udata = params->udata;
2076	u32 uidx = params->uidx;
2077	struct mlx5_ib_resources *devr = &dev->devr;
2078	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2079	struct mlx5_core_dev *mdev = dev->mdev;
2080	struct mlx5_ib_cq *send_cq;
2081	struct mlx5_ib_cq *recv_cq;
2082	unsigned long flags;
2083	struct mlx5_ib_qp_base *base;
2084	int ts_format;
2085	int mlx5_st;
2086	void *qpc;
2087	u32 *in;
2088	int err;
2089
2090	spin_lock_init(&qp->sq.lock);
2091	spin_lock_init(&qp->rq.lock);
2092
2093	mlx5_st = to_mlx5_st(qp->type);
2094	if (mlx5_st < 0)
2095		return -EINVAL;
2096
2097	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2098		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2099
2100	base = &qp->trans_qp.base;
2101
2102	qp->has_rq = qp_has_rq(init_attr);
2103	err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
2104	if (err) {
2105		mlx5_ib_dbg(dev, "err %d\n", err);
2106		return err;
2107	}
2108
2109	if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
2110	    ucmd->rq_wqe_count != qp->rq.wqe_cnt)
2111		return -EINVAL;
2112
2113	if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
2114		return -EINVAL;
2115
2116	ts_format = get_qp_ts_format(dev, to_mcq(init_attr->send_cq),
2117				     to_mcq(init_attr->recv_cq));
2118
2119	if (ts_format < 0)
2120		return ts_format;
2121
2122	err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
2123			      &inlen, base, ucmd);
2124	if (err)
2125		return err;
2126
2127	if (MLX5_CAP_GEN(mdev, ece_support))
2128		MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
2129	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2130
2131	MLX5_SET(qpc, qpc, st, mlx5_st);
2132	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2133	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
2134
2135	if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
2136		MLX5_SET(qpc, qpc, wq_signature, 1);
2137
2138	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2139		MLX5_SET(qpc, qpc, cd_master, 1);
2140	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2141		MLX5_SET(qpc, qpc, cd_slave_send, 1);
2142	if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE)
2143		configure_requester_scat_cqe(dev, qp, init_attr, qpc);
2144
2145	if (qp->rq.wqe_cnt) {
2146		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2147		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2148	}
2149
2150	if (qp->flags_en & MLX5_QP_FLAG_DCI_STREAM) {
2151		MLX5_SET(qpc, qpc, log_num_dci_stream_channels,
2152			 ucmd->dci_streams.log_num_concurent);
2153		MLX5_SET(qpc, qpc, log_num_dci_errored_streams,
2154			 ucmd->dci_streams.log_num_errored);
2155	}
2156
2157	MLX5_SET(qpc, qpc, ts_format, ts_format);
2158	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
2159
2160	MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2161
2162	/* Set default resources */
2163	if (init_attr->srq) {
2164		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2165		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2166			 to_msrq(init_attr->srq)->msrq.srqn);
2167	} else {
2168		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2169		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2170			 to_msrq(devr->s1)->msrq.srqn);
2171	}
2172
2173	if (init_attr->send_cq)
2174		MLX5_SET(qpc, qpc, cqn_snd,
2175			 to_mcq(init_attr->send_cq)->mcq.cqn);
2176
2177	if (init_attr->recv_cq)
2178		MLX5_SET(qpc, qpc, cqn_rcv,
2179			 to_mcq(init_attr->recv_cq)->mcq.cqn);
2180
2181	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2182
2183	/* 0xffffff means we ask to work with cqe version 0 */
2184	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2185		MLX5_SET(qpc, qpc, user_index, uidx);
2186
2187	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING) {
2188		MLX5_SET(qpc, qpc, end_padding_mode,
2189			 MLX5_WQ_END_PAD_MODE_ALIGN);
2190		/* Special case to clean flag */
2191		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2192	}
2193
2194	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2195
2196	kvfree(in);
2197	if (err)
2198		goto err_create;
2199
2200	base->container_mibqp = qp;
2201	base->mqp.event = mlx5_ib_qp_event;
2202	if (MLX5_CAP_GEN(mdev, ece_support))
2203		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2204
2205	get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
2206		&send_cq, &recv_cq);
2207	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2208	mlx5_ib_lock_cqs(send_cq, recv_cq);
2209	/* Maintain device to QPs access, needed for further handling via reset
2210	 * flow
2211	 */
2212	list_add_tail(&qp->qps_list, &dev->qp_list);
2213	/* Maintain CQ to QPs access, needed for further handling via reset flow
2214	 */
2215	if (send_cq)
2216		list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2217	if (recv_cq)
2218		list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2219	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2220	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2221
2222	return 0;
2223
2224err_create:
2225	destroy_qp(dev, qp, base, udata);
2226	return err;
2227}
2228
2229static int create_user_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2230			  struct mlx5_ib_qp *qp,
2231			  struct mlx5_create_qp_params *params)
2232{
2233	struct ib_qp_init_attr *init_attr = params->attr;
2234	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2235	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2236	struct ib_udata *udata = params->udata;
2237	u32 uidx = params->uidx;
2238	struct mlx5_ib_resources *devr = &dev->devr;
2239	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2240	struct mlx5_core_dev *mdev = dev->mdev;
2241	struct mlx5_ib_cq *send_cq;
2242	struct mlx5_ib_cq *recv_cq;
2243	unsigned long flags;
2244	struct mlx5_ib_qp_base *base;
2245	int ts_format;
2246	int mlx5_st;
2247	void *qpc;
2248	u32 *in;
2249	int err;
2250
2251	spin_lock_init(&qp->sq.lock);
2252	spin_lock_init(&qp->rq.lock);
2253
2254	mlx5_st = to_mlx5_st(qp->type);
2255	if (mlx5_st < 0)
2256		return -EINVAL;
2257
2258	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2259		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2260
2261	if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
2262		qp->underlay_qpn = init_attr->source_qpn;
2263
2264	base = (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2265		qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
2266	       &qp->raw_packet_qp.rq.base :
2267	       &qp->trans_qp.base;
2268
2269	qp->has_rq = qp_has_rq(init_attr);
2270	err = set_rq_size(dev, &init_attr->cap, qp->has_rq, qp, ucmd);
2271	if (err) {
2272		mlx5_ib_dbg(dev, "err %d\n", err);
2273		return err;
2274	}
2275
2276	if (ucmd->rq_wqe_shift != qp->rq.wqe_shift ||
2277	    ucmd->rq_wqe_count != qp->rq.wqe_cnt)
2278		return -EINVAL;
2279
2280	if (ucmd->sq_wqe_count > (1 << MLX5_CAP_GEN(mdev, log_max_qp_sz)))
2281		return -EINVAL;
2282
2283	if (init_attr->qp_type != IB_QPT_RAW_PACKET) {
2284		ts_format = get_qp_ts_format(dev, to_mcq(init_attr->send_cq),
2285					     to_mcq(init_attr->recv_cq));
2286		if (ts_format < 0)
2287			return ts_format;
2288	}
2289
2290	err = _create_user_qp(dev, pd, qp, udata, init_attr, &in, &params->resp,
2291			      &inlen, base, ucmd);
2292	if (err)
2293		return err;
2294
2295	if (is_sqp(init_attr->qp_type))
2296		qp->port = init_attr->port_num;
2297
2298	if (MLX5_CAP_GEN(mdev, ece_support))
2299		MLX5_SET(create_qp_in, in, ece, ucmd->ece_options);
2300	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2301
2302	MLX5_SET(qpc, qpc, st, mlx5_st);
2303	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2304	MLX5_SET(qpc, qpc, pd, to_mpd(pd)->pdn);
2305
2306	if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE)
2307		MLX5_SET(qpc, qpc, wq_signature, 1);
2308
2309	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2310		MLX5_SET(qpc, qpc, block_lb_mc, 1);
2311
2312	if (qp->flags & IB_QP_CREATE_CROSS_CHANNEL)
2313		MLX5_SET(qpc, qpc, cd_master, 1);
2314	if (qp->flags & IB_QP_CREATE_MANAGED_SEND)
2315		MLX5_SET(qpc, qpc, cd_slave_send, 1);
2316	if (qp->flags & IB_QP_CREATE_MANAGED_RECV)
2317		MLX5_SET(qpc, qpc, cd_slave_receive, 1);
2318	if (qp->flags_en & MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE)
2319		MLX5_SET(qpc, qpc, req_e2e_credit_mode, 1);
2320	if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
2321	    (init_attr->qp_type == IB_QPT_RC ||
2322	     init_attr->qp_type == IB_QPT_UC)) {
2323		int rcqe_sz = mlx5_ib_get_cqe_size(init_attr->recv_cq);
2324
2325		MLX5_SET(qpc, qpc, cs_res,
2326			 rcqe_sz == 128 ? MLX5_RES_SCAT_DATA64_CQE :
2327					  MLX5_RES_SCAT_DATA32_CQE);
2328	}
2329	if ((qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) &&
2330	    (qp->type == MLX5_IB_QPT_DCI || qp->type == IB_QPT_RC))
2331		configure_requester_scat_cqe(dev, qp, init_attr, qpc);
2332
2333	if (qp->rq.wqe_cnt) {
2334		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2335		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2336	}
2337
2338	if (init_attr->qp_type != IB_QPT_RAW_PACKET)
2339		MLX5_SET(qpc, qpc, ts_format, ts_format);
2340
2341	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, init_attr));
2342
2343	if (qp->sq.wqe_cnt) {
2344		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2345	} else {
2346		MLX5_SET(qpc, qpc, no_sq, 1);
2347		if (init_attr->srq &&
2348		    init_attr->srq->srq_type == IB_SRQT_TM)
2349			MLX5_SET(qpc, qpc, offload_type,
2350				 MLX5_QPC_OFFLOAD_TYPE_RNDV);
2351	}
2352
2353	/* Set default resources */
2354	switch (init_attr->qp_type) {
2355	case IB_QPT_XRC_INI:
2356		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(devr->c0)->mcq.cqn);
2357		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2358		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s0)->msrq.srqn);
2359		break;
2360	default:
2361		if (init_attr->srq) {
2362			MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2363			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(init_attr->srq)->msrq.srqn);
2364		} else {
2365			MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2366			MLX5_SET(qpc, qpc, srqn_rmpn_xrqn, to_msrq(devr->s1)->msrq.srqn);
2367		}
2368	}
2369
2370	if (init_attr->send_cq)
2371		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(init_attr->send_cq)->mcq.cqn);
2372
2373	if (init_attr->recv_cq)
2374		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(init_attr->recv_cq)->mcq.cqn);
2375
2376	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2377
2378	/* 0xffffff means we ask to work with cqe version 0 */
2379	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2380		MLX5_SET(qpc, qpc, user_index, uidx);
2381
2382	if (qp->flags & IB_QP_CREATE_PCI_WRITE_END_PADDING &&
2383	    init_attr->qp_type != IB_QPT_RAW_PACKET) {
2384		MLX5_SET(qpc, qpc, end_padding_mode,
2385			 MLX5_WQ_END_PAD_MODE_ALIGN);
2386		/* Special case to clean flag */
2387		qp->flags &= ~IB_QP_CREATE_PCI_WRITE_END_PADDING;
2388	}
2389
2390	if (init_attr->qp_type == IB_QPT_RAW_PACKET ||
2391	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2392		qp->raw_packet_qp.sq.ubuffer.buf_addr = ucmd->sq_buf_addr;
2393		raw_packet_qp_copy_info(qp, &qp->raw_packet_qp);
2394		err = create_raw_packet_qp(dev, qp, in, inlen, pd, udata,
2395					   &params->resp, init_attr);
2396	} else
2397		err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2398
2399	kvfree(in);
2400	if (err)
2401		goto err_create;
2402
2403	base->container_mibqp = qp;
2404	base->mqp.event = mlx5_ib_qp_event;
2405	if (MLX5_CAP_GEN(mdev, ece_support))
2406		params->resp.ece_options = MLX5_GET(create_qp_out, out, ece);
2407
2408	get_cqs(qp->type, init_attr->send_cq, init_attr->recv_cq,
2409		&send_cq, &recv_cq);
2410	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2411	mlx5_ib_lock_cqs(send_cq, recv_cq);
2412	/* Maintain device to QPs access, needed for further handling via reset
2413	 * flow
2414	 */
2415	list_add_tail(&qp->qps_list, &dev->qp_list);
2416	/* Maintain CQ to QPs access, needed for further handling via reset flow
2417	 */
2418	if (send_cq)
2419		list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2420	if (recv_cq)
2421		list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2422	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2423	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2424
2425	return 0;
2426
2427err_create:
2428	destroy_qp(dev, qp, base, udata);
2429	return err;
2430}
2431
2432static int create_kernel_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2433			    struct mlx5_ib_qp *qp,
2434			    struct mlx5_create_qp_params *params)
2435{
2436	struct ib_qp_init_attr *attr = params->attr;
2437	u32 uidx = params->uidx;
2438	struct mlx5_ib_resources *devr = &dev->devr;
2439	u32 out[MLX5_ST_SZ_DW(create_qp_out)] = {};
2440	int inlen = MLX5_ST_SZ_BYTES(create_qp_in);
2441	struct mlx5_core_dev *mdev = dev->mdev;
2442	struct mlx5_ib_cq *send_cq;
2443	struct mlx5_ib_cq *recv_cq;
2444	unsigned long flags;
2445	struct mlx5_ib_qp_base *base;
2446	int mlx5_st;
2447	void *qpc;
2448	u32 *in;
2449	int err;
2450
2451	spin_lock_init(&qp->sq.lock);
2452	spin_lock_init(&qp->rq.lock);
2453
2454	mlx5_st = to_mlx5_st(qp->type);
2455	if (mlx5_st < 0)
2456		return -EINVAL;
2457
2458	if (attr->sq_sig_type == IB_SIGNAL_ALL_WR)
2459		qp->sq_signal_bits = MLX5_WQE_CTRL_CQ_UPDATE;
2460
2461	base = &qp->trans_qp.base;
2462
2463	qp->has_rq = qp_has_rq(attr);
2464	err = set_rq_size(dev, &attr->cap, qp->has_rq, qp, NULL);
2465	if (err) {
2466		mlx5_ib_dbg(dev, "err %d\n", err);
2467		return err;
2468	}
2469
2470	err = _create_kernel_qp(dev, attr, qp, &in, &inlen, base);
2471	if (err)
2472		return err;
2473
2474	if (is_sqp(attr->qp_type))
2475		qp->port = attr->port_num;
2476
2477	qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
2478
2479	MLX5_SET(qpc, qpc, st, mlx5_st);
2480	MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
2481
2482	if (attr->qp_type != MLX5_IB_QPT_REG_UMR)
2483		MLX5_SET(qpc, qpc, pd, to_mpd(pd ? pd : devr->p0)->pdn);
2484	else
2485		MLX5_SET(qpc, qpc, latency_sensitive, 1);
2486
2487
2488	if (qp->flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
2489		MLX5_SET(qpc, qpc, block_lb_mc, 1);
2490
2491	if (qp->rq.wqe_cnt) {
2492		MLX5_SET(qpc, qpc, log_rq_stride, qp->rq.wqe_shift - 4);
2493		MLX5_SET(qpc, qpc, log_rq_size, ilog2(qp->rq.wqe_cnt));
2494	}
2495
2496	MLX5_SET(qpc, qpc, rq_type, get_rx_type(qp, attr));
2497
2498	if (qp->sq.wqe_cnt)
2499		MLX5_SET(qpc, qpc, log_sq_size, ilog2(qp->sq.wqe_cnt));
2500	else
2501		MLX5_SET(qpc, qpc, no_sq, 1);
2502
2503	if (attr->srq) {
2504		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn0);
2505		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2506			 to_msrq(attr->srq)->msrq.srqn);
2507	} else {
2508		MLX5_SET(qpc, qpc, xrcd, devr->xrcdn1);
2509		MLX5_SET(qpc, qpc, srqn_rmpn_xrqn,
2510			 to_msrq(devr->s1)->msrq.srqn);
2511	}
2512
2513	if (attr->send_cq)
2514		MLX5_SET(qpc, qpc, cqn_snd, to_mcq(attr->send_cq)->mcq.cqn);
2515
2516	if (attr->recv_cq)
2517		MLX5_SET(qpc, qpc, cqn_rcv, to_mcq(attr->recv_cq)->mcq.cqn);
2518
2519	MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
2520
2521	/* 0xffffff means we ask to work with cqe version 0 */
2522	if (MLX5_CAP_GEN(mdev, cqe_version) == MLX5_CQE_VERSION_V1)
2523		MLX5_SET(qpc, qpc, user_index, uidx);
2524
2525	/* we use IB_QP_CREATE_IPOIB_UD_LSO to indicates ipoib qp */
2526	if (qp->flags & IB_QP_CREATE_IPOIB_UD_LSO)
2527		MLX5_SET(qpc, qpc, ulp_stateless_offload_mode, 1);
2528
2529	if (qp->flags & IB_QP_CREATE_INTEGRITY_EN &&
2530	    MLX5_CAP_GEN(mdev, go_back_n))
2531		MLX5_SET(qpc, qpc, retry_mode, MLX5_QP_RM_GO_BACK_N);
2532
2533	err = mlx5_qpc_create_qp(dev, &base->mqp, in, inlen, out);
2534	kvfree(in);
2535	if (err)
2536		goto err_create;
2537
2538	base->container_mibqp = qp;
2539	base->mqp.event = mlx5_ib_qp_event;
2540
2541	get_cqs(qp->type, attr->send_cq, attr->recv_cq,
2542		&send_cq, &recv_cq);
2543	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2544	mlx5_ib_lock_cqs(send_cq, recv_cq);
2545	/* Maintain device to QPs access, needed for further handling via reset
2546	 * flow
2547	 */
2548	list_add_tail(&qp->qps_list, &dev->qp_list);
2549	/* Maintain CQ to QPs access, needed for further handling via reset flow
2550	 */
2551	if (send_cq)
2552		list_add_tail(&qp->cq_send_list, &send_cq->list_send_qp);
2553	if (recv_cq)
2554		list_add_tail(&qp->cq_recv_list, &recv_cq->list_recv_qp);
2555	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2556	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2557
2558	return 0;
2559
2560err_create:
2561	destroy_qp(dev, qp, base, NULL);
2562	return err;
2563}
2564
2565static void mlx5_ib_lock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2566	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
2567{
2568	if (send_cq) {
2569		if (recv_cq) {
2570			if (send_cq->mcq.cqn < recv_cq->mcq.cqn)  {
2571				spin_lock(&send_cq->lock);
2572				spin_lock_nested(&recv_cq->lock,
2573						 SINGLE_DEPTH_NESTING);
2574			} else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2575				spin_lock(&send_cq->lock);
2576				__acquire(&recv_cq->lock);
2577			} else {
2578				spin_lock(&recv_cq->lock);
2579				spin_lock_nested(&send_cq->lock,
2580						 SINGLE_DEPTH_NESTING);
2581			}
2582		} else {
2583			spin_lock(&send_cq->lock);
2584			__acquire(&recv_cq->lock);
2585		}
2586	} else if (recv_cq) {
2587		spin_lock(&recv_cq->lock);
2588		__acquire(&send_cq->lock);
2589	} else {
2590		__acquire(&send_cq->lock);
2591		__acquire(&recv_cq->lock);
2592	}
2593}
2594
2595static void mlx5_ib_unlock_cqs(struct mlx5_ib_cq *send_cq, struct mlx5_ib_cq *recv_cq)
2596	__releases(&send_cq->lock) __releases(&recv_cq->lock)
2597{
2598	if (send_cq) {
2599		if (recv_cq) {
2600			if (send_cq->mcq.cqn < recv_cq->mcq.cqn)  {
2601				spin_unlock(&recv_cq->lock);
2602				spin_unlock(&send_cq->lock);
2603			} else if (send_cq->mcq.cqn == recv_cq->mcq.cqn) {
2604				__release(&recv_cq->lock);
2605				spin_unlock(&send_cq->lock);
2606			} else {
2607				spin_unlock(&send_cq->lock);
2608				spin_unlock(&recv_cq->lock);
2609			}
2610		} else {
2611			__release(&recv_cq->lock);
2612			spin_unlock(&send_cq->lock);
2613		}
2614	} else if (recv_cq) {
2615		__release(&send_cq->lock);
2616		spin_unlock(&recv_cq->lock);
2617	} else {
2618		__release(&recv_cq->lock);
2619		__release(&send_cq->lock);
2620	}
2621}
2622
2623static void get_cqs(enum ib_qp_type qp_type,
2624		    struct ib_cq *ib_send_cq, struct ib_cq *ib_recv_cq,
2625		    struct mlx5_ib_cq **send_cq, struct mlx5_ib_cq **recv_cq)
2626{
2627	switch (qp_type) {
2628	case IB_QPT_XRC_TGT:
2629		*send_cq = NULL;
2630		*recv_cq = NULL;
2631		break;
2632	case MLX5_IB_QPT_REG_UMR:
2633	case IB_QPT_XRC_INI:
2634		*send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2635		*recv_cq = NULL;
2636		break;
2637
2638	case IB_QPT_SMI:
2639	case MLX5_IB_QPT_HW_GSI:
2640	case IB_QPT_RC:
2641	case IB_QPT_UC:
2642	case IB_QPT_UD:
2643	case IB_QPT_RAW_PACKET:
2644		*send_cq = ib_send_cq ? to_mcq(ib_send_cq) : NULL;
2645		*recv_cq = ib_recv_cq ? to_mcq(ib_recv_cq) : NULL;
2646		break;
2647	default:
2648		*send_cq = NULL;
2649		*recv_cq = NULL;
2650		break;
2651	}
2652}
2653
2654static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2655				const struct mlx5_modify_raw_qp_param *raw_qp_param,
2656				u8 lag_tx_affinity);
2657
2658static void destroy_qp_common(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2659			      struct ib_udata *udata)
2660{
2661	struct mlx5_ib_cq *send_cq, *recv_cq;
2662	struct mlx5_ib_qp_base *base;
2663	unsigned long flags;
2664	int err;
2665
2666	if (qp->is_rss) {
2667		destroy_rss_raw_qp_tir(dev, qp);
2668		return;
2669	}
2670
2671	base = (qp->type == IB_QPT_RAW_PACKET ||
2672		qp->flags & IB_QP_CREATE_SOURCE_QPN) ?
2673		       &qp->raw_packet_qp.rq.base :
2674		       &qp->trans_qp.base;
2675
2676	if (qp->state != IB_QPS_RESET) {
2677		if (qp->type != IB_QPT_RAW_PACKET &&
2678		    !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) {
2679			err = mlx5_core_qp_modify(dev, MLX5_CMD_OP_2RST_QP, 0,
2680						  NULL, &base->mqp, NULL);
2681		} else {
2682			struct mlx5_modify_raw_qp_param raw_qp_param = {
2683				.operation = MLX5_CMD_OP_2RST_QP
2684			};
2685
2686			err = modify_raw_packet_qp(dev, qp, &raw_qp_param, 0);
2687		}
2688		if (err)
2689			mlx5_ib_warn(dev, "mlx5_ib: modify QP 0x%06x to RESET failed\n",
2690				     base->mqp.qpn);
2691	}
2692
2693	get_cqs(qp->type, qp->ibqp.send_cq, qp->ibqp.recv_cq, &send_cq,
2694		&recv_cq);
2695
2696	spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
2697	mlx5_ib_lock_cqs(send_cq, recv_cq);
2698	/* del from lists under both locks above to protect reset flow paths */
2699	list_del(&qp->qps_list);
2700	if (send_cq)
2701		list_del(&qp->cq_send_list);
2702
2703	if (recv_cq)
2704		list_del(&qp->cq_recv_list);
2705
2706	if (!udata) {
2707		__mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
2708				   qp->ibqp.srq ? to_msrq(qp->ibqp.srq) : NULL);
2709		if (send_cq != recv_cq)
2710			__mlx5_ib_cq_clean(send_cq, base->mqp.qpn,
2711					   NULL);
2712	}
2713	mlx5_ib_unlock_cqs(send_cq, recv_cq);
2714	spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
2715
2716	if (qp->type == IB_QPT_RAW_PACKET ||
2717	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
2718		destroy_raw_packet_qp(dev, qp);
2719	} else {
2720		err = mlx5_core_destroy_qp(dev, &base->mqp);
2721		if (err)
2722			mlx5_ib_warn(dev, "failed to destroy QP 0x%x\n",
2723				     base->mqp.qpn);
2724	}
2725
2726	destroy_qp(dev, qp, base, udata);
2727}
2728
2729static int create_dct(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2730		      struct mlx5_ib_qp *qp,
2731		      struct mlx5_create_qp_params *params)
2732{
2733	struct ib_qp_init_attr *attr = params->attr;
2734	struct mlx5_ib_create_qp *ucmd = params->ucmd;
2735	u32 uidx = params->uidx;
2736	void *dctc;
2737
2738	if (mlx5_lag_is_active(dev->mdev) && !MLX5_CAP_GEN(dev->mdev, lag_dct))
2739		return -EOPNOTSUPP;
2740
2741	qp->dct.in = kzalloc(MLX5_ST_SZ_BYTES(create_dct_in), GFP_KERNEL);
2742	if (!qp->dct.in)
2743		return -ENOMEM;
2744
2745	MLX5_SET(create_dct_in, qp->dct.in, uid, to_mpd(pd)->uid);
2746	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
2747	MLX5_SET(dctc, dctc, pd, to_mpd(pd)->pdn);
2748	MLX5_SET(dctc, dctc, srqn_xrqn, to_msrq(attr->srq)->msrq.srqn);
2749	MLX5_SET(dctc, dctc, cqn, to_mcq(attr->recv_cq)->mcq.cqn);
2750	MLX5_SET64(dctc, dctc, dc_access_key, ucmd->access_key);
2751	MLX5_SET(dctc, dctc, user_index, uidx);
2752	if (MLX5_CAP_GEN(dev->mdev, ece_support))
2753		MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
2754
2755	if (qp->flags_en & MLX5_QP_FLAG_SCATTER_CQE) {
2756		int rcqe_sz = mlx5_ib_get_cqe_size(attr->recv_cq);
2757
2758		if (rcqe_sz == 128)
2759			MLX5_SET(dctc, dctc, cs_res, MLX5_RES_SCAT_DATA64_CQE);
2760	}
2761
2762	qp->state = IB_QPS_RESET;
2763	return 0;
2764}
2765
2766static int check_qp_type(struct mlx5_ib_dev *dev, struct ib_qp_init_attr *attr,
2767			 enum ib_qp_type *type)
2768{
2769	if (attr->qp_type == IB_QPT_DRIVER && !MLX5_CAP_GEN(dev->mdev, dct))
2770		goto out;
2771
2772	switch (attr->qp_type) {
2773	case IB_QPT_XRC_TGT:
2774	case IB_QPT_XRC_INI:
2775		if (!MLX5_CAP_GEN(dev->mdev, xrc))
2776			goto out;
2777		fallthrough;
2778	case IB_QPT_RC:
2779	case IB_QPT_UC:
2780	case IB_QPT_SMI:
2781	case MLX5_IB_QPT_HW_GSI:
2782	case IB_QPT_DRIVER:
2783	case IB_QPT_GSI:
2784	case IB_QPT_RAW_PACKET:
2785	case IB_QPT_UD:
2786	case MLX5_IB_QPT_REG_UMR:
2787		break;
2788	default:
2789		goto out;
2790	}
2791
2792	*type = attr->qp_type;
2793	return 0;
2794
2795out:
2796	mlx5_ib_dbg(dev, "Unsupported QP type %d\n", attr->qp_type);
2797	return -EOPNOTSUPP;
2798}
2799
2800static int check_valid_flow(struct mlx5_ib_dev *dev, struct ib_pd *pd,
2801			    struct ib_qp_init_attr *attr,
2802			    struct ib_udata *udata)
2803{
2804	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
2805		udata, struct mlx5_ib_ucontext, ibucontext);
2806
2807	if (!udata) {
2808		/* Kernel create_qp callers */
2809		if (attr->rwq_ind_tbl)
2810			return -EOPNOTSUPP;
2811
2812		switch (attr->qp_type) {
2813		case IB_QPT_RAW_PACKET:
2814		case IB_QPT_DRIVER:
2815			return -EOPNOTSUPP;
2816		default:
2817			return 0;
2818		}
2819	}
2820
2821	/* Userspace create_qp callers */
2822	if (attr->qp_type == IB_QPT_RAW_PACKET && !ucontext->cqe_version) {
2823		mlx5_ib_dbg(dev,
2824			"Raw Packet QP is only supported for CQE version > 0\n");
2825		return -EINVAL;
2826	}
2827
2828	if (attr->qp_type != IB_QPT_RAW_PACKET && attr->rwq_ind_tbl) {
2829		mlx5_ib_dbg(dev,
2830			    "Wrong QP type %d for the RWQ indirect table\n",
2831			    attr->qp_type);
2832		return -EINVAL;
2833	}
2834
2835	/*
2836	 * We don't need to see this warning, it means that kernel code
2837	 * missing ib_pd. Placed here to catch developer's mistakes.
2838	 */
2839	WARN_ONCE(!pd && attr->qp_type != IB_QPT_XRC_TGT,
2840		  "There is a missing PD pointer assignment\n");
2841	return 0;
2842}
2843
2844static void process_vendor_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2845				bool cond, struct mlx5_ib_qp *qp)
2846{
2847	if (!(*flags & flag))
2848		return;
2849
2850	if (cond) {
2851		qp->flags_en |= flag;
2852		*flags &= ~flag;
2853		return;
2854	}
2855
2856	switch (flag) {
2857	case MLX5_QP_FLAG_SCATTER_CQE:
2858	case MLX5_QP_FLAG_ALLOW_SCATTER_CQE:
2859		/*
2860		 * We don't return error if these flags were provided,
2861		 * and mlx5 doesn't have right capability.
2862		 */
2863		*flags &= ~(MLX5_QP_FLAG_SCATTER_CQE |
2864			    MLX5_QP_FLAG_ALLOW_SCATTER_CQE);
2865		return;
2866	default:
2867		break;
2868	}
2869	mlx5_ib_dbg(dev, "Vendor create QP flag 0x%X is not supported\n", flag);
2870}
2871
2872static int process_vendor_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2873				void *ucmd, struct ib_qp_init_attr *attr)
2874{
2875	struct mlx5_core_dev *mdev = dev->mdev;
2876	bool cond;
2877	int flags;
2878
2879	if (attr->rwq_ind_tbl)
2880		flags = ((struct mlx5_ib_create_qp_rss *)ucmd)->flags;
2881	else
2882		flags = ((struct mlx5_ib_create_qp *)ucmd)->flags;
2883
2884	switch (flags & (MLX5_QP_FLAG_TYPE_DCT | MLX5_QP_FLAG_TYPE_DCI)) {
2885	case MLX5_QP_FLAG_TYPE_DCI:
2886		qp->type = MLX5_IB_QPT_DCI;
2887		break;
2888	case MLX5_QP_FLAG_TYPE_DCT:
2889		qp->type = MLX5_IB_QPT_DCT;
2890		break;
2891	default:
2892		if (qp->type != IB_QPT_DRIVER)
2893			break;
2894		/*
2895		 * It is IB_QPT_DRIVER and or no subtype or
2896		 * wrong subtype were provided.
2897		 */
2898		return -EINVAL;
2899	}
2900
2901	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCI, true, qp);
2902	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TYPE_DCT, true, qp);
2903	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_DCI_STREAM,
2904			    MLX5_CAP_GEN(mdev, log_max_dci_stream_channels),
2905			    qp);
2906
2907	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SIGNATURE, true, qp);
2908	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_SCATTER_CQE,
2909			    MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
2910	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_ALLOW_SCATTER_CQE,
2911			    MLX5_CAP_GEN(mdev, sctr_data_cqe), qp);
2912
2913	if (qp->type == IB_QPT_RAW_PACKET) {
2914		cond = MLX5_CAP_ETH(mdev, tunnel_stateless_vxlan) ||
2915		       MLX5_CAP_ETH(mdev, tunnel_stateless_gre) ||
2916		       MLX5_CAP_ETH(mdev, tunnel_stateless_geneve_rx);
2917		process_vendor_flag(dev, &flags, MLX5_QP_FLAG_TUNNEL_OFFLOADS,
2918				    cond, qp);
2919		process_vendor_flag(dev, &flags,
2920				    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC, true,
2921				    qp);
2922		process_vendor_flag(dev, &flags,
2923				    MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC, true,
2924				    qp);
2925	}
2926
2927	if (qp->type == IB_QPT_RC)
2928		process_vendor_flag(dev, &flags,
2929				    MLX5_QP_FLAG_PACKET_BASED_CREDIT_MODE,
2930				    MLX5_CAP_GEN(mdev, qp_packet_based), qp);
2931
2932	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_BFREG_INDEX, true, qp);
2933	process_vendor_flag(dev, &flags, MLX5_QP_FLAG_UAR_PAGE_INDEX, true, qp);
2934
2935	cond = qp->flags_en & ~(MLX5_QP_FLAG_TUNNEL_OFFLOADS |
2936				MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_UC |
2937				MLX5_QP_FLAG_TIR_ALLOW_SELF_LB_MC);
2938	if (attr->rwq_ind_tbl && cond) {
2939		mlx5_ib_dbg(dev, "RSS RAW QP has unsupported flags 0x%X\n",
2940			    cond);
2941		return -EINVAL;
2942	}
2943
2944	if (flags)
2945		mlx5_ib_dbg(dev, "udata has unsupported flags 0x%X\n", flags);
2946
2947	return (flags) ? -EINVAL : 0;
2948	}
2949
2950static void process_create_flag(struct mlx5_ib_dev *dev, int *flags, int flag,
2951				bool cond, struct mlx5_ib_qp *qp)
2952{
2953	if (!(*flags & flag))
2954		return;
2955
2956	if (cond) {
2957		qp->flags |= flag;
2958		*flags &= ~flag;
2959		return;
2960	}
2961
2962	if (flag == MLX5_IB_QP_CREATE_WC_TEST) {
2963		/*
2964		 * Special case, if condition didn't meet, it won't be error,
2965		 * just different in-kernel flow.
2966		 */
2967		*flags &= ~MLX5_IB_QP_CREATE_WC_TEST;
2968		return;
2969	}
2970	mlx5_ib_dbg(dev, "Verbs create QP flag 0x%X is not supported\n", flag);
2971}
2972
2973static int process_create_flags(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
2974				struct ib_qp_init_attr *attr)
2975{
2976	enum ib_qp_type qp_type = qp->type;
2977	struct mlx5_core_dev *mdev = dev->mdev;
2978	int create_flags = attr->create_flags;
2979	bool cond;
2980
2981	if (qp_type == MLX5_IB_QPT_DCT)
2982		return (create_flags) ? -EINVAL : 0;
2983
2984	if (qp_type == IB_QPT_RAW_PACKET && attr->rwq_ind_tbl)
2985		return (create_flags) ? -EINVAL : 0;
2986
2987	process_create_flag(dev, &create_flags, IB_QP_CREATE_NETIF_QP,
2988			    mlx5_get_flow_namespace(dev->mdev,
2989						    MLX5_FLOW_NAMESPACE_BYPASS),
2990			    qp);
2991	process_create_flag(dev, &create_flags,
2992			    IB_QP_CREATE_INTEGRITY_EN,
2993			    MLX5_CAP_GEN(mdev, sho), qp);
2994	process_create_flag(dev, &create_flags,
2995			    IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK,
2996			    MLX5_CAP_GEN(mdev, block_lb_mc), qp);
2997	process_create_flag(dev, &create_flags, IB_QP_CREATE_CROSS_CHANNEL,
2998			    MLX5_CAP_GEN(mdev, cd), qp);
2999	process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_SEND,
3000			    MLX5_CAP_GEN(mdev, cd), qp);
3001	process_create_flag(dev, &create_flags, IB_QP_CREATE_MANAGED_RECV,
3002			    MLX5_CAP_GEN(mdev, cd), qp);
3003
3004	if (qp_type == IB_QPT_UD) {
3005		process_create_flag(dev, &create_flags,
3006				    IB_QP_CREATE_IPOIB_UD_LSO,
3007				    MLX5_CAP_GEN(mdev, ipoib_basic_offloads),
3008				    qp);
3009		cond = MLX5_CAP_GEN(mdev, port_type) == MLX5_CAP_PORT_TYPE_IB;
3010		process_create_flag(dev, &create_flags, IB_QP_CREATE_SOURCE_QPN,
3011				    cond, qp);
3012	}
3013
3014	if (qp_type == IB_QPT_RAW_PACKET) {
3015		cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
3016		       MLX5_CAP_ETH(mdev, scatter_fcs);
3017		process_create_flag(dev, &create_flags,
3018				    IB_QP_CREATE_SCATTER_FCS, cond, qp);
3019
3020		cond = MLX5_CAP_GEN(mdev, eth_net_offloads) &&
3021		       MLX5_CAP_ETH(mdev, vlan_cap);
3022		process_create_flag(dev, &create_flags,
3023				    IB_QP_CREATE_CVLAN_STRIPPING, cond, qp);
3024	}
3025
3026	process_create_flag(dev, &create_flags,
3027			    IB_QP_CREATE_PCI_WRITE_END_PADDING,
3028			    MLX5_CAP_GEN(mdev, end_pad), qp);
3029
3030	process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_WC_TEST,
3031			    qp_type != MLX5_IB_QPT_REG_UMR, qp);
3032	process_create_flag(dev, &create_flags, MLX5_IB_QP_CREATE_SQPN_QP1,
3033			    true, qp);
3034
3035	if (create_flags) {
3036		mlx5_ib_dbg(dev, "Create QP has unsupported flags 0x%X\n",
3037			    create_flags);
3038		return -EOPNOTSUPP;
3039	}
3040	return 0;
3041}
3042
3043static int process_udata_size(struct mlx5_ib_dev *dev,
3044			      struct mlx5_create_qp_params *params)
3045{
3046	size_t ucmd = sizeof(struct mlx5_ib_create_qp);
3047	struct ib_udata *udata = params->udata;
3048	size_t outlen = udata->outlen;
3049	size_t inlen = udata->inlen;
3050
3051	params->outlen = min(outlen, sizeof(struct mlx5_ib_create_qp_resp));
3052	params->ucmd_size = ucmd;
3053	if (!params->is_rss_raw) {
3054		/* User has old rdma-core, which doesn't support ECE */
3055		size_t min_inlen =
3056			offsetof(struct mlx5_ib_create_qp, ece_options);
3057
3058		/*
3059		 * We will check in check_ucmd_data() that user
3060		 * cleared everything after inlen.
3061		 */
3062		params->inlen = (inlen < min_inlen) ? 0 : min(inlen, ucmd);
3063		goto out;
3064	}
3065
3066	/* RSS RAW QP */
3067	if (inlen < offsetofend(struct mlx5_ib_create_qp_rss, flags))
3068		return -EINVAL;
3069
3070	if (outlen < offsetofend(struct mlx5_ib_create_qp_resp, bfreg_index))
3071		return -EINVAL;
3072
3073	ucmd = sizeof(struct mlx5_ib_create_qp_rss);
3074	params->ucmd_size = ucmd;
3075	if (inlen > ucmd && !ib_is_udata_cleared(udata, ucmd, inlen - ucmd))
3076		return -EINVAL;
3077
3078	params->inlen = min(ucmd, inlen);
3079out:
3080	if (!params->inlen)
3081		mlx5_ib_dbg(dev, "udata is too small\n");
3082
3083	return (params->inlen) ? 0 : -EINVAL;
3084}
3085
3086static int create_qp(struct mlx5_ib_dev *dev, struct ib_pd *pd,
3087		     struct mlx5_ib_qp *qp,
3088		     struct mlx5_create_qp_params *params)
3089{
3090	int err;
3091
3092	if (params->is_rss_raw) {
3093		err = create_rss_raw_qp_tir(dev, pd, qp, params);
3094		goto out;
3095	}
3096
3097	switch (qp->type) {
3098	case MLX5_IB_QPT_DCT:
3099		err = create_dct(dev, pd, qp, params);
3100		rdma_restrack_no_track(&qp->ibqp.res);
3101		break;
3102	case MLX5_IB_QPT_DCI:
3103		err = create_dci(dev, pd, qp, params);
3104		break;
3105	case IB_QPT_XRC_TGT:
3106		err = create_xrc_tgt_qp(dev, qp, params);
3107		break;
3108	case IB_QPT_GSI:
3109		err = mlx5_ib_create_gsi(pd, qp, params->attr);
3110		break;
3111	case MLX5_IB_QPT_HW_GSI:
3112	case MLX5_IB_QPT_REG_UMR:
3113		rdma_restrack_no_track(&qp->ibqp.res);
3114		fallthrough;
3115	default:
3116		if (params->udata)
3117			err = create_user_qp(dev, pd, qp, params);
3118		else
3119			err = create_kernel_qp(dev, pd, qp, params);
3120	}
3121
3122out:
3123	if (err) {
3124		mlx5_ib_err(dev, "Create QP type %d failed\n", qp->type);
3125		return err;
3126	}
3127
3128	if (is_qp0(qp->type))
3129		qp->ibqp.qp_num = 0;
3130	else if (is_qp1(qp->type))
3131		qp->ibqp.qp_num = 1;
3132	else
3133		qp->ibqp.qp_num = qp->trans_qp.base.mqp.qpn;
3134
3135	mlx5_ib_dbg(dev,
3136		"QP type %d, ib qpn 0x%X, mlx qpn 0x%x, rcqn 0x%x, scqn 0x%x, ece 0x%x\n",
3137		qp->type, qp->ibqp.qp_num, qp->trans_qp.base.mqp.qpn,
3138		params->attr->recv_cq ? to_mcq(params->attr->recv_cq)->mcq.cqn :
3139					-1,
3140		params->attr->send_cq ? to_mcq(params->attr->send_cq)->mcq.cqn :
3141					-1,
3142		params->resp.ece_options);
3143
3144	return 0;
3145}
3146
3147static int check_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3148			 struct ib_qp_init_attr *attr)
3149{
3150	int ret = 0;
3151
3152	switch (qp->type) {
3153	case MLX5_IB_QPT_DCT:
3154		ret = (!attr->srq || !attr->recv_cq) ? -EINVAL : 0;
3155		break;
3156	case MLX5_IB_QPT_DCI:
3157		ret = (attr->cap.max_recv_wr || attr->cap.max_recv_sge) ?
3158			      -EINVAL :
3159			      0;
3160		break;
3161	case IB_QPT_RAW_PACKET:
3162		ret = (attr->rwq_ind_tbl && attr->send_cq) ? -EINVAL : 0;
3163		break;
3164	default:
3165		break;
3166	}
3167
3168	if (ret)
3169		mlx5_ib_dbg(dev, "QP type %d has wrong attributes\n", qp->type);
3170
3171	return ret;
3172}
3173
3174static int get_qp_uidx(struct mlx5_ib_qp *qp,
3175		       struct mlx5_create_qp_params *params)
3176{
3177	struct mlx5_ib_create_qp *ucmd = params->ucmd;
3178	struct ib_udata *udata = params->udata;
3179	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
3180		udata, struct mlx5_ib_ucontext, ibucontext);
3181
3182	if (params->is_rss_raw)
3183		return 0;
3184
3185	return get_qp_user_index(ucontext, ucmd, sizeof(*ucmd), &params->uidx);
3186}
3187
3188static int mlx5_ib_destroy_dct(struct mlx5_ib_qp *mqp)
3189{
3190	struct mlx5_ib_dev *dev = to_mdev(mqp->ibqp.device);
3191
3192	if (mqp->state == IB_QPS_RTR) {
3193		int err;
3194
3195		err = mlx5_core_destroy_dct(dev, &mqp->dct.mdct);
3196		if (err) {
3197			mlx5_ib_warn(dev, "failed to destroy DCT %d\n", err);
3198			return err;
3199		}
3200	}
3201
3202	kfree(mqp->dct.in);
3203	return 0;
3204}
3205
3206static int check_ucmd_data(struct mlx5_ib_dev *dev,
3207			   struct mlx5_create_qp_params *params)
3208{
3209	struct ib_udata *udata = params->udata;
3210	size_t size, last;
3211	int ret;
3212
3213	if (params->is_rss_raw)
3214		/*
3215		 * These QPs don't have "reserved" field in their
3216		 * create_qp input struct, so their data is always valid.
3217		 */
3218		last = sizeof(struct mlx5_ib_create_qp_rss);
3219	else
3220		last = offsetof(struct mlx5_ib_create_qp, reserved);
3221
3222	if (udata->inlen <= last)
3223		return 0;
3224
3225	/*
3226	 * User provides different create_qp structures based on the
3227	 * flow and we need to know if he cleared memory after our
3228	 * struct create_qp ends.
3229	 */
3230	size = udata->inlen - last;
3231	ret = ib_is_udata_cleared(params->udata, last, size);
3232	if (!ret)
3233		mlx5_ib_dbg(
3234			dev,
3235			"udata is not cleared, inlen = %zu, ucmd = %zu, last = %zu, size = %zu\n",
3236			udata->inlen, params->ucmd_size, last, size);
3237	return ret ? 0 : -EINVAL;
3238}
3239
3240int mlx5_ib_create_qp(struct ib_qp *ibqp, struct ib_qp_init_attr *attr,
3241		      struct ib_udata *udata)
3242{
3243	struct mlx5_create_qp_params params = {};
3244	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
3245	struct mlx5_ib_qp *qp = to_mqp(ibqp);
3246	struct ib_pd *pd = ibqp->pd;
3247	enum ib_qp_type type;
3248	int err;
3249
3250	err = check_qp_type(dev, attr, &type);
3251	if (err)
3252		return err;
3253
3254	err = check_valid_flow(dev, pd, attr, udata);
3255	if (err)
3256		return err;
3257
3258	params.udata = udata;
3259	params.uidx = MLX5_IB_DEFAULT_UIDX;
3260	params.attr = attr;
3261	params.is_rss_raw = !!attr->rwq_ind_tbl;
3262
3263	if (udata) {
3264		err = process_udata_size(dev, &params);
3265		if (err)
3266			return err;
3267
3268		err = check_ucmd_data(dev, &params);
3269		if (err)
3270			return err;
3271
3272		params.ucmd = kzalloc(params.ucmd_size, GFP_KERNEL);
3273		if (!params.ucmd)
3274			return -ENOMEM;
3275
3276		err = ib_copy_from_udata(params.ucmd, udata, params.inlen);
3277		if (err)
3278			goto free_ucmd;
3279	}
3280
3281	mutex_init(&qp->mutex);
3282	qp->type = type;
3283	if (udata) {
3284		err = process_vendor_flags(dev, qp, params.ucmd, attr);
3285		if (err)
3286			goto free_ucmd;
3287
3288		err = get_qp_uidx(qp, &params);
3289		if (err)
3290			goto free_ucmd;
3291	}
3292	err = process_create_flags(dev, qp, attr);
3293	if (err)
3294		goto free_ucmd;
3295
3296	err = check_qp_attr(dev, qp, attr);
3297	if (err)
3298		goto free_ucmd;
3299
3300	err = create_qp(dev, pd, qp, &params);
3301	if (err)
3302		goto free_ucmd;
3303
3304	kfree(params.ucmd);
3305	params.ucmd = NULL;
3306
3307	if (udata)
3308		/*
3309		 * It is safe to copy response for all user create QP flows,
3310		 * including MLX5_IB_QPT_DCT, which doesn't need it.
3311		 * In that case, resp will be filled with zeros.
3312		 */
3313		err = ib_copy_to_udata(udata, &params.resp, params.outlen);
3314	if (err)
3315		goto destroy_qp;
3316
3317	return 0;
3318
3319destroy_qp:
3320	switch (qp->type) {
3321	case MLX5_IB_QPT_DCT:
3322		mlx5_ib_destroy_dct(qp);
3323		break;
3324	case IB_QPT_GSI:
3325		mlx5_ib_destroy_gsi(qp);
3326		break;
3327	default:
3328		destroy_qp_common(dev, qp, udata);
3329	}
3330
3331free_ucmd:
3332	kfree(params.ucmd);
3333	return err;
3334}
3335
3336int mlx5_ib_destroy_qp(struct ib_qp *qp, struct ib_udata *udata)
3337{
3338	struct mlx5_ib_dev *dev = to_mdev(qp->device);
3339	struct mlx5_ib_qp *mqp = to_mqp(qp);
3340
3341	if (mqp->type == IB_QPT_GSI)
3342		return mlx5_ib_destroy_gsi(mqp);
3343
3344	if (mqp->type == MLX5_IB_QPT_DCT)
3345		return mlx5_ib_destroy_dct(mqp);
3346
3347	destroy_qp_common(dev, mqp, udata);
3348	return 0;
3349}
3350
3351static int set_qpc_atomic_flags(struct mlx5_ib_qp *qp,
3352				const struct ib_qp_attr *attr, int attr_mask,
3353				void *qpc)
3354{
3355	struct mlx5_ib_dev *dev = to_mdev(qp->ibqp.device);
3356	u8 dest_rd_atomic;
3357	u32 access_flags;
3358
3359	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
3360		dest_rd_atomic = attr->max_dest_rd_atomic;
3361	else
3362		dest_rd_atomic = qp->trans_qp.resp_depth;
3363
3364	if (attr_mask & IB_QP_ACCESS_FLAGS)
3365		access_flags = attr->qp_access_flags;
3366	else
3367		access_flags = qp->trans_qp.atomic_rd_en;
3368
3369	if (!dest_rd_atomic)
3370		access_flags &= IB_ACCESS_REMOTE_WRITE;
3371
3372	MLX5_SET(qpc, qpc, rre, !!(access_flags & IB_ACCESS_REMOTE_READ));
3373
3374	if (access_flags & IB_ACCESS_REMOTE_ATOMIC) {
3375		int atomic_mode;
3376
3377		atomic_mode = get_atomic_mode(dev, qp->type);
3378		if (atomic_mode < 0)
3379			return -EOPNOTSUPP;
3380
3381		MLX5_SET(qpc, qpc, rae, 1);
3382		MLX5_SET(qpc, qpc, atomic_mode, atomic_mode);
3383	}
3384
3385	MLX5_SET(qpc, qpc, rwe, !!(access_flags & IB_ACCESS_REMOTE_WRITE));
3386	return 0;
3387}
3388
3389enum {
3390	MLX5_PATH_FLAG_FL	= 1 << 0,
3391	MLX5_PATH_FLAG_FREE_AR	= 1 << 1,
3392	MLX5_PATH_FLAG_COUNTER	= 1 << 2,
3393};
3394
3395static int mlx5_to_ib_rate_map(u8 rate)
3396{
3397	static const int rates[] = { IB_RATE_PORT_CURRENT, IB_RATE_56_GBPS,
3398				     IB_RATE_25_GBPS,	   IB_RATE_100_GBPS,
3399				     IB_RATE_200_GBPS,	   IB_RATE_50_GBPS,
3400				     IB_RATE_400_GBPS };
3401
3402	if (rate < ARRAY_SIZE(rates))
3403		return rates[rate];
3404
3405	return rate - MLX5_STAT_RATE_OFFSET;
3406}
3407
3408static int ib_to_mlx5_rate_map(u8 rate)
3409{
3410	switch (rate) {
3411	case IB_RATE_PORT_CURRENT:
3412		return 0;
3413	case IB_RATE_56_GBPS:
3414		return 1;
3415	case IB_RATE_25_GBPS:
3416		return 2;
3417	case IB_RATE_100_GBPS:
3418		return 3;
3419	case IB_RATE_200_GBPS:
3420		return 4;
3421	case IB_RATE_50_GBPS:
3422		return 5;
3423	case IB_RATE_400_GBPS:
3424		return 6;
3425	default:
3426		return rate + MLX5_STAT_RATE_OFFSET;
3427	}
3428
3429	return 0;
3430}
3431
3432static int ib_rate_to_mlx5(struct mlx5_ib_dev *dev, u8 rate)
3433{
3434	u32 stat_rate_support;
3435
3436	if (rate == IB_RATE_PORT_CURRENT)
3437		return 0;
3438
3439	if (rate < IB_RATE_2_5_GBPS || rate > IB_RATE_800_GBPS)
3440		return -EINVAL;
3441
3442	stat_rate_support = MLX5_CAP_GEN(dev->mdev, stat_rate_support);
3443	while (rate != IB_RATE_PORT_CURRENT &&
3444	       !(1 << ib_to_mlx5_rate_map(rate) & stat_rate_support))
3445		--rate;
3446
3447	return ib_to_mlx5_rate_map(rate);
3448}
3449
3450static int modify_raw_packet_eth_prio(struct mlx5_core_dev *dev,
3451				      struct mlx5_ib_sq *sq, u8 sl,
3452				      struct ib_pd *pd)
3453{
3454	void *in;
3455	void *tisc;
3456	int inlen;
3457	int err;
3458
3459	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
3460	in = kvzalloc(inlen, GFP_KERNEL);
3461	if (!in)
3462		return -ENOMEM;
3463
3464	MLX5_SET(modify_tis_in, in, bitmask.prio, 1);
3465	MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
3466
3467	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3468	MLX5_SET(tisc, tisc, prio, ((sl & 0x7) << 1));
3469
3470	err = mlx5_core_modify_tis(dev, sq->tisn, in);
3471
3472	kvfree(in);
3473
3474	return err;
3475}
3476
3477static int modify_raw_packet_tx_affinity(struct mlx5_core_dev *dev,
3478					 struct mlx5_ib_sq *sq, u8 tx_affinity,
3479					 struct ib_pd *pd)
3480{
3481	void *in;
3482	void *tisc;
3483	int inlen;
3484	int err;
3485
3486	inlen = MLX5_ST_SZ_BYTES(modify_tis_in);
3487	in = kvzalloc(inlen, GFP_KERNEL);
3488	if (!in)
3489		return -ENOMEM;
3490
3491	MLX5_SET(modify_tis_in, in, bitmask.lag_tx_port_affinity, 1);
3492	MLX5_SET(modify_tis_in, in, uid, to_mpd(pd)->uid);
3493
3494	tisc = MLX5_ADDR_OF(modify_tis_in, in, ctx);
3495	MLX5_SET(tisc, tisc, lag_tx_port_affinity, tx_affinity);
3496
3497	err = mlx5_core_modify_tis(dev, sq->tisn, in);
3498
3499	kvfree(in);
3500
3501	return err;
3502}
3503
3504static void mlx5_set_path_udp_sport(void *path, const struct rdma_ah_attr *ah,
3505				    u32 lqpn, u32 rqpn)
3506
3507{
3508	u32 fl = ah->grh.flow_label;
3509
3510	if (!fl)
3511		fl = rdma_calc_flow_label(lqpn, rqpn);
3512
3513	MLX5_SET(ads, path, udp_sport, rdma_flow_label_to_udp_sport(fl));
3514}
3515
3516static int mlx5_set_path(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3517			 const struct rdma_ah_attr *ah, void *path, u8 port,
3518			 int attr_mask, u32 path_flags,
3519			 const struct ib_qp_attr *attr, bool alt)
3520{
3521	const struct ib_global_route *grh = rdma_ah_read_grh(ah);
3522	int err;
3523	enum ib_gid_type gid_type;
3524	u8 ah_flags = rdma_ah_get_ah_flags(ah);
3525	u8 sl = rdma_ah_get_sl(ah);
3526
3527	if (attr_mask & IB_QP_PKEY_INDEX)
3528		MLX5_SET(ads, path, pkey_index,
3529			 alt ? attr->alt_pkey_index : attr->pkey_index);
3530
3531	if (ah_flags & IB_AH_GRH) {
3532		const struct ib_port_immutable *immutable;
3533
3534		immutable = ib_port_immutable_read(&dev->ib_dev, port);
3535		if (grh->sgid_index >= immutable->gid_tbl_len) {
3536			pr_err("sgid_index (%u) too large. max is %d\n",
3537			       grh->sgid_index,
3538			       immutable->gid_tbl_len);
3539			return -EINVAL;
3540		}
3541	}
3542
3543	if (ah->type == RDMA_AH_ATTR_TYPE_ROCE) {
3544		if (!(ah_flags & IB_AH_GRH))
3545			return -EINVAL;
3546
3547		ether_addr_copy(MLX5_ADDR_OF(ads, path, rmac_47_32),
3548				ah->roce.dmac);
3549		if ((qp->type == IB_QPT_RC ||
3550		     qp->type == IB_QPT_UC ||
3551		     qp->type == IB_QPT_XRC_INI ||
3552		     qp->type == IB_QPT_XRC_TGT) &&
3553		    (grh->sgid_attr->gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP) &&
3554		    (attr_mask & IB_QP_DEST_QPN))
3555			mlx5_set_path_udp_sport(path, ah,
3556						qp->ibqp.qp_num,
3557						attr->dest_qp_num);
3558		MLX5_SET(ads, path, eth_prio, sl & 0x7);
3559		gid_type = ah->grh.sgid_attr->gid_type;
3560		if (gid_type == IB_GID_TYPE_ROCE_UDP_ENCAP)
3561			MLX5_SET(ads, path, dscp, grh->traffic_class >> 2);
3562	} else {
3563		MLX5_SET(ads, path, fl, !!(path_flags & MLX5_PATH_FLAG_FL));
3564		MLX5_SET(ads, path, free_ar,
3565			 !!(path_flags & MLX5_PATH_FLAG_FREE_AR));
3566		MLX5_SET(ads, path, rlid, rdma_ah_get_dlid(ah));
3567		MLX5_SET(ads, path, mlid, rdma_ah_get_path_bits(ah));
3568		MLX5_SET(ads, path, grh, !!(ah_flags & IB_AH_GRH));
3569		MLX5_SET(ads, path, sl, sl);
3570	}
3571
3572	if (ah_flags & IB_AH_GRH) {
3573		MLX5_SET(ads, path, src_addr_index, grh->sgid_index);
3574		MLX5_SET(ads, path, hop_limit, grh->hop_limit);
3575		MLX5_SET(ads, path, tclass, grh->traffic_class);
3576		MLX5_SET(ads, path, flow_label, grh->flow_label);
3577		memcpy(MLX5_ADDR_OF(ads, path, rgid_rip), grh->dgid.raw,
3578		       sizeof(grh->dgid.raw));
3579	}
3580
3581	err = ib_rate_to_mlx5(dev, rdma_ah_get_static_rate(ah));
3582	if (err < 0)
3583		return err;
3584	MLX5_SET(ads, path, stat_rate, err);
3585	MLX5_SET(ads, path, vhca_port_num, port);
3586
3587	if (attr_mask & IB_QP_TIMEOUT)
3588		MLX5_SET(ads, path, ack_timeout,
3589			 alt ? attr->alt_timeout : attr->timeout);
3590
3591	if ((qp->type == IB_QPT_RAW_PACKET) && qp->sq.wqe_cnt)
3592		return modify_raw_packet_eth_prio(dev->mdev,
3593						  &qp->raw_packet_qp.sq,
3594						  sl & 0xf, qp->ibqp.pd);
3595
3596	return 0;
3597}
3598
3599static enum mlx5_qp_optpar opt_mask[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE][MLX5_QP_ST_MAX] = {
3600	[MLX5_QP_STATE_INIT] = {
3601		[MLX5_QP_STATE_INIT] = {
3602			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE		|
3603					  MLX5_QP_OPTPAR_RAE		|
3604					  MLX5_QP_OPTPAR_RWE		|
3605					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3606					  MLX5_QP_OPTPAR_PRI_PORT	|
3607					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3608			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE		|
3609					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3610					  MLX5_QP_OPTPAR_PRI_PORT	|
3611					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3612			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX	|
3613					  MLX5_QP_OPTPAR_Q_KEY		|
3614					  MLX5_QP_OPTPAR_PRI_PORT,
3615			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE		|
3616					  MLX5_QP_OPTPAR_RAE		|
3617					  MLX5_QP_OPTPAR_RWE		|
3618					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3619					  MLX5_QP_OPTPAR_PRI_PORT	|
3620					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3621		},
3622		[MLX5_QP_STATE_RTR] = {
3623			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
3624					  MLX5_QP_OPTPAR_RRE            |
3625					  MLX5_QP_OPTPAR_RAE            |
3626					  MLX5_QP_OPTPAR_RWE            |
3627					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3628					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3629			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH  |
3630					  MLX5_QP_OPTPAR_RWE            |
3631					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3632					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3633			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_PKEY_INDEX     |
3634					  MLX5_QP_OPTPAR_Q_KEY,
3635			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_PKEY_INDEX	|
3636					   MLX5_QP_OPTPAR_Q_KEY,
3637			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH |
3638					  MLX5_QP_OPTPAR_RRE            |
3639					  MLX5_QP_OPTPAR_RAE            |
3640					  MLX5_QP_OPTPAR_RWE            |
3641					  MLX5_QP_OPTPAR_PKEY_INDEX	|
3642					  MLX5_QP_OPTPAR_LAG_TX_AFF,
3643		},
3644	},
3645	[MLX5_QP_STATE_RTR] = {
3646		[MLX5_QP_STATE_RTS] = {
3647			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH	|
3648					  MLX5_QP_OPTPAR_RRE		|
3649					  MLX5_QP_OPTPAR_RAE		|
3650					  MLX5_QP_OPTPAR_RWE		|
3651					  MLX5_QP_OPTPAR_PM_STATE	|
3652					  MLX5_QP_OPTPAR_RNR_TIMEOUT,
3653			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH	|
3654					  MLX5_QP_OPTPAR_RWE		|
3655					  MLX5_QP_OPTPAR_PM_STATE,
3656			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3657			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_ALT_ADDR_PATH	|
3658					  MLX5_QP_OPTPAR_RRE		|
3659					  MLX5_QP_OPTPAR_RAE		|
3660					  MLX5_QP_OPTPAR_RWE		|
3661					  MLX5_QP_OPTPAR_PM_STATE	|
3662					  MLX5_QP_OPTPAR_RNR_TIMEOUT,
3663		},
3664	},
3665	[MLX5_QP_STATE_RTS] = {
3666		[MLX5_QP_STATE_RTS] = {
3667			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RRE		|
3668					  MLX5_QP_OPTPAR_RAE		|
3669					  MLX5_QP_OPTPAR_RWE		|
3670					  MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3671					  MLX5_QP_OPTPAR_PM_STATE	|
3672					  MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3673			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE		|
3674					  MLX5_QP_OPTPAR_PM_STATE	|
3675					  MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3676			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY		|
3677					  MLX5_QP_OPTPAR_SRQN		|
3678					  MLX5_QP_OPTPAR_CQN_RCV,
3679			[MLX5_QP_ST_XRC] = MLX5_QP_OPTPAR_RRE		|
3680					  MLX5_QP_OPTPAR_RAE		|
3681					  MLX5_QP_OPTPAR_RWE		|
3682					  MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3683					  MLX5_QP_OPTPAR_PM_STATE	|
3684					  MLX5_QP_OPTPAR_ALT_ADDR_PATH,
3685		},
3686	},
3687	[MLX5_QP_STATE_SQER] = {
3688		[MLX5_QP_STATE_RTS] = {
3689			[MLX5_QP_ST_UD]	 = MLX5_QP_OPTPAR_Q_KEY,
3690			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
3691			[MLX5_QP_ST_UC]	 = MLX5_QP_OPTPAR_RWE,
3692			[MLX5_QP_ST_RC]	 = MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3693					   MLX5_QP_OPTPAR_RWE		|
3694					   MLX5_QP_OPTPAR_RAE		|
3695					   MLX5_QP_OPTPAR_RRE,
3696			[MLX5_QP_ST_XRC]  = MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3697					   MLX5_QP_OPTPAR_RWE		|
3698					   MLX5_QP_OPTPAR_RAE		|
3699					   MLX5_QP_OPTPAR_RRE,
3700		},
3701	},
3702	[MLX5_QP_STATE_SQD] = {
3703		[MLX5_QP_STATE_RTS] = {
3704			[MLX5_QP_ST_UD] = MLX5_QP_OPTPAR_Q_KEY,
3705			[MLX5_QP_ST_MLX] = MLX5_QP_OPTPAR_Q_KEY,
3706			[MLX5_QP_ST_UC] = MLX5_QP_OPTPAR_RWE,
3707			[MLX5_QP_ST_RC] = MLX5_QP_OPTPAR_RNR_TIMEOUT	|
3708					  MLX5_QP_OPTPAR_RWE		|
3709					  MLX5_QP_OPTPAR_RAE		|
3710					  MLX5_QP_OPTPAR_RRE,
3711		},
3712	},
3713};
3714
3715static int ib_nr_to_mlx5_nr(int ib_mask)
3716{
3717	switch (ib_mask) {
3718	case IB_QP_STATE:
3719		return 0;
3720	case IB_QP_CUR_STATE:
3721		return 0;
3722	case IB_QP_EN_SQD_ASYNC_NOTIFY:
3723		return 0;
3724	case IB_QP_ACCESS_FLAGS:
3725		return MLX5_QP_OPTPAR_RWE | MLX5_QP_OPTPAR_RRE |
3726			MLX5_QP_OPTPAR_RAE;
3727	case IB_QP_PKEY_INDEX:
3728		return MLX5_QP_OPTPAR_PKEY_INDEX;
3729	case IB_QP_PORT:
3730		return MLX5_QP_OPTPAR_PRI_PORT;
3731	case IB_QP_QKEY:
3732		return MLX5_QP_OPTPAR_Q_KEY;
3733	case IB_QP_AV:
3734		return MLX5_QP_OPTPAR_PRIMARY_ADDR_PATH |
3735			MLX5_QP_OPTPAR_PRI_PORT;
3736	case IB_QP_PATH_MTU:
3737		return 0;
3738	case IB_QP_TIMEOUT:
3739		return MLX5_QP_OPTPAR_ACK_TIMEOUT;
3740	case IB_QP_RETRY_CNT:
3741		return MLX5_QP_OPTPAR_RETRY_COUNT;
3742	case IB_QP_RNR_RETRY:
3743		return MLX5_QP_OPTPAR_RNR_RETRY;
3744	case IB_QP_RQ_PSN:
3745		return 0;
3746	case IB_QP_MAX_QP_RD_ATOMIC:
3747		return MLX5_QP_OPTPAR_SRA_MAX;
3748	case IB_QP_ALT_PATH:
3749		return MLX5_QP_OPTPAR_ALT_ADDR_PATH;
3750	case IB_QP_MIN_RNR_TIMER:
3751		return MLX5_QP_OPTPAR_RNR_TIMEOUT;
3752	case IB_QP_SQ_PSN:
3753		return 0;
3754	case IB_QP_MAX_DEST_RD_ATOMIC:
3755		return MLX5_QP_OPTPAR_RRA_MAX | MLX5_QP_OPTPAR_RWE |
3756			MLX5_QP_OPTPAR_RRE | MLX5_QP_OPTPAR_RAE;
3757	case IB_QP_PATH_MIG_STATE:
3758		return MLX5_QP_OPTPAR_PM_STATE;
3759	case IB_QP_CAP:
3760		return 0;
3761	case IB_QP_DEST_QPN:
3762		return 0;
3763	}
3764	return 0;
3765}
3766
3767static int ib_mask_to_mlx5_opt(int ib_mask)
3768{
3769	int result = 0;
3770	int i;
3771
3772	for (i = 0; i < 8 * sizeof(int); i++) {
3773		if ((1 << i) & ib_mask)
3774			result |= ib_nr_to_mlx5_nr(1 << i);
3775	}
3776
3777	return result;
3778}
3779
3780static int modify_raw_packet_qp_rq(
3781	struct mlx5_ib_dev *dev, struct mlx5_ib_rq *rq, int new_state,
3782	const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
3783{
3784	void *in;
3785	void *rqc;
3786	int inlen;
3787	int err;
3788
3789	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
3790	in = kvzalloc(inlen, GFP_KERNEL);
3791	if (!in)
3792		return -ENOMEM;
3793
3794	MLX5_SET(modify_rq_in, in, rq_state, rq->state);
3795	MLX5_SET(modify_rq_in, in, uid, to_mpd(pd)->uid);
3796
3797	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
3798	MLX5_SET(rqc, rqc, state, new_state);
3799
3800	if (raw_qp_param->set_mask & MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID) {
3801		if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
3802			MLX5_SET64(modify_rq_in, in, modify_bitmask,
3803				   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
3804			MLX5_SET(rqc, rqc, counter_set_id, raw_qp_param->rq_q_ctr_id);
3805		} else
3806			dev_info_once(
3807				&dev->ib_dev.dev,
3808				"RAW PACKET QP counters are not supported on current FW\n");
3809	}
3810
3811	err = mlx5_core_modify_rq(dev->mdev, rq->base.mqp.qpn, in);
3812	if (err)
3813		goto out;
3814
3815	rq->state = new_state;
3816
3817out:
3818	kvfree(in);
3819	return err;
3820}
3821
3822static int modify_raw_packet_qp_sq(
3823	struct mlx5_core_dev *dev, struct mlx5_ib_sq *sq, int new_state,
3824	const struct mlx5_modify_raw_qp_param *raw_qp_param, struct ib_pd *pd)
3825{
3826	struct mlx5_ib_qp *ibqp = sq->base.container_mibqp;
3827	struct mlx5_rate_limit old_rl = ibqp->rl;
3828	struct mlx5_rate_limit new_rl = old_rl;
3829	bool new_rate_added = false;
3830	u16 rl_index = 0;
3831	void *in;
3832	void *sqc;
3833	int inlen;
3834	int err;
3835
3836	inlen = MLX5_ST_SZ_BYTES(modify_sq_in);
3837	in = kvzalloc(inlen, GFP_KERNEL);
3838	if (!in)
3839		return -ENOMEM;
3840
3841	MLX5_SET(modify_sq_in, in, uid, to_mpd(pd)->uid);
3842	MLX5_SET(modify_sq_in, in, sq_state, sq->state);
3843
3844	sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx);
3845	MLX5_SET(sqc, sqc, state, new_state);
3846
3847	if (raw_qp_param->set_mask & MLX5_RAW_QP_RATE_LIMIT) {
3848		if (new_state != MLX5_SQC_STATE_RDY)
3849			pr_warn("%s: Rate limit can only be changed when SQ is moving to RDY\n",
3850				__func__);
3851		else
3852			new_rl = raw_qp_param->rl;
3853	}
3854
3855	if (!mlx5_rl_are_equal(&old_rl, &new_rl)) {
3856		if (new_rl.rate) {
3857			err = mlx5_rl_add_rate(dev, &rl_index, &new_rl);
3858			if (err) {
3859				pr_err("Failed configuring rate limit(err %d): \
3860				       rate %u, max_burst_sz %u, typical_pkt_sz %u\n",
3861				       err, new_rl.rate, new_rl.max_burst_sz,
3862				       new_rl.typical_pkt_sz);
3863
3864				goto out;
3865			}
3866			new_rate_added = true;
3867		}
3868
3869		MLX5_SET64(modify_sq_in, in, modify_bitmask, 1);
3870		/* index 0 means no limit */
3871		MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, rl_index);
3872	}
3873
3874	err = mlx5_core_modify_sq(dev, sq->base.mqp.qpn, in);
3875	if (err) {
3876		/* Remove new rate from table if failed */
3877		if (new_rate_added)
3878			mlx5_rl_remove_rate(dev, &new_rl);
3879		goto out;
3880	}
3881
3882	/* Only remove the old rate after new rate was set */
3883	if ((old_rl.rate && !mlx5_rl_are_equal(&old_rl, &new_rl)) ||
3884	    (new_state != MLX5_SQC_STATE_RDY)) {
3885		mlx5_rl_remove_rate(dev, &old_rl);
3886		if (new_state != MLX5_SQC_STATE_RDY)
3887			memset(&new_rl, 0, sizeof(new_rl));
3888	}
3889
3890	ibqp->rl = new_rl;
3891	sq->state = new_state;
3892
3893out:
3894	kvfree(in);
3895	return err;
3896}
3897
3898static int modify_raw_packet_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
3899				const struct mlx5_modify_raw_qp_param *raw_qp_param,
3900				u8 tx_affinity)
3901{
3902	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
3903	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
3904	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
3905	int modify_rq = !!qp->rq.wqe_cnt;
3906	int modify_sq = !!qp->sq.wqe_cnt;
3907	int rq_state;
3908	int sq_state;
3909	int err;
3910
3911	switch (raw_qp_param->operation) {
3912	case MLX5_CMD_OP_RST2INIT_QP:
3913		rq_state = MLX5_RQC_STATE_RDY;
3914		sq_state = MLX5_SQC_STATE_RST;
3915		break;
3916	case MLX5_CMD_OP_2ERR_QP:
3917		rq_state = MLX5_RQC_STATE_ERR;
3918		sq_state = MLX5_SQC_STATE_ERR;
3919		break;
3920	case MLX5_CMD_OP_2RST_QP:
3921		rq_state = MLX5_RQC_STATE_RST;
3922		sq_state = MLX5_SQC_STATE_RST;
3923		break;
3924	case MLX5_CMD_OP_RTR2RTS_QP:
3925	case MLX5_CMD_OP_RTS2RTS_QP:
3926		if (raw_qp_param->set_mask & ~MLX5_RAW_QP_RATE_LIMIT)
3927			return -EINVAL;
3928
3929		modify_rq = 0;
3930		sq_state = MLX5_SQC_STATE_RDY;
3931		break;
3932	case MLX5_CMD_OP_INIT2INIT_QP:
3933	case MLX5_CMD_OP_INIT2RTR_QP:
3934		if (raw_qp_param->set_mask)
3935			return -EINVAL;
3936		else
3937			return 0;
3938	default:
3939		WARN_ON(1);
3940		return -EINVAL;
3941	}
3942
3943	if (modify_rq) {
3944		err =  modify_raw_packet_qp_rq(dev, rq, rq_state, raw_qp_param,
3945					       qp->ibqp.pd);
3946		if (err)
3947			return err;
3948	}
3949
3950	if (modify_sq) {
3951		struct mlx5_flow_handle *flow_rule;
3952
3953		if (tx_affinity) {
3954			err = modify_raw_packet_tx_affinity(dev->mdev, sq,
3955							    tx_affinity,
3956							    qp->ibqp.pd);
3957			if (err)
3958				return err;
3959		}
3960
3961		flow_rule = create_flow_rule_vport_sq(dev, sq,
3962						      raw_qp_param->port);
3963		if (IS_ERR(flow_rule))
3964			return PTR_ERR(flow_rule);
3965
3966		err = modify_raw_packet_qp_sq(dev->mdev, sq, sq_state,
3967					      raw_qp_param, qp->ibqp.pd);
3968		if (err) {
3969			if (flow_rule)
3970				mlx5_del_flow_rules(flow_rule);
3971			return err;
3972		}
3973
3974		if (flow_rule) {
3975			destroy_flow_rule_vport_sq(sq);
3976			sq->flow_rule = flow_rule;
3977		}
3978
3979		return err;
3980	}
3981
3982	return 0;
3983}
3984
3985static unsigned int get_tx_affinity_rr(struct mlx5_ib_dev *dev,
3986				       struct ib_udata *udata)
3987{
3988	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
3989		udata, struct mlx5_ib_ucontext, ibucontext);
3990	u8 port_num = mlx5_core_native_port_num(dev->mdev) - 1;
3991	atomic_t *tx_port_affinity;
3992
3993	if (ucontext)
3994		tx_port_affinity = &ucontext->tx_port_affinity;
3995	else
3996		tx_port_affinity = &dev->port[port_num].roce.tx_port_affinity;
3997
3998	return (unsigned int)atomic_add_return(1, tx_port_affinity) %
3999		(dev->lag_active ? dev->lag_ports : MLX5_CAP_GEN(dev->mdev, num_lag_ports)) + 1;
4000}
4001
4002static bool qp_supports_affinity(struct mlx5_ib_qp *qp)
4003{
4004	if ((qp->type == IB_QPT_RC) || (qp->type == IB_QPT_UD) ||
4005	    (qp->type == IB_QPT_UC) || (qp->type == IB_QPT_RAW_PACKET) ||
4006	    (qp->type == IB_QPT_XRC_INI) || (qp->type == IB_QPT_XRC_TGT) ||
4007	    (qp->type == MLX5_IB_QPT_DCI))
4008		return true;
4009	return false;
4010}
4011
4012static unsigned int get_tx_affinity(struct ib_qp *qp,
4013				    const struct ib_qp_attr *attr,
4014				    int attr_mask, u8 init,
4015				    struct ib_udata *udata)
4016{
4017	struct mlx5_ib_ucontext *ucontext = rdma_udata_to_drv_context(
4018		udata, struct mlx5_ib_ucontext, ibucontext);
4019	struct mlx5_ib_dev *dev = to_mdev(qp->device);
4020	struct mlx5_ib_qp *mqp = to_mqp(qp);
4021	struct mlx5_ib_qp_base *qp_base;
4022	unsigned int tx_affinity;
4023
4024	if (!(mlx5_ib_lag_should_assign_affinity(dev) &&
4025	      qp_supports_affinity(mqp)))
4026		return 0;
4027
4028	if (mqp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
4029		tx_affinity = mqp->gsi_lag_port;
4030	else if (init)
4031		tx_affinity = get_tx_affinity_rr(dev, udata);
4032	else if ((attr_mask & IB_QP_AV) && attr->xmit_slave)
4033		tx_affinity =
4034			mlx5_lag_get_slave_port(dev->mdev, attr->xmit_slave);
4035	else
4036		return 0;
4037
4038	qp_base = &mqp->trans_qp.base;
4039	if (ucontext)
4040		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x ucontext %p\n",
4041			    tx_affinity, qp_base->mqp.qpn, ucontext);
4042	else
4043		mlx5_ib_dbg(dev, "Set tx affinity 0x%x to qpn 0x%x\n",
4044			    tx_affinity, qp_base->mqp.qpn);
4045	return tx_affinity;
4046}
4047
4048static int __mlx5_ib_qp_set_raw_qp_counter(struct mlx5_ib_qp *qp, u32 set_id,
4049					   struct mlx5_core_dev *mdev)
4050{
4051	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4052	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4053	u32 in[MLX5_ST_SZ_DW(modify_rq_in)] = {};
4054	void *rqc;
4055
4056	if (!qp->rq.wqe_cnt)
4057		return 0;
4058
4059	MLX5_SET(modify_rq_in, in, rq_state, rq->state);
4060	MLX5_SET(modify_rq_in, in, uid, to_mpd(qp->ibqp.pd)->uid);
4061
4062	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
4063	MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY);
4064
4065	MLX5_SET64(modify_rq_in, in, modify_bitmask,
4066		   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
4067	MLX5_SET(rqc, rqc, counter_set_id, set_id);
4068
4069	return mlx5_core_modify_rq(mdev, rq->base.mqp.qpn, in);
4070}
4071
4072static int __mlx5_ib_qp_set_counter(struct ib_qp *qp,
4073				    struct rdma_counter *counter)
4074{
4075	struct mlx5_ib_dev *dev = to_mdev(qp->device);
4076	u32 in[MLX5_ST_SZ_DW(rts2rts_qp_in)] = {};
4077	struct mlx5_ib_qp *mqp = to_mqp(qp);
4078	struct mlx5_ib_qp_base *base;
4079	u32 set_id;
4080	u32 *qpc;
4081
4082	if (counter)
4083		set_id = counter->id;
4084	else
4085		set_id = mlx5_ib_get_counters_id(dev, mqp->port - 1);
4086
4087	if (mqp->type == IB_QPT_RAW_PACKET)
4088		return __mlx5_ib_qp_set_raw_qp_counter(mqp, set_id, dev->mdev);
4089
4090	base = &mqp->trans_qp.base;
4091	MLX5_SET(rts2rts_qp_in, in, opcode, MLX5_CMD_OP_RTS2RTS_QP);
4092	MLX5_SET(rts2rts_qp_in, in, qpn, base->mqp.qpn);
4093	MLX5_SET(rts2rts_qp_in, in, uid, base->mqp.uid);
4094	MLX5_SET(rts2rts_qp_in, in, opt_param_mask,
4095		 MLX5_QP_OPTPAR_COUNTER_SET_ID);
4096
4097	qpc = MLX5_ADDR_OF(rts2rts_qp_in, in, qpc);
4098	MLX5_SET(qpc, qpc, counter_set_id, set_id);
4099	return mlx5_cmd_exec_in(dev->mdev, rts2rts_qp, in);
4100}
4101
4102static int __mlx5_ib_modify_qp(struct ib_qp *ibqp,
4103			       const struct ib_qp_attr *attr, int attr_mask,
4104			       enum ib_qp_state cur_state,
4105			       enum ib_qp_state new_state,
4106			       const struct mlx5_ib_modify_qp *ucmd,
4107			       struct mlx5_ib_modify_qp_resp *resp,
4108			       struct ib_udata *udata)
4109{
4110	static const u16 optab[MLX5_QP_NUM_STATE][MLX5_QP_NUM_STATE] = {
4111		[MLX5_QP_STATE_RST] = {
4112			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4113			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4114			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_RST2INIT_QP,
4115		},
4116		[MLX5_QP_STATE_INIT]  = {
4117			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4118			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4119			[MLX5_QP_STATE_INIT]	= MLX5_CMD_OP_INIT2INIT_QP,
4120			[MLX5_QP_STATE_RTR]	= MLX5_CMD_OP_INIT2RTR_QP,
4121		},
4122		[MLX5_QP_STATE_RTR]   = {
4123			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4124			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4125			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTR2RTS_QP,
4126		},
4127		[MLX5_QP_STATE_RTS]   = {
4128			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4129			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4130			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_RTS2RTS_QP,
4131		},
4132		[MLX5_QP_STATE_SQD] = {
4133			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4134			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4135			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQD_RTS_QP,
4136		},
4137		[MLX5_QP_STATE_SQER] = {
4138			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4139			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4140			[MLX5_QP_STATE_RTS]	= MLX5_CMD_OP_SQERR2RTS_QP,
4141		},
4142		[MLX5_QP_STATE_ERR] = {
4143			[MLX5_QP_STATE_RST]	= MLX5_CMD_OP_2RST_QP,
4144			[MLX5_QP_STATE_ERR]	= MLX5_CMD_OP_2ERR_QP,
4145		}
4146	};
4147
4148	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4149	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4150	struct mlx5_ib_qp_base *base = &qp->trans_qp.base;
4151	struct mlx5_ib_cq *send_cq, *recv_cq;
4152	struct mlx5_ib_pd *pd;
4153	enum mlx5_qp_state mlx5_cur, mlx5_new;
4154	void *qpc, *pri_path, *alt_path;
4155	enum mlx5_qp_optpar optpar = 0;
4156	u32 set_id = 0;
4157	int mlx5_st;
4158	int err;
4159	u16 op;
4160	u8 tx_affinity = 0;
4161
4162	mlx5_st = to_mlx5_st(qp->type);
4163	if (mlx5_st < 0)
4164		return -EINVAL;
4165
4166	qpc = kzalloc(MLX5_ST_SZ_BYTES(qpc), GFP_KERNEL);
4167	if (!qpc)
4168		return -ENOMEM;
4169
4170	pd = to_mpd(qp->ibqp.pd);
4171	MLX5_SET(qpc, qpc, st, mlx5_st);
4172
4173	if (!(attr_mask & IB_QP_PATH_MIG_STATE)) {
4174		MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
4175	} else {
4176		switch (attr->path_mig_state) {
4177		case IB_MIG_MIGRATED:
4178			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_MIGRATED);
4179			break;
4180		case IB_MIG_REARM:
4181			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_REARM);
4182			break;
4183		case IB_MIG_ARMED:
4184			MLX5_SET(qpc, qpc, pm_state, MLX5_QP_PM_ARMED);
4185			break;
4186		}
4187	}
4188
4189	tx_affinity = get_tx_affinity(ibqp, attr, attr_mask,
4190				      cur_state == IB_QPS_RESET &&
4191				      new_state == IB_QPS_INIT, udata);
4192
4193	MLX5_SET(qpc, qpc, lag_tx_port_affinity, tx_affinity);
4194	if (tx_affinity && new_state == IB_QPS_RTR &&
4195	    MLX5_CAP_GEN(dev->mdev, init2_lag_tx_port_affinity))
4196		optpar |= MLX5_QP_OPTPAR_LAG_TX_AFF;
4197
4198	if (is_sqp(qp->type)) {
4199		MLX5_SET(qpc, qpc, mtu, IB_MTU_256);
4200		MLX5_SET(qpc, qpc, log_msg_max, 8);
4201	} else if ((qp->type == IB_QPT_UD &&
4202		    !(qp->flags & IB_QP_CREATE_SOURCE_QPN)) ||
4203		   qp->type == MLX5_IB_QPT_REG_UMR) {
4204		MLX5_SET(qpc, qpc, mtu, IB_MTU_4096);
4205		MLX5_SET(qpc, qpc, log_msg_max, 12);
4206	} else if (attr_mask & IB_QP_PATH_MTU) {
4207		if (attr->path_mtu < IB_MTU_256 ||
4208		    attr->path_mtu > IB_MTU_4096) {
4209			mlx5_ib_warn(dev, "invalid mtu %d\n", attr->path_mtu);
4210			err = -EINVAL;
4211			goto out;
4212		}
4213		MLX5_SET(qpc, qpc, mtu, attr->path_mtu);
4214		MLX5_SET(qpc, qpc, log_msg_max,
4215			 MLX5_CAP_GEN(dev->mdev, log_max_msg));
4216	}
4217
4218	if (attr_mask & IB_QP_DEST_QPN)
4219		MLX5_SET(qpc, qpc, remote_qpn, attr->dest_qp_num);
4220
4221	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
4222	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
4223
4224	if (attr_mask & IB_QP_PKEY_INDEX)
4225		MLX5_SET(ads, pri_path, pkey_index, attr->pkey_index);
4226
4227	/* todo implement counter_index functionality */
4228
4229	if (is_sqp(qp->type))
4230		MLX5_SET(ads, pri_path, vhca_port_num, qp->port);
4231
4232	if (attr_mask & IB_QP_PORT)
4233		MLX5_SET(ads, pri_path, vhca_port_num, attr->port_num);
4234
4235	if (attr_mask & IB_QP_AV) {
4236		err = mlx5_set_path(dev, qp, &attr->ah_attr, pri_path,
4237				    attr_mask & IB_QP_PORT ? attr->port_num :
4238							     qp->port,
4239				    attr_mask, 0, attr, false);
4240		if (err)
4241			goto out;
4242	}
4243
4244	if (attr_mask & IB_QP_TIMEOUT)
4245		MLX5_SET(ads, pri_path, ack_timeout, attr->timeout);
4246
4247	if (attr_mask & IB_QP_ALT_PATH) {
4248		err = mlx5_set_path(dev, qp, &attr->alt_ah_attr, alt_path,
4249				    attr->alt_port_num,
4250				    attr_mask | IB_QP_PKEY_INDEX |
4251					    IB_QP_TIMEOUT,
4252				    0, attr, true);
4253		if (err)
4254			goto out;
4255	}
4256
4257	get_cqs(qp->type, qp->ibqp.send_cq, qp->ibqp.recv_cq,
4258		&send_cq, &recv_cq);
4259
4260	MLX5_SET(qpc, qpc, pd, pd ? pd->pdn : to_mpd(dev->devr.p0)->pdn);
4261	if (send_cq)
4262		MLX5_SET(qpc, qpc, cqn_snd, send_cq->mcq.cqn);
4263	if (recv_cq)
4264		MLX5_SET(qpc, qpc, cqn_rcv, recv_cq->mcq.cqn);
4265
4266	MLX5_SET(qpc, qpc, log_ack_req_freq, MLX5_IB_ACK_REQ_FREQ);
4267
4268	if (attr_mask & IB_QP_RNR_RETRY)
4269		MLX5_SET(qpc, qpc, rnr_retry, attr->rnr_retry);
4270
4271	if (attr_mask & IB_QP_RETRY_CNT)
4272		MLX5_SET(qpc, qpc, retry_count, attr->retry_cnt);
4273
4274	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC && attr->max_rd_atomic)
4275		MLX5_SET(qpc, qpc, log_sra_max, ilog2(attr->max_rd_atomic));
4276
4277	if (attr_mask & IB_QP_SQ_PSN)
4278		MLX5_SET(qpc, qpc, next_send_psn, attr->sq_psn);
4279
4280	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC && attr->max_dest_rd_atomic)
4281		MLX5_SET(qpc, qpc, log_rra_max,
4282			 ilog2(attr->max_dest_rd_atomic));
4283
4284	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
4285		err = set_qpc_atomic_flags(qp, attr, attr_mask, qpc);
4286		if (err)
4287			goto out;
4288	}
4289
4290	if (attr_mask & IB_QP_MIN_RNR_TIMER)
4291		MLX5_SET(qpc, qpc, min_rnr_nak, attr->min_rnr_timer);
4292
4293	if (attr_mask & IB_QP_RQ_PSN)
4294		MLX5_SET(qpc, qpc, next_rcv_psn, attr->rq_psn);
4295
4296	if (attr_mask & IB_QP_QKEY)
4297		MLX5_SET(qpc, qpc, q_key, attr->qkey);
4298
4299	if (qp->rq.wqe_cnt && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
4300		MLX5_SET64(qpc, qpc, dbr_addr, qp->db.dma);
4301
4302	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4303		u8 port_num = (attr_mask & IB_QP_PORT ? attr->port_num :
4304			       qp->port) - 1;
4305
4306		/* Underlay port should be used - index 0 function per port */
4307		if (qp->flags & IB_QP_CREATE_SOURCE_QPN)
4308			port_num = 0;
4309
4310		if (ibqp->counter)
4311			set_id = ibqp->counter->id;
4312		else
4313			set_id = mlx5_ib_get_counters_id(dev, port_num);
4314		MLX5_SET(qpc, qpc, counter_set_id, set_id);
4315	}
4316
4317	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
4318		MLX5_SET(qpc, qpc, rlky, 1);
4319
4320	if (qp->flags & MLX5_IB_QP_CREATE_SQPN_QP1)
4321		MLX5_SET(qpc, qpc, deth_sqpn, 1);
4322
4323	mlx5_cur = to_mlx5_state(cur_state);
4324	mlx5_new = to_mlx5_state(new_state);
4325
4326	if (mlx5_cur >= MLX5_QP_NUM_STATE || mlx5_new >= MLX5_QP_NUM_STATE ||
4327	    !optab[mlx5_cur][mlx5_new]) {
4328		err = -EINVAL;
4329		goto out;
4330	}
4331
4332	op = optab[mlx5_cur][mlx5_new];
4333	optpar |= ib_mask_to_mlx5_opt(attr_mask);
4334	optpar &= opt_mask[mlx5_cur][mlx5_new][mlx5_st];
4335
4336	if (qp->type == IB_QPT_RAW_PACKET ||
4337	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
4338		struct mlx5_modify_raw_qp_param raw_qp_param = {};
4339
4340		raw_qp_param.operation = op;
4341		if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4342			raw_qp_param.rq_q_ctr_id = set_id;
4343			raw_qp_param.set_mask |= MLX5_RAW_QP_MOD_SET_RQ_Q_CTR_ID;
4344		}
4345
4346		if (attr_mask & IB_QP_PORT)
4347			raw_qp_param.port = attr->port_num;
4348
4349		if (attr_mask & IB_QP_RATE_LIMIT) {
4350			raw_qp_param.rl.rate = attr->rate_limit;
4351
4352			if (ucmd->burst_info.max_burst_sz) {
4353				if (attr->rate_limit &&
4354				    MLX5_CAP_QOS(dev->mdev, packet_pacing_burst_bound)) {
4355					raw_qp_param.rl.max_burst_sz =
4356						ucmd->burst_info.max_burst_sz;
4357				} else {
4358					err = -EINVAL;
4359					goto out;
4360				}
4361			}
4362
4363			if (ucmd->burst_info.typical_pkt_sz) {
4364				if (attr->rate_limit &&
4365				    MLX5_CAP_QOS(dev->mdev, packet_pacing_typical_size)) {
4366					raw_qp_param.rl.typical_pkt_sz =
4367						ucmd->burst_info.typical_pkt_sz;
4368				} else {
4369					err = -EINVAL;
4370					goto out;
4371				}
4372			}
4373
4374			raw_qp_param.set_mask |= MLX5_RAW_QP_RATE_LIMIT;
4375		}
4376
4377		err = modify_raw_packet_qp(dev, qp, &raw_qp_param, tx_affinity);
4378	} else {
4379		if (udata) {
4380			/* For the kernel flows, the resp will stay zero */
4381			resp->ece_options =
4382				MLX5_CAP_GEN(dev->mdev, ece_support) ?
4383					ucmd->ece_options : 0;
4384			resp->response_length = sizeof(*resp);
4385		}
4386		err = mlx5_core_qp_modify(dev, op, optpar, qpc, &base->mqp,
4387					  &resp->ece_options);
4388	}
4389
4390	if (err)
4391		goto out;
4392
4393	qp->state = new_state;
4394
4395	if (attr_mask & IB_QP_ACCESS_FLAGS)
4396		qp->trans_qp.atomic_rd_en = attr->qp_access_flags;
4397	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
4398		qp->trans_qp.resp_depth = attr->max_dest_rd_atomic;
4399	if (attr_mask & IB_QP_PORT)
4400		qp->port = attr->port_num;
4401	if (attr_mask & IB_QP_ALT_PATH)
4402		qp->trans_qp.alt_port = attr->alt_port_num;
4403
4404	/*
4405	 * If we moved a kernel QP to RESET, clean up all old CQ
4406	 * entries and reinitialize the QP.
4407	 */
4408	if (new_state == IB_QPS_RESET &&
4409	    !ibqp->uobject && qp->type != IB_QPT_XRC_TGT) {
4410		mlx5_ib_cq_clean(recv_cq, base->mqp.qpn,
4411				 ibqp->srq ? to_msrq(ibqp->srq) : NULL);
4412		if (send_cq != recv_cq)
4413			mlx5_ib_cq_clean(send_cq, base->mqp.qpn, NULL);
4414
4415		qp->rq.head = 0;
4416		qp->rq.tail = 0;
4417		qp->sq.head = 0;
4418		qp->sq.tail = 0;
4419		qp->sq.cur_post = 0;
4420		if (qp->sq.wqe_cnt)
4421			qp->sq.cur_edge = get_sq_edge(&qp->sq, 0);
4422		qp->sq.last_poll = 0;
4423		qp->db.db[MLX5_RCV_DBR] = 0;
4424		qp->db.db[MLX5_SND_DBR] = 0;
4425	}
4426
4427	if ((new_state == IB_QPS_RTS) && qp->counter_pending) {
4428		err = __mlx5_ib_qp_set_counter(ibqp, ibqp->counter);
4429		if (!err)
4430			qp->counter_pending = 0;
4431	}
4432
4433out:
4434	kfree(qpc);
4435	return err;
4436}
4437
4438static inline bool is_valid_mask(int mask, int req, int opt)
4439{
4440	if ((mask & req) != req)
4441		return false;
4442
4443	if (mask & ~(req | opt))
4444		return false;
4445
4446	return true;
4447}
4448
4449/* check valid transition for driver QP types
4450 * for now the only QP type that this function supports is DCI
4451 */
4452static bool modify_dci_qp_is_ok(enum ib_qp_state cur_state, enum ib_qp_state new_state,
4453				enum ib_qp_attr_mask attr_mask)
4454{
4455	int req = IB_QP_STATE;
4456	int opt = 0;
4457
4458	if (new_state == IB_QPS_RESET) {
4459		return is_valid_mask(attr_mask, req, opt);
4460	} else if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4461		req |= IB_QP_PKEY_INDEX | IB_QP_PORT;
4462		return is_valid_mask(attr_mask, req, opt);
4463	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_INIT) {
4464		opt = IB_QP_PKEY_INDEX | IB_QP_PORT;
4465		return is_valid_mask(attr_mask, req, opt);
4466	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4467		req |= IB_QP_PATH_MTU;
4468		opt = IB_QP_PKEY_INDEX | IB_QP_AV;
4469		return is_valid_mask(attr_mask, req, opt);
4470	} else if (cur_state == IB_QPS_RTR && new_state == IB_QPS_RTS) {
4471		req |= IB_QP_TIMEOUT | IB_QP_RETRY_CNT | IB_QP_RNR_RETRY |
4472		       IB_QP_MAX_QP_RD_ATOMIC | IB_QP_SQ_PSN;
4473		opt = IB_QP_MIN_RNR_TIMER;
4474		return is_valid_mask(attr_mask, req, opt);
4475	} else if (cur_state == IB_QPS_RTS && new_state == IB_QPS_RTS) {
4476		opt = IB_QP_MIN_RNR_TIMER;
4477		return is_valid_mask(attr_mask, req, opt);
4478	} else if (cur_state != IB_QPS_RESET && new_state == IB_QPS_ERR) {
4479		return is_valid_mask(attr_mask, req, opt);
4480	}
4481	return false;
4482}
4483
4484/* mlx5_ib_modify_dct: modify a DCT QP
4485 * valid transitions are:
4486 * RESET to INIT: must set access_flags, pkey_index and port
4487 * INIT  to RTR : must set min_rnr_timer, tclass, flow_label,
4488 *			   mtu, gid_index and hop_limit
4489 * Other transitions and attributes are illegal
4490 */
4491static int mlx5_ib_modify_dct(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4492			      int attr_mask, struct mlx5_ib_modify_qp *ucmd,
4493			      struct ib_udata *udata)
4494{
4495	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4496	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4497	enum ib_qp_state cur_state, new_state;
4498	int required = IB_QP_STATE;
4499	void *dctc;
4500	int err;
4501
4502	if (!(attr_mask & IB_QP_STATE))
4503		return -EINVAL;
4504
4505	cur_state = qp->state;
4506	new_state = attr->qp_state;
4507
4508	dctc = MLX5_ADDR_OF(create_dct_in, qp->dct.in, dct_context_entry);
4509	if (MLX5_CAP_GEN(dev->mdev, ece_support) && ucmd->ece_options)
4510		/*
4511		 * DCT doesn't initialize QP till modify command is executed,
4512		 * so we need to overwrite previously set ECE field if user
4513		 * provided any value except zero, which means not set/not
4514		 * valid.
4515		 */
4516		MLX5_SET(dctc, dctc, ece, ucmd->ece_options);
4517
4518	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
4519		u16 set_id;
4520
4521		required |= IB_QP_ACCESS_FLAGS | IB_QP_PKEY_INDEX | IB_QP_PORT;
4522		if (!is_valid_mask(attr_mask, required, 0))
4523			return -EINVAL;
4524
4525		if (attr->port_num == 0 ||
4526		    attr->port_num > dev->num_ports) {
4527			mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4528				    attr->port_num, dev->num_ports);
4529			return -EINVAL;
4530		}
4531		if (attr->qp_access_flags & IB_ACCESS_REMOTE_READ)
4532			MLX5_SET(dctc, dctc, rre, 1);
4533		if (attr->qp_access_flags & IB_ACCESS_REMOTE_WRITE)
4534			MLX5_SET(dctc, dctc, rwe, 1);
4535		if (attr->qp_access_flags & IB_ACCESS_REMOTE_ATOMIC) {
4536			int atomic_mode;
4537
4538			atomic_mode = get_atomic_mode(dev, MLX5_IB_QPT_DCT);
4539			if (atomic_mode < 0)
4540				return -EOPNOTSUPP;
4541
4542			MLX5_SET(dctc, dctc, atomic_mode, atomic_mode);
4543			MLX5_SET(dctc, dctc, rae, 1);
4544		}
4545		MLX5_SET(dctc, dctc, pkey_index, attr->pkey_index);
4546		if (mlx5_lag_is_active(dev->mdev))
4547			MLX5_SET(dctc, dctc, port,
4548				 get_tx_affinity_rr(dev, udata));
4549		else
4550			MLX5_SET(dctc, dctc, port, attr->port_num);
4551
4552		set_id = mlx5_ib_get_counters_id(dev, attr->port_num - 1);
4553		MLX5_SET(dctc, dctc, counter_set_id, set_id);
4554	} else if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
4555		struct mlx5_ib_modify_qp_resp resp = {};
4556		u32 out[MLX5_ST_SZ_DW(create_dct_out)] = {};
4557		u32 min_resp_len = offsetofend(typeof(resp), dctn);
4558
4559		if (udata->outlen < min_resp_len)
4560			return -EINVAL;
4561		/*
4562		 * If we don't have enough space for the ECE options,
4563		 * simply indicate it with resp.response_length.
4564		 */
4565		resp.response_length = (udata->outlen < sizeof(resp)) ?
4566					       min_resp_len :
4567					       sizeof(resp);
4568
4569		required |= IB_QP_MIN_RNR_TIMER | IB_QP_AV | IB_QP_PATH_MTU;
4570		if (!is_valid_mask(attr_mask, required, 0))
4571			return -EINVAL;
4572		MLX5_SET(dctc, dctc, min_rnr_nak, attr->min_rnr_timer);
4573		MLX5_SET(dctc, dctc, tclass, attr->ah_attr.grh.traffic_class);
4574		MLX5_SET(dctc, dctc, flow_label, attr->ah_attr.grh.flow_label);
4575		MLX5_SET(dctc, dctc, mtu, attr->path_mtu);
4576		MLX5_SET(dctc, dctc, my_addr_index, attr->ah_attr.grh.sgid_index);
4577		MLX5_SET(dctc, dctc, hop_limit, attr->ah_attr.grh.hop_limit);
4578		if (attr->ah_attr.type == RDMA_AH_ATTR_TYPE_ROCE)
4579			MLX5_SET(dctc, dctc, eth_prio, attr->ah_attr.sl & 0x7);
4580
4581		err = mlx5_core_create_dct(dev, &qp->dct.mdct, qp->dct.in,
4582					   MLX5_ST_SZ_BYTES(create_dct_in), out,
4583					   sizeof(out));
4584		err = mlx5_cmd_check(dev->mdev, err, qp->dct.in, out);
4585		if (err)
4586			return err;
4587		resp.dctn = qp->dct.mdct.mqp.qpn;
4588		if (MLX5_CAP_GEN(dev->mdev, ece_support))
4589			resp.ece_options = MLX5_GET(create_dct_out, out, ece);
4590		err = ib_copy_to_udata(udata, &resp, resp.response_length);
4591		if (err) {
4592			mlx5_core_destroy_dct(dev, &qp->dct.mdct);
4593			return err;
4594		}
4595	} else {
4596		mlx5_ib_warn(dev, "Modify DCT: Invalid transition from %d to %d\n", cur_state, new_state);
4597		return -EINVAL;
4598	}
4599
4600	qp->state = new_state;
4601	return 0;
4602}
4603
4604static bool mlx5_ib_modify_qp_allowed(struct mlx5_ib_dev *dev,
4605				      struct mlx5_ib_qp *qp)
4606{
4607	if (dev->profile != &raw_eth_profile)
4608		return true;
4609
4610	if (qp->type == IB_QPT_RAW_PACKET || qp->type == MLX5_IB_QPT_REG_UMR)
4611		return true;
4612
4613	/* Internal QP used for wc testing, with NOPs in wq */
4614	if (qp->flags & MLX5_IB_QP_CREATE_WC_TEST)
4615		return true;
4616
4617	return false;
4618}
4619
4620static int validate_rd_atomic(struct mlx5_ib_dev *dev, struct ib_qp_attr *attr,
4621			      int attr_mask, enum ib_qp_type qp_type)
4622{
4623	int log_max_ra_res;
4624	int log_max_ra_req;
4625
4626	if (qp_type == MLX5_IB_QPT_DCI) {
4627		log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
4628						   log_max_ra_res_dc);
4629		log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
4630						   log_max_ra_req_dc);
4631	} else {
4632		log_max_ra_res = 1 << MLX5_CAP_GEN(dev->mdev,
4633						   log_max_ra_res_qp);
4634		log_max_ra_req = 1 << MLX5_CAP_GEN(dev->mdev,
4635						   log_max_ra_req_qp);
4636	}
4637
4638	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
4639	    attr->max_rd_atomic > log_max_ra_res) {
4640		mlx5_ib_dbg(dev, "invalid max_rd_atomic value %d\n",
4641			    attr->max_rd_atomic);
4642		return false;
4643	}
4644
4645	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
4646	    attr->max_dest_rd_atomic > log_max_ra_req) {
4647		mlx5_ib_dbg(dev, "invalid max_dest_rd_atomic value %d\n",
4648			    attr->max_dest_rd_atomic);
4649		return false;
4650	}
4651	return true;
4652}
4653
4654int mlx5_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
4655		      int attr_mask, struct ib_udata *udata)
4656{
4657	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
4658	struct mlx5_ib_modify_qp_resp resp = {};
4659	struct mlx5_ib_qp *qp = to_mqp(ibqp);
4660	struct mlx5_ib_modify_qp ucmd = {};
4661	enum ib_qp_type qp_type;
4662	enum ib_qp_state cur_state, new_state;
4663	int err = -EINVAL;
4664
4665	if (!mlx5_ib_modify_qp_allowed(dev, qp))
4666		return -EOPNOTSUPP;
4667
4668	if (attr_mask & ~(IB_QP_ATTR_STANDARD_BITS | IB_QP_RATE_LIMIT))
4669		return -EOPNOTSUPP;
4670
4671	if (ibqp->rwq_ind_tbl)
4672		return -ENOSYS;
4673
4674	if (udata && udata->inlen) {
4675		if (udata->inlen < offsetofend(typeof(ucmd), ece_options))
4676			return -EINVAL;
4677
4678		if (udata->inlen > sizeof(ucmd) &&
4679		    !ib_is_udata_cleared(udata, sizeof(ucmd),
4680					 udata->inlen - sizeof(ucmd)))
4681			return -EOPNOTSUPP;
4682
4683		if (ib_copy_from_udata(&ucmd, udata,
4684				       min(udata->inlen, sizeof(ucmd))))
4685			return -EFAULT;
4686
4687		if (ucmd.comp_mask ||
4688		    memchr_inv(&ucmd.burst_info.reserved, 0,
4689			       sizeof(ucmd.burst_info.reserved)))
4690			return -EOPNOTSUPP;
4691
4692	}
4693
4694	if (qp->type == IB_QPT_GSI)
4695		return mlx5_ib_gsi_modify_qp(ibqp, attr, attr_mask);
4696
4697	qp_type = (qp->type == MLX5_IB_QPT_HW_GSI) ? IB_QPT_GSI : qp->type;
4698
4699	if (qp_type == MLX5_IB_QPT_DCT)
4700		return mlx5_ib_modify_dct(ibqp, attr, attr_mask, &ucmd, udata);
4701
4702	mutex_lock(&qp->mutex);
4703
4704	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
4705	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
4706
4707	if (qp->flags & IB_QP_CREATE_SOURCE_QPN) {
4708		if (attr_mask & ~(IB_QP_STATE | IB_QP_CUR_STATE)) {
4709			mlx5_ib_dbg(dev, "invalid attr_mask 0x%x when underlay QP is used\n",
4710				    attr_mask);
4711			goto out;
4712		}
4713	} else if (qp_type != MLX5_IB_QPT_REG_UMR &&
4714		   qp_type != MLX5_IB_QPT_DCI &&
4715		   !ib_modify_qp_is_ok(cur_state, new_state, qp_type,
4716				       attr_mask)) {
4717		mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4718			    cur_state, new_state, qp->type, attr_mask);
4719		goto out;
4720	} else if (qp_type == MLX5_IB_QPT_DCI &&
4721		   !modify_dci_qp_is_ok(cur_state, new_state, attr_mask)) {
4722		mlx5_ib_dbg(dev, "invalid QP state transition from %d to %d, qp_type %d, attr_mask 0x%x\n",
4723			    cur_state, new_state, qp_type, attr_mask);
4724		goto out;
4725	}
4726
4727	if ((attr_mask & IB_QP_PORT) &&
4728	    (attr->port_num == 0 ||
4729	     attr->port_num > dev->num_ports)) {
4730		mlx5_ib_dbg(dev, "invalid port number %d. number of ports is %d\n",
4731			    attr->port_num, dev->num_ports);
4732		goto out;
4733	}
4734
4735	if ((attr_mask & IB_QP_PKEY_INDEX) &&
4736	    attr->pkey_index >= dev->pkey_table_len) {
4737		mlx5_ib_dbg(dev, "invalid pkey index %d\n", attr->pkey_index);
4738		goto out;
4739	}
4740
4741	if (!validate_rd_atomic(dev, attr, attr_mask, qp_type))
4742		goto out;
4743
4744	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
4745		err = 0;
4746		goto out;
4747	}
4748
4749	err = __mlx5_ib_modify_qp(ibqp, attr, attr_mask, cur_state,
4750				  new_state, &ucmd, &resp, udata);
4751
4752	/* resp.response_length is set in ECE supported flows only */
4753	if (!err && resp.response_length &&
4754	    udata->outlen >= resp.response_length)
4755		/* Return -EFAULT to the user and expect him to destroy QP. */
4756		err = ib_copy_to_udata(udata, &resp, resp.response_length);
4757
4758out:
4759	mutex_unlock(&qp->mutex);
4760	return err;
4761}
4762
4763static inline enum ib_qp_state to_ib_qp_state(enum mlx5_qp_state mlx5_state)
4764{
4765	switch (mlx5_state) {
4766	case MLX5_QP_STATE_RST:      return IB_QPS_RESET;
4767	case MLX5_QP_STATE_INIT:     return IB_QPS_INIT;
4768	case MLX5_QP_STATE_RTR:      return IB_QPS_RTR;
4769	case MLX5_QP_STATE_RTS:      return IB_QPS_RTS;
4770	case MLX5_QP_STATE_SQ_DRAINING:
4771	case MLX5_QP_STATE_SQD:      return IB_QPS_SQD;
4772	case MLX5_QP_STATE_SQER:     return IB_QPS_SQE;
4773	case MLX5_QP_STATE_ERR:      return IB_QPS_ERR;
4774	default:		     return -1;
4775	}
4776}
4777
4778static inline enum ib_mig_state to_ib_mig_state(int mlx5_mig_state)
4779{
4780	switch (mlx5_mig_state) {
4781	case MLX5_QP_PM_ARMED:		return IB_MIG_ARMED;
4782	case MLX5_QP_PM_REARM:		return IB_MIG_REARM;
4783	case MLX5_QP_PM_MIGRATED:	return IB_MIG_MIGRATED;
4784	default: return -1;
4785	}
4786}
4787
4788static void to_rdma_ah_attr(struct mlx5_ib_dev *ibdev,
4789			    struct rdma_ah_attr *ah_attr, void *path)
4790{
4791	int port = MLX5_GET(ads, path, vhca_port_num);
4792	int static_rate;
4793
4794	memset(ah_attr, 0, sizeof(*ah_attr));
4795
4796	if (!port || port > ibdev->num_ports)
4797		return;
4798
4799	ah_attr->type = rdma_ah_find_type(&ibdev->ib_dev, port);
4800
4801	rdma_ah_set_port_num(ah_attr, port);
4802	rdma_ah_set_sl(ah_attr, MLX5_GET(ads, path, sl));
4803
4804	rdma_ah_set_dlid(ah_attr, MLX5_GET(ads, path, rlid));
4805	rdma_ah_set_path_bits(ah_attr, MLX5_GET(ads, path, mlid));
4806
4807	static_rate = MLX5_GET(ads, path, stat_rate);
4808	rdma_ah_set_static_rate(ah_attr, mlx5_to_ib_rate_map(static_rate));
4809	if (MLX5_GET(ads, path, grh) ||
4810	    ah_attr->type == RDMA_AH_ATTR_TYPE_ROCE) {
4811		rdma_ah_set_grh(ah_attr, NULL, MLX5_GET(ads, path, flow_label),
4812				MLX5_GET(ads, path, src_addr_index),
4813				MLX5_GET(ads, path, hop_limit),
4814				MLX5_GET(ads, path, tclass));
4815		rdma_ah_set_dgid_raw(ah_attr, MLX5_ADDR_OF(ads, path, rgid_rip));
4816	}
4817}
4818
4819static int query_raw_packet_qp_sq_state(struct mlx5_ib_dev *dev,
4820					struct mlx5_ib_sq *sq,
4821					u8 *sq_state)
4822{
4823	int err;
4824
4825	err = mlx5_core_query_sq_state(dev->mdev, sq->base.mqp.qpn, sq_state);
4826	if (err)
4827		goto out;
4828	sq->state = *sq_state;
4829
4830out:
4831	return err;
4832}
4833
4834static int query_raw_packet_qp_rq_state(struct mlx5_ib_dev *dev,
4835					struct mlx5_ib_rq *rq,
4836					u8 *rq_state)
4837{
4838	void *out;
4839	void *rqc;
4840	int inlen;
4841	int err;
4842
4843	inlen = MLX5_ST_SZ_BYTES(query_rq_out);
4844	out = kvzalloc(inlen, GFP_KERNEL);
4845	if (!out)
4846		return -ENOMEM;
4847
4848	err = mlx5_core_query_rq(dev->mdev, rq->base.mqp.qpn, out);
4849	if (err)
4850		goto out;
4851
4852	rqc = MLX5_ADDR_OF(query_rq_out, out, rq_context);
4853	*rq_state = MLX5_GET(rqc, rqc, state);
4854	rq->state = *rq_state;
4855
4856out:
4857	kvfree(out);
4858	return err;
4859}
4860
4861static int sqrq_state_to_qp_state(u8 sq_state, u8 rq_state,
4862				  struct mlx5_ib_qp *qp, u8 *qp_state)
4863{
4864	static const u8 sqrq_trans[MLX5_RQ_NUM_STATE][MLX5_SQ_NUM_STATE] = {
4865		[MLX5_RQC_STATE_RST] = {
4866			[MLX5_SQC_STATE_RST]	= IB_QPS_RESET,
4867			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE_BAD,
4868			[MLX5_SQC_STATE_ERR]	= MLX5_QP_STATE_BAD,
4869			[MLX5_SQ_STATE_NA]	= IB_QPS_RESET,
4870		},
4871		[MLX5_RQC_STATE_RDY] = {
4872			[MLX5_SQC_STATE_RST]	= MLX5_QP_STATE,
4873			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE,
4874			[MLX5_SQC_STATE_ERR]	= IB_QPS_SQE,
4875			[MLX5_SQ_STATE_NA]	= MLX5_QP_STATE,
4876		},
4877		[MLX5_RQC_STATE_ERR] = {
4878			[MLX5_SQC_STATE_RST]    = MLX5_QP_STATE_BAD,
4879			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE_BAD,
4880			[MLX5_SQC_STATE_ERR]	= IB_QPS_ERR,
4881			[MLX5_SQ_STATE_NA]	= IB_QPS_ERR,
4882		},
4883		[MLX5_RQ_STATE_NA] = {
4884			[MLX5_SQC_STATE_RST]    = MLX5_QP_STATE,
4885			[MLX5_SQC_STATE_RDY]	= MLX5_QP_STATE,
4886			[MLX5_SQC_STATE_ERR]	= MLX5_QP_STATE,
4887			[MLX5_SQ_STATE_NA]	= MLX5_QP_STATE_BAD,
4888		},
4889	};
4890
4891	*qp_state = sqrq_trans[rq_state][sq_state];
4892
4893	if (*qp_state == MLX5_QP_STATE_BAD) {
4894		WARN(1, "Buggy Raw Packet QP state, SQ 0x%x state: 0x%x, RQ 0x%x state: 0x%x",
4895		     qp->raw_packet_qp.sq.base.mqp.qpn, sq_state,
4896		     qp->raw_packet_qp.rq.base.mqp.qpn, rq_state);
4897		return -EINVAL;
4898	}
4899
4900	if (*qp_state == MLX5_QP_STATE)
4901		*qp_state = qp->state;
4902
4903	return 0;
4904}
4905
4906static int query_raw_packet_qp_state(struct mlx5_ib_dev *dev,
4907				     struct mlx5_ib_qp *qp,
4908				     u8 *raw_packet_qp_state)
4909{
4910	struct mlx5_ib_raw_packet_qp *raw_packet_qp = &qp->raw_packet_qp;
4911	struct mlx5_ib_sq *sq = &raw_packet_qp->sq;
4912	struct mlx5_ib_rq *rq = &raw_packet_qp->rq;
4913	int err;
4914	u8 sq_state = MLX5_SQ_STATE_NA;
4915	u8 rq_state = MLX5_RQ_STATE_NA;
4916
4917	if (qp->sq.wqe_cnt) {
4918		err = query_raw_packet_qp_sq_state(dev, sq, &sq_state);
4919		if (err)
4920			return err;
4921	}
4922
4923	if (qp->rq.wqe_cnt) {
4924		err = query_raw_packet_qp_rq_state(dev, rq, &rq_state);
4925		if (err)
4926			return err;
4927	}
4928
4929	return sqrq_state_to_qp_state(sq_state, rq_state, qp,
4930				      raw_packet_qp_state);
4931}
4932
4933static int query_qp_attr(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *qp,
4934			 struct ib_qp_attr *qp_attr)
4935{
4936	int outlen = MLX5_ST_SZ_BYTES(query_qp_out);
4937	void *qpc, *pri_path, *alt_path;
4938	u32 *outb;
4939	int err;
4940
4941	outb = kzalloc(outlen, GFP_KERNEL);
4942	if (!outb)
4943		return -ENOMEM;
4944
4945	err = mlx5_core_qp_query(dev, &qp->trans_qp.base.mqp, outb, outlen,
4946				 false);
4947	if (err)
4948		goto out;
4949
4950	qpc = MLX5_ADDR_OF(query_qp_out, outb, qpc);
4951
4952	qp->state = to_ib_qp_state(MLX5_GET(qpc, qpc, state));
4953	if (MLX5_GET(qpc, qpc, state) == MLX5_QP_STATE_SQ_DRAINING)
4954		qp_attr->sq_draining = 1;
4955
4956	qp_attr->path_mtu = MLX5_GET(qpc, qpc, mtu);
4957	qp_attr->path_mig_state = to_ib_mig_state(MLX5_GET(qpc, qpc, pm_state));
4958	qp_attr->qkey = MLX5_GET(qpc, qpc, q_key);
4959	qp_attr->rq_psn = MLX5_GET(qpc, qpc, next_rcv_psn);
4960	qp_attr->sq_psn = MLX5_GET(qpc, qpc, next_send_psn);
4961	qp_attr->dest_qp_num = MLX5_GET(qpc, qpc, remote_qpn);
4962
4963	if (MLX5_GET(qpc, qpc, rre))
4964		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_READ;
4965	if (MLX5_GET(qpc, qpc, rwe))
4966		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_WRITE;
4967	if (MLX5_GET(qpc, qpc, rae))
4968		qp_attr->qp_access_flags |= IB_ACCESS_REMOTE_ATOMIC;
4969
4970	qp_attr->max_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_sra_max);
4971	qp_attr->max_dest_rd_atomic = 1 << MLX5_GET(qpc, qpc, log_rra_max);
4972	qp_attr->min_rnr_timer = MLX5_GET(qpc, qpc, min_rnr_nak);
4973	qp_attr->retry_cnt = MLX5_GET(qpc, qpc, retry_count);
4974	qp_attr->rnr_retry = MLX5_GET(qpc, qpc, rnr_retry);
4975
4976	pri_path = MLX5_ADDR_OF(qpc, qpc, primary_address_path);
4977	alt_path = MLX5_ADDR_OF(qpc, qpc, secondary_address_path);
4978
4979	if (qp->type == IB_QPT_RC || qp->type == IB_QPT_UC ||
4980	    qp->type == IB_QPT_XRC_INI || qp->type == IB_QPT_XRC_TGT) {
4981		to_rdma_ah_attr(dev, &qp_attr->ah_attr, pri_path);
4982		to_rdma_ah_attr(dev, &qp_attr->alt_ah_attr, alt_path);
4983		qp_attr->alt_pkey_index = MLX5_GET(ads, alt_path, pkey_index);
4984		qp_attr->alt_port_num = MLX5_GET(ads, alt_path, vhca_port_num);
4985	}
4986
4987	qp_attr->pkey_index = MLX5_GET(ads, pri_path, pkey_index);
4988	qp_attr->port_num = MLX5_GET(ads, pri_path, vhca_port_num);
4989	qp_attr->timeout = MLX5_GET(ads, pri_path, ack_timeout);
4990	qp_attr->alt_timeout = MLX5_GET(ads, alt_path, ack_timeout);
4991
4992out:
4993	kfree(outb);
4994	return err;
4995}
4996
4997static int mlx5_ib_dct_query_qp(struct mlx5_ib_dev *dev, struct mlx5_ib_qp *mqp,
4998				struct ib_qp_attr *qp_attr, int qp_attr_mask,
4999				struct ib_qp_init_attr *qp_init_attr)
5000{
5001	struct mlx5_core_dct	*dct = &mqp->dct.mdct;
5002	u32 *out;
5003	u32 access_flags = 0;
5004	int outlen = MLX5_ST_SZ_BYTES(query_dct_out);
5005	void *dctc;
5006	int err;
5007	int supported_mask = IB_QP_STATE |
5008			     IB_QP_ACCESS_FLAGS |
5009			     IB_QP_PORT |
5010			     IB_QP_MIN_RNR_TIMER |
5011			     IB_QP_AV |
5012			     IB_QP_PATH_MTU |
5013			     IB_QP_PKEY_INDEX;
5014
5015	if (qp_attr_mask & ~supported_mask)
5016		return -EINVAL;
5017	if (mqp->state != IB_QPS_RTR)
5018		return -EINVAL;
5019
5020	out = kzalloc(outlen, GFP_KERNEL);
5021	if (!out)
5022		return -ENOMEM;
5023
5024	err = mlx5_core_dct_query(dev, dct, out, outlen);
5025	if (err)
5026		goto out;
5027
5028	dctc = MLX5_ADDR_OF(query_dct_out, out, dct_context_entry);
5029
5030	if (qp_attr_mask & IB_QP_STATE)
5031		qp_attr->qp_state = IB_QPS_RTR;
5032
5033	if (qp_attr_mask & IB_QP_ACCESS_FLAGS) {
5034		if (MLX5_GET(dctc, dctc, rre))
5035			access_flags |= IB_ACCESS_REMOTE_READ;
5036		if (MLX5_GET(dctc, dctc, rwe))
5037			access_flags |= IB_ACCESS_REMOTE_WRITE;
5038		if (MLX5_GET(dctc, dctc, rae))
5039			access_flags |= IB_ACCESS_REMOTE_ATOMIC;
5040		qp_attr->qp_access_flags = access_flags;
5041	}
5042
5043	if (qp_attr_mask & IB_QP_PORT)
5044		qp_attr->port_num = MLX5_GET(dctc, dctc, port);
5045	if (qp_attr_mask & IB_QP_MIN_RNR_TIMER)
5046		qp_attr->min_rnr_timer = MLX5_GET(dctc, dctc, min_rnr_nak);
5047	if (qp_attr_mask & IB_QP_AV) {
5048		qp_attr->ah_attr.grh.traffic_class = MLX5_GET(dctc, dctc, tclass);
5049		qp_attr->ah_attr.grh.flow_label = MLX5_GET(dctc, dctc, flow_label);
5050		qp_attr->ah_attr.grh.sgid_index = MLX5_GET(dctc, dctc, my_addr_index);
5051		qp_attr->ah_attr.grh.hop_limit = MLX5_GET(dctc, dctc, hop_limit);
5052	}
5053	if (qp_attr_mask & IB_QP_PATH_MTU)
5054		qp_attr->path_mtu = MLX5_GET(dctc, dctc, mtu);
5055	if (qp_attr_mask & IB_QP_PKEY_INDEX)
5056		qp_attr->pkey_index = MLX5_GET(dctc, dctc, pkey_index);
5057out:
5058	kfree(out);
5059	return err;
5060}
5061
5062int mlx5_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr,
5063		     int qp_attr_mask, struct ib_qp_init_attr *qp_init_attr)
5064{
5065	struct mlx5_ib_dev *dev = to_mdev(ibqp->device);
5066	struct mlx5_ib_qp *qp = to_mqp(ibqp);
5067	int err = 0;
5068	u8 raw_packet_qp_state;
5069
5070	if (ibqp->rwq_ind_tbl)
5071		return -ENOSYS;
5072
5073	if (qp->type == IB_QPT_GSI)
5074		return mlx5_ib_gsi_query_qp(ibqp, qp_attr, qp_attr_mask,
5075					    qp_init_attr);
5076
5077	/* Not all of output fields are applicable, make sure to zero them */
5078	memset(qp_init_attr, 0, sizeof(*qp_init_attr));
5079	memset(qp_attr, 0, sizeof(*qp_attr));
5080
5081	if (unlikely(qp->type == MLX5_IB_QPT_DCT))
5082		return mlx5_ib_dct_query_qp(dev, qp, qp_attr,
5083					    qp_attr_mask, qp_init_attr);
5084
5085	mutex_lock(&qp->mutex);
5086
5087	if (qp->type == IB_QPT_RAW_PACKET ||
5088	    qp->flags & IB_QP_CREATE_SOURCE_QPN) {
5089		err = query_raw_packet_qp_state(dev, qp, &raw_packet_qp_state);
5090		if (err)
5091			goto out;
5092		qp->state = raw_packet_qp_state;
5093		qp_attr->port_num = 1;
5094	} else {
5095		err = query_qp_attr(dev, qp, qp_attr);
5096		if (err)
5097			goto out;
5098	}
5099
5100	qp_attr->qp_state	     = qp->state;
5101	qp_attr->cur_qp_state	     = qp_attr->qp_state;
5102	qp_attr->cap.max_recv_wr     = qp->rq.wqe_cnt;
5103	qp_attr->cap.max_recv_sge    = qp->rq.max_gs;
5104
5105	if (!ibqp->uobject) {
5106		qp_attr->cap.max_send_wr  = qp->sq.max_post;
5107		qp_attr->cap.max_send_sge = qp->sq.max_gs;
5108		qp_init_attr->qp_context = ibqp->qp_context;
5109	} else {
5110		qp_attr->cap.max_send_wr  = 0;
5111		qp_attr->cap.max_send_sge = 0;
5112	}
5113
5114	qp_init_attr->qp_type = qp->type;
5115	qp_init_attr->recv_cq = ibqp->recv_cq;
5116	qp_init_attr->send_cq = ibqp->send_cq;
5117	qp_init_attr->srq = ibqp->srq;
5118	qp_attr->cap.max_inline_data = qp->max_inline_data;
5119
5120	qp_init_attr->cap	     = qp_attr->cap;
5121
5122	qp_init_attr->create_flags = qp->flags;
5123
5124	qp_init_attr->sq_sig_type = qp->sq_signal_bits & MLX5_WQE_CTRL_CQ_UPDATE ?
5125		IB_SIGNAL_ALL_WR : IB_SIGNAL_REQ_WR;
5126
5127out:
5128	mutex_unlock(&qp->mutex);
5129	return err;
5130}
5131
5132int mlx5_ib_alloc_xrcd(struct ib_xrcd *ibxrcd, struct ib_udata *udata)
5133{
5134	struct mlx5_ib_dev *dev = to_mdev(ibxrcd->device);
5135	struct mlx5_ib_xrcd *xrcd = to_mxrcd(ibxrcd);
5136
5137	if (!MLX5_CAP_GEN(dev->mdev, xrc))
5138		return -EOPNOTSUPP;
5139
5140	return mlx5_cmd_xrcd_alloc(dev->mdev, &xrcd->xrcdn, 0);
5141}
5142
5143int mlx5_ib_dealloc_xrcd(struct ib_xrcd *xrcd, struct ib_udata *udata)
5144{
5145	struct mlx5_ib_dev *dev = to_mdev(xrcd->device);
5146	u32 xrcdn = to_mxrcd(xrcd)->xrcdn;
5147
5148	return mlx5_cmd_xrcd_dealloc(dev->mdev, xrcdn, 0);
5149}
5150
5151static void mlx5_ib_wq_event(struct mlx5_core_qp *core_qp, int type)
5152{
5153	struct mlx5_ib_rwq *rwq = to_mibrwq(core_qp);
5154	struct mlx5_ib_dev *dev = to_mdev(rwq->ibwq.device);
5155	struct ib_event event;
5156
5157	if (rwq->ibwq.event_handler) {
5158		event.device     = rwq->ibwq.device;
5159		event.element.wq = &rwq->ibwq;
5160		switch (type) {
5161		case MLX5_EVENT_TYPE_WQ_CATAS_ERROR:
5162			event.event = IB_EVENT_WQ_FATAL;
5163			break;
5164		default:
5165			mlx5_ib_warn(dev, "Unexpected event type %d on WQ %06x\n", type, core_qp->qpn);
5166			return;
5167		}
5168
5169		rwq->ibwq.event_handler(&event, rwq->ibwq.wq_context);
5170	}
5171}
5172
5173static int set_delay_drop(struct mlx5_ib_dev *dev)
5174{
5175	int err = 0;
5176
5177	mutex_lock(&dev->delay_drop.lock);
5178	if (dev->delay_drop.activate)
5179		goto out;
5180
5181	err = mlx5_core_set_delay_drop(dev, dev->delay_drop.timeout);
5182	if (err)
5183		goto out;
5184
5185	dev->delay_drop.activate = true;
5186out:
5187	mutex_unlock(&dev->delay_drop.lock);
5188
5189	if (!err)
5190		atomic_inc(&dev->delay_drop.rqs_cnt);
5191	return err;
5192}
5193
5194static int  create_rq(struct mlx5_ib_rwq *rwq, struct ib_pd *pd,
5195		      struct ib_wq_init_attr *init_attr)
5196{
5197	struct mlx5_ib_dev *dev;
5198	int has_net_offloads;
5199	__be64 *rq_pas0;
5200	int ts_format;
5201	void *in;
5202	void *rqc;
5203	void *wq;
5204	int inlen;
5205	int err;
5206
5207	dev = to_mdev(pd->device);
5208
5209	ts_format = get_rq_ts_format(dev, to_mcq(init_attr->cq));
5210	if (ts_format < 0)
5211		return ts_format;
5212
5213	inlen = MLX5_ST_SZ_BYTES(create_rq_in) + sizeof(u64) * rwq->rq_num_pas;
5214	in = kvzalloc(inlen, GFP_KERNEL);
5215	if (!in)
5216		return -ENOMEM;
5217
5218	MLX5_SET(create_rq_in, in, uid, to_mpd(pd)->uid);
5219	rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
5220	MLX5_SET(rqc,  rqc, mem_rq_type,
5221		 MLX5_RQC_MEM_RQ_TYPE_MEMORY_RQ_INLINE);
5222	MLX5_SET(rqc, rqc, ts_format, ts_format);
5223	MLX5_SET(rqc, rqc, user_index, rwq->user_index);
5224	MLX5_SET(rqc,  rqc, cqn, to_mcq(init_attr->cq)->mcq.cqn);
5225	MLX5_SET(rqc,  rqc, state, MLX5_RQC_STATE_RST);
5226	MLX5_SET(rqc,  rqc, flush_in_error_en, 1);
5227	wq = MLX5_ADDR_OF(rqc, rqc, wq);
5228	MLX5_SET(wq, wq, wq_type,
5229		 rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ ?
5230		 MLX5_WQ_TYPE_CYCLIC_STRIDING_RQ : MLX5_WQ_TYPE_CYCLIC);
5231	if (init_attr->create_flags & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5232		if (!MLX5_CAP_GEN(dev->mdev, end_pad)) {
5233			mlx5_ib_dbg(dev, "Scatter end padding is not supported\n");
5234			err = -EOPNOTSUPP;
5235			goto out;
5236		} else {
5237			MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN);
5238		}
5239	}
5240	MLX5_SET(wq, wq, log_wq_stride, rwq->log_rq_stride);
5241	if (rwq->create_flags & MLX5_IB_WQ_FLAGS_STRIDING_RQ) {
5242		/*
5243		 * In Firmware number of strides in each WQE is:
5244		 *   "512 * 2^single_wqe_log_num_of_strides"
5245		 * Values 3 to 8 are accepted as 10 to 15, 9 to 18 are
5246		 * accepted as 0 to 9
5247		 */
5248		static const u8 fw_map[] = { 10, 11, 12, 13, 14, 15, 0, 1,
5249					     2,  3,  4,  5,  6,  7,  8, 9 };
5250		MLX5_SET(wq, wq, two_byte_shift_en, rwq->two_byte_shift_en);
5251		MLX5_SET(wq, wq, log_wqe_stride_size,
5252			 rwq->single_stride_log_num_of_bytes -
5253			 MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES);
5254		MLX5_SET(wq, wq, log_wqe_num_of_strides,
5255			 fw_map[rwq->log_num_strides -
5256				MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES]);
5257	}
5258	MLX5_SET(wq, wq, log_wq_sz, rwq->log_rq_size);
5259	MLX5_SET(wq, wq, pd, to_mpd(pd)->pdn);
5260	MLX5_SET(wq, wq, page_offset, rwq->rq_page_offset);
5261	MLX5_SET(wq, wq, log_wq_pg_sz, rwq->log_page_size);
5262	MLX5_SET(wq, wq, wq_signature, rwq->wq_sig);
5263	MLX5_SET64(wq, wq, dbr_addr, rwq->db.dma);
5264	has_net_offloads = MLX5_CAP_GEN(dev->mdev, eth_net_offloads);
5265	if (init_attr->create_flags & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5266		if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5267			mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5268			err = -EOPNOTSUPP;
5269			goto out;
5270		}
5271	} else {
5272		MLX5_SET(rqc, rqc, vsd, 1);
5273	}
5274	if (init_attr->create_flags & IB_WQ_FLAGS_SCATTER_FCS) {
5275		if (!(has_net_offloads && MLX5_CAP_ETH(dev->mdev, scatter_fcs))) {
5276			mlx5_ib_dbg(dev, "Scatter FCS is not supported\n");
5277			err = -EOPNOTSUPP;
5278			goto out;
5279		}
5280		MLX5_SET(rqc, rqc, scatter_fcs, 1);
5281	}
5282	if (init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5283		if (!(dev->ib_dev.attrs.raw_packet_caps &
5284		      IB_RAW_PACKET_CAP_DELAY_DROP)) {
5285			mlx5_ib_dbg(dev, "Delay drop is not supported\n");
5286			err = -EOPNOTSUPP;
5287			goto out;
5288		}
5289		MLX5_SET(rqc, rqc, delay_drop_en, 1);
5290	}
5291	rq_pas0 = (__be64 *)MLX5_ADDR_OF(wq, wq, pas);
5292	mlx5_ib_populate_pas(rwq->umem, 1UL << rwq->page_shift, rq_pas0, 0);
5293	err = mlx5_core_create_rq_tracked(dev, in, inlen, &rwq->core_qp);
5294	if (!err && init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP) {
5295		err = set_delay_drop(dev);
5296		if (err) {
5297			mlx5_ib_warn(dev, "Failed to enable delay drop err=%d\n",
5298				     err);
5299			mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5300		} else {
5301			rwq->create_flags |= MLX5_IB_WQ_FLAGS_DELAY_DROP;
5302		}
5303	}
5304out:
5305	kvfree(in);
5306	return err;
5307}
5308
5309static int set_user_rq_size(struct mlx5_ib_dev *dev,
5310			    struct ib_wq_init_attr *wq_init_attr,
5311			    struct mlx5_ib_create_wq *ucmd,
5312			    struct mlx5_ib_rwq *rwq)
5313{
5314	/* Sanity check RQ size before proceeding */
5315	if (wq_init_attr->max_wr > (1 << MLX5_CAP_GEN(dev->mdev, log_max_wq_sz)))
5316		return -EINVAL;
5317
5318	if (!ucmd->rq_wqe_count)
5319		return -EINVAL;
5320
5321	rwq->wqe_count = ucmd->rq_wqe_count;
5322	rwq->wqe_shift = ucmd->rq_wqe_shift;
5323	if (check_shl_overflow(rwq->wqe_count, rwq->wqe_shift, &rwq->buf_size))
5324		return -EINVAL;
5325
5326	rwq->log_rq_stride = rwq->wqe_shift;
5327	rwq->log_rq_size = ilog2(rwq->wqe_count);
5328	return 0;
5329}
5330
5331static bool log_of_strides_valid(struct mlx5_ib_dev *dev, u32 log_num_strides)
5332{
5333	if ((log_num_strides > MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES) ||
5334	    (log_num_strides < MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
5335		return false;
5336
5337	if (!MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) &&
5338	    (log_num_strides < MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES))
5339		return false;
5340
5341	return true;
5342}
5343
5344static int prepare_user_rq(struct ib_pd *pd,
5345			   struct ib_wq_init_attr *init_attr,
5346			   struct ib_udata *udata,
5347			   struct mlx5_ib_rwq *rwq)
5348{
5349	struct mlx5_ib_dev *dev = to_mdev(pd->device);
5350	struct mlx5_ib_create_wq ucmd = {};
5351	int err;
5352	size_t required_cmd_sz;
5353
5354	required_cmd_sz = offsetofend(struct mlx5_ib_create_wq,
5355				      single_stride_log_num_of_bytes);
5356	if (udata->inlen < required_cmd_sz) {
5357		mlx5_ib_dbg(dev, "invalid inlen\n");
5358		return -EINVAL;
5359	}
5360
5361	if (udata->inlen > sizeof(ucmd) &&
5362	    !ib_is_udata_cleared(udata, sizeof(ucmd),
5363				 udata->inlen - sizeof(ucmd))) {
5364		mlx5_ib_dbg(dev, "inlen is not supported\n");
5365		return -EOPNOTSUPP;
5366	}
5367
5368	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen))) {
5369		mlx5_ib_dbg(dev, "copy failed\n");
5370		return -EFAULT;
5371	}
5372
5373	if (ucmd.comp_mask & (~MLX5_IB_CREATE_WQ_STRIDING_RQ)) {
5374		mlx5_ib_dbg(dev, "invalid comp mask\n");
5375		return -EOPNOTSUPP;
5376	} else if (ucmd.comp_mask & MLX5_IB_CREATE_WQ_STRIDING_RQ) {
5377		if (!MLX5_CAP_GEN(dev->mdev, striding_rq)) {
5378			mlx5_ib_dbg(dev, "Striding RQ is not supported\n");
5379			return -EOPNOTSUPP;
5380		}
5381		if ((ucmd.single_stride_log_num_of_bytes <
5382		    MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES) ||
5383		    (ucmd.single_stride_log_num_of_bytes >
5384		     MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES)) {
5385			mlx5_ib_dbg(dev, "Invalid log stride size (%u. Range is %u - %u)\n",
5386				    ucmd.single_stride_log_num_of_bytes,
5387				    MLX5_MIN_SINGLE_STRIDE_LOG_NUM_BYTES,
5388				    MLX5_MAX_SINGLE_STRIDE_LOG_NUM_BYTES);
5389			return -EINVAL;
5390		}
5391		if (!log_of_strides_valid(dev,
5392					  ucmd.single_wqe_log_num_of_strides)) {
5393			mlx5_ib_dbg(
5394				dev,
5395				"Invalid log num strides (%u. Range is %u - %u)\n",
5396				ucmd.single_wqe_log_num_of_strides,
5397				MLX5_CAP_GEN(dev->mdev, ext_stride_num_range) ?
5398					MLX5_EXT_MIN_SINGLE_WQE_LOG_NUM_STRIDES :
5399					MLX5_MIN_SINGLE_WQE_LOG_NUM_STRIDES,
5400				MLX5_MAX_SINGLE_WQE_LOG_NUM_STRIDES);
5401			return -EINVAL;
5402		}
5403		rwq->single_stride_log_num_of_bytes =
5404			ucmd.single_stride_log_num_of_bytes;
5405		rwq->log_num_strides = ucmd.single_wqe_log_num_of_strides;
5406		rwq->two_byte_shift_en = !!ucmd.two_byte_shift_en;
5407		rwq->create_flags |= MLX5_IB_WQ_FLAGS_STRIDING_RQ;
5408	}
5409
5410	err = set_user_rq_size(dev, init_attr, &ucmd, rwq);
5411	if (err) {
5412		mlx5_ib_dbg(dev, "err %d\n", err);
5413		return err;
5414	}
5415
5416	err = create_user_rq(dev, pd, udata, rwq, &ucmd);
5417	if (err) {
5418		mlx5_ib_dbg(dev, "err %d\n", err);
5419		return err;
5420	}
5421
5422	rwq->user_index = ucmd.user_index;
5423	return 0;
5424}
5425
5426struct ib_wq *mlx5_ib_create_wq(struct ib_pd *pd,
5427				struct ib_wq_init_attr *init_attr,
5428				struct ib_udata *udata)
5429{
5430	struct mlx5_ib_dev *dev;
5431	struct mlx5_ib_rwq *rwq;
5432	struct mlx5_ib_create_wq_resp resp = {};
5433	size_t min_resp_len;
5434	int err;
5435
5436	if (!udata)
5437		return ERR_PTR(-ENOSYS);
5438
5439	min_resp_len = offsetofend(struct mlx5_ib_create_wq_resp, reserved);
5440	if (udata->outlen && udata->outlen < min_resp_len)
5441		return ERR_PTR(-EINVAL);
5442
5443	if (!capable(CAP_SYS_RAWIO) &&
5444	    init_attr->create_flags & IB_WQ_FLAGS_DELAY_DROP)
5445		return ERR_PTR(-EPERM);
5446
5447	dev = to_mdev(pd->device);
5448	switch (init_attr->wq_type) {
5449	case IB_WQT_RQ:
5450		rwq = kzalloc(sizeof(*rwq), GFP_KERNEL);
5451		if (!rwq)
5452			return ERR_PTR(-ENOMEM);
5453		err = prepare_user_rq(pd, init_attr, udata, rwq);
5454		if (err)
5455			goto err;
5456		err = create_rq(rwq, pd, init_attr);
5457		if (err)
5458			goto err_user_rq;
5459		break;
5460	default:
5461		mlx5_ib_dbg(dev, "unsupported wq type %d\n",
5462			    init_attr->wq_type);
5463		return ERR_PTR(-EINVAL);
5464	}
5465
5466	rwq->ibwq.wq_num = rwq->core_qp.qpn;
5467	rwq->ibwq.state = IB_WQS_RESET;
5468	if (udata->outlen) {
5469		resp.response_length = offsetofend(
5470			struct mlx5_ib_create_wq_resp, response_length);
5471		err = ib_copy_to_udata(udata, &resp, resp.response_length);
5472		if (err)
5473			goto err_copy;
5474	}
5475
5476	rwq->core_qp.event = mlx5_ib_wq_event;
5477	rwq->ibwq.event_handler = init_attr->event_handler;
5478	return &rwq->ibwq;
5479
5480err_copy:
5481	mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5482err_user_rq:
5483	destroy_user_rq(dev, pd, rwq, udata);
5484err:
5485	kfree(rwq);
5486	return ERR_PTR(err);
5487}
5488
5489int mlx5_ib_destroy_wq(struct ib_wq *wq, struct ib_udata *udata)
5490{
5491	struct mlx5_ib_dev *dev = to_mdev(wq->device);
5492	struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5493	int ret;
5494
5495	ret = mlx5_core_destroy_rq_tracked(dev, &rwq->core_qp);
5496	if (ret)
5497		return ret;
5498	destroy_user_rq(dev, wq->pd, rwq, udata);
5499	kfree(rwq);
5500	return 0;
5501}
5502
5503int mlx5_ib_create_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_table,
5504				 struct ib_rwq_ind_table_init_attr *init_attr,
5505				 struct ib_udata *udata)
5506{
5507	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl =
5508		to_mrwq_ind_table(ib_rwq_ind_table);
5509	struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_table->device);
5510	int sz = 1 << init_attr->log_ind_tbl_size;
5511	struct mlx5_ib_create_rwq_ind_tbl_resp resp = {};
5512	size_t min_resp_len;
5513	int inlen;
5514	int err;
5515	int i;
5516	u32 *in;
5517	void *rqtc;
5518
5519	if (udata->inlen > 0 &&
5520	    !ib_is_udata_cleared(udata, 0,
5521				 udata->inlen))
5522		return -EOPNOTSUPP;
5523
5524	if (init_attr->log_ind_tbl_size >
5525	    MLX5_CAP_GEN(dev->mdev, log_max_rqt_size)) {
5526		mlx5_ib_dbg(dev, "log_ind_tbl_size = %d is bigger than supported = %d\n",
5527			    init_attr->log_ind_tbl_size,
5528			    MLX5_CAP_GEN(dev->mdev, log_max_rqt_size));
5529		return -EINVAL;
5530	}
5531
5532	min_resp_len =
5533		offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp, reserved);
5534	if (udata->outlen && udata->outlen < min_resp_len)
5535		return -EINVAL;
5536
5537	inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz;
5538	in = kvzalloc(inlen, GFP_KERNEL);
5539	if (!in)
5540		return -ENOMEM;
5541
5542	rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context);
5543
5544	MLX5_SET(rqtc, rqtc, rqt_actual_size, sz);
5545	MLX5_SET(rqtc, rqtc, rqt_max_size, sz);
5546
5547	for (i = 0; i < sz; i++)
5548		MLX5_SET(rqtc, rqtc, rq_num[i], init_attr->ind_tbl[i]->wq_num);
5549
5550	rwq_ind_tbl->uid = to_mpd(init_attr->ind_tbl[0]->pd)->uid;
5551	MLX5_SET(create_rqt_in, in, uid, rwq_ind_tbl->uid);
5552
5553	err = mlx5_core_create_rqt(dev->mdev, in, inlen, &rwq_ind_tbl->rqtn);
5554	kvfree(in);
5555	if (err)
5556		return err;
5557
5558	rwq_ind_tbl->ib_rwq_ind_tbl.ind_tbl_num = rwq_ind_tbl->rqtn;
5559	if (udata->outlen) {
5560		resp.response_length =
5561			offsetofend(struct mlx5_ib_create_rwq_ind_tbl_resp,
5562				    response_length);
5563		err = ib_copy_to_udata(udata, &resp, resp.response_length);
5564		if (err)
5565			goto err_copy;
5566	}
5567
5568	return 0;
5569
5570err_copy:
5571	mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
5572	return err;
5573}
5574
5575int mlx5_ib_destroy_rwq_ind_table(struct ib_rwq_ind_table *ib_rwq_ind_tbl)
5576{
5577	struct mlx5_ib_rwq_ind_table *rwq_ind_tbl = to_mrwq_ind_table(ib_rwq_ind_tbl);
5578	struct mlx5_ib_dev *dev = to_mdev(ib_rwq_ind_tbl->device);
5579
5580	return mlx5_cmd_destroy_rqt(dev->mdev, rwq_ind_tbl->rqtn, rwq_ind_tbl->uid);
5581}
5582
5583int mlx5_ib_modify_wq(struct ib_wq *wq, struct ib_wq_attr *wq_attr,
5584		      u32 wq_attr_mask, struct ib_udata *udata)
5585{
5586	struct mlx5_ib_dev *dev = to_mdev(wq->device);
5587	struct mlx5_ib_rwq *rwq = to_mrwq(wq);
5588	struct mlx5_ib_modify_wq ucmd = {};
5589	size_t required_cmd_sz;
5590	int curr_wq_state;
5591	int wq_state;
5592	int inlen;
5593	int err;
5594	void *rqc;
5595	void *in;
5596
5597	required_cmd_sz = offsetofend(struct mlx5_ib_modify_wq, reserved);
5598	if (udata->inlen < required_cmd_sz)
5599		return -EINVAL;
5600
5601	if (udata->inlen > sizeof(ucmd) &&
5602	    !ib_is_udata_cleared(udata, sizeof(ucmd),
5603				 udata->inlen - sizeof(ucmd)))
5604		return -EOPNOTSUPP;
5605
5606	if (ib_copy_from_udata(&ucmd, udata, min(sizeof(ucmd), udata->inlen)))
5607		return -EFAULT;
5608
5609	if (ucmd.comp_mask || ucmd.reserved)
5610		return -EOPNOTSUPP;
5611
5612	inlen = MLX5_ST_SZ_BYTES(modify_rq_in);
5613	in = kvzalloc(inlen, GFP_KERNEL);
5614	if (!in)
5615		return -ENOMEM;
5616
5617	rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx);
5618
5619	curr_wq_state = wq_attr->curr_wq_state;
5620	wq_state = wq_attr->wq_state;
5621	if (curr_wq_state == IB_WQS_ERR)
5622		curr_wq_state = MLX5_RQC_STATE_ERR;
5623	if (wq_state == IB_WQS_ERR)
5624		wq_state = MLX5_RQC_STATE_ERR;
5625	MLX5_SET(modify_rq_in, in, rq_state, curr_wq_state);
5626	MLX5_SET(modify_rq_in, in, uid, to_mpd(wq->pd)->uid);
5627	MLX5_SET(rqc, rqc, state, wq_state);
5628
5629	if (wq_attr_mask & IB_WQ_FLAGS) {
5630		if (wq_attr->flags_mask & IB_WQ_FLAGS_CVLAN_STRIPPING) {
5631			if (!(MLX5_CAP_GEN(dev->mdev, eth_net_offloads) &&
5632			      MLX5_CAP_ETH(dev->mdev, vlan_cap))) {
5633				mlx5_ib_dbg(dev, "VLAN offloads are not supported\n");
5634				err = -EOPNOTSUPP;
5635				goto out;
5636			}
5637			MLX5_SET64(modify_rq_in, in, modify_bitmask,
5638				   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD);
5639			MLX5_SET(rqc, rqc, vsd,
5640				 (wq_attr->flags & IB_WQ_FLAGS_CVLAN_STRIPPING) ? 0 : 1);
5641		}
5642
5643		if (wq_attr->flags_mask & IB_WQ_FLAGS_PCI_WRITE_END_PADDING) {
5644			mlx5_ib_dbg(dev, "Modifying scatter end padding is not supported\n");
5645			err = -EOPNOTSUPP;
5646			goto out;
5647		}
5648	}
5649
5650	if (curr_wq_state == IB_WQS_RESET && wq_state == IB_WQS_RDY) {
5651		u16 set_id;
5652
5653		set_id = mlx5_ib_get_counters_id(dev, 0);
5654		if (MLX5_CAP_GEN(dev->mdev, modify_rq_counter_set_id)) {
5655			MLX5_SET64(modify_rq_in, in, modify_bitmask,
5656				   MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_RQ_COUNTER_SET_ID);
5657			MLX5_SET(rqc, rqc, counter_set_id, set_id);
5658		} else
5659			dev_info_once(
5660				&dev->ib_dev.dev,
5661				"Receive WQ counters are not supported on current FW\n");
5662	}
5663
5664	err = mlx5_core_modify_rq(dev->mdev, rwq->core_qp.qpn, in);
5665	if (!err)
5666		rwq->ibwq.state = (wq_state == MLX5_RQC_STATE_ERR) ? IB_WQS_ERR : wq_state;
5667
5668out:
5669	kvfree(in);
5670	return err;
5671}
5672
5673struct mlx5_ib_drain_cqe {
5674	struct ib_cqe cqe;
5675	struct completion done;
5676};
5677
5678static void mlx5_ib_drain_qp_done(struct ib_cq *cq, struct ib_wc *wc)
5679{
5680	struct mlx5_ib_drain_cqe *cqe = container_of(wc->wr_cqe,
5681						     struct mlx5_ib_drain_cqe,
5682						     cqe);
5683
5684	complete(&cqe->done);
5685}
5686
5687/* This function returns only once the drained WR was completed */
5688static void handle_drain_completion(struct ib_cq *cq,
5689				    struct mlx5_ib_drain_cqe *sdrain,
5690				    struct mlx5_ib_dev *dev)
5691{
5692	struct mlx5_core_dev *mdev = dev->mdev;
5693
5694	if (cq->poll_ctx == IB_POLL_DIRECT) {
5695		while (wait_for_completion_timeout(&sdrain->done, HZ / 10) <= 0)
5696			ib_process_cq_direct(cq, -1);
5697		return;
5698	}
5699
5700	if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5701		struct mlx5_ib_cq *mcq = to_mcq(cq);
5702		bool triggered = false;
5703		unsigned long flags;
5704
5705		spin_lock_irqsave(&dev->reset_flow_resource_lock, flags);
5706		/* Make sure that the CQ handler won't run if wasn't run yet */
5707		if (!mcq->mcq.reset_notify_added)
5708			mcq->mcq.reset_notify_added = 1;
5709		else
5710			triggered = true;
5711		spin_unlock_irqrestore(&dev->reset_flow_resource_lock, flags);
5712
5713		if (triggered) {
5714			/* Wait for any scheduled/running task to be ended */
5715			switch (cq->poll_ctx) {
5716			case IB_POLL_SOFTIRQ:
5717				irq_poll_disable(&cq->iop);
5718				irq_poll_enable(&cq->iop);
5719				break;
5720			case IB_POLL_WORKQUEUE:
5721				cancel_work_sync(&cq->work);
5722				break;
5723			default:
5724				WARN_ON_ONCE(1);
5725			}
5726		}
5727
5728		/* Run the CQ handler - this makes sure that the drain WR will
5729		 * be processed if wasn't processed yet.
5730		 */
5731		mcq->mcq.comp(&mcq->mcq, NULL);
5732	}
5733
5734	wait_for_completion(&sdrain->done);
5735}
5736
5737void mlx5_ib_drain_sq(struct ib_qp *qp)
5738{
5739	struct ib_cq *cq = qp->send_cq;
5740	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5741	struct mlx5_ib_drain_cqe sdrain;
5742	const struct ib_send_wr *bad_swr;
5743	struct ib_rdma_wr swr = {
5744		.wr = {
5745			.next = NULL,
5746			{ .wr_cqe	= &sdrain.cqe, },
5747			.opcode	= IB_WR_RDMA_WRITE,
5748		},
5749	};
5750	int ret;
5751	struct mlx5_ib_dev *dev = to_mdev(qp->device);
5752	struct mlx5_core_dev *mdev = dev->mdev;
5753
5754	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
5755	if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5756		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5757		return;
5758	}
5759
5760	sdrain.cqe.done = mlx5_ib_drain_qp_done;
5761	init_completion(&sdrain.done);
5762
5763	ret = mlx5_ib_post_send_drain(qp, &swr.wr, &bad_swr);
5764	if (ret) {
5765		WARN_ONCE(ret, "failed to drain send queue: %d\n", ret);
5766		return;
5767	}
5768
5769	handle_drain_completion(cq, &sdrain, dev);
5770}
5771
5772void mlx5_ib_drain_rq(struct ib_qp *qp)
5773{
5774	struct ib_cq *cq = qp->recv_cq;
5775	struct ib_qp_attr attr = { .qp_state = IB_QPS_ERR };
5776	struct mlx5_ib_drain_cqe rdrain;
5777	struct ib_recv_wr rwr = {};
5778	const struct ib_recv_wr *bad_rwr;
5779	int ret;
5780	struct mlx5_ib_dev *dev = to_mdev(qp->device);
5781	struct mlx5_core_dev *mdev = dev->mdev;
5782
5783	ret = ib_modify_qp(qp, &attr, IB_QP_STATE);
5784	if (ret && mdev->state != MLX5_DEVICE_STATE_INTERNAL_ERROR) {
5785		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5786		return;
5787	}
5788
5789	rwr.wr_cqe = &rdrain.cqe;
5790	rdrain.cqe.done = mlx5_ib_drain_qp_done;
5791	init_completion(&rdrain.done);
5792
5793	ret = mlx5_ib_post_recv_drain(qp, &rwr, &bad_rwr);
5794	if (ret) {
5795		WARN_ONCE(ret, "failed to drain recv queue: %d\n", ret);
5796		return;
5797	}
5798
5799	handle_drain_completion(cq, &rdrain, dev);
5800}
5801
5802/*
5803 * Bind a qp to a counter. If @counter is NULL then bind the qp to
5804 * the default counter
5805 */
5806int mlx5_ib_qp_set_counter(struct ib_qp *qp, struct rdma_counter *counter)
5807{
5808	struct mlx5_ib_dev *dev = to_mdev(qp->device);
5809	struct mlx5_ib_qp *mqp = to_mqp(qp);
5810	int err = 0;
5811
5812	mutex_lock(&mqp->mutex);
5813	if (mqp->state == IB_QPS_RESET) {
5814		qp->counter = counter;
5815		goto out;
5816	}
5817
5818	if (!MLX5_CAP_GEN(dev->mdev, rts2rts_qp_counters_set_id)) {
5819		err = -EOPNOTSUPP;
5820		goto out;
5821	}
5822
5823	if (mqp->state == IB_QPS_RTS) {
5824		err = __mlx5_ib_qp_set_counter(qp, counter);
5825		if (!err)
5826			qp->counter = counter;
5827
5828		goto out;
5829	}
5830
5831	mqp->counter_pending = 1;
5832	qp->counter = counter;
5833
5834out:
5835	mutex_unlock(&mqp->mutex);
5836	return err;
5837}
5838
5839int mlx5_ib_qp_event_init(void)
5840{
5841	mlx5_ib_qp_event_wq = alloc_ordered_workqueue("mlx5_ib_qp_event_wq", 0);
5842	if (!mlx5_ib_qp_event_wq)
5843		return -ENOMEM;
5844
5845	return 0;
5846}
5847
5848void mlx5_ib_qp_event_cleanup(void)
5849{
5850	destroy_workqueue(mlx5_ib_qp_event_wq);
5851}