1/*
   2 * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
   3 * Copyright (c) 2007, 2008 Mellanox Technologies. All rights reserved.
   4 *
   5 * This software is available to you under a choice of one of two
   6 * licenses.  You may choose to be licensed under the terms of the GNU
   7 * General Public License (GPL) Version 2, available from the file
   8 * COPYING in the main directory of this source tree, or the
   9 * OpenIB.org BSD license below:
  10 *
  11 *     Redistribution and use in source and binary forms, with or
  12 *     without modification, are permitted provided that the following
  13 *     conditions are met:
  14 *
  15 *      - Redistributions of source code must retain the above
  16 *        copyright notice, this list of conditions and the following
  17 *        disclaimer.
  18 *
  19 *      - Redistributions in binary form must reproduce the above
  20 *        copyright notice, this list of conditions and the following
  21 *        disclaimer in the documentation and/or other materials
  22 *        provided with the distribution.
  23 *
  24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  31 * SOFTWARE.
  32 */
  33
  34#include <linux/log2.h>
 
 
  35#include <linux/slab.h>
  36#include <linux/netdevice.h>
  37
  38#include <rdma/ib_cache.h>
  39#include <rdma/ib_pack.h>
  40#include <rdma/ib_addr.h>
 
 
  41
 
  42#include <linux/mlx4/qp.h>
  43
  44#include "mlx4_ib.h"
  45#include "user.h"
 
 
 
 
 
 
 
  46
  47enum {
  48	MLX4_IB_ACK_REQ_FREQ	= 8,
  49};
  50
  51enum {
  52	MLX4_IB_DEFAULT_SCHED_QUEUE	= 0x83,
  53	MLX4_IB_DEFAULT_QP0_SCHED_QUEUE	= 0x3f,
  54	MLX4_IB_LINK_TYPE_IB		= 0,
  55	MLX4_IB_LINK_TYPE_ETH		= 1
  56};
  57
  58enum {
  59	/*
  60	 * Largest possible UD header: send with GRH and immediate
  61	 * data plus 18 bytes for an Ethernet header with VLAN/802.1Q
  62	 * tag.  (LRH would only use 8 bytes, so Ethernet is the
  63	 * biggest case)
  64	 */
  65	MLX4_IB_UD_HEADER_SIZE		= 82,
  66	MLX4_IB_LSO_HEADER_SPARE	= 128,
  67};
  68
  69enum {
  70	MLX4_IB_IBOE_ETHERTYPE		= 0x8915
  71};
  72
  73struct mlx4_ib_sqp {
  74	struct mlx4_ib_qp	qp;
  75	int			pkey_index;
  76	u32			qkey;
  77	u32			send_psn;
  78	struct ib_ud_header	ud_header;
  79	u8			header_buf[MLX4_IB_UD_HEADER_SIZE];
 
  80};
  81
  82enum {
  83	MLX4_IB_MIN_SQ_STRIDE	= 6,
  84	MLX4_IB_CACHE_LINE_SIZE	= 64,
  85};
  86
 
 
 
 
 
 
 
 
 
  87static const __be32 mlx4_ib_opcode[] = {
  88	[IB_WR_SEND]				= cpu_to_be32(MLX4_OPCODE_SEND),
  89	[IB_WR_LSO]				= cpu_to_be32(MLX4_OPCODE_LSO),
  90	[IB_WR_SEND_WITH_IMM]			= cpu_to_be32(MLX4_OPCODE_SEND_IMM),
  91	[IB_WR_RDMA_WRITE]			= cpu_to_be32(MLX4_OPCODE_RDMA_WRITE),
  92	[IB_WR_RDMA_WRITE_WITH_IMM]		= cpu_to_be32(MLX4_OPCODE_RDMA_WRITE_IMM),
  93	[IB_WR_RDMA_READ]			= cpu_to_be32(MLX4_OPCODE_RDMA_READ),
  94	[IB_WR_ATOMIC_CMP_AND_SWP]		= cpu_to_be32(MLX4_OPCODE_ATOMIC_CS),
  95	[IB_WR_ATOMIC_FETCH_AND_ADD]		= cpu_to_be32(MLX4_OPCODE_ATOMIC_FA),
  96	[IB_WR_SEND_WITH_INV]			= cpu_to_be32(MLX4_OPCODE_SEND_INVAL),
  97	[IB_WR_LOCAL_INV]			= cpu_to_be32(MLX4_OPCODE_LOCAL_INVAL),
  98	[IB_WR_FAST_REG_MR]			= cpu_to_be32(MLX4_OPCODE_FMR),
  99	[IB_WR_MASKED_ATOMIC_CMP_AND_SWP]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_CS),
 100	[IB_WR_MASKED_ATOMIC_FETCH_AND_ADD]	= cpu_to_be32(MLX4_OPCODE_MASKED_ATOMIC_FA),
 101};
 102
 
 
 
 
 
 103static struct mlx4_ib_sqp *to_msqp(struct mlx4_ib_qp *mqp)
 104{
 105	return container_of(mqp, struct mlx4_ib_sqp, qp);
 106}
 107
 
 
 
 
 
 
 
 
 
 
 108static int is_sqp(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 109{
 110	return qp->mqp.qpn >= dev->dev->caps.sqp_start &&
 111		qp->mqp.qpn <= dev->dev->caps.sqp_start + 3;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 112}
 113
 
 114static int is_qp0(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 115{
 116	return qp->mqp.qpn >= dev->dev->caps.sqp_start &&
 117		qp->mqp.qpn <= dev->dev->caps.sqp_start + 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 118}
 119
 120static void *get_wqe(struct mlx4_ib_qp *qp, int offset)
 121{
 122	return mlx4_buf_offset(&qp->buf, offset);
 123}
 124
 125static void *get_recv_wqe(struct mlx4_ib_qp *qp, int n)
 126{
 127	return get_wqe(qp, qp->rq.offset + (n << qp->rq.wqe_shift));
 128}
 129
 130static void *get_send_wqe(struct mlx4_ib_qp *qp, int n)
 131{
 132	return get_wqe(qp, qp->sq.offset + (n << qp->sq.wqe_shift));
 133}
 134
 135/*
 136 * Stamp a SQ WQE so that it is invalid if prefetched by marking the
 137 * first four bytes of every 64 byte chunk with
 138 *     0x7FFFFFF | (invalid_ownership_value << 31).
 139 *
 140 * When the max work request size is less than or equal to the WQE
 141 * basic block size, as an optimization, we can stamp all WQEs with
 142 * 0xffffffff, and skip the very first chunk of each WQE.
 143 */
 144static void stamp_send_wqe(struct mlx4_ib_qp *qp, int n, int size)
 145{
 146	__be32 *wqe;
 147	int i;
 148	int s;
 149	int ind;
 150	void *buf;
 151	__be32 stamp;
 152	struct mlx4_wqe_ctrl_seg *ctrl;
 153
 154	if (qp->sq_max_wqes_per_wr > 1) {
 155		s = roundup(size, 1U << qp->sq.wqe_shift);
 156		for (i = 0; i < s; i += 64) {
 157			ind = (i >> qp->sq.wqe_shift) + n;
 158			stamp = ind & qp->sq.wqe_cnt ? cpu_to_be32(0x7fffffff) :
 159						       cpu_to_be32(0xffffffff);
 160			buf = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
 161			wqe = buf + (i & ((1 << qp->sq.wqe_shift) - 1));
 162			*wqe = stamp;
 163		}
 164	} else {
 165		ctrl = buf = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
 166		s = (ctrl->fence_size & 0x3f) << 4;
 167		for (i = 64; i < s; i += 64) {
 168			wqe = buf + i;
 169			*wqe = cpu_to_be32(0xffffffff);
 170		}
 171	}
 172}
 173
 174static void post_nop_wqe(struct mlx4_ib_qp *qp, int n, int size)
 175{
 176	struct mlx4_wqe_ctrl_seg *ctrl;
 177	struct mlx4_wqe_inline_seg *inl;
 178	void *wqe;
 179	int s;
 180
 181	ctrl = wqe = get_send_wqe(qp, n & (qp->sq.wqe_cnt - 1));
 182	s = sizeof(struct mlx4_wqe_ctrl_seg);
 183
 184	if (qp->ibqp.qp_type == IB_QPT_UD) {
 185		struct mlx4_wqe_datagram_seg *dgram = wqe + sizeof *ctrl;
 186		struct mlx4_av *av = (struct mlx4_av *)dgram->av;
 187		memset(dgram, 0, sizeof *dgram);
 188		av->port_pd = cpu_to_be32((qp->port << 24) | to_mpd(qp->ibqp.pd)->pdn);
 189		s += sizeof(struct mlx4_wqe_datagram_seg);
 190	}
 191
 192	/* Pad the remainder of the WQE with an inline data segment. */
 193	if (size > s) {
 194		inl = wqe + s;
 195		inl->byte_count = cpu_to_be32(1 << 31 | (size - s - sizeof *inl));
 196	}
 197	ctrl->srcrb_flags = 0;
 198	ctrl->fence_size = size / 16;
 199	/*
 200	 * Make sure descriptor is fully written before setting ownership bit
 201	 * (because HW can start executing as soon as we do).
 202	 */
 203	wmb();
 204
 205	ctrl->owner_opcode = cpu_to_be32(MLX4_OPCODE_NOP | MLX4_WQE_CTRL_NEC) |
 206		(n & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0);
 207
 208	stamp_send_wqe(qp, n + qp->sq_spare_wqes, size);
 209}
 210
 211/* Post NOP WQE to prevent wrap-around in the middle of WR */
 212static inline unsigned pad_wraparound(struct mlx4_ib_qp *qp, int ind)
 213{
 214	unsigned s = qp->sq.wqe_cnt - (ind & (qp->sq.wqe_cnt - 1));
 215	if (unlikely(s < qp->sq_max_wqes_per_wr)) {
 216		post_nop_wqe(qp, ind, s << qp->sq.wqe_shift);
 217		ind += s;
 218	}
 219	return ind;
 220}
 221
 222static void mlx4_ib_qp_event(struct mlx4_qp *qp, enum mlx4_event type)
 223{
 224	struct ib_event event;
 225	struct ib_qp *ibqp = &to_mibqp(qp)->ibqp;
 226
 227	if (type == MLX4_EVENT_TYPE_PATH_MIG)
 228		to_mibqp(qp)->port = to_mibqp(qp)->alt_port;
 229
 230	if (ibqp->event_handler) {
 231		event.device     = ibqp->device;
 232		event.element.qp = ibqp;
 233		switch (type) {
 234		case MLX4_EVENT_TYPE_PATH_MIG:
 235			event.event = IB_EVENT_PATH_MIG;
 236			break;
 237		case MLX4_EVENT_TYPE_COMM_EST:
 238			event.event = IB_EVENT_COMM_EST;
 239			break;
 240		case MLX4_EVENT_TYPE_SQ_DRAINED:
 241			event.event = IB_EVENT_SQ_DRAINED;
 242			break;
 243		case MLX4_EVENT_TYPE_SRQ_QP_LAST_WQE:
 244			event.event = IB_EVENT_QP_LAST_WQE_REACHED;
 245			break;
 246		case MLX4_EVENT_TYPE_WQ_CATAS_ERROR:
 247			event.event = IB_EVENT_QP_FATAL;
 248			break;
 249		case MLX4_EVENT_TYPE_PATH_MIG_FAILED:
 250			event.event = IB_EVENT_PATH_MIG_ERR;
 251			break;
 252		case MLX4_EVENT_TYPE_WQ_INVAL_REQ_ERROR:
 253			event.event = IB_EVENT_QP_REQ_ERR;
 254			break;
 255		case MLX4_EVENT_TYPE_WQ_ACCESS_ERROR:
 256			event.event = IB_EVENT_QP_ACCESS_ERR;
 257			break;
 258		default:
 259			printk(KERN_WARNING "mlx4_ib: Unexpected event type %d "
 260			       "on QP %06x\n", type, qp->qpn);
 261			return;
 262		}
 263
 264		ibqp->event_handler(&event, ibqp->qp_context);
 265	}
 266}
 267
 268static int send_wqe_overhead(enum ib_qp_type type, u32 flags)
 
 
 
 
 
 
 269{
 270	/*
 271	 * UD WQEs must have a datagram segment.
 272	 * RC and UC WQEs might have a remote address segment.
 273	 * MLX WQEs need two extra inline data segments (for the UD
 274	 * header and space for the ICRC).
 275	 */
 276	switch (type) {
 277	case IB_QPT_UD:
 278		return sizeof (struct mlx4_wqe_ctrl_seg) +
 279			sizeof (struct mlx4_wqe_datagram_seg) +
 280			((flags & MLX4_IB_QP_LSO) ? MLX4_IB_LSO_HEADER_SPARE : 0);
 281	case IB_QPT_UC:
 
 
 
 
 
 
 
 
 
 
 282		return sizeof (struct mlx4_wqe_ctrl_seg) +
 283			sizeof (struct mlx4_wqe_raddr_seg);
 284	case IB_QPT_RC:
 285		return sizeof (struct mlx4_wqe_ctrl_seg) +
 286			sizeof (struct mlx4_wqe_atomic_seg) +
 287			sizeof (struct mlx4_wqe_raddr_seg);
 288	case IB_QPT_SMI:
 289	case IB_QPT_GSI:
 290		return sizeof (struct mlx4_wqe_ctrl_seg) +
 291			ALIGN(MLX4_IB_UD_HEADER_SIZE +
 292			      DIV_ROUND_UP(MLX4_IB_UD_HEADER_SIZE,
 293					   MLX4_INLINE_ALIGN) *
 294			      sizeof (struct mlx4_wqe_inline_seg),
 295			      sizeof (struct mlx4_wqe_data_seg)) +
 296			ALIGN(4 +
 297			      sizeof (struct mlx4_wqe_inline_seg),
 298			      sizeof (struct mlx4_wqe_data_seg));
 299	default:
 300		return sizeof (struct mlx4_wqe_ctrl_seg);
 301	}
 302}
 303
 304static int set_rq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
 305		       int is_user, int has_srq, struct mlx4_ib_qp *qp)
 
 306{
 307	/* Sanity check RQ size before proceeding */
 308	if (cap->max_recv_wr  > dev->dev->caps.max_wqes  ||
 309	    cap->max_recv_sge > dev->dev->caps.max_rq_sg)
 310		return -EINVAL;
 311
 312	if (has_srq) {
 313		/* QPs attached to an SRQ should have no RQ */
 314		if (cap->max_recv_wr)
 315			return -EINVAL;
 316
 317		qp->rq.wqe_cnt = qp->rq.max_gs = 0;
 318	} else {
 
 
 
 
 319		/* HW requires >= 1 RQ entry with >= 1 gather entry */
 320		if (is_user && (!cap->max_recv_wr || !cap->max_recv_sge))
 
 321			return -EINVAL;
 322
 323		qp->rq.wqe_cnt	 = roundup_pow_of_two(max(1U, cap->max_recv_wr));
 324		qp->rq.max_gs	 = roundup_pow_of_two(max(1U, cap->max_recv_sge));
 325		qp->rq.wqe_shift = ilog2(qp->rq.max_gs * sizeof (struct mlx4_wqe_data_seg));
 
 326	}
 327
 328	cap->max_recv_wr  = qp->rq.max_post = qp->rq.wqe_cnt;
 329	cap->max_recv_sge = qp->rq.max_gs;
 
 
 
 
 
 
 
 
 
 330
 331	return 0;
 332}
 333
 334static int set_kernel_sq_size(struct mlx4_ib_dev *dev, struct ib_qp_cap *cap,
 335			      enum ib_qp_type type, struct mlx4_ib_qp *qp)
 336{
 337	int s;
 338
 339	/* Sanity check SQ size before proceeding */
 340	if (cap->max_send_wr	 > dev->dev->caps.max_wqes  ||
 341	    cap->max_send_sge	 > dev->dev->caps.max_sq_sg ||
 342	    cap->max_inline_data + send_wqe_overhead(type, qp->flags) +
 343	    sizeof (struct mlx4_wqe_inline_seg) > dev->dev->caps.max_sq_desc_sz)
 344		return -EINVAL;
 345
 346	/*
 347	 * For MLX transport we need 2 extra S/G entries:
 348	 * one for the header and one for the checksum at the end
 349	 */
 350	if ((type == IB_QPT_SMI || type == IB_QPT_GSI) &&
 
 351	    cap->max_send_sge + 2 > dev->dev->caps.max_sq_sg)
 352		return -EINVAL;
 353
 354	s = max(cap->max_send_sge * sizeof (struct mlx4_wqe_data_seg),
 355		cap->max_inline_data + sizeof (struct mlx4_wqe_inline_seg)) +
 356		send_wqe_overhead(type, qp->flags);
 357
 358	if (s > dev->dev->caps.max_sq_desc_sz)
 359		return -EINVAL;
 360
 361	/*
 362	 * Hermon supports shrinking WQEs, such that a single work
 363	 * request can include multiple units of 1 << wqe_shift.  This
 364	 * way, work requests can differ in size, and do not have to
 365	 * be a power of 2 in size, saving memory and speeding up send
 366	 * WR posting.  Unfortunately, if we do this then the
 367	 * wqe_index field in CQEs can't be used to look up the WR ID
 368	 * anymore, so we do this only if selective signaling is off.
 369	 *
 370	 * Further, on 32-bit platforms, we can't use vmap() to make
 371	 * the QP buffer virtually contiguous.  Thus we have to use
 372	 * constant-sized WRs to make sure a WR is always fully within
 373	 * a single page-sized chunk.
 374	 *
 375	 * Finally, we use NOP work requests to pad the end of the
 376	 * work queue, to avoid wrap-around in the middle of WR.  We
 377	 * set NEC bit to avoid getting completions with error for
 378	 * these NOP WRs, but since NEC is only supported starting
 379	 * with firmware 2.2.232, we use constant-sized WRs for older
 380	 * firmware.
 381	 *
 382	 * And, since MLX QPs only support SEND, we use constant-sized
 383	 * WRs in this case.
 384	 *
 385	 * We look for the smallest value of wqe_shift such that the
 386	 * resulting number of wqes does not exceed device
 387	 * capabilities.
 388	 *
 389	 * We set WQE size to at least 64 bytes, this way stamping
 390	 * invalidates each WQE.
 391	 */
 392	if (dev->dev->caps.fw_ver >= MLX4_FW_VER_WQE_CTRL_NEC &&
 393	    qp->sq_signal_bits && BITS_PER_LONG == 64 &&
 394	    type != IB_QPT_SMI && type != IB_QPT_GSI)
 395		qp->sq.wqe_shift = ilog2(64);
 396	else
 397		qp->sq.wqe_shift = ilog2(roundup_pow_of_two(s));
 398
 399	for (;;) {
 400		qp->sq_max_wqes_per_wr = DIV_ROUND_UP(s, 1U << qp->sq.wqe_shift);
 401
 402		/*
 403		 * We need to leave 2 KB + 1 WR of headroom in the SQ to
 404		 * allow HW to prefetch.
 405		 */
 406		qp->sq_spare_wqes = (2048 >> qp->sq.wqe_shift) + qp->sq_max_wqes_per_wr;
 407		qp->sq.wqe_cnt = roundup_pow_of_two(cap->max_send_wr *
 408						    qp->sq_max_wqes_per_wr +
 409						    qp->sq_spare_wqes);
 410
 411		if (qp->sq.wqe_cnt <= dev->dev->caps.max_wqes)
 412			break;
 413
 414		if (qp->sq_max_wqes_per_wr <= 1)
 415			return -EINVAL;
 416
 417		++qp->sq.wqe_shift;
 418	}
 419
 420	qp->sq.max_gs = (min(dev->dev->caps.max_sq_desc_sz,
 421			     (qp->sq_max_wqes_per_wr << qp->sq.wqe_shift)) -
 422			 send_wqe_overhead(type, qp->flags)) /
 423		sizeof (struct mlx4_wqe_data_seg);
 424
 425	qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
 426		(qp->sq.wqe_cnt << qp->sq.wqe_shift);
 427	if (qp->rq.wqe_shift > qp->sq.wqe_shift) {
 428		qp->rq.offset = 0;
 429		qp->sq.offset = qp->rq.wqe_cnt << qp->rq.wqe_shift;
 430	} else {
 431		qp->rq.offset = qp->sq.wqe_cnt << qp->sq.wqe_shift;
 432		qp->sq.offset = 0;
 433	}
 434
 435	cap->max_send_wr  = qp->sq.max_post =
 436		(qp->sq.wqe_cnt - qp->sq_spare_wqes) / qp->sq_max_wqes_per_wr;
 437	cap->max_send_sge = min(qp->sq.max_gs,
 438				min(dev->dev->caps.max_sq_sg,
 439				    dev->dev->caps.max_rq_sg));
 440	/* We don't support inline sends for kernel QPs (yet) */
 441	cap->max_inline_data = 0;
 442
 443	return 0;
 444}
 445
 446static int set_user_sq_size(struct mlx4_ib_dev *dev,
 447			    struct mlx4_ib_qp *qp,
 448			    struct mlx4_ib_create_qp *ucmd)
 449{
 450	/* Sanity check SQ size before proceeding */
 451	if ((1 << ucmd->log_sq_bb_count) > dev->dev->caps.max_wqes	 ||
 452	    ucmd->log_sq_stride >
 453		ilog2(roundup_pow_of_two(dev->dev->caps.max_sq_desc_sz)) ||
 454	    ucmd->log_sq_stride < MLX4_IB_MIN_SQ_STRIDE)
 455		return -EINVAL;
 456
 457	qp->sq.wqe_cnt   = 1 << ucmd->log_sq_bb_count;
 458	qp->sq.wqe_shift = ucmd->log_sq_stride;
 459
 460	qp->buf_size = (qp->rq.wqe_cnt << qp->rq.wqe_shift) +
 461		(qp->sq.wqe_cnt << qp->sq.wqe_shift);
 462
 463	return 0;
 464}
 465
 466static int create_qp_common(struct mlx4_ib_dev *dev, struct ib_pd *pd,
 467			    struct ib_qp_init_attr *init_attr,
 468			    struct ib_udata *udata, int sqpn, struct mlx4_ib_qp *qp)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 469{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 470	int qpn;
 471	int err;
 
 
 
 
 
 
 
 
 
 
 
 472
 473	mutex_init(&qp->mutex);
 474	spin_lock_init(&qp->sq.lock);
 475	spin_lock_init(&qp->rq.lock);
 476	INIT_LIST_HEAD(&qp->gid_list);
 
 477
 478	qp->state	 = IB_QPS_RESET;
 479	if (init_attr->sq_sig_type == IB_SIGNAL_ALL_WR)
 480		qp->sq_signal_bits = cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
 
 
 
 
 
 481
 482	err = set_rq_size(dev, &init_attr->cap, !!pd->uobject, !!init_attr->srq, qp);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 483	if (err)
 484		goto err;
 485
 486	if (pd->uobject) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 487		struct mlx4_ib_create_qp ucmd;
 
 
 
 
 
 488
 489		if (ib_copy_from_udata(&ucmd, udata, sizeof ucmd)) {
 490			err = -EFAULT;
 491			goto err;
 492		}
 493
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 494		qp->sq_no_prefetch = ucmd.sq_no_prefetch;
 495
 496		err = set_user_sq_size(dev, qp, &ucmd);
 497		if (err)
 498			goto err;
 499
 500		qp->umem = ib_umem_get(pd->uobject->context, ucmd.buf_addr,
 501				       qp->buf_size, 0, 0);
 502		if (IS_ERR(qp->umem)) {
 503			err = PTR_ERR(qp->umem);
 504			goto err;
 505		}
 506
 507		err = mlx4_mtt_init(dev->dev, ib_umem_page_count(qp->umem),
 508				    ilog2(qp->umem->page_size), &qp->mtt);
 
 
 509		if (err)
 510			goto err_buf;
 511
 512		err = mlx4_ib_umem_write_mtt(dev, &qp->mtt, qp->umem);
 513		if (err)
 514			goto err_mtt;
 515
 516		if (!init_attr->srq) {
 517			err = mlx4_ib_db_map_user(to_mucontext(pd->uobject->context),
 518						  ucmd.db_addr, &qp->db);
 519			if (err)
 520				goto err_mtt;
 521		}
 
 522	} else {
 523		qp->sq_no_prefetch = 0;
 
 
 
 524
 525		if (init_attr->create_flags & IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK)
 526			qp->flags |= MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK;
 527
 528		if (init_attr->create_flags & IB_QP_CREATE_IPOIB_UD_LSO)
 529			qp->flags |= MLX4_IB_QP_LSO;
 530
 531		err = set_kernel_sq_size(dev, &init_attr->cap, init_attr->qp_type, qp);
 
 
 
 
 
 
 
 
 532		if (err)
 533			goto err;
 534
 535		if (!init_attr->srq) {
 536			err = mlx4_db_alloc(dev->dev, &qp->db, 0);
 537			if (err)
 538				goto err;
 539
 540			*qp->db.db = 0;
 541		}
 542
 543		if (mlx4_buf_alloc(dev->dev, qp->buf_size, PAGE_SIZE * 2, &qp->buf)) {
 
 544			err = -ENOMEM;
 545			goto err_db;
 546		}
 547
 548		err = mlx4_mtt_init(dev->dev, qp->buf.npages, qp->buf.page_shift,
 549				    &qp->mtt);
 550		if (err)
 551			goto err_buf;
 552
 553		err = mlx4_buf_write_mtt(dev->dev, &qp->mtt, &qp->buf);
 554		if (err)
 555			goto err_mtt;
 556
 557		qp->sq.wrid  = kmalloc(qp->sq.wqe_cnt * sizeof (u64), GFP_KERNEL);
 558		qp->rq.wrid  = kmalloc(qp->rq.wqe_cnt * sizeof (u64), GFP_KERNEL);
 559
 
 560		if (!qp->sq.wrid || !qp->rq.wrid) {
 561			err = -ENOMEM;
 562			goto err_wrid;
 563		}
 
 564	}
 565
 566	if (sqpn) {
 567		qpn = sqpn;
 
 
 
 
 
 
 568	} else {
 569		err = mlx4_qp_reserve_range(dev->dev, 1, 1, &qpn);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 570		if (err)
 571			goto err_wrid;
 572	}
 573
 
 
 
 574	err = mlx4_qp_alloc(dev->dev, qpn, &qp->mqp);
 575	if (err)
 576		goto err_qpn;
 577
 
 
 
 578	/*
 579	 * Hardware wants QPN written in big-endian order (after
 580	 * shifting) for send doorbell.  Precompute this value to save
 581	 * a little bit when posting sends.
 582	 */
 583	qp->doorbell_qpn = swab32(qp->mqp.qpn << 8);
 584
 585	qp->mqp.event = mlx4_ib_qp_event;
 586
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 587	return 0;
 588
 589err_qpn:
 590	if (!sqpn)
 591		mlx4_qp_release_range(dev->dev, qpn, 1);
 592
 
 
 
 
 
 
 593err_wrid:
 594	if (pd->uobject) {
 595		if (!init_attr->srq)
 596			mlx4_ib_db_unmap_user(to_mucontext(pd->uobject->context),
 597					      &qp->db);
 598	} else {
 599		kfree(qp->sq.wrid);
 600		kfree(qp->rq.wrid);
 601	}
 602
 603err_mtt:
 604	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
 605
 606err_buf:
 607	if (pd->uobject)
 608		ib_umem_release(qp->umem);
 609	else
 610		mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
 
 611
 612err_db:
 613	if (!pd->uobject && !init_attr->srq)
 614		mlx4_db_free(dev->dev, &qp->db);
 615
 616err:
 
 
 
 
 617	return err;
 618}
 619
 620static enum mlx4_qp_state to_mlx4_state(enum ib_qp_state state)
 621{
 622	switch (state) {
 623	case IB_QPS_RESET:	return MLX4_QP_STATE_RST;
 624	case IB_QPS_INIT:	return MLX4_QP_STATE_INIT;
 625	case IB_QPS_RTR:	return MLX4_QP_STATE_RTR;
 626	case IB_QPS_RTS:	return MLX4_QP_STATE_RTS;
 627	case IB_QPS_SQD:	return MLX4_QP_STATE_SQD;
 628	case IB_QPS_SQE:	return MLX4_QP_STATE_SQER;
 629	case IB_QPS_ERR:	return MLX4_QP_STATE_ERR;
 630	default:		return -1;
 631	}
 632}
 633
 634static void mlx4_ib_lock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
 635	__acquires(&send_cq->lock) __acquires(&recv_cq->lock)
 636{
 637	if (send_cq == recv_cq) {
 638		spin_lock_irq(&send_cq->lock);
 639		__acquire(&recv_cq->lock);
 640	} else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
 641		spin_lock_irq(&send_cq->lock);
 642		spin_lock_nested(&recv_cq->lock, SINGLE_DEPTH_NESTING);
 643	} else {
 644		spin_lock_irq(&recv_cq->lock);
 645		spin_lock_nested(&send_cq->lock, SINGLE_DEPTH_NESTING);
 646	}
 647}
 648
 649static void mlx4_ib_unlock_cqs(struct mlx4_ib_cq *send_cq, struct mlx4_ib_cq *recv_cq)
 650	__releases(&send_cq->lock) __releases(&recv_cq->lock)
 651{
 652	if (send_cq == recv_cq) {
 653		__release(&recv_cq->lock);
 654		spin_unlock_irq(&send_cq->lock);
 655	} else if (send_cq->mcq.cqn < recv_cq->mcq.cqn) {
 656		spin_unlock(&recv_cq->lock);
 657		spin_unlock_irq(&send_cq->lock);
 658	} else {
 659		spin_unlock(&send_cq->lock);
 660		spin_unlock_irq(&recv_cq->lock);
 661	}
 662}
 663
 664static void del_gid_entries(struct mlx4_ib_qp *qp)
 665{
 666	struct mlx4_ib_gid_entry *ge, *tmp;
 667
 668	list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
 669		list_del(&ge->list);
 670		kfree(ge);
 671	}
 672}
 673
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 674static void destroy_qp_common(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp,
 675			      int is_user)
 
 676{
 677	struct mlx4_ib_cq *send_cq, *recv_cq;
 
 678
 679	if (qp->state != IB_QPS_RESET)
 680		if (mlx4_qp_modify(dev->dev, NULL, to_mlx4_state(qp->state),
 681				   MLX4_QP_STATE_RST, NULL, 0, 0, &qp->mqp))
 682			printk(KERN_WARNING "mlx4_ib: modify QP %06x to RESET failed.\n",
 683			       qp->mqp.qpn);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 684
 685	send_cq = to_mcq(qp->ibqp.send_cq);
 686	recv_cq = to_mcq(qp->ibqp.recv_cq);
 687
 
 688	mlx4_ib_lock_cqs(send_cq, recv_cq);
 689
 690	if (!is_user) {
 
 
 
 
 691		__mlx4_ib_cq_clean(recv_cq, qp->mqp.qpn,
 692				 qp->ibqp.srq ? to_msrq(qp->ibqp.srq): NULL);
 693		if (send_cq != recv_cq)
 694			__mlx4_ib_cq_clean(send_cq, qp->mqp.qpn, NULL);
 695	}
 696
 697	mlx4_qp_remove(dev->dev, &qp->mqp);
 698
 699	mlx4_ib_unlock_cqs(send_cq, recv_cq);
 
 700
 701	mlx4_qp_free(dev->dev, &qp->mqp);
 702
 703	if (!is_sqp(dev, qp))
 704		mlx4_qp_release_range(dev->dev, qp->mqp.qpn, 1);
 
 
 
 
 
 
 
 
 
 
 
 705
 706	mlx4_mtt_cleanup(dev->dev, &qp->mtt);
 707
 708	if (is_user) {
 709		if (!qp->ibqp.srq)
 710			mlx4_ib_db_unmap_user(to_mucontext(qp->ibqp.uobject->context),
 711					      &qp->db);
 712		ib_umem_release(qp->umem);
 
 
 
 
 
 713	} else {
 714		kfree(qp->sq.wrid);
 715		kfree(qp->rq.wrid);
 
 
 
 716		mlx4_buf_free(dev->dev, qp->buf_size, &qp->buf);
 717		if (!qp->ibqp.srq)
 718			mlx4_db_free(dev->dev, &qp->db);
 719	}
 
 720
 721	del_gid_entries(qp);
 722}
 723
 724struct ib_qp *mlx4_ib_create_qp(struct ib_pd *pd,
 725				struct ib_qp_init_attr *init_attr,
 726				struct ib_udata *udata)
 727{
 728	struct mlx4_ib_dev *dev = to_mdev(pd->device);
 729	struct mlx4_ib_sqp *sqp;
 730	struct mlx4_ib_qp *qp;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 731	int err;
 
 
 
 
 
 732
 733	/*
 734	 * We only support LSO and multicast loopback blocking, and
 735	 * only for kernel UD QPs.
 736	 */
 737	if (init_attr->create_flags & ~(IB_QP_CREATE_IPOIB_UD_LSO |
 738					IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK))
 
 
 
 
 739		return ERR_PTR(-EINVAL);
 740
 741	if (init_attr->create_flags &&
 742	    (pd->uobject || init_attr->qp_type != IB_QPT_UD))
 743		return ERR_PTR(-EINVAL);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 744
 745	switch (init_attr->qp_type) {
 
 
 
 
 
 
 
 
 
 
 746	case IB_QPT_RC:
 747	case IB_QPT_UC:
 748	case IB_QPT_UD:
 749	{
 750		qp = kzalloc(sizeof *qp, GFP_KERNEL);
 751		if (!qp)
 752			return ERR_PTR(-ENOMEM);
 753
 754		err = create_qp_common(dev, pd, init_attr, udata, 0, qp);
 
 
 
 
 755		if (err) {
 756			kfree(qp);
 757			return ERR_PTR(err);
 758		}
 759
 760		qp->ibqp.qp_num = qp->mqp.qpn;
 
 761
 762		break;
 763	}
 764	case IB_QPT_SMI:
 765	case IB_QPT_GSI:
 766	{
 
 
 767		/* Userspace is not allowed to create special QPs: */
 768		if (pd->uobject)
 769			return ERR_PTR(-EINVAL);
 
 
 
 
 770
 771		sqp = kzalloc(sizeof *sqp, GFP_KERNEL);
 772		if (!sqp)
 773			return ERR_PTR(-ENOMEM);
 774
 775		qp = &sqp->qp;
 776
 777		err = create_qp_common(dev, pd, init_attr, udata,
 778				       dev->dev->caps.sqp_start +
 779				       (init_attr->qp_type == IB_QPT_SMI ? 0 : 2) +
 780				       init_attr->port_num - 1,
 781				       qp);
 782		if (err) {
 783			kfree(sqp);
 784			return ERR_PTR(err);
 785		}
 786
 787		qp->port	= init_attr->port_num;
 788		qp->ibqp.qp_num = init_attr->qp_type == IB_QPT_SMI ? 0 : 1;
 789
 790		break;
 791	}
 792	default:
 793		/* Don't support raw QPs */
 794		return ERR_PTR(-EINVAL);
 795	}
 796
 797	return &qp->ibqp;
 798}
 799
 800int mlx4_ib_destroy_qp(struct ib_qp *qp)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 801{
 802	struct mlx4_ib_dev *dev = to_mdev(qp->device);
 803	struct mlx4_ib_qp *mqp = to_mqp(qp);
 804
 805	if (is_qp0(dev, mqp))
 806		mlx4_CLOSE_PORT(dev->dev, mqp->port);
 807
 808	destroy_qp_common(dev, mqp, !!qp->pd->uobject);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 809
 810	if (is_sqp(dev, mqp))
 811		kfree(to_msqp(mqp));
 812	else
 813		kfree(mqp);
 814
 815	return 0;
 816}
 817
 818static int to_mlx4_st(enum ib_qp_type type)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 819{
 820	switch (type) {
 821	case IB_QPT_RC:		return MLX4_QP_ST_RC;
 822	case IB_QPT_UC:		return MLX4_QP_ST_UC;
 823	case IB_QPT_UD:		return MLX4_QP_ST_UD;
 824	case IB_QPT_SMI:
 825	case IB_QPT_GSI:	return MLX4_QP_ST_MLX;
 826	default:		return -1;
 
 
 
 
 
 
 
 
 
 
 
 
 827	}
 828}
 829
 830static __be32 to_mlx4_access_flags(struct mlx4_ib_qp *qp, const struct ib_qp_attr *attr,
 831				   int attr_mask)
 832{
 833	u8 dest_rd_atomic;
 834	u32 access_flags;
 835	u32 hw_access_flags = 0;
 836
 837	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
 838		dest_rd_atomic = attr->max_dest_rd_atomic;
 839	else
 840		dest_rd_atomic = qp->resp_depth;
 841
 842	if (attr_mask & IB_QP_ACCESS_FLAGS)
 843		access_flags = attr->qp_access_flags;
 844	else
 845		access_flags = qp->atomic_rd_en;
 846
 847	if (!dest_rd_atomic)
 848		access_flags &= IB_ACCESS_REMOTE_WRITE;
 849
 850	if (access_flags & IB_ACCESS_REMOTE_READ)
 851		hw_access_flags |= MLX4_QP_BIT_RRE;
 852	if (access_flags & IB_ACCESS_REMOTE_ATOMIC)
 853		hw_access_flags |= MLX4_QP_BIT_RAE;
 854	if (access_flags & IB_ACCESS_REMOTE_WRITE)
 855		hw_access_flags |= MLX4_QP_BIT_RWE;
 856
 857	return cpu_to_be32(hw_access_flags);
 858}
 859
 860static void store_sqp_attrs(struct mlx4_ib_sqp *sqp, const struct ib_qp_attr *attr,
 861			    int attr_mask)
 862{
 863	if (attr_mask & IB_QP_PKEY_INDEX)
 864		sqp->pkey_index = attr->pkey_index;
 865	if (attr_mask & IB_QP_QKEY)
 866		sqp->qkey = attr->qkey;
 867	if (attr_mask & IB_QP_SQ_PSN)
 868		sqp->send_psn = attr->sq_psn;
 869}
 870
 871static void mlx4_set_sched(struct mlx4_qp_path *path, u8 port)
 872{
 873	path->sched_queue = (path->sched_queue & 0xbf) | ((port - 1) << 6);
 874}
 875
 876static int mlx4_set_path(struct mlx4_ib_dev *dev, const struct ib_ah_attr *ah,
 877			 struct mlx4_qp_path *path, u8 port)
 
 
 878{
 879	int err;
 880	int is_eth = rdma_port_get_link_layer(&dev->ib_dev, port) ==
 881		IB_LINK_LAYER_ETHERNET;
 882	u8 mac[6];
 883	int is_mcast;
 884	u16 vlan_tag;
 885	int vidx;
 
 
 886
 887	path->grh_mylmc     = ah->src_path_bits & 0x7f;
 888	path->rlid	    = cpu_to_be16(ah->dlid);
 889	if (ah->static_rate) {
 890		path->static_rate = ah->static_rate + MLX4_STAT_RATE_OFFSET;
 
 891		while (path->static_rate > IB_RATE_2_5_GBPS + MLX4_STAT_RATE_OFFSET &&
 892		       !(1 << path->static_rate & dev->dev->caps.stat_rate_support))
 893			--path->static_rate;
 894	} else
 895		path->static_rate = 0;
 896
 897	if (ah->ah_flags & IB_AH_GRH) {
 898		if (ah->grh.sgid_index >= dev->dev->caps.gid_table_len[port]) {
 899			printk(KERN_ERR "sgid_index (%u) too large. max is %d\n",
 900			       ah->grh.sgid_index, dev->dev->caps.gid_table_len[port] - 1);
 
 
 
 
 
 
 901			return -1;
 902		}
 903
 904		path->grh_mylmc |= 1 << 7;
 905		path->mgid_index = ah->grh.sgid_index;
 906		path->hop_limit  = ah->grh.hop_limit;
 907		path->tclass_flowlabel =
 908			cpu_to_be32((ah->grh.traffic_class << 20) |
 909				    (ah->grh.flow_label));
 910		memcpy(path->rgid, ah->grh.dgid.raw, 16);
 911	}
 912
 913	if (is_eth) {
 914		path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
 915			((port - 1) << 6) | ((ah->sl & 7) << 3) | ((ah->sl & 8) >> 1);
 916
 917		if (!(ah->ah_flags & IB_AH_GRH))
 918			return -1;
 919
 920		err = mlx4_ib_resolve_grh(dev, ah, mac, &is_mcast, port);
 921		if (err)
 922			return err;
 923
 924		memcpy(path->dmac, mac, 6);
 925		path->ackto = MLX4_IB_LINK_TYPE_ETH;
 926		/* use index 0 into MAC table for IBoE */
 927		path->grh_mylmc &= 0x80;
 928
 929		vlan_tag = rdma_get_vlan_id(&dev->iboe.gid_table[port - 1][ah->grh.sgid_index]);
 930		if (vlan_tag < 0x1000) {
 931			if (mlx4_find_cached_vlan(dev->dev, port, vlan_tag, &vidx))
 932				return -ENOENT;
 933
 934			path->vlan_index = vidx;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 935			path->fl = 1 << 6;
 
 
 
 
 
 
 936		}
 937	} else
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 938		path->sched_queue = MLX4_IB_DEFAULT_SCHED_QUEUE |
 939			((port - 1) << 6) | ((ah->sl & 0xf) << 2);
 
 940
 941	return 0;
 942}
 943
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 944static void update_mcg_macs(struct mlx4_ib_dev *dev, struct mlx4_ib_qp *qp)
 945{
 946	struct mlx4_ib_gid_entry *ge, *tmp;
 947
 948	list_for_each_entry_safe(ge, tmp, &qp->gid_list, list) {
 949		if (!ge->added && mlx4_ib_add_mc(dev, qp, &ge->gid)) {
 950			ge->added = 1;
 951			ge->port = qp->port;
 952		}
 953	}
 954}
 955
 956static int __mlx4_ib_modify_qp(struct ib_qp *ibqp,
 957			       const struct ib_qp_attr *attr, int attr_mask,
 958			       enum ib_qp_state cur_state, enum ib_qp_state new_state)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 959{
 960	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
 961	struct mlx4_ib_qp *qp = to_mqp(ibqp);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 962	struct mlx4_qp_context *context;
 963	enum mlx4_qp_optpar optpar = 0;
 964	int sqd_event;
 
 965	int err = -EINVAL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 966
 967	context = kzalloc(sizeof *context, GFP_KERNEL);
 968	if (!context)
 969		return -ENOMEM;
 970
 971	context->flags = cpu_to_be32((to_mlx4_state(new_state) << 28) |
 972				     (to_mlx4_st(ibqp->qp_type) << 16));
 973
 974	if (!(attr_mask & IB_QP_PATH_MIG_STATE))
 975		context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
 976	else {
 977		optpar |= MLX4_QP_OPTPAR_PM_STATE;
 978		switch (attr->path_mig_state) {
 979		case IB_MIG_MIGRATED:
 980			context->flags |= cpu_to_be32(MLX4_QP_PM_MIGRATED << 11);
 981			break;
 982		case IB_MIG_REARM:
 983			context->flags |= cpu_to_be32(MLX4_QP_PM_REARM << 11);
 984			break;
 985		case IB_MIG_ARMED:
 986			context->flags |= cpu_to_be32(MLX4_QP_PM_ARMED << 11);
 987			break;
 988		}
 989	}
 990
 991	if (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI)
 
 
 
 
 
 
 992		context->mtu_msgmax = (IB_MTU_4096 << 5) | 11;
 993	else if (ibqp->qp_type == IB_QPT_UD) {
 
 
 994		if (qp->flags & MLX4_IB_QP_LSO)
 995			context->mtu_msgmax = (IB_MTU_4096 << 5) |
 996					      ilog2(dev->dev->caps.max_gso_sz);
 997		else
 998			context->mtu_msgmax = (IB_MTU_4096 << 5) | 12;
 999	} else if (attr_mask & IB_QP_PATH_MTU) {
1000		if (attr->path_mtu < IB_MTU_256 || attr->path_mtu > IB_MTU_4096) {
1001			printk(KERN_ERR "path MTU (%u) is invalid\n",
1002			       attr->path_mtu);
1003			goto out;
1004		}
1005		context->mtu_msgmax = (attr->path_mtu << 5) |
1006			ilog2(dev->dev->caps.max_msg_sz);
1007	}
1008
1009	if (qp->rq.wqe_cnt)
1010		context->rq_size_stride = ilog2(qp->rq.wqe_cnt) << 3;
1011	context->rq_size_stride |= qp->rq.wqe_shift - 4;
 
 
1012
1013	if (qp->sq.wqe_cnt)
1014		context->sq_size_stride = ilog2(qp->sq.wqe_cnt) << 3;
1015	context->sq_size_stride |= qp->sq.wqe_shift - 4;
1016
1017	if (cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
 
 
 
1018		context->sq_size_stride |= !!qp->sq_no_prefetch << 7;
 
 
 
 
1019
1020	if (qp->ibqp.uobject)
1021		context->usr_page = cpu_to_be32(to_mucontext(ibqp->uobject->context)->uar.index);
 
1022	else
1023		context->usr_page = cpu_to_be32(dev->priv_uar.index);
 
1024
1025	if (attr_mask & IB_QP_DEST_QPN)
1026		context->remote_qpn = cpu_to_be32(attr->dest_qp_num);
1027
1028	if (attr_mask & IB_QP_PORT) {
1029		if (cur_state == IB_QPS_SQD && new_state == IB_QPS_SQD &&
1030		    !(attr_mask & IB_QP_AV)) {
1031			mlx4_set_sched(&context->pri_path, attr->port_num);
1032			optpar |= MLX4_QP_OPTPAR_SCHED_QUEUE;
1033		}
1034	}
1035
1036	if (cur_state == IB_QPS_INIT && new_state == IB_QPS_RTR) {
1037		if (dev->counters[qp->port - 1] != -1) {
1038			context->pri_path.counter_index =
1039						dev->counters[qp->port - 1];
 
 
 
 
 
 
 
 
1040			optpar |= MLX4_QP_OPTPAR_COUNTER_INDEX;
 
 
 
 
 
 
1041		} else
1042			context->pri_path.counter_index = 0xff;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1043	}
1044
1045	if (attr_mask & IB_QP_PKEY_INDEX) {
 
 
1046		context->pri_path.pkey_index = attr->pkey_index;
1047		optpar |= MLX4_QP_OPTPAR_PKEY_INDEX;
1048	}
1049
1050	if (attr_mask & IB_QP_AV) {
1051		if (mlx4_set_path(dev, &attr->ah_attr, &context->pri_path,
1052				  attr_mask & IB_QP_PORT ? attr->port_num : qp->port))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1053			goto out;
1054
1055		optpar |= (MLX4_QP_OPTPAR_PRIMARY_ADDR_PATH |
1056			   MLX4_QP_OPTPAR_SCHED_QUEUE);
 
 
 
 
 
 
 
 
 
 
 
 
1057	}
1058
1059	if (attr_mask & IB_QP_TIMEOUT) {
1060		context->pri_path.ackto |= attr->timeout << 3;
1061		optpar |= MLX4_QP_OPTPAR_ACK_TIMEOUT;
1062	}
1063
1064	if (attr_mask & IB_QP_ALT_PATH) {
1065		if (attr->alt_port_num == 0 ||
1066		    attr->alt_port_num > dev->dev->caps.num_ports)
1067			goto out;
1068
1069		if (attr->alt_pkey_index >=
1070		    dev->dev->caps.pkey_table_len[attr->alt_port_num])
1071			goto out;
1072
1073		if (mlx4_set_path(dev, &attr->alt_ah_attr, &context->alt_path,
1074				  attr->alt_port_num))
 
1075			goto out;
1076
1077		context->alt_path.pkey_index = attr->alt_pkey_index;
1078		context->alt_path.ackto = attr->alt_timeout << 3;
1079		optpar |= MLX4_QP_OPTPAR_ALT_ADDR_PATH;
1080	}
1081
1082	context->pd	    = cpu_to_be32(to_mpd(ibqp->pd)->pdn);
1083	context->params1    = cpu_to_be32(MLX4_IB_ACK_REQ_FREQ << 28);
 
 
 
 
 
 
 
 
 
1084
1085	/* Set "fast registration enabled" for all kernel QPs */
1086	if (!qp->ibqp.uobject)
1087		context->params1 |= cpu_to_be32(1 << 11);
1088
1089	if (attr_mask & IB_QP_RNR_RETRY) {
1090		context->params1 |= cpu_to_be32(attr->rnr_retry << 13);
1091		optpar |= MLX4_QP_OPTPAR_RNR_RETRY;
1092	}
1093
1094	if (attr_mask & IB_QP_RETRY_CNT) {
1095		context->params1 |= cpu_to_be32(attr->retry_cnt << 16);
1096		optpar |= MLX4_QP_OPTPAR_RETRY_COUNT;
1097	}
1098
1099	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC) {
1100		if (attr->max_rd_atomic)
1101			context->params1 |=
1102				cpu_to_be32(fls(attr->max_rd_atomic - 1) << 21);
1103		optpar |= MLX4_QP_OPTPAR_SRA_MAX;
1104	}
1105
1106	if (attr_mask & IB_QP_SQ_PSN)
1107		context->next_send_psn = cpu_to_be32(attr->sq_psn);
1108
1109	context->cqn_send = cpu_to_be32(to_mcq(ibqp->send_cq)->mcq.cqn);
1110
1111	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC) {
1112		if (attr->max_dest_rd_atomic)
1113			context->params2 |=
1114				cpu_to_be32(fls(attr->max_dest_rd_atomic - 1) << 21);
1115		optpar |= MLX4_QP_OPTPAR_RRA_MAX;
1116	}
1117
1118	if (attr_mask & (IB_QP_ACCESS_FLAGS | IB_QP_MAX_DEST_RD_ATOMIC)) {
1119		context->params2 |= to_mlx4_access_flags(qp, attr, attr_mask);
1120		optpar |= MLX4_QP_OPTPAR_RWE | MLX4_QP_OPTPAR_RRE | MLX4_QP_OPTPAR_RAE;
1121	}
1122
1123	if (ibqp->srq)
1124		context->params2 |= cpu_to_be32(MLX4_QP_BIT_RIC);
1125
1126	if (attr_mask & IB_QP_MIN_RNR_TIMER) {
1127		context->rnr_nextrecvpsn |= cpu_to_be32(attr->min_rnr_timer << 24);
1128		optpar |= MLX4_QP_OPTPAR_RNR_TIMEOUT;
1129	}
1130	if (attr_mask & IB_QP_RQ_PSN)
1131		context->rnr_nextrecvpsn |= cpu_to_be32(attr->rq_psn);
1132
1133	context->cqn_recv = cpu_to_be32(to_mcq(ibqp->recv_cq)->mcq.cqn);
1134
1135	if (attr_mask & IB_QP_QKEY) {
1136		context->qkey = cpu_to_be32(attr->qkey);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1137		optpar |= MLX4_QP_OPTPAR_Q_KEY;
1138	}
1139
1140	if (ibqp->srq)
1141		context->srqn = cpu_to_be32(1 << 24 | to_msrq(ibqp->srq)->msrq.srqn);
1142
1143	if (!ibqp->srq && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
 
 
 
1144		context->db_rec_addr = cpu_to_be64(qp->db.dma);
1145
1146	if (cur_state == IB_QPS_INIT &&
1147	    new_state == IB_QPS_RTR  &&
1148	    (ibqp->qp_type == IB_QPT_GSI || ibqp->qp_type == IB_QPT_SMI ||
1149	     ibqp->qp_type == IB_QPT_UD)) {
1150		context->pri_path.sched_queue = (qp->port - 1) << 6;
1151		if (is_qp0(dev, qp))
 
 
1152			context->pri_path.sched_queue |= MLX4_IB_DEFAULT_QP0_SCHED_QUEUE;
1153		else
 
 
 
 
1154			context->pri_path.sched_queue |= MLX4_IB_DEFAULT_SCHED_QUEUE;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1155	}
1156
1157	if (cur_state == IB_QPS_RTS && new_state == IB_QPS_SQD	&&
1158	    attr_mask & IB_QP_EN_SQD_ASYNC_NOTIFY && attr->en_sqd_async_notify)
1159		sqd_event = 1;
1160	else
1161		sqd_event = 0;
1162
1163	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT)
1164		context->rlkey |= (1 << 4);
 
 
1165
1166	/*
1167	 * Before passing a kernel QP to the HW, make sure that the
1168	 * ownership bits of the send queue are set and the SQ
1169	 * headroom is stamped so that the hardware doesn't start
1170	 * processing stale work requests.
1171	 */
1172	if (!ibqp->uobject && cur_state == IB_QPS_RESET && new_state == IB_QPS_INIT) {
 
 
1173		struct mlx4_wqe_ctrl_seg *ctrl;
1174		int i;
1175
1176		for (i = 0; i < qp->sq.wqe_cnt; ++i) {
1177			ctrl = get_send_wqe(qp, i);
1178			ctrl->owner_opcode = cpu_to_be32(1 << 31);
1179			if (qp->sq_max_wqes_per_wr == 1)
1180				ctrl->fence_size = 1 << (qp->sq.wqe_shift - 4);
1181
1182			stamp_send_wqe(qp, i, 1 << qp->sq.wqe_shift);
1183		}
1184	}
1185
 
 
 
 
 
 
 
1186	err = mlx4_qp_modify(dev->dev, &qp->mtt, to_mlx4_state(cur_state),
1187			     to_mlx4_state(new_state), context, optpar,
1188			     sqd_event, &qp->mqp);
1189	if (err)
1190		goto out;
1191
1192	qp->state = new_state;
1193
1194	if (attr_mask & IB_QP_ACCESS_FLAGS)
1195		qp->atomic_rd_en = attr->qp_access_flags;
1196	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC)
1197		qp->resp_depth = attr->max_dest_rd_atomic;
1198	if (attr_mask & IB_QP_PORT) {
1199		qp->port = attr->port_num;
1200		update_mcg_macs(dev, qp);
1201	}
1202	if (attr_mask & IB_QP_ALT_PATH)
1203		qp->alt_port = attr->alt_port_num;
1204
1205	if (is_sqp(dev, qp))
1206		store_sqp_attrs(to_msqp(qp), attr, attr_mask);
1207
1208	/*
1209	 * If we moved QP0 to RTR, bring the IB link up; if we moved
1210	 * QP0 to RESET or ERROR, bring the link back down.
1211	 */
1212	if (is_qp0(dev, qp)) {
1213		if (cur_state != IB_QPS_RTR && new_state == IB_QPS_RTR)
1214			if (mlx4_INIT_PORT(dev->dev, qp->port))
1215				printk(KERN_WARNING "INIT_PORT failed for port %d\n",
1216				       qp->port);
1217
1218		if (cur_state != IB_QPS_RESET && cur_state != IB_QPS_ERR &&
1219		    (new_state == IB_QPS_RESET || new_state == IB_QPS_ERR))
1220			mlx4_CLOSE_PORT(dev->dev, qp->port);
1221	}
1222
1223	/*
1224	 * If we moved a kernel QP to RESET, clean up all old CQ
1225	 * entries and reinitialize the QP.
1226	 */
1227	if (new_state == IB_QPS_RESET && !ibqp->uobject) {
1228		mlx4_ib_cq_clean(to_mcq(ibqp->recv_cq), qp->mqp.qpn,
1229				 ibqp->srq ? to_msrq(ibqp->srq): NULL);
1230		if (ibqp->send_cq != ibqp->recv_cq)
1231			mlx4_ib_cq_clean(to_mcq(ibqp->send_cq), qp->mqp.qpn, NULL);
1232
1233		qp->rq.head = 0;
1234		qp->rq.tail = 0;
1235		qp->sq.head = 0;
1236		qp->sq.tail = 0;
1237		qp->sq_next_wqe = 0;
1238		if (!ibqp->srq)
1239			*qp->db.db  = 0;
1240	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1241
 
 
 
 
 
 
 
1242out:
 
 
 
 
1243	kfree(context);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1244	return err;
1245}
1246
1247int mlx4_ib_modify_qp(struct ib_qp *ibqp, struct ib_qp_attr *attr,
1248		      int attr_mask, struct ib_udata *udata)
 
 
 
 
 
1249{
1250	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
1251	struct mlx4_ib_qp *qp = to_mqp(ibqp);
1252	enum ib_qp_state cur_state, new_state;
1253	int err = -EINVAL;
1254
1255	mutex_lock(&qp->mutex);
1256
1257	cur_state = attr_mask & IB_QP_CUR_STATE ? attr->cur_qp_state : qp->state;
1258	new_state = attr_mask & IB_QP_STATE ? attr->qp_state : cur_state;
1259
1260	if (!ib_modify_qp_is_ok(cur_state, new_state, ibqp->qp_type, attr_mask))
 
 
 
 
 
 
1261		goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1262
1263	if ((attr_mask & IB_QP_PORT) &&
1264	    (attr->port_num == 0 || attr->port_num > dev->dev->caps.num_ports)) {
 
 
 
 
1265		goto out;
1266	}
1267
 
 
 
 
 
1268	if (attr_mask & IB_QP_PKEY_INDEX) {
1269		int p = attr_mask & IB_QP_PORT ? attr->port_num : qp->port;
1270		if (attr->pkey_index >= dev->dev->caps.pkey_table_len[p])
 
 
 
 
1271			goto out;
 
1272	}
1273
1274	if (attr_mask & IB_QP_MAX_QP_RD_ATOMIC &&
1275	    attr->max_rd_atomic > dev->dev->caps.max_qp_init_rdma) {
 
 
 
 
1276		goto out;
1277	}
1278
1279	if (attr_mask & IB_QP_MAX_DEST_RD_ATOMIC &&
1280	    attr->max_dest_rd_atomic > dev->dev->caps.max_qp_dest_rdma) {
 
 
 
 
1281		goto out;
1282	}
1283
1284	if (cur_state == new_state && cur_state == IB_QPS_RESET) {
1285		err = 0;
1286		goto out;
1287	}
1288
1289	err = __mlx4_ib_modify_qp(ibqp, attr, attr_mask, cur_state, new_state);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1290
1291out:
1292	mutex_unlock(&qp->mutex);
1293	return err;
1294}
1295
1296static int build_mlx_header(struct mlx4_ib_sqp *sqp, struct ib_send_wr *wr,
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1297			    void *wqe, unsigned *mlx_seg_len)
1298{
1299	struct ib_device *ib_dev = sqp->qp.ibqp.device;
 
1300	struct mlx4_wqe_mlx_seg *mlx = wqe;
 
1301	struct mlx4_wqe_inline_seg *inl = wqe + sizeof *mlx;
1302	struct mlx4_ib_ah *ah = to_mah(wr->wr.ud.ah);
1303	union ib_gid sgid;
1304	u16 pkey;
1305	int send_size;
1306	int header_size;
1307	int spc;
1308	int i;
1309	int is_eth;
1310	int is_vlan = 0;
1311	int is_grh;
1312	u16 vlan;
 
 
 
1313
1314	send_size = 0;
1315	for (i = 0; i < wr->num_sge; ++i)
1316		send_size += wr->sg_list[i].length;
1317
1318	is_eth = rdma_port_get_link_layer(sqp->qp.ibqp.device, sqp->qp.port) == IB_LINK_LAYER_ETHERNET;
1319	is_grh = mlx4_ib_ah_grh_present(ah);
1320	if (is_eth) {
1321		ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.ib.port_pd) >> 24,
1322				  ah->av.ib.gid_index, &sgid);
1323		vlan = rdma_get_vlan_id(&sgid);
1324		is_vlan = vlan < 0x1000;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1325	}
1326	ib_ud_header_init(send_size, !is_eth, is_eth, is_vlan, is_grh, 0, &sqp->ud_header);
 
 
 
1327
1328	if (!is_eth) {
1329		sqp->ud_header.lrh.service_level =
1330			be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 28;
1331		sqp->ud_header.lrh.destination_lid = ah->av.ib.dlid;
1332		sqp->ud_header.lrh.source_lid = cpu_to_be16(ah->av.ib.g_slid & 0x7f);
1333	}
1334
1335	if (is_grh) {
1336		sqp->ud_header.grh.traffic_class =
1337			(be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 20) & 0xff;
1338		sqp->ud_header.grh.flow_label    =
1339			ah->av.ib.sl_tclass_flowlabel & cpu_to_be32(0xfffff);
1340		sqp->ud_header.grh.hop_limit     = ah->av.ib.hop_limit;
1341		ib_get_cached_gid(ib_dev, be32_to_cpu(ah->av.ib.port_pd) >> 24,
1342				  ah->av.ib.gid_index, &sqp->ud_header.grh.source_gid);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1343		memcpy(sqp->ud_header.grh.destination_gid.raw,
1344		       ah->av.ib.dgid, 16);
1345	}
1346
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1347	mlx->flags &= cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE);
1348
1349	if (!is_eth) {
1350		mlx->flags |= cpu_to_be32((!sqp->qp.ibqp.qp_num ? MLX4_WQE_MLX_VL15 : 0) |
1351					  (sqp->ud_header.lrh.destination_lid ==
1352					   IB_LID_PERMISSIVE ? MLX4_WQE_MLX_SLR : 0) |
1353					  (sqp->ud_header.lrh.service_level << 8));
 
 
1354		mlx->rlid = sqp->ud_header.lrh.destination_lid;
1355	}
1356
1357	switch (wr->opcode) {
1358	case IB_WR_SEND:
1359		sqp->ud_header.bth.opcode	 = IB_OPCODE_UD_SEND_ONLY;
1360		sqp->ud_header.immediate_present = 0;
1361		break;
1362	case IB_WR_SEND_WITH_IMM:
1363		sqp->ud_header.bth.opcode	 = IB_OPCODE_UD_SEND_ONLY_WITH_IMMEDIATE;
1364		sqp->ud_header.immediate_present = 1;
1365		sqp->ud_header.immediate_data    = wr->ex.imm_data;
1366		break;
1367	default:
1368		return -EINVAL;
1369	}
1370
1371	if (is_eth) {
1372		u8 *smac;
 
 
1373
 
 
 
 
 
 
1374		memcpy(sqp->ud_header.eth.dmac_h, ah->av.eth.mac, 6);
1375		/* FIXME: cache smac value? */
1376		smac = to_mdev(sqp->qp.ibqp.device)->iboe.netdevs[sqp->qp.port - 1]->dev_addr;
1377		memcpy(sqp->ud_header.eth.smac_h, smac, 6);
 
 
1378		if (!memcmp(sqp->ud_header.eth.smac_h, sqp->ud_header.eth.dmac_h, 6))
1379			mlx->flags |= cpu_to_be32(MLX4_WQE_CTRL_FORCE_LOOPBACK);
1380		if (!is_vlan) {
1381			sqp->ud_header.eth.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
1382		} else {
1383			u16 pcp;
1384
1385			sqp->ud_header.vlan.type = cpu_to_be16(MLX4_IB_IBOE_ETHERTYPE);
1386			pcp = (be32_to_cpu(ah->av.ib.sl_tclass_flowlabel) >> 27 & 3) << 13;
1387			sqp->ud_header.vlan.tag = cpu_to_be16(vlan | pcp);
1388		}
1389	} else {
1390		sqp->ud_header.lrh.virtual_lane    = !sqp->qp.ibqp.qp_num ? 15 : 0;
 
 
 
 
 
1391		if (sqp->ud_header.lrh.destination_lid == IB_LID_PERMISSIVE)
1392			sqp->ud_header.lrh.source_lid = IB_LID_PERMISSIVE;
1393	}
1394	sqp->ud_header.bth.solicited_event = !!(wr->send_flags & IB_SEND_SOLICITED);
1395	if (!sqp->qp.ibqp.qp_num)
1396		ib_get_cached_pkey(ib_dev, sqp->qp.port, sqp->pkey_index, &pkey);
 
1397	else
1398		ib_get_cached_pkey(ib_dev, sqp->qp.port, wr->wr.ud.pkey_index, &pkey);
 
 
 
 
1399	sqp->ud_header.bth.pkey = cpu_to_be16(pkey);
1400	sqp->ud_header.bth.destination_qpn = cpu_to_be32(wr->wr.ud.remote_qpn);
1401	sqp->ud_header.bth.psn = cpu_to_be32((sqp->send_psn++) & ((1 << 24) - 1));
1402	sqp->ud_header.deth.qkey = cpu_to_be32(wr->wr.ud.remote_qkey & 0x80000000 ?
1403					       sqp->qkey : wr->wr.ud.remote_qkey);
1404	sqp->ud_header.deth.source_qpn = cpu_to_be32(sqp->qp.ibqp.qp_num);
1405
1406	header_size = ib_ud_header_pack(&sqp->ud_header, sqp->header_buf);
1407
1408	if (0) {
1409		printk(KERN_ERR "built UD header of size %d:\n", header_size);
1410		for (i = 0; i < header_size / 4; ++i) {
1411			if (i % 8 == 0)
1412				printk("  [%02x] ", i * 4);
1413			printk(" %08x",
1414			       be32_to_cpu(((__be32 *) sqp->header_buf)[i]));
1415			if ((i + 1) % 8 == 0)
1416				printk("\n");
1417		}
1418		printk("\n");
1419	}
1420
1421	/*
1422	 * Inline data segments may not cross a 64 byte boundary.  If
1423	 * our UD header is bigger than the space available up to the
1424	 * next 64 byte boundary in the WQE, use two inline data
1425	 * segments to hold the UD header.
1426	 */
1427	spc = MLX4_INLINE_ALIGN -
1428		((unsigned long) (inl + 1) & (MLX4_INLINE_ALIGN - 1));
1429	if (header_size <= spc) {
1430		inl->byte_count = cpu_to_be32(1 << 31 | header_size);
1431		memcpy(inl + 1, sqp->header_buf, header_size);
1432		i = 1;
1433	} else {
1434		inl->byte_count = cpu_to_be32(1 << 31 | spc);
1435		memcpy(inl + 1, sqp->header_buf, spc);
1436
1437		inl = (void *) (inl + 1) + spc;
1438		memcpy(inl + 1, sqp->header_buf + spc, header_size - spc);
1439		/*
1440		 * Need a barrier here to make sure all the data is
1441		 * visible before the byte_count field is set.
1442		 * Otherwise the HCA prefetcher could grab the 64-byte
1443		 * chunk with this inline segment and get a valid (!=
1444		 * 0xffffffff) byte count but stale data, and end up
1445		 * generating a packet with bad headers.
1446		 *
1447		 * The first inline segment's byte_count field doesn't
1448		 * need a barrier, because it comes after a
1449		 * control/MLX segment and therefore is at an offset
1450		 * of 16 mod 64.
1451		 */
1452		wmb();
1453		inl->byte_count = cpu_to_be32(1 << 31 | (header_size - spc));
1454		i = 2;
1455	}
1456
1457	*mlx_seg_len =
1458		ALIGN(i * sizeof (struct mlx4_wqe_inline_seg) + header_size, 16);
1459	return 0;
1460}
1461
1462static int mlx4_wq_overflow(struct mlx4_ib_wq *wq, int nreq, struct ib_cq *ib_cq)
1463{
1464	unsigned cur;
1465	struct mlx4_ib_cq *cq;
1466
1467	cur = wq->head - wq->tail;
1468	if (likely(cur + nreq < wq->max_post))
1469		return 0;
1470
1471	cq = to_mcq(ib_cq);
1472	spin_lock(&cq->lock);
1473	cur = wq->head - wq->tail;
1474	spin_unlock(&cq->lock);
1475
1476	return cur + nreq >= wq->max_post;
1477}
1478
1479static __be32 convert_access(int acc)
1480{
1481	return (acc & IB_ACCESS_REMOTE_ATOMIC ? cpu_to_be32(MLX4_WQE_FMR_PERM_ATOMIC)       : 0) |
1482	       (acc & IB_ACCESS_REMOTE_WRITE  ? cpu_to_be32(MLX4_WQE_FMR_PERM_REMOTE_WRITE) : 0) |
1483	       (acc & IB_ACCESS_REMOTE_READ   ? cpu_to_be32(MLX4_WQE_FMR_PERM_REMOTE_READ)  : 0) |
 
 
 
1484	       (acc & IB_ACCESS_LOCAL_WRITE   ? cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_WRITE)  : 0) |
1485		cpu_to_be32(MLX4_WQE_FMR_PERM_LOCAL_READ);
1486}
1487
1488static void set_fmr_seg(struct mlx4_wqe_fmr_seg *fseg, struct ib_send_wr *wr)
 
1489{
1490	struct mlx4_ib_fast_reg_page_list *mfrpl = to_mfrpl(wr->wr.fast_reg.page_list);
1491	int i;
1492
1493	for (i = 0; i < wr->wr.fast_reg.page_list_len; ++i)
1494		mfrpl->mapped_page_list[i] =
1495			cpu_to_be64(wr->wr.fast_reg.page_list->page_list[i] |
1496				    MLX4_MTT_FLAG_PRESENT);
1497
1498	fseg->flags		= convert_access(wr->wr.fast_reg.access_flags);
1499	fseg->mem_key		= cpu_to_be32(wr->wr.fast_reg.rkey);
1500	fseg->buf_list		= cpu_to_be64(mfrpl->map);
1501	fseg->start_addr	= cpu_to_be64(wr->wr.fast_reg.iova_start);
1502	fseg->reg_len		= cpu_to_be64(wr->wr.fast_reg.length);
1503	fseg->offset		= 0; /* XXX -- is this just for ZBVA? */
1504	fseg->page_size		= cpu_to_be32(wr->wr.fast_reg.page_shift);
1505	fseg->reserved[0]	= 0;
1506	fseg->reserved[1]	= 0;
1507}
1508
1509static void set_local_inv_seg(struct mlx4_wqe_local_inval_seg *iseg, u32 rkey)
1510{
1511	iseg->flags	= 0;
1512	iseg->mem_key	= cpu_to_be32(rkey);
1513	iseg->guest_id	= 0;
1514	iseg->pa	= 0;
1515}
1516
1517static __always_inline void set_raddr_seg(struct mlx4_wqe_raddr_seg *rseg,
1518					  u64 remote_addr, u32 rkey)
1519{
1520	rseg->raddr    = cpu_to_be64(remote_addr);
1521	rseg->rkey     = cpu_to_be32(rkey);
1522	rseg->reserved = 0;
1523}
1524
1525static void set_atomic_seg(struct mlx4_wqe_atomic_seg *aseg, struct ib_send_wr *wr)
 
1526{
1527	if (wr->opcode == IB_WR_ATOMIC_CMP_AND_SWP) {
1528		aseg->swap_add = cpu_to_be64(wr->wr.atomic.swap);
1529		aseg->compare  = cpu_to_be64(wr->wr.atomic.compare_add);
1530	} else if (wr->opcode == IB_WR_MASKED_ATOMIC_FETCH_AND_ADD) {
1531		aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add);
1532		aseg->compare  = cpu_to_be64(wr->wr.atomic.compare_add_mask);
1533	} else {
1534		aseg->swap_add = cpu_to_be64(wr->wr.atomic.compare_add);
1535		aseg->compare  = 0;
1536	}
1537
1538}
1539
1540static void set_masked_atomic_seg(struct mlx4_wqe_masked_atomic_seg *aseg,
1541				  struct ib_send_wr *wr)
1542{
1543	aseg->swap_add		= cpu_to_be64(wr->wr.atomic.swap);
1544	aseg->swap_add_mask	= cpu_to_be64(wr->wr.atomic.swap_mask);
1545	aseg->compare		= cpu_to_be64(wr->wr.atomic.compare_add);
1546	aseg->compare_mask	= cpu_to_be64(wr->wr.atomic.compare_add_mask);
1547}
1548
1549static void set_datagram_seg(struct mlx4_wqe_datagram_seg *dseg,
1550			     struct ib_send_wr *wr, __be16 *vlan)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1551{
1552	memcpy(dseg->av, &to_mah(wr->wr.ud.ah)->av, sizeof (struct mlx4_av));
1553	dseg->dqpn = cpu_to_be32(wr->wr.ud.remote_qpn);
1554	dseg->qkey = cpu_to_be32(wr->wr.ud.remote_qkey);
1555	dseg->vlan = to_mah(wr->wr.ud.ah)->av.eth.vlan;
1556	memcpy(dseg->mac, to_mah(wr->wr.ud.ah)->av.eth.mac, 6);
1557	*vlan = dseg->vlan;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1558}
1559
1560static void set_mlx_icrc_seg(void *dseg)
1561{
1562	u32 *t = dseg;
1563	struct mlx4_wqe_inline_seg *iseg = dseg;
1564
1565	t[1] = 0;
1566
1567	/*
1568	 * Need a barrier here before writing the byte_count field to
1569	 * make sure that all the data is visible before the
1570	 * byte_count field is set.  Otherwise, if the segment begins
1571	 * a new cacheline, the HCA prefetcher could grab the 64-byte
1572	 * chunk and get a valid (!= * 0xffffffff) byte count but
1573	 * stale data, and end up sending the wrong data.
1574	 */
1575	wmb();
1576
1577	iseg->byte_count = cpu_to_be32((1 << 31) | 4);
1578}
1579
1580static void set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
1581{
1582	dseg->lkey       = cpu_to_be32(sg->lkey);
1583	dseg->addr       = cpu_to_be64(sg->addr);
1584
1585	/*
1586	 * Need a barrier here before writing the byte_count field to
1587	 * make sure that all the data is visible before the
1588	 * byte_count field is set.  Otherwise, if the segment begins
1589	 * a new cacheline, the HCA prefetcher could grab the 64-byte
1590	 * chunk and get a valid (!= * 0xffffffff) byte count but
1591	 * stale data, and end up sending the wrong data.
1592	 */
1593	wmb();
1594
1595	dseg->byte_count = cpu_to_be32(sg->length);
1596}
1597
1598static void __set_data_seg(struct mlx4_wqe_data_seg *dseg, struct ib_sge *sg)
1599{
1600	dseg->byte_count = cpu_to_be32(sg->length);
1601	dseg->lkey       = cpu_to_be32(sg->lkey);
1602	dseg->addr       = cpu_to_be64(sg->addr);
1603}
1604
1605static int build_lso_seg(struct mlx4_wqe_lso_seg *wqe, struct ib_send_wr *wr,
1606			 struct mlx4_ib_qp *qp, unsigned *lso_seg_len,
1607			 __be32 *lso_hdr_sz, __be32 *blh)
1608{
1609	unsigned halign = ALIGN(sizeof *wqe + wr->wr.ud.hlen, 16);
1610
1611	if (unlikely(halign > MLX4_IB_CACHE_LINE_SIZE))
1612		*blh = cpu_to_be32(1 << 6);
1613
1614	if (unlikely(!(qp->flags & MLX4_IB_QP_LSO) &&
1615		     wr->num_sge > qp->sq.max_gs - (halign >> 4)))
1616		return -EINVAL;
1617
1618	memcpy(wqe->header, wr->wr.ud.header, wr->wr.ud.hlen);
1619
1620	*lso_hdr_sz  = cpu_to_be32((wr->wr.ud.mss - wr->wr.ud.hlen) << 16 |
1621				   wr->wr.ud.hlen);
1622	*lso_seg_len = halign;
1623	return 0;
1624}
1625
1626static __be32 send_ieth(struct ib_send_wr *wr)
1627{
1628	switch (wr->opcode) {
1629	case IB_WR_SEND_WITH_IMM:
1630	case IB_WR_RDMA_WRITE_WITH_IMM:
1631		return wr->ex.imm_data;
1632
1633	case IB_WR_SEND_WITH_INV:
1634		return cpu_to_be32(wr->ex.invalidate_rkey);
1635
1636	default:
1637		return 0;
1638	}
1639}
1640
1641int mlx4_ib_post_send(struct ib_qp *ibqp, struct ib_send_wr *wr,
1642		      struct ib_send_wr **bad_wr)
 
 
 
 
 
 
 
1643{
1644	struct mlx4_ib_qp *qp = to_mqp(ibqp);
1645	void *wqe;
1646	struct mlx4_wqe_ctrl_seg *ctrl;
1647	struct mlx4_wqe_data_seg *dseg;
1648	unsigned long flags;
1649	int nreq;
1650	int err = 0;
1651	unsigned ind;
1652	int uninitialized_var(stamp);
1653	int uninitialized_var(size);
1654	unsigned uninitialized_var(seglen);
1655	__be32 dummy;
1656	__be32 *lso_wqe;
1657	__be32 uninitialized_var(lso_hdr_sz);
1658	__be32 blh;
1659	int i;
1660	__be16 vlan = cpu_to_be16(0xffff);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1661
1662	spin_lock_irqsave(&qp->sq.lock, flags);
 
 
 
 
 
 
 
1663
1664	ind = qp->sq_next_wqe;
1665
1666	for (nreq = 0; wr; ++nreq, wr = wr->next) {
1667		lso_wqe = &dummy;
1668		blh = 0;
1669
1670		if (mlx4_wq_overflow(&qp->sq, nreq, qp->ibqp.send_cq)) {
1671			err = -ENOMEM;
1672			*bad_wr = wr;
1673			goto out;
1674		}
1675
1676		if (unlikely(wr->num_sge > qp->sq.max_gs)) {
1677			err = -EINVAL;
1678			*bad_wr = wr;
1679			goto out;
1680		}
1681
1682		ctrl = wqe = get_send_wqe(qp, ind & (qp->sq.wqe_cnt - 1));
1683		qp->sq.wrid[(qp->sq.head + nreq) & (qp->sq.wqe_cnt - 1)] = wr->wr_id;
1684
1685		ctrl->srcrb_flags =
1686			(wr->send_flags & IB_SEND_SIGNALED ?
1687			 cpu_to_be32(MLX4_WQE_CTRL_CQ_UPDATE) : 0) |
1688			(wr->send_flags & IB_SEND_SOLICITED ?
1689			 cpu_to_be32(MLX4_WQE_CTRL_SOLICITED) : 0) |
1690			((wr->send_flags & IB_SEND_IP_CSUM) ?
1691			 cpu_to_be32(MLX4_WQE_CTRL_IP_CSUM |
1692				     MLX4_WQE_CTRL_TCP_UDP_CSUM) : 0) |
1693			qp->sq_signal_bits;
1694
1695		ctrl->imm = send_ieth(wr);
1696
1697		wqe += sizeof *ctrl;
1698		size = sizeof *ctrl / 16;
1699
1700		switch (ibqp->qp_type) {
1701		case IB_QPT_RC:
1702		case IB_QPT_UC:
1703			switch (wr->opcode) {
1704			case IB_WR_ATOMIC_CMP_AND_SWP:
1705			case IB_WR_ATOMIC_FETCH_AND_ADD:
1706			case IB_WR_MASKED_ATOMIC_FETCH_AND_ADD:
1707				set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
1708					      wr->wr.atomic.rkey);
1709				wqe  += sizeof (struct mlx4_wqe_raddr_seg);
1710
1711				set_atomic_seg(wqe, wr);
1712				wqe  += sizeof (struct mlx4_wqe_atomic_seg);
1713
1714				size += (sizeof (struct mlx4_wqe_raddr_seg) +
1715					 sizeof (struct mlx4_wqe_atomic_seg)) / 16;
1716
1717				break;
1718
1719			case IB_WR_MASKED_ATOMIC_CMP_AND_SWP:
1720				set_raddr_seg(wqe, wr->wr.atomic.remote_addr,
1721					      wr->wr.atomic.rkey);
1722				wqe  += sizeof (struct mlx4_wqe_raddr_seg);
1723
1724				set_masked_atomic_seg(wqe, wr);
1725				wqe  += sizeof (struct mlx4_wqe_masked_atomic_seg);
1726
1727				size += (sizeof (struct mlx4_wqe_raddr_seg) +
1728					 sizeof (struct mlx4_wqe_masked_atomic_seg)) / 16;
1729
1730				break;
1731
1732			case IB_WR_RDMA_READ:
1733			case IB_WR_RDMA_WRITE:
1734			case IB_WR_RDMA_WRITE_WITH_IMM:
1735				set_raddr_seg(wqe, wr->wr.rdma.remote_addr,
1736					      wr->wr.rdma.rkey);
1737				wqe  += sizeof (struct mlx4_wqe_raddr_seg);
1738				size += sizeof (struct mlx4_wqe_raddr_seg) / 16;
1739				break;
1740
1741			case IB_WR_LOCAL_INV:
1742				ctrl->srcrb_flags |=
1743					cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
1744				set_local_inv_seg(wqe, wr->ex.invalidate_rkey);
1745				wqe  += sizeof (struct mlx4_wqe_local_inval_seg);
1746				size += sizeof (struct mlx4_wqe_local_inval_seg) / 16;
1747				break;
1748
1749			case IB_WR_FAST_REG_MR:
1750				ctrl->srcrb_flags |=
1751					cpu_to_be32(MLX4_WQE_CTRL_STRONG_ORDER);
1752				set_fmr_seg(wqe, wr);
1753				wqe  += sizeof (struct mlx4_wqe_fmr_seg);
1754				size += sizeof (struct mlx4_wqe_fmr_seg) / 16;
1755				break;
1756
1757			default:
1758				/* No extra segments required for sends */
1759				break;
1760			}
1761			break;
1762
1763		case IB_QPT_UD:
1764			set_datagram_seg(wqe, wr, &vlan);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1765			wqe  += sizeof (struct mlx4_wqe_datagram_seg);
1766			size += sizeof (struct mlx4_wqe_datagram_seg) / 16;
1767
1768			if (wr->opcode == IB_WR_LSO) {
1769				err = build_lso_seg(wqe, wr, qp, &seglen, &lso_hdr_sz, &blh);
 
1770				if (unlikely(err)) {
1771					*bad_wr = wr;
1772					goto out;
1773				}
1774				lso_wqe = (__be32 *) wqe;
1775				wqe  += seglen;
1776				size += seglen / 16;
1777			}
1778			break;
1779
1780		case IB_QPT_SMI:
1781		case IB_QPT_GSI:
1782			err = build_mlx_header(to_msqp(qp), wr, ctrl, &seglen);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1783			if (unlikely(err)) {
1784				*bad_wr = wr;
1785				goto out;
1786			}
1787			wqe  += seglen;
1788			size += seglen / 16;
1789			break;
1790
1791		default:
1792			break;
1793		}
1794
1795		/*
1796		 * Write data segments in reverse order, so as to
1797		 * overwrite cacheline stamp last within each
1798		 * cacheline.  This avoids issues with WQE
1799		 * prefetching.
1800		 */
1801
1802		dseg = wqe;
1803		dseg += wr->num_sge - 1;
1804		size += wr->num_sge * (sizeof (struct mlx4_wqe_data_seg) / 16);
1805
1806		/* Add one more inline data segment for ICRC for MLX sends */
1807		if (unlikely(qp->ibqp.qp_type == IB_QPT_SMI ||
1808			     qp->ibqp.qp_type == IB_QPT_GSI)) {
 
 
1809			set_mlx_icrc_seg(dseg + 1);
1810			size += sizeof (struct mlx4_wqe_data_seg) / 16;
1811		}
1812
1813		for (i = wr->num_sge - 1; i >= 0; --i, --dseg)
1814			set_data_seg(dseg, wr->sg_list + i);
1815
1816		/*
1817		 * Possibly overwrite stamping in cacheline with LSO
1818		 * segment only after making sure all data segments
1819		 * are written.
1820		 */
1821		wmb();
1822		*lso_wqe = lso_hdr_sz;
1823
1824		ctrl->fence_size = (wr->send_flags & IB_SEND_FENCE ?
1825				    MLX4_WQE_CTRL_FENCE : 0) | size;
1826
1827		if (be16_to_cpu(vlan) < 0x1000) {
1828			ctrl->ins_vlan = 1 << 6;
1829			ctrl->vlan_tag = vlan;
1830		}
1831
1832		/*
1833		 * Make sure descriptor is fully written before
1834		 * setting ownership bit (because HW can start
1835		 * executing as soon as we do).
1836		 */
1837		wmb();
1838
1839		if (wr->opcode < 0 || wr->opcode >= ARRAY_SIZE(mlx4_ib_opcode)) {
 
1840			err = -EINVAL;
1841			goto out;
1842		}
1843
1844		ctrl->owner_opcode = mlx4_ib_opcode[wr->opcode] |
1845			(ind & qp->sq.wqe_cnt ? cpu_to_be32(1 << 31) : 0) | blh;
1846
1847		stamp = ind + qp->sq_spare_wqes;
1848		ind += DIV_ROUND_UP(size * 16, 1U << qp->sq.wqe_shift);
1849
1850		/*
1851		 * We can improve latency by not stamping the last
1852		 * send queue WQE until after ringing the doorbell, so
1853		 * only stamp here if there are still more WQEs to post.
1854		 *
1855		 * Same optimization applies to padding with NOP wqe
1856		 * in case of WQE shrinking (used to prevent wrap-around
1857		 * in the middle of WR).
1858		 */
1859		if (wr->next) {
1860			stamp_send_wqe(qp, stamp, size * 16);
1861			ind = pad_wraparound(qp, ind);
1862		}
1863	}
1864
1865out:
1866	if (likely(nreq)) {
1867		qp->sq.head += nreq;
1868
1869		/*
1870		 * Make sure that descriptors are written before
1871		 * doorbell record.
1872		 */
1873		wmb();
1874
1875		writel(qp->doorbell_qpn,
1876		       to_mdev(ibqp->device)->uar_map + MLX4_SEND_DOORBELL);
1877
1878		/*
1879		 * Make sure doorbells don't leak out of SQ spinlock
1880		 * and reach the HCA out of order.
1881		 */
1882		mmiowb();
1883
1884		stamp_send_wqe(qp, stamp, size * 16);
1885
1886		ind = pad_wraparound(qp, ind);
1887		qp->sq_next_wqe = ind;
1888	}
1889
1890	spin_unlock_irqrestore(&qp->sq.lock, flags);
1891
1892	return err;
1893}
1894
1895int mlx4_ib_post_recv(struct ib_qp *ibqp, struct ib_recv_wr *wr,
1896		      struct ib_recv_wr **bad_wr)
 
 
 
 
 
 
1897{
1898	struct mlx4_ib_qp *qp = to_mqp(ibqp);
1899	struct mlx4_wqe_data_seg *scat;
1900	unsigned long flags;
1901	int err = 0;
1902	int nreq;
1903	int ind;
 
1904	int i;
 
1905
 
1906	spin_lock_irqsave(&qp->rq.lock, flags);
1907
 
 
 
 
 
 
 
 
1908	ind = qp->rq.head & (qp->rq.wqe_cnt - 1);
1909
1910	for (nreq = 0; wr; ++nreq, wr = wr->next) {
1911		if (mlx4_wq_overflow(&qp->rq, nreq, qp->ibqp.recv_cq)) {
1912			err = -ENOMEM;
1913			*bad_wr = wr;
1914			goto out;
1915		}
1916
1917		if (unlikely(wr->num_sge > qp->rq.max_gs)) {
1918			err = -EINVAL;
1919			*bad_wr = wr;
1920			goto out;
1921		}
1922
1923		scat = get_recv_wqe(qp, ind);
1924
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1925		for (i = 0; i < wr->num_sge; ++i)
1926			__set_data_seg(scat + i, wr->sg_list + i);
1927
1928		if (i < qp->rq.max_gs) {
1929			scat[i].byte_count = 0;
1930			scat[i].lkey       = cpu_to_be32(MLX4_INVALID_LKEY);
1931			scat[i].addr       = 0;
1932		}
1933
1934		qp->rq.wrid[ind] = wr->wr_id;
1935
1936		ind = (ind + 1) & (qp->rq.wqe_cnt - 1);
1937	}
1938
1939out:
1940	if (likely(nreq)) {
1941		qp->rq.head += nreq;
1942
1943		/*
1944		 * Make sure that descriptors are written before
1945		 * doorbell record.
1946		 */
1947		wmb();
1948
1949		*qp->db.db = cpu_to_be32(qp->rq.head & 0xffff);
1950	}
1951
1952	spin_unlock_irqrestore(&qp->rq.lock, flags);
1953
1954	return err;
1955}
1956
 
 
 
 
 
 
1957static inline enum ib_qp_state to_ib_qp_state(enum mlx4_qp_state mlx4_state)
1958{
1959	switch (mlx4_state) {
1960	case MLX4_QP_STATE_RST:      return IB_QPS_RESET;
1961	case MLX4_QP_STATE_INIT:     return IB_QPS_INIT;
1962	case MLX4_QP_STATE_RTR:      return IB_QPS_RTR;
1963	case MLX4_QP_STATE_RTS:      return IB_QPS_RTS;
1964	case MLX4_QP_STATE_SQ_DRAINING:
1965	case MLX4_QP_STATE_SQD:      return IB_QPS_SQD;
1966	case MLX4_QP_STATE_SQER:     return IB_QPS_SQE;
1967	case MLX4_QP_STATE_ERR:      return IB_QPS_ERR;
1968	default:		     return -1;
1969	}
1970}
1971
1972static inline enum ib_mig_state to_ib_mig_state(int mlx4_mig_state)
1973{
1974	switch (mlx4_mig_state) {
1975	case MLX4_QP_PM_ARMED:		return IB_MIG_ARMED;
1976	case MLX4_QP_PM_REARM:		return IB_MIG_REARM;
1977	case MLX4_QP_PM_MIGRATED:	return IB_MIG_MIGRATED;
1978	default: return -1;
1979	}
1980}
1981
1982static int to_ib_qp_access_flags(int mlx4_flags)
1983{
1984	int ib_flags = 0;
1985
1986	if (mlx4_flags & MLX4_QP_BIT_RRE)
1987		ib_flags |= IB_ACCESS_REMOTE_READ;
1988	if (mlx4_flags & MLX4_QP_BIT_RWE)
1989		ib_flags |= IB_ACCESS_REMOTE_WRITE;
1990	if (mlx4_flags & MLX4_QP_BIT_RAE)
1991		ib_flags |= IB_ACCESS_REMOTE_ATOMIC;
1992
1993	return ib_flags;
1994}
1995
1996static void to_ib_ah_attr(struct mlx4_ib_dev *ibdev, struct ib_ah_attr *ib_ah_attr,
1997				struct mlx4_qp_path *path)
 
1998{
1999	struct mlx4_dev *dev = ibdev->dev;
2000	int is_eth;
2001
2002	memset(ib_ah_attr, 0, sizeof *ib_ah_attr);
2003	ib_ah_attr->port_num	  = path->sched_queue & 0x40 ? 2 : 1;
2004
2005	if (ib_ah_attr->port_num == 0 || ib_ah_attr->port_num > dev->caps.num_ports)
 
2006		return;
 
2007
2008	is_eth = rdma_port_get_link_layer(&ibdev->ib_dev, ib_ah_attr->port_num) ==
2009		IB_LINK_LAYER_ETHERNET;
2010	if (is_eth)
2011		ib_ah_attr->sl = ((path->sched_queue >> 3) & 0x7) |
2012		((path->sched_queue & 4) << 1);
2013	else
2014		ib_ah_attr->sl = (path->sched_queue >> 2) & 0xf;
 
2015
2016	ib_ah_attr->dlid	  = be16_to_cpu(path->rlid);
2017	ib_ah_attr->src_path_bits = path->grh_mylmc & 0x7f;
2018	ib_ah_attr->static_rate   = path->static_rate ? path->static_rate - 5 : 0;
2019	ib_ah_attr->ah_flags      = (path->grh_mylmc & (1 << 7)) ? IB_AH_GRH : 0;
2020	if (ib_ah_attr->ah_flags) {
2021		ib_ah_attr->grh.sgid_index = path->mgid_index;
2022		ib_ah_attr->grh.hop_limit  = path->hop_limit;
2023		ib_ah_attr->grh.traffic_class =
2024			(be32_to_cpu(path->tclass_flowlabel) >> 20) & 0xff;
2025		ib_ah_attr->grh.flow_label =
2026			be32_to_cpu(path->tclass_flowlabel) & 0xfffff;
2027		memcpy(ib_ah_attr->grh.dgid.raw,
2028			path->rgid, sizeof ib_ah_attr->grh.dgid.raw);
2029	}
2030}
2031
2032int mlx4_ib_query_qp(struct ib_qp *ibqp, struct ib_qp_attr *qp_attr, int qp_attr_mask,
2033		     struct ib_qp_init_attr *qp_init_attr)
2034{
2035	struct mlx4_ib_dev *dev = to_mdev(ibqp->device);
2036	struct mlx4_ib_qp *qp = to_mqp(ibqp);
2037	struct mlx4_qp_context context;
2038	int mlx4_state;
2039	int err = 0;
2040
 
 
 
2041	mutex_lock(&qp->mutex);
2042
2043	if (qp->state == IB_QPS_RESET) {
2044		qp_attr->qp_state = IB_QPS_RESET;
2045		goto done;
2046	}
2047
2048	err = mlx4_qp_query(dev->dev, &qp->mqp, &context);
2049	if (err) {
2050		err = -EINVAL;
2051		goto out;
2052	}
2053
2054	mlx4_state = be32_to_cpu(context.flags) >> 28;
2055
2056	qp->state		     = to_ib_qp_state(mlx4_state);
2057	qp_attr->qp_state	     = qp->state;
2058	qp_attr->path_mtu	     = context.mtu_msgmax >> 5;
2059	qp_attr->path_mig_state	     =
2060		to_ib_mig_state((be32_to_cpu(context.flags) >> 11) & 0x3);
2061	qp_attr->qkey		     = be32_to_cpu(context.qkey);
2062	qp_attr->rq_psn		     = be32_to_cpu(context.rnr_nextrecvpsn) & 0xffffff;
2063	qp_attr->sq_psn		     = be32_to_cpu(context.next_send_psn) & 0xffffff;
2064	qp_attr->dest_qp_num	     = be32_to_cpu(context.remote_qpn) & 0xffffff;
2065	qp_attr->qp_access_flags     =
2066		to_ib_qp_access_flags(be32_to_cpu(context.params2));
2067
2068	if (qp->ibqp.qp_type == IB_QPT_RC || qp->ibqp.qp_type == IB_QPT_UC) {
2069		to_ib_ah_attr(dev, &qp_attr->ah_attr, &context.pri_path);
2070		to_ib_ah_attr(dev, &qp_attr->alt_ah_attr, &context.alt_path);
2071		qp_attr->alt_pkey_index = context.alt_path.pkey_index & 0x7f;
2072		qp_attr->alt_port_num	= qp_attr->alt_ah_attr.port_num;
 
2073	}
2074
2075	qp_attr->pkey_index = context.pri_path.pkey_index & 0x7f;
2076	if (qp_attr->qp_state == IB_QPS_INIT)
2077		qp_attr->port_num = qp->port;
2078	else
2079		qp_attr->port_num = context.pri_path.sched_queue & 0x40 ? 2 : 1;
2080
2081	/* qp_attr->en_sqd_async_notify is only applicable in modify qp */
2082	qp_attr->sq_draining = mlx4_state == MLX4_QP_STATE_SQ_DRAINING;
2083
2084	qp_attr->max_rd_atomic = 1 << ((be32_to_cpu(context.params1) >> 21) & 0x7);
2085
2086	qp_attr->max_dest_rd_atomic =
2087		1 << ((be32_to_cpu(context.params2) >> 21) & 0x7);
2088	qp_attr->min_rnr_timer	    =
2089		(be32_to_cpu(context.rnr_nextrecvpsn) >> 24) & 0x1f;
2090	qp_attr->timeout	    = context.pri_path.ackto >> 3;
2091	qp_attr->retry_cnt	    = (be32_to_cpu(context.params1) >> 16) & 0x7;
2092	qp_attr->rnr_retry	    = (be32_to_cpu(context.params1) >> 13) & 0x7;
2093	qp_attr->alt_timeout	    = context.alt_path.ackto >> 3;
2094
2095done:
2096	qp_attr->cur_qp_state	     = qp_attr->qp_state;
2097	qp_attr->cap.max_recv_wr     = qp->rq.wqe_cnt;
2098	qp_attr->cap.max_recv_sge    = qp->rq.max_gs;
2099
2100	if (!ibqp->uobject) {
2101		qp_attr->cap.max_send_wr  = qp->sq.wqe_cnt;
2102		qp_attr->cap.max_send_sge = qp->sq.max_gs;
2103	} else {
2104		qp_attr->cap.max_send_wr  = 0;
2105		qp_attr->cap.max_send_sge = 0;
2106	}
2107
2108	/*
2109	 * We don't support inline sends for kernel QPs (yet), and we
2110	 * don't know what userspace's value should be.
2111	 */
2112	qp_attr->cap.max_inline_data = 0;
2113
2114	qp_init_attr->cap	     = qp_attr->cap;
2115
2116	qp_init_attr->create_flags = 0;
2117	if (qp->flags & MLX4_IB_QP_BLOCK_MULTICAST_LOOPBACK)
2118		qp_init_attr->create_flags |= IB_QP_CREATE_BLOCK_MULTICAST_LOOPBACK;
2119
2120	if (qp->flags & MLX4_IB_QP_LSO)
2121		qp_init_attr->create_flags |= IB_QP_CREATE_IPOIB_UD_LSO;
2122
 
 
 
 
 
 
 
2123out:
2124	mutex_unlock(&qp->mutex);
2125	return err;
2126}
2127