1// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2/* Copyright (c) 2015 - 2021 Intel Corporation */
   3#include <linux/etherdevice.h>
   4
   5#include "osdep.h"
   6#include "hmc.h"
   7#include "defs.h"
   8#include "type.h"
   9#include "ws.h"
  10#include "protos.h"
  11
  12/**
  13 * irdma_get_qp_from_list - get next qp from a list
  14 * @head: Listhead of qp's
  15 * @qp: current qp
  16 */
  17struct irdma_sc_qp *irdma_get_qp_from_list(struct list_head *head,
  18					   struct irdma_sc_qp *qp)
  19{
  20	struct list_head *lastentry;
  21	struct list_head *entry = NULL;
  22
  23	if (list_empty(head))
  24		return NULL;
  25
  26	if (!qp) {
  27		entry = head->next;
  28	} else {
  29		lastentry = &qp->list;
  30		entry = lastentry->next;
  31		if (entry == head)
  32			return NULL;
  33	}
  34
  35	return container_of(entry, struct irdma_sc_qp, list);
  36}
  37
  38/**
  39 * irdma_sc_suspend_resume_qps - suspend/resume all qp's on VSI
  40 * @vsi: the VSI struct pointer
  41 * @op: Set to IRDMA_OP_RESUME or IRDMA_OP_SUSPEND
  42 */
  43void irdma_sc_suspend_resume_qps(struct irdma_sc_vsi *vsi, u8 op)
  44{
  45	struct irdma_sc_qp *qp = NULL;
  46	u8 i;
  47
  48	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
  49		mutex_lock(&vsi->qos[i].qos_mutex);
  50		qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
  51		while (qp) {
  52			if (op == IRDMA_OP_RESUME) {
  53				if (!qp->dev->ws_add(vsi, i)) {
  54					qp->qs_handle =
  55						vsi->qos[qp->user_pri].qs_handle;
  56					irdma_cqp_qp_suspend_resume(qp, op);
  57				} else {
  58					irdma_cqp_qp_suspend_resume(qp, op);
  59					irdma_modify_qp_to_err(qp);
  60				}
  61			} else if (op == IRDMA_OP_SUSPEND) {
  62				/* issue cqp suspend command */
  63				if (!irdma_cqp_qp_suspend_resume(qp, op))
  64					atomic_inc(&vsi->qp_suspend_reqs);
  65			}
  66			qp = irdma_get_qp_from_list(&vsi->qos[i].qplist, qp);
  67		}
  68		mutex_unlock(&vsi->qos[i].qos_mutex);
  69	}
  70}
  71
  72static void irdma_set_qos_info(struct irdma_sc_vsi  *vsi,
  73			       struct irdma_l2params *l2p)
  74{
  75	u8 i;
  76
  77	vsi->qos_rel_bw = l2p->vsi_rel_bw;
  78	vsi->qos_prio_type = l2p->vsi_prio_type;
  79	vsi->dscp_mode = l2p->dscp_mode;
  80	if (l2p->dscp_mode) {
  81		memcpy(vsi->dscp_map, l2p->dscp_map, sizeof(vsi->dscp_map));
  82		for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
  83			l2p->up2tc[i] = i;
  84	}
  85	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
  86		if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
  87			vsi->qos[i].qs_handle = l2p->qs_handle_list[i];
  88		vsi->qos[i].traffic_class = l2p->up2tc[i];
  89		vsi->qos[i].rel_bw =
  90			l2p->tc_info[vsi->qos[i].traffic_class].rel_bw;
  91		vsi->qos[i].prio_type =
  92			l2p->tc_info[vsi->qos[i].traffic_class].prio_type;
  93		vsi->qos[i].valid = false;
  94	}
  95}
  96
  97/**
  98 * irdma_change_l2params - given the new l2 parameters, change all qp
  99 * @vsi: RDMA VSI pointer
 100 * @l2params: New parameters from l2
 101 */
 102void irdma_change_l2params(struct irdma_sc_vsi *vsi,
 103			   struct irdma_l2params *l2params)
 104{
 105	if (l2params->mtu_changed) {
 106		vsi->mtu = l2params->mtu;
 107		if (vsi->ieq)
 108			irdma_reinitialize_ieq(vsi);
 109	}
 110
 111	if (!l2params->tc_changed)
 112		return;
 113
 114	vsi->tc_change_pending = false;
 115	irdma_set_qos_info(vsi, l2params);
 116	irdma_sc_suspend_resume_qps(vsi, IRDMA_OP_RESUME);
 117}
 118
 119/**
 120 * irdma_qp_rem_qos - remove qp from qos lists during destroy qp
 121 * @qp: qp to be removed from qos
 122 */
 123void irdma_qp_rem_qos(struct irdma_sc_qp *qp)
 124{
 125	struct irdma_sc_vsi *vsi = qp->vsi;
 126
 127	ibdev_dbg(to_ibdev(qp->dev),
 128		  "DCB: DCB: Remove qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",
 129		  qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,
 130		  qp->on_qoslist);
 131	mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);
 132	if (qp->on_qoslist) {
 133		qp->on_qoslist = false;
 134		list_del(&qp->list);
 135	}
 136	mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex);
 137}
 138
 139/**
 140 * irdma_qp_add_qos - called during setctx for qp to be added to qos
 141 * @qp: qp to be added to qos
 142 */
 143void irdma_qp_add_qos(struct irdma_sc_qp *qp)
 144{
 145	struct irdma_sc_vsi *vsi = qp->vsi;
 146
 147	ibdev_dbg(to_ibdev(qp->dev),
 148		  "DCB: DCB: Add qp[%d] UP[%d] qset[%d] on_qoslist[%d]\n",
 149		  qp->qp_uk.qp_id, qp->user_pri, qp->qs_handle,
 150		  qp->on_qoslist);
 151	mutex_lock(&vsi->qos[qp->user_pri].qos_mutex);
 152	if (!qp->on_qoslist) {
 153		list_add(&qp->list, &vsi->qos[qp->user_pri].qplist);
 154		qp->on_qoslist = true;
 155		qp->qs_handle = vsi->qos[qp->user_pri].qs_handle;
 156	}
 157	mutex_unlock(&vsi->qos[qp->user_pri].qos_mutex);
 158}
 159
 160/**
 161 * irdma_sc_pd_init - initialize sc pd struct
 162 * @dev: sc device struct
 163 * @pd: sc pd ptr
 164 * @pd_id: pd_id for allocated pd
 165 * @abi_ver: User/Kernel ABI version
 166 */
 167void irdma_sc_pd_init(struct irdma_sc_dev *dev, struct irdma_sc_pd *pd, u32 pd_id,
 168		      int abi_ver)
 169{
 170	pd->pd_id = pd_id;
 171	pd->abi_ver = abi_ver;
 172	pd->dev = dev;
 173}
 174
 175/**
 176 * irdma_sc_add_arp_cache_entry - cqp wqe add arp cache entry
 177 * @cqp: struct for cqp hw
 178 * @info: arp entry information
 179 * @scratch: u64 saved to be used during cqp completion
 180 * @post_sq: flag for cqp db to ring
 181 */
 182static int irdma_sc_add_arp_cache_entry(struct irdma_sc_cqp *cqp,
 183					struct irdma_add_arp_cache_entry_info *info,
 184					u64 scratch, bool post_sq)
 185{
 186	__le64 *wqe;
 187	u64 hdr;
 188
 189	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 190	if (!wqe)
 191		return -ENOMEM;
 192	set_64bit_val(wqe, 8, info->reach_max);
 193	set_64bit_val(wqe, 16, ether_addr_to_u64(info->mac_addr));
 194
 195	hdr = info->arp_index |
 196	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
 197	      FIELD_PREP(IRDMA_CQPSQ_MAT_PERMANENT, (info->permanent ? 1 : 0)) |
 198	      FIELD_PREP(IRDMA_CQPSQ_MAT_ENTRYVALID, 1) |
 199	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 200	dma_wmb(); /* make sure WQE is written before valid bit is set */
 201
 202	set_64bit_val(wqe, 24, hdr);
 203
 204	print_hex_dump_debug("WQE: ARP_CACHE_ENTRY WQE", DUMP_PREFIX_OFFSET,
 205			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
 206	if (post_sq)
 207		irdma_sc_cqp_post_sq(cqp);
 208
 209	return 0;
 210}
 211
 212/**
 213 * irdma_sc_del_arp_cache_entry - dele arp cache entry
 214 * @cqp: struct for cqp hw
 215 * @scratch: u64 saved to be used during cqp completion
 216 * @arp_index: arp index to delete arp entry
 217 * @post_sq: flag for cqp db to ring
 218 */
 219static int irdma_sc_del_arp_cache_entry(struct irdma_sc_cqp *cqp, u64 scratch,
 220					u16 arp_index, bool post_sq)
 221{
 222	__le64 *wqe;
 223	u64 hdr;
 224
 225	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 226	if (!wqe)
 227		return -ENOMEM;
 228
 229	hdr = arp_index |
 230	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_ARP) |
 231	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 232	dma_wmb(); /* make sure WQE is written before valid bit is set */
 233
 234	set_64bit_val(wqe, 24, hdr);
 235
 236	print_hex_dump_debug("WQE: ARP_CACHE_DEL_ENTRY WQE",
 237			     DUMP_PREFIX_OFFSET, 16, 8, wqe,
 238			     IRDMA_CQP_WQE_SIZE * 8, false);
 239	if (post_sq)
 240		irdma_sc_cqp_post_sq(cqp);
 241
 242	return 0;
 243}
 244
 245/**
 246 * irdma_sc_manage_apbvt_entry - for adding and deleting apbvt entries
 247 * @cqp: struct for cqp hw
 248 * @info: info for apbvt entry to add or delete
 249 * @scratch: u64 saved to be used during cqp completion
 250 * @post_sq: flag for cqp db to ring
 251 */
 252static int irdma_sc_manage_apbvt_entry(struct irdma_sc_cqp *cqp,
 253				       struct irdma_apbvt_info *info,
 254				       u64 scratch, bool post_sq)
 255{
 256	__le64 *wqe;
 257	u64 hdr;
 258
 259	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 260	if (!wqe)
 261		return -ENOMEM;
 262
 263	set_64bit_val(wqe, 16, info->port);
 264
 265	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_APBVT) |
 266	      FIELD_PREP(IRDMA_CQPSQ_MAPT_ADDPORT, info->add) |
 267	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 268	dma_wmb(); /* make sure WQE is written before valid bit is set */
 269
 270	set_64bit_val(wqe, 24, hdr);
 271
 272	print_hex_dump_debug("WQE: MANAGE_APBVT WQE", DUMP_PREFIX_OFFSET, 16,
 273			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
 274	if (post_sq)
 275		irdma_sc_cqp_post_sq(cqp);
 276
 277	return 0;
 278}
 279
 280/**
 281 * irdma_sc_manage_qhash_table_entry - manage quad hash entries
 282 * @cqp: struct for cqp hw
 283 * @info: info for quad hash to manage
 284 * @scratch: u64 saved to be used during cqp completion
 285 * @post_sq: flag for cqp db to ring
 286 *
 287 * This is called before connection establishment is started.
 288 * For passive connections, when listener is created, it will
 289 * call with entry type of  IRDMA_QHASH_TYPE_TCP_SYN with local
 290 * ip address and tcp port. When SYN is received (passive
 291 * connections) or sent (active connections), this routine is
 292 * called with entry type of IRDMA_QHASH_TYPE_TCP_ESTABLISHED
 293 * and quad is passed in info.
 294 *
 295 * When iwarp connection is done and its state moves to RTS, the
 296 * quad hash entry in the hardware will point to iwarp's qp
 297 * number and requires no calls from the driver.
 298 */
 299static int
 300irdma_sc_manage_qhash_table_entry(struct irdma_sc_cqp *cqp,
 301				  struct irdma_qhash_table_info *info,
 302				  u64 scratch, bool post_sq)
 303{
 304	__le64 *wqe;
 305	u64 qw1 = 0;
 306	u64 qw2 = 0;
 307	u64 temp;
 308	struct irdma_sc_vsi *vsi = info->vsi;
 309
 310	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 311	if (!wqe)
 312		return -ENOMEM;
 313
 314	set_64bit_val(wqe, 0, ether_addr_to_u64(info->mac_addr));
 315
 316	qw1 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QPN, info->qp_num) |
 317	      FIELD_PREP(IRDMA_CQPSQ_QHASH_DEST_PORT, info->dest_port);
 318	if (info->ipv4_valid) {
 319		set_64bit_val(wqe, 48,
 320			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[0]));
 321	} else {
 322		set_64bit_val(wqe, 56,
 323			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->dest_ip[0]) |
 324			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->dest_ip[1]));
 325
 326		set_64bit_val(wqe, 48,
 327			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->dest_ip[2]) |
 328			      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->dest_ip[3]));
 329	}
 330	qw2 = FIELD_PREP(IRDMA_CQPSQ_QHASH_QS_HANDLE,
 331			 vsi->qos[info->user_pri].qs_handle);
 332	if (info->vlan_valid)
 333		qw2 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANID, info->vlan_id);
 334	set_64bit_val(wqe, 16, qw2);
 335	if (info->entry_type == IRDMA_QHASH_TYPE_TCP_ESTABLISHED) {
 336		qw1 |= FIELD_PREP(IRDMA_CQPSQ_QHASH_SRC_PORT, info->src_port);
 337		if (!info->ipv4_valid) {
 338			set_64bit_val(wqe, 40,
 339				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR0, info->src_ip[0]) |
 340				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR1, info->src_ip[1]));
 341			set_64bit_val(wqe, 32,
 342				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR2, info->src_ip[2]) |
 343				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[3]));
 344		} else {
 345			set_64bit_val(wqe, 32,
 346				      FIELD_PREP(IRDMA_CQPSQ_QHASH_ADDR3, info->src_ip[0]));
 347		}
 348	}
 349
 350	set_64bit_val(wqe, 8, qw1);
 351	temp = FIELD_PREP(IRDMA_CQPSQ_QHASH_WQEVALID, cqp->polarity) |
 352	       FIELD_PREP(IRDMA_CQPSQ_QHASH_OPCODE,
 353			  IRDMA_CQP_OP_MANAGE_QUAD_HASH_TABLE_ENTRY) |
 354	       FIELD_PREP(IRDMA_CQPSQ_QHASH_MANAGE, info->manage) |
 355	       FIELD_PREP(IRDMA_CQPSQ_QHASH_IPV4VALID, info->ipv4_valid) |
 356	       FIELD_PREP(IRDMA_CQPSQ_QHASH_VLANVALID, info->vlan_valid) |
 357	       FIELD_PREP(IRDMA_CQPSQ_QHASH_ENTRYTYPE, info->entry_type);
 358	dma_wmb(); /* make sure WQE is written before valid bit is set */
 359
 360	set_64bit_val(wqe, 24, temp);
 361
 362	print_hex_dump_debug("WQE: MANAGE_QHASH WQE", DUMP_PREFIX_OFFSET, 16,
 363			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
 364	if (post_sq)
 365		irdma_sc_cqp_post_sq(cqp);
 366
 367	return 0;
 368}
 369
 370/**
 371 * irdma_sc_qp_init - initialize qp
 372 * @qp: sc qp
 373 * @info: initialization qp info
 374 */
 375int irdma_sc_qp_init(struct irdma_sc_qp *qp, struct irdma_qp_init_info *info)
 376{
 377	int ret_code;
 378	u32 pble_obj_cnt;
 379	u16 wqe_size;
 380
 381	if (info->qp_uk_init_info.max_sq_frag_cnt >
 382	    info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags ||
 383	    info->qp_uk_init_info.max_rq_frag_cnt >
 384	    info->pd->dev->hw_attrs.uk_attrs.max_hw_wq_frags)
 385		return -EINVAL;
 386
 387	qp->dev = info->pd->dev;
 388	qp->vsi = info->vsi;
 389	qp->ieq_qp = info->vsi->exception_lan_q;
 390	qp->sq_pa = info->sq_pa;
 391	qp->rq_pa = info->rq_pa;
 392	qp->hw_host_ctx_pa = info->host_ctx_pa;
 393	qp->q2_pa = info->q2_pa;
 394	qp->shadow_area_pa = info->shadow_area_pa;
 395	qp->q2_buf = info->q2;
 396	qp->pd = info->pd;
 397	qp->hw_host_ctx = info->host_ctx;
 398	info->qp_uk_init_info.wqe_alloc_db = qp->pd->dev->wqe_alloc_db;
 399	ret_code = irdma_uk_qp_init(&qp->qp_uk, &info->qp_uk_init_info);
 400	if (ret_code)
 401		return ret_code;
 402
 403	qp->virtual_map = info->virtual_map;
 404	pble_obj_cnt = info->pd->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
 405
 406	if ((info->virtual_map && info->sq_pa >= pble_obj_cnt) ||
 407	    (info->virtual_map && info->rq_pa >= pble_obj_cnt))
 408		return -EINVAL;
 409
 410	qp->llp_stream_handle = (void *)(-1);
 411	qp->hw_sq_size = irdma_get_encoded_wqe_size(qp->qp_uk.sq_ring.size,
 412						    IRDMA_QUEUE_TYPE_SQ_RQ);
 413	ibdev_dbg(to_ibdev(qp->dev),
 414		  "WQE: hw_sq_size[%04d] sq_ring.size[%04d]\n",
 415		  qp->hw_sq_size, qp->qp_uk.sq_ring.size);
 416	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1 && qp->pd->abi_ver > 4)
 417		wqe_size = IRDMA_WQE_SIZE_128;
 418	else
 419		ret_code = irdma_fragcnt_to_wqesize_rq(qp->qp_uk.max_rq_frag_cnt,
 420						       &wqe_size);
 421	if (ret_code)
 422		return ret_code;
 423
 424	qp->hw_rq_size = irdma_get_encoded_wqe_size(qp->qp_uk.rq_size *
 425				(wqe_size / IRDMA_QP_WQE_MIN_SIZE), IRDMA_QUEUE_TYPE_SQ_RQ);
 426	ibdev_dbg(to_ibdev(qp->dev),
 427		  "WQE: hw_rq_size[%04d] qp_uk.rq_size[%04d] wqe_size[%04d]\n",
 428		  qp->hw_rq_size, qp->qp_uk.rq_size, wqe_size);
 429	qp->sq_tph_val = info->sq_tph_val;
 430	qp->rq_tph_val = info->rq_tph_val;
 431	qp->sq_tph_en = info->sq_tph_en;
 432	qp->rq_tph_en = info->rq_tph_en;
 433	qp->rcv_tph_en = info->rcv_tph_en;
 434	qp->xmit_tph_en = info->xmit_tph_en;
 435	qp->qp_uk.first_sq_wq = info->qp_uk_init_info.first_sq_wq;
 436	qp->qs_handle = qp->vsi->qos[qp->user_pri].qs_handle;
 437
 438	return 0;
 439}
 440
 441/**
 442 * irdma_sc_qp_create - create qp
 443 * @qp: sc qp
 444 * @info: qp create info
 445 * @scratch: u64 saved to be used during cqp completion
 446 * @post_sq: flag for cqp db to ring
 447 */
 448int irdma_sc_qp_create(struct irdma_sc_qp *qp, struct irdma_create_qp_info *info,
 449		       u64 scratch, bool post_sq)
 450{
 451	struct irdma_sc_cqp *cqp;
 452	__le64 *wqe;
 453	u64 hdr;
 454
 455	cqp = qp->dev->cqp;
 456	if (qp->qp_uk.qp_id < cqp->dev->hw_attrs.min_hw_qp_id ||
 457	    qp->qp_uk.qp_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt)
 458		return -EINVAL;
 459
 460	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 461	if (!wqe)
 462		return -ENOMEM;
 463
 464	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
 465	set_64bit_val(wqe, 40, qp->shadow_area_pa);
 466
 467	hdr = qp->qp_uk.qp_id |
 468	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_QP) |
 469	      FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, (info->ord_valid ? 1 : 0)) |
 470	      FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |
 471	      FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |
 472	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
 473	      FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |
 474	      FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |
 475	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |
 476	      FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,
 477			 info->arp_cache_idx_valid) |
 478	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |
 479	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 480	dma_wmb(); /* make sure WQE is written before valid bit is set */
 481
 482	set_64bit_val(wqe, 24, hdr);
 483
 484	print_hex_dump_debug("WQE: QP_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
 485			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
 486	if (post_sq)
 487		irdma_sc_cqp_post_sq(cqp);
 488
 489	return 0;
 490}
 491
 492/**
 493 * irdma_sc_qp_modify - modify qp cqp wqe
 494 * @qp: sc qp
 495 * @info: modify qp info
 496 * @scratch: u64 saved to be used during cqp completion
 497 * @post_sq: flag for cqp db to ring
 498 */
 499int irdma_sc_qp_modify(struct irdma_sc_qp *qp, struct irdma_modify_qp_info *info,
 500		       u64 scratch, bool post_sq)
 501{
 502	__le64 *wqe;
 503	struct irdma_sc_cqp *cqp;
 504	u64 hdr;
 505	u8 term_actions = 0;
 506	u8 term_len = 0;
 507
 508	cqp = qp->dev->cqp;
 509	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 510	if (!wqe)
 511		return -ENOMEM;
 512
 513	if (info->next_iwarp_state == IRDMA_QP_STATE_TERMINATE) {
 514		if (info->dont_send_fin)
 515			term_actions += IRDMAQP_TERM_SEND_TERM_ONLY;
 516		if (info->dont_send_term)
 517			term_actions += IRDMAQP_TERM_SEND_FIN_ONLY;
 518		if (term_actions == IRDMAQP_TERM_SEND_TERM_AND_FIN ||
 519		    term_actions == IRDMAQP_TERM_SEND_TERM_ONLY)
 520			term_len = info->termlen;
 521	}
 522
 523	set_64bit_val(wqe, 8,
 524		      FIELD_PREP(IRDMA_CQPSQ_QP_NEWMSS, info->new_mss) |
 525		      FIELD_PREP(IRDMA_CQPSQ_QP_TERMLEN, term_len));
 526	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
 527	set_64bit_val(wqe, 40, qp->shadow_area_pa);
 528
 529	hdr = qp->qp_uk.qp_id |
 530	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_QP) |
 531	      FIELD_PREP(IRDMA_CQPSQ_QP_ORDVALID, info->ord_valid) |
 532	      FIELD_PREP(IRDMA_CQPSQ_QP_TOECTXVALID, info->tcp_ctx_valid) |
 533	      FIELD_PREP(IRDMA_CQPSQ_QP_CACHEDVARVALID,
 534			 info->cached_var_valid) |
 535	      FIELD_PREP(IRDMA_CQPSQ_QP_VQ, qp->virtual_map) |
 536	      FIELD_PREP(IRDMA_CQPSQ_QP_FORCELOOPBACK, info->force_lpb) |
 537	      FIELD_PREP(IRDMA_CQPSQ_QP_CQNUMVALID, info->cq_num_valid) |
 538	      FIELD_PREP(IRDMA_CQPSQ_QP_MACVALID, info->mac_valid) |
 539	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
 540	      FIELD_PREP(IRDMA_CQPSQ_QP_MSSCHANGE, info->mss_change) |
 541	      FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY,
 542			 info->remove_hash_idx) |
 543	      FIELD_PREP(IRDMA_CQPSQ_QP_TERMACT, term_actions) |
 544	      FIELD_PREP(IRDMA_CQPSQ_QP_RESETCON, info->reset_tcp_conn) |
 545	      FIELD_PREP(IRDMA_CQPSQ_QP_ARPTABIDXVALID,
 546			 info->arp_cache_idx_valid) |
 547	      FIELD_PREP(IRDMA_CQPSQ_QP_NEXTIWSTATE, info->next_iwarp_state) |
 548	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 549	dma_wmb(); /* make sure WQE is written before valid bit is set */
 550
 551	set_64bit_val(wqe, 24, hdr);
 552
 553	print_hex_dump_debug("WQE: QP_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,
 554			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
 555	if (post_sq)
 556		irdma_sc_cqp_post_sq(cqp);
 557
 558	return 0;
 559}
 560
 561/**
 562 * irdma_sc_qp_destroy - cqp destroy qp
 563 * @qp: sc qp
 564 * @scratch: u64 saved to be used during cqp completion
 565 * @remove_hash_idx: flag if to remove hash idx
 566 * @ignore_mw_bnd: memory window bind flag
 567 * @post_sq: flag for cqp db to ring
 568 */
 569int irdma_sc_qp_destroy(struct irdma_sc_qp *qp, u64 scratch,
 570			bool remove_hash_idx, bool ignore_mw_bnd, bool post_sq)
 571{
 572	__le64 *wqe;
 573	struct irdma_sc_cqp *cqp;
 574	u64 hdr;
 575
 576	cqp = qp->dev->cqp;
 577	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 578	if (!wqe)
 579		return -ENOMEM;
 580
 581	set_64bit_val(wqe, 16, qp->hw_host_ctx_pa);
 582	set_64bit_val(wqe, 40, qp->shadow_area_pa);
 583
 584	hdr = qp->qp_uk.qp_id |
 585	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_QP) |
 586	      FIELD_PREP(IRDMA_CQPSQ_QP_QPTYPE, qp->qp_uk.qp_type) |
 587	      FIELD_PREP(IRDMA_CQPSQ_QP_IGNOREMWBOUND, ignore_mw_bnd) |
 588	      FIELD_PREP(IRDMA_CQPSQ_QP_REMOVEHASHENTRY, remove_hash_idx) |
 589	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 590	dma_wmb(); /* make sure WQE is written before valid bit is set */
 591
 592	set_64bit_val(wqe, 24, hdr);
 593
 594	print_hex_dump_debug("WQE: QP_DESTROY WQE", DUMP_PREFIX_OFFSET, 16, 8,
 595			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
 596	if (post_sq)
 597		irdma_sc_cqp_post_sq(cqp);
 598
 599	return 0;
 600}
 601
 602/**
 603 * irdma_sc_get_encoded_ird_size -
 604 * @ird_size: IRD size
 605 * The ird from the connection is rounded to a supported HW setting and then encoded
 606 * for ird_size field of qp_ctx. Consumers are expected to provide valid ird size based
 607 * on hardware attributes. IRD size defaults to a value of 4 in case of invalid input
 608 */
 609static u8 irdma_sc_get_encoded_ird_size(u16 ird_size)
 610{
 611	switch (ird_size ?
 612		roundup_pow_of_two(2 * ird_size) : 4) {
 613	case 256:
 614		return IRDMA_IRD_HW_SIZE_256;
 615	case 128:
 616		return IRDMA_IRD_HW_SIZE_128;
 617	case 64:
 618	case 32:
 619		return IRDMA_IRD_HW_SIZE_64;
 620	case 16:
 621	case 8:
 622		return IRDMA_IRD_HW_SIZE_16;
 623	case 4:
 624	default:
 625		break;
 626	}
 627
 628	return IRDMA_IRD_HW_SIZE_4;
 629}
 630
 631/**
 632 * irdma_sc_qp_setctx_roce - set qp's context
 633 * @qp: sc qp
 634 * @qp_ctx: context ptr
 635 * @info: ctx info
 636 */
 637void irdma_sc_qp_setctx_roce(struct irdma_sc_qp *qp, __le64 *qp_ctx,
 638			     struct irdma_qp_host_ctx_info *info)
 639{
 640	struct irdma_roce_offload_info *roce_info;
 641	struct irdma_udp_offload_info *udp;
 642	u8 push_mode_en;
 643	u32 push_idx;
 644
 645	roce_info = info->roce_info;
 646	udp = info->udp_info;
 647	qp->user_pri = info->user_pri;
 648	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
 649		push_mode_en = 0;
 650		push_idx = 0;
 651	} else {
 652		push_mode_en = 1;
 653		push_idx = qp->push_idx;
 654	}
 655	set_64bit_val(qp_ctx, 0,
 656		      FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |
 657		      FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |
 658		      FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |
 659		      FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |
 660		      FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |
 661		      FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |
 662		      FIELD_PREP(IRDMAQPC_PMENA, push_mode_en) |
 663		      FIELD_PREP(IRDMAQPC_PDIDXHI, roce_info->pd_id >> 16) |
 664		      FIELD_PREP(IRDMAQPC_DC_TCP_EN, roce_info->dctcp_en) |
 665		      FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID, roce_info->err_rq_idx_valid) |
 666		      FIELD_PREP(IRDMAQPC_ISQP1, roce_info->is_qp1) |
 667		      FIELD_PREP(IRDMAQPC_ROCE_TVER, roce_info->roce_tver) |
 668		      FIELD_PREP(IRDMAQPC_IPV4, udp->ipv4) |
 669		      FIELD_PREP(IRDMAQPC_INSERTVLANTAG, udp->insert_vlan_tag));
 670	set_64bit_val(qp_ctx, 8, qp->sq_pa);
 671	set_64bit_val(qp_ctx, 16, qp->rq_pa);
 672	if ((roce_info->dcqcn_en || roce_info->dctcp_en) &&
 673	    !(udp->tos & 0x03))
 674		udp->tos |= ECN_CODE_PT_VAL;
 675	set_64bit_val(qp_ctx, 24,
 676		      FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
 677		      FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size) |
 678		      FIELD_PREP(IRDMAQPC_TTL, udp->ttl) | FIELD_PREP(IRDMAQPC_TOS, udp->tos) |
 679		      FIELD_PREP(IRDMAQPC_SRCPORTNUM, udp->src_port) |
 680		      FIELD_PREP(IRDMAQPC_DESTPORTNUM, udp->dst_port));
 681	set_64bit_val(qp_ctx, 32,
 682		      FIELD_PREP(IRDMAQPC_DESTIPADDR2, udp->dest_ip_addr[2]) |
 683		      FIELD_PREP(IRDMAQPC_DESTIPADDR3, udp->dest_ip_addr[3]));
 684	set_64bit_val(qp_ctx, 40,
 685		      FIELD_PREP(IRDMAQPC_DESTIPADDR0, udp->dest_ip_addr[0]) |
 686		      FIELD_PREP(IRDMAQPC_DESTIPADDR1, udp->dest_ip_addr[1]));
 687	set_64bit_val(qp_ctx, 48,
 688		      FIELD_PREP(IRDMAQPC_SNDMSS, udp->snd_mss) |
 689		      FIELD_PREP(IRDMAQPC_VLANTAG, udp->vlan_tag) |
 690		      FIELD_PREP(IRDMAQPC_ARPIDX, udp->arp_idx));
 691	set_64bit_val(qp_ctx, 56,
 692		      FIELD_PREP(IRDMAQPC_PKEY, roce_info->p_key) |
 693		      FIELD_PREP(IRDMAQPC_PDIDX, roce_info->pd_id) |
 694		      FIELD_PREP(IRDMAQPC_ACKCREDITS, roce_info->ack_credits) |
 695		      FIELD_PREP(IRDMAQPC_FLOWLABEL, udp->flow_label));
 696	set_64bit_val(qp_ctx, 64,
 697		      FIELD_PREP(IRDMAQPC_QKEY, roce_info->qkey) |
 698		      FIELD_PREP(IRDMAQPC_DESTQP, roce_info->dest_qp));
 699	set_64bit_val(qp_ctx, 80,
 700		      FIELD_PREP(IRDMAQPC_PSNNXT, udp->psn_nxt) |
 701		      FIELD_PREP(IRDMAQPC_LSN, udp->lsn));
 702	set_64bit_val(qp_ctx, 88,
 703		      FIELD_PREP(IRDMAQPC_EPSN, udp->epsn));
 704	set_64bit_val(qp_ctx, 96,
 705		      FIELD_PREP(IRDMAQPC_PSNMAX, udp->psn_max) |
 706		      FIELD_PREP(IRDMAQPC_PSNUNA, udp->psn_una));
 707	set_64bit_val(qp_ctx, 112,
 708		      FIELD_PREP(IRDMAQPC_CWNDROCE, udp->cwnd));
 709	set_64bit_val(qp_ctx, 128,
 710		      FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, roce_info->err_rq_idx) |
 711		      FIELD_PREP(IRDMAQPC_RNRNAK_THRESH, udp->rnr_nak_thresh) |
 712		      FIELD_PREP(IRDMAQPC_REXMIT_THRESH, udp->rexmit_thresh) |
 713		      FIELD_PREP(IRDMAQPC_RTOMIN, roce_info->rtomin));
 714	set_64bit_val(qp_ctx, 136,
 715		      FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |
 716		      FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
 717	set_64bit_val(qp_ctx, 144,
 718		      FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
 719	set_64bit_val(qp_ctx, 152, ether_addr_to_u64(roce_info->mac_addr) << 16);
 720	set_64bit_val(qp_ctx, 160,
 721		      FIELD_PREP(IRDMAQPC_ORDSIZE, roce_info->ord_size) |
 722		      FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(roce_info->ird_size)) |
 723		      FIELD_PREP(IRDMAQPC_WRRDRSPOK, roce_info->wr_rdresp_en) |
 724		      FIELD_PREP(IRDMAQPC_RDOK, roce_info->rd_en) |
 725		      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
 726		      FIELD_PREP(IRDMAQPC_BINDEN, roce_info->bind_en) |
 727		      FIELD_PREP(IRDMAQPC_FASTREGEN, roce_info->fast_reg_en) |
 728		      FIELD_PREP(IRDMAQPC_DCQCNENABLE, roce_info->dcqcn_en) |
 729		      FIELD_PREP(IRDMAQPC_RCVNOICRC, roce_info->rcv_no_icrc) |
 730		      FIELD_PREP(IRDMAQPC_FW_CC_ENABLE, roce_info->fw_cc_enable) |
 731		      FIELD_PREP(IRDMAQPC_UDPRIVCQENABLE, roce_info->udprivcq_en) |
 732		      FIELD_PREP(IRDMAQPC_PRIVEN, roce_info->priv_mode_en) |
 733		      FIELD_PREP(IRDMAQPC_TIMELYENABLE, roce_info->timely_en));
 734	set_64bit_val(qp_ctx, 168,
 735		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
 736	set_64bit_val(qp_ctx, 176,
 737		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
 738		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
 739		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle));
 740	set_64bit_val(qp_ctx, 184,
 741		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, udp->local_ipaddr[3]) |
 742		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, udp->local_ipaddr[2]));
 743	set_64bit_val(qp_ctx, 192,
 744		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, udp->local_ipaddr[1]) |
 745		      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, udp->local_ipaddr[0]));
 746	set_64bit_val(qp_ctx, 200,
 747		      FIELD_PREP(IRDMAQPC_THIGH, roce_info->t_high) |
 748		      FIELD_PREP(IRDMAQPC_TLOW, roce_info->t_low));
 749	set_64bit_val(qp_ctx, 208,
 750		      FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));
 751
 752	print_hex_dump_debug("WQE: QP_HOST CTX WQE", DUMP_PREFIX_OFFSET, 16,
 753			     8, qp_ctx, IRDMA_QP_CTX_SIZE, false);
 754}
 755
 756/* irdma_sc_alloc_local_mac_entry - allocate a mac entry
 757 * @cqp: struct for cqp hw
 758 * @scratch: u64 saved to be used during cqp completion
 759 * @post_sq: flag for cqp db to ring
 760 */
 761static int irdma_sc_alloc_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,
 762					  bool post_sq)
 763{
 764	__le64 *wqe;
 765	u64 hdr;
 766
 767	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 768	if (!wqe)
 769		return -ENOMEM;
 770
 771	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,
 772			 IRDMA_CQP_OP_ALLOCATE_LOC_MAC_TABLE_ENTRY) |
 773	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 774
 775	dma_wmb(); /* make sure WQE is written before valid bit is set */
 776
 777	set_64bit_val(wqe, 24, hdr);
 778
 779	print_hex_dump_debug("WQE: ALLOCATE_LOCAL_MAC WQE",
 780			     DUMP_PREFIX_OFFSET, 16, 8, wqe,
 781			     IRDMA_CQP_WQE_SIZE * 8, false);
 782
 783	if (post_sq)
 784		irdma_sc_cqp_post_sq(cqp);
 785	return 0;
 786}
 787
 788/**
 789 * irdma_sc_add_local_mac_entry - add mac enry
 790 * @cqp: struct for cqp hw
 791 * @info:mac addr info
 792 * @scratch: u64 saved to be used during cqp completion
 793 * @post_sq: flag for cqp db to ring
 794 */
 795static int irdma_sc_add_local_mac_entry(struct irdma_sc_cqp *cqp,
 796					struct irdma_local_mac_entry_info *info,
 797					u64 scratch, bool post_sq)
 798{
 799	__le64 *wqe;
 800	u64 header;
 801
 802	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 803	if (!wqe)
 804		return -ENOMEM;
 805
 806	set_64bit_val(wqe, 32, ether_addr_to_u64(info->mac_addr));
 807
 808	header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, info->entry_idx) |
 809		 FIELD_PREP(IRDMA_CQPSQ_OPCODE,
 810			    IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |
 811		 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
 812
 813	dma_wmb(); /* make sure WQE is written before valid bit is set */
 814
 815	set_64bit_val(wqe, 24, header);
 816
 817	print_hex_dump_debug("WQE: ADD_LOCAL_MAC WQE", DUMP_PREFIX_OFFSET, 16,
 818			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
 819
 820	if (post_sq)
 821		irdma_sc_cqp_post_sq(cqp);
 822	return 0;
 823}
 824
 825/**
 826 * irdma_sc_del_local_mac_entry - cqp wqe to dele local mac
 827 * @cqp: struct for cqp hw
 828 * @scratch: u64 saved to be used during cqp completion
 829 * @entry_idx: index of mac entry
 830 * @ignore_ref_count: to force mac adde delete
 831 * @post_sq: flag for cqp db to ring
 832 */
 833static int irdma_sc_del_local_mac_entry(struct irdma_sc_cqp *cqp, u64 scratch,
 834					u16 entry_idx, u8 ignore_ref_count,
 835					bool post_sq)
 836{
 837	__le64 *wqe;
 838	u64 header;
 839
 840	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
 841	if (!wqe)
 842		return -ENOMEM;
 843	header = FIELD_PREP(IRDMA_CQPSQ_MLM_TABLEIDX, entry_idx) |
 844		 FIELD_PREP(IRDMA_CQPSQ_OPCODE,
 845			    IRDMA_CQP_OP_MANAGE_LOC_MAC_TABLE) |
 846		 FIELD_PREP(IRDMA_CQPSQ_MLM_FREEENTRY, 1) |
 847		 FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
 848		 FIELD_PREP(IRDMA_CQPSQ_MLM_IGNORE_REF_CNT, ignore_ref_count);
 849
 850	dma_wmb(); /* make sure WQE is written before valid bit is set */
 851
 852	set_64bit_val(wqe, 24, header);
 853
 854	print_hex_dump_debug("WQE: DEL_LOCAL_MAC_IPADDR WQE",
 855			     DUMP_PREFIX_OFFSET, 16, 8, wqe,
 856			     IRDMA_CQP_WQE_SIZE * 8, false);
 857
 858	if (post_sq)
 859		irdma_sc_cqp_post_sq(cqp);
 860	return 0;
 861}
 862
 863/**
 864 * irdma_sc_qp_setctx - set qp's context
 865 * @qp: sc qp
 866 * @qp_ctx: context ptr
 867 * @info: ctx info
 868 */
 869void irdma_sc_qp_setctx(struct irdma_sc_qp *qp, __le64 *qp_ctx,
 870			struct irdma_qp_host_ctx_info *info)
 871{
 872	struct irdma_iwarp_offload_info *iw;
 873	struct irdma_tcp_offload_info *tcp;
 874	struct irdma_sc_dev *dev;
 875	u8 push_mode_en;
 876	u32 push_idx;
 877	u64 qw0, qw3, qw7 = 0, qw16 = 0;
 878	u64 mac = 0;
 879
 880	iw = info->iwarp_info;
 881	tcp = info->tcp_info;
 882	dev = qp->dev;
 883	if (iw->rcv_mark_en) {
 884		qp->pfpdu.marker_len = 4;
 885		qp->pfpdu.rcv_start_seq = tcp->rcv_nxt;
 886	}
 887	qp->user_pri = info->user_pri;
 888	if (qp->push_idx == IRDMA_INVALID_PUSH_PAGE_INDEX) {
 889		push_mode_en = 0;
 890		push_idx = 0;
 891	} else {
 892		push_mode_en = 1;
 893		push_idx = qp->push_idx;
 894	}
 895	qw0 = FIELD_PREP(IRDMAQPC_RQWQESIZE, qp->qp_uk.rq_wqe_size) |
 896	      FIELD_PREP(IRDMAQPC_RCVTPHEN, qp->rcv_tph_en) |
 897	      FIELD_PREP(IRDMAQPC_XMITTPHEN, qp->xmit_tph_en) |
 898	      FIELD_PREP(IRDMAQPC_RQTPHEN, qp->rq_tph_en) |
 899	      FIELD_PREP(IRDMAQPC_SQTPHEN, qp->sq_tph_en) |
 900	      FIELD_PREP(IRDMAQPC_PPIDX, push_idx) |
 901	      FIELD_PREP(IRDMAQPC_PMENA, push_mode_en);
 902
 903	set_64bit_val(qp_ctx, 8, qp->sq_pa);
 904	set_64bit_val(qp_ctx, 16, qp->rq_pa);
 905
 906	qw3 = FIELD_PREP(IRDMAQPC_RQSIZE, qp->hw_rq_size) |
 907	      FIELD_PREP(IRDMAQPC_SQSIZE, qp->hw_sq_size);
 908	if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
 909		qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX,
 910				  qp->src_mac_addr_idx);
 911	set_64bit_val(qp_ctx, 136,
 912		      FIELD_PREP(IRDMAQPC_TXCQNUM, info->send_cq_num) |
 913		      FIELD_PREP(IRDMAQPC_RXCQNUM, info->rcv_cq_num));
 914	set_64bit_val(qp_ctx, 168,
 915		      FIELD_PREP(IRDMAQPC_QPCOMPCTX, info->qp_compl_ctx));
 916	set_64bit_val(qp_ctx, 176,
 917		      FIELD_PREP(IRDMAQPC_SQTPHVAL, qp->sq_tph_val) |
 918		      FIELD_PREP(IRDMAQPC_RQTPHVAL, qp->rq_tph_val) |
 919		      FIELD_PREP(IRDMAQPC_QSHANDLE, qp->qs_handle) |
 920		      FIELD_PREP(IRDMAQPC_EXCEPTION_LAN_QUEUE, qp->ieq_qp));
 921	if (info->iwarp_info_valid) {
 922		qw0 |= FIELD_PREP(IRDMAQPC_DDP_VER, iw->ddp_ver) |
 923		       FIELD_PREP(IRDMAQPC_RDMAP_VER, iw->rdmap_ver) |
 924		       FIELD_PREP(IRDMAQPC_DC_TCP_EN, iw->dctcp_en) |
 925		       FIELD_PREP(IRDMAQPC_ECN_EN, iw->ecn_en) |
 926		       FIELD_PREP(IRDMAQPC_IBRDENABLE, iw->ib_rd_en) |
 927		       FIELD_PREP(IRDMAQPC_PDIDXHI, iw->pd_id >> 16) |
 928		       FIELD_PREP(IRDMAQPC_ERR_RQ_IDX_VALID,
 929				  iw->err_rq_idx_valid);
 930		qw7 |= FIELD_PREP(IRDMAQPC_PDIDX, iw->pd_id);
 931		qw16 |= FIELD_PREP(IRDMAQPC_ERR_RQ_IDX, iw->err_rq_idx) |
 932			FIELD_PREP(IRDMAQPC_RTOMIN, iw->rtomin);
 933		set_64bit_val(qp_ctx, 144,
 934			      FIELD_PREP(IRDMAQPC_Q2ADDR, qp->q2_pa >> 8) |
 935			      FIELD_PREP(IRDMAQPC_STAT_INDEX, info->stats_idx));
 936
 937		if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2)
 938			mac = ether_addr_to_u64(iw->mac_addr);
 939
 940		set_64bit_val(qp_ctx, 152,
 941			      mac << 16 | FIELD_PREP(IRDMAQPC_LASTBYTESENT, iw->last_byte_sent));
 942		set_64bit_val(qp_ctx, 160,
 943			      FIELD_PREP(IRDMAQPC_ORDSIZE, iw->ord_size) |
 944			      FIELD_PREP(IRDMAQPC_IRDSIZE, irdma_sc_get_encoded_ird_size(iw->ird_size)) |
 945			      FIELD_PREP(IRDMAQPC_WRRDRSPOK, iw->wr_rdresp_en) |
 946			      FIELD_PREP(IRDMAQPC_RDOK, iw->rd_en) |
 947			      FIELD_PREP(IRDMAQPC_SNDMARKERS, iw->snd_mark_en) |
 948			      FIELD_PREP(IRDMAQPC_BINDEN, iw->bind_en) |
 949			      FIELD_PREP(IRDMAQPC_FASTREGEN, iw->fast_reg_en) |
 950			      FIELD_PREP(IRDMAQPC_PRIVEN, iw->priv_mode_en) |
 951			      FIELD_PREP(IRDMAQPC_USESTATSINSTANCE, info->stats_idx_valid) |
 952			      FIELD_PREP(IRDMAQPC_IWARPMODE, 1) |
 953			      FIELD_PREP(IRDMAQPC_RCVMARKERS, iw->rcv_mark_en) |
 954			      FIELD_PREP(IRDMAQPC_ALIGNHDRS, iw->align_hdrs) |
 955			      FIELD_PREP(IRDMAQPC_RCVNOMPACRC, iw->rcv_no_mpa_crc) |
 956			      FIELD_PREP(IRDMAQPC_RCVMARKOFFSET, iw->rcv_mark_offset || !tcp ? iw->rcv_mark_offset : tcp->rcv_nxt) |
 957			      FIELD_PREP(IRDMAQPC_SNDMARKOFFSET, iw->snd_mark_offset || !tcp ? iw->snd_mark_offset : tcp->snd_nxt) |
 958			      FIELD_PREP(IRDMAQPC_TIMELYENABLE, iw->timely_en));
 959	}
 960	if (info->tcp_info_valid) {
 961		qw0 |= FIELD_PREP(IRDMAQPC_IPV4, tcp->ipv4) |
 962		       FIELD_PREP(IRDMAQPC_NONAGLE, tcp->no_nagle) |
 963		       FIELD_PREP(IRDMAQPC_INSERTVLANTAG,
 964				  tcp->insert_vlan_tag) |
 965		       FIELD_PREP(IRDMAQPC_TIMESTAMP, tcp->time_stamp) |
 966		       FIELD_PREP(IRDMAQPC_LIMIT, tcp->cwnd_inc_limit) |
 967		       FIELD_PREP(IRDMAQPC_DROPOOOSEG, tcp->drop_ooo_seg) |
 968		       FIELD_PREP(IRDMAQPC_DUPACK_THRESH, tcp->dup_ack_thresh);
 969
 970		if ((iw->ecn_en || iw->dctcp_en) && !(tcp->tos & 0x03))
 971			tcp->tos |= ECN_CODE_PT_VAL;
 972
 973		qw3 |= FIELD_PREP(IRDMAQPC_TTL, tcp->ttl) |
 974		       FIELD_PREP(IRDMAQPC_AVOIDSTRETCHACK, tcp->avoid_stretch_ack) |
 975		       FIELD_PREP(IRDMAQPC_TOS, tcp->tos) |
 976		       FIELD_PREP(IRDMAQPC_SRCPORTNUM, tcp->src_port) |
 977		       FIELD_PREP(IRDMAQPC_DESTPORTNUM, tcp->dst_port);
 978		if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
 979			qw3 |= FIELD_PREP(IRDMAQPC_GEN1_SRCMACADDRIDX, tcp->src_mac_addr_idx);
 980
 981			qp->src_mac_addr_idx = tcp->src_mac_addr_idx;
 982		}
 983		set_64bit_val(qp_ctx, 32,
 984			      FIELD_PREP(IRDMAQPC_DESTIPADDR2, tcp->dest_ip_addr[2]) |
 985			      FIELD_PREP(IRDMAQPC_DESTIPADDR3, tcp->dest_ip_addr[3]));
 986		set_64bit_val(qp_ctx, 40,
 987			      FIELD_PREP(IRDMAQPC_DESTIPADDR0, tcp->dest_ip_addr[0]) |
 988			      FIELD_PREP(IRDMAQPC_DESTIPADDR1, tcp->dest_ip_addr[1]));
 989		set_64bit_val(qp_ctx, 48,
 990			      FIELD_PREP(IRDMAQPC_SNDMSS, tcp->snd_mss) |
 991			      FIELD_PREP(IRDMAQPC_SYN_RST_HANDLING, tcp->syn_rst_handling) |
 992			      FIELD_PREP(IRDMAQPC_VLANTAG, tcp->vlan_tag) |
 993			      FIELD_PREP(IRDMAQPC_ARPIDX, tcp->arp_idx));
 994		qw7 |= FIELD_PREP(IRDMAQPC_FLOWLABEL, tcp->flow_label) |
 995		       FIELD_PREP(IRDMAQPC_WSCALE, tcp->wscale) |
 996		       FIELD_PREP(IRDMAQPC_IGNORE_TCP_OPT,
 997				  tcp->ignore_tcp_opt) |
 998		       FIELD_PREP(IRDMAQPC_IGNORE_TCP_UNS_OPT,
 999				  tcp->ignore_tcp_uns_opt) |
1000		       FIELD_PREP(IRDMAQPC_TCPSTATE, tcp->tcp_state) |
1001		       FIELD_PREP(IRDMAQPC_RCVSCALE, tcp->rcv_wscale) |
1002		       FIELD_PREP(IRDMAQPC_SNDSCALE, tcp->snd_wscale);
1003		set_64bit_val(qp_ctx, 72,
1004			      FIELD_PREP(IRDMAQPC_TIMESTAMP_RECENT, tcp->time_stamp_recent) |
1005			      FIELD_PREP(IRDMAQPC_TIMESTAMP_AGE, tcp->time_stamp_age));
1006		set_64bit_val(qp_ctx, 80,
1007			      FIELD_PREP(IRDMAQPC_SNDNXT, tcp->snd_nxt) |
1008			      FIELD_PREP(IRDMAQPC_SNDWND, tcp->snd_wnd));
1009		set_64bit_val(qp_ctx, 88,
1010			      FIELD_PREP(IRDMAQPC_RCVNXT, tcp->rcv_nxt) |
1011			      FIELD_PREP(IRDMAQPC_RCVWND, tcp->rcv_wnd));
1012		set_64bit_val(qp_ctx, 96,
1013			      FIELD_PREP(IRDMAQPC_SNDMAX, tcp->snd_max) |
1014			      FIELD_PREP(IRDMAQPC_SNDUNA, tcp->snd_una));
1015		set_64bit_val(qp_ctx, 104,
1016			      FIELD_PREP(IRDMAQPC_SRTT, tcp->srtt) |
1017			      FIELD_PREP(IRDMAQPC_RTTVAR, tcp->rtt_var));
1018		set_64bit_val(qp_ctx, 112,
1019			      FIELD_PREP(IRDMAQPC_SSTHRESH, tcp->ss_thresh) |
1020			      FIELD_PREP(IRDMAQPC_CWND, tcp->cwnd));
1021		set_64bit_val(qp_ctx, 120,
1022			      FIELD_PREP(IRDMAQPC_SNDWL1, tcp->snd_wl1) |
1023			      FIELD_PREP(IRDMAQPC_SNDWL2, tcp->snd_wl2));
1024		qw16 |= FIELD_PREP(IRDMAQPC_MAXSNDWND, tcp->max_snd_window) |
1025			FIELD_PREP(IRDMAQPC_REXMIT_THRESH, tcp->rexmit_thresh);
1026		set_64bit_val(qp_ctx, 184,
1027			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR3, tcp->local_ipaddr[3]) |
1028			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR2, tcp->local_ipaddr[2]));
1029		set_64bit_val(qp_ctx, 192,
1030			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR1, tcp->local_ipaddr[1]) |
1031			      FIELD_PREP(IRDMAQPC_LOCAL_IPADDR0, tcp->local_ipaddr[0]));
1032		set_64bit_val(qp_ctx, 200,
1033			      FIELD_PREP(IRDMAQPC_THIGH, iw->t_high) |
1034			      FIELD_PREP(IRDMAQPC_TLOW, iw->t_low));
1035		set_64bit_val(qp_ctx, 208,
1036			      FIELD_PREP(IRDMAQPC_REMENDPOINTIDX, info->rem_endpoint_idx));
1037	}
1038
1039	set_64bit_val(qp_ctx, 0, qw0);
1040	set_64bit_val(qp_ctx, 24, qw3);
1041	set_64bit_val(qp_ctx, 56, qw7);
1042	set_64bit_val(qp_ctx, 128, qw16);
1043
1044	print_hex_dump_debug("WQE: QP_HOST CTX", DUMP_PREFIX_OFFSET, 16, 8,
1045			     qp_ctx, IRDMA_QP_CTX_SIZE, false);
1046}
1047
1048/**
1049 * irdma_sc_alloc_stag - mr stag alloc
1050 * @dev: sc device struct
1051 * @info: stag info
1052 * @scratch: u64 saved to be used during cqp completion
1053 * @post_sq: flag for cqp db to ring
1054 */
1055static int irdma_sc_alloc_stag(struct irdma_sc_dev *dev,
1056			       struct irdma_allocate_stag_info *info,
1057			       u64 scratch, bool post_sq)
1058{
1059	__le64 *wqe;
1060	struct irdma_sc_cqp *cqp;
1061	u64 hdr;
1062	enum irdma_page_size page_size;
1063
1064	if (!info->total_len && !info->all_memory)
1065		return -EINVAL;
1066
1067	if (info->page_size == 0x40000000)
1068		page_size = IRDMA_PAGE_SIZE_1G;
1069	else if (info->page_size == 0x200000)
1070		page_size = IRDMA_PAGE_SIZE_2M;
1071	else
1072		page_size = IRDMA_PAGE_SIZE_4K;
1073
1074	cqp = dev->cqp;
1075	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1076	if (!wqe)
1077		return -ENOMEM;
1078
1079	set_64bit_val(wqe, 8,
1080		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID) |
1081		      FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len));
1082	set_64bit_val(wqe, 16,
1083		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1084	set_64bit_val(wqe, 40,
1085		      FIELD_PREP(IRDMA_CQPSQ_STAG_HMCFNIDX, info->hmc_fcn_index));
1086
1087	if (info->chunk_size)
1088		set_64bit_val(wqe, 48,
1089			      FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_idx));
1090
1091	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |
1092	      FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |
1093	      FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |
1094	      FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |
1095	      FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |
1096	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, info->remote_access) |
1097	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |
1098	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |
1099	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1100	dma_wmb(); /* make sure WQE is written before valid bit is set */
1101
1102	set_64bit_val(wqe, 24, hdr);
1103
1104	print_hex_dump_debug("WQE: ALLOC_STAG WQE", DUMP_PREFIX_OFFSET, 16, 8,
1105			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1106	if (post_sq)
1107		irdma_sc_cqp_post_sq(cqp);
1108
1109	return 0;
1110}
1111
1112/**
1113 * irdma_sc_mr_reg_non_shared - non-shared mr registration
1114 * @dev: sc device struct
1115 * @info: mr info
1116 * @scratch: u64 saved to be used during cqp completion
1117 * @post_sq: flag for cqp db to ring
1118 */
1119static int irdma_sc_mr_reg_non_shared(struct irdma_sc_dev *dev,
1120				      struct irdma_reg_ns_stag_info *info,
1121				      u64 scratch, bool post_sq)
1122{
1123	__le64 *wqe;
1124	u64 fbo;
1125	struct irdma_sc_cqp *cqp;
1126	u64 hdr;
1127	u32 pble_obj_cnt;
1128	bool remote_access;
1129	u8 addr_type;
1130	enum irdma_page_size page_size;
1131
1132	if (!info->total_len && !info->all_memory)
1133		return -EINVAL;
1134
1135	if (info->page_size == 0x40000000)
1136		page_size = IRDMA_PAGE_SIZE_1G;
1137	else if (info->page_size == 0x200000)
1138		page_size = IRDMA_PAGE_SIZE_2M;
1139	else if (info->page_size == 0x1000)
1140		page_size = IRDMA_PAGE_SIZE_4K;
1141	else
1142		return -EINVAL;
1143
1144	if (info->access_rights & (IRDMA_ACCESS_FLAGS_REMOTEREAD_ONLY |
1145				   IRDMA_ACCESS_FLAGS_REMOTEWRITE_ONLY))
1146		remote_access = true;
1147	else
1148		remote_access = false;
1149
1150	pble_obj_cnt = dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
1151	if (info->chunk_size && info->first_pm_pbl_index >= pble_obj_cnt)
1152		return -EINVAL;
1153
1154	cqp = dev->cqp;
1155	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1156	if (!wqe)
1157		return -ENOMEM;
1158	fbo = info->va & (info->page_size - 1);
1159
1160	set_64bit_val(wqe, 0,
1161		      (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED ?
1162		      info->va : fbo));
1163	set_64bit_val(wqe, 8,
1164		      FIELD_PREP(IRDMA_CQPSQ_STAG_STAGLEN, info->total_len) |
1165		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1166	set_64bit_val(wqe, 16,
1167		      FIELD_PREP(IRDMA_CQPSQ_STAG_KEY, info->stag_key) |
1168		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1169	if (!info->chunk_size) {
1170		set_64bit_val(wqe, 32, info->reg_addr_pa);
1171		set_64bit_val(wqe, 48, 0);
1172	} else {
1173		set_64bit_val(wqe, 32, 0);
1174		set_64bit_val(wqe, 48,
1175			      FIELD_PREP(IRDMA_CQPSQ_STAG_FIRSTPMPBLIDX, info->first_pm_pbl_index));
1176	}
1177	set_64bit_val(wqe, 40, info->hmc_fcn_index);
1178	set_64bit_val(wqe, 56, 0);
1179
1180	addr_type = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ? 1 : 0;
1181	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_REG_MR) |
1182	      FIELD_PREP(IRDMA_CQPSQ_STAG_MR, 1) |
1183	      FIELD_PREP(IRDMA_CQPSQ_STAG_LPBLSIZE, info->chunk_size) |
1184	      FIELD_PREP(IRDMA_CQPSQ_STAG_HPAGESIZE, page_size) |
1185	      FIELD_PREP(IRDMA_CQPSQ_STAG_ARIGHTS, info->access_rights) |
1186	      FIELD_PREP(IRDMA_CQPSQ_STAG_REMACCENABLED, remote_access) |
1187	      FIELD_PREP(IRDMA_CQPSQ_STAG_VABASEDTO, addr_type) |
1188	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEHMCFNIDX, info->use_hmc_fcn_index) |
1189	      FIELD_PREP(IRDMA_CQPSQ_STAG_USEPFRID, info->use_pf_rid) |
1190	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1191	dma_wmb(); /* make sure WQE is written before valid bit is set */
1192
1193	set_64bit_val(wqe, 24, hdr);
1194
1195	print_hex_dump_debug("WQE: MR_REG_NS WQE", DUMP_PREFIX_OFFSET, 16, 8,
1196			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1197	if (post_sq)
1198		irdma_sc_cqp_post_sq(cqp);
1199
1200	return 0;
1201}
1202
1203/**
1204 * irdma_sc_dealloc_stag - deallocate stag
1205 * @dev: sc device struct
1206 * @info: dealloc stag info
1207 * @scratch: u64 saved to be used during cqp completion
1208 * @post_sq: flag for cqp db to ring
1209 */
1210static int irdma_sc_dealloc_stag(struct irdma_sc_dev *dev,
1211				 struct irdma_dealloc_stag_info *info,
1212				 u64 scratch, bool post_sq)
1213{
1214	u64 hdr;
1215	__le64 *wqe;
1216	struct irdma_sc_cqp *cqp;
1217
1218	cqp = dev->cqp;
1219	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1220	if (!wqe)
1221		return -ENOMEM;
1222
1223	set_64bit_val(wqe, 8,
1224		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1225	set_64bit_val(wqe, 16,
1226		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->stag_idx));
1227
1228	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DEALLOC_STAG) |
1229	      FIELD_PREP(IRDMA_CQPSQ_STAG_MR, info->mr) |
1230	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1231	dma_wmb(); /* make sure WQE is written before valid bit is set */
1232
1233	set_64bit_val(wqe, 24, hdr);
1234
1235	print_hex_dump_debug("WQE: DEALLOC_STAG WQE", DUMP_PREFIX_OFFSET, 16,
1236			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1237	if (post_sq)
1238		irdma_sc_cqp_post_sq(cqp);
1239
1240	return 0;
1241}
1242
1243/**
1244 * irdma_sc_mw_alloc - mw allocate
1245 * @dev: sc device struct
1246 * @info: memory window allocation information
1247 * @scratch: u64 saved to be used during cqp completion
1248 * @post_sq: flag for cqp db to ring
1249 */
1250static int irdma_sc_mw_alloc(struct irdma_sc_dev *dev,
1251			     struct irdma_mw_alloc_info *info, u64 scratch,
1252			     bool post_sq)
1253{
1254	u64 hdr;
1255	struct irdma_sc_cqp *cqp;
1256	__le64 *wqe;
1257
1258	cqp = dev->cqp;
1259	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
1260	if (!wqe)
1261		return -ENOMEM;
1262
1263	set_64bit_val(wqe, 8,
1264		      FLD_LS_64(dev, info->pd_id, IRDMA_CQPSQ_STAG_PDID));
1265	set_64bit_val(wqe, 16,
1266		      FIELD_PREP(IRDMA_CQPSQ_STAG_IDX, info->mw_stag_index));
1267
1268	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_ALLOC_STAG) |
1269	      FIELD_PREP(IRDMA_CQPSQ_STAG_MWTYPE, info->mw_wide) |
1270	      FIELD_PREP(IRDMA_CQPSQ_STAG_MW1_BIND_DONT_VLDT_KEY,
1271			 info->mw1_bind_dont_vldt_key) |
1272	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
1273	dma_wmb(); /* make sure WQE is written before valid bit is set */
1274
1275	set_64bit_val(wqe, 24, hdr);
1276
1277	print_hex_dump_debug("WQE: MW_ALLOC WQE", DUMP_PREFIX_OFFSET, 16, 8,
1278			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
1279	if (post_sq)
1280		irdma_sc_cqp_post_sq(cqp);
1281
1282	return 0;
1283}
1284
1285/**
1286 * irdma_sc_mr_fast_register - Posts RDMA fast register mr WR to iwarp qp
1287 * @qp: sc qp struct
1288 * @info: fast mr info
1289 * @post_sq: flag for cqp db to ring
1290 */
1291int irdma_sc_mr_fast_register(struct irdma_sc_qp *qp,
1292			      struct irdma_fast_reg_stag_info *info,
1293			      bool post_sq)
1294{
1295	u64 temp, hdr;
1296	__le64 *wqe;
1297	u32 wqe_idx;
1298	enum irdma_page_size page_size;
1299	struct irdma_post_sq_info sq_info = {};
1300
1301	if (info->page_size == 0x40000000)
1302		page_size = IRDMA_PAGE_SIZE_1G;
1303	else if (info->page_size == 0x200000)
1304		page_size = IRDMA_PAGE_SIZE_2M;
1305	else
1306		page_size = IRDMA_PAGE_SIZE_4K;
1307
1308	sq_info.wr_id = info->wr_id;
1309	sq_info.signaled = info->signaled;
 
1310
1311	wqe = irdma_qp_get_next_send_wqe(&qp->qp_uk, &wqe_idx,
1312					 IRDMA_QP_WQE_MIN_QUANTA, 0, &sq_info);
1313	if (!wqe)
1314		return -ENOMEM;
1315
1316	irdma_clr_wqes(&qp->qp_uk, wqe_idx);
1317
1318	ibdev_dbg(to_ibdev(qp->dev),
1319		  "MR: wr_id[%llxh] wqe_idx[%04d] location[%p]\n",
1320		  info->wr_id, wqe_idx,
1321		  &qp->qp_uk.sq_wrtrk_array[wqe_idx].wrid);
1322
1323	temp = (info->addr_type == IRDMA_ADDR_TYPE_VA_BASED) ?
1324		(uintptr_t)info->va : info->fbo;
1325	set_64bit_val(wqe, 0, temp);
1326
1327	temp = FIELD_GET(IRDMAQPSQ_FIRSTPMPBLIDXHI,
1328			 info->first_pm_pbl_index >> 16);
1329	set_64bit_val(wqe, 8,
1330		      FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXHI, temp) |
1331		      FIELD_PREP(IRDMAQPSQ_PBLADDR >> IRDMA_HW_PAGE_SHIFT, info->reg_addr_pa));
1332	set_64bit_val(wqe, 16,
1333		      info->total_len |
1334		      FIELD_PREP(IRDMAQPSQ_FIRSTPMPBLIDXLO, info->first_pm_pbl_index));
1335
1336	hdr = FIELD_PREP(IRDMAQPSQ_STAGKEY, info->stag_key) |
1337	      FIELD_PREP(IRDMAQPSQ_STAGINDEX, info->stag_idx) |
1338	      FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_FAST_REGISTER) |
1339	      FIELD_PREP(IRDMAQPSQ_LPBLSIZE, info->chunk_size) |
1340	      FIELD_PREP(IRDMAQPSQ_HPAGESIZE, page_size) |
1341	      FIELD_PREP(IRDMAQPSQ_STAGRIGHTS, info->access_rights) |
1342	      FIELD_PREP(IRDMAQPSQ_VABASEDTO, info->addr_type) |
 
1343	      FIELD_PREP(IRDMAQPSQ_READFENCE, info->read_fence) |
1344	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, info->local_fence) |
1345	      FIELD_PREP(IRDMAQPSQ_SIGCOMPL, info->signaled) |
1346	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1347	dma_wmb(); /* make sure WQE is written before valid bit is set */
1348
1349	set_64bit_val(wqe, 24, hdr);
1350
1351	print_hex_dump_debug("WQE: FAST_REG WQE", DUMP_PREFIX_OFFSET, 16, 8,
1352			     wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1353
1354	if (post_sq)
1355		irdma_uk_qp_post_wr(&qp->qp_uk);
 
 
 
 
1356
1357	return 0;
1358}
1359
1360/**
1361 * irdma_sc_gen_rts_ae - request AE generated after RTS
1362 * @qp: sc qp struct
1363 */
1364static void irdma_sc_gen_rts_ae(struct irdma_sc_qp *qp)
1365{
1366	__le64 *wqe;
1367	u64 hdr;
1368	struct irdma_qp_uk *qp_uk;
1369
1370	qp_uk = &qp->qp_uk;
1371
1372	wqe = qp_uk->sq_base[1].elem;
1373
1374	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_NOP) |
1375	      FIELD_PREP(IRDMAQPSQ_LOCALFENCE, 1) |
1376	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1377	dma_wmb(); /* make sure WQE is written before valid bit is set */
1378
1379	set_64bit_val(wqe, 24, hdr);
1380	print_hex_dump_debug("QP: NOP W/LOCAL FENCE WQE", DUMP_PREFIX_OFFSET,
1381			     16, 8, wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1382
1383	wqe = qp_uk->sq_base[2].elem;
1384	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_GEN_RTS_AE) |
1385	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1386	dma_wmb(); /* make sure WQE is written before valid bit is set */
1387
1388	set_64bit_val(wqe, 24, hdr);
1389	print_hex_dump_debug("QP: CONN EST WQE", DUMP_PREFIX_OFFSET, 16, 8,
1390			     wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1391}
1392
1393/**
1394 * irdma_sc_send_lsmm - send last streaming mode message
1395 * @qp: sc qp struct
1396 * @lsmm_buf: buffer with lsmm message
1397 * @size: size of lsmm buffer
1398 * @stag: stag of lsmm buffer
1399 */
1400void irdma_sc_send_lsmm(struct irdma_sc_qp *qp, void *lsmm_buf, u32 size,
1401			irdma_stag stag)
1402{
1403	__le64 *wqe;
1404	u64 hdr;
1405	struct irdma_qp_uk *qp_uk;
1406
1407	qp_uk = &qp->qp_uk;
1408	wqe = qp_uk->sq_base->elem;
1409
1410	set_64bit_val(wqe, 0, (uintptr_t)lsmm_buf);
1411	if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1412		set_64bit_val(wqe, 8,
1413			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_LEN, size) |
1414			      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, stag));
1415	} else {
1416		set_64bit_val(wqe, 8,
1417			      FIELD_PREP(IRDMAQPSQ_FRAG_LEN, size) |
1418			      FIELD_PREP(IRDMAQPSQ_FRAG_STAG, stag) |
1419			      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1420	}
1421	set_64bit_val(wqe, 16, 0);
1422
1423	hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_SEND) |
1424	      FIELD_PREP(IRDMAQPSQ_STREAMMODE, 1) |
1425	      FIELD_PREP(IRDMAQPSQ_WAITFORRCVPDU, 1) |
1426	      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1427	dma_wmb(); /* make sure WQE is written before valid bit is set */
1428
1429	set_64bit_val(wqe, 24, hdr);
1430
1431	print_hex_dump_debug("WQE: SEND_LSMM WQE", DUMP_PREFIX_OFFSET, 16, 8,
1432			     wqe, IRDMA_QP_WQE_MIN_SIZE, false);
1433
1434	if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
1435		irdma_sc_gen_rts_ae(qp);
1436}
1437
1438/**
1439 * irdma_sc_send_rtt - send last read0 or write0
1440 * @qp: sc qp struct
1441 * @read: Do read0 or write0
1442 */
1443void irdma_sc_send_rtt(struct irdma_sc_qp *qp, bool read)
1444{
1445	__le64 *wqe;
1446	u64 hdr;
1447	struct irdma_qp_uk *qp_uk;
1448
1449	qp_uk = &qp->qp_uk;
1450	wqe = qp_uk->sq_base->elem;
1451
1452	set_64bit_val(wqe, 0, 0);
1453	set_64bit_val(wqe, 16, 0);
1454	if (read) {
1455		if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1456			set_64bit_val(wqe, 8,
1457				      FIELD_PREP(IRDMAQPSQ_GEN1_FRAG_STAG, 0xabcd));
1458		} else {
1459			set_64bit_val(wqe, 8,
1460				      (u64)0xabcd | FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1461		}
1462		hdr = FIELD_PREP(IRDMAQPSQ_REMSTAG, 0x1234) |
1463		      FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_READ) |
1464		      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1465
1466	} else {
1467		if (qp->qp_uk.uk_attrs->hw_rev == IRDMA_GEN_1) {
1468			set_64bit_val(wqe, 8, 0);
1469		} else {
1470			set_64bit_val(wqe, 8,
1471				      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity));
1472		}
1473		hdr = FIELD_PREP(IRDMAQPSQ_OPCODE, IRDMAQP_OP_RDMA_WRITE) |
1474		      FIELD_PREP(IRDMAQPSQ_VALID, qp->qp_uk.swqe_polarity);
1475	}
1476
1477	dma_wmb(); /* make sure WQE is written before valid bit is set */
1478
1479	set_64bit_val(wqe, 24, hdr);
1480
1481	print_hex_dump_debug("WQE: RTR WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe,
1482			     IRDMA_QP_WQE_MIN_SIZE, false);
1483
1484	if (qp->dev->hw_attrs.uk_attrs.feature_flags & IRDMA_FEATURE_RTS_AE)
1485		irdma_sc_gen_rts_ae(qp);
1486}
1487
1488/**
1489 * irdma_iwarp_opcode - determine if incoming is rdma layer
1490 * @info: aeq info for the packet
1491 * @pkt: packet for error
1492 */
1493static u32 irdma_iwarp_opcode(struct irdma_aeqe_info *info, u8 *pkt)
1494{
1495	__be16 *mpa;
1496	u32 opcode = 0xffffffff;
1497
1498	if (info->q2_data_written) {
1499		mpa = (__be16 *)pkt;
1500		opcode = ntohs(mpa[1]) & 0xf;
1501	}
1502
1503	return opcode;
1504}
1505
1506/**
1507 * irdma_locate_mpa - return pointer to mpa in the pkt
1508 * @pkt: packet with data
1509 */
1510static u8 *irdma_locate_mpa(u8 *pkt)
1511{
1512	/* skip over ethernet header */
1513	pkt += IRDMA_MAC_HLEN;
1514
1515	/* Skip over IP and TCP headers */
1516	pkt += 4 * (pkt[0] & 0x0f);
1517	pkt += 4 * ((pkt[12] >> 4) & 0x0f);
1518
1519	return pkt;
1520}
1521
1522/**
1523 * irdma_bld_termhdr_ctrl - setup terminate hdr control fields
1524 * @qp: sc qp ptr for pkt
1525 * @hdr: term hdr
1526 * @opcode: flush opcode for termhdr
1527 * @layer_etype: error layer + error type
1528 * @err: error cod ein the header
1529 */
1530static void irdma_bld_termhdr_ctrl(struct irdma_sc_qp *qp,
1531				   struct irdma_terminate_hdr *hdr,
1532				   enum irdma_flush_opcode opcode,
1533				   u8 layer_etype, u8 err)
1534{
1535	qp->flush_code = opcode;
1536	hdr->layer_etype = layer_etype;
1537	hdr->error_code = err;
1538}
1539
1540/**
1541 * irdma_bld_termhdr_ddp_rdma - setup ddp and rdma hdrs in terminate hdr
1542 * @pkt: ptr to mpa in offending pkt
1543 * @hdr: term hdr
1544 * @copy_len: offending pkt length to be copied to term hdr
1545 * @is_tagged: DDP tagged or untagged
1546 */
1547static void irdma_bld_termhdr_ddp_rdma(u8 *pkt, struct irdma_terminate_hdr *hdr,
1548				       int *copy_len, u8 *is_tagged)
1549{
1550	u16 ddp_seg_len;
1551
1552	ddp_seg_len = ntohs(*(__be16 *)pkt);
1553	if (ddp_seg_len) {
1554		*copy_len = 2;
1555		hdr->hdrct = DDP_LEN_FLAG;
1556		if (pkt[2] & 0x80) {
1557			*is_tagged = 1;
1558			if (ddp_seg_len >= TERM_DDP_LEN_TAGGED) {
1559				*copy_len += TERM_DDP_LEN_TAGGED;
1560				hdr->hdrct |= DDP_HDR_FLAG;
1561			}
1562		} else {
1563			if (ddp_seg_len >= TERM_DDP_LEN_UNTAGGED) {
1564				*copy_len += TERM_DDP_LEN_UNTAGGED;
1565				hdr->hdrct |= DDP_HDR_FLAG;
1566			}
1567			if (ddp_seg_len >= (TERM_DDP_LEN_UNTAGGED + TERM_RDMA_LEN) &&
1568			    ((pkt[3] & RDMA_OPCODE_M) == RDMA_READ_REQ_OPCODE)) {
1569				*copy_len += TERM_RDMA_LEN;
1570				hdr->hdrct |= RDMA_HDR_FLAG;
1571			}
1572		}
1573	}
1574}
1575
1576/**
1577 * irdma_bld_terminate_hdr - build terminate message header
1578 * @qp: qp associated with received terminate AE
1579 * @info: the struct contiaing AE information
1580 */
1581static int irdma_bld_terminate_hdr(struct irdma_sc_qp *qp,
1582				   struct irdma_aeqe_info *info)
1583{
1584	u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
1585	int copy_len = 0;
1586	u8 is_tagged = 0;
1587	u32 opcode;
1588	struct irdma_terminate_hdr *termhdr;
1589
1590	termhdr = (struct irdma_terminate_hdr *)qp->q2_buf;
1591	memset(termhdr, 0, Q2_BAD_FRAME_OFFSET);
1592
1593	if (info->q2_data_written) {
1594		pkt = irdma_locate_mpa(pkt);
1595		irdma_bld_termhdr_ddp_rdma(pkt, termhdr, &copy_len, &is_tagged);
1596	}
1597
1598	opcode = irdma_iwarp_opcode(info, pkt);
1599	qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;
1600	qp->sq_flush_code = info->sq;
1601	qp->rq_flush_code = info->rq;
1602
1603	switch (info->ae_id) {
1604	case IRDMA_AE_AMP_UNALLOCATED_STAG:
1605		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1606		if (opcode == IRDMA_OP_TYPE_RDMA_WRITE)
1607			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1608					       (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1609					       DDP_TAGGED_INV_STAG);
1610		else
1611			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1612					       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1613					       RDMAP_INV_STAG);
1614		break;
1615	case IRDMA_AE_AMP_BOUNDS_VIOLATION:
1616		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1617		if (info->q2_data_written)
1618			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1619					       (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1620					       DDP_TAGGED_BOUNDS);
1621		else
1622			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1623					       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1624					       RDMAP_INV_BOUNDS);
1625		break;
1626	case IRDMA_AE_AMP_BAD_PD:
1627		switch (opcode) {
1628		case IRDMA_OP_TYPE_RDMA_WRITE:
1629			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_PROT_ERR,
1630					       (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1631					       DDP_TAGGED_UNASSOC_STAG);
1632			break;
1633		case IRDMA_OP_TYPE_SEND_INV:
1634		case IRDMA_OP_TYPE_SEND_SOL_INV:
1635			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1636					       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1637					       RDMAP_CANT_INV_STAG);
1638			break;
1639		default:
1640			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1641					       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1642					       RDMAP_UNASSOC_STAG);
1643		}
1644		break;
1645	case IRDMA_AE_AMP_INVALID_STAG:
1646		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1647		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1648				       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1649				       RDMAP_INV_STAG);
1650		break;
1651	case IRDMA_AE_AMP_BAD_QP:
1652		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_QP_OP_ERR,
1653				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1654				       DDP_UNTAGGED_INV_QN);
1655		break;
1656	case IRDMA_AE_AMP_BAD_STAG_KEY:
1657	case IRDMA_AE_AMP_BAD_STAG_INDEX:
1658		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1659		switch (opcode) {
1660		case IRDMA_OP_TYPE_SEND_INV:
1661		case IRDMA_OP_TYPE_SEND_SOL_INV:
1662			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,
1663					       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1664					       RDMAP_CANT_INV_STAG);
1665			break;
1666		default:
1667			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1668					       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1669					       RDMAP_INV_STAG);
1670		}
1671		break;
1672	case IRDMA_AE_AMP_RIGHTS_VIOLATION:
1673	case IRDMA_AE_AMP_INVALIDATE_NO_REMOTE_ACCESS_RIGHTS:
1674	case IRDMA_AE_PRIV_OPERATION_DENIED:
1675		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1676		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1677				       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1678				       RDMAP_ACCESS);
1679		break;
1680	case IRDMA_AE_AMP_TO_WRAP:
1681		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1682		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_ACCESS_ERR,
1683				       (LAYER_RDMA << 4) | RDMAP_REMOTE_PROT,
1684				       RDMAP_TO_WRAP);
1685		break;
1686	case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
1687		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1688				       (LAYER_MPA << 4) | DDP_LLP, MPA_CRC);
1689		break;
1690	case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
1691		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,
1692				       (LAYER_DDP << 4) | DDP_CATASTROPHIC,
1693				       DDP_CATASTROPHIC_LOCAL);
1694		break;
1695	case IRDMA_AE_LCE_QP_CATASTROPHIC:
1696	case IRDMA_AE_DDP_NO_L_BIT:
1697		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,
1698				       (LAYER_DDP << 4) | DDP_CATASTROPHIC,
1699				       DDP_CATASTROPHIC_LOCAL);
1700		break;
1701	case IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN:
1702		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1703				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1704				       DDP_UNTAGGED_INV_MSN_RANGE);
1705		break;
1706	case IRDMA_AE_DDP_UBE_DDP_MESSAGE_TOO_LONG_FOR_AVAILABLE_BUFFER:
1707		qp->event_type = IRDMA_QP_EVENT_ACCESS_ERR;
1708		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_LOC_LEN_ERR,
1709				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1710				       DDP_UNTAGGED_INV_TOO_LONG);
1711		break;
1712	case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
1713		if (is_tagged)
1714			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1715					       (LAYER_DDP << 4) | DDP_TAGGED_BUF,
1716					       DDP_TAGGED_INV_DDP_VER);
1717		else
1718			irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1719					       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1720					       DDP_UNTAGGED_INV_DDP_VER);
1721		break;
1722	case IRDMA_AE_DDP_UBE_INVALID_MO:
1723		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1724				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1725				       DDP_UNTAGGED_INV_MO);
1726		break;
1727	case IRDMA_AE_DDP_UBE_INVALID_MSN_NO_BUFFER_AVAILABLE:
1728		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_REM_OP_ERR,
1729				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1730				       DDP_UNTAGGED_INV_MSN_NO_BUF);
1731		break;
1732	case IRDMA_AE_DDP_UBE_INVALID_QN:
1733		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1734				       (LAYER_DDP << 4) | DDP_UNTAGGED_BUF,
1735				       DDP_UNTAGGED_INV_QN);
1736		break;
1737	case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
1738		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_GENERAL_ERR,
1739				       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1740				       RDMAP_INV_RDMAP_VER);
1741		break;
1742	default:
1743		irdma_bld_termhdr_ctrl(qp, termhdr, FLUSH_FATAL_ERR,
1744				       (LAYER_RDMA << 4) | RDMAP_REMOTE_OP,
1745				       RDMAP_UNSPECIFIED);
1746		break;
1747	}
1748
1749	if (copy_len)
1750		memcpy(termhdr + 1, pkt, copy_len);
1751
1752	return sizeof(struct irdma_terminate_hdr) + copy_len;
1753}
1754
1755/**
1756 * irdma_terminate_send_fin() - Send fin for terminate message
1757 * @qp: qp associated with received terminate AE
1758 */
1759void irdma_terminate_send_fin(struct irdma_sc_qp *qp)
1760{
1761	irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
1762			     IRDMAQP_TERM_SEND_FIN_ONLY, 0);
1763}
1764
1765/**
1766 * irdma_terminate_connection() - Bad AE and send terminate to remote QP
1767 * @qp: qp associated with received terminate AE
1768 * @info: the struct contiaing AE information
1769 */
1770void irdma_terminate_connection(struct irdma_sc_qp *qp,
1771				struct irdma_aeqe_info *info)
1772{
1773	u8 termlen = 0;
1774
1775	if (qp->term_flags & IRDMA_TERM_SENT)
1776		return;
1777
1778	termlen = irdma_bld_terminate_hdr(qp, info);
1779	irdma_terminate_start_timer(qp);
1780	qp->term_flags |= IRDMA_TERM_SENT;
1781	irdma_term_modify_qp(qp, IRDMA_QP_STATE_TERMINATE,
1782			     IRDMAQP_TERM_SEND_TERM_ONLY, termlen);
1783}
1784
1785/**
1786 * irdma_terminate_received - handle terminate received AE
1787 * @qp: qp associated with received terminate AE
1788 * @info: the struct contiaing AE information
1789 */
1790void irdma_terminate_received(struct irdma_sc_qp *qp,
1791			      struct irdma_aeqe_info *info)
1792{
1793	u8 *pkt = qp->q2_buf + Q2_BAD_FRAME_OFFSET;
1794	__be32 *mpa;
1795	u8 ddp_ctl;
1796	u8 rdma_ctl;
1797	u16 aeq_id = 0;
1798	struct irdma_terminate_hdr *termhdr;
1799
1800	mpa = (__be32 *)irdma_locate_mpa(pkt);
1801	if (info->q2_data_written) {
1802		/* did not validate the frame - do it now */
1803		ddp_ctl = (ntohl(mpa[0]) >> 8) & 0xff;
1804		rdma_ctl = ntohl(mpa[0]) & 0xff;
1805		if ((ddp_ctl & 0xc0) != 0x40)
1806			aeq_id = IRDMA_AE_LCE_QP_CATASTROPHIC;
1807		else if ((ddp_ctl & 0x03) != 1)
1808			aeq_id = IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION;
1809		else if (ntohl(mpa[2]) != 2)
1810			aeq_id = IRDMA_AE_DDP_UBE_INVALID_QN;
1811		else if (ntohl(mpa[3]) != 1)
1812			aeq_id = IRDMA_AE_DDP_INVALID_MSN_GAP_IN_MSN;
1813		else if (ntohl(mpa[4]) != 0)
1814			aeq_id = IRDMA_AE_DDP_UBE_INVALID_MO;
1815		else if ((rdma_ctl & 0xc0) != 0x40)
1816			aeq_id = IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION;
1817
1818		info->ae_id = aeq_id;
1819		if (info->ae_id) {
1820			/* Bad terminate recvd - send back a terminate */
1821			irdma_terminate_connection(qp, info);
1822			return;
1823		}
1824	}
1825
1826	qp->term_flags |= IRDMA_TERM_RCVD;
1827	qp->event_type = IRDMA_QP_EVENT_CATASTROPHIC;
1828	termhdr = (struct irdma_terminate_hdr *)&mpa[5];
1829	if (termhdr->layer_etype == RDMAP_REMOTE_PROT ||
1830	    termhdr->layer_etype == RDMAP_REMOTE_OP) {
1831		irdma_terminate_done(qp, 0);
1832	} else {
1833		irdma_terminate_start_timer(qp);
1834		irdma_terminate_send_fin(qp);
1835	}
1836}
1837
1838static int irdma_null_ws_add(struct irdma_sc_vsi *vsi, u8 user_pri)
1839{
1840	return 0;
1841}
1842
1843static void irdma_null_ws_remove(struct irdma_sc_vsi *vsi, u8 user_pri)
1844{
1845	/* do nothing */
1846}
1847
1848static void irdma_null_ws_reset(struct irdma_sc_vsi *vsi)
1849{
1850	/* do nothing */
1851}
1852
1853/**
1854 * irdma_sc_vsi_init - Init the vsi structure
1855 * @vsi: pointer to vsi structure to initialize
1856 * @info: the info used to initialize the vsi struct
1857 */
1858void irdma_sc_vsi_init(struct irdma_sc_vsi  *vsi,
1859		       struct irdma_vsi_init_info *info)
1860{
1861	int i;
1862
1863	vsi->dev = info->dev;
1864	vsi->back_vsi = info->back_vsi;
1865	vsi->register_qset = info->register_qset;
1866	vsi->unregister_qset = info->unregister_qset;
1867	vsi->mtu = info->params->mtu;
1868	vsi->exception_lan_q = info->exception_lan_q;
1869	vsi->vsi_idx = info->pf_data_vsi_num;
 
 
1870
1871	irdma_set_qos_info(vsi, info->params);
1872	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++) {
1873		mutex_init(&vsi->qos[i].qos_mutex);
1874		INIT_LIST_HEAD(&vsi->qos[i].qplist);
1875	}
1876	if (vsi->register_qset) {
1877		vsi->dev->ws_add = irdma_ws_add;
1878		vsi->dev->ws_remove = irdma_ws_remove;
1879		vsi->dev->ws_reset = irdma_ws_reset;
1880	} else {
1881		vsi->dev->ws_add = irdma_null_ws_add;
1882		vsi->dev->ws_remove = irdma_null_ws_remove;
1883		vsi->dev->ws_reset = irdma_null_ws_reset;
1884	}
1885}
1886
1887/**
1888 * irdma_get_stats_idx - Return stats index
1889 * @vsi: pointer to the vsi
1890 */
1891static u8 irdma_get_stats_idx(struct irdma_sc_vsi *vsi)
1892{
1893	struct irdma_stats_inst_info stats_info = {};
1894	struct irdma_sc_dev *dev = vsi->dev;
1895	u8 i;
 
1896
1897	if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1898		if (!irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_ALLOCATE,
1899					      &stats_info))
1900			return stats_info.stats_idx;
1901	}
1902
1903	for (i = 0; i < IRDMA_MAX_STATS_COUNT_GEN_1; i++) {
1904		if (!dev->stats_idx_array[i]) {
1905			dev->stats_idx_array[i] = true;
1906			return i;
 
 
 
1907		}
1908	}
1909
1910	return IRDMA_INVALID_STATS_IDX;
1911}
1912
1913/**
1914 * irdma_hw_stats_init_gen1 - Initialize stat reg table used for gen1
1915 * @vsi: vsi structure where hw_regs are set
1916 *
1917 * Populate the HW stats table
1918 */
1919static void irdma_hw_stats_init_gen1(struct irdma_sc_vsi *vsi)
1920{
1921	struct irdma_sc_dev *dev = vsi->dev;
1922	const struct irdma_hw_stat_map *map;
1923	u64 *stat_reg = vsi->hw_stats_regs;
1924	u64 *regs = dev->hw_stats_regs;
1925	u16 i, stats_reg_set = vsi->stats_idx;
1926
1927	map = dev->hw_stats_map;
1928
1929	/* First 4 stat instances are reserved for port level statistics. */
1930	stats_reg_set += vsi->stats_inst_alloc ? IRDMA_FIRST_NON_PF_STAT : 0;
1931
1932	for (i = 0; i < dev->hw_attrs.max_stat_idx; i++) {
1933		if (map[i].bitmask <= IRDMA_MAX_STATS_32)
1934			stat_reg[i] = regs[i] + stats_reg_set * sizeof(u32);
1935		else
1936			stat_reg[i] = regs[i] + stats_reg_set * sizeof(u64);
1937	}
1938}
1939
1940/**
1941 * irdma_vsi_stats_init - Initialize the vsi statistics
1942 * @vsi: pointer to the vsi structure
1943 * @info: The info structure used for initialization
1944 */
1945int irdma_vsi_stats_init(struct irdma_sc_vsi *vsi,
1946			 struct irdma_vsi_stats_info *info)
1947{
 
1948	struct irdma_dma_mem *stats_buff_mem;
1949
1950	vsi->pestat = info->pestat;
1951	vsi->pestat->hw = vsi->dev->hw;
1952	vsi->pestat->vsi = vsi;
1953	stats_buff_mem = &vsi->pestat->gather_info.stats_buff_mem;
1954	stats_buff_mem->size = ALIGN(IRDMA_GATHER_STATS_BUF_SIZE * 2, 1);
1955	stats_buff_mem->va = dma_alloc_coherent(vsi->pestat->hw->device,
1956						stats_buff_mem->size,
1957						&stats_buff_mem->pa,
1958						GFP_KERNEL);
1959	if (!stats_buff_mem->va)
1960		return -ENOMEM;
1961
1962	vsi->pestat->gather_info.gather_stats_va = stats_buff_mem->va;
1963	vsi->pestat->gather_info.last_gather_stats_va =
1964		(void *)((uintptr_t)stats_buff_mem->va +
1965			 IRDMA_GATHER_STATS_BUF_SIZE);
1966
1967	irdma_hw_stats_start_timer(vsi);
 
 
 
 
1968
1969	/* when stat allocation is not required default to fcn_id. */
1970	vsi->stats_idx = info->fcn_id;
1971	if (info->alloc_stats_inst) {
1972		u8 stats_idx = irdma_get_stats_idx(vsi);
1973
1974		if (stats_idx != IRDMA_INVALID_STATS_IDX) {
1975			vsi->stats_inst_alloc = true;
1976			vsi->stats_idx = stats_idx;
1977			vsi->pestat->gather_info.use_stats_inst = true;
1978			vsi->pestat->gather_info.stats_inst_index = stats_idx;
1979		}
1980	}
1981
1982	if (vsi->dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
1983		irdma_hw_stats_init_gen1(vsi);
1984
1985	return 0;
 
 
 
 
 
 
 
1986}
1987
1988/**
1989 * irdma_vsi_stats_free - Free the vsi stats
1990 * @vsi: pointer to the vsi structure
1991 */
1992void irdma_vsi_stats_free(struct irdma_sc_vsi *vsi)
1993{
1994	struct irdma_stats_inst_info stats_info = {};
 
1995	struct irdma_sc_dev *dev = vsi->dev;
1996	u8 stats_idx = vsi->stats_idx;
1997
1998	if (dev->hw_attrs.uk_attrs.hw_rev >= IRDMA_GEN_2) {
1999		if (vsi->stats_inst_alloc) {
2000			stats_info.stats_idx = vsi->stats_idx;
2001			irdma_cqp_stats_inst_cmd(vsi, IRDMA_OP_STATS_FREE,
2002						 &stats_info);
2003		}
2004	} else {
2005		if (vsi->stats_inst_alloc &&
2006		    stats_idx < vsi->dev->hw_attrs.max_stat_inst)
2007			vsi->dev->stats_idx_array[stats_idx] = false;
2008	}
2009
2010	if (!vsi->pestat)
2011		return;
2012	irdma_hw_stats_stop_timer(vsi);
2013	dma_free_coherent(vsi->pestat->hw->device,
2014			  vsi->pestat->gather_info.stats_buff_mem.size,
2015			  vsi->pestat->gather_info.stats_buff_mem.va,
2016			  vsi->pestat->gather_info.stats_buff_mem.pa);
2017	vsi->pestat->gather_info.stats_buff_mem.va = NULL;
2018}
2019
2020/**
2021 * irdma_get_encoded_wqe_size - given wq size, returns hardware encoded size
2022 * @wqsize: size of the wq (sq, rq) to encoded_size
2023 * @queue_type: queue type selected for the calculation algorithm
2024 */
2025u8 irdma_get_encoded_wqe_size(u32 wqsize, enum irdma_queue_type queue_type)
2026{
2027	u8 encoded_size = 0;
2028
2029	/* cqp sq's hw coded value starts from 1 for size of 4
2030	 * while it starts from 0 for qp' wq's.
2031	 */
2032	if (queue_type == IRDMA_QUEUE_TYPE_CQP)
2033		encoded_size = 1;
2034	wqsize >>= 2;
2035	while (wqsize >>= 1)
2036		encoded_size++;
2037
2038	return encoded_size;
2039}
2040
2041/**
2042 * irdma_sc_gather_stats - collect the statistics
2043 * @cqp: struct for cqp hw
2044 * @info: gather stats info structure
2045 * @scratch: u64 saved to be used during cqp completion
2046 */
2047static int irdma_sc_gather_stats(struct irdma_sc_cqp *cqp,
2048				 struct irdma_stats_gather_info *info,
2049				 u64 scratch)
2050{
2051	__le64 *wqe;
2052	u64 temp;
2053
2054	if (info->stats_buff_mem.size < IRDMA_GATHER_STATS_BUF_SIZE)
2055		return -ENOMEM;
2056
2057	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2058	if (!wqe)
2059		return -ENOMEM;
2060
2061	set_64bit_val(wqe, 40,
2062		      FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fcn_index));
2063	set_64bit_val(wqe, 32, info->stats_buff_mem.pa);
2064
2065	temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |
2066	       FIELD_PREP(IRDMA_CQPSQ_STATS_USE_INST, info->use_stats_inst) |
2067	       FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX,
2068			  info->stats_inst_index) |
2069	       FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,
2070			  info->use_hmc_fcn_index) |
2071	       FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_GATHER_STATS);
2072	dma_wmb(); /* make sure WQE is written before valid bit is set */
2073
2074	set_64bit_val(wqe, 24, temp);
2075
2076	print_hex_dump_debug("STATS: GATHER_STATS WQE", DUMP_PREFIX_OFFSET,
2077			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2078
2079	irdma_sc_cqp_post_sq(cqp);
2080	ibdev_dbg(to_ibdev(cqp->dev),
2081		  "STATS: CQP SQ head 0x%x tail 0x%x size 0x%x\n",
2082		  cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);
2083
2084	return 0;
2085}
2086
2087/**
2088 * irdma_sc_manage_stats_inst - allocate or free stats instance
2089 * @cqp: struct for cqp hw
2090 * @info: stats info structure
2091 * @alloc: alloc vs. delete flag
2092 * @scratch: u64 saved to be used during cqp completion
2093 */
2094static int irdma_sc_manage_stats_inst(struct irdma_sc_cqp *cqp,
2095				      struct irdma_stats_inst_info *info,
2096				      bool alloc, u64 scratch)
2097{
2098	__le64 *wqe;
2099	u64 temp;
2100
2101	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2102	if (!wqe)
2103		return -ENOMEM;
2104
2105	set_64bit_val(wqe, 40,
2106		      FIELD_PREP(IRDMA_CQPSQ_STATS_HMC_FCN_INDEX, info->hmc_fn_id));
2107	temp = FIELD_PREP(IRDMA_CQPSQ_STATS_WQEVALID, cqp->polarity) |
2108	       FIELD_PREP(IRDMA_CQPSQ_STATS_ALLOC_INST, alloc) |
2109	       FIELD_PREP(IRDMA_CQPSQ_STATS_USE_HMC_FCN_INDEX,
2110			  info->use_hmc_fcn_index) |
2111	       FIELD_PREP(IRDMA_CQPSQ_STATS_INST_INDEX, info->stats_idx) |
2112	       FIELD_PREP(IRDMA_CQPSQ_STATS_OP, IRDMA_CQP_OP_MANAGE_STATS);
2113
2114	dma_wmb(); /* make sure WQE is written before valid bit is set */
2115
2116	set_64bit_val(wqe, 24, temp);
2117
2118	print_hex_dump_debug("WQE: MANAGE_STATS WQE", DUMP_PREFIX_OFFSET, 16,
2119			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2120
2121	irdma_sc_cqp_post_sq(cqp);
2122	return 0;
2123}
2124
2125/**
2126 * irdma_sc_set_up_map - set the up map table
2127 * @cqp: struct for cqp hw
2128 * @info: User priority map info
2129 * @scratch: u64 saved to be used during cqp completion
2130 */
2131static int irdma_sc_set_up_map(struct irdma_sc_cqp *cqp,
2132			       struct irdma_up_info *info, u64 scratch)
2133{
2134	__le64 *wqe;
2135	u64 temp = 0;
2136	int i;
2137
2138	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2139	if (!wqe)
2140		return -ENOMEM;
2141
2142	for (i = 0; i < IRDMA_MAX_USER_PRIORITY; i++)
2143		temp |= (u64)info->map[i] << (i * 8);
2144
2145	set_64bit_val(wqe, 0, temp);
2146	set_64bit_val(wqe, 40,
2147		      FIELD_PREP(IRDMA_CQPSQ_UP_CNPOVERRIDE, info->cnp_up_override) |
2148		      FIELD_PREP(IRDMA_CQPSQ_UP_HMCFCNIDX, info->hmc_fcn_idx));
2149
2150	temp = FIELD_PREP(IRDMA_CQPSQ_UP_WQEVALID, cqp->polarity) |
2151	       FIELD_PREP(IRDMA_CQPSQ_UP_USEVLAN, info->use_vlan) |
2152	       FIELD_PREP(IRDMA_CQPSQ_UP_USEOVERRIDE,
2153			  info->use_cnp_up_override) |
2154	       FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_UP_MAP);
2155	dma_wmb(); /* make sure WQE is written before valid bit is set */
2156
2157	set_64bit_val(wqe, 24, temp);
2158
2159	print_hex_dump_debug("WQE: UPMAP WQE", DUMP_PREFIX_OFFSET, 16, 8, wqe,
2160			     IRDMA_CQP_WQE_SIZE * 8, false);
2161	irdma_sc_cqp_post_sq(cqp);
2162
2163	return 0;
2164}
2165
2166/**
2167 * irdma_sc_manage_ws_node - create/modify/destroy WS node
2168 * @cqp: struct for cqp hw
2169 * @info: node info structure
2170 * @node_op: 0 for add 1 for modify, 2 for delete
2171 * @scratch: u64 saved to be used during cqp completion
2172 */
2173static int irdma_sc_manage_ws_node(struct irdma_sc_cqp *cqp,
2174				   struct irdma_ws_node_info *info,
2175				   enum irdma_ws_node_op node_op, u64 scratch)
2176{
2177	__le64 *wqe;
2178	u64 temp = 0;
2179
2180	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2181	if (!wqe)
2182		return -ENOMEM;
2183
2184	set_64bit_val(wqe, 32,
2185		      FIELD_PREP(IRDMA_CQPSQ_WS_VSI, info->vsi) |
2186		      FIELD_PREP(IRDMA_CQPSQ_WS_WEIGHT, info->weight));
2187
2188	temp = FIELD_PREP(IRDMA_CQPSQ_WS_WQEVALID, cqp->polarity) |
2189	       FIELD_PREP(IRDMA_CQPSQ_WS_NODEOP, node_op) |
2190	       FIELD_PREP(IRDMA_CQPSQ_WS_ENABLENODE, info->enable) |
2191	       FIELD_PREP(IRDMA_CQPSQ_WS_NODETYPE, info->type_leaf) |
2192	       FIELD_PREP(IRDMA_CQPSQ_WS_PRIOTYPE, info->prio_type) |
2193	       FIELD_PREP(IRDMA_CQPSQ_WS_TC, info->tc) |
2194	       FIELD_PREP(IRDMA_CQPSQ_WS_OP, IRDMA_CQP_OP_WORK_SCHED_NODE) |
2195	       FIELD_PREP(IRDMA_CQPSQ_WS_PARENTID, info->parent_id) |
2196	       FIELD_PREP(IRDMA_CQPSQ_WS_NODEID, info->id);
2197	dma_wmb(); /* make sure WQE is written before valid bit is set */
2198
2199	set_64bit_val(wqe, 24, temp);
2200
2201	print_hex_dump_debug("WQE: MANAGE_WS WQE", DUMP_PREFIX_OFFSET, 16, 8,
2202			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2203	irdma_sc_cqp_post_sq(cqp);
2204
2205	return 0;
2206}
2207
2208/**
2209 * irdma_sc_qp_flush_wqes - flush qp's wqe
2210 * @qp: sc qp
2211 * @info: dlush information
2212 * @scratch: u64 saved to be used during cqp completion
2213 * @post_sq: flag for cqp db to ring
2214 */
2215int irdma_sc_qp_flush_wqes(struct irdma_sc_qp *qp,
2216			   struct irdma_qp_flush_info *info, u64 scratch,
2217			   bool post_sq)
2218{
2219	u64 temp = 0;
2220	__le64 *wqe;
2221	struct irdma_sc_cqp *cqp;
2222	u64 hdr;
2223	bool flush_sq = false, flush_rq = false;
2224
2225	if (info->rq && !qp->flush_rq)
2226		flush_rq = true;
2227	if (info->sq && !qp->flush_sq)
2228		flush_sq = true;
2229	qp->flush_sq |= flush_sq;
2230	qp->flush_rq |= flush_rq;
2231
2232	if (!flush_sq && !flush_rq) {
2233		ibdev_dbg(to_ibdev(qp->dev),
2234			  "CQP: Additional flush request ignored for qp %x\n",
2235			  qp->qp_uk.qp_id);
2236		return -EALREADY;
2237	}
2238
2239	cqp = qp->pd->dev->cqp;
2240	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2241	if (!wqe)
2242		return -ENOMEM;
2243
2244	if (info->userflushcode) {
2245		if (flush_rq)
2246			temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMNERR,
2247					   info->rq_minor_code) |
2248				FIELD_PREP(IRDMA_CQPSQ_FWQE_RQMJERR,
2249					   info->rq_major_code);
2250		if (flush_sq)
2251			temp |= FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMNERR,
2252					   info->sq_minor_code) |
2253				FIELD_PREP(IRDMA_CQPSQ_FWQE_SQMJERR,
2254					   info->sq_major_code);
2255	}
2256	set_64bit_val(wqe, 16, temp);
2257
2258	temp = (info->generate_ae) ?
2259		info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,
2260					   info->ae_src) : 0;
2261	set_64bit_val(wqe, 8, temp);
2262
2263	hdr = qp->qp_uk.qp_id |
2264	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_FLUSH_WQES) |
2265	      FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, info->generate_ae) |
2266	      FIELD_PREP(IRDMA_CQPSQ_FWQE_USERFLCODE, info->userflushcode) |
2267	      FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHSQ, flush_sq) |
2268	      FIELD_PREP(IRDMA_CQPSQ_FWQE_FLUSHRQ, flush_rq) |
2269	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2270	dma_wmb(); /* make sure WQE is written before valid bit is set */
2271
2272	set_64bit_val(wqe, 24, hdr);
2273
2274	print_hex_dump_debug("WQE: QP_FLUSH WQE", DUMP_PREFIX_OFFSET, 16, 8,
2275			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2276	if (post_sq)
2277		irdma_sc_cqp_post_sq(cqp);
2278
2279	return 0;
2280}
2281
2282/**
2283 * irdma_sc_gen_ae - generate AE, uses flush WQE CQP OP
2284 * @qp: sc qp
2285 * @info: gen ae information
2286 * @scratch: u64 saved to be used during cqp completion
2287 * @post_sq: flag for cqp db to ring
2288 */
2289static int irdma_sc_gen_ae(struct irdma_sc_qp *qp,
2290			   struct irdma_gen_ae_info *info, u64 scratch,
2291			   bool post_sq)
2292{
2293	u64 temp;
2294	__le64 *wqe;
2295	struct irdma_sc_cqp *cqp;
2296	u64 hdr;
2297
2298	cqp = qp->pd->dev->cqp;
2299	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2300	if (!wqe)
2301		return -ENOMEM;
2302
2303	temp = info->ae_code | FIELD_PREP(IRDMA_CQPSQ_FWQE_AESOURCE,
2304					  info->ae_src);
2305	set_64bit_val(wqe, 8, temp);
2306
2307	hdr = qp->qp_uk.qp_id | FIELD_PREP(IRDMA_CQPSQ_OPCODE,
2308					   IRDMA_CQP_OP_GEN_AE) |
2309	      FIELD_PREP(IRDMA_CQPSQ_FWQE_GENERATE_AE, 1) |
2310	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2311	dma_wmb(); /* make sure WQE is written before valid bit is set */
2312
2313	set_64bit_val(wqe, 24, hdr);
2314
2315	print_hex_dump_debug("WQE: GEN_AE WQE", DUMP_PREFIX_OFFSET, 16, 8,
2316			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2317	if (post_sq)
2318		irdma_sc_cqp_post_sq(cqp);
2319
2320	return 0;
2321}
2322
2323/*** irdma_sc_qp_upload_context - upload qp's context
2324 * @dev: sc device struct
2325 * @info: upload context info ptr for return
2326 * @scratch: u64 saved to be used during cqp completion
2327 * @post_sq: flag for cqp db to ring
2328 */
2329static int irdma_sc_qp_upload_context(struct irdma_sc_dev *dev,
2330				      struct irdma_upload_context_info *info,
2331				      u64 scratch, bool post_sq)
2332{
2333	__le64 *wqe;
2334	struct irdma_sc_cqp *cqp;
2335	u64 hdr;
2336
2337	cqp = dev->cqp;
2338	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2339	if (!wqe)
2340		return -ENOMEM;
2341
2342	set_64bit_val(wqe, 16, info->buf_pa);
2343
2344	hdr = FIELD_PREP(IRDMA_CQPSQ_UCTX_QPID, info->qp_id) |
2345	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPLOAD_CONTEXT) |
2346	      FIELD_PREP(IRDMA_CQPSQ_UCTX_QPTYPE, info->qp_type) |
2347	      FIELD_PREP(IRDMA_CQPSQ_UCTX_RAWFORMAT, info->raw_format) |
2348	      FIELD_PREP(IRDMA_CQPSQ_UCTX_FREEZEQP, info->freeze_qp) |
2349	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2350	dma_wmb(); /* make sure WQE is written before valid bit is set */
2351
2352	set_64bit_val(wqe, 24, hdr);
2353
2354	print_hex_dump_debug("WQE: QP_UPLOAD_CTX WQE", DUMP_PREFIX_OFFSET, 16,
2355			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2356	if (post_sq)
2357		irdma_sc_cqp_post_sq(cqp);
2358
2359	return 0;
2360}
2361
2362/**
2363 * irdma_sc_manage_push_page - Handle push page
2364 * @cqp: struct for cqp hw
2365 * @info: push page info
2366 * @scratch: u64 saved to be used during cqp completion
2367 * @post_sq: flag for cqp db to ring
2368 */
2369static int irdma_sc_manage_push_page(struct irdma_sc_cqp *cqp,
2370				     struct irdma_cqp_manage_push_page_info *info,
2371				     u64 scratch, bool post_sq)
2372{
2373	__le64 *wqe;
2374	u64 hdr;
2375
2376	if (info->free_page &&
2377	    info->push_idx >= cqp->dev->hw_attrs.max_hw_device_pages)
2378		return -EINVAL;
2379
2380	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2381	if (!wqe)
2382		return -ENOMEM;
2383
2384	set_64bit_val(wqe, 16, info->qs_handle);
2385	hdr = FIELD_PREP(IRDMA_CQPSQ_MPP_PPIDX, info->push_idx) |
2386	      FIELD_PREP(IRDMA_CQPSQ_MPP_PPTYPE, info->push_page_type) |
2387	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MANAGE_PUSH_PAGES) |
2388	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
2389	      FIELD_PREP(IRDMA_CQPSQ_MPP_FREE_PAGE, info->free_page);
2390	dma_wmb(); /* make sure WQE is written before valid bit is set */
2391
2392	set_64bit_val(wqe, 24, hdr);
2393
2394	print_hex_dump_debug("WQE: MANAGE_PUSH_PAGES WQE", DUMP_PREFIX_OFFSET,
2395			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2396	if (post_sq)
2397		irdma_sc_cqp_post_sq(cqp);
2398
2399	return 0;
2400}
2401
2402/**
2403 * irdma_sc_suspend_qp - suspend qp for param change
2404 * @cqp: struct for cqp hw
2405 * @qp: sc qp struct
2406 * @scratch: u64 saved to be used during cqp completion
2407 */
2408static int irdma_sc_suspend_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp,
2409			       u64 scratch)
2410{
2411	u64 hdr;
2412	__le64 *wqe;
2413
2414	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2415	if (!wqe)
2416		return -ENOMEM;
2417
2418	hdr = FIELD_PREP(IRDMA_CQPSQ_SUSPENDQP_QPID, qp->qp_uk.qp_id) |
2419	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_SUSPEND_QP) |
2420	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2421	dma_wmb(); /* make sure WQE is written before valid bit is set */
2422
2423	set_64bit_val(wqe, 24, hdr);
2424
2425	print_hex_dump_debug("WQE: SUSPEND_QP WQE", DUMP_PREFIX_OFFSET, 16, 8,
2426			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2427	irdma_sc_cqp_post_sq(cqp);
2428
2429	return 0;
2430}
2431
2432/**
2433 * irdma_sc_resume_qp - resume qp after suspend
2434 * @cqp: struct for cqp hw
2435 * @qp: sc qp struct
2436 * @scratch: u64 saved to be used during cqp completion
2437 */
2438static int irdma_sc_resume_qp(struct irdma_sc_cqp *cqp, struct irdma_sc_qp *qp,
2439			      u64 scratch)
2440{
2441	u64 hdr;
2442	__le64 *wqe;
2443
2444	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2445	if (!wqe)
2446		return -ENOMEM;
2447
2448	set_64bit_val(wqe, 16,
2449		      FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QSHANDLE, qp->qs_handle));
2450
2451	hdr = FIELD_PREP(IRDMA_CQPSQ_RESUMEQP_QPID, qp->qp_uk.qp_id) |
2452	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_RESUME_QP) |
2453	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2454	dma_wmb(); /* make sure WQE is written before valid bit is set */
2455
2456	set_64bit_val(wqe, 24, hdr);
2457
2458	print_hex_dump_debug("WQE: RESUME_QP WQE", DUMP_PREFIX_OFFSET, 16, 8,
2459			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2460	irdma_sc_cqp_post_sq(cqp);
2461
2462	return 0;
2463}
2464
2465/**
2466 * irdma_sc_cq_ack - acknowledge completion q
2467 * @cq: cq struct
2468 */
2469static inline void irdma_sc_cq_ack(struct irdma_sc_cq *cq)
2470{
2471	writel(cq->cq_uk.cq_id, cq->cq_uk.cq_ack_db);
2472}
2473
2474/**
2475 * irdma_sc_cq_init - initialize completion q
2476 * @cq: cq struct
2477 * @info: cq initialization info
2478 */
2479int irdma_sc_cq_init(struct irdma_sc_cq *cq, struct irdma_cq_init_info *info)
2480{
2481	u32 pble_obj_cnt;
2482
2483	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
2484	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
2485		return -EINVAL;
2486
2487	cq->cq_pa = info->cq_base_pa;
2488	cq->dev = info->dev;
2489	cq->ceq_id = info->ceq_id;
2490	info->cq_uk_init_info.cqe_alloc_db = cq->dev->cq_arm_db;
2491	info->cq_uk_init_info.cq_ack_db = cq->dev->cq_ack_db;
2492	irdma_uk_cq_init(&cq->cq_uk, &info->cq_uk_init_info);
2493
2494	cq->virtual_map = info->virtual_map;
2495	cq->pbl_chunk_size = info->pbl_chunk_size;
2496	cq->ceqe_mask = info->ceqe_mask;
2497	cq->cq_type = (info->type) ? info->type : IRDMA_CQ_TYPE_IWARP;
2498	cq->shadow_area_pa = info->shadow_area_pa;
2499	cq->shadow_read_threshold = info->shadow_read_threshold;
2500	cq->ceq_id_valid = info->ceq_id_valid;
2501	cq->tph_en = info->tph_en;
2502	cq->tph_val = info->tph_val;
2503	cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2504	cq->vsi = info->vsi;
2505
2506	return 0;
2507}
2508
2509/**
2510 * irdma_sc_cq_create - create completion q
2511 * @cq: cq struct
2512 * @scratch: u64 saved to be used during cqp completion
2513 * @check_overflow: flag for overflow check
2514 * @post_sq: flag for cqp db to ring
2515 */
2516static int irdma_sc_cq_create(struct irdma_sc_cq *cq, u64 scratch,
2517			      bool check_overflow, bool post_sq)
2518{
2519	__le64 *wqe;
2520	struct irdma_sc_cqp *cqp;
2521	u64 hdr;
2522	struct irdma_sc_ceq *ceq;
2523	int ret_code = 0;
2524
2525	cqp = cq->dev->cqp;
2526	if (cq->cq_uk.cq_id >= cqp->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt)
2527		return -EINVAL;
2528
2529	if (cq->ceq_id >= cq->dev->hmc_fpm_misc.max_ceqs)
2530		return -EINVAL;
2531
2532	ceq = cq->dev->ceq[cq->ceq_id];
2533	if (ceq && ceq->reg_cq)
2534		ret_code = irdma_sc_add_cq_ctx(ceq, cq);
2535
2536	if (ret_code)
2537		return ret_code;
2538
2539	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2540	if (!wqe) {
2541		if (ceq && ceq->reg_cq)
2542			irdma_sc_remove_cq_ctx(ceq, cq);
2543		return -ENOMEM;
2544	}
2545
2546	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2547	set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2548	set_64bit_val(wqe, 16,
2549		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, cq->shadow_read_threshold));
2550	set_64bit_val(wqe, 32, (cq->virtual_map ? 0 : cq->cq_pa));
2551	set_64bit_val(wqe, 40, cq->shadow_area_pa);
2552	set_64bit_val(wqe, 48,
2553		      FIELD_PREP(IRDMA_CQPSQ_CQ_FIRSTPMPBLIDX, (cq->virtual_map ? cq->first_pm_pbl_idx : 0)));
2554	set_64bit_val(wqe, 56,
2555		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
2556		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
2557
2558	hdr = FLD_LS_64(cq->dev, cq->cq_uk.cq_id, IRDMA_CQPSQ_CQ_CQID) |
2559	      FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
2560			IRDMA_CQPSQ_CQ_CEQID) |
2561	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CQ) |
2562	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
2563	      FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, check_overflow) |
2564	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |
2565	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2566	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |
2567	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2568	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,
2569			 cq->cq_uk.avoid_mem_cflct) |
2570	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2571	dma_wmb(); /* make sure WQE is written before valid bit is set */
2572
2573	set_64bit_val(wqe, 24, hdr);
2574
2575	print_hex_dump_debug("WQE: CQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
2576			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2577	if (post_sq)
2578		irdma_sc_cqp_post_sq(cqp);
2579
2580	return 0;
2581}
2582
2583/**
2584 * irdma_sc_cq_destroy - destroy completion q
2585 * @cq: cq struct
2586 * @scratch: u64 saved to be used during cqp completion
2587 * @post_sq: flag for cqp db to ring
2588 */
2589int irdma_sc_cq_destroy(struct irdma_sc_cq *cq, u64 scratch, bool post_sq)
2590{
2591	struct irdma_sc_cqp *cqp;
2592	__le64 *wqe;
2593	u64 hdr;
2594	struct irdma_sc_ceq *ceq;
2595
2596	cqp = cq->dev->cqp;
2597	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2598	if (!wqe)
2599		return -ENOMEM;
2600
2601	ceq = cq->dev->ceq[cq->ceq_id];
2602	if (ceq && ceq->reg_cq)
2603		irdma_sc_remove_cq_ctx(ceq, cq);
2604
2605	set_64bit_val(wqe, 0, cq->cq_uk.cq_size);
2606	set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2607	set_64bit_val(wqe, 40, cq->shadow_area_pa);
2608	set_64bit_val(wqe, 48,
2609		      (cq->virtual_map ? cq->first_pm_pbl_idx : 0));
2610
2611	hdr = cq->cq_uk.cq_id |
2612	      FLD_LS_64(cq->dev, (cq->ceq_id_valid ? cq->ceq_id : 0),
2613			IRDMA_CQPSQ_CQ_CEQID) |
2614	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |
2615	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, cq->pbl_chunk_size) |
2616	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, cq->virtual_map) |
2617	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2618	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, cq->ceq_id_valid) |
2619	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2620	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, cq->cq_uk.avoid_mem_cflct) |
2621	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2622	dma_wmb(); /* make sure WQE is written before valid bit is set */
2623
2624	set_64bit_val(wqe, 24, hdr);
2625
2626	print_hex_dump_debug("WQE: CQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16, 8,
2627			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2628	if (post_sq)
2629		irdma_sc_cqp_post_sq(cqp);
2630
2631	return 0;
2632}
2633
2634/**
2635 * irdma_sc_cq_resize - set resized cq buffer info
2636 * @cq: resized cq
2637 * @info: resized cq buffer info
2638 */
2639void irdma_sc_cq_resize(struct irdma_sc_cq *cq, struct irdma_modify_cq_info *info)
2640{
2641	cq->virtual_map = info->virtual_map;
2642	cq->cq_pa = info->cq_pa;
2643	cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
2644	cq->pbl_chunk_size = info->pbl_chunk_size;
2645	irdma_uk_cq_resize(&cq->cq_uk, info->cq_base, info->cq_size);
2646}
2647
2648/**
2649 * irdma_sc_cq_modify - modify a Completion Queue
2650 * @cq: cq struct
2651 * @info: modification info struct
2652 * @scratch: u64 saved to be used during cqp completion
2653 * @post_sq: flag to post to sq
2654 */
2655static int irdma_sc_cq_modify(struct irdma_sc_cq *cq,
2656			      struct irdma_modify_cq_info *info, u64 scratch,
2657			      bool post_sq)
2658{
2659	struct irdma_sc_cqp *cqp;
2660	__le64 *wqe;
2661	u64 hdr;
2662	u32 pble_obj_cnt;
2663
2664	pble_obj_cnt = cq->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
2665	if (info->cq_resize && info->virtual_map &&
2666	    info->first_pm_pbl_idx >= pble_obj_cnt)
2667		return -EINVAL;
2668
2669	cqp = cq->dev->cqp;
2670	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
2671	if (!wqe)
2672		return -ENOMEM;
2673
2674	set_64bit_val(wqe, 0, info->cq_size);
2675	set_64bit_val(wqe, 8, (uintptr_t)cq >> 1);
2676	set_64bit_val(wqe, 16,
2677		      FIELD_PREP(IRDMA_CQPSQ_CQ_SHADOW_READ_THRESHOLD, info->shadow_read_threshold));
2678	set_64bit_val(wqe, 32, info->cq_pa);
2679	set_64bit_val(wqe, 40, cq->shadow_area_pa);
2680	set_64bit_val(wqe, 48, info->first_pm_pbl_idx);
2681	set_64bit_val(wqe, 56,
2682		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, cq->tph_val) |
2683		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, cq->vsi->vsi_idx));
2684
2685	hdr = cq->cq_uk.cq_id |
2686	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_MODIFY_CQ) |
2687	      FIELD_PREP(IRDMA_CQPSQ_CQ_CQRESIZE, info->cq_resize) |
2688	      FIELD_PREP(IRDMA_CQPSQ_CQ_LPBLSIZE, info->pbl_chunk_size) |
2689	      FIELD_PREP(IRDMA_CQPSQ_CQ_CHKOVERFLOW, info->check_overflow) |
2690	      FIELD_PREP(IRDMA_CQPSQ_CQ_VIRTMAP, info->virtual_map) |
2691	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, cq->ceqe_mask) |
2692	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, cq->tph_en) |
2693	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT,
2694			 cq->cq_uk.avoid_mem_cflct) |
2695	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
2696	dma_wmb(); /* make sure WQE is written before valid bit is set */
2697
2698	set_64bit_val(wqe, 24, hdr);
2699
2700	print_hex_dump_debug("WQE: CQ_MODIFY WQE", DUMP_PREFIX_OFFSET, 16, 8,
2701			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
2702	if (post_sq)
2703		irdma_sc_cqp_post_sq(cqp);
2704
2705	return 0;
2706}
2707
2708/**
2709 * irdma_check_cqp_progress - check cqp processing progress
2710 * @timeout: timeout info struct
2711 * @dev: sc device struct
2712 */
2713void irdma_check_cqp_progress(struct irdma_cqp_timeout *timeout, struct irdma_sc_dev *dev)
2714{
2715	u64 completed_ops = atomic64_read(&dev->cqp->completed_ops);
2716
2717	if (timeout->compl_cqp_cmds != completed_ops) {
2718		timeout->compl_cqp_cmds = completed_ops;
2719		timeout->count = 0;
2720	} else if (timeout->compl_cqp_cmds != dev->cqp->requested_ops) {
2721		timeout->count++;
 
 
2722	}
2723}
2724
2725/**
2726 * irdma_get_cqp_reg_info - get head and tail for cqp using registers
2727 * @cqp: struct for cqp hw
2728 * @val: cqp tail register value
2729 * @tail: wqtail register value
2730 * @error: cqp processing err
2731 */
2732static inline void irdma_get_cqp_reg_info(struct irdma_sc_cqp *cqp, u32 *val,
2733					  u32 *tail, u32 *error)
2734{
2735	*val = readl(cqp->dev->hw_regs[IRDMA_CQPTAIL]);
2736	*tail = FIELD_GET(IRDMA_CQPTAIL_WQTAIL, *val);
2737	*error = FIELD_GET(IRDMA_CQPTAIL_CQP_OP_ERR, *val);
2738}
2739
2740/**
2741 * irdma_cqp_poll_registers - poll cqp registers
2742 * @cqp: struct for cqp hw
2743 * @tail: wqtail register value
2744 * @count: how many times to try for completion
2745 */
2746static int irdma_cqp_poll_registers(struct irdma_sc_cqp *cqp, u32 tail,
2747				    u32 count)
2748{
2749	u32 i = 0;
2750	u32 newtail, error, val;
2751
2752	while (i++ < count) {
2753		irdma_get_cqp_reg_info(cqp, &val, &newtail, &error);
2754		if (error) {
2755			error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
2756			ibdev_dbg(to_ibdev(cqp->dev),
2757				  "CQP: CQPERRCODES error_code[x%08X]\n",
2758				  error);
2759			return -EIO;
2760		}
2761		if (newtail != tail) {
2762			/* SUCCESS */
2763			IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
2764			atomic64_inc(&cqp->completed_ops);
2765			return 0;
2766		}
2767		udelay(cqp->dev->hw_attrs.max_sleep_count);
2768	}
2769
2770	return -ETIMEDOUT;
2771}
2772
2773/**
2774 * irdma_sc_decode_fpm_commit - decode a 64 bit value into count and base
2775 * @dev: sc device struct
2776 * @buf: pointer to commit buffer
2777 * @buf_idx: buffer index
2778 * @obj_info: object info pointer
2779 * @rsrc_idx: indexs of memory resource
2780 */
2781static u64 irdma_sc_decode_fpm_commit(struct irdma_sc_dev *dev, __le64 *buf,
2782				      u32 buf_idx, struct irdma_hmc_obj_info *obj_info,
2783				      u32 rsrc_idx)
2784{
2785	u64 temp;
2786
2787	get_64bit_val(buf, buf_idx, &temp);
2788
2789	switch (rsrc_idx) {
2790	case IRDMA_HMC_IW_QP:
2791		obj_info[rsrc_idx].cnt = (u32)FIELD_GET(IRDMA_COMMIT_FPM_QPCNT, temp);
2792		break;
2793	case IRDMA_HMC_IW_CQ:
2794		obj_info[rsrc_idx].cnt = (u32)FLD_RS_64(dev, temp, IRDMA_COMMIT_FPM_CQCNT);
2795		break;
2796	case IRDMA_HMC_IW_APBVT_ENTRY:
2797		obj_info[rsrc_idx].cnt = 1;
2798		break;
2799	default:
2800		obj_info[rsrc_idx].cnt = (u32)temp;
2801		break;
2802	}
2803
2804	obj_info[rsrc_idx].base = (temp >> IRDMA_COMMIT_FPM_BASE_S) * 512;
2805
2806	return temp;
2807}
2808
2809/**
2810 * irdma_sc_parse_fpm_commit_buf - parse fpm commit buffer
2811 * @dev: pointer to dev struct
2812 * @buf: ptr to fpm commit buffer
2813 * @info: ptr to irdma_hmc_obj_info struct
2814 * @sd: number of SDs for HMC objects
2815 *
2816 * parses fpm commit info and copy base value
2817 * of hmc objects in hmc_info
2818 */
2819static void
2820irdma_sc_parse_fpm_commit_buf(struct irdma_sc_dev *dev, __le64 *buf,
2821			      struct irdma_hmc_obj_info *info, u32 *sd)
2822{
2823	u64 size;
2824	u32 i;
2825	u64 max_base = 0;
2826	u32 last_hmc_obj = 0;
2827
2828	irdma_sc_decode_fpm_commit(dev, buf, 0, info,
2829				   IRDMA_HMC_IW_QP);
2830	irdma_sc_decode_fpm_commit(dev, buf, 8, info,
2831				   IRDMA_HMC_IW_CQ);
2832	/* skiping RSRVD */
2833	irdma_sc_decode_fpm_commit(dev, buf, 24, info,
2834				   IRDMA_HMC_IW_HTE);
2835	irdma_sc_decode_fpm_commit(dev, buf, 32, info,
2836				   IRDMA_HMC_IW_ARP);
2837	irdma_sc_decode_fpm_commit(dev, buf, 40, info,
2838				   IRDMA_HMC_IW_APBVT_ENTRY);
2839	irdma_sc_decode_fpm_commit(dev, buf, 48, info,
2840				   IRDMA_HMC_IW_MR);
2841	irdma_sc_decode_fpm_commit(dev, buf, 56, info,
2842				   IRDMA_HMC_IW_XF);
2843	irdma_sc_decode_fpm_commit(dev, buf, 64, info,
2844				   IRDMA_HMC_IW_XFFL);
2845	irdma_sc_decode_fpm_commit(dev, buf, 72, info,
2846				   IRDMA_HMC_IW_Q1);
2847	irdma_sc_decode_fpm_commit(dev, buf, 80, info,
2848				   IRDMA_HMC_IW_Q1FL);
2849	irdma_sc_decode_fpm_commit(dev, buf, 88, info,
2850				   IRDMA_HMC_IW_TIMER);
2851	irdma_sc_decode_fpm_commit(dev, buf, 112, info,
2852				   IRDMA_HMC_IW_PBLE);
2853	/* skipping RSVD. */
2854	if (dev->hw_attrs.uk_attrs.hw_rev != IRDMA_GEN_1) {
2855		irdma_sc_decode_fpm_commit(dev, buf, 96, info,
2856					   IRDMA_HMC_IW_FSIMC);
2857		irdma_sc_decode_fpm_commit(dev, buf, 104, info,
2858					   IRDMA_HMC_IW_FSIAV);
2859		irdma_sc_decode_fpm_commit(dev, buf, 128, info,
2860					   IRDMA_HMC_IW_RRF);
2861		irdma_sc_decode_fpm_commit(dev, buf, 136, info,
2862					   IRDMA_HMC_IW_RRFFL);
2863		irdma_sc_decode_fpm_commit(dev, buf, 144, info,
2864					   IRDMA_HMC_IW_HDR);
2865		irdma_sc_decode_fpm_commit(dev, buf, 152, info,
2866					   IRDMA_HMC_IW_MD);
2867		irdma_sc_decode_fpm_commit(dev, buf, 160, info,
2868					   IRDMA_HMC_IW_OOISC);
2869		irdma_sc_decode_fpm_commit(dev, buf, 168, info,
2870					   IRDMA_HMC_IW_OOISCFFL);
2871	}
2872
2873	/* searching for the last object in HMC to find the size of the HMC area. */
2874	for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++) {
2875		if (info[i].base > max_base) {
2876			max_base = info[i].base;
2877			last_hmc_obj = i;
2878		}
2879	}
2880
2881	size = info[last_hmc_obj].cnt * info[last_hmc_obj].size +
2882	       info[last_hmc_obj].base;
2883
2884	if (size & 0x1FFFFF)
2885		*sd = (u32)((size >> 21) + 1); /* add 1 for remainder */
2886	else
2887		*sd = (u32)(size >> 21);
2888
2889}
2890
2891/**
2892 * irdma_sc_decode_fpm_query() - Decode a 64 bit value into max count and size
2893 * @buf: ptr to fpm query buffer
2894 * @buf_idx: index into buf
2895 * @obj_info: ptr to irdma_hmc_obj_info struct
2896 * @rsrc_idx: resource index into info
2897 *
2898 * Decode a 64 bit value from fpm query buffer into max count and size
2899 */
2900static u64 irdma_sc_decode_fpm_query(__le64 *buf, u32 buf_idx,
2901				     struct irdma_hmc_obj_info *obj_info,
2902				     u32 rsrc_idx)
2903{
2904	u64 temp;
2905	u32 size;
2906
2907	get_64bit_val(buf, buf_idx, &temp);
2908	obj_info[rsrc_idx].max_cnt = (u32)temp;
2909	size = (u32)(temp >> 32);
2910	obj_info[rsrc_idx].size = BIT_ULL(size);
2911
2912	return temp;
2913}
2914
2915/**
2916 * irdma_sc_parse_fpm_query_buf() - parses fpm query buffer
2917 * @dev: ptr to shared code device
2918 * @buf: ptr to fpm query buffer
2919 * @hmc_info: ptr to irdma_hmc_obj_info struct
2920 * @hmc_fpm_misc: ptr to fpm data
2921 *
2922 * parses fpm query buffer and copy max_cnt and
2923 * size value of hmc objects in hmc_info
2924 */
2925static int irdma_sc_parse_fpm_query_buf(struct irdma_sc_dev *dev, __le64 *buf,
2926					struct irdma_hmc_info *hmc_info,
2927					struct irdma_hmc_fpm_misc *hmc_fpm_misc)
2928{
2929	struct irdma_hmc_obj_info *obj_info;
2930	u64 temp;
2931	u32 size;
2932	u16 max_pe_sds;
2933
2934	obj_info = hmc_info->hmc_obj;
2935
2936	get_64bit_val(buf, 0, &temp);
2937	hmc_info->first_sd_index = (u16)FIELD_GET(IRDMA_QUERY_FPM_FIRST_PE_SD_INDEX, temp);
2938	max_pe_sds = (u16)FIELD_GET(IRDMA_QUERY_FPM_MAX_PE_SDS, temp);
2939
2940	hmc_fpm_misc->max_sds = max_pe_sds;
2941	hmc_info->sd_table.sd_cnt = max_pe_sds + hmc_info->first_sd_index;
2942	get_64bit_val(buf, 8, &temp);
2943	obj_info[IRDMA_HMC_IW_QP].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_QPS, temp);
2944	size = (u32)(temp >> 32);
2945	obj_info[IRDMA_HMC_IW_QP].size = BIT_ULL(size);
2946
2947	get_64bit_val(buf, 16, &temp);
2948	obj_info[IRDMA_HMC_IW_CQ].max_cnt = (u32)FIELD_GET(IRDMA_QUERY_FPM_MAX_CQS, temp);
2949	size = (u32)(temp >> 32);
2950	obj_info[IRDMA_HMC_IW_CQ].size = BIT_ULL(size);
2951
2952	irdma_sc_decode_fpm_query(buf, 32, obj_info, IRDMA_HMC_IW_HTE);
2953	irdma_sc_decode_fpm_query(buf, 40, obj_info, IRDMA_HMC_IW_ARP);
2954
2955	obj_info[IRDMA_HMC_IW_APBVT_ENTRY].size = 8192;
2956	obj_info[IRDMA_HMC_IW_APBVT_ENTRY].max_cnt = 1;
2957
2958	irdma_sc_decode_fpm_query(buf, 48, obj_info, IRDMA_HMC_IW_MR);
2959	irdma_sc_decode_fpm_query(buf, 56, obj_info, IRDMA_HMC_IW_XF);
2960
2961	get_64bit_val(buf, 64, &temp);
2962	obj_info[IRDMA_HMC_IW_XFFL].max_cnt = (u32)temp;
2963	obj_info[IRDMA_HMC_IW_XFFL].size = 4;
2964	hmc_fpm_misc->xf_block_size = FIELD_GET(IRDMA_QUERY_FPM_XFBLOCKSIZE, temp);
2965	if (!hmc_fpm_misc->xf_block_size)
2966		return -EINVAL;
2967
2968	irdma_sc_decode_fpm_query(buf, 72, obj_info, IRDMA_HMC_IW_Q1);
2969	get_64bit_val(buf, 80, &temp);
2970	obj_info[IRDMA_HMC_IW_Q1FL].max_cnt = (u32)temp;
2971	obj_info[IRDMA_HMC_IW_Q1FL].size = 4;
2972
2973	hmc_fpm_misc->q1_block_size = FIELD_GET(IRDMA_QUERY_FPM_Q1BLOCKSIZE, temp);
2974	if (!hmc_fpm_misc->q1_block_size)
2975		return -EINVAL;
2976
2977	irdma_sc_decode_fpm_query(buf, 88, obj_info, IRDMA_HMC_IW_TIMER);
2978
2979	get_64bit_val(buf, 112, &temp);
2980	obj_info[IRDMA_HMC_IW_PBLE].max_cnt = (u32)temp;
2981	obj_info[IRDMA_HMC_IW_PBLE].size = 8;
2982
2983	get_64bit_val(buf, 120, &temp);
2984	hmc_fpm_misc->max_ceqs = FIELD_GET(IRDMA_QUERY_FPM_MAX_CEQS, temp);
2985	hmc_fpm_misc->ht_multiplier = FIELD_GET(IRDMA_QUERY_FPM_HTMULTIPLIER, temp);
2986	hmc_fpm_misc->timer_bucket = FIELD_GET(IRDMA_QUERY_FPM_TIMERBUCKET, temp);
2987	if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
2988		return 0;
2989	irdma_sc_decode_fpm_query(buf, 96, obj_info, IRDMA_HMC_IW_FSIMC);
2990	irdma_sc_decode_fpm_query(buf, 104, obj_info, IRDMA_HMC_IW_FSIAV);
2991	irdma_sc_decode_fpm_query(buf, 128, obj_info, IRDMA_HMC_IW_RRF);
2992
2993	get_64bit_val(buf, 136, &temp);
2994	obj_info[IRDMA_HMC_IW_RRFFL].max_cnt = (u32)temp;
2995	obj_info[IRDMA_HMC_IW_RRFFL].size = 4;
2996	hmc_fpm_misc->rrf_block_size = FIELD_GET(IRDMA_QUERY_FPM_RRFBLOCKSIZE, temp);
2997	if (!hmc_fpm_misc->rrf_block_size &&
2998	    obj_info[IRDMA_HMC_IW_RRFFL].max_cnt)
2999		return -EINVAL;
3000
3001	irdma_sc_decode_fpm_query(buf, 144, obj_info, IRDMA_HMC_IW_HDR);
3002	irdma_sc_decode_fpm_query(buf, 152, obj_info, IRDMA_HMC_IW_MD);
3003	irdma_sc_decode_fpm_query(buf, 160, obj_info, IRDMA_HMC_IW_OOISC);
3004
3005	get_64bit_val(buf, 168, &temp);
3006	obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt = (u32)temp;
3007	obj_info[IRDMA_HMC_IW_OOISCFFL].size = 4;
3008	hmc_fpm_misc->ooiscf_block_size = FIELD_GET(IRDMA_QUERY_FPM_OOISCFBLOCKSIZE, temp);
3009	if (!hmc_fpm_misc->ooiscf_block_size &&
3010	    obj_info[IRDMA_HMC_IW_OOISCFFL].max_cnt)
3011		return -EINVAL;
3012
3013	return 0;
3014}
3015
3016/**
3017 * irdma_sc_find_reg_cq - find cq ctx index
3018 * @ceq: ceq sc structure
3019 * @cq: cq sc structure
3020 */
3021static u32 irdma_sc_find_reg_cq(struct irdma_sc_ceq *ceq,
3022				struct irdma_sc_cq *cq)
3023{
3024	u32 i;
3025
3026	for (i = 0; i < ceq->reg_cq_size; i++) {
3027		if (cq == ceq->reg_cq[i])
3028			return i;
3029	}
3030
3031	return IRDMA_INVALID_CQ_IDX;
3032}
3033
3034/**
3035 * irdma_sc_add_cq_ctx - add cq ctx tracking for ceq
3036 * @ceq: ceq sc structure
3037 * @cq: cq sc structure
3038 */
3039int irdma_sc_add_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)
3040{
3041	unsigned long flags;
3042
3043	spin_lock_irqsave(&ceq->req_cq_lock, flags);
3044
3045	if (ceq->reg_cq_size == ceq->elem_cnt) {
3046		spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3047		return -ENOMEM;
3048	}
3049
3050	ceq->reg_cq[ceq->reg_cq_size++] = cq;
3051
3052	spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3053
3054	return 0;
3055}
3056
3057/**
3058 * irdma_sc_remove_cq_ctx - remove cq ctx tracking for ceq
3059 * @ceq: ceq sc structure
3060 * @cq: cq sc structure
3061 */
3062void irdma_sc_remove_cq_ctx(struct irdma_sc_ceq *ceq, struct irdma_sc_cq *cq)
3063{
3064	unsigned long flags;
3065	u32 cq_ctx_idx;
3066
3067	spin_lock_irqsave(&ceq->req_cq_lock, flags);
3068	cq_ctx_idx = irdma_sc_find_reg_cq(ceq, cq);
3069	if (cq_ctx_idx == IRDMA_INVALID_CQ_IDX)
3070		goto exit;
3071
3072	ceq->reg_cq_size--;
3073	if (cq_ctx_idx != ceq->reg_cq_size)
3074		ceq->reg_cq[cq_ctx_idx] = ceq->reg_cq[ceq->reg_cq_size];
3075	ceq->reg_cq[ceq->reg_cq_size] = NULL;
3076
3077exit:
3078	spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3079}
3080
3081/**
3082 * irdma_sc_cqp_init - Initialize buffers for a control Queue Pair
3083 * @cqp: IWARP control queue pair pointer
3084 * @info: IWARP control queue pair init info pointer
3085 *
3086 * Initializes the object and context buffers for a control Queue Pair.
3087 */
3088int irdma_sc_cqp_init(struct irdma_sc_cqp *cqp,
3089		      struct irdma_cqp_init_info *info)
3090{
3091	u8 hw_sq_size;
3092
3093	if (info->sq_size > IRDMA_CQP_SW_SQSIZE_2048 ||
3094	    info->sq_size < IRDMA_CQP_SW_SQSIZE_4 ||
3095	    ((info->sq_size & (info->sq_size - 1))))
3096		return -EINVAL;
3097
3098	hw_sq_size = irdma_get_encoded_wqe_size(info->sq_size,
3099						IRDMA_QUEUE_TYPE_CQP);
3100	cqp->size = sizeof(*cqp);
3101	cqp->sq_size = info->sq_size;
3102	cqp->hw_sq_size = hw_sq_size;
3103	cqp->sq_base = info->sq;
3104	cqp->host_ctx = info->host_ctx;
3105	cqp->sq_pa = info->sq_pa;
3106	cqp->host_ctx_pa = info->host_ctx_pa;
3107	cqp->dev = info->dev;
3108	cqp->struct_ver = info->struct_ver;
3109	cqp->hw_maj_ver = info->hw_maj_ver;
3110	cqp->hw_min_ver = info->hw_min_ver;
3111	cqp->scratch_array = info->scratch_array;
3112	cqp->polarity = 0;
3113	cqp->en_datacenter_tcp = info->en_datacenter_tcp;
3114	cqp->ena_vf_count = info->ena_vf_count;
3115	cqp->hmc_profile = info->hmc_profile;
3116	cqp->ceqs_per_vf = info->ceqs_per_vf;
3117	cqp->disable_packed = info->disable_packed;
3118	cqp->rocev2_rto_policy = info->rocev2_rto_policy;
3119	cqp->protocol_used = info->protocol_used;
3120	memcpy(&cqp->dcqcn_params, &info->dcqcn_params, sizeof(cqp->dcqcn_params));
3121	info->dev->cqp = cqp;
3122
3123	IRDMA_RING_INIT(cqp->sq_ring, cqp->sq_size);
3124	cqp->requested_ops = 0;
3125	atomic64_set(&cqp->completed_ops, 0);
3126	/* for the cqp commands backlog. */
3127	INIT_LIST_HEAD(&cqp->dev->cqp_cmd_head);
3128
3129	writel(0, cqp->dev->hw_regs[IRDMA_CQPTAIL]);
3130	writel(0, cqp->dev->hw_regs[IRDMA_CQPDB]);
3131	writel(0, cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3132
3133	ibdev_dbg(to_ibdev(cqp->dev),
3134		  "WQE: sq_size[%04d] hw_sq_size[%04d] sq_base[%p] sq_pa[%pK] cqp[%p] polarity[x%04x]\n",
3135		  cqp->sq_size, cqp->hw_sq_size, cqp->sq_base,
3136		  (u64 *)(uintptr_t)cqp->sq_pa, cqp, cqp->polarity);
3137	return 0;
3138}
3139
3140/**
3141 * irdma_sc_cqp_create - create cqp during bringup
3142 * @cqp: struct for cqp hw
3143 * @maj_err: If error, major err number
3144 * @min_err: If error, minor err number
3145 */
3146int irdma_sc_cqp_create(struct irdma_sc_cqp *cqp, u16 *maj_err, u16 *min_err)
3147{
3148	u64 temp;
3149	u8 hw_rev;
3150	u32 cnt = 0, p1, p2, val = 0, err_code;
3151	int ret_code;
3152
3153	hw_rev = cqp->dev->hw_attrs.uk_attrs.hw_rev;
3154	cqp->sdbuf.size = ALIGN(IRDMA_UPDATE_SD_BUFF_SIZE * cqp->sq_size,
3155				IRDMA_SD_BUF_ALIGNMENT);
3156	cqp->sdbuf.va = dma_alloc_coherent(cqp->dev->hw->device,
3157					   cqp->sdbuf.size, &cqp->sdbuf.pa,
3158					   GFP_KERNEL);
3159	if (!cqp->sdbuf.va)
3160		return -ENOMEM;
3161
3162	spin_lock_init(&cqp->dev->cqp_lock);
3163
3164	temp = FIELD_PREP(IRDMA_CQPHC_SQSIZE, cqp->hw_sq_size) |
3165	       FIELD_PREP(IRDMA_CQPHC_SVER, cqp->struct_ver) |
3166	       FIELD_PREP(IRDMA_CQPHC_DISABLE_PFPDUS, cqp->disable_packed) |
3167	       FIELD_PREP(IRDMA_CQPHC_CEQPERVF, cqp->ceqs_per_vf);
3168	if (hw_rev >= IRDMA_GEN_2) {
3169		temp |= FIELD_PREP(IRDMA_CQPHC_ROCEV2_RTO_POLICY,
3170				   cqp->rocev2_rto_policy) |
3171			FIELD_PREP(IRDMA_CQPHC_PROTOCOL_USED,
3172				   cqp->protocol_used);
3173	}
3174
3175	set_64bit_val(cqp->host_ctx, 0, temp);
3176	set_64bit_val(cqp->host_ctx, 8, cqp->sq_pa);
3177
3178	temp = FIELD_PREP(IRDMA_CQPHC_ENABLED_VFS, cqp->ena_vf_count) |
3179	       FIELD_PREP(IRDMA_CQPHC_HMC_PROFILE, cqp->hmc_profile);
3180	set_64bit_val(cqp->host_ctx, 16, temp);
3181	set_64bit_val(cqp->host_ctx, 24, (uintptr_t)cqp);
3182	temp = FIELD_PREP(IRDMA_CQPHC_HW_MAJVER, cqp->hw_maj_ver) |
3183	       FIELD_PREP(IRDMA_CQPHC_HW_MINVER, cqp->hw_min_ver);
3184	if (hw_rev >= IRDMA_GEN_2) {
3185		temp |= FIELD_PREP(IRDMA_CQPHC_MIN_RATE, cqp->dcqcn_params.min_rate) |
3186			FIELD_PREP(IRDMA_CQPHC_MIN_DEC_FACTOR, cqp->dcqcn_params.min_dec_factor);
3187	}
3188	set_64bit_val(cqp->host_ctx, 32, temp);
3189	set_64bit_val(cqp->host_ctx, 40, 0);
3190	temp = 0;
3191	if (hw_rev >= IRDMA_GEN_2) {
3192		temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_T, cqp->dcqcn_params.dcqcn_t) |
3193			FIELD_PREP(IRDMA_CQPHC_RAI_FACTOR, cqp->dcqcn_params.rai_factor) |
3194			FIELD_PREP(IRDMA_CQPHC_HAI_FACTOR, cqp->dcqcn_params.hai_factor);
3195	}
3196	set_64bit_val(cqp->host_ctx, 48, temp);
3197	temp = 0;
3198	if (hw_rev >= IRDMA_GEN_2) {
3199		temp |= FIELD_PREP(IRDMA_CQPHC_DCQCN_B, cqp->dcqcn_params.dcqcn_b) |
3200			FIELD_PREP(IRDMA_CQPHC_DCQCN_F, cqp->dcqcn_params.dcqcn_f) |
3201			FIELD_PREP(IRDMA_CQPHC_CC_CFG_VALID, cqp->dcqcn_params.cc_cfg_valid) |
3202			FIELD_PREP(IRDMA_CQPHC_RREDUCE_MPERIOD, cqp->dcqcn_params.rreduce_mperiod);
3203	}
3204	set_64bit_val(cqp->host_ctx, 56, temp);
3205	print_hex_dump_debug("WQE: CQP_HOST_CTX WQE", DUMP_PREFIX_OFFSET, 16,
3206			     8, cqp->host_ctx, IRDMA_CQP_CTX_SIZE * 8, false);
3207	p1 = cqp->host_ctx_pa >> 32;
3208	p2 = (u32)cqp->host_ctx_pa;
3209
3210	writel(p1, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);
3211	writel(p2, cqp->dev->hw_regs[IRDMA_CCQPLOW]);
3212
3213	do {
3214		if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
3215			ret_code = -ETIMEDOUT;
3216			goto err;
3217		}
3218		udelay(cqp->dev->hw_attrs.max_sleep_count);
3219		val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3220	} while (!val);
3221
3222	if (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_ERR)) {
3223		ret_code = -EOPNOTSUPP;
3224		goto err;
3225	}
3226
3227	cqp->process_cqp_sds = irdma_update_sds_noccq;
3228	return 0;
3229
3230err:
3231	dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size,
3232			  cqp->sdbuf.va, cqp->sdbuf.pa);
3233	cqp->sdbuf.va = NULL;
3234	err_code = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
3235	*min_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MINOR_CODE, err_code);
3236	*maj_err = FIELD_GET(IRDMA_CQPERRCODES_CQP_MAJOR_CODE, err_code);
3237	return ret_code;
3238}
3239
3240/**
3241 * irdma_sc_cqp_post_sq - post of cqp's sq
3242 * @cqp: struct for cqp hw
3243 */
3244void irdma_sc_cqp_post_sq(struct irdma_sc_cqp *cqp)
3245{
3246	writel(IRDMA_RING_CURRENT_HEAD(cqp->sq_ring), cqp->dev->cqp_db);
3247
3248	ibdev_dbg(to_ibdev(cqp->dev),
3249		  "WQE: CQP SQ head 0x%x tail 0x%x size 0x%x\n",
3250		  cqp->sq_ring.head, cqp->sq_ring.tail, cqp->sq_ring.size);
3251}
3252
3253/**
3254 * irdma_sc_cqp_get_next_send_wqe_idx - get next wqe on cqp sq
3255 * and pass back index
3256 * @cqp: CQP HW structure
3257 * @scratch: private data for CQP WQE
3258 * @wqe_idx: WQE index of CQP SQ
3259 */
3260__le64 *irdma_sc_cqp_get_next_send_wqe_idx(struct irdma_sc_cqp *cqp, u64 scratch,
3261					   u32 *wqe_idx)
3262{
3263	__le64 *wqe = NULL;
3264	int ret_code;
3265
3266	if (IRDMA_RING_FULL_ERR(cqp->sq_ring)) {
3267		ibdev_dbg(to_ibdev(cqp->dev),
3268			  "WQE: CQP SQ is full, head 0x%x tail 0x%x size 0x%x\n",
3269			  cqp->sq_ring.head, cqp->sq_ring.tail,
3270			  cqp->sq_ring.size);
3271		return NULL;
3272	}
3273	IRDMA_ATOMIC_RING_MOVE_HEAD(cqp->sq_ring, *wqe_idx, ret_code);
3274	if (ret_code)
3275		return NULL;
3276
3277	cqp->requested_ops++;
3278	if (!*wqe_idx)
3279		cqp->polarity = !cqp->polarity;
3280	wqe = cqp->sq_base[*wqe_idx].elem;
3281	cqp->scratch_array[*wqe_idx] = scratch;
3282	IRDMA_CQP_INIT_WQE(wqe);
3283
3284	return wqe;
3285}
3286
3287/**
3288 * irdma_sc_cqp_destroy - destroy cqp during close
3289 * @cqp: struct for cqp hw
3290 */
3291int irdma_sc_cqp_destroy(struct irdma_sc_cqp *cqp)
3292{
3293	u32 cnt = 0, val;
3294	int ret_code = 0;
3295
3296	writel(0, cqp->dev->hw_regs[IRDMA_CCQPHIGH]);
3297	writel(0, cqp->dev->hw_regs[IRDMA_CCQPLOW]);
3298	do {
3299		if (cnt++ > cqp->dev->hw_attrs.max_done_count) {
3300			ret_code = -ETIMEDOUT;
3301			break;
3302		}
3303		udelay(cqp->dev->hw_attrs.max_sleep_count);
3304		val = readl(cqp->dev->hw_regs[IRDMA_CCQPSTATUS]);
3305	} while (FLD_RS_32(cqp->dev, val, IRDMA_CCQPSTATUS_CCQP_DONE));
3306
3307	dma_free_coherent(cqp->dev->hw->device, cqp->sdbuf.size,
3308			  cqp->sdbuf.va, cqp->sdbuf.pa);
3309	cqp->sdbuf.va = NULL;
3310	return ret_code;
3311}
3312
3313/**
3314 * irdma_sc_ccq_arm - enable intr for control cq
3315 * @ccq: ccq sc struct
3316 */
3317void irdma_sc_ccq_arm(struct irdma_sc_cq *ccq)
3318{
3319	u64 temp_val;
3320	u16 sw_cq_sel;
3321	u8 arm_next_se;
3322	u8 arm_seq_num;
3323
3324	get_64bit_val(ccq->cq_uk.shadow_area, 32, &temp_val);
3325	sw_cq_sel = (u16)FIELD_GET(IRDMA_CQ_DBSA_SW_CQ_SELECT, temp_val);
3326	arm_next_se = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_NEXT_SE, temp_val);
3327	arm_seq_num = (u8)FIELD_GET(IRDMA_CQ_DBSA_ARM_SEQ_NUM, temp_val);
3328	arm_seq_num++;
3329	temp_val = FIELD_PREP(IRDMA_CQ_DBSA_ARM_SEQ_NUM, arm_seq_num) |
3330		   FIELD_PREP(IRDMA_CQ_DBSA_SW_CQ_SELECT, sw_cq_sel) |
3331		   FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT_SE, arm_next_se) |
3332		   FIELD_PREP(IRDMA_CQ_DBSA_ARM_NEXT, 1);
3333	set_64bit_val(ccq->cq_uk.shadow_area, 32, temp_val);
3334
3335	dma_wmb(); /* make sure shadow area is updated before arming */
3336
3337	writel(ccq->cq_uk.cq_id, ccq->dev->cq_arm_db);
3338}
3339
3340/**
3341 * irdma_sc_ccq_get_cqe_info - get ccq's cq entry
3342 * @ccq: ccq sc struct
3343 * @info: completion q entry to return
3344 */
3345int irdma_sc_ccq_get_cqe_info(struct irdma_sc_cq *ccq,
3346			      struct irdma_ccq_cqe_info *info)
3347{
3348	u64 qp_ctx, temp, temp1;
3349	__le64 *cqe;
3350	struct irdma_sc_cqp *cqp;
3351	u32 wqe_idx;
3352	u32 error;
3353	u8 polarity;
3354	int ret_code = 0;
3355
3356	if (ccq->cq_uk.avoid_mem_cflct)
3357		cqe = IRDMA_GET_CURRENT_EXTENDED_CQ_ELEM(&ccq->cq_uk);
3358	else
3359		cqe = IRDMA_GET_CURRENT_CQ_ELEM(&ccq->cq_uk);
3360
3361	get_64bit_val(cqe, 24, &temp);
3362	polarity = (u8)FIELD_GET(IRDMA_CQ_VALID, temp);
3363	if (polarity != ccq->cq_uk.polarity)
3364		return -ENOENT;
3365
3366	/* Ensure CEQE contents are read after valid bit is checked */
3367	dma_rmb();
3368
3369	get_64bit_val(cqe, 8, &qp_ctx);
3370	cqp = (struct irdma_sc_cqp *)(unsigned long)qp_ctx;
3371	info->error = (bool)FIELD_GET(IRDMA_CQ_ERROR, temp);
3372	info->maj_err_code = IRDMA_CQPSQ_MAJ_NO_ERROR;
3373	info->min_err_code = (u16)FIELD_GET(IRDMA_CQ_MINERR, temp);
3374	if (info->error) {
3375		info->maj_err_code = (u16)FIELD_GET(IRDMA_CQ_MAJERR, temp);
3376		error = readl(cqp->dev->hw_regs[IRDMA_CQPERRCODES]);
3377		ibdev_dbg(to_ibdev(cqp->dev),
3378			  "CQP: CQPERRCODES error_code[x%08X]\n", error);
3379	}
3380
3381	wqe_idx = (u32)FIELD_GET(IRDMA_CQ_WQEIDX, temp);
3382	info->scratch = cqp->scratch_array[wqe_idx];
3383
3384	get_64bit_val(cqe, 16, &temp1);
3385	info->op_ret_val = (u32)FIELD_GET(IRDMA_CCQ_OPRETVAL, temp1);
3386	get_64bit_val(cqp->sq_base[wqe_idx].elem, 24, &temp1);
3387	info->op_code = (u8)FIELD_GET(IRDMA_CQPSQ_OPCODE, temp1);
3388	info->cqp = cqp;
3389
3390	/*  move the head for cq */
3391	IRDMA_RING_MOVE_HEAD(ccq->cq_uk.cq_ring, ret_code);
3392	if (!IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring))
3393		ccq->cq_uk.polarity ^= 1;
3394
3395	/* update cq tail in cq shadow memory also */
3396	IRDMA_RING_MOVE_TAIL(ccq->cq_uk.cq_ring);
3397	set_64bit_val(ccq->cq_uk.shadow_area, 0,
3398		      IRDMA_RING_CURRENT_HEAD(ccq->cq_uk.cq_ring));
3399
3400	dma_wmb(); /* make sure shadow area is updated before moving tail */
3401
3402	IRDMA_RING_MOVE_TAIL(cqp->sq_ring);
3403	atomic64_inc(&cqp->completed_ops);
3404
3405	return ret_code;
3406}
3407
3408/**
3409 * irdma_sc_poll_for_cqp_op_done - Waits for last write to complete in CQP SQ
3410 * @cqp: struct for cqp hw
3411 * @op_code: cqp opcode for completion
3412 * @compl_info: completion q entry to return
3413 */
3414int irdma_sc_poll_for_cqp_op_done(struct irdma_sc_cqp *cqp, u8 op_code,
3415				  struct irdma_ccq_cqe_info *compl_info)
3416{
3417	struct irdma_ccq_cqe_info info = {};
3418	struct irdma_sc_cq *ccq;
3419	int ret_code = 0;
3420	u32 cnt = 0;
3421
3422	ccq = cqp->dev->ccq;
3423	while (1) {
3424		if (cnt++ > 100 * cqp->dev->hw_attrs.max_done_count)
3425			return -ETIMEDOUT;
3426
3427		if (irdma_sc_ccq_get_cqe_info(ccq, &info)) {
3428			udelay(cqp->dev->hw_attrs.max_sleep_count);
3429			continue;
3430		}
3431		if (info.error && info.op_code != IRDMA_CQP_OP_QUERY_STAG) {
3432			ret_code = -EIO;
3433			break;
3434		}
3435		/* make sure op code matches*/
3436		if (op_code == info.op_code)
3437			break;
3438		ibdev_dbg(to_ibdev(cqp->dev),
3439			  "WQE: opcode mismatch for my op code 0x%x, returned opcode %x\n",
3440			  op_code, info.op_code);
3441	}
3442
3443	if (compl_info)
3444		memcpy(compl_info, &info, sizeof(*compl_info));
3445
3446	return ret_code;
3447}
3448
3449/**
3450 * irdma_sc_manage_hmc_pm_func_table - manage of function table
3451 * @cqp: struct for cqp hw
3452 * @scratch: u64 saved to be used during cqp completion
3453 * @info: info for the manage function table operation
3454 * @post_sq: flag for cqp db to ring
3455 */
3456static int irdma_sc_manage_hmc_pm_func_table(struct irdma_sc_cqp *cqp,
3457					     struct irdma_hmc_fcn_info *info,
3458					     u64 scratch, bool post_sq)
3459{
3460	__le64 *wqe;
3461	u64 hdr;
3462
3463	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3464	if (!wqe)
3465		return -ENOMEM;
3466
3467	set_64bit_val(wqe, 0, 0);
3468	set_64bit_val(wqe, 8, 0);
3469	set_64bit_val(wqe, 16, 0);
3470	set_64bit_val(wqe, 32, 0);
3471	set_64bit_val(wqe, 40, 0);
3472	set_64bit_val(wqe, 48, 0);
3473	set_64bit_val(wqe, 56, 0);
3474
3475	hdr = FIELD_PREP(IRDMA_CQPSQ_MHMC_VFIDX, info->vf_id) |
3476	      FIELD_PREP(IRDMA_CQPSQ_OPCODE,
3477			 IRDMA_CQP_OP_MANAGE_HMC_PM_FUNC_TABLE) |
3478	      FIELD_PREP(IRDMA_CQPSQ_MHMC_FREEPMFN, info->free_fcn) |
3479	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3480	dma_wmb(); /* make sure WQE is written before valid bit is set */
3481
3482	set_64bit_val(wqe, 24, hdr);
3483
3484	print_hex_dump_debug("WQE: MANAGE_HMC_PM_FUNC_TABLE WQE",
3485			     DUMP_PREFIX_OFFSET, 16, 8, wqe,
3486			     IRDMA_CQP_WQE_SIZE * 8, false);
3487	if (post_sq)
3488		irdma_sc_cqp_post_sq(cqp);
3489
3490	return 0;
3491}
3492
3493/**
3494 * irdma_sc_commit_fpm_val_done - wait for cqp eqe completion
3495 * for fpm commit
3496 * @cqp: struct for cqp hw
3497 */
3498static int irdma_sc_commit_fpm_val_done(struct irdma_sc_cqp *cqp)
3499{
3500	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_COMMIT_FPM_VAL,
3501					     NULL);
3502}
3503
3504/**
3505 * irdma_sc_commit_fpm_val - cqp wqe for commit fpm values
3506 * @cqp: struct for cqp hw
3507 * @scratch: u64 saved to be used during cqp completion
3508 * @hmc_fn_id: hmc function id
3509 * @commit_fpm_mem: Memory for fpm values
3510 * @post_sq: flag for cqp db to ring
3511 * @wait_type: poll ccq or cqp registers for cqp completion
3512 */
3513static int irdma_sc_commit_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,
3514				   u8 hmc_fn_id,
3515				   struct irdma_dma_mem *commit_fpm_mem,
3516				   bool post_sq, u8 wait_type)
3517{
3518	__le64 *wqe;
3519	u64 hdr;
3520	u32 tail, val, error;
3521	int ret_code = 0;
3522
3523	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3524	if (!wqe)
3525		return -ENOMEM;
3526
3527	set_64bit_val(wqe, 16, hmc_fn_id);
3528	set_64bit_val(wqe, 32, commit_fpm_mem->pa);
3529
3530	hdr = FIELD_PREP(IRDMA_CQPSQ_BUFSIZE, IRDMA_COMMIT_FPM_BUF_SIZE) |
3531	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_COMMIT_FPM_VAL) |
3532	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3533
3534	dma_wmb(); /* make sure WQE is written before valid bit is set */
3535
3536	set_64bit_val(wqe, 24, hdr);
3537
3538	print_hex_dump_debug("WQE: COMMIT_FPM_VAL WQE", DUMP_PREFIX_OFFSET,
3539			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3540	irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
3541
3542	if (post_sq) {
3543		irdma_sc_cqp_post_sq(cqp);
3544		if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)
3545			ret_code = irdma_cqp_poll_registers(cqp, tail,
3546							    cqp->dev->hw_attrs.max_done_count);
3547		else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)
3548			ret_code = irdma_sc_commit_fpm_val_done(cqp);
3549	}
3550
3551	return ret_code;
3552}
3553
3554/**
3555 * irdma_sc_query_fpm_val_done - poll for cqp wqe completion for
3556 * query fpm
3557 * @cqp: struct for cqp hw
3558 */
3559static int irdma_sc_query_fpm_val_done(struct irdma_sc_cqp *cqp)
3560{
3561	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_QUERY_FPM_VAL,
3562					     NULL);
3563}
3564
3565/**
3566 * irdma_sc_query_fpm_val - cqp wqe query fpm values
3567 * @cqp: struct for cqp hw
3568 * @scratch: u64 saved to be used during cqp completion
3569 * @hmc_fn_id: hmc function id
3570 * @query_fpm_mem: memory for return fpm values
3571 * @post_sq: flag for cqp db to ring
3572 * @wait_type: poll ccq or cqp registers for cqp completion
3573 */
3574static int irdma_sc_query_fpm_val(struct irdma_sc_cqp *cqp, u64 scratch,
3575				  u8 hmc_fn_id,
3576				  struct irdma_dma_mem *query_fpm_mem,
3577				  bool post_sq, u8 wait_type)
3578{
3579	__le64 *wqe;
3580	u64 hdr;
3581	u32 tail, val, error;
3582	int ret_code = 0;
3583
3584	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3585	if (!wqe)
3586		return -ENOMEM;
3587
3588	set_64bit_val(wqe, 16, hmc_fn_id);
3589	set_64bit_val(wqe, 32, query_fpm_mem->pa);
3590
3591	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_QUERY_FPM_VAL) |
3592	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3593	dma_wmb(); /* make sure WQE is written before valid bit is set */
3594
3595	set_64bit_val(wqe, 24, hdr);
3596
3597	print_hex_dump_debug("WQE: QUERY_FPM WQE", DUMP_PREFIX_OFFSET, 16, 8,
3598			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3599	irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
3600
3601	if (post_sq) {
3602		irdma_sc_cqp_post_sq(cqp);
3603		if (wait_type == IRDMA_CQP_WAIT_POLL_REGS)
3604			ret_code = irdma_cqp_poll_registers(cqp, tail,
3605							    cqp->dev->hw_attrs.max_done_count);
3606		else if (wait_type == IRDMA_CQP_WAIT_POLL_CQ)
3607			ret_code = irdma_sc_query_fpm_val_done(cqp);
3608	}
3609
3610	return ret_code;
3611}
3612
3613/**
3614 * irdma_sc_ceq_init - initialize ceq
3615 * @ceq: ceq sc structure
3616 * @info: ceq initialization info
3617 */
3618int irdma_sc_ceq_init(struct irdma_sc_ceq *ceq,
3619		      struct irdma_ceq_init_info *info)
3620{
3621	u32 pble_obj_cnt;
3622
3623	if (info->elem_cnt < info->dev->hw_attrs.min_hw_ceq_size ||
3624	    info->elem_cnt > info->dev->hw_attrs.max_hw_ceq_size)
3625		return -EINVAL;
3626
3627	if (info->ceq_id >= info->dev->hmc_fpm_misc.max_ceqs)
3628		return -EINVAL;
3629	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
3630
3631	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
3632		return -EINVAL;
3633
3634	ceq->size = sizeof(*ceq);
3635	ceq->ceqe_base = (struct irdma_ceqe *)info->ceqe_base;
3636	ceq->ceq_id = info->ceq_id;
3637	ceq->dev = info->dev;
3638	ceq->elem_cnt = info->elem_cnt;
3639	ceq->ceq_elem_pa = info->ceqe_pa;
3640	ceq->virtual_map = info->virtual_map;
3641	ceq->itr_no_expire = info->itr_no_expire;
3642	ceq->reg_cq = info->reg_cq;
3643	ceq->reg_cq_size = 0;
3644	spin_lock_init(&ceq->req_cq_lock);
3645	ceq->pbl_chunk_size = (ceq->virtual_map ? info->pbl_chunk_size : 0);
3646	ceq->first_pm_pbl_idx = (ceq->virtual_map ? info->first_pm_pbl_idx : 0);
3647	ceq->pbl_list = (ceq->virtual_map ? info->pbl_list : NULL);
3648	ceq->tph_en = info->tph_en;
3649	ceq->tph_val = info->tph_val;
3650	ceq->vsi = info->vsi;
3651	ceq->polarity = 1;
3652	IRDMA_RING_INIT(ceq->ceq_ring, ceq->elem_cnt);
3653	ceq->dev->ceq[info->ceq_id] = ceq;
3654
3655	return 0;
3656}
3657
3658/**
3659 * irdma_sc_ceq_create - create ceq wqe
3660 * @ceq: ceq sc structure
3661 * @scratch: u64 saved to be used during cqp completion
3662 * @post_sq: flag for cqp db to ring
3663 */
3664
3665static int irdma_sc_ceq_create(struct irdma_sc_ceq *ceq, u64 scratch,
3666			       bool post_sq)
3667{
3668	struct irdma_sc_cqp *cqp;
3669	__le64 *wqe;
3670	u64 hdr;
3671
3672	cqp = ceq->dev->cqp;
3673	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3674	if (!wqe)
3675		return -ENOMEM;
3676	set_64bit_val(wqe, 16, ceq->elem_cnt);
3677	set_64bit_val(wqe, 32,
3678		      (ceq->virtual_map ? 0 : ceq->ceq_elem_pa));
3679	set_64bit_val(wqe, 48,
3680		      (ceq->virtual_map ? ceq->first_pm_pbl_idx : 0));
3681	set_64bit_val(wqe, 56,
3682		      FIELD_PREP(IRDMA_CQPSQ_TPHVAL, ceq->tph_val) |
3683		      FIELD_PREP(IRDMA_CQPSQ_VSIIDX, ceq->vsi->vsi_idx));
3684	hdr = FIELD_PREP(IRDMA_CQPSQ_CEQ_CEQID, ceq->ceq_id) |
3685	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_CEQ) |
3686	      FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |
3687	      FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |
3688	      FIELD_PREP(IRDMA_CQPSQ_CEQ_ITRNOEXPIRE, ceq->itr_no_expire) |
3689	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |
3690	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3691	dma_wmb(); /* make sure WQE is written before valid bit is set */
3692
3693	set_64bit_val(wqe, 24, hdr);
3694
3695	print_hex_dump_debug("WQE: CEQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
3696			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3697	if (post_sq)
3698		irdma_sc_cqp_post_sq(cqp);
3699
3700	return 0;
3701}
3702
3703/**
3704 * irdma_sc_cceq_create_done - poll for control ceq wqe to complete
3705 * @ceq: ceq sc structure
3706 */
3707static int irdma_sc_cceq_create_done(struct irdma_sc_ceq *ceq)
3708{
3709	struct irdma_sc_cqp *cqp;
3710
3711	cqp = ceq->dev->cqp;
3712	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CEQ,
3713					     NULL);
3714}
3715
3716/**
3717 * irdma_sc_cceq_destroy_done - poll for destroy cceq to complete
3718 * @ceq: ceq sc structure
3719 */
3720int irdma_sc_cceq_destroy_done(struct irdma_sc_ceq *ceq)
3721{
3722	struct irdma_sc_cqp *cqp;
3723
3724	if (ceq->reg_cq)
3725		irdma_sc_remove_cq_ctx(ceq, ceq->dev->ccq);
3726
3727	cqp = ceq->dev->cqp;
3728	cqp->process_cqp_sds = irdma_update_sds_noccq;
3729
3730	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_DESTROY_CEQ,
3731					     NULL);
3732}
3733
3734/**
3735 * irdma_sc_cceq_create - create cceq
3736 * @ceq: ceq sc structure
3737 * @scratch: u64 saved to be used during cqp completion
3738 */
3739int irdma_sc_cceq_create(struct irdma_sc_ceq *ceq, u64 scratch)
3740{
3741	int ret_code;
3742	struct irdma_sc_dev *dev = ceq->dev;
3743
3744	dev->ccq->vsi = ceq->vsi;
3745	if (ceq->reg_cq) {
3746		ret_code = irdma_sc_add_cq_ctx(ceq, ceq->dev->ccq);
3747		if (ret_code)
3748			return ret_code;
3749	}
3750
3751	ret_code = irdma_sc_ceq_create(ceq, scratch, true);
3752	if (!ret_code)
3753		return irdma_sc_cceq_create_done(ceq);
3754
3755	return ret_code;
3756}
3757
3758/**
3759 * irdma_sc_ceq_destroy - destroy ceq
3760 * @ceq: ceq sc structure
3761 * @scratch: u64 saved to be used during cqp completion
3762 * @post_sq: flag for cqp db to ring
3763 */
3764int irdma_sc_ceq_destroy(struct irdma_sc_ceq *ceq, u64 scratch, bool post_sq)
3765{
3766	struct irdma_sc_cqp *cqp;
3767	__le64 *wqe;
3768	u64 hdr;
3769
3770	cqp = ceq->dev->cqp;
3771	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3772	if (!wqe)
3773		return -ENOMEM;
3774
3775	set_64bit_val(wqe, 16, ceq->elem_cnt);
3776	set_64bit_val(wqe, 48, ceq->first_pm_pbl_idx);
3777	hdr = ceq->ceq_id |
3778	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CEQ) |
3779	      FIELD_PREP(IRDMA_CQPSQ_CEQ_LPBLSIZE, ceq->pbl_chunk_size) |
3780	      FIELD_PREP(IRDMA_CQPSQ_CEQ_VMAP, ceq->virtual_map) |
3781	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ceq->tph_en) |
3782	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3783	dma_wmb(); /* make sure WQE is written before valid bit is set */
3784
3785	set_64bit_val(wqe, 24, hdr);
3786
3787	print_hex_dump_debug("WQE: CEQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
3788			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3789	if (post_sq)
3790		irdma_sc_cqp_post_sq(cqp);
3791
3792	return 0;
3793}
3794
3795/**
3796 * irdma_sc_process_ceq - process ceq
3797 * @dev: sc device struct
3798 * @ceq: ceq sc structure
3799 *
3800 * It is expected caller serializes this function with cleanup_ceqes()
3801 * because these functions manipulate the same ceq
3802 */
3803void *irdma_sc_process_ceq(struct irdma_sc_dev *dev, struct irdma_sc_ceq *ceq)
3804{
3805	u64 temp;
3806	__le64 *ceqe;
3807	struct irdma_sc_cq *cq = NULL;
3808	struct irdma_sc_cq *temp_cq;
3809	u8 polarity;
3810	u32 cq_idx;
3811	unsigned long flags;
3812
3813	do {
3814		cq_idx = 0;
3815		ceqe = IRDMA_GET_CURRENT_CEQ_ELEM(ceq);
3816		get_64bit_val(ceqe, 0, &temp);
3817		polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);
3818		if (polarity != ceq->polarity)
3819			return NULL;
3820
3821		temp_cq = (struct irdma_sc_cq *)(unsigned long)(temp << 1);
3822		if (!temp_cq) {
3823			cq_idx = IRDMA_INVALID_CQ_IDX;
3824			IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
3825
3826			if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))
3827				ceq->polarity ^= 1;
3828			continue;
3829		}
3830
3831		cq = temp_cq;
3832		if (ceq->reg_cq) {
3833			spin_lock_irqsave(&ceq->req_cq_lock, flags);
3834			cq_idx = irdma_sc_find_reg_cq(ceq, cq);
3835			spin_unlock_irqrestore(&ceq->req_cq_lock, flags);
3836		}
3837
3838		IRDMA_RING_MOVE_TAIL(ceq->ceq_ring);
3839		if (!IRDMA_RING_CURRENT_TAIL(ceq->ceq_ring))
3840			ceq->polarity ^= 1;
3841	} while (cq_idx == IRDMA_INVALID_CQ_IDX);
3842
3843	if (cq)
3844		irdma_sc_cq_ack(cq);
3845	return cq;
3846}
3847
3848/**
3849 * irdma_sc_cleanup_ceqes - clear the valid ceqes ctx matching the cq
3850 * @cq: cq for which the ceqes need to be cleaned up
3851 * @ceq: ceq ptr
3852 *
3853 * The function is called after the cq is destroyed to cleanup
3854 * its pending ceqe entries. It is expected caller serializes this
3855 * function with process_ceq() in interrupt context.
3856 */
3857void irdma_sc_cleanup_ceqes(struct irdma_sc_cq *cq, struct irdma_sc_ceq *ceq)
3858{
3859	struct irdma_sc_cq *next_cq;
3860	u8 ceq_polarity = ceq->polarity;
3861	__le64 *ceqe;
3862	u8 polarity;
3863	u64 temp;
3864	int next;
3865	u32 i;
3866
3867	next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, 0);
3868
3869	for (i = 1; i <= IRDMA_RING_SIZE(*ceq); i++) {
3870		ceqe = IRDMA_GET_CEQ_ELEM_AT_POS(ceq, next);
3871
3872		get_64bit_val(ceqe, 0, &temp);
3873		polarity = (u8)FIELD_GET(IRDMA_CEQE_VALID, temp);
3874		if (polarity != ceq_polarity)
3875			return;
3876
3877		next_cq = (struct irdma_sc_cq *)(unsigned long)(temp << 1);
3878		if (cq == next_cq)
3879			set_64bit_val(ceqe, 0, temp & IRDMA_CEQE_VALID);
3880
3881		next = IRDMA_RING_GET_NEXT_TAIL(ceq->ceq_ring, i);
3882		if (!next)
3883			ceq_polarity ^= 1;
3884	}
3885}
3886
3887/**
3888 * irdma_sc_aeq_init - initialize aeq
3889 * @aeq: aeq structure ptr
3890 * @info: aeq initialization info
3891 */
3892int irdma_sc_aeq_init(struct irdma_sc_aeq *aeq,
3893		      struct irdma_aeq_init_info *info)
3894{
3895	u32 pble_obj_cnt;
3896
3897	if (info->elem_cnt < info->dev->hw_attrs.min_hw_aeq_size ||
3898	    info->elem_cnt > info->dev->hw_attrs.max_hw_aeq_size)
3899		return -EINVAL;
3900
3901	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
3902
3903	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
3904		return -EINVAL;
3905
3906	aeq->size = sizeof(*aeq);
3907	aeq->polarity = 1;
3908	aeq->aeqe_base = (struct irdma_sc_aeqe *)info->aeqe_base;
3909	aeq->dev = info->dev;
3910	aeq->elem_cnt = info->elem_cnt;
3911	aeq->aeq_elem_pa = info->aeq_elem_pa;
3912	IRDMA_RING_INIT(aeq->aeq_ring, aeq->elem_cnt);
3913	aeq->virtual_map = info->virtual_map;
3914	aeq->pbl_list = (aeq->virtual_map ? info->pbl_list : NULL);
3915	aeq->pbl_chunk_size = (aeq->virtual_map ? info->pbl_chunk_size : 0);
3916	aeq->first_pm_pbl_idx = (aeq->virtual_map ? info->first_pm_pbl_idx : 0);
3917	aeq->msix_idx = info->msix_idx;
3918	info->dev->aeq = aeq;
3919
3920	return 0;
3921}
3922
3923/**
3924 * irdma_sc_aeq_create - create aeq
3925 * @aeq: aeq structure ptr
3926 * @scratch: u64 saved to be used during cqp completion
3927 * @post_sq: flag for cqp db to ring
3928 */
3929static int irdma_sc_aeq_create(struct irdma_sc_aeq *aeq, u64 scratch,
3930			       bool post_sq)
3931{
3932	__le64 *wqe;
3933	struct irdma_sc_cqp *cqp;
3934	u64 hdr;
3935
3936	cqp = aeq->dev->cqp;
3937	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3938	if (!wqe)
3939		return -ENOMEM;
3940	set_64bit_val(wqe, 16, aeq->elem_cnt);
3941	set_64bit_val(wqe, 32,
3942		      (aeq->virtual_map ? 0 : aeq->aeq_elem_pa));
3943	set_64bit_val(wqe, 48,
3944		      (aeq->virtual_map ? aeq->first_pm_pbl_idx : 0));
3945
3946	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_CREATE_AEQ) |
3947	      FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |
3948	      FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |
3949	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3950	dma_wmb(); /* make sure WQE is written before valid bit is set */
3951
3952	set_64bit_val(wqe, 24, hdr);
3953
3954	print_hex_dump_debug("WQE: AEQ_CREATE WQE", DUMP_PREFIX_OFFSET, 16, 8,
3955			     wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3956	if (post_sq)
3957		irdma_sc_cqp_post_sq(cqp);
3958
3959	return 0;
3960}
3961
3962/**
3963 * irdma_sc_aeq_destroy - destroy aeq during close
3964 * @aeq: aeq structure ptr
3965 * @scratch: u64 saved to be used during cqp completion
3966 * @post_sq: flag for cqp db to ring
3967 */
3968static int irdma_sc_aeq_destroy(struct irdma_sc_aeq *aeq, u64 scratch,
3969				bool post_sq)
3970{
3971	__le64 *wqe;
3972	struct irdma_sc_cqp *cqp;
3973	struct irdma_sc_dev *dev;
3974	u64 hdr;
3975
3976	dev = aeq->dev;
3977	writel(0, dev->hw_regs[IRDMA_PFINT_AEQCTL]);
3978
3979	cqp = dev->cqp;
3980	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
3981	if (!wqe)
3982		return -ENOMEM;
3983	set_64bit_val(wqe, 16, aeq->elem_cnt);
3984	set_64bit_val(wqe, 48, aeq->first_pm_pbl_idx);
3985	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_AEQ) |
3986	      FIELD_PREP(IRDMA_CQPSQ_AEQ_LPBLSIZE, aeq->pbl_chunk_size) |
3987	      FIELD_PREP(IRDMA_CQPSQ_AEQ_VMAP, aeq->virtual_map) |
3988	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
3989	dma_wmb(); /* make sure WQE is written before valid bit is set */
3990
3991	set_64bit_val(wqe, 24, hdr);
3992
3993	print_hex_dump_debug("WQE: AEQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
3994			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
3995	if (post_sq)
3996		irdma_sc_cqp_post_sq(cqp);
3997	return 0;
3998}
3999
4000/**
4001 * irdma_sc_get_next_aeqe - get next aeq entry
4002 * @aeq: aeq structure ptr
4003 * @info: aeqe info to be returned
4004 */
4005int irdma_sc_get_next_aeqe(struct irdma_sc_aeq *aeq,
4006			   struct irdma_aeqe_info *info)
4007{
4008	u64 temp, compl_ctx;
4009	__le64 *aeqe;
 
4010	u8 ae_src;
4011	u8 polarity;
4012
4013	aeqe = IRDMA_GET_CURRENT_AEQ_ELEM(aeq);
 
4014	get_64bit_val(aeqe, 8, &temp);
4015	polarity = (u8)FIELD_GET(IRDMA_AEQE_VALID, temp);
4016
4017	if (aeq->polarity != polarity)
4018		return -ENOENT;
4019
4020	/* Ensure AEQE contents are read after valid bit is checked */
4021	dma_rmb();
4022
4023	get_64bit_val(aeqe, 0, &compl_ctx);
4024
4025	print_hex_dump_debug("WQE: AEQ_ENTRY WQE", DUMP_PREFIX_OFFSET, 16, 8,
4026			     aeqe, 16, false);
4027
4028	ae_src = (u8)FIELD_GET(IRDMA_AEQE_AESRC, temp);
4029	info->wqe_idx = (u16)FIELD_GET(IRDMA_AEQE_WQDESCIDX, temp);
4030	info->qp_cq_id = (u32)FIELD_GET(IRDMA_AEQE_QPCQID_LOW, temp) |
4031			 ((u32)FIELD_GET(IRDMA_AEQE_QPCQID_HI, temp) << 18);
4032	info->ae_id = (u16)FIELD_GET(IRDMA_AEQE_AECODE, temp);
4033	info->tcp_state = (u8)FIELD_GET(IRDMA_AEQE_TCPSTATE, temp);
4034	info->iwarp_state = (u8)FIELD_GET(IRDMA_AEQE_IWSTATE, temp);
4035	info->q2_data_written = (u8)FIELD_GET(IRDMA_AEQE_Q2DATA, temp);
4036	info->aeqe_overflow = (bool)FIELD_GET(IRDMA_AEQE_OVERFLOW, temp);
4037
4038	info->ae_src = ae_src;
4039	switch (info->ae_id) {
4040	case IRDMA_AE_PRIV_OPERATION_DENIED:
4041	case IRDMA_AE_AMP_INVALIDATE_TYPE1_MW:
4042	case IRDMA_AE_AMP_MWBIND_ZERO_BASED_TYPE1_MW:
4043	case IRDMA_AE_AMP_FASTREG_INVALID_PBL_HPS_CFG:
4044	case IRDMA_AE_AMP_FASTREG_PBLE_MISMATCH:
4045	case IRDMA_AE_UDA_XMIT_DGRAM_TOO_LONG:
4046	case IRDMA_AE_UDA_XMIT_BAD_PD:
4047	case IRDMA_AE_UDA_XMIT_DGRAM_TOO_SHORT:
4048	case IRDMA_AE_BAD_CLOSE:
4049	case IRDMA_AE_RDMA_READ_WHILE_ORD_ZERO:
4050	case IRDMA_AE_STAG_ZERO_INVALID:
4051	case IRDMA_AE_IB_RREQ_AND_Q1_FULL:
4052	case IRDMA_AE_IB_INVALID_REQUEST:
4053	case IRDMA_AE_WQE_UNEXPECTED_OPCODE:
4054	case IRDMA_AE_IB_REMOTE_ACCESS_ERROR:
4055	case IRDMA_AE_IB_REMOTE_OP_ERROR:
4056	case IRDMA_AE_DDP_UBE_INVALID_DDP_VERSION:
4057	case IRDMA_AE_DDP_UBE_INVALID_MO:
4058	case IRDMA_AE_DDP_UBE_INVALID_QN:
4059	case IRDMA_AE_DDP_NO_L_BIT:
4060	case IRDMA_AE_RDMAP_ROE_INVALID_RDMAP_VERSION:
4061	case IRDMA_AE_RDMAP_ROE_UNEXPECTED_OPCODE:
4062	case IRDMA_AE_ROE_INVALID_RDMA_READ_REQUEST:
4063	case IRDMA_AE_ROE_INVALID_RDMA_WRITE_OR_READ_RESP:
4064	case IRDMA_AE_ROCE_RSP_LENGTH_ERROR:
4065	case IRDMA_AE_INVALID_ARP_ENTRY:
4066	case IRDMA_AE_INVALID_TCP_OPTION_RCVD:
4067	case IRDMA_AE_STALE_ARP_ENTRY:
4068	case IRDMA_AE_INVALID_AH_ENTRY:
4069	case IRDMA_AE_LLP_RECEIVED_MPA_CRC_ERROR:
4070	case IRDMA_AE_LLP_SEGMENT_TOO_SMALL:
4071	case IRDMA_AE_LLP_TOO_MANY_RETRIES:
4072	case IRDMA_AE_LLP_DOUBT_REACHABILITY:
4073	case IRDMA_AE_LLP_CONNECTION_ESTABLISHED:
4074	case IRDMA_AE_RESET_SENT:
4075	case IRDMA_AE_TERMINATE_SENT:
4076	case IRDMA_AE_RESET_NOT_SENT:
4077	case IRDMA_AE_LCE_QP_CATASTROPHIC:
4078	case IRDMA_AE_QP_SUSPEND_COMPLETE:
4079	case IRDMA_AE_UDA_L4LEN_INVALID:
4080		info->qp = true;
4081		info->compl_ctx = compl_ctx;
4082		break;
4083	case IRDMA_AE_LCE_CQ_CATASTROPHIC:
4084		info->cq = true;
4085		info->compl_ctx = compl_ctx << 1;
4086		ae_src = IRDMA_AE_SOURCE_RSVD;
4087		break;
4088	case IRDMA_AE_ROCE_EMPTY_MCG:
4089	case IRDMA_AE_ROCE_BAD_MC_IP_ADDR:
4090	case IRDMA_AE_ROCE_BAD_MC_QPID:
4091	case IRDMA_AE_MCG_QP_PROTOCOL_MISMATCH:
4092		fallthrough;
4093	case IRDMA_AE_LLP_CONNECTION_RESET:
4094	case IRDMA_AE_LLP_SYN_RECEIVED:
4095	case IRDMA_AE_LLP_FIN_RECEIVED:
4096	case IRDMA_AE_LLP_CLOSE_COMPLETE:
4097	case IRDMA_AE_LLP_TERMINATE_RECEIVED:
4098	case IRDMA_AE_RDMAP_ROE_BAD_LLP_CLOSE:
4099		ae_src = IRDMA_AE_SOURCE_RSVD;
4100		info->qp = true;
4101		info->compl_ctx = compl_ctx;
4102		break;
4103	default:
4104		break;
4105	}
4106
4107	switch (ae_src) {
4108	case IRDMA_AE_SOURCE_RQ:
4109	case IRDMA_AE_SOURCE_RQ_0011:
4110		info->qp = true;
4111		info->rq = true;
 
4112		info->compl_ctx = compl_ctx;
4113		break;
4114	case IRDMA_AE_SOURCE_CQ:
4115	case IRDMA_AE_SOURCE_CQ_0110:
4116	case IRDMA_AE_SOURCE_CQ_1010:
4117	case IRDMA_AE_SOURCE_CQ_1110:
4118		info->cq = true;
4119		info->compl_ctx = compl_ctx << 1;
4120		break;
4121	case IRDMA_AE_SOURCE_SQ:
4122	case IRDMA_AE_SOURCE_SQ_0111:
4123		info->qp = true;
4124		info->sq = true;
 
4125		info->compl_ctx = compl_ctx;
4126		break;
4127	case IRDMA_AE_SOURCE_IN_RR_WR:
4128	case IRDMA_AE_SOURCE_IN_RR_WR_1011:
4129		info->qp = true;
4130		info->compl_ctx = compl_ctx;
4131		info->in_rdrsp_wr = true;
4132		break;
4133	case IRDMA_AE_SOURCE_OUT_RR:
4134	case IRDMA_AE_SOURCE_OUT_RR_1111:
4135		info->qp = true;
4136		info->compl_ctx = compl_ctx;
4137		info->out_rdrsp = true;
4138		break;
4139	case IRDMA_AE_SOURCE_RSVD:
4140	default:
4141		break;
4142	}
4143
4144	IRDMA_RING_MOVE_TAIL(aeq->aeq_ring);
4145	if (!IRDMA_RING_CURRENT_TAIL(aeq->aeq_ring))
4146		aeq->polarity ^= 1;
4147
4148	return 0;
4149}
4150
4151/**
4152 * irdma_sc_repost_aeq_entries - repost completed aeq entries
4153 * @dev: sc device struct
4154 * @count: allocate count
4155 */
4156void irdma_sc_repost_aeq_entries(struct irdma_sc_dev *dev, u32 count)
4157{
4158	writel(count, dev->hw_regs[IRDMA_AEQALLOC]);
4159}
4160
4161/**
4162 * irdma_sc_ccq_init - initialize control cq
4163 * @cq: sc's cq ctruct
4164 * @info: info for control cq initialization
4165 */
4166int irdma_sc_ccq_init(struct irdma_sc_cq *cq, struct irdma_ccq_init_info *info)
4167{
4168	u32 pble_obj_cnt;
4169
4170	if (info->num_elem < info->dev->hw_attrs.uk_attrs.min_hw_cq_size ||
4171	    info->num_elem > info->dev->hw_attrs.uk_attrs.max_hw_cq_size)
4172		return -EINVAL;
4173
4174	if (info->ceq_id >= info->dev->hmc_fpm_misc.max_ceqs)
4175		return -EINVAL;
4176
4177	pble_obj_cnt = info->dev->hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt;
4178
4179	if (info->virtual_map && info->first_pm_pbl_idx >= pble_obj_cnt)
4180		return -EINVAL;
4181
4182	cq->cq_pa = info->cq_pa;
4183	cq->cq_uk.cq_base = info->cq_base;
4184	cq->shadow_area_pa = info->shadow_area_pa;
4185	cq->cq_uk.shadow_area = info->shadow_area;
4186	cq->shadow_read_threshold = info->shadow_read_threshold;
4187	cq->dev = info->dev;
4188	cq->ceq_id = info->ceq_id;
4189	cq->cq_uk.cq_size = info->num_elem;
4190	cq->cq_type = IRDMA_CQ_TYPE_CQP;
4191	cq->ceqe_mask = info->ceqe_mask;
4192	IRDMA_RING_INIT(cq->cq_uk.cq_ring, info->num_elem);
4193	cq->cq_uk.cq_id = 0; /* control cq is id 0 always */
4194	cq->ceq_id_valid = info->ceq_id_valid;
4195	cq->tph_en = info->tph_en;
4196	cq->tph_val = info->tph_val;
4197	cq->cq_uk.avoid_mem_cflct = info->avoid_mem_cflct;
4198	cq->pbl_list = info->pbl_list;
4199	cq->virtual_map = info->virtual_map;
4200	cq->pbl_chunk_size = info->pbl_chunk_size;
4201	cq->first_pm_pbl_idx = info->first_pm_pbl_idx;
4202	cq->cq_uk.polarity = true;
4203	cq->vsi = info->vsi;
4204	cq->cq_uk.cq_ack_db = cq->dev->cq_ack_db;
4205
4206	/* Only applicable to CQs other than CCQ so initialize to zero */
4207	cq->cq_uk.cqe_alloc_db = NULL;
4208
4209	info->dev->ccq = cq;
4210	return 0;
4211}
4212
4213/**
4214 * irdma_sc_ccq_create_done - poll cqp for ccq create
4215 * @ccq: ccq sc struct
4216 */
4217static inline int irdma_sc_ccq_create_done(struct irdma_sc_cq *ccq)
4218{
4219	struct irdma_sc_cqp *cqp;
4220
4221	cqp = ccq->dev->cqp;
4222
4223	return irdma_sc_poll_for_cqp_op_done(cqp, IRDMA_CQP_OP_CREATE_CQ, NULL);
4224}
4225
4226/**
4227 * irdma_sc_ccq_create - create control cq
4228 * @ccq: ccq sc struct
4229 * @scratch: u64 saved to be used during cqp completion
4230 * @check_overflow: overlow flag for ccq
4231 * @post_sq: flag for cqp db to ring
4232 */
4233int irdma_sc_ccq_create(struct irdma_sc_cq *ccq, u64 scratch,
4234			bool check_overflow, bool post_sq)
4235{
4236	int ret_code;
4237
4238	ret_code = irdma_sc_cq_create(ccq, scratch, check_overflow, post_sq);
4239	if (ret_code)
4240		return ret_code;
4241
4242	if (post_sq) {
4243		ret_code = irdma_sc_ccq_create_done(ccq);
4244		if (ret_code)
4245			return ret_code;
4246	}
4247	ccq->dev->cqp->process_cqp_sds = irdma_cqp_sds_cmd;
4248
4249	return 0;
4250}
4251
4252/**
4253 * irdma_sc_ccq_destroy - destroy ccq during close
4254 * @ccq: ccq sc struct
4255 * @scratch: u64 saved to be used during cqp completion
4256 * @post_sq: flag for cqp db to ring
4257 */
4258int irdma_sc_ccq_destroy(struct irdma_sc_cq *ccq, u64 scratch, bool post_sq)
4259{
4260	struct irdma_sc_cqp *cqp;
4261	__le64 *wqe;
4262	u64 hdr;
4263	int ret_code = 0;
4264	u32 tail, val, error;
4265
4266	cqp = ccq->dev->cqp;
4267	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4268	if (!wqe)
4269		return -ENOMEM;
4270
4271	set_64bit_val(wqe, 0, ccq->cq_uk.cq_size);
4272	set_64bit_val(wqe, 8, (uintptr_t)ccq >> 1);
4273	set_64bit_val(wqe, 40, ccq->shadow_area_pa);
4274
4275	hdr = ccq->cq_uk.cq_id |
4276	      FLD_LS_64(ccq->dev, (ccq->ceq_id_valid ? ccq->ceq_id : 0),
4277			IRDMA_CQPSQ_CQ_CEQID) |
4278	      FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_DESTROY_CQ) |
4279	      FIELD_PREP(IRDMA_CQPSQ_CQ_ENCEQEMASK, ccq->ceqe_mask) |
4280	      FIELD_PREP(IRDMA_CQPSQ_CQ_CEQIDVALID, ccq->ceq_id_valid) |
4281	      FIELD_PREP(IRDMA_CQPSQ_TPHEN, ccq->tph_en) |
4282	      FIELD_PREP(IRDMA_CQPSQ_CQ_AVOIDMEMCNFLCT, ccq->cq_uk.avoid_mem_cflct) |
4283	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
4284	dma_wmb(); /* make sure WQE is written before valid bit is set */
4285
4286	set_64bit_val(wqe, 24, hdr);
4287
4288	print_hex_dump_debug("WQE: CCQ_DESTROY WQE", DUMP_PREFIX_OFFSET, 16,
4289			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4290	irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4291
4292	if (post_sq) {
4293		irdma_sc_cqp_post_sq(cqp);
4294		ret_code = irdma_cqp_poll_registers(cqp, tail,
4295						    cqp->dev->hw_attrs.max_done_count);
4296	}
4297
4298	cqp->process_cqp_sds = irdma_update_sds_noccq;
4299
4300	return ret_code;
4301}
4302
4303/**
4304 * irdma_sc_init_iw_hmc() - queries fpm values using cqp and populates hmc_info
4305 * @dev : ptr to irdma_dev struct
4306 * @hmc_fn_id: hmc function id
4307 */
4308int irdma_sc_init_iw_hmc(struct irdma_sc_dev *dev, u8 hmc_fn_id)
4309{
4310	struct irdma_hmc_info *hmc_info;
4311	struct irdma_hmc_fpm_misc *hmc_fpm_misc;
4312	struct irdma_dma_mem query_fpm_mem;
4313	int ret_code = 0;
4314	u8 wait_type;
4315
4316	hmc_info = dev->hmc_info;
4317	hmc_fpm_misc = &dev->hmc_fpm_misc;
4318	query_fpm_mem.pa = dev->fpm_query_buf_pa;
4319	query_fpm_mem.va = dev->fpm_query_buf;
4320	hmc_info->hmc_fn_id = hmc_fn_id;
4321	wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS;
4322
4323	ret_code = irdma_sc_query_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,
4324					  &query_fpm_mem, true, wait_type);
4325	if (ret_code)
4326		return ret_code;
4327
4328	/* parse the fpm_query_buf and fill hmc obj info */
4329	ret_code = irdma_sc_parse_fpm_query_buf(dev, query_fpm_mem.va, hmc_info,
4330						hmc_fpm_misc);
4331
4332	print_hex_dump_debug("HMC: QUERY FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4333			     8, query_fpm_mem.va, IRDMA_QUERY_FPM_BUF_SIZE,
4334			     false);
4335	return ret_code;
4336}
4337
4338/**
4339 * irdma_sc_cfg_iw_fpm() - commits hmc obj cnt values using cqp
4340 * command and populates fpm base address in hmc_info
4341 * @dev : ptr to irdma_dev struct
4342 * @hmc_fn_id: hmc function id
4343 */
4344static int irdma_sc_cfg_iw_fpm(struct irdma_sc_dev *dev, u8 hmc_fn_id)
4345{
4346	struct irdma_hmc_info *hmc_info;
4347	struct irdma_hmc_obj_info *obj_info;
4348	__le64 *buf;
4349	struct irdma_dma_mem commit_fpm_mem;
4350	int ret_code = 0;
4351	u8 wait_type;
4352
4353	hmc_info = dev->hmc_info;
4354	obj_info = hmc_info->hmc_obj;
4355	buf = dev->fpm_commit_buf;
4356
4357	set_64bit_val(buf, 0, (u64)obj_info[IRDMA_HMC_IW_QP].cnt);
4358	set_64bit_val(buf, 8, (u64)obj_info[IRDMA_HMC_IW_CQ].cnt);
4359	set_64bit_val(buf, 16, (u64)0); /* RSRVD */
4360	set_64bit_val(buf, 24, (u64)obj_info[IRDMA_HMC_IW_HTE].cnt);
4361	set_64bit_val(buf, 32, (u64)obj_info[IRDMA_HMC_IW_ARP].cnt);
4362	set_64bit_val(buf, 40, (u64)0); /* RSVD */
4363	set_64bit_val(buf, 48, (u64)obj_info[IRDMA_HMC_IW_MR].cnt);
4364	set_64bit_val(buf, 56, (u64)obj_info[IRDMA_HMC_IW_XF].cnt);
4365	set_64bit_val(buf, 64, (u64)obj_info[IRDMA_HMC_IW_XFFL].cnt);
4366	set_64bit_val(buf, 72, (u64)obj_info[IRDMA_HMC_IW_Q1].cnt);
4367	set_64bit_val(buf, 80, (u64)obj_info[IRDMA_HMC_IW_Q1FL].cnt);
4368	set_64bit_val(buf, 88,
4369		      (u64)obj_info[IRDMA_HMC_IW_TIMER].cnt);
4370	set_64bit_val(buf, 96,
4371		      (u64)obj_info[IRDMA_HMC_IW_FSIMC].cnt);
4372	set_64bit_val(buf, 104,
4373		      (u64)obj_info[IRDMA_HMC_IW_FSIAV].cnt);
4374	set_64bit_val(buf, 112,
4375		      (u64)obj_info[IRDMA_HMC_IW_PBLE].cnt);
4376	set_64bit_val(buf, 120, (u64)0); /* RSVD */
4377	set_64bit_val(buf, 128, (u64)obj_info[IRDMA_HMC_IW_RRF].cnt);
4378	set_64bit_val(buf, 136,
4379		      (u64)obj_info[IRDMA_HMC_IW_RRFFL].cnt);
4380	set_64bit_val(buf, 144, (u64)obj_info[IRDMA_HMC_IW_HDR].cnt);
4381	set_64bit_val(buf, 152, (u64)obj_info[IRDMA_HMC_IW_MD].cnt);
4382	set_64bit_val(buf, 160,
4383		      (u64)obj_info[IRDMA_HMC_IW_OOISC].cnt);
4384	set_64bit_val(buf, 168,
4385		      (u64)obj_info[IRDMA_HMC_IW_OOISCFFL].cnt);
4386
4387	commit_fpm_mem.pa = dev->fpm_commit_buf_pa;
4388	commit_fpm_mem.va = dev->fpm_commit_buf;
4389
4390	wait_type = (u8)IRDMA_CQP_WAIT_POLL_REGS;
4391	print_hex_dump_debug("HMC: COMMIT FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4392			     8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE,
4393			     false);
4394	ret_code = irdma_sc_commit_fpm_val(dev->cqp, 0, hmc_info->hmc_fn_id,
4395					   &commit_fpm_mem, true, wait_type);
4396	if (!ret_code)
4397		irdma_sc_parse_fpm_commit_buf(dev, dev->fpm_commit_buf,
4398					      hmc_info->hmc_obj,
4399					      &hmc_info->sd_table.sd_cnt);
4400	print_hex_dump_debug("HMC: COMMIT FPM BUFFER", DUMP_PREFIX_OFFSET, 16,
4401			     8, commit_fpm_mem.va, IRDMA_COMMIT_FPM_BUF_SIZE,
4402			     false);
4403
4404	return ret_code;
4405}
4406
4407/**
4408 * cqp_sds_wqe_fill - fill cqp wqe doe sd
4409 * @cqp: struct for cqp hw
4410 * @info: sd info for wqe
4411 * @scratch: u64 saved to be used during cqp completion
4412 */
4413static int cqp_sds_wqe_fill(struct irdma_sc_cqp *cqp,
4414			    struct irdma_update_sds_info *info, u64 scratch)
4415{
4416	u64 data;
4417	u64 hdr;
4418	__le64 *wqe;
4419	int mem_entries, wqe_entries;
4420	struct irdma_dma_mem *sdbuf = &cqp->sdbuf;
4421	u64 offset = 0;
4422	u32 wqe_idx;
4423
4424	wqe = irdma_sc_cqp_get_next_send_wqe_idx(cqp, scratch, &wqe_idx);
4425	if (!wqe)
4426		return -ENOMEM;
4427
4428	wqe_entries = (info->cnt > 3) ? 3 : info->cnt;
4429	mem_entries = info->cnt - wqe_entries;
4430
4431	if (mem_entries) {
4432		offset = wqe_idx * IRDMA_UPDATE_SD_BUFF_SIZE;
4433		memcpy(((char *)sdbuf->va + offset), &info->entry[3], mem_entries << 4);
4434
4435		data = (u64)sdbuf->pa + offset;
4436	} else {
4437		data = 0;
4438	}
4439	data |= FIELD_PREP(IRDMA_CQPSQ_UPESD_HMCFNID, info->hmc_fn_id);
4440	set_64bit_val(wqe, 16, data);
4441
4442	switch (wqe_entries) {
4443	case 3:
4444		set_64bit_val(wqe, 48,
4445			      (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[2].cmd) |
4446			       FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));
4447
4448		set_64bit_val(wqe, 56, info->entry[2].data);
4449		fallthrough;
4450	case 2:
4451		set_64bit_val(wqe, 32,
4452			      (FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[1].cmd) |
4453			       FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_VALID, 1)));
4454
4455		set_64bit_val(wqe, 40, info->entry[1].data);
4456		fallthrough;
4457	case 1:
4458		set_64bit_val(wqe, 0,
4459			      FIELD_PREP(IRDMA_CQPSQ_UPESD_SDCMD, info->entry[0].cmd));
4460
4461		set_64bit_val(wqe, 8, info->entry[0].data);
4462		break;
4463	default:
4464		break;
4465	}
4466
4467	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE, IRDMA_CQP_OP_UPDATE_PE_SDS) |
4468	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity) |
4469	      FIELD_PREP(IRDMA_CQPSQ_UPESD_ENTRY_COUNT, mem_entries);
4470	dma_wmb(); /* make sure WQE is written before valid bit is set */
4471
4472	set_64bit_val(wqe, 24, hdr);
4473
4474	if (mem_entries)
4475		print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE Buffer",
4476				     DUMP_PREFIX_OFFSET, 16, 8,
4477				     (char *)sdbuf->va + offset,
4478				     mem_entries << 4, false);
4479
4480	print_hex_dump_debug("WQE: UPDATE_PE_SDS WQE", DUMP_PREFIX_OFFSET, 16,
4481			     8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4482
4483	return 0;
4484}
4485
4486/**
4487 * irdma_update_pe_sds - cqp wqe for sd
4488 * @dev: ptr to irdma_dev struct
4489 * @info: sd info for sd's
4490 * @scratch: u64 saved to be used during cqp completion
4491 */
4492static int irdma_update_pe_sds(struct irdma_sc_dev *dev,
4493			       struct irdma_update_sds_info *info, u64 scratch)
4494{
4495	struct irdma_sc_cqp *cqp = dev->cqp;
4496	int ret_code;
4497
4498	ret_code = cqp_sds_wqe_fill(cqp, info, scratch);
4499	if (!ret_code)
4500		irdma_sc_cqp_post_sq(cqp);
4501
4502	return ret_code;
4503}
4504
4505/**
4506 * irdma_update_sds_noccq - update sd before ccq created
4507 * @dev: sc device struct
4508 * @info: sd info for sd's
4509 */
4510int irdma_update_sds_noccq(struct irdma_sc_dev *dev,
4511			   struct irdma_update_sds_info *info)
4512{
4513	u32 error, val, tail;
4514	struct irdma_sc_cqp *cqp = dev->cqp;
4515	int ret_code;
4516
4517	ret_code = cqp_sds_wqe_fill(cqp, info, 0);
4518	if (ret_code)
4519		return ret_code;
4520
4521	irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4522
4523	irdma_sc_cqp_post_sq(cqp);
4524	return irdma_cqp_poll_registers(cqp, tail,
4525					cqp->dev->hw_attrs.max_done_count);
4526}
4527
4528/**
4529 * irdma_sc_static_hmc_pages_allocated - cqp wqe to allocate hmc pages
4530 * @cqp: struct for cqp hw
4531 * @scratch: u64 saved to be used during cqp completion
4532 * @hmc_fn_id: hmc function id
4533 * @post_sq: flag for cqp db to ring
4534 * @poll_registers: flag to poll register for cqp completion
4535 */
4536int irdma_sc_static_hmc_pages_allocated(struct irdma_sc_cqp *cqp, u64 scratch,
4537					u8 hmc_fn_id, bool post_sq,
4538					bool poll_registers)
4539{
4540	u64 hdr;
4541	__le64 *wqe;
4542	u32 tail, val, error;
4543
4544	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4545	if (!wqe)
4546		return -ENOMEM;
4547
4548	set_64bit_val(wqe, 16,
4549		      FIELD_PREP(IRDMA_SHMC_PAGE_ALLOCATED_HMC_FN_ID, hmc_fn_id));
4550
4551	hdr = FIELD_PREP(IRDMA_CQPSQ_OPCODE,
4552			 IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED) |
4553	      FIELD_PREP(IRDMA_CQPSQ_WQEVALID, cqp->polarity);
4554	dma_wmb(); /* make sure WQE is written before valid bit is set */
4555
4556	set_64bit_val(wqe, 24, hdr);
4557
4558	print_hex_dump_debug("WQE: SHMC_PAGES_ALLOCATED WQE",
4559			     DUMP_PREFIX_OFFSET, 16, 8, wqe,
4560			     IRDMA_CQP_WQE_SIZE * 8, false);
4561	irdma_get_cqp_reg_info(cqp, &val, &tail, &error);
4562
4563	if (post_sq) {
4564		irdma_sc_cqp_post_sq(cqp);
4565		if (poll_registers)
4566			/* check for cqp sq tail update */
4567			return irdma_cqp_poll_registers(cqp, tail,
4568							cqp->dev->hw_attrs.max_done_count);
4569		else
4570			return irdma_sc_poll_for_cqp_op_done(cqp,
4571							     IRDMA_CQP_OP_SHMC_PAGES_ALLOCATED,
4572							     NULL);
4573	}
4574
4575	return 0;
4576}
4577
4578/**
4579 * irdma_cqp_ring_full - check if cqp ring is full
4580 * @cqp: struct for cqp hw
4581 */
4582static bool irdma_cqp_ring_full(struct irdma_sc_cqp *cqp)
4583{
4584	return IRDMA_RING_FULL_ERR(cqp->sq_ring);
4585}
4586
4587/**
4588 * irdma_est_sd - returns approximate number of SDs for HMC
4589 * @dev: sc device struct
4590 * @hmc_info: hmc structure, size and count for HMC objects
4591 */
4592static u32 irdma_est_sd(struct irdma_sc_dev *dev,
4593			struct irdma_hmc_info *hmc_info)
4594{
4595	int i;
4596	u64 size = 0;
4597	u64 sd;
4598
4599	for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
4600		if (i != IRDMA_HMC_IW_PBLE)
4601			size += round_up(hmc_info->hmc_obj[i].cnt *
4602					 hmc_info->hmc_obj[i].size, 512);
4603	size += round_up(hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt *
4604			 hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].size, 512);
4605	if (size & 0x1FFFFF)
4606		sd = (size >> 21) + 1; /* add 1 for remainder */
4607	else
4608		sd = size >> 21;
4609	if (sd > 0xFFFFFFFF) {
4610		ibdev_dbg(to_ibdev(dev), "HMC: sd overflow[%lld]\n", sd);
4611		sd = 0xFFFFFFFF - 1;
4612	}
4613
4614	return (u32)sd;
4615}
4616
4617/**
4618 * irdma_sc_query_rdma_features_done - poll cqp for query features done
4619 * @cqp: struct for cqp hw
4620 */
4621static int irdma_sc_query_rdma_features_done(struct irdma_sc_cqp *cqp)
4622{
4623	return irdma_sc_poll_for_cqp_op_done(cqp,
4624					     IRDMA_CQP_OP_QUERY_RDMA_FEATURES,
4625					     NULL);
4626}
4627
4628/**
4629 * irdma_sc_query_rdma_features - query RDMA features and FW ver
4630 * @cqp: struct for cqp hw
4631 * @buf: buffer to hold query info
4632 * @scratch: u64 saved to be used during cqp completion
4633 */
4634static int irdma_sc_query_rdma_features(struct irdma_sc_cqp *cqp,
4635					struct irdma_dma_mem *buf, u64 scratch)
4636{
4637	__le64 *wqe;
4638	u64 temp;
4639
4640	wqe = irdma_sc_cqp_get_next_send_wqe(cqp, scratch);
4641	if (!wqe)
4642		return -ENOMEM;
4643
4644	temp = buf->pa;
4645	set_64bit_val(wqe, 32, temp);
4646
4647	temp = FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_WQEVALID,
4648			  cqp->polarity) |
4649	       FIELD_PREP(IRDMA_CQPSQ_QUERY_RDMA_FEATURES_BUF_LEN, buf->size) |
4650	       FIELD_PREP(IRDMA_CQPSQ_UP_OP, IRDMA_CQP_OP_QUERY_RDMA_FEATURES);
4651	dma_wmb(); /* make sure WQE is written before valid bit is set */
4652
4653	set_64bit_val(wqe, 24, temp);
4654
4655	print_hex_dump_debug("WQE: QUERY RDMA FEATURES", DUMP_PREFIX_OFFSET,
4656			     16, 8, wqe, IRDMA_CQP_WQE_SIZE * 8, false);
4657	irdma_sc_cqp_post_sq(cqp);
4658
4659	return 0;
4660}
4661
4662/**
4663 * irdma_get_rdma_features - get RDMA features
4664 * @dev: sc device struct
4665 */
4666int irdma_get_rdma_features(struct irdma_sc_dev *dev)
4667{
4668	int ret_code;
4669	struct irdma_dma_mem feat_buf;
4670	u64 temp;
4671	u16 byte_idx, feat_type, feat_cnt, feat_idx;
4672
4673	feat_buf.size = ALIGN(IRDMA_FEATURE_BUF_SIZE,
4674			      IRDMA_FEATURE_BUF_ALIGNMENT);
4675	feat_buf.va = dma_alloc_coherent(dev->hw->device, feat_buf.size,
4676					 &feat_buf.pa, GFP_KERNEL);
4677	if (!feat_buf.va)
4678		return -ENOMEM;
4679
4680	ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
4681	if (!ret_code)
4682		ret_code = irdma_sc_query_rdma_features_done(dev->cqp);
4683	if (ret_code)
4684		goto exit;
4685
4686	get_64bit_val(feat_buf.va, 0, &temp);
4687	feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);
4688	if (feat_cnt < 2) {
4689		ret_code = -EINVAL;
4690		goto exit;
4691	} else if (feat_cnt > IRDMA_MAX_FEATURES) {
4692		ibdev_dbg(to_ibdev(dev),
4693			  "DEV: feature buf size insufficient, retrying with larger buffer\n");
4694		dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,
4695				  feat_buf.pa);
4696		feat_buf.va = NULL;
4697		feat_buf.size = ALIGN(8 * feat_cnt,
4698				      IRDMA_FEATURE_BUF_ALIGNMENT);
4699		feat_buf.va = dma_alloc_coherent(dev->hw->device,
4700						 feat_buf.size, &feat_buf.pa,
4701						 GFP_KERNEL);
4702		if (!feat_buf.va)
4703			return -ENOMEM;
4704
4705		ret_code = irdma_sc_query_rdma_features(dev->cqp, &feat_buf, 0);
4706		if (!ret_code)
4707			ret_code = irdma_sc_query_rdma_features_done(dev->cqp);
4708		if (ret_code)
4709			goto exit;
4710
4711		get_64bit_val(feat_buf.va, 0, &temp);
4712		feat_cnt = (u16)FIELD_GET(IRDMA_FEATURE_CNT, temp);
4713		if (feat_cnt < 2) {
4714			ret_code = -EINVAL;
4715			goto exit;
4716		}
4717	}
4718
4719	print_hex_dump_debug("WQE: QUERY RDMA FEATURES", DUMP_PREFIX_OFFSET,
4720			     16, 8, feat_buf.va, feat_cnt * 8, false);
4721
4722	for (byte_idx = 0, feat_idx = 0; feat_idx < min(feat_cnt, (u16)IRDMA_MAX_FEATURES);
4723	     feat_idx++, byte_idx += 8) {
4724		get_64bit_val(feat_buf.va, byte_idx, &temp);
4725		feat_type = FIELD_GET(IRDMA_FEATURE_TYPE, temp);
4726		if (feat_type >= IRDMA_MAX_FEATURES) {
4727			ibdev_dbg(to_ibdev(dev),
4728				  "DEV: found unrecognized feature type %d\n",
4729				  feat_type);
4730			continue;
4731		}
4732		dev->feature_info[feat_type] = temp;
4733	}
4734exit:
4735	dma_free_coherent(dev->hw->device, feat_buf.size, feat_buf.va,
4736			  feat_buf.pa);
4737	feat_buf.va = NULL;
4738	return ret_code;
4739}
4740
4741static u32 irdma_q1_cnt(struct irdma_sc_dev *dev,
4742			struct irdma_hmc_info *hmc_info, u32 qpwanted)
4743{
4744	u32 q1_cnt;
4745
4746	if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1) {
4747		q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted);
4748	} else {
4749		if (dev->cqp->protocol_used != IRDMA_IWARP_PROTOCOL_ONLY)
4750			q1_cnt = roundup_pow_of_two(dev->hw_attrs.max_hw_ird * 2 * qpwanted + 512);
4751		else
4752			q1_cnt = dev->hw_attrs.max_hw_ird * 2 * qpwanted;
4753	}
4754
4755	return q1_cnt;
4756}
4757
4758static void cfg_fpm_value_gen_1(struct irdma_sc_dev *dev,
4759				struct irdma_hmc_info *hmc_info, u32 qpwanted)
4760{
4761	hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt = roundup_pow_of_two(qpwanted * dev->hw_attrs.max_hw_wqes);
4762}
4763
4764static void cfg_fpm_value_gen_2(struct irdma_sc_dev *dev,
4765				struct irdma_hmc_info *hmc_info, u32 qpwanted)
4766{
4767	struct irdma_hmc_fpm_misc *hmc_fpm_misc = &dev->hmc_fpm_misc;
4768
4769	hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt =
4770		4 * hmc_fpm_misc->xf_block_size * qpwanted;
4771
4772	hmc_info->hmc_obj[IRDMA_HMC_IW_HDR].cnt = qpwanted;
4773
4774	if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].max_cnt)
4775		hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt = 32 * qpwanted;
4776	if (hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].max_cnt)
4777		hmc_info->hmc_obj[IRDMA_HMC_IW_RRFFL].cnt =
4778			hmc_info->hmc_obj[IRDMA_HMC_IW_RRF].cnt /
4779			hmc_fpm_misc->rrf_block_size;
4780	if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].max_cnt)
4781		hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt = 32 * qpwanted;
4782	if (hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].max_cnt)
4783		hmc_info->hmc_obj[IRDMA_HMC_IW_OOISCFFL].cnt =
4784			hmc_info->hmc_obj[IRDMA_HMC_IW_OOISC].cnt /
4785			hmc_fpm_misc->ooiscf_block_size;
4786}
4787
4788/**
4789 * irdma_cfg_fpm_val - configure HMC objects
4790 * @dev: sc device struct
4791 * @qp_count: desired qp count
4792 */
4793int irdma_cfg_fpm_val(struct irdma_sc_dev *dev, u32 qp_count)
4794{
4795	struct irdma_virt_mem virt_mem;
4796	u32 i, mem_size;
4797	u32 qpwanted, mrwanted, pblewanted;
4798	u32 powerof2, hte;
4799	u32 sd_needed;
4800	u32 sd_diff;
4801	u32 loop_count = 0;
4802	struct irdma_hmc_info *hmc_info;
4803	struct irdma_hmc_fpm_misc *hmc_fpm_misc;
4804	int ret_code = 0;
4805
4806	hmc_info = dev->hmc_info;
4807	hmc_fpm_misc = &dev->hmc_fpm_misc;
4808
4809	ret_code = irdma_sc_init_iw_hmc(dev, dev->hmc_fn_id);
4810	if (ret_code) {
4811		ibdev_dbg(to_ibdev(dev),
4812			  "HMC: irdma_sc_init_iw_hmc returned error_code = %d\n",
4813			  ret_code);
4814		return ret_code;
4815	}
4816
4817	for (i = IRDMA_HMC_IW_QP; i < IRDMA_HMC_IW_MAX; i++)
4818		hmc_info->hmc_obj[i].cnt = hmc_info->hmc_obj[i].max_cnt;
4819	sd_needed = irdma_est_sd(dev, hmc_info);
4820	ibdev_dbg(to_ibdev(dev),
4821		  "HMC: FW max resources sd_needed[%08d] first_sd_index[%04d]\n",
4822		  sd_needed, hmc_info->first_sd_index);
4823	ibdev_dbg(to_ibdev(dev), "HMC: sd count %d where max sd is %d\n",
4824		  hmc_info->sd_table.sd_cnt, hmc_fpm_misc->max_sds);
4825
4826	qpwanted = min(qp_count, hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt);
4827
4828	powerof2 = 1;
4829	while (powerof2 <= qpwanted)
4830		powerof2 *= 2;
4831	powerof2 /= 2;
4832	qpwanted = powerof2;
4833
4834	mrwanted = hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt;
4835	pblewanted = hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt;
4836
4837	ibdev_dbg(to_ibdev(dev),
4838		  "HMC: req_qp=%d max_sd=%d, max_qp = %d, max_cq=%d, max_mr=%d, max_pble=%d, mc=%d, av=%d\n",
4839		  qp_count, hmc_fpm_misc->max_sds,
4840		  hmc_info->hmc_obj[IRDMA_HMC_IW_QP].max_cnt,
4841		  hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].max_cnt,
4842		  hmc_info->hmc_obj[IRDMA_HMC_IW_MR].max_cnt,
4843		  hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].max_cnt,
4844		  hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt,
4845		  hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt);
4846	hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt =
4847		hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].max_cnt;
4848	hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt =
4849		hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].max_cnt;
4850	hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].cnt =
4851		hmc_info->hmc_obj[IRDMA_HMC_IW_ARP].max_cnt;
4852
4853	hmc_info->hmc_obj[IRDMA_HMC_IW_APBVT_ENTRY].cnt = 1;
4854
4855	while (irdma_q1_cnt(dev, hmc_info, qpwanted) > hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].max_cnt)
4856		qpwanted /= 2;
4857
4858	do {
4859		++loop_count;
4860		hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt = qpwanted;
4861		hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt =
4862			min(2 * qpwanted, hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt);
4863		hmc_info->hmc_obj[IRDMA_HMC_IW_RESERVED].cnt = 0; /* Reserved */
4864		hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt = mrwanted;
4865
4866		hte = round_up(qpwanted + hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt, 512);
4867		powerof2 = 1;
4868		while (powerof2 < hte)
4869			powerof2 *= 2;
4870		hmc_info->hmc_obj[IRDMA_HMC_IW_HTE].cnt =
4871			powerof2 * hmc_fpm_misc->ht_multiplier;
4872		if (dev->hw_attrs.uk_attrs.hw_rev == IRDMA_GEN_1)
4873			cfg_fpm_value_gen_1(dev, hmc_info, qpwanted);
4874		else
4875			cfg_fpm_value_gen_2(dev, hmc_info, qpwanted);
4876
4877		hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt = irdma_q1_cnt(dev, hmc_info, qpwanted);
4878		hmc_info->hmc_obj[IRDMA_HMC_IW_XFFL].cnt =
4879			hmc_info->hmc_obj[IRDMA_HMC_IW_XF].cnt / hmc_fpm_misc->xf_block_size;
4880		hmc_info->hmc_obj[IRDMA_HMC_IW_Q1FL].cnt =
4881			hmc_info->hmc_obj[IRDMA_HMC_IW_Q1].cnt / hmc_fpm_misc->q1_block_size;
4882		hmc_info->hmc_obj[IRDMA_HMC_IW_TIMER].cnt =
4883			(round_up(qpwanted, 512) / 512 + 1) * hmc_fpm_misc->timer_bucket;
4884
4885		hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;
4886		sd_needed = irdma_est_sd(dev, hmc_info);
4887		ibdev_dbg(to_ibdev(dev),
4888			  "HMC: sd_needed = %d, hmc_fpm_misc->max_sds=%d, mrwanted=%d, pblewanted=%d qpwanted=%d\n",
4889			  sd_needed, hmc_fpm_misc->max_sds, mrwanted,
4890			  pblewanted, qpwanted);
4891
4892		/* Do not reduce resources further. All objects fit with max SDs */
4893		if (sd_needed <= hmc_fpm_misc->max_sds)
4894			break;
4895
4896		sd_diff = sd_needed - hmc_fpm_misc->max_sds;
4897		if (sd_diff > 128) {
4898			if (!(loop_count % 2) && qpwanted > 128) {
4899				qpwanted /= 2;
4900			} else {
4901				mrwanted /= 2;
4902				pblewanted /= 2;
4903			}
4904			continue;
4905		}
4906		if (dev->cqp->hmc_profile != IRDMA_HMC_PROFILE_FAVOR_VF &&
4907		    pblewanted > (512 * FPM_MULTIPLIER * sd_diff)) {
4908			pblewanted -= 256 * FPM_MULTIPLIER * sd_diff;
4909			continue;
4910		} else if (pblewanted > (100 * FPM_MULTIPLIER)) {
4911			pblewanted -= 10 * FPM_MULTIPLIER;
4912		} else if (pblewanted > FPM_MULTIPLIER) {
4913			pblewanted -= FPM_MULTIPLIER;
4914		} else if (qpwanted <= 128) {
4915			if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt > 256)
4916				hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt /= 2;
4917			if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)
4918				hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;
4919		}
4920		if (mrwanted > FPM_MULTIPLIER)
4921			mrwanted -= FPM_MULTIPLIER;
4922		if (!(loop_count % 10) && qpwanted > 128) {
4923			qpwanted /= 2;
4924			if (hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt > 256)
4925				hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt /= 2;
4926		}
4927	} while (loop_count < 2000);
4928
4929	if (sd_needed > hmc_fpm_misc->max_sds) {
4930		ibdev_dbg(to_ibdev(dev),
4931			  "HMC: cfg_fpm failed loop_cnt=%d, sd_needed=%d, max sd count %d\n",
4932			  loop_count, sd_needed, hmc_info->sd_table.sd_cnt);
4933		return -EINVAL;
4934	}
4935
4936	if (loop_count > 1 && sd_needed < hmc_fpm_misc->max_sds) {
4937		pblewanted += (hmc_fpm_misc->max_sds - sd_needed) * 256 *
4938			      FPM_MULTIPLIER;
4939		hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt = pblewanted;
4940		sd_needed = irdma_est_sd(dev, hmc_info);
4941	}
4942
4943	ibdev_dbg(to_ibdev(dev),
4944		  "HMC: loop_cnt=%d, sd_needed=%d, qpcnt = %d, cqcnt=%d, mrcnt=%d, pblecnt=%d, mc=%d, ah=%d, max sd count %d, first sd index %d\n",
4945		  loop_count, sd_needed,
4946		  hmc_info->hmc_obj[IRDMA_HMC_IW_QP].cnt,
4947		  hmc_info->hmc_obj[IRDMA_HMC_IW_CQ].cnt,
4948		  hmc_info->hmc_obj[IRDMA_HMC_IW_MR].cnt,
4949		  hmc_info->hmc_obj[IRDMA_HMC_IW_PBLE].cnt,
4950		  hmc_info->hmc_obj[IRDMA_HMC_IW_FSIMC].cnt,
4951		  hmc_info->hmc_obj[IRDMA_HMC_IW_FSIAV].cnt,
4952		  hmc_info->sd_table.sd_cnt, hmc_info->first_sd_index);
4953
4954	ret_code = irdma_sc_cfg_iw_fpm(dev, dev->hmc_fn_id);
4955	if (ret_code) {
4956		ibdev_dbg(to_ibdev(dev),
4957			  "HMC: cfg_iw_fpm returned error_code[x%08X]\n",
4958			  readl(dev->hw_regs[IRDMA_CQPERRCODES]));
4959		return ret_code;
4960	}
4961
4962	mem_size = sizeof(struct irdma_hmc_sd_entry) *
4963		   (hmc_info->sd_table.sd_cnt + hmc_info->first_sd_index + 1);
4964	virt_mem.size = mem_size;
4965	virt_mem.va = kzalloc(virt_mem.size, GFP_KERNEL);
4966	if (!virt_mem.va) {
4967		ibdev_dbg(to_ibdev(dev),
4968			  "HMC: failed to allocate memory for sd_entry buffer\n");
4969		return -ENOMEM;
4970	}
4971	hmc_info->sd_table.sd_entry = virt_mem.va;
4972
4973	return ret_code;
4974}
4975
4976/**
4977 * irdma_exec_cqp_cmd - execute cqp cmd when wqe are available
4978 * @dev: rdma device
4979 * @pcmdinfo: cqp command info
4980 */
4981static int irdma_exec_cqp_cmd(struct irdma_sc_dev *dev,
4982			      struct cqp_cmds_info *pcmdinfo)
4983{
4984	int status;
4985	struct irdma_dma_mem val_mem;
4986	bool alloc = false;
4987
4988	dev->cqp_cmd_stats[pcmdinfo->cqp_cmd]++;
4989	switch (pcmdinfo->cqp_cmd) {
4990	case IRDMA_OP_CEQ_DESTROY:
4991		status = irdma_sc_ceq_destroy(pcmdinfo->in.u.ceq_destroy.ceq,
4992					      pcmdinfo->in.u.ceq_destroy.scratch,
4993					      pcmdinfo->post_sq);
4994		break;
4995	case IRDMA_OP_AEQ_DESTROY:
4996		status = irdma_sc_aeq_destroy(pcmdinfo->in.u.aeq_destroy.aeq,
4997					      pcmdinfo->in.u.aeq_destroy.scratch,
4998					      pcmdinfo->post_sq);
4999
5000		break;
5001	case IRDMA_OP_CEQ_CREATE:
5002		status = irdma_sc_ceq_create(pcmdinfo->in.u.ceq_create.ceq,
5003					     pcmdinfo->in.u.ceq_create.scratch,
5004					     pcmdinfo->post_sq);
5005		break;
5006	case IRDMA_OP_AEQ_CREATE:
5007		status = irdma_sc_aeq_create(pcmdinfo->in.u.aeq_create.aeq,
5008					     pcmdinfo->in.u.aeq_create.scratch,
5009					     pcmdinfo->post_sq);
5010		break;
5011	case IRDMA_OP_QP_UPLOAD_CONTEXT:
5012		status = irdma_sc_qp_upload_context(pcmdinfo->in.u.qp_upload_context.dev,
5013						    &pcmdinfo->in.u.qp_upload_context.info,
5014						    pcmdinfo->in.u.qp_upload_context.scratch,
5015						    pcmdinfo->post_sq);
5016		break;
5017	case IRDMA_OP_CQ_CREATE:
5018		status = irdma_sc_cq_create(pcmdinfo->in.u.cq_create.cq,
5019					    pcmdinfo->in.u.cq_create.scratch,
5020					    pcmdinfo->in.u.cq_create.check_overflow,
5021					    pcmdinfo->post_sq);
5022		break;
5023	case IRDMA_OP_CQ_MODIFY:
5024		status = irdma_sc_cq_modify(pcmdinfo->in.u.cq_modify.cq,
5025					    &pcmdinfo->in.u.cq_modify.info,
5026					    pcmdinfo->in.u.cq_modify.scratch,
5027					    pcmdinfo->post_sq);
5028		break;
5029	case IRDMA_OP_CQ_DESTROY:
5030		status = irdma_sc_cq_destroy(pcmdinfo->in.u.cq_destroy.cq,
5031					     pcmdinfo->in.u.cq_destroy.scratch,
5032					     pcmdinfo->post_sq);
5033		break;
5034	case IRDMA_OP_QP_FLUSH_WQES:
5035		status = irdma_sc_qp_flush_wqes(pcmdinfo->in.u.qp_flush_wqes.qp,
5036						&pcmdinfo->in.u.qp_flush_wqes.info,
5037						pcmdinfo->in.u.qp_flush_wqes.scratch,
5038						pcmdinfo->post_sq);
5039		break;
5040	case IRDMA_OP_GEN_AE:
5041		status = irdma_sc_gen_ae(pcmdinfo->in.u.gen_ae.qp,
5042					 &pcmdinfo->in.u.gen_ae.info,
5043					 pcmdinfo->in.u.gen_ae.scratch,
5044					 pcmdinfo->post_sq);
5045		break;
5046	case IRDMA_OP_MANAGE_PUSH_PAGE:
5047		status = irdma_sc_manage_push_page(pcmdinfo->in.u.manage_push_page.cqp,
5048						   &pcmdinfo->in.u.manage_push_page.info,
5049						   pcmdinfo->in.u.manage_push_page.scratch,
5050						   pcmdinfo->post_sq);
5051		break;
5052	case IRDMA_OP_UPDATE_PE_SDS:
5053		status = irdma_update_pe_sds(pcmdinfo->in.u.update_pe_sds.dev,
5054					     &pcmdinfo->in.u.update_pe_sds.info,
5055					     pcmdinfo->in.u.update_pe_sds.scratch);
5056		break;
5057	case IRDMA_OP_MANAGE_HMC_PM_FUNC_TABLE:
5058		/* switch to calling through the call table */
5059		status =
5060			irdma_sc_manage_hmc_pm_func_table(pcmdinfo->in.u.manage_hmc_pm.dev->cqp,
5061							  &pcmdinfo->in.u.manage_hmc_pm.info,
5062							  pcmdinfo->in.u.manage_hmc_pm.scratch,
5063							  true);
5064		break;
5065	case IRDMA_OP_SUSPEND:
5066		status = irdma_sc_suspend_qp(pcmdinfo->in.u.suspend_resume.cqp,
5067					     pcmdinfo->in.u.suspend_resume.qp,
5068					     pcmdinfo->in.u.suspend_resume.scratch);
5069		break;
5070	case IRDMA_OP_RESUME:
5071		status = irdma_sc_resume_qp(pcmdinfo->in.u.suspend_resume.cqp,
5072					    pcmdinfo->in.u.suspend_resume.qp,
5073					    pcmdinfo->in.u.suspend_resume.scratch);
5074		break;
5075	case IRDMA_OP_QUERY_FPM_VAL:
5076		val_mem.pa = pcmdinfo->in.u.query_fpm_val.fpm_val_pa;
5077		val_mem.va = pcmdinfo->in.u.query_fpm_val.fpm_val_va;
5078		status = irdma_sc_query_fpm_val(pcmdinfo->in.u.query_fpm_val.cqp,
5079						pcmdinfo->in.u.query_fpm_val.scratch,
5080						pcmdinfo->in.u.query_fpm_val.hmc_fn_id,
5081						&val_mem, true, IRDMA_CQP_WAIT_EVENT);
5082		break;
5083	case IRDMA_OP_COMMIT_FPM_VAL:
5084		val_mem.pa = pcmdinfo->in.u.commit_fpm_val.fpm_val_pa;
5085		val_mem.va = pcmdinfo->in.u.commit_fpm_val.fpm_val_va;
5086		status = irdma_sc_commit_fpm_val(pcmdinfo->in.u.commit_fpm_val.cqp,
5087						 pcmdinfo->in.u.commit_fpm_val.scratch,
5088						 pcmdinfo->in.u.commit_fpm_val.hmc_fn_id,
5089						 &val_mem,
5090						 true,
5091						 IRDMA_CQP_WAIT_EVENT);
5092		break;
5093	case IRDMA_OP_STATS_ALLOCATE:
5094		alloc = true;
5095		fallthrough;
5096	case IRDMA_OP_STATS_FREE:
5097		status = irdma_sc_manage_stats_inst(pcmdinfo->in.u.stats_manage.cqp,
5098						    &pcmdinfo->in.u.stats_manage.info,
5099						    alloc,
5100						    pcmdinfo->in.u.stats_manage.scratch);
5101		break;
5102	case IRDMA_OP_STATS_GATHER:
5103		status = irdma_sc_gather_stats(pcmdinfo->in.u.stats_gather.cqp,
5104					       &pcmdinfo->in.u.stats_gather.info,
5105					       pcmdinfo->in.u.stats_gather.scratch);
5106		break;
5107	case IRDMA_OP_WS_MODIFY_NODE:
5108		status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5109						 &pcmdinfo->in.u.ws_node.info,
5110						 IRDMA_MODIFY_NODE,
5111						 pcmdinfo->in.u.ws_node.scratch);
5112		break;
5113	case IRDMA_OP_WS_DELETE_NODE:
5114		status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5115						 &pcmdinfo->in.u.ws_node.info,
5116						 IRDMA_DEL_NODE,
5117						 pcmdinfo->in.u.ws_node.scratch);
5118		break;
5119	case IRDMA_OP_WS_ADD_NODE:
5120		status = irdma_sc_manage_ws_node(pcmdinfo->in.u.ws_node.cqp,
5121						 &pcmdinfo->in.u.ws_node.info,
5122						 IRDMA_ADD_NODE,
5123						 pcmdinfo->in.u.ws_node.scratch);
5124		break;
5125	case IRDMA_OP_SET_UP_MAP:
5126		status = irdma_sc_set_up_map(pcmdinfo->in.u.up_map.cqp,
5127					     &pcmdinfo->in.u.up_map.info,
5128					     pcmdinfo->in.u.up_map.scratch);
5129		break;
5130	case IRDMA_OP_QUERY_RDMA_FEATURES:
5131		status = irdma_sc_query_rdma_features(pcmdinfo->in.u.query_rdma.cqp,
5132						      &pcmdinfo->in.u.query_rdma.query_buff_mem,
5133						      pcmdinfo->in.u.query_rdma.scratch);
5134		break;
5135	case IRDMA_OP_DELETE_ARP_CACHE_ENTRY:
5136		status = irdma_sc_del_arp_cache_entry(pcmdinfo->in.u.del_arp_cache_entry.cqp,
5137						      pcmdinfo->in.u.del_arp_cache_entry.scratch,
5138						      pcmdinfo->in.u.del_arp_cache_entry.arp_index,
5139						      pcmdinfo->post_sq);
5140		break;
5141	case IRDMA_OP_MANAGE_APBVT_ENTRY:
5142		status = irdma_sc_manage_apbvt_entry(pcmdinfo->in.u.manage_apbvt_entry.cqp,
5143						     &pcmdinfo->in.u.manage_apbvt_entry.info,
5144						     pcmdinfo->in.u.manage_apbvt_entry.scratch,
5145						     pcmdinfo->post_sq);
5146		break;
5147	case IRDMA_OP_MANAGE_QHASH_TABLE_ENTRY:
5148		status = irdma_sc_manage_qhash_table_entry(pcmdinfo->in.u.manage_qhash_table_entry.cqp,
5149							   &pcmdinfo->in.u.manage_qhash_table_entry.info,
5150							   pcmdinfo->in.u.manage_qhash_table_entry.scratch,
5151							   pcmdinfo->post_sq);
5152		break;
5153	case IRDMA_OP_QP_MODIFY:
5154		status = irdma_sc_qp_modify(pcmdinfo->in.u.qp_modify.qp,
5155					    &pcmdinfo->in.u.qp_modify.info,
5156					    pcmdinfo->in.u.qp_modify.scratch,
5157					    pcmdinfo->post_sq);
5158		break;
5159	case IRDMA_OP_QP_CREATE:
5160		status = irdma_sc_qp_create(pcmdinfo->in.u.qp_create.qp,
5161					    &pcmdinfo->in.u.qp_create.info,
5162					    pcmdinfo->in.u.qp_create.scratch,
5163					    pcmdinfo->post_sq);
5164		break;
5165	case IRDMA_OP_QP_DESTROY:
5166		status = irdma_sc_qp_destroy(pcmdinfo->in.u.qp_destroy.qp,
5167					     pcmdinfo->in.u.qp_destroy.scratch,
5168					     pcmdinfo->in.u.qp_destroy.remove_hash_idx,
5169					     pcmdinfo->in.u.qp_destroy.ignore_mw_bnd,
5170					     pcmdinfo->post_sq);
5171		break;
5172	case IRDMA_OP_ALLOC_STAG:
5173		status = irdma_sc_alloc_stag(pcmdinfo->in.u.alloc_stag.dev,
5174					     &pcmdinfo->in.u.alloc_stag.info,
5175					     pcmdinfo->in.u.alloc_stag.scratch,
5176					     pcmdinfo->post_sq);
5177		break;
5178	case IRDMA_OP_MR_REG_NON_SHARED:
5179		status = irdma_sc_mr_reg_non_shared(pcmdinfo->in.u.mr_reg_non_shared.dev,
5180						    &pcmdinfo->in.u.mr_reg_non_shared.info,
5181						    pcmdinfo->in.u.mr_reg_non_shared.scratch,
5182						    pcmdinfo->post_sq);
5183		break;
5184	case IRDMA_OP_DEALLOC_STAG:
5185		status = irdma_sc_dealloc_stag(pcmdinfo->in.u.dealloc_stag.dev,
5186					       &pcmdinfo->in.u.dealloc_stag.info,
5187					       pcmdinfo->in.u.dealloc_stag.scratch,
5188					       pcmdinfo->post_sq);
5189		break;
5190	case IRDMA_OP_MW_ALLOC:
5191		status = irdma_sc_mw_alloc(pcmdinfo->in.u.mw_alloc.dev,
5192					   &pcmdinfo->in.u.mw_alloc.info,
5193					   pcmdinfo->in.u.mw_alloc.scratch,
5194					   pcmdinfo->post_sq);
5195		break;
5196	case IRDMA_OP_ADD_ARP_CACHE_ENTRY:
5197		status = irdma_sc_add_arp_cache_entry(pcmdinfo->in.u.add_arp_cache_entry.cqp,
5198						      &pcmdinfo->in.u.add_arp_cache_entry.info,
5199						      pcmdinfo->in.u.add_arp_cache_entry.scratch,
5200						      pcmdinfo->post_sq);
5201		break;
5202	case IRDMA_OP_ALLOC_LOCAL_MAC_ENTRY:
5203		status = irdma_sc_alloc_local_mac_entry(pcmdinfo->in.u.alloc_local_mac_entry.cqp,
5204							pcmdinfo->in.u.alloc_local_mac_entry.scratch,
5205							pcmdinfo->post_sq);
5206		break;
5207	case IRDMA_OP_ADD_LOCAL_MAC_ENTRY:
5208		status = irdma_sc_add_local_mac_entry(pcmdinfo->in.u.add_local_mac_entry.cqp,
5209						      &pcmdinfo->in.u.add_local_mac_entry.info,
5210						      pcmdinfo->in.u.add_local_mac_entry.scratch,
5211						      pcmdinfo->post_sq);
5212		break;
5213	case IRDMA_OP_DELETE_LOCAL_MAC_ENTRY:
5214		status = irdma_sc_del_local_mac_entry(pcmdinfo->in.u.del_local_mac_entry.cqp,
5215						      pcmdinfo->in.u.del_local_mac_entry.scratch,
5216						      pcmdinfo->in.u.del_local_mac_entry.entry_idx,
5217						      pcmdinfo->in.u.del_local_mac_entry.ignore_ref_count,
5218						      pcmdinfo->post_sq);
5219		break;
5220	case IRDMA_OP_AH_CREATE:
5221		status = irdma_sc_create_ah(pcmdinfo->in.u.ah_create.cqp,
5222					    &pcmdinfo->in.u.ah_create.info,
5223					    pcmdinfo->in.u.ah_create.scratch);
5224		break;
5225	case IRDMA_OP_AH_DESTROY:
5226		status = irdma_sc_destroy_ah(pcmdinfo->in.u.ah_destroy.cqp,
5227					     &pcmdinfo->in.u.ah_destroy.info,
5228					     pcmdinfo->in.u.ah_destroy.scratch);
5229		break;
5230	case IRDMA_OP_MC_CREATE:
5231		status = irdma_sc_create_mcast_grp(pcmdinfo->in.u.mc_create.cqp,
5232						   &pcmdinfo->in.u.mc_create.info,
5233						   pcmdinfo->in.u.mc_create.scratch);
5234		break;
5235	case IRDMA_OP_MC_DESTROY:
5236		status = irdma_sc_destroy_mcast_grp(pcmdinfo->in.u.mc_destroy.cqp,
5237						    &pcmdinfo->in.u.mc_destroy.info,
5238						    pcmdinfo->in.u.mc_destroy.scratch);
5239		break;
5240	case IRDMA_OP_MC_MODIFY:
5241		status = irdma_sc_modify_mcast_grp(pcmdinfo->in.u.mc_modify.cqp,
5242						   &pcmdinfo->in.u.mc_modify.info,
5243						   pcmdinfo->in.u.mc_modify.scratch);
5244		break;
5245	default:
5246		status = -EOPNOTSUPP;
5247		break;
5248	}
5249
5250	return status;
5251}
5252
5253/**
5254 * irdma_process_cqp_cmd - process all cqp commands
5255 * @dev: sc device struct
5256 * @pcmdinfo: cqp command info
5257 */
5258int irdma_process_cqp_cmd(struct irdma_sc_dev *dev,
5259			  struct cqp_cmds_info *pcmdinfo)
5260{
5261	int status = 0;
5262	unsigned long flags;
5263
5264	spin_lock_irqsave(&dev->cqp_lock, flags);
5265	if (list_empty(&dev->cqp_cmd_head) && !irdma_cqp_ring_full(dev->cqp))
5266		status = irdma_exec_cqp_cmd(dev, pcmdinfo);
5267	else
5268		list_add_tail(&pcmdinfo->cqp_cmd_entry, &dev->cqp_cmd_head);
5269	spin_unlock_irqrestore(&dev->cqp_lock, flags);
5270	return status;
5271}
5272
5273/**
5274 * irdma_process_bh - called from tasklet for cqp list
5275 * @dev: sc device struct
5276 */
5277int irdma_process_bh(struct irdma_sc_dev *dev)
5278{
5279	int status = 0;
5280	struct cqp_cmds_info *pcmdinfo;
5281	unsigned long flags;
5282
5283	spin_lock_irqsave(&dev->cqp_lock, flags);
5284	while (!list_empty(&dev->cqp_cmd_head) &&
5285	       !irdma_cqp_ring_full(dev->cqp)) {
5286		pcmdinfo = (struct cqp_cmds_info *)irdma_remove_cqp_head(dev);
5287		status = irdma_exec_cqp_cmd(dev, pcmdinfo);
5288		if (status)
5289			break;
5290	}
5291	spin_unlock_irqrestore(&dev->cqp_lock, flags);
5292	return status;
5293}
5294
5295/**
5296 * irdma_cfg_aeq- Configure AEQ interrupt
5297 * @dev: pointer to the device structure
5298 * @idx: vector index
5299 * @enable: True to enable, False disables
5300 */
5301void irdma_cfg_aeq(struct irdma_sc_dev *dev, u32 idx, bool enable)
5302{
5303	u32 reg_val;
5304
5305	reg_val = FIELD_PREP(IRDMA_PFINT_AEQCTL_CAUSE_ENA, enable) |
5306		  FIELD_PREP(IRDMA_PFINT_AEQCTL_MSIX_INDX, idx) |
5307		  FIELD_PREP(IRDMA_PFINT_AEQCTL_ITR_INDX, 3);
5308	writel(reg_val, dev->hw_regs[IRDMA_PFINT_AEQCTL]);
5309}
5310
5311/**
5312 * sc_vsi_update_stats - Update statistics
5313 * @vsi: sc_vsi instance to update
5314 */
5315void sc_vsi_update_stats(struct irdma_sc_vsi *vsi)
5316{
5317	struct irdma_gather_stats *gather_stats;
5318	struct irdma_gather_stats *last_gather_stats;
5319
5320	gather_stats = vsi->pestat->gather_info.gather_stats_va;
5321	last_gather_stats = vsi->pestat->gather_info.last_gather_stats_va;
5322	irdma_update_stats(&vsi->pestat->hw_stats, gather_stats,
5323			   last_gather_stats, vsi->dev->hw_stats_map,
5324			   vsi->dev->hw_attrs.max_stat_idx);
5325}
5326
5327/**
5328 * irdma_wait_pe_ready - Check if firmware is ready
5329 * @dev: provides access to registers
5330 */
5331static int irdma_wait_pe_ready(struct irdma_sc_dev *dev)
5332{
5333	u32 statuscpu0;
5334	u32 statuscpu1;
5335	u32 statuscpu2;
5336	u32 retrycount = 0;
5337
5338	do {
5339		statuscpu0 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS0]);
5340		statuscpu1 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS1]);
5341		statuscpu2 = readl(dev->hw_regs[IRDMA_GLPE_CPUSTATUS2]);
5342		if (statuscpu0 == 0x80 && statuscpu1 == 0x80 &&
5343		    statuscpu2 == 0x80)
5344			return 0;
5345		mdelay(1000);
5346	} while (retrycount++ < dev->hw_attrs.max_pe_ready_count);
5347	return -1;
5348}
5349
5350static inline void irdma_sc_init_hw(struct irdma_sc_dev *dev)
5351{
5352	switch (dev->hw_attrs.uk_attrs.hw_rev) {
5353	case IRDMA_GEN_1:
5354		i40iw_init_hw(dev);
5355		break;
5356	case IRDMA_GEN_2:
5357		icrdma_init_hw(dev);
5358		break;
5359	}
5360}
5361
5362/**
5363 * irdma_sc_dev_init - Initialize control part of device
5364 * @ver: version
5365 * @dev: Device pointer
5366 * @info: Device init info
5367 */
5368int irdma_sc_dev_init(enum irdma_vers ver, struct irdma_sc_dev *dev,
5369		      struct irdma_device_init_info *info)
5370{
5371	u32 val;
5372	int ret_code = 0;
5373	u8 db_size;
5374
5375	INIT_LIST_HEAD(&dev->cqp_cmd_head); /* for CQP command backlog */
5376	mutex_init(&dev->ws_mutex);
5377	dev->hmc_fn_id = info->hmc_fn_id;
5378	dev->fpm_query_buf_pa = info->fpm_query_buf_pa;
5379	dev->fpm_query_buf = info->fpm_query_buf;
5380	dev->fpm_commit_buf_pa = info->fpm_commit_buf_pa;
5381	dev->fpm_commit_buf = info->fpm_commit_buf;
5382	dev->hw = info->hw;
5383	dev->hw->hw_addr = info->bar0;
5384	/* Setup the hardware limits, hmc may limit further */
5385	dev->hw_attrs.min_hw_qp_id = IRDMA_MIN_IW_QP_ID;
5386	dev->hw_attrs.min_hw_aeq_size = IRDMA_MIN_AEQ_ENTRIES;
5387	dev->hw_attrs.max_hw_aeq_size = IRDMA_MAX_AEQ_ENTRIES;
5388	dev->hw_attrs.min_hw_ceq_size = IRDMA_MIN_CEQ_ENTRIES;
5389	dev->hw_attrs.max_hw_ceq_size = IRDMA_MAX_CEQ_ENTRIES;
5390	dev->hw_attrs.uk_attrs.min_hw_cq_size = IRDMA_MIN_CQ_SIZE;
5391	dev->hw_attrs.uk_attrs.max_hw_cq_size = IRDMA_MAX_CQ_SIZE;
5392	dev->hw_attrs.uk_attrs.max_hw_wq_frags = IRDMA_MAX_WQ_FRAGMENT_COUNT;
5393	dev->hw_attrs.uk_attrs.max_hw_read_sges = IRDMA_MAX_SGE_RD;
5394	dev->hw_attrs.max_hw_outbound_msg_size = IRDMA_MAX_OUTBOUND_MSG_SIZE;
5395	dev->hw_attrs.max_mr_size = IRDMA_MAX_MR_SIZE;
5396	dev->hw_attrs.max_hw_inbound_msg_size = IRDMA_MAX_INBOUND_MSG_SIZE;
5397	dev->hw_attrs.max_hw_device_pages = IRDMA_MAX_PUSH_PAGE_COUNT;
5398	dev->hw_attrs.uk_attrs.max_hw_inline = IRDMA_MAX_INLINE_DATA_SIZE;
5399	dev->hw_attrs.max_hw_wqes = IRDMA_MAX_WQ_ENTRIES;
5400	dev->hw_attrs.max_qp_wr = IRDMA_MAX_QP_WRS(IRDMA_MAX_QUANTA_PER_WR);
5401
5402	dev->hw_attrs.uk_attrs.max_hw_rq_quanta = IRDMA_QP_SW_MAX_RQ_QUANTA;
5403	dev->hw_attrs.uk_attrs.max_hw_wq_quanta = IRDMA_QP_SW_MAX_WQ_QUANTA;
5404	dev->hw_attrs.max_hw_pds = IRDMA_MAX_PDS;
5405	dev->hw_attrs.max_hw_ena_vf_count = IRDMA_MAX_PE_ENA_VF_COUNT;
5406
5407	dev->hw_attrs.max_pe_ready_count = 14;
5408	dev->hw_attrs.max_done_count = IRDMA_DONE_COUNT;
5409	dev->hw_attrs.max_sleep_count = IRDMA_SLEEP_COUNT;
5410	dev->hw_attrs.max_cqp_compl_wait_time_ms = CQP_COMPL_WAIT_TIME_MS;
5411
5412	dev->hw_attrs.uk_attrs.hw_rev = ver;
5413	irdma_sc_init_hw(dev);
5414
5415	if (irdma_wait_pe_ready(dev))
5416		return -ETIMEDOUT;
5417
5418	val = readl(dev->hw_regs[IRDMA_GLPCI_LBARCTRL]);
5419	db_size = (u8)FIELD_GET(IRDMA_GLPCI_LBARCTRL_PE_DB_SIZE, val);
5420	if (db_size != IRDMA_PE_DB_SIZE_4M && db_size != IRDMA_PE_DB_SIZE_8M) {
5421		ibdev_dbg(to_ibdev(dev),
5422			  "DEV: RDMA PE doorbell is not enabled in CSR val 0x%x db_size=%d\n",
5423			  val, db_size);
5424		return -ENODEV;
5425	}
5426	dev->db_addr = dev->hw->hw_addr + (uintptr_t)dev->hw_regs[IRDMA_DB_ADDR_OFFSET];
5427
5428	return ret_code;
5429}
5430
5431/**
5432 * irdma_stat_val - Extract HW counter value from statistics buffer
5433 * @stats_val: pointer to statistics buffer
5434 * @byteoff: byte offset of counter value in the buffer (8B-aligned)
5435 * @bitoff: bit offset of counter value within 8B entry
5436 * @bitmask: maximum counter value (e.g. 0xffffff for 24-bit counter)
5437 */
5438static inline u64 irdma_stat_val(const u64 *stats_val, u16 byteoff, u8 bitoff,
5439				 u64 bitmask)
5440{
5441	u16 idx = byteoff / sizeof(*stats_val);
5442
5443	return (stats_val[idx] >> bitoff) & bitmask;
5444}
5445
5446/**
5447 * irdma_stat_delta - Calculate counter delta
5448 * @new_val: updated counter value
5449 * @old_val: last counter value
5450 * @max_val: maximum counter value (e.g. 0xffffff for 24-bit counter)
5451 */
5452static inline u64 irdma_stat_delta(u64 new_val, u64 old_val, u64 max_val)
5453{
5454	if (new_val >= old_val)
5455		return new_val - old_val;
5456
5457	/* roll-over case */
5458	return max_val - old_val + new_val + 1;
5459}
5460
5461/**
5462 * irdma_update_stats - Update statistics
5463 * @hw_stats: hw_stats instance to update
5464 * @gather_stats: updated stat counters
5465 * @last_gather_stats: last stat counters
5466 * @map: HW stat map (hw_stats => gather_stats)
5467 * @max_stat_idx: number of HW stats
5468 */
5469void irdma_update_stats(struct irdma_dev_hw_stats *hw_stats,
5470			struct irdma_gather_stats *gather_stats,
5471			struct irdma_gather_stats *last_gather_stats,
5472			const struct irdma_hw_stat_map *map, u16 max_stat_idx)
5473{
5474	u64 *stats_val = hw_stats->stats_val;
5475	u16 i;
5476
5477	for (i = 0; i < max_stat_idx; i++) {
5478		u64 new_val = irdma_stat_val(gather_stats->val, map[i].byteoff,
5479					     map[i].bitoff, map[i].bitmask);
5480		u64 last_val = irdma_stat_val(last_gather_stats->val,
5481					      map[i].byteoff, map[i].bitoff,
5482					      map[i].bitmask);
5483
5484		stats_val[i] +=
5485			irdma_stat_delta(new_val, last_val, map[i].bitmask);
5486	}
5487
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5488	memcpy(last_gather_stats, gather_stats, sizeof(*last_gather_stats));
5489}