1// SPDX-License-Identifier: GPL-2.0 OR BSD-2-Clause
   2/*
   3 * Copyright 2018-2021 Amazon.com, Inc. or its affiliates. All rights reserved.
   4 */
   5
   6#include "efa_com.h"
   7#include "efa_regs_defs.h"
   8
   9#define ADMIN_CMD_TIMEOUT_US 30000000 /* usecs */
  10
  11#define EFA_REG_READ_TIMEOUT_US 50000 /* usecs */
  12#define EFA_MMIO_READ_INVALID 0xffffffff
  13
  14#define EFA_POLL_INTERVAL_MS 100 /* msecs */
  15
  16#define EFA_ASYNC_QUEUE_DEPTH 16
  17#define EFA_ADMIN_QUEUE_DEPTH 32
  18
  19#define EFA_CTRL_MAJOR          0
  20#define EFA_CTRL_MINOR          0
  21#define EFA_CTRL_SUB_MINOR      1
  22
  23enum efa_cmd_status {
  24	EFA_CMD_SUBMITTED,
  25	EFA_CMD_COMPLETED,
  26};
  27
  28struct efa_comp_ctx {
  29	struct completion wait_event;
  30	struct efa_admin_acq_entry *user_cqe;
  31	u32 comp_size;
  32	enum efa_cmd_status status;
  33	u8 cmd_opcode;
  34	u8 occupied;
  35};
  36
  37static const char *efa_com_cmd_str(u8 cmd)
  38{
  39#define EFA_CMD_STR_CASE(_cmd) case EFA_ADMIN_##_cmd: return #_cmd
  40
  41	switch (cmd) {
  42	EFA_CMD_STR_CASE(CREATE_QP);
  43	EFA_CMD_STR_CASE(MODIFY_QP);
  44	EFA_CMD_STR_CASE(QUERY_QP);
  45	EFA_CMD_STR_CASE(DESTROY_QP);
  46	EFA_CMD_STR_CASE(CREATE_AH);
  47	EFA_CMD_STR_CASE(DESTROY_AH);
  48	EFA_CMD_STR_CASE(REG_MR);
  49	EFA_CMD_STR_CASE(DEREG_MR);
  50	EFA_CMD_STR_CASE(CREATE_CQ);
  51	EFA_CMD_STR_CASE(DESTROY_CQ);
  52	EFA_CMD_STR_CASE(GET_FEATURE);
  53	EFA_CMD_STR_CASE(SET_FEATURE);
  54	EFA_CMD_STR_CASE(GET_STATS);
  55	EFA_CMD_STR_CASE(ALLOC_PD);
  56	EFA_CMD_STR_CASE(DEALLOC_PD);
  57	EFA_CMD_STR_CASE(ALLOC_UAR);
  58	EFA_CMD_STR_CASE(DEALLOC_UAR);
  59	EFA_CMD_STR_CASE(CREATE_EQ);
  60	EFA_CMD_STR_CASE(DESTROY_EQ);
  61	default: return "unknown command opcode";
  62	}
  63#undef EFA_CMD_STR_CASE
  64}
  65
  66void efa_com_set_dma_addr(dma_addr_t addr, u32 *addr_high, u32 *addr_low)
  67{
  68	*addr_low = lower_32_bits(addr);
  69	*addr_high = upper_32_bits(addr);
  70}
  71
  72static u32 efa_com_reg_read32(struct efa_com_dev *edev, u16 offset)
  73{
  74	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
  75	struct efa_admin_mmio_req_read_less_resp *read_resp;
  76	unsigned long exp_time;
  77	u32 mmio_read_reg = 0;
  78	u32 err;
  79
  80	read_resp = mmio_read->read_resp;
  81
  82	spin_lock(&mmio_read->lock);
  83	mmio_read->seq_num++;
  84
  85	/* trash DMA req_id to identify when hardware is done */
  86	read_resp->req_id = mmio_read->seq_num + 0x9aL;
  87	EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REG_OFF, offset);
  88	EFA_SET(&mmio_read_reg, EFA_REGS_MMIO_REG_READ_REQ_ID,
  89		mmio_read->seq_num);
  90
  91	writel(mmio_read_reg, edev->reg_bar + EFA_REGS_MMIO_REG_READ_OFF);
  92
  93	exp_time = jiffies + usecs_to_jiffies(mmio_read->mmio_read_timeout);
  94	do {
  95		if (READ_ONCE(read_resp->req_id) == mmio_read->seq_num)
  96			break;
  97		udelay(1);
  98	} while (time_is_after_jiffies(exp_time));
  99
 100	if (read_resp->req_id != mmio_read->seq_num) {
 101		ibdev_err_ratelimited(
 102			edev->efa_dev,
 103			"Reading register timed out. expected: req id[%u] offset[%#x] actual: req id[%u] offset[%#x]\n",
 104			mmio_read->seq_num, offset, read_resp->req_id,
 105			read_resp->reg_off);
 106		err = EFA_MMIO_READ_INVALID;
 107		goto out;
 108	}
 109
 110	if (read_resp->reg_off != offset) {
 111		ibdev_err_ratelimited(
 112			edev->efa_dev,
 113			"Reading register failed: wrong offset provided\n");
 114		err = EFA_MMIO_READ_INVALID;
 115		goto out;
 116	}
 117
 118	err = read_resp->reg_val;
 119out:
 120	spin_unlock(&mmio_read->lock);
 121	return err;
 122}
 123
 124static int efa_com_admin_init_sq(struct efa_com_dev *edev)
 125{
 126	struct efa_com_admin_queue *aq = &edev->aq;
 127	struct efa_com_admin_sq *sq = &aq->sq;
 128	u16 size = aq->depth * sizeof(*sq->entries);
 129	u32 aq_caps = 0;
 130	u32 addr_high;
 131	u32 addr_low;
 132
 133	sq->entries =
 134		dma_alloc_coherent(aq->dmadev, size, &sq->dma_addr, GFP_KERNEL);
 135	if (!sq->entries)
 136		return -ENOMEM;
 137
 138	spin_lock_init(&sq->lock);
 139
 140	sq->cc = 0;
 141	sq->pc = 0;
 142	sq->phase = 1;
 143
 144	sq->db_addr = (u32 __iomem *)(edev->reg_bar + EFA_REGS_AQ_PROD_DB_OFF);
 145
 146	addr_high = upper_32_bits(sq->dma_addr);
 147	addr_low = lower_32_bits(sq->dma_addr);
 148
 149	writel(addr_low, edev->reg_bar + EFA_REGS_AQ_BASE_LO_OFF);
 150	writel(addr_high, edev->reg_bar + EFA_REGS_AQ_BASE_HI_OFF);
 151
 152	EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_DEPTH, aq->depth);
 153	EFA_SET(&aq_caps, EFA_REGS_AQ_CAPS_AQ_ENTRY_SIZE,
 154		sizeof(struct efa_admin_aq_entry));
 155
 156	writel(aq_caps, edev->reg_bar + EFA_REGS_AQ_CAPS_OFF);
 157
 158	return 0;
 159}
 160
 161static int efa_com_admin_init_cq(struct efa_com_dev *edev)
 162{
 163	struct efa_com_admin_queue *aq = &edev->aq;
 164	struct efa_com_admin_cq *cq = &aq->cq;
 165	u16 size = aq->depth * sizeof(*cq->entries);
 166	u32 acq_caps = 0;
 167	u32 addr_high;
 168	u32 addr_low;
 169
 170	cq->entries =
 171		dma_alloc_coherent(aq->dmadev, size, &cq->dma_addr, GFP_KERNEL);
 172	if (!cq->entries)
 173		return -ENOMEM;
 174
 175	spin_lock_init(&cq->lock);
 176
 177	cq->cc = 0;
 178	cq->phase = 1;
 179
 180	addr_high = upper_32_bits(cq->dma_addr);
 181	addr_low = lower_32_bits(cq->dma_addr);
 182
 183	writel(addr_low, edev->reg_bar + EFA_REGS_ACQ_BASE_LO_OFF);
 184	writel(addr_high, edev->reg_bar + EFA_REGS_ACQ_BASE_HI_OFF);
 185
 186	EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_DEPTH, aq->depth);
 187	EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_ENTRY_SIZE,
 188		sizeof(struct efa_admin_acq_entry));
 189	EFA_SET(&acq_caps, EFA_REGS_ACQ_CAPS_ACQ_MSIX_VECTOR,
 190		aq->msix_vector_idx);
 191
 192	writel(acq_caps, edev->reg_bar + EFA_REGS_ACQ_CAPS_OFF);
 193
 194	return 0;
 195}
 196
 197static int efa_com_admin_init_aenq(struct efa_com_dev *edev,
 198				   struct efa_aenq_handlers *aenq_handlers)
 199{
 200	struct efa_com_aenq *aenq = &edev->aenq;
 201	u32 addr_low, addr_high;
 202	u32 aenq_caps = 0;
 203	u16 size;
 204
 205	if (!aenq_handlers) {
 206		ibdev_err(edev->efa_dev, "aenq handlers pointer is NULL\n");
 207		return -EINVAL;
 208	}
 209
 210	size = EFA_ASYNC_QUEUE_DEPTH * sizeof(*aenq->entries);
 211	aenq->entries = dma_alloc_coherent(edev->dmadev, size, &aenq->dma_addr,
 212					   GFP_KERNEL);
 213	if (!aenq->entries)
 214		return -ENOMEM;
 215
 216	aenq->aenq_handlers = aenq_handlers;
 217	aenq->depth = EFA_ASYNC_QUEUE_DEPTH;
 218	aenq->cc = 0;
 219	aenq->phase = 1;
 220
 221	addr_low = lower_32_bits(aenq->dma_addr);
 222	addr_high = upper_32_bits(aenq->dma_addr);
 223
 224	writel(addr_low, edev->reg_bar + EFA_REGS_AENQ_BASE_LO_OFF);
 225	writel(addr_high, edev->reg_bar + EFA_REGS_AENQ_BASE_HI_OFF);
 226
 227	EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_DEPTH, aenq->depth);
 228	EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_ENTRY_SIZE,
 229		sizeof(struct efa_admin_aenq_entry));
 230	EFA_SET(&aenq_caps, EFA_REGS_AENQ_CAPS_AENQ_MSIX_VECTOR,
 231		aenq->msix_vector_idx);
 232	writel(aenq_caps, edev->reg_bar + EFA_REGS_AENQ_CAPS_OFF);
 233
 234	/*
 235	 * Init cons_db to mark that all entries in the queue
 236	 * are initially available
 237	 */
 238	writel(edev->aenq.cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF);
 239
 240	return 0;
 241}
 242
 243/* ID to be used with efa_com_get_comp_ctx */
 244static u16 efa_com_alloc_ctx_id(struct efa_com_admin_queue *aq)
 245{
 246	u16 ctx_id;
 247
 248	spin_lock(&aq->comp_ctx_lock);
 249	ctx_id = aq->comp_ctx_pool[aq->comp_ctx_pool_next];
 250	aq->comp_ctx_pool_next++;
 251	spin_unlock(&aq->comp_ctx_lock);
 252
 253	return ctx_id;
 254}
 255
 256static void efa_com_dealloc_ctx_id(struct efa_com_admin_queue *aq,
 257				   u16 ctx_id)
 258{
 259	spin_lock(&aq->comp_ctx_lock);
 260	aq->comp_ctx_pool_next--;
 261	aq->comp_ctx_pool[aq->comp_ctx_pool_next] = ctx_id;
 262	spin_unlock(&aq->comp_ctx_lock);
 263}
 264
 265static inline void efa_com_put_comp_ctx(struct efa_com_admin_queue *aq,
 266					struct efa_comp_ctx *comp_ctx)
 267{
 268	u16 cmd_id = EFA_GET(&comp_ctx->user_cqe->acq_common_descriptor.command,
 269			     EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);
 270	u16 ctx_id = cmd_id & (aq->depth - 1);
 271
 272	ibdev_dbg(aq->efa_dev, "Put completion command_id %#x\n", cmd_id);
 273	comp_ctx->occupied = 0;
 274	efa_com_dealloc_ctx_id(aq, ctx_id);
 275}
 276
 277static struct efa_comp_ctx *efa_com_get_comp_ctx(struct efa_com_admin_queue *aq,
 278						 u16 cmd_id, bool capture)
 279{
 280	u16 ctx_id = cmd_id & (aq->depth - 1);
 281
 282	if (aq->comp_ctx[ctx_id].occupied && capture) {
 283		ibdev_err_ratelimited(
 284			aq->efa_dev,
 285			"Completion context for command_id %#x is occupied\n",
 286			cmd_id);
 287		return NULL;
 288	}
 289
 290	if (capture) {
 291		aq->comp_ctx[ctx_id].occupied = 1;
 292		ibdev_dbg(aq->efa_dev,
 293			  "Take completion ctxt for command_id %#x\n", cmd_id);
 294	}
 295
 296	return &aq->comp_ctx[ctx_id];
 297}
 298
 299static struct efa_comp_ctx *__efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
 300						       struct efa_admin_aq_entry *cmd,
 301						       size_t cmd_size_in_bytes,
 302						       struct efa_admin_acq_entry *comp,
 303						       size_t comp_size_in_bytes)
 304{
 305	struct efa_admin_aq_entry *aqe;
 306	struct efa_comp_ctx *comp_ctx;
 307	u16 queue_size_mask;
 308	u16 cmd_id;
 309	u16 ctx_id;
 310	u16 pi;
 311
 312	queue_size_mask = aq->depth - 1;
 313	pi = aq->sq.pc & queue_size_mask;
 314
 315	ctx_id = efa_com_alloc_ctx_id(aq);
 316
 317	/* cmd_id LSBs are the ctx_id and MSBs are entropy bits from pc */
 318	cmd_id = ctx_id & queue_size_mask;
 319	cmd_id |= aq->sq.pc & ~queue_size_mask;
 320	cmd_id &= EFA_ADMIN_AQ_COMMON_DESC_COMMAND_ID_MASK;
 321
 322	cmd->aq_common_descriptor.command_id = cmd_id;
 323	EFA_SET(&cmd->aq_common_descriptor.flags,
 324		EFA_ADMIN_AQ_COMMON_DESC_PHASE, aq->sq.phase);
 325
 326	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, true);
 327	if (!comp_ctx) {
 328		efa_com_dealloc_ctx_id(aq, ctx_id);
 329		return ERR_PTR(-EINVAL);
 330	}
 331
 332	comp_ctx->status = EFA_CMD_SUBMITTED;
 333	comp_ctx->comp_size = comp_size_in_bytes;
 334	comp_ctx->user_cqe = comp;
 335	comp_ctx->cmd_opcode = cmd->aq_common_descriptor.opcode;
 336
 337	reinit_completion(&comp_ctx->wait_event);
 338
 339	aqe = &aq->sq.entries[pi];
 340	memset(aqe, 0, sizeof(*aqe));
 341	memcpy(aqe, cmd, cmd_size_in_bytes);
 342
 343	aq->sq.pc++;
 344	atomic64_inc(&aq->stats.submitted_cmd);
 345
 346	if ((aq->sq.pc & queue_size_mask) == 0)
 347		aq->sq.phase = !aq->sq.phase;
 348
 349	/* barrier not needed in case of writel */
 350	writel(aq->sq.pc, aq->sq.db_addr);
 351
 352	return comp_ctx;
 353}
 354
 355static inline int efa_com_init_comp_ctxt(struct efa_com_admin_queue *aq)
 356{
 357	size_t pool_size = aq->depth * sizeof(*aq->comp_ctx_pool);
 358	size_t size = aq->depth * sizeof(struct efa_comp_ctx);
 359	struct efa_comp_ctx *comp_ctx;
 360	u16 i;
 361
 362	aq->comp_ctx = devm_kzalloc(aq->dmadev, size, GFP_KERNEL);
 363	aq->comp_ctx_pool = devm_kzalloc(aq->dmadev, pool_size, GFP_KERNEL);
 364	if (!aq->comp_ctx || !aq->comp_ctx_pool) {
 365		devm_kfree(aq->dmadev, aq->comp_ctx_pool);
 366		devm_kfree(aq->dmadev, aq->comp_ctx);
 367		return -ENOMEM;
 368	}
 369
 370	for (i = 0; i < aq->depth; i++) {
 371		comp_ctx = efa_com_get_comp_ctx(aq, i, false);
 372		if (comp_ctx)
 373			init_completion(&comp_ctx->wait_event);
 374
 375		aq->comp_ctx_pool[i] = i;
 376	}
 377
 378	spin_lock_init(&aq->comp_ctx_lock);
 379
 380	aq->comp_ctx_pool_next = 0;
 381
 382	return 0;
 383}
 384
 385static struct efa_comp_ctx *efa_com_submit_admin_cmd(struct efa_com_admin_queue *aq,
 386						     struct efa_admin_aq_entry *cmd,
 387						     size_t cmd_size_in_bytes,
 388						     struct efa_admin_acq_entry *comp,
 389						     size_t comp_size_in_bytes)
 390{
 391	struct efa_comp_ctx *comp_ctx;
 392
 393	spin_lock(&aq->sq.lock);
 394	if (!test_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state)) {
 395		ibdev_err_ratelimited(aq->efa_dev, "Admin queue is closed\n");
 396		spin_unlock(&aq->sq.lock);
 397		return ERR_PTR(-ENODEV);
 398	}
 399
 400	comp_ctx = __efa_com_submit_admin_cmd(aq, cmd, cmd_size_in_bytes, comp,
 401					      comp_size_in_bytes);
 402	spin_unlock(&aq->sq.lock);
 403	if (IS_ERR(comp_ctx))
 404		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
 405
 406	return comp_ctx;
 407}
 408
 409static void efa_com_handle_single_admin_completion(struct efa_com_admin_queue *aq,
 410						   struct efa_admin_acq_entry *cqe)
 411{
 412	struct efa_comp_ctx *comp_ctx;
 413	u16 cmd_id;
 414
 415	cmd_id = EFA_GET(&cqe->acq_common_descriptor.command,
 416			 EFA_ADMIN_ACQ_COMMON_DESC_COMMAND_ID);
 417
 418	comp_ctx = efa_com_get_comp_ctx(aq, cmd_id, false);
 419	if (!comp_ctx) {
 420		ibdev_err(aq->efa_dev,
 421			  "comp_ctx is NULL. Changing the admin queue running state\n");
 422		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
 423		return;
 424	}
 425
 426	comp_ctx->status = EFA_CMD_COMPLETED;
 427	memcpy(comp_ctx->user_cqe, cqe, comp_ctx->comp_size);
 428
 429	if (!test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))
 430		complete(&comp_ctx->wait_event);
 431}
 432
 433static void efa_com_handle_admin_completion(struct efa_com_admin_queue *aq)
 434{
 435	struct efa_admin_acq_entry *cqe;
 436	u16 queue_size_mask;
 437	u16 comp_num = 0;
 438	u8 phase;
 439	u16 ci;
 440
 441	queue_size_mask = aq->depth - 1;
 442
 443	ci = aq->cq.cc & queue_size_mask;
 444	phase = aq->cq.phase;
 445
 446	cqe = &aq->cq.entries[ci];
 447
 448	/* Go over all the completions */
 449	while ((READ_ONCE(cqe->acq_common_descriptor.flags) &
 450		EFA_ADMIN_ACQ_COMMON_DESC_PHASE_MASK) == phase) {
 451		/*
 452		 * Do not read the rest of the completion entry before the
 453		 * phase bit was validated
 454		 */
 455		dma_rmb();
 456		efa_com_handle_single_admin_completion(aq, cqe);
 457
 458		ci++;
 459		comp_num++;
 460		if (ci == aq->depth) {
 461			ci = 0;
 462			phase = !phase;
 463		}
 464
 465		cqe = &aq->cq.entries[ci];
 466	}
 467
 468	aq->cq.cc += comp_num;
 469	aq->cq.phase = phase;
 470	aq->sq.cc += comp_num;
 471	atomic64_add(comp_num, &aq->stats.completed_cmd);
 472}
 473
 474static int efa_com_comp_status_to_errno(u8 comp_status)
 475{
 476	switch (comp_status) {
 477	case EFA_ADMIN_SUCCESS:
 478		return 0;
 479	case EFA_ADMIN_RESOURCE_ALLOCATION_FAILURE:
 480		return -ENOMEM;
 481	case EFA_ADMIN_UNSUPPORTED_OPCODE:
 482		return -EOPNOTSUPP;
 483	case EFA_ADMIN_BAD_OPCODE:
 484	case EFA_ADMIN_MALFORMED_REQUEST:
 485	case EFA_ADMIN_ILLEGAL_PARAMETER:
 486	case EFA_ADMIN_UNKNOWN_ERROR:
 487		return -EINVAL;
 488	default:
 489		return -EINVAL;
 490	}
 491}
 492
 493static int efa_com_wait_and_process_admin_cq_polling(struct efa_comp_ctx *comp_ctx,
 494						     struct efa_com_admin_queue *aq)
 495{
 496	unsigned long timeout;
 497	unsigned long flags;
 498	int err;
 499
 500	timeout = jiffies + usecs_to_jiffies(aq->completion_timeout);
 501
 502	while (1) {
 503		spin_lock_irqsave(&aq->cq.lock, flags);
 504		efa_com_handle_admin_completion(aq);
 505		spin_unlock_irqrestore(&aq->cq.lock, flags);
 506
 507		if (comp_ctx->status != EFA_CMD_SUBMITTED)
 508			break;
 509
 510		if (time_is_before_jiffies(timeout)) {
 511			ibdev_err_ratelimited(
 512				aq->efa_dev,
 513				"Wait for completion (polling) timeout\n");
 514			/* EFA didn't have any completion */
 515			atomic64_inc(&aq->stats.no_completion);
 516
 517			clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
 518			err = -ETIME;
 519			goto out;
 520		}
 521
 522		msleep(aq->poll_interval);
 523	}
 524
 525	err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status);
 526out:
 527	efa_com_put_comp_ctx(aq, comp_ctx);
 528	return err;
 529}
 530
 531static int efa_com_wait_and_process_admin_cq_interrupts(struct efa_comp_ctx *comp_ctx,
 532							struct efa_com_admin_queue *aq)
 533{
 534	unsigned long flags;
 535	int err;
 536
 537	wait_for_completion_timeout(&comp_ctx->wait_event,
 538				    usecs_to_jiffies(aq->completion_timeout));
 539
 540	/*
 541	 * In case the command wasn't completed find out the root cause.
 542	 * There might be 2 kinds of errors
 543	 * 1) No completion (timeout reached)
 544	 * 2) There is completion but the device didn't get any msi-x interrupt.
 545	 */
 546	if (comp_ctx->status == EFA_CMD_SUBMITTED) {
 547		spin_lock_irqsave(&aq->cq.lock, flags);
 548		efa_com_handle_admin_completion(aq);
 549		spin_unlock_irqrestore(&aq->cq.lock, flags);
 550
 551		atomic64_inc(&aq->stats.no_completion);
 552
 553		if (comp_ctx->status == EFA_CMD_COMPLETED)
 554			ibdev_err_ratelimited(
 555				aq->efa_dev,
 556				"The device sent a completion but the driver didn't receive any MSI-X interrupt for admin cmd %s(%d) status %d (ctx: 0x%p, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",
 557				efa_com_cmd_str(comp_ctx->cmd_opcode),
 558				comp_ctx->cmd_opcode, comp_ctx->status,
 559				comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);
 560		else
 561			ibdev_err_ratelimited(
 562				aq->efa_dev,
 563				"The device didn't send any completion for admin cmd %s(%d) status %d (ctx 0x%p, sq producer: %d, sq consumer: %d, cq consumer: %d)\n",
 564				efa_com_cmd_str(comp_ctx->cmd_opcode),
 565				comp_ctx->cmd_opcode, comp_ctx->status,
 566				comp_ctx, aq->sq.pc, aq->sq.cc, aq->cq.cc);
 567
 568		clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
 569		err = -ETIME;
 570		goto out;
 571	}
 572
 573	err = efa_com_comp_status_to_errno(comp_ctx->user_cqe->acq_common_descriptor.status);
 574out:
 575	efa_com_put_comp_ctx(aq, comp_ctx);
 576	return err;
 577}
 578
 579/*
 580 * There are two types to wait for completion.
 581 * Polling mode - wait until the completion is available.
 582 * Async mode - wait on wait queue until the completion is ready
 583 * (or the timeout expired).
 584 * It is expected that the IRQ called efa_com_handle_admin_completion
 585 * to mark the completions.
 586 */
 587static int efa_com_wait_and_process_admin_cq(struct efa_comp_ctx *comp_ctx,
 588					     struct efa_com_admin_queue *aq)
 589{
 590	if (test_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state))
 591		return efa_com_wait_and_process_admin_cq_polling(comp_ctx, aq);
 592
 593	return efa_com_wait_and_process_admin_cq_interrupts(comp_ctx, aq);
 594}
 595
 596/**
 597 * efa_com_cmd_exec - Execute admin command
 598 * @aq: admin queue.
 599 * @cmd: the admin command to execute.
 600 * @cmd_size: the command size.
 601 * @comp: command completion return entry.
 602 * @comp_size: command completion size.
 603 * Submit an admin command and then wait until the device will return a
 604 * completion.
 605 * The completion will be copied into comp.
 606 *
 607 * @return - 0 on success, negative value on failure.
 608 */
 609int efa_com_cmd_exec(struct efa_com_admin_queue *aq,
 610		     struct efa_admin_aq_entry *cmd,
 611		     size_t cmd_size,
 612		     struct efa_admin_acq_entry *comp,
 613		     size_t comp_size)
 614{
 615	struct efa_comp_ctx *comp_ctx;
 616	int err;
 617
 618	might_sleep();
 619
 620	/* In case of queue FULL */
 621	down(&aq->avail_cmds);
 622
 623	ibdev_dbg(aq->efa_dev, "%s (opcode %d)\n",
 624		  efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
 625		  cmd->aq_common_descriptor.opcode);
 626	comp_ctx = efa_com_submit_admin_cmd(aq, cmd, cmd_size, comp, comp_size);
 627	if (IS_ERR(comp_ctx)) {
 628		ibdev_err_ratelimited(
 629			aq->efa_dev,
 630			"Failed to submit command %s (opcode %u) err %ld\n",
 631			efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
 632			cmd->aq_common_descriptor.opcode, PTR_ERR(comp_ctx));
 633
 634		up(&aq->avail_cmds);
 635		atomic64_inc(&aq->stats.cmd_err);
 636		return PTR_ERR(comp_ctx);
 637	}
 638
 639	err = efa_com_wait_and_process_admin_cq(comp_ctx, aq);
 640	if (err) {
 641		ibdev_err_ratelimited(
 642			aq->efa_dev,
 643			"Failed to process command %s (opcode %u) comp_status %d err %d\n",
 644			efa_com_cmd_str(cmd->aq_common_descriptor.opcode),
 645			cmd->aq_common_descriptor.opcode,
 646			comp_ctx->user_cqe->acq_common_descriptor.status, err);
 647		atomic64_inc(&aq->stats.cmd_err);
 648	}
 649
 650	up(&aq->avail_cmds);
 651
 652	return err;
 653}
 654
 655/**
 656 * efa_com_admin_destroy - Destroy the admin and the async events queues.
 657 * @edev: EFA communication layer struct
 658 */
 659void efa_com_admin_destroy(struct efa_com_dev *edev)
 660{
 661	struct efa_com_admin_queue *aq = &edev->aq;
 662	struct efa_com_aenq *aenq = &edev->aenq;
 663	struct efa_com_admin_cq *cq = &aq->cq;
 664	struct efa_com_admin_sq *sq = &aq->sq;
 665	u16 size;
 666
 667	clear_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
 668
 669	devm_kfree(edev->dmadev, aq->comp_ctx_pool);
 670	devm_kfree(edev->dmadev, aq->comp_ctx);
 671
 672	size = aq->depth * sizeof(*sq->entries);
 673	dma_free_coherent(edev->dmadev, size, sq->entries, sq->dma_addr);
 674
 675	size = aq->depth * sizeof(*cq->entries);
 676	dma_free_coherent(edev->dmadev, size, cq->entries, cq->dma_addr);
 677
 678	size = aenq->depth * sizeof(*aenq->entries);
 679	dma_free_coherent(edev->dmadev, size, aenq->entries, aenq->dma_addr);
 680}
 681
 682/**
 683 * efa_com_set_admin_polling_mode - Set the admin completion queue polling mode
 684 * @edev: EFA communication layer struct
 685 * @polling: Enable/Disable polling mode
 686 *
 687 * Set the admin completion mode.
 688 */
 689void efa_com_set_admin_polling_mode(struct efa_com_dev *edev, bool polling)
 690{
 691	u32 mask_value = 0;
 692
 693	if (polling)
 694		EFA_SET(&mask_value, EFA_REGS_INTR_MASK_EN, 1);
 695
 696	writel(mask_value, edev->reg_bar + EFA_REGS_INTR_MASK_OFF);
 697	if (polling)
 698		set_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);
 699	else
 700		clear_bit(EFA_AQ_STATE_POLLING_BIT, &edev->aq.state);
 701}
 702
 703static void efa_com_stats_init(struct efa_com_dev *edev)
 704{
 705	atomic64_t *s = (atomic64_t *)&edev->aq.stats;
 706	int i;
 707
 708	for (i = 0; i < sizeof(edev->aq.stats) / sizeof(*s); i++, s++)
 709		atomic64_set(s, 0);
 710}
 711
 712/**
 713 * efa_com_admin_init - Init the admin and the async queues
 714 * @edev: EFA communication layer struct
 715 * @aenq_handlers: Those handlers to be called upon event.
 716 *
 717 * Initialize the admin submission and completion queues.
 718 * Initialize the asynchronous events notification queues.
 719 *
 720 * @return - 0 on success, negative value on failure.
 721 */
 722int efa_com_admin_init(struct efa_com_dev *edev,
 723		       struct efa_aenq_handlers *aenq_handlers)
 724{
 725	struct efa_com_admin_queue *aq = &edev->aq;
 726	u32 timeout;
 727	u32 dev_sts;
 728	u32 cap;
 729	int err;
 730
 731	dev_sts = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
 732	if (!EFA_GET(&dev_sts, EFA_REGS_DEV_STS_READY)) {
 733		ibdev_err(edev->efa_dev,
 734			  "Device isn't ready, abort com init %#x\n", dev_sts);
 735		return -ENODEV;
 736	}
 737
 738	aq->depth = EFA_ADMIN_QUEUE_DEPTH;
 739
 740	aq->dmadev = edev->dmadev;
 741	aq->efa_dev = edev->efa_dev;
 742	set_bit(EFA_AQ_STATE_POLLING_BIT, &aq->state);
 743
 744	sema_init(&aq->avail_cmds, aq->depth);
 745
 746	efa_com_stats_init(edev);
 747
 748	err = efa_com_init_comp_ctxt(aq);
 749	if (err)
 750		return err;
 751
 752	err = efa_com_admin_init_sq(edev);
 753	if (err)
 754		goto err_destroy_comp_ctxt;
 755
 756	err = efa_com_admin_init_cq(edev);
 757	if (err)
 758		goto err_destroy_sq;
 759
 760	efa_com_set_admin_polling_mode(edev, false);
 761
 762	err = efa_com_admin_init_aenq(edev, aenq_handlers);
 763	if (err)
 764		goto err_destroy_cq;
 765
 766	cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
 767	timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO);
 768	if (timeout)
 769		/* the resolution of timeout reg is 100ms */
 770		aq->completion_timeout = timeout * 100000;
 771	else
 772		aq->completion_timeout = ADMIN_CMD_TIMEOUT_US;
 773
 774	aq->poll_interval = EFA_POLL_INTERVAL_MS;
 775
 776	set_bit(EFA_AQ_STATE_RUNNING_BIT, &aq->state);
 777
 778	return 0;
 779
 780err_destroy_cq:
 781	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->cq.entries),
 782			  aq->cq.entries, aq->cq.dma_addr);
 783err_destroy_sq:
 784	dma_free_coherent(edev->dmadev, aq->depth * sizeof(*aq->sq.entries),
 785			  aq->sq.entries, aq->sq.dma_addr);
 786err_destroy_comp_ctxt:
 787	devm_kfree(edev->dmadev, aq->comp_ctx);
 788
 789	return err;
 790}
 791
 792/**
 793 * efa_com_admin_q_comp_intr_handler - admin queue interrupt handler
 794 * @edev: EFA communication layer struct
 795 *
 796 * This method goes over the admin completion queue and wakes up
 797 * all the pending threads that wait on the commands wait event.
 798 *
 799 * Note: Should be called after MSI-X interrupt.
 800 */
 801void efa_com_admin_q_comp_intr_handler(struct efa_com_dev *edev)
 802{
 803	unsigned long flags;
 804
 805	spin_lock_irqsave(&edev->aq.cq.lock, flags);
 806	efa_com_handle_admin_completion(&edev->aq);
 807	spin_unlock_irqrestore(&edev->aq.cq.lock, flags);
 808}
 809
 810/*
 811 * efa_handle_specific_aenq_event:
 812 * return the handler that is relevant to the specific event group
 813 */
 814static efa_aenq_handler efa_com_get_specific_aenq_cb(struct efa_com_dev *edev,
 815						     u16 group)
 816{
 817	struct efa_aenq_handlers *aenq_handlers = edev->aenq.aenq_handlers;
 818
 819	if (group < EFA_MAX_HANDLERS && aenq_handlers->handlers[group])
 820		return aenq_handlers->handlers[group];
 821
 822	return aenq_handlers->unimplemented_handler;
 823}
 824
 825/**
 826 * efa_com_aenq_intr_handler - AENQ interrupt handler
 827 * @edev: EFA communication layer struct
 828 * @data: Data of interrupt handler.
 829 *
 830 * Go over the async event notification queue and call the proper aenq handler.
 831 */
 832void efa_com_aenq_intr_handler(struct efa_com_dev *edev, void *data)
 833{
 834	struct efa_admin_aenq_common_desc *aenq_common;
 835	struct efa_com_aenq *aenq = &edev->aenq;
 836	struct efa_admin_aenq_entry *aenq_e;
 837	efa_aenq_handler handler_cb;
 838	u32 processed = 0;
 839	u8 phase;
 840	u32 ci;
 841
 842	ci = aenq->cc & (aenq->depth - 1);
 843	phase = aenq->phase;
 844	aenq_e = &aenq->entries[ci]; /* Get first entry */
 845	aenq_common = &aenq_e->aenq_common_desc;
 846
 847	/* Go over all the events */
 848	while ((READ_ONCE(aenq_common->flags) &
 849		EFA_ADMIN_AENQ_COMMON_DESC_PHASE_MASK) == phase) {
 850		/*
 851		 * Do not read the rest of the completion entry before the
 852		 * phase bit was validated
 853		 */
 854		dma_rmb();
 855
 856		/* Handle specific event*/
 857		handler_cb = efa_com_get_specific_aenq_cb(edev,
 858							  aenq_common->group);
 859		handler_cb(data, aenq_e); /* call the actual event handler*/
 860
 861		/* Get next event entry */
 862		ci++;
 863		processed++;
 864
 865		if (ci == aenq->depth) {
 866			ci = 0;
 867			phase = !phase;
 868		}
 869		aenq_e = &aenq->entries[ci];
 870		aenq_common = &aenq_e->aenq_common_desc;
 871	}
 872
 873	aenq->cc += processed;
 874	aenq->phase = phase;
 875
 876	/* Don't update aenq doorbell if there weren't any processed events */
 877	if (!processed)
 878		return;
 879
 880	/* barrier not needed in case of writel */
 881	writel(aenq->cc, edev->reg_bar + EFA_REGS_AENQ_CONS_DB_OFF);
 882}
 883
 884static void efa_com_mmio_reg_read_resp_addr_init(struct efa_com_dev *edev)
 885{
 886	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
 887	u32 addr_high;
 888	u32 addr_low;
 889
 890	/* dma_addr_bits is unknown at this point */
 891	addr_high = (mmio_read->read_resp_dma_addr >> 32) & GENMASK(31, 0);
 892	addr_low = mmio_read->read_resp_dma_addr & GENMASK(31, 0);
 893
 894	writel(addr_high, edev->reg_bar + EFA_REGS_MMIO_RESP_HI_OFF);
 895	writel(addr_low, edev->reg_bar + EFA_REGS_MMIO_RESP_LO_OFF);
 896}
 897
 898int efa_com_mmio_reg_read_init(struct efa_com_dev *edev)
 899{
 900	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
 901
 902	spin_lock_init(&mmio_read->lock);
 903	mmio_read->read_resp =
 904		dma_alloc_coherent(edev->dmadev, sizeof(*mmio_read->read_resp),
 905				   &mmio_read->read_resp_dma_addr, GFP_KERNEL);
 906	if (!mmio_read->read_resp)
 907		return -ENOMEM;
 908
 909	efa_com_mmio_reg_read_resp_addr_init(edev);
 910
 911	mmio_read->read_resp->req_id = 0;
 912	mmio_read->seq_num = 0;
 913	mmio_read->mmio_read_timeout = EFA_REG_READ_TIMEOUT_US;
 914
 915	return 0;
 916}
 917
 918void efa_com_mmio_reg_read_destroy(struct efa_com_dev *edev)
 919{
 920	struct efa_com_mmio_read *mmio_read = &edev->mmio_read;
 921
 922	dma_free_coherent(edev->dmadev, sizeof(*mmio_read->read_resp),
 923			  mmio_read->read_resp, mmio_read->read_resp_dma_addr);
 924}
 925
 926int efa_com_validate_version(struct efa_com_dev *edev)
 927{
 928	u32 min_ctrl_ver = 0;
 929	u32 ctrl_ver_masked;
 930	u32 min_ver = 0;
 931	u32 ctrl_ver;
 932	u32 ver;
 933
 934	/*
 935	 * Make sure the EFA version and the controller version are at least
 936	 * as the driver expects
 937	 */
 938	ver = efa_com_reg_read32(edev, EFA_REGS_VERSION_OFF);
 939	ctrl_ver = efa_com_reg_read32(edev,
 940				      EFA_REGS_CONTROLLER_VERSION_OFF);
 941
 942	ibdev_dbg(edev->efa_dev, "efa device version: %d.%d\n",
 943		  EFA_GET(&ver, EFA_REGS_VERSION_MAJOR_VERSION),
 944		  EFA_GET(&ver, EFA_REGS_VERSION_MINOR_VERSION));
 945
 946	EFA_SET(&min_ver, EFA_REGS_VERSION_MAJOR_VERSION,
 947		EFA_ADMIN_API_VERSION_MAJOR);
 948	EFA_SET(&min_ver, EFA_REGS_VERSION_MINOR_VERSION,
 949		EFA_ADMIN_API_VERSION_MINOR);
 950	if (ver < min_ver) {
 951		ibdev_err(edev->efa_dev,
 952			  "EFA version is lower than the minimal version the driver supports\n");
 953		return -EOPNOTSUPP;
 954	}
 955
 956	ibdev_dbg(
 957		edev->efa_dev,
 958		"efa controller version: %d.%d.%d implementation version %d\n",
 959		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION),
 960		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION),
 961		EFA_GET(&ctrl_ver,
 962			EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION),
 963		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_IMPL_ID));
 964
 965	ctrl_ver_masked =
 966		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION) |
 967		EFA_GET(&ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION) |
 968		EFA_GET(&ctrl_ver,
 969			EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION);
 970
 971	EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MAJOR_VERSION,
 972		EFA_CTRL_MAJOR);
 973	EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_MINOR_VERSION,
 974		EFA_CTRL_MINOR);
 975	EFA_SET(&min_ctrl_ver, EFA_REGS_CONTROLLER_VERSION_SUBMINOR_VERSION,
 976		EFA_CTRL_SUB_MINOR);
 977	/* Validate the ctrl version without the implementation ID */
 978	if (ctrl_ver_masked < min_ctrl_ver) {
 979		ibdev_err(edev->efa_dev,
 980			  "EFA ctrl version is lower than the minimal ctrl version the driver supports\n");
 981		return -EOPNOTSUPP;
 982	}
 983
 984	return 0;
 985}
 986
 987/**
 988 * efa_com_get_dma_width - Retrieve physical dma address width the device
 989 * supports.
 990 * @edev: EFA communication layer struct
 991 *
 992 * Retrieve the maximum physical address bits the device can handle.
 993 *
 994 * @return: > 0 on Success and negative value otherwise.
 995 */
 996int efa_com_get_dma_width(struct efa_com_dev *edev)
 997{
 998	u32 caps = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
 999	int width;
1000
1001	width = EFA_GET(&caps, EFA_REGS_CAPS_DMA_ADDR_WIDTH);
1002
1003	ibdev_dbg(edev->efa_dev, "DMA width: %d\n", width);
1004
1005	if (width < 32 || width > 64) {
1006		ibdev_err(edev->efa_dev, "DMA width illegal value: %d\n", width);
1007		return -EINVAL;
1008	}
1009
1010	edev->dma_addr_bits = width;
1011
1012	return width;
1013}
1014
1015static int wait_for_reset_state(struct efa_com_dev *edev, u32 timeout, int on)
1016{
1017	u32 val, i;
1018
1019	for (i = 0; i < timeout; i++) {
1020		val = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
1021
1022		if (EFA_GET(&val, EFA_REGS_DEV_STS_RESET_IN_PROGRESS) == on)
1023			return 0;
1024
1025		ibdev_dbg(edev->efa_dev, "Reset indication val %d\n", val);
1026		msleep(EFA_POLL_INTERVAL_MS);
1027	}
1028
1029	return -ETIME;
1030}
1031
1032/**
1033 * efa_com_dev_reset - Perform device FLR to the device.
1034 * @edev: EFA communication layer struct
1035 * @reset_reason: Specify what is the trigger for the reset in case of an error.
1036 *
1037 * @return - 0 on success, negative value on failure.
1038 */
1039int efa_com_dev_reset(struct efa_com_dev *edev,
1040		      enum efa_regs_reset_reason_types reset_reason)
1041{
1042	u32 stat, timeout, cap;
1043	u32 reset_val = 0;
1044	int err;
1045
1046	stat = efa_com_reg_read32(edev, EFA_REGS_DEV_STS_OFF);
1047	cap = efa_com_reg_read32(edev, EFA_REGS_CAPS_OFF);
1048
1049	if (!EFA_GET(&stat, EFA_REGS_DEV_STS_READY)) {
1050		ibdev_err(edev->efa_dev,
1051			  "Device isn't ready, can't reset device\n");
1052		return -EINVAL;
1053	}
1054
1055	timeout = EFA_GET(&cap, EFA_REGS_CAPS_RESET_TIMEOUT);
1056	if (!timeout) {
1057		ibdev_err(edev->efa_dev, "Invalid timeout value\n");
1058		return -EINVAL;
1059	}
1060
1061	/* start reset */
1062	EFA_SET(&reset_val, EFA_REGS_DEV_CTL_DEV_RESET, 1);
1063	EFA_SET(&reset_val, EFA_REGS_DEV_CTL_RESET_REASON, reset_reason);
1064	writel(reset_val, edev->reg_bar + EFA_REGS_DEV_CTL_OFF);
1065
1066	/* reset clears the mmio readless address, restore it */
1067	efa_com_mmio_reg_read_resp_addr_init(edev);
1068
1069	err = wait_for_reset_state(edev, timeout, 1);
1070	if (err) {
1071		ibdev_err(edev->efa_dev, "Reset indication didn't turn on\n");
1072		return err;
1073	}
1074
1075	/* reset done */
1076	writel(0, edev->reg_bar + EFA_REGS_DEV_CTL_OFF);
1077	err = wait_for_reset_state(edev, timeout, 0);
1078	if (err) {
1079		ibdev_err(edev->efa_dev, "Reset indication didn't turn off\n");
1080		return err;
1081	}
1082
1083	timeout = EFA_GET(&cap, EFA_REGS_CAPS_ADMIN_CMD_TO);
1084	if (timeout)
1085		/* the resolution of timeout reg is 100ms */
1086		edev->aq.completion_timeout = timeout * 100000;
1087	else
1088		edev->aq.completion_timeout = ADMIN_CMD_TIMEOUT_US;
1089
1090	return 0;
1091}
1092
1093static int efa_com_create_eq(struct efa_com_dev *edev,
1094			     struct efa_com_create_eq_params *params,
1095			     struct efa_com_create_eq_result *result)
1096{
1097	struct efa_com_admin_queue *aq = &edev->aq;
1098	struct efa_admin_create_eq_resp resp = {};
1099	struct efa_admin_create_eq_cmd cmd = {};
1100	int err;
1101
1102	cmd.aq_common_descriptor.opcode = EFA_ADMIN_CREATE_EQ;
1103	EFA_SET(&cmd.caps, EFA_ADMIN_CREATE_EQ_CMD_ENTRY_SIZE_WORDS,
1104		params->entry_size_in_bytes / 4);
1105	cmd.depth = params->depth;
1106	cmd.event_bitmask = params->event_bitmask;
1107	cmd.msix_vec = params->msix_vec;
1108
1109	efa_com_set_dma_addr(params->dma_addr, &cmd.ba.mem_addr_high,
1110			     &cmd.ba.mem_addr_low);
1111
1112	err = efa_com_cmd_exec(aq,
1113			       (struct efa_admin_aq_entry *)&cmd,
1114			       sizeof(cmd),
1115			       (struct efa_admin_acq_entry *)&resp,
1116			       sizeof(resp));
1117	if (err) {
1118		ibdev_err_ratelimited(edev->efa_dev,
1119				      "Failed to create eq[%d]\n", err);
1120		return err;
1121	}
1122
1123	result->eqn = resp.eqn;
1124
1125	return 0;
1126}
1127
1128static void efa_com_destroy_eq(struct efa_com_dev *edev,
1129			       struct efa_com_destroy_eq_params *params)
1130{
1131	struct efa_com_admin_queue *aq = &edev->aq;
1132	struct efa_admin_destroy_eq_resp resp = {};
1133	struct efa_admin_destroy_eq_cmd cmd = {};
1134	int err;
1135
1136	cmd.aq_common_descriptor.opcode = EFA_ADMIN_DESTROY_EQ;
1137	cmd.eqn = params->eqn;
1138
1139	err = efa_com_cmd_exec(aq,
1140			       (struct efa_admin_aq_entry *)&cmd,
1141			       sizeof(cmd),
1142			       (struct efa_admin_acq_entry *)&resp,
1143			       sizeof(resp));
1144	if (err)
1145		ibdev_err_ratelimited(edev->efa_dev,
1146				      "Failed to destroy EQ-%u [%d]\n", cmd.eqn,
1147				      err);
1148}
1149
1150static void efa_com_arm_eq(struct efa_com_dev *edev, struct efa_com_eq *eeq)
1151{
1152	u32 val = 0;
1153
1154	EFA_SET(&val, EFA_REGS_EQ_DB_EQN, eeq->eqn);
1155	EFA_SET(&val, EFA_REGS_EQ_DB_ARM, 1);
1156
1157	writel(val, edev->reg_bar + EFA_REGS_EQ_DB_OFF);
1158}
1159
1160void efa_com_eq_comp_intr_handler(struct efa_com_dev *edev,
1161				  struct efa_com_eq *eeq)
1162{
1163	struct efa_admin_eqe *eqe;
1164	u32 processed = 0;
1165	u8 phase;
1166	u32 ci;
1167
1168	ci = eeq->cc & (eeq->depth - 1);
1169	phase = eeq->phase;
1170	eqe = &eeq->eqes[ci];
1171
1172	/* Go over all the events */
1173	while ((READ_ONCE(eqe->common) & EFA_ADMIN_EQE_PHASE_MASK) == phase) {
1174		/*
1175		 * Do not read the rest of the completion entry before the
1176		 * phase bit was validated
1177		 */
1178		dma_rmb();
1179
1180		eeq->cb(eeq, eqe);
1181
1182		/* Get next event entry */
1183		ci++;
1184		processed++;
1185
1186		if (ci == eeq->depth) {
1187			ci = 0;
1188			phase = !phase;
1189		}
1190
1191		eqe = &eeq->eqes[ci];
1192	}
1193
1194	eeq->cc += processed;
1195	eeq->phase = phase;
1196	efa_com_arm_eq(eeq->edev, eeq);
1197}
1198
1199void efa_com_eq_destroy(struct efa_com_dev *edev, struct efa_com_eq *eeq)
1200{
1201	struct efa_com_destroy_eq_params params = {
1202		.eqn = eeq->eqn,
1203	};
1204
1205	efa_com_destroy_eq(edev, &params);
1206	dma_free_coherent(edev->dmadev, eeq->depth * sizeof(*eeq->eqes),
1207			  eeq->eqes, eeq->dma_addr);
1208}
1209
1210int efa_com_eq_init(struct efa_com_dev *edev, struct efa_com_eq *eeq,
1211		    efa_eqe_handler cb, u16 depth, u8 msix_vec)
1212{
1213	struct efa_com_create_eq_params params = {};
1214	struct efa_com_create_eq_result result = {};
1215	int err;
1216
1217	params.depth = depth;
1218	params.entry_size_in_bytes = sizeof(*eeq->eqes);
1219	EFA_SET(&params.event_bitmask,
1220		EFA_ADMIN_CREATE_EQ_CMD_COMPLETION_EVENTS, 1);
1221	params.msix_vec = msix_vec;
1222
1223	eeq->eqes = dma_alloc_coherent(edev->dmadev,
1224				       params.depth * sizeof(*eeq->eqes),
1225				       &params.dma_addr, GFP_KERNEL);
1226	if (!eeq->eqes)
1227		return -ENOMEM;
1228
1229	err = efa_com_create_eq(edev, &params, &result);
1230	if (err)
1231		goto err_free_coherent;
1232
1233	eeq->eqn = result.eqn;
1234	eeq->edev = edev;
1235	eeq->dma_addr = params.dma_addr;
1236	eeq->phase = 1;
1237	eeq->depth = params.depth;
1238	eeq->cb = cb;
1239	efa_com_arm_eq(edev, eeq);
1240
1241	return 0;
1242
1243err_free_coherent:
1244	dma_free_coherent(edev->dmadev, params.depth * sizeof(*eeq->eqes),
1245			  eeq->eqes, params.dma_addr);
1246	return err;
1247}