1// SPDX-License-Identifier: ISC
   2/*
   3 * Copyright (c) 2005-2011 Atheros Communications Inc.
   4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
   5 * Copyright (c) 2018 The Linux Foundation. All rights reserved.
   6 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
   7 */
   8
   9#include "hif.h"
  10#include "ce.h"
  11#include "debug.h"
  12
  13/*
  14 * Support for Copy Engine hardware, which is mainly used for
  15 * communication between Host and Target over a PCIe interconnect.
  16 */
  17
  18/*
  19 * A single CopyEngine (CE) comprises two "rings":
  20 *   a source ring
  21 *   a destination ring
  22 *
  23 * Each ring consists of a number of descriptors which specify
  24 * an address, length, and meta-data.
  25 *
  26 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
  27 * controls one ring and the other side controls the other ring.
  28 * The source side chooses when to initiate a transfer and it
  29 * chooses what to send (buffer address, length). The destination
  30 * side keeps a supply of "anonymous receive buffers" available and
  31 * it handles incoming data as it arrives (when the destination
  32 * receives an interrupt).
  33 *
  34 * The sender may send a simple buffer (address/length) or it may
  35 * send a small list of buffers.  When a small list is sent, hardware
  36 * "gathers" these and they end up in a single destination buffer
  37 * with a single interrupt.
  38 *
  39 * There are several "contexts" managed by this layer -- more, it
  40 * may seem -- than should be needed. These are provided mainly for
  41 * maximum flexibility and especially to facilitate a simpler HIF
  42 * implementation. There are per-CopyEngine recv, send, and watermark
  43 * contexts. These are supplied by the caller when a recv, send,
  44 * or watermark handler is established and they are echoed back to
  45 * the caller when the respective callbacks are invoked. There is
  46 * also a per-transfer context supplied by the caller when a buffer
  47 * (or sendlist) is sent and when a buffer is enqueued for recv.
  48 * These per-transfer contexts are echoed back to the caller when
  49 * the buffer is sent/received.
  50 */
  51
  52static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
  53					struct ath10k_ce_pipe *ce_state)
  54{
  55	u32 ce_id = ce_state->id;
  56	u32 addr = 0;
  57
  58	switch (ce_id) {
  59	case 0:
  60		addr = 0x00032000;
  61		break;
  62	case 3:
  63		addr = 0x0003200C;
  64		break;
  65	case 4:
  66		addr = 0x00032010;
  67		break;
  68	case 5:
  69		addr = 0x00032014;
  70		break;
  71	case 7:
  72		addr = 0x0003201C;
  73		break;
  74	default:
  75		ath10k_warn(ar, "invalid CE id: %d", ce_id);
  76		break;
  77	}
  78	return addr;
  79}
  80
  81static inline unsigned int
  82ath10k_set_ring_byte(unsigned int offset,
  83		     struct ath10k_hw_ce_regs_addr_map *addr_map)
  84{
  85	return ((offset << addr_map->lsb) & addr_map->mask);
  86}
  87
  88static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
  89{
  90	struct ath10k_ce *ce = ath10k_ce_priv(ar);
  91
  92	return ce->bus_ops->read32(ar, offset);
  93}
  94
  95static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
  96{
  97	struct ath10k_ce *ce = ath10k_ce_priv(ar);
  98
  99	ce->bus_ops->write32(ar, offset, value);
 100}
 101
 102static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
 103						       u32 ce_ctrl_addr,
 104						       unsigned int n)
 105{
 106	ath10k_ce_write32(ar, ce_ctrl_addr +
 107			  ar->hw_ce_regs->dst_wr_index_addr, n);
 108}
 109
 110static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
 111						      u32 ce_ctrl_addr)
 112{
 113	return ath10k_ce_read32(ar, ce_ctrl_addr +
 114				ar->hw_ce_regs->dst_wr_index_addr);
 115}
 116
 117static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
 118						      u32 ce_ctrl_addr,
 119						      unsigned int n)
 120{
 121	ath10k_ce_write32(ar, ce_ctrl_addr +
 122			  ar->hw_ce_regs->sr_wr_index_addr, n);
 123}
 124
 125static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
 126						     u32 ce_ctrl_addr)
 127{
 128	return ath10k_ce_read32(ar, ce_ctrl_addr +
 129				ar->hw_ce_regs->sr_wr_index_addr);
 130}
 131
 132static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
 133							 u32 ce_id)
 134{
 135	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 136
 137	return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
 138}
 139
 140static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
 141						    u32 ce_ctrl_addr)
 142{
 143	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 144	u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 145	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 146	u32 index;
 147
 148	if (ar->hw_params.rri_on_ddr &&
 149	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 150		index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
 151	else
 152		index = ath10k_ce_read32(ar, ce_ctrl_addr +
 153					 ar->hw_ce_regs->current_srri_addr);
 154
 155	return index;
 156}
 157
 158static inline void
 159ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
 160					  struct ath10k_ce_pipe *ce_state,
 161					  unsigned int value)
 162{
 163	ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
 164}
 165
 166static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
 167						    u32 ce_id,
 168						    u64 addr)
 169{
 170	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 171	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 172	u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 173	u32 addr_lo = lower_32_bits(addr);
 174
 175	ath10k_ce_write32(ar, ce_ctrl_addr +
 176			  ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
 177
 178	if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
 179		ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
 180							    addr);
 181	}
 182}
 183
 184static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
 185						u32 ce_ctrl_addr,
 186						u64 addr)
 187{
 188	u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 189
 190	ath10k_ce_write32(ar, ce_ctrl_addr +
 191			  ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
 192}
 193
 194static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
 195					       u32 ce_ctrl_addr,
 196					       unsigned int n)
 197{
 198	ath10k_ce_write32(ar, ce_ctrl_addr +
 199			  ar->hw_ce_regs->sr_size_addr, n);
 200}
 201
 202static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
 203					       u32 ce_ctrl_addr,
 204					       unsigned int n)
 205{
 206	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 207
 208	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 209					  ctrl_regs->addr);
 210
 211	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 212			  (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
 213			  ath10k_set_ring_byte(n, ctrl_regs->dmax));
 214}
 215
 216static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
 217						    u32 ce_ctrl_addr,
 218						    unsigned int n)
 219{
 220	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 221
 222	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 223					  ctrl_regs->addr);
 224
 225	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 226			  (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
 227			  ath10k_set_ring_byte(n, ctrl_regs->src_ring));
 228}
 229
 230static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
 231						     u32 ce_ctrl_addr,
 232						     unsigned int n)
 233{
 234	struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 235
 236	u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 237					  ctrl_regs->addr);
 238
 239	ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 240			  (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
 241			  ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
 242}
 243
 244static inline
 245	u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
 246{
 247	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 248
 249	return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
 250		CE_DDR_RRI_MASK;
 251}
 252
 253static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
 254						     u32 ce_ctrl_addr)
 255{
 256	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 257	u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 258	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 259	u32 index;
 260
 261	if (ar->hw_params.rri_on_ddr &&
 262	    (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 263		index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
 264	else
 265		index = ath10k_ce_read32(ar, ce_ctrl_addr +
 266					 ar->hw_ce_regs->current_drri_addr);
 267
 268	return index;
 269}
 270
 271static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
 272						     u32 ce_id,
 273						     u64 addr)
 274{
 275	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 276	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 277	u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 278	u32 addr_lo = lower_32_bits(addr);
 279
 280	ath10k_ce_write32(ar, ce_ctrl_addr +
 281			  ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
 282
 283	if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
 284		ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
 285							     addr);
 286	}
 287}
 288
 289static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
 290						 u32 ce_ctrl_addr,
 291						 u64 addr)
 292{
 293	u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 294	u32 reg_value;
 295
 296	reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
 297				     ar->hw_ce_regs->dr_base_addr_hi);
 298	reg_value &= ~CE_DESC_ADDR_HI_MASK;
 299	reg_value |= addr_hi;
 300	ath10k_ce_write32(ar, ce_ctrl_addr +
 301			  ar->hw_ce_regs->dr_base_addr_hi, reg_value);
 302}
 303
 304static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
 305						u32 ce_ctrl_addr,
 306						unsigned int n)
 307{
 308	ath10k_ce_write32(ar, ce_ctrl_addr +
 309			  ar->hw_ce_regs->dr_size_addr, n);
 310}
 311
 312static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
 313						   u32 ce_ctrl_addr,
 314						   unsigned int n)
 315{
 316	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 317	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 318
 319	ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 320			  (addr & ~(srcr_wm->wm_high->mask)) |
 321			  (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
 322}
 323
 324static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
 325						  u32 ce_ctrl_addr,
 326						  unsigned int n)
 327{
 328	struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 329	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 330
 331	ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 332			  (addr & ~(srcr_wm->wm_low->mask)) |
 333			  (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
 334}
 335
 336static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
 337						    u32 ce_ctrl_addr,
 338						    unsigned int n)
 339{
 340	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 341	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 342
 343	ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 344			  (addr & ~(dstr_wm->wm_high->mask)) |
 345			  (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
 346}
 347
 348static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
 349						   u32 ce_ctrl_addr,
 350						   unsigned int n)
 351{
 352	struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 353	u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 354
 355	ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 356			  (addr & ~(dstr_wm->wm_low->mask)) |
 357			  (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
 358}
 359
 360static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
 361							u32 ce_ctrl_addr)
 362{
 363	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 364
 365	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 366					    ar->hw_ce_regs->host_ie_addr);
 367
 368	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 369			  host_ie_addr | host_ie->copy_complete->mask);
 370}
 371
 372static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
 373							u32 ce_ctrl_addr)
 374{
 375	struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 376
 377	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 378					    ar->hw_ce_regs->host_ie_addr);
 379
 380	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 381			  host_ie_addr & ~(host_ie->copy_complete->mask));
 382}
 383
 384static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
 385						    u32 ce_ctrl_addr)
 386{
 387	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 388
 389	u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 390					    ar->hw_ce_regs->host_ie_addr);
 391
 392	ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 393			  host_ie_addr & ~(wm_regs->wm_mask));
 394}
 395
 396static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
 397						u32 ce_ctrl_addr)
 398{
 399	struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
 400
 401	u32 misc_ie_addr = ath10k_ce_read32(ar,
 402			ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
 403
 404	ath10k_ce_write32(ar,
 405			  ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
 406			  misc_ie_addr & ~(misc_regs->err_mask));
 407}
 408
 409static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
 410						     u32 ce_ctrl_addr,
 411						     unsigned int mask)
 412{
 413	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 414
 415	ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
 416}
 417
 418/*
 419 * Guts of ath10k_ce_send.
 420 * The caller takes responsibility for any needed locking.
 421 */
 422static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 423				  void *per_transfer_context,
 424				  dma_addr_t buffer,
 425				  unsigned int nbytes,
 426				  unsigned int transfer_id,
 427				  unsigned int flags)
 428{
 429	struct ath10k *ar = ce_state->ar;
 430	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 431	struct ce_desc *desc, sdesc;
 432	unsigned int nentries_mask = src_ring->nentries_mask;
 433	unsigned int sw_index = src_ring->sw_index;
 434	unsigned int write_index = src_ring->write_index;
 435	u32 ctrl_addr = ce_state->ctrl_addr;
 436	u32 desc_flags = 0;
 437	int ret = 0;
 438
 439	if (nbytes > ce_state->src_sz_max)
 440		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 441			    __func__, nbytes, ce_state->src_sz_max);
 442
 443	if (unlikely(CE_RING_DELTA(nentries_mask,
 444				   write_index, sw_index - 1) <= 0)) {
 445		ret = -ENOSR;
 446		goto exit;
 447	}
 448
 449	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
 450				   write_index);
 451
 452	desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 453
 454	if (flags & CE_SEND_FLAG_GATHER)
 455		desc_flags |= CE_DESC_FLAGS_GATHER;
 456	if (flags & CE_SEND_FLAG_BYTE_SWAP)
 457		desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 458
 459	sdesc.addr   = __cpu_to_le32(buffer);
 460	sdesc.nbytes = __cpu_to_le16(nbytes);
 461	sdesc.flags  = __cpu_to_le16(desc_flags);
 462
 463	*desc = sdesc;
 464
 465	src_ring->per_transfer_context[write_index] = per_transfer_context;
 466
 467	/* Update Source Ring Write Index */
 468	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 469
 470	/* WORKAROUND */
 471	if (!(flags & CE_SEND_FLAG_GATHER))
 472		ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
 473
 474	src_ring->write_index = write_index;
 475exit:
 476	return ret;
 477}
 478
 479static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
 480				     void *per_transfer_context,
 481				     dma_addr_t buffer,
 482				     unsigned int nbytes,
 483				     unsigned int transfer_id,
 484				     unsigned int flags)
 485{
 486	struct ath10k *ar = ce_state->ar;
 487	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 488	struct ce_desc_64 *desc, sdesc;
 489	unsigned int nentries_mask = src_ring->nentries_mask;
 490	unsigned int sw_index;
 491	unsigned int write_index = src_ring->write_index;
 492	u32 ctrl_addr = ce_state->ctrl_addr;
 493	__le32 *addr;
 494	u32 desc_flags = 0;
 495	int ret = 0;
 496
 497	if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
 498		return -ESHUTDOWN;
 499
 500	if (nbytes > ce_state->src_sz_max)
 501		ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 502			    __func__, nbytes, ce_state->src_sz_max);
 503
 504	if (ar->hw_params.rri_on_ddr)
 505		sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
 506	else
 507		sw_index = src_ring->sw_index;
 508
 509	if (unlikely(CE_RING_DELTA(nentries_mask,
 510				   write_index, sw_index - 1) <= 0)) {
 511		ret = -ENOSR;
 512		goto exit;
 513	}
 514
 515	desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
 516				      write_index);
 517
 518	desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 519
 520	if (flags & CE_SEND_FLAG_GATHER)
 521		desc_flags |= CE_DESC_FLAGS_GATHER;
 522
 523	if (flags & CE_SEND_FLAG_BYTE_SWAP)
 524		desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 525
 526	addr = (__le32 *)&sdesc.addr;
 527
 528	flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
 529	addr[0] = __cpu_to_le32(buffer);
 530	addr[1] = __cpu_to_le32(flags);
 531	if (flags & CE_SEND_FLAG_GATHER)
 532		addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
 533	else
 534		addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
 535
 536	sdesc.nbytes = __cpu_to_le16(nbytes);
 537	sdesc.flags  = __cpu_to_le16(desc_flags);
 538
 539	*desc = sdesc;
 540
 541	src_ring->per_transfer_context[write_index] = per_transfer_context;
 542
 543	/* Update Source Ring Write Index */
 544	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 545
 546	if (!(flags & CE_SEND_FLAG_GATHER)) {
 547		if (ar->hw_params.shadow_reg_support)
 548			ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
 549								  write_index);
 550		else
 551			ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
 552							   write_index);
 553	}
 554
 555	src_ring->write_index = write_index;
 556exit:
 557	return ret;
 558}
 559
 560int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 561			  void *per_transfer_context,
 562			  dma_addr_t buffer,
 563			  unsigned int nbytes,
 564			  unsigned int transfer_id,
 565			  unsigned int flags)
 566{
 567	return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
 568				    buffer, nbytes, transfer_id, flags);
 569}
 570EXPORT_SYMBOL(ath10k_ce_send_nolock);
 571
 572void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 573{
 574	struct ath10k *ar = pipe->ar;
 575	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 576	struct ath10k_ce_ring *src_ring = pipe->src_ring;
 577	u32 ctrl_addr = pipe->ctrl_addr;
 578
 579	lockdep_assert_held(&ce->ce_lock);
 580
 581	/*
 582	 * This function must be called only if there is an incomplete
 583	 * scatter-gather transfer (before index register is updated)
 584	 * that needs to be cleaned up.
 585	 */
 586	if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
 587		return;
 588
 589	if (WARN_ON_ONCE(src_ring->write_index ==
 590			 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
 591		return;
 592
 593	src_ring->write_index--;
 594	src_ring->write_index &= src_ring->nentries_mask;
 595
 596	src_ring->per_transfer_context[src_ring->write_index] = NULL;
 597}
 598EXPORT_SYMBOL(__ath10k_ce_send_revert);
 599
 600int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 601		   void *per_transfer_context,
 602		   dma_addr_t buffer,
 603		   unsigned int nbytes,
 604		   unsigned int transfer_id,
 605		   unsigned int flags)
 606{
 607	struct ath10k *ar = ce_state->ar;
 608	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 609	int ret;
 610
 611	spin_lock_bh(&ce->ce_lock);
 612	ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
 613				    buffer, nbytes, transfer_id, flags);
 614	spin_unlock_bh(&ce->ce_lock);
 615
 616	return ret;
 617}
 618EXPORT_SYMBOL(ath10k_ce_send);
 619
 620int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 621{
 622	struct ath10k *ar = pipe->ar;
 623	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 624	int delta;
 625
 626	spin_lock_bh(&ce->ce_lock);
 627	delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
 628			      pipe->src_ring->write_index,
 629			      pipe->src_ring->sw_index - 1);
 630	spin_unlock_bh(&ce->ce_lock);
 631
 632	return delta;
 633}
 634EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
 635
 636int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 637{
 638	struct ath10k *ar = pipe->ar;
 639	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 640	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 641	unsigned int nentries_mask = dest_ring->nentries_mask;
 642	unsigned int write_index = dest_ring->write_index;
 643	unsigned int sw_index = dest_ring->sw_index;
 644
 645	lockdep_assert_held(&ce->ce_lock);
 646
 647	return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
 648}
 649EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
 650
 651static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 652				   dma_addr_t paddr)
 653{
 654	struct ath10k *ar = pipe->ar;
 655	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 656	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 657	unsigned int nentries_mask = dest_ring->nentries_mask;
 658	unsigned int write_index = dest_ring->write_index;
 659	unsigned int sw_index = dest_ring->sw_index;
 660	struct ce_desc *base = dest_ring->base_addr_owner_space;
 661	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
 662	u32 ctrl_addr = pipe->ctrl_addr;
 663
 664	lockdep_assert_held(&ce->ce_lock);
 665
 666	if ((pipe->id != 5) &&
 667	    CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 668		return -ENOSPC;
 669
 670	desc->addr = __cpu_to_le32(paddr);
 671	desc->nbytes = 0;
 672
 673	dest_ring->per_transfer_context[write_index] = ctx;
 674	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 675	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 676	dest_ring->write_index = write_index;
 677
 678	return 0;
 679}
 680
 681static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
 682				      void *ctx,
 683				      dma_addr_t paddr)
 684{
 685	struct ath10k *ar = pipe->ar;
 686	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 687	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 688	unsigned int nentries_mask = dest_ring->nentries_mask;
 689	unsigned int write_index = dest_ring->write_index;
 690	unsigned int sw_index = dest_ring->sw_index;
 691	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 692	struct ce_desc_64 *desc =
 693			CE_DEST_RING_TO_DESC_64(base, write_index);
 694	u32 ctrl_addr = pipe->ctrl_addr;
 695
 696	lockdep_assert_held(&ce->ce_lock);
 697
 698	if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 699		return -ENOSPC;
 700
 701	desc->addr = __cpu_to_le64(paddr);
 702	desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
 703
 704	desc->nbytes = 0;
 705
 706	dest_ring->per_transfer_context[write_index] = ctx;
 707	write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 708	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 709	dest_ring->write_index = write_index;
 710
 711	return 0;
 712}
 713
 714void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 715{
 716	struct ath10k *ar = pipe->ar;
 717	struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 718	unsigned int nentries_mask = dest_ring->nentries_mask;
 719	unsigned int write_index = dest_ring->write_index;
 720	u32 ctrl_addr = pipe->ctrl_addr;
 721	u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
 722
 723	/* Prevent CE ring stuck issue that will occur when ring is full.
 724	 * Make sure that write index is 1 less than read index.
 725	 */
 726	if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
 727		nentries -= 1;
 728
 729	write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
 730	ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 731	dest_ring->write_index = write_index;
 732}
 733EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
 734
 735int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 736			  dma_addr_t paddr)
 737{
 738	struct ath10k *ar = pipe->ar;
 739	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 740	int ret;
 741
 742	spin_lock_bh(&ce->ce_lock);
 743	ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
 744	spin_unlock_bh(&ce->ce_lock);
 745
 746	return ret;
 747}
 748EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
 749
 750/*
 751 * Guts of ath10k_ce_completed_recv_next.
 752 * The caller takes responsibility for any necessary locking.
 753 */
 754static int
 755	 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 756					       void **per_transfer_contextp,
 757					       unsigned int *nbytesp)
 758{
 759	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 760	unsigned int nentries_mask = dest_ring->nentries_mask;
 761	unsigned int sw_index = dest_ring->sw_index;
 762
 763	struct ce_desc *base = dest_ring->base_addr_owner_space;
 764	struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 765	struct ce_desc sdesc;
 766	u16 nbytes;
 767
 768	/* Copy in one go for performance reasons */
 769	sdesc = *desc;
 770
 771	nbytes = __le16_to_cpu(sdesc.nbytes);
 772	if (nbytes == 0) {
 773		/*
 774		 * This closes a relatively unusual race where the Host
 775		 * sees the updated DRRI before the update to the
 776		 * corresponding descriptor has completed. We treat this
 777		 * as a descriptor that is not yet done.
 778		 */
 779		return -EIO;
 780	}
 781
 782	desc->nbytes = 0;
 783
 784	/* Return data from completed destination descriptor */
 785	*nbytesp = nbytes;
 786
 787	if (per_transfer_contextp)
 788		*per_transfer_contextp =
 789			dest_ring->per_transfer_context[sw_index];
 790
 791	/* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 792	 * So update transfer context all CEs except CE5.
 793	 */
 794	if (ce_state->id != 5)
 795		dest_ring->per_transfer_context[sw_index] = NULL;
 796
 797	/* Update sw_index */
 798	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 799	dest_ring->sw_index = sw_index;
 800
 801	return 0;
 802}
 803
 804static int
 805_ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
 806					 void **per_transfer_contextp,
 807					 unsigned int *nbytesp)
 808{
 809	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 810	unsigned int nentries_mask = dest_ring->nentries_mask;
 811	unsigned int sw_index = dest_ring->sw_index;
 812	struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 813	struct ce_desc_64 *desc =
 814		CE_DEST_RING_TO_DESC_64(base, sw_index);
 815	struct ce_desc_64 sdesc;
 816	u16 nbytes;
 817
 818	/* Copy in one go for performance reasons */
 819	sdesc = *desc;
 820
 821	nbytes = __le16_to_cpu(sdesc.nbytes);
 822	if (nbytes == 0) {
 823		/* This closes a relatively unusual race where the Host
 824		 * sees the updated DRRI before the update to the
 825		 * corresponding descriptor has completed. We treat this
 826		 * as a descriptor that is not yet done.
 827		 */
 828		return -EIO;
 829	}
 830
 831	desc->nbytes = 0;
 832
 833	/* Return data from completed destination descriptor */
 834	*nbytesp = nbytes;
 835
 836	if (per_transfer_contextp)
 837		*per_transfer_contextp =
 838			dest_ring->per_transfer_context[sw_index];
 839
 840	/* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 841	 * So update transfer context all CEs except CE5.
 842	 */
 843	if (ce_state->id != 5)
 844		dest_ring->per_transfer_context[sw_index] = NULL;
 845
 846	/* Update sw_index */
 847	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 848	dest_ring->sw_index = sw_index;
 849
 850	return 0;
 851}
 852
 853int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 854					 void **per_transfer_ctx,
 855					 unsigned int *nbytesp)
 856{
 857	return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 858							    per_transfer_ctx,
 859							    nbytesp);
 860}
 861EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
 862
 863int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 864				  void **per_transfer_contextp,
 865				  unsigned int *nbytesp)
 866{
 867	struct ath10k *ar = ce_state->ar;
 868	struct ath10k_ce *ce = ath10k_ce_priv(ar);
 869	int ret;
 870
 871	spin_lock_bh(&ce->ce_lock);
 872	ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 873						   per_transfer_contextp,
 874						   nbytesp);
 875
 876	spin_unlock_bh(&ce->ce_lock);
 877
 878	return ret;
 879}
 880EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
 881
 882static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 883				       void **per_transfer_contextp,
 884				       dma_addr_t *bufferp)
 885{
 886	struct ath10k_ce_ring *dest_ring;
 887	unsigned int nentries_mask;
 888	unsigned int sw_index;
 889	unsigned int write_index;
 890	int ret;
 891	struct ath10k *ar;
 892	struct ath10k_ce *ce;
 893
 894	dest_ring = ce_state->dest_ring;
 895
 896	if (!dest_ring)
 897		return -EIO;
 898
 899	ar = ce_state->ar;
 900	ce = ath10k_ce_priv(ar);
 901
 902	spin_lock_bh(&ce->ce_lock);
 903
 904	nentries_mask = dest_ring->nentries_mask;
 905	sw_index = dest_ring->sw_index;
 906	write_index = dest_ring->write_index;
 907	if (write_index != sw_index) {
 908		struct ce_desc *base = dest_ring->base_addr_owner_space;
 909		struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 910
 911		/* Return data from completed destination descriptor */
 912		*bufferp = __le32_to_cpu(desc->addr);
 913
 914		if (per_transfer_contextp)
 915			*per_transfer_contextp =
 916				dest_ring->per_transfer_context[sw_index];
 917
 918		/* sanity */
 919		dest_ring->per_transfer_context[sw_index] = NULL;
 920		desc->nbytes = 0;
 921
 922		/* Update sw_index */
 923		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 924		dest_ring->sw_index = sw_index;
 925		ret = 0;
 926	} else {
 927		ret = -EIO;
 928	}
 929
 930	spin_unlock_bh(&ce->ce_lock);
 931
 932	return ret;
 933}
 934
 935static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
 936					  void **per_transfer_contextp,
 937					  dma_addr_t *bufferp)
 938{
 939	struct ath10k_ce_ring *dest_ring;
 940	unsigned int nentries_mask;
 941	unsigned int sw_index;
 942	unsigned int write_index;
 943	int ret;
 944	struct ath10k *ar;
 945	struct ath10k_ce *ce;
 946
 947	dest_ring = ce_state->dest_ring;
 948
 949	if (!dest_ring)
 950		return -EIO;
 951
 952	ar = ce_state->ar;
 953	ce = ath10k_ce_priv(ar);
 954
 955	spin_lock_bh(&ce->ce_lock);
 956
 957	nentries_mask = dest_ring->nentries_mask;
 958	sw_index = dest_ring->sw_index;
 959	write_index = dest_ring->write_index;
 960	if (write_index != sw_index) {
 961		struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 962		struct ce_desc_64 *desc =
 963			CE_DEST_RING_TO_DESC_64(base, sw_index);
 964
 965		/* Return data from completed destination descriptor */
 966		*bufferp = __le64_to_cpu(desc->addr);
 967
 968		if (per_transfer_contextp)
 969			*per_transfer_contextp =
 970				dest_ring->per_transfer_context[sw_index];
 971
 972		/* sanity */
 973		dest_ring->per_transfer_context[sw_index] = NULL;
 974		desc->nbytes = 0;
 975
 976		/* Update sw_index */
 977		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 978		dest_ring->sw_index = sw_index;
 979		ret = 0;
 980	} else {
 981		ret = -EIO;
 982	}
 983
 984	spin_unlock_bh(&ce->ce_lock);
 985
 986	return ret;
 987}
 988
 989int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 990			       void **per_transfer_contextp,
 991			       dma_addr_t *bufferp)
 992{
 993	return ce_state->ops->ce_revoke_recv_next(ce_state,
 994						  per_transfer_contextp,
 995						  bufferp);
 996}
 997EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
 998
 999/*
1000 * Guts of ath10k_ce_completed_send_next.
1001 * The caller takes responsibility for any necessary locking.
1002 */
1003static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1004						 void **per_transfer_contextp)
1005{
1006	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1007	u32 ctrl_addr = ce_state->ctrl_addr;
1008	struct ath10k *ar = ce_state->ar;
1009	unsigned int nentries_mask = src_ring->nentries_mask;
1010	unsigned int sw_index = src_ring->sw_index;
1011	unsigned int read_index;
1012	struct ce_desc *desc;
1013
1014	if (src_ring->hw_index == sw_index) {
1015		/*
1016		 * The SW completion index has caught up with the cached
1017		 * version of the HW completion index.
1018		 * Update the cached HW completion index to see whether
1019		 * the SW has really caught up to the HW, or if the cached
1020		 * value of the HW index has become stale.
1021		 */
1022
1023		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1024		if (read_index == 0xffffffff)
1025			return -ENODEV;
1026
1027		read_index &= nentries_mask;
1028		src_ring->hw_index = read_index;
1029	}
1030
1031	if (ar->hw_params.rri_on_ddr)
1032		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1033	else
1034		read_index = src_ring->hw_index;
1035
1036	if (read_index == sw_index)
1037		return -EIO;
1038
1039	if (per_transfer_contextp)
1040		*per_transfer_contextp =
1041			src_ring->per_transfer_context[sw_index];
1042
1043	/* sanity */
1044	src_ring->per_transfer_context[sw_index] = NULL;
1045	desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1046				   sw_index);
1047	desc->nbytes = 0;
1048
1049	/* Update sw_index */
1050	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1051	src_ring->sw_index = sw_index;
1052
1053	return 0;
1054}
1055
1056static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1057						    void **per_transfer_contextp)
1058{
1059	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1060	u32 ctrl_addr = ce_state->ctrl_addr;
1061	struct ath10k *ar = ce_state->ar;
1062	unsigned int nentries_mask = src_ring->nentries_mask;
1063	unsigned int sw_index = src_ring->sw_index;
1064	unsigned int read_index;
1065	struct ce_desc_64 *desc;
1066
1067	if (src_ring->hw_index == sw_index) {
1068		/*
1069		 * The SW completion index has caught up with the cached
1070		 * version of the HW completion index.
1071		 * Update the cached HW completion index to see whether
1072		 * the SW has really caught up to the HW, or if the cached
1073		 * value of the HW index has become stale.
1074		 */
1075
1076		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1077		if (read_index == 0xffffffff)
1078			return -ENODEV;
1079
1080		read_index &= nentries_mask;
1081		src_ring->hw_index = read_index;
1082	}
1083
1084	if (ar->hw_params.rri_on_ddr)
1085		read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1086	else
1087		read_index = src_ring->hw_index;
1088
1089	if (read_index == sw_index)
1090		return -EIO;
1091
1092	if (per_transfer_contextp)
1093		*per_transfer_contextp =
1094			src_ring->per_transfer_context[sw_index];
1095
1096	/* sanity */
1097	src_ring->per_transfer_context[sw_index] = NULL;
1098	desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1099				      sw_index);
1100	desc->nbytes = 0;
1101
1102	/* Update sw_index */
1103	sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1104	src_ring->sw_index = sw_index;
1105
1106	return 0;
1107}
1108
1109int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1110					 void **per_transfer_contextp)
1111{
1112	return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1113							    per_transfer_contextp);
1114}
1115EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1116
1117static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1118					struct ath10k_ce_ring *src_ring,
1119					u32 sw_index,
1120					dma_addr_t *bufferp,
1121					u32 *nbytesp,
1122					u32 *transfer_idp)
1123{
1124		struct ce_desc *base = src_ring->base_addr_owner_space;
1125		struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1126
1127		/* Return data from completed source descriptor */
1128		*bufferp = __le32_to_cpu(desc->addr);
1129		*nbytesp = __le16_to_cpu(desc->nbytes);
1130		*transfer_idp = MS(__le16_to_cpu(desc->flags),
1131				   CE_DESC_FLAGS_META_DATA);
1132}
1133
1134static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1135					   struct ath10k_ce_ring *src_ring,
1136					   u32 sw_index,
1137					   dma_addr_t *bufferp,
1138					   u32 *nbytesp,
1139					   u32 *transfer_idp)
1140{
1141		struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1142		struct ce_desc_64 *desc =
1143			CE_SRC_RING_TO_DESC_64(base, sw_index);
1144
1145		/* Return data from completed source descriptor */
1146		*bufferp = __le64_to_cpu(desc->addr);
1147		*nbytesp = __le16_to_cpu(desc->nbytes);
1148		*transfer_idp = MS(__le16_to_cpu(desc->flags),
1149				   CE_DESC_FLAGS_META_DATA);
1150}
1151
1152/* NB: Modeled after ath10k_ce_completed_send_next */
1153int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1154			       void **per_transfer_contextp,
1155			       dma_addr_t *bufferp,
1156			       unsigned int *nbytesp,
1157			       unsigned int *transfer_idp)
1158{
1159	struct ath10k_ce_ring *src_ring;
1160	unsigned int nentries_mask;
1161	unsigned int sw_index;
1162	unsigned int write_index;
1163	int ret;
1164	struct ath10k *ar;
1165	struct ath10k_ce *ce;
1166
1167	src_ring = ce_state->src_ring;
1168
1169	if (!src_ring)
1170		return -EIO;
1171
1172	ar = ce_state->ar;
1173	ce = ath10k_ce_priv(ar);
1174
1175	spin_lock_bh(&ce->ce_lock);
1176
1177	nentries_mask = src_ring->nentries_mask;
1178	sw_index = src_ring->sw_index;
1179	write_index = src_ring->write_index;
1180
1181	if (write_index != sw_index) {
1182		ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1183						    bufferp, nbytesp,
1184						    transfer_idp);
1185
1186		if (per_transfer_contextp)
1187			*per_transfer_contextp =
1188				src_ring->per_transfer_context[sw_index];
1189
1190		/* sanity */
1191		src_ring->per_transfer_context[sw_index] = NULL;
1192
1193		/* Update sw_index */
1194		sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1195		src_ring->sw_index = sw_index;
1196		ret = 0;
1197	} else {
1198		ret = -EIO;
1199	}
1200
1201	spin_unlock_bh(&ce->ce_lock);
1202
1203	return ret;
1204}
1205EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1206
1207int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1208				  void **per_transfer_contextp)
1209{
1210	struct ath10k *ar = ce_state->ar;
1211	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1212	int ret;
1213
1214	spin_lock_bh(&ce->ce_lock);
1215	ret = ath10k_ce_completed_send_next_nolock(ce_state,
1216						   per_transfer_contextp);
1217	spin_unlock_bh(&ce->ce_lock);
1218
1219	return ret;
1220}
1221EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1222
1223/*
1224 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1225 *
1226 * Invokes registered callbacks for recv_complete,
1227 * send_complete, and watermarks.
1228 */
1229void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1230{
1231	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1232	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1233	struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1234	u32 ctrl_addr = ce_state->ctrl_addr;
1235
1236	/*
1237	 * Clear before handling
1238	 *
1239	 * Misc CE interrupts are not being handled, but still need
1240	 * to be cleared.
1241	 *
1242	 * NOTE: When the last copy engine interrupt is cleared the
1243	 * hardware will go to sleep.  Once this happens any access to
1244	 * the CE registers can cause a hardware fault.
1245	 */
1246	ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1247					  wm_regs->cc_mask | wm_regs->wm_mask);
1248
1249	if (ce_state->recv_cb)
1250		ce_state->recv_cb(ce_state);
1251
1252	if (ce_state->send_cb)
1253		ce_state->send_cb(ce_state);
1254}
1255EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1256
1257/*
1258 * Handler for per-engine interrupts on ALL active CEs.
1259 * This is used in cases where the system is sharing a
1260 * single interrupt for all CEs
1261 */
1262
1263void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1264{
1265	int ce_id;
1266	u32 intr_summary;
1267
1268	intr_summary = ath10k_ce_interrupt_summary(ar);
1269
1270	for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1271		if (intr_summary & (1 << ce_id))
1272			intr_summary &= ~(1 << ce_id);
1273		else
1274			/* no intr pending on this CE */
1275			continue;
1276
1277		ath10k_ce_per_engine_service(ar, ce_id);
1278	}
1279}
1280EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1281
1282/*
1283 * Adjust interrupts for the copy complete handler.
1284 * If it's needed for either send or recv, then unmask
1285 * this interrupt; otherwise, mask it.
1286 *
1287 * Called with ce_lock held.
1288 */
1289static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1290{
1291	u32 ctrl_addr = ce_state->ctrl_addr;
1292	struct ath10k *ar = ce_state->ar;
1293	bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1294
1295	if ((!disable_copy_compl_intr) &&
1296	    (ce_state->send_cb || ce_state->recv_cb))
1297		ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1298	else
1299		ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1300
1301	ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1302}
1303
1304void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1305{
1306	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1307	struct ath10k_ce_pipe *ce_state;
1308	u32 ctrl_addr;
1309
1310	ce_state  = &ce->ce_states[ce_id];
1311	if (ce_state->attr_flags & CE_ATTR_POLL)
1312		return;
1313
1314	ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1315
1316	ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1317	ath10k_ce_error_intr_disable(ar, ctrl_addr);
1318	ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1319}
1320EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1321
1322void ath10k_ce_disable_interrupts(struct ath10k *ar)
1323{
1324	int ce_id;
1325
1326	for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1327		ath10k_ce_disable_interrupt(ar, ce_id);
1328}
1329EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1330
1331void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1332{
1333	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1334	struct ath10k_ce_pipe *ce_state;
1335
1336	ce_state  = &ce->ce_states[ce_id];
1337	if (ce_state->attr_flags & CE_ATTR_POLL)
1338		return;
1339
1340	ath10k_ce_per_engine_handler_adjust(ce_state);
1341}
1342EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1343
1344void ath10k_ce_enable_interrupts(struct ath10k *ar)
1345{
1346	int ce_id;
1347
1348	/* Enable interrupts for copy engine that
1349	 * are not using polling mode.
1350	 */
1351	for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1352		ath10k_ce_enable_interrupt(ar, ce_id);
1353}
1354EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1355
1356static int ath10k_ce_init_src_ring(struct ath10k *ar,
1357				   unsigned int ce_id,
1358				   const struct ce_attr *attr)
1359{
1360	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1361	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1362	struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1363	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1364
1365	nentries = roundup_pow_of_two(attr->src_nentries);
1366
1367	if (ar->hw_params.target_64bit)
1368		memset(src_ring->base_addr_owner_space, 0,
1369		       nentries * sizeof(struct ce_desc_64));
1370	else
1371		memset(src_ring->base_addr_owner_space, 0,
1372		       nentries * sizeof(struct ce_desc));
1373
1374	src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1375	src_ring->sw_index &= src_ring->nentries_mask;
1376	src_ring->hw_index = src_ring->sw_index;
1377
1378	src_ring->write_index =
1379		ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1380	src_ring->write_index &= src_ring->nentries_mask;
1381
1382	ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1383					 src_ring->base_addr_ce_space);
1384	ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1385	ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1386	ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1387	ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1388	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1389
1390	ath10k_dbg(ar, ATH10K_DBG_BOOT,
1391		   "boot init ce src ring id %d entries %d base_addr %pK\n",
1392		   ce_id, nentries, src_ring->base_addr_owner_space);
1393
1394	return 0;
1395}
1396
1397static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1398				    unsigned int ce_id,
1399				    const struct ce_attr *attr)
1400{
1401	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1402	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1403	struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1404	u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1405
1406	nentries = roundup_pow_of_two(attr->dest_nentries);
1407
1408	if (ar->hw_params.target_64bit)
1409		memset(dest_ring->base_addr_owner_space, 0,
1410		       nentries * sizeof(struct ce_desc_64));
1411	else
1412		memset(dest_ring->base_addr_owner_space, 0,
1413		       nentries * sizeof(struct ce_desc));
1414
1415	dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1416	dest_ring->sw_index &= dest_ring->nentries_mask;
1417	dest_ring->write_index =
1418		ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1419	dest_ring->write_index &= dest_ring->nentries_mask;
1420
1421	ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1422					  dest_ring->base_addr_ce_space);
1423	ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1424	ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1425	ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1426	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1427
1428	ath10k_dbg(ar, ATH10K_DBG_BOOT,
1429		   "boot ce dest ring id %d entries %d base_addr %pK\n",
1430		   ce_id, nentries, dest_ring->base_addr_owner_space);
1431
1432	return 0;
1433}
1434
1435static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1436				       struct ath10k_ce_ring *src_ring,
1437				       u32 nentries)
1438{
1439	src_ring->shadow_base_unaligned = kcalloc(nentries,
1440						  sizeof(struct ce_desc_64),
1441						  GFP_KERNEL);
1442	if (!src_ring->shadow_base_unaligned)
1443		return -ENOMEM;
1444
1445	src_ring->shadow_base = (struct ce_desc_64 *)
1446			PTR_ALIGN(src_ring->shadow_base_unaligned,
1447				  CE_DESC_RING_ALIGN);
1448	return 0;
1449}
1450
1451static struct ath10k_ce_ring *
1452ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1453			 const struct ce_attr *attr)
1454{
1455	struct ath10k_ce_ring *src_ring;
1456	u32 nentries = attr->src_nentries;
1457	dma_addr_t base_addr;
1458	int ret;
1459
1460	nentries = roundup_pow_of_two(nentries);
1461
1462	src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1463				       nentries), GFP_KERNEL);
1464	if (src_ring == NULL)
1465		return ERR_PTR(-ENOMEM);
1466
1467	src_ring->nentries = nentries;
1468	src_ring->nentries_mask = nentries - 1;
1469
1470	/*
1471	 * Legacy platforms that do not support cache
1472	 * coherent DMA are unsupported
1473	 */
1474	src_ring->base_addr_owner_space_unaligned =
1475		dma_alloc_coherent(ar->dev,
1476				   (nentries * sizeof(struct ce_desc) +
1477				    CE_DESC_RING_ALIGN),
1478				   &base_addr, GFP_KERNEL);
1479	if (!src_ring->base_addr_owner_space_unaligned) {
1480		kfree(src_ring);
1481		return ERR_PTR(-ENOMEM);
1482	}
1483
1484	src_ring->base_addr_ce_space_unaligned = base_addr;
1485
1486	src_ring->base_addr_owner_space =
1487			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1488				  CE_DESC_RING_ALIGN);
1489	src_ring->base_addr_ce_space =
1490			ALIGN(src_ring->base_addr_ce_space_unaligned,
1491			      CE_DESC_RING_ALIGN);
1492
1493	if (ar->hw_params.shadow_reg_support) {
1494		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1495		if (ret) {
1496			dma_free_coherent(ar->dev,
1497					  (nentries * sizeof(struct ce_desc) +
1498					   CE_DESC_RING_ALIGN),
1499					  src_ring->base_addr_owner_space_unaligned,
1500					  base_addr);
1501			kfree(src_ring);
1502			return ERR_PTR(ret);
1503		}
1504	}
1505
1506	return src_ring;
1507}
1508
1509static struct ath10k_ce_ring *
1510ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1511			    const struct ce_attr *attr)
1512{
1513	struct ath10k_ce_ring *src_ring;
1514	u32 nentries = attr->src_nentries;
1515	dma_addr_t base_addr;
1516	int ret;
1517
1518	nentries = roundup_pow_of_two(nentries);
1519
1520	src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1521				       nentries), GFP_KERNEL);
1522	if (!src_ring)
1523		return ERR_PTR(-ENOMEM);
1524
1525	src_ring->nentries = nentries;
1526	src_ring->nentries_mask = nentries - 1;
1527
1528	/* Legacy platforms that do not support cache
1529	 * coherent DMA are unsupported
1530	 */
1531	src_ring->base_addr_owner_space_unaligned =
1532		dma_alloc_coherent(ar->dev,
1533				   (nentries * sizeof(struct ce_desc_64) +
1534				    CE_DESC_RING_ALIGN),
1535				   &base_addr, GFP_KERNEL);
1536	if (!src_ring->base_addr_owner_space_unaligned) {
1537		kfree(src_ring);
1538		return ERR_PTR(-ENOMEM);
1539	}
1540
1541	src_ring->base_addr_ce_space_unaligned = base_addr;
1542
1543	src_ring->base_addr_owner_space =
1544			PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1545				  CE_DESC_RING_ALIGN);
1546	src_ring->base_addr_ce_space =
1547			ALIGN(src_ring->base_addr_ce_space_unaligned,
1548			      CE_DESC_RING_ALIGN);
1549
1550	if (ar->hw_params.shadow_reg_support) {
1551		ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1552		if (ret) {
1553			dma_free_coherent(ar->dev,
1554					  (nentries * sizeof(struct ce_desc_64) +
1555					   CE_DESC_RING_ALIGN),
1556					  src_ring->base_addr_owner_space_unaligned,
1557					  base_addr);
1558			kfree(src_ring);
1559			return ERR_PTR(ret);
1560		}
1561	}
1562
1563	return src_ring;
1564}
1565
1566static struct ath10k_ce_ring *
1567ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1568			  const struct ce_attr *attr)
1569{
1570	struct ath10k_ce_ring *dest_ring;
1571	u32 nentries;
1572	dma_addr_t base_addr;
1573
1574	nentries = roundup_pow_of_two(attr->dest_nentries);
1575
1576	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1577					nentries), GFP_KERNEL);
1578	if (dest_ring == NULL)
1579		return ERR_PTR(-ENOMEM);
1580
1581	dest_ring->nentries = nentries;
1582	dest_ring->nentries_mask = nentries - 1;
1583
1584	/*
1585	 * Legacy platforms that do not support cache
1586	 * coherent DMA are unsupported
1587	 */
1588	dest_ring->base_addr_owner_space_unaligned =
1589		dma_alloc_coherent(ar->dev,
1590				   (nentries * sizeof(struct ce_desc) +
1591				    CE_DESC_RING_ALIGN),
1592				   &base_addr, GFP_KERNEL);
1593	if (!dest_ring->base_addr_owner_space_unaligned) {
1594		kfree(dest_ring);
1595		return ERR_PTR(-ENOMEM);
1596	}
1597
1598	dest_ring->base_addr_ce_space_unaligned = base_addr;
1599
1600	dest_ring->base_addr_owner_space =
1601			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1602				  CE_DESC_RING_ALIGN);
1603	dest_ring->base_addr_ce_space =
1604				ALIGN(dest_ring->base_addr_ce_space_unaligned,
1605				      CE_DESC_RING_ALIGN);
1606
1607	return dest_ring;
1608}
1609
1610static struct ath10k_ce_ring *
1611ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1612			     const struct ce_attr *attr)
1613{
1614	struct ath10k_ce_ring *dest_ring;
1615	u32 nentries;
1616	dma_addr_t base_addr;
1617
1618	nentries = roundup_pow_of_two(attr->dest_nentries);
1619
1620	dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1621					nentries), GFP_KERNEL);
1622	if (!dest_ring)
1623		return ERR_PTR(-ENOMEM);
1624
1625	dest_ring->nentries = nentries;
1626	dest_ring->nentries_mask = nentries - 1;
1627
1628	/* Legacy platforms that do not support cache
1629	 * coherent DMA are unsupported
1630	 */
1631	dest_ring->base_addr_owner_space_unaligned =
1632		dma_alloc_coherent(ar->dev,
1633				   (nentries * sizeof(struct ce_desc_64) +
1634				    CE_DESC_RING_ALIGN),
1635				   &base_addr, GFP_KERNEL);
1636	if (!dest_ring->base_addr_owner_space_unaligned) {
1637		kfree(dest_ring);
1638		return ERR_PTR(-ENOMEM);
1639	}
1640
1641	dest_ring->base_addr_ce_space_unaligned = base_addr;
1642
1643	/* Correctly initialize memory to 0 to prevent garbage
1644	 * data crashing system when download firmware
1645	 */
1646	dest_ring->base_addr_owner_space =
1647			PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1648				  CE_DESC_RING_ALIGN);
1649	dest_ring->base_addr_ce_space =
1650			ALIGN(dest_ring->base_addr_ce_space_unaligned,
1651			      CE_DESC_RING_ALIGN);
1652
1653	return dest_ring;
1654}
1655
1656/*
1657 * Initialize a Copy Engine based on caller-supplied attributes.
1658 * This may be called once to initialize both source and destination
1659 * rings or it may be called twice for separate source and destination
1660 * initialization. It may be that only one side or the other is
1661 * initialized by software/firmware.
1662 */
1663int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1664			const struct ce_attr *attr)
1665{
1666	int ret;
1667
1668	if (attr->src_nentries) {
1669		ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1670		if (ret) {
1671			ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1672				   ce_id, ret);
1673			return ret;
1674		}
1675	}
1676
1677	if (attr->dest_nentries) {
1678		ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1679		if (ret) {
1680			ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1681				   ce_id, ret);
1682			return ret;
1683		}
1684	}
1685
1686	return 0;
1687}
1688EXPORT_SYMBOL(ath10k_ce_init_pipe);
1689
1690static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1691{
1692	u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1693
1694	ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1695	ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1696	ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1697	ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1698}
1699
1700static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1701{
1702	u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1703
1704	ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1705	ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1706	ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1707}
1708
1709void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1710{
1711	ath10k_ce_deinit_src_ring(ar, ce_id);
1712	ath10k_ce_deinit_dest_ring(ar, ce_id);
1713}
1714EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1715
1716static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1717{
1718	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1719	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1720
1721	if (ce_state->src_ring) {
1722		if (ar->hw_params.shadow_reg_support)
1723			kfree(ce_state->src_ring->shadow_base_unaligned);
1724		dma_free_coherent(ar->dev,
1725				  (ce_state->src_ring->nentries *
1726				   sizeof(struct ce_desc) +
1727				   CE_DESC_RING_ALIGN),
1728				  ce_state->src_ring->base_addr_owner_space,
1729				  ce_state->src_ring->base_addr_ce_space);
1730		kfree(ce_state->src_ring);
1731	}
1732
1733	if (ce_state->dest_ring) {
1734		dma_free_coherent(ar->dev,
1735				  (ce_state->dest_ring->nentries *
1736				   sizeof(struct ce_desc) +
1737				   CE_DESC_RING_ALIGN),
1738				  ce_state->dest_ring->base_addr_owner_space,
1739				  ce_state->dest_ring->base_addr_ce_space);
1740		kfree(ce_state->dest_ring);
1741	}
1742
1743	ce_state->src_ring = NULL;
1744	ce_state->dest_ring = NULL;
1745}
1746
1747static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1748{
1749	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1750	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1751
1752	if (ce_state->src_ring) {
1753		if (ar->hw_params.shadow_reg_support)
1754			kfree(ce_state->src_ring->shadow_base_unaligned);
1755		dma_free_coherent(ar->dev,
1756				  (ce_state->src_ring->nentries *
1757				   sizeof(struct ce_desc_64) +
1758				   CE_DESC_RING_ALIGN),
1759				  ce_state->src_ring->base_addr_owner_space,
1760				  ce_state->src_ring->base_addr_ce_space);
1761		kfree(ce_state->src_ring);
1762	}
1763
1764	if (ce_state->dest_ring) {
1765		dma_free_coherent(ar->dev,
1766				  (ce_state->dest_ring->nentries *
1767				   sizeof(struct ce_desc_64) +
1768				   CE_DESC_RING_ALIGN),
1769				  ce_state->dest_ring->base_addr_owner_space,
1770				  ce_state->dest_ring->base_addr_ce_space);
1771		kfree(ce_state->dest_ring);
1772	}
1773
1774	ce_state->src_ring = NULL;
1775	ce_state->dest_ring = NULL;
1776}
1777
1778void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1779{
1780	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1781	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1782
1783	ce_state->ops->ce_free_pipe(ar, ce_id);
1784}
1785EXPORT_SYMBOL(ath10k_ce_free_pipe);
1786
1787void ath10k_ce_dump_registers(struct ath10k *ar,
1788			      struct ath10k_fw_crash_data *crash_data)
1789{
1790	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1791	struct ath10k_ce_crash_data ce_data;
1792	u32 addr, id;
1793
1794	lockdep_assert_held(&ar->dump_mutex);
1795
1796	ath10k_err(ar, "Copy Engine register dump:\n");
1797
1798	spin_lock_bh(&ce->ce_lock);
1799	for (id = 0; id < CE_COUNT; id++) {
1800		addr = ath10k_ce_base_address(ar, id);
1801		ce_data.base_addr = cpu_to_le32(addr);
1802
1803		ce_data.src_wr_idx =
1804			cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1805		ce_data.src_r_idx =
1806			cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1807		ce_data.dst_wr_idx =
1808			cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1809		ce_data.dst_r_idx =
1810			cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1811
1812		if (crash_data)
1813			crash_data->ce_crash_data[id] = ce_data;
1814
1815		ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1816			   le32_to_cpu(ce_data.base_addr),
1817			   le32_to_cpu(ce_data.src_wr_idx),
1818			   le32_to_cpu(ce_data.src_r_idx),
1819			   le32_to_cpu(ce_data.dst_wr_idx),
1820			   le32_to_cpu(ce_data.dst_r_idx));
1821	}
1822
1823	spin_unlock_bh(&ce->ce_lock);
1824}
1825EXPORT_SYMBOL(ath10k_ce_dump_registers);
1826
1827static const struct ath10k_ce_ops ce_ops = {
1828	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1829	.ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1830	.ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1831	.ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1832	.ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1833	.ce_extract_desc_data = ath10k_ce_extract_desc_data,
1834	.ce_free_pipe = _ath10k_ce_free_pipe,
1835	.ce_send_nolock = _ath10k_ce_send_nolock,
1836	.ce_set_src_ring_base_addr_hi = NULL,
1837	.ce_set_dest_ring_base_addr_hi = NULL,
1838	.ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1839};
1840
1841static const struct ath10k_ce_ops ce_64_ops = {
1842	.ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1843	.ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1844	.ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1845	.ce_completed_recv_next_nolock =
1846				_ath10k_ce_completed_recv_next_nolock_64,
1847	.ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1848	.ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1849	.ce_free_pipe = _ath10k_ce_free_pipe_64,
1850	.ce_send_nolock = _ath10k_ce_send_nolock_64,
1851	.ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1852	.ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1853	.ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1854};
1855
1856static void ath10k_ce_set_ops(struct ath10k *ar,
1857			      struct ath10k_ce_pipe *ce_state)
1858{
1859	switch (ar->hw_rev) {
1860	case ATH10K_HW_WCN3990:
1861		ce_state->ops = &ce_64_ops;
1862		break;
1863	default:
1864		ce_state->ops = &ce_ops;
1865		break;
1866	}
1867}
1868
1869int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1870			 const struct ce_attr *attr)
1871{
1872	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1873	struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1874	int ret;
1875
1876	ath10k_ce_set_ops(ar, ce_state);
1877	/* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1878	 * additional TX locking checks.
1879	 *
1880	 * For the lack of a better place do the check here.
1881	 */
1882	BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1883		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1884	BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1885		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1886	BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1887		     (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1888
1889	ce_state->ar = ar;
1890	ce_state->id = ce_id;
1891	ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1892	ce_state->attr_flags = attr->flags;
1893	ce_state->src_sz_max = attr->src_sz_max;
1894
1895	if (attr->src_nentries)
1896		ce_state->send_cb = attr->send_cb;
1897
1898	if (attr->dest_nentries)
1899		ce_state->recv_cb = attr->recv_cb;
1900
1901	if (attr->src_nentries) {
1902		ce_state->src_ring =
1903			ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1904		if (IS_ERR(ce_state->src_ring)) {
1905			ret = PTR_ERR(ce_state->src_ring);
1906			ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1907				   ce_id, ret);
1908			ce_state->src_ring = NULL;
1909			return ret;
1910		}
1911	}
1912
1913	if (attr->dest_nentries) {
1914		ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1915									ce_id,
1916									attr);
1917		if (IS_ERR(ce_state->dest_ring)) {
1918			ret = PTR_ERR(ce_state->dest_ring);
1919			ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1920				   ce_id, ret);
1921			ce_state->dest_ring = NULL;
1922			return ret;
1923		}
1924	}
1925
1926	return 0;
1927}
1928EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1929
1930void ath10k_ce_alloc_rri(struct ath10k *ar)
1931{
1932	int i;
1933	u32 value;
1934	u32 ctrl1_regs;
1935	u32 ce_base_addr;
1936	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1937
1938	ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1939					   (CE_COUNT * sizeof(u32)),
1940					   &ce->paddr_rri, GFP_KERNEL);
1941
1942	if (!ce->vaddr_rri)
1943		return;
1944
1945	ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
1946			  lower_32_bits(ce->paddr_rri));
1947	ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
1948			  (upper_32_bits(ce->paddr_rri) &
1949			  CE_DESC_ADDR_HI_MASK));
1950
1951	for (i = 0; i < CE_COUNT; i++) {
1952		ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1953		ce_base_addr = ath10k_ce_base_address(ar, i);
1954		value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
1955		value |= ar->hw_ce_regs->upd->mask;
1956		ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
1957	}
1958}
1959EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1960
1961void ath10k_ce_free_rri(struct ath10k *ar)
1962{
1963	struct ath10k_ce *ce = ath10k_ce_priv(ar);
1964
1965	dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
1966			  ce->vaddr_rri,
1967			  ce->paddr_rri);
1968}
1969EXPORT_SYMBOL(ath10k_ce_free_rri);