Loading...
1// SPDX-License-Identifier: ISC
2/*
3 * Copyright (c) 2005-2011 Atheros Communications Inc.
4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
5 * Copyright (c) 2018 The Linux Foundation. All rights reserved.
6 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
7 */
8
9#include "hif.h"
10#include "ce.h"
11#include "debug.h"
12
13/*
14 * Support for Copy Engine hardware, which is mainly used for
15 * communication between Host and Target over a PCIe interconnect.
16 */
17
18/*
19 * A single CopyEngine (CE) comprises two "rings":
20 * a source ring
21 * a destination ring
22 *
23 * Each ring consists of a number of descriptors which specify
24 * an address, length, and meta-data.
25 *
26 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
27 * controls one ring and the other side controls the other ring.
28 * The source side chooses when to initiate a transfer and it
29 * chooses what to send (buffer address, length). The destination
30 * side keeps a supply of "anonymous receive buffers" available and
31 * it handles incoming data as it arrives (when the destination
32 * receives an interrupt).
33 *
34 * The sender may send a simple buffer (address/length) or it may
35 * send a small list of buffers. When a small list is sent, hardware
36 * "gathers" these and they end up in a single destination buffer
37 * with a single interrupt.
38 *
39 * There are several "contexts" managed by this layer -- more, it
40 * may seem -- than should be needed. These are provided mainly for
41 * maximum flexibility and especially to facilitate a simpler HIF
42 * implementation. There are per-CopyEngine recv, send, and watermark
43 * contexts. These are supplied by the caller when a recv, send,
44 * or watermark handler is established and they are echoed back to
45 * the caller when the respective callbacks are invoked. There is
46 * also a per-transfer context supplied by the caller when a buffer
47 * (or sendlist) is sent and when a buffer is enqueued for recv.
48 * These per-transfer contexts are echoed back to the caller when
49 * the buffer is sent/received.
50 */
51
52static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
53 struct ath10k_ce_pipe *ce_state)
54{
55 u32 ce_id = ce_state->id;
56 u32 addr = 0;
57
58 switch (ce_id) {
59 case 0:
60 addr = 0x00032000;
61 break;
62 case 3:
63 addr = 0x0003200C;
64 break;
65 case 4:
66 addr = 0x00032010;
67 break;
68 case 5:
69 addr = 0x00032014;
70 break;
71 case 7:
72 addr = 0x0003201C;
73 break;
74 default:
75 ath10k_warn(ar, "invalid CE id: %d", ce_id);
76 break;
77 }
78 return addr;
79}
80
81static inline unsigned int
82ath10k_set_ring_byte(unsigned int offset,
83 struct ath10k_hw_ce_regs_addr_map *addr_map)
84{
85 return ((offset << addr_map->lsb) & addr_map->mask);
86}
87
88static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
89{
90 struct ath10k_ce *ce = ath10k_ce_priv(ar);
91
92 return ce->bus_ops->read32(ar, offset);
93}
94
95static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
96{
97 struct ath10k_ce *ce = ath10k_ce_priv(ar);
98
99 ce->bus_ops->write32(ar, offset, value);
100}
101
102static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
103 u32 ce_ctrl_addr,
104 unsigned int n)
105{
106 ath10k_ce_write32(ar, ce_ctrl_addr +
107 ar->hw_ce_regs->dst_wr_index_addr, n);
108}
109
110static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
111 u32 ce_ctrl_addr)
112{
113 return ath10k_ce_read32(ar, ce_ctrl_addr +
114 ar->hw_ce_regs->dst_wr_index_addr);
115}
116
117static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
118 u32 ce_ctrl_addr,
119 unsigned int n)
120{
121 ath10k_ce_write32(ar, ce_ctrl_addr +
122 ar->hw_ce_regs->sr_wr_index_addr, n);
123}
124
125static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
126 u32 ce_ctrl_addr)
127{
128 return ath10k_ce_read32(ar, ce_ctrl_addr +
129 ar->hw_ce_regs->sr_wr_index_addr);
130}
131
132static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
133 u32 ce_id)
134{
135 struct ath10k_ce *ce = ath10k_ce_priv(ar);
136
137 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
138}
139
140static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
141 u32 ce_ctrl_addr)
142{
143 struct ath10k_ce *ce = ath10k_ce_priv(ar);
144 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
145 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
146 u32 index;
147
148 if (ar->hw_params.rri_on_ddr &&
149 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
150 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
151 else
152 index = ath10k_ce_read32(ar, ce_ctrl_addr +
153 ar->hw_ce_regs->current_srri_addr);
154
155 return index;
156}
157
158static inline void
159ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
160 struct ath10k_ce_pipe *ce_state,
161 unsigned int value)
162{
163 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
164}
165
166static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
167 u32 ce_id,
168 u64 addr)
169{
170 struct ath10k_ce *ce = ath10k_ce_priv(ar);
171 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
172 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
173 u32 addr_lo = lower_32_bits(addr);
174
175 ath10k_ce_write32(ar, ce_ctrl_addr +
176 ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
177
178 if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
179 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
180 addr);
181 }
182}
183
184static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
185 u32 ce_ctrl_addr,
186 u64 addr)
187{
188 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
189
190 ath10k_ce_write32(ar, ce_ctrl_addr +
191 ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
192}
193
194static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
195 u32 ce_ctrl_addr,
196 unsigned int n)
197{
198 ath10k_ce_write32(ar, ce_ctrl_addr +
199 ar->hw_ce_regs->sr_size_addr, n);
200}
201
202static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
203 u32 ce_ctrl_addr,
204 unsigned int n)
205{
206 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
207
208 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
209 ctrl_regs->addr);
210
211 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
212 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) |
213 ath10k_set_ring_byte(n, ctrl_regs->dmax));
214}
215
216static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
217 u32 ce_ctrl_addr,
218 unsigned int n)
219{
220 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
221
222 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
223 ctrl_regs->addr);
224
225 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
226 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
227 ath10k_set_ring_byte(n, ctrl_regs->src_ring));
228}
229
230static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
231 u32 ce_ctrl_addr,
232 unsigned int n)
233{
234 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
235
236 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
237 ctrl_regs->addr);
238
239 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
240 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
241 ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
242}
243
244static inline
245 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
246{
247 struct ath10k_ce *ce = ath10k_ce_priv(ar);
248
249 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
250 CE_DDR_RRI_MASK;
251}
252
253static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
254 u32 ce_ctrl_addr)
255{
256 struct ath10k_ce *ce = ath10k_ce_priv(ar);
257 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
258 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
259 u32 index;
260
261 if (ar->hw_params.rri_on_ddr &&
262 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
263 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
264 else
265 index = ath10k_ce_read32(ar, ce_ctrl_addr +
266 ar->hw_ce_regs->current_drri_addr);
267
268 return index;
269}
270
271static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
272 u32 ce_id,
273 u64 addr)
274{
275 struct ath10k_ce *ce = ath10k_ce_priv(ar);
276 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
277 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
278 u32 addr_lo = lower_32_bits(addr);
279
280 ath10k_ce_write32(ar, ce_ctrl_addr +
281 ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
282
283 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
284 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
285 addr);
286 }
287}
288
289static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
290 u32 ce_ctrl_addr,
291 u64 addr)
292{
293 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
294 u32 reg_value;
295
296 reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
297 ar->hw_ce_regs->dr_base_addr_hi);
298 reg_value &= ~CE_DESC_ADDR_HI_MASK;
299 reg_value |= addr_hi;
300 ath10k_ce_write32(ar, ce_ctrl_addr +
301 ar->hw_ce_regs->dr_base_addr_hi, reg_value);
302}
303
304static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
305 u32 ce_ctrl_addr,
306 unsigned int n)
307{
308 ath10k_ce_write32(ar, ce_ctrl_addr +
309 ar->hw_ce_regs->dr_size_addr, n);
310}
311
312static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
313 u32 ce_ctrl_addr,
314 unsigned int n)
315{
316 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
317 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
318
319 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
320 (addr & ~(srcr_wm->wm_high->mask)) |
321 (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
322}
323
324static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
325 u32 ce_ctrl_addr,
326 unsigned int n)
327{
328 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
329 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
330
331 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
332 (addr & ~(srcr_wm->wm_low->mask)) |
333 (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
334}
335
336static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
337 u32 ce_ctrl_addr,
338 unsigned int n)
339{
340 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
341 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
342
343 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
344 (addr & ~(dstr_wm->wm_high->mask)) |
345 (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
346}
347
348static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
349 u32 ce_ctrl_addr,
350 unsigned int n)
351{
352 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
353 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
354
355 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
356 (addr & ~(dstr_wm->wm_low->mask)) |
357 (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
358}
359
360static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
361 u32 ce_ctrl_addr)
362{
363 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
364
365 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
366 ar->hw_ce_regs->host_ie_addr);
367
368 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
369 host_ie_addr | host_ie->copy_complete->mask);
370}
371
372static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
373 u32 ce_ctrl_addr)
374{
375 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
376
377 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
378 ar->hw_ce_regs->host_ie_addr);
379
380 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
381 host_ie_addr & ~(host_ie->copy_complete->mask));
382}
383
384static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
385 u32 ce_ctrl_addr)
386{
387 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
388
389 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
390 ar->hw_ce_regs->host_ie_addr);
391
392 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
393 host_ie_addr & ~(wm_regs->wm_mask));
394}
395
396static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
397 u32 ce_ctrl_addr)
398{
399 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
400
401 u32 misc_ie_addr = ath10k_ce_read32(ar,
402 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
403
404 ath10k_ce_write32(ar,
405 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
406 misc_ie_addr & ~(misc_regs->err_mask));
407}
408
409static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
410 u32 ce_ctrl_addr,
411 unsigned int mask)
412{
413 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
414
415 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
416}
417
418/*
419 * Guts of ath10k_ce_send.
420 * The caller takes responsibility for any needed locking.
421 */
422static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
423 void *per_transfer_context,
424 dma_addr_t buffer,
425 unsigned int nbytes,
426 unsigned int transfer_id,
427 unsigned int flags)
428{
429 struct ath10k *ar = ce_state->ar;
430 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
431 struct ce_desc *desc, sdesc;
432 unsigned int nentries_mask = src_ring->nentries_mask;
433 unsigned int sw_index = src_ring->sw_index;
434 unsigned int write_index = src_ring->write_index;
435 u32 ctrl_addr = ce_state->ctrl_addr;
436 u32 desc_flags = 0;
437 int ret = 0;
438
439 if (nbytes > ce_state->src_sz_max)
440 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
441 __func__, nbytes, ce_state->src_sz_max);
442
443 if (unlikely(CE_RING_DELTA(nentries_mask,
444 write_index, sw_index - 1) <= 0)) {
445 ret = -ENOSR;
446 goto exit;
447 }
448
449 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
450 write_index);
451
452 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
453
454 if (flags & CE_SEND_FLAG_GATHER)
455 desc_flags |= CE_DESC_FLAGS_GATHER;
456 if (flags & CE_SEND_FLAG_BYTE_SWAP)
457 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
458
459 sdesc.addr = __cpu_to_le32(buffer);
460 sdesc.nbytes = __cpu_to_le16(nbytes);
461 sdesc.flags = __cpu_to_le16(desc_flags);
462
463 *desc = sdesc;
464
465 src_ring->per_transfer_context[write_index] = per_transfer_context;
466
467 /* Update Source Ring Write Index */
468 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
469
470 /* WORKAROUND */
471 if (!(flags & CE_SEND_FLAG_GATHER))
472 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
473
474 src_ring->write_index = write_index;
475exit:
476 return ret;
477}
478
479static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
480 void *per_transfer_context,
481 dma_addr_t buffer,
482 unsigned int nbytes,
483 unsigned int transfer_id,
484 unsigned int flags)
485{
486 struct ath10k *ar = ce_state->ar;
487 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
488 struct ce_desc_64 *desc, sdesc;
489 unsigned int nentries_mask = src_ring->nentries_mask;
490 unsigned int sw_index;
491 unsigned int write_index = src_ring->write_index;
492 u32 ctrl_addr = ce_state->ctrl_addr;
493 __le32 *addr;
494 u32 desc_flags = 0;
495 int ret = 0;
496
497 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
498 return -ESHUTDOWN;
499
500 if (nbytes > ce_state->src_sz_max)
501 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
502 __func__, nbytes, ce_state->src_sz_max);
503
504 if (ar->hw_params.rri_on_ddr)
505 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
506 else
507 sw_index = src_ring->sw_index;
508
509 if (unlikely(CE_RING_DELTA(nentries_mask,
510 write_index, sw_index - 1) <= 0)) {
511 ret = -ENOSR;
512 goto exit;
513 }
514
515 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
516 write_index);
517
518 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
519
520 if (flags & CE_SEND_FLAG_GATHER)
521 desc_flags |= CE_DESC_FLAGS_GATHER;
522
523 if (flags & CE_SEND_FLAG_BYTE_SWAP)
524 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
525
526 addr = (__le32 *)&sdesc.addr;
527
528 flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
529 addr[0] = __cpu_to_le32(buffer);
530 addr[1] = __cpu_to_le32(flags);
531 if (flags & CE_SEND_FLAG_GATHER)
532 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
533 else
534 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
535
536 sdesc.nbytes = __cpu_to_le16(nbytes);
537 sdesc.flags = __cpu_to_le16(desc_flags);
538
539 *desc = sdesc;
540
541 src_ring->per_transfer_context[write_index] = per_transfer_context;
542
543 /* Update Source Ring Write Index */
544 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
545
546 if (!(flags & CE_SEND_FLAG_GATHER)) {
547 if (ar->hw_params.shadow_reg_support)
548 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
549 write_index);
550 else
551 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
552 write_index);
553 }
554
555 src_ring->write_index = write_index;
556exit:
557 return ret;
558}
559
560int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
561 void *per_transfer_context,
562 dma_addr_t buffer,
563 unsigned int nbytes,
564 unsigned int transfer_id,
565 unsigned int flags)
566{
567 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
568 buffer, nbytes, transfer_id, flags);
569}
570EXPORT_SYMBOL(ath10k_ce_send_nolock);
571
572void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
573{
574 struct ath10k *ar = pipe->ar;
575 struct ath10k_ce *ce = ath10k_ce_priv(ar);
576 struct ath10k_ce_ring *src_ring = pipe->src_ring;
577 u32 ctrl_addr = pipe->ctrl_addr;
578
579 lockdep_assert_held(&ce->ce_lock);
580
581 /*
582 * This function must be called only if there is an incomplete
583 * scatter-gather transfer (before index register is updated)
584 * that needs to be cleaned up.
585 */
586 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
587 return;
588
589 if (WARN_ON_ONCE(src_ring->write_index ==
590 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
591 return;
592
593 src_ring->write_index--;
594 src_ring->write_index &= src_ring->nentries_mask;
595
596 src_ring->per_transfer_context[src_ring->write_index] = NULL;
597}
598EXPORT_SYMBOL(__ath10k_ce_send_revert);
599
600int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
601 void *per_transfer_context,
602 dma_addr_t buffer,
603 unsigned int nbytes,
604 unsigned int transfer_id,
605 unsigned int flags)
606{
607 struct ath10k *ar = ce_state->ar;
608 struct ath10k_ce *ce = ath10k_ce_priv(ar);
609 int ret;
610
611 spin_lock_bh(&ce->ce_lock);
612 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
613 buffer, nbytes, transfer_id, flags);
614 spin_unlock_bh(&ce->ce_lock);
615
616 return ret;
617}
618EXPORT_SYMBOL(ath10k_ce_send);
619
620int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
621{
622 struct ath10k *ar = pipe->ar;
623 struct ath10k_ce *ce = ath10k_ce_priv(ar);
624 int delta;
625
626 spin_lock_bh(&ce->ce_lock);
627 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
628 pipe->src_ring->write_index,
629 pipe->src_ring->sw_index - 1);
630 spin_unlock_bh(&ce->ce_lock);
631
632 return delta;
633}
634EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
635
636int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
637{
638 struct ath10k *ar = pipe->ar;
639 struct ath10k_ce *ce = ath10k_ce_priv(ar);
640 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
641 unsigned int nentries_mask = dest_ring->nentries_mask;
642 unsigned int write_index = dest_ring->write_index;
643 unsigned int sw_index = dest_ring->sw_index;
644
645 lockdep_assert_held(&ce->ce_lock);
646
647 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
648}
649EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
650
651static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
652 dma_addr_t paddr)
653{
654 struct ath10k *ar = pipe->ar;
655 struct ath10k_ce *ce = ath10k_ce_priv(ar);
656 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
657 unsigned int nentries_mask = dest_ring->nentries_mask;
658 unsigned int write_index = dest_ring->write_index;
659 unsigned int sw_index = dest_ring->sw_index;
660 struct ce_desc *base = dest_ring->base_addr_owner_space;
661 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
662 u32 ctrl_addr = pipe->ctrl_addr;
663
664 lockdep_assert_held(&ce->ce_lock);
665
666 if ((pipe->id != 5) &&
667 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
668 return -ENOSPC;
669
670 desc->addr = __cpu_to_le32(paddr);
671 desc->nbytes = 0;
672
673 dest_ring->per_transfer_context[write_index] = ctx;
674 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
675 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
676 dest_ring->write_index = write_index;
677
678 return 0;
679}
680
681static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
682 void *ctx,
683 dma_addr_t paddr)
684{
685 struct ath10k *ar = pipe->ar;
686 struct ath10k_ce *ce = ath10k_ce_priv(ar);
687 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
688 unsigned int nentries_mask = dest_ring->nentries_mask;
689 unsigned int write_index = dest_ring->write_index;
690 unsigned int sw_index = dest_ring->sw_index;
691 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
692 struct ce_desc_64 *desc =
693 CE_DEST_RING_TO_DESC_64(base, write_index);
694 u32 ctrl_addr = pipe->ctrl_addr;
695
696 lockdep_assert_held(&ce->ce_lock);
697
698 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
699 return -ENOSPC;
700
701 desc->addr = __cpu_to_le64(paddr);
702 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
703
704 desc->nbytes = 0;
705
706 dest_ring->per_transfer_context[write_index] = ctx;
707 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
708 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
709 dest_ring->write_index = write_index;
710
711 return 0;
712}
713
714void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
715{
716 struct ath10k *ar = pipe->ar;
717 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
718 unsigned int nentries_mask = dest_ring->nentries_mask;
719 unsigned int write_index = dest_ring->write_index;
720 u32 ctrl_addr = pipe->ctrl_addr;
721 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
722
723 /* Prevent CE ring stuck issue that will occur when ring is full.
724 * Make sure that write index is 1 less than read index.
725 */
726 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
727 nentries -= 1;
728
729 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
730 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
731 dest_ring->write_index = write_index;
732}
733EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
734
735int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
736 dma_addr_t paddr)
737{
738 struct ath10k *ar = pipe->ar;
739 struct ath10k_ce *ce = ath10k_ce_priv(ar);
740 int ret;
741
742 spin_lock_bh(&ce->ce_lock);
743 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
744 spin_unlock_bh(&ce->ce_lock);
745
746 return ret;
747}
748EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
749
750/*
751 * Guts of ath10k_ce_completed_recv_next.
752 * The caller takes responsibility for any necessary locking.
753 */
754static int
755 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
756 void **per_transfer_contextp,
757 unsigned int *nbytesp)
758{
759 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
760 unsigned int nentries_mask = dest_ring->nentries_mask;
761 unsigned int sw_index = dest_ring->sw_index;
762
763 struct ce_desc *base = dest_ring->base_addr_owner_space;
764 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
765 struct ce_desc sdesc;
766 u16 nbytes;
767
768 /* Copy in one go for performance reasons */
769 sdesc = *desc;
770
771 nbytes = __le16_to_cpu(sdesc.nbytes);
772 if (nbytes == 0) {
773 /*
774 * This closes a relatively unusual race where the Host
775 * sees the updated DRRI before the update to the
776 * corresponding descriptor has completed. We treat this
777 * as a descriptor that is not yet done.
778 */
779 return -EIO;
780 }
781
782 desc->nbytes = 0;
783
784 /* Return data from completed destination descriptor */
785 *nbytesp = nbytes;
786
787 if (per_transfer_contextp)
788 *per_transfer_contextp =
789 dest_ring->per_transfer_context[sw_index];
790
791 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
792 * So update transfer context all CEs except CE5.
793 */
794 if (ce_state->id != 5)
795 dest_ring->per_transfer_context[sw_index] = NULL;
796
797 /* Update sw_index */
798 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
799 dest_ring->sw_index = sw_index;
800
801 return 0;
802}
803
804static int
805_ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
806 void **per_transfer_contextp,
807 unsigned int *nbytesp)
808{
809 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
810 unsigned int nentries_mask = dest_ring->nentries_mask;
811 unsigned int sw_index = dest_ring->sw_index;
812 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
813 struct ce_desc_64 *desc =
814 CE_DEST_RING_TO_DESC_64(base, sw_index);
815 struct ce_desc_64 sdesc;
816 u16 nbytes;
817
818 /* Copy in one go for performance reasons */
819 sdesc = *desc;
820
821 nbytes = __le16_to_cpu(sdesc.nbytes);
822 if (nbytes == 0) {
823 /* This closes a relatively unusual race where the Host
824 * sees the updated DRRI before the update to the
825 * corresponding descriptor has completed. We treat this
826 * as a descriptor that is not yet done.
827 */
828 return -EIO;
829 }
830
831 desc->nbytes = 0;
832
833 /* Return data from completed destination descriptor */
834 *nbytesp = nbytes;
835
836 if (per_transfer_contextp)
837 *per_transfer_contextp =
838 dest_ring->per_transfer_context[sw_index];
839
840 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
841 * So update transfer context all CEs except CE5.
842 */
843 if (ce_state->id != 5)
844 dest_ring->per_transfer_context[sw_index] = NULL;
845
846 /* Update sw_index */
847 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
848 dest_ring->sw_index = sw_index;
849
850 return 0;
851}
852
853int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
854 void **per_transfer_ctx,
855 unsigned int *nbytesp)
856{
857 return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
858 per_transfer_ctx,
859 nbytesp);
860}
861EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
862
863int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
864 void **per_transfer_contextp,
865 unsigned int *nbytesp)
866{
867 struct ath10k *ar = ce_state->ar;
868 struct ath10k_ce *ce = ath10k_ce_priv(ar);
869 int ret;
870
871 spin_lock_bh(&ce->ce_lock);
872 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
873 per_transfer_contextp,
874 nbytesp);
875
876 spin_unlock_bh(&ce->ce_lock);
877
878 return ret;
879}
880EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
881
882static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
883 void **per_transfer_contextp,
884 dma_addr_t *bufferp)
885{
886 struct ath10k_ce_ring *dest_ring;
887 unsigned int nentries_mask;
888 unsigned int sw_index;
889 unsigned int write_index;
890 int ret;
891 struct ath10k *ar;
892 struct ath10k_ce *ce;
893
894 dest_ring = ce_state->dest_ring;
895
896 if (!dest_ring)
897 return -EIO;
898
899 ar = ce_state->ar;
900 ce = ath10k_ce_priv(ar);
901
902 spin_lock_bh(&ce->ce_lock);
903
904 nentries_mask = dest_ring->nentries_mask;
905 sw_index = dest_ring->sw_index;
906 write_index = dest_ring->write_index;
907 if (write_index != sw_index) {
908 struct ce_desc *base = dest_ring->base_addr_owner_space;
909 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
910
911 /* Return data from completed destination descriptor */
912 *bufferp = __le32_to_cpu(desc->addr);
913
914 if (per_transfer_contextp)
915 *per_transfer_contextp =
916 dest_ring->per_transfer_context[sw_index];
917
918 /* sanity */
919 dest_ring->per_transfer_context[sw_index] = NULL;
920 desc->nbytes = 0;
921
922 /* Update sw_index */
923 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
924 dest_ring->sw_index = sw_index;
925 ret = 0;
926 } else {
927 ret = -EIO;
928 }
929
930 spin_unlock_bh(&ce->ce_lock);
931
932 return ret;
933}
934
935static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
936 void **per_transfer_contextp,
937 dma_addr_t *bufferp)
938{
939 struct ath10k_ce_ring *dest_ring;
940 unsigned int nentries_mask;
941 unsigned int sw_index;
942 unsigned int write_index;
943 int ret;
944 struct ath10k *ar;
945 struct ath10k_ce *ce;
946
947 dest_ring = ce_state->dest_ring;
948
949 if (!dest_ring)
950 return -EIO;
951
952 ar = ce_state->ar;
953 ce = ath10k_ce_priv(ar);
954
955 spin_lock_bh(&ce->ce_lock);
956
957 nentries_mask = dest_ring->nentries_mask;
958 sw_index = dest_ring->sw_index;
959 write_index = dest_ring->write_index;
960 if (write_index != sw_index) {
961 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
962 struct ce_desc_64 *desc =
963 CE_DEST_RING_TO_DESC_64(base, sw_index);
964
965 /* Return data from completed destination descriptor */
966 *bufferp = __le64_to_cpu(desc->addr);
967
968 if (per_transfer_contextp)
969 *per_transfer_contextp =
970 dest_ring->per_transfer_context[sw_index];
971
972 /* sanity */
973 dest_ring->per_transfer_context[sw_index] = NULL;
974 desc->nbytes = 0;
975
976 /* Update sw_index */
977 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
978 dest_ring->sw_index = sw_index;
979 ret = 0;
980 } else {
981 ret = -EIO;
982 }
983
984 spin_unlock_bh(&ce->ce_lock);
985
986 return ret;
987}
988
989int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
990 void **per_transfer_contextp,
991 dma_addr_t *bufferp)
992{
993 return ce_state->ops->ce_revoke_recv_next(ce_state,
994 per_transfer_contextp,
995 bufferp);
996}
997EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
998
999/*
1000 * Guts of ath10k_ce_completed_send_next.
1001 * The caller takes responsibility for any necessary locking.
1002 */
1003static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1004 void **per_transfer_contextp)
1005{
1006 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1007 u32 ctrl_addr = ce_state->ctrl_addr;
1008 struct ath10k *ar = ce_state->ar;
1009 unsigned int nentries_mask = src_ring->nentries_mask;
1010 unsigned int sw_index = src_ring->sw_index;
1011 unsigned int read_index;
1012 struct ce_desc *desc;
1013
1014 if (src_ring->hw_index == sw_index) {
1015 /*
1016 * The SW completion index has caught up with the cached
1017 * version of the HW completion index.
1018 * Update the cached HW completion index to see whether
1019 * the SW has really caught up to the HW, or if the cached
1020 * value of the HW index has become stale.
1021 */
1022
1023 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1024 if (read_index == 0xffffffff)
1025 return -ENODEV;
1026
1027 read_index &= nentries_mask;
1028 src_ring->hw_index = read_index;
1029 }
1030
1031 if (ar->hw_params.rri_on_ddr)
1032 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1033 else
1034 read_index = src_ring->hw_index;
1035
1036 if (read_index == sw_index)
1037 return -EIO;
1038
1039 if (per_transfer_contextp)
1040 *per_transfer_contextp =
1041 src_ring->per_transfer_context[sw_index];
1042
1043 /* sanity */
1044 src_ring->per_transfer_context[sw_index] = NULL;
1045 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1046 sw_index);
1047 desc->nbytes = 0;
1048
1049 /* Update sw_index */
1050 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1051 src_ring->sw_index = sw_index;
1052
1053 return 0;
1054}
1055
1056static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1057 void **per_transfer_contextp)
1058{
1059 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1060 u32 ctrl_addr = ce_state->ctrl_addr;
1061 struct ath10k *ar = ce_state->ar;
1062 unsigned int nentries_mask = src_ring->nentries_mask;
1063 unsigned int sw_index = src_ring->sw_index;
1064 unsigned int read_index;
1065 struct ce_desc_64 *desc;
1066
1067 if (src_ring->hw_index == sw_index) {
1068 /*
1069 * The SW completion index has caught up with the cached
1070 * version of the HW completion index.
1071 * Update the cached HW completion index to see whether
1072 * the SW has really caught up to the HW, or if the cached
1073 * value of the HW index has become stale.
1074 */
1075
1076 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1077 if (read_index == 0xffffffff)
1078 return -ENODEV;
1079
1080 read_index &= nentries_mask;
1081 src_ring->hw_index = read_index;
1082 }
1083
1084 if (ar->hw_params.rri_on_ddr)
1085 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1086 else
1087 read_index = src_ring->hw_index;
1088
1089 if (read_index == sw_index)
1090 return -EIO;
1091
1092 if (per_transfer_contextp)
1093 *per_transfer_contextp =
1094 src_ring->per_transfer_context[sw_index];
1095
1096 /* sanity */
1097 src_ring->per_transfer_context[sw_index] = NULL;
1098 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1099 sw_index);
1100 desc->nbytes = 0;
1101
1102 /* Update sw_index */
1103 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1104 src_ring->sw_index = sw_index;
1105
1106 return 0;
1107}
1108
1109int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1110 void **per_transfer_contextp)
1111{
1112 return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1113 per_transfer_contextp);
1114}
1115EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1116
1117static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1118 struct ath10k_ce_ring *src_ring,
1119 u32 sw_index,
1120 dma_addr_t *bufferp,
1121 u32 *nbytesp,
1122 u32 *transfer_idp)
1123{
1124 struct ce_desc *base = src_ring->base_addr_owner_space;
1125 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1126
1127 /* Return data from completed source descriptor */
1128 *bufferp = __le32_to_cpu(desc->addr);
1129 *nbytesp = __le16_to_cpu(desc->nbytes);
1130 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1131 CE_DESC_FLAGS_META_DATA);
1132}
1133
1134static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1135 struct ath10k_ce_ring *src_ring,
1136 u32 sw_index,
1137 dma_addr_t *bufferp,
1138 u32 *nbytesp,
1139 u32 *transfer_idp)
1140{
1141 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1142 struct ce_desc_64 *desc =
1143 CE_SRC_RING_TO_DESC_64(base, sw_index);
1144
1145 /* Return data from completed source descriptor */
1146 *bufferp = __le64_to_cpu(desc->addr);
1147 *nbytesp = __le16_to_cpu(desc->nbytes);
1148 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1149 CE_DESC_FLAGS_META_DATA);
1150}
1151
1152/* NB: Modeled after ath10k_ce_completed_send_next */
1153int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1154 void **per_transfer_contextp,
1155 dma_addr_t *bufferp,
1156 unsigned int *nbytesp,
1157 unsigned int *transfer_idp)
1158{
1159 struct ath10k_ce_ring *src_ring;
1160 unsigned int nentries_mask;
1161 unsigned int sw_index;
1162 unsigned int write_index;
1163 int ret;
1164 struct ath10k *ar;
1165 struct ath10k_ce *ce;
1166
1167 src_ring = ce_state->src_ring;
1168
1169 if (!src_ring)
1170 return -EIO;
1171
1172 ar = ce_state->ar;
1173 ce = ath10k_ce_priv(ar);
1174
1175 spin_lock_bh(&ce->ce_lock);
1176
1177 nentries_mask = src_ring->nentries_mask;
1178 sw_index = src_ring->sw_index;
1179 write_index = src_ring->write_index;
1180
1181 if (write_index != sw_index) {
1182 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1183 bufferp, nbytesp,
1184 transfer_idp);
1185
1186 if (per_transfer_contextp)
1187 *per_transfer_contextp =
1188 src_ring->per_transfer_context[sw_index];
1189
1190 /* sanity */
1191 src_ring->per_transfer_context[sw_index] = NULL;
1192
1193 /* Update sw_index */
1194 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1195 src_ring->sw_index = sw_index;
1196 ret = 0;
1197 } else {
1198 ret = -EIO;
1199 }
1200
1201 spin_unlock_bh(&ce->ce_lock);
1202
1203 return ret;
1204}
1205EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1206
1207int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1208 void **per_transfer_contextp)
1209{
1210 struct ath10k *ar = ce_state->ar;
1211 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1212 int ret;
1213
1214 spin_lock_bh(&ce->ce_lock);
1215 ret = ath10k_ce_completed_send_next_nolock(ce_state,
1216 per_transfer_contextp);
1217 spin_unlock_bh(&ce->ce_lock);
1218
1219 return ret;
1220}
1221EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1222
1223/*
1224 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1225 *
1226 * Invokes registered callbacks for recv_complete,
1227 * send_complete, and watermarks.
1228 */
1229void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1230{
1231 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1232 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1233 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1234 u32 ctrl_addr = ce_state->ctrl_addr;
1235
1236 /*
1237 * Clear before handling
1238 *
1239 * Misc CE interrupts are not being handled, but still need
1240 * to be cleared.
1241 *
1242 * NOTE: When the last copy engine interrupt is cleared the
1243 * hardware will go to sleep. Once this happens any access to
1244 * the CE registers can cause a hardware fault.
1245 */
1246 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1247 wm_regs->cc_mask | wm_regs->wm_mask);
1248
1249 if (ce_state->recv_cb)
1250 ce_state->recv_cb(ce_state);
1251
1252 if (ce_state->send_cb)
1253 ce_state->send_cb(ce_state);
1254}
1255EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1256
1257/*
1258 * Handler for per-engine interrupts on ALL active CEs.
1259 * This is used in cases where the system is sharing a
1260 * single interrupt for all CEs
1261 */
1262
1263void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1264{
1265 int ce_id;
1266 u32 intr_summary;
1267
1268 intr_summary = ath10k_ce_interrupt_summary(ar);
1269
1270 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1271 if (intr_summary & (1 << ce_id))
1272 intr_summary &= ~(1 << ce_id);
1273 else
1274 /* no intr pending on this CE */
1275 continue;
1276
1277 ath10k_ce_per_engine_service(ar, ce_id);
1278 }
1279}
1280EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1281
1282/*
1283 * Adjust interrupts for the copy complete handler.
1284 * If it's needed for either send or recv, then unmask
1285 * this interrupt; otherwise, mask it.
1286 *
1287 * Called with ce_lock held.
1288 */
1289static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1290{
1291 u32 ctrl_addr = ce_state->ctrl_addr;
1292 struct ath10k *ar = ce_state->ar;
1293 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1294
1295 if ((!disable_copy_compl_intr) &&
1296 (ce_state->send_cb || ce_state->recv_cb))
1297 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1298 else
1299 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1300
1301 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1302}
1303
1304void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1305{
1306 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1307 struct ath10k_ce_pipe *ce_state;
1308 u32 ctrl_addr;
1309
1310 ce_state = &ce->ce_states[ce_id];
1311 if (ce_state->attr_flags & CE_ATTR_POLL)
1312 return;
1313
1314 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1315
1316 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1317 ath10k_ce_error_intr_disable(ar, ctrl_addr);
1318 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1319}
1320EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1321
1322void ath10k_ce_disable_interrupts(struct ath10k *ar)
1323{
1324 int ce_id;
1325
1326 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1327 ath10k_ce_disable_interrupt(ar, ce_id);
1328}
1329EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1330
1331void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1332{
1333 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1334 struct ath10k_ce_pipe *ce_state;
1335
1336 ce_state = &ce->ce_states[ce_id];
1337 if (ce_state->attr_flags & CE_ATTR_POLL)
1338 return;
1339
1340 ath10k_ce_per_engine_handler_adjust(ce_state);
1341}
1342EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1343
1344void ath10k_ce_enable_interrupts(struct ath10k *ar)
1345{
1346 int ce_id;
1347
1348 /* Enable interrupts for copy engine that
1349 * are not using polling mode.
1350 */
1351 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1352 ath10k_ce_enable_interrupt(ar, ce_id);
1353}
1354EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1355
1356static int ath10k_ce_init_src_ring(struct ath10k *ar,
1357 unsigned int ce_id,
1358 const struct ce_attr *attr)
1359{
1360 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1361 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1362 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1363 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1364
1365 nentries = roundup_pow_of_two(attr->src_nentries);
1366
1367 if (ar->hw_params.target_64bit)
1368 memset(src_ring->base_addr_owner_space, 0,
1369 nentries * sizeof(struct ce_desc_64));
1370 else
1371 memset(src_ring->base_addr_owner_space, 0,
1372 nentries * sizeof(struct ce_desc));
1373
1374 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1375 src_ring->sw_index &= src_ring->nentries_mask;
1376 src_ring->hw_index = src_ring->sw_index;
1377
1378 src_ring->write_index =
1379 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1380 src_ring->write_index &= src_ring->nentries_mask;
1381
1382 ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1383 src_ring->base_addr_ce_space);
1384 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1385 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1386 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1387 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1388 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1389
1390 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1391 "boot init ce src ring id %d entries %d base_addr %pK\n",
1392 ce_id, nentries, src_ring->base_addr_owner_space);
1393
1394 return 0;
1395}
1396
1397static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1398 unsigned int ce_id,
1399 const struct ce_attr *attr)
1400{
1401 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1402 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1403 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1404 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1405
1406 nentries = roundup_pow_of_two(attr->dest_nentries);
1407
1408 if (ar->hw_params.target_64bit)
1409 memset(dest_ring->base_addr_owner_space, 0,
1410 nentries * sizeof(struct ce_desc_64));
1411 else
1412 memset(dest_ring->base_addr_owner_space, 0,
1413 nentries * sizeof(struct ce_desc));
1414
1415 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1416 dest_ring->sw_index &= dest_ring->nentries_mask;
1417 dest_ring->write_index =
1418 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1419 dest_ring->write_index &= dest_ring->nentries_mask;
1420
1421 ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1422 dest_ring->base_addr_ce_space);
1423 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1424 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1425 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1426 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1427
1428 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1429 "boot ce dest ring id %d entries %d base_addr %pK\n",
1430 ce_id, nentries, dest_ring->base_addr_owner_space);
1431
1432 return 0;
1433}
1434
1435static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1436 struct ath10k_ce_ring *src_ring,
1437 u32 nentries)
1438{
1439 src_ring->shadow_base_unaligned = kcalloc(nentries,
1440 sizeof(struct ce_desc_64),
1441 GFP_KERNEL);
1442 if (!src_ring->shadow_base_unaligned)
1443 return -ENOMEM;
1444
1445 src_ring->shadow_base = (struct ce_desc_64 *)
1446 PTR_ALIGN(src_ring->shadow_base_unaligned,
1447 CE_DESC_RING_ALIGN);
1448 return 0;
1449}
1450
1451static struct ath10k_ce_ring *
1452ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1453 const struct ce_attr *attr)
1454{
1455 struct ath10k_ce_ring *src_ring;
1456 u32 nentries = attr->src_nentries;
1457 dma_addr_t base_addr;
1458 int ret;
1459
1460 nentries = roundup_pow_of_two(nentries);
1461
1462 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1463 nentries), GFP_KERNEL);
1464 if (src_ring == NULL)
1465 return ERR_PTR(-ENOMEM);
1466
1467 src_ring->nentries = nentries;
1468 src_ring->nentries_mask = nentries - 1;
1469
1470 /*
1471 * Legacy platforms that do not support cache
1472 * coherent DMA are unsupported
1473 */
1474 src_ring->base_addr_owner_space_unaligned =
1475 dma_alloc_coherent(ar->dev,
1476 (nentries * sizeof(struct ce_desc) +
1477 CE_DESC_RING_ALIGN),
1478 &base_addr, GFP_KERNEL);
1479 if (!src_ring->base_addr_owner_space_unaligned) {
1480 kfree(src_ring);
1481 return ERR_PTR(-ENOMEM);
1482 }
1483
1484 src_ring->base_addr_ce_space_unaligned = base_addr;
1485
1486 src_ring->base_addr_owner_space =
1487 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1488 CE_DESC_RING_ALIGN);
1489 src_ring->base_addr_ce_space =
1490 ALIGN(src_ring->base_addr_ce_space_unaligned,
1491 CE_DESC_RING_ALIGN);
1492
1493 if (ar->hw_params.shadow_reg_support) {
1494 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1495 if (ret) {
1496 dma_free_coherent(ar->dev,
1497 (nentries * sizeof(struct ce_desc) +
1498 CE_DESC_RING_ALIGN),
1499 src_ring->base_addr_owner_space_unaligned,
1500 base_addr);
1501 kfree(src_ring);
1502 return ERR_PTR(ret);
1503 }
1504 }
1505
1506 return src_ring;
1507}
1508
1509static struct ath10k_ce_ring *
1510ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1511 const struct ce_attr *attr)
1512{
1513 struct ath10k_ce_ring *src_ring;
1514 u32 nentries = attr->src_nentries;
1515 dma_addr_t base_addr;
1516 int ret;
1517
1518 nentries = roundup_pow_of_two(nentries);
1519
1520 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1521 nentries), GFP_KERNEL);
1522 if (!src_ring)
1523 return ERR_PTR(-ENOMEM);
1524
1525 src_ring->nentries = nentries;
1526 src_ring->nentries_mask = nentries - 1;
1527
1528 /* Legacy platforms that do not support cache
1529 * coherent DMA are unsupported
1530 */
1531 src_ring->base_addr_owner_space_unaligned =
1532 dma_alloc_coherent(ar->dev,
1533 (nentries * sizeof(struct ce_desc_64) +
1534 CE_DESC_RING_ALIGN),
1535 &base_addr, GFP_KERNEL);
1536 if (!src_ring->base_addr_owner_space_unaligned) {
1537 kfree(src_ring);
1538 return ERR_PTR(-ENOMEM);
1539 }
1540
1541 src_ring->base_addr_ce_space_unaligned = base_addr;
1542
1543 src_ring->base_addr_owner_space =
1544 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1545 CE_DESC_RING_ALIGN);
1546 src_ring->base_addr_ce_space =
1547 ALIGN(src_ring->base_addr_ce_space_unaligned,
1548 CE_DESC_RING_ALIGN);
1549
1550 if (ar->hw_params.shadow_reg_support) {
1551 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1552 if (ret) {
1553 dma_free_coherent(ar->dev,
1554 (nentries * sizeof(struct ce_desc_64) +
1555 CE_DESC_RING_ALIGN),
1556 src_ring->base_addr_owner_space_unaligned,
1557 base_addr);
1558 kfree(src_ring);
1559 return ERR_PTR(ret);
1560 }
1561 }
1562
1563 return src_ring;
1564}
1565
1566static struct ath10k_ce_ring *
1567ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1568 const struct ce_attr *attr)
1569{
1570 struct ath10k_ce_ring *dest_ring;
1571 u32 nentries;
1572 dma_addr_t base_addr;
1573
1574 nentries = roundup_pow_of_two(attr->dest_nentries);
1575
1576 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1577 nentries), GFP_KERNEL);
1578 if (dest_ring == NULL)
1579 return ERR_PTR(-ENOMEM);
1580
1581 dest_ring->nentries = nentries;
1582 dest_ring->nentries_mask = nentries - 1;
1583
1584 /*
1585 * Legacy platforms that do not support cache
1586 * coherent DMA are unsupported
1587 */
1588 dest_ring->base_addr_owner_space_unaligned =
1589 dma_alloc_coherent(ar->dev,
1590 (nentries * sizeof(struct ce_desc) +
1591 CE_DESC_RING_ALIGN),
1592 &base_addr, GFP_KERNEL);
1593 if (!dest_ring->base_addr_owner_space_unaligned) {
1594 kfree(dest_ring);
1595 return ERR_PTR(-ENOMEM);
1596 }
1597
1598 dest_ring->base_addr_ce_space_unaligned = base_addr;
1599
1600 dest_ring->base_addr_owner_space =
1601 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1602 CE_DESC_RING_ALIGN);
1603 dest_ring->base_addr_ce_space =
1604 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1605 CE_DESC_RING_ALIGN);
1606
1607 return dest_ring;
1608}
1609
1610static struct ath10k_ce_ring *
1611ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1612 const struct ce_attr *attr)
1613{
1614 struct ath10k_ce_ring *dest_ring;
1615 u32 nentries;
1616 dma_addr_t base_addr;
1617
1618 nentries = roundup_pow_of_two(attr->dest_nentries);
1619
1620 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1621 nentries), GFP_KERNEL);
1622 if (!dest_ring)
1623 return ERR_PTR(-ENOMEM);
1624
1625 dest_ring->nentries = nentries;
1626 dest_ring->nentries_mask = nentries - 1;
1627
1628 /* Legacy platforms that do not support cache
1629 * coherent DMA are unsupported
1630 */
1631 dest_ring->base_addr_owner_space_unaligned =
1632 dma_alloc_coherent(ar->dev,
1633 (nentries * sizeof(struct ce_desc_64) +
1634 CE_DESC_RING_ALIGN),
1635 &base_addr, GFP_KERNEL);
1636 if (!dest_ring->base_addr_owner_space_unaligned) {
1637 kfree(dest_ring);
1638 return ERR_PTR(-ENOMEM);
1639 }
1640
1641 dest_ring->base_addr_ce_space_unaligned = base_addr;
1642
1643 /* Correctly initialize memory to 0 to prevent garbage
1644 * data crashing system when download firmware
1645 */
1646 dest_ring->base_addr_owner_space =
1647 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1648 CE_DESC_RING_ALIGN);
1649 dest_ring->base_addr_ce_space =
1650 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1651 CE_DESC_RING_ALIGN);
1652
1653 return dest_ring;
1654}
1655
1656/*
1657 * Initialize a Copy Engine based on caller-supplied attributes.
1658 * This may be called once to initialize both source and destination
1659 * rings or it may be called twice for separate source and destination
1660 * initialization. It may be that only one side or the other is
1661 * initialized by software/firmware.
1662 */
1663int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1664 const struct ce_attr *attr)
1665{
1666 int ret;
1667
1668 if (attr->src_nentries) {
1669 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1670 if (ret) {
1671 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1672 ce_id, ret);
1673 return ret;
1674 }
1675 }
1676
1677 if (attr->dest_nentries) {
1678 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1679 if (ret) {
1680 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1681 ce_id, ret);
1682 return ret;
1683 }
1684 }
1685
1686 return 0;
1687}
1688EXPORT_SYMBOL(ath10k_ce_init_pipe);
1689
1690static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1691{
1692 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1693
1694 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1695 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1696 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1697 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1698}
1699
1700static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1701{
1702 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1703
1704 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1705 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1706 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1707}
1708
1709void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1710{
1711 ath10k_ce_deinit_src_ring(ar, ce_id);
1712 ath10k_ce_deinit_dest_ring(ar, ce_id);
1713}
1714EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1715
1716static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1717{
1718 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1719 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1720
1721 if (ce_state->src_ring) {
1722 if (ar->hw_params.shadow_reg_support)
1723 kfree(ce_state->src_ring->shadow_base_unaligned);
1724 dma_free_coherent(ar->dev,
1725 (ce_state->src_ring->nentries *
1726 sizeof(struct ce_desc) +
1727 CE_DESC_RING_ALIGN),
1728 ce_state->src_ring->base_addr_owner_space,
1729 ce_state->src_ring->base_addr_ce_space);
1730 kfree(ce_state->src_ring);
1731 }
1732
1733 if (ce_state->dest_ring) {
1734 dma_free_coherent(ar->dev,
1735 (ce_state->dest_ring->nentries *
1736 sizeof(struct ce_desc) +
1737 CE_DESC_RING_ALIGN),
1738 ce_state->dest_ring->base_addr_owner_space,
1739 ce_state->dest_ring->base_addr_ce_space);
1740 kfree(ce_state->dest_ring);
1741 }
1742
1743 ce_state->src_ring = NULL;
1744 ce_state->dest_ring = NULL;
1745}
1746
1747static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1748{
1749 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1750 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1751
1752 if (ce_state->src_ring) {
1753 if (ar->hw_params.shadow_reg_support)
1754 kfree(ce_state->src_ring->shadow_base_unaligned);
1755 dma_free_coherent(ar->dev,
1756 (ce_state->src_ring->nentries *
1757 sizeof(struct ce_desc_64) +
1758 CE_DESC_RING_ALIGN),
1759 ce_state->src_ring->base_addr_owner_space,
1760 ce_state->src_ring->base_addr_ce_space);
1761 kfree(ce_state->src_ring);
1762 }
1763
1764 if (ce_state->dest_ring) {
1765 dma_free_coherent(ar->dev,
1766 (ce_state->dest_ring->nentries *
1767 sizeof(struct ce_desc_64) +
1768 CE_DESC_RING_ALIGN),
1769 ce_state->dest_ring->base_addr_owner_space,
1770 ce_state->dest_ring->base_addr_ce_space);
1771 kfree(ce_state->dest_ring);
1772 }
1773
1774 ce_state->src_ring = NULL;
1775 ce_state->dest_ring = NULL;
1776}
1777
1778void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1779{
1780 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1781 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1782
1783 ce_state->ops->ce_free_pipe(ar, ce_id);
1784}
1785EXPORT_SYMBOL(ath10k_ce_free_pipe);
1786
1787void ath10k_ce_dump_registers(struct ath10k *ar,
1788 struct ath10k_fw_crash_data *crash_data)
1789{
1790 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1791 struct ath10k_ce_crash_data ce_data;
1792 u32 addr, id;
1793
1794 lockdep_assert_held(&ar->dump_mutex);
1795
1796 ath10k_err(ar, "Copy Engine register dump:\n");
1797
1798 spin_lock_bh(&ce->ce_lock);
1799 for (id = 0; id < CE_COUNT; id++) {
1800 addr = ath10k_ce_base_address(ar, id);
1801 ce_data.base_addr = cpu_to_le32(addr);
1802
1803 ce_data.src_wr_idx =
1804 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1805 ce_data.src_r_idx =
1806 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1807 ce_data.dst_wr_idx =
1808 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1809 ce_data.dst_r_idx =
1810 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1811
1812 if (crash_data)
1813 crash_data->ce_crash_data[id] = ce_data;
1814
1815 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1816 le32_to_cpu(ce_data.base_addr),
1817 le32_to_cpu(ce_data.src_wr_idx),
1818 le32_to_cpu(ce_data.src_r_idx),
1819 le32_to_cpu(ce_data.dst_wr_idx),
1820 le32_to_cpu(ce_data.dst_r_idx));
1821 }
1822
1823 spin_unlock_bh(&ce->ce_lock);
1824}
1825EXPORT_SYMBOL(ath10k_ce_dump_registers);
1826
1827static const struct ath10k_ce_ops ce_ops = {
1828 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1829 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1830 .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1831 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1832 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1833 .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1834 .ce_free_pipe = _ath10k_ce_free_pipe,
1835 .ce_send_nolock = _ath10k_ce_send_nolock,
1836 .ce_set_src_ring_base_addr_hi = NULL,
1837 .ce_set_dest_ring_base_addr_hi = NULL,
1838 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1839};
1840
1841static const struct ath10k_ce_ops ce_64_ops = {
1842 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1843 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1844 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1845 .ce_completed_recv_next_nolock =
1846 _ath10k_ce_completed_recv_next_nolock_64,
1847 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1848 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1849 .ce_free_pipe = _ath10k_ce_free_pipe_64,
1850 .ce_send_nolock = _ath10k_ce_send_nolock_64,
1851 .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1852 .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1853 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1854};
1855
1856static void ath10k_ce_set_ops(struct ath10k *ar,
1857 struct ath10k_ce_pipe *ce_state)
1858{
1859 switch (ar->hw_rev) {
1860 case ATH10K_HW_WCN3990:
1861 ce_state->ops = &ce_64_ops;
1862 break;
1863 default:
1864 ce_state->ops = &ce_ops;
1865 break;
1866 }
1867}
1868
1869int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1870 const struct ce_attr *attr)
1871{
1872 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1873 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1874 int ret;
1875
1876 ath10k_ce_set_ops(ar, ce_state);
1877 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1878 * additional TX locking checks.
1879 *
1880 * For the lack of a better place do the check here.
1881 */
1882 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1883 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1884 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1885 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1886 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1887 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1888
1889 ce_state->ar = ar;
1890 ce_state->id = ce_id;
1891 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1892 ce_state->attr_flags = attr->flags;
1893 ce_state->src_sz_max = attr->src_sz_max;
1894
1895 if (attr->src_nentries)
1896 ce_state->send_cb = attr->send_cb;
1897
1898 if (attr->dest_nentries)
1899 ce_state->recv_cb = attr->recv_cb;
1900
1901 if (attr->src_nentries) {
1902 ce_state->src_ring =
1903 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1904 if (IS_ERR(ce_state->src_ring)) {
1905 ret = PTR_ERR(ce_state->src_ring);
1906 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1907 ce_id, ret);
1908 ce_state->src_ring = NULL;
1909 return ret;
1910 }
1911 }
1912
1913 if (attr->dest_nentries) {
1914 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1915 ce_id,
1916 attr);
1917 if (IS_ERR(ce_state->dest_ring)) {
1918 ret = PTR_ERR(ce_state->dest_ring);
1919 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1920 ce_id, ret);
1921 ce_state->dest_ring = NULL;
1922 return ret;
1923 }
1924 }
1925
1926 return 0;
1927}
1928EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1929
1930void ath10k_ce_alloc_rri(struct ath10k *ar)
1931{
1932 int i;
1933 u32 value;
1934 u32 ctrl1_regs;
1935 u32 ce_base_addr;
1936 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1937
1938 ce->vaddr_rri = dma_alloc_coherent(ar->dev,
1939 (CE_COUNT * sizeof(u32)),
1940 &ce->paddr_rri, GFP_KERNEL);
1941
1942 if (!ce->vaddr_rri)
1943 return;
1944
1945 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
1946 lower_32_bits(ce->paddr_rri));
1947 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
1948 (upper_32_bits(ce->paddr_rri) &
1949 CE_DESC_ADDR_HI_MASK));
1950
1951 for (i = 0; i < CE_COUNT; i++) {
1952 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
1953 ce_base_addr = ath10k_ce_base_address(ar, i);
1954 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
1955 value |= ar->hw_ce_regs->upd->mask;
1956 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
1957 }
1958}
1959EXPORT_SYMBOL(ath10k_ce_alloc_rri);
1960
1961void ath10k_ce_free_rri(struct ath10k *ar)
1962{
1963 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1964
1965 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
1966 ce->vaddr_rri,
1967 ce->paddr_rri);
1968}
1969EXPORT_SYMBOL(ath10k_ce_free_rri);
1/*
2 * Copyright (c) 2005-2011 Atheros Communications Inc.
3 * Copyright (c) 2011-2013 Qualcomm Atheros, Inc.
4 *
5 * Permission to use, copy, modify, and/or distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18#include "hif.h"
19#include "pci.h"
20#include "ce.h"
21#include "debug.h"
22
23/*
24 * Support for Copy Engine hardware, which is mainly used for
25 * communication between Host and Target over a PCIe interconnect.
26 */
27
28/*
29 * A single CopyEngine (CE) comprises two "rings":
30 * a source ring
31 * a destination ring
32 *
33 * Each ring consists of a number of descriptors which specify
34 * an address, length, and meta-data.
35 *
36 * Typically, one side of the PCIe interconnect (Host or Target)
37 * controls one ring and the other side controls the other ring.
38 * The source side chooses when to initiate a transfer and it
39 * chooses what to send (buffer address, length). The destination
40 * side keeps a supply of "anonymous receive buffers" available and
41 * it handles incoming data as it arrives (when the destination
42 * recieves an interrupt).
43 *
44 * The sender may send a simple buffer (address/length) or it may
45 * send a small list of buffers. When a small list is sent, hardware
46 * "gathers" these and they end up in a single destination buffer
47 * with a single interrupt.
48 *
49 * There are several "contexts" managed by this layer -- more, it
50 * may seem -- than should be needed. These are provided mainly for
51 * maximum flexibility and especially to facilitate a simpler HIF
52 * implementation. There are per-CopyEngine recv, send, and watermark
53 * contexts. These are supplied by the caller when a recv, send,
54 * or watermark handler is established and they are echoed back to
55 * the caller when the respective callbacks are invoked. There is
56 * also a per-transfer context supplied by the caller when a buffer
57 * (or sendlist) is sent and when a buffer is enqueued for recv.
58 * These per-transfer contexts are echoed back to the caller when
59 * the buffer is sent/received.
60 */
61
62static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
63 u32 ce_ctrl_addr,
64 unsigned int n)
65{
66 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS, n);
67}
68
69static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
70 u32 ce_ctrl_addr)
71{
72 return ath10k_pci_read32(ar, ce_ctrl_addr + DST_WR_INDEX_ADDRESS);
73}
74
75static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
76 u32 ce_ctrl_addr,
77 unsigned int n)
78{
79 ath10k_pci_write32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS, n);
80}
81
82static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
83 u32 ce_ctrl_addr)
84{
85 return ath10k_pci_read32(ar, ce_ctrl_addr + SR_WR_INDEX_ADDRESS);
86}
87
88static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
89 u32 ce_ctrl_addr)
90{
91 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_SRRI_ADDRESS);
92}
93
94static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
95 u32 ce_ctrl_addr,
96 unsigned int addr)
97{
98 ath10k_pci_write32(ar, ce_ctrl_addr + SR_BA_ADDRESS, addr);
99}
100
101static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
102 u32 ce_ctrl_addr,
103 unsigned int n)
104{
105 ath10k_pci_write32(ar, ce_ctrl_addr + SR_SIZE_ADDRESS, n);
106}
107
108static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
109 u32 ce_ctrl_addr,
110 unsigned int n)
111{
112 u32 ctrl1_addr = ath10k_pci_read32((ar),
113 (ce_ctrl_addr) + CE_CTRL1_ADDRESS);
114
115 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
116 (ctrl1_addr & ~CE_CTRL1_DMAX_LENGTH_MASK) |
117 CE_CTRL1_DMAX_LENGTH_SET(n));
118}
119
120static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
121 u32 ce_ctrl_addr,
122 unsigned int n)
123{
124 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS);
125
126 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
127 (ctrl1_addr & ~CE_CTRL1_SRC_RING_BYTE_SWAP_EN_MASK) |
128 CE_CTRL1_SRC_RING_BYTE_SWAP_EN_SET(n));
129}
130
131static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
132 u32 ce_ctrl_addr,
133 unsigned int n)
134{
135 u32 ctrl1_addr = ath10k_pci_read32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS);
136
137 ath10k_pci_write32(ar, ce_ctrl_addr + CE_CTRL1_ADDRESS,
138 (ctrl1_addr & ~CE_CTRL1_DST_RING_BYTE_SWAP_EN_MASK) |
139 CE_CTRL1_DST_RING_BYTE_SWAP_EN_SET(n));
140}
141
142static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
143 u32 ce_ctrl_addr)
144{
145 return ath10k_pci_read32(ar, ce_ctrl_addr + CURRENT_DRRI_ADDRESS);
146}
147
148static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
149 u32 ce_ctrl_addr,
150 u32 addr)
151{
152 ath10k_pci_write32(ar, ce_ctrl_addr + DR_BA_ADDRESS, addr);
153}
154
155static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
156 u32 ce_ctrl_addr,
157 unsigned int n)
158{
159 ath10k_pci_write32(ar, ce_ctrl_addr + DR_SIZE_ADDRESS, n);
160}
161
162static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
163 u32 ce_ctrl_addr,
164 unsigned int n)
165{
166 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS);
167
168 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS,
169 (addr & ~SRC_WATERMARK_HIGH_MASK) |
170 SRC_WATERMARK_HIGH_SET(n));
171}
172
173static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
174 u32 ce_ctrl_addr,
175 unsigned int n)
176{
177 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS);
178
179 ath10k_pci_write32(ar, ce_ctrl_addr + SRC_WATERMARK_ADDRESS,
180 (addr & ~SRC_WATERMARK_LOW_MASK) |
181 SRC_WATERMARK_LOW_SET(n));
182}
183
184static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
185 u32 ce_ctrl_addr,
186 unsigned int n)
187{
188 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS);
189
190 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS,
191 (addr & ~DST_WATERMARK_HIGH_MASK) |
192 DST_WATERMARK_HIGH_SET(n));
193}
194
195static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
196 u32 ce_ctrl_addr,
197 unsigned int n)
198{
199 u32 addr = ath10k_pci_read32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS);
200
201 ath10k_pci_write32(ar, ce_ctrl_addr + DST_WATERMARK_ADDRESS,
202 (addr & ~DST_WATERMARK_LOW_MASK) |
203 DST_WATERMARK_LOW_SET(n));
204}
205
206static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
207 u32 ce_ctrl_addr)
208{
209 u32 host_ie_addr = ath10k_pci_read32(ar,
210 ce_ctrl_addr + HOST_IE_ADDRESS);
211
212 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
213 host_ie_addr | HOST_IE_COPY_COMPLETE_MASK);
214}
215
216static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
217 u32 ce_ctrl_addr)
218{
219 u32 host_ie_addr = ath10k_pci_read32(ar,
220 ce_ctrl_addr + HOST_IE_ADDRESS);
221
222 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
223 host_ie_addr & ~HOST_IE_COPY_COMPLETE_MASK);
224}
225
226static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
227 u32 ce_ctrl_addr)
228{
229 u32 host_ie_addr = ath10k_pci_read32(ar,
230 ce_ctrl_addr + HOST_IE_ADDRESS);
231
232 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IE_ADDRESS,
233 host_ie_addr & ~CE_WATERMARK_MASK);
234}
235
236static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
237 u32 ce_ctrl_addr)
238{
239 u32 misc_ie_addr = ath10k_pci_read32(ar,
240 ce_ctrl_addr + MISC_IE_ADDRESS);
241
242 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS,
243 misc_ie_addr | CE_ERROR_MASK);
244}
245
246static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
247 u32 ce_ctrl_addr)
248{
249 u32 misc_ie_addr = ath10k_pci_read32(ar,
250 ce_ctrl_addr + MISC_IE_ADDRESS);
251
252 ath10k_pci_write32(ar, ce_ctrl_addr + MISC_IE_ADDRESS,
253 misc_ie_addr & ~CE_ERROR_MASK);
254}
255
256static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
257 u32 ce_ctrl_addr,
258 unsigned int mask)
259{
260 ath10k_pci_write32(ar, ce_ctrl_addr + HOST_IS_ADDRESS, mask);
261}
262
263
264/*
265 * Guts of ath10k_ce_send, used by both ath10k_ce_send and
266 * ath10k_ce_sendlist_send.
267 * The caller takes responsibility for any needed locking.
268 */
269int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
270 void *per_transfer_context,
271 u32 buffer,
272 unsigned int nbytes,
273 unsigned int transfer_id,
274 unsigned int flags)
275{
276 struct ath10k *ar = ce_state->ar;
277 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
278 struct ce_desc *desc, *sdesc;
279 unsigned int nentries_mask = src_ring->nentries_mask;
280 unsigned int sw_index = src_ring->sw_index;
281 unsigned int write_index = src_ring->write_index;
282 u32 ctrl_addr = ce_state->ctrl_addr;
283 u32 desc_flags = 0;
284 int ret = 0;
285
286 if (nbytes > ce_state->src_sz_max)
287 ath10k_warn("%s: send more we can (nbytes: %d, max: %d)\n",
288 __func__, nbytes, ce_state->src_sz_max);
289
290 ret = ath10k_pci_wake(ar);
291 if (ret)
292 return ret;
293
294 if (unlikely(CE_RING_DELTA(nentries_mask,
295 write_index, sw_index - 1) <= 0)) {
296 ret = -ENOSR;
297 goto exit;
298 }
299
300 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
301 write_index);
302 sdesc = CE_SRC_RING_TO_DESC(src_ring->shadow_base, write_index);
303
304 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
305
306 if (flags & CE_SEND_FLAG_GATHER)
307 desc_flags |= CE_DESC_FLAGS_GATHER;
308 if (flags & CE_SEND_FLAG_BYTE_SWAP)
309 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
310
311 sdesc->addr = __cpu_to_le32(buffer);
312 sdesc->nbytes = __cpu_to_le16(nbytes);
313 sdesc->flags = __cpu_to_le16(desc_flags);
314
315 *desc = *sdesc;
316
317 src_ring->per_transfer_context[write_index] = per_transfer_context;
318
319 /* Update Source Ring Write Index */
320 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
321
322 /* WORKAROUND */
323 if (!(flags & CE_SEND_FLAG_GATHER))
324 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
325
326 src_ring->write_index = write_index;
327exit:
328 ath10k_pci_sleep(ar);
329 return ret;
330}
331
332int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
333 void *per_transfer_context,
334 u32 buffer,
335 unsigned int nbytes,
336 unsigned int transfer_id,
337 unsigned int flags)
338{
339 struct ath10k *ar = ce_state->ar;
340 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
341 int ret;
342
343 spin_lock_bh(&ar_pci->ce_lock);
344 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
345 buffer, nbytes, transfer_id, flags);
346 spin_unlock_bh(&ar_pci->ce_lock);
347
348 return ret;
349}
350
351int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
352{
353 struct ath10k *ar = pipe->ar;
354 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
355 int delta;
356
357 spin_lock_bh(&ar_pci->ce_lock);
358 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
359 pipe->src_ring->write_index,
360 pipe->src_ring->sw_index - 1);
361 spin_unlock_bh(&ar_pci->ce_lock);
362
363 return delta;
364}
365
366int ath10k_ce_recv_buf_enqueue(struct ath10k_ce_pipe *ce_state,
367 void *per_recv_context,
368 u32 buffer)
369{
370 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
371 u32 ctrl_addr = ce_state->ctrl_addr;
372 struct ath10k *ar = ce_state->ar;
373 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
374 unsigned int nentries_mask = dest_ring->nentries_mask;
375 unsigned int write_index;
376 unsigned int sw_index;
377 int ret;
378
379 spin_lock_bh(&ar_pci->ce_lock);
380 write_index = dest_ring->write_index;
381 sw_index = dest_ring->sw_index;
382
383 ret = ath10k_pci_wake(ar);
384 if (ret)
385 goto out;
386
387 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) > 0) {
388 struct ce_desc *base = dest_ring->base_addr_owner_space;
389 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
390
391 /* Update destination descriptor */
392 desc->addr = __cpu_to_le32(buffer);
393 desc->nbytes = 0;
394
395 dest_ring->per_transfer_context[write_index] =
396 per_recv_context;
397
398 /* Update Destination Ring Write Index */
399 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
400 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
401 dest_ring->write_index = write_index;
402 ret = 0;
403 } else {
404 ret = -EIO;
405 }
406 ath10k_pci_sleep(ar);
407
408out:
409 spin_unlock_bh(&ar_pci->ce_lock);
410
411 return ret;
412}
413
414/*
415 * Guts of ath10k_ce_completed_recv_next.
416 * The caller takes responsibility for any necessary locking.
417 */
418static int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
419 void **per_transfer_contextp,
420 u32 *bufferp,
421 unsigned int *nbytesp,
422 unsigned int *transfer_idp,
423 unsigned int *flagsp)
424{
425 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
426 unsigned int nentries_mask = dest_ring->nentries_mask;
427 unsigned int sw_index = dest_ring->sw_index;
428
429 struct ce_desc *base = dest_ring->base_addr_owner_space;
430 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
431 struct ce_desc sdesc;
432 u16 nbytes;
433
434 /* Copy in one go for performance reasons */
435 sdesc = *desc;
436
437 nbytes = __le16_to_cpu(sdesc.nbytes);
438 if (nbytes == 0) {
439 /*
440 * This closes a relatively unusual race where the Host
441 * sees the updated DRRI before the update to the
442 * corresponding descriptor has completed. We treat this
443 * as a descriptor that is not yet done.
444 */
445 return -EIO;
446 }
447
448 desc->nbytes = 0;
449
450 /* Return data from completed destination descriptor */
451 *bufferp = __le32_to_cpu(sdesc.addr);
452 *nbytesp = nbytes;
453 *transfer_idp = MS(__le16_to_cpu(sdesc.flags), CE_DESC_FLAGS_META_DATA);
454
455 if (__le16_to_cpu(sdesc.flags) & CE_DESC_FLAGS_BYTE_SWAP)
456 *flagsp = CE_RECV_FLAG_SWAPPED;
457 else
458 *flagsp = 0;
459
460 if (per_transfer_contextp)
461 *per_transfer_contextp =
462 dest_ring->per_transfer_context[sw_index];
463
464 /* sanity */
465 dest_ring->per_transfer_context[sw_index] = NULL;
466
467 /* Update sw_index */
468 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
469 dest_ring->sw_index = sw_index;
470
471 return 0;
472}
473
474int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
475 void **per_transfer_contextp,
476 u32 *bufferp,
477 unsigned int *nbytesp,
478 unsigned int *transfer_idp,
479 unsigned int *flagsp)
480{
481 struct ath10k *ar = ce_state->ar;
482 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
483 int ret;
484
485 spin_lock_bh(&ar_pci->ce_lock);
486 ret = ath10k_ce_completed_recv_next_nolock(ce_state,
487 per_transfer_contextp,
488 bufferp, nbytesp,
489 transfer_idp, flagsp);
490 spin_unlock_bh(&ar_pci->ce_lock);
491
492 return ret;
493}
494
495int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
496 void **per_transfer_contextp,
497 u32 *bufferp)
498{
499 struct ath10k_ce_ring *dest_ring;
500 unsigned int nentries_mask;
501 unsigned int sw_index;
502 unsigned int write_index;
503 int ret;
504 struct ath10k *ar;
505 struct ath10k_pci *ar_pci;
506
507 dest_ring = ce_state->dest_ring;
508
509 if (!dest_ring)
510 return -EIO;
511
512 ar = ce_state->ar;
513 ar_pci = ath10k_pci_priv(ar);
514
515 spin_lock_bh(&ar_pci->ce_lock);
516
517 nentries_mask = dest_ring->nentries_mask;
518 sw_index = dest_ring->sw_index;
519 write_index = dest_ring->write_index;
520 if (write_index != sw_index) {
521 struct ce_desc *base = dest_ring->base_addr_owner_space;
522 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
523
524 /* Return data from completed destination descriptor */
525 *bufferp = __le32_to_cpu(desc->addr);
526
527 if (per_transfer_contextp)
528 *per_transfer_contextp =
529 dest_ring->per_transfer_context[sw_index];
530
531 /* sanity */
532 dest_ring->per_transfer_context[sw_index] = NULL;
533
534 /* Update sw_index */
535 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
536 dest_ring->sw_index = sw_index;
537 ret = 0;
538 } else {
539 ret = -EIO;
540 }
541
542 spin_unlock_bh(&ar_pci->ce_lock);
543
544 return ret;
545}
546
547/*
548 * Guts of ath10k_ce_completed_send_next.
549 * The caller takes responsibility for any necessary locking.
550 */
551static int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
552 void **per_transfer_contextp,
553 u32 *bufferp,
554 unsigned int *nbytesp,
555 unsigned int *transfer_idp)
556{
557 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
558 u32 ctrl_addr = ce_state->ctrl_addr;
559 struct ath10k *ar = ce_state->ar;
560 unsigned int nentries_mask = src_ring->nentries_mask;
561 unsigned int sw_index = src_ring->sw_index;
562 struct ce_desc *sdesc, *sbase;
563 unsigned int read_index;
564 int ret;
565
566 if (src_ring->hw_index == sw_index) {
567 /*
568 * The SW completion index has caught up with the cached
569 * version of the HW completion index.
570 * Update the cached HW completion index to see whether
571 * the SW has really caught up to the HW, or if the cached
572 * value of the HW index has become stale.
573 */
574
575 ret = ath10k_pci_wake(ar);
576 if (ret)
577 return ret;
578
579 src_ring->hw_index =
580 ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
581 src_ring->hw_index &= nentries_mask;
582
583 ath10k_pci_sleep(ar);
584 }
585
586 read_index = src_ring->hw_index;
587
588 if ((read_index == sw_index) || (read_index == 0xffffffff))
589 return -EIO;
590
591 sbase = src_ring->shadow_base;
592 sdesc = CE_SRC_RING_TO_DESC(sbase, sw_index);
593
594 /* Return data from completed source descriptor */
595 *bufferp = __le32_to_cpu(sdesc->addr);
596 *nbytesp = __le16_to_cpu(sdesc->nbytes);
597 *transfer_idp = MS(__le16_to_cpu(sdesc->flags),
598 CE_DESC_FLAGS_META_DATA);
599
600 if (per_transfer_contextp)
601 *per_transfer_contextp =
602 src_ring->per_transfer_context[sw_index];
603
604 /* sanity */
605 src_ring->per_transfer_context[sw_index] = NULL;
606
607 /* Update sw_index */
608 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
609 src_ring->sw_index = sw_index;
610
611 return 0;
612}
613
614/* NB: Modeled after ath10k_ce_completed_send_next */
615int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
616 void **per_transfer_contextp,
617 u32 *bufferp,
618 unsigned int *nbytesp,
619 unsigned int *transfer_idp)
620{
621 struct ath10k_ce_ring *src_ring;
622 unsigned int nentries_mask;
623 unsigned int sw_index;
624 unsigned int write_index;
625 int ret;
626 struct ath10k *ar;
627 struct ath10k_pci *ar_pci;
628
629 src_ring = ce_state->src_ring;
630
631 if (!src_ring)
632 return -EIO;
633
634 ar = ce_state->ar;
635 ar_pci = ath10k_pci_priv(ar);
636
637 spin_lock_bh(&ar_pci->ce_lock);
638
639 nentries_mask = src_ring->nentries_mask;
640 sw_index = src_ring->sw_index;
641 write_index = src_ring->write_index;
642
643 if (write_index != sw_index) {
644 struct ce_desc *base = src_ring->base_addr_owner_space;
645 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
646
647 /* Return data from completed source descriptor */
648 *bufferp = __le32_to_cpu(desc->addr);
649 *nbytesp = __le16_to_cpu(desc->nbytes);
650 *transfer_idp = MS(__le16_to_cpu(desc->flags),
651 CE_DESC_FLAGS_META_DATA);
652
653 if (per_transfer_contextp)
654 *per_transfer_contextp =
655 src_ring->per_transfer_context[sw_index];
656
657 /* sanity */
658 src_ring->per_transfer_context[sw_index] = NULL;
659
660 /* Update sw_index */
661 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
662 src_ring->sw_index = sw_index;
663 ret = 0;
664 } else {
665 ret = -EIO;
666 }
667
668 spin_unlock_bh(&ar_pci->ce_lock);
669
670 return ret;
671}
672
673int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
674 void **per_transfer_contextp,
675 u32 *bufferp,
676 unsigned int *nbytesp,
677 unsigned int *transfer_idp)
678{
679 struct ath10k *ar = ce_state->ar;
680 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
681 int ret;
682
683 spin_lock_bh(&ar_pci->ce_lock);
684 ret = ath10k_ce_completed_send_next_nolock(ce_state,
685 per_transfer_contextp,
686 bufferp, nbytesp,
687 transfer_idp);
688 spin_unlock_bh(&ar_pci->ce_lock);
689
690 return ret;
691}
692
693/*
694 * Guts of interrupt handler for per-engine interrupts on a particular CE.
695 *
696 * Invokes registered callbacks for recv_complete,
697 * send_complete, and watermarks.
698 */
699void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
700{
701 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
702 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
703 u32 ctrl_addr = ce_state->ctrl_addr;
704 int ret;
705
706 ret = ath10k_pci_wake(ar);
707 if (ret)
708 return;
709
710 spin_lock_bh(&ar_pci->ce_lock);
711
712 /* Clear the copy-complete interrupts that will be handled here. */
713 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
714 HOST_IS_COPY_COMPLETE_MASK);
715
716 spin_unlock_bh(&ar_pci->ce_lock);
717
718 if (ce_state->recv_cb)
719 ce_state->recv_cb(ce_state);
720
721 if (ce_state->send_cb)
722 ce_state->send_cb(ce_state);
723
724 spin_lock_bh(&ar_pci->ce_lock);
725
726 /*
727 * Misc CE interrupts are not being handled, but still need
728 * to be cleared.
729 */
730 ath10k_ce_engine_int_status_clear(ar, ctrl_addr, CE_WATERMARK_MASK);
731
732 spin_unlock_bh(&ar_pci->ce_lock);
733 ath10k_pci_sleep(ar);
734}
735
736/*
737 * Handler for per-engine interrupts on ALL active CEs.
738 * This is used in cases where the system is sharing a
739 * single interrput for all CEs
740 */
741
742void ath10k_ce_per_engine_service_any(struct ath10k *ar)
743{
744 int ce_id, ret;
745 u32 intr_summary;
746
747 ret = ath10k_pci_wake(ar);
748 if (ret)
749 return;
750
751 intr_summary = CE_INTERRUPT_SUMMARY(ar);
752
753 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
754 if (intr_summary & (1 << ce_id))
755 intr_summary &= ~(1 << ce_id);
756 else
757 /* no intr pending on this CE */
758 continue;
759
760 ath10k_ce_per_engine_service(ar, ce_id);
761 }
762
763 ath10k_pci_sleep(ar);
764}
765
766/*
767 * Adjust interrupts for the copy complete handler.
768 * If it's needed for either send or recv, then unmask
769 * this interrupt; otherwise, mask it.
770 *
771 * Called with ce_lock held.
772 */
773static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state,
774 int disable_copy_compl_intr)
775{
776 u32 ctrl_addr = ce_state->ctrl_addr;
777 struct ath10k *ar = ce_state->ar;
778 int ret;
779
780 ret = ath10k_pci_wake(ar);
781 if (ret)
782 return;
783
784 if ((!disable_copy_compl_intr) &&
785 (ce_state->send_cb || ce_state->recv_cb))
786 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
787 else
788 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
789
790 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
791
792 ath10k_pci_sleep(ar);
793}
794
795int ath10k_ce_disable_interrupts(struct ath10k *ar)
796{
797 int ce_id, ret;
798
799 ret = ath10k_pci_wake(ar);
800 if (ret)
801 return ret;
802
803 for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
804 u32 ctrl_addr = ath10k_ce_base_address(ce_id);
805
806 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
807 ath10k_ce_error_intr_disable(ar, ctrl_addr);
808 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
809 }
810
811 ath10k_pci_sleep(ar);
812
813 return 0;
814}
815
816void ath10k_ce_send_cb_register(struct ath10k_ce_pipe *ce_state,
817 void (*send_cb)(struct ath10k_ce_pipe *),
818 int disable_interrupts)
819{
820 struct ath10k *ar = ce_state->ar;
821 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
822
823 spin_lock_bh(&ar_pci->ce_lock);
824 ce_state->send_cb = send_cb;
825 ath10k_ce_per_engine_handler_adjust(ce_state, disable_interrupts);
826 spin_unlock_bh(&ar_pci->ce_lock);
827}
828
829void ath10k_ce_recv_cb_register(struct ath10k_ce_pipe *ce_state,
830 void (*recv_cb)(struct ath10k_ce_pipe *))
831{
832 struct ath10k *ar = ce_state->ar;
833 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
834
835 spin_lock_bh(&ar_pci->ce_lock);
836 ce_state->recv_cb = recv_cb;
837 ath10k_ce_per_engine_handler_adjust(ce_state, 0);
838 spin_unlock_bh(&ar_pci->ce_lock);
839}
840
841static int ath10k_ce_init_src_ring(struct ath10k *ar,
842 unsigned int ce_id,
843 struct ath10k_ce_pipe *ce_state,
844 const struct ce_attr *attr)
845{
846 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
847 struct ath10k_ce_ring *src_ring;
848 unsigned int nentries = attr->src_nentries;
849 unsigned int ce_nbytes;
850 u32 ctrl_addr = ath10k_ce_base_address(ce_id);
851 dma_addr_t base_addr;
852 char *ptr;
853
854 nentries = roundup_pow_of_two(nentries);
855
856 if (ce_state->src_ring) {
857 WARN_ON(ce_state->src_ring->nentries != nentries);
858 return 0;
859 }
860
861 ce_nbytes = sizeof(struct ath10k_ce_ring) + (nentries * sizeof(void *));
862 ptr = kzalloc(ce_nbytes, GFP_KERNEL);
863 if (ptr == NULL)
864 return -ENOMEM;
865
866 ce_state->src_ring = (struct ath10k_ce_ring *)ptr;
867 src_ring = ce_state->src_ring;
868
869 ptr += sizeof(struct ath10k_ce_ring);
870 src_ring->nentries = nentries;
871 src_ring->nentries_mask = nentries - 1;
872
873 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
874 src_ring->sw_index &= src_ring->nentries_mask;
875 src_ring->hw_index = src_ring->sw_index;
876
877 src_ring->write_index =
878 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
879 src_ring->write_index &= src_ring->nentries_mask;
880
881 src_ring->per_transfer_context = (void **)ptr;
882
883 /*
884 * Legacy platforms that do not support cache
885 * coherent DMA are unsupported
886 */
887 src_ring->base_addr_owner_space_unaligned =
888 pci_alloc_consistent(ar_pci->pdev,
889 (nentries * sizeof(struct ce_desc) +
890 CE_DESC_RING_ALIGN),
891 &base_addr);
892 if (!src_ring->base_addr_owner_space_unaligned) {
893 kfree(ce_state->src_ring);
894 ce_state->src_ring = NULL;
895 return -ENOMEM;
896 }
897
898 src_ring->base_addr_ce_space_unaligned = base_addr;
899
900 src_ring->base_addr_owner_space = PTR_ALIGN(
901 src_ring->base_addr_owner_space_unaligned,
902 CE_DESC_RING_ALIGN);
903 src_ring->base_addr_ce_space = ALIGN(
904 src_ring->base_addr_ce_space_unaligned,
905 CE_DESC_RING_ALIGN);
906
907 /*
908 * Also allocate a shadow src ring in regular
909 * mem to use for faster access.
910 */
911 src_ring->shadow_base_unaligned =
912 kmalloc((nentries * sizeof(struct ce_desc) +
913 CE_DESC_RING_ALIGN), GFP_KERNEL);
914 if (!src_ring->shadow_base_unaligned) {
915 pci_free_consistent(ar_pci->pdev,
916 (nentries * sizeof(struct ce_desc) +
917 CE_DESC_RING_ALIGN),
918 src_ring->base_addr_owner_space,
919 src_ring->base_addr_ce_space);
920 kfree(ce_state->src_ring);
921 ce_state->src_ring = NULL;
922 return -ENOMEM;
923 }
924
925 src_ring->shadow_base = PTR_ALIGN(
926 src_ring->shadow_base_unaligned,
927 CE_DESC_RING_ALIGN);
928
929 ath10k_ce_src_ring_base_addr_set(ar, ctrl_addr,
930 src_ring->base_addr_ce_space);
931 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
932 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
933 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
934 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
935 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
936
937 ath10k_dbg(ATH10K_DBG_BOOT,
938 "boot ce src ring id %d entries %d base_addr %p\n",
939 ce_id, nentries, src_ring->base_addr_owner_space);
940
941 return 0;
942}
943
944static int ath10k_ce_init_dest_ring(struct ath10k *ar,
945 unsigned int ce_id,
946 struct ath10k_ce_pipe *ce_state,
947 const struct ce_attr *attr)
948{
949 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
950 struct ath10k_ce_ring *dest_ring;
951 unsigned int nentries = attr->dest_nentries;
952 unsigned int ce_nbytes;
953 u32 ctrl_addr = ath10k_ce_base_address(ce_id);
954 dma_addr_t base_addr;
955 char *ptr;
956
957 nentries = roundup_pow_of_two(nentries);
958
959 if (ce_state->dest_ring) {
960 WARN_ON(ce_state->dest_ring->nentries != nentries);
961 return 0;
962 }
963
964 ce_nbytes = sizeof(struct ath10k_ce_ring) + (nentries * sizeof(void *));
965 ptr = kzalloc(ce_nbytes, GFP_KERNEL);
966 if (ptr == NULL)
967 return -ENOMEM;
968
969 ce_state->dest_ring = (struct ath10k_ce_ring *)ptr;
970 dest_ring = ce_state->dest_ring;
971
972 ptr += sizeof(struct ath10k_ce_ring);
973 dest_ring->nentries = nentries;
974 dest_ring->nentries_mask = nentries - 1;
975
976 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
977 dest_ring->sw_index &= dest_ring->nentries_mask;
978 dest_ring->write_index =
979 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
980 dest_ring->write_index &= dest_ring->nentries_mask;
981
982 dest_ring->per_transfer_context = (void **)ptr;
983
984 /*
985 * Legacy platforms that do not support cache
986 * coherent DMA are unsupported
987 */
988 dest_ring->base_addr_owner_space_unaligned =
989 pci_alloc_consistent(ar_pci->pdev,
990 (nentries * sizeof(struct ce_desc) +
991 CE_DESC_RING_ALIGN),
992 &base_addr);
993 if (!dest_ring->base_addr_owner_space_unaligned) {
994 kfree(ce_state->dest_ring);
995 ce_state->dest_ring = NULL;
996 return -ENOMEM;
997 }
998
999 dest_ring->base_addr_ce_space_unaligned = base_addr;
1000
1001 /*
1002 * Correctly initialize memory to 0 to prevent garbage
1003 * data crashing system when download firmware
1004 */
1005 memset(dest_ring->base_addr_owner_space_unaligned, 0,
1006 nentries * sizeof(struct ce_desc) + CE_DESC_RING_ALIGN);
1007
1008 dest_ring->base_addr_owner_space = PTR_ALIGN(
1009 dest_ring->base_addr_owner_space_unaligned,
1010 CE_DESC_RING_ALIGN);
1011 dest_ring->base_addr_ce_space = ALIGN(
1012 dest_ring->base_addr_ce_space_unaligned,
1013 CE_DESC_RING_ALIGN);
1014
1015 ath10k_ce_dest_ring_base_addr_set(ar, ctrl_addr,
1016 dest_ring->base_addr_ce_space);
1017 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1018 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1019 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1020 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1021
1022 ath10k_dbg(ATH10K_DBG_BOOT,
1023 "boot ce dest ring id %d entries %d base_addr %p\n",
1024 ce_id, nentries, dest_ring->base_addr_owner_space);
1025
1026 return 0;
1027}
1028
1029static struct ath10k_ce_pipe *ath10k_ce_init_state(struct ath10k *ar,
1030 unsigned int ce_id,
1031 const struct ce_attr *attr)
1032{
1033 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1034 struct ath10k_ce_pipe *ce_state = &ar_pci->ce_states[ce_id];
1035 u32 ctrl_addr = ath10k_ce_base_address(ce_id);
1036
1037 spin_lock_bh(&ar_pci->ce_lock);
1038
1039 ce_state->ar = ar;
1040 ce_state->id = ce_id;
1041 ce_state->ctrl_addr = ctrl_addr;
1042 ce_state->attr_flags = attr->flags;
1043 ce_state->src_sz_max = attr->src_sz_max;
1044
1045 spin_unlock_bh(&ar_pci->ce_lock);
1046
1047 return ce_state;
1048}
1049
1050/*
1051 * Initialize a Copy Engine based on caller-supplied attributes.
1052 * This may be called once to initialize both source and destination
1053 * rings or it may be called twice for separate source and destination
1054 * initialization. It may be that only one side or the other is
1055 * initialized by software/firmware.
1056 */
1057struct ath10k_ce_pipe *ath10k_ce_init(struct ath10k *ar,
1058 unsigned int ce_id,
1059 const struct ce_attr *attr)
1060{
1061 struct ath10k_ce_pipe *ce_state;
1062 int ret;
1063
1064 /*
1065 * Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1066 * additional TX locking checks.
1067 *
1068 * For the lack of a better place do the check here.
1069 */
1070 BUILD_BUG_ON(2*TARGET_NUM_MSDU_DESC >
1071 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1072 BUILD_BUG_ON(2*TARGET_10X_NUM_MSDU_DESC >
1073 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1074
1075 ret = ath10k_pci_wake(ar);
1076 if (ret)
1077 return NULL;
1078
1079 ce_state = ath10k_ce_init_state(ar, ce_id, attr);
1080 if (!ce_state) {
1081 ath10k_err("Failed to initialize CE state for ID: %d\n", ce_id);
1082 goto out;
1083 }
1084
1085 if (attr->src_nentries) {
1086 ret = ath10k_ce_init_src_ring(ar, ce_id, ce_state, attr);
1087 if (ret) {
1088 ath10k_err("Failed to initialize CE src ring for ID: %d (%d)\n",
1089 ce_id, ret);
1090 ath10k_ce_deinit(ce_state);
1091 ce_state = NULL;
1092 goto out;
1093 }
1094 }
1095
1096 if (attr->dest_nentries) {
1097 ret = ath10k_ce_init_dest_ring(ar, ce_id, ce_state, attr);
1098 if (ret) {
1099 ath10k_err("Failed to initialize CE dest ring for ID: %d (%d)\n",
1100 ce_id, ret);
1101 ath10k_ce_deinit(ce_state);
1102 ce_state = NULL;
1103 goto out;
1104 }
1105 }
1106
1107out:
1108 ath10k_pci_sleep(ar);
1109 return ce_state;
1110}
1111
1112void ath10k_ce_deinit(struct ath10k_ce_pipe *ce_state)
1113{
1114 struct ath10k *ar = ce_state->ar;
1115 struct ath10k_pci *ar_pci = ath10k_pci_priv(ar);
1116
1117 if (ce_state->src_ring) {
1118 kfree(ce_state->src_ring->shadow_base_unaligned);
1119 pci_free_consistent(ar_pci->pdev,
1120 (ce_state->src_ring->nentries *
1121 sizeof(struct ce_desc) +
1122 CE_DESC_RING_ALIGN),
1123 ce_state->src_ring->base_addr_owner_space,
1124 ce_state->src_ring->base_addr_ce_space);
1125 kfree(ce_state->src_ring);
1126 }
1127
1128 if (ce_state->dest_ring) {
1129 pci_free_consistent(ar_pci->pdev,
1130 (ce_state->dest_ring->nentries *
1131 sizeof(struct ce_desc) +
1132 CE_DESC_RING_ALIGN),
1133 ce_state->dest_ring->base_addr_owner_space,
1134 ce_state->dest_ring->base_addr_ce_space);
1135 kfree(ce_state->dest_ring);
1136 }
1137
1138 ce_state->src_ring = NULL;
1139 ce_state->dest_ring = NULL;
1140}