Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: ISC
2/*
3 * Copyright (c) 2005-2011 Atheros Communications Inc.
4 * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
5 * Copyright (c) 2018 The Linux Foundation. All rights reserved.
6 */
7
8#include "hif.h"
9#include "ce.h"
10#include "debug.h"
11
12/*
13 * Support for Copy Engine hardware, which is mainly used for
14 * communication between Host and Target over a PCIe interconnect.
15 */
16
17/*
18 * A single CopyEngine (CE) comprises two "rings":
19 * a source ring
20 * a destination ring
21 *
22 * Each ring consists of a number of descriptors which specify
23 * an address, length, and meta-data.
24 *
25 * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
26 * controls one ring and the other side controls the other ring.
27 * The source side chooses when to initiate a transfer and it
28 * chooses what to send (buffer address, length). The destination
29 * side keeps a supply of "anonymous receive buffers" available and
30 * it handles incoming data as it arrives (when the destination
31 * receives an interrupt).
32 *
33 * The sender may send a simple buffer (address/length) or it may
34 * send a small list of buffers. When a small list is sent, hardware
35 * "gathers" these and they end up in a single destination buffer
36 * with a single interrupt.
37 *
38 * There are several "contexts" managed by this layer -- more, it
39 * may seem -- than should be needed. These are provided mainly for
40 * maximum flexibility and especially to facilitate a simpler HIF
41 * implementation. There are per-CopyEngine recv, send, and watermark
42 * contexts. These are supplied by the caller when a recv, send,
43 * or watermark handler is established and they are echoed back to
44 * the caller when the respective callbacks are invoked. There is
45 * also a per-transfer context supplied by the caller when a buffer
46 * (or sendlist) is sent and when a buffer is enqueued for recv.
47 * These per-transfer contexts are echoed back to the caller when
48 * the buffer is sent/received.
49 */
50
51static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
52 struct ath10k_ce_pipe *ce_state)
53{
54 u32 ce_id = ce_state->id;
55 u32 addr = 0;
56
57 switch (ce_id) {
58 case 0:
59 addr = 0x00032000;
60 break;
61 case 3:
62 addr = 0x0003200C;
63 break;
64 case 4:
65 addr = 0x00032010;
66 break;
67 case 5:
68 addr = 0x00032014;
69 break;
70 case 7:
71 addr = 0x0003201C;
72 break;
73 default:
74 ath10k_warn(ar, "invalid CE id: %d", ce_id);
75 break;
76 }
77 return addr;
78}
79
80static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
81 struct ath10k_ce_pipe *ce_state)
82{
83 u32 ce_id = ce_state->id;
84 u32 addr = 0;
85
86 switch (ce_id) {
87 case 1:
88 addr = 0x00032034;
89 break;
90 case 2:
91 addr = 0x00032038;
92 break;
93 case 5:
94 addr = 0x00032044;
95 break;
96 case 7:
97 addr = 0x0003204C;
98 break;
99 case 8:
100 addr = 0x00032050;
101 break;
102 case 9:
103 addr = 0x00032054;
104 break;
105 case 10:
106 addr = 0x00032058;
107 break;
108 case 11:
109 addr = 0x0003205C;
110 break;
111 default:
112 ath10k_warn(ar, "invalid CE id: %d", ce_id);
113 break;
114 }
115
116 return addr;
117}
118
119static inline unsigned int
120ath10k_set_ring_byte(unsigned int offset,
121 struct ath10k_hw_ce_regs_addr_map *addr_map)
122{
123 return ((offset << addr_map->lsb) & addr_map->mask);
124}
125
126static inline unsigned int
127ath10k_get_ring_byte(unsigned int offset,
128 struct ath10k_hw_ce_regs_addr_map *addr_map)
129{
130 return ((offset & addr_map->mask) >> (addr_map->lsb));
131}
132
133static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
134{
135 struct ath10k_ce *ce = ath10k_ce_priv(ar);
136
137 return ce->bus_ops->read32(ar, offset);
138}
139
140static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
141{
142 struct ath10k_ce *ce = ath10k_ce_priv(ar);
143
144 ce->bus_ops->write32(ar, offset, value);
145}
146
147static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
148 u32 ce_ctrl_addr,
149 unsigned int n)
150{
151 ath10k_ce_write32(ar, ce_ctrl_addr +
152 ar->hw_ce_regs->dst_wr_index_addr, n);
153}
154
155static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
156 u32 ce_ctrl_addr)
157{
158 return ath10k_ce_read32(ar, ce_ctrl_addr +
159 ar->hw_ce_regs->dst_wr_index_addr);
160}
161
162static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
163 u32 ce_ctrl_addr,
164 unsigned int n)
165{
166 ath10k_ce_write32(ar, ce_ctrl_addr +
167 ar->hw_ce_regs->sr_wr_index_addr, n);
168}
169
170static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
171 u32 ce_ctrl_addr)
172{
173 return ath10k_ce_read32(ar, ce_ctrl_addr +
174 ar->hw_ce_regs->sr_wr_index_addr);
175}
176
177static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
178 u32 ce_id)
179{
180 struct ath10k_ce *ce = ath10k_ce_priv(ar);
181
182 return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
183}
184
185static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
186 u32 ce_ctrl_addr)
187{
188 struct ath10k_ce *ce = ath10k_ce_priv(ar);
189 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
190 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
191 u32 index;
192
193 if (ar->hw_params.rri_on_ddr &&
194 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
195 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
196 else
197 index = ath10k_ce_read32(ar, ce_ctrl_addr +
198 ar->hw_ce_regs->current_srri_addr);
199
200 return index;
201}
202
203static inline void
204ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
205 struct ath10k_ce_pipe *ce_state,
206 unsigned int value)
207{
208 ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
209}
210
211static inline void
212ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar,
213 struct ath10k_ce_pipe *ce_state,
214 unsigned int value)
215{
216 ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);
217}
218
219static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
220 u32 ce_id,
221 u64 addr)
222{
223 struct ath10k_ce *ce = ath10k_ce_priv(ar);
224 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
225 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
226 u32 addr_lo = lower_32_bits(addr);
227
228 ath10k_ce_write32(ar, ce_ctrl_addr +
229 ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
230
231 if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
232 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
233 addr);
234 }
235}
236
237static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
238 u32 ce_ctrl_addr,
239 u64 addr)
240{
241 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
242
243 ath10k_ce_write32(ar, ce_ctrl_addr +
244 ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
245}
246
247static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
248 u32 ce_ctrl_addr,
249 unsigned int n)
250{
251 ath10k_ce_write32(ar, ce_ctrl_addr +
252 ar->hw_ce_regs->sr_size_addr, n);
253}
254
255static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
256 u32 ce_ctrl_addr,
257 unsigned int n)
258{
259 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
260
261 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
262 ctrl_regs->addr);
263
264 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
265 (ctrl1_addr & ~(ctrl_regs->dmax->mask)) |
266 ath10k_set_ring_byte(n, ctrl_regs->dmax));
267}
268
269static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
270 u32 ce_ctrl_addr,
271 unsigned int n)
272{
273 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
274
275 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
276 ctrl_regs->addr);
277
278 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
279 (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
280 ath10k_set_ring_byte(n, ctrl_regs->src_ring));
281}
282
283static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
284 u32 ce_ctrl_addr,
285 unsigned int n)
286{
287 struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
288
289 u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
290 ctrl_regs->addr);
291
292 ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
293 (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
294 ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
295}
296
297static inline
298 u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
299{
300 struct ath10k_ce *ce = ath10k_ce_priv(ar);
301
302 return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
303 CE_DDR_RRI_MASK;
304}
305
306static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
307 u32 ce_ctrl_addr)
308{
309 struct ath10k_ce *ce = ath10k_ce_priv(ar);
310 u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
311 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
312 u32 index;
313
314 if (ar->hw_params.rri_on_ddr &&
315 (ce_state->attr_flags & CE_ATTR_DIS_INTR))
316 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
317 else
318 index = ath10k_ce_read32(ar, ce_ctrl_addr +
319 ar->hw_ce_regs->current_drri_addr);
320
321 return index;
322}
323
324static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
325 u32 ce_id,
326 u64 addr)
327{
328 struct ath10k_ce *ce = ath10k_ce_priv(ar);
329 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
330 u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
331 u32 addr_lo = lower_32_bits(addr);
332
333 ath10k_ce_write32(ar, ce_ctrl_addr +
334 ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
335
336 if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
337 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
338 addr);
339 }
340}
341
342static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
343 u32 ce_ctrl_addr,
344 u64 addr)
345{
346 u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
347 u32 reg_value;
348
349 reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
350 ar->hw_ce_regs->dr_base_addr_hi);
351 reg_value &= ~CE_DESC_ADDR_HI_MASK;
352 reg_value |= addr_hi;
353 ath10k_ce_write32(ar, ce_ctrl_addr +
354 ar->hw_ce_regs->dr_base_addr_hi, reg_value);
355}
356
357static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
358 u32 ce_ctrl_addr,
359 unsigned int n)
360{
361 ath10k_ce_write32(ar, ce_ctrl_addr +
362 ar->hw_ce_regs->dr_size_addr, n);
363}
364
365static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
366 u32 ce_ctrl_addr,
367 unsigned int n)
368{
369 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
370 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
371
372 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
373 (addr & ~(srcr_wm->wm_high->mask)) |
374 (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
375}
376
377static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
378 u32 ce_ctrl_addr,
379 unsigned int n)
380{
381 struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
382 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
383
384 ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
385 (addr & ~(srcr_wm->wm_low->mask)) |
386 (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
387}
388
389static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
390 u32 ce_ctrl_addr,
391 unsigned int n)
392{
393 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
394 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
395
396 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
397 (addr & ~(dstr_wm->wm_high->mask)) |
398 (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
399}
400
401static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
402 u32 ce_ctrl_addr,
403 unsigned int n)
404{
405 struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
406 u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
407
408 ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
409 (addr & ~(dstr_wm->wm_low->mask)) |
410 (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
411}
412
413static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
414 u32 ce_ctrl_addr)
415{
416 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
417
418 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
419 ar->hw_ce_regs->host_ie_addr);
420
421 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
422 host_ie_addr | host_ie->copy_complete->mask);
423}
424
425static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
426 u32 ce_ctrl_addr)
427{
428 struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
429
430 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
431 ar->hw_ce_regs->host_ie_addr);
432
433 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
434 host_ie_addr & ~(host_ie->copy_complete->mask));
435}
436
437static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
438 u32 ce_ctrl_addr)
439{
440 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
441
442 u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
443 ar->hw_ce_regs->host_ie_addr);
444
445 ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
446 host_ie_addr & ~(wm_regs->wm_mask));
447}
448
449static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
450 u32 ce_ctrl_addr)
451{
452 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
453
454 u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
455 ar->hw_ce_regs->misc_ie_addr);
456
457 ath10k_ce_write32(ar,
458 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
459 misc_ie_addr | misc_regs->err_mask);
460}
461
462static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
463 u32 ce_ctrl_addr)
464{
465 struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
466
467 u32 misc_ie_addr = ath10k_ce_read32(ar,
468 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
469
470 ath10k_ce_write32(ar,
471 ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
472 misc_ie_addr & ~(misc_regs->err_mask));
473}
474
475static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
476 u32 ce_ctrl_addr,
477 unsigned int mask)
478{
479 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
480
481 ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
482}
483
484/*
485 * Guts of ath10k_ce_send.
486 * The caller takes responsibility for any needed locking.
487 */
488static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
489 void *per_transfer_context,
490 dma_addr_t buffer,
491 unsigned int nbytes,
492 unsigned int transfer_id,
493 unsigned int flags)
494{
495 struct ath10k *ar = ce_state->ar;
496 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
497 struct ce_desc *desc, sdesc;
498 unsigned int nentries_mask = src_ring->nentries_mask;
499 unsigned int sw_index = src_ring->sw_index;
500 unsigned int write_index = src_ring->write_index;
501 u32 ctrl_addr = ce_state->ctrl_addr;
502 u32 desc_flags = 0;
503 int ret = 0;
504
505 if (nbytes > ce_state->src_sz_max)
506 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
507 __func__, nbytes, ce_state->src_sz_max);
508
509 if (unlikely(CE_RING_DELTA(nentries_mask,
510 write_index, sw_index - 1) <= 0)) {
511 ret = -ENOSR;
512 goto exit;
513 }
514
515 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
516 write_index);
517
518 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
519
520 if (flags & CE_SEND_FLAG_GATHER)
521 desc_flags |= CE_DESC_FLAGS_GATHER;
522 if (flags & CE_SEND_FLAG_BYTE_SWAP)
523 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
524
525 sdesc.addr = __cpu_to_le32(buffer);
526 sdesc.nbytes = __cpu_to_le16(nbytes);
527 sdesc.flags = __cpu_to_le16(desc_flags);
528
529 *desc = sdesc;
530
531 src_ring->per_transfer_context[write_index] = per_transfer_context;
532
533 /* Update Source Ring Write Index */
534 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
535
536 /* WORKAROUND */
537 if (!(flags & CE_SEND_FLAG_GATHER))
538 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
539
540 src_ring->write_index = write_index;
541exit:
542 return ret;
543}
544
545static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
546 void *per_transfer_context,
547 dma_addr_t buffer,
548 unsigned int nbytes,
549 unsigned int transfer_id,
550 unsigned int flags)
551{
552 struct ath10k *ar = ce_state->ar;
553 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
554 struct ce_desc_64 *desc, sdesc;
555 unsigned int nentries_mask = src_ring->nentries_mask;
556 unsigned int sw_index;
557 unsigned int write_index = src_ring->write_index;
558 u32 ctrl_addr = ce_state->ctrl_addr;
559 __le32 *addr;
560 u32 desc_flags = 0;
561 int ret = 0;
562
563 if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
564 return -ESHUTDOWN;
565
566 if (nbytes > ce_state->src_sz_max)
567 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
568 __func__, nbytes, ce_state->src_sz_max);
569
570 if (ar->hw_params.rri_on_ddr)
571 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
572 else
573 sw_index = src_ring->sw_index;
574
575 if (unlikely(CE_RING_DELTA(nentries_mask,
576 write_index, sw_index - 1) <= 0)) {
577 ret = -ENOSR;
578 goto exit;
579 }
580
581 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
582 write_index);
583
584 desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
585
586 if (flags & CE_SEND_FLAG_GATHER)
587 desc_flags |= CE_DESC_FLAGS_GATHER;
588
589 if (flags & CE_SEND_FLAG_BYTE_SWAP)
590 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
591
592 addr = (__le32 *)&sdesc.addr;
593
594 flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
595 addr[0] = __cpu_to_le32(buffer);
596 addr[1] = __cpu_to_le32(flags);
597 if (flags & CE_SEND_FLAG_GATHER)
598 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
599 else
600 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
601
602 sdesc.nbytes = __cpu_to_le16(nbytes);
603 sdesc.flags = __cpu_to_le16(desc_flags);
604
605 *desc = sdesc;
606
607 src_ring->per_transfer_context[write_index] = per_transfer_context;
608
609 /* Update Source Ring Write Index */
610 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
611
612 if (!(flags & CE_SEND_FLAG_GATHER)) {
613 if (ar->hw_params.shadow_reg_support)
614 ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
615 write_index);
616 else
617 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
618 write_index);
619 }
620
621 src_ring->write_index = write_index;
622exit:
623 return ret;
624}
625
626int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
627 void *per_transfer_context,
628 dma_addr_t buffer,
629 unsigned int nbytes,
630 unsigned int transfer_id,
631 unsigned int flags)
632{
633 return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
634 buffer, nbytes, transfer_id, flags);
635}
636EXPORT_SYMBOL(ath10k_ce_send_nolock);
637
638void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
639{
640 struct ath10k *ar = pipe->ar;
641 struct ath10k_ce *ce = ath10k_ce_priv(ar);
642 struct ath10k_ce_ring *src_ring = pipe->src_ring;
643 u32 ctrl_addr = pipe->ctrl_addr;
644
645 lockdep_assert_held(&ce->ce_lock);
646
647 /*
648 * This function must be called only if there is an incomplete
649 * scatter-gather transfer (before index register is updated)
650 * that needs to be cleaned up.
651 */
652 if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
653 return;
654
655 if (WARN_ON_ONCE(src_ring->write_index ==
656 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
657 return;
658
659 src_ring->write_index--;
660 src_ring->write_index &= src_ring->nentries_mask;
661
662 src_ring->per_transfer_context[src_ring->write_index] = NULL;
663}
664EXPORT_SYMBOL(__ath10k_ce_send_revert);
665
666int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
667 void *per_transfer_context,
668 dma_addr_t buffer,
669 unsigned int nbytes,
670 unsigned int transfer_id,
671 unsigned int flags)
672{
673 struct ath10k *ar = ce_state->ar;
674 struct ath10k_ce *ce = ath10k_ce_priv(ar);
675 int ret;
676
677 spin_lock_bh(&ce->ce_lock);
678 ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
679 buffer, nbytes, transfer_id, flags);
680 spin_unlock_bh(&ce->ce_lock);
681
682 return ret;
683}
684EXPORT_SYMBOL(ath10k_ce_send);
685
686int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
687{
688 struct ath10k *ar = pipe->ar;
689 struct ath10k_ce *ce = ath10k_ce_priv(ar);
690 int delta;
691
692 spin_lock_bh(&ce->ce_lock);
693 delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
694 pipe->src_ring->write_index,
695 pipe->src_ring->sw_index - 1);
696 spin_unlock_bh(&ce->ce_lock);
697
698 return delta;
699}
700EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
701
702int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
703{
704 struct ath10k *ar = pipe->ar;
705 struct ath10k_ce *ce = ath10k_ce_priv(ar);
706 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
707 unsigned int nentries_mask = dest_ring->nentries_mask;
708 unsigned int write_index = dest_ring->write_index;
709 unsigned int sw_index = dest_ring->sw_index;
710
711 lockdep_assert_held(&ce->ce_lock);
712
713 return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
714}
715EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
716
717static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
718 dma_addr_t paddr)
719{
720 struct ath10k *ar = pipe->ar;
721 struct ath10k_ce *ce = ath10k_ce_priv(ar);
722 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
723 unsigned int nentries_mask = dest_ring->nentries_mask;
724 unsigned int write_index = dest_ring->write_index;
725 unsigned int sw_index = dest_ring->sw_index;
726 struct ce_desc *base = dest_ring->base_addr_owner_space;
727 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
728 u32 ctrl_addr = pipe->ctrl_addr;
729
730 lockdep_assert_held(&ce->ce_lock);
731
732 if ((pipe->id != 5) &&
733 CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
734 return -ENOSPC;
735
736 desc->addr = __cpu_to_le32(paddr);
737 desc->nbytes = 0;
738
739 dest_ring->per_transfer_context[write_index] = ctx;
740 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
741 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
742 dest_ring->write_index = write_index;
743
744 return 0;
745}
746
747static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
748 void *ctx,
749 dma_addr_t paddr)
750{
751 struct ath10k *ar = pipe->ar;
752 struct ath10k_ce *ce = ath10k_ce_priv(ar);
753 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
754 unsigned int nentries_mask = dest_ring->nentries_mask;
755 unsigned int write_index = dest_ring->write_index;
756 unsigned int sw_index = dest_ring->sw_index;
757 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
758 struct ce_desc_64 *desc =
759 CE_DEST_RING_TO_DESC_64(base, write_index);
760 u32 ctrl_addr = pipe->ctrl_addr;
761
762 lockdep_assert_held(&ce->ce_lock);
763
764 if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
765 return -ENOSPC;
766
767 desc->addr = __cpu_to_le64(paddr);
768 desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
769
770 desc->nbytes = 0;
771
772 dest_ring->per_transfer_context[write_index] = ctx;
773 write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
774 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
775 dest_ring->write_index = write_index;
776
777 return 0;
778}
779
780void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
781{
782 struct ath10k *ar = pipe->ar;
783 struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
784 unsigned int nentries_mask = dest_ring->nentries_mask;
785 unsigned int write_index = dest_ring->write_index;
786 u32 ctrl_addr = pipe->ctrl_addr;
787 u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
788
789 /* Prevent CE ring stuck issue that will occur when ring is full.
790 * Make sure that write index is 1 less than read index.
791 */
792 if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
793 nentries -= 1;
794
795 write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
796 ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
797 dest_ring->write_index = write_index;
798}
799EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
800
801int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
802 dma_addr_t paddr)
803{
804 struct ath10k *ar = pipe->ar;
805 struct ath10k_ce *ce = ath10k_ce_priv(ar);
806 int ret;
807
808 spin_lock_bh(&ce->ce_lock);
809 ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
810 spin_unlock_bh(&ce->ce_lock);
811
812 return ret;
813}
814EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
815
816/*
817 * Guts of ath10k_ce_completed_recv_next.
818 * The caller takes responsibility for any necessary locking.
819 */
820static int
821 _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
822 void **per_transfer_contextp,
823 unsigned int *nbytesp)
824{
825 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
826 unsigned int nentries_mask = dest_ring->nentries_mask;
827 unsigned int sw_index = dest_ring->sw_index;
828
829 struct ce_desc *base = dest_ring->base_addr_owner_space;
830 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
831 struct ce_desc sdesc;
832 u16 nbytes;
833
834 /* Copy in one go for performance reasons */
835 sdesc = *desc;
836
837 nbytes = __le16_to_cpu(sdesc.nbytes);
838 if (nbytes == 0) {
839 /*
840 * This closes a relatively unusual race where the Host
841 * sees the updated DRRI before the update to the
842 * corresponding descriptor has completed. We treat this
843 * as a descriptor that is not yet done.
844 */
845 return -EIO;
846 }
847
848 desc->nbytes = 0;
849
850 /* Return data from completed destination descriptor */
851 *nbytesp = nbytes;
852
853 if (per_transfer_contextp)
854 *per_transfer_contextp =
855 dest_ring->per_transfer_context[sw_index];
856
857 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
858 * So update transfer context all CEs except CE5.
859 */
860 if (ce_state->id != 5)
861 dest_ring->per_transfer_context[sw_index] = NULL;
862
863 /* Update sw_index */
864 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
865 dest_ring->sw_index = sw_index;
866
867 return 0;
868}
869
870static int
871_ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
872 void **per_transfer_contextp,
873 unsigned int *nbytesp)
874{
875 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
876 unsigned int nentries_mask = dest_ring->nentries_mask;
877 unsigned int sw_index = dest_ring->sw_index;
878 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
879 struct ce_desc_64 *desc =
880 CE_DEST_RING_TO_DESC_64(base, sw_index);
881 struct ce_desc_64 sdesc;
882 u16 nbytes;
883
884 /* Copy in one go for performance reasons */
885 sdesc = *desc;
886
887 nbytes = __le16_to_cpu(sdesc.nbytes);
888 if (nbytes == 0) {
889 /* This closes a relatively unusual race where the Host
890 * sees the updated DRRI before the update to the
891 * corresponding descriptor has completed. We treat this
892 * as a descriptor that is not yet done.
893 */
894 return -EIO;
895 }
896
897 desc->nbytes = 0;
898
899 /* Return data from completed destination descriptor */
900 *nbytesp = nbytes;
901
902 if (per_transfer_contextp)
903 *per_transfer_contextp =
904 dest_ring->per_transfer_context[sw_index];
905
906 /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
907 * So update transfer context all CEs except CE5.
908 */
909 if (ce_state->id != 5)
910 dest_ring->per_transfer_context[sw_index] = NULL;
911
912 /* Update sw_index */
913 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
914 dest_ring->sw_index = sw_index;
915
916 return 0;
917}
918
919int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
920 void **per_transfer_ctx,
921 unsigned int *nbytesp)
922{
923 return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
924 per_transfer_ctx,
925 nbytesp);
926}
927EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
928
929int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
930 void **per_transfer_contextp,
931 unsigned int *nbytesp)
932{
933 struct ath10k *ar = ce_state->ar;
934 struct ath10k_ce *ce = ath10k_ce_priv(ar);
935 int ret;
936
937 spin_lock_bh(&ce->ce_lock);
938 ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
939 per_transfer_contextp,
940 nbytesp);
941
942 spin_unlock_bh(&ce->ce_lock);
943
944 return ret;
945}
946EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
947
948static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
949 void **per_transfer_contextp,
950 dma_addr_t *bufferp)
951{
952 struct ath10k_ce_ring *dest_ring;
953 unsigned int nentries_mask;
954 unsigned int sw_index;
955 unsigned int write_index;
956 int ret;
957 struct ath10k *ar;
958 struct ath10k_ce *ce;
959
960 dest_ring = ce_state->dest_ring;
961
962 if (!dest_ring)
963 return -EIO;
964
965 ar = ce_state->ar;
966 ce = ath10k_ce_priv(ar);
967
968 spin_lock_bh(&ce->ce_lock);
969
970 nentries_mask = dest_ring->nentries_mask;
971 sw_index = dest_ring->sw_index;
972 write_index = dest_ring->write_index;
973 if (write_index != sw_index) {
974 struct ce_desc *base = dest_ring->base_addr_owner_space;
975 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
976
977 /* Return data from completed destination descriptor */
978 *bufferp = __le32_to_cpu(desc->addr);
979
980 if (per_transfer_contextp)
981 *per_transfer_contextp =
982 dest_ring->per_transfer_context[sw_index];
983
984 /* sanity */
985 dest_ring->per_transfer_context[sw_index] = NULL;
986 desc->nbytes = 0;
987
988 /* Update sw_index */
989 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
990 dest_ring->sw_index = sw_index;
991 ret = 0;
992 } else {
993 ret = -EIO;
994 }
995
996 spin_unlock_bh(&ce->ce_lock);
997
998 return ret;
999}
1000
1001static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
1002 void **per_transfer_contextp,
1003 dma_addr_t *bufferp)
1004{
1005 struct ath10k_ce_ring *dest_ring;
1006 unsigned int nentries_mask;
1007 unsigned int sw_index;
1008 unsigned int write_index;
1009 int ret;
1010 struct ath10k *ar;
1011 struct ath10k_ce *ce;
1012
1013 dest_ring = ce_state->dest_ring;
1014
1015 if (!dest_ring)
1016 return -EIO;
1017
1018 ar = ce_state->ar;
1019 ce = ath10k_ce_priv(ar);
1020
1021 spin_lock_bh(&ce->ce_lock);
1022
1023 nentries_mask = dest_ring->nentries_mask;
1024 sw_index = dest_ring->sw_index;
1025 write_index = dest_ring->write_index;
1026 if (write_index != sw_index) {
1027 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
1028 struct ce_desc_64 *desc =
1029 CE_DEST_RING_TO_DESC_64(base, sw_index);
1030
1031 /* Return data from completed destination descriptor */
1032 *bufferp = __le64_to_cpu(desc->addr);
1033
1034 if (per_transfer_contextp)
1035 *per_transfer_contextp =
1036 dest_ring->per_transfer_context[sw_index];
1037
1038 /* sanity */
1039 dest_ring->per_transfer_context[sw_index] = NULL;
1040 desc->nbytes = 0;
1041
1042 /* Update sw_index */
1043 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1044 dest_ring->sw_index = sw_index;
1045 ret = 0;
1046 } else {
1047 ret = -EIO;
1048 }
1049
1050 spin_unlock_bh(&ce->ce_lock);
1051
1052 return ret;
1053}
1054
1055int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
1056 void **per_transfer_contextp,
1057 dma_addr_t *bufferp)
1058{
1059 return ce_state->ops->ce_revoke_recv_next(ce_state,
1060 per_transfer_contextp,
1061 bufferp);
1062}
1063EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
1064
1065/*
1066 * Guts of ath10k_ce_completed_send_next.
1067 * The caller takes responsibility for any necessary locking.
1068 */
1069static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1070 void **per_transfer_contextp)
1071{
1072 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1073 u32 ctrl_addr = ce_state->ctrl_addr;
1074 struct ath10k *ar = ce_state->ar;
1075 unsigned int nentries_mask = src_ring->nentries_mask;
1076 unsigned int sw_index = src_ring->sw_index;
1077 unsigned int read_index;
1078 struct ce_desc *desc;
1079
1080 if (src_ring->hw_index == sw_index) {
1081 /*
1082 * The SW completion index has caught up with the cached
1083 * version of the HW completion index.
1084 * Update the cached HW completion index to see whether
1085 * the SW has really caught up to the HW, or if the cached
1086 * value of the HW index has become stale.
1087 */
1088
1089 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1090 if (read_index == 0xffffffff)
1091 return -ENODEV;
1092
1093 read_index &= nentries_mask;
1094 src_ring->hw_index = read_index;
1095 }
1096
1097 if (ar->hw_params.rri_on_ddr)
1098 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1099 else
1100 read_index = src_ring->hw_index;
1101
1102 if (read_index == sw_index)
1103 return -EIO;
1104
1105 if (per_transfer_contextp)
1106 *per_transfer_contextp =
1107 src_ring->per_transfer_context[sw_index];
1108
1109 /* sanity */
1110 src_ring->per_transfer_context[sw_index] = NULL;
1111 desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1112 sw_index);
1113 desc->nbytes = 0;
1114
1115 /* Update sw_index */
1116 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1117 src_ring->sw_index = sw_index;
1118
1119 return 0;
1120}
1121
1122static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1123 void **per_transfer_contextp)
1124{
1125 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1126 u32 ctrl_addr = ce_state->ctrl_addr;
1127 struct ath10k *ar = ce_state->ar;
1128 unsigned int nentries_mask = src_ring->nentries_mask;
1129 unsigned int sw_index = src_ring->sw_index;
1130 unsigned int read_index;
1131 struct ce_desc_64 *desc;
1132
1133 if (src_ring->hw_index == sw_index) {
1134 /*
1135 * The SW completion index has caught up with the cached
1136 * version of the HW completion index.
1137 * Update the cached HW completion index to see whether
1138 * the SW has really caught up to the HW, or if the cached
1139 * value of the HW index has become stale.
1140 */
1141
1142 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1143 if (read_index == 0xffffffff)
1144 return -ENODEV;
1145
1146 read_index &= nentries_mask;
1147 src_ring->hw_index = read_index;
1148 }
1149
1150 if (ar->hw_params.rri_on_ddr)
1151 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1152 else
1153 read_index = src_ring->hw_index;
1154
1155 if (read_index == sw_index)
1156 return -EIO;
1157
1158 if (per_transfer_contextp)
1159 *per_transfer_contextp =
1160 src_ring->per_transfer_context[sw_index];
1161
1162 /* sanity */
1163 src_ring->per_transfer_context[sw_index] = NULL;
1164 desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1165 sw_index);
1166 desc->nbytes = 0;
1167
1168 /* Update sw_index */
1169 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1170 src_ring->sw_index = sw_index;
1171
1172 return 0;
1173}
1174
1175int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1176 void **per_transfer_contextp)
1177{
1178 return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1179 per_transfer_contextp);
1180}
1181EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1182
1183static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1184 struct ath10k_ce_ring *src_ring,
1185 u32 sw_index,
1186 dma_addr_t *bufferp,
1187 u32 *nbytesp,
1188 u32 *transfer_idp)
1189{
1190 struct ce_desc *base = src_ring->base_addr_owner_space;
1191 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1192
1193 /* Return data from completed source descriptor */
1194 *bufferp = __le32_to_cpu(desc->addr);
1195 *nbytesp = __le16_to_cpu(desc->nbytes);
1196 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1197 CE_DESC_FLAGS_META_DATA);
1198}
1199
1200static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1201 struct ath10k_ce_ring *src_ring,
1202 u32 sw_index,
1203 dma_addr_t *bufferp,
1204 u32 *nbytesp,
1205 u32 *transfer_idp)
1206{
1207 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1208 struct ce_desc_64 *desc =
1209 CE_SRC_RING_TO_DESC_64(base, sw_index);
1210
1211 /* Return data from completed source descriptor */
1212 *bufferp = __le64_to_cpu(desc->addr);
1213 *nbytesp = __le16_to_cpu(desc->nbytes);
1214 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1215 CE_DESC_FLAGS_META_DATA);
1216}
1217
1218/* NB: Modeled after ath10k_ce_completed_send_next */
1219int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1220 void **per_transfer_contextp,
1221 dma_addr_t *bufferp,
1222 unsigned int *nbytesp,
1223 unsigned int *transfer_idp)
1224{
1225 struct ath10k_ce_ring *src_ring;
1226 unsigned int nentries_mask;
1227 unsigned int sw_index;
1228 unsigned int write_index;
1229 int ret;
1230 struct ath10k *ar;
1231 struct ath10k_ce *ce;
1232
1233 src_ring = ce_state->src_ring;
1234
1235 if (!src_ring)
1236 return -EIO;
1237
1238 ar = ce_state->ar;
1239 ce = ath10k_ce_priv(ar);
1240
1241 spin_lock_bh(&ce->ce_lock);
1242
1243 nentries_mask = src_ring->nentries_mask;
1244 sw_index = src_ring->sw_index;
1245 write_index = src_ring->write_index;
1246
1247 if (write_index != sw_index) {
1248 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1249 bufferp, nbytesp,
1250 transfer_idp);
1251
1252 if (per_transfer_contextp)
1253 *per_transfer_contextp =
1254 src_ring->per_transfer_context[sw_index];
1255
1256 /* sanity */
1257 src_ring->per_transfer_context[sw_index] = NULL;
1258
1259 /* Update sw_index */
1260 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1261 src_ring->sw_index = sw_index;
1262 ret = 0;
1263 } else {
1264 ret = -EIO;
1265 }
1266
1267 spin_unlock_bh(&ce->ce_lock);
1268
1269 return ret;
1270}
1271EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1272
1273int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1274 void **per_transfer_contextp)
1275{
1276 struct ath10k *ar = ce_state->ar;
1277 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1278 int ret;
1279
1280 spin_lock_bh(&ce->ce_lock);
1281 ret = ath10k_ce_completed_send_next_nolock(ce_state,
1282 per_transfer_contextp);
1283 spin_unlock_bh(&ce->ce_lock);
1284
1285 return ret;
1286}
1287EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1288
1289/*
1290 * Guts of interrupt handler for per-engine interrupts on a particular CE.
1291 *
1292 * Invokes registered callbacks for recv_complete,
1293 * send_complete, and watermarks.
1294 */
1295void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1296{
1297 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1298 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1299 struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1300 u32 ctrl_addr = ce_state->ctrl_addr;
1301
1302 /*
1303 * Clear before handling
1304 *
1305 * Misc CE interrupts are not being handled, but still need
1306 * to be cleared.
1307 *
1308 * NOTE: When the last copy engine interrupt is cleared the
1309 * hardware will go to sleep. Once this happens any access to
1310 * the CE registers can cause a hardware fault.
1311 */
1312 ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1313 wm_regs->cc_mask | wm_regs->wm_mask);
1314
1315 if (ce_state->recv_cb)
1316 ce_state->recv_cb(ce_state);
1317
1318 if (ce_state->send_cb)
1319 ce_state->send_cb(ce_state);
1320}
1321EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1322
1323/*
1324 * Handler for per-engine interrupts on ALL active CEs.
1325 * This is used in cases where the system is sharing a
1326 * single interrput for all CEs
1327 */
1328
1329void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1330{
1331 int ce_id;
1332 u32 intr_summary;
1333
1334 intr_summary = ath10k_ce_interrupt_summary(ar);
1335
1336 for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1337 if (intr_summary & (1 << ce_id))
1338 intr_summary &= ~(1 << ce_id);
1339 else
1340 /* no intr pending on this CE */
1341 continue;
1342
1343 ath10k_ce_per_engine_service(ar, ce_id);
1344 }
1345}
1346EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1347
1348/*
1349 * Adjust interrupts for the copy complete handler.
1350 * If it's needed for either send or recv, then unmask
1351 * this interrupt; otherwise, mask it.
1352 *
1353 * Called with ce_lock held.
1354 */
1355static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1356{
1357 u32 ctrl_addr = ce_state->ctrl_addr;
1358 struct ath10k *ar = ce_state->ar;
1359 bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1360
1361 if ((!disable_copy_compl_intr) &&
1362 (ce_state->send_cb || ce_state->recv_cb))
1363 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1364 else
1365 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1366
1367 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1368}
1369
1370void ath10k_ce_disable_interrupt(struct ath10k *ar, int ce_id)
1371{
1372 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1373 struct ath10k_ce_pipe *ce_state;
1374 u32 ctrl_addr;
1375
1376 ce_state = &ce->ce_states[ce_id];
1377 if (ce_state->attr_flags & CE_ATTR_POLL)
1378 return;
1379
1380 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1381
1382 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1383 ath10k_ce_error_intr_disable(ar, ctrl_addr);
1384 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1385}
1386EXPORT_SYMBOL(ath10k_ce_disable_interrupt);
1387
1388void ath10k_ce_disable_interrupts(struct ath10k *ar)
1389{
1390 int ce_id;
1391
1392 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1393 ath10k_ce_disable_interrupt(ar, ce_id);
1394}
1395EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1396
1397void ath10k_ce_enable_interrupt(struct ath10k *ar, int ce_id)
1398{
1399 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1400 struct ath10k_ce_pipe *ce_state;
1401
1402 ce_state = &ce->ce_states[ce_id];
1403 if (ce_state->attr_flags & CE_ATTR_POLL)
1404 return;
1405
1406 ath10k_ce_per_engine_handler_adjust(ce_state);
1407}
1408EXPORT_SYMBOL(ath10k_ce_enable_interrupt);
1409
1410void ath10k_ce_enable_interrupts(struct ath10k *ar)
1411{
1412 int ce_id;
1413
1414 /* Enable interrupts for copy engine that
1415 * are not using polling mode.
1416 */
1417 for (ce_id = 0; ce_id < CE_COUNT; ce_id++)
1418 ath10k_ce_enable_interrupt(ar, ce_id);
1419}
1420EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1421
1422static int ath10k_ce_init_src_ring(struct ath10k *ar,
1423 unsigned int ce_id,
1424 const struct ce_attr *attr)
1425{
1426 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1427 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1428 struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1429 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1430
1431 nentries = roundup_pow_of_two(attr->src_nentries);
1432
1433 if (ar->hw_params.target_64bit)
1434 memset(src_ring->base_addr_owner_space, 0,
1435 nentries * sizeof(struct ce_desc_64));
1436 else
1437 memset(src_ring->base_addr_owner_space, 0,
1438 nentries * sizeof(struct ce_desc));
1439
1440 src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1441 src_ring->sw_index &= src_ring->nentries_mask;
1442 src_ring->hw_index = src_ring->sw_index;
1443
1444 src_ring->write_index =
1445 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1446 src_ring->write_index &= src_ring->nentries_mask;
1447
1448 ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1449 src_ring->base_addr_ce_space);
1450 ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1451 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1452 ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1453 ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1454 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1455
1456 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1457 "boot init ce src ring id %d entries %d base_addr %pK\n",
1458 ce_id, nentries, src_ring->base_addr_owner_space);
1459
1460 return 0;
1461}
1462
1463static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1464 unsigned int ce_id,
1465 const struct ce_attr *attr)
1466{
1467 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1468 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1469 struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1470 u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1471
1472 nentries = roundup_pow_of_two(attr->dest_nentries);
1473
1474 if (ar->hw_params.target_64bit)
1475 memset(dest_ring->base_addr_owner_space, 0,
1476 nentries * sizeof(struct ce_desc_64));
1477 else
1478 memset(dest_ring->base_addr_owner_space, 0,
1479 nentries * sizeof(struct ce_desc));
1480
1481 dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1482 dest_ring->sw_index &= dest_ring->nentries_mask;
1483 dest_ring->write_index =
1484 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1485 dest_ring->write_index &= dest_ring->nentries_mask;
1486
1487 ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1488 dest_ring->base_addr_ce_space);
1489 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1490 ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1491 ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1492 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1493
1494 ath10k_dbg(ar, ATH10K_DBG_BOOT,
1495 "boot ce dest ring id %d entries %d base_addr %pK\n",
1496 ce_id, nentries, dest_ring->base_addr_owner_space);
1497
1498 return 0;
1499}
1500
1501static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1502 struct ath10k_ce_ring *src_ring,
1503 u32 nentries)
1504{
1505 src_ring->shadow_base_unaligned = kcalloc(nentries,
1506 sizeof(struct ce_desc_64),
1507 GFP_KERNEL);
1508 if (!src_ring->shadow_base_unaligned)
1509 return -ENOMEM;
1510
1511 src_ring->shadow_base = (struct ce_desc_64 *)
1512 PTR_ALIGN(src_ring->shadow_base_unaligned,
1513 CE_DESC_RING_ALIGN);
1514 return 0;
1515}
1516
1517static struct ath10k_ce_ring *
1518ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1519 const struct ce_attr *attr)
1520{
1521 struct ath10k_ce_ring *src_ring;
1522 u32 nentries = attr->src_nentries;
1523 dma_addr_t base_addr;
1524 int ret;
1525
1526 nentries = roundup_pow_of_two(nentries);
1527
1528 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1529 nentries), GFP_KERNEL);
1530 if (src_ring == NULL)
1531 return ERR_PTR(-ENOMEM);
1532
1533 src_ring->nentries = nentries;
1534 src_ring->nentries_mask = nentries - 1;
1535
1536 /*
1537 * Legacy platforms that do not support cache
1538 * coherent DMA are unsupported
1539 */
1540 src_ring->base_addr_owner_space_unaligned =
1541 dma_alloc_coherent(ar->dev,
1542 (nentries * sizeof(struct ce_desc) +
1543 CE_DESC_RING_ALIGN),
1544 &base_addr, GFP_KERNEL);
1545 if (!src_ring->base_addr_owner_space_unaligned) {
1546 kfree(src_ring);
1547 return ERR_PTR(-ENOMEM);
1548 }
1549
1550 src_ring->base_addr_ce_space_unaligned = base_addr;
1551
1552 src_ring->base_addr_owner_space =
1553 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1554 CE_DESC_RING_ALIGN);
1555 src_ring->base_addr_ce_space =
1556 ALIGN(src_ring->base_addr_ce_space_unaligned,
1557 CE_DESC_RING_ALIGN);
1558
1559 if (ar->hw_params.shadow_reg_support) {
1560 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1561 if (ret) {
1562 dma_free_coherent(ar->dev,
1563 (nentries * sizeof(struct ce_desc) +
1564 CE_DESC_RING_ALIGN),
1565 src_ring->base_addr_owner_space_unaligned,
1566 base_addr);
1567 kfree(src_ring);
1568 return ERR_PTR(ret);
1569 }
1570 }
1571
1572 return src_ring;
1573}
1574
1575static struct ath10k_ce_ring *
1576ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1577 const struct ce_attr *attr)
1578{
1579 struct ath10k_ce_ring *src_ring;
1580 u32 nentries = attr->src_nentries;
1581 dma_addr_t base_addr;
1582 int ret;
1583
1584 nentries = roundup_pow_of_two(nentries);
1585
1586 src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1587 nentries), GFP_KERNEL);
1588 if (!src_ring)
1589 return ERR_PTR(-ENOMEM);
1590
1591 src_ring->nentries = nentries;
1592 src_ring->nentries_mask = nentries - 1;
1593
1594 /* Legacy platforms that do not support cache
1595 * coherent DMA are unsupported
1596 */
1597 src_ring->base_addr_owner_space_unaligned =
1598 dma_alloc_coherent(ar->dev,
1599 (nentries * sizeof(struct ce_desc_64) +
1600 CE_DESC_RING_ALIGN),
1601 &base_addr, GFP_KERNEL);
1602 if (!src_ring->base_addr_owner_space_unaligned) {
1603 kfree(src_ring);
1604 return ERR_PTR(-ENOMEM);
1605 }
1606
1607 src_ring->base_addr_ce_space_unaligned = base_addr;
1608
1609 src_ring->base_addr_owner_space =
1610 PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1611 CE_DESC_RING_ALIGN);
1612 src_ring->base_addr_ce_space =
1613 ALIGN(src_ring->base_addr_ce_space_unaligned,
1614 CE_DESC_RING_ALIGN);
1615
1616 if (ar->hw_params.shadow_reg_support) {
1617 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1618 if (ret) {
1619 dma_free_coherent(ar->dev,
1620 (nentries * sizeof(struct ce_desc_64) +
1621 CE_DESC_RING_ALIGN),
1622 src_ring->base_addr_owner_space_unaligned,
1623 base_addr);
1624 kfree(src_ring);
1625 return ERR_PTR(ret);
1626 }
1627 }
1628
1629 return src_ring;
1630}
1631
1632static struct ath10k_ce_ring *
1633ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1634 const struct ce_attr *attr)
1635{
1636 struct ath10k_ce_ring *dest_ring;
1637 u32 nentries;
1638 dma_addr_t base_addr;
1639
1640 nentries = roundup_pow_of_two(attr->dest_nentries);
1641
1642 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1643 nentries), GFP_KERNEL);
1644 if (dest_ring == NULL)
1645 return ERR_PTR(-ENOMEM);
1646
1647 dest_ring->nentries = nentries;
1648 dest_ring->nentries_mask = nentries - 1;
1649
1650 /*
1651 * Legacy platforms that do not support cache
1652 * coherent DMA are unsupported
1653 */
1654 dest_ring->base_addr_owner_space_unaligned =
1655 dma_alloc_coherent(ar->dev,
1656 (nentries * sizeof(struct ce_desc) +
1657 CE_DESC_RING_ALIGN),
1658 &base_addr, GFP_KERNEL);
1659 if (!dest_ring->base_addr_owner_space_unaligned) {
1660 kfree(dest_ring);
1661 return ERR_PTR(-ENOMEM);
1662 }
1663
1664 dest_ring->base_addr_ce_space_unaligned = base_addr;
1665
1666 dest_ring->base_addr_owner_space =
1667 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1668 CE_DESC_RING_ALIGN);
1669 dest_ring->base_addr_ce_space =
1670 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1671 CE_DESC_RING_ALIGN);
1672
1673 return dest_ring;
1674}
1675
1676static struct ath10k_ce_ring *
1677ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1678 const struct ce_attr *attr)
1679{
1680 struct ath10k_ce_ring *dest_ring;
1681 u32 nentries;
1682 dma_addr_t base_addr;
1683
1684 nentries = roundup_pow_of_two(attr->dest_nentries);
1685
1686 dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1687 nentries), GFP_KERNEL);
1688 if (!dest_ring)
1689 return ERR_PTR(-ENOMEM);
1690
1691 dest_ring->nentries = nentries;
1692 dest_ring->nentries_mask = nentries - 1;
1693
1694 /* Legacy platforms that do not support cache
1695 * coherent DMA are unsupported
1696 */
1697 dest_ring->base_addr_owner_space_unaligned =
1698 dma_alloc_coherent(ar->dev,
1699 (nentries * sizeof(struct ce_desc_64) +
1700 CE_DESC_RING_ALIGN),
1701 &base_addr, GFP_KERNEL);
1702 if (!dest_ring->base_addr_owner_space_unaligned) {
1703 kfree(dest_ring);
1704 return ERR_PTR(-ENOMEM);
1705 }
1706
1707 dest_ring->base_addr_ce_space_unaligned = base_addr;
1708
1709 /* Correctly initialize memory to 0 to prevent garbage
1710 * data crashing system when download firmware
1711 */
1712 dest_ring->base_addr_owner_space =
1713 PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1714 CE_DESC_RING_ALIGN);
1715 dest_ring->base_addr_ce_space =
1716 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1717 CE_DESC_RING_ALIGN);
1718
1719 return dest_ring;
1720}
1721
1722/*
1723 * Initialize a Copy Engine based on caller-supplied attributes.
1724 * This may be called once to initialize both source and destination
1725 * rings or it may be called twice for separate source and destination
1726 * initialization. It may be that only one side or the other is
1727 * initialized by software/firmware.
1728 */
1729int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1730 const struct ce_attr *attr)
1731{
1732 int ret;
1733
1734 if (attr->src_nentries) {
1735 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1736 if (ret) {
1737 ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1738 ce_id, ret);
1739 return ret;
1740 }
1741 }
1742
1743 if (attr->dest_nentries) {
1744 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1745 if (ret) {
1746 ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1747 ce_id, ret);
1748 return ret;
1749 }
1750 }
1751
1752 return 0;
1753}
1754EXPORT_SYMBOL(ath10k_ce_init_pipe);
1755
1756static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1757{
1758 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1759
1760 ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1761 ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1762 ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1763 ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1764}
1765
1766static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1767{
1768 u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1769
1770 ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1771 ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1772 ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1773}
1774
1775void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1776{
1777 ath10k_ce_deinit_src_ring(ar, ce_id);
1778 ath10k_ce_deinit_dest_ring(ar, ce_id);
1779}
1780EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1781
1782static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1783{
1784 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1785 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1786
1787 if (ce_state->src_ring) {
1788 if (ar->hw_params.shadow_reg_support)
1789 kfree(ce_state->src_ring->shadow_base_unaligned);
1790 dma_free_coherent(ar->dev,
1791 (ce_state->src_ring->nentries *
1792 sizeof(struct ce_desc) +
1793 CE_DESC_RING_ALIGN),
1794 ce_state->src_ring->base_addr_owner_space,
1795 ce_state->src_ring->base_addr_ce_space);
1796 kfree(ce_state->src_ring);
1797 }
1798
1799 if (ce_state->dest_ring) {
1800 dma_free_coherent(ar->dev,
1801 (ce_state->dest_ring->nentries *
1802 sizeof(struct ce_desc) +
1803 CE_DESC_RING_ALIGN),
1804 ce_state->dest_ring->base_addr_owner_space,
1805 ce_state->dest_ring->base_addr_ce_space);
1806 kfree(ce_state->dest_ring);
1807 }
1808
1809 ce_state->src_ring = NULL;
1810 ce_state->dest_ring = NULL;
1811}
1812
1813static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1814{
1815 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1816 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1817
1818 if (ce_state->src_ring) {
1819 if (ar->hw_params.shadow_reg_support)
1820 kfree(ce_state->src_ring->shadow_base_unaligned);
1821 dma_free_coherent(ar->dev,
1822 (ce_state->src_ring->nentries *
1823 sizeof(struct ce_desc_64) +
1824 CE_DESC_RING_ALIGN),
1825 ce_state->src_ring->base_addr_owner_space,
1826 ce_state->src_ring->base_addr_ce_space);
1827 kfree(ce_state->src_ring);
1828 }
1829
1830 if (ce_state->dest_ring) {
1831 dma_free_coherent(ar->dev,
1832 (ce_state->dest_ring->nentries *
1833 sizeof(struct ce_desc_64) +
1834 CE_DESC_RING_ALIGN),
1835 ce_state->dest_ring->base_addr_owner_space,
1836 ce_state->dest_ring->base_addr_ce_space);
1837 kfree(ce_state->dest_ring);
1838 }
1839
1840 ce_state->src_ring = NULL;
1841 ce_state->dest_ring = NULL;
1842}
1843
1844void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1845{
1846 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1847 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1848
1849 ce_state->ops->ce_free_pipe(ar, ce_id);
1850}
1851EXPORT_SYMBOL(ath10k_ce_free_pipe);
1852
1853void ath10k_ce_dump_registers(struct ath10k *ar,
1854 struct ath10k_fw_crash_data *crash_data)
1855{
1856 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1857 struct ath10k_ce_crash_data ce_data;
1858 u32 addr, id;
1859
1860 lockdep_assert_held(&ar->dump_mutex);
1861
1862 ath10k_err(ar, "Copy Engine register dump:\n");
1863
1864 spin_lock_bh(&ce->ce_lock);
1865 for (id = 0; id < CE_COUNT; id++) {
1866 addr = ath10k_ce_base_address(ar, id);
1867 ce_data.base_addr = cpu_to_le32(addr);
1868
1869 ce_data.src_wr_idx =
1870 cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1871 ce_data.src_r_idx =
1872 cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1873 ce_data.dst_wr_idx =
1874 cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1875 ce_data.dst_r_idx =
1876 cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1877
1878 if (crash_data)
1879 crash_data->ce_crash_data[id] = ce_data;
1880
1881 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1882 le32_to_cpu(ce_data.base_addr),
1883 le32_to_cpu(ce_data.src_wr_idx),
1884 le32_to_cpu(ce_data.src_r_idx),
1885 le32_to_cpu(ce_data.dst_wr_idx),
1886 le32_to_cpu(ce_data.dst_r_idx));
1887 }
1888
1889 spin_unlock_bh(&ce->ce_lock);
1890}
1891EXPORT_SYMBOL(ath10k_ce_dump_registers);
1892
1893static const struct ath10k_ce_ops ce_ops = {
1894 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1895 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1896 .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1897 .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1898 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1899 .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1900 .ce_free_pipe = _ath10k_ce_free_pipe,
1901 .ce_send_nolock = _ath10k_ce_send_nolock,
1902 .ce_set_src_ring_base_addr_hi = NULL,
1903 .ce_set_dest_ring_base_addr_hi = NULL,
1904 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1905};
1906
1907static const struct ath10k_ce_ops ce_64_ops = {
1908 .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1909 .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1910 .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1911 .ce_completed_recv_next_nolock =
1912 _ath10k_ce_completed_recv_next_nolock_64,
1913 .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1914 .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1915 .ce_free_pipe = _ath10k_ce_free_pipe_64,
1916 .ce_send_nolock = _ath10k_ce_send_nolock_64,
1917 .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1918 .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1919 .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1920};
1921
1922static void ath10k_ce_set_ops(struct ath10k *ar,
1923 struct ath10k_ce_pipe *ce_state)
1924{
1925 switch (ar->hw_rev) {
1926 case ATH10K_HW_WCN3990:
1927 ce_state->ops = &ce_64_ops;
1928 break;
1929 default:
1930 ce_state->ops = &ce_ops;
1931 break;
1932 }
1933}
1934
1935int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1936 const struct ce_attr *attr)
1937{
1938 struct ath10k_ce *ce = ath10k_ce_priv(ar);
1939 struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1940 int ret;
1941
1942 ath10k_ce_set_ops(ar, ce_state);
1943 /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1944 * additional TX locking checks.
1945 *
1946 * For the lack of a better place do the check here.
1947 */
1948 BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1949 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1950 BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1951 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1952 BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1953 (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1954
1955 ce_state->ar = ar;
1956 ce_state->id = ce_id;
1957 ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1958 ce_state->attr_flags = attr->flags;
1959 ce_state->src_sz_max = attr->src_sz_max;
1960
1961 if (attr->src_nentries)
1962 ce_state->send_cb = attr->send_cb;
1963
1964 if (attr->dest_nentries)
1965 ce_state->recv_cb = attr->recv_cb;
1966
1967 if (attr->src_nentries) {
1968 ce_state->src_ring =
1969 ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1970 if (IS_ERR(ce_state->src_ring)) {
1971 ret = PTR_ERR(ce_state->src_ring);
1972 ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1973 ce_id, ret);
1974 ce_state->src_ring = NULL;
1975 return ret;
1976 }
1977 }
1978
1979 if (attr->dest_nentries) {
1980 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1981 ce_id,
1982 attr);
1983 if (IS_ERR(ce_state->dest_ring)) {
1984 ret = PTR_ERR(ce_state->dest_ring);
1985 ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1986 ce_id, ret);
1987 ce_state->dest_ring = NULL;
1988 return ret;
1989 }
1990 }
1991
1992 return 0;
1993}
1994EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1995
1996void ath10k_ce_alloc_rri(struct ath10k *ar)
1997{
1998 int i;
1999 u32 value;
2000 u32 ctrl1_regs;
2001 u32 ce_base_addr;
2002 struct ath10k_ce *ce = ath10k_ce_priv(ar);
2003
2004 ce->vaddr_rri = dma_alloc_coherent(ar->dev,
2005 (CE_COUNT * sizeof(u32)),
2006 &ce->paddr_rri, GFP_KERNEL);
2007
2008 if (!ce->vaddr_rri)
2009 return;
2010
2011 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
2012 lower_32_bits(ce->paddr_rri));
2013 ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
2014 (upper_32_bits(ce->paddr_rri) &
2015 CE_DESC_ADDR_HI_MASK));
2016
2017 for (i = 0; i < CE_COUNT; i++) {
2018 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
2019 ce_base_addr = ath10k_ce_base_address(ar, i);
2020 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
2021 value |= ar->hw_ce_regs->upd->mask;
2022 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
2023 }
2024}
2025EXPORT_SYMBOL(ath10k_ce_alloc_rri);
2026
2027void ath10k_ce_free_rri(struct ath10k *ar)
2028{
2029 struct ath10k_ce *ce = ath10k_ce_priv(ar);
2030
2031 dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
2032 ce->vaddr_rri,
2033 ce->paddr_rri);
2034}
2035EXPORT_SYMBOL(ath10k_ce_free_rri);