Loading...
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * mtu3_qmu.c - Queue Management Unit driver for device controller
4 *
5 * Copyright (C) 2016 MediaTek Inc.
6 *
7 * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
8 */
9
10/*
11 * Queue Management Unit (QMU) is designed to unload SW effort
12 * to serve DMA interrupts.
13 * By preparing General Purpose Descriptor (GPD) and Buffer Descriptor (BD),
14 * SW links data buffers and triggers QMU to send / receive data to
15 * host / from device at a time.
16 * And now only GPD is supported.
17 *
18 * For more detailed information, please refer to QMU Programming Guide
19 */
20
21#include <linux/dmapool.h>
22#include <linux/iopoll.h>
23
24#include "mtu3.h"
25#include "mtu3_trace.h"
26
27#define QMU_CHECKSUM_LEN 16
28
29#define GPD_FLAGS_HWO BIT(0)
30#define GPD_FLAGS_BDP BIT(1)
31#define GPD_FLAGS_BPS BIT(2)
32#define GPD_FLAGS_ZLP BIT(6)
33#define GPD_FLAGS_IOC BIT(7)
34#define GET_GPD_HWO(gpd) (le32_to_cpu((gpd)->dw0_info) & GPD_FLAGS_HWO)
35
36#define GPD_RX_BUF_LEN_OG(x) (((x) & 0xffff) << 16)
37#define GPD_RX_BUF_LEN_EL(x) (((x) & 0xfffff) << 12)
38#define GPD_RX_BUF_LEN(mtu, x) \
39({ \
40 typeof(x) x_ = (x); \
41 ((mtu)->gen2cp) ? GPD_RX_BUF_LEN_EL(x_) : GPD_RX_BUF_LEN_OG(x_); \
42})
43
44#define GPD_DATA_LEN_OG(x) ((x) & 0xffff)
45#define GPD_DATA_LEN_EL(x) ((x) & 0xfffff)
46#define GPD_DATA_LEN(mtu, x) \
47({ \
48 typeof(x) x_ = (x); \
49 ((mtu)->gen2cp) ? GPD_DATA_LEN_EL(x_) : GPD_DATA_LEN_OG(x_); \
50})
51
52#define GPD_EXT_FLAG_ZLP BIT(29)
53#define GPD_EXT_NGP_OG(x) (((x) & 0xf) << 20)
54#define GPD_EXT_BUF_OG(x) (((x) & 0xf) << 16)
55#define GPD_EXT_NGP_EL(x) (((x) & 0xf) << 28)
56#define GPD_EXT_BUF_EL(x) (((x) & 0xf) << 24)
57#define GPD_EXT_NGP(mtu, x) \
58({ \
59 typeof(x) x_ = (x); \
60 ((mtu)->gen2cp) ? GPD_EXT_NGP_EL(x_) : GPD_EXT_NGP_OG(x_); \
61})
62
63#define GPD_EXT_BUF(mtu, x) \
64({ \
65 typeof(x) x_ = (x); \
66 ((mtu)->gen2cp) ? GPD_EXT_BUF_EL(x_) : GPD_EXT_BUF_OG(x_); \
67})
68
69#define HILO_GEN64(hi, lo) (((u64)(hi) << 32) + (lo))
70#define HILO_DMA(hi, lo) \
71 ((dma_addr_t)HILO_GEN64((le32_to_cpu(hi)), (le32_to_cpu(lo))))
72
73static dma_addr_t read_txq_cur_addr(void __iomem *mbase, u8 epnum)
74{
75 u32 txcpr;
76 u32 txhiar;
77
78 txcpr = mtu3_readl(mbase, USB_QMU_TQCPR(epnum));
79 txhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));
80
81 return HILO_DMA(QMU_CUR_GPD_ADDR_HI(txhiar), txcpr);
82}
83
84static dma_addr_t read_rxq_cur_addr(void __iomem *mbase, u8 epnum)
85{
86 u32 rxcpr;
87 u32 rxhiar;
88
89 rxcpr = mtu3_readl(mbase, USB_QMU_RQCPR(epnum));
90 rxhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));
91
92 return HILO_DMA(QMU_CUR_GPD_ADDR_HI(rxhiar), rxcpr);
93}
94
95static void write_txq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)
96{
97 u32 tqhiar;
98
99 mtu3_writel(mbase, USB_QMU_TQSAR(epnum),
100 cpu_to_le32(lower_32_bits(dma)));
101 tqhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));
102 tqhiar &= ~QMU_START_ADDR_HI_MSK;
103 tqhiar |= QMU_START_ADDR_HI(upper_32_bits(dma));
104 mtu3_writel(mbase, USB_QMU_TQHIAR(epnum), tqhiar);
105}
106
107static void write_rxq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)
108{
109 u32 rqhiar;
110
111 mtu3_writel(mbase, USB_QMU_RQSAR(epnum),
112 cpu_to_le32(lower_32_bits(dma)));
113 rqhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));
114 rqhiar &= ~QMU_START_ADDR_HI_MSK;
115 rqhiar |= QMU_START_ADDR_HI(upper_32_bits(dma));
116 mtu3_writel(mbase, USB_QMU_RQHIAR(epnum), rqhiar);
117}
118
119static struct qmu_gpd *gpd_dma_to_virt(struct mtu3_gpd_ring *ring,
120 dma_addr_t dma_addr)
121{
122 dma_addr_t dma_base = ring->dma;
123 struct qmu_gpd *gpd_head = ring->start;
124 u32 offset = (dma_addr - dma_base) / sizeof(*gpd_head);
125
126 if (offset >= MAX_GPD_NUM)
127 return NULL;
128
129 return gpd_head + offset;
130}
131
132static dma_addr_t gpd_virt_to_dma(struct mtu3_gpd_ring *ring,
133 struct qmu_gpd *gpd)
134{
135 dma_addr_t dma_base = ring->dma;
136 struct qmu_gpd *gpd_head = ring->start;
137 u32 offset;
138
139 offset = gpd - gpd_head;
140 if (offset >= MAX_GPD_NUM)
141 return 0;
142
143 return dma_base + (offset * sizeof(*gpd));
144}
145
146static void gpd_ring_init(struct mtu3_gpd_ring *ring, struct qmu_gpd *gpd)
147{
148 ring->start = gpd;
149 ring->enqueue = gpd;
150 ring->dequeue = gpd;
151 ring->end = gpd + MAX_GPD_NUM - 1;
152}
153
154static void reset_gpd_list(struct mtu3_ep *mep)
155{
156 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
157 struct qmu_gpd *gpd = ring->start;
158
159 if (gpd) {
160 gpd->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
161 gpd_ring_init(ring, gpd);
162 }
163}
164
165int mtu3_gpd_ring_alloc(struct mtu3_ep *mep)
166{
167 struct qmu_gpd *gpd;
168 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
169
170 /* software own all gpds as default */
171 gpd = dma_pool_zalloc(mep->mtu->qmu_gpd_pool, GFP_ATOMIC, &ring->dma);
172 if (gpd == NULL)
173 return -ENOMEM;
174
175 gpd_ring_init(ring, gpd);
176
177 return 0;
178}
179
180void mtu3_gpd_ring_free(struct mtu3_ep *mep)
181{
182 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
183
184 dma_pool_free(mep->mtu->qmu_gpd_pool,
185 ring->start, ring->dma);
186 memset(ring, 0, sizeof(*ring));
187}
188
189void mtu3_qmu_resume(struct mtu3_ep *mep)
190{
191 struct mtu3 *mtu = mep->mtu;
192 void __iomem *mbase = mtu->mac_base;
193 int epnum = mep->epnum;
194 u32 offset;
195
196 offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
197
198 mtu3_writel(mbase, offset, QMU_Q_RESUME);
199 if (!(mtu3_readl(mbase, offset) & QMU_Q_ACTIVE))
200 mtu3_writel(mbase, offset, QMU_Q_RESUME);
201}
202
203static struct qmu_gpd *advance_enq_gpd(struct mtu3_gpd_ring *ring)
204{
205 if (ring->enqueue < ring->end)
206 ring->enqueue++;
207 else
208 ring->enqueue = ring->start;
209
210 return ring->enqueue;
211}
212
213/* @dequeue may be NULL if ring is unallocated or freed */
214static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring)
215{
216 if (ring->dequeue < ring->end)
217 ring->dequeue++;
218 else
219 ring->dequeue = ring->start;
220
221 return ring->dequeue;
222}
223
224/* check if a ring is emtpy */
225static bool gpd_ring_empty(struct mtu3_gpd_ring *ring)
226{
227 struct qmu_gpd *enq = ring->enqueue;
228 struct qmu_gpd *next;
229
230 if (ring->enqueue < ring->end)
231 next = enq + 1;
232 else
233 next = ring->start;
234
235 /* one gpd is reserved to simplify gpd preparation */
236 return next == ring->dequeue;
237}
238
239int mtu3_prepare_transfer(struct mtu3_ep *mep)
240{
241 return gpd_ring_empty(&mep->gpd_ring);
242}
243
244static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
245{
246 struct qmu_gpd *enq;
247 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
248 struct qmu_gpd *gpd = ring->enqueue;
249 struct usb_request *req = &mreq->request;
250 struct mtu3 *mtu = mep->mtu;
251 dma_addr_t enq_dma;
252 u32 ext_addr;
253
254 gpd->dw0_info = 0; /* SW own it */
255 gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));
256 ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));
257 gpd->dw3_info = cpu_to_le32(GPD_DATA_LEN(mtu, req->length));
258
259 /* get the next GPD */
260 enq = advance_enq_gpd(ring);
261 enq_dma = gpd_virt_to_dma(ring, enq);
262 dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",
263 mep->epnum, gpd, enq, &enq_dma);
264
265 enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
266 gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
267 ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));
268 gpd->dw0_info = cpu_to_le32(ext_addr);
269
270 if (req->zero) {
271 if (mtu->gen2cp)
272 gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_ZLP);
273 else
274 gpd->dw3_info |= cpu_to_le32(GPD_EXT_FLAG_ZLP);
275 }
276
277 /* prevent reorder, make sure GPD's HWO is set last */
278 mb();
279 gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);
280
281 mreq->gpd = gpd;
282 trace_mtu3_prepare_gpd(mep, gpd);
283
284 return 0;
285}
286
287static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
288{
289 struct qmu_gpd *enq;
290 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
291 struct qmu_gpd *gpd = ring->enqueue;
292 struct usb_request *req = &mreq->request;
293 struct mtu3 *mtu = mep->mtu;
294 dma_addr_t enq_dma;
295 u32 ext_addr;
296
297 gpd->dw0_info = 0; /* SW own it */
298 gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));
299 ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));
300 gpd->dw0_info = cpu_to_le32(GPD_RX_BUF_LEN(mtu, req->length));
301
302 /* get the next GPD */
303 enq = advance_enq_gpd(ring);
304 enq_dma = gpd_virt_to_dma(ring, enq);
305 dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",
306 mep->epnum, gpd, enq, &enq_dma);
307
308 enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
309 gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
310 ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));
311 gpd->dw3_info = cpu_to_le32(ext_addr);
312 /* prevent reorder, make sure GPD's HWO is set last */
313 mb();
314 gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);
315
316 mreq->gpd = gpd;
317 trace_mtu3_prepare_gpd(mep, gpd);
318
319 return 0;
320}
321
322void mtu3_insert_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
323{
324
325 if (mep->is_in)
326 mtu3_prepare_tx_gpd(mep, mreq);
327 else
328 mtu3_prepare_rx_gpd(mep, mreq);
329}
330
331int mtu3_qmu_start(struct mtu3_ep *mep)
332{
333 struct mtu3 *mtu = mep->mtu;
334 void __iomem *mbase = mtu->mac_base;
335 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
336 u8 epnum = mep->epnum;
337
338 if (mep->is_in) {
339 /* set QMU start address */
340 write_txq_start_addr(mbase, epnum, ring->dma);
341 mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_DMAREQEN);
342 /* send zero length packet according to ZLP flag in GPD */
343 mtu3_setbits(mbase, U3D_QCR1, QMU_TX_ZLP(epnum));
344 mtu3_writel(mbase, U3D_TQERRIESR0,
345 QMU_TX_LEN_ERR(epnum) | QMU_TX_CS_ERR(epnum));
346
347 if (mtu3_readl(mbase, USB_QMU_TQCSR(epnum)) & QMU_Q_ACTIVE) {
348 dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);
349 return 0;
350 }
351 mtu3_writel(mbase, USB_QMU_TQCSR(epnum), QMU_Q_START);
352
353 } else {
354 write_rxq_start_addr(mbase, epnum, ring->dma);
355 mtu3_setbits(mbase, MU3D_EP_RXCR0(epnum), RX_DMAREQEN);
356 /* don't expect ZLP */
357 mtu3_clrbits(mbase, U3D_QCR3, QMU_RX_ZLP(epnum));
358 /* move to next GPD when receive ZLP */
359 mtu3_setbits(mbase, U3D_QCR3, QMU_RX_COZ(epnum));
360 mtu3_writel(mbase, U3D_RQERRIESR0,
361 QMU_RX_LEN_ERR(epnum) | QMU_RX_CS_ERR(epnum));
362 mtu3_writel(mbase, U3D_RQERRIESR1, QMU_RX_ZLP_ERR(epnum));
363
364 if (mtu3_readl(mbase, USB_QMU_RQCSR(epnum)) & QMU_Q_ACTIVE) {
365 dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);
366 return 0;
367 }
368 mtu3_writel(mbase, USB_QMU_RQCSR(epnum), QMU_Q_START);
369 }
370
371 return 0;
372}
373
374/* may called in atomic context */
375void mtu3_qmu_stop(struct mtu3_ep *mep)
376{
377 struct mtu3 *mtu = mep->mtu;
378 void __iomem *mbase = mtu->mac_base;
379 int epnum = mep->epnum;
380 u32 value = 0;
381 u32 qcsr;
382 int ret;
383
384 qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
385
386 if (!(mtu3_readl(mbase, qcsr) & QMU_Q_ACTIVE)) {
387 dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);
388 return;
389 }
390 mtu3_writel(mbase, qcsr, QMU_Q_STOP);
391
392 if (mep->is_in)
393 mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_FLUSHFIFO);
394
395 ret = readl_poll_timeout_atomic(mbase + qcsr, value,
396 !(value & QMU_Q_ACTIVE), 1, 1000);
397 if (ret) {
398 dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);
399 return;
400 }
401
402 /* flush fifo again to make sure the fifo is empty */
403 if (mep->is_in)
404 mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_FLUSHFIFO);
405
406 dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);
407}
408
409void mtu3_qmu_flush(struct mtu3_ep *mep)
410{
411
412 dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,
413 ((mep->is_in) ? "TX" : "RX"));
414
415 /*Stop QMU */
416 mtu3_qmu_stop(mep);
417 reset_gpd_list(mep);
418}
419
420/*
421 * QMU can't transfer zero length packet directly (a hardware limit
422 * on old SoCs), so when needs to send ZLP, we intentionally trigger
423 * a length error interrupt, and in the ISR sends a ZLP by BMU.
424 */
425static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)
426{
427 struct mtu3_ep *mep = mtu->in_eps + epnum;
428 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
429 void __iomem *mbase = mtu->mac_base;
430 struct qmu_gpd *gpd_current = NULL;
431 struct mtu3_request *mreq;
432 dma_addr_t cur_gpd_dma;
433 u32 txcsr = 0;
434 int ret;
435
436 mreq = next_request(mep);
437 if (mreq && mreq->request.length != 0)
438 return;
439
440 cur_gpd_dma = read_txq_cur_addr(mbase, epnum);
441 gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
442
443 if (GPD_DATA_LEN(mtu, le32_to_cpu(gpd_current->dw3_info)) != 0) {
444 dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);
445 return;
446 }
447
448 dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, mreq);
449 trace_mtu3_zlp_exp_gpd(mep, gpd_current);
450
451 mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
452
453 ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),
454 txcsr, !(txcsr & TX_FIFOFULL), 1, 1000);
455 if (ret) {
456 dev_err(mtu->dev, "%s wait for fifo empty fail\n", __func__);
457 return;
458 }
459 mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);
460 /* prevent reorder, make sure GPD's HWO is set last */
461 mb();
462 /* by pass the current GDP */
463 gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO);
464
465 /*enable DMAREQEN, switch back to QMU mode */
466 mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
467 mtu3_qmu_resume(mep);
468}
469
470/*
471 * when rx error happens (except zlperr), QMU will stop, and RQCPR saves
472 * the GPD encountered error, Done irq will arise after resuming QMU again.
473 */
474static void qmu_error_rx(struct mtu3 *mtu, u8 epnum)
475{
476 struct mtu3_ep *mep = mtu->out_eps + epnum;
477 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
478 struct qmu_gpd *gpd_current = NULL;
479 struct mtu3_request *mreq;
480 dma_addr_t cur_gpd_dma;
481
482 cur_gpd_dma = read_rxq_cur_addr(mtu->mac_base, epnum);
483 gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
484
485 mreq = next_request(mep);
486 if (!mreq || mreq->gpd != gpd_current) {
487 dev_err(mtu->dev, "no correct RX req is found\n");
488 return;
489 }
490
491 mreq->request.status = -EAGAIN;
492
493 /* by pass the current GDP */
494 gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO);
495 mtu3_qmu_resume(mep);
496
497 dev_dbg(mtu->dev, "%s EP%d, current=%p, req=%p\n",
498 __func__, epnum, gpd_current, mreq);
499}
500
501/*
502 * NOTE: request list maybe is already empty as following case:
503 * queue_tx --> qmu_interrupt(clear interrupt pending, schedule tasklet)-->
504 * queue_tx --> process_tasklet(meanwhile, the second one is transferred,
505 * tasklet process both of them)-->qmu_interrupt for second one.
506 * To avoid upper case, put qmu_done_tx in ISR directly to process it.
507 */
508static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)
509{
510 struct mtu3_ep *mep = mtu->in_eps + epnum;
511 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
512 void __iomem *mbase = mtu->mac_base;
513 struct qmu_gpd *gpd = ring->dequeue;
514 struct qmu_gpd *gpd_current = NULL;
515 struct usb_request *request = NULL;
516 struct mtu3_request *mreq;
517 dma_addr_t cur_gpd_dma;
518
519 /*transfer phy address got from QMU register to virtual address */
520 cur_gpd_dma = read_txq_cur_addr(mbase, epnum);
521 gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
522
523 dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
524 __func__, epnum, gpd, gpd_current, ring->enqueue);
525
526 while (gpd && gpd != gpd_current && !GET_GPD_HWO(gpd)) {
527
528 mreq = next_request(mep);
529
530 if (mreq == NULL || mreq->gpd != gpd) {
531 dev_err(mtu->dev, "no correct TX req is found\n");
532 break;
533 }
534
535 request = &mreq->request;
536 request->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));
537 trace_mtu3_complete_gpd(mep, gpd);
538 mtu3_req_complete(mep, request, 0);
539
540 gpd = advance_deq_gpd(ring);
541 }
542
543 dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
544 __func__, epnum, ring->dequeue, ring->enqueue);
545
546}
547
548static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)
549{
550 struct mtu3_ep *mep = mtu->out_eps + epnum;
551 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
552 void __iomem *mbase = mtu->mac_base;
553 struct qmu_gpd *gpd = ring->dequeue;
554 struct qmu_gpd *gpd_current = NULL;
555 struct usb_request *req = NULL;
556 struct mtu3_request *mreq;
557 dma_addr_t cur_gpd_dma;
558
559 cur_gpd_dma = read_rxq_cur_addr(mbase, epnum);
560 gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
561
562 dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
563 __func__, epnum, gpd, gpd_current, ring->enqueue);
564
565 while (gpd && gpd != gpd_current && !GET_GPD_HWO(gpd)) {
566
567 mreq = next_request(mep);
568
569 if (mreq == NULL || mreq->gpd != gpd) {
570 dev_err(mtu->dev, "no correct RX req is found\n");
571 break;
572 }
573 req = &mreq->request;
574
575 req->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));
576 trace_mtu3_complete_gpd(mep, gpd);
577 mtu3_req_complete(mep, req, 0);
578
579 gpd = advance_deq_gpd(ring);
580 }
581
582 dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
583 __func__, epnum, ring->dequeue, ring->enqueue);
584}
585
586static void qmu_done_isr(struct mtu3 *mtu, u32 done_status)
587{
588 int i;
589
590 for (i = 1; i < mtu->num_eps; i++) {
591 if (done_status & QMU_RX_DONE_INT(i))
592 qmu_done_rx(mtu, i);
593 if (done_status & QMU_TX_DONE_INT(i))
594 qmu_done_tx(mtu, i);
595 }
596}
597
598static void qmu_exception_isr(struct mtu3 *mtu, u32 qmu_status)
599{
600 void __iomem *mbase = mtu->mac_base;
601 u32 errval;
602 int i;
603
604 if ((qmu_status & RXQ_CSERR_INT) || (qmu_status & RXQ_LENERR_INT)) {
605 errval = mtu3_readl(mbase, U3D_RQERRIR0);
606 mtu3_writel(mbase, U3D_RQERRIR0, errval);
607
608 for (i = 1; i < mtu->num_eps; i++) {
609 if (errval & QMU_RX_CS_ERR(i))
610 dev_err(mtu->dev, "Rx %d CS error!\n", i);
611
612 if (errval & QMU_RX_LEN_ERR(i))
613 dev_err(mtu->dev, "RX %d Length error\n", i);
614
615 if (errval & (QMU_RX_CS_ERR(i) | QMU_RX_LEN_ERR(i)))
616 qmu_error_rx(mtu, i);
617 }
618 }
619
620 if (qmu_status & RXQ_ZLPERR_INT) {
621 errval = mtu3_readl(mbase, U3D_RQERRIR1);
622 for (i = 1; i < mtu->num_eps; i++) {
623 if (errval & QMU_RX_ZLP_ERR(i))
624 dev_dbg(mtu->dev, "RX EP%d Recv ZLP\n", i);
625 }
626 mtu3_writel(mbase, U3D_RQERRIR1, errval);
627 }
628
629 if ((qmu_status & TXQ_CSERR_INT) || (qmu_status & TXQ_LENERR_INT)) {
630 errval = mtu3_readl(mbase, U3D_TQERRIR0);
631 for (i = 1; i < mtu->num_eps; i++) {
632 if (errval & QMU_TX_CS_ERR(i))
633 dev_err(mtu->dev, "Tx %d checksum error!\n", i);
634
635 if (errval & QMU_TX_LEN_ERR(i))
636 qmu_tx_zlp_error_handler(mtu, i);
637 }
638 mtu3_writel(mbase, U3D_TQERRIR0, errval);
639 }
640}
641
642irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu)
643{
644 void __iomem *mbase = mtu->mac_base;
645 u32 qmu_status;
646 u32 qmu_done_status;
647
648 /* U3D_QISAR1 is read update */
649 qmu_status = mtu3_readl(mbase, U3D_QISAR1);
650 qmu_status &= mtu3_readl(mbase, U3D_QIER1);
651
652 qmu_done_status = mtu3_readl(mbase, U3D_QISAR0);
653 qmu_done_status &= mtu3_readl(mbase, U3D_QIER0);
654 mtu3_writel(mbase, U3D_QISAR0, qmu_done_status); /* W1C */
655 dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n",
656 (qmu_done_status & 0xFFFF), qmu_done_status >> 16,
657 qmu_status);
658 trace_mtu3_qmu_isr(qmu_done_status, qmu_status);
659
660 if (qmu_done_status)
661 qmu_done_isr(mtu, qmu_done_status);
662
663 if (qmu_status)
664 qmu_exception_isr(mtu, qmu_status);
665
666 return IRQ_HANDLED;
667}
668
669int mtu3_qmu_init(struct mtu3 *mtu)
670{
671
672 compiletime_assert(QMU_GPD_SIZE == 16, "QMU_GPD size SHOULD be 16B");
673
674 mtu->qmu_gpd_pool = dma_pool_create("QMU_GPD", mtu->dev,
675 QMU_GPD_RING_SIZE, QMU_GPD_SIZE, 0);
676
677 if (!mtu->qmu_gpd_pool)
678 return -ENOMEM;
679
680 return 0;
681}
682
683void mtu3_qmu_exit(struct mtu3 *mtu)
684{
685 dma_pool_destroy(mtu->qmu_gpd_pool);
686}
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * mtu3_qmu.c - Queue Management Unit driver for device controller
4 *
5 * Copyright (C) 2016 MediaTek Inc.
6 *
7 * Author: Chunfeng Yun <chunfeng.yun@mediatek.com>
8 */
9
10/*
11 * Queue Management Unit (QMU) is designed to unload SW effort
12 * to serve DMA interrupts.
13 * By preparing General Purpose Descriptor (GPD) and Buffer Descriptor (BD),
14 * SW links data buffers and triggers QMU to send / receive data to
15 * host / from device at a time.
16 * And now only GPD is supported.
17 *
18 * For more detailed information, please refer to QMU Programming Guide
19 */
20
21#include <linux/dmapool.h>
22#include <linux/iopoll.h>
23
24#include "mtu3.h"
25#include "mtu3_trace.h"
26
27#define QMU_CHECKSUM_LEN 16
28
29#define GPD_FLAGS_HWO BIT(0)
30#define GPD_FLAGS_BDP BIT(1)
31#define GPD_FLAGS_BPS BIT(2)
32#define GPD_FLAGS_ZLP BIT(6)
33#define GPD_FLAGS_IOC BIT(7)
34#define GET_GPD_HWO(gpd) (le32_to_cpu((gpd)->dw0_info) & GPD_FLAGS_HWO)
35
36#define GPD_RX_BUF_LEN_OG(x) (((x) & 0xffff) << 16)
37#define GPD_RX_BUF_LEN_EL(x) (((x) & 0xfffff) << 12)
38#define GPD_RX_BUF_LEN(mtu, x) \
39({ \
40 typeof(x) x_ = (x); \
41 ((mtu)->gen2cp) ? GPD_RX_BUF_LEN_EL(x_) : GPD_RX_BUF_LEN_OG(x_); \
42})
43
44#define GPD_DATA_LEN_OG(x) ((x) & 0xffff)
45#define GPD_DATA_LEN_EL(x) ((x) & 0xfffff)
46#define GPD_DATA_LEN(mtu, x) \
47({ \
48 typeof(x) x_ = (x); \
49 ((mtu)->gen2cp) ? GPD_DATA_LEN_EL(x_) : GPD_DATA_LEN_OG(x_); \
50})
51
52#define GPD_EXT_FLAG_ZLP BIT(29)
53#define GPD_EXT_NGP_OG(x) (((x) & 0xf) << 20)
54#define GPD_EXT_BUF_OG(x) (((x) & 0xf) << 16)
55#define GPD_EXT_NGP_EL(x) (((x) & 0xf) << 28)
56#define GPD_EXT_BUF_EL(x) (((x) & 0xf) << 24)
57#define GPD_EXT_NGP(mtu, x) \
58({ \
59 typeof(x) x_ = (x); \
60 ((mtu)->gen2cp) ? GPD_EXT_NGP_EL(x_) : GPD_EXT_NGP_OG(x_); \
61})
62
63#define GPD_EXT_BUF(mtu, x) \
64({ \
65 typeof(x) x_ = (x); \
66 ((mtu)->gen2cp) ? GPD_EXT_BUF_EL(x_) : GPD_EXT_BUF_OG(x_); \
67})
68
69#define HILO_GEN64(hi, lo) (((u64)(hi) << 32) + (lo))
70#define HILO_DMA(hi, lo) \
71 ((dma_addr_t)HILO_GEN64((le32_to_cpu(hi)), (le32_to_cpu(lo))))
72
73static dma_addr_t read_txq_cur_addr(void __iomem *mbase, u8 epnum)
74{
75 u32 txcpr;
76 u32 txhiar;
77
78 txcpr = mtu3_readl(mbase, USB_QMU_TQCPR(epnum));
79 txhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));
80
81 return HILO_DMA(QMU_CUR_GPD_ADDR_HI(txhiar), txcpr);
82}
83
84static dma_addr_t read_rxq_cur_addr(void __iomem *mbase, u8 epnum)
85{
86 u32 rxcpr;
87 u32 rxhiar;
88
89 rxcpr = mtu3_readl(mbase, USB_QMU_RQCPR(epnum));
90 rxhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));
91
92 return HILO_DMA(QMU_CUR_GPD_ADDR_HI(rxhiar), rxcpr);
93}
94
95static void write_txq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)
96{
97 u32 tqhiar;
98
99 mtu3_writel(mbase, USB_QMU_TQSAR(epnum),
100 cpu_to_le32(lower_32_bits(dma)));
101 tqhiar = mtu3_readl(mbase, USB_QMU_TQHIAR(epnum));
102 tqhiar &= ~QMU_START_ADDR_HI_MSK;
103 tqhiar |= QMU_START_ADDR_HI(upper_32_bits(dma));
104 mtu3_writel(mbase, USB_QMU_TQHIAR(epnum), tqhiar);
105}
106
107static void write_rxq_start_addr(void __iomem *mbase, u8 epnum, dma_addr_t dma)
108{
109 u32 rqhiar;
110
111 mtu3_writel(mbase, USB_QMU_RQSAR(epnum),
112 cpu_to_le32(lower_32_bits(dma)));
113 rqhiar = mtu3_readl(mbase, USB_QMU_RQHIAR(epnum));
114 rqhiar &= ~QMU_START_ADDR_HI_MSK;
115 rqhiar |= QMU_START_ADDR_HI(upper_32_bits(dma));
116 mtu3_writel(mbase, USB_QMU_RQHIAR(epnum), rqhiar);
117}
118
119static struct qmu_gpd *gpd_dma_to_virt(struct mtu3_gpd_ring *ring,
120 dma_addr_t dma_addr)
121{
122 dma_addr_t dma_base = ring->dma;
123 struct qmu_gpd *gpd_head = ring->start;
124 u32 offset = (dma_addr - dma_base) / sizeof(*gpd_head);
125
126 if (offset >= MAX_GPD_NUM)
127 return NULL;
128
129 return gpd_head + offset;
130}
131
132static dma_addr_t gpd_virt_to_dma(struct mtu3_gpd_ring *ring,
133 struct qmu_gpd *gpd)
134{
135 dma_addr_t dma_base = ring->dma;
136 struct qmu_gpd *gpd_head = ring->start;
137 u32 offset;
138
139 offset = gpd - gpd_head;
140 if (offset >= MAX_GPD_NUM)
141 return 0;
142
143 return dma_base + (offset * sizeof(*gpd));
144}
145
146static void gpd_ring_init(struct mtu3_gpd_ring *ring, struct qmu_gpd *gpd)
147{
148 ring->start = gpd;
149 ring->enqueue = gpd;
150 ring->dequeue = gpd;
151 ring->end = gpd + MAX_GPD_NUM - 1;
152}
153
154static void reset_gpd_list(struct mtu3_ep *mep)
155{
156 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
157 struct qmu_gpd *gpd = ring->start;
158
159 if (gpd) {
160 gpd->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
161 gpd_ring_init(ring, gpd);
162 }
163}
164
165int mtu3_gpd_ring_alloc(struct mtu3_ep *mep)
166{
167 struct qmu_gpd *gpd;
168 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
169
170 /* software own all gpds as default */
171 gpd = dma_pool_zalloc(mep->mtu->qmu_gpd_pool, GFP_ATOMIC, &ring->dma);
172 if (gpd == NULL)
173 return -ENOMEM;
174
175 gpd_ring_init(ring, gpd);
176
177 return 0;
178}
179
180void mtu3_gpd_ring_free(struct mtu3_ep *mep)
181{
182 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
183
184 dma_pool_free(mep->mtu->qmu_gpd_pool,
185 ring->start, ring->dma);
186 memset(ring, 0, sizeof(*ring));
187}
188
189void mtu3_qmu_resume(struct mtu3_ep *mep)
190{
191 struct mtu3 *mtu = mep->mtu;
192 void __iomem *mbase = mtu->mac_base;
193 int epnum = mep->epnum;
194 u32 offset;
195
196 offset = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
197
198 mtu3_writel(mbase, offset, QMU_Q_RESUME);
199 if (!(mtu3_readl(mbase, offset) & QMU_Q_ACTIVE))
200 mtu3_writel(mbase, offset, QMU_Q_RESUME);
201}
202
203static struct qmu_gpd *advance_enq_gpd(struct mtu3_gpd_ring *ring)
204{
205 if (ring->enqueue < ring->end)
206 ring->enqueue++;
207 else
208 ring->enqueue = ring->start;
209
210 return ring->enqueue;
211}
212
213static struct qmu_gpd *advance_deq_gpd(struct mtu3_gpd_ring *ring)
214{
215 if (ring->dequeue < ring->end)
216 ring->dequeue++;
217 else
218 ring->dequeue = ring->start;
219
220 return ring->dequeue;
221}
222
223/* check if a ring is emtpy */
224static int gpd_ring_empty(struct mtu3_gpd_ring *ring)
225{
226 struct qmu_gpd *enq = ring->enqueue;
227 struct qmu_gpd *next;
228
229 if (ring->enqueue < ring->end)
230 next = enq + 1;
231 else
232 next = ring->start;
233
234 /* one gpd is reserved to simplify gpd preparation */
235 return next == ring->dequeue;
236}
237
238int mtu3_prepare_transfer(struct mtu3_ep *mep)
239{
240 return gpd_ring_empty(&mep->gpd_ring);
241}
242
243static int mtu3_prepare_tx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
244{
245 struct qmu_gpd *enq;
246 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
247 struct qmu_gpd *gpd = ring->enqueue;
248 struct usb_request *req = &mreq->request;
249 struct mtu3 *mtu = mep->mtu;
250 dma_addr_t enq_dma;
251 u32 ext_addr;
252
253 gpd->dw0_info = 0; /* SW own it */
254 gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));
255 ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));
256 gpd->dw3_info = cpu_to_le32(GPD_DATA_LEN(mtu, req->length));
257
258 /* get the next GPD */
259 enq = advance_enq_gpd(ring);
260 enq_dma = gpd_virt_to_dma(ring, enq);
261 dev_dbg(mep->mtu->dev, "TX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",
262 mep->epnum, gpd, enq, &enq_dma);
263
264 enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
265 gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
266 ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));
267 gpd->dw0_info = cpu_to_le32(ext_addr);
268
269 if (req->zero) {
270 if (mtu->gen2cp)
271 gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_ZLP);
272 else
273 gpd->dw3_info |= cpu_to_le32(GPD_EXT_FLAG_ZLP);
274 }
275
276 /* prevent reorder, make sure GPD's HWO is set last */
277 mb();
278 gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);
279
280 mreq->gpd = gpd;
281 trace_mtu3_prepare_gpd(mep, gpd);
282
283 return 0;
284}
285
286static int mtu3_prepare_rx_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
287{
288 struct qmu_gpd *enq;
289 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
290 struct qmu_gpd *gpd = ring->enqueue;
291 struct usb_request *req = &mreq->request;
292 struct mtu3 *mtu = mep->mtu;
293 dma_addr_t enq_dma;
294 u32 ext_addr;
295
296 gpd->dw0_info = 0; /* SW own it */
297 gpd->buffer = cpu_to_le32(lower_32_bits(req->dma));
298 ext_addr = GPD_EXT_BUF(mtu, upper_32_bits(req->dma));
299 gpd->dw0_info = cpu_to_le32(GPD_RX_BUF_LEN(mtu, req->length));
300
301 /* get the next GPD */
302 enq = advance_enq_gpd(ring);
303 enq_dma = gpd_virt_to_dma(ring, enq);
304 dev_dbg(mep->mtu->dev, "RX-EP%d queue gpd=%p, enq=%p, qdma=%pad\n",
305 mep->epnum, gpd, enq, &enq_dma);
306
307 enq->dw0_info &= cpu_to_le32(~GPD_FLAGS_HWO);
308 gpd->next_gpd = cpu_to_le32(lower_32_bits(enq_dma));
309 ext_addr |= GPD_EXT_NGP(mtu, upper_32_bits(enq_dma));
310 gpd->dw3_info = cpu_to_le32(ext_addr);
311 /* prevent reorder, make sure GPD's HWO is set last */
312 mb();
313 gpd->dw0_info |= cpu_to_le32(GPD_FLAGS_IOC | GPD_FLAGS_HWO);
314
315 mreq->gpd = gpd;
316 trace_mtu3_prepare_gpd(mep, gpd);
317
318 return 0;
319}
320
321void mtu3_insert_gpd(struct mtu3_ep *mep, struct mtu3_request *mreq)
322{
323
324 if (mep->is_in)
325 mtu3_prepare_tx_gpd(mep, mreq);
326 else
327 mtu3_prepare_rx_gpd(mep, mreq);
328}
329
330int mtu3_qmu_start(struct mtu3_ep *mep)
331{
332 struct mtu3 *mtu = mep->mtu;
333 void __iomem *mbase = mtu->mac_base;
334 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
335 u8 epnum = mep->epnum;
336
337 if (mep->is_in) {
338 /* set QMU start address */
339 write_txq_start_addr(mbase, epnum, ring->dma);
340 mtu3_setbits(mbase, MU3D_EP_TXCR0(epnum), TX_DMAREQEN);
341 /* send zero length packet according to ZLP flag in GPD */
342 mtu3_setbits(mbase, U3D_QCR1, QMU_TX_ZLP(epnum));
343 mtu3_writel(mbase, U3D_TQERRIESR0,
344 QMU_TX_LEN_ERR(epnum) | QMU_TX_CS_ERR(epnum));
345
346 if (mtu3_readl(mbase, USB_QMU_TQCSR(epnum)) & QMU_Q_ACTIVE) {
347 dev_warn(mtu->dev, "Tx %d Active Now!\n", epnum);
348 return 0;
349 }
350 mtu3_writel(mbase, USB_QMU_TQCSR(epnum), QMU_Q_START);
351
352 } else {
353 write_rxq_start_addr(mbase, epnum, ring->dma);
354 mtu3_setbits(mbase, MU3D_EP_RXCR0(epnum), RX_DMAREQEN);
355 /* don't expect ZLP */
356 mtu3_clrbits(mbase, U3D_QCR3, QMU_RX_ZLP(epnum));
357 /* move to next GPD when receive ZLP */
358 mtu3_setbits(mbase, U3D_QCR3, QMU_RX_COZ(epnum));
359 mtu3_writel(mbase, U3D_RQERRIESR0,
360 QMU_RX_LEN_ERR(epnum) | QMU_RX_CS_ERR(epnum));
361 mtu3_writel(mbase, U3D_RQERRIESR1, QMU_RX_ZLP_ERR(epnum));
362
363 if (mtu3_readl(mbase, USB_QMU_RQCSR(epnum)) & QMU_Q_ACTIVE) {
364 dev_warn(mtu->dev, "Rx %d Active Now!\n", epnum);
365 return 0;
366 }
367 mtu3_writel(mbase, USB_QMU_RQCSR(epnum), QMU_Q_START);
368 }
369
370 return 0;
371}
372
373/* may called in atomic context */
374void mtu3_qmu_stop(struct mtu3_ep *mep)
375{
376 struct mtu3 *mtu = mep->mtu;
377 void __iomem *mbase = mtu->mac_base;
378 int epnum = mep->epnum;
379 u32 value = 0;
380 u32 qcsr;
381 int ret;
382
383 qcsr = mep->is_in ? USB_QMU_TQCSR(epnum) : USB_QMU_RQCSR(epnum);
384
385 if (!(mtu3_readl(mbase, qcsr) & QMU_Q_ACTIVE)) {
386 dev_dbg(mtu->dev, "%s's qmu is inactive now!\n", mep->name);
387 return;
388 }
389 mtu3_writel(mbase, qcsr, QMU_Q_STOP);
390
391 ret = readl_poll_timeout_atomic(mbase + qcsr, value,
392 !(value & QMU_Q_ACTIVE), 1, 1000);
393 if (ret) {
394 dev_err(mtu->dev, "stop %s's qmu failed\n", mep->name);
395 return;
396 }
397
398 dev_dbg(mtu->dev, "%s's qmu stop now!\n", mep->name);
399}
400
401void mtu3_qmu_flush(struct mtu3_ep *mep)
402{
403
404 dev_dbg(mep->mtu->dev, "%s flush QMU %s\n", __func__,
405 ((mep->is_in) ? "TX" : "RX"));
406
407 /*Stop QMU */
408 mtu3_qmu_stop(mep);
409 reset_gpd_list(mep);
410}
411
412/*
413 * QMU can't transfer zero length packet directly (a hardware limit
414 * on old SoCs), so when needs to send ZLP, we intentionally trigger
415 * a length error interrupt, and in the ISR sends a ZLP by BMU.
416 */
417static void qmu_tx_zlp_error_handler(struct mtu3 *mtu, u8 epnum)
418{
419 struct mtu3_ep *mep = mtu->in_eps + epnum;
420 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
421 void __iomem *mbase = mtu->mac_base;
422 struct qmu_gpd *gpd_current = NULL;
423 struct mtu3_request *mreq;
424 dma_addr_t cur_gpd_dma;
425 u32 txcsr = 0;
426 int ret;
427
428 mreq = next_request(mep);
429 if (mreq && mreq->request.length != 0)
430 return;
431
432 cur_gpd_dma = read_txq_cur_addr(mbase, epnum);
433 gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
434
435 if (GPD_DATA_LEN(mtu, le32_to_cpu(gpd_current->dw3_info)) != 0) {
436 dev_err(mtu->dev, "TX EP%d buffer length error(!=0)\n", epnum);
437 return;
438 }
439
440 dev_dbg(mtu->dev, "%s send ZLP for req=%p\n", __func__, mreq);
441 trace_mtu3_zlp_exp_gpd(mep, gpd_current);
442
443 mtu3_clrbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
444
445 ret = readl_poll_timeout_atomic(mbase + MU3D_EP_TXCR0(mep->epnum),
446 txcsr, !(txcsr & TX_FIFOFULL), 1, 1000);
447 if (ret) {
448 dev_err(mtu->dev, "%s wait for fifo empty fail\n", __func__);
449 return;
450 }
451 mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_TXPKTRDY);
452 /* prevent reorder, make sure GPD's HWO is set last */
453 mb();
454 /* by pass the current GDP */
455 gpd_current->dw0_info |= cpu_to_le32(GPD_FLAGS_BPS | GPD_FLAGS_HWO);
456
457 /*enable DMAREQEN, switch back to QMU mode */
458 mtu3_setbits(mbase, MU3D_EP_TXCR0(mep->epnum), TX_DMAREQEN);
459 mtu3_qmu_resume(mep);
460}
461
462/*
463 * NOTE: request list maybe is already empty as following case:
464 * queue_tx --> qmu_interrupt(clear interrupt pending, schedule tasklet)-->
465 * queue_tx --> process_tasklet(meanwhile, the second one is transferred,
466 * tasklet process both of them)-->qmu_interrupt for second one.
467 * To avoid upper case, put qmu_done_tx in ISR directly to process it.
468 */
469static void qmu_done_tx(struct mtu3 *mtu, u8 epnum)
470{
471 struct mtu3_ep *mep = mtu->in_eps + epnum;
472 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
473 void __iomem *mbase = mtu->mac_base;
474 struct qmu_gpd *gpd = ring->dequeue;
475 struct qmu_gpd *gpd_current = NULL;
476 struct usb_request *request = NULL;
477 struct mtu3_request *mreq;
478 dma_addr_t cur_gpd_dma;
479
480 /*transfer phy address got from QMU register to virtual address */
481 cur_gpd_dma = read_txq_cur_addr(mbase, epnum);
482 gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
483
484 dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
485 __func__, epnum, gpd, gpd_current, ring->enqueue);
486
487 while (gpd != gpd_current && !GET_GPD_HWO(gpd)) {
488
489 mreq = next_request(mep);
490
491 if (mreq == NULL || mreq->gpd != gpd) {
492 dev_err(mtu->dev, "no correct TX req is found\n");
493 break;
494 }
495
496 request = &mreq->request;
497 request->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));
498 trace_mtu3_complete_gpd(mep, gpd);
499 mtu3_req_complete(mep, request, 0);
500
501 gpd = advance_deq_gpd(ring);
502 }
503
504 dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
505 __func__, epnum, ring->dequeue, ring->enqueue);
506
507}
508
509static void qmu_done_rx(struct mtu3 *mtu, u8 epnum)
510{
511 struct mtu3_ep *mep = mtu->out_eps + epnum;
512 struct mtu3_gpd_ring *ring = &mep->gpd_ring;
513 void __iomem *mbase = mtu->mac_base;
514 struct qmu_gpd *gpd = ring->dequeue;
515 struct qmu_gpd *gpd_current = NULL;
516 struct usb_request *req = NULL;
517 struct mtu3_request *mreq;
518 dma_addr_t cur_gpd_dma;
519
520 cur_gpd_dma = read_rxq_cur_addr(mbase, epnum);
521 gpd_current = gpd_dma_to_virt(ring, cur_gpd_dma);
522
523 dev_dbg(mtu->dev, "%s EP%d, last=%p, current=%p, enq=%p\n",
524 __func__, epnum, gpd, gpd_current, ring->enqueue);
525
526 while (gpd != gpd_current && !GET_GPD_HWO(gpd)) {
527
528 mreq = next_request(mep);
529
530 if (mreq == NULL || mreq->gpd != gpd) {
531 dev_err(mtu->dev, "no correct RX req is found\n");
532 break;
533 }
534 req = &mreq->request;
535
536 req->actual = GPD_DATA_LEN(mtu, le32_to_cpu(gpd->dw3_info));
537 trace_mtu3_complete_gpd(mep, gpd);
538 mtu3_req_complete(mep, req, 0);
539
540 gpd = advance_deq_gpd(ring);
541 }
542
543 dev_dbg(mtu->dev, "%s EP%d, deq=%p, enq=%p, complete\n",
544 __func__, epnum, ring->dequeue, ring->enqueue);
545}
546
547static void qmu_done_isr(struct mtu3 *mtu, u32 done_status)
548{
549 int i;
550
551 for (i = 1; i < mtu->num_eps; i++) {
552 if (done_status & QMU_RX_DONE_INT(i))
553 qmu_done_rx(mtu, i);
554 if (done_status & QMU_TX_DONE_INT(i))
555 qmu_done_tx(mtu, i);
556 }
557}
558
559static void qmu_exception_isr(struct mtu3 *mtu, u32 qmu_status)
560{
561 void __iomem *mbase = mtu->mac_base;
562 u32 errval;
563 int i;
564
565 if ((qmu_status & RXQ_CSERR_INT) || (qmu_status & RXQ_LENERR_INT)) {
566 errval = mtu3_readl(mbase, U3D_RQERRIR0);
567 for (i = 1; i < mtu->num_eps; i++) {
568 if (errval & QMU_RX_CS_ERR(i))
569 dev_err(mtu->dev, "Rx %d CS error!\n", i);
570
571 if (errval & QMU_RX_LEN_ERR(i))
572 dev_err(mtu->dev, "RX %d Length error\n", i);
573 }
574 mtu3_writel(mbase, U3D_RQERRIR0, errval);
575 }
576
577 if (qmu_status & RXQ_ZLPERR_INT) {
578 errval = mtu3_readl(mbase, U3D_RQERRIR1);
579 for (i = 1; i < mtu->num_eps; i++) {
580 if (errval & QMU_RX_ZLP_ERR(i))
581 dev_dbg(mtu->dev, "RX EP%d Recv ZLP\n", i);
582 }
583 mtu3_writel(mbase, U3D_RQERRIR1, errval);
584 }
585
586 if ((qmu_status & TXQ_CSERR_INT) || (qmu_status & TXQ_LENERR_INT)) {
587 errval = mtu3_readl(mbase, U3D_TQERRIR0);
588 for (i = 1; i < mtu->num_eps; i++) {
589 if (errval & QMU_TX_CS_ERR(i))
590 dev_err(mtu->dev, "Tx %d checksum error!\n", i);
591
592 if (errval & QMU_TX_LEN_ERR(i))
593 qmu_tx_zlp_error_handler(mtu, i);
594 }
595 mtu3_writel(mbase, U3D_TQERRIR0, errval);
596 }
597}
598
599irqreturn_t mtu3_qmu_isr(struct mtu3 *mtu)
600{
601 void __iomem *mbase = mtu->mac_base;
602 u32 qmu_status;
603 u32 qmu_done_status;
604
605 /* U3D_QISAR1 is read update */
606 qmu_status = mtu3_readl(mbase, U3D_QISAR1);
607 qmu_status &= mtu3_readl(mbase, U3D_QIER1);
608
609 qmu_done_status = mtu3_readl(mbase, U3D_QISAR0);
610 qmu_done_status &= mtu3_readl(mbase, U3D_QIER0);
611 mtu3_writel(mbase, U3D_QISAR0, qmu_done_status); /* W1C */
612 dev_dbg(mtu->dev, "=== QMUdone[tx=%x, rx=%x] QMUexp[%x] ===\n",
613 (qmu_done_status & 0xFFFF), qmu_done_status >> 16,
614 qmu_status);
615 trace_mtu3_qmu_isr(qmu_done_status, qmu_status);
616
617 if (qmu_done_status)
618 qmu_done_isr(mtu, qmu_done_status);
619
620 if (qmu_status)
621 qmu_exception_isr(mtu, qmu_status);
622
623 return IRQ_HANDLED;
624}
625
626int mtu3_qmu_init(struct mtu3 *mtu)
627{
628
629 compiletime_assert(QMU_GPD_SIZE == 16, "QMU_GPD size SHOULD be 16B");
630
631 mtu->qmu_gpd_pool = dma_pool_create("QMU_GPD", mtu->dev,
632 QMU_GPD_RING_SIZE, QMU_GPD_SIZE, 0);
633
634 if (!mtu->qmu_gpd_pool)
635 return -ENOMEM;
636
637 return 0;
638}
639
640void mtu3_qmu_exit(struct mtu3 *mtu)
641{
642 dma_pool_destroy(mtu->qmu_gpd_pool);
643}