Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Cadence CDNSP DRD Driver.
4 *
5 * Copyright (C) 2020 Cadence.
6 *
7 * Author: Pawel Laszczak <pawell@cadence.com>
8 *
9 * Code based on Linux XHCI driver.
10 * Origin: Copyright (C) 2008 Intel Corp
11 */
12
13/*
14 * Ring initialization rules:
15 * 1. Each segment is initialized to zero, except for link TRBs.
16 * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or
17 * Consumer Cycle State (CCS), depending on ring function.
18 * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment.
19 *
20 * Ring behavior rules:
21 * 1. A ring is empty if enqueue == dequeue. This means there will always be at
22 * least one free TRB in the ring. This is useful if you want to turn that
23 * into a link TRB and expand the ring.
24 * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a
25 * link TRB, then load the pointer with the address in the link TRB. If the
26 * link TRB had its toggle bit set, you may need to update the ring cycle
27 * state (see cycle bit rules). You may have to do this multiple times
28 * until you reach a non-link TRB.
29 * 3. A ring is full if enqueue++ (for the definition of increment above)
30 * equals the dequeue pointer.
31 *
32 * Cycle bit rules:
33 * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
34 * in a link TRB, it must toggle the ring cycle state.
35 * 2. When a producer increments an enqueue pointer and encounters a toggle bit
36 * in a link TRB, it must toggle the ring cycle state.
37 *
38 * Producer rules:
39 * 1. Check if ring is full before you enqueue.
40 * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing.
41 * Update enqueue pointer between each write (which may update the ring
42 * cycle state).
43 * 3. Notify consumer. If SW is producer, it rings the doorbell for command
44 * and endpoint rings. If controller is the producer for the event ring,
45 * and it generates an interrupt according to interrupt modulation rules.
46 *
47 * Consumer rules:
48 * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state,
49 * the TRB is owned by the consumer.
50 * 2. Update dequeue pointer (which may update the ring cycle state) and
51 * continue processing TRBs until you reach a TRB which is not owned by you.
52 * 3. Notify the producer. SW is the consumer for the event ring, and it
53 * updates event ring dequeue pointer. Controller is the consumer for the
54 * command and endpoint rings; it generates events on the event ring
55 * for these.
56 */
57
58#include <linux/scatterlist.h>
59#include <linux/dma-mapping.h>
60#include <linux/delay.h>
61#include <linux/slab.h>
62#include <linux/irq.h>
63
64#include "cdnsp-trace.h"
65#include "cdnsp-gadget.h"
66
67/*
68 * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
69 * address of the TRB.
70 */
71dma_addr_t cdnsp_trb_virt_to_dma(struct cdnsp_segment *seg,
72 union cdnsp_trb *trb)
73{
74 unsigned long segment_offset = trb - seg->trbs;
75
76 if (trb < seg->trbs || segment_offset >= TRBS_PER_SEGMENT)
77 return 0;
78
79 return seg->dma + (segment_offset * sizeof(*trb));
80}
81
82static bool cdnsp_trb_is_noop(union cdnsp_trb *trb)
83{
84 return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
85}
86
87static bool cdnsp_trb_is_link(union cdnsp_trb *trb)
88{
89 return TRB_TYPE_LINK_LE32(trb->link.control);
90}
91
92bool cdnsp_last_trb_on_seg(struct cdnsp_segment *seg, union cdnsp_trb *trb)
93{
94 return trb == &seg->trbs[TRBS_PER_SEGMENT - 1];
95}
96
97bool cdnsp_last_trb_on_ring(struct cdnsp_ring *ring,
98 struct cdnsp_segment *seg,
99 union cdnsp_trb *trb)
100{
101 return cdnsp_last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg);
102}
103
104static bool cdnsp_link_trb_toggles_cycle(union cdnsp_trb *trb)
105{
106 return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
107}
108
109static void cdnsp_trb_to_noop(union cdnsp_trb *trb, u32 noop_type)
110{
111 if (cdnsp_trb_is_link(trb)) {
112 /* Unchain chained link TRBs. */
113 trb->link.control &= cpu_to_le32(~TRB_CHAIN);
114 } else {
115 trb->generic.field[0] = 0;
116 trb->generic.field[1] = 0;
117 trb->generic.field[2] = 0;
118 /* Preserve only the cycle bit of this TRB. */
119 trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
120 trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type));
121 }
122}
123
124/*
125 * Updates trb to point to the next TRB in the ring, and updates seg if the next
126 * TRB is in a new segment. This does not skip over link TRBs, and it does not
127 * effect the ring dequeue or enqueue pointers.
128 */
129static void cdnsp_next_trb(struct cdnsp_device *pdev,
130 struct cdnsp_ring *ring,
131 struct cdnsp_segment **seg,
132 union cdnsp_trb **trb)
133{
134 if (cdnsp_trb_is_link(*trb)) {
135 *seg = (*seg)->next;
136 *trb = ((*seg)->trbs);
137 } else {
138 (*trb)++;
139 }
140}
141
142/*
143 * See Cycle bit rules. SW is the consumer for the event ring only.
144 * Don't make a ring full of link TRBs. That would be dumb and this would loop.
145 */
146void cdnsp_inc_deq(struct cdnsp_device *pdev, struct cdnsp_ring *ring)
147{
148 /* event ring doesn't have link trbs, check for last trb. */
149 if (ring->type == TYPE_EVENT) {
150 if (!cdnsp_last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
151 ring->dequeue++;
152 goto out;
153 }
154
155 if (cdnsp_last_trb_on_ring(ring, ring->deq_seg, ring->dequeue))
156 ring->cycle_state ^= 1;
157
158 ring->deq_seg = ring->deq_seg->next;
159 ring->dequeue = ring->deq_seg->trbs;
160 goto out;
161 }
162
163 /* All other rings have link trbs. */
164 if (!cdnsp_trb_is_link(ring->dequeue)) {
165 ring->dequeue++;
166 ring->num_trbs_free++;
167 }
168 while (cdnsp_trb_is_link(ring->dequeue)) {
169 ring->deq_seg = ring->deq_seg->next;
170 ring->dequeue = ring->deq_seg->trbs;
171 }
172out:
173 trace_cdnsp_inc_deq(ring);
174}
175
176/*
177 * See Cycle bit rules. SW is the consumer for the event ring only.
178 * Don't make a ring full of link TRBs. That would be dumb and this would loop.
179 *
180 * If we've just enqueued a TRB that is in the middle of a TD (meaning the
181 * chain bit is set), then set the chain bit in all the following link TRBs.
182 * If we've enqueued the last TRB in a TD, make sure the following link TRBs
183 * have their chain bit cleared (so that each Link TRB is a separate TD).
184 *
185 * @more_trbs_coming: Will you enqueue more TRBs before ringing the doorbell.
186 */
187static void cdnsp_inc_enq(struct cdnsp_device *pdev,
188 struct cdnsp_ring *ring,
189 bool more_trbs_coming)
190{
191 union cdnsp_trb *next;
192 u32 chain;
193
194 chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
195
196 /* If this is not event ring, there is one less usable TRB. */
197 if (!cdnsp_trb_is_link(ring->enqueue))
198 ring->num_trbs_free--;
199 next = ++(ring->enqueue);
200
201 /* Update the dequeue pointer further if that was a link TRB */
202 while (cdnsp_trb_is_link(next)) {
203 /*
204 * If the caller doesn't plan on enqueuing more TDs before
205 * ringing the doorbell, then we don't want to give the link TRB
206 * to the hardware just yet. We'll give the link TRB back in
207 * cdnsp_prepare_ring() just before we enqueue the TD at the
208 * top of the ring.
209 */
210 if (!chain && !more_trbs_coming)
211 break;
212
213 next->link.control &= cpu_to_le32(~TRB_CHAIN);
214 next->link.control |= cpu_to_le32(chain);
215
216 /* Give this link TRB to the hardware */
217 wmb();
218 next->link.control ^= cpu_to_le32(TRB_CYCLE);
219
220 /* Toggle the cycle bit after the last ring segment. */
221 if (cdnsp_link_trb_toggles_cycle(next))
222 ring->cycle_state ^= 1;
223
224 ring->enq_seg = ring->enq_seg->next;
225 ring->enqueue = ring->enq_seg->trbs;
226 next = ring->enqueue;
227 }
228
229 trace_cdnsp_inc_enq(ring);
230}
231
232/*
233 * Check to see if there's room to enqueue num_trbs on the ring and make sure
234 * enqueue pointer will not advance into dequeue segment.
235 */
236static bool cdnsp_room_on_ring(struct cdnsp_device *pdev,
237 struct cdnsp_ring *ring,
238 unsigned int num_trbs)
239{
240 int num_trbs_in_deq_seg;
241
242 if (ring->num_trbs_free < num_trbs)
243 return false;
244
245 if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) {
246 num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs;
247
248 if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg)
249 return false;
250 }
251
252 return true;
253}
254
255/*
256 * Workaround for L1: controller has issue with resuming from L1 after
257 * setting doorbell for endpoint during L1 state. This function forces
258 * resume signal in such case.
259 */
260static void cdnsp_force_l0_go(struct cdnsp_device *pdev)
261{
262 if (pdev->active_port == &pdev->usb2_port && pdev->gadget.lpm_capable)
263 cdnsp_set_link_state(pdev, &pdev->active_port->regs->portsc, XDEV_U0);
264}
265
266/* Ring the doorbell after placing a command on the ring. */
267void cdnsp_ring_cmd_db(struct cdnsp_device *pdev)
268{
269 writel(DB_VALUE_CMD, &pdev->dba->cmd_db);
270}
271
272/*
273 * Ring the doorbell after placing a transfer on the ring.
274 * Returns true if doorbell was set, otherwise false.
275 */
276static bool cdnsp_ring_ep_doorbell(struct cdnsp_device *pdev,
277 struct cdnsp_ep *pep,
278 unsigned int stream_id)
279{
280 __le32 __iomem *reg_addr = &pdev->dba->ep_db;
281 unsigned int ep_state = pep->ep_state;
282 unsigned int db_value;
283
284 /*
285 * Don't ring the doorbell for this endpoint if endpoint is halted or
286 * disabled.
287 */
288 if (ep_state & EP_HALTED || !(ep_state & EP_ENABLED))
289 return false;
290
291 /* For stream capable endpoints driver can ring doorbell only twice. */
292 if (pep->ep_state & EP_HAS_STREAMS) {
293 if (pep->stream_info.drbls_count >= 2)
294 return false;
295
296 pep->stream_info.drbls_count++;
297 }
298
299 pep->ep_state &= ~EP_STOPPED;
300
301 if (pep->idx == 0 && pdev->ep0_stage == CDNSP_DATA_STAGE &&
302 !pdev->ep0_expect_in)
303 db_value = DB_VALUE_EP0_OUT(pep->idx, stream_id);
304 else
305 db_value = DB_VALUE(pep->idx, stream_id);
306
307 trace_cdnsp_tr_drbl(pep, stream_id);
308
309 writel(db_value, reg_addr);
310
311 cdnsp_force_l0_go(pdev);
312
313 /* Doorbell was set. */
314 return true;
315}
316
317/*
318 * Get the right ring for the given pep and stream_id.
319 * If the endpoint supports streams, boundary check the USB request's stream ID.
320 * If the endpoint doesn't support streams, return the singular endpoint ring.
321 */
322static struct cdnsp_ring *cdnsp_get_transfer_ring(struct cdnsp_device *pdev,
323 struct cdnsp_ep *pep,
324 unsigned int stream_id)
325{
326 if (!(pep->ep_state & EP_HAS_STREAMS))
327 return pep->ring;
328
329 if (stream_id == 0 || stream_id >= pep->stream_info.num_streams) {
330 dev_err(pdev->dev, "ERR: %s ring doesn't exist for SID: %d.\n",
331 pep->name, stream_id);
332 return NULL;
333 }
334
335 return pep->stream_info.stream_rings[stream_id];
336}
337
338static struct cdnsp_ring *
339 cdnsp_request_to_transfer_ring(struct cdnsp_device *pdev,
340 struct cdnsp_request *preq)
341{
342 return cdnsp_get_transfer_ring(pdev, preq->pep,
343 preq->request.stream_id);
344}
345
346/* Ring the doorbell for any rings with pending requests. */
347void cdnsp_ring_doorbell_for_active_rings(struct cdnsp_device *pdev,
348 struct cdnsp_ep *pep)
349{
350 struct cdnsp_stream_info *stream_info;
351 unsigned int stream_id;
352 int ret;
353
354 if (pep->ep_state & EP_DIS_IN_RROGRESS)
355 return;
356
357 /* A ring has pending Request if its TD list is not empty. */
358 if (!(pep->ep_state & EP_HAS_STREAMS) && pep->number) {
359 if (pep->ring && !list_empty(&pep->ring->td_list))
360 cdnsp_ring_ep_doorbell(pdev, pep, 0);
361 return;
362 }
363
364 stream_info = &pep->stream_info;
365
366 for (stream_id = 1; stream_id < stream_info->num_streams; stream_id++) {
367 struct cdnsp_td *td, *td_temp;
368 struct cdnsp_ring *ep_ring;
369
370 if (stream_info->drbls_count >= 2)
371 return;
372
373 ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);
374 if (!ep_ring)
375 continue;
376
377 if (!ep_ring->stream_active || ep_ring->stream_rejected)
378 continue;
379
380 list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,
381 td_list) {
382 if (td->drbl)
383 continue;
384
385 ret = cdnsp_ring_ep_doorbell(pdev, pep, stream_id);
386 if (ret)
387 td->drbl = 1;
388 }
389 }
390}
391
392/*
393 * Get the hw dequeue pointer controller stopped on, either directly from the
394 * endpoint context, or if streams are in use from the stream context.
395 * The returned hw_dequeue contains the lowest four bits with cycle state
396 * and possible stream context type.
397 */
398static u64 cdnsp_get_hw_deq(struct cdnsp_device *pdev,
399 unsigned int ep_index,
400 unsigned int stream_id)
401{
402 struct cdnsp_stream_ctx *st_ctx;
403 struct cdnsp_ep *pep;
404
405 pep = &pdev->eps[ep_index];
406
407 if (pep->ep_state & EP_HAS_STREAMS) {
408 st_ctx = &pep->stream_info.stream_ctx_array[stream_id];
409 return le64_to_cpu(st_ctx->stream_ring);
410 }
411
412 return le64_to_cpu(pep->out_ctx->deq);
413}
414
415/*
416 * Move the controller endpoint ring dequeue pointer past cur_td.
417 * Record the new state of the controller endpoint ring dequeue segment,
418 * dequeue pointer, and new consumer cycle state in state.
419 * Update internal representation of the ring's dequeue pointer.
420 *
421 * We do this in three jumps:
422 * - First we update our new ring state to be the same as when the
423 * controller stopped.
424 * - Then we traverse the ring to find the segment that contains
425 * the last TRB in the TD. We toggle the controller new cycle state
426 * when we pass any link TRBs with the toggle cycle bit set.
427 * - Finally we move the dequeue state one TRB further, toggling the cycle bit
428 * if we've moved it past a link TRB with the toggle cycle bit set.
429 */
430static void cdnsp_find_new_dequeue_state(struct cdnsp_device *pdev,
431 struct cdnsp_ep *pep,
432 unsigned int stream_id,
433 struct cdnsp_td *cur_td,
434 struct cdnsp_dequeue_state *state)
435{
436 bool td_last_trb_found = false;
437 struct cdnsp_segment *new_seg;
438 struct cdnsp_ring *ep_ring;
439 union cdnsp_trb *new_deq;
440 bool cycle_found = false;
441 u64 hw_dequeue;
442
443 ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);
444 if (!ep_ring)
445 return;
446
447 /*
448 * Dig out the cycle state saved by the controller during the
449 * stop endpoint command.
450 */
451 hw_dequeue = cdnsp_get_hw_deq(pdev, pep->idx, stream_id);
452 new_seg = ep_ring->deq_seg;
453 new_deq = ep_ring->dequeue;
454 state->new_cycle_state = hw_dequeue & 0x1;
455 state->stream_id = stream_id;
456
457 /*
458 * We want to find the pointer, segment and cycle state of the new trb
459 * (the one after current TD's last_trb). We know the cycle state at
460 * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are
461 * found.
462 */
463 do {
464 if (!cycle_found && cdnsp_trb_virt_to_dma(new_seg, new_deq)
465 == (dma_addr_t)(hw_dequeue & ~0xf)) {
466 cycle_found = true;
467
468 if (td_last_trb_found)
469 break;
470 }
471
472 if (new_deq == cur_td->last_trb)
473 td_last_trb_found = true;
474
475 if (cycle_found && cdnsp_trb_is_link(new_deq) &&
476 cdnsp_link_trb_toggles_cycle(new_deq))
477 state->new_cycle_state ^= 0x1;
478
479 cdnsp_next_trb(pdev, ep_ring, &new_seg, &new_deq);
480
481 /* Search wrapped around, bail out. */
482 if (new_deq == pep->ring->dequeue) {
483 dev_err(pdev->dev,
484 "Error: Failed finding new dequeue state\n");
485 state->new_deq_seg = NULL;
486 state->new_deq_ptr = NULL;
487 return;
488 }
489
490 } while (!cycle_found || !td_last_trb_found);
491
492 state->new_deq_seg = new_seg;
493 state->new_deq_ptr = new_deq;
494
495 trace_cdnsp_new_deq_state(state);
496}
497
498/*
499 * flip_cycle means flip the cycle bit of all but the first and last TRB.
500 * (The last TRB actually points to the ring enqueue pointer, which is not part
501 * of this TD.) This is used to remove partially enqueued isoc TDs from a ring.
502 */
503static void cdnsp_td_to_noop(struct cdnsp_device *pdev,
504 struct cdnsp_ring *ep_ring,
505 struct cdnsp_td *td,
506 bool flip_cycle)
507{
508 struct cdnsp_segment *seg = td->start_seg;
509 union cdnsp_trb *trb = td->first_trb;
510
511 while (1) {
512 cdnsp_trb_to_noop(trb, TRB_TR_NOOP);
513
514 /* flip cycle if asked to */
515 if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
516 trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
517
518 if (trb == td->last_trb)
519 break;
520
521 cdnsp_next_trb(pdev, ep_ring, &seg, &trb);
522 }
523}
524
525/*
526 * This TD is defined by the TRBs starting at start_trb in start_seg and ending
527 * at end_trb, which may be in another segment. If the suspect DMA address is a
528 * TRB in this TD, this function returns that TRB's segment. Otherwise it
529 * returns 0.
530 */
531static struct cdnsp_segment *cdnsp_trb_in_td(struct cdnsp_device *pdev,
532 struct cdnsp_segment *start_seg,
533 union cdnsp_trb *start_trb,
534 union cdnsp_trb *end_trb,
535 dma_addr_t suspect_dma)
536{
537 struct cdnsp_segment *cur_seg;
538 union cdnsp_trb *temp_trb;
539 dma_addr_t end_seg_dma;
540 dma_addr_t end_trb_dma;
541 dma_addr_t start_dma;
542
543 start_dma = cdnsp_trb_virt_to_dma(start_seg, start_trb);
544 cur_seg = start_seg;
545
546 do {
547 if (start_dma == 0)
548 return NULL;
549
550 temp_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1];
551 /* We may get an event for a Link TRB in the middle of a TD */
552 end_seg_dma = cdnsp_trb_virt_to_dma(cur_seg, temp_trb);
553 /* If the end TRB isn't in this segment, this is set to 0 */
554 end_trb_dma = cdnsp_trb_virt_to_dma(cur_seg, end_trb);
555
556 trace_cdnsp_looking_trb_in_td(suspect_dma, start_dma,
557 end_trb_dma, cur_seg->dma,
558 end_seg_dma);
559
560 if (end_trb_dma > 0) {
561 /*
562 * The end TRB is in this segment, so suspect should
563 * be here
564 */
565 if (start_dma <= end_trb_dma) {
566 if (suspect_dma >= start_dma &&
567 suspect_dma <= end_trb_dma) {
568 return cur_seg;
569 }
570 } else {
571 /*
572 * Case for one segment with a
573 * TD wrapped around to the top
574 */
575 if ((suspect_dma >= start_dma &&
576 suspect_dma <= end_seg_dma) ||
577 (suspect_dma >= cur_seg->dma &&
578 suspect_dma <= end_trb_dma)) {
579 return cur_seg;
580 }
581 }
582
583 return NULL;
584 }
585
586 /* Might still be somewhere in this segment */
587 if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
588 return cur_seg;
589
590 cur_seg = cur_seg->next;
591 start_dma = cdnsp_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]);
592 } while (cur_seg != start_seg);
593
594 return NULL;
595}
596
597static void cdnsp_unmap_td_bounce_buffer(struct cdnsp_device *pdev,
598 struct cdnsp_ring *ring,
599 struct cdnsp_td *td)
600{
601 struct cdnsp_segment *seg = td->bounce_seg;
602 struct cdnsp_request *preq;
603 size_t len;
604
605 if (!seg)
606 return;
607
608 preq = td->preq;
609
610 trace_cdnsp_bounce_unmap(td->preq, seg->bounce_len, seg->bounce_offs,
611 seg->bounce_dma, 0);
612
613 if (!preq->direction) {
614 dma_unmap_single(pdev->dev, seg->bounce_dma,
615 ring->bounce_buf_len, DMA_TO_DEVICE);
616 return;
617 }
618
619 dma_unmap_single(pdev->dev, seg->bounce_dma, ring->bounce_buf_len,
620 DMA_FROM_DEVICE);
621
622 /* For in transfers we need to copy the data from bounce to sg */
623 len = sg_pcopy_from_buffer(preq->request.sg, preq->request.num_sgs,
624 seg->bounce_buf, seg->bounce_len,
625 seg->bounce_offs);
626 if (len != seg->bounce_len)
627 dev_warn(pdev->dev, "WARN Wrong bounce buffer read length: %zu != %d\n",
628 len, seg->bounce_len);
629
630 seg->bounce_len = 0;
631 seg->bounce_offs = 0;
632}
633
634static int cdnsp_cmd_set_deq(struct cdnsp_device *pdev,
635 struct cdnsp_ep *pep,
636 struct cdnsp_dequeue_state *deq_state)
637{
638 struct cdnsp_ring *ep_ring;
639 int ret;
640
641 if (!deq_state->new_deq_ptr || !deq_state->new_deq_seg) {
642 cdnsp_ring_doorbell_for_active_rings(pdev, pep);
643 return 0;
644 }
645
646 cdnsp_queue_new_dequeue_state(pdev, pep, deq_state);
647 cdnsp_ring_cmd_db(pdev);
648 ret = cdnsp_wait_for_cmd_compl(pdev);
649
650 trace_cdnsp_handle_cmd_set_deq(cdnsp_get_slot_ctx(&pdev->out_ctx));
651 trace_cdnsp_handle_cmd_set_deq_ep(pep->out_ctx);
652
653 /*
654 * Update the ring's dequeue segment and dequeue pointer
655 * to reflect the new position.
656 */
657 ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id);
658
659 if (cdnsp_trb_is_link(ep_ring->dequeue)) {
660 ep_ring->deq_seg = ep_ring->deq_seg->next;
661 ep_ring->dequeue = ep_ring->deq_seg->trbs;
662 }
663
664 while (ep_ring->dequeue != deq_state->new_deq_ptr) {
665 ep_ring->num_trbs_free++;
666 ep_ring->dequeue++;
667
668 if (cdnsp_trb_is_link(ep_ring->dequeue)) {
669 if (ep_ring->dequeue == deq_state->new_deq_ptr)
670 break;
671
672 ep_ring->deq_seg = ep_ring->deq_seg->next;
673 ep_ring->dequeue = ep_ring->deq_seg->trbs;
674 }
675 }
676
677 /*
678 * Probably there was TIMEOUT during handling Set Dequeue Pointer
679 * command. It's critical error and controller will be stopped.
680 */
681 if (ret)
682 return -ESHUTDOWN;
683
684 /* Restart any rings with pending requests */
685 cdnsp_ring_doorbell_for_active_rings(pdev, pep);
686
687 return 0;
688}
689
690int cdnsp_remove_request(struct cdnsp_device *pdev,
691 struct cdnsp_request *preq,
692 struct cdnsp_ep *pep)
693{
694 struct cdnsp_dequeue_state deq_state;
695 struct cdnsp_td *cur_td = NULL;
696 struct cdnsp_ring *ep_ring;
697 struct cdnsp_segment *seg;
698 int status = -ECONNRESET;
699 int ret = 0;
700 u64 hw_deq;
701
702 memset(&deq_state, 0, sizeof(deq_state));
703
704 trace_cdnsp_remove_request(pep->out_ctx);
705 trace_cdnsp_remove_request_td(preq);
706
707 cur_td = &preq->td;
708 ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);
709
710 /*
711 * If we stopped on the TD we need to cancel, then we have to
712 * move the controller endpoint ring dequeue pointer past
713 * this TD.
714 */
715 hw_deq = cdnsp_get_hw_deq(pdev, pep->idx, preq->request.stream_id);
716 hw_deq &= ~0xf;
717
718 seg = cdnsp_trb_in_td(pdev, cur_td->start_seg, cur_td->first_trb,
719 cur_td->last_trb, hw_deq);
720
721 if (seg && (pep->ep_state & EP_ENABLED) &&
722 !(pep->ep_state & EP_DIS_IN_RROGRESS))
723 cdnsp_find_new_dequeue_state(pdev, pep, preq->request.stream_id,
724 cur_td, &deq_state);
725 else
726 cdnsp_td_to_noop(pdev, ep_ring, cur_td, false);
727
728 /*
729 * The event handler won't see a completion for this TD anymore,
730 * so remove it from the endpoint ring's TD list.
731 */
732 list_del_init(&cur_td->td_list);
733 ep_ring->num_tds--;
734 pep->stream_info.td_count--;
735
736 /*
737 * During disconnecting all endpoint will be disabled so we don't
738 * have to worry about updating dequeue pointer.
739 */
740 if (pdev->cdnsp_state & CDNSP_STATE_DISCONNECT_PENDING ||
741 pep->ep_state & EP_DIS_IN_RROGRESS) {
742 status = -ESHUTDOWN;
743 ret = cdnsp_cmd_set_deq(pdev, pep, &deq_state);
744 }
745
746 cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, cur_td);
747 cdnsp_gadget_giveback(pep, cur_td->preq, status);
748
749 return ret;
750}
751
752static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id)
753{
754 struct cdnsp_port *port = pdev->active_port;
755 u8 old_port = 0;
756
757 if (port && port->port_num == port_id)
758 return 0;
759
760 if (port)
761 old_port = port->port_num;
762
763 if (port_id == pdev->usb2_port.port_num) {
764 port = &pdev->usb2_port;
765 } else if (port_id == pdev->usb3_port.port_num) {
766 port = &pdev->usb3_port;
767 } else {
768 dev_err(pdev->dev, "Port event with invalid port ID %d\n",
769 port_id);
770 return -EINVAL;
771 }
772
773 if (port_id != old_port) {
774 cdnsp_disable_slot(pdev);
775 pdev->active_port = port;
776 cdnsp_enable_slot(pdev);
777 }
778
779 if (port_id == pdev->usb2_port.port_num)
780 cdnsp_set_usb2_hardware_lpm(pdev, NULL, 1);
781 else
782 writel(PORT_U1_TIMEOUT(1) | PORT_U2_TIMEOUT(1),
783 &pdev->usb3_port.regs->portpmsc);
784
785 return 0;
786}
787
788static void cdnsp_handle_port_status(struct cdnsp_device *pdev,
789 union cdnsp_trb *event)
790{
791 struct cdnsp_port_regs __iomem *port_regs;
792 u32 portsc, cmd_regs;
793 bool port2 = false;
794 u32 link_state;
795 u32 port_id;
796
797 /* Port status change events always have a successful completion code */
798 if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
799 dev_err(pdev->dev, "ERR: incorrect PSC event\n");
800
801 port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
802
803 if (cdnsp_update_port_id(pdev, port_id))
804 goto cleanup;
805
806 port_regs = pdev->active_port->regs;
807
808 if (port_id == pdev->usb2_port.port_num)
809 port2 = true;
810
811new_event:
812 portsc = readl(&port_regs->portsc);
813 writel(cdnsp_port_state_to_neutral(portsc) |
814 (portsc & PORT_CHANGE_BITS), &port_regs->portsc);
815
816 trace_cdnsp_handle_port_status(pdev->active_port->port_num, portsc);
817
818 pdev->gadget.speed = cdnsp_port_speed(portsc);
819 link_state = portsc & PORT_PLS_MASK;
820
821 /* Port Link State change detected. */
822 if ((portsc & PORT_PLC)) {
823 if (!(pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) &&
824 link_state == XDEV_RESUME) {
825 cmd_regs = readl(&pdev->op_regs->command);
826 if (!(cmd_regs & CMD_R_S))
827 goto cleanup;
828
829 if (DEV_SUPERSPEED_ANY(portsc)) {
830 cdnsp_set_link_state(pdev, &port_regs->portsc,
831 XDEV_U0);
832
833 cdnsp_resume_gadget(pdev);
834 }
835 }
836
837 if ((pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) &&
838 link_state == XDEV_U0) {
839 pdev->cdnsp_state &= ~CDNSP_WAKEUP_PENDING;
840
841 cdnsp_force_header_wakeup(pdev, 1);
842 cdnsp_ring_cmd_db(pdev);
843 cdnsp_wait_for_cmd_compl(pdev);
844 }
845
846 if (link_state == XDEV_U0 && pdev->link_state == XDEV_U3 &&
847 !DEV_SUPERSPEED_ANY(portsc))
848 cdnsp_resume_gadget(pdev);
849
850 if (link_state == XDEV_U3 && pdev->link_state != XDEV_U3)
851 cdnsp_suspend_gadget(pdev);
852
853 pdev->link_state = link_state;
854 }
855
856 if (portsc & PORT_CSC) {
857 /* Detach device. */
858 if (pdev->gadget.connected && !(portsc & PORT_CONNECT))
859 cdnsp_disconnect_gadget(pdev);
860
861 /* Attach device. */
862 if (portsc & PORT_CONNECT) {
863 if (!port2)
864 cdnsp_irq_reset(pdev);
865
866 usb_gadget_set_state(&pdev->gadget, USB_STATE_ATTACHED);
867 }
868 }
869
870 /* Port reset. */
871 if ((portsc & (PORT_RC | PORT_WRC)) && (portsc & PORT_CONNECT)) {
872 cdnsp_irq_reset(pdev);
873 pdev->u1_allowed = 0;
874 pdev->u2_allowed = 0;
875 pdev->may_wakeup = 0;
876 }
877
878 if (portsc & PORT_CEC)
879 dev_err(pdev->dev, "Port Over Current detected\n");
880
881 if (portsc & PORT_CEC)
882 dev_err(pdev->dev, "Port Configure Error detected\n");
883
884 if (readl(&port_regs->portsc) & PORT_CHANGE_BITS)
885 goto new_event;
886
887cleanup:
888 cdnsp_inc_deq(pdev, pdev->event_ring);
889}
890
891static void cdnsp_td_cleanup(struct cdnsp_device *pdev,
892 struct cdnsp_td *td,
893 struct cdnsp_ring *ep_ring,
894 int *status)
895{
896 struct cdnsp_request *preq = td->preq;
897
898 /* if a bounce buffer was used to align this td then unmap it */
899 cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, td);
900
901 /*
902 * If the controller said we transferred more data than the buffer
903 * length, Play it safe and say we didn't transfer anything.
904 */
905 if (preq->request.actual > preq->request.length) {
906 preq->request.actual = 0;
907 *status = 0;
908 }
909
910 list_del_init(&td->td_list);
911 ep_ring->num_tds--;
912 preq->pep->stream_info.td_count--;
913
914 cdnsp_gadget_giveback(preq->pep, preq, *status);
915}
916
917static void cdnsp_finish_td(struct cdnsp_device *pdev,
918 struct cdnsp_td *td,
919 struct cdnsp_transfer_event *event,
920 struct cdnsp_ep *ep,
921 int *status)
922{
923 struct cdnsp_ring *ep_ring;
924 u32 trb_comp_code;
925
926 ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
927 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
928
929 if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
930 trb_comp_code == COMP_STOPPED ||
931 trb_comp_code == COMP_STOPPED_SHORT_PACKET) {
932 /*
933 * The Endpoint Stop Command completion will take care of any
934 * stopped TDs. A stopped TD may be restarted, so don't update
935 * the ring dequeue pointer or take this TD off any lists yet.
936 */
937 return;
938 }
939
940 /* Update ring dequeue pointer */
941 while (ep_ring->dequeue != td->last_trb)
942 cdnsp_inc_deq(pdev, ep_ring);
943
944 cdnsp_inc_deq(pdev, ep_ring);
945
946 cdnsp_td_cleanup(pdev, td, ep_ring, status);
947}
948
949/* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
950static int cdnsp_sum_trb_lengths(struct cdnsp_device *pdev,
951 struct cdnsp_ring *ring,
952 union cdnsp_trb *stop_trb)
953{
954 struct cdnsp_segment *seg = ring->deq_seg;
955 union cdnsp_trb *trb = ring->dequeue;
956 u32 sum;
957
958 for (sum = 0; trb != stop_trb; cdnsp_next_trb(pdev, ring, &seg, &trb)) {
959 if (!cdnsp_trb_is_noop(trb) && !cdnsp_trb_is_link(trb))
960 sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
961 }
962 return sum;
963}
964
965static int cdnsp_giveback_first_trb(struct cdnsp_device *pdev,
966 struct cdnsp_ep *pep,
967 unsigned int stream_id,
968 int start_cycle,
969 struct cdnsp_generic_trb *start_trb)
970{
971 /*
972 * Pass all the TRBs to the hardware at once and make sure this write
973 * isn't reordered.
974 */
975 wmb();
976
977 if (start_cycle)
978 start_trb->field[3] |= cpu_to_le32(start_cycle);
979 else
980 start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
981
982 if ((pep->ep_state & EP_HAS_STREAMS) &&
983 !pep->stream_info.first_prime_det) {
984 trace_cdnsp_wait_for_prime(pep, stream_id);
985 return 0;
986 }
987
988 return cdnsp_ring_ep_doorbell(pdev, pep, stream_id);
989}
990
991/*
992 * Process control tds, update USB request status and actual_length.
993 */
994static void cdnsp_process_ctrl_td(struct cdnsp_device *pdev,
995 struct cdnsp_td *td,
996 union cdnsp_trb *event_trb,
997 struct cdnsp_transfer_event *event,
998 struct cdnsp_ep *pep,
999 int *status)
1000{
1001 struct cdnsp_ring *ep_ring;
1002 u32 remaining;
1003 u32 trb_type;
1004
1005 trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event_trb->generic.field[3]));
1006 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1007 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1008
1009 /*
1010 * if on data stage then update the actual_length of the USB
1011 * request and flag it as set, so it won't be overwritten in the event
1012 * for the last TRB.
1013 */
1014 if (trb_type == TRB_DATA) {
1015 td->request_length_set = true;
1016 td->preq->request.actual = td->preq->request.length - remaining;
1017 }
1018
1019 /* at status stage */
1020 if (!td->request_length_set)
1021 td->preq->request.actual = td->preq->request.length;
1022
1023 if (pdev->ep0_stage == CDNSP_DATA_STAGE && pep->number == 0 &&
1024 pdev->three_stage_setup) {
1025 td = list_entry(ep_ring->td_list.next, struct cdnsp_td,
1026 td_list);
1027 pdev->ep0_stage = CDNSP_STATUS_STAGE;
1028
1029 cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state,
1030 &td->last_trb->generic);
1031 return;
1032 }
1033
1034 *status = 0;
1035
1036 cdnsp_finish_td(pdev, td, event, pep, status);
1037}
1038
1039/*
1040 * Process isochronous tds, update usb request status and actual_length.
1041 */
1042static void cdnsp_process_isoc_td(struct cdnsp_device *pdev,
1043 struct cdnsp_td *td,
1044 union cdnsp_trb *ep_trb,
1045 struct cdnsp_transfer_event *event,
1046 struct cdnsp_ep *pep,
1047 int status)
1048{
1049 struct cdnsp_request *preq = td->preq;
1050 u32 remaining, requested, ep_trb_len;
1051 bool sum_trbs_for_length = false;
1052 struct cdnsp_ring *ep_ring;
1053 u32 trb_comp_code;
1054 u32 td_length;
1055
1056 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1057 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1058 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1059 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
1060
1061 requested = preq->request.length;
1062
1063 /* handle completion code */
1064 switch (trb_comp_code) {
1065 case COMP_SUCCESS:
1066 preq->request.status = 0;
1067 break;
1068 case COMP_SHORT_PACKET:
1069 preq->request.status = 0;
1070 sum_trbs_for_length = true;
1071 break;
1072 case COMP_ISOCH_BUFFER_OVERRUN:
1073 case COMP_BABBLE_DETECTED_ERROR:
1074 preq->request.status = -EOVERFLOW;
1075 break;
1076 case COMP_STOPPED:
1077 sum_trbs_for_length = true;
1078 break;
1079 case COMP_STOPPED_SHORT_PACKET:
1080 /* field normally containing residue now contains transferred */
1081 preq->request.status = 0;
1082 requested = remaining;
1083 break;
1084 case COMP_STOPPED_LENGTH_INVALID:
1085 requested = 0;
1086 remaining = 0;
1087 break;
1088 default:
1089 sum_trbs_for_length = true;
1090 preq->request.status = -1;
1091 break;
1092 }
1093
1094 if (sum_trbs_for_length) {
1095 td_length = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb);
1096 td_length += ep_trb_len - remaining;
1097 } else {
1098 td_length = requested;
1099 }
1100
1101 td->preq->request.actual += td_length;
1102
1103 cdnsp_finish_td(pdev, td, event, pep, &status);
1104}
1105
1106static void cdnsp_skip_isoc_td(struct cdnsp_device *pdev,
1107 struct cdnsp_td *td,
1108 struct cdnsp_transfer_event *event,
1109 struct cdnsp_ep *pep,
1110 int status)
1111{
1112 struct cdnsp_ring *ep_ring;
1113
1114 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1115 td->preq->request.status = -EXDEV;
1116 td->preq->request.actual = 0;
1117
1118 /* Update ring dequeue pointer */
1119 while (ep_ring->dequeue != td->last_trb)
1120 cdnsp_inc_deq(pdev, ep_ring);
1121
1122 cdnsp_inc_deq(pdev, ep_ring);
1123
1124 cdnsp_td_cleanup(pdev, td, ep_ring, &status);
1125}
1126
1127/*
1128 * Process bulk and interrupt tds, update usb request status and actual_length.
1129 */
1130static void cdnsp_process_bulk_intr_td(struct cdnsp_device *pdev,
1131 struct cdnsp_td *td,
1132 union cdnsp_trb *ep_trb,
1133 struct cdnsp_transfer_event *event,
1134 struct cdnsp_ep *ep,
1135 int *status)
1136{
1137 u32 remaining, requested, ep_trb_len;
1138 struct cdnsp_ring *ep_ring;
1139 u32 trb_comp_code;
1140
1141 ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1142 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1143 remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1144 ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
1145 requested = td->preq->request.length;
1146
1147 switch (trb_comp_code) {
1148 case COMP_SUCCESS:
1149 case COMP_SHORT_PACKET:
1150 *status = 0;
1151 break;
1152 case COMP_STOPPED_SHORT_PACKET:
1153 td->preq->request.actual = remaining;
1154 goto finish_td;
1155 case COMP_STOPPED_LENGTH_INVALID:
1156 /* Stopped on ep trb with invalid length, exclude it. */
1157 ep_trb_len = 0;
1158 remaining = 0;
1159 break;
1160 }
1161
1162 if (ep_trb == td->last_trb)
1163 ep_trb_len = requested - remaining;
1164 else
1165 ep_trb_len = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb) +
1166 ep_trb_len - remaining;
1167 td->preq->request.actual = ep_trb_len;
1168
1169finish_td:
1170 ep->stream_info.drbls_count--;
1171
1172 cdnsp_finish_td(pdev, td, event, ep, status);
1173}
1174
1175static void cdnsp_handle_tx_nrdy(struct cdnsp_device *pdev,
1176 struct cdnsp_transfer_event *event)
1177{
1178 struct cdnsp_generic_trb *generic;
1179 struct cdnsp_ring *ep_ring;
1180 struct cdnsp_ep *pep;
1181 int cur_stream;
1182 int ep_index;
1183 int host_sid;
1184 int dev_sid;
1185
1186 generic = (struct cdnsp_generic_trb *)event;
1187 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1188 dev_sid = TRB_TO_DEV_STREAM(le32_to_cpu(generic->field[0]));
1189 host_sid = TRB_TO_HOST_STREAM(le32_to_cpu(generic->field[2]));
1190
1191 pep = &pdev->eps[ep_index];
1192
1193 if (!(pep->ep_state & EP_HAS_STREAMS))
1194 return;
1195
1196 if (host_sid == STREAM_PRIME_ACK) {
1197 pep->stream_info.first_prime_det = 1;
1198 for (cur_stream = 1; cur_stream < pep->stream_info.num_streams;
1199 cur_stream++) {
1200 ep_ring = pep->stream_info.stream_rings[cur_stream];
1201 ep_ring->stream_active = 1;
1202 ep_ring->stream_rejected = 0;
1203 }
1204 }
1205
1206 if (host_sid == STREAM_REJECTED) {
1207 struct cdnsp_td *td, *td_temp;
1208
1209 pep->stream_info.drbls_count--;
1210 ep_ring = pep->stream_info.stream_rings[dev_sid];
1211 ep_ring->stream_active = 0;
1212 ep_ring->stream_rejected = 1;
1213
1214 list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,
1215 td_list) {
1216 td->drbl = 0;
1217 }
1218 }
1219
1220 cdnsp_ring_doorbell_for_active_rings(pdev, pep);
1221}
1222
1223/*
1224 * If this function returns an error condition, it means it got a Transfer
1225 * event with a corrupted TRB DMA address or endpoint is disabled.
1226 */
1227static int cdnsp_handle_tx_event(struct cdnsp_device *pdev,
1228 struct cdnsp_transfer_event *event)
1229{
1230 const struct usb_endpoint_descriptor *desc;
1231 bool handling_skipped_tds = false;
1232 struct cdnsp_segment *ep_seg;
1233 struct cdnsp_ring *ep_ring;
1234 int status = -EINPROGRESS;
1235 union cdnsp_trb *ep_trb;
1236 dma_addr_t ep_trb_dma;
1237 struct cdnsp_ep *pep;
1238 struct cdnsp_td *td;
1239 u32 trb_comp_code;
1240 int invalidate;
1241 int ep_index;
1242
1243 invalidate = le32_to_cpu(event->flags) & TRB_EVENT_INVALIDATE;
1244 ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1245 trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1246 ep_trb_dma = le64_to_cpu(event->buffer);
1247
1248 pep = &pdev->eps[ep_index];
1249 ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1250
1251 /*
1252 * If device is disconnect then all requests will be dequeued
1253 * by upper layers as part of disconnect sequence.
1254 * We don't want handle such event to avoid racing.
1255 */
1256 if (invalidate || !pdev->gadget.connected)
1257 goto cleanup;
1258
1259 if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_DISABLED) {
1260 trace_cdnsp_ep_disabled(pep->out_ctx);
1261 goto err_out;
1262 }
1263
1264 /* Some transfer events don't always point to a trb*/
1265 if (!ep_ring) {
1266 switch (trb_comp_code) {
1267 case COMP_INVALID_STREAM_TYPE_ERROR:
1268 case COMP_INVALID_STREAM_ID_ERROR:
1269 case COMP_RING_UNDERRUN:
1270 case COMP_RING_OVERRUN:
1271 goto cleanup;
1272 default:
1273 dev_err(pdev->dev, "ERROR: %s event for unknown ring\n",
1274 pep->name);
1275 goto err_out;
1276 }
1277 }
1278
1279 /* Look for some error cases that need special treatment. */
1280 switch (trb_comp_code) {
1281 case COMP_BABBLE_DETECTED_ERROR:
1282 status = -EOVERFLOW;
1283 break;
1284 case COMP_RING_UNDERRUN:
1285 case COMP_RING_OVERRUN:
1286 /*
1287 * When the Isoch ring is empty, the controller will generate
1288 * a Ring Overrun Event for IN Isoch endpoint or Ring
1289 * Underrun Event for OUT Isoch endpoint.
1290 */
1291 goto cleanup;
1292 case COMP_MISSED_SERVICE_ERROR:
1293 /*
1294 * When encounter missed service error, one or more isoc tds
1295 * may be missed by controller.
1296 * Set skip flag of the ep_ring; Complete the missed tds as
1297 * short transfer when process the ep_ring next time.
1298 */
1299 pep->skip = true;
1300 break;
1301 }
1302
1303 do {
1304 /*
1305 * This TRB should be in the TD at the head of this ring's TD
1306 * list.
1307 */
1308 if (list_empty(&ep_ring->td_list)) {
1309 /*
1310 * Don't print warnings if it's due to a stopped
1311 * endpoint generating an extra completion event, or
1312 * a event for the last TRB of a short TD we already
1313 * got a short event for.
1314 * The short TD is already removed from the TD list.
1315 */
1316 if (!(trb_comp_code == COMP_STOPPED ||
1317 trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
1318 ep_ring->last_td_was_short))
1319 trace_cdnsp_trb_without_td(ep_ring,
1320 (struct cdnsp_generic_trb *)event);
1321
1322 if (pep->skip) {
1323 pep->skip = false;
1324 trace_cdnsp_ep_list_empty_with_skip(pep, 0);
1325 }
1326
1327 goto cleanup;
1328 }
1329
1330 td = list_entry(ep_ring->td_list.next, struct cdnsp_td,
1331 td_list);
1332
1333 /* Is this a TRB in the currently executing TD? */
1334 ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg,
1335 ep_ring->dequeue, td->last_trb,
1336 ep_trb_dma);
1337
1338 desc = td->preq->pep->endpoint.desc;
1339
1340 if (ep_seg) {
1341 ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma)
1342 / sizeof(*ep_trb)];
1343
1344 trace_cdnsp_handle_transfer(ep_ring,
1345 (struct cdnsp_generic_trb *)ep_trb);
1346
1347 if (pep->skip && usb_endpoint_xfer_isoc(desc) &&
1348 td->last_trb != ep_trb)
1349 return -EAGAIN;
1350 }
1351
1352 /*
1353 * Skip the Force Stopped Event. The event_trb(ep_trb_dma)
1354 * of FSE is not in the current TD pointed by ep_ring->dequeue
1355 * because that the hardware dequeue pointer still at the
1356 * previous TRB of the current TD. The previous TRB maybe a
1357 * Link TD or the last TRB of the previous TD. The command
1358 * completion handle will take care the rest.
1359 */
1360 if (!ep_seg && (trb_comp_code == COMP_STOPPED ||
1361 trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) {
1362 pep->skip = false;
1363 goto cleanup;
1364 }
1365
1366 if (!ep_seg) {
1367 if (!pep->skip || !usb_endpoint_xfer_isoc(desc)) {
1368 /* Something is busted, give up! */
1369 dev_err(pdev->dev,
1370 "ERROR Transfer event TRB DMA ptr not "
1371 "part of current TD ep_index %d "
1372 "comp_code %u\n", ep_index,
1373 trb_comp_code);
1374 return -EINVAL;
1375 }
1376
1377 cdnsp_skip_isoc_td(pdev, td, event, pep, status);
1378 goto cleanup;
1379 }
1380
1381 if (trb_comp_code == COMP_SHORT_PACKET)
1382 ep_ring->last_td_was_short = true;
1383 else
1384 ep_ring->last_td_was_short = false;
1385
1386 if (pep->skip) {
1387 pep->skip = false;
1388 cdnsp_skip_isoc_td(pdev, td, event, pep, status);
1389 goto cleanup;
1390 }
1391
1392 if (cdnsp_trb_is_noop(ep_trb))
1393 goto cleanup;
1394
1395 if (usb_endpoint_xfer_control(desc))
1396 cdnsp_process_ctrl_td(pdev, td, ep_trb, event, pep,
1397 &status);
1398 else if (usb_endpoint_xfer_isoc(desc))
1399 cdnsp_process_isoc_td(pdev, td, ep_trb, event, pep,
1400 status);
1401 else
1402 cdnsp_process_bulk_intr_td(pdev, td, ep_trb, event, pep,
1403 &status);
1404cleanup:
1405 handling_skipped_tds = pep->skip;
1406
1407 /*
1408 * Do not update event ring dequeue pointer if we're in a loop
1409 * processing missed tds.
1410 */
1411 if (!handling_skipped_tds)
1412 cdnsp_inc_deq(pdev, pdev->event_ring);
1413
1414 /*
1415 * If ep->skip is set, it means there are missed tds on the
1416 * endpoint ring need to take care of.
1417 * Process them as short transfer until reach the td pointed by
1418 * the event.
1419 */
1420 } while (handling_skipped_tds);
1421 return 0;
1422
1423err_out:
1424 dev_err(pdev->dev, "@%016llx %08x %08x %08x %08x\n",
1425 (unsigned long long)
1426 cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1427 pdev->event_ring->dequeue),
1428 lower_32_bits(le64_to_cpu(event->buffer)),
1429 upper_32_bits(le64_to_cpu(event->buffer)),
1430 le32_to_cpu(event->transfer_len),
1431 le32_to_cpu(event->flags));
1432 return -EINVAL;
1433}
1434
1435/*
1436 * This function handles all events on the event ring.
1437 * Returns true for "possibly more events to process" (caller should call
1438 * again), otherwise false if done.
1439 */
1440static bool cdnsp_handle_event(struct cdnsp_device *pdev)
1441{
1442 unsigned int comp_code;
1443 union cdnsp_trb *event;
1444 bool update_ptrs = true;
1445 u32 cycle_bit;
1446 int ret = 0;
1447 u32 flags;
1448
1449 event = pdev->event_ring->dequeue;
1450 flags = le32_to_cpu(event->event_cmd.flags);
1451 cycle_bit = (flags & TRB_CYCLE);
1452
1453 /* Does the controller or driver own the TRB? */
1454 if (cycle_bit != pdev->event_ring->cycle_state)
1455 return false;
1456
1457 trace_cdnsp_handle_event(pdev->event_ring, &event->generic);
1458
1459 /*
1460 * Barrier between reading the TRB_CYCLE (valid) flag above and any
1461 * reads of the event's flags/data below.
1462 */
1463 rmb();
1464
1465 switch (flags & TRB_TYPE_BITMASK) {
1466 case TRB_TYPE(TRB_COMPLETION):
1467 /*
1468 * Command can't be handled in interrupt context so just
1469 * increment command ring dequeue pointer.
1470 */
1471 cdnsp_inc_deq(pdev, pdev->cmd_ring);
1472 break;
1473 case TRB_TYPE(TRB_PORT_STATUS):
1474 cdnsp_handle_port_status(pdev, event);
1475 update_ptrs = false;
1476 break;
1477 case TRB_TYPE(TRB_TRANSFER):
1478 ret = cdnsp_handle_tx_event(pdev, &event->trans_event);
1479 if (ret >= 0)
1480 update_ptrs = false;
1481 break;
1482 case TRB_TYPE(TRB_SETUP):
1483 pdev->ep0_stage = CDNSP_SETUP_STAGE;
1484 pdev->setup_id = TRB_SETUPID_TO_TYPE(flags);
1485 pdev->setup_speed = TRB_SETUP_SPEEDID(flags);
1486 pdev->setup = *((struct usb_ctrlrequest *)
1487 &event->trans_event.buffer);
1488
1489 cdnsp_setup_analyze(pdev);
1490 break;
1491 case TRB_TYPE(TRB_ENDPOINT_NRDY):
1492 cdnsp_handle_tx_nrdy(pdev, &event->trans_event);
1493 break;
1494 case TRB_TYPE(TRB_HC_EVENT): {
1495 comp_code = GET_COMP_CODE(le32_to_cpu(event->generic.field[2]));
1496
1497 switch (comp_code) {
1498 case COMP_EVENT_RING_FULL_ERROR:
1499 dev_err(pdev->dev, "Event Ring Full\n");
1500 break;
1501 default:
1502 dev_err(pdev->dev, "Controller error code 0x%02x\n",
1503 comp_code);
1504 }
1505
1506 break;
1507 }
1508 case TRB_TYPE(TRB_MFINDEX_WRAP):
1509 case TRB_TYPE(TRB_DRB_OVERFLOW):
1510 break;
1511 default:
1512 dev_warn(pdev->dev, "ERROR unknown event type %ld\n",
1513 TRB_FIELD_TO_TYPE(flags));
1514 }
1515
1516 if (update_ptrs)
1517 /* Update SW event ring dequeue pointer. */
1518 cdnsp_inc_deq(pdev, pdev->event_ring);
1519
1520 /*
1521 * Caller will call us again to check if there are more items
1522 * on the event ring.
1523 */
1524 return true;
1525}
1526
1527irqreturn_t cdnsp_thread_irq_handler(int irq, void *data)
1528{
1529 struct cdnsp_device *pdev = (struct cdnsp_device *)data;
1530 union cdnsp_trb *event_ring_deq;
1531 unsigned long flags;
1532 int counter = 0;
1533
1534 local_bh_disable();
1535 spin_lock_irqsave(&pdev->lock, flags);
1536
1537 if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) {
1538 /*
1539 * While removing or stopping driver there may still be deferred
1540 * not handled interrupt which should not be treated as error.
1541 * Driver should simply ignore it.
1542 */
1543 if (pdev->gadget_driver)
1544 cdnsp_died(pdev);
1545
1546 spin_unlock_irqrestore(&pdev->lock, flags);
1547 local_bh_enable();
1548 return IRQ_HANDLED;
1549 }
1550
1551 event_ring_deq = pdev->event_ring->dequeue;
1552
1553 while (cdnsp_handle_event(pdev)) {
1554 if (++counter >= TRBS_PER_EV_DEQ_UPDATE) {
1555 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 0);
1556 event_ring_deq = pdev->event_ring->dequeue;
1557 counter = 0;
1558 }
1559 }
1560
1561 cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1);
1562
1563 spin_unlock_irqrestore(&pdev->lock, flags);
1564 local_bh_enable();
1565
1566 return IRQ_HANDLED;
1567}
1568
1569irqreturn_t cdnsp_irq_handler(int irq, void *priv)
1570{
1571 struct cdnsp_device *pdev = (struct cdnsp_device *)priv;
1572 u32 irq_pending;
1573 u32 status;
1574
1575 status = readl(&pdev->op_regs->status);
1576
1577 if (status == ~(u32)0) {
1578 cdnsp_died(pdev);
1579 return IRQ_HANDLED;
1580 }
1581
1582 if (!(status & STS_EINT))
1583 return IRQ_NONE;
1584
1585 writel(status | STS_EINT, &pdev->op_regs->status);
1586 irq_pending = readl(&pdev->ir_set->irq_pending);
1587 irq_pending |= IMAN_IP;
1588 writel(irq_pending, &pdev->ir_set->irq_pending);
1589
1590 if (status & STS_FATAL) {
1591 cdnsp_died(pdev);
1592 return IRQ_HANDLED;
1593 }
1594
1595 return IRQ_WAKE_THREAD;
1596}
1597
1598/*
1599 * Generic function for queuing a TRB on a ring.
1600 * The caller must have checked to make sure there's room on the ring.
1601 *
1602 * @more_trbs_coming: Will you enqueue more TRBs before setting doorbell?
1603 */
1604static void cdnsp_queue_trb(struct cdnsp_device *pdev, struct cdnsp_ring *ring,
1605 bool more_trbs_coming, u32 field1, u32 field2,
1606 u32 field3, u32 field4)
1607{
1608 struct cdnsp_generic_trb *trb;
1609
1610 trb = &ring->enqueue->generic;
1611
1612 trb->field[0] = cpu_to_le32(field1);
1613 trb->field[1] = cpu_to_le32(field2);
1614 trb->field[2] = cpu_to_le32(field3);
1615 trb->field[3] = cpu_to_le32(field4);
1616
1617 trace_cdnsp_queue_trb(ring, trb);
1618 cdnsp_inc_enq(pdev, ring, more_trbs_coming);
1619}
1620
1621/*
1622 * Does various checks on the endpoint ring, and makes it ready to
1623 * queue num_trbs.
1624 */
1625static int cdnsp_prepare_ring(struct cdnsp_device *pdev,
1626 struct cdnsp_ring *ep_ring,
1627 u32 ep_state, unsigned
1628 int num_trbs,
1629 gfp_t mem_flags)
1630{
1631 unsigned int num_trbs_needed;
1632
1633 /* Make sure the endpoint has been added to controller schedule. */
1634 switch (ep_state) {
1635 case EP_STATE_STOPPED:
1636 case EP_STATE_RUNNING:
1637 case EP_STATE_HALTED:
1638 break;
1639 default:
1640 dev_err(pdev->dev, "ERROR: incorrect endpoint state\n");
1641 return -EINVAL;
1642 }
1643
1644 while (1) {
1645 if (cdnsp_room_on_ring(pdev, ep_ring, num_trbs))
1646 break;
1647
1648 trace_cdnsp_no_room_on_ring("try ring expansion");
1649
1650 num_trbs_needed = num_trbs - ep_ring->num_trbs_free;
1651 if (cdnsp_ring_expansion(pdev, ep_ring, num_trbs_needed,
1652 mem_flags)) {
1653 dev_err(pdev->dev, "Ring expansion failed\n");
1654 return -ENOMEM;
1655 }
1656 }
1657
1658 while (cdnsp_trb_is_link(ep_ring->enqueue)) {
1659 ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN);
1660 /* The cycle bit must be set as the last operation. */
1661 wmb();
1662 ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
1663
1664 /* Toggle the cycle bit after the last ring segment. */
1665 if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue))
1666 ep_ring->cycle_state ^= 1;
1667 ep_ring->enq_seg = ep_ring->enq_seg->next;
1668 ep_ring->enqueue = ep_ring->enq_seg->trbs;
1669 }
1670 return 0;
1671}
1672
1673static int cdnsp_prepare_transfer(struct cdnsp_device *pdev,
1674 struct cdnsp_request *preq,
1675 unsigned int num_trbs)
1676{
1677 struct cdnsp_ring *ep_ring;
1678 int ret;
1679
1680 ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep,
1681 preq->request.stream_id);
1682 if (!ep_ring)
1683 return -EINVAL;
1684
1685 ret = cdnsp_prepare_ring(pdev, ep_ring,
1686 GET_EP_CTX_STATE(preq->pep->out_ctx),
1687 num_trbs, GFP_ATOMIC);
1688 if (ret)
1689 return ret;
1690
1691 INIT_LIST_HEAD(&preq->td.td_list);
1692 preq->td.preq = preq;
1693
1694 /* Add this TD to the tail of the endpoint ring's TD list. */
1695 list_add_tail(&preq->td.td_list, &ep_ring->td_list);
1696 ep_ring->num_tds++;
1697 preq->pep->stream_info.td_count++;
1698
1699 preq->td.start_seg = ep_ring->enq_seg;
1700 preq->td.first_trb = ep_ring->enqueue;
1701
1702 return 0;
1703}
1704
1705static unsigned int cdnsp_count_trbs(u64 addr, u64 len)
1706{
1707 unsigned int num_trbs;
1708
1709 num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
1710 TRB_MAX_BUFF_SIZE);
1711 if (num_trbs == 0)
1712 num_trbs++;
1713
1714 return num_trbs;
1715}
1716
1717static unsigned int count_trbs_needed(struct cdnsp_request *preq)
1718{
1719 return cdnsp_count_trbs(preq->request.dma, preq->request.length);
1720}
1721
1722static unsigned int count_sg_trbs_needed(struct cdnsp_request *preq)
1723{
1724 unsigned int i, len, full_len, num_trbs = 0;
1725 struct scatterlist *sg;
1726
1727 full_len = preq->request.length;
1728
1729 for_each_sg(preq->request.sg, sg, preq->request.num_sgs, i) {
1730 len = sg_dma_len(sg);
1731 num_trbs += cdnsp_count_trbs(sg_dma_address(sg), len);
1732 len = min(len, full_len);
1733 full_len -= len;
1734 if (full_len == 0)
1735 break;
1736 }
1737
1738 return num_trbs;
1739}
1740
1741static void cdnsp_check_trb_math(struct cdnsp_request *preq, int running_total)
1742{
1743 if (running_total != preq->request.length)
1744 dev_err(preq->pep->pdev->dev,
1745 "%s - Miscalculated tx length, "
1746 "queued %#x, asked for %#x (%d)\n",
1747 preq->pep->name, running_total,
1748 preq->request.length, preq->request.actual);
1749}
1750
1751/*
1752 * TD size is the number of max packet sized packets remaining in the TD
1753 * (*not* including this TRB).
1754 *
1755 * Total TD packet count = total_packet_count =
1756 * DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
1757 *
1758 * Packets transferred up to and including this TRB = packets_transferred =
1759 * rounddown(total bytes transferred including this TRB / wMaxPacketSize)
1760 *
1761 * TD size = total_packet_count - packets_transferred
1762 *
1763 * It must fit in bits 21:17, so it can't be bigger than 31.
1764 * This is taken care of in the TRB_TD_SIZE() macro
1765 *
1766 * The last TRB in a TD must have the TD size set to zero.
1767 */
1768static u32 cdnsp_td_remainder(struct cdnsp_device *pdev,
1769 int transferred,
1770 int trb_buff_len,
1771 unsigned int td_total_len,
1772 struct cdnsp_request *preq,
1773 bool more_trbs_coming,
1774 bool zlp)
1775{
1776 u32 maxp, total_packet_count;
1777
1778 /* Before ZLP driver needs set TD_SIZE = 1. */
1779 if (zlp)
1780 return 1;
1781
1782 /* One TRB with a zero-length data packet. */
1783 if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
1784 trb_buff_len == td_total_len)
1785 return 0;
1786
1787 maxp = usb_endpoint_maxp(preq->pep->endpoint.desc);
1788 total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
1789
1790 /* Queuing functions don't count the current TRB into transferred. */
1791 return (total_packet_count - ((transferred + trb_buff_len) / maxp));
1792}
1793
1794static int cdnsp_align_td(struct cdnsp_device *pdev,
1795 struct cdnsp_request *preq, u32 enqd_len,
1796 u32 *trb_buff_len, struct cdnsp_segment *seg)
1797{
1798 struct device *dev = pdev->dev;
1799 unsigned int unalign;
1800 unsigned int max_pkt;
1801 u32 new_buff_len;
1802
1803 max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc);
1804 unalign = (enqd_len + *trb_buff_len) % max_pkt;
1805
1806 /* We got lucky, last normal TRB data on segment is packet aligned. */
1807 if (unalign == 0)
1808 return 0;
1809
1810 /* Is the last nornal TRB alignable by splitting it. */
1811 if (*trb_buff_len > unalign) {
1812 *trb_buff_len -= unalign;
1813 trace_cdnsp_bounce_align_td_split(preq, *trb_buff_len,
1814 enqd_len, 0, unalign);
1815 return 0;
1816 }
1817
1818 /*
1819 * We want enqd_len + trb_buff_len to sum up to a number aligned to
1820 * number which is divisible by the endpoint's wMaxPacketSize. IOW:
1821 * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0.
1822 */
1823 new_buff_len = max_pkt - (enqd_len % max_pkt);
1824
1825 if (new_buff_len > (preq->request.length - enqd_len))
1826 new_buff_len = (preq->request.length - enqd_len);
1827
1828 /* Create a max max_pkt sized bounce buffer pointed to by last trb. */
1829 if (preq->direction) {
1830 sg_pcopy_to_buffer(preq->request.sg,
1831 preq->request.num_mapped_sgs,
1832 seg->bounce_buf, new_buff_len, enqd_len);
1833 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
1834 max_pkt, DMA_TO_DEVICE);
1835 } else {
1836 seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
1837 max_pkt, DMA_FROM_DEVICE);
1838 }
1839
1840 if (dma_mapping_error(dev, seg->bounce_dma)) {
1841 /* Try without aligning.*/
1842 dev_warn(pdev->dev,
1843 "Failed mapping bounce buffer, not aligning\n");
1844 return 0;
1845 }
1846
1847 *trb_buff_len = new_buff_len;
1848 seg->bounce_len = new_buff_len;
1849 seg->bounce_offs = enqd_len;
1850
1851 trace_cdnsp_bounce_map(preq, new_buff_len, enqd_len, seg->bounce_dma,
1852 unalign);
1853
1854 /*
1855 * Bounce buffer successful aligned and seg->bounce_dma will be used
1856 * in transfer TRB as new transfer buffer address.
1857 */
1858 return 1;
1859}
1860
1861int cdnsp_queue_bulk_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq)
1862{
1863 unsigned int enqd_len, block_len, trb_buff_len, full_len;
1864 unsigned int start_cycle, num_sgs = 0;
1865 struct cdnsp_generic_trb *start_trb;
1866 u32 field, length_field, remainder;
1867 struct scatterlist *sg = NULL;
1868 bool more_trbs_coming = true;
1869 bool need_zero_pkt = false;
1870 bool zero_len_trb = false;
1871 struct cdnsp_ring *ring;
1872 bool first_trb = true;
1873 unsigned int num_trbs;
1874 struct cdnsp_ep *pep;
1875 u64 addr, send_addr;
1876 int sent_len, ret;
1877
1878 ring = cdnsp_request_to_transfer_ring(pdev, preq);
1879 if (!ring)
1880 return -EINVAL;
1881
1882 full_len = preq->request.length;
1883
1884 if (preq->request.num_sgs) {
1885 num_sgs = preq->request.num_sgs;
1886 sg = preq->request.sg;
1887 addr = (u64)sg_dma_address(sg);
1888 block_len = sg_dma_len(sg);
1889 num_trbs = count_sg_trbs_needed(preq);
1890 } else {
1891 num_trbs = count_trbs_needed(preq);
1892 addr = (u64)preq->request.dma;
1893 block_len = full_len;
1894 }
1895
1896 pep = preq->pep;
1897
1898 /* Deal with request.zero - need one more td/trb. */
1899 if (preq->request.zero && preq->request.length &&
1900 IS_ALIGNED(full_len, usb_endpoint_maxp(pep->endpoint.desc))) {
1901 need_zero_pkt = true;
1902 num_trbs++;
1903 }
1904
1905 ret = cdnsp_prepare_transfer(pdev, preq, num_trbs);
1906 if (ret)
1907 return ret;
1908
1909 /*
1910 * workaround 1: STOP EP command on LINK TRB with TC bit set to 1
1911 * causes that internal cycle bit can have incorrect state after
1912 * command complete. In consequence empty transfer ring can be
1913 * incorrectly detected when EP is resumed.
1914 * NOP TRB before LINK TRB avoid such scenario. STOP EP command is
1915 * then on NOP TRB and internal cycle bit is not changed and have
1916 * correct value.
1917 */
1918 if (pep->wa1_nop_trb) {
1919 field = le32_to_cpu(pep->wa1_nop_trb->trans_event.flags);
1920 field ^= TRB_CYCLE;
1921
1922 pep->wa1_nop_trb->trans_event.flags = cpu_to_le32(field);
1923 pep->wa1_nop_trb = NULL;
1924 }
1925
1926 /*
1927 * Don't give the first TRB to the hardware (by toggling the cycle bit)
1928 * until we've finished creating all the other TRBs. The ring's cycle
1929 * state may change as we enqueue the other TRBs, so save it too.
1930 */
1931 start_trb = &ring->enqueue->generic;
1932 start_cycle = ring->cycle_state;
1933 send_addr = addr;
1934
1935 /* Queue the TRBs, even if they are zero-length */
1936 for (enqd_len = 0; zero_len_trb || first_trb || enqd_len < full_len;
1937 enqd_len += trb_buff_len) {
1938 field = TRB_TYPE(TRB_NORMAL);
1939
1940 /* TRB buffer should not cross 64KB boundaries */
1941 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
1942 trb_buff_len = min(trb_buff_len, block_len);
1943 if (enqd_len + trb_buff_len > full_len)
1944 trb_buff_len = full_len - enqd_len;
1945
1946 /* Don't change the cycle bit of the first TRB until later */
1947 if (first_trb) {
1948 first_trb = false;
1949 if (start_cycle == 0)
1950 field |= TRB_CYCLE;
1951 } else {
1952 field |= ring->cycle_state;
1953 }
1954
1955 /*
1956 * Chain all the TRBs together; clear the chain bit in the last
1957 * TRB to indicate it's the last TRB in the chain.
1958 */
1959 if (enqd_len + trb_buff_len < full_len || need_zero_pkt) {
1960 field |= TRB_CHAIN;
1961 if (cdnsp_trb_is_link(ring->enqueue + 1)) {
1962 if (cdnsp_align_td(pdev, preq, enqd_len,
1963 &trb_buff_len,
1964 ring->enq_seg)) {
1965 send_addr = ring->enq_seg->bounce_dma;
1966 /* Assuming TD won't span 2 segs */
1967 preq->td.bounce_seg = ring->enq_seg;
1968 }
1969 }
1970 }
1971
1972 if (enqd_len + trb_buff_len >= full_len) {
1973 if (need_zero_pkt && !zero_len_trb) {
1974 zero_len_trb = true;
1975 } else {
1976 zero_len_trb = false;
1977 field &= ~TRB_CHAIN;
1978 field |= TRB_IOC;
1979 more_trbs_coming = false;
1980 need_zero_pkt = false;
1981 preq->td.last_trb = ring->enqueue;
1982 }
1983 }
1984
1985 /* Only set interrupt on short packet for OUT endpoints. */
1986 if (!preq->direction)
1987 field |= TRB_ISP;
1988
1989 /* Set the TRB length, TD size, and interrupter fields. */
1990 remainder = cdnsp_td_remainder(pdev, enqd_len, trb_buff_len,
1991 full_len, preq,
1992 more_trbs_coming,
1993 zero_len_trb);
1994
1995 length_field = TRB_LEN(trb_buff_len) | TRB_TD_SIZE(remainder) |
1996 TRB_INTR_TARGET(0);
1997
1998 cdnsp_queue_trb(pdev, ring, more_trbs_coming,
1999 lower_32_bits(send_addr),
2000 upper_32_bits(send_addr),
2001 length_field,
2002 field);
2003
2004 addr += trb_buff_len;
2005 sent_len = trb_buff_len;
2006 while (sg && sent_len >= block_len) {
2007 /* New sg entry */
2008 --num_sgs;
2009 sent_len -= block_len;
2010 if (num_sgs != 0) {
2011 sg = sg_next(sg);
2012 block_len = sg_dma_len(sg);
2013 addr = (u64)sg_dma_address(sg);
2014 addr += sent_len;
2015 }
2016 }
2017 block_len -= sent_len;
2018 send_addr = addr;
2019 }
2020
2021 if (cdnsp_trb_is_link(ring->enqueue + 1)) {
2022 field = TRB_TYPE(TRB_TR_NOOP) | TRB_IOC;
2023 if (!ring->cycle_state)
2024 field |= TRB_CYCLE;
2025
2026 pep->wa1_nop_trb = ring->enqueue;
2027
2028 cdnsp_queue_trb(pdev, ring, 0, 0x0, 0x0,
2029 TRB_INTR_TARGET(0), field);
2030 }
2031
2032 cdnsp_check_trb_math(preq, enqd_len);
2033 ret = cdnsp_giveback_first_trb(pdev, pep, preq->request.stream_id,
2034 start_cycle, start_trb);
2035
2036 if (ret)
2037 preq->td.drbl = 1;
2038
2039 return 0;
2040}
2041
2042int cdnsp_queue_ctrl_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq)
2043{
2044 u32 field, length_field, zlp = 0;
2045 struct cdnsp_ep *pep = preq->pep;
2046 struct cdnsp_ring *ep_ring;
2047 int num_trbs;
2048 u32 maxp;
2049 int ret;
2050
2051 ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);
2052 if (!ep_ring)
2053 return -EINVAL;
2054
2055 /* 1 TRB for data, 1 for status */
2056 num_trbs = (pdev->three_stage_setup) ? 2 : 1;
2057
2058 maxp = usb_endpoint_maxp(pep->endpoint.desc);
2059
2060 if (preq->request.zero && preq->request.length &&
2061 (preq->request.length % maxp == 0)) {
2062 num_trbs++;
2063 zlp = 1;
2064 }
2065
2066 ret = cdnsp_prepare_transfer(pdev, preq, num_trbs);
2067 if (ret)
2068 return ret;
2069
2070 /* If there's data, queue data TRBs */
2071 if (preq->request.length > 0) {
2072 field = TRB_TYPE(TRB_DATA);
2073
2074 if (zlp)
2075 field |= TRB_CHAIN;
2076 else
2077 field |= TRB_IOC | (pdev->ep0_expect_in ? 0 : TRB_ISP);
2078
2079 if (pdev->ep0_expect_in)
2080 field |= TRB_DIR_IN;
2081
2082 length_field = TRB_LEN(preq->request.length) |
2083 TRB_TD_SIZE(zlp) | TRB_INTR_TARGET(0);
2084
2085 cdnsp_queue_trb(pdev, ep_ring, true,
2086 lower_32_bits(preq->request.dma),
2087 upper_32_bits(preq->request.dma), length_field,
2088 field | ep_ring->cycle_state |
2089 TRB_SETUPID(pdev->setup_id) |
2090 pdev->setup_speed);
2091
2092 if (zlp) {
2093 field = TRB_TYPE(TRB_NORMAL) | TRB_IOC;
2094
2095 if (!pdev->ep0_expect_in)
2096 field = TRB_ISP;
2097
2098 cdnsp_queue_trb(pdev, ep_ring, true,
2099 lower_32_bits(preq->request.dma),
2100 upper_32_bits(preq->request.dma), 0,
2101 field | ep_ring->cycle_state |
2102 TRB_SETUPID(pdev->setup_id) |
2103 pdev->setup_speed);
2104 }
2105
2106 pdev->ep0_stage = CDNSP_DATA_STAGE;
2107 }
2108
2109 /* Save the DMA address of the last TRB in the TD. */
2110 preq->td.last_trb = ep_ring->enqueue;
2111
2112 /* Queue status TRB. */
2113 if (preq->request.length == 0)
2114 field = ep_ring->cycle_state;
2115 else
2116 field = (ep_ring->cycle_state ^ 1);
2117
2118 if (preq->request.length > 0 && pdev->ep0_expect_in)
2119 field |= TRB_DIR_IN;
2120
2121 if (pep->ep_state & EP0_HALTED_STATUS) {
2122 pep->ep_state &= ~EP0_HALTED_STATUS;
2123 field |= TRB_SETUPSTAT(TRB_SETUPSTAT_STALL);
2124 } else {
2125 field |= TRB_SETUPSTAT(TRB_SETUPSTAT_ACK);
2126 }
2127
2128 cdnsp_queue_trb(pdev, ep_ring, false, 0, 0, TRB_INTR_TARGET(0),
2129 field | TRB_IOC | TRB_SETUPID(pdev->setup_id) |
2130 TRB_TYPE(TRB_STATUS) | pdev->setup_speed);
2131
2132 cdnsp_ring_ep_doorbell(pdev, pep, preq->request.stream_id);
2133
2134 return 0;
2135}
2136
2137int cdnsp_cmd_stop_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
2138{
2139 u32 ep_state = GET_EP_CTX_STATE(pep->out_ctx);
2140 int ret = 0;
2141
2142 if (ep_state == EP_STATE_STOPPED || ep_state == EP_STATE_DISABLED ||
2143 ep_state == EP_STATE_HALTED) {
2144 trace_cdnsp_ep_stopped_or_disabled(pep->out_ctx);
2145 goto ep_stopped;
2146 }
2147
2148 cdnsp_queue_stop_endpoint(pdev, pep->idx);
2149 cdnsp_ring_cmd_db(pdev);
2150 ret = cdnsp_wait_for_cmd_compl(pdev);
2151
2152 trace_cdnsp_handle_cmd_stop_ep(pep->out_ctx);
2153
2154ep_stopped:
2155 pep->ep_state |= EP_STOPPED;
2156 return ret;
2157}
2158
2159/*
2160 * The transfer burst count field of the isochronous TRB defines the number of
2161 * bursts that are required to move all packets in this TD. Only SuperSpeed
2162 * devices can burst up to bMaxBurst number of packets per service interval.
2163 * This field is zero based, meaning a value of zero in the field means one
2164 * burst. Basically, for everything but SuperSpeed devices, this field will be
2165 * zero.
2166 */
2167static unsigned int cdnsp_get_burst_count(struct cdnsp_device *pdev,
2168 struct cdnsp_request *preq,
2169 unsigned int total_packet_count)
2170{
2171 unsigned int max_burst;
2172
2173 if (pdev->gadget.speed < USB_SPEED_SUPER)
2174 return 0;
2175
2176 max_burst = preq->pep->endpoint.comp_desc->bMaxBurst;
2177 return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1;
2178}
2179
2180/*
2181 * Returns the number of packets in the last "burst" of packets. This field is
2182 * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so
2183 * the last burst packet count is equal to the total number of packets in the
2184 * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst
2185 * must contain (bMaxBurst + 1) number of packets, but the last burst can
2186 * contain 1 to (bMaxBurst + 1) packets.
2187 */
2188static unsigned int
2189 cdnsp_get_last_burst_packet_count(struct cdnsp_device *pdev,
2190 struct cdnsp_request *preq,
2191 unsigned int total_packet_count)
2192{
2193 unsigned int max_burst;
2194 unsigned int residue;
2195
2196 if (pdev->gadget.speed >= USB_SPEED_SUPER) {
2197 /* bMaxBurst is zero based: 0 means 1 packet per burst. */
2198 max_burst = preq->pep->endpoint.comp_desc->bMaxBurst;
2199 residue = total_packet_count % (max_burst + 1);
2200
2201 /*
2202 * If residue is zero, the last burst contains (max_burst + 1)
2203 * number of packets, but the TLBPC field is zero-based.
2204 */
2205 if (residue == 0)
2206 return max_burst;
2207
2208 return residue - 1;
2209 }
2210 if (total_packet_count == 0)
2211 return 0;
2212
2213 return total_packet_count - 1;
2214}
2215
2216/* Queue function isoc transfer */
2217int cdnsp_queue_isoc_tx(struct cdnsp_device *pdev,
2218 struct cdnsp_request *preq)
2219{
2220 unsigned int trb_buff_len, td_len, td_remain_len, block_len;
2221 unsigned int burst_count, last_burst_pkt;
2222 unsigned int total_pkt_count, max_pkt;
2223 struct cdnsp_generic_trb *start_trb;
2224 struct scatterlist *sg = NULL;
2225 bool more_trbs_coming = true;
2226 struct cdnsp_ring *ep_ring;
2227 unsigned int num_sgs = 0;
2228 int running_total = 0;
2229 u32 field, length_field;
2230 u64 addr, send_addr;
2231 int start_cycle;
2232 int trbs_per_td;
2233 int i, sent_len, ret;
2234
2235 ep_ring = preq->pep->ring;
2236
2237 td_len = preq->request.length;
2238
2239 if (preq->request.num_sgs) {
2240 num_sgs = preq->request.num_sgs;
2241 sg = preq->request.sg;
2242 addr = (u64)sg_dma_address(sg);
2243 block_len = sg_dma_len(sg);
2244 trbs_per_td = count_sg_trbs_needed(preq);
2245 } else {
2246 addr = (u64)preq->request.dma;
2247 block_len = td_len;
2248 trbs_per_td = count_trbs_needed(preq);
2249 }
2250
2251 ret = cdnsp_prepare_transfer(pdev, preq, trbs_per_td);
2252 if (ret)
2253 return ret;
2254
2255 start_trb = &ep_ring->enqueue->generic;
2256 start_cycle = ep_ring->cycle_state;
2257 td_remain_len = td_len;
2258 send_addr = addr;
2259
2260 max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc);
2261 total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
2262
2263 /* A zero-length transfer still involves at least one packet. */
2264 if (total_pkt_count == 0)
2265 total_pkt_count++;
2266
2267 burst_count = cdnsp_get_burst_count(pdev, preq, total_pkt_count);
2268 last_burst_pkt = cdnsp_get_last_burst_packet_count(pdev, preq,
2269 total_pkt_count);
2270
2271 /*
2272 * Set isoc specific data for the first TRB in a TD.
2273 * Prevent HW from getting the TRBs by keeping the cycle state
2274 * inverted in the first TDs isoc TRB.
2275 */
2276 field = TRB_TYPE(TRB_ISOC) | TRB_TLBPC(last_burst_pkt) |
2277 TRB_SIA | TRB_TBC(burst_count);
2278
2279 if (!start_cycle)
2280 field |= TRB_CYCLE;
2281
2282 /* Fill the rest of the TRB fields, and remaining normal TRBs. */
2283 for (i = 0; i < trbs_per_td; i++) {
2284 u32 remainder;
2285
2286 /* Calculate TRB length. */
2287 trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
2288 trb_buff_len = min(trb_buff_len, block_len);
2289 if (trb_buff_len > td_remain_len)
2290 trb_buff_len = td_remain_len;
2291
2292 /* Set the TRB length, TD size, & interrupter fields. */
2293 remainder = cdnsp_td_remainder(pdev, running_total,
2294 trb_buff_len, td_len, preq,
2295 more_trbs_coming, 0);
2296
2297 length_field = TRB_LEN(trb_buff_len) | TRB_TD_SIZE(remainder) |
2298 TRB_INTR_TARGET(0);
2299
2300 /* Only first TRB is isoc, overwrite otherwise. */
2301 if (i) {
2302 field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state;
2303 length_field |= TRB_TD_SIZE(remainder);
2304 } else {
2305 length_field |= TRB_TD_SIZE_TBC(burst_count);
2306 }
2307
2308 /* Only set interrupt on short packet for OUT EPs. */
2309 if (usb_endpoint_dir_out(preq->pep->endpoint.desc))
2310 field |= TRB_ISP;
2311
2312 /* Set the chain bit for all except the last TRB. */
2313 if (i < trbs_per_td - 1) {
2314 more_trbs_coming = true;
2315 field |= TRB_CHAIN;
2316 } else {
2317 more_trbs_coming = false;
2318 preq->td.last_trb = ep_ring->enqueue;
2319 field |= TRB_IOC;
2320 }
2321
2322 cdnsp_queue_trb(pdev, ep_ring, more_trbs_coming,
2323 lower_32_bits(send_addr), upper_32_bits(send_addr),
2324 length_field, field);
2325
2326 running_total += trb_buff_len;
2327 addr += trb_buff_len;
2328 td_remain_len -= trb_buff_len;
2329
2330 sent_len = trb_buff_len;
2331 while (sg && sent_len >= block_len) {
2332 /* New sg entry */
2333 --num_sgs;
2334 sent_len -= block_len;
2335 if (num_sgs != 0) {
2336 sg = sg_next(sg);
2337 block_len = sg_dma_len(sg);
2338 addr = (u64)sg_dma_address(sg);
2339 addr += sent_len;
2340 }
2341 }
2342 block_len -= sent_len;
2343 send_addr = addr;
2344 }
2345
2346 /* Check TD length */
2347 if (running_total != td_len) {
2348 dev_err(pdev->dev, "ISOC TD length unmatch\n");
2349 ret = -EINVAL;
2350 goto cleanup;
2351 }
2352
2353 cdnsp_giveback_first_trb(pdev, preq->pep, preq->request.stream_id,
2354 start_cycle, start_trb);
2355
2356 return 0;
2357
2358cleanup:
2359 /* Clean up a partially enqueued isoc transfer. */
2360 list_del_init(&preq->td.td_list);
2361 ep_ring->num_tds--;
2362
2363 /*
2364 * Use the first TD as a temporary variable to turn the TDs we've
2365 * queued into No-ops with a software-owned cycle bit.
2366 * That way the hardware won't accidentally start executing bogus TDs
2367 * when we partially overwrite them.
2368 * td->first_trb and td->start_seg are already set.
2369 */
2370 preq->td.last_trb = ep_ring->enqueue;
2371 /* Every TRB except the first & last will have its cycle bit flipped. */
2372 cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true);
2373
2374 /* Reset the ring enqueue back to the first TRB and its cycle bit. */
2375 ep_ring->enqueue = preq->td.first_trb;
2376 ep_ring->enq_seg = preq->td.start_seg;
2377 ep_ring->cycle_state = start_cycle;
2378 return ret;
2379}
2380
2381/**** Command Ring Operations ****/
2382/*
2383 * Generic function for queuing a command TRB on the command ring.
2384 * Driver queue only one command to ring in the moment.
2385 */
2386static void cdnsp_queue_command(struct cdnsp_device *pdev,
2387 u32 field1,
2388 u32 field2,
2389 u32 field3,
2390 u32 field4)
2391{
2392 cdnsp_prepare_ring(pdev, pdev->cmd_ring, EP_STATE_RUNNING, 1,
2393 GFP_ATOMIC);
2394
2395 pdev->cmd.command_trb = pdev->cmd_ring->enqueue;
2396
2397 cdnsp_queue_trb(pdev, pdev->cmd_ring, false, field1, field2,
2398 field3, field4 | pdev->cmd_ring->cycle_state);
2399}
2400
2401/* Queue a slot enable or disable request on the command ring */
2402void cdnsp_queue_slot_control(struct cdnsp_device *pdev, u32 trb_type)
2403{
2404 cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(trb_type) |
2405 SLOT_ID_FOR_TRB(pdev->slot_id));
2406}
2407
2408/* Queue an address device command TRB */
2409void cdnsp_queue_address_device(struct cdnsp_device *pdev,
2410 dma_addr_t in_ctx_ptr,
2411 enum cdnsp_setup_dev setup)
2412{
2413 cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr),
2414 upper_32_bits(in_ctx_ptr), 0,
2415 TRB_TYPE(TRB_ADDR_DEV) |
2416 SLOT_ID_FOR_TRB(pdev->slot_id) |
2417 (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0));
2418}
2419
2420/* Queue a reset device command TRB */
2421void cdnsp_queue_reset_device(struct cdnsp_device *pdev)
2422{
2423 cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_RESET_DEV) |
2424 SLOT_ID_FOR_TRB(pdev->slot_id));
2425}
2426
2427/* Queue a configure endpoint command TRB */
2428void cdnsp_queue_configure_endpoint(struct cdnsp_device *pdev,
2429 dma_addr_t in_ctx_ptr)
2430{
2431 cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr),
2432 upper_32_bits(in_ctx_ptr), 0,
2433 TRB_TYPE(TRB_CONFIG_EP) |
2434 SLOT_ID_FOR_TRB(pdev->slot_id));
2435}
2436
2437/*
2438 * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop
2439 * activity on an endpoint that is about to be suspended.
2440 */
2441void cdnsp_queue_stop_endpoint(struct cdnsp_device *pdev, unsigned int ep_index)
2442{
2443 cdnsp_queue_command(pdev, 0, 0, 0, SLOT_ID_FOR_TRB(pdev->slot_id) |
2444 EP_ID_FOR_TRB(ep_index) | TRB_TYPE(TRB_STOP_RING));
2445}
2446
2447/* Set Transfer Ring Dequeue Pointer command. */
2448void cdnsp_queue_new_dequeue_state(struct cdnsp_device *pdev,
2449 struct cdnsp_ep *pep,
2450 struct cdnsp_dequeue_state *deq_state)
2451{
2452 u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id);
2453 u32 trb_slot_id = SLOT_ID_FOR_TRB(pdev->slot_id);
2454 u32 type = TRB_TYPE(TRB_SET_DEQ);
2455 u32 trb_sct = 0;
2456 dma_addr_t addr;
2457
2458 addr = cdnsp_trb_virt_to_dma(deq_state->new_deq_seg,
2459 deq_state->new_deq_ptr);
2460
2461 if (deq_state->stream_id)
2462 trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
2463
2464 cdnsp_queue_command(pdev, lower_32_bits(addr) | trb_sct |
2465 deq_state->new_cycle_state, upper_32_bits(addr),
2466 trb_stream_id, trb_slot_id |
2467 EP_ID_FOR_TRB(pep->idx) | type);
2468}
2469
2470void cdnsp_queue_reset_ep(struct cdnsp_device *pdev, unsigned int ep_index)
2471{
2472 return cdnsp_queue_command(pdev, 0, 0, 0,
2473 SLOT_ID_FOR_TRB(pdev->slot_id) |
2474 EP_ID_FOR_TRB(ep_index) |
2475 TRB_TYPE(TRB_RESET_EP));
2476}
2477
2478/*
2479 * Queue a halt endpoint request on the command ring.
2480 */
2481void cdnsp_queue_halt_endpoint(struct cdnsp_device *pdev, unsigned int ep_index)
2482{
2483 cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_HALT_ENDPOINT) |
2484 SLOT_ID_FOR_TRB(pdev->slot_id) |
2485 EP_ID_FOR_TRB(ep_index));
2486}
2487
2488void cdnsp_force_header_wakeup(struct cdnsp_device *pdev, int intf_num)
2489{
2490 u32 lo, mid;
2491
2492 lo = TRB_FH_TO_PACKET_TYPE(TRB_FH_TR_PACKET) |
2493 TRB_FH_TO_DEVICE_ADDRESS(pdev->device_address);
2494 mid = TRB_FH_TR_PACKET_DEV_NOT |
2495 TRB_FH_TO_NOT_TYPE(TRB_FH_TR_PACKET_FUNCTION_WAKE) |
2496 TRB_FH_TO_INTERFACE(intf_num);
2497
2498 cdnsp_queue_command(pdev, lo, mid, 0,
2499 TRB_TYPE(TRB_FORCE_HEADER) | SET_PORT_ID(2));
2500}