Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2/*
3 * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
4 *
5 * Copyright (C) 2004-2013 Synopsys, Inc.
6 */
7
8/*
9 * This file contains the interrupt handlers for Host mode
10 */
11#include <linux/kernel.h>
12#include <linux/module.h>
13#include <linux/spinlock.h>
14#include <linux/interrupt.h>
15#include <linux/dma-mapping.h>
16#include <linux/io.h>
17#include <linux/slab.h>
18#include <linux/usb.h>
19
20#include <linux/usb/hcd.h>
21#include <linux/usb/ch11.h>
22
23#include "core.h"
24#include "hcd.h"
25
26/*
27 * If we get this many NAKs on a split transaction we'll slow down
28 * retransmission. A 1 here means delay after the first NAK.
29 */
30#define DWC2_NAKS_BEFORE_DELAY 3
31
32/* This function is for debug only */
33static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
34{
35 u16 curr_frame_number = hsotg->frame_number;
36 u16 expected = dwc2_frame_num_inc(hsotg->last_frame_num, 1);
37
38 if (expected != curr_frame_number)
39 dwc2_sch_vdbg(hsotg, "MISSED SOF %04x != %04x\n",
40 expected, curr_frame_number);
41
42#ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
43 if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
44 if (expected != curr_frame_number) {
45 hsotg->frame_num_array[hsotg->frame_num_idx] =
46 curr_frame_number;
47 hsotg->last_frame_num_array[hsotg->frame_num_idx] =
48 hsotg->last_frame_num;
49 hsotg->frame_num_idx++;
50 }
51 } else if (!hsotg->dumped_frame_num_array) {
52 int i;
53
54 dev_info(hsotg->dev, "Frame Last Frame\n");
55 dev_info(hsotg->dev, "----- ----------\n");
56 for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
57 dev_info(hsotg->dev, "0x%04x 0x%04x\n",
58 hsotg->frame_num_array[i],
59 hsotg->last_frame_num_array[i]);
60 }
61 hsotg->dumped_frame_num_array = 1;
62 }
63#endif
64 hsotg->last_frame_num = curr_frame_number;
65}
66
67static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
68 struct dwc2_host_chan *chan,
69 struct dwc2_qtd *qtd)
70{
71 struct usb_device *root_hub = dwc2_hsotg_to_hcd(hsotg)->self.root_hub;
72 struct urb *usb_urb;
73
74 if (!chan->qh)
75 return;
76
77 if (chan->qh->dev_speed == USB_SPEED_HIGH)
78 return;
79
80 if (!qtd->urb)
81 return;
82
83 usb_urb = qtd->urb->priv;
84 if (!usb_urb || !usb_urb->dev || !usb_urb->dev->tt)
85 return;
86
87 /*
88 * The root hub doesn't really have a TT, but Linux thinks it
89 * does because how could you have a "high speed hub" that
90 * directly talks directly to low speed devices without a TT?
91 * It's all lies. Lies, I tell you.
92 */
93 if (usb_urb->dev->tt->hub == root_hub)
94 return;
95
96 if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
97 chan->qh->tt_buffer_dirty = 1;
98 if (usb_hub_clear_tt_buffer(usb_urb))
99 /* Clear failed; let's hope things work anyway */
100 chan->qh->tt_buffer_dirty = 0;
101 }
102}
103
104/*
105 * Handles the start-of-frame interrupt in host mode. Non-periodic
106 * transactions may be queued to the DWC_otg controller for the current
107 * (micro)frame. Periodic transactions may be queued to the controller
108 * for the next (micro)frame.
109 */
110static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
111{
112 struct list_head *qh_entry;
113 struct dwc2_qh *qh;
114 enum dwc2_transaction_type tr_type;
115
116 /* Clear interrupt */
117 dwc2_writel(hsotg, GINTSTS_SOF, GINTSTS);
118
119#ifdef DEBUG_SOF
120 dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
121#endif
122
123 hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
124
125 dwc2_track_missed_sofs(hsotg);
126
127 /* Determine whether any periodic QHs should be executed */
128 qh_entry = hsotg->periodic_sched_inactive.next;
129 while (qh_entry != &hsotg->periodic_sched_inactive) {
130 qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
131 qh_entry = qh_entry->next;
132 if (dwc2_frame_num_le(qh->next_active_frame,
133 hsotg->frame_number)) {
134 dwc2_sch_vdbg(hsotg, "QH=%p ready fn=%04x, nxt=%04x\n",
135 qh, hsotg->frame_number,
136 qh->next_active_frame);
137
138 /*
139 * Move QH to the ready list to be executed next
140 * (micro)frame
141 */
142 list_move_tail(&qh->qh_list_entry,
143 &hsotg->periodic_sched_ready);
144 }
145 }
146 tr_type = dwc2_hcd_select_transactions(hsotg);
147 if (tr_type != DWC2_TRANSACTION_NONE)
148 dwc2_hcd_queue_transactions(hsotg, tr_type);
149}
150
151/*
152 * Handles the Rx FIFO Level Interrupt, which indicates that there is
153 * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
154 * memory if the DWC_otg controller is operating in Slave mode.
155 */
156static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
157{
158 u32 grxsts, chnum, bcnt, dpid, pktsts;
159 struct dwc2_host_chan *chan;
160
161 if (dbg_perio())
162 dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
163
164 grxsts = dwc2_readl(hsotg, GRXSTSP);
165 chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
166 chan = hsotg->hc_ptr_array[chnum];
167 if (!chan) {
168 dev_err(hsotg->dev, "Unable to get corresponding channel\n");
169 return;
170 }
171
172 bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
173 dpid = (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT;
174 pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
175
176 /* Packet Status */
177 if (dbg_perio()) {
178 dev_vdbg(hsotg->dev, " Ch num = %d\n", chnum);
179 dev_vdbg(hsotg->dev, " Count = %d\n", bcnt);
180 dev_vdbg(hsotg->dev, " DPID = %d, chan.dpid = %d\n", dpid,
181 chan->data_pid_start);
182 dev_vdbg(hsotg->dev, " PStatus = %d\n", pktsts);
183 }
184
185 switch (pktsts) {
186 case GRXSTS_PKTSTS_HCHIN:
187 /* Read the data into the host buffer */
188 if (bcnt > 0) {
189 dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);
190
191 /* Update the HC fields for the next packet received */
192 chan->xfer_count += bcnt;
193 chan->xfer_buf += bcnt;
194 }
195 break;
196 case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
197 case GRXSTS_PKTSTS_DATATOGGLEERR:
198 case GRXSTS_PKTSTS_HCHHALTED:
199 /* Handled in interrupt, just ignore data */
200 break;
201 default:
202 dev_err(hsotg->dev,
203 "RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
204 break;
205 }
206}
207
208/*
209 * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
210 * data packets may be written to the FIFO for OUT transfers. More requests
211 * may be written to the non-periodic request queue for IN transfers. This
212 * interrupt is enabled only in Slave mode.
213 */
214static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
215{
216 dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
217 dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
218}
219
220/*
221 * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
222 * packets may be written to the FIFO for OUT transfers. More requests may be
223 * written to the periodic request queue for IN transfers. This interrupt is
224 * enabled only in Slave mode.
225 */
226static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
227{
228 if (dbg_perio())
229 dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
230 dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
231}
232
233static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
234 u32 *hprt0_modify)
235{
236 struct dwc2_core_params *params = &hsotg->params;
237 int do_reset = 0;
238 u32 usbcfg;
239 u32 prtspd;
240 u32 hcfg;
241 u32 fslspclksel;
242 u32 hfir;
243
244 dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
245
246 /* Every time when port enables calculate HFIR.FrInterval */
247 hfir = dwc2_readl(hsotg, HFIR);
248 hfir &= ~HFIR_FRINT_MASK;
249 hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
250 HFIR_FRINT_MASK;
251 dwc2_writel(hsotg, hfir, HFIR);
252
253 /* Check if we need to adjust the PHY clock speed for low power */
254 if (!params->host_support_fs_ls_low_power) {
255 /* Port has been enabled, set the reset change flag */
256 hsotg->flags.b.port_reset_change = 1;
257 return;
258 }
259
260 usbcfg = dwc2_readl(hsotg, GUSBCFG);
261 prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
262
263 if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
264 /* Low power */
265 if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
266 /* Set PHY low power clock select for FS/LS devices */
267 usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
268 dwc2_writel(hsotg, usbcfg, GUSBCFG);
269 do_reset = 1;
270 }
271
272 hcfg = dwc2_readl(hsotg, HCFG);
273 fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
274 HCFG_FSLSPCLKSEL_SHIFT;
275
276 if (prtspd == HPRT0_SPD_LOW_SPEED &&
277 params->host_ls_low_power_phy_clk) {
278 /* 6 MHZ */
279 dev_vdbg(hsotg->dev,
280 "FS_PHY programming HCFG to 6 MHz\n");
281 if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
282 fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
283 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
284 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
285 dwc2_writel(hsotg, hcfg, HCFG);
286 do_reset = 1;
287 }
288 } else {
289 /* 48 MHZ */
290 dev_vdbg(hsotg->dev,
291 "FS_PHY programming HCFG to 48 MHz\n");
292 if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
293 fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
294 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
295 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
296 dwc2_writel(hsotg, hcfg, HCFG);
297 do_reset = 1;
298 }
299 }
300 } else {
301 /* Not low power */
302 if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
303 usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
304 dwc2_writel(hsotg, usbcfg, GUSBCFG);
305 do_reset = 1;
306 }
307 }
308
309 if (do_reset) {
310 *hprt0_modify |= HPRT0_RST;
311 dwc2_writel(hsotg, *hprt0_modify, HPRT0);
312 queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
313 msecs_to_jiffies(60));
314 } else {
315 /* Port has been enabled, set the reset change flag */
316 hsotg->flags.b.port_reset_change = 1;
317 }
318}
319
320/*
321 * There are multiple conditions that can cause a port interrupt. This function
322 * determines which interrupt conditions have occurred and handles them
323 * appropriately.
324 */
325static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
326{
327 u32 hprt0;
328 u32 hprt0_modify;
329
330 dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
331
332 hprt0 = dwc2_readl(hsotg, HPRT0);
333 hprt0_modify = hprt0;
334
335 /*
336 * Clear appropriate bits in HPRT0 to clear the interrupt bit in
337 * GINTSTS
338 */
339 hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
340 HPRT0_OVRCURRCHG);
341
342 /*
343 * Port Connect Detected
344 * Set flag and clear if detected
345 */
346 if (hprt0 & HPRT0_CONNDET) {
347 dwc2_writel(hsotg, hprt0_modify | HPRT0_CONNDET, HPRT0);
348
349 dev_vdbg(hsotg->dev,
350 "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
351 hprt0);
352 dwc2_hcd_connect(hsotg);
353
354 /*
355 * The Hub driver asserts a reset when it sees port connect
356 * status change flag
357 */
358 }
359
360 /*
361 * Port Enable Changed
362 * Clear if detected - Set internal flag if disabled
363 */
364 if (hprt0 & HPRT0_ENACHG) {
365 dwc2_writel(hsotg, hprt0_modify | HPRT0_ENACHG, HPRT0);
366 dev_vdbg(hsotg->dev,
367 " --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
368 hprt0, !!(hprt0 & HPRT0_ENA));
369 if (hprt0 & HPRT0_ENA) {
370 hsotg->new_connection = true;
371 dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
372 } else {
373 hsotg->flags.b.port_enable_change = 1;
374 if (hsotg->params.dma_desc_fs_enable) {
375 u32 hcfg;
376
377 hsotg->params.dma_desc_enable = false;
378 hsotg->new_connection = false;
379 hcfg = dwc2_readl(hsotg, HCFG);
380 hcfg &= ~HCFG_DESCDMA;
381 dwc2_writel(hsotg, hcfg, HCFG);
382 }
383 }
384 }
385
386 /* Overcurrent Change Interrupt */
387 if (hprt0 & HPRT0_OVRCURRCHG) {
388 dwc2_writel(hsotg, hprt0_modify | HPRT0_OVRCURRCHG,
389 HPRT0);
390 dev_vdbg(hsotg->dev,
391 " --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
392 hprt0);
393 hsotg->flags.b.port_over_current_change = 1;
394 }
395}
396
397/*
398 * Gets the actual length of a transfer after the transfer halts. halt_status
399 * holds the reason for the halt.
400 *
401 * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
402 * is set to 1 upon return if less than the requested number of bytes were
403 * transferred. short_read may also be NULL on entry, in which case it remains
404 * unchanged.
405 */
406static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
407 struct dwc2_host_chan *chan, int chnum,
408 struct dwc2_qtd *qtd,
409 enum dwc2_halt_status halt_status,
410 int *short_read)
411{
412 u32 hctsiz, count, length;
413
414 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
415
416 if (halt_status == DWC2_HC_XFER_COMPLETE) {
417 if (chan->ep_is_in) {
418 count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
419 TSIZ_XFERSIZE_SHIFT;
420 length = chan->xfer_len - count;
421 if (short_read)
422 *short_read = (count != 0);
423 } else if (chan->qh->do_split) {
424 length = qtd->ssplit_out_xfer_count;
425 } else {
426 length = chan->xfer_len;
427 }
428 } else {
429 /*
430 * Must use the hctsiz.pktcnt field to determine how much data
431 * has been transferred. This field reflects the number of
432 * packets that have been transferred via the USB. This is
433 * always an integral number of packets if the transfer was
434 * halted before its normal completion. (Can't use the
435 * hctsiz.xfersize field because that reflects the number of
436 * bytes transferred via the AHB, not the USB).
437 */
438 count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
439 length = (chan->start_pkt_count - count) * chan->max_packet;
440 }
441
442 return length;
443}
444
445/**
446 * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
447 * Complete interrupt on the host channel. Updates the actual_length field
448 * of the URB based on the number of bytes transferred via the host channel.
449 * Sets the URB status if the data transfer is finished.
450 *
451 * @hsotg: Programming view of the DWC_otg controller
452 * @chan: Programming view of host channel
453 * @chnum: Channel number
454 * @urb: Processing URB
455 * @qtd: Queue transfer descriptor
456 *
457 * Return: 1 if the data transfer specified by the URB is completely finished,
458 * 0 otherwise
459 */
460static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
461 struct dwc2_host_chan *chan, int chnum,
462 struct dwc2_hcd_urb *urb,
463 struct dwc2_qtd *qtd)
464{
465 u32 hctsiz;
466 int xfer_done = 0;
467 int short_read = 0;
468 int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
469 DWC2_HC_XFER_COMPLETE,
470 &short_read);
471
472 if (urb->actual_length + xfer_length > urb->length) {
473 dev_dbg(hsotg->dev, "%s(): trimming xfer length\n", __func__);
474 xfer_length = urb->length - urb->actual_length;
475 }
476
477 dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
478 urb->actual_length, xfer_length);
479 urb->actual_length += xfer_length;
480
481 if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
482 (urb->flags & URB_SEND_ZERO_PACKET) &&
483 urb->actual_length >= urb->length &&
484 !(urb->length % chan->max_packet)) {
485 xfer_done = 0;
486 } else if (short_read || urb->actual_length >= urb->length) {
487 xfer_done = 1;
488 urb->status = 0;
489 }
490
491 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
492 dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
493 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
494 dev_vdbg(hsotg->dev, " chan->xfer_len %d\n", chan->xfer_len);
495 dev_vdbg(hsotg->dev, " hctsiz.xfersize %d\n",
496 (hctsiz & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
497 dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n", urb->length);
498 dev_vdbg(hsotg->dev, " urb->actual_length %d\n", urb->actual_length);
499 dev_vdbg(hsotg->dev, " short_read %d, xfer_done %d\n", short_read,
500 xfer_done);
501
502 return xfer_done;
503}
504
505/*
506 * Save the starting data toggle for the next transfer. The data toggle is
507 * saved in the QH for non-control transfers and it's saved in the QTD for
508 * control transfers.
509 */
510void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
511 struct dwc2_host_chan *chan, int chnum,
512 struct dwc2_qtd *qtd)
513{
514 u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
515 u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
516
517 if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
518 if (WARN(!chan || !chan->qh,
519 "chan->qh must be specified for non-control eps\n"))
520 return;
521
522 if (pid == TSIZ_SC_MC_PID_DATA0)
523 chan->qh->data_toggle = DWC2_HC_PID_DATA0;
524 else
525 chan->qh->data_toggle = DWC2_HC_PID_DATA1;
526 } else {
527 if (WARN(!qtd,
528 "qtd must be specified for control eps\n"))
529 return;
530
531 if (pid == TSIZ_SC_MC_PID_DATA0)
532 qtd->data_toggle = DWC2_HC_PID_DATA0;
533 else
534 qtd->data_toggle = DWC2_HC_PID_DATA1;
535 }
536}
537
538/**
539 * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
540 * the transfer is stopped for any reason. The fields of the current entry in
541 * the frame descriptor array are set based on the transfer state and the input
542 * halt_status. Completes the Isochronous URB if all the URB frames have been
543 * completed.
544 *
545 * @hsotg: Programming view of the DWC_otg controller
546 * @chan: Programming view of host channel
547 * @chnum: Channel number
548 * @halt_status: Reason for halting a host channel
549 * @qtd: Queue transfer descriptor
550 *
551 * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
552 * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
553 */
554static enum dwc2_halt_status dwc2_update_isoc_urb_state(
555 struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
556 int chnum, struct dwc2_qtd *qtd,
557 enum dwc2_halt_status halt_status)
558{
559 struct dwc2_hcd_iso_packet_desc *frame_desc;
560 struct dwc2_hcd_urb *urb = qtd->urb;
561
562 if (!urb)
563 return DWC2_HC_XFER_NO_HALT_STATUS;
564
565 frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
566
567 switch (halt_status) {
568 case DWC2_HC_XFER_COMPLETE:
569 frame_desc->status = 0;
570 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
571 chan, chnum, qtd, halt_status, NULL);
572 break;
573 case DWC2_HC_XFER_FRAME_OVERRUN:
574 urb->error_count++;
575 if (chan->ep_is_in)
576 frame_desc->status = -ENOSR;
577 else
578 frame_desc->status = -ECOMM;
579 frame_desc->actual_length = 0;
580 break;
581 case DWC2_HC_XFER_BABBLE_ERR:
582 urb->error_count++;
583 frame_desc->status = -EOVERFLOW;
584 /* Don't need to update actual_length in this case */
585 break;
586 case DWC2_HC_XFER_XACT_ERR:
587 urb->error_count++;
588 frame_desc->status = -EPROTO;
589 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
590 chan, chnum, qtd, halt_status, NULL);
591
592 /* Skip whole frame */
593 if (chan->qh->do_split &&
594 chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
595 hsotg->params.host_dma) {
596 qtd->complete_split = 0;
597 qtd->isoc_split_offset = 0;
598 }
599
600 break;
601 default:
602 dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
603 halt_status);
604 break;
605 }
606
607 if (++qtd->isoc_frame_index == urb->packet_count) {
608 /*
609 * urb->status is not used for isoc transfers. The individual
610 * frame_desc statuses are used instead.
611 */
612 dwc2_host_complete(hsotg, qtd, 0);
613 halt_status = DWC2_HC_XFER_URB_COMPLETE;
614 } else {
615 halt_status = DWC2_HC_XFER_COMPLETE;
616 }
617
618 return halt_status;
619}
620
621/*
622 * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
623 * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
624 * still linked to the QH, the QH is added to the end of the inactive
625 * non-periodic schedule. For periodic QHs, removes the QH from the periodic
626 * schedule if no more QTDs are linked to the QH.
627 */
628static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
629 int free_qtd)
630{
631 int continue_split = 0;
632 struct dwc2_qtd *qtd;
633
634 if (dbg_qh(qh))
635 dev_vdbg(hsotg->dev, " %s(%p,%p,%d)\n", __func__,
636 hsotg, qh, free_qtd);
637
638 if (list_empty(&qh->qtd_list)) {
639 dev_dbg(hsotg->dev, "## QTD list empty ##\n");
640 goto no_qtd;
641 }
642
643 qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
644
645 if (qtd->complete_split)
646 continue_split = 1;
647 else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
648 qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
649 continue_split = 1;
650
651 if (free_qtd) {
652 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
653 continue_split = 0;
654 }
655
656no_qtd:
657 qh->channel = NULL;
658 dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
659}
660
661/**
662 * dwc2_release_channel() - Releases a host channel for use by other transfers
663 *
664 * @hsotg: The HCD state structure
665 * @chan: The host channel to release
666 * @qtd: The QTD associated with the host channel. This QTD may be
667 * freed if the transfer is complete or an error has occurred.
668 * @halt_status: Reason the channel is being released. This status
669 * determines the actions taken by this function.
670 *
671 * Also attempts to select and queue more transactions since at least one host
672 * channel is available.
673 */
674static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
675 struct dwc2_host_chan *chan,
676 struct dwc2_qtd *qtd,
677 enum dwc2_halt_status halt_status)
678{
679 enum dwc2_transaction_type tr_type;
680 u32 haintmsk;
681 int free_qtd = 0;
682
683 if (dbg_hc(chan))
684 dev_vdbg(hsotg->dev, " %s: channel %d, halt_status %d\n",
685 __func__, chan->hc_num, halt_status);
686
687 switch (halt_status) {
688 case DWC2_HC_XFER_URB_COMPLETE:
689 free_qtd = 1;
690 break;
691 case DWC2_HC_XFER_AHB_ERR:
692 case DWC2_HC_XFER_STALL:
693 case DWC2_HC_XFER_BABBLE_ERR:
694 free_qtd = 1;
695 break;
696 case DWC2_HC_XFER_XACT_ERR:
697 if (qtd && qtd->error_count >= 3) {
698 dev_vdbg(hsotg->dev,
699 " Complete URB with transaction error\n");
700 free_qtd = 1;
701 dwc2_host_complete(hsotg, qtd, -EPROTO);
702 }
703 break;
704 case DWC2_HC_XFER_URB_DEQUEUE:
705 /*
706 * The QTD has already been removed and the QH has been
707 * deactivated. Don't want to do anything except release the
708 * host channel and try to queue more transfers.
709 */
710 goto cleanup;
711 case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
712 dev_vdbg(hsotg->dev, " Complete URB with I/O error\n");
713 free_qtd = 1;
714 dwc2_host_complete(hsotg, qtd, -EIO);
715 break;
716 case DWC2_HC_XFER_NO_HALT_STATUS:
717 default:
718 break;
719 }
720
721 dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);
722
723cleanup:
724 /*
725 * Release the host channel for use by other transfers. The cleanup
726 * function clears the channel interrupt enables and conditions, so
727 * there's no need to clear the Channel Halted interrupt separately.
728 */
729 if (!list_empty(&chan->hc_list_entry))
730 list_del(&chan->hc_list_entry);
731 dwc2_hc_cleanup(hsotg, chan);
732 list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
733
734 if (hsotg->params.uframe_sched) {
735 hsotg->available_host_channels++;
736 } else {
737 switch (chan->ep_type) {
738 case USB_ENDPOINT_XFER_CONTROL:
739 case USB_ENDPOINT_XFER_BULK:
740 hsotg->non_periodic_channels--;
741 break;
742 default:
743 /*
744 * Don't release reservations for periodic channels
745 * here. That's done when a periodic transfer is
746 * descheduled (i.e. when the QH is removed from the
747 * periodic schedule).
748 */
749 break;
750 }
751 }
752
753 haintmsk = dwc2_readl(hsotg, HAINTMSK);
754 haintmsk &= ~(1 << chan->hc_num);
755 dwc2_writel(hsotg, haintmsk, HAINTMSK);
756
757 /* Try to queue more transfers now that there's a free channel */
758 tr_type = dwc2_hcd_select_transactions(hsotg);
759 if (tr_type != DWC2_TRANSACTION_NONE)
760 dwc2_hcd_queue_transactions(hsotg, tr_type);
761}
762
763/*
764 * Halts a host channel. If the channel cannot be halted immediately because
765 * the request queue is full, this function ensures that the FIFO empty
766 * interrupt for the appropriate queue is enabled so that the halt request can
767 * be queued when there is space in the request queue.
768 *
769 * This function may also be called in DMA mode. In that case, the channel is
770 * simply released since the core always halts the channel automatically in
771 * DMA mode.
772 */
773static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
774 struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
775 enum dwc2_halt_status halt_status)
776{
777 if (dbg_hc(chan))
778 dev_vdbg(hsotg->dev, "%s()\n", __func__);
779
780 if (hsotg->params.host_dma) {
781 if (dbg_hc(chan))
782 dev_vdbg(hsotg->dev, "DMA enabled\n");
783 dwc2_release_channel(hsotg, chan, qtd, halt_status);
784 return;
785 }
786
787 /* Slave mode processing */
788 dwc2_hc_halt(hsotg, chan, halt_status);
789
790 if (chan->halt_on_queue) {
791 u32 gintmsk;
792
793 dev_vdbg(hsotg->dev, "Halt on queue\n");
794 if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
795 chan->ep_type == USB_ENDPOINT_XFER_BULK) {
796 dev_vdbg(hsotg->dev, "control/bulk\n");
797 /*
798 * Make sure the Non-periodic Tx FIFO empty interrupt
799 * is enabled so that the non-periodic schedule will
800 * be processed
801 */
802 gintmsk = dwc2_readl(hsotg, GINTMSK);
803 gintmsk |= GINTSTS_NPTXFEMP;
804 dwc2_writel(hsotg, gintmsk, GINTMSK);
805 } else {
806 dev_vdbg(hsotg->dev, "isoc/intr\n");
807 /*
808 * Move the QH from the periodic queued schedule to
809 * the periodic assigned schedule. This allows the
810 * halt to be queued when the periodic schedule is
811 * processed.
812 */
813 list_move_tail(&chan->qh->qh_list_entry,
814 &hsotg->periodic_sched_assigned);
815
816 /*
817 * Make sure the Periodic Tx FIFO Empty interrupt is
818 * enabled so that the periodic schedule will be
819 * processed
820 */
821 gintmsk = dwc2_readl(hsotg, GINTMSK);
822 gintmsk |= GINTSTS_PTXFEMP;
823 dwc2_writel(hsotg, gintmsk, GINTMSK);
824 }
825 }
826}
827
828/*
829 * Performs common cleanup for non-periodic transfers after a Transfer
830 * Complete interrupt. This function should be called after any endpoint type
831 * specific handling is finished to release the host channel.
832 */
833static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
834 struct dwc2_host_chan *chan,
835 int chnum, struct dwc2_qtd *qtd,
836 enum dwc2_halt_status halt_status)
837{
838 dev_vdbg(hsotg->dev, "%s()\n", __func__);
839
840 qtd->error_count = 0;
841
842 if (chan->hcint & HCINTMSK_NYET) {
843 /*
844 * Got a NYET on the last transaction of the transfer. This
845 * means that the endpoint should be in the PING state at the
846 * beginning of the next transfer.
847 */
848 dev_vdbg(hsotg->dev, "got NYET\n");
849 chan->qh->ping_state = 1;
850 }
851
852 /*
853 * Always halt and release the host channel to make it available for
854 * more transfers. There may still be more phases for a control
855 * transfer or more data packets for a bulk transfer at this point,
856 * but the host channel is still halted. A channel will be reassigned
857 * to the transfer when the non-periodic schedule is processed after
858 * the channel is released. This allows transactions to be queued
859 * properly via dwc2_hcd_queue_transactions, which also enables the
860 * Tx FIFO Empty interrupt if necessary.
861 */
862 if (chan->ep_is_in) {
863 /*
864 * IN transfers in Slave mode require an explicit disable to
865 * halt the channel. (In DMA mode, this call simply releases
866 * the channel.)
867 */
868 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
869 } else {
870 /*
871 * The channel is automatically disabled by the core for OUT
872 * transfers in Slave mode
873 */
874 dwc2_release_channel(hsotg, chan, qtd, halt_status);
875 }
876}
877
878/*
879 * Performs common cleanup for periodic transfers after a Transfer Complete
880 * interrupt. This function should be called after any endpoint type specific
881 * handling is finished to release the host channel.
882 */
883static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
884 struct dwc2_host_chan *chan, int chnum,
885 struct dwc2_qtd *qtd,
886 enum dwc2_halt_status halt_status)
887{
888 u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
889
890 qtd->error_count = 0;
891
892 if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
893 /* Core halts channel in these cases */
894 dwc2_release_channel(hsotg, chan, qtd, halt_status);
895 else
896 /* Flush any outstanding requests from the Tx queue */
897 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
898}
899
900static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
901 struct dwc2_host_chan *chan, int chnum,
902 struct dwc2_qtd *qtd)
903{
904 struct dwc2_hcd_iso_packet_desc *frame_desc;
905 u32 len;
906 u32 hctsiz;
907 u32 pid;
908
909 if (!qtd->urb)
910 return 0;
911
912 frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
913 len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
914 DWC2_HC_XFER_COMPLETE, NULL);
915 if (!len && !qtd->isoc_split_offset) {
916 qtd->complete_split = 0;
917 return 0;
918 }
919
920 frame_desc->actual_length += len;
921
922 if (chan->align_buf) {
923 dev_vdbg(hsotg->dev, "non-aligned buffer\n");
924 dma_unmap_single(hsotg->dev, chan->qh->dw_align_buf_dma,
925 DWC2_KMEM_UNALIGNED_BUF_SIZE, DMA_FROM_DEVICE);
926 memcpy(qtd->urb->buf + (chan->xfer_dma - qtd->urb->dma),
927 chan->qh->dw_align_buf, len);
928 }
929
930 qtd->isoc_split_offset += len;
931
932 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
933 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
934
935 if (frame_desc->actual_length >= frame_desc->length || pid == 0) {
936 frame_desc->status = 0;
937 qtd->isoc_frame_index++;
938 qtd->complete_split = 0;
939 qtd->isoc_split_offset = 0;
940 }
941
942 if (qtd->isoc_frame_index == qtd->urb->packet_count) {
943 dwc2_host_complete(hsotg, qtd, 0);
944 dwc2_release_channel(hsotg, chan, qtd,
945 DWC2_HC_XFER_URB_COMPLETE);
946 } else {
947 dwc2_release_channel(hsotg, chan, qtd,
948 DWC2_HC_XFER_NO_HALT_STATUS);
949 }
950
951 return 1; /* Indicates that channel released */
952}
953
954/*
955 * Handles a host channel Transfer Complete interrupt. This handler may be
956 * called in either DMA mode or Slave mode.
957 */
958static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
959 struct dwc2_host_chan *chan, int chnum,
960 struct dwc2_qtd *qtd)
961{
962 struct dwc2_hcd_urb *urb = qtd->urb;
963 enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
964 int pipe_type;
965 int urb_xfer_done;
966
967 if (dbg_hc(chan))
968 dev_vdbg(hsotg->dev,
969 "--Host Channel %d Interrupt: Transfer Complete--\n",
970 chnum);
971
972 if (!urb)
973 goto handle_xfercomp_done;
974
975 pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
976
977 if (hsotg->params.dma_desc_enable) {
978 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
979 if (pipe_type == USB_ENDPOINT_XFER_ISOC)
980 /* Do not disable the interrupt, just clear it */
981 return;
982 goto handle_xfercomp_done;
983 }
984
985 /* Handle xfer complete on CSPLIT */
986 if (chan->qh->do_split) {
987 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
988 hsotg->params.host_dma) {
989 if (qtd->complete_split &&
990 dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
991 qtd))
992 goto handle_xfercomp_done;
993 } else {
994 qtd->complete_split = 0;
995 }
996 }
997
998 /* Update the QTD and URB states */
999 switch (pipe_type) {
1000 case USB_ENDPOINT_XFER_CONTROL:
1001 switch (qtd->control_phase) {
1002 case DWC2_CONTROL_SETUP:
1003 if (urb->length > 0)
1004 qtd->control_phase = DWC2_CONTROL_DATA;
1005 else
1006 qtd->control_phase = DWC2_CONTROL_STATUS;
1007 dev_vdbg(hsotg->dev,
1008 " Control setup transaction done\n");
1009 halt_status = DWC2_HC_XFER_COMPLETE;
1010 break;
1011 case DWC2_CONTROL_DATA:
1012 urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1013 chnum, urb, qtd);
1014 if (urb_xfer_done) {
1015 qtd->control_phase = DWC2_CONTROL_STATUS;
1016 dev_vdbg(hsotg->dev,
1017 " Control data transfer done\n");
1018 } else {
1019 dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1020 qtd);
1021 }
1022 halt_status = DWC2_HC_XFER_COMPLETE;
1023 break;
1024 case DWC2_CONTROL_STATUS:
1025 dev_vdbg(hsotg->dev, " Control transfer complete\n");
1026 if (urb->status == -EINPROGRESS)
1027 urb->status = 0;
1028 dwc2_host_complete(hsotg, qtd, urb->status);
1029 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1030 break;
1031 }
1032
1033 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1034 halt_status);
1035 break;
1036 case USB_ENDPOINT_XFER_BULK:
1037 dev_vdbg(hsotg->dev, " Bulk transfer complete\n");
1038 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1039 qtd);
1040 if (urb_xfer_done) {
1041 dwc2_host_complete(hsotg, qtd, urb->status);
1042 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1043 } else {
1044 halt_status = DWC2_HC_XFER_COMPLETE;
1045 }
1046
1047 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1048 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1049 halt_status);
1050 break;
1051 case USB_ENDPOINT_XFER_INT:
1052 dev_vdbg(hsotg->dev, " Interrupt transfer complete\n");
1053 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1054 qtd);
1055
1056 /*
1057 * Interrupt URB is done on the first transfer complete
1058 * interrupt
1059 */
1060 if (urb_xfer_done) {
1061 dwc2_host_complete(hsotg, qtd, urb->status);
1062 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1063 } else {
1064 halt_status = DWC2_HC_XFER_COMPLETE;
1065 }
1066
1067 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1068 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1069 halt_status);
1070 break;
1071 case USB_ENDPOINT_XFER_ISOC:
1072 if (dbg_perio())
1073 dev_vdbg(hsotg->dev, " Isochronous transfer complete\n");
1074 if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1075 halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1076 chnum, qtd,
1077 DWC2_HC_XFER_COMPLETE);
1078 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1079 halt_status);
1080 break;
1081 }
1082
1083handle_xfercomp_done:
1084 disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1085}
1086
1087/*
1088 * Handles a host channel STALL interrupt. This handler may be called in
1089 * either DMA mode or Slave mode.
1090 */
1091static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1092 struct dwc2_host_chan *chan, int chnum,
1093 struct dwc2_qtd *qtd)
1094{
1095 struct dwc2_hcd_urb *urb = qtd->urb;
1096 int pipe_type;
1097
1098 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1099 chnum);
1100
1101 if (hsotg->params.dma_desc_enable) {
1102 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1103 DWC2_HC_XFER_STALL);
1104 goto handle_stall_done;
1105 }
1106
1107 if (!urb)
1108 goto handle_stall_halt;
1109
1110 pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1111
1112 if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1113 dwc2_host_complete(hsotg, qtd, -EPIPE);
1114
1115 if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1116 pipe_type == USB_ENDPOINT_XFER_INT) {
1117 dwc2_host_complete(hsotg, qtd, -EPIPE);
1118 /*
1119 * USB protocol requires resetting the data toggle for bulk
1120 * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1121 * setup command is issued to the endpoint. Anticipate the
1122 * CLEAR_FEATURE command since a STALL has occurred and reset
1123 * the data toggle now.
1124 */
1125 chan->qh->data_toggle = 0;
1126 }
1127
1128handle_stall_halt:
1129 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);
1130
1131handle_stall_done:
1132 disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1133}
1134
1135/*
1136 * Updates the state of the URB when a transfer has been stopped due to an
1137 * abnormal condition before the transfer completes. Modifies the
1138 * actual_length field of the URB to reflect the number of bytes that have
1139 * actually been transferred via the host channel.
1140 */
1141static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1142 struct dwc2_host_chan *chan, int chnum,
1143 struct dwc2_hcd_urb *urb,
1144 struct dwc2_qtd *qtd,
1145 enum dwc2_halt_status halt_status)
1146{
1147 u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1148 qtd, halt_status, NULL);
1149 u32 hctsiz;
1150
1151 if (urb->actual_length + xfer_length > urb->length) {
1152 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1153 xfer_length = urb->length - urb->actual_length;
1154 }
1155
1156 urb->actual_length += xfer_length;
1157
1158 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1159 dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1160 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1161 dev_vdbg(hsotg->dev, " chan->start_pkt_count %d\n",
1162 chan->start_pkt_count);
1163 dev_vdbg(hsotg->dev, " hctsiz.pktcnt %d\n",
1164 (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1165 dev_vdbg(hsotg->dev, " chan->max_packet %d\n", chan->max_packet);
1166 dev_vdbg(hsotg->dev, " bytes_transferred %d\n",
1167 xfer_length);
1168 dev_vdbg(hsotg->dev, " urb->actual_length %d\n",
1169 urb->actual_length);
1170 dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n",
1171 urb->length);
1172}
1173
1174/*
1175 * Handles a host channel NAK interrupt. This handler may be called in either
1176 * DMA mode or Slave mode.
1177 */
1178static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1179 struct dwc2_host_chan *chan, int chnum,
1180 struct dwc2_qtd *qtd)
1181{
1182 if (!qtd) {
1183 dev_dbg(hsotg->dev, "%s: qtd is NULL\n", __func__);
1184 return;
1185 }
1186
1187 if (!qtd->urb) {
1188 dev_dbg(hsotg->dev, "%s: qtd->urb is NULL\n", __func__);
1189 return;
1190 }
1191
1192 if (dbg_hc(chan))
1193 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1194 chnum);
1195
1196 /*
1197 * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1198 * interrupt. Re-start the SSPLIT transfer.
1199 *
1200 * Normally for non-periodic transfers we'll retry right away, but to
1201 * avoid interrupt storms we'll wait before retrying if we've got
1202 * several NAKs. If we didn't do this we'd retry directly from the
1203 * interrupt handler and could end up quickly getting another
1204 * interrupt (another NAK), which we'd retry. Note that we do not
1205 * delay retries for IN parts of control requests, as those are expected
1206 * to complete fairly quickly, and if we delay them we risk confusing
1207 * the device and cause it issue STALL.
1208 *
1209 * Note that in DMA mode software only gets involved to re-send NAKed
1210 * transfers for split transactions, so we only need to apply this
1211 * delaying logic when handling splits. In non-DMA mode presumably we
1212 * might want a similar delay if someone can demonstrate this problem
1213 * affects that code path too.
1214 */
1215 if (chan->do_split) {
1216 if (chan->complete_split)
1217 qtd->error_count = 0;
1218 qtd->complete_split = 0;
1219 qtd->num_naks++;
1220 qtd->qh->want_wait = qtd->num_naks >= DWC2_NAKS_BEFORE_DELAY &&
1221 !(chan->ep_type == USB_ENDPOINT_XFER_CONTROL &&
1222 chan->ep_is_in);
1223 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1224 goto handle_nak_done;
1225 }
1226
1227 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1228 case USB_ENDPOINT_XFER_CONTROL:
1229 case USB_ENDPOINT_XFER_BULK:
1230 if (hsotg->params.host_dma && chan->ep_is_in) {
1231 /*
1232 * NAK interrupts are enabled on bulk/control IN
1233 * transfers in DMA mode for the sole purpose of
1234 * resetting the error count after a transaction error
1235 * occurs. The core will continue transferring data.
1236 */
1237 qtd->error_count = 0;
1238 break;
1239 }
1240
1241 /*
1242 * NAK interrupts normally occur during OUT transfers in DMA
1243 * or Slave mode. For IN transfers, more requests will be
1244 * queued as request queue space is available.
1245 */
1246 qtd->error_count = 0;
1247
1248 if (!chan->qh->ping_state) {
1249 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1250 qtd, DWC2_HC_XFER_NAK);
1251 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1252
1253 if (chan->speed == USB_SPEED_HIGH)
1254 chan->qh->ping_state = 1;
1255 }
1256
1257 /*
1258 * Halt the channel so the transfer can be re-started from
1259 * the appropriate point or the PING protocol will
1260 * start/continue
1261 */
1262 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1263 break;
1264 case USB_ENDPOINT_XFER_INT:
1265 qtd->error_count = 0;
1266 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1267 break;
1268 case USB_ENDPOINT_XFER_ISOC:
1269 /* Should never get called for isochronous transfers */
1270 dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1271 break;
1272 }
1273
1274handle_nak_done:
1275 disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1276}
1277
1278/*
1279 * Handles a host channel ACK interrupt. This interrupt is enabled when
1280 * performing the PING protocol in Slave mode, when errors occur during
1281 * either Slave mode or DMA mode, and during Start Split transactions.
1282 */
1283static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1284 struct dwc2_host_chan *chan, int chnum,
1285 struct dwc2_qtd *qtd)
1286{
1287 struct dwc2_hcd_iso_packet_desc *frame_desc;
1288
1289 if (dbg_hc(chan))
1290 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1291 chnum);
1292
1293 if (chan->do_split) {
1294 /* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1295 if (!chan->ep_is_in &&
1296 chan->data_pid_start != DWC2_HC_PID_SETUP)
1297 qtd->ssplit_out_xfer_count = chan->xfer_len;
1298
1299 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1300 qtd->complete_split = 1;
1301 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1302 } else {
1303 /* ISOC OUT */
1304 switch (chan->xact_pos) {
1305 case DWC2_HCSPLT_XACTPOS_ALL:
1306 break;
1307 case DWC2_HCSPLT_XACTPOS_END:
1308 qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1309 qtd->isoc_split_offset = 0;
1310 break;
1311 case DWC2_HCSPLT_XACTPOS_BEGIN:
1312 case DWC2_HCSPLT_XACTPOS_MID:
1313 /*
1314 * For BEGIN or MID, calculate the length for
1315 * the next microframe to determine the correct
1316 * SSPLIT token, either MID or END
1317 */
1318 frame_desc = &qtd->urb->iso_descs[
1319 qtd->isoc_frame_index];
1320 qtd->isoc_split_offset += 188;
1321
1322 if (frame_desc->length - qtd->isoc_split_offset
1323 <= 188)
1324 qtd->isoc_split_pos =
1325 DWC2_HCSPLT_XACTPOS_END;
1326 else
1327 qtd->isoc_split_pos =
1328 DWC2_HCSPLT_XACTPOS_MID;
1329 break;
1330 }
1331 }
1332 } else {
1333 qtd->error_count = 0;
1334
1335 if (chan->qh->ping_state) {
1336 chan->qh->ping_state = 0;
1337 /*
1338 * Halt the channel so the transfer can be re-started
1339 * from the appropriate point. This only happens in
1340 * Slave mode. In DMA mode, the ping_state is cleared
1341 * when the transfer is started because the core
1342 * automatically executes the PING, then the transfer.
1343 */
1344 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1345 }
1346 }
1347
1348 /*
1349 * If the ACK occurred when _not_ in the PING state, let the channel
1350 * continue transferring data after clearing the error count
1351 */
1352 disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1353}
1354
1355/*
1356 * Handles a host channel NYET interrupt. This interrupt should only occur on
1357 * Bulk and Control OUT endpoints and for complete split transactions. If a
1358 * NYET occurs at the same time as a Transfer Complete interrupt, it is
1359 * handled in the xfercomp interrupt handler, not here. This handler may be
1360 * called in either DMA mode or Slave mode.
1361 */
1362static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1363 struct dwc2_host_chan *chan, int chnum,
1364 struct dwc2_qtd *qtd)
1365{
1366 if (dbg_hc(chan))
1367 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1368 chnum);
1369
1370 /*
1371 * NYET on CSPLIT
1372 * re-do the CSPLIT immediately on non-periodic
1373 */
1374 if (chan->do_split && chan->complete_split) {
1375 if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1376 hsotg->params.host_dma) {
1377 qtd->complete_split = 0;
1378 qtd->isoc_split_offset = 0;
1379 qtd->isoc_frame_index++;
1380 if (qtd->urb &&
1381 qtd->isoc_frame_index == qtd->urb->packet_count) {
1382 dwc2_host_complete(hsotg, qtd, 0);
1383 dwc2_release_channel(hsotg, chan, qtd,
1384 DWC2_HC_XFER_URB_COMPLETE);
1385 } else {
1386 dwc2_release_channel(hsotg, chan, qtd,
1387 DWC2_HC_XFER_NO_HALT_STATUS);
1388 }
1389 goto handle_nyet_done;
1390 }
1391
1392 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1393 chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1394 struct dwc2_qh *qh = chan->qh;
1395 bool past_end;
1396
1397 if (!hsotg->params.uframe_sched) {
1398 int frnum = dwc2_hcd_get_frame_number(hsotg);
1399
1400 /* Don't have num_hs_transfers; simple logic */
1401 past_end = dwc2_full_frame_num(frnum) !=
1402 dwc2_full_frame_num(qh->next_active_frame);
1403 } else {
1404 int end_frnum;
1405
1406 /*
1407 * Figure out the end frame based on
1408 * schedule.
1409 *
1410 * We don't want to go on trying again
1411 * and again forever. Let's stop when
1412 * we've done all the transfers that
1413 * were scheduled.
1414 *
1415 * We're going to be comparing
1416 * start_active_frame and
1417 * next_active_frame, both of which
1418 * are 1 before the time the packet
1419 * goes on the wire, so that cancels
1420 * out. Basically if had 1 transfer
1421 * and we saw 1 NYET then we're done.
1422 * We're getting a NYET here so if
1423 * next >= (start + num_transfers)
1424 * we're done. The complexity is that
1425 * for all but ISOC_OUT we skip one
1426 * slot.
1427 */
1428 end_frnum = dwc2_frame_num_inc(
1429 qh->start_active_frame,
1430 qh->num_hs_transfers);
1431
1432 if (qh->ep_type != USB_ENDPOINT_XFER_ISOC ||
1433 qh->ep_is_in)
1434 end_frnum =
1435 dwc2_frame_num_inc(end_frnum, 1);
1436
1437 past_end = dwc2_frame_num_le(
1438 end_frnum, qh->next_active_frame);
1439 }
1440
1441 if (past_end) {
1442 /* Treat this as a transaction error. */
1443#if 0
1444 /*
1445 * Todo: Fix system performance so this can
1446 * be treated as an error. Right now complete
1447 * splits cannot be scheduled precisely enough
1448 * due to other system activity, so this error
1449 * occurs regularly in Slave mode.
1450 */
1451 qtd->error_count++;
1452#endif
1453 qtd->complete_split = 0;
1454 dwc2_halt_channel(hsotg, chan, qtd,
1455 DWC2_HC_XFER_XACT_ERR);
1456 /* Todo: add support for isoc release */
1457 goto handle_nyet_done;
1458 }
1459 }
1460
1461 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1462 goto handle_nyet_done;
1463 }
1464
1465 chan->qh->ping_state = 1;
1466 qtd->error_count = 0;
1467
1468 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
1469 DWC2_HC_XFER_NYET);
1470 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1471
1472 /*
1473 * Halt the channel and re-start the transfer so the PING protocol
1474 * will start
1475 */
1476 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1477
1478handle_nyet_done:
1479 disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1480}
1481
1482/*
1483 * Handles a host channel babble interrupt. This handler may be called in
1484 * either DMA mode or Slave mode.
1485 */
1486static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1487 struct dwc2_host_chan *chan, int chnum,
1488 struct dwc2_qtd *qtd)
1489{
1490 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1491 chnum);
1492
1493 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1494
1495 if (hsotg->params.dma_desc_enable) {
1496 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1497 DWC2_HC_XFER_BABBLE_ERR);
1498 goto disable_int;
1499 }
1500
1501 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1502 dwc2_host_complete(hsotg, qtd, -EOVERFLOW);
1503 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
1504 } else {
1505 enum dwc2_halt_status halt_status;
1506
1507 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1508 qtd, DWC2_HC_XFER_BABBLE_ERR);
1509 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1510 }
1511
1512disable_int:
1513 disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1514}
1515
1516/*
1517 * Handles a host channel AHB error interrupt. This handler is only called in
1518 * DMA mode.
1519 */
1520static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1521 struct dwc2_host_chan *chan, int chnum,
1522 struct dwc2_qtd *qtd)
1523{
1524 struct dwc2_hcd_urb *urb = qtd->urb;
1525 char *pipetype, *speed;
1526 u32 hcchar;
1527 u32 hcsplt;
1528 u32 hctsiz;
1529 u32 hc_dma;
1530
1531 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1532 chnum);
1533
1534 if (!urb)
1535 goto handle_ahberr_halt;
1536
1537 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1538
1539 hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1540 hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1541 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1542 hc_dma = dwc2_readl(hsotg, HCDMA(chnum));
1543
1544 dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1545 dev_err(hsotg->dev, " hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1546 dev_err(hsotg->dev, " hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1547 dev_err(hsotg->dev, " Device address: %d\n",
1548 dwc2_hcd_get_dev_addr(&urb->pipe_info));
1549 dev_err(hsotg->dev, " Endpoint: %d, %s\n",
1550 dwc2_hcd_get_ep_num(&urb->pipe_info),
1551 dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1552
1553 switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
1554 case USB_ENDPOINT_XFER_CONTROL:
1555 pipetype = "CONTROL";
1556 break;
1557 case USB_ENDPOINT_XFER_BULK:
1558 pipetype = "BULK";
1559 break;
1560 case USB_ENDPOINT_XFER_INT:
1561 pipetype = "INTERRUPT";
1562 break;
1563 case USB_ENDPOINT_XFER_ISOC:
1564 pipetype = "ISOCHRONOUS";
1565 break;
1566 default:
1567 pipetype = "UNKNOWN";
1568 break;
1569 }
1570
1571 dev_err(hsotg->dev, " Endpoint type: %s\n", pipetype);
1572
1573 switch (chan->speed) {
1574 case USB_SPEED_HIGH:
1575 speed = "HIGH";
1576 break;
1577 case USB_SPEED_FULL:
1578 speed = "FULL";
1579 break;
1580 case USB_SPEED_LOW:
1581 speed = "LOW";
1582 break;
1583 default:
1584 speed = "UNKNOWN";
1585 break;
1586 }
1587
1588 dev_err(hsotg->dev, " Speed: %s\n", speed);
1589
1590 dev_err(hsotg->dev, " Max packet size: %d (mult %d)\n",
1591 dwc2_hcd_get_maxp(&urb->pipe_info),
1592 dwc2_hcd_get_maxp_mult(&urb->pipe_info));
1593 dev_err(hsotg->dev, " Data buffer length: %d\n", urb->length);
1594 dev_err(hsotg->dev, " Transfer buffer: %p, Transfer DMA: %08lx\n",
1595 urb->buf, (unsigned long)urb->dma);
1596 dev_err(hsotg->dev, " Setup buffer: %p, Setup DMA: %08lx\n",
1597 urb->setup_packet, (unsigned long)urb->setup_dma);
1598 dev_err(hsotg->dev, " Interval: %d\n", urb->interval);
1599
1600 /* Core halts the channel for Descriptor DMA mode */
1601 if (hsotg->params.dma_desc_enable) {
1602 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1603 DWC2_HC_XFER_AHB_ERR);
1604 goto handle_ahberr_done;
1605 }
1606
1607 dwc2_host_complete(hsotg, qtd, -EIO);
1608
1609handle_ahberr_halt:
1610 /*
1611 * Force a channel halt. Don't call dwc2_halt_channel because that won't
1612 * write to the HCCHARn register in DMA mode to force the halt.
1613 */
1614 dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);
1615
1616handle_ahberr_done:
1617 disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1618}
1619
1620/*
1621 * Handles a host channel transaction error interrupt. This handler may be
1622 * called in either DMA mode or Slave mode.
1623 */
1624static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1625 struct dwc2_host_chan *chan, int chnum,
1626 struct dwc2_qtd *qtd)
1627{
1628 dev_dbg(hsotg->dev,
1629 "--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1630
1631 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1632
1633 if (hsotg->params.dma_desc_enable) {
1634 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1635 DWC2_HC_XFER_XACT_ERR);
1636 goto handle_xacterr_done;
1637 }
1638
1639 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1640 case USB_ENDPOINT_XFER_CONTROL:
1641 case USB_ENDPOINT_XFER_BULK:
1642 qtd->error_count++;
1643 if (!chan->qh->ping_state) {
1644 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1645 qtd, DWC2_HC_XFER_XACT_ERR);
1646 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1647 if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1648 chan->qh->ping_state = 1;
1649 }
1650
1651 /*
1652 * Halt the channel so the transfer can be re-started from
1653 * the appropriate point or the PING protocol will start
1654 */
1655 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1656 break;
1657 case USB_ENDPOINT_XFER_INT:
1658 qtd->error_count++;
1659 if (chan->do_split && chan->complete_split)
1660 qtd->complete_split = 0;
1661 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1662 break;
1663 case USB_ENDPOINT_XFER_ISOC:
1664 {
1665 enum dwc2_halt_status halt_status;
1666
1667 halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1668 chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1669 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1670 }
1671 break;
1672 }
1673
1674handle_xacterr_done:
1675 disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1676}
1677
1678/*
1679 * Handles a host channel frame overrun interrupt. This handler may be called
1680 * in either DMA mode or Slave mode.
1681 */
1682static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1683 struct dwc2_host_chan *chan, int chnum,
1684 struct dwc2_qtd *qtd)
1685{
1686 enum dwc2_halt_status halt_status;
1687
1688 if (dbg_hc(chan))
1689 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1690 chnum);
1691
1692 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1693
1694 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1695 case USB_ENDPOINT_XFER_CONTROL:
1696 case USB_ENDPOINT_XFER_BULK:
1697 break;
1698 case USB_ENDPOINT_XFER_INT:
1699 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1700 break;
1701 case USB_ENDPOINT_XFER_ISOC:
1702 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1703 qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1704 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1705 break;
1706 }
1707
1708 disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1709}
1710
1711/*
1712 * Handles a host channel data toggle error interrupt. This handler may be
1713 * called in either DMA mode or Slave mode.
1714 */
1715static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1716 struct dwc2_host_chan *chan, int chnum,
1717 struct dwc2_qtd *qtd)
1718{
1719 dev_dbg(hsotg->dev,
1720 "--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1721
1722 if (chan->ep_is_in)
1723 qtd->error_count = 0;
1724 else
1725 dev_err(hsotg->dev,
1726 "Data Toggle Error on OUT transfer, channel %d\n",
1727 chnum);
1728
1729 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1730 disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1731}
1732
1733/*
1734 * For debug only. It checks that a valid halt status is set and that
1735 * HCCHARn.chdis is clear. If there's a problem, corrective action is
1736 * taken and a warning is issued.
1737 *
1738 * Return: true if halt status is ok, false otherwise
1739 */
1740static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1741 struct dwc2_host_chan *chan, int chnum,
1742 struct dwc2_qtd *qtd)
1743{
1744#ifdef DEBUG
1745 u32 hcchar;
1746 u32 hctsiz;
1747 u32 hcintmsk;
1748 u32 hcsplt;
1749
1750 if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1751 /*
1752 * This code is here only as a check. This condition should
1753 * never happen. Ignore the halt if it does occur.
1754 */
1755 hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1756 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1757 hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1758 hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1759 dev_dbg(hsotg->dev,
1760 "%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1761 __func__);
1762 dev_dbg(hsotg->dev,
1763 "channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1764 chnum, hcchar, hctsiz);
1765 dev_dbg(hsotg->dev,
1766 "hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1767 chan->hcint, hcintmsk, hcsplt);
1768 if (qtd)
1769 dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1770 qtd->complete_split);
1771 dev_warn(hsotg->dev,
1772 "%s: no halt status, channel %d, ignoring interrupt\n",
1773 __func__, chnum);
1774 return false;
1775 }
1776
1777 /*
1778 * This code is here only as a check. hcchar.chdis should never be set
1779 * when the halt interrupt occurs. Halt the channel again if it does
1780 * occur.
1781 */
1782 hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1783 if (hcchar & HCCHAR_CHDIS) {
1784 dev_warn(hsotg->dev,
1785 "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1786 __func__, hcchar);
1787 chan->halt_pending = 0;
1788 dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
1789 return false;
1790 }
1791#endif
1792
1793 return true;
1794}
1795
1796/*
1797 * Handles a host Channel Halted interrupt in DMA mode. This handler
1798 * determines the reason the channel halted and proceeds accordingly.
1799 */
1800static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1801 struct dwc2_host_chan *chan, int chnum,
1802 struct dwc2_qtd *qtd)
1803{
1804 u32 hcintmsk;
1805 int out_nak_enh = 0;
1806
1807 if (dbg_hc(chan))
1808 dev_vdbg(hsotg->dev,
1809 "--Host Channel %d Interrupt: DMA Channel Halted--\n",
1810 chnum);
1811
1812 /*
1813 * For core with OUT NAK enhancement, the flow for high-speed
1814 * CONTROL/BULK OUT is handled a little differently
1815 */
1816 if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1817 if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1818 (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1819 chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1820 out_nak_enh = 1;
1821 }
1822 }
1823
1824 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1825 (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1826 !hsotg->params.dma_desc_enable)) {
1827 if (hsotg->params.dma_desc_enable)
1828 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1829 chan->halt_status);
1830 else
1831 /*
1832 * Just release the channel. A dequeue can happen on a
1833 * transfer timeout. In the case of an AHB Error, the
1834 * channel was forced to halt because there's no way to
1835 * gracefully recover.
1836 */
1837 dwc2_release_channel(hsotg, chan, qtd,
1838 chan->halt_status);
1839 return;
1840 }
1841
1842 hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1843
1844 if (chan->hcint & HCINTMSK_XFERCOMPL) {
1845 /*
1846 * Todo: This is here because of a possible hardware bug. Spec
1847 * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1848 * interrupt w/ACK bit set should occur, but I only see the
1849 * XFERCOMP bit, even with it masked out. This is a workaround
1850 * for that behavior. Should fix this when hardware is fixed.
1851 */
1852 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1853 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1854 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1855 } else if (chan->hcint & HCINTMSK_STALL) {
1856 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1857 } else if ((chan->hcint & HCINTMSK_XACTERR) &&
1858 !hsotg->params.dma_desc_enable) {
1859 if (out_nak_enh) {
1860 if (chan->hcint &
1861 (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1862 dev_vdbg(hsotg->dev,
1863 "XactErr with NYET/NAK/ACK\n");
1864 qtd->error_count = 0;
1865 } else {
1866 dev_vdbg(hsotg->dev,
1867 "XactErr without NYET/NAK/ACK\n");
1868 }
1869 }
1870
1871 /*
1872 * Must handle xacterr before nak or ack. Could get a xacterr
1873 * at the same time as either of these on a BULK/CONTROL OUT
1874 * that started with a PING. The xacterr takes precedence.
1875 */
1876 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1877 } else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1878 hsotg->params.dma_desc_enable) {
1879 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1880 } else if ((chan->hcint & HCINTMSK_AHBERR) &&
1881 hsotg->params.dma_desc_enable) {
1882 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1883 } else if (chan->hcint & HCINTMSK_BBLERR) {
1884 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1885 } else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1886 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1887 } else if (!out_nak_enh) {
1888 if (chan->hcint & HCINTMSK_NYET) {
1889 /*
1890 * Must handle nyet before nak or ack. Could get a nyet
1891 * at the same time as either of those on a BULK/CONTROL
1892 * OUT that started with a PING. The nyet takes
1893 * precedence.
1894 */
1895 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1896 } else if ((chan->hcint & HCINTMSK_NAK) &&
1897 !(hcintmsk & HCINTMSK_NAK)) {
1898 /*
1899 * If nak is not masked, it's because a non-split IN
1900 * transfer is in an error state. In that case, the nak
1901 * is handled by the nak interrupt handler, not here.
1902 * Handle nak here for BULK/CONTROL OUT transfers, which
1903 * halt on a NAK to allow rewinding the buffer pointer.
1904 */
1905 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1906 } else if ((chan->hcint & HCINTMSK_ACK) &&
1907 !(hcintmsk & HCINTMSK_ACK)) {
1908 /*
1909 * If ack is not masked, it's because a non-split IN
1910 * transfer is in an error state. In that case, the ack
1911 * is handled by the ack interrupt handler, not here.
1912 * Handle ack here for split transfers. Start splits
1913 * halt on ACK.
1914 */
1915 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1916 } else {
1917 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1918 chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1919 /*
1920 * A periodic transfer halted with no other
1921 * channel interrupts set. Assume it was halted
1922 * by the core because it could not be completed
1923 * in its scheduled (micro)frame.
1924 */
1925 dev_dbg(hsotg->dev,
1926 "%s: Halt channel %d (assume incomplete periodic transfer)\n",
1927 __func__, chnum);
1928 dwc2_halt_channel(hsotg, chan, qtd,
1929 DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1930 } else {
1931 dev_err(hsotg->dev,
1932 "%s: Channel %d - ChHltd set, but reason is unknown\n",
1933 __func__, chnum);
1934 dev_err(hsotg->dev,
1935 "hcint 0x%08x, intsts 0x%08x\n",
1936 chan->hcint,
1937 dwc2_readl(hsotg, GINTSTS));
1938 goto error;
1939 }
1940 }
1941 } else {
1942 dev_info(hsotg->dev,
1943 "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1944 chan->hcint);
1945error:
1946 /* Failthrough: use 3-strikes rule */
1947 qtd->error_count++;
1948 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1949 qtd, DWC2_HC_XFER_XACT_ERR);
1950 /*
1951 * We can get here after a completed transaction
1952 * (urb->actual_length >= urb->length) which was not reported
1953 * as completed. If that is the case, and we do not abort
1954 * the transfer, a transfer of size 0 will be enqueued
1955 * subsequently. If urb->actual_length is not DMA-aligned,
1956 * the buffer will then point to an unaligned address, and
1957 * the resulting behavior is undefined. Bail out in that
1958 * situation.
1959 */
1960 if (qtd->urb->actual_length >= qtd->urb->length)
1961 qtd->error_count = 3;
1962 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1963 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1964 }
1965}
1966
1967/*
1968 * Handles a host channel Channel Halted interrupt
1969 *
1970 * In slave mode, this handler is called only when the driver specifically
1971 * requests a halt. This occurs during handling other host channel interrupts
1972 * (e.g. nak, xacterr, stall, nyet, etc.).
1973 *
1974 * In DMA mode, this is the interrupt that occurs when the core has finished
1975 * processing a transfer on a channel. Other host channel interrupts (except
1976 * ahberr) are disabled in DMA mode.
1977 */
1978static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
1979 struct dwc2_host_chan *chan, int chnum,
1980 struct dwc2_qtd *qtd)
1981{
1982 if (dbg_hc(chan))
1983 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
1984 chnum);
1985
1986 if (hsotg->params.host_dma) {
1987 dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
1988 } else {
1989 if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
1990 return;
1991 dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
1992 }
1993}
1994
1995/*
1996 * Check if the given qtd is still the top of the list (and thus valid).
1997 *
1998 * If dwc2_hcd_qtd_unlink_and_free() has been called since we grabbed
1999 * the qtd from the top of the list, this will return false (otherwise true).
2000 */
2001static bool dwc2_check_qtd_still_ok(struct dwc2_qtd *qtd, struct dwc2_qh *qh)
2002{
2003 struct dwc2_qtd *cur_head;
2004
2005 if (!qh)
2006 return false;
2007
2008 cur_head = list_first_entry(&qh->qtd_list, struct dwc2_qtd,
2009 qtd_list_entry);
2010 return (cur_head == qtd);
2011}
2012
2013/* Handles interrupt for a specific Host Channel */
2014static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
2015{
2016 struct dwc2_qtd *qtd;
2017 struct dwc2_host_chan *chan;
2018 u32 hcint, hcintraw, hcintmsk;
2019
2020 chan = hsotg->hc_ptr_array[chnum];
2021
2022 hcintraw = dwc2_readl(hsotg, HCINT(chnum));
2023 hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
2024 hcint = hcintraw & hcintmsk;
2025 dwc2_writel(hsotg, hcint, HCINT(chnum));
2026
2027 if (!chan) {
2028 dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
2029 return;
2030 }
2031
2032 if (dbg_hc(chan)) {
2033 dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
2034 chnum);
2035 dev_vdbg(hsotg->dev,
2036 " hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2037 hcintraw, hcintmsk, hcint);
2038 }
2039
2040 /*
2041 * If we got an interrupt after someone called
2042 * dwc2_hcd_endpoint_disable() we don't want to crash below
2043 */
2044 if (!chan->qh) {
2045 dev_warn(hsotg->dev, "Interrupt on disabled channel\n");
2046 return;
2047 }
2048
2049 chan->hcint = hcintraw;
2050
2051 /*
2052 * If the channel was halted due to a dequeue, the qtd list might
2053 * be empty or at least the first entry will not be the active qtd.
2054 * In this case, take a shortcut and just release the channel.
2055 */
2056 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
2057 /*
2058 * If the channel was halted, this should be the only
2059 * interrupt unmasked
2060 */
2061 WARN_ON(hcint != HCINTMSK_CHHLTD);
2062 if (hsotg->params.dma_desc_enable)
2063 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
2064 chan->halt_status);
2065 else
2066 dwc2_release_channel(hsotg, chan, NULL,
2067 chan->halt_status);
2068 return;
2069 }
2070
2071 if (list_empty(&chan->qh->qtd_list)) {
2072 /*
2073 * TODO: Will this ever happen with the
2074 * DWC2_HC_XFER_URB_DEQUEUE handling above?
2075 */
2076 dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
2077 chnum);
2078 dev_dbg(hsotg->dev,
2079 " hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2080 chan->hcint, hcintmsk, hcint);
2081 chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
2082 disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
2083 chan->hcint = 0;
2084 return;
2085 }
2086
2087 qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
2088 qtd_list_entry);
2089
2090 if (!hsotg->params.host_dma) {
2091 if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
2092 hcint &= ~HCINTMSK_CHHLTD;
2093 }
2094
2095 if (hcint & HCINTMSK_XFERCOMPL) {
2096 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
2097 /*
2098 * If NYET occurred at same time as Xfer Complete, the NYET is
2099 * handled by the Xfer Complete interrupt handler. Don't want
2100 * to call the NYET interrupt handler in this case.
2101 */
2102 hcint &= ~HCINTMSK_NYET;
2103 }
2104
2105 if (hcint & HCINTMSK_CHHLTD) {
2106 dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2107 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2108 goto exit;
2109 }
2110 if (hcint & HCINTMSK_AHBERR) {
2111 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2112 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2113 goto exit;
2114 }
2115 if (hcint & HCINTMSK_STALL) {
2116 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2117 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2118 goto exit;
2119 }
2120 if (hcint & HCINTMSK_NAK) {
2121 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2122 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2123 goto exit;
2124 }
2125 if (hcint & HCINTMSK_ACK) {
2126 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2127 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2128 goto exit;
2129 }
2130 if (hcint & HCINTMSK_NYET) {
2131 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2132 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2133 goto exit;
2134 }
2135 if (hcint & HCINTMSK_XACTERR) {
2136 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2137 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2138 goto exit;
2139 }
2140 if (hcint & HCINTMSK_BBLERR) {
2141 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2142 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2143 goto exit;
2144 }
2145 if (hcint & HCINTMSK_FRMOVRUN) {
2146 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2147 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2148 goto exit;
2149 }
2150 if (hcint & HCINTMSK_DATATGLERR) {
2151 dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2152 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2153 goto exit;
2154 }
2155
2156exit:
2157 chan->hcint = 0;
2158}
2159
2160/*
2161 * This interrupt indicates that one or more host channels has a pending
2162 * interrupt. There are multiple conditions that can cause each host channel
2163 * interrupt. This function determines which conditions have occurred for each
2164 * host channel interrupt and handles them appropriately.
2165 */
2166static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2167{
2168 u32 haint;
2169 int i;
2170 struct dwc2_host_chan *chan, *chan_tmp;
2171
2172 haint = dwc2_readl(hsotg, HAINT);
2173 if (dbg_perio()) {
2174 dev_vdbg(hsotg->dev, "%s()\n", __func__);
2175
2176 dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2177 }
2178
2179 /*
2180 * According to USB 2.0 spec section 11.18.8, a host must
2181 * issue complete-split transactions in a microframe for a
2182 * set of full-/low-speed endpoints in the same relative
2183 * order as the start-splits were issued in a microframe for.
2184 */
2185 list_for_each_entry_safe(chan, chan_tmp, &hsotg->split_order,
2186 split_order_list_entry) {
2187 int hc_num = chan->hc_num;
2188
2189 if (haint & (1 << hc_num)) {
2190 dwc2_hc_n_intr(hsotg, hc_num);
2191 haint &= ~(1 << hc_num);
2192 }
2193 }
2194
2195 for (i = 0; i < hsotg->params.host_channels; i++) {
2196 if (haint & (1 << i))
2197 dwc2_hc_n_intr(hsotg, i);
2198 }
2199}
2200
2201/* This function handles interrupts for the HCD */
2202irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2203{
2204 u32 gintsts, dbg_gintsts;
2205 irqreturn_t retval = IRQ_HANDLED;
2206
2207 if (!dwc2_is_controller_alive(hsotg)) {
2208 dev_warn(hsotg->dev, "Controller is dead\n");
2209 return retval;
2210 } else {
2211 retval = IRQ_NONE;
2212 }
2213
2214 spin_lock(&hsotg->lock);
2215
2216 /* Check if HOST Mode */
2217 if (dwc2_is_host_mode(hsotg)) {
2218 gintsts = dwc2_read_core_intr(hsotg);
2219 if (!gintsts) {
2220 spin_unlock(&hsotg->lock);
2221 return retval;
2222 }
2223
2224 retval = IRQ_HANDLED;
2225
2226 dbg_gintsts = gintsts;
2227#ifndef DEBUG_SOF
2228 dbg_gintsts &= ~GINTSTS_SOF;
2229#endif
2230 if (!dbg_perio())
2231 dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
2232 GINTSTS_PTXFEMP);
2233
2234 /* Only print if there are any non-suppressed interrupts left */
2235 if (dbg_gintsts)
2236 dev_vdbg(hsotg->dev,
2237 "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2238 gintsts);
2239
2240 if (gintsts & GINTSTS_SOF)
2241 dwc2_sof_intr(hsotg);
2242 if (gintsts & GINTSTS_RXFLVL)
2243 dwc2_rx_fifo_level_intr(hsotg);
2244 if (gintsts & GINTSTS_NPTXFEMP)
2245 dwc2_np_tx_fifo_empty_intr(hsotg);
2246 if (gintsts & GINTSTS_PRTINT)
2247 dwc2_port_intr(hsotg);
2248 if (gintsts & GINTSTS_HCHINT)
2249 dwc2_hc_intr(hsotg);
2250 if (gintsts & GINTSTS_PTXFEMP)
2251 dwc2_perio_tx_fifo_empty_intr(hsotg);
2252
2253 if (dbg_gintsts) {
2254 dev_vdbg(hsotg->dev,
2255 "DWC OTG HCD Finished Servicing Interrupts\n");
2256 dev_vdbg(hsotg->dev,
2257 "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2258 dwc2_readl(hsotg, GINTSTS),
2259 dwc2_readl(hsotg, GINTMSK));
2260 }
2261 }
2262
2263 spin_unlock(&hsotg->lock);
2264
2265 return retval;
2266}