Loading...
1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2/*
3 * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
4 *
5 * Copyright (C) 2004-2013 Synopsys, Inc.
6 */
7
8/*
9 * This file contains the interrupt handlers for Host mode
10 */
11#include <linux/kernel.h>
12#include <linux/module.h>
13#include <linux/spinlock.h>
14#include <linux/interrupt.h>
15#include <linux/dma-mapping.h>
16#include <linux/io.h>
17#include <linux/slab.h>
18#include <linux/usb.h>
19
20#include <linux/usb/hcd.h>
21#include <linux/usb/ch11.h>
22
23#include "core.h"
24#include "hcd.h"
25
26/*
27 * If we get this many NAKs on a split transaction we'll slow down
28 * retransmission. A 1 here means delay after the first NAK.
29 */
30#define DWC2_NAKS_BEFORE_DELAY 3
31
32/* This function is for debug only */
33static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
34{
35 u16 curr_frame_number = hsotg->frame_number;
36 u16 expected = dwc2_frame_num_inc(hsotg->last_frame_num, 1);
37
38 if (expected != curr_frame_number)
39 dwc2_sch_vdbg(hsotg, "MISSED SOF %04x != %04x\n",
40 expected, curr_frame_number);
41
42#ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
43 if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
44 if (expected != curr_frame_number) {
45 hsotg->frame_num_array[hsotg->frame_num_idx] =
46 curr_frame_number;
47 hsotg->last_frame_num_array[hsotg->frame_num_idx] =
48 hsotg->last_frame_num;
49 hsotg->frame_num_idx++;
50 }
51 } else if (!hsotg->dumped_frame_num_array) {
52 int i;
53
54 dev_info(hsotg->dev, "Frame Last Frame\n");
55 dev_info(hsotg->dev, "----- ----------\n");
56 for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
57 dev_info(hsotg->dev, "0x%04x 0x%04x\n",
58 hsotg->frame_num_array[i],
59 hsotg->last_frame_num_array[i]);
60 }
61 hsotg->dumped_frame_num_array = 1;
62 }
63#endif
64 hsotg->last_frame_num = curr_frame_number;
65}
66
67static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
68 struct dwc2_host_chan *chan,
69 struct dwc2_qtd *qtd)
70{
71 struct usb_device *root_hub = dwc2_hsotg_to_hcd(hsotg)->self.root_hub;
72 struct urb *usb_urb;
73
74 if (!chan->qh)
75 return;
76
77 if (chan->qh->dev_speed == USB_SPEED_HIGH)
78 return;
79
80 if (!qtd->urb)
81 return;
82
83 usb_urb = qtd->urb->priv;
84 if (!usb_urb || !usb_urb->dev || !usb_urb->dev->tt)
85 return;
86
87 /*
88 * The root hub doesn't really have a TT, but Linux thinks it
89 * does because how could you have a "high speed hub" that
90 * directly talks directly to low speed devices without a TT?
91 * It's all lies. Lies, I tell you.
92 */
93 if (usb_urb->dev->tt->hub == root_hub)
94 return;
95
96 if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
97 chan->qh->tt_buffer_dirty = 1;
98 if (usb_hub_clear_tt_buffer(usb_urb))
99 /* Clear failed; let's hope things work anyway */
100 chan->qh->tt_buffer_dirty = 0;
101 }
102}
103
104/*
105 * Handles the start-of-frame interrupt in host mode. Non-periodic
106 * transactions may be queued to the DWC_otg controller for the current
107 * (micro)frame. Periodic transactions may be queued to the controller
108 * for the next (micro)frame.
109 */
110static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
111{
112 struct list_head *qh_entry;
113 struct dwc2_qh *qh;
114 enum dwc2_transaction_type tr_type;
115
116 /* Clear interrupt */
117 dwc2_writel(hsotg, GINTSTS_SOF, GINTSTS);
118
119#ifdef DEBUG_SOF
120 dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
121#endif
122
123 hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
124
125 dwc2_track_missed_sofs(hsotg);
126
127 /* Determine whether any periodic QHs should be executed */
128 qh_entry = hsotg->periodic_sched_inactive.next;
129 while (qh_entry != &hsotg->periodic_sched_inactive) {
130 qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
131 qh_entry = qh_entry->next;
132 if (dwc2_frame_num_le(qh->next_active_frame,
133 hsotg->frame_number)) {
134 dwc2_sch_vdbg(hsotg, "QH=%p ready fn=%04x, nxt=%04x\n",
135 qh, hsotg->frame_number,
136 qh->next_active_frame);
137
138 /*
139 * Move QH to the ready list to be executed next
140 * (micro)frame
141 */
142 list_move_tail(&qh->qh_list_entry,
143 &hsotg->periodic_sched_ready);
144 }
145 }
146 tr_type = dwc2_hcd_select_transactions(hsotg);
147 if (tr_type != DWC2_TRANSACTION_NONE)
148 dwc2_hcd_queue_transactions(hsotg, tr_type);
149}
150
151/*
152 * Handles the Rx FIFO Level Interrupt, which indicates that there is
153 * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
154 * memory if the DWC_otg controller is operating in Slave mode.
155 */
156static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
157{
158 u32 grxsts, chnum, bcnt, dpid, pktsts;
159 struct dwc2_host_chan *chan;
160
161 if (dbg_perio())
162 dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
163
164 grxsts = dwc2_readl(hsotg, GRXSTSP);
165 chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
166 chan = hsotg->hc_ptr_array[chnum];
167 if (!chan) {
168 dev_err(hsotg->dev, "Unable to get corresponding channel\n");
169 return;
170 }
171
172 bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
173 dpid = (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT;
174 pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
175
176 /* Packet Status */
177 if (dbg_perio()) {
178 dev_vdbg(hsotg->dev, " Ch num = %d\n", chnum);
179 dev_vdbg(hsotg->dev, " Count = %d\n", bcnt);
180 dev_vdbg(hsotg->dev, " DPID = %d, chan.dpid = %d\n", dpid,
181 chan->data_pid_start);
182 dev_vdbg(hsotg->dev, " PStatus = %d\n", pktsts);
183 }
184
185 switch (pktsts) {
186 case GRXSTS_PKTSTS_HCHIN:
187 /* Read the data into the host buffer */
188 if (bcnt > 0) {
189 dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);
190
191 /* Update the HC fields for the next packet received */
192 chan->xfer_count += bcnt;
193 chan->xfer_buf += bcnt;
194 }
195 break;
196 case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
197 case GRXSTS_PKTSTS_DATATOGGLEERR:
198 case GRXSTS_PKTSTS_HCHHALTED:
199 /* Handled in interrupt, just ignore data */
200 break;
201 default:
202 dev_err(hsotg->dev,
203 "RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
204 break;
205 }
206}
207
208/*
209 * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
210 * data packets may be written to the FIFO for OUT transfers. More requests
211 * may be written to the non-periodic request queue for IN transfers. This
212 * interrupt is enabled only in Slave mode.
213 */
214static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
215{
216 dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
217 dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
218}
219
220/*
221 * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
222 * packets may be written to the FIFO for OUT transfers. More requests may be
223 * written to the periodic request queue for IN transfers. This interrupt is
224 * enabled only in Slave mode.
225 */
226static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
227{
228 if (dbg_perio())
229 dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
230 dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
231}
232
233static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
234 u32 *hprt0_modify)
235{
236 struct dwc2_core_params *params = &hsotg->params;
237 int do_reset = 0;
238 u32 usbcfg;
239 u32 prtspd;
240 u32 hcfg;
241 u32 fslspclksel;
242 u32 hfir;
243
244 dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
245
246 /* Every time when port enables calculate HFIR.FrInterval */
247 hfir = dwc2_readl(hsotg, HFIR);
248 hfir &= ~HFIR_FRINT_MASK;
249 hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
250 HFIR_FRINT_MASK;
251 dwc2_writel(hsotg, hfir, HFIR);
252
253 /* Check if we need to adjust the PHY clock speed for low power */
254 if (!params->host_support_fs_ls_low_power) {
255 /* Port has been enabled, set the reset change flag */
256 hsotg->flags.b.port_reset_change = 1;
257 return;
258 }
259
260 usbcfg = dwc2_readl(hsotg, GUSBCFG);
261 prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
262
263 if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
264 /* Low power */
265 if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
266 /* Set PHY low power clock select for FS/LS devices */
267 usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
268 dwc2_writel(hsotg, usbcfg, GUSBCFG);
269 do_reset = 1;
270 }
271
272 hcfg = dwc2_readl(hsotg, HCFG);
273 fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
274 HCFG_FSLSPCLKSEL_SHIFT;
275
276 if (prtspd == HPRT0_SPD_LOW_SPEED &&
277 params->host_ls_low_power_phy_clk) {
278 /* 6 MHZ */
279 dev_vdbg(hsotg->dev,
280 "FS_PHY programming HCFG to 6 MHz\n");
281 if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
282 fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
283 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
284 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
285 dwc2_writel(hsotg, hcfg, HCFG);
286 do_reset = 1;
287 }
288 } else {
289 /* 48 MHZ */
290 dev_vdbg(hsotg->dev,
291 "FS_PHY programming HCFG to 48 MHz\n");
292 if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
293 fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
294 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
295 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
296 dwc2_writel(hsotg, hcfg, HCFG);
297 do_reset = 1;
298 }
299 }
300 } else {
301 /* Not low power */
302 if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
303 usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
304 dwc2_writel(hsotg, usbcfg, GUSBCFG);
305 do_reset = 1;
306 }
307 }
308
309 if (do_reset) {
310 *hprt0_modify |= HPRT0_RST;
311 dwc2_writel(hsotg, *hprt0_modify, HPRT0);
312 queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
313 msecs_to_jiffies(60));
314 } else {
315 /* Port has been enabled, set the reset change flag */
316 hsotg->flags.b.port_reset_change = 1;
317 }
318}
319
320/*
321 * There are multiple conditions that can cause a port interrupt. This function
322 * determines which interrupt conditions have occurred and handles them
323 * appropriately.
324 */
325static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
326{
327 u32 hprt0;
328 u32 hprt0_modify;
329
330 dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
331
332 hprt0 = dwc2_readl(hsotg, HPRT0);
333 hprt0_modify = hprt0;
334
335 /*
336 * Clear appropriate bits in HPRT0 to clear the interrupt bit in
337 * GINTSTS
338 */
339 hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
340 HPRT0_OVRCURRCHG);
341
342 /*
343 * Port Connect Detected
344 * Set flag and clear if detected
345 */
346 if (hprt0 & HPRT0_CONNDET) {
347 dwc2_writel(hsotg, hprt0_modify | HPRT0_CONNDET, HPRT0);
348
349 dev_vdbg(hsotg->dev,
350 "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
351 hprt0);
352 dwc2_hcd_connect(hsotg);
353
354 /*
355 * The Hub driver asserts a reset when it sees port connect
356 * status change flag
357 */
358 }
359
360 /*
361 * Port Enable Changed
362 * Clear if detected - Set internal flag if disabled
363 */
364 if (hprt0 & HPRT0_ENACHG) {
365 dwc2_writel(hsotg, hprt0_modify | HPRT0_ENACHG, HPRT0);
366 dev_vdbg(hsotg->dev,
367 " --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
368 hprt0, !!(hprt0 & HPRT0_ENA));
369 if (hprt0 & HPRT0_ENA) {
370 hsotg->new_connection = true;
371 dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
372 } else {
373 hsotg->flags.b.port_enable_change = 1;
374 if (hsotg->params.dma_desc_fs_enable) {
375 u32 hcfg;
376
377 hsotg->params.dma_desc_enable = false;
378 hsotg->new_connection = false;
379 hcfg = dwc2_readl(hsotg, HCFG);
380 hcfg &= ~HCFG_DESCDMA;
381 dwc2_writel(hsotg, hcfg, HCFG);
382 }
383 }
384 }
385
386 /* Overcurrent Change Interrupt */
387 if (hprt0 & HPRT0_OVRCURRCHG) {
388 dwc2_writel(hsotg, hprt0_modify | HPRT0_OVRCURRCHG,
389 HPRT0);
390 dev_vdbg(hsotg->dev,
391 " --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
392 hprt0);
393 hsotg->flags.b.port_over_current_change = 1;
394 }
395}
396
397/*
398 * Gets the actual length of a transfer after the transfer halts. halt_status
399 * holds the reason for the halt.
400 *
401 * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
402 * is set to 1 upon return if less than the requested number of bytes were
403 * transferred. short_read may also be NULL on entry, in which case it remains
404 * unchanged.
405 */
406static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
407 struct dwc2_host_chan *chan, int chnum,
408 struct dwc2_qtd *qtd,
409 enum dwc2_halt_status halt_status,
410 int *short_read)
411{
412 u32 hctsiz, count, length;
413
414 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
415
416 if (halt_status == DWC2_HC_XFER_COMPLETE) {
417 if (chan->ep_is_in) {
418 count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
419 TSIZ_XFERSIZE_SHIFT;
420 length = chan->xfer_len - count;
421 if (short_read)
422 *short_read = (count != 0);
423 } else if (chan->qh->do_split) {
424 length = qtd->ssplit_out_xfer_count;
425 } else {
426 length = chan->xfer_len;
427 }
428 } else {
429 /*
430 * Must use the hctsiz.pktcnt field to determine how much data
431 * has been transferred. This field reflects the number of
432 * packets that have been transferred via the USB. This is
433 * always an integral number of packets if the transfer was
434 * halted before its normal completion. (Can't use the
435 * hctsiz.xfersize field because that reflects the number of
436 * bytes transferred via the AHB, not the USB).
437 */
438 count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
439 length = (chan->start_pkt_count - count) * chan->max_packet;
440 }
441
442 return length;
443}
444
445/**
446 * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
447 * Complete interrupt on the host channel. Updates the actual_length field
448 * of the URB based on the number of bytes transferred via the host channel.
449 * Sets the URB status if the data transfer is finished.
450 *
451 * @hsotg: Programming view of the DWC_otg controller
452 * @chan: Programming view of host channel
453 * @chnum: Channel number
454 * @urb: Processing URB
455 * @qtd: Queue transfer descriptor
456 *
457 * Return: 1 if the data transfer specified by the URB is completely finished,
458 * 0 otherwise
459 */
460static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
461 struct dwc2_host_chan *chan, int chnum,
462 struct dwc2_hcd_urb *urb,
463 struct dwc2_qtd *qtd)
464{
465 u32 hctsiz;
466 int xfer_done = 0;
467 int short_read = 0;
468 int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
469 DWC2_HC_XFER_COMPLETE,
470 &short_read);
471
472 if (urb->actual_length + xfer_length > urb->length) {
473 dev_dbg(hsotg->dev, "%s(): trimming xfer length\n", __func__);
474 xfer_length = urb->length - urb->actual_length;
475 }
476
477 dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
478 urb->actual_length, xfer_length);
479 urb->actual_length += xfer_length;
480
481 if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
482 (urb->flags & URB_SEND_ZERO_PACKET) &&
483 urb->actual_length >= urb->length &&
484 !(urb->length % chan->max_packet)) {
485 xfer_done = 0;
486 } else if (short_read || urb->actual_length >= urb->length) {
487 xfer_done = 1;
488 urb->status = 0;
489 }
490
491 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
492 dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
493 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
494 dev_vdbg(hsotg->dev, " chan->xfer_len %d\n", chan->xfer_len);
495 dev_vdbg(hsotg->dev, " hctsiz.xfersize %d\n",
496 (hctsiz & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
497 dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n", urb->length);
498 dev_vdbg(hsotg->dev, " urb->actual_length %d\n", urb->actual_length);
499 dev_vdbg(hsotg->dev, " short_read %d, xfer_done %d\n", short_read,
500 xfer_done);
501
502 return xfer_done;
503}
504
505/*
506 * Save the starting data toggle for the next transfer. The data toggle is
507 * saved in the QH for non-control transfers and it's saved in the QTD for
508 * control transfers.
509 */
510void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
511 struct dwc2_host_chan *chan, int chnum,
512 struct dwc2_qtd *qtd)
513{
514 u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
515 u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
516
517 if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
518 if (WARN(!chan || !chan->qh,
519 "chan->qh must be specified for non-control eps\n"))
520 return;
521
522 if (pid == TSIZ_SC_MC_PID_DATA0)
523 chan->qh->data_toggle = DWC2_HC_PID_DATA0;
524 else
525 chan->qh->data_toggle = DWC2_HC_PID_DATA1;
526 } else {
527 if (WARN(!qtd,
528 "qtd must be specified for control eps\n"))
529 return;
530
531 if (pid == TSIZ_SC_MC_PID_DATA0)
532 qtd->data_toggle = DWC2_HC_PID_DATA0;
533 else
534 qtd->data_toggle = DWC2_HC_PID_DATA1;
535 }
536}
537
538/**
539 * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
540 * the transfer is stopped for any reason. The fields of the current entry in
541 * the frame descriptor array are set based on the transfer state and the input
542 * halt_status. Completes the Isochronous URB if all the URB frames have been
543 * completed.
544 *
545 * @hsotg: Programming view of the DWC_otg controller
546 * @chan: Programming view of host channel
547 * @chnum: Channel number
548 * @halt_status: Reason for halting a host channel
549 * @qtd: Queue transfer descriptor
550 *
551 * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
552 * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
553 */
554static enum dwc2_halt_status dwc2_update_isoc_urb_state(
555 struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
556 int chnum, struct dwc2_qtd *qtd,
557 enum dwc2_halt_status halt_status)
558{
559 struct dwc2_hcd_iso_packet_desc *frame_desc;
560 struct dwc2_hcd_urb *urb = qtd->urb;
561
562 if (!urb)
563 return DWC2_HC_XFER_NO_HALT_STATUS;
564
565 frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
566
567 switch (halt_status) {
568 case DWC2_HC_XFER_COMPLETE:
569 frame_desc->status = 0;
570 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
571 chan, chnum, qtd, halt_status, NULL);
572 break;
573 case DWC2_HC_XFER_FRAME_OVERRUN:
574 urb->error_count++;
575 if (chan->ep_is_in)
576 frame_desc->status = -ENOSR;
577 else
578 frame_desc->status = -ECOMM;
579 frame_desc->actual_length = 0;
580 break;
581 case DWC2_HC_XFER_BABBLE_ERR:
582 urb->error_count++;
583 frame_desc->status = -EOVERFLOW;
584 /* Don't need to update actual_length in this case */
585 break;
586 case DWC2_HC_XFER_XACT_ERR:
587 urb->error_count++;
588 frame_desc->status = -EPROTO;
589 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
590 chan, chnum, qtd, halt_status, NULL);
591
592 /* Skip whole frame */
593 if (chan->qh->do_split &&
594 chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
595 hsotg->params.host_dma) {
596 qtd->complete_split = 0;
597 qtd->isoc_split_offset = 0;
598 }
599
600 break;
601 default:
602 dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
603 halt_status);
604 break;
605 }
606
607 if (++qtd->isoc_frame_index == urb->packet_count) {
608 /*
609 * urb->status is not used for isoc transfers. The individual
610 * frame_desc statuses are used instead.
611 */
612 dwc2_host_complete(hsotg, qtd, 0);
613 halt_status = DWC2_HC_XFER_URB_COMPLETE;
614 } else {
615 halt_status = DWC2_HC_XFER_COMPLETE;
616 }
617
618 return halt_status;
619}
620
621/*
622 * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
623 * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
624 * still linked to the QH, the QH is added to the end of the inactive
625 * non-periodic schedule. For periodic QHs, removes the QH from the periodic
626 * schedule if no more QTDs are linked to the QH.
627 */
628static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
629 int free_qtd)
630{
631 int continue_split = 0;
632 struct dwc2_qtd *qtd;
633
634 if (dbg_qh(qh))
635 dev_vdbg(hsotg->dev, " %s(%p,%p,%d)\n", __func__,
636 hsotg, qh, free_qtd);
637
638 if (list_empty(&qh->qtd_list)) {
639 dev_dbg(hsotg->dev, "## QTD list empty ##\n");
640 goto no_qtd;
641 }
642
643 qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
644
645 if (qtd->complete_split)
646 continue_split = 1;
647 else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
648 qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
649 continue_split = 1;
650
651 if (free_qtd) {
652 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
653 continue_split = 0;
654 }
655
656no_qtd:
657 qh->channel = NULL;
658 dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
659}
660
661/**
662 * dwc2_release_channel() - Releases a host channel for use by other transfers
663 *
664 * @hsotg: The HCD state structure
665 * @chan: The host channel to release
666 * @qtd: The QTD associated with the host channel. This QTD may be
667 * freed if the transfer is complete or an error has occurred.
668 * @halt_status: Reason the channel is being released. This status
669 * determines the actions taken by this function.
670 *
671 * Also attempts to select and queue more transactions since at least one host
672 * channel is available.
673 */
674static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
675 struct dwc2_host_chan *chan,
676 struct dwc2_qtd *qtd,
677 enum dwc2_halt_status halt_status)
678{
679 enum dwc2_transaction_type tr_type;
680 u32 haintmsk;
681 int free_qtd = 0;
682
683 if (dbg_hc(chan))
684 dev_vdbg(hsotg->dev, " %s: channel %d, halt_status %d\n",
685 __func__, chan->hc_num, halt_status);
686
687 switch (halt_status) {
688 case DWC2_HC_XFER_URB_COMPLETE:
689 free_qtd = 1;
690 break;
691 case DWC2_HC_XFER_AHB_ERR:
692 case DWC2_HC_XFER_STALL:
693 case DWC2_HC_XFER_BABBLE_ERR:
694 free_qtd = 1;
695 break;
696 case DWC2_HC_XFER_XACT_ERR:
697 if (qtd && qtd->error_count >= 3) {
698 dev_vdbg(hsotg->dev,
699 " Complete URB with transaction error\n");
700 free_qtd = 1;
701 dwc2_host_complete(hsotg, qtd, -EPROTO);
702 }
703 break;
704 case DWC2_HC_XFER_URB_DEQUEUE:
705 /*
706 * The QTD has already been removed and the QH has been
707 * deactivated. Don't want to do anything except release the
708 * host channel and try to queue more transfers.
709 */
710 goto cleanup;
711 case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
712 dev_vdbg(hsotg->dev, " Complete URB with I/O error\n");
713 free_qtd = 1;
714 dwc2_host_complete(hsotg, qtd, -EIO);
715 break;
716 case DWC2_HC_XFER_NO_HALT_STATUS:
717 default:
718 break;
719 }
720
721 dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);
722
723cleanup:
724 /*
725 * Release the host channel for use by other transfers. The cleanup
726 * function clears the channel interrupt enables and conditions, so
727 * there's no need to clear the Channel Halted interrupt separately.
728 */
729 if (!list_empty(&chan->hc_list_entry))
730 list_del(&chan->hc_list_entry);
731 dwc2_hc_cleanup(hsotg, chan);
732 list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
733
734 if (hsotg->params.uframe_sched) {
735 hsotg->available_host_channels++;
736 } else {
737 switch (chan->ep_type) {
738 case USB_ENDPOINT_XFER_CONTROL:
739 case USB_ENDPOINT_XFER_BULK:
740 hsotg->non_periodic_channels--;
741 break;
742 default:
743 /*
744 * Don't release reservations for periodic channels
745 * here. That's done when a periodic transfer is
746 * descheduled (i.e. when the QH is removed from the
747 * periodic schedule).
748 */
749 break;
750 }
751 }
752
753 haintmsk = dwc2_readl(hsotg, HAINTMSK);
754 haintmsk &= ~(1 << chan->hc_num);
755 dwc2_writel(hsotg, haintmsk, HAINTMSK);
756
757 /* Try to queue more transfers now that there's a free channel */
758 tr_type = dwc2_hcd_select_transactions(hsotg);
759 if (tr_type != DWC2_TRANSACTION_NONE)
760 dwc2_hcd_queue_transactions(hsotg, tr_type);
761}
762
763/*
764 * Halts a host channel. If the channel cannot be halted immediately because
765 * the request queue is full, this function ensures that the FIFO empty
766 * interrupt for the appropriate queue is enabled so that the halt request can
767 * be queued when there is space in the request queue.
768 *
769 * This function may also be called in DMA mode. In that case, the channel is
770 * simply released since the core always halts the channel automatically in
771 * DMA mode.
772 */
773static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
774 struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
775 enum dwc2_halt_status halt_status)
776{
777 if (dbg_hc(chan))
778 dev_vdbg(hsotg->dev, "%s()\n", __func__);
779
780 if (hsotg->params.host_dma) {
781 if (dbg_hc(chan))
782 dev_vdbg(hsotg->dev, "DMA enabled\n");
783 dwc2_release_channel(hsotg, chan, qtd, halt_status);
784 return;
785 }
786
787 /* Slave mode processing */
788 dwc2_hc_halt(hsotg, chan, halt_status);
789
790 if (chan->halt_on_queue) {
791 u32 gintmsk;
792
793 dev_vdbg(hsotg->dev, "Halt on queue\n");
794 if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
795 chan->ep_type == USB_ENDPOINT_XFER_BULK) {
796 dev_vdbg(hsotg->dev, "control/bulk\n");
797 /*
798 * Make sure the Non-periodic Tx FIFO empty interrupt
799 * is enabled so that the non-periodic schedule will
800 * be processed
801 */
802 gintmsk = dwc2_readl(hsotg, GINTMSK);
803 gintmsk |= GINTSTS_NPTXFEMP;
804 dwc2_writel(hsotg, gintmsk, GINTMSK);
805 } else {
806 dev_vdbg(hsotg->dev, "isoc/intr\n");
807 /*
808 * Move the QH from the periodic queued schedule to
809 * the periodic assigned schedule. This allows the
810 * halt to be queued when the periodic schedule is
811 * processed.
812 */
813 list_move_tail(&chan->qh->qh_list_entry,
814 &hsotg->periodic_sched_assigned);
815
816 /*
817 * Make sure the Periodic Tx FIFO Empty interrupt is
818 * enabled so that the periodic schedule will be
819 * processed
820 */
821 gintmsk = dwc2_readl(hsotg, GINTMSK);
822 gintmsk |= GINTSTS_PTXFEMP;
823 dwc2_writel(hsotg, gintmsk, GINTMSK);
824 }
825 }
826}
827
828/*
829 * Performs common cleanup for non-periodic transfers after a Transfer
830 * Complete interrupt. This function should be called after any endpoint type
831 * specific handling is finished to release the host channel.
832 */
833static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
834 struct dwc2_host_chan *chan,
835 int chnum, struct dwc2_qtd *qtd,
836 enum dwc2_halt_status halt_status)
837{
838 dev_vdbg(hsotg->dev, "%s()\n", __func__);
839
840 qtd->error_count = 0;
841
842 if (chan->hcint & HCINTMSK_NYET) {
843 /*
844 * Got a NYET on the last transaction of the transfer. This
845 * means that the endpoint should be in the PING state at the
846 * beginning of the next transfer.
847 */
848 dev_vdbg(hsotg->dev, "got NYET\n");
849 chan->qh->ping_state = 1;
850 }
851
852 /*
853 * Always halt and release the host channel to make it available for
854 * more transfers. There may still be more phases for a control
855 * transfer or more data packets for a bulk transfer at this point,
856 * but the host channel is still halted. A channel will be reassigned
857 * to the transfer when the non-periodic schedule is processed after
858 * the channel is released. This allows transactions to be queued
859 * properly via dwc2_hcd_queue_transactions, which also enables the
860 * Tx FIFO Empty interrupt if necessary.
861 */
862 if (chan->ep_is_in) {
863 /*
864 * IN transfers in Slave mode require an explicit disable to
865 * halt the channel. (In DMA mode, this call simply releases
866 * the channel.)
867 */
868 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
869 } else {
870 /*
871 * The channel is automatically disabled by the core for OUT
872 * transfers in Slave mode
873 */
874 dwc2_release_channel(hsotg, chan, qtd, halt_status);
875 }
876}
877
878/*
879 * Performs common cleanup for periodic transfers after a Transfer Complete
880 * interrupt. This function should be called after any endpoint type specific
881 * handling is finished to release the host channel.
882 */
883static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
884 struct dwc2_host_chan *chan, int chnum,
885 struct dwc2_qtd *qtd,
886 enum dwc2_halt_status halt_status)
887{
888 u32 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
889
890 qtd->error_count = 0;
891
892 if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
893 /* Core halts channel in these cases */
894 dwc2_release_channel(hsotg, chan, qtd, halt_status);
895 else
896 /* Flush any outstanding requests from the Tx queue */
897 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
898}
899
900static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
901 struct dwc2_host_chan *chan, int chnum,
902 struct dwc2_qtd *qtd)
903{
904 struct dwc2_hcd_iso_packet_desc *frame_desc;
905 u32 len;
906 u32 hctsiz;
907 u32 pid;
908
909 if (!qtd->urb)
910 return 0;
911
912 frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
913 len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
914 DWC2_HC_XFER_COMPLETE, NULL);
915 if (!len && !qtd->isoc_split_offset) {
916 qtd->complete_split = 0;
917 return 0;
918 }
919
920 frame_desc->actual_length += len;
921
922 if (chan->align_buf) {
923 dev_vdbg(hsotg->dev, "non-aligned buffer\n");
924 dma_unmap_single(hsotg->dev, chan->qh->dw_align_buf_dma,
925 DWC2_KMEM_UNALIGNED_BUF_SIZE, DMA_FROM_DEVICE);
926 memcpy(qtd->urb->buf + (chan->xfer_dma - qtd->urb->dma),
927 chan->qh->dw_align_buf, len);
928 }
929
930 qtd->isoc_split_offset += len;
931
932 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
933 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
934
935 if (frame_desc->actual_length >= frame_desc->length || pid == 0) {
936 frame_desc->status = 0;
937 qtd->isoc_frame_index++;
938 qtd->complete_split = 0;
939 qtd->isoc_split_offset = 0;
940 }
941
942 if (qtd->isoc_frame_index == qtd->urb->packet_count) {
943 dwc2_host_complete(hsotg, qtd, 0);
944 dwc2_release_channel(hsotg, chan, qtd,
945 DWC2_HC_XFER_URB_COMPLETE);
946 } else {
947 dwc2_release_channel(hsotg, chan, qtd,
948 DWC2_HC_XFER_NO_HALT_STATUS);
949 }
950
951 return 1; /* Indicates that channel released */
952}
953
954/*
955 * Handles a host channel Transfer Complete interrupt. This handler may be
956 * called in either DMA mode or Slave mode.
957 */
958static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
959 struct dwc2_host_chan *chan, int chnum,
960 struct dwc2_qtd *qtd)
961{
962 struct dwc2_hcd_urb *urb = qtd->urb;
963 enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
964 int pipe_type;
965 int urb_xfer_done;
966
967 if (dbg_hc(chan))
968 dev_vdbg(hsotg->dev,
969 "--Host Channel %d Interrupt: Transfer Complete--\n",
970 chnum);
971
972 if (!urb)
973 goto handle_xfercomp_done;
974
975 pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
976
977 if (hsotg->params.dma_desc_enable) {
978 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
979 if (pipe_type == USB_ENDPOINT_XFER_ISOC)
980 /* Do not disable the interrupt, just clear it */
981 return;
982 goto handle_xfercomp_done;
983 }
984
985 /* Handle xfer complete on CSPLIT */
986 if (chan->qh->do_split) {
987 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
988 hsotg->params.host_dma) {
989 if (qtd->complete_split &&
990 dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
991 qtd))
992 goto handle_xfercomp_done;
993 } else {
994 qtd->complete_split = 0;
995 }
996 }
997
998 /* Update the QTD and URB states */
999 switch (pipe_type) {
1000 case USB_ENDPOINT_XFER_CONTROL:
1001 switch (qtd->control_phase) {
1002 case DWC2_CONTROL_SETUP:
1003 if (urb->length > 0)
1004 qtd->control_phase = DWC2_CONTROL_DATA;
1005 else
1006 qtd->control_phase = DWC2_CONTROL_STATUS;
1007 dev_vdbg(hsotg->dev,
1008 " Control setup transaction done\n");
1009 halt_status = DWC2_HC_XFER_COMPLETE;
1010 break;
1011 case DWC2_CONTROL_DATA:
1012 urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1013 chnum, urb, qtd);
1014 if (urb_xfer_done) {
1015 qtd->control_phase = DWC2_CONTROL_STATUS;
1016 dev_vdbg(hsotg->dev,
1017 " Control data transfer done\n");
1018 } else {
1019 dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1020 qtd);
1021 }
1022 halt_status = DWC2_HC_XFER_COMPLETE;
1023 break;
1024 case DWC2_CONTROL_STATUS:
1025 dev_vdbg(hsotg->dev, " Control transfer complete\n");
1026 if (urb->status == -EINPROGRESS)
1027 urb->status = 0;
1028 dwc2_host_complete(hsotg, qtd, urb->status);
1029 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1030 break;
1031 }
1032
1033 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1034 halt_status);
1035 break;
1036 case USB_ENDPOINT_XFER_BULK:
1037 dev_vdbg(hsotg->dev, " Bulk transfer complete\n");
1038 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1039 qtd);
1040 if (urb_xfer_done) {
1041 dwc2_host_complete(hsotg, qtd, urb->status);
1042 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1043 } else {
1044 halt_status = DWC2_HC_XFER_COMPLETE;
1045 }
1046
1047 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1048 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1049 halt_status);
1050 break;
1051 case USB_ENDPOINT_XFER_INT:
1052 dev_vdbg(hsotg->dev, " Interrupt transfer complete\n");
1053 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1054 qtd);
1055
1056 /*
1057 * Interrupt URB is done on the first transfer complete
1058 * interrupt
1059 */
1060 if (urb_xfer_done) {
1061 dwc2_host_complete(hsotg, qtd, urb->status);
1062 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1063 } else {
1064 halt_status = DWC2_HC_XFER_COMPLETE;
1065 }
1066
1067 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1068 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1069 halt_status);
1070 break;
1071 case USB_ENDPOINT_XFER_ISOC:
1072 if (dbg_perio())
1073 dev_vdbg(hsotg->dev, " Isochronous transfer complete\n");
1074 if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1075 halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1076 chnum, qtd,
1077 DWC2_HC_XFER_COMPLETE);
1078 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1079 halt_status);
1080 break;
1081 }
1082
1083handle_xfercomp_done:
1084 disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1085}
1086
1087/*
1088 * Handles a host channel STALL interrupt. This handler may be called in
1089 * either DMA mode or Slave mode.
1090 */
1091static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1092 struct dwc2_host_chan *chan, int chnum,
1093 struct dwc2_qtd *qtd)
1094{
1095 struct dwc2_hcd_urb *urb = qtd->urb;
1096 int pipe_type;
1097
1098 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1099 chnum);
1100
1101 if (hsotg->params.dma_desc_enable) {
1102 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1103 DWC2_HC_XFER_STALL);
1104 goto handle_stall_done;
1105 }
1106
1107 if (!urb)
1108 goto handle_stall_halt;
1109
1110 pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1111
1112 if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1113 dwc2_host_complete(hsotg, qtd, -EPIPE);
1114
1115 if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1116 pipe_type == USB_ENDPOINT_XFER_INT) {
1117 dwc2_host_complete(hsotg, qtd, -EPIPE);
1118 /*
1119 * USB protocol requires resetting the data toggle for bulk
1120 * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1121 * setup command is issued to the endpoint. Anticipate the
1122 * CLEAR_FEATURE command since a STALL has occurred and reset
1123 * the data toggle now.
1124 */
1125 chan->qh->data_toggle = 0;
1126 }
1127
1128handle_stall_halt:
1129 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);
1130
1131handle_stall_done:
1132 disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1133}
1134
1135/*
1136 * Updates the state of the URB when a transfer has been stopped due to an
1137 * abnormal condition before the transfer completes. Modifies the
1138 * actual_length field of the URB to reflect the number of bytes that have
1139 * actually been transferred via the host channel.
1140 */
1141static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1142 struct dwc2_host_chan *chan, int chnum,
1143 struct dwc2_hcd_urb *urb,
1144 struct dwc2_qtd *qtd,
1145 enum dwc2_halt_status halt_status)
1146{
1147 u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1148 qtd, halt_status, NULL);
1149 u32 hctsiz;
1150
1151 if (urb->actual_length + xfer_length > urb->length) {
1152 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1153 xfer_length = urb->length - urb->actual_length;
1154 }
1155
1156 urb->actual_length += xfer_length;
1157
1158 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1159 dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1160 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1161 dev_vdbg(hsotg->dev, " chan->start_pkt_count %d\n",
1162 chan->start_pkt_count);
1163 dev_vdbg(hsotg->dev, " hctsiz.pktcnt %d\n",
1164 (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1165 dev_vdbg(hsotg->dev, " chan->max_packet %d\n", chan->max_packet);
1166 dev_vdbg(hsotg->dev, " bytes_transferred %d\n",
1167 xfer_length);
1168 dev_vdbg(hsotg->dev, " urb->actual_length %d\n",
1169 urb->actual_length);
1170 dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n",
1171 urb->length);
1172}
1173
1174/*
1175 * Handles a host channel NAK interrupt. This handler may be called in either
1176 * DMA mode or Slave mode.
1177 */
1178static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1179 struct dwc2_host_chan *chan, int chnum,
1180 struct dwc2_qtd *qtd)
1181{
1182 if (!qtd) {
1183 dev_dbg(hsotg->dev, "%s: qtd is NULL\n", __func__);
1184 return;
1185 }
1186
1187 if (!qtd->urb) {
1188 dev_dbg(hsotg->dev, "%s: qtd->urb is NULL\n", __func__);
1189 return;
1190 }
1191
1192 if (dbg_hc(chan))
1193 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1194 chnum);
1195
1196 /*
1197 * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1198 * interrupt. Re-start the SSPLIT transfer.
1199 *
1200 * Normally for non-periodic transfers we'll retry right away, but to
1201 * avoid interrupt storms we'll wait before retrying if we've got
1202 * several NAKs. If we didn't do this we'd retry directly from the
1203 * interrupt handler and could end up quickly getting another
1204 * interrupt (another NAK), which we'd retry. Note that we do not
1205 * delay retries for IN parts of control requests, as those are expected
1206 * to complete fairly quickly, and if we delay them we risk confusing
1207 * the device and cause it issue STALL.
1208 *
1209 * Note that in DMA mode software only gets involved to re-send NAKed
1210 * transfers for split transactions, so we only need to apply this
1211 * delaying logic when handling splits. In non-DMA mode presumably we
1212 * might want a similar delay if someone can demonstrate this problem
1213 * affects that code path too.
1214 */
1215 if (chan->do_split) {
1216 if (chan->complete_split)
1217 qtd->error_count = 0;
1218 qtd->complete_split = 0;
1219 qtd->num_naks++;
1220 qtd->qh->want_wait = qtd->num_naks >= DWC2_NAKS_BEFORE_DELAY &&
1221 !(chan->ep_type == USB_ENDPOINT_XFER_CONTROL &&
1222 chan->ep_is_in);
1223 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1224 goto handle_nak_done;
1225 }
1226
1227 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1228 case USB_ENDPOINT_XFER_CONTROL:
1229 case USB_ENDPOINT_XFER_BULK:
1230 if (hsotg->params.host_dma && chan->ep_is_in) {
1231 /*
1232 * NAK interrupts are enabled on bulk/control IN
1233 * transfers in DMA mode for the sole purpose of
1234 * resetting the error count after a transaction error
1235 * occurs. The core will continue transferring data.
1236 */
1237 qtd->error_count = 0;
1238 break;
1239 }
1240
1241 /*
1242 * NAK interrupts normally occur during OUT transfers in DMA
1243 * or Slave mode. For IN transfers, more requests will be
1244 * queued as request queue space is available.
1245 */
1246 qtd->error_count = 0;
1247
1248 if (!chan->qh->ping_state) {
1249 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1250 qtd, DWC2_HC_XFER_NAK);
1251 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1252
1253 if (chan->speed == USB_SPEED_HIGH)
1254 chan->qh->ping_state = 1;
1255 }
1256
1257 /*
1258 * Halt the channel so the transfer can be re-started from
1259 * the appropriate point or the PING protocol will
1260 * start/continue
1261 */
1262 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1263 break;
1264 case USB_ENDPOINT_XFER_INT:
1265 qtd->error_count = 0;
1266 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1267 break;
1268 case USB_ENDPOINT_XFER_ISOC:
1269 /* Should never get called for isochronous transfers */
1270 dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1271 break;
1272 }
1273
1274handle_nak_done:
1275 disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1276}
1277
1278/*
1279 * Handles a host channel ACK interrupt. This interrupt is enabled when
1280 * performing the PING protocol in Slave mode, when errors occur during
1281 * either Slave mode or DMA mode, and during Start Split transactions.
1282 */
1283static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1284 struct dwc2_host_chan *chan, int chnum,
1285 struct dwc2_qtd *qtd)
1286{
1287 struct dwc2_hcd_iso_packet_desc *frame_desc;
1288
1289 if (dbg_hc(chan))
1290 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1291 chnum);
1292
1293 if (chan->do_split) {
1294 /* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1295 if (!chan->ep_is_in &&
1296 chan->data_pid_start != DWC2_HC_PID_SETUP)
1297 qtd->ssplit_out_xfer_count = chan->xfer_len;
1298
1299 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1300 qtd->complete_split = 1;
1301 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1302 } else {
1303 /* ISOC OUT */
1304 switch (chan->xact_pos) {
1305 case DWC2_HCSPLT_XACTPOS_ALL:
1306 break;
1307 case DWC2_HCSPLT_XACTPOS_END:
1308 qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1309 qtd->isoc_split_offset = 0;
1310 break;
1311 case DWC2_HCSPLT_XACTPOS_BEGIN:
1312 case DWC2_HCSPLT_XACTPOS_MID:
1313 /*
1314 * For BEGIN or MID, calculate the length for
1315 * the next microframe to determine the correct
1316 * SSPLIT token, either MID or END
1317 */
1318 frame_desc = &qtd->urb->iso_descs[
1319 qtd->isoc_frame_index];
1320 qtd->isoc_split_offset += 188;
1321
1322 if (frame_desc->length - qtd->isoc_split_offset
1323 <= 188)
1324 qtd->isoc_split_pos =
1325 DWC2_HCSPLT_XACTPOS_END;
1326 else
1327 qtd->isoc_split_pos =
1328 DWC2_HCSPLT_XACTPOS_MID;
1329 break;
1330 }
1331 }
1332 } else {
1333 qtd->error_count = 0;
1334
1335 if (chan->qh->ping_state) {
1336 chan->qh->ping_state = 0;
1337 /*
1338 * Halt the channel so the transfer can be re-started
1339 * from the appropriate point. This only happens in
1340 * Slave mode. In DMA mode, the ping_state is cleared
1341 * when the transfer is started because the core
1342 * automatically executes the PING, then the transfer.
1343 */
1344 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1345 }
1346 }
1347
1348 /*
1349 * If the ACK occurred when _not_ in the PING state, let the channel
1350 * continue transferring data after clearing the error count
1351 */
1352 disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1353}
1354
1355/*
1356 * Handles a host channel NYET interrupt. This interrupt should only occur on
1357 * Bulk and Control OUT endpoints and for complete split transactions. If a
1358 * NYET occurs at the same time as a Transfer Complete interrupt, it is
1359 * handled in the xfercomp interrupt handler, not here. This handler may be
1360 * called in either DMA mode or Slave mode.
1361 */
1362static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1363 struct dwc2_host_chan *chan, int chnum,
1364 struct dwc2_qtd *qtd)
1365{
1366 if (dbg_hc(chan))
1367 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1368 chnum);
1369
1370 /*
1371 * NYET on CSPLIT
1372 * re-do the CSPLIT immediately on non-periodic
1373 */
1374 if (chan->do_split && chan->complete_split) {
1375 if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1376 hsotg->params.host_dma) {
1377 qtd->complete_split = 0;
1378 qtd->isoc_split_offset = 0;
1379 qtd->isoc_frame_index++;
1380 if (qtd->urb &&
1381 qtd->isoc_frame_index == qtd->urb->packet_count) {
1382 dwc2_host_complete(hsotg, qtd, 0);
1383 dwc2_release_channel(hsotg, chan, qtd,
1384 DWC2_HC_XFER_URB_COMPLETE);
1385 } else {
1386 dwc2_release_channel(hsotg, chan, qtd,
1387 DWC2_HC_XFER_NO_HALT_STATUS);
1388 }
1389 goto handle_nyet_done;
1390 }
1391
1392 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1393 chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1394 struct dwc2_qh *qh = chan->qh;
1395 bool past_end;
1396
1397 if (!hsotg->params.uframe_sched) {
1398 int frnum = dwc2_hcd_get_frame_number(hsotg);
1399
1400 /* Don't have num_hs_transfers; simple logic */
1401 past_end = dwc2_full_frame_num(frnum) !=
1402 dwc2_full_frame_num(qh->next_active_frame);
1403 } else {
1404 int end_frnum;
1405
1406 /*
1407 * Figure out the end frame based on
1408 * schedule.
1409 *
1410 * We don't want to go on trying again
1411 * and again forever. Let's stop when
1412 * we've done all the transfers that
1413 * were scheduled.
1414 *
1415 * We're going to be comparing
1416 * start_active_frame and
1417 * next_active_frame, both of which
1418 * are 1 before the time the packet
1419 * goes on the wire, so that cancels
1420 * out. Basically if had 1 transfer
1421 * and we saw 1 NYET then we're done.
1422 * We're getting a NYET here so if
1423 * next >= (start + num_transfers)
1424 * we're done. The complexity is that
1425 * for all but ISOC_OUT we skip one
1426 * slot.
1427 */
1428 end_frnum = dwc2_frame_num_inc(
1429 qh->start_active_frame,
1430 qh->num_hs_transfers);
1431
1432 if (qh->ep_type != USB_ENDPOINT_XFER_ISOC ||
1433 qh->ep_is_in)
1434 end_frnum =
1435 dwc2_frame_num_inc(end_frnum, 1);
1436
1437 past_end = dwc2_frame_num_le(
1438 end_frnum, qh->next_active_frame);
1439 }
1440
1441 if (past_end) {
1442 /* Treat this as a transaction error. */
1443#if 0
1444 /*
1445 * Todo: Fix system performance so this can
1446 * be treated as an error. Right now complete
1447 * splits cannot be scheduled precisely enough
1448 * due to other system activity, so this error
1449 * occurs regularly in Slave mode.
1450 */
1451 qtd->error_count++;
1452#endif
1453 qtd->complete_split = 0;
1454 dwc2_halt_channel(hsotg, chan, qtd,
1455 DWC2_HC_XFER_XACT_ERR);
1456 /* Todo: add support for isoc release */
1457 goto handle_nyet_done;
1458 }
1459 }
1460
1461 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1462 goto handle_nyet_done;
1463 }
1464
1465 chan->qh->ping_state = 1;
1466 qtd->error_count = 0;
1467
1468 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
1469 DWC2_HC_XFER_NYET);
1470 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1471
1472 /*
1473 * Halt the channel and re-start the transfer so the PING protocol
1474 * will start
1475 */
1476 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1477
1478handle_nyet_done:
1479 disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1480}
1481
1482/*
1483 * Handles a host channel babble interrupt. This handler may be called in
1484 * either DMA mode or Slave mode.
1485 */
1486static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1487 struct dwc2_host_chan *chan, int chnum,
1488 struct dwc2_qtd *qtd)
1489{
1490 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1491 chnum);
1492
1493 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1494
1495 if (hsotg->params.dma_desc_enable) {
1496 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1497 DWC2_HC_XFER_BABBLE_ERR);
1498 goto disable_int;
1499 }
1500
1501 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1502 dwc2_host_complete(hsotg, qtd, -EOVERFLOW);
1503 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
1504 } else {
1505 enum dwc2_halt_status halt_status;
1506
1507 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1508 qtd, DWC2_HC_XFER_BABBLE_ERR);
1509 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1510 }
1511
1512disable_int:
1513 disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1514}
1515
1516/*
1517 * Handles a host channel AHB error interrupt. This handler is only called in
1518 * DMA mode.
1519 */
1520static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1521 struct dwc2_host_chan *chan, int chnum,
1522 struct dwc2_qtd *qtd)
1523{
1524 struct dwc2_hcd_urb *urb = qtd->urb;
1525 char *pipetype, *speed;
1526 u32 hcchar;
1527 u32 hcsplt;
1528 u32 hctsiz;
1529 u32 hc_dma;
1530
1531 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1532 chnum);
1533
1534 if (!urb)
1535 goto handle_ahberr_halt;
1536
1537 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1538
1539 hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1540 hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1541 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1542 hc_dma = dwc2_readl(hsotg, HCDMA(chnum));
1543
1544 dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1545 dev_err(hsotg->dev, " hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1546 dev_err(hsotg->dev, " hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1547 dev_err(hsotg->dev, " Device address: %d\n",
1548 dwc2_hcd_get_dev_addr(&urb->pipe_info));
1549 dev_err(hsotg->dev, " Endpoint: %d, %s\n",
1550 dwc2_hcd_get_ep_num(&urb->pipe_info),
1551 dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1552
1553 switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
1554 case USB_ENDPOINT_XFER_CONTROL:
1555 pipetype = "CONTROL";
1556 break;
1557 case USB_ENDPOINT_XFER_BULK:
1558 pipetype = "BULK";
1559 break;
1560 case USB_ENDPOINT_XFER_INT:
1561 pipetype = "INTERRUPT";
1562 break;
1563 case USB_ENDPOINT_XFER_ISOC:
1564 pipetype = "ISOCHRONOUS";
1565 break;
1566 default:
1567 pipetype = "UNKNOWN";
1568 break;
1569 }
1570
1571 dev_err(hsotg->dev, " Endpoint type: %s\n", pipetype);
1572
1573 switch (chan->speed) {
1574 case USB_SPEED_HIGH:
1575 speed = "HIGH";
1576 break;
1577 case USB_SPEED_FULL:
1578 speed = "FULL";
1579 break;
1580 case USB_SPEED_LOW:
1581 speed = "LOW";
1582 break;
1583 default:
1584 speed = "UNKNOWN";
1585 break;
1586 }
1587
1588 dev_err(hsotg->dev, " Speed: %s\n", speed);
1589
1590 dev_err(hsotg->dev, " Max packet size: %d (mult %d)\n",
1591 dwc2_hcd_get_maxp(&urb->pipe_info),
1592 dwc2_hcd_get_maxp_mult(&urb->pipe_info));
1593 dev_err(hsotg->dev, " Data buffer length: %d\n", urb->length);
1594 dev_err(hsotg->dev, " Transfer buffer: %p, Transfer DMA: %08lx\n",
1595 urb->buf, (unsigned long)urb->dma);
1596 dev_err(hsotg->dev, " Setup buffer: %p, Setup DMA: %08lx\n",
1597 urb->setup_packet, (unsigned long)urb->setup_dma);
1598 dev_err(hsotg->dev, " Interval: %d\n", urb->interval);
1599
1600 /* Core halts the channel for Descriptor DMA mode */
1601 if (hsotg->params.dma_desc_enable) {
1602 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1603 DWC2_HC_XFER_AHB_ERR);
1604 goto handle_ahberr_done;
1605 }
1606
1607 dwc2_host_complete(hsotg, qtd, -EIO);
1608
1609handle_ahberr_halt:
1610 /*
1611 * Force a channel halt. Don't call dwc2_halt_channel because that won't
1612 * write to the HCCHARn register in DMA mode to force the halt.
1613 */
1614 dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);
1615
1616handle_ahberr_done:
1617 disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1618}
1619
1620/*
1621 * Handles a host channel transaction error interrupt. This handler may be
1622 * called in either DMA mode or Slave mode.
1623 */
1624static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1625 struct dwc2_host_chan *chan, int chnum,
1626 struct dwc2_qtd *qtd)
1627{
1628 dev_dbg(hsotg->dev,
1629 "--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1630
1631 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1632
1633 if (hsotg->params.dma_desc_enable) {
1634 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1635 DWC2_HC_XFER_XACT_ERR);
1636 goto handle_xacterr_done;
1637 }
1638
1639 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1640 case USB_ENDPOINT_XFER_CONTROL:
1641 case USB_ENDPOINT_XFER_BULK:
1642 qtd->error_count++;
1643 if (!chan->qh->ping_state) {
1644 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1645 qtd, DWC2_HC_XFER_XACT_ERR);
1646 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1647 if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1648 chan->qh->ping_state = 1;
1649 }
1650
1651 /*
1652 * Halt the channel so the transfer can be re-started from
1653 * the appropriate point or the PING protocol will start
1654 */
1655 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1656 break;
1657 case USB_ENDPOINT_XFER_INT:
1658 qtd->error_count++;
1659 if (chan->do_split && chan->complete_split)
1660 qtd->complete_split = 0;
1661 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1662 break;
1663 case USB_ENDPOINT_XFER_ISOC:
1664 {
1665 enum dwc2_halt_status halt_status;
1666
1667 halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1668 chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1669 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1670 }
1671 break;
1672 }
1673
1674handle_xacterr_done:
1675 disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1676}
1677
1678/*
1679 * Handles a host channel frame overrun interrupt. This handler may be called
1680 * in either DMA mode or Slave mode.
1681 */
1682static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1683 struct dwc2_host_chan *chan, int chnum,
1684 struct dwc2_qtd *qtd)
1685{
1686 enum dwc2_halt_status halt_status;
1687
1688 if (dbg_hc(chan))
1689 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1690 chnum);
1691
1692 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1693
1694 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1695 case USB_ENDPOINT_XFER_CONTROL:
1696 case USB_ENDPOINT_XFER_BULK:
1697 break;
1698 case USB_ENDPOINT_XFER_INT:
1699 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1700 break;
1701 case USB_ENDPOINT_XFER_ISOC:
1702 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1703 qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1704 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1705 break;
1706 }
1707
1708 disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1709}
1710
1711/*
1712 * Handles a host channel data toggle error interrupt. This handler may be
1713 * called in either DMA mode or Slave mode.
1714 */
1715static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1716 struct dwc2_host_chan *chan, int chnum,
1717 struct dwc2_qtd *qtd)
1718{
1719 dev_dbg(hsotg->dev,
1720 "--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1721
1722 if (chan->ep_is_in)
1723 qtd->error_count = 0;
1724 else
1725 dev_err(hsotg->dev,
1726 "Data Toggle Error on OUT transfer, channel %d\n",
1727 chnum);
1728
1729 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1730 disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1731}
1732
1733/*
1734 * For debug only. It checks that a valid halt status is set and that
1735 * HCCHARn.chdis is clear. If there's a problem, corrective action is
1736 * taken and a warning is issued.
1737 *
1738 * Return: true if halt status is ok, false otherwise
1739 */
1740static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1741 struct dwc2_host_chan *chan, int chnum,
1742 struct dwc2_qtd *qtd)
1743{
1744#ifdef DEBUG
1745 u32 hcchar;
1746 u32 hctsiz;
1747 u32 hcintmsk;
1748 u32 hcsplt;
1749
1750 if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1751 /*
1752 * This code is here only as a check. This condition should
1753 * never happen. Ignore the halt if it does occur.
1754 */
1755 hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1756 hctsiz = dwc2_readl(hsotg, HCTSIZ(chnum));
1757 hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1758 hcsplt = dwc2_readl(hsotg, HCSPLT(chnum));
1759 dev_dbg(hsotg->dev,
1760 "%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1761 __func__);
1762 dev_dbg(hsotg->dev,
1763 "channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1764 chnum, hcchar, hctsiz);
1765 dev_dbg(hsotg->dev,
1766 "hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1767 chan->hcint, hcintmsk, hcsplt);
1768 if (qtd)
1769 dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1770 qtd->complete_split);
1771 dev_warn(hsotg->dev,
1772 "%s: no halt status, channel %d, ignoring interrupt\n",
1773 __func__, chnum);
1774 return false;
1775 }
1776
1777 /*
1778 * This code is here only as a check. hcchar.chdis should never be set
1779 * when the halt interrupt occurs. Halt the channel again if it does
1780 * occur.
1781 */
1782 hcchar = dwc2_readl(hsotg, HCCHAR(chnum));
1783 if (hcchar & HCCHAR_CHDIS) {
1784 dev_warn(hsotg->dev,
1785 "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1786 __func__, hcchar);
1787 chan->halt_pending = 0;
1788 dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
1789 return false;
1790 }
1791#endif
1792
1793 return true;
1794}
1795
1796/*
1797 * Handles a host Channel Halted interrupt in DMA mode. This handler
1798 * determines the reason the channel halted and proceeds accordingly.
1799 */
1800static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1801 struct dwc2_host_chan *chan, int chnum,
1802 struct dwc2_qtd *qtd)
1803{
1804 u32 hcintmsk;
1805 int out_nak_enh = 0;
1806
1807 if (dbg_hc(chan))
1808 dev_vdbg(hsotg->dev,
1809 "--Host Channel %d Interrupt: DMA Channel Halted--\n",
1810 chnum);
1811
1812 /*
1813 * For core with OUT NAK enhancement, the flow for high-speed
1814 * CONTROL/BULK OUT is handled a little differently
1815 */
1816 if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1817 if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1818 (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1819 chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1820 out_nak_enh = 1;
1821 }
1822 }
1823
1824 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1825 (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1826 !hsotg->params.dma_desc_enable)) {
1827 if (hsotg->params.dma_desc_enable)
1828 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1829 chan->halt_status);
1830 else
1831 /*
1832 * Just release the channel. A dequeue can happen on a
1833 * transfer timeout. In the case of an AHB Error, the
1834 * channel was forced to halt because there's no way to
1835 * gracefully recover.
1836 */
1837 dwc2_release_channel(hsotg, chan, qtd,
1838 chan->halt_status);
1839 return;
1840 }
1841
1842 hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
1843
1844 if (chan->hcint & HCINTMSK_XFERCOMPL) {
1845 /*
1846 * Todo: This is here because of a possible hardware bug. Spec
1847 * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1848 * interrupt w/ACK bit set should occur, but I only see the
1849 * XFERCOMP bit, even with it masked out. This is a workaround
1850 * for that behavior. Should fix this when hardware is fixed.
1851 */
1852 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1853 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1854 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1855 } else if (chan->hcint & HCINTMSK_STALL) {
1856 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1857 } else if ((chan->hcint & HCINTMSK_XACTERR) &&
1858 !hsotg->params.dma_desc_enable) {
1859 if (out_nak_enh) {
1860 if (chan->hcint &
1861 (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1862 dev_vdbg(hsotg->dev,
1863 "XactErr with NYET/NAK/ACK\n");
1864 qtd->error_count = 0;
1865 } else {
1866 dev_vdbg(hsotg->dev,
1867 "XactErr without NYET/NAK/ACK\n");
1868 }
1869 }
1870
1871 /*
1872 * Must handle xacterr before nak or ack. Could get a xacterr
1873 * at the same time as either of these on a BULK/CONTROL OUT
1874 * that started with a PING. The xacterr takes precedence.
1875 */
1876 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1877 } else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1878 hsotg->params.dma_desc_enable) {
1879 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1880 } else if ((chan->hcint & HCINTMSK_AHBERR) &&
1881 hsotg->params.dma_desc_enable) {
1882 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1883 } else if (chan->hcint & HCINTMSK_BBLERR) {
1884 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1885 } else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1886 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1887 } else if (!out_nak_enh) {
1888 if (chan->hcint & HCINTMSK_NYET) {
1889 /*
1890 * Must handle nyet before nak or ack. Could get a nyet
1891 * at the same time as either of those on a BULK/CONTROL
1892 * OUT that started with a PING. The nyet takes
1893 * precedence.
1894 */
1895 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1896 } else if ((chan->hcint & HCINTMSK_NAK) &&
1897 !(hcintmsk & HCINTMSK_NAK)) {
1898 /*
1899 * If nak is not masked, it's because a non-split IN
1900 * transfer is in an error state. In that case, the nak
1901 * is handled by the nak interrupt handler, not here.
1902 * Handle nak here for BULK/CONTROL OUT transfers, which
1903 * halt on a NAK to allow rewinding the buffer pointer.
1904 */
1905 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1906 } else if ((chan->hcint & HCINTMSK_ACK) &&
1907 !(hcintmsk & HCINTMSK_ACK)) {
1908 /*
1909 * If ack is not masked, it's because a non-split IN
1910 * transfer is in an error state. In that case, the ack
1911 * is handled by the ack interrupt handler, not here.
1912 * Handle ack here for split transfers. Start splits
1913 * halt on ACK.
1914 */
1915 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1916 } else {
1917 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1918 chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1919 /*
1920 * A periodic transfer halted with no other
1921 * channel interrupts set. Assume it was halted
1922 * by the core because it could not be completed
1923 * in its scheduled (micro)frame.
1924 */
1925 dev_dbg(hsotg->dev,
1926 "%s: Halt channel %d (assume incomplete periodic transfer)\n",
1927 __func__, chnum);
1928 dwc2_halt_channel(hsotg, chan, qtd,
1929 DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1930 } else {
1931 dev_err(hsotg->dev,
1932 "%s: Channel %d - ChHltd set, but reason is unknown\n",
1933 __func__, chnum);
1934 dev_err(hsotg->dev,
1935 "hcint 0x%08x, intsts 0x%08x\n",
1936 chan->hcint,
1937 dwc2_readl(hsotg, GINTSTS));
1938 goto error;
1939 }
1940 }
1941 } else {
1942 dev_info(hsotg->dev,
1943 "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1944 chan->hcint);
1945error:
1946 /* Failthrough: use 3-strikes rule */
1947 qtd->error_count++;
1948 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1949 qtd, DWC2_HC_XFER_XACT_ERR);
1950 /*
1951 * We can get here after a completed transaction
1952 * (urb->actual_length >= urb->length) which was not reported
1953 * as completed. If that is the case, and we do not abort
1954 * the transfer, a transfer of size 0 will be enqueued
1955 * subsequently. If urb->actual_length is not DMA-aligned,
1956 * the buffer will then point to an unaligned address, and
1957 * the resulting behavior is undefined. Bail out in that
1958 * situation.
1959 */
1960 if (qtd->urb->actual_length >= qtd->urb->length)
1961 qtd->error_count = 3;
1962 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1963 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1964 }
1965}
1966
1967/*
1968 * Handles a host channel Channel Halted interrupt
1969 *
1970 * In slave mode, this handler is called only when the driver specifically
1971 * requests a halt. This occurs during handling other host channel interrupts
1972 * (e.g. nak, xacterr, stall, nyet, etc.).
1973 *
1974 * In DMA mode, this is the interrupt that occurs when the core has finished
1975 * processing a transfer on a channel. Other host channel interrupts (except
1976 * ahberr) are disabled in DMA mode.
1977 */
1978static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
1979 struct dwc2_host_chan *chan, int chnum,
1980 struct dwc2_qtd *qtd)
1981{
1982 if (dbg_hc(chan))
1983 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
1984 chnum);
1985
1986 if (hsotg->params.host_dma) {
1987 dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
1988 } else {
1989 if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
1990 return;
1991 dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
1992 }
1993}
1994
1995/*
1996 * Check if the given qtd is still the top of the list (and thus valid).
1997 *
1998 * If dwc2_hcd_qtd_unlink_and_free() has been called since we grabbed
1999 * the qtd from the top of the list, this will return false (otherwise true).
2000 */
2001static bool dwc2_check_qtd_still_ok(struct dwc2_qtd *qtd, struct dwc2_qh *qh)
2002{
2003 struct dwc2_qtd *cur_head;
2004
2005 if (!qh)
2006 return false;
2007
2008 cur_head = list_first_entry(&qh->qtd_list, struct dwc2_qtd,
2009 qtd_list_entry);
2010 return (cur_head == qtd);
2011}
2012
2013/* Handles interrupt for a specific Host Channel */
2014static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
2015{
2016 struct dwc2_qtd *qtd;
2017 struct dwc2_host_chan *chan;
2018 u32 hcint, hcintraw, hcintmsk;
2019
2020 chan = hsotg->hc_ptr_array[chnum];
2021
2022 hcintraw = dwc2_readl(hsotg, HCINT(chnum));
2023 hcintmsk = dwc2_readl(hsotg, HCINTMSK(chnum));
2024 hcint = hcintraw & hcintmsk;
2025 dwc2_writel(hsotg, hcint, HCINT(chnum));
2026
2027 if (!chan) {
2028 dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
2029 return;
2030 }
2031
2032 if (dbg_hc(chan)) {
2033 dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
2034 chnum);
2035 dev_vdbg(hsotg->dev,
2036 " hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2037 hcintraw, hcintmsk, hcint);
2038 }
2039
2040 /*
2041 * If we got an interrupt after someone called
2042 * dwc2_hcd_endpoint_disable() we don't want to crash below
2043 */
2044 if (!chan->qh) {
2045 dev_warn(hsotg->dev, "Interrupt on disabled channel\n");
2046 return;
2047 }
2048
2049 chan->hcint = hcintraw;
2050
2051 /*
2052 * If the channel was halted due to a dequeue, the qtd list might
2053 * be empty or at least the first entry will not be the active qtd.
2054 * In this case, take a shortcut and just release the channel.
2055 */
2056 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
2057 /*
2058 * If the channel was halted, this should be the only
2059 * interrupt unmasked
2060 */
2061 WARN_ON(hcint != HCINTMSK_CHHLTD);
2062 if (hsotg->params.dma_desc_enable)
2063 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
2064 chan->halt_status);
2065 else
2066 dwc2_release_channel(hsotg, chan, NULL,
2067 chan->halt_status);
2068 return;
2069 }
2070
2071 if (list_empty(&chan->qh->qtd_list)) {
2072 /*
2073 * TODO: Will this ever happen with the
2074 * DWC2_HC_XFER_URB_DEQUEUE handling above?
2075 */
2076 dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
2077 chnum);
2078 dev_dbg(hsotg->dev,
2079 " hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2080 chan->hcint, hcintmsk, hcint);
2081 chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
2082 disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
2083 chan->hcint = 0;
2084 return;
2085 }
2086
2087 qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
2088 qtd_list_entry);
2089
2090 if (!hsotg->params.host_dma) {
2091 if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
2092 hcint &= ~HCINTMSK_CHHLTD;
2093 }
2094
2095 if (hcint & HCINTMSK_XFERCOMPL) {
2096 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
2097 /*
2098 * If NYET occurred at same time as Xfer Complete, the NYET is
2099 * handled by the Xfer Complete interrupt handler. Don't want
2100 * to call the NYET interrupt handler in this case.
2101 */
2102 hcint &= ~HCINTMSK_NYET;
2103 }
2104
2105 if (hcint & HCINTMSK_CHHLTD) {
2106 dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2107 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2108 goto exit;
2109 }
2110 if (hcint & HCINTMSK_AHBERR) {
2111 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2112 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2113 goto exit;
2114 }
2115 if (hcint & HCINTMSK_STALL) {
2116 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2117 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2118 goto exit;
2119 }
2120 if (hcint & HCINTMSK_NAK) {
2121 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2122 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2123 goto exit;
2124 }
2125 if (hcint & HCINTMSK_ACK) {
2126 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2127 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2128 goto exit;
2129 }
2130 if (hcint & HCINTMSK_NYET) {
2131 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2132 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2133 goto exit;
2134 }
2135 if (hcint & HCINTMSK_XACTERR) {
2136 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2137 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2138 goto exit;
2139 }
2140 if (hcint & HCINTMSK_BBLERR) {
2141 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2142 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2143 goto exit;
2144 }
2145 if (hcint & HCINTMSK_FRMOVRUN) {
2146 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2147 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2148 goto exit;
2149 }
2150 if (hcint & HCINTMSK_DATATGLERR) {
2151 dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2152 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2153 goto exit;
2154 }
2155
2156exit:
2157 chan->hcint = 0;
2158}
2159
2160/*
2161 * This interrupt indicates that one or more host channels has a pending
2162 * interrupt. There are multiple conditions that can cause each host channel
2163 * interrupt. This function determines which conditions have occurred for each
2164 * host channel interrupt and handles them appropriately.
2165 */
2166static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2167{
2168 u32 haint;
2169 int i;
2170 struct dwc2_host_chan *chan, *chan_tmp;
2171
2172 haint = dwc2_readl(hsotg, HAINT);
2173 if (dbg_perio()) {
2174 dev_vdbg(hsotg->dev, "%s()\n", __func__);
2175
2176 dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2177 }
2178
2179 /*
2180 * According to USB 2.0 spec section 11.18.8, a host must
2181 * issue complete-split transactions in a microframe for a
2182 * set of full-/low-speed endpoints in the same relative
2183 * order as the start-splits were issued in a microframe for.
2184 */
2185 list_for_each_entry_safe(chan, chan_tmp, &hsotg->split_order,
2186 split_order_list_entry) {
2187 int hc_num = chan->hc_num;
2188
2189 if (haint & (1 << hc_num)) {
2190 dwc2_hc_n_intr(hsotg, hc_num);
2191 haint &= ~(1 << hc_num);
2192 }
2193 }
2194
2195 for (i = 0; i < hsotg->params.host_channels; i++) {
2196 if (haint & (1 << i))
2197 dwc2_hc_n_intr(hsotg, i);
2198 }
2199}
2200
2201/* This function handles interrupts for the HCD */
2202irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2203{
2204 u32 gintsts, dbg_gintsts;
2205 irqreturn_t retval = IRQ_HANDLED;
2206
2207 if (!dwc2_is_controller_alive(hsotg)) {
2208 dev_warn(hsotg->dev, "Controller is dead\n");
2209 return retval;
2210 } else {
2211 retval = IRQ_NONE;
2212 }
2213
2214 spin_lock(&hsotg->lock);
2215
2216 /* Check if HOST Mode */
2217 if (dwc2_is_host_mode(hsotg)) {
2218 gintsts = dwc2_read_core_intr(hsotg);
2219 if (!gintsts) {
2220 spin_unlock(&hsotg->lock);
2221 return retval;
2222 }
2223
2224 retval = IRQ_HANDLED;
2225
2226 dbg_gintsts = gintsts;
2227#ifndef DEBUG_SOF
2228 dbg_gintsts &= ~GINTSTS_SOF;
2229#endif
2230 if (!dbg_perio())
2231 dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
2232 GINTSTS_PTXFEMP);
2233
2234 /* Only print if there are any non-suppressed interrupts left */
2235 if (dbg_gintsts)
2236 dev_vdbg(hsotg->dev,
2237 "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2238 gintsts);
2239
2240 if (gintsts & GINTSTS_SOF)
2241 dwc2_sof_intr(hsotg);
2242 if (gintsts & GINTSTS_RXFLVL)
2243 dwc2_rx_fifo_level_intr(hsotg);
2244 if (gintsts & GINTSTS_NPTXFEMP)
2245 dwc2_np_tx_fifo_empty_intr(hsotg);
2246 if (gintsts & GINTSTS_PRTINT)
2247 dwc2_port_intr(hsotg);
2248 if (gintsts & GINTSTS_HCHINT)
2249 dwc2_hc_intr(hsotg);
2250 if (gintsts & GINTSTS_PTXFEMP)
2251 dwc2_perio_tx_fifo_empty_intr(hsotg);
2252
2253 if (dbg_gintsts) {
2254 dev_vdbg(hsotg->dev,
2255 "DWC OTG HCD Finished Servicing Interrupts\n");
2256 dev_vdbg(hsotg->dev,
2257 "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2258 dwc2_readl(hsotg, GINTSTS),
2259 dwc2_readl(hsotg, GINTMSK));
2260 }
2261 }
2262
2263 spin_unlock(&hsotg->lock);
2264
2265 return retval;
2266}
1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2/*
3 * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
4 *
5 * Copyright (C) 2004-2013 Synopsys, Inc.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions, and the following disclaimer,
12 * without modification.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. The names of the above-listed copyright holders may not be used
17 * to endorse or promote products derived from this software without
18 * specific prior written permission.
19 *
20 * ALTERNATIVELY, this software may be distributed under the terms of the
21 * GNU General Public License ("GPL") as published by the Free Software
22 * Foundation; either version 2 of the License, or (at your option) any
23 * later version.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
26 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
27 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
29 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 */
37
38/*
39 * This file contains the interrupt handlers for Host mode
40 */
41#include <linux/kernel.h>
42#include <linux/module.h>
43#include <linux/spinlock.h>
44#include <linux/interrupt.h>
45#include <linux/dma-mapping.h>
46#include <linux/io.h>
47#include <linux/slab.h>
48#include <linux/usb.h>
49
50#include <linux/usb/hcd.h>
51#include <linux/usb/ch11.h>
52
53#include "core.h"
54#include "hcd.h"
55
56/*
57 * If we get this many NAKs on a split transaction we'll slow down
58 * retransmission. A 1 here means delay after the first NAK.
59 */
60#define DWC2_NAKS_BEFORE_DELAY 3
61
62/* This function is for debug only */
63static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
64{
65 u16 curr_frame_number = hsotg->frame_number;
66 u16 expected = dwc2_frame_num_inc(hsotg->last_frame_num, 1);
67
68 if (expected != curr_frame_number)
69 dwc2_sch_vdbg(hsotg, "MISSED SOF %04x != %04x\n",
70 expected, curr_frame_number);
71
72#ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
73 if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
74 if (expected != curr_frame_number) {
75 hsotg->frame_num_array[hsotg->frame_num_idx] =
76 curr_frame_number;
77 hsotg->last_frame_num_array[hsotg->frame_num_idx] =
78 hsotg->last_frame_num;
79 hsotg->frame_num_idx++;
80 }
81 } else if (!hsotg->dumped_frame_num_array) {
82 int i;
83
84 dev_info(hsotg->dev, "Frame Last Frame\n");
85 dev_info(hsotg->dev, "----- ----------\n");
86 for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
87 dev_info(hsotg->dev, "0x%04x 0x%04x\n",
88 hsotg->frame_num_array[i],
89 hsotg->last_frame_num_array[i]);
90 }
91 hsotg->dumped_frame_num_array = 1;
92 }
93#endif
94 hsotg->last_frame_num = curr_frame_number;
95}
96
97static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
98 struct dwc2_host_chan *chan,
99 struct dwc2_qtd *qtd)
100{
101 struct usb_device *root_hub = dwc2_hsotg_to_hcd(hsotg)->self.root_hub;
102 struct urb *usb_urb;
103
104 if (!chan->qh)
105 return;
106
107 if (chan->qh->dev_speed == USB_SPEED_HIGH)
108 return;
109
110 if (!qtd->urb)
111 return;
112
113 usb_urb = qtd->urb->priv;
114 if (!usb_urb || !usb_urb->dev || !usb_urb->dev->tt)
115 return;
116
117 /*
118 * The root hub doesn't really have a TT, but Linux thinks it
119 * does because how could you have a "high speed hub" that
120 * directly talks directly to low speed devices without a TT?
121 * It's all lies. Lies, I tell you.
122 */
123 if (usb_urb->dev->tt->hub == root_hub)
124 return;
125
126 if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
127 chan->qh->tt_buffer_dirty = 1;
128 if (usb_hub_clear_tt_buffer(usb_urb))
129 /* Clear failed; let's hope things work anyway */
130 chan->qh->tt_buffer_dirty = 0;
131 }
132}
133
134/*
135 * Handles the start-of-frame interrupt in host mode. Non-periodic
136 * transactions may be queued to the DWC_otg controller for the current
137 * (micro)frame. Periodic transactions may be queued to the controller
138 * for the next (micro)frame.
139 */
140static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
141{
142 struct list_head *qh_entry;
143 struct dwc2_qh *qh;
144 enum dwc2_transaction_type tr_type;
145
146 /* Clear interrupt */
147 dwc2_writel(GINTSTS_SOF, hsotg->regs + GINTSTS);
148
149#ifdef DEBUG_SOF
150 dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
151#endif
152
153 hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
154
155 dwc2_track_missed_sofs(hsotg);
156
157 /* Determine whether any periodic QHs should be executed */
158 qh_entry = hsotg->periodic_sched_inactive.next;
159 while (qh_entry != &hsotg->periodic_sched_inactive) {
160 qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
161 qh_entry = qh_entry->next;
162 if (dwc2_frame_num_le(qh->next_active_frame,
163 hsotg->frame_number)) {
164 dwc2_sch_vdbg(hsotg, "QH=%p ready fn=%04x, nxt=%04x\n",
165 qh, hsotg->frame_number,
166 qh->next_active_frame);
167
168 /*
169 * Move QH to the ready list to be executed next
170 * (micro)frame
171 */
172 list_move_tail(&qh->qh_list_entry,
173 &hsotg->periodic_sched_ready);
174 }
175 }
176 tr_type = dwc2_hcd_select_transactions(hsotg);
177 if (tr_type != DWC2_TRANSACTION_NONE)
178 dwc2_hcd_queue_transactions(hsotg, tr_type);
179}
180
181/*
182 * Handles the Rx FIFO Level Interrupt, which indicates that there is
183 * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
184 * memory if the DWC_otg controller is operating in Slave mode.
185 */
186static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
187{
188 u32 grxsts, chnum, bcnt, dpid, pktsts;
189 struct dwc2_host_chan *chan;
190
191 if (dbg_perio())
192 dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
193
194 grxsts = dwc2_readl(hsotg->regs + GRXSTSP);
195 chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
196 chan = hsotg->hc_ptr_array[chnum];
197 if (!chan) {
198 dev_err(hsotg->dev, "Unable to get corresponding channel\n");
199 return;
200 }
201
202 bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
203 dpid = (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT;
204 pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
205
206 /* Packet Status */
207 if (dbg_perio()) {
208 dev_vdbg(hsotg->dev, " Ch num = %d\n", chnum);
209 dev_vdbg(hsotg->dev, " Count = %d\n", bcnt);
210 dev_vdbg(hsotg->dev, " DPID = %d, chan.dpid = %d\n", dpid,
211 chan->data_pid_start);
212 dev_vdbg(hsotg->dev, " PStatus = %d\n", pktsts);
213 }
214
215 switch (pktsts) {
216 case GRXSTS_PKTSTS_HCHIN:
217 /* Read the data into the host buffer */
218 if (bcnt > 0) {
219 dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);
220
221 /* Update the HC fields for the next packet received */
222 chan->xfer_count += bcnt;
223 chan->xfer_buf += bcnt;
224 }
225 break;
226 case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
227 case GRXSTS_PKTSTS_DATATOGGLEERR:
228 case GRXSTS_PKTSTS_HCHHALTED:
229 /* Handled in interrupt, just ignore data */
230 break;
231 default:
232 dev_err(hsotg->dev,
233 "RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
234 break;
235 }
236}
237
238/*
239 * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
240 * data packets may be written to the FIFO for OUT transfers. More requests
241 * may be written to the non-periodic request queue for IN transfers. This
242 * interrupt is enabled only in Slave mode.
243 */
244static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
245{
246 dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
247 dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
248}
249
250/*
251 * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
252 * packets may be written to the FIFO for OUT transfers. More requests may be
253 * written to the periodic request queue for IN transfers. This interrupt is
254 * enabled only in Slave mode.
255 */
256static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
257{
258 if (dbg_perio())
259 dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
260 dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
261}
262
263static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
264 u32 *hprt0_modify)
265{
266 struct dwc2_core_params *params = &hsotg->params;
267 int do_reset = 0;
268 u32 usbcfg;
269 u32 prtspd;
270 u32 hcfg;
271 u32 fslspclksel;
272 u32 hfir;
273
274 dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
275
276 /* Every time when port enables calculate HFIR.FrInterval */
277 hfir = dwc2_readl(hsotg->regs + HFIR);
278 hfir &= ~HFIR_FRINT_MASK;
279 hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
280 HFIR_FRINT_MASK;
281 dwc2_writel(hfir, hsotg->regs + HFIR);
282
283 /* Check if we need to adjust the PHY clock speed for low power */
284 if (!params->host_support_fs_ls_low_power) {
285 /* Port has been enabled, set the reset change flag */
286 hsotg->flags.b.port_reset_change = 1;
287 return;
288 }
289
290 usbcfg = dwc2_readl(hsotg->regs + GUSBCFG);
291 prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
292
293 if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
294 /* Low power */
295 if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
296 /* Set PHY low power clock select for FS/LS devices */
297 usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
298 dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
299 do_reset = 1;
300 }
301
302 hcfg = dwc2_readl(hsotg->regs + HCFG);
303 fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
304 HCFG_FSLSPCLKSEL_SHIFT;
305
306 if (prtspd == HPRT0_SPD_LOW_SPEED &&
307 params->host_ls_low_power_phy_clk) {
308 /* 6 MHZ */
309 dev_vdbg(hsotg->dev,
310 "FS_PHY programming HCFG to 6 MHz\n");
311 if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
312 fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
313 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
314 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
315 dwc2_writel(hcfg, hsotg->regs + HCFG);
316 do_reset = 1;
317 }
318 } else {
319 /* 48 MHZ */
320 dev_vdbg(hsotg->dev,
321 "FS_PHY programming HCFG to 48 MHz\n");
322 if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
323 fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
324 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
325 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
326 dwc2_writel(hcfg, hsotg->regs + HCFG);
327 do_reset = 1;
328 }
329 }
330 } else {
331 /* Not low power */
332 if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
333 usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
334 dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
335 do_reset = 1;
336 }
337 }
338
339 if (do_reset) {
340 *hprt0_modify |= HPRT0_RST;
341 dwc2_writel(*hprt0_modify, hsotg->regs + HPRT0);
342 queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
343 msecs_to_jiffies(60));
344 } else {
345 /* Port has been enabled, set the reset change flag */
346 hsotg->flags.b.port_reset_change = 1;
347 }
348}
349
350/*
351 * There are multiple conditions that can cause a port interrupt. This function
352 * determines which interrupt conditions have occurred and handles them
353 * appropriately.
354 */
355static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
356{
357 u32 hprt0;
358 u32 hprt0_modify;
359
360 dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
361
362 hprt0 = dwc2_readl(hsotg->regs + HPRT0);
363 hprt0_modify = hprt0;
364
365 /*
366 * Clear appropriate bits in HPRT0 to clear the interrupt bit in
367 * GINTSTS
368 */
369 hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
370 HPRT0_OVRCURRCHG);
371
372 /*
373 * Port Connect Detected
374 * Set flag and clear if detected
375 */
376 if (hprt0 & HPRT0_CONNDET) {
377 dwc2_writel(hprt0_modify | HPRT0_CONNDET, hsotg->regs + HPRT0);
378
379 dev_vdbg(hsotg->dev,
380 "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
381 hprt0);
382 dwc2_hcd_connect(hsotg);
383
384 /*
385 * The Hub driver asserts a reset when it sees port connect
386 * status change flag
387 */
388 }
389
390 /*
391 * Port Enable Changed
392 * Clear if detected - Set internal flag if disabled
393 */
394 if (hprt0 & HPRT0_ENACHG) {
395 dwc2_writel(hprt0_modify | HPRT0_ENACHG, hsotg->regs + HPRT0);
396 dev_vdbg(hsotg->dev,
397 " --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
398 hprt0, !!(hprt0 & HPRT0_ENA));
399 if (hprt0 & HPRT0_ENA) {
400 hsotg->new_connection = true;
401 dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
402 } else {
403 hsotg->flags.b.port_enable_change = 1;
404 if (hsotg->params.dma_desc_fs_enable) {
405 u32 hcfg;
406
407 hsotg->params.dma_desc_enable = false;
408 hsotg->new_connection = false;
409 hcfg = dwc2_readl(hsotg->regs + HCFG);
410 hcfg &= ~HCFG_DESCDMA;
411 dwc2_writel(hcfg, hsotg->regs + HCFG);
412 }
413 }
414 }
415
416 /* Overcurrent Change Interrupt */
417 if (hprt0 & HPRT0_OVRCURRCHG) {
418 dwc2_writel(hprt0_modify | HPRT0_OVRCURRCHG,
419 hsotg->regs + HPRT0);
420 dev_vdbg(hsotg->dev,
421 " --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
422 hprt0);
423 hsotg->flags.b.port_over_current_change = 1;
424 }
425}
426
427/*
428 * Gets the actual length of a transfer after the transfer halts. halt_status
429 * holds the reason for the halt.
430 *
431 * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
432 * is set to 1 upon return if less than the requested number of bytes were
433 * transferred. short_read may also be NULL on entry, in which case it remains
434 * unchanged.
435 */
436static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
437 struct dwc2_host_chan *chan, int chnum,
438 struct dwc2_qtd *qtd,
439 enum dwc2_halt_status halt_status,
440 int *short_read)
441{
442 u32 hctsiz, count, length;
443
444 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
445
446 if (halt_status == DWC2_HC_XFER_COMPLETE) {
447 if (chan->ep_is_in) {
448 count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
449 TSIZ_XFERSIZE_SHIFT;
450 length = chan->xfer_len - count;
451 if (short_read)
452 *short_read = (count != 0);
453 } else if (chan->qh->do_split) {
454 length = qtd->ssplit_out_xfer_count;
455 } else {
456 length = chan->xfer_len;
457 }
458 } else {
459 /*
460 * Must use the hctsiz.pktcnt field to determine how much data
461 * has been transferred. This field reflects the number of
462 * packets that have been transferred via the USB. This is
463 * always an integral number of packets if the transfer was
464 * halted before its normal completion. (Can't use the
465 * hctsiz.xfersize field because that reflects the number of
466 * bytes transferred via the AHB, not the USB).
467 */
468 count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
469 length = (chan->start_pkt_count - count) * chan->max_packet;
470 }
471
472 return length;
473}
474
475/**
476 * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
477 * Complete interrupt on the host channel. Updates the actual_length field
478 * of the URB based on the number of bytes transferred via the host channel.
479 * Sets the URB status if the data transfer is finished.
480 *
481 * Return: 1 if the data transfer specified by the URB is completely finished,
482 * 0 otherwise
483 */
484static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
485 struct dwc2_host_chan *chan, int chnum,
486 struct dwc2_hcd_urb *urb,
487 struct dwc2_qtd *qtd)
488{
489 u32 hctsiz;
490 int xfer_done = 0;
491 int short_read = 0;
492 int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
493 DWC2_HC_XFER_COMPLETE,
494 &short_read);
495
496 if (urb->actual_length + xfer_length > urb->length) {
497 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
498 xfer_length = urb->length - urb->actual_length;
499 }
500
501 dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
502 urb->actual_length, xfer_length);
503 urb->actual_length += xfer_length;
504
505 if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
506 (urb->flags & URB_SEND_ZERO_PACKET) &&
507 urb->actual_length >= urb->length &&
508 !(urb->length % chan->max_packet)) {
509 xfer_done = 0;
510 } else if (short_read || urb->actual_length >= urb->length) {
511 xfer_done = 1;
512 urb->status = 0;
513 }
514
515 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
516 dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
517 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
518 dev_vdbg(hsotg->dev, " chan->xfer_len %d\n", chan->xfer_len);
519 dev_vdbg(hsotg->dev, " hctsiz.xfersize %d\n",
520 (hctsiz & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
521 dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n", urb->length);
522 dev_vdbg(hsotg->dev, " urb->actual_length %d\n", urb->actual_length);
523 dev_vdbg(hsotg->dev, " short_read %d, xfer_done %d\n", short_read,
524 xfer_done);
525
526 return xfer_done;
527}
528
529/*
530 * Save the starting data toggle for the next transfer. The data toggle is
531 * saved in the QH for non-control transfers and it's saved in the QTD for
532 * control transfers.
533 */
534void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
535 struct dwc2_host_chan *chan, int chnum,
536 struct dwc2_qtd *qtd)
537{
538 u32 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
539 u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
540
541 if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
542 if (WARN(!chan || !chan->qh,
543 "chan->qh must be specified for non-control eps\n"))
544 return;
545
546 if (pid == TSIZ_SC_MC_PID_DATA0)
547 chan->qh->data_toggle = DWC2_HC_PID_DATA0;
548 else
549 chan->qh->data_toggle = DWC2_HC_PID_DATA1;
550 } else {
551 if (WARN(!qtd,
552 "qtd must be specified for control eps\n"))
553 return;
554
555 if (pid == TSIZ_SC_MC_PID_DATA0)
556 qtd->data_toggle = DWC2_HC_PID_DATA0;
557 else
558 qtd->data_toggle = DWC2_HC_PID_DATA1;
559 }
560}
561
562/**
563 * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
564 * the transfer is stopped for any reason. The fields of the current entry in
565 * the frame descriptor array are set based on the transfer state and the input
566 * halt_status. Completes the Isochronous URB if all the URB frames have been
567 * completed.
568 *
569 * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
570 * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
571 */
572static enum dwc2_halt_status dwc2_update_isoc_urb_state(
573 struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
574 int chnum, struct dwc2_qtd *qtd,
575 enum dwc2_halt_status halt_status)
576{
577 struct dwc2_hcd_iso_packet_desc *frame_desc;
578 struct dwc2_hcd_urb *urb = qtd->urb;
579
580 if (!urb)
581 return DWC2_HC_XFER_NO_HALT_STATUS;
582
583 frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
584
585 switch (halt_status) {
586 case DWC2_HC_XFER_COMPLETE:
587 frame_desc->status = 0;
588 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
589 chan, chnum, qtd, halt_status, NULL);
590 break;
591 case DWC2_HC_XFER_FRAME_OVERRUN:
592 urb->error_count++;
593 if (chan->ep_is_in)
594 frame_desc->status = -ENOSR;
595 else
596 frame_desc->status = -ECOMM;
597 frame_desc->actual_length = 0;
598 break;
599 case DWC2_HC_XFER_BABBLE_ERR:
600 urb->error_count++;
601 frame_desc->status = -EOVERFLOW;
602 /* Don't need to update actual_length in this case */
603 break;
604 case DWC2_HC_XFER_XACT_ERR:
605 urb->error_count++;
606 frame_desc->status = -EPROTO;
607 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
608 chan, chnum, qtd, halt_status, NULL);
609
610 /* Skip whole frame */
611 if (chan->qh->do_split &&
612 chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
613 hsotg->params.host_dma) {
614 qtd->complete_split = 0;
615 qtd->isoc_split_offset = 0;
616 }
617
618 break;
619 default:
620 dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
621 halt_status);
622 break;
623 }
624
625 if (++qtd->isoc_frame_index == urb->packet_count) {
626 /*
627 * urb->status is not used for isoc transfers. The individual
628 * frame_desc statuses are used instead.
629 */
630 dwc2_host_complete(hsotg, qtd, 0);
631 halt_status = DWC2_HC_XFER_URB_COMPLETE;
632 } else {
633 halt_status = DWC2_HC_XFER_COMPLETE;
634 }
635
636 return halt_status;
637}
638
639/*
640 * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
641 * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
642 * still linked to the QH, the QH is added to the end of the inactive
643 * non-periodic schedule. For periodic QHs, removes the QH from the periodic
644 * schedule if no more QTDs are linked to the QH.
645 */
646static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
647 int free_qtd)
648{
649 int continue_split = 0;
650 struct dwc2_qtd *qtd;
651
652 if (dbg_qh(qh))
653 dev_vdbg(hsotg->dev, " %s(%p,%p,%d)\n", __func__,
654 hsotg, qh, free_qtd);
655
656 if (list_empty(&qh->qtd_list)) {
657 dev_dbg(hsotg->dev, "## QTD list empty ##\n");
658 goto no_qtd;
659 }
660
661 qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
662
663 if (qtd->complete_split)
664 continue_split = 1;
665 else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
666 qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
667 continue_split = 1;
668
669 if (free_qtd) {
670 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
671 continue_split = 0;
672 }
673
674no_qtd:
675 qh->channel = NULL;
676 dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
677}
678
679/**
680 * dwc2_release_channel() - Releases a host channel for use by other transfers
681 *
682 * @hsotg: The HCD state structure
683 * @chan: The host channel to release
684 * @qtd: The QTD associated with the host channel. This QTD may be
685 * freed if the transfer is complete or an error has occurred.
686 * @halt_status: Reason the channel is being released. This status
687 * determines the actions taken by this function.
688 *
689 * Also attempts to select and queue more transactions since at least one host
690 * channel is available.
691 */
692static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
693 struct dwc2_host_chan *chan,
694 struct dwc2_qtd *qtd,
695 enum dwc2_halt_status halt_status)
696{
697 enum dwc2_transaction_type tr_type;
698 u32 haintmsk;
699 int free_qtd = 0;
700
701 if (dbg_hc(chan))
702 dev_vdbg(hsotg->dev, " %s: channel %d, halt_status %d\n",
703 __func__, chan->hc_num, halt_status);
704
705 switch (halt_status) {
706 case DWC2_HC_XFER_URB_COMPLETE:
707 free_qtd = 1;
708 break;
709 case DWC2_HC_XFER_AHB_ERR:
710 case DWC2_HC_XFER_STALL:
711 case DWC2_HC_XFER_BABBLE_ERR:
712 free_qtd = 1;
713 break;
714 case DWC2_HC_XFER_XACT_ERR:
715 if (qtd && qtd->error_count >= 3) {
716 dev_vdbg(hsotg->dev,
717 " Complete URB with transaction error\n");
718 free_qtd = 1;
719 dwc2_host_complete(hsotg, qtd, -EPROTO);
720 }
721 break;
722 case DWC2_HC_XFER_URB_DEQUEUE:
723 /*
724 * The QTD has already been removed and the QH has been
725 * deactivated. Don't want to do anything except release the
726 * host channel and try to queue more transfers.
727 */
728 goto cleanup;
729 case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
730 dev_vdbg(hsotg->dev, " Complete URB with I/O error\n");
731 free_qtd = 1;
732 dwc2_host_complete(hsotg, qtd, -EIO);
733 break;
734 case DWC2_HC_XFER_NO_HALT_STATUS:
735 default:
736 break;
737 }
738
739 dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);
740
741cleanup:
742 /*
743 * Release the host channel for use by other transfers. The cleanup
744 * function clears the channel interrupt enables and conditions, so
745 * there's no need to clear the Channel Halted interrupt separately.
746 */
747 if (!list_empty(&chan->hc_list_entry))
748 list_del(&chan->hc_list_entry);
749 dwc2_hc_cleanup(hsotg, chan);
750 list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
751
752 if (hsotg->params.uframe_sched) {
753 hsotg->available_host_channels++;
754 } else {
755 switch (chan->ep_type) {
756 case USB_ENDPOINT_XFER_CONTROL:
757 case USB_ENDPOINT_XFER_BULK:
758 hsotg->non_periodic_channels--;
759 break;
760 default:
761 /*
762 * Don't release reservations for periodic channels
763 * here. That's done when a periodic transfer is
764 * descheduled (i.e. when the QH is removed from the
765 * periodic schedule).
766 */
767 break;
768 }
769 }
770
771 haintmsk = dwc2_readl(hsotg->regs + HAINTMSK);
772 haintmsk &= ~(1 << chan->hc_num);
773 dwc2_writel(haintmsk, hsotg->regs + HAINTMSK);
774
775 /* Try to queue more transfers now that there's a free channel */
776 tr_type = dwc2_hcd_select_transactions(hsotg);
777 if (tr_type != DWC2_TRANSACTION_NONE)
778 dwc2_hcd_queue_transactions(hsotg, tr_type);
779}
780
781/*
782 * Halts a host channel. If the channel cannot be halted immediately because
783 * the request queue is full, this function ensures that the FIFO empty
784 * interrupt for the appropriate queue is enabled so that the halt request can
785 * be queued when there is space in the request queue.
786 *
787 * This function may also be called in DMA mode. In that case, the channel is
788 * simply released since the core always halts the channel automatically in
789 * DMA mode.
790 */
791static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
792 struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
793 enum dwc2_halt_status halt_status)
794{
795 if (dbg_hc(chan))
796 dev_vdbg(hsotg->dev, "%s()\n", __func__);
797
798 if (hsotg->params.host_dma) {
799 if (dbg_hc(chan))
800 dev_vdbg(hsotg->dev, "DMA enabled\n");
801 dwc2_release_channel(hsotg, chan, qtd, halt_status);
802 return;
803 }
804
805 /* Slave mode processing */
806 dwc2_hc_halt(hsotg, chan, halt_status);
807
808 if (chan->halt_on_queue) {
809 u32 gintmsk;
810
811 dev_vdbg(hsotg->dev, "Halt on queue\n");
812 if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
813 chan->ep_type == USB_ENDPOINT_XFER_BULK) {
814 dev_vdbg(hsotg->dev, "control/bulk\n");
815 /*
816 * Make sure the Non-periodic Tx FIFO empty interrupt
817 * is enabled so that the non-periodic schedule will
818 * be processed
819 */
820 gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
821 gintmsk |= GINTSTS_NPTXFEMP;
822 dwc2_writel(gintmsk, hsotg->regs + GINTMSK);
823 } else {
824 dev_vdbg(hsotg->dev, "isoc/intr\n");
825 /*
826 * Move the QH from the periodic queued schedule to
827 * the periodic assigned schedule. This allows the
828 * halt to be queued when the periodic schedule is
829 * processed.
830 */
831 list_move_tail(&chan->qh->qh_list_entry,
832 &hsotg->periodic_sched_assigned);
833
834 /*
835 * Make sure the Periodic Tx FIFO Empty interrupt is
836 * enabled so that the periodic schedule will be
837 * processed
838 */
839 gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
840 gintmsk |= GINTSTS_PTXFEMP;
841 dwc2_writel(gintmsk, hsotg->regs + GINTMSK);
842 }
843 }
844}
845
846/*
847 * Performs common cleanup for non-periodic transfers after a Transfer
848 * Complete interrupt. This function should be called after any endpoint type
849 * specific handling is finished to release the host channel.
850 */
851static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
852 struct dwc2_host_chan *chan,
853 int chnum, struct dwc2_qtd *qtd,
854 enum dwc2_halt_status halt_status)
855{
856 dev_vdbg(hsotg->dev, "%s()\n", __func__);
857
858 qtd->error_count = 0;
859
860 if (chan->hcint & HCINTMSK_NYET) {
861 /*
862 * Got a NYET on the last transaction of the transfer. This
863 * means that the endpoint should be in the PING state at the
864 * beginning of the next transfer.
865 */
866 dev_vdbg(hsotg->dev, "got NYET\n");
867 chan->qh->ping_state = 1;
868 }
869
870 /*
871 * Always halt and release the host channel to make it available for
872 * more transfers. There may still be more phases for a control
873 * transfer or more data packets for a bulk transfer at this point,
874 * but the host channel is still halted. A channel will be reassigned
875 * to the transfer when the non-periodic schedule is processed after
876 * the channel is released. This allows transactions to be queued
877 * properly via dwc2_hcd_queue_transactions, which also enables the
878 * Tx FIFO Empty interrupt if necessary.
879 */
880 if (chan->ep_is_in) {
881 /*
882 * IN transfers in Slave mode require an explicit disable to
883 * halt the channel. (In DMA mode, this call simply releases
884 * the channel.)
885 */
886 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
887 } else {
888 /*
889 * The channel is automatically disabled by the core for OUT
890 * transfers in Slave mode
891 */
892 dwc2_release_channel(hsotg, chan, qtd, halt_status);
893 }
894}
895
896/*
897 * Performs common cleanup for periodic transfers after a Transfer Complete
898 * interrupt. This function should be called after any endpoint type specific
899 * handling is finished to release the host channel.
900 */
901static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
902 struct dwc2_host_chan *chan, int chnum,
903 struct dwc2_qtd *qtd,
904 enum dwc2_halt_status halt_status)
905{
906 u32 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
907
908 qtd->error_count = 0;
909
910 if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
911 /* Core halts channel in these cases */
912 dwc2_release_channel(hsotg, chan, qtd, halt_status);
913 else
914 /* Flush any outstanding requests from the Tx queue */
915 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
916}
917
918static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
919 struct dwc2_host_chan *chan, int chnum,
920 struct dwc2_qtd *qtd)
921{
922 struct dwc2_hcd_iso_packet_desc *frame_desc;
923 u32 len;
924 u32 hctsiz;
925 u32 pid;
926
927 if (!qtd->urb)
928 return 0;
929
930 frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
931 len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
932 DWC2_HC_XFER_COMPLETE, NULL);
933 if (!len) {
934 qtd->complete_split = 0;
935 qtd->isoc_split_offset = 0;
936 return 0;
937 }
938
939 frame_desc->actual_length += len;
940
941 qtd->isoc_split_offset += len;
942
943 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
944 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
945
946 if (frame_desc->actual_length >= frame_desc->length || pid == 0) {
947 frame_desc->status = 0;
948 qtd->isoc_frame_index++;
949 qtd->complete_split = 0;
950 qtd->isoc_split_offset = 0;
951 }
952
953 if (qtd->isoc_frame_index == qtd->urb->packet_count) {
954 dwc2_host_complete(hsotg, qtd, 0);
955 dwc2_release_channel(hsotg, chan, qtd,
956 DWC2_HC_XFER_URB_COMPLETE);
957 } else {
958 dwc2_release_channel(hsotg, chan, qtd,
959 DWC2_HC_XFER_NO_HALT_STATUS);
960 }
961
962 return 1; /* Indicates that channel released */
963}
964
965/*
966 * Handles a host channel Transfer Complete interrupt. This handler may be
967 * called in either DMA mode or Slave mode.
968 */
969static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
970 struct dwc2_host_chan *chan, int chnum,
971 struct dwc2_qtd *qtd)
972{
973 struct dwc2_hcd_urb *urb = qtd->urb;
974 enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
975 int pipe_type;
976 int urb_xfer_done;
977
978 if (dbg_hc(chan))
979 dev_vdbg(hsotg->dev,
980 "--Host Channel %d Interrupt: Transfer Complete--\n",
981 chnum);
982
983 if (!urb)
984 goto handle_xfercomp_done;
985
986 pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
987
988 if (hsotg->params.dma_desc_enable) {
989 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
990 if (pipe_type == USB_ENDPOINT_XFER_ISOC)
991 /* Do not disable the interrupt, just clear it */
992 return;
993 goto handle_xfercomp_done;
994 }
995
996 /* Handle xfer complete on CSPLIT */
997 if (chan->qh->do_split) {
998 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
999 hsotg->params.host_dma) {
1000 if (qtd->complete_split &&
1001 dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
1002 qtd))
1003 goto handle_xfercomp_done;
1004 } else {
1005 qtd->complete_split = 0;
1006 }
1007 }
1008
1009 /* Update the QTD and URB states */
1010 switch (pipe_type) {
1011 case USB_ENDPOINT_XFER_CONTROL:
1012 switch (qtd->control_phase) {
1013 case DWC2_CONTROL_SETUP:
1014 if (urb->length > 0)
1015 qtd->control_phase = DWC2_CONTROL_DATA;
1016 else
1017 qtd->control_phase = DWC2_CONTROL_STATUS;
1018 dev_vdbg(hsotg->dev,
1019 " Control setup transaction done\n");
1020 halt_status = DWC2_HC_XFER_COMPLETE;
1021 break;
1022 case DWC2_CONTROL_DATA:
1023 urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1024 chnum, urb, qtd);
1025 if (urb_xfer_done) {
1026 qtd->control_phase = DWC2_CONTROL_STATUS;
1027 dev_vdbg(hsotg->dev,
1028 " Control data transfer done\n");
1029 } else {
1030 dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1031 qtd);
1032 }
1033 halt_status = DWC2_HC_XFER_COMPLETE;
1034 break;
1035 case DWC2_CONTROL_STATUS:
1036 dev_vdbg(hsotg->dev, " Control transfer complete\n");
1037 if (urb->status == -EINPROGRESS)
1038 urb->status = 0;
1039 dwc2_host_complete(hsotg, qtd, urb->status);
1040 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1041 break;
1042 }
1043
1044 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1045 halt_status);
1046 break;
1047 case USB_ENDPOINT_XFER_BULK:
1048 dev_vdbg(hsotg->dev, " Bulk transfer complete\n");
1049 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1050 qtd);
1051 if (urb_xfer_done) {
1052 dwc2_host_complete(hsotg, qtd, urb->status);
1053 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1054 } else {
1055 halt_status = DWC2_HC_XFER_COMPLETE;
1056 }
1057
1058 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1059 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1060 halt_status);
1061 break;
1062 case USB_ENDPOINT_XFER_INT:
1063 dev_vdbg(hsotg->dev, " Interrupt transfer complete\n");
1064 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1065 qtd);
1066
1067 /*
1068 * Interrupt URB is done on the first transfer complete
1069 * interrupt
1070 */
1071 if (urb_xfer_done) {
1072 dwc2_host_complete(hsotg, qtd, urb->status);
1073 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1074 } else {
1075 halt_status = DWC2_HC_XFER_COMPLETE;
1076 }
1077
1078 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1079 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1080 halt_status);
1081 break;
1082 case USB_ENDPOINT_XFER_ISOC:
1083 if (dbg_perio())
1084 dev_vdbg(hsotg->dev, " Isochronous transfer complete\n");
1085 if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1086 halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1087 chnum, qtd,
1088 DWC2_HC_XFER_COMPLETE);
1089 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1090 halt_status);
1091 break;
1092 }
1093
1094handle_xfercomp_done:
1095 disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1096}
1097
1098/*
1099 * Handles a host channel STALL interrupt. This handler may be called in
1100 * either DMA mode or Slave mode.
1101 */
1102static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1103 struct dwc2_host_chan *chan, int chnum,
1104 struct dwc2_qtd *qtd)
1105{
1106 struct dwc2_hcd_urb *urb = qtd->urb;
1107 int pipe_type;
1108
1109 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1110 chnum);
1111
1112 if (hsotg->params.dma_desc_enable) {
1113 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1114 DWC2_HC_XFER_STALL);
1115 goto handle_stall_done;
1116 }
1117
1118 if (!urb)
1119 goto handle_stall_halt;
1120
1121 pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1122
1123 if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1124 dwc2_host_complete(hsotg, qtd, -EPIPE);
1125
1126 if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1127 pipe_type == USB_ENDPOINT_XFER_INT) {
1128 dwc2_host_complete(hsotg, qtd, -EPIPE);
1129 /*
1130 * USB protocol requires resetting the data toggle for bulk
1131 * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1132 * setup command is issued to the endpoint. Anticipate the
1133 * CLEAR_FEATURE command since a STALL has occurred and reset
1134 * the data toggle now.
1135 */
1136 chan->qh->data_toggle = 0;
1137 }
1138
1139handle_stall_halt:
1140 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);
1141
1142handle_stall_done:
1143 disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1144}
1145
1146/*
1147 * Updates the state of the URB when a transfer has been stopped due to an
1148 * abnormal condition before the transfer completes. Modifies the
1149 * actual_length field of the URB to reflect the number of bytes that have
1150 * actually been transferred via the host channel.
1151 */
1152static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1153 struct dwc2_host_chan *chan, int chnum,
1154 struct dwc2_hcd_urb *urb,
1155 struct dwc2_qtd *qtd,
1156 enum dwc2_halt_status halt_status)
1157{
1158 u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1159 qtd, halt_status, NULL);
1160 u32 hctsiz;
1161
1162 if (urb->actual_length + xfer_length > urb->length) {
1163 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1164 xfer_length = urb->length - urb->actual_length;
1165 }
1166
1167 urb->actual_length += xfer_length;
1168
1169 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
1170 dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1171 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1172 dev_vdbg(hsotg->dev, " chan->start_pkt_count %d\n",
1173 chan->start_pkt_count);
1174 dev_vdbg(hsotg->dev, " hctsiz.pktcnt %d\n",
1175 (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1176 dev_vdbg(hsotg->dev, " chan->max_packet %d\n", chan->max_packet);
1177 dev_vdbg(hsotg->dev, " bytes_transferred %d\n",
1178 xfer_length);
1179 dev_vdbg(hsotg->dev, " urb->actual_length %d\n",
1180 urb->actual_length);
1181 dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n",
1182 urb->length);
1183}
1184
1185/*
1186 * Handles a host channel NAK interrupt. This handler may be called in either
1187 * DMA mode or Slave mode.
1188 */
1189static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1190 struct dwc2_host_chan *chan, int chnum,
1191 struct dwc2_qtd *qtd)
1192{
1193 if (!qtd) {
1194 dev_dbg(hsotg->dev, "%s: qtd is NULL\n", __func__);
1195 return;
1196 }
1197
1198 if (!qtd->urb) {
1199 dev_dbg(hsotg->dev, "%s: qtd->urb is NULL\n", __func__);
1200 return;
1201 }
1202
1203 if (dbg_hc(chan))
1204 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1205 chnum);
1206
1207 /*
1208 * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1209 * interrupt. Re-start the SSPLIT transfer.
1210 *
1211 * Normally for non-periodic transfers we'll retry right away, but to
1212 * avoid interrupt storms we'll wait before retrying if we've got
1213 * several NAKs. If we didn't do this we'd retry directly from the
1214 * interrupt handler and could end up quickly getting another
1215 * interrupt (another NAK), which we'd retry.
1216 *
1217 * Note that in DMA mode software only gets involved to re-send NAKed
1218 * transfers for split transactions, so we only need to apply this
1219 * delaying logic when handling splits. In non-DMA mode presumably we
1220 * might want a similar delay if someone can demonstrate this problem
1221 * affects that code path too.
1222 */
1223 if (chan->do_split) {
1224 if (chan->complete_split)
1225 qtd->error_count = 0;
1226 qtd->complete_split = 0;
1227 qtd->num_naks++;
1228 qtd->qh->want_wait = qtd->num_naks >= DWC2_NAKS_BEFORE_DELAY;
1229 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1230 goto handle_nak_done;
1231 }
1232
1233 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1234 case USB_ENDPOINT_XFER_CONTROL:
1235 case USB_ENDPOINT_XFER_BULK:
1236 if (hsotg->params.host_dma && chan->ep_is_in) {
1237 /*
1238 * NAK interrupts are enabled on bulk/control IN
1239 * transfers in DMA mode for the sole purpose of
1240 * resetting the error count after a transaction error
1241 * occurs. The core will continue transferring data.
1242 */
1243 qtd->error_count = 0;
1244 break;
1245 }
1246
1247 /*
1248 * NAK interrupts normally occur during OUT transfers in DMA
1249 * or Slave mode. For IN transfers, more requests will be
1250 * queued as request queue space is available.
1251 */
1252 qtd->error_count = 0;
1253
1254 if (!chan->qh->ping_state) {
1255 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1256 qtd, DWC2_HC_XFER_NAK);
1257 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1258
1259 if (chan->speed == USB_SPEED_HIGH)
1260 chan->qh->ping_state = 1;
1261 }
1262
1263 /*
1264 * Halt the channel so the transfer can be re-started from
1265 * the appropriate point or the PING protocol will
1266 * start/continue
1267 */
1268 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1269 break;
1270 case USB_ENDPOINT_XFER_INT:
1271 qtd->error_count = 0;
1272 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1273 break;
1274 case USB_ENDPOINT_XFER_ISOC:
1275 /* Should never get called for isochronous transfers */
1276 dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1277 break;
1278 }
1279
1280handle_nak_done:
1281 disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1282}
1283
1284/*
1285 * Handles a host channel ACK interrupt. This interrupt is enabled when
1286 * performing the PING protocol in Slave mode, when errors occur during
1287 * either Slave mode or DMA mode, and during Start Split transactions.
1288 */
1289static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1290 struct dwc2_host_chan *chan, int chnum,
1291 struct dwc2_qtd *qtd)
1292{
1293 struct dwc2_hcd_iso_packet_desc *frame_desc;
1294
1295 if (dbg_hc(chan))
1296 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1297 chnum);
1298
1299 if (chan->do_split) {
1300 /* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1301 if (!chan->ep_is_in &&
1302 chan->data_pid_start != DWC2_HC_PID_SETUP)
1303 qtd->ssplit_out_xfer_count = chan->xfer_len;
1304
1305 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1306 qtd->complete_split = 1;
1307 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1308 } else {
1309 /* ISOC OUT */
1310 switch (chan->xact_pos) {
1311 case DWC2_HCSPLT_XACTPOS_ALL:
1312 break;
1313 case DWC2_HCSPLT_XACTPOS_END:
1314 qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1315 qtd->isoc_split_offset = 0;
1316 break;
1317 case DWC2_HCSPLT_XACTPOS_BEGIN:
1318 case DWC2_HCSPLT_XACTPOS_MID:
1319 /*
1320 * For BEGIN or MID, calculate the length for
1321 * the next microframe to determine the correct
1322 * SSPLIT token, either MID or END
1323 */
1324 frame_desc = &qtd->urb->iso_descs[
1325 qtd->isoc_frame_index];
1326 qtd->isoc_split_offset += 188;
1327
1328 if (frame_desc->length - qtd->isoc_split_offset
1329 <= 188)
1330 qtd->isoc_split_pos =
1331 DWC2_HCSPLT_XACTPOS_END;
1332 else
1333 qtd->isoc_split_pos =
1334 DWC2_HCSPLT_XACTPOS_MID;
1335 break;
1336 }
1337 }
1338 } else {
1339 qtd->error_count = 0;
1340
1341 if (chan->qh->ping_state) {
1342 chan->qh->ping_state = 0;
1343 /*
1344 * Halt the channel so the transfer can be re-started
1345 * from the appropriate point. This only happens in
1346 * Slave mode. In DMA mode, the ping_state is cleared
1347 * when the transfer is started because the core
1348 * automatically executes the PING, then the transfer.
1349 */
1350 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1351 }
1352 }
1353
1354 /*
1355 * If the ACK occurred when _not_ in the PING state, let the channel
1356 * continue transferring data after clearing the error count
1357 */
1358 disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1359}
1360
1361/*
1362 * Handles a host channel NYET interrupt. This interrupt should only occur on
1363 * Bulk and Control OUT endpoints and for complete split transactions. If a
1364 * NYET occurs at the same time as a Transfer Complete interrupt, it is
1365 * handled in the xfercomp interrupt handler, not here. This handler may be
1366 * called in either DMA mode or Slave mode.
1367 */
1368static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1369 struct dwc2_host_chan *chan, int chnum,
1370 struct dwc2_qtd *qtd)
1371{
1372 if (dbg_hc(chan))
1373 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1374 chnum);
1375
1376 /*
1377 * NYET on CSPLIT
1378 * re-do the CSPLIT immediately on non-periodic
1379 */
1380 if (chan->do_split && chan->complete_split) {
1381 if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1382 hsotg->params.host_dma) {
1383 qtd->complete_split = 0;
1384 qtd->isoc_split_offset = 0;
1385 qtd->isoc_frame_index++;
1386 if (qtd->urb &&
1387 qtd->isoc_frame_index == qtd->urb->packet_count) {
1388 dwc2_host_complete(hsotg, qtd, 0);
1389 dwc2_release_channel(hsotg, chan, qtd,
1390 DWC2_HC_XFER_URB_COMPLETE);
1391 } else {
1392 dwc2_release_channel(hsotg, chan, qtd,
1393 DWC2_HC_XFER_NO_HALT_STATUS);
1394 }
1395 goto handle_nyet_done;
1396 }
1397
1398 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1399 chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1400 struct dwc2_qh *qh = chan->qh;
1401 bool past_end;
1402
1403 if (!hsotg->params.uframe_sched) {
1404 int frnum = dwc2_hcd_get_frame_number(hsotg);
1405
1406 /* Don't have num_hs_transfers; simple logic */
1407 past_end = dwc2_full_frame_num(frnum) !=
1408 dwc2_full_frame_num(qh->next_active_frame);
1409 } else {
1410 int end_frnum;
1411
1412 /*
1413 * Figure out the end frame based on
1414 * schedule.
1415 *
1416 * We don't want to go on trying again
1417 * and again forever. Let's stop when
1418 * we've done all the transfers that
1419 * were scheduled.
1420 *
1421 * We're going to be comparing
1422 * start_active_frame and
1423 * next_active_frame, both of which
1424 * are 1 before the time the packet
1425 * goes on the wire, so that cancels
1426 * out. Basically if had 1 transfer
1427 * and we saw 1 NYET then we're done.
1428 * We're getting a NYET here so if
1429 * next >= (start + num_transfers)
1430 * we're done. The complexity is that
1431 * for all but ISOC_OUT we skip one
1432 * slot.
1433 */
1434 end_frnum = dwc2_frame_num_inc(
1435 qh->start_active_frame,
1436 qh->num_hs_transfers);
1437
1438 if (qh->ep_type != USB_ENDPOINT_XFER_ISOC ||
1439 qh->ep_is_in)
1440 end_frnum =
1441 dwc2_frame_num_inc(end_frnum, 1);
1442
1443 past_end = dwc2_frame_num_le(
1444 end_frnum, qh->next_active_frame);
1445 }
1446
1447 if (past_end) {
1448 /* Treat this as a transaction error. */
1449#if 0
1450 /*
1451 * Todo: Fix system performance so this can
1452 * be treated as an error. Right now complete
1453 * splits cannot be scheduled precisely enough
1454 * due to other system activity, so this error
1455 * occurs regularly in Slave mode.
1456 */
1457 qtd->error_count++;
1458#endif
1459 qtd->complete_split = 0;
1460 dwc2_halt_channel(hsotg, chan, qtd,
1461 DWC2_HC_XFER_XACT_ERR);
1462 /* Todo: add support for isoc release */
1463 goto handle_nyet_done;
1464 }
1465 }
1466
1467 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1468 goto handle_nyet_done;
1469 }
1470
1471 chan->qh->ping_state = 1;
1472 qtd->error_count = 0;
1473
1474 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
1475 DWC2_HC_XFER_NYET);
1476 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1477
1478 /*
1479 * Halt the channel and re-start the transfer so the PING protocol
1480 * will start
1481 */
1482 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1483
1484handle_nyet_done:
1485 disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1486}
1487
1488/*
1489 * Handles a host channel babble interrupt. This handler may be called in
1490 * either DMA mode or Slave mode.
1491 */
1492static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1493 struct dwc2_host_chan *chan, int chnum,
1494 struct dwc2_qtd *qtd)
1495{
1496 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1497 chnum);
1498
1499 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1500
1501 if (hsotg->params.dma_desc_enable) {
1502 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1503 DWC2_HC_XFER_BABBLE_ERR);
1504 goto disable_int;
1505 }
1506
1507 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1508 dwc2_host_complete(hsotg, qtd, -EOVERFLOW);
1509 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
1510 } else {
1511 enum dwc2_halt_status halt_status;
1512
1513 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1514 qtd, DWC2_HC_XFER_BABBLE_ERR);
1515 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1516 }
1517
1518disable_int:
1519 disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1520}
1521
1522/*
1523 * Handles a host channel AHB error interrupt. This handler is only called in
1524 * DMA mode.
1525 */
1526static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1527 struct dwc2_host_chan *chan, int chnum,
1528 struct dwc2_qtd *qtd)
1529{
1530 struct dwc2_hcd_urb *urb = qtd->urb;
1531 char *pipetype, *speed;
1532 u32 hcchar;
1533 u32 hcsplt;
1534 u32 hctsiz;
1535 u32 hc_dma;
1536
1537 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1538 chnum);
1539
1540 if (!urb)
1541 goto handle_ahberr_halt;
1542
1543 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1544
1545 hcchar = dwc2_readl(hsotg->regs + HCCHAR(chnum));
1546 hcsplt = dwc2_readl(hsotg->regs + HCSPLT(chnum));
1547 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
1548 hc_dma = dwc2_readl(hsotg->regs + HCDMA(chnum));
1549
1550 dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1551 dev_err(hsotg->dev, " hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1552 dev_err(hsotg->dev, " hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1553 dev_err(hsotg->dev, " Device address: %d\n",
1554 dwc2_hcd_get_dev_addr(&urb->pipe_info));
1555 dev_err(hsotg->dev, " Endpoint: %d, %s\n",
1556 dwc2_hcd_get_ep_num(&urb->pipe_info),
1557 dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1558
1559 switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
1560 case USB_ENDPOINT_XFER_CONTROL:
1561 pipetype = "CONTROL";
1562 break;
1563 case USB_ENDPOINT_XFER_BULK:
1564 pipetype = "BULK";
1565 break;
1566 case USB_ENDPOINT_XFER_INT:
1567 pipetype = "INTERRUPT";
1568 break;
1569 case USB_ENDPOINT_XFER_ISOC:
1570 pipetype = "ISOCHRONOUS";
1571 break;
1572 default:
1573 pipetype = "UNKNOWN";
1574 break;
1575 }
1576
1577 dev_err(hsotg->dev, " Endpoint type: %s\n", pipetype);
1578
1579 switch (chan->speed) {
1580 case USB_SPEED_HIGH:
1581 speed = "HIGH";
1582 break;
1583 case USB_SPEED_FULL:
1584 speed = "FULL";
1585 break;
1586 case USB_SPEED_LOW:
1587 speed = "LOW";
1588 break;
1589 default:
1590 speed = "UNKNOWN";
1591 break;
1592 }
1593
1594 dev_err(hsotg->dev, " Speed: %s\n", speed);
1595
1596 dev_err(hsotg->dev, " Max packet size: %d\n",
1597 dwc2_hcd_get_mps(&urb->pipe_info));
1598 dev_err(hsotg->dev, " Data buffer length: %d\n", urb->length);
1599 dev_err(hsotg->dev, " Transfer buffer: %p, Transfer DMA: %08lx\n",
1600 urb->buf, (unsigned long)urb->dma);
1601 dev_err(hsotg->dev, " Setup buffer: %p, Setup DMA: %08lx\n",
1602 urb->setup_packet, (unsigned long)urb->setup_dma);
1603 dev_err(hsotg->dev, " Interval: %d\n", urb->interval);
1604
1605 /* Core halts the channel for Descriptor DMA mode */
1606 if (hsotg->params.dma_desc_enable) {
1607 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1608 DWC2_HC_XFER_AHB_ERR);
1609 goto handle_ahberr_done;
1610 }
1611
1612 dwc2_host_complete(hsotg, qtd, -EIO);
1613
1614handle_ahberr_halt:
1615 /*
1616 * Force a channel halt. Don't call dwc2_halt_channel because that won't
1617 * write to the HCCHARn register in DMA mode to force the halt.
1618 */
1619 dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);
1620
1621handle_ahberr_done:
1622 disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1623}
1624
1625/*
1626 * Handles a host channel transaction error interrupt. This handler may be
1627 * called in either DMA mode or Slave mode.
1628 */
1629static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1630 struct dwc2_host_chan *chan, int chnum,
1631 struct dwc2_qtd *qtd)
1632{
1633 dev_dbg(hsotg->dev,
1634 "--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1635
1636 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1637
1638 if (hsotg->params.dma_desc_enable) {
1639 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1640 DWC2_HC_XFER_XACT_ERR);
1641 goto handle_xacterr_done;
1642 }
1643
1644 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1645 case USB_ENDPOINT_XFER_CONTROL:
1646 case USB_ENDPOINT_XFER_BULK:
1647 qtd->error_count++;
1648 if (!chan->qh->ping_state) {
1649 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1650 qtd, DWC2_HC_XFER_XACT_ERR);
1651 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1652 if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1653 chan->qh->ping_state = 1;
1654 }
1655
1656 /*
1657 * Halt the channel so the transfer can be re-started from
1658 * the appropriate point or the PING protocol will start
1659 */
1660 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1661 break;
1662 case USB_ENDPOINT_XFER_INT:
1663 qtd->error_count++;
1664 if (chan->do_split && chan->complete_split)
1665 qtd->complete_split = 0;
1666 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1667 break;
1668 case USB_ENDPOINT_XFER_ISOC:
1669 {
1670 enum dwc2_halt_status halt_status;
1671
1672 halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1673 chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1674 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1675 }
1676 break;
1677 }
1678
1679handle_xacterr_done:
1680 disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1681}
1682
1683/*
1684 * Handles a host channel frame overrun interrupt. This handler may be called
1685 * in either DMA mode or Slave mode.
1686 */
1687static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1688 struct dwc2_host_chan *chan, int chnum,
1689 struct dwc2_qtd *qtd)
1690{
1691 enum dwc2_halt_status halt_status;
1692
1693 if (dbg_hc(chan))
1694 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1695 chnum);
1696
1697 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1698
1699 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1700 case USB_ENDPOINT_XFER_CONTROL:
1701 case USB_ENDPOINT_XFER_BULK:
1702 break;
1703 case USB_ENDPOINT_XFER_INT:
1704 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1705 break;
1706 case USB_ENDPOINT_XFER_ISOC:
1707 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1708 qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1709 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1710 break;
1711 }
1712
1713 disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1714}
1715
1716/*
1717 * Handles a host channel data toggle error interrupt. This handler may be
1718 * called in either DMA mode or Slave mode.
1719 */
1720static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1721 struct dwc2_host_chan *chan, int chnum,
1722 struct dwc2_qtd *qtd)
1723{
1724 dev_dbg(hsotg->dev,
1725 "--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1726
1727 if (chan->ep_is_in)
1728 qtd->error_count = 0;
1729 else
1730 dev_err(hsotg->dev,
1731 "Data Toggle Error on OUT transfer, channel %d\n",
1732 chnum);
1733
1734 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1735 disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1736}
1737
1738/*
1739 * For debug only. It checks that a valid halt status is set and that
1740 * HCCHARn.chdis is clear. If there's a problem, corrective action is
1741 * taken and a warning is issued.
1742 *
1743 * Return: true if halt status is ok, false otherwise
1744 */
1745static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1746 struct dwc2_host_chan *chan, int chnum,
1747 struct dwc2_qtd *qtd)
1748{
1749#ifdef DEBUG
1750 u32 hcchar;
1751 u32 hctsiz;
1752 u32 hcintmsk;
1753 u32 hcsplt;
1754
1755 if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1756 /*
1757 * This code is here only as a check. This condition should
1758 * never happen. Ignore the halt if it does occur.
1759 */
1760 hcchar = dwc2_readl(hsotg->regs + HCCHAR(chnum));
1761 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
1762 hcintmsk = dwc2_readl(hsotg->regs + HCINTMSK(chnum));
1763 hcsplt = dwc2_readl(hsotg->regs + HCSPLT(chnum));
1764 dev_dbg(hsotg->dev,
1765 "%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1766 __func__);
1767 dev_dbg(hsotg->dev,
1768 "channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1769 chnum, hcchar, hctsiz);
1770 dev_dbg(hsotg->dev,
1771 "hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1772 chan->hcint, hcintmsk, hcsplt);
1773 if (qtd)
1774 dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1775 qtd->complete_split);
1776 dev_warn(hsotg->dev,
1777 "%s: no halt status, channel %d, ignoring interrupt\n",
1778 __func__, chnum);
1779 return false;
1780 }
1781
1782 /*
1783 * This code is here only as a check. hcchar.chdis should never be set
1784 * when the halt interrupt occurs. Halt the channel again if it does
1785 * occur.
1786 */
1787 hcchar = dwc2_readl(hsotg->regs + HCCHAR(chnum));
1788 if (hcchar & HCCHAR_CHDIS) {
1789 dev_warn(hsotg->dev,
1790 "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1791 __func__, hcchar);
1792 chan->halt_pending = 0;
1793 dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
1794 return false;
1795 }
1796#endif
1797
1798 return true;
1799}
1800
1801/*
1802 * Handles a host Channel Halted interrupt in DMA mode. This handler
1803 * determines the reason the channel halted and proceeds accordingly.
1804 */
1805static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1806 struct dwc2_host_chan *chan, int chnum,
1807 struct dwc2_qtd *qtd)
1808{
1809 u32 hcintmsk;
1810 int out_nak_enh = 0;
1811
1812 if (dbg_hc(chan))
1813 dev_vdbg(hsotg->dev,
1814 "--Host Channel %d Interrupt: DMA Channel Halted--\n",
1815 chnum);
1816
1817 /*
1818 * For core with OUT NAK enhancement, the flow for high-speed
1819 * CONTROL/BULK OUT is handled a little differently
1820 */
1821 if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1822 if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1823 (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1824 chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1825 out_nak_enh = 1;
1826 }
1827 }
1828
1829 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1830 (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1831 !hsotg->params.dma_desc_enable)) {
1832 if (hsotg->params.dma_desc_enable)
1833 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1834 chan->halt_status);
1835 else
1836 /*
1837 * Just release the channel. A dequeue can happen on a
1838 * transfer timeout. In the case of an AHB Error, the
1839 * channel was forced to halt because there's no way to
1840 * gracefully recover.
1841 */
1842 dwc2_release_channel(hsotg, chan, qtd,
1843 chan->halt_status);
1844 return;
1845 }
1846
1847 hcintmsk = dwc2_readl(hsotg->regs + HCINTMSK(chnum));
1848
1849 if (chan->hcint & HCINTMSK_XFERCOMPL) {
1850 /*
1851 * Todo: This is here because of a possible hardware bug. Spec
1852 * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1853 * interrupt w/ACK bit set should occur, but I only see the
1854 * XFERCOMP bit, even with it masked out. This is a workaround
1855 * for that behavior. Should fix this when hardware is fixed.
1856 */
1857 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1858 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1859 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1860 } else if (chan->hcint & HCINTMSK_STALL) {
1861 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1862 } else if ((chan->hcint & HCINTMSK_XACTERR) &&
1863 !hsotg->params.dma_desc_enable) {
1864 if (out_nak_enh) {
1865 if (chan->hcint &
1866 (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1867 dev_vdbg(hsotg->dev,
1868 "XactErr with NYET/NAK/ACK\n");
1869 qtd->error_count = 0;
1870 } else {
1871 dev_vdbg(hsotg->dev,
1872 "XactErr without NYET/NAK/ACK\n");
1873 }
1874 }
1875
1876 /*
1877 * Must handle xacterr before nak or ack. Could get a xacterr
1878 * at the same time as either of these on a BULK/CONTROL OUT
1879 * that started with a PING. The xacterr takes precedence.
1880 */
1881 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1882 } else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1883 hsotg->params.dma_desc_enable) {
1884 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1885 } else if ((chan->hcint & HCINTMSK_AHBERR) &&
1886 hsotg->params.dma_desc_enable) {
1887 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1888 } else if (chan->hcint & HCINTMSK_BBLERR) {
1889 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1890 } else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1891 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1892 } else if (!out_nak_enh) {
1893 if (chan->hcint & HCINTMSK_NYET) {
1894 /*
1895 * Must handle nyet before nak or ack. Could get a nyet
1896 * at the same time as either of those on a BULK/CONTROL
1897 * OUT that started with a PING. The nyet takes
1898 * precedence.
1899 */
1900 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1901 } else if ((chan->hcint & HCINTMSK_NAK) &&
1902 !(hcintmsk & HCINTMSK_NAK)) {
1903 /*
1904 * If nak is not masked, it's because a non-split IN
1905 * transfer is in an error state. In that case, the nak
1906 * is handled by the nak interrupt handler, not here.
1907 * Handle nak here for BULK/CONTROL OUT transfers, which
1908 * halt on a NAK to allow rewinding the buffer pointer.
1909 */
1910 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1911 } else if ((chan->hcint & HCINTMSK_ACK) &&
1912 !(hcintmsk & HCINTMSK_ACK)) {
1913 /*
1914 * If ack is not masked, it's because a non-split IN
1915 * transfer is in an error state. In that case, the ack
1916 * is handled by the ack interrupt handler, not here.
1917 * Handle ack here for split transfers. Start splits
1918 * halt on ACK.
1919 */
1920 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1921 } else {
1922 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1923 chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1924 /*
1925 * A periodic transfer halted with no other
1926 * channel interrupts set. Assume it was halted
1927 * by the core because it could not be completed
1928 * in its scheduled (micro)frame.
1929 */
1930 dev_dbg(hsotg->dev,
1931 "%s: Halt channel %d (assume incomplete periodic transfer)\n",
1932 __func__, chnum);
1933 dwc2_halt_channel(hsotg, chan, qtd,
1934 DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1935 } else {
1936 dev_err(hsotg->dev,
1937 "%s: Channel %d - ChHltd set, but reason is unknown\n",
1938 __func__, chnum);
1939 dev_err(hsotg->dev,
1940 "hcint 0x%08x, intsts 0x%08x\n",
1941 chan->hcint,
1942 dwc2_readl(hsotg->regs + GINTSTS));
1943 goto error;
1944 }
1945 }
1946 } else {
1947 dev_info(hsotg->dev,
1948 "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1949 chan->hcint);
1950error:
1951 /* Failthrough: use 3-strikes rule */
1952 qtd->error_count++;
1953 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1954 qtd, DWC2_HC_XFER_XACT_ERR);
1955 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1956 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1957 }
1958}
1959
1960/*
1961 * Handles a host channel Channel Halted interrupt
1962 *
1963 * In slave mode, this handler is called only when the driver specifically
1964 * requests a halt. This occurs during handling other host channel interrupts
1965 * (e.g. nak, xacterr, stall, nyet, etc.).
1966 *
1967 * In DMA mode, this is the interrupt that occurs when the core has finished
1968 * processing a transfer on a channel. Other host channel interrupts (except
1969 * ahberr) are disabled in DMA mode.
1970 */
1971static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
1972 struct dwc2_host_chan *chan, int chnum,
1973 struct dwc2_qtd *qtd)
1974{
1975 if (dbg_hc(chan))
1976 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
1977 chnum);
1978
1979 if (hsotg->params.host_dma) {
1980 dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
1981 } else {
1982 if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
1983 return;
1984 dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
1985 }
1986}
1987
1988/*
1989 * Check if the given qtd is still the top of the list (and thus valid).
1990 *
1991 * If dwc2_hcd_qtd_unlink_and_free() has been called since we grabbed
1992 * the qtd from the top of the list, this will return false (otherwise true).
1993 */
1994static bool dwc2_check_qtd_still_ok(struct dwc2_qtd *qtd, struct dwc2_qh *qh)
1995{
1996 struct dwc2_qtd *cur_head;
1997
1998 if (!qh)
1999 return false;
2000
2001 cur_head = list_first_entry(&qh->qtd_list, struct dwc2_qtd,
2002 qtd_list_entry);
2003 return (cur_head == qtd);
2004}
2005
2006/* Handles interrupt for a specific Host Channel */
2007static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
2008{
2009 struct dwc2_qtd *qtd;
2010 struct dwc2_host_chan *chan;
2011 u32 hcint, hcintmsk;
2012
2013 chan = hsotg->hc_ptr_array[chnum];
2014
2015 hcint = dwc2_readl(hsotg->regs + HCINT(chnum));
2016 hcintmsk = dwc2_readl(hsotg->regs + HCINTMSK(chnum));
2017 if (!chan) {
2018 dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
2019 dwc2_writel(hcint, hsotg->regs + HCINT(chnum));
2020 return;
2021 }
2022
2023 if (dbg_hc(chan)) {
2024 dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
2025 chnum);
2026 dev_vdbg(hsotg->dev,
2027 " hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2028 hcint, hcintmsk, hcint & hcintmsk);
2029 }
2030
2031 dwc2_writel(hcint, hsotg->regs + HCINT(chnum));
2032
2033 /*
2034 * If we got an interrupt after someone called
2035 * dwc2_hcd_endpoint_disable() we don't want to crash below
2036 */
2037 if (!chan->qh) {
2038 dev_warn(hsotg->dev, "Interrupt on disabled channel\n");
2039 return;
2040 }
2041
2042 chan->hcint = hcint;
2043 hcint &= hcintmsk;
2044
2045 /*
2046 * If the channel was halted due to a dequeue, the qtd list might
2047 * be empty or at least the first entry will not be the active qtd.
2048 * In this case, take a shortcut and just release the channel.
2049 */
2050 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
2051 /*
2052 * If the channel was halted, this should be the only
2053 * interrupt unmasked
2054 */
2055 WARN_ON(hcint != HCINTMSK_CHHLTD);
2056 if (hsotg->params.dma_desc_enable)
2057 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
2058 chan->halt_status);
2059 else
2060 dwc2_release_channel(hsotg, chan, NULL,
2061 chan->halt_status);
2062 return;
2063 }
2064
2065 if (list_empty(&chan->qh->qtd_list)) {
2066 /*
2067 * TODO: Will this ever happen with the
2068 * DWC2_HC_XFER_URB_DEQUEUE handling above?
2069 */
2070 dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
2071 chnum);
2072 dev_dbg(hsotg->dev,
2073 " hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2074 chan->hcint, hcintmsk, hcint);
2075 chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
2076 disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
2077 chan->hcint = 0;
2078 return;
2079 }
2080
2081 qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
2082 qtd_list_entry);
2083
2084 if (!hsotg->params.host_dma) {
2085 if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
2086 hcint &= ~HCINTMSK_CHHLTD;
2087 }
2088
2089 if (hcint & HCINTMSK_XFERCOMPL) {
2090 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
2091 /*
2092 * If NYET occurred at same time as Xfer Complete, the NYET is
2093 * handled by the Xfer Complete interrupt handler. Don't want
2094 * to call the NYET interrupt handler in this case.
2095 */
2096 hcint &= ~HCINTMSK_NYET;
2097 }
2098
2099 if (hcint & HCINTMSK_CHHLTD) {
2100 dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2101 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2102 goto exit;
2103 }
2104 if (hcint & HCINTMSK_AHBERR) {
2105 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2106 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2107 goto exit;
2108 }
2109 if (hcint & HCINTMSK_STALL) {
2110 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2111 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2112 goto exit;
2113 }
2114 if (hcint & HCINTMSK_NAK) {
2115 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2116 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2117 goto exit;
2118 }
2119 if (hcint & HCINTMSK_ACK) {
2120 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2121 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2122 goto exit;
2123 }
2124 if (hcint & HCINTMSK_NYET) {
2125 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2126 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2127 goto exit;
2128 }
2129 if (hcint & HCINTMSK_XACTERR) {
2130 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2131 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2132 goto exit;
2133 }
2134 if (hcint & HCINTMSK_BBLERR) {
2135 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2136 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2137 goto exit;
2138 }
2139 if (hcint & HCINTMSK_FRMOVRUN) {
2140 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2141 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2142 goto exit;
2143 }
2144 if (hcint & HCINTMSK_DATATGLERR) {
2145 dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2146 if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2147 goto exit;
2148 }
2149
2150exit:
2151 chan->hcint = 0;
2152}
2153
2154/*
2155 * This interrupt indicates that one or more host channels has a pending
2156 * interrupt. There are multiple conditions that can cause each host channel
2157 * interrupt. This function determines which conditions have occurred for each
2158 * host channel interrupt and handles them appropriately.
2159 */
2160static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2161{
2162 u32 haint;
2163 int i;
2164 struct dwc2_host_chan *chan, *chan_tmp;
2165
2166 haint = dwc2_readl(hsotg->regs + HAINT);
2167 if (dbg_perio()) {
2168 dev_vdbg(hsotg->dev, "%s()\n", __func__);
2169
2170 dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2171 }
2172
2173 /*
2174 * According to USB 2.0 spec section 11.18.8, a host must
2175 * issue complete-split transactions in a microframe for a
2176 * set of full-/low-speed endpoints in the same relative
2177 * order as the start-splits were issued in a microframe for.
2178 */
2179 list_for_each_entry_safe(chan, chan_tmp, &hsotg->split_order,
2180 split_order_list_entry) {
2181 int hc_num = chan->hc_num;
2182
2183 if (haint & (1 << hc_num)) {
2184 dwc2_hc_n_intr(hsotg, hc_num);
2185 haint &= ~(1 << hc_num);
2186 }
2187 }
2188
2189 for (i = 0; i < hsotg->params.host_channels; i++) {
2190 if (haint & (1 << i))
2191 dwc2_hc_n_intr(hsotg, i);
2192 }
2193}
2194
2195/* This function handles interrupts for the HCD */
2196irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2197{
2198 u32 gintsts, dbg_gintsts;
2199 irqreturn_t retval = IRQ_NONE;
2200
2201 if (!dwc2_is_controller_alive(hsotg)) {
2202 dev_warn(hsotg->dev, "Controller is dead\n");
2203 return retval;
2204 }
2205
2206 spin_lock(&hsotg->lock);
2207
2208 /* Check if HOST Mode */
2209 if (dwc2_is_host_mode(hsotg)) {
2210 gintsts = dwc2_read_core_intr(hsotg);
2211 if (!gintsts) {
2212 spin_unlock(&hsotg->lock);
2213 return retval;
2214 }
2215
2216 retval = IRQ_HANDLED;
2217
2218 dbg_gintsts = gintsts;
2219#ifndef DEBUG_SOF
2220 dbg_gintsts &= ~GINTSTS_SOF;
2221#endif
2222 if (!dbg_perio())
2223 dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
2224 GINTSTS_PTXFEMP);
2225
2226 /* Only print if there are any non-suppressed interrupts left */
2227 if (dbg_gintsts)
2228 dev_vdbg(hsotg->dev,
2229 "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2230 gintsts);
2231
2232 if (gintsts & GINTSTS_SOF)
2233 dwc2_sof_intr(hsotg);
2234 if (gintsts & GINTSTS_RXFLVL)
2235 dwc2_rx_fifo_level_intr(hsotg);
2236 if (gintsts & GINTSTS_NPTXFEMP)
2237 dwc2_np_tx_fifo_empty_intr(hsotg);
2238 if (gintsts & GINTSTS_PRTINT)
2239 dwc2_port_intr(hsotg);
2240 if (gintsts & GINTSTS_HCHINT)
2241 dwc2_hc_intr(hsotg);
2242 if (gintsts & GINTSTS_PTXFEMP)
2243 dwc2_perio_tx_fifo_empty_intr(hsotg);
2244
2245 if (dbg_gintsts) {
2246 dev_vdbg(hsotg->dev,
2247 "DWC OTG HCD Finished Servicing Interrupts\n");
2248 dev_vdbg(hsotg->dev,
2249 "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2250 dwc2_readl(hsotg->regs + GINTSTS),
2251 dwc2_readl(hsotg->regs + GINTMSK));
2252 }
2253 }
2254
2255 spin_unlock(&hsotg->lock);
2256
2257 return retval;
2258}