Loading...
Note: File does not exist in v3.1.
1/*
2 * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
3 *
4 * Copyright (C) 2004-2013 Synopsys, Inc.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions, and the following disclaimer,
11 * without modification.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 * 3. The names of the above-listed copyright holders may not be used
16 * to endorse or promote products derived from this software without
17 * specific prior written permission.
18 *
19 * ALTERNATIVELY, this software may be distributed under the terms of the
20 * GNU General Public License ("GPL") as published by the Free Software
21 * Foundation; either version 2 of the License, or (at your option) any
22 * later version.
23 *
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
25 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
26 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
27 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
28 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
29 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
30 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
31 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
32 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
33 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
34 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
35 */
36
37/*
38 * This file contains the interrupt handlers for Host mode
39 */
40#include <linux/kernel.h>
41#include <linux/module.h>
42#include <linux/spinlock.h>
43#include <linux/interrupt.h>
44#include <linux/dma-mapping.h>
45#include <linux/io.h>
46#include <linux/slab.h>
47#include <linux/usb.h>
48
49#include <linux/usb/hcd.h>
50#include <linux/usb/ch11.h>
51
52#include "core.h"
53#include "hcd.h"
54
55/* This function is for debug only */
56static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
57{
58#ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
59 u16 curr_frame_number = hsotg->frame_number;
60
61 if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
62 if (((hsotg->last_frame_num + 1) & HFNUM_MAX_FRNUM) !=
63 curr_frame_number) {
64 hsotg->frame_num_array[hsotg->frame_num_idx] =
65 curr_frame_number;
66 hsotg->last_frame_num_array[hsotg->frame_num_idx] =
67 hsotg->last_frame_num;
68 hsotg->frame_num_idx++;
69 }
70 } else if (!hsotg->dumped_frame_num_array) {
71 int i;
72
73 dev_info(hsotg->dev, "Frame Last Frame\n");
74 dev_info(hsotg->dev, "----- ----------\n");
75 for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
76 dev_info(hsotg->dev, "0x%04x 0x%04x\n",
77 hsotg->frame_num_array[i],
78 hsotg->last_frame_num_array[i]);
79 }
80 hsotg->dumped_frame_num_array = 1;
81 }
82 hsotg->last_frame_num = curr_frame_number;
83#endif
84}
85
86static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
87 struct dwc2_host_chan *chan,
88 struct dwc2_qtd *qtd)
89{
90 struct urb *usb_urb;
91
92 if (!chan->qh)
93 return;
94
95 if (chan->qh->dev_speed == USB_SPEED_HIGH)
96 return;
97
98 if (!qtd->urb)
99 return;
100
101 usb_urb = qtd->urb->priv;
102 if (!usb_urb || !usb_urb->dev || !usb_urb->dev->tt)
103 return;
104
105 if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
106 chan->qh->tt_buffer_dirty = 1;
107 if (usb_hub_clear_tt_buffer(usb_urb))
108 /* Clear failed; let's hope things work anyway */
109 chan->qh->tt_buffer_dirty = 0;
110 }
111}
112
113/*
114 * Handles the start-of-frame interrupt in host mode. Non-periodic
115 * transactions may be queued to the DWC_otg controller for the current
116 * (micro)frame. Periodic transactions may be queued to the controller
117 * for the next (micro)frame.
118 */
119static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
120{
121 struct list_head *qh_entry;
122 struct dwc2_qh *qh;
123 enum dwc2_transaction_type tr_type;
124
125#ifdef DEBUG_SOF
126 dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
127#endif
128
129 hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
130
131 dwc2_track_missed_sofs(hsotg);
132
133 /* Determine whether any periodic QHs should be executed */
134 qh_entry = hsotg->periodic_sched_inactive.next;
135 while (qh_entry != &hsotg->periodic_sched_inactive) {
136 qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
137 qh_entry = qh_entry->next;
138 if (dwc2_frame_num_le(qh->sched_frame, hsotg->frame_number))
139 /*
140 * Move QH to the ready list to be executed next
141 * (micro)frame
142 */
143 list_move(&qh->qh_list_entry,
144 &hsotg->periodic_sched_ready);
145 }
146 tr_type = dwc2_hcd_select_transactions(hsotg);
147 if (tr_type != DWC2_TRANSACTION_NONE)
148 dwc2_hcd_queue_transactions(hsotg, tr_type);
149
150 /* Clear interrupt */
151 writel(GINTSTS_SOF, hsotg->regs + GINTSTS);
152}
153
154/*
155 * Handles the Rx FIFO Level Interrupt, which indicates that there is
156 * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
157 * memory if the DWC_otg controller is operating in Slave mode.
158 */
159static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
160{
161 u32 grxsts, chnum, bcnt, dpid, pktsts;
162 struct dwc2_host_chan *chan;
163
164 if (dbg_perio())
165 dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
166
167 grxsts = readl(hsotg->regs + GRXSTSP);
168 chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
169 chan = hsotg->hc_ptr_array[chnum];
170 if (!chan) {
171 dev_err(hsotg->dev, "Unable to get corresponding channel\n");
172 return;
173 }
174
175 bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
176 dpid = (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT;
177 pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
178
179 /* Packet Status */
180 if (dbg_perio()) {
181 dev_vdbg(hsotg->dev, " Ch num = %d\n", chnum);
182 dev_vdbg(hsotg->dev, " Count = %d\n", bcnt);
183 dev_vdbg(hsotg->dev, " DPID = %d, chan.dpid = %d\n", dpid,
184 chan->data_pid_start);
185 dev_vdbg(hsotg->dev, " PStatus = %d\n", pktsts);
186 }
187
188 switch (pktsts) {
189 case GRXSTS_PKTSTS_HCHIN:
190 /* Read the data into the host buffer */
191 if (bcnt > 0) {
192 dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);
193
194 /* Update the HC fields for the next packet received */
195 chan->xfer_count += bcnt;
196 chan->xfer_buf += bcnt;
197 }
198 break;
199 case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
200 case GRXSTS_PKTSTS_DATATOGGLEERR:
201 case GRXSTS_PKTSTS_HCHHALTED:
202 /* Handled in interrupt, just ignore data */
203 break;
204 default:
205 dev_err(hsotg->dev,
206 "RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
207 break;
208 }
209}
210
211/*
212 * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
213 * data packets may be written to the FIFO for OUT transfers. More requests
214 * may be written to the non-periodic request queue for IN transfers. This
215 * interrupt is enabled only in Slave mode.
216 */
217static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
218{
219 dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
220 dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
221}
222
223/*
224 * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
225 * packets may be written to the FIFO for OUT transfers. More requests may be
226 * written to the periodic request queue for IN transfers. This interrupt is
227 * enabled only in Slave mode.
228 */
229static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
230{
231 if (dbg_perio())
232 dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
233 dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
234}
235
236static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
237 u32 *hprt0_modify)
238{
239 struct dwc2_core_params *params = hsotg->core_params;
240 int do_reset = 0;
241 u32 usbcfg;
242 u32 prtspd;
243 u32 hcfg;
244 u32 fslspclksel;
245 u32 hfir;
246
247 dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
248
249 /* Every time when port enables calculate HFIR.FrInterval */
250 hfir = readl(hsotg->regs + HFIR);
251 hfir &= ~HFIR_FRINT_MASK;
252 hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
253 HFIR_FRINT_MASK;
254 writel(hfir, hsotg->regs + HFIR);
255
256 /* Check if we need to adjust the PHY clock speed for low power */
257 if (!params->host_support_fs_ls_low_power) {
258 /* Port has been enabled, set the reset change flag */
259 hsotg->flags.b.port_reset_change = 1;
260 return;
261 }
262
263 usbcfg = readl(hsotg->regs + GUSBCFG);
264 prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
265
266 if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
267 /* Low power */
268 if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
269 /* Set PHY low power clock select for FS/LS devices */
270 usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
271 writel(usbcfg, hsotg->regs + GUSBCFG);
272 do_reset = 1;
273 }
274
275 hcfg = readl(hsotg->regs + HCFG);
276 fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
277 HCFG_FSLSPCLKSEL_SHIFT;
278
279 if (prtspd == HPRT0_SPD_LOW_SPEED &&
280 params->host_ls_low_power_phy_clk ==
281 DWC2_HOST_LS_LOW_POWER_PHY_CLK_PARAM_6MHZ) {
282 /* 6 MHZ */
283 dev_vdbg(hsotg->dev,
284 "FS_PHY programming HCFG to 6 MHz\n");
285 if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
286 fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
287 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
288 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
289 writel(hcfg, hsotg->regs + HCFG);
290 do_reset = 1;
291 }
292 } else {
293 /* 48 MHZ */
294 dev_vdbg(hsotg->dev,
295 "FS_PHY programming HCFG to 48 MHz\n");
296 if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
297 fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
298 hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
299 hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
300 writel(hcfg, hsotg->regs + HCFG);
301 do_reset = 1;
302 }
303 }
304 } else {
305 /* Not low power */
306 if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
307 usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
308 writel(usbcfg, hsotg->regs + GUSBCFG);
309 do_reset = 1;
310 }
311 }
312
313 if (do_reset) {
314 *hprt0_modify |= HPRT0_RST;
315 queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
316 msecs_to_jiffies(60));
317 } else {
318 /* Port has been enabled, set the reset change flag */
319 hsotg->flags.b.port_reset_change = 1;
320 }
321}
322
323/*
324 * There are multiple conditions that can cause a port interrupt. This function
325 * determines which interrupt conditions have occurred and handles them
326 * appropriately.
327 */
328static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
329{
330 u32 hprt0;
331 u32 hprt0_modify;
332
333 dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
334
335 hprt0 = readl(hsotg->regs + HPRT0);
336 hprt0_modify = hprt0;
337
338 /*
339 * Clear appropriate bits in HPRT0 to clear the interrupt bit in
340 * GINTSTS
341 */
342 hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
343 HPRT0_OVRCURRCHG);
344
345 /*
346 * Port Connect Detected
347 * Set flag and clear if detected
348 */
349 if (hprt0 & HPRT0_CONNDET) {
350 dev_vdbg(hsotg->dev,
351 "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
352 hprt0);
353 hsotg->flags.b.port_connect_status_change = 1;
354 hsotg->flags.b.port_connect_status = 1;
355 hprt0_modify |= HPRT0_CONNDET;
356
357 /*
358 * The Hub driver asserts a reset when it sees port connect
359 * status change flag
360 */
361 }
362
363 /*
364 * Port Enable Changed
365 * Clear if detected - Set internal flag if disabled
366 */
367 if (hprt0 & HPRT0_ENACHG) {
368 dev_vdbg(hsotg->dev,
369 " --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
370 hprt0, !!(hprt0 & HPRT0_ENA));
371 hprt0_modify |= HPRT0_ENACHG;
372 if (hprt0 & HPRT0_ENA)
373 dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
374 else
375 hsotg->flags.b.port_enable_change = 1;
376 }
377
378 /* Overcurrent Change Interrupt */
379 if (hprt0 & HPRT0_OVRCURRCHG) {
380 dev_vdbg(hsotg->dev,
381 " --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
382 hprt0);
383 hsotg->flags.b.port_over_current_change = 1;
384 hprt0_modify |= HPRT0_OVRCURRCHG;
385 }
386
387 /* Clear Port Interrupts */
388 writel(hprt0_modify, hsotg->regs + HPRT0);
389}
390
391/*
392 * Gets the actual length of a transfer after the transfer halts. halt_status
393 * holds the reason for the halt.
394 *
395 * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
396 * is set to 1 upon return if less than the requested number of bytes were
397 * transferred. short_read may also be NULL on entry, in which case it remains
398 * unchanged.
399 */
400static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
401 struct dwc2_host_chan *chan, int chnum,
402 struct dwc2_qtd *qtd,
403 enum dwc2_halt_status halt_status,
404 int *short_read)
405{
406 u32 hctsiz, count, length;
407
408 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
409
410 if (halt_status == DWC2_HC_XFER_COMPLETE) {
411 if (chan->ep_is_in) {
412 count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
413 TSIZ_XFERSIZE_SHIFT;
414 length = chan->xfer_len - count;
415 if (short_read != NULL)
416 *short_read = (count != 0);
417 } else if (chan->qh->do_split) {
418 length = qtd->ssplit_out_xfer_count;
419 } else {
420 length = chan->xfer_len;
421 }
422 } else {
423 /*
424 * Must use the hctsiz.pktcnt field to determine how much data
425 * has been transferred. This field reflects the number of
426 * packets that have been transferred via the USB. This is
427 * always an integral number of packets if the transfer was
428 * halted before its normal completion. (Can't use the
429 * hctsiz.xfersize field because that reflects the number of
430 * bytes transferred via the AHB, not the USB).
431 */
432 count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
433 length = (chan->start_pkt_count - count) * chan->max_packet;
434 }
435
436 return length;
437}
438
439/**
440 * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
441 * Complete interrupt on the host channel. Updates the actual_length field
442 * of the URB based on the number of bytes transferred via the host channel.
443 * Sets the URB status if the data transfer is finished.
444 *
445 * Return: 1 if the data transfer specified by the URB is completely finished,
446 * 0 otherwise
447 */
448static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
449 struct dwc2_host_chan *chan, int chnum,
450 struct dwc2_hcd_urb *urb,
451 struct dwc2_qtd *qtd)
452{
453 u32 hctsiz;
454 int xfer_done = 0;
455 int short_read = 0;
456 int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
457 DWC2_HC_XFER_COMPLETE,
458 &short_read);
459
460 if (urb->actual_length + xfer_length > urb->length) {
461 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
462 xfer_length = urb->length - urb->actual_length;
463 }
464
465 /* Non DWORD-aligned buffer case handling */
466 if (chan->align_buf && xfer_length && chan->ep_is_in) {
467 dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
468 dma_sync_single_for_cpu(hsotg->dev, urb->dma, urb->length,
469 DMA_FROM_DEVICE);
470 memcpy(urb->buf + urb->actual_length, chan->qh->dw_align_buf,
471 xfer_length);
472 dma_sync_single_for_device(hsotg->dev, urb->dma, urb->length,
473 DMA_FROM_DEVICE);
474 }
475
476 dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
477 urb->actual_length, xfer_length);
478 urb->actual_length += xfer_length;
479
480 if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
481 (urb->flags & URB_SEND_ZERO_PACKET) &&
482 urb->actual_length >= urb->length &&
483 !(urb->length % chan->max_packet)) {
484 xfer_done = 0;
485 } else if (short_read || urb->actual_length >= urb->length) {
486 xfer_done = 1;
487 urb->status = 0;
488 }
489
490 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
491 dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
492 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
493 dev_vdbg(hsotg->dev, " chan->xfer_len %d\n", chan->xfer_len);
494 dev_vdbg(hsotg->dev, " hctsiz.xfersize %d\n",
495 (hctsiz & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
496 dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n", urb->length);
497 dev_vdbg(hsotg->dev, " urb->actual_length %d\n", urb->actual_length);
498 dev_vdbg(hsotg->dev, " short_read %d, xfer_done %d\n", short_read,
499 xfer_done);
500
501 return xfer_done;
502}
503
504/*
505 * Save the starting data toggle for the next transfer. The data toggle is
506 * saved in the QH for non-control transfers and it's saved in the QTD for
507 * control transfers.
508 */
509void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
510 struct dwc2_host_chan *chan, int chnum,
511 struct dwc2_qtd *qtd)
512{
513 u32 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
514 u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
515
516 if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
517 if (pid == TSIZ_SC_MC_PID_DATA0)
518 chan->qh->data_toggle = DWC2_HC_PID_DATA0;
519 else
520 chan->qh->data_toggle = DWC2_HC_PID_DATA1;
521 } else {
522 if (pid == TSIZ_SC_MC_PID_DATA0)
523 qtd->data_toggle = DWC2_HC_PID_DATA0;
524 else
525 qtd->data_toggle = DWC2_HC_PID_DATA1;
526 }
527}
528
529/**
530 * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
531 * the transfer is stopped for any reason. The fields of the current entry in
532 * the frame descriptor array are set based on the transfer state and the input
533 * halt_status. Completes the Isochronous URB if all the URB frames have been
534 * completed.
535 *
536 * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
537 * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
538 */
539static enum dwc2_halt_status dwc2_update_isoc_urb_state(
540 struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
541 int chnum, struct dwc2_qtd *qtd,
542 enum dwc2_halt_status halt_status)
543{
544 struct dwc2_hcd_iso_packet_desc *frame_desc;
545 struct dwc2_hcd_urb *urb = qtd->urb;
546
547 if (!urb)
548 return DWC2_HC_XFER_NO_HALT_STATUS;
549
550 frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
551
552 switch (halt_status) {
553 case DWC2_HC_XFER_COMPLETE:
554 frame_desc->status = 0;
555 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
556 chan, chnum, qtd, halt_status, NULL);
557
558 /* Non DWORD-aligned buffer case handling */
559 if (chan->align_buf && frame_desc->actual_length &&
560 chan->ep_is_in) {
561 dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n",
562 __func__);
563 dma_sync_single_for_cpu(hsotg->dev, urb->dma,
564 urb->length, DMA_FROM_DEVICE);
565 memcpy(urb->buf + frame_desc->offset +
566 qtd->isoc_split_offset, chan->qh->dw_align_buf,
567 frame_desc->actual_length);
568 dma_sync_single_for_device(hsotg->dev, urb->dma,
569 urb->length,
570 DMA_FROM_DEVICE);
571 }
572 break;
573 case DWC2_HC_XFER_FRAME_OVERRUN:
574 urb->error_count++;
575 if (chan->ep_is_in)
576 frame_desc->status = -ENOSR;
577 else
578 frame_desc->status = -ECOMM;
579 frame_desc->actual_length = 0;
580 break;
581 case DWC2_HC_XFER_BABBLE_ERR:
582 urb->error_count++;
583 frame_desc->status = -EOVERFLOW;
584 /* Don't need to update actual_length in this case */
585 break;
586 case DWC2_HC_XFER_XACT_ERR:
587 urb->error_count++;
588 frame_desc->status = -EPROTO;
589 frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
590 chan, chnum, qtd, halt_status, NULL);
591
592 /* Non DWORD-aligned buffer case handling */
593 if (chan->align_buf && frame_desc->actual_length &&
594 chan->ep_is_in) {
595 dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n",
596 __func__);
597 dma_sync_single_for_cpu(hsotg->dev, urb->dma,
598 urb->length, DMA_FROM_DEVICE);
599 memcpy(urb->buf + frame_desc->offset +
600 qtd->isoc_split_offset, chan->qh->dw_align_buf,
601 frame_desc->actual_length);
602 dma_sync_single_for_device(hsotg->dev, urb->dma,
603 urb->length,
604 DMA_FROM_DEVICE);
605 }
606
607 /* Skip whole frame */
608 if (chan->qh->do_split &&
609 chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
610 hsotg->core_params->dma_enable > 0) {
611 qtd->complete_split = 0;
612 qtd->isoc_split_offset = 0;
613 }
614
615 break;
616 default:
617 dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
618 halt_status);
619 break;
620 }
621
622 if (++qtd->isoc_frame_index == urb->packet_count) {
623 /*
624 * urb->status is not used for isoc transfers. The individual
625 * frame_desc statuses are used instead.
626 */
627 dwc2_host_complete(hsotg, qtd, 0);
628 halt_status = DWC2_HC_XFER_URB_COMPLETE;
629 } else {
630 halt_status = DWC2_HC_XFER_COMPLETE;
631 }
632
633 return halt_status;
634}
635
636/*
637 * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
638 * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
639 * still linked to the QH, the QH is added to the end of the inactive
640 * non-periodic schedule. For periodic QHs, removes the QH from the periodic
641 * schedule if no more QTDs are linked to the QH.
642 */
643static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
644 int free_qtd)
645{
646 int continue_split = 0;
647 struct dwc2_qtd *qtd;
648
649 if (dbg_qh(qh))
650 dev_vdbg(hsotg->dev, " %s(%p,%p,%d)\n", __func__,
651 hsotg, qh, free_qtd);
652
653 if (list_empty(&qh->qtd_list)) {
654 dev_dbg(hsotg->dev, "## QTD list empty ##\n");
655 goto no_qtd;
656 }
657
658 qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
659
660 if (qtd->complete_split)
661 continue_split = 1;
662 else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
663 qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
664 continue_split = 1;
665
666 if (free_qtd) {
667 dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
668 continue_split = 0;
669 }
670
671no_qtd:
672 if (qh->channel)
673 qh->channel->align_buf = 0;
674 qh->channel = NULL;
675 dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
676}
677
678/**
679 * dwc2_release_channel() - Releases a host channel for use by other transfers
680 *
681 * @hsotg: The HCD state structure
682 * @chan: The host channel to release
683 * @qtd: The QTD associated with the host channel. This QTD may be
684 * freed if the transfer is complete or an error has occurred.
685 * @halt_status: Reason the channel is being released. This status
686 * determines the actions taken by this function.
687 *
688 * Also attempts to select and queue more transactions since at least one host
689 * channel is available.
690 */
691static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
692 struct dwc2_host_chan *chan,
693 struct dwc2_qtd *qtd,
694 enum dwc2_halt_status halt_status)
695{
696 enum dwc2_transaction_type tr_type;
697 u32 haintmsk;
698 int free_qtd = 0;
699
700 if (dbg_hc(chan))
701 dev_vdbg(hsotg->dev, " %s: channel %d, halt_status %d\n",
702 __func__, chan->hc_num, halt_status);
703
704 switch (halt_status) {
705 case DWC2_HC_XFER_URB_COMPLETE:
706 free_qtd = 1;
707 break;
708 case DWC2_HC_XFER_AHB_ERR:
709 case DWC2_HC_XFER_STALL:
710 case DWC2_HC_XFER_BABBLE_ERR:
711 free_qtd = 1;
712 break;
713 case DWC2_HC_XFER_XACT_ERR:
714 if (qtd && qtd->error_count >= 3) {
715 dev_vdbg(hsotg->dev,
716 " Complete URB with transaction error\n");
717 free_qtd = 1;
718 dwc2_host_complete(hsotg, qtd, -EPROTO);
719 }
720 break;
721 case DWC2_HC_XFER_URB_DEQUEUE:
722 /*
723 * The QTD has already been removed and the QH has been
724 * deactivated. Don't want to do anything except release the
725 * host channel and try to queue more transfers.
726 */
727 goto cleanup;
728 case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
729 dev_vdbg(hsotg->dev, " Complete URB with I/O error\n");
730 free_qtd = 1;
731 dwc2_host_complete(hsotg, qtd, -EIO);
732 break;
733 case DWC2_HC_XFER_NO_HALT_STATUS:
734 default:
735 break;
736 }
737
738 dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);
739
740cleanup:
741 /*
742 * Release the host channel for use by other transfers. The cleanup
743 * function clears the channel interrupt enables and conditions, so
744 * there's no need to clear the Channel Halted interrupt separately.
745 */
746 if (!list_empty(&chan->hc_list_entry))
747 list_del(&chan->hc_list_entry);
748 dwc2_hc_cleanup(hsotg, chan);
749 list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
750
751 if (hsotg->core_params->uframe_sched > 0) {
752 hsotg->available_host_channels++;
753 } else {
754 switch (chan->ep_type) {
755 case USB_ENDPOINT_XFER_CONTROL:
756 case USB_ENDPOINT_XFER_BULK:
757 hsotg->non_periodic_channels--;
758 break;
759 default:
760 /*
761 * Don't release reservations for periodic channels
762 * here. That's done when a periodic transfer is
763 * descheduled (i.e. when the QH is removed from the
764 * periodic schedule).
765 */
766 break;
767 }
768 }
769
770 haintmsk = readl(hsotg->regs + HAINTMSK);
771 haintmsk &= ~(1 << chan->hc_num);
772 writel(haintmsk, hsotg->regs + HAINTMSK);
773
774 /* Try to queue more transfers now that there's a free channel */
775 tr_type = dwc2_hcd_select_transactions(hsotg);
776 if (tr_type != DWC2_TRANSACTION_NONE)
777 dwc2_hcd_queue_transactions(hsotg, tr_type);
778}
779
780/*
781 * Halts a host channel. If the channel cannot be halted immediately because
782 * the request queue is full, this function ensures that the FIFO empty
783 * interrupt for the appropriate queue is enabled so that the halt request can
784 * be queued when there is space in the request queue.
785 *
786 * This function may also be called in DMA mode. In that case, the channel is
787 * simply released since the core always halts the channel automatically in
788 * DMA mode.
789 */
790static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
791 struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
792 enum dwc2_halt_status halt_status)
793{
794 if (dbg_hc(chan))
795 dev_vdbg(hsotg->dev, "%s()\n", __func__);
796
797 if (hsotg->core_params->dma_enable > 0) {
798 if (dbg_hc(chan))
799 dev_vdbg(hsotg->dev, "DMA enabled\n");
800 dwc2_release_channel(hsotg, chan, qtd, halt_status);
801 return;
802 }
803
804 /* Slave mode processing */
805 dwc2_hc_halt(hsotg, chan, halt_status);
806
807 if (chan->halt_on_queue) {
808 u32 gintmsk;
809
810 dev_vdbg(hsotg->dev, "Halt on queue\n");
811 if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
812 chan->ep_type == USB_ENDPOINT_XFER_BULK) {
813 dev_vdbg(hsotg->dev, "control/bulk\n");
814 /*
815 * Make sure the Non-periodic Tx FIFO empty interrupt
816 * is enabled so that the non-periodic schedule will
817 * be processed
818 */
819 gintmsk = readl(hsotg->regs + GINTMSK);
820 gintmsk |= GINTSTS_NPTXFEMP;
821 writel(gintmsk, hsotg->regs + GINTMSK);
822 } else {
823 dev_vdbg(hsotg->dev, "isoc/intr\n");
824 /*
825 * Move the QH from the periodic queued schedule to
826 * the periodic assigned schedule. This allows the
827 * halt to be queued when the periodic schedule is
828 * processed.
829 */
830 list_move(&chan->qh->qh_list_entry,
831 &hsotg->periodic_sched_assigned);
832
833 /*
834 * Make sure the Periodic Tx FIFO Empty interrupt is
835 * enabled so that the periodic schedule will be
836 * processed
837 */
838 gintmsk = readl(hsotg->regs + GINTMSK);
839 gintmsk |= GINTSTS_PTXFEMP;
840 writel(gintmsk, hsotg->regs + GINTMSK);
841 }
842 }
843}
844
845/*
846 * Performs common cleanup for non-periodic transfers after a Transfer
847 * Complete interrupt. This function should be called after any endpoint type
848 * specific handling is finished to release the host channel.
849 */
850static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
851 struct dwc2_host_chan *chan,
852 int chnum, struct dwc2_qtd *qtd,
853 enum dwc2_halt_status halt_status)
854{
855 dev_vdbg(hsotg->dev, "%s()\n", __func__);
856
857 qtd->error_count = 0;
858
859 if (chan->hcint & HCINTMSK_NYET) {
860 /*
861 * Got a NYET on the last transaction of the transfer. This
862 * means that the endpoint should be in the PING state at the
863 * beginning of the next transfer.
864 */
865 dev_vdbg(hsotg->dev, "got NYET\n");
866 chan->qh->ping_state = 1;
867 }
868
869 /*
870 * Always halt and release the host channel to make it available for
871 * more transfers. There may still be more phases for a control
872 * transfer or more data packets for a bulk transfer at this point,
873 * but the host channel is still halted. A channel will be reassigned
874 * to the transfer when the non-periodic schedule is processed after
875 * the channel is released. This allows transactions to be queued
876 * properly via dwc2_hcd_queue_transactions, which also enables the
877 * Tx FIFO Empty interrupt if necessary.
878 */
879 if (chan->ep_is_in) {
880 /*
881 * IN transfers in Slave mode require an explicit disable to
882 * halt the channel. (In DMA mode, this call simply releases
883 * the channel.)
884 */
885 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
886 } else {
887 /*
888 * The channel is automatically disabled by the core for OUT
889 * transfers in Slave mode
890 */
891 dwc2_release_channel(hsotg, chan, qtd, halt_status);
892 }
893}
894
895/*
896 * Performs common cleanup for periodic transfers after a Transfer Complete
897 * interrupt. This function should be called after any endpoint type specific
898 * handling is finished to release the host channel.
899 */
900static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
901 struct dwc2_host_chan *chan, int chnum,
902 struct dwc2_qtd *qtd,
903 enum dwc2_halt_status halt_status)
904{
905 u32 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
906
907 qtd->error_count = 0;
908
909 if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
910 /* Core halts channel in these cases */
911 dwc2_release_channel(hsotg, chan, qtd, halt_status);
912 else
913 /* Flush any outstanding requests from the Tx queue */
914 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
915}
916
917static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
918 struct dwc2_host_chan *chan, int chnum,
919 struct dwc2_qtd *qtd)
920{
921 struct dwc2_hcd_iso_packet_desc *frame_desc;
922 u32 len;
923
924 if (!qtd->urb)
925 return 0;
926
927 frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
928 len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
929 DWC2_HC_XFER_COMPLETE, NULL);
930 if (!len) {
931 qtd->complete_split = 0;
932 qtd->isoc_split_offset = 0;
933 return 0;
934 }
935
936 frame_desc->actual_length += len;
937
938 if (chan->align_buf) {
939 dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
940 dma_sync_single_for_cpu(hsotg->dev, qtd->urb->dma,
941 qtd->urb->length, DMA_FROM_DEVICE);
942 memcpy(qtd->urb->buf + frame_desc->offset +
943 qtd->isoc_split_offset, chan->qh->dw_align_buf, len);
944 dma_sync_single_for_device(hsotg->dev, qtd->urb->dma,
945 qtd->urb->length, DMA_FROM_DEVICE);
946 }
947
948 qtd->isoc_split_offset += len;
949
950 if (frame_desc->actual_length >= frame_desc->length) {
951 frame_desc->status = 0;
952 qtd->isoc_frame_index++;
953 qtd->complete_split = 0;
954 qtd->isoc_split_offset = 0;
955 }
956
957 if (qtd->isoc_frame_index == qtd->urb->packet_count) {
958 dwc2_host_complete(hsotg, qtd, 0);
959 dwc2_release_channel(hsotg, chan, qtd,
960 DWC2_HC_XFER_URB_COMPLETE);
961 } else {
962 dwc2_release_channel(hsotg, chan, qtd,
963 DWC2_HC_XFER_NO_HALT_STATUS);
964 }
965
966 return 1; /* Indicates that channel released */
967}
968
969/*
970 * Handles a host channel Transfer Complete interrupt. This handler may be
971 * called in either DMA mode or Slave mode.
972 */
973static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
974 struct dwc2_host_chan *chan, int chnum,
975 struct dwc2_qtd *qtd)
976{
977 struct dwc2_hcd_urb *urb = qtd->urb;
978 enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
979 int pipe_type;
980 int urb_xfer_done;
981
982 if (dbg_hc(chan))
983 dev_vdbg(hsotg->dev,
984 "--Host Channel %d Interrupt: Transfer Complete--\n",
985 chnum);
986
987 if (!urb)
988 goto handle_xfercomp_done;
989
990 pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
991
992 if (hsotg->core_params->dma_desc_enable > 0) {
993 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
994 if (pipe_type == USB_ENDPOINT_XFER_ISOC)
995 /* Do not disable the interrupt, just clear it */
996 return;
997 goto handle_xfercomp_done;
998 }
999
1000 /* Handle xfer complete on CSPLIT */
1001 if (chan->qh->do_split) {
1002 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
1003 hsotg->core_params->dma_enable > 0) {
1004 if (qtd->complete_split &&
1005 dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
1006 qtd))
1007 goto handle_xfercomp_done;
1008 } else {
1009 qtd->complete_split = 0;
1010 }
1011 }
1012
1013 /* Update the QTD and URB states */
1014 switch (pipe_type) {
1015 case USB_ENDPOINT_XFER_CONTROL:
1016 switch (qtd->control_phase) {
1017 case DWC2_CONTROL_SETUP:
1018 if (urb->length > 0)
1019 qtd->control_phase = DWC2_CONTROL_DATA;
1020 else
1021 qtd->control_phase = DWC2_CONTROL_STATUS;
1022 dev_vdbg(hsotg->dev,
1023 " Control setup transaction done\n");
1024 halt_status = DWC2_HC_XFER_COMPLETE;
1025 break;
1026 case DWC2_CONTROL_DATA:
1027 urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1028 chnum, urb, qtd);
1029 if (urb_xfer_done) {
1030 qtd->control_phase = DWC2_CONTROL_STATUS;
1031 dev_vdbg(hsotg->dev,
1032 " Control data transfer done\n");
1033 } else {
1034 dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1035 qtd);
1036 }
1037 halt_status = DWC2_HC_XFER_COMPLETE;
1038 break;
1039 case DWC2_CONTROL_STATUS:
1040 dev_vdbg(hsotg->dev, " Control transfer complete\n");
1041 if (urb->status == -EINPROGRESS)
1042 urb->status = 0;
1043 dwc2_host_complete(hsotg, qtd, urb->status);
1044 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1045 break;
1046 }
1047
1048 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1049 halt_status);
1050 break;
1051 case USB_ENDPOINT_XFER_BULK:
1052 dev_vdbg(hsotg->dev, " Bulk transfer complete\n");
1053 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1054 qtd);
1055 if (urb_xfer_done) {
1056 dwc2_host_complete(hsotg, qtd, urb->status);
1057 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1058 } else {
1059 halt_status = DWC2_HC_XFER_COMPLETE;
1060 }
1061
1062 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1063 dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1064 halt_status);
1065 break;
1066 case USB_ENDPOINT_XFER_INT:
1067 dev_vdbg(hsotg->dev, " Interrupt transfer complete\n");
1068 urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1069 qtd);
1070
1071 /*
1072 * Interrupt URB is done on the first transfer complete
1073 * interrupt
1074 */
1075 if (urb_xfer_done) {
1076 dwc2_host_complete(hsotg, qtd, urb->status);
1077 halt_status = DWC2_HC_XFER_URB_COMPLETE;
1078 } else {
1079 halt_status = DWC2_HC_XFER_COMPLETE;
1080 }
1081
1082 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1083 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1084 halt_status);
1085 break;
1086 case USB_ENDPOINT_XFER_ISOC:
1087 if (dbg_perio())
1088 dev_vdbg(hsotg->dev, " Isochronous transfer complete\n");
1089 if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1090 halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1091 chnum, qtd, DWC2_HC_XFER_COMPLETE);
1092 dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1093 halt_status);
1094 break;
1095 }
1096
1097handle_xfercomp_done:
1098 disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1099}
1100
1101/*
1102 * Handles a host channel STALL interrupt. This handler may be called in
1103 * either DMA mode or Slave mode.
1104 */
1105static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1106 struct dwc2_host_chan *chan, int chnum,
1107 struct dwc2_qtd *qtd)
1108{
1109 struct dwc2_hcd_urb *urb = qtd->urb;
1110 int pipe_type;
1111
1112 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1113 chnum);
1114
1115 if (hsotg->core_params->dma_desc_enable > 0) {
1116 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1117 DWC2_HC_XFER_STALL);
1118 goto handle_stall_done;
1119 }
1120
1121 if (!urb)
1122 goto handle_stall_halt;
1123
1124 pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1125
1126 if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1127 dwc2_host_complete(hsotg, qtd, -EPIPE);
1128
1129 if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1130 pipe_type == USB_ENDPOINT_XFER_INT) {
1131 dwc2_host_complete(hsotg, qtd, -EPIPE);
1132 /*
1133 * USB protocol requires resetting the data toggle for bulk
1134 * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1135 * setup command is issued to the endpoint. Anticipate the
1136 * CLEAR_FEATURE command since a STALL has occurred and reset
1137 * the data toggle now.
1138 */
1139 chan->qh->data_toggle = 0;
1140 }
1141
1142handle_stall_halt:
1143 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);
1144
1145handle_stall_done:
1146 disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1147}
1148
1149/*
1150 * Updates the state of the URB when a transfer has been stopped due to an
1151 * abnormal condition before the transfer completes. Modifies the
1152 * actual_length field of the URB to reflect the number of bytes that have
1153 * actually been transferred via the host channel.
1154 */
1155static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1156 struct dwc2_host_chan *chan, int chnum,
1157 struct dwc2_hcd_urb *urb,
1158 struct dwc2_qtd *qtd,
1159 enum dwc2_halt_status halt_status)
1160{
1161 u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1162 qtd, halt_status, NULL);
1163 u32 hctsiz;
1164
1165 if (urb->actual_length + xfer_length > urb->length) {
1166 dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1167 xfer_length = urb->length - urb->actual_length;
1168 }
1169
1170 /* Non DWORD-aligned buffer case handling */
1171 if (chan->align_buf && xfer_length && chan->ep_is_in) {
1172 dev_vdbg(hsotg->dev, "%s(): non-aligned buffer\n", __func__);
1173 dma_sync_single_for_cpu(hsotg->dev, urb->dma, urb->length,
1174 DMA_FROM_DEVICE);
1175 memcpy(urb->buf + urb->actual_length, chan->qh->dw_align_buf,
1176 xfer_length);
1177 dma_sync_single_for_device(hsotg->dev, urb->dma, urb->length,
1178 DMA_FROM_DEVICE);
1179 }
1180
1181 urb->actual_length += xfer_length;
1182
1183 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
1184 dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1185 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1186 dev_vdbg(hsotg->dev, " chan->start_pkt_count %d\n",
1187 chan->start_pkt_count);
1188 dev_vdbg(hsotg->dev, " hctsiz.pktcnt %d\n",
1189 (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1190 dev_vdbg(hsotg->dev, " chan->max_packet %d\n", chan->max_packet);
1191 dev_vdbg(hsotg->dev, " bytes_transferred %d\n",
1192 xfer_length);
1193 dev_vdbg(hsotg->dev, " urb->actual_length %d\n",
1194 urb->actual_length);
1195 dev_vdbg(hsotg->dev, " urb->transfer_buffer_length %d\n",
1196 urb->length);
1197}
1198
1199/*
1200 * Handles a host channel NAK interrupt. This handler may be called in either
1201 * DMA mode or Slave mode.
1202 */
1203static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1204 struct dwc2_host_chan *chan, int chnum,
1205 struct dwc2_qtd *qtd)
1206{
1207 if (dbg_hc(chan))
1208 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1209 chnum);
1210
1211 /*
1212 * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1213 * interrupt. Re-start the SSPLIT transfer.
1214 */
1215 if (chan->do_split) {
1216 if (chan->complete_split)
1217 qtd->error_count = 0;
1218 qtd->complete_split = 0;
1219 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1220 goto handle_nak_done;
1221 }
1222
1223 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1224 case USB_ENDPOINT_XFER_CONTROL:
1225 case USB_ENDPOINT_XFER_BULK:
1226 if (hsotg->core_params->dma_enable > 0 && chan->ep_is_in) {
1227 /*
1228 * NAK interrupts are enabled on bulk/control IN
1229 * transfers in DMA mode for the sole purpose of
1230 * resetting the error count after a transaction error
1231 * occurs. The core will continue transferring data.
1232 */
1233 qtd->error_count = 0;
1234 break;
1235 }
1236
1237 /*
1238 * NAK interrupts normally occur during OUT transfers in DMA
1239 * or Slave mode. For IN transfers, more requests will be
1240 * queued as request queue space is available.
1241 */
1242 qtd->error_count = 0;
1243
1244 if (!chan->qh->ping_state) {
1245 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1246 qtd, DWC2_HC_XFER_NAK);
1247 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1248
1249 if (chan->speed == USB_SPEED_HIGH)
1250 chan->qh->ping_state = 1;
1251 }
1252
1253 /*
1254 * Halt the channel so the transfer can be re-started from
1255 * the appropriate point or the PING protocol will
1256 * start/continue
1257 */
1258 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1259 break;
1260 case USB_ENDPOINT_XFER_INT:
1261 qtd->error_count = 0;
1262 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1263 break;
1264 case USB_ENDPOINT_XFER_ISOC:
1265 /* Should never get called for isochronous transfers */
1266 dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1267 break;
1268 }
1269
1270handle_nak_done:
1271 disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1272}
1273
1274/*
1275 * Handles a host channel ACK interrupt. This interrupt is enabled when
1276 * performing the PING protocol in Slave mode, when errors occur during
1277 * either Slave mode or DMA mode, and during Start Split transactions.
1278 */
1279static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1280 struct dwc2_host_chan *chan, int chnum,
1281 struct dwc2_qtd *qtd)
1282{
1283 struct dwc2_hcd_iso_packet_desc *frame_desc;
1284
1285 if (dbg_hc(chan))
1286 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1287 chnum);
1288
1289 if (chan->do_split) {
1290 /* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1291 if (!chan->ep_is_in &&
1292 chan->data_pid_start != DWC2_HC_PID_SETUP)
1293 qtd->ssplit_out_xfer_count = chan->xfer_len;
1294
1295 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1296 qtd->complete_split = 1;
1297 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1298 } else {
1299 /* ISOC OUT */
1300 switch (chan->xact_pos) {
1301 case DWC2_HCSPLT_XACTPOS_ALL:
1302 break;
1303 case DWC2_HCSPLT_XACTPOS_END:
1304 qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1305 qtd->isoc_split_offset = 0;
1306 break;
1307 case DWC2_HCSPLT_XACTPOS_BEGIN:
1308 case DWC2_HCSPLT_XACTPOS_MID:
1309 /*
1310 * For BEGIN or MID, calculate the length for
1311 * the next microframe to determine the correct
1312 * SSPLIT token, either MID or END
1313 */
1314 frame_desc = &qtd->urb->iso_descs[
1315 qtd->isoc_frame_index];
1316 qtd->isoc_split_offset += 188;
1317
1318 if (frame_desc->length - qtd->isoc_split_offset
1319 <= 188)
1320 qtd->isoc_split_pos =
1321 DWC2_HCSPLT_XACTPOS_END;
1322 else
1323 qtd->isoc_split_pos =
1324 DWC2_HCSPLT_XACTPOS_MID;
1325 break;
1326 }
1327 }
1328 } else {
1329 qtd->error_count = 0;
1330
1331 if (chan->qh->ping_state) {
1332 chan->qh->ping_state = 0;
1333 /*
1334 * Halt the channel so the transfer can be re-started
1335 * from the appropriate point. This only happens in
1336 * Slave mode. In DMA mode, the ping_state is cleared
1337 * when the transfer is started because the core
1338 * automatically executes the PING, then the transfer.
1339 */
1340 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1341 }
1342 }
1343
1344 /*
1345 * If the ACK occurred when _not_ in the PING state, let the channel
1346 * continue transferring data after clearing the error count
1347 */
1348 disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1349}
1350
1351/*
1352 * Handles a host channel NYET interrupt. This interrupt should only occur on
1353 * Bulk and Control OUT endpoints and for complete split transactions. If a
1354 * NYET occurs at the same time as a Transfer Complete interrupt, it is
1355 * handled in the xfercomp interrupt handler, not here. This handler may be
1356 * called in either DMA mode or Slave mode.
1357 */
1358static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1359 struct dwc2_host_chan *chan, int chnum,
1360 struct dwc2_qtd *qtd)
1361{
1362 if (dbg_hc(chan))
1363 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1364 chnum);
1365
1366 /*
1367 * NYET on CSPLIT
1368 * re-do the CSPLIT immediately on non-periodic
1369 */
1370 if (chan->do_split && chan->complete_split) {
1371 if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1372 hsotg->core_params->dma_enable > 0) {
1373 qtd->complete_split = 0;
1374 qtd->isoc_split_offset = 0;
1375 qtd->isoc_frame_index++;
1376 if (qtd->urb &&
1377 qtd->isoc_frame_index == qtd->urb->packet_count) {
1378 dwc2_host_complete(hsotg, qtd, 0);
1379 dwc2_release_channel(hsotg, chan, qtd,
1380 DWC2_HC_XFER_URB_COMPLETE);
1381 } else {
1382 dwc2_release_channel(hsotg, chan, qtd,
1383 DWC2_HC_XFER_NO_HALT_STATUS);
1384 }
1385 goto handle_nyet_done;
1386 }
1387
1388 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1389 chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1390 int frnum = dwc2_hcd_get_frame_number(hsotg);
1391
1392 if (dwc2_full_frame_num(frnum) !=
1393 dwc2_full_frame_num(chan->qh->sched_frame)) {
1394 /*
1395 * No longer in the same full speed frame.
1396 * Treat this as a transaction error.
1397 */
1398#if 0
1399 /*
1400 * Todo: Fix system performance so this can
1401 * be treated as an error. Right now complete
1402 * splits cannot be scheduled precisely enough
1403 * due to other system activity, so this error
1404 * occurs regularly in Slave mode.
1405 */
1406 qtd->error_count++;
1407#endif
1408 qtd->complete_split = 0;
1409 dwc2_halt_channel(hsotg, chan, qtd,
1410 DWC2_HC_XFER_XACT_ERR);
1411 /* Todo: add support for isoc release */
1412 goto handle_nyet_done;
1413 }
1414 }
1415
1416 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1417 goto handle_nyet_done;
1418 }
1419
1420 chan->qh->ping_state = 1;
1421 qtd->error_count = 0;
1422
1423 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
1424 DWC2_HC_XFER_NYET);
1425 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1426
1427 /*
1428 * Halt the channel and re-start the transfer so the PING protocol
1429 * will start
1430 */
1431 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1432
1433handle_nyet_done:
1434 disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1435}
1436
1437/*
1438 * Handles a host channel babble interrupt. This handler may be called in
1439 * either DMA mode or Slave mode.
1440 */
1441static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1442 struct dwc2_host_chan *chan, int chnum,
1443 struct dwc2_qtd *qtd)
1444{
1445 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1446 chnum);
1447
1448 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1449
1450 if (hsotg->core_params->dma_desc_enable > 0) {
1451 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1452 DWC2_HC_XFER_BABBLE_ERR);
1453 goto disable_int;
1454 }
1455
1456 if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1457 dwc2_host_complete(hsotg, qtd, -EOVERFLOW);
1458 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
1459 } else {
1460 enum dwc2_halt_status halt_status;
1461
1462 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1463 qtd, DWC2_HC_XFER_BABBLE_ERR);
1464 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1465 }
1466
1467disable_int:
1468 disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1469}
1470
1471/*
1472 * Handles a host channel AHB error interrupt. This handler is only called in
1473 * DMA mode.
1474 */
1475static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1476 struct dwc2_host_chan *chan, int chnum,
1477 struct dwc2_qtd *qtd)
1478{
1479 struct dwc2_hcd_urb *urb = qtd->urb;
1480 char *pipetype, *speed;
1481 u32 hcchar;
1482 u32 hcsplt;
1483 u32 hctsiz;
1484 u32 hc_dma;
1485
1486 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1487 chnum);
1488
1489 if (!urb)
1490 goto handle_ahberr_halt;
1491
1492 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1493
1494 hcchar = readl(hsotg->regs + HCCHAR(chnum));
1495 hcsplt = readl(hsotg->regs + HCSPLT(chnum));
1496 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
1497 hc_dma = readl(hsotg->regs + HCDMA(chnum));
1498
1499 dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1500 dev_err(hsotg->dev, " hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1501 dev_err(hsotg->dev, " hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1502 dev_err(hsotg->dev, " Device address: %d\n",
1503 dwc2_hcd_get_dev_addr(&urb->pipe_info));
1504 dev_err(hsotg->dev, " Endpoint: %d, %s\n",
1505 dwc2_hcd_get_ep_num(&urb->pipe_info),
1506 dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1507
1508 switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
1509 case USB_ENDPOINT_XFER_CONTROL:
1510 pipetype = "CONTROL";
1511 break;
1512 case USB_ENDPOINT_XFER_BULK:
1513 pipetype = "BULK";
1514 break;
1515 case USB_ENDPOINT_XFER_INT:
1516 pipetype = "INTERRUPT";
1517 break;
1518 case USB_ENDPOINT_XFER_ISOC:
1519 pipetype = "ISOCHRONOUS";
1520 break;
1521 default:
1522 pipetype = "UNKNOWN";
1523 break;
1524 }
1525
1526 dev_err(hsotg->dev, " Endpoint type: %s\n", pipetype);
1527
1528 switch (chan->speed) {
1529 case USB_SPEED_HIGH:
1530 speed = "HIGH";
1531 break;
1532 case USB_SPEED_FULL:
1533 speed = "FULL";
1534 break;
1535 case USB_SPEED_LOW:
1536 speed = "LOW";
1537 break;
1538 default:
1539 speed = "UNKNOWN";
1540 break;
1541 }
1542
1543 dev_err(hsotg->dev, " Speed: %s\n", speed);
1544
1545 dev_err(hsotg->dev, " Max packet size: %d\n",
1546 dwc2_hcd_get_mps(&urb->pipe_info));
1547 dev_err(hsotg->dev, " Data buffer length: %d\n", urb->length);
1548 dev_err(hsotg->dev, " Transfer buffer: %p, Transfer DMA: %08lx\n",
1549 urb->buf, (unsigned long)urb->dma);
1550 dev_err(hsotg->dev, " Setup buffer: %p, Setup DMA: %08lx\n",
1551 urb->setup_packet, (unsigned long)urb->setup_dma);
1552 dev_err(hsotg->dev, " Interval: %d\n", urb->interval);
1553
1554 /* Core halts the channel for Descriptor DMA mode */
1555 if (hsotg->core_params->dma_desc_enable > 0) {
1556 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1557 DWC2_HC_XFER_AHB_ERR);
1558 goto handle_ahberr_done;
1559 }
1560
1561 dwc2_host_complete(hsotg, qtd, -EIO);
1562
1563handle_ahberr_halt:
1564 /*
1565 * Force a channel halt. Don't call dwc2_halt_channel because that won't
1566 * write to the HCCHARn register in DMA mode to force the halt.
1567 */
1568 dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);
1569
1570handle_ahberr_done:
1571 disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1572}
1573
1574/*
1575 * Handles a host channel transaction error interrupt. This handler may be
1576 * called in either DMA mode or Slave mode.
1577 */
1578static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1579 struct dwc2_host_chan *chan, int chnum,
1580 struct dwc2_qtd *qtd)
1581{
1582 dev_dbg(hsotg->dev,
1583 "--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1584
1585 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1586
1587 if (hsotg->core_params->dma_desc_enable > 0) {
1588 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1589 DWC2_HC_XFER_XACT_ERR);
1590 goto handle_xacterr_done;
1591 }
1592
1593 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1594 case USB_ENDPOINT_XFER_CONTROL:
1595 case USB_ENDPOINT_XFER_BULK:
1596 qtd->error_count++;
1597 if (!chan->qh->ping_state) {
1598
1599 dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1600 qtd, DWC2_HC_XFER_XACT_ERR);
1601 dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1602 if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1603 chan->qh->ping_state = 1;
1604 }
1605
1606 /*
1607 * Halt the channel so the transfer can be re-started from
1608 * the appropriate point or the PING protocol will start
1609 */
1610 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1611 break;
1612 case USB_ENDPOINT_XFER_INT:
1613 qtd->error_count++;
1614 if (chan->do_split && chan->complete_split)
1615 qtd->complete_split = 0;
1616 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1617 break;
1618 case USB_ENDPOINT_XFER_ISOC:
1619 {
1620 enum dwc2_halt_status halt_status;
1621
1622 halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1623 chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1624 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1625 }
1626 break;
1627 }
1628
1629handle_xacterr_done:
1630 disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1631}
1632
1633/*
1634 * Handles a host channel frame overrun interrupt. This handler may be called
1635 * in either DMA mode or Slave mode.
1636 */
1637static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1638 struct dwc2_host_chan *chan, int chnum,
1639 struct dwc2_qtd *qtd)
1640{
1641 enum dwc2_halt_status halt_status;
1642
1643 if (dbg_hc(chan))
1644 dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1645 chnum);
1646
1647 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1648
1649 switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1650 case USB_ENDPOINT_XFER_CONTROL:
1651 case USB_ENDPOINT_XFER_BULK:
1652 break;
1653 case USB_ENDPOINT_XFER_INT:
1654 dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1655 break;
1656 case USB_ENDPOINT_XFER_ISOC:
1657 halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1658 qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1659 dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1660 break;
1661 }
1662
1663 disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1664}
1665
1666/*
1667 * Handles a host channel data toggle error interrupt. This handler may be
1668 * called in either DMA mode or Slave mode.
1669 */
1670static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1671 struct dwc2_host_chan *chan, int chnum,
1672 struct dwc2_qtd *qtd)
1673{
1674 dev_dbg(hsotg->dev,
1675 "--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1676
1677 if (chan->ep_is_in)
1678 qtd->error_count = 0;
1679 else
1680 dev_err(hsotg->dev,
1681 "Data Toggle Error on OUT transfer, channel %d\n",
1682 chnum);
1683
1684 dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1685 disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1686}
1687
1688/*
1689 * For debug only. It checks that a valid halt status is set and that
1690 * HCCHARn.chdis is clear. If there's a problem, corrective action is
1691 * taken and a warning is issued.
1692 *
1693 * Return: true if halt status is ok, false otherwise
1694 */
1695static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1696 struct dwc2_host_chan *chan, int chnum,
1697 struct dwc2_qtd *qtd)
1698{
1699#ifdef DEBUG
1700 u32 hcchar;
1701 u32 hctsiz;
1702 u32 hcintmsk;
1703 u32 hcsplt;
1704
1705 if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1706 /*
1707 * This code is here only as a check. This condition should
1708 * never happen. Ignore the halt if it does occur.
1709 */
1710 hcchar = readl(hsotg->regs + HCCHAR(chnum));
1711 hctsiz = readl(hsotg->regs + HCTSIZ(chnum));
1712 hcintmsk = readl(hsotg->regs + HCINTMSK(chnum));
1713 hcsplt = readl(hsotg->regs + HCSPLT(chnum));
1714 dev_dbg(hsotg->dev,
1715 "%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1716 __func__);
1717 dev_dbg(hsotg->dev,
1718 "channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1719 chnum, hcchar, hctsiz);
1720 dev_dbg(hsotg->dev,
1721 "hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1722 chan->hcint, hcintmsk, hcsplt);
1723 if (qtd)
1724 dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1725 qtd->complete_split);
1726 dev_warn(hsotg->dev,
1727 "%s: no halt status, channel %d, ignoring interrupt\n",
1728 __func__, chnum);
1729 return false;
1730 }
1731
1732 /*
1733 * This code is here only as a check. hcchar.chdis should never be set
1734 * when the halt interrupt occurs. Halt the channel again if it does
1735 * occur.
1736 */
1737 hcchar = readl(hsotg->regs + HCCHAR(chnum));
1738 if (hcchar & HCCHAR_CHDIS) {
1739 dev_warn(hsotg->dev,
1740 "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1741 __func__, hcchar);
1742 chan->halt_pending = 0;
1743 dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
1744 return false;
1745 }
1746#endif
1747
1748 return true;
1749}
1750
1751/*
1752 * Handles a host Channel Halted interrupt in DMA mode. This handler
1753 * determines the reason the channel halted and proceeds accordingly.
1754 */
1755static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1756 struct dwc2_host_chan *chan, int chnum,
1757 struct dwc2_qtd *qtd)
1758{
1759 u32 hcintmsk;
1760 int out_nak_enh = 0;
1761
1762 if (dbg_hc(chan))
1763 dev_vdbg(hsotg->dev,
1764 "--Host Channel %d Interrupt: DMA Channel Halted--\n",
1765 chnum);
1766
1767 /*
1768 * For core with OUT NAK enhancement, the flow for high-speed
1769 * CONTROL/BULK OUT is handled a little differently
1770 */
1771 if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1772 if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1773 (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1774 chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1775 out_nak_enh = 1;
1776 }
1777 }
1778
1779 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1780 (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1781 hsotg->core_params->dma_desc_enable <= 0)) {
1782 if (hsotg->core_params->dma_desc_enable > 0)
1783 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1784 chan->halt_status);
1785 else
1786 /*
1787 * Just release the channel. A dequeue can happen on a
1788 * transfer timeout. In the case of an AHB Error, the
1789 * channel was forced to halt because there's no way to
1790 * gracefully recover.
1791 */
1792 dwc2_release_channel(hsotg, chan, qtd,
1793 chan->halt_status);
1794 return;
1795 }
1796
1797 hcintmsk = readl(hsotg->regs + HCINTMSK(chnum));
1798
1799 if (chan->hcint & HCINTMSK_XFERCOMPL) {
1800 /*
1801 * Todo: This is here because of a possible hardware bug. Spec
1802 * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1803 * interrupt w/ACK bit set should occur, but I only see the
1804 * XFERCOMP bit, even with it masked out. This is a workaround
1805 * for that behavior. Should fix this when hardware is fixed.
1806 */
1807 if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1808 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1809 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1810 } else if (chan->hcint & HCINTMSK_STALL) {
1811 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1812 } else if ((chan->hcint & HCINTMSK_XACTERR) &&
1813 hsotg->core_params->dma_desc_enable <= 0) {
1814 if (out_nak_enh) {
1815 if (chan->hcint &
1816 (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1817 dev_vdbg(hsotg->dev,
1818 "XactErr with NYET/NAK/ACK\n");
1819 qtd->error_count = 0;
1820 } else {
1821 dev_vdbg(hsotg->dev,
1822 "XactErr without NYET/NAK/ACK\n");
1823 }
1824 }
1825
1826 /*
1827 * Must handle xacterr before nak or ack. Could get a xacterr
1828 * at the same time as either of these on a BULK/CONTROL OUT
1829 * that started with a PING. The xacterr takes precedence.
1830 */
1831 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1832 } else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1833 hsotg->core_params->dma_desc_enable > 0) {
1834 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1835 } else if ((chan->hcint & HCINTMSK_AHBERR) &&
1836 hsotg->core_params->dma_desc_enable > 0) {
1837 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1838 } else if (chan->hcint & HCINTMSK_BBLERR) {
1839 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1840 } else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1841 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1842 } else if (!out_nak_enh) {
1843 if (chan->hcint & HCINTMSK_NYET) {
1844 /*
1845 * Must handle nyet before nak or ack. Could get a nyet
1846 * at the same time as either of those on a BULK/CONTROL
1847 * OUT that started with a PING. The nyet takes
1848 * precedence.
1849 */
1850 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1851 } else if ((chan->hcint & HCINTMSK_NAK) &&
1852 !(hcintmsk & HCINTMSK_NAK)) {
1853 /*
1854 * If nak is not masked, it's because a non-split IN
1855 * transfer is in an error state. In that case, the nak
1856 * is handled by the nak interrupt handler, not here.
1857 * Handle nak here for BULK/CONTROL OUT transfers, which
1858 * halt on a NAK to allow rewinding the buffer pointer.
1859 */
1860 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1861 } else if ((chan->hcint & HCINTMSK_ACK) &&
1862 !(hcintmsk & HCINTMSK_ACK)) {
1863 /*
1864 * If ack is not masked, it's because a non-split IN
1865 * transfer is in an error state. In that case, the ack
1866 * is handled by the ack interrupt handler, not here.
1867 * Handle ack here for split transfers. Start splits
1868 * halt on ACK.
1869 */
1870 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1871 } else {
1872 if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1873 chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1874 /*
1875 * A periodic transfer halted with no other
1876 * channel interrupts set. Assume it was halted
1877 * by the core because it could not be completed
1878 * in its scheduled (micro)frame.
1879 */
1880 dev_dbg(hsotg->dev,
1881 "%s: Halt channel %d (assume incomplete periodic transfer)\n",
1882 __func__, chnum);
1883 dwc2_halt_channel(hsotg, chan, qtd,
1884 DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1885 } else {
1886 dev_err(hsotg->dev,
1887 "%s: Channel %d - ChHltd set, but reason is unknown\n",
1888 __func__, chnum);
1889 dev_err(hsotg->dev,
1890 "hcint 0x%08x, intsts 0x%08x\n",
1891 chan->hcint,
1892 readl(hsotg->regs + GINTSTS));
1893 }
1894 }
1895 } else {
1896 dev_info(hsotg->dev,
1897 "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1898 chan->hcint);
1899 }
1900}
1901
1902/*
1903 * Handles a host channel Channel Halted interrupt
1904 *
1905 * In slave mode, this handler is called only when the driver specifically
1906 * requests a halt. This occurs during handling other host channel interrupts
1907 * (e.g. nak, xacterr, stall, nyet, etc.).
1908 *
1909 * In DMA mode, this is the interrupt that occurs when the core has finished
1910 * processing a transfer on a channel. Other host channel interrupts (except
1911 * ahberr) are disabled in DMA mode.
1912 */
1913static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
1914 struct dwc2_host_chan *chan, int chnum,
1915 struct dwc2_qtd *qtd)
1916{
1917 if (dbg_hc(chan))
1918 dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
1919 chnum);
1920
1921 if (hsotg->core_params->dma_enable > 0) {
1922 dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
1923 } else {
1924 if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
1925 return;
1926 dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
1927 }
1928}
1929
1930/* Handles interrupt for a specific Host Channel */
1931static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
1932{
1933 struct dwc2_qtd *qtd;
1934 struct dwc2_host_chan *chan;
1935 u32 hcint, hcintmsk;
1936
1937 chan = hsotg->hc_ptr_array[chnum];
1938
1939 hcint = readl(hsotg->regs + HCINT(chnum));
1940 hcintmsk = readl(hsotg->regs + HCINTMSK(chnum));
1941 if (!chan) {
1942 dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
1943 writel(hcint, hsotg->regs + HCINT(chnum));
1944 return;
1945 }
1946
1947 if (dbg_hc(chan)) {
1948 dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
1949 chnum);
1950 dev_vdbg(hsotg->dev,
1951 " hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
1952 hcint, hcintmsk, hcint & hcintmsk);
1953 }
1954
1955 writel(hcint, hsotg->regs + HCINT(chnum));
1956 chan->hcint = hcint;
1957 hcint &= hcintmsk;
1958
1959 /*
1960 * If the channel was halted due to a dequeue, the qtd list might
1961 * be empty or at least the first entry will not be the active qtd.
1962 * In this case, take a shortcut and just release the channel.
1963 */
1964 if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
1965 /*
1966 * If the channel was halted, this should be the only
1967 * interrupt unmasked
1968 */
1969 WARN_ON(hcint != HCINTMSK_CHHLTD);
1970 if (hsotg->core_params->dma_desc_enable > 0)
1971 dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1972 chan->halt_status);
1973 else
1974 dwc2_release_channel(hsotg, chan, NULL,
1975 chan->halt_status);
1976 return;
1977 }
1978
1979 if (list_empty(&chan->qh->qtd_list)) {
1980 /*
1981 * TODO: Will this ever happen with the
1982 * DWC2_HC_XFER_URB_DEQUEUE handling above?
1983 */
1984 dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
1985 chnum);
1986 dev_dbg(hsotg->dev,
1987 " hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
1988 chan->hcint, hcintmsk, hcint);
1989 chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
1990 disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
1991 chan->hcint = 0;
1992 return;
1993 }
1994
1995 qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
1996 qtd_list_entry);
1997
1998 if (hsotg->core_params->dma_enable <= 0) {
1999 if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
2000 hcint &= ~HCINTMSK_CHHLTD;
2001 }
2002
2003 if (hcint & HCINTMSK_XFERCOMPL) {
2004 dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
2005 /*
2006 * If NYET occurred at same time as Xfer Complete, the NYET is
2007 * handled by the Xfer Complete interrupt handler. Don't want
2008 * to call the NYET interrupt handler in this case.
2009 */
2010 hcint &= ~HCINTMSK_NYET;
2011 }
2012 if (hcint & HCINTMSK_CHHLTD)
2013 dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2014 if (hcint & HCINTMSK_AHBERR)
2015 dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2016 if (hcint & HCINTMSK_STALL)
2017 dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2018 if (hcint & HCINTMSK_NAK)
2019 dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2020 if (hcint & HCINTMSK_ACK)
2021 dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2022 if (hcint & HCINTMSK_NYET)
2023 dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2024 if (hcint & HCINTMSK_XACTERR)
2025 dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2026 if (hcint & HCINTMSK_BBLERR)
2027 dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2028 if (hcint & HCINTMSK_FRMOVRUN)
2029 dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2030 if (hcint & HCINTMSK_DATATGLERR)
2031 dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2032
2033 chan->hcint = 0;
2034}
2035
2036/*
2037 * This interrupt indicates that one or more host channels has a pending
2038 * interrupt. There are multiple conditions that can cause each host channel
2039 * interrupt. This function determines which conditions have occurred for each
2040 * host channel interrupt and handles them appropriately.
2041 */
2042static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2043{
2044 u32 haint;
2045 int i;
2046
2047 haint = readl(hsotg->regs + HAINT);
2048 if (dbg_perio()) {
2049 dev_vdbg(hsotg->dev, "%s()\n", __func__);
2050
2051 dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2052 }
2053
2054 for (i = 0; i < hsotg->core_params->host_channels; i++) {
2055 if (haint & (1 << i))
2056 dwc2_hc_n_intr(hsotg, i);
2057 }
2058}
2059
2060/* This function handles interrupts for the HCD */
2061irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2062{
2063 u32 gintsts, dbg_gintsts;
2064 irqreturn_t retval = IRQ_NONE;
2065
2066 if (!dwc2_is_controller_alive(hsotg)) {
2067 dev_warn(hsotg->dev, "Controller is dead\n");
2068 return retval;
2069 }
2070
2071 spin_lock(&hsotg->lock);
2072
2073 /* Check if HOST Mode */
2074 if (dwc2_is_host_mode(hsotg)) {
2075 gintsts = dwc2_read_core_intr(hsotg);
2076 if (!gintsts) {
2077 spin_unlock(&hsotg->lock);
2078 return retval;
2079 }
2080
2081 retval = IRQ_HANDLED;
2082
2083 dbg_gintsts = gintsts;
2084#ifndef DEBUG_SOF
2085 dbg_gintsts &= ~GINTSTS_SOF;
2086#endif
2087 if (!dbg_perio())
2088 dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
2089 GINTSTS_PTXFEMP);
2090
2091 /* Only print if there are any non-suppressed interrupts left */
2092 if (dbg_gintsts)
2093 dev_vdbg(hsotg->dev,
2094 "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2095 gintsts);
2096
2097 if (gintsts & GINTSTS_SOF)
2098 dwc2_sof_intr(hsotg);
2099 if (gintsts & GINTSTS_RXFLVL)
2100 dwc2_rx_fifo_level_intr(hsotg);
2101 if (gintsts & GINTSTS_NPTXFEMP)
2102 dwc2_np_tx_fifo_empty_intr(hsotg);
2103 if (gintsts & GINTSTS_PRTINT)
2104 dwc2_port_intr(hsotg);
2105 if (gintsts & GINTSTS_HCHINT)
2106 dwc2_hc_intr(hsotg);
2107 if (gintsts & GINTSTS_PTXFEMP)
2108 dwc2_perio_tx_fifo_empty_intr(hsotg);
2109
2110 if (dbg_gintsts) {
2111 dev_vdbg(hsotg->dev,
2112 "DWC OTG HCD Finished Servicing Interrupts\n");
2113 dev_vdbg(hsotg->dev,
2114 "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2115 readl(hsotg->regs + GINTSTS),
2116 readl(hsotg->regs + GINTMSK));
2117 }
2118 }
2119
2120 spin_unlock(&hsotg->lock);
2121
2122 return retval;
2123}