1// SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
   2/*
   3 * hcd_intr.c - DesignWare HS OTG Controller host-mode interrupt handling
   4 *
   5 * Copyright (C) 2004-2013 Synopsys, Inc.
   6 *
   7 * Redistribution and use in source and binary forms, with or without
   8 * modification, are permitted provided that the following conditions
   9 * are met:
  10 * 1. Redistributions of source code must retain the above copyright
  11 *    notice, this list of conditions, and the following disclaimer,
  12 *    without modification.
  13 * 2. Redistributions in binary form must reproduce the above copyright
  14 *    notice, this list of conditions and the following disclaimer in the
  15 *    documentation and/or other materials provided with the distribution.
  16 * 3. The names of the above-listed copyright holders may not be used
  17 *    to endorse or promote products derived from this software without
  18 *    specific prior written permission.
  19 *
  20 * ALTERNATIVELY, this software may be distributed under the terms of the
  21 * GNU General Public License ("GPL") as published by the Free Software
  22 * Foundation; either version 2 of the License, or (at your option) any
  23 * later version.
  24 *
  25 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
  26 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
  27 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
  28 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
  29 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
  30 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
  31 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
  32 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
  33 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
  34 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
  35 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  36 */
  37
  38/*
  39 * This file contains the interrupt handlers for Host mode
  40 */
  41#include <linux/kernel.h>
  42#include <linux/module.h>
  43#include <linux/spinlock.h>
  44#include <linux/interrupt.h>
  45#include <linux/dma-mapping.h>
  46#include <linux/io.h>
  47#include <linux/slab.h>
  48#include <linux/usb.h>
  49
  50#include <linux/usb/hcd.h>
  51#include <linux/usb/ch11.h>
  52
  53#include "core.h"
  54#include "hcd.h"
  55
  56/*
  57 * If we get this many NAKs on a split transaction we'll slow down
  58 * retransmission.  A 1 here means delay after the first NAK.
  59 */
  60#define DWC2_NAKS_BEFORE_DELAY		3
  61
  62/* This function is for debug only */
  63static void dwc2_track_missed_sofs(struct dwc2_hsotg *hsotg)
  64{
  65	u16 curr_frame_number = hsotg->frame_number;
  66	u16 expected = dwc2_frame_num_inc(hsotg->last_frame_num, 1);
  67
  68	if (expected != curr_frame_number)
  69		dwc2_sch_vdbg(hsotg, "MISSED SOF %04x != %04x\n",
  70			      expected, curr_frame_number);
  71
  72#ifdef CONFIG_USB_DWC2_TRACK_MISSED_SOFS
  73	if (hsotg->frame_num_idx < FRAME_NUM_ARRAY_SIZE) {
  74		if (expected != curr_frame_number) {
  75			hsotg->frame_num_array[hsotg->frame_num_idx] =
  76					curr_frame_number;
  77			hsotg->last_frame_num_array[hsotg->frame_num_idx] =
  78					hsotg->last_frame_num;
  79			hsotg->frame_num_idx++;
  80		}
  81	} else if (!hsotg->dumped_frame_num_array) {
  82		int i;
  83
  84		dev_info(hsotg->dev, "Frame     Last Frame\n");
  85		dev_info(hsotg->dev, "-----     ----------\n");
  86		for (i = 0; i < FRAME_NUM_ARRAY_SIZE; i++) {
  87			dev_info(hsotg->dev, "0x%04x    0x%04x\n",
  88				 hsotg->frame_num_array[i],
  89				 hsotg->last_frame_num_array[i]);
  90		}
  91		hsotg->dumped_frame_num_array = 1;
  92	}
  93#endif
  94	hsotg->last_frame_num = curr_frame_number;
  95}
  96
  97static void dwc2_hc_handle_tt_clear(struct dwc2_hsotg *hsotg,
  98				    struct dwc2_host_chan *chan,
  99				    struct dwc2_qtd *qtd)
 100{
 101	struct usb_device *root_hub = dwc2_hsotg_to_hcd(hsotg)->self.root_hub;
 102	struct urb *usb_urb;
 103
 104	if (!chan->qh)
 105		return;
 106
 107	if (chan->qh->dev_speed == USB_SPEED_HIGH)
 108		return;
 109
 110	if (!qtd->urb)
 111		return;
 112
 113	usb_urb = qtd->urb->priv;
 114	if (!usb_urb || !usb_urb->dev || !usb_urb->dev->tt)
 115		return;
 116
 117	/*
 118	 * The root hub doesn't really have a TT, but Linux thinks it
 119	 * does because how could you have a "high speed hub" that
 120	 * directly talks directly to low speed devices without a TT?
 121	 * It's all lies.  Lies, I tell you.
 122	 */
 123	if (usb_urb->dev->tt->hub == root_hub)
 124		return;
 125
 126	if (qtd->urb->status != -EPIPE && qtd->urb->status != -EREMOTEIO) {
 127		chan->qh->tt_buffer_dirty = 1;
 128		if (usb_hub_clear_tt_buffer(usb_urb))
 129			/* Clear failed; let's hope things work anyway */
 130			chan->qh->tt_buffer_dirty = 0;
 131	}
 132}
 133
 134/*
 135 * Handles the start-of-frame interrupt in host mode. Non-periodic
 136 * transactions may be queued to the DWC_otg controller for the current
 137 * (micro)frame. Periodic transactions may be queued to the controller
 138 * for the next (micro)frame.
 139 */
 140static void dwc2_sof_intr(struct dwc2_hsotg *hsotg)
 141{
 142	struct list_head *qh_entry;
 143	struct dwc2_qh *qh;
 144	enum dwc2_transaction_type tr_type;
 145
 146	/* Clear interrupt */
 147	dwc2_writel(GINTSTS_SOF, hsotg->regs + GINTSTS);
 148
 149#ifdef DEBUG_SOF
 150	dev_vdbg(hsotg->dev, "--Start of Frame Interrupt--\n");
 151#endif
 152
 153	hsotg->frame_number = dwc2_hcd_get_frame_number(hsotg);
 154
 155	dwc2_track_missed_sofs(hsotg);
 156
 157	/* Determine whether any periodic QHs should be executed */
 158	qh_entry = hsotg->periodic_sched_inactive.next;
 159	while (qh_entry != &hsotg->periodic_sched_inactive) {
 160		qh = list_entry(qh_entry, struct dwc2_qh, qh_list_entry);
 161		qh_entry = qh_entry->next;
 162		if (dwc2_frame_num_le(qh->next_active_frame,
 163				      hsotg->frame_number)) {
 164			dwc2_sch_vdbg(hsotg, "QH=%p ready fn=%04x, nxt=%04x\n",
 165				      qh, hsotg->frame_number,
 166				      qh->next_active_frame);
 167
 168			/*
 169			 * Move QH to the ready list to be executed next
 170			 * (micro)frame
 171			 */
 172			list_move_tail(&qh->qh_list_entry,
 173				       &hsotg->periodic_sched_ready);
 174		}
 175	}
 176	tr_type = dwc2_hcd_select_transactions(hsotg);
 177	if (tr_type != DWC2_TRANSACTION_NONE)
 178		dwc2_hcd_queue_transactions(hsotg, tr_type);
 179}
 180
 181/*
 182 * Handles the Rx FIFO Level Interrupt, which indicates that there is
 183 * at least one packet in the Rx FIFO. The packets are moved from the FIFO to
 184 * memory if the DWC_otg controller is operating in Slave mode.
 185 */
 186static void dwc2_rx_fifo_level_intr(struct dwc2_hsotg *hsotg)
 187{
 188	u32 grxsts, chnum, bcnt, dpid, pktsts;
 189	struct dwc2_host_chan *chan;
 190
 191	if (dbg_perio())
 192		dev_vdbg(hsotg->dev, "--RxFIFO Level Interrupt--\n");
 193
 194	grxsts = dwc2_readl(hsotg->regs + GRXSTSP);
 195	chnum = (grxsts & GRXSTS_HCHNUM_MASK) >> GRXSTS_HCHNUM_SHIFT;
 196	chan = hsotg->hc_ptr_array[chnum];
 197	if (!chan) {
 198		dev_err(hsotg->dev, "Unable to get corresponding channel\n");
 199		return;
 200	}
 201
 202	bcnt = (grxsts & GRXSTS_BYTECNT_MASK) >> GRXSTS_BYTECNT_SHIFT;
 203	dpid = (grxsts & GRXSTS_DPID_MASK) >> GRXSTS_DPID_SHIFT;
 204	pktsts = (grxsts & GRXSTS_PKTSTS_MASK) >> GRXSTS_PKTSTS_SHIFT;
 205
 206	/* Packet Status */
 207	if (dbg_perio()) {
 208		dev_vdbg(hsotg->dev, "    Ch num = %d\n", chnum);
 209		dev_vdbg(hsotg->dev, "    Count = %d\n", bcnt);
 210		dev_vdbg(hsotg->dev, "    DPID = %d, chan.dpid = %d\n", dpid,
 211			 chan->data_pid_start);
 212		dev_vdbg(hsotg->dev, "    PStatus = %d\n", pktsts);
 213	}
 214
 215	switch (pktsts) {
 216	case GRXSTS_PKTSTS_HCHIN:
 217		/* Read the data into the host buffer */
 218		if (bcnt > 0) {
 219			dwc2_read_packet(hsotg, chan->xfer_buf, bcnt);
 220
 221			/* Update the HC fields for the next packet received */
 222			chan->xfer_count += bcnt;
 223			chan->xfer_buf += bcnt;
 224		}
 225		break;
 226	case GRXSTS_PKTSTS_HCHIN_XFER_COMP:
 227	case GRXSTS_PKTSTS_DATATOGGLEERR:
 228	case GRXSTS_PKTSTS_HCHHALTED:
 229		/* Handled in interrupt, just ignore data */
 230		break;
 231	default:
 232		dev_err(hsotg->dev,
 233			"RxFIFO Level Interrupt: Unknown status %d\n", pktsts);
 234		break;
 235	}
 236}
 237
 238/*
 239 * This interrupt occurs when the non-periodic Tx FIFO is half-empty. More
 240 * data packets may be written to the FIFO for OUT transfers. More requests
 241 * may be written to the non-periodic request queue for IN transfers. This
 242 * interrupt is enabled only in Slave mode.
 243 */
 244static void dwc2_np_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
 245{
 246	dev_vdbg(hsotg->dev, "--Non-Periodic TxFIFO Empty Interrupt--\n");
 247	dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_NON_PERIODIC);
 248}
 249
 250/*
 251 * This interrupt occurs when the periodic Tx FIFO is half-empty. More data
 252 * packets may be written to the FIFO for OUT transfers. More requests may be
 253 * written to the periodic request queue for IN transfers. This interrupt is
 254 * enabled only in Slave mode.
 255 */
 256static void dwc2_perio_tx_fifo_empty_intr(struct dwc2_hsotg *hsotg)
 257{
 258	if (dbg_perio())
 259		dev_vdbg(hsotg->dev, "--Periodic TxFIFO Empty Interrupt--\n");
 260	dwc2_hcd_queue_transactions(hsotg, DWC2_TRANSACTION_PERIODIC);
 261}
 262
 263static void dwc2_hprt0_enable(struct dwc2_hsotg *hsotg, u32 hprt0,
 264			      u32 *hprt0_modify)
 265{
 266	struct dwc2_core_params *params = &hsotg->params;
 267	int do_reset = 0;
 268	u32 usbcfg;
 269	u32 prtspd;
 270	u32 hcfg;
 271	u32 fslspclksel;
 272	u32 hfir;
 273
 274	dev_vdbg(hsotg->dev, "%s(%p)\n", __func__, hsotg);
 275
 276	/* Every time when port enables calculate HFIR.FrInterval */
 277	hfir = dwc2_readl(hsotg->regs + HFIR);
 278	hfir &= ~HFIR_FRINT_MASK;
 279	hfir |= dwc2_calc_frame_interval(hsotg) << HFIR_FRINT_SHIFT &
 280		HFIR_FRINT_MASK;
 281	dwc2_writel(hfir, hsotg->regs + HFIR);
 282
 283	/* Check if we need to adjust the PHY clock speed for low power */
 284	if (!params->host_support_fs_ls_low_power) {
 285		/* Port has been enabled, set the reset change flag */
 286		hsotg->flags.b.port_reset_change = 1;
 287		return;
 288	}
 289
 290	usbcfg = dwc2_readl(hsotg->regs + GUSBCFG);
 291	prtspd = (hprt0 & HPRT0_SPD_MASK) >> HPRT0_SPD_SHIFT;
 292
 293	if (prtspd == HPRT0_SPD_LOW_SPEED || prtspd == HPRT0_SPD_FULL_SPEED) {
 294		/* Low power */
 295		if (!(usbcfg & GUSBCFG_PHY_LP_CLK_SEL)) {
 296			/* Set PHY low power clock select for FS/LS devices */
 297			usbcfg |= GUSBCFG_PHY_LP_CLK_SEL;
 298			dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
 299			do_reset = 1;
 300		}
 301
 302		hcfg = dwc2_readl(hsotg->regs + HCFG);
 303		fslspclksel = (hcfg & HCFG_FSLSPCLKSEL_MASK) >>
 304			      HCFG_FSLSPCLKSEL_SHIFT;
 305
 306		if (prtspd == HPRT0_SPD_LOW_SPEED &&
 307		    params->host_ls_low_power_phy_clk) {
 
 308			/* 6 MHZ */
 309			dev_vdbg(hsotg->dev,
 310				 "FS_PHY programming HCFG to 6 MHz\n");
 311			if (fslspclksel != HCFG_FSLSPCLKSEL_6_MHZ) {
 312				fslspclksel = HCFG_FSLSPCLKSEL_6_MHZ;
 313				hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
 314				hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
 315				dwc2_writel(hcfg, hsotg->regs + HCFG);
 316				do_reset = 1;
 317			}
 318		} else {
 319			/* 48 MHZ */
 320			dev_vdbg(hsotg->dev,
 321				 "FS_PHY programming HCFG to 48 MHz\n");
 322			if (fslspclksel != HCFG_FSLSPCLKSEL_48_MHZ) {
 323				fslspclksel = HCFG_FSLSPCLKSEL_48_MHZ;
 324				hcfg &= ~HCFG_FSLSPCLKSEL_MASK;
 325				hcfg |= fslspclksel << HCFG_FSLSPCLKSEL_SHIFT;
 326				dwc2_writel(hcfg, hsotg->regs + HCFG);
 327				do_reset = 1;
 328			}
 329		}
 330	} else {
 331		/* Not low power */
 332		if (usbcfg & GUSBCFG_PHY_LP_CLK_SEL) {
 333			usbcfg &= ~GUSBCFG_PHY_LP_CLK_SEL;
 334			dwc2_writel(usbcfg, hsotg->regs + GUSBCFG);
 335			do_reset = 1;
 336		}
 337	}
 338
 339	if (do_reset) {
 340		*hprt0_modify |= HPRT0_RST;
 341		dwc2_writel(*hprt0_modify, hsotg->regs + HPRT0);
 342		queue_delayed_work(hsotg->wq_otg, &hsotg->reset_work,
 343				   msecs_to_jiffies(60));
 344	} else {
 345		/* Port has been enabled, set the reset change flag */
 346		hsotg->flags.b.port_reset_change = 1;
 347	}
 348}
 349
 350/*
 351 * There are multiple conditions that can cause a port interrupt. This function
 352 * determines which interrupt conditions have occurred and handles them
 353 * appropriately.
 354 */
 355static void dwc2_port_intr(struct dwc2_hsotg *hsotg)
 356{
 357	u32 hprt0;
 358	u32 hprt0_modify;
 359
 360	dev_vdbg(hsotg->dev, "--Port Interrupt--\n");
 361
 362	hprt0 = dwc2_readl(hsotg->regs + HPRT0);
 363	hprt0_modify = hprt0;
 364
 365	/*
 366	 * Clear appropriate bits in HPRT0 to clear the interrupt bit in
 367	 * GINTSTS
 368	 */
 369	hprt0_modify &= ~(HPRT0_ENA | HPRT0_CONNDET | HPRT0_ENACHG |
 370			  HPRT0_OVRCURRCHG);
 371
 372	/*
 373	 * Port Connect Detected
 374	 * Set flag and clear if detected
 375	 */
 376	if (hprt0 & HPRT0_CONNDET) {
 377		dwc2_writel(hprt0_modify | HPRT0_CONNDET, hsotg->regs + HPRT0);
 378
 379		dev_vdbg(hsotg->dev,
 380			 "--Port Interrupt HPRT0=0x%08x Port Connect Detected--\n",
 381			 hprt0);
 382		dwc2_hcd_connect(hsotg);
 383
 384		/*
 385		 * The Hub driver asserts a reset when it sees port connect
 386		 * status change flag
 387		 */
 388	}
 389
 390	/*
 391	 * Port Enable Changed
 392	 * Clear if detected - Set internal flag if disabled
 393	 */
 394	if (hprt0 & HPRT0_ENACHG) {
 395		dwc2_writel(hprt0_modify | HPRT0_ENACHG, hsotg->regs + HPRT0);
 396		dev_vdbg(hsotg->dev,
 397			 "  --Port Interrupt HPRT0=0x%08x Port Enable Changed (now %d)--\n",
 398			 hprt0, !!(hprt0 & HPRT0_ENA));
 399		if (hprt0 & HPRT0_ENA) {
 400			hsotg->new_connection = true;
 401			dwc2_hprt0_enable(hsotg, hprt0, &hprt0_modify);
 402		} else {
 403			hsotg->flags.b.port_enable_change = 1;
 404			if (hsotg->params.dma_desc_fs_enable) {
 405				u32 hcfg;
 406
 407				hsotg->params.dma_desc_enable = false;
 408				hsotg->new_connection = false;
 409				hcfg = dwc2_readl(hsotg->regs + HCFG);
 410				hcfg &= ~HCFG_DESCDMA;
 411				dwc2_writel(hcfg, hsotg->regs + HCFG);
 412			}
 413		}
 414	}
 415
 416	/* Overcurrent Change Interrupt */
 417	if (hprt0 & HPRT0_OVRCURRCHG) {
 418		dwc2_writel(hprt0_modify | HPRT0_OVRCURRCHG,
 419			    hsotg->regs + HPRT0);
 420		dev_vdbg(hsotg->dev,
 421			 "  --Port Interrupt HPRT0=0x%08x Port Overcurrent Changed--\n",
 422			 hprt0);
 423		hsotg->flags.b.port_over_current_change = 1;
 424	}
 425}
 426
 427/*
 428 * Gets the actual length of a transfer after the transfer halts. halt_status
 429 * holds the reason for the halt.
 430 *
 431 * For IN transfers where halt_status is DWC2_HC_XFER_COMPLETE, *short_read
 432 * is set to 1 upon return if less than the requested number of bytes were
 433 * transferred. short_read may also be NULL on entry, in which case it remains
 434 * unchanged.
 435 */
 436static u32 dwc2_get_actual_xfer_length(struct dwc2_hsotg *hsotg,
 437				       struct dwc2_host_chan *chan, int chnum,
 438				       struct dwc2_qtd *qtd,
 439				       enum dwc2_halt_status halt_status,
 440				       int *short_read)
 441{
 442	u32 hctsiz, count, length;
 443
 444	hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
 445
 446	if (halt_status == DWC2_HC_XFER_COMPLETE) {
 447		if (chan->ep_is_in) {
 448			count = (hctsiz & TSIZ_XFERSIZE_MASK) >>
 449				TSIZ_XFERSIZE_SHIFT;
 450			length = chan->xfer_len - count;
 451			if (short_read)
 452				*short_read = (count != 0);
 453		} else if (chan->qh->do_split) {
 454			length = qtd->ssplit_out_xfer_count;
 455		} else {
 456			length = chan->xfer_len;
 457		}
 458	} else {
 459		/*
 460		 * Must use the hctsiz.pktcnt field to determine how much data
 461		 * has been transferred. This field reflects the number of
 462		 * packets that have been transferred via the USB. This is
 463		 * always an integral number of packets if the transfer was
 464		 * halted before its normal completion. (Can't use the
 465		 * hctsiz.xfersize field because that reflects the number of
 466		 * bytes transferred via the AHB, not the USB).
 467		 */
 468		count = (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT;
 469		length = (chan->start_pkt_count - count) * chan->max_packet;
 470	}
 471
 472	return length;
 473}
 474
 475/**
 476 * dwc2_update_urb_state() - Updates the state of the URB after a Transfer
 477 * Complete interrupt on the host channel. Updates the actual_length field
 478 * of the URB based on the number of bytes transferred via the host channel.
 479 * Sets the URB status if the data transfer is finished.
 480 *
 481 * Return: 1 if the data transfer specified by the URB is completely finished,
 482 * 0 otherwise
 483 */
 484static int dwc2_update_urb_state(struct dwc2_hsotg *hsotg,
 485				 struct dwc2_host_chan *chan, int chnum,
 486				 struct dwc2_hcd_urb *urb,
 487				 struct dwc2_qtd *qtd)
 488{
 489	u32 hctsiz;
 490	int xfer_done = 0;
 491	int short_read = 0;
 492	int xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
 493						      DWC2_HC_XFER_COMPLETE,
 494						      &short_read);
 495
 496	if (urb->actual_length + xfer_length > urb->length) {
 497		dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
 498		xfer_length = urb->length - urb->actual_length;
 499	}
 500
 501	dev_vdbg(hsotg->dev, "urb->actual_length=%d xfer_length=%d\n",
 502		 urb->actual_length, xfer_length);
 503	urb->actual_length += xfer_length;
 504
 505	if (xfer_length && chan->ep_type == USB_ENDPOINT_XFER_BULK &&
 506	    (urb->flags & URB_SEND_ZERO_PACKET) &&
 507	    urb->actual_length >= urb->length &&
 508	    !(urb->length % chan->max_packet)) {
 509		xfer_done = 0;
 510	} else if (short_read || urb->actual_length >= urb->length) {
 511		xfer_done = 1;
 512		urb->status = 0;
 513	}
 514
 515	hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
 516	dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
 517		 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
 518	dev_vdbg(hsotg->dev, "  chan->xfer_len %d\n", chan->xfer_len);
 519	dev_vdbg(hsotg->dev, "  hctsiz.xfersize %d\n",
 520		 (hctsiz & TSIZ_XFERSIZE_MASK) >> TSIZ_XFERSIZE_SHIFT);
 521	dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n", urb->length);
 522	dev_vdbg(hsotg->dev, "  urb->actual_length %d\n", urb->actual_length);
 523	dev_vdbg(hsotg->dev, "  short_read %d, xfer_done %d\n", short_read,
 524		 xfer_done);
 525
 526	return xfer_done;
 527}
 528
 529/*
 530 * Save the starting data toggle for the next transfer. The data toggle is
 531 * saved in the QH for non-control transfers and it's saved in the QTD for
 532 * control transfers.
 533 */
 534void dwc2_hcd_save_data_toggle(struct dwc2_hsotg *hsotg,
 535			       struct dwc2_host_chan *chan, int chnum,
 536			       struct dwc2_qtd *qtd)
 537{
 538	u32 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
 539	u32 pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
 540
 541	if (chan->ep_type != USB_ENDPOINT_XFER_CONTROL) {
 542		if (WARN(!chan || !chan->qh,
 543			 "chan->qh must be specified for non-control eps\n"))
 544			return;
 545
 546		if (pid == TSIZ_SC_MC_PID_DATA0)
 547			chan->qh->data_toggle = DWC2_HC_PID_DATA0;
 548		else
 549			chan->qh->data_toggle = DWC2_HC_PID_DATA1;
 550	} else {
 551		if (WARN(!qtd,
 552			 "qtd must be specified for control eps\n"))
 553			return;
 554
 555		if (pid == TSIZ_SC_MC_PID_DATA0)
 556			qtd->data_toggle = DWC2_HC_PID_DATA0;
 557		else
 558			qtd->data_toggle = DWC2_HC_PID_DATA1;
 559	}
 560}
 561
 562/**
 563 * dwc2_update_isoc_urb_state() - Updates the state of an Isochronous URB when
 564 * the transfer is stopped for any reason. The fields of the current entry in
 565 * the frame descriptor array are set based on the transfer state and the input
 566 * halt_status. Completes the Isochronous URB if all the URB frames have been
 567 * completed.
 568 *
 569 * Return: DWC2_HC_XFER_COMPLETE if there are more frames remaining to be
 570 * transferred in the URB. Otherwise return DWC2_HC_XFER_URB_COMPLETE.
 571 */
 572static enum dwc2_halt_status dwc2_update_isoc_urb_state(
 573		struct dwc2_hsotg *hsotg, struct dwc2_host_chan *chan,
 574		int chnum, struct dwc2_qtd *qtd,
 575		enum dwc2_halt_status halt_status)
 576{
 577	struct dwc2_hcd_iso_packet_desc *frame_desc;
 578	struct dwc2_hcd_urb *urb = qtd->urb;
 579
 580	if (!urb)
 581		return DWC2_HC_XFER_NO_HALT_STATUS;
 582
 583	frame_desc = &urb->iso_descs[qtd->isoc_frame_index];
 584
 585	switch (halt_status) {
 586	case DWC2_HC_XFER_COMPLETE:
 587		frame_desc->status = 0;
 588		frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
 589					chan, chnum, qtd, halt_status, NULL);
 590		break;
 591	case DWC2_HC_XFER_FRAME_OVERRUN:
 592		urb->error_count++;
 593		if (chan->ep_is_in)
 594			frame_desc->status = -ENOSR;
 595		else
 596			frame_desc->status = -ECOMM;
 597		frame_desc->actual_length = 0;
 598		break;
 599	case DWC2_HC_XFER_BABBLE_ERR:
 600		urb->error_count++;
 601		frame_desc->status = -EOVERFLOW;
 602		/* Don't need to update actual_length in this case */
 603		break;
 604	case DWC2_HC_XFER_XACT_ERR:
 605		urb->error_count++;
 606		frame_desc->status = -EPROTO;
 607		frame_desc->actual_length = dwc2_get_actual_xfer_length(hsotg,
 608					chan, chnum, qtd, halt_status, NULL);
 609
 610		/* Skip whole frame */
 611		if (chan->qh->do_split &&
 612		    chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
 613		    hsotg->params.host_dma) {
 614			qtd->complete_split = 0;
 615			qtd->isoc_split_offset = 0;
 616		}
 617
 618		break;
 619	default:
 620		dev_err(hsotg->dev, "Unhandled halt_status (%d)\n",
 621			halt_status);
 622		break;
 623	}
 624
 625	if (++qtd->isoc_frame_index == urb->packet_count) {
 626		/*
 627		 * urb->status is not used for isoc transfers. The individual
 628		 * frame_desc statuses are used instead.
 629		 */
 630		dwc2_host_complete(hsotg, qtd, 0);
 631		halt_status = DWC2_HC_XFER_URB_COMPLETE;
 632	} else {
 633		halt_status = DWC2_HC_XFER_COMPLETE;
 634	}
 635
 636	return halt_status;
 637}
 638
 639/*
 640 * Frees the first QTD in the QH's list if free_qtd is 1. For non-periodic
 641 * QHs, removes the QH from the active non-periodic schedule. If any QTDs are
 642 * still linked to the QH, the QH is added to the end of the inactive
 643 * non-periodic schedule. For periodic QHs, removes the QH from the periodic
 644 * schedule if no more QTDs are linked to the QH.
 645 */
 646static void dwc2_deactivate_qh(struct dwc2_hsotg *hsotg, struct dwc2_qh *qh,
 647			       int free_qtd)
 648{
 649	int continue_split = 0;
 650	struct dwc2_qtd *qtd;
 651
 652	if (dbg_qh(qh))
 653		dev_vdbg(hsotg->dev, "  %s(%p,%p,%d)\n", __func__,
 654			 hsotg, qh, free_qtd);
 655
 656	if (list_empty(&qh->qtd_list)) {
 657		dev_dbg(hsotg->dev, "## QTD list empty ##\n");
 658		goto no_qtd;
 659	}
 660
 661	qtd = list_first_entry(&qh->qtd_list, struct dwc2_qtd, qtd_list_entry);
 662
 663	if (qtd->complete_split)
 664		continue_split = 1;
 665	else if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_MID ||
 666		 qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_END)
 667		continue_split = 1;
 668
 669	if (free_qtd) {
 670		dwc2_hcd_qtd_unlink_and_free(hsotg, qtd, qh);
 671		continue_split = 0;
 672	}
 673
 674no_qtd:
 675	qh->channel = NULL;
 676	dwc2_hcd_qh_deactivate(hsotg, qh, continue_split);
 677}
 678
 679/**
 680 * dwc2_release_channel() - Releases a host channel for use by other transfers
 681 *
 682 * @hsotg:       The HCD state structure
 683 * @chan:        The host channel to release
 684 * @qtd:         The QTD associated with the host channel. This QTD may be
 685 *               freed if the transfer is complete or an error has occurred.
 686 * @halt_status: Reason the channel is being released. This status
 687 *               determines the actions taken by this function.
 688 *
 689 * Also attempts to select and queue more transactions since at least one host
 690 * channel is available.
 691 */
 692static void dwc2_release_channel(struct dwc2_hsotg *hsotg,
 693				 struct dwc2_host_chan *chan,
 694				 struct dwc2_qtd *qtd,
 695				 enum dwc2_halt_status halt_status)
 696{
 697	enum dwc2_transaction_type tr_type;
 698	u32 haintmsk;
 699	int free_qtd = 0;
 700
 701	if (dbg_hc(chan))
 702		dev_vdbg(hsotg->dev, "  %s: channel %d, halt_status %d\n",
 703			 __func__, chan->hc_num, halt_status);
 704
 705	switch (halt_status) {
 706	case DWC2_HC_XFER_URB_COMPLETE:
 707		free_qtd = 1;
 708		break;
 709	case DWC2_HC_XFER_AHB_ERR:
 710	case DWC2_HC_XFER_STALL:
 711	case DWC2_HC_XFER_BABBLE_ERR:
 712		free_qtd = 1;
 713		break;
 714	case DWC2_HC_XFER_XACT_ERR:
 715		if (qtd && qtd->error_count >= 3) {
 716			dev_vdbg(hsotg->dev,
 717				 "  Complete URB with transaction error\n");
 718			free_qtd = 1;
 719			dwc2_host_complete(hsotg, qtd, -EPROTO);
 720		}
 721		break;
 722	case DWC2_HC_XFER_URB_DEQUEUE:
 723		/*
 724		 * The QTD has already been removed and the QH has been
 725		 * deactivated. Don't want to do anything except release the
 726		 * host channel and try to queue more transfers.
 727		 */
 728		goto cleanup;
 729	case DWC2_HC_XFER_PERIODIC_INCOMPLETE:
 730		dev_vdbg(hsotg->dev, "  Complete URB with I/O error\n");
 731		free_qtd = 1;
 732		dwc2_host_complete(hsotg, qtd, -EIO);
 733		break;
 734	case DWC2_HC_XFER_NO_HALT_STATUS:
 735	default:
 736		break;
 737	}
 738
 739	dwc2_deactivate_qh(hsotg, chan->qh, free_qtd);
 740
 741cleanup:
 742	/*
 743	 * Release the host channel for use by other transfers. The cleanup
 744	 * function clears the channel interrupt enables and conditions, so
 745	 * there's no need to clear the Channel Halted interrupt separately.
 746	 */
 747	if (!list_empty(&chan->hc_list_entry))
 748		list_del(&chan->hc_list_entry);
 749	dwc2_hc_cleanup(hsotg, chan);
 750	list_add_tail(&chan->hc_list_entry, &hsotg->free_hc_list);
 751
 752	if (hsotg->params.uframe_sched) {
 753		hsotg->available_host_channels++;
 754	} else {
 755		switch (chan->ep_type) {
 756		case USB_ENDPOINT_XFER_CONTROL:
 757		case USB_ENDPOINT_XFER_BULK:
 758			hsotg->non_periodic_channels--;
 759			break;
 760		default:
 761			/*
 762			 * Don't release reservations for periodic channels
 763			 * here. That's done when a periodic transfer is
 764			 * descheduled (i.e. when the QH is removed from the
 765			 * periodic schedule).
 766			 */
 767			break;
 768		}
 769	}
 770
 771	haintmsk = dwc2_readl(hsotg->regs + HAINTMSK);
 772	haintmsk &= ~(1 << chan->hc_num);
 773	dwc2_writel(haintmsk, hsotg->regs + HAINTMSK);
 774
 775	/* Try to queue more transfers now that there's a free channel */
 776	tr_type = dwc2_hcd_select_transactions(hsotg);
 777	if (tr_type != DWC2_TRANSACTION_NONE)
 778		dwc2_hcd_queue_transactions(hsotg, tr_type);
 779}
 780
 781/*
 782 * Halts a host channel. If the channel cannot be halted immediately because
 783 * the request queue is full, this function ensures that the FIFO empty
 784 * interrupt for the appropriate queue is enabled so that the halt request can
 785 * be queued when there is space in the request queue.
 786 *
 787 * This function may also be called in DMA mode. In that case, the channel is
 788 * simply released since the core always halts the channel automatically in
 789 * DMA mode.
 790 */
 791static void dwc2_halt_channel(struct dwc2_hsotg *hsotg,
 792			      struct dwc2_host_chan *chan, struct dwc2_qtd *qtd,
 793			      enum dwc2_halt_status halt_status)
 794{
 795	if (dbg_hc(chan))
 796		dev_vdbg(hsotg->dev, "%s()\n", __func__);
 797
 798	if (hsotg->params.host_dma) {
 799		if (dbg_hc(chan))
 800			dev_vdbg(hsotg->dev, "DMA enabled\n");
 801		dwc2_release_channel(hsotg, chan, qtd, halt_status);
 802		return;
 803	}
 804
 805	/* Slave mode processing */
 806	dwc2_hc_halt(hsotg, chan, halt_status);
 807
 808	if (chan->halt_on_queue) {
 809		u32 gintmsk;
 810
 811		dev_vdbg(hsotg->dev, "Halt on queue\n");
 812		if (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
 813		    chan->ep_type == USB_ENDPOINT_XFER_BULK) {
 814			dev_vdbg(hsotg->dev, "control/bulk\n");
 815			/*
 816			 * Make sure the Non-periodic Tx FIFO empty interrupt
 817			 * is enabled so that the non-periodic schedule will
 818			 * be processed
 819			 */
 820			gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
 821			gintmsk |= GINTSTS_NPTXFEMP;
 822			dwc2_writel(gintmsk, hsotg->regs + GINTMSK);
 823		} else {
 824			dev_vdbg(hsotg->dev, "isoc/intr\n");
 825			/*
 826			 * Move the QH from the periodic queued schedule to
 827			 * the periodic assigned schedule. This allows the
 828			 * halt to be queued when the periodic schedule is
 829			 * processed.
 830			 */
 831			list_move_tail(&chan->qh->qh_list_entry,
 832				       &hsotg->periodic_sched_assigned);
 833
 834			/*
 835			 * Make sure the Periodic Tx FIFO Empty interrupt is
 836			 * enabled so that the periodic schedule will be
 837			 * processed
 838			 */
 839			gintmsk = dwc2_readl(hsotg->regs + GINTMSK);
 840			gintmsk |= GINTSTS_PTXFEMP;
 841			dwc2_writel(gintmsk, hsotg->regs + GINTMSK);
 842		}
 843	}
 844}
 845
 846/*
 847 * Performs common cleanup for non-periodic transfers after a Transfer
 848 * Complete interrupt. This function should be called after any endpoint type
 849 * specific handling is finished to release the host channel.
 850 */
 851static void dwc2_complete_non_periodic_xfer(struct dwc2_hsotg *hsotg,
 852					    struct dwc2_host_chan *chan,
 853					    int chnum, struct dwc2_qtd *qtd,
 854					    enum dwc2_halt_status halt_status)
 855{
 856	dev_vdbg(hsotg->dev, "%s()\n", __func__);
 857
 858	qtd->error_count = 0;
 859
 860	if (chan->hcint & HCINTMSK_NYET) {
 861		/*
 862		 * Got a NYET on the last transaction of the transfer. This
 863		 * means that the endpoint should be in the PING state at the
 864		 * beginning of the next transfer.
 865		 */
 866		dev_vdbg(hsotg->dev, "got NYET\n");
 867		chan->qh->ping_state = 1;
 868	}
 869
 870	/*
 871	 * Always halt and release the host channel to make it available for
 872	 * more transfers. There may still be more phases for a control
 873	 * transfer or more data packets for a bulk transfer at this point,
 874	 * but the host channel is still halted. A channel will be reassigned
 875	 * to the transfer when the non-periodic schedule is processed after
 876	 * the channel is released. This allows transactions to be queued
 877	 * properly via dwc2_hcd_queue_transactions, which also enables the
 878	 * Tx FIFO Empty interrupt if necessary.
 879	 */
 880	if (chan->ep_is_in) {
 881		/*
 882		 * IN transfers in Slave mode require an explicit disable to
 883		 * halt the channel. (In DMA mode, this call simply releases
 884		 * the channel.)
 885		 */
 886		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
 887	} else {
 888		/*
 889		 * The channel is automatically disabled by the core for OUT
 890		 * transfers in Slave mode
 891		 */
 892		dwc2_release_channel(hsotg, chan, qtd, halt_status);
 893	}
 894}
 895
 896/*
 897 * Performs common cleanup for periodic transfers after a Transfer Complete
 898 * interrupt. This function should be called after any endpoint type specific
 899 * handling is finished to release the host channel.
 900 */
 901static void dwc2_complete_periodic_xfer(struct dwc2_hsotg *hsotg,
 902					struct dwc2_host_chan *chan, int chnum,
 903					struct dwc2_qtd *qtd,
 904					enum dwc2_halt_status halt_status)
 905{
 906	u32 hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
 907
 908	qtd->error_count = 0;
 909
 910	if (!chan->ep_is_in || (hctsiz & TSIZ_PKTCNT_MASK) == 0)
 911		/* Core halts channel in these cases */
 912		dwc2_release_channel(hsotg, chan, qtd, halt_status);
 913	else
 914		/* Flush any outstanding requests from the Tx queue */
 915		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
 916}
 917
 918static int dwc2_xfercomp_isoc_split_in(struct dwc2_hsotg *hsotg,
 919				       struct dwc2_host_chan *chan, int chnum,
 920				       struct dwc2_qtd *qtd)
 921{
 922	struct dwc2_hcd_iso_packet_desc *frame_desc;
 923	u32 len;
 924	u32 hctsiz;
 925	u32 pid;
 926
 927	if (!qtd->urb)
 928		return 0;
 929
 930	frame_desc = &qtd->urb->iso_descs[qtd->isoc_frame_index];
 931	len = dwc2_get_actual_xfer_length(hsotg, chan, chnum, qtd,
 932					  DWC2_HC_XFER_COMPLETE, NULL);
 933	if (!len) {
 934		qtd->complete_split = 0;
 935		qtd->isoc_split_offset = 0;
 936		return 0;
 937	}
 938
 939	frame_desc->actual_length += len;
 940
 941	qtd->isoc_split_offset += len;
 942
 943	hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
 944	pid = (hctsiz & TSIZ_SC_MC_PID_MASK) >> TSIZ_SC_MC_PID_SHIFT;
 945
 946	if (frame_desc->actual_length >= frame_desc->length || pid == 0) {
 947		frame_desc->status = 0;
 948		qtd->isoc_frame_index++;
 949		qtd->complete_split = 0;
 950		qtd->isoc_split_offset = 0;
 951	}
 952
 953	if (qtd->isoc_frame_index == qtd->urb->packet_count) {
 954		dwc2_host_complete(hsotg, qtd, 0);
 955		dwc2_release_channel(hsotg, chan, qtd,
 956				     DWC2_HC_XFER_URB_COMPLETE);
 957	} else {
 958		dwc2_release_channel(hsotg, chan, qtd,
 959				     DWC2_HC_XFER_NO_HALT_STATUS);
 960	}
 961
 962	return 1;	/* Indicates that channel released */
 963}
 964
 965/*
 966 * Handles a host channel Transfer Complete interrupt. This handler may be
 967 * called in either DMA mode or Slave mode.
 968 */
 969static void dwc2_hc_xfercomp_intr(struct dwc2_hsotg *hsotg,
 970				  struct dwc2_host_chan *chan, int chnum,
 971				  struct dwc2_qtd *qtd)
 972{
 973	struct dwc2_hcd_urb *urb = qtd->urb;
 974	enum dwc2_halt_status halt_status = DWC2_HC_XFER_COMPLETE;
 975	int pipe_type;
 976	int urb_xfer_done;
 977
 978	if (dbg_hc(chan))
 979		dev_vdbg(hsotg->dev,
 980			 "--Host Channel %d Interrupt: Transfer Complete--\n",
 981			 chnum);
 982
 983	if (!urb)
 984		goto handle_xfercomp_done;
 985
 986	pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
 987
 988	if (hsotg->params.dma_desc_enable) {
 989		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum, halt_status);
 990		if (pipe_type == USB_ENDPOINT_XFER_ISOC)
 991			/* Do not disable the interrupt, just clear it */
 992			return;
 993		goto handle_xfercomp_done;
 994	}
 995
 996	/* Handle xfer complete on CSPLIT */
 997	if (chan->qh->do_split) {
 998		if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && chan->ep_is_in &&
 999		    hsotg->params.host_dma) {
1000			if (qtd->complete_split &&
1001			    dwc2_xfercomp_isoc_split_in(hsotg, chan, chnum,
1002							qtd))
1003				goto handle_xfercomp_done;
1004		} else {
1005			qtd->complete_split = 0;
1006		}
1007	}
1008
1009	/* Update the QTD and URB states */
1010	switch (pipe_type) {
1011	case USB_ENDPOINT_XFER_CONTROL:
1012		switch (qtd->control_phase) {
1013		case DWC2_CONTROL_SETUP:
1014			if (urb->length > 0)
1015				qtd->control_phase = DWC2_CONTROL_DATA;
1016			else
1017				qtd->control_phase = DWC2_CONTROL_STATUS;
1018			dev_vdbg(hsotg->dev,
1019				 "  Control setup transaction done\n");
1020			halt_status = DWC2_HC_XFER_COMPLETE;
1021			break;
1022		case DWC2_CONTROL_DATA:
1023			urb_xfer_done = dwc2_update_urb_state(hsotg, chan,
1024							      chnum, urb, qtd);
1025			if (urb_xfer_done) {
1026				qtd->control_phase = DWC2_CONTROL_STATUS;
1027				dev_vdbg(hsotg->dev,
1028					 "  Control data transfer done\n");
1029			} else {
1030				dwc2_hcd_save_data_toggle(hsotg, chan, chnum,
1031							  qtd);
1032			}
1033			halt_status = DWC2_HC_XFER_COMPLETE;
1034			break;
1035		case DWC2_CONTROL_STATUS:
1036			dev_vdbg(hsotg->dev, "  Control transfer complete\n");
1037			if (urb->status == -EINPROGRESS)
1038				urb->status = 0;
1039			dwc2_host_complete(hsotg, qtd, urb->status);
1040			halt_status = DWC2_HC_XFER_URB_COMPLETE;
1041			break;
1042		}
1043
1044		dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1045						halt_status);
1046		break;
1047	case USB_ENDPOINT_XFER_BULK:
1048		dev_vdbg(hsotg->dev, "  Bulk transfer complete\n");
1049		urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1050						      qtd);
1051		if (urb_xfer_done) {
1052			dwc2_host_complete(hsotg, qtd, urb->status);
1053			halt_status = DWC2_HC_XFER_URB_COMPLETE;
1054		} else {
1055			halt_status = DWC2_HC_XFER_COMPLETE;
1056		}
1057
1058		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1059		dwc2_complete_non_periodic_xfer(hsotg, chan, chnum, qtd,
1060						halt_status);
1061		break;
1062	case USB_ENDPOINT_XFER_INT:
1063		dev_vdbg(hsotg->dev, "  Interrupt transfer complete\n");
1064		urb_xfer_done = dwc2_update_urb_state(hsotg, chan, chnum, urb,
1065						      qtd);
1066
1067		/*
1068		 * Interrupt URB is done on the first transfer complete
1069		 * interrupt
1070		 */
1071		if (urb_xfer_done) {
1072			dwc2_host_complete(hsotg, qtd, urb->status);
1073			halt_status = DWC2_HC_XFER_URB_COMPLETE;
1074		} else {
1075			halt_status = DWC2_HC_XFER_COMPLETE;
1076		}
1077
1078		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1079		dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1080					    halt_status);
1081		break;
1082	case USB_ENDPOINT_XFER_ISOC:
1083		if (dbg_perio())
1084			dev_vdbg(hsotg->dev, "  Isochronous transfer complete\n");
1085		if (qtd->isoc_split_pos == DWC2_HCSPLT_XACTPOS_ALL)
1086			halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1087							chnum, qtd,
1088							DWC2_HC_XFER_COMPLETE);
1089		dwc2_complete_periodic_xfer(hsotg, chan, chnum, qtd,
1090					    halt_status);
1091		break;
1092	}
1093
1094handle_xfercomp_done:
1095	disable_hc_int(hsotg, chnum, HCINTMSK_XFERCOMPL);
1096}
1097
1098/*
1099 * Handles a host channel STALL interrupt. This handler may be called in
1100 * either DMA mode or Slave mode.
1101 */
1102static void dwc2_hc_stall_intr(struct dwc2_hsotg *hsotg,
1103			       struct dwc2_host_chan *chan, int chnum,
1104			       struct dwc2_qtd *qtd)
1105{
1106	struct dwc2_hcd_urb *urb = qtd->urb;
1107	int pipe_type;
1108
1109	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: STALL Received--\n",
1110		chnum);
1111
1112	if (hsotg->params.dma_desc_enable) {
1113		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1114					    DWC2_HC_XFER_STALL);
1115		goto handle_stall_done;
1116	}
1117
1118	if (!urb)
1119		goto handle_stall_halt;
1120
1121	pipe_type = dwc2_hcd_get_pipe_type(&urb->pipe_info);
1122
1123	if (pipe_type == USB_ENDPOINT_XFER_CONTROL)
1124		dwc2_host_complete(hsotg, qtd, -EPIPE);
1125
1126	if (pipe_type == USB_ENDPOINT_XFER_BULK ||
1127	    pipe_type == USB_ENDPOINT_XFER_INT) {
1128		dwc2_host_complete(hsotg, qtd, -EPIPE);
1129		/*
1130		 * USB protocol requires resetting the data toggle for bulk
1131		 * and interrupt endpoints when a CLEAR_FEATURE(ENDPOINT_HALT)
1132		 * setup command is issued to the endpoint. Anticipate the
1133		 * CLEAR_FEATURE command since a STALL has occurred and reset
1134		 * the data toggle now.
1135		 */
1136		chan->qh->data_toggle = 0;
1137	}
1138
1139handle_stall_halt:
1140	dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_STALL);
1141
1142handle_stall_done:
1143	disable_hc_int(hsotg, chnum, HCINTMSK_STALL);
1144}
1145
1146/*
1147 * Updates the state of the URB when a transfer has been stopped due to an
1148 * abnormal condition before the transfer completes. Modifies the
1149 * actual_length field of the URB to reflect the number of bytes that have
1150 * actually been transferred via the host channel.
1151 */
1152static void dwc2_update_urb_state_abn(struct dwc2_hsotg *hsotg,
1153				      struct dwc2_host_chan *chan, int chnum,
1154				      struct dwc2_hcd_urb *urb,
1155				      struct dwc2_qtd *qtd,
1156				      enum dwc2_halt_status halt_status)
1157{
1158	u32 xfer_length = dwc2_get_actual_xfer_length(hsotg, chan, chnum,
1159						      qtd, halt_status, NULL);
1160	u32 hctsiz;
1161
1162	if (urb->actual_length + xfer_length > urb->length) {
1163		dev_warn(hsotg->dev, "%s(): trimming xfer length\n", __func__);
1164		xfer_length = urb->length - urb->actual_length;
1165	}
1166
1167	urb->actual_length += xfer_length;
1168
1169	hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
1170	dev_vdbg(hsotg->dev, "DWC_otg: %s: %s, channel %d\n",
1171		 __func__, (chan->ep_is_in ? "IN" : "OUT"), chnum);
1172	dev_vdbg(hsotg->dev, "  chan->start_pkt_count %d\n",
1173		 chan->start_pkt_count);
1174	dev_vdbg(hsotg->dev, "  hctsiz.pktcnt %d\n",
1175		 (hctsiz & TSIZ_PKTCNT_MASK) >> TSIZ_PKTCNT_SHIFT);
1176	dev_vdbg(hsotg->dev, "  chan->max_packet %d\n", chan->max_packet);
1177	dev_vdbg(hsotg->dev, "  bytes_transferred %d\n",
1178		 xfer_length);
1179	dev_vdbg(hsotg->dev, "  urb->actual_length %d\n",
1180		 urb->actual_length);
1181	dev_vdbg(hsotg->dev, "  urb->transfer_buffer_length %d\n",
1182		 urb->length);
1183}
1184
1185/*
1186 * Handles a host channel NAK interrupt. This handler may be called in either
1187 * DMA mode or Slave mode.
1188 */
1189static void dwc2_hc_nak_intr(struct dwc2_hsotg *hsotg,
1190			     struct dwc2_host_chan *chan, int chnum,
1191			     struct dwc2_qtd *qtd)
1192{
1193	if (!qtd) {
1194		dev_dbg(hsotg->dev, "%s: qtd is NULL\n", __func__);
1195		return;
1196	}
1197
1198	if (!qtd->urb) {
1199		dev_dbg(hsotg->dev, "%s: qtd->urb is NULL\n", __func__);
1200		return;
1201	}
1202
1203	if (dbg_hc(chan))
1204		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NAK Received--\n",
1205			 chnum);
1206
1207	/*
1208	 * Handle NAK for IN/OUT SSPLIT/CSPLIT transfers, bulk, control, and
1209	 * interrupt. Re-start the SSPLIT transfer.
1210	 *
1211	 * Normally for non-periodic transfers we'll retry right away, but to
1212	 * avoid interrupt storms we'll wait before retrying if we've got
1213	 * several NAKs. If we didn't do this we'd retry directly from the
1214	 * interrupt handler and could end up quickly getting another
1215	 * interrupt (another NAK), which we'd retry.
1216	 *
1217	 * Note that in DMA mode software only gets involved to re-send NAKed
1218	 * transfers for split transactions, so we only need to apply this
1219	 * delaying logic when handling splits. In non-DMA mode presumably we
1220	 * might want a similar delay if someone can demonstrate this problem
1221	 * affects that code path too.
1222	 */
1223	if (chan->do_split) {
1224		if (chan->complete_split)
1225			qtd->error_count = 0;
1226		qtd->complete_split = 0;
1227		qtd->num_naks++;
1228		qtd->qh->want_wait = qtd->num_naks >= DWC2_NAKS_BEFORE_DELAY;
1229		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1230		goto handle_nak_done;
1231	}
1232
1233	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1234	case USB_ENDPOINT_XFER_CONTROL:
1235	case USB_ENDPOINT_XFER_BULK:
1236		if (hsotg->params.host_dma && chan->ep_is_in) {
1237			/*
1238			 * NAK interrupts are enabled on bulk/control IN
1239			 * transfers in DMA mode for the sole purpose of
1240			 * resetting the error count after a transaction error
1241			 * occurs. The core will continue transferring data.
1242			 */
1243			qtd->error_count = 0;
1244			break;
1245		}
1246
1247		/*
1248		 * NAK interrupts normally occur during OUT transfers in DMA
1249		 * or Slave mode. For IN transfers, more requests will be
1250		 * queued as request queue space is available.
1251		 */
1252		qtd->error_count = 0;
1253
1254		if (!chan->qh->ping_state) {
1255			dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1256						  qtd, DWC2_HC_XFER_NAK);
1257			dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1258
1259			if (chan->speed == USB_SPEED_HIGH)
1260				chan->qh->ping_state = 1;
1261		}
1262
1263		/*
1264		 * Halt the channel so the transfer can be re-started from
1265		 * the appropriate point or the PING protocol will
1266		 * start/continue
1267		 */
1268		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1269		break;
1270	case USB_ENDPOINT_XFER_INT:
1271		qtd->error_count = 0;
1272		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NAK);
1273		break;
1274	case USB_ENDPOINT_XFER_ISOC:
1275		/* Should never get called for isochronous transfers */
1276		dev_err(hsotg->dev, "NACK interrupt for ISOC transfer\n");
1277		break;
1278	}
1279
1280handle_nak_done:
1281	disable_hc_int(hsotg, chnum, HCINTMSK_NAK);
1282}
1283
1284/*
1285 * Handles a host channel ACK interrupt. This interrupt is enabled when
1286 * performing the PING protocol in Slave mode, when errors occur during
1287 * either Slave mode or DMA mode, and during Start Split transactions.
1288 */
1289static void dwc2_hc_ack_intr(struct dwc2_hsotg *hsotg,
1290			     struct dwc2_host_chan *chan, int chnum,
1291			     struct dwc2_qtd *qtd)
1292{
1293	struct dwc2_hcd_iso_packet_desc *frame_desc;
1294
1295	if (dbg_hc(chan))
1296		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: ACK Received--\n",
1297			 chnum);
1298
1299	if (chan->do_split) {
1300		/* Handle ACK on SSPLIT. ACK should not occur in CSPLIT. */
1301		if (!chan->ep_is_in &&
1302		    chan->data_pid_start != DWC2_HC_PID_SETUP)
1303			qtd->ssplit_out_xfer_count = chan->xfer_len;
1304
1305		if (chan->ep_type != USB_ENDPOINT_XFER_ISOC || chan->ep_is_in) {
1306			qtd->complete_split = 1;
1307			dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1308		} else {
1309			/* ISOC OUT */
1310			switch (chan->xact_pos) {
1311			case DWC2_HCSPLT_XACTPOS_ALL:
1312				break;
1313			case DWC2_HCSPLT_XACTPOS_END:
1314				qtd->isoc_split_pos = DWC2_HCSPLT_XACTPOS_ALL;
1315				qtd->isoc_split_offset = 0;
1316				break;
1317			case DWC2_HCSPLT_XACTPOS_BEGIN:
1318			case DWC2_HCSPLT_XACTPOS_MID:
1319				/*
1320				 * For BEGIN or MID, calculate the length for
1321				 * the next microframe to determine the correct
1322				 * SSPLIT token, either MID or END
1323				 */
1324				frame_desc = &qtd->urb->iso_descs[
1325						qtd->isoc_frame_index];
1326				qtd->isoc_split_offset += 188;
1327
1328				if (frame_desc->length - qtd->isoc_split_offset
1329							<= 188)
1330					qtd->isoc_split_pos =
1331							DWC2_HCSPLT_XACTPOS_END;
1332				else
1333					qtd->isoc_split_pos =
1334							DWC2_HCSPLT_XACTPOS_MID;
1335				break;
1336			}
1337		}
1338	} else {
1339		qtd->error_count = 0;
1340
1341		if (chan->qh->ping_state) {
1342			chan->qh->ping_state = 0;
1343			/*
1344			 * Halt the channel so the transfer can be re-started
1345			 * from the appropriate point. This only happens in
1346			 * Slave mode. In DMA mode, the ping_state is cleared
1347			 * when the transfer is started because the core
1348			 * automatically executes the PING, then the transfer.
1349			 */
1350			dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_ACK);
1351		}
1352	}
1353
1354	/*
1355	 * If the ACK occurred when _not_ in the PING state, let the channel
1356	 * continue transferring data after clearing the error count
1357	 */
1358	disable_hc_int(hsotg, chnum, HCINTMSK_ACK);
1359}
1360
1361/*
1362 * Handles a host channel NYET interrupt. This interrupt should only occur on
1363 * Bulk and Control OUT endpoints and for complete split transactions. If a
1364 * NYET occurs at the same time as a Transfer Complete interrupt, it is
1365 * handled in the xfercomp interrupt handler, not here. This handler may be
1366 * called in either DMA mode or Slave mode.
1367 */
1368static void dwc2_hc_nyet_intr(struct dwc2_hsotg *hsotg,
1369			      struct dwc2_host_chan *chan, int chnum,
1370			      struct dwc2_qtd *qtd)
1371{
1372	if (dbg_hc(chan))
1373		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: NYET Received--\n",
1374			 chnum);
1375
1376	/*
1377	 * NYET on CSPLIT
1378	 * re-do the CSPLIT immediately on non-periodic
1379	 */
1380	if (chan->do_split && chan->complete_split) {
1381		if (chan->ep_is_in && chan->ep_type == USB_ENDPOINT_XFER_ISOC &&
1382		    hsotg->params.host_dma) {
1383			qtd->complete_split = 0;
1384			qtd->isoc_split_offset = 0;
1385			qtd->isoc_frame_index++;
1386			if (qtd->urb &&
1387			    qtd->isoc_frame_index == qtd->urb->packet_count) {
1388				dwc2_host_complete(hsotg, qtd, 0);
1389				dwc2_release_channel(hsotg, chan, qtd,
1390						     DWC2_HC_XFER_URB_COMPLETE);
1391			} else {
1392				dwc2_release_channel(hsotg, chan, qtd,
1393						DWC2_HC_XFER_NO_HALT_STATUS);
1394			}
1395			goto handle_nyet_done;
1396		}
1397
1398		if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1399		    chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1400			struct dwc2_qh *qh = chan->qh;
1401			bool past_end;
1402
1403			if (!hsotg->params.uframe_sched) {
1404				int frnum = dwc2_hcd_get_frame_number(hsotg);
1405
1406				/* Don't have num_hs_transfers; simple logic */
1407				past_end = dwc2_full_frame_num(frnum) !=
1408				     dwc2_full_frame_num(qh->next_active_frame);
1409			} else {
1410				int end_frnum;
1411
1412				/*
1413				 * Figure out the end frame based on
1414				 * schedule.
1415				 *
1416				 * We don't want to go on trying again
1417				 * and again forever. Let's stop when
1418				 * we've done all the transfers that
1419				 * were scheduled.
1420				 *
1421				 * We're going to be comparing
1422				 * start_active_frame and
1423				 * next_active_frame, both of which
1424				 * are 1 before the time the packet
1425				 * goes on the wire, so that cancels
1426				 * out. Basically if had 1 transfer
1427				 * and we saw 1 NYET then we're done.
1428				 * We're getting a NYET here so if
1429				 * next >= (start + num_transfers)
1430				 * we're done. The complexity is that
1431				 * for all but ISOC_OUT we skip one
1432				 * slot.
1433				 */
1434				end_frnum = dwc2_frame_num_inc(
1435					qh->start_active_frame,
1436					qh->num_hs_transfers);
1437
1438				if (qh->ep_type != USB_ENDPOINT_XFER_ISOC ||
1439				    qh->ep_is_in)
1440					end_frnum =
1441					       dwc2_frame_num_inc(end_frnum, 1);
1442
1443				past_end = dwc2_frame_num_le(
1444					end_frnum, qh->next_active_frame);
1445			}
1446
1447			if (past_end) {
1448				/* Treat this as a transaction error. */
1449#if 0
1450				/*
1451				 * Todo: Fix system performance so this can
1452				 * be treated as an error. Right now complete
1453				 * splits cannot be scheduled precisely enough
1454				 * due to other system activity, so this error
1455				 * occurs regularly in Slave mode.
1456				 */
1457				qtd->error_count++;
1458#endif
1459				qtd->complete_split = 0;
1460				dwc2_halt_channel(hsotg, chan, qtd,
1461						  DWC2_HC_XFER_XACT_ERR);
1462				/* Todo: add support for isoc release */
1463				goto handle_nyet_done;
1464			}
1465		}
1466
1467		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1468		goto handle_nyet_done;
1469	}
1470
1471	chan->qh->ping_state = 1;
1472	qtd->error_count = 0;
1473
1474	dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb, qtd,
1475				  DWC2_HC_XFER_NYET);
1476	dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1477
1478	/*
1479	 * Halt the channel and re-start the transfer so the PING protocol
1480	 * will start
1481	 */
1482	dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_NYET);
1483
1484handle_nyet_done:
1485	disable_hc_int(hsotg, chnum, HCINTMSK_NYET);
1486}
1487
1488/*
1489 * Handles a host channel babble interrupt. This handler may be called in
1490 * either DMA mode or Slave mode.
1491 */
1492static void dwc2_hc_babble_intr(struct dwc2_hsotg *hsotg,
1493				struct dwc2_host_chan *chan, int chnum,
1494				struct dwc2_qtd *qtd)
1495{
1496	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Babble Error--\n",
1497		chnum);
1498
1499	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1500
1501	if (hsotg->params.dma_desc_enable) {
1502		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1503					    DWC2_HC_XFER_BABBLE_ERR);
1504		goto disable_int;
1505	}
1506
1507	if (chan->ep_type != USB_ENDPOINT_XFER_ISOC) {
1508		dwc2_host_complete(hsotg, qtd, -EOVERFLOW);
1509		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_BABBLE_ERR);
1510	} else {
1511		enum dwc2_halt_status halt_status;
1512
1513		halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1514						qtd, DWC2_HC_XFER_BABBLE_ERR);
1515		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1516	}
1517
1518disable_int:
1519	disable_hc_int(hsotg, chnum, HCINTMSK_BBLERR);
1520}
1521
1522/*
1523 * Handles a host channel AHB error interrupt. This handler is only called in
1524 * DMA mode.
1525 */
1526static void dwc2_hc_ahberr_intr(struct dwc2_hsotg *hsotg,
1527				struct dwc2_host_chan *chan, int chnum,
1528				struct dwc2_qtd *qtd)
1529{
1530	struct dwc2_hcd_urb *urb = qtd->urb;
1531	char *pipetype, *speed;
1532	u32 hcchar;
1533	u32 hcsplt;
1534	u32 hctsiz;
1535	u32 hc_dma;
1536
1537	dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: AHB Error--\n",
1538		chnum);
1539
1540	if (!urb)
1541		goto handle_ahberr_halt;
1542
1543	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1544
1545	hcchar = dwc2_readl(hsotg->regs + HCCHAR(chnum));
1546	hcsplt = dwc2_readl(hsotg->regs + HCSPLT(chnum));
1547	hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
1548	hc_dma = dwc2_readl(hsotg->regs + HCDMA(chnum));
1549
1550	dev_err(hsotg->dev, "AHB ERROR, Channel %d\n", chnum);
1551	dev_err(hsotg->dev, "  hcchar 0x%08x, hcsplt 0x%08x\n", hcchar, hcsplt);
1552	dev_err(hsotg->dev, "  hctsiz 0x%08x, hc_dma 0x%08x\n", hctsiz, hc_dma);
1553	dev_err(hsotg->dev, "  Device address: %d\n",
1554		dwc2_hcd_get_dev_addr(&urb->pipe_info));
1555	dev_err(hsotg->dev, "  Endpoint: %d, %s\n",
1556		dwc2_hcd_get_ep_num(&urb->pipe_info),
1557		dwc2_hcd_is_pipe_in(&urb->pipe_info) ? "IN" : "OUT");
1558
1559	switch (dwc2_hcd_get_pipe_type(&urb->pipe_info)) {
1560	case USB_ENDPOINT_XFER_CONTROL:
1561		pipetype = "CONTROL";
1562		break;
1563	case USB_ENDPOINT_XFER_BULK:
1564		pipetype = "BULK";
1565		break;
1566	case USB_ENDPOINT_XFER_INT:
1567		pipetype = "INTERRUPT";
1568		break;
1569	case USB_ENDPOINT_XFER_ISOC:
1570		pipetype = "ISOCHRONOUS";
1571		break;
1572	default:
1573		pipetype = "UNKNOWN";
1574		break;
1575	}
1576
1577	dev_err(hsotg->dev, "  Endpoint type: %s\n", pipetype);
1578
1579	switch (chan->speed) {
1580	case USB_SPEED_HIGH:
1581		speed = "HIGH";
1582		break;
1583	case USB_SPEED_FULL:
1584		speed = "FULL";
1585		break;
1586	case USB_SPEED_LOW:
1587		speed = "LOW";
1588		break;
1589	default:
1590		speed = "UNKNOWN";
1591		break;
1592	}
1593
1594	dev_err(hsotg->dev, "  Speed: %s\n", speed);
1595
1596	dev_err(hsotg->dev, "  Max packet size: %d\n",
1597		dwc2_hcd_get_mps(&urb->pipe_info));
1598	dev_err(hsotg->dev, "  Data buffer length: %d\n", urb->length);
1599	dev_err(hsotg->dev, "  Transfer buffer: %p, Transfer DMA: %08lx\n",
1600		urb->buf, (unsigned long)urb->dma);
1601	dev_err(hsotg->dev, "  Setup buffer: %p, Setup DMA: %08lx\n",
1602		urb->setup_packet, (unsigned long)urb->setup_dma);
1603	dev_err(hsotg->dev, "  Interval: %d\n", urb->interval);
1604
1605	/* Core halts the channel for Descriptor DMA mode */
1606	if (hsotg->params.dma_desc_enable) {
1607		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1608					    DWC2_HC_XFER_AHB_ERR);
1609		goto handle_ahberr_done;
1610	}
1611
1612	dwc2_host_complete(hsotg, qtd, -EIO);
1613
1614handle_ahberr_halt:
1615	/*
1616	 * Force a channel halt. Don't call dwc2_halt_channel because that won't
1617	 * write to the HCCHARn register in DMA mode to force the halt.
1618	 */
1619	dwc2_hc_halt(hsotg, chan, DWC2_HC_XFER_AHB_ERR);
1620
1621handle_ahberr_done:
1622	disable_hc_int(hsotg, chnum, HCINTMSK_AHBERR);
1623}
1624
1625/*
1626 * Handles a host channel transaction error interrupt. This handler may be
1627 * called in either DMA mode or Slave mode.
1628 */
1629static void dwc2_hc_xacterr_intr(struct dwc2_hsotg *hsotg,
1630				 struct dwc2_host_chan *chan, int chnum,
1631				 struct dwc2_qtd *qtd)
1632{
1633	dev_dbg(hsotg->dev,
1634		"--Host Channel %d Interrupt: Transaction Error--\n", chnum);
1635
1636	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1637
1638	if (hsotg->params.dma_desc_enable) {
1639		dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1640					    DWC2_HC_XFER_XACT_ERR);
1641		goto handle_xacterr_done;
1642	}
1643
1644	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1645	case USB_ENDPOINT_XFER_CONTROL:
1646	case USB_ENDPOINT_XFER_BULK:
1647		qtd->error_count++;
1648		if (!chan->qh->ping_state) {
 
1649			dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1650						  qtd, DWC2_HC_XFER_XACT_ERR);
1651			dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1652			if (!chan->ep_is_in && chan->speed == USB_SPEED_HIGH)
1653				chan->qh->ping_state = 1;
1654		}
1655
1656		/*
1657		 * Halt the channel so the transfer can be re-started from
1658		 * the appropriate point or the PING protocol will start
1659		 */
1660		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1661		break;
1662	case USB_ENDPOINT_XFER_INT:
1663		qtd->error_count++;
1664		if (chan->do_split && chan->complete_split)
1665			qtd->complete_split = 0;
1666		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1667		break;
1668	case USB_ENDPOINT_XFER_ISOC:
1669		{
1670			enum dwc2_halt_status halt_status;
1671
1672			halt_status = dwc2_update_isoc_urb_state(hsotg, chan,
1673					 chnum, qtd, DWC2_HC_XFER_XACT_ERR);
1674			dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1675		}
1676		break;
1677	}
1678
1679handle_xacterr_done:
1680	disable_hc_int(hsotg, chnum, HCINTMSK_XACTERR);
1681}
1682
1683/*
1684 * Handles a host channel frame overrun interrupt. This handler may be called
1685 * in either DMA mode or Slave mode.
1686 */
1687static void dwc2_hc_frmovrun_intr(struct dwc2_hsotg *hsotg,
1688				  struct dwc2_host_chan *chan, int chnum,
1689				  struct dwc2_qtd *qtd)
1690{
1691	enum dwc2_halt_status halt_status;
1692
1693	if (dbg_hc(chan))
1694		dev_dbg(hsotg->dev, "--Host Channel %d Interrupt: Frame Overrun--\n",
1695			chnum);
1696
1697	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1698
1699	switch (dwc2_hcd_get_pipe_type(&qtd->urb->pipe_info)) {
1700	case USB_ENDPOINT_XFER_CONTROL:
1701	case USB_ENDPOINT_XFER_BULK:
1702		break;
1703	case USB_ENDPOINT_XFER_INT:
1704		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1705		break;
1706	case USB_ENDPOINT_XFER_ISOC:
1707		halt_status = dwc2_update_isoc_urb_state(hsotg, chan, chnum,
1708					qtd, DWC2_HC_XFER_FRAME_OVERRUN);
1709		dwc2_halt_channel(hsotg, chan, qtd, halt_status);
1710		break;
1711	}
1712
1713	disable_hc_int(hsotg, chnum, HCINTMSK_FRMOVRUN);
1714}
1715
1716/*
1717 * Handles a host channel data toggle error interrupt. This handler may be
1718 * called in either DMA mode or Slave mode.
1719 */
1720static void dwc2_hc_datatglerr_intr(struct dwc2_hsotg *hsotg,
1721				    struct dwc2_host_chan *chan, int chnum,
1722				    struct dwc2_qtd *qtd)
1723{
1724	dev_dbg(hsotg->dev,
1725		"--Host Channel %d Interrupt: Data Toggle Error--\n", chnum);
1726
1727	if (chan->ep_is_in)
1728		qtd->error_count = 0;
1729	else
1730		dev_err(hsotg->dev,
1731			"Data Toggle Error on OUT transfer, channel %d\n",
1732			chnum);
1733
1734	dwc2_hc_handle_tt_clear(hsotg, chan, qtd);
1735	disable_hc_int(hsotg, chnum, HCINTMSK_DATATGLERR);
1736}
1737
1738/*
1739 * For debug only. It checks that a valid halt status is set and that
1740 * HCCHARn.chdis is clear. If there's a problem, corrective action is
1741 * taken and a warning is issued.
1742 *
1743 * Return: true if halt status is ok, false otherwise
1744 */
1745static bool dwc2_halt_status_ok(struct dwc2_hsotg *hsotg,
1746				struct dwc2_host_chan *chan, int chnum,
1747				struct dwc2_qtd *qtd)
1748{
1749#ifdef DEBUG
1750	u32 hcchar;
1751	u32 hctsiz;
1752	u32 hcintmsk;
1753	u32 hcsplt;
1754
1755	if (chan->halt_status == DWC2_HC_XFER_NO_HALT_STATUS) {
1756		/*
1757		 * This code is here only as a check. This condition should
1758		 * never happen. Ignore the halt if it does occur.
1759		 */
1760		hcchar = dwc2_readl(hsotg->regs + HCCHAR(chnum));
1761		hctsiz = dwc2_readl(hsotg->regs + HCTSIZ(chnum));
1762		hcintmsk = dwc2_readl(hsotg->regs + HCINTMSK(chnum));
1763		hcsplt = dwc2_readl(hsotg->regs + HCSPLT(chnum));
1764		dev_dbg(hsotg->dev,
1765			"%s: chan->halt_status DWC2_HC_XFER_NO_HALT_STATUS,\n",
1766			 __func__);
1767		dev_dbg(hsotg->dev,
1768			"channel %d, hcchar 0x%08x, hctsiz 0x%08x,\n",
1769			chnum, hcchar, hctsiz);
1770		dev_dbg(hsotg->dev,
1771			"hcint 0x%08x, hcintmsk 0x%08x, hcsplt 0x%08x,\n",
1772			chan->hcint, hcintmsk, hcsplt);
1773		if (qtd)
1774			dev_dbg(hsotg->dev, "qtd->complete_split %d\n",
1775				qtd->complete_split);
1776		dev_warn(hsotg->dev,
1777			 "%s: no halt status, channel %d, ignoring interrupt\n",
1778			 __func__, chnum);
1779		return false;
1780	}
1781
1782	/*
1783	 * This code is here only as a check. hcchar.chdis should never be set
1784	 * when the halt interrupt occurs. Halt the channel again if it does
1785	 * occur.
1786	 */
1787	hcchar = dwc2_readl(hsotg->regs + HCCHAR(chnum));
1788	if (hcchar & HCCHAR_CHDIS) {
1789		dev_warn(hsotg->dev,
1790			 "%s: hcchar.chdis set unexpectedly, hcchar 0x%08x, trying to halt again\n",
1791			 __func__, hcchar);
1792		chan->halt_pending = 0;
1793		dwc2_halt_channel(hsotg, chan, qtd, chan->halt_status);
1794		return false;
1795	}
1796#endif
1797
1798	return true;
1799}
1800
1801/*
1802 * Handles a host Channel Halted interrupt in DMA mode. This handler
1803 * determines the reason the channel halted and proceeds accordingly.
1804 */
1805static void dwc2_hc_chhltd_intr_dma(struct dwc2_hsotg *hsotg,
1806				    struct dwc2_host_chan *chan, int chnum,
1807				    struct dwc2_qtd *qtd)
1808{
1809	u32 hcintmsk;
1810	int out_nak_enh = 0;
1811
1812	if (dbg_hc(chan))
1813		dev_vdbg(hsotg->dev,
1814			 "--Host Channel %d Interrupt: DMA Channel Halted--\n",
1815			 chnum);
1816
1817	/*
1818	 * For core with OUT NAK enhancement, the flow for high-speed
1819	 * CONTROL/BULK OUT is handled a little differently
1820	 */
1821	if (hsotg->hw_params.snpsid >= DWC2_CORE_REV_2_71a) {
1822		if (chan->speed == USB_SPEED_HIGH && !chan->ep_is_in &&
1823		    (chan->ep_type == USB_ENDPOINT_XFER_CONTROL ||
1824		     chan->ep_type == USB_ENDPOINT_XFER_BULK)) {
1825			out_nak_enh = 1;
1826		}
1827	}
1828
1829	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE ||
1830	    (chan->halt_status == DWC2_HC_XFER_AHB_ERR &&
1831	     !hsotg->params.dma_desc_enable)) {
1832		if (hsotg->params.dma_desc_enable)
1833			dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
1834						    chan->halt_status);
1835		else
1836			/*
1837			 * Just release the channel. A dequeue can happen on a
1838			 * transfer timeout. In the case of an AHB Error, the
1839			 * channel was forced to halt because there's no way to
1840			 * gracefully recover.
1841			 */
1842			dwc2_release_channel(hsotg, chan, qtd,
1843					     chan->halt_status);
1844		return;
1845	}
1846
1847	hcintmsk = dwc2_readl(hsotg->regs + HCINTMSK(chnum));
1848
1849	if (chan->hcint & HCINTMSK_XFERCOMPL) {
1850		/*
1851		 * Todo: This is here because of a possible hardware bug. Spec
1852		 * says that on SPLIT-ISOC OUT transfers in DMA mode that a HALT
1853		 * interrupt w/ACK bit set should occur, but I only see the
1854		 * XFERCOMP bit, even with it masked out. This is a workaround
1855		 * for that behavior. Should fix this when hardware is fixed.
1856		 */
1857		if (chan->ep_type == USB_ENDPOINT_XFER_ISOC && !chan->ep_is_in)
1858			dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1859		dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
1860	} else if (chan->hcint & HCINTMSK_STALL) {
1861		dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
1862	} else if ((chan->hcint & HCINTMSK_XACTERR) &&
1863		   !hsotg->params.dma_desc_enable) {
1864		if (out_nak_enh) {
1865			if (chan->hcint &
1866			    (HCINTMSK_NYET | HCINTMSK_NAK | HCINTMSK_ACK)) {
1867				dev_vdbg(hsotg->dev,
1868					 "XactErr with NYET/NAK/ACK\n");
1869				qtd->error_count = 0;
1870			} else {
1871				dev_vdbg(hsotg->dev,
1872					 "XactErr without NYET/NAK/ACK\n");
1873			}
1874		}
1875
1876		/*
1877		 * Must handle xacterr before nak or ack. Could get a xacterr
1878		 * at the same time as either of these on a BULK/CONTROL OUT
1879		 * that started with a PING. The xacterr takes precedence.
1880		 */
1881		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1882	} else if ((chan->hcint & HCINTMSK_XCS_XACT) &&
1883		   hsotg->params.dma_desc_enable) {
1884		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
1885	} else if ((chan->hcint & HCINTMSK_AHBERR) &&
1886		   hsotg->params.dma_desc_enable) {
1887		dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
1888	} else if (chan->hcint & HCINTMSK_BBLERR) {
1889		dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
1890	} else if (chan->hcint & HCINTMSK_FRMOVRUN) {
1891		dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
1892	} else if (!out_nak_enh) {
1893		if (chan->hcint & HCINTMSK_NYET) {
1894			/*
1895			 * Must handle nyet before nak or ack. Could get a nyet
1896			 * at the same time as either of those on a BULK/CONTROL
1897			 * OUT that started with a PING. The nyet takes
1898			 * precedence.
1899			 */
1900			dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
1901		} else if ((chan->hcint & HCINTMSK_NAK) &&
1902			   !(hcintmsk & HCINTMSK_NAK)) {
1903			/*
1904			 * If nak is not masked, it's because a non-split IN
1905			 * transfer is in an error state. In that case, the nak
1906			 * is handled by the nak interrupt handler, not here.
1907			 * Handle nak here for BULK/CONTROL OUT transfers, which
1908			 * halt on a NAK to allow rewinding the buffer pointer.
1909			 */
1910			dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
1911		} else if ((chan->hcint & HCINTMSK_ACK) &&
1912			   !(hcintmsk & HCINTMSK_ACK)) {
1913			/*
1914			 * If ack is not masked, it's because a non-split IN
1915			 * transfer is in an error state. In that case, the ack
1916			 * is handled by the ack interrupt handler, not here.
1917			 * Handle ack here for split transfers. Start splits
1918			 * halt on ACK.
1919			 */
1920			dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
1921		} else {
1922			if (chan->ep_type == USB_ENDPOINT_XFER_INT ||
1923			    chan->ep_type == USB_ENDPOINT_XFER_ISOC) {
1924				/*
1925				 * A periodic transfer halted with no other
1926				 * channel interrupts set. Assume it was halted
1927				 * by the core because it could not be completed
1928				 * in its scheduled (micro)frame.
1929				 */
1930				dev_dbg(hsotg->dev,
1931					"%s: Halt channel %d (assume incomplete periodic transfer)\n",
1932					__func__, chnum);
1933				dwc2_halt_channel(hsotg, chan, qtd,
1934					DWC2_HC_XFER_PERIODIC_INCOMPLETE);
1935			} else {
1936				dev_err(hsotg->dev,
1937					"%s: Channel %d - ChHltd set, but reason is unknown\n",
1938					__func__, chnum);
1939				dev_err(hsotg->dev,
1940					"hcint 0x%08x, intsts 0x%08x\n",
1941					chan->hcint,
1942					dwc2_readl(hsotg->regs + GINTSTS));
1943				goto error;
1944			}
1945		}
1946	} else {
1947		dev_info(hsotg->dev,
1948			 "NYET/NAK/ACK/other in non-error case, 0x%08x\n",
1949			 chan->hcint);
1950error:
1951		/* Failthrough: use 3-strikes rule */
1952		qtd->error_count++;
1953		dwc2_update_urb_state_abn(hsotg, chan, chnum, qtd->urb,
1954					  qtd, DWC2_HC_XFER_XACT_ERR);
1955		dwc2_hcd_save_data_toggle(hsotg, chan, chnum, qtd);
1956		dwc2_halt_channel(hsotg, chan, qtd, DWC2_HC_XFER_XACT_ERR);
1957	}
1958}
1959
1960/*
1961 * Handles a host channel Channel Halted interrupt
1962 *
1963 * In slave mode, this handler is called only when the driver specifically
1964 * requests a halt. This occurs during handling other host channel interrupts
1965 * (e.g. nak, xacterr, stall, nyet, etc.).
1966 *
1967 * In DMA mode, this is the interrupt that occurs when the core has finished
1968 * processing a transfer on a channel. Other host channel interrupts (except
1969 * ahberr) are disabled in DMA mode.
1970 */
1971static void dwc2_hc_chhltd_intr(struct dwc2_hsotg *hsotg,
1972				struct dwc2_host_chan *chan, int chnum,
1973				struct dwc2_qtd *qtd)
1974{
1975	if (dbg_hc(chan))
1976		dev_vdbg(hsotg->dev, "--Host Channel %d Interrupt: Channel Halted--\n",
1977			 chnum);
1978
1979	if (hsotg->params.host_dma) {
1980		dwc2_hc_chhltd_intr_dma(hsotg, chan, chnum, qtd);
1981	} else {
1982		if (!dwc2_halt_status_ok(hsotg, chan, chnum, qtd))
1983			return;
1984		dwc2_release_channel(hsotg, chan, qtd, chan->halt_status);
1985	}
1986}
1987
1988/*
1989 * Check if the given qtd is still the top of the list (and thus valid).
1990 *
1991 * If dwc2_hcd_qtd_unlink_and_free() has been called since we grabbed
1992 * the qtd from the top of the list, this will return false (otherwise true).
1993 */
1994static bool dwc2_check_qtd_still_ok(struct dwc2_qtd *qtd, struct dwc2_qh *qh)
1995{
1996	struct dwc2_qtd *cur_head;
1997
1998	if (!qh)
1999		return false;
2000
2001	cur_head = list_first_entry(&qh->qtd_list, struct dwc2_qtd,
2002				    qtd_list_entry);
2003	return (cur_head == qtd);
2004}
2005
2006/* Handles interrupt for a specific Host Channel */
2007static void dwc2_hc_n_intr(struct dwc2_hsotg *hsotg, int chnum)
2008{
2009	struct dwc2_qtd *qtd;
2010	struct dwc2_host_chan *chan;
2011	u32 hcint, hcintmsk;
2012
2013	chan = hsotg->hc_ptr_array[chnum];
2014
2015	hcint = dwc2_readl(hsotg->regs + HCINT(chnum));
2016	hcintmsk = dwc2_readl(hsotg->regs + HCINTMSK(chnum));
2017	if (!chan) {
2018		dev_err(hsotg->dev, "## hc_ptr_array for channel is NULL ##\n");
2019		dwc2_writel(hcint, hsotg->regs + HCINT(chnum));
2020		return;
2021	}
2022
2023	if (dbg_hc(chan)) {
2024		dev_vdbg(hsotg->dev, "--Host Channel Interrupt--, Channel %d\n",
2025			 chnum);
2026		dev_vdbg(hsotg->dev,
2027			 "  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2028			 hcint, hcintmsk, hcint & hcintmsk);
2029	}
2030
2031	dwc2_writel(hcint, hsotg->regs + HCINT(chnum));
2032
2033	/*
2034	 * If we got an interrupt after someone called
2035	 * dwc2_hcd_endpoint_disable() we don't want to crash below
2036	 */
2037	if (!chan->qh) {
2038		dev_warn(hsotg->dev, "Interrupt on disabled channel\n");
2039		return;
2040	}
2041
2042	chan->hcint = hcint;
2043	hcint &= hcintmsk;
2044
2045	/*
2046	 * If the channel was halted due to a dequeue, the qtd list might
2047	 * be empty or at least the first entry will not be the active qtd.
2048	 * In this case, take a shortcut and just release the channel.
2049	 */
2050	if (chan->halt_status == DWC2_HC_XFER_URB_DEQUEUE) {
2051		/*
2052		 * If the channel was halted, this should be the only
2053		 * interrupt unmasked
2054		 */
2055		WARN_ON(hcint != HCINTMSK_CHHLTD);
2056		if (hsotg->params.dma_desc_enable)
2057			dwc2_hcd_complete_xfer_ddma(hsotg, chan, chnum,
2058						    chan->halt_status);
2059		else
2060			dwc2_release_channel(hsotg, chan, NULL,
2061					     chan->halt_status);
2062		return;
2063	}
2064
2065	if (list_empty(&chan->qh->qtd_list)) {
2066		/*
2067		 * TODO: Will this ever happen with the
2068		 * DWC2_HC_XFER_URB_DEQUEUE handling above?
2069		 */
2070		dev_dbg(hsotg->dev, "## no QTD queued for channel %d ##\n",
2071			chnum);
2072		dev_dbg(hsotg->dev,
2073			"  hcint 0x%08x, hcintmsk 0x%08x, hcint&hcintmsk 0x%08x\n",
2074			chan->hcint, hcintmsk, hcint);
2075		chan->halt_status = DWC2_HC_XFER_NO_HALT_STATUS;
2076		disable_hc_int(hsotg, chnum, HCINTMSK_CHHLTD);
2077		chan->hcint = 0;
2078		return;
2079	}
2080
2081	qtd = list_first_entry(&chan->qh->qtd_list, struct dwc2_qtd,
2082			       qtd_list_entry);
2083
2084	if (!hsotg->params.host_dma) {
2085		if ((hcint & HCINTMSK_CHHLTD) && hcint != HCINTMSK_CHHLTD)
2086			hcint &= ~HCINTMSK_CHHLTD;
2087	}
2088
2089	if (hcint & HCINTMSK_XFERCOMPL) {
2090		dwc2_hc_xfercomp_intr(hsotg, chan, chnum, qtd);
2091		/*
2092		 * If NYET occurred at same time as Xfer Complete, the NYET is
2093		 * handled by the Xfer Complete interrupt handler. Don't want
2094		 * to call the NYET interrupt handler in this case.
2095		 */
2096		hcint &= ~HCINTMSK_NYET;
2097	}
2098
2099	if (hcint & HCINTMSK_CHHLTD) {
2100		dwc2_hc_chhltd_intr(hsotg, chan, chnum, qtd);
2101		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2102			goto exit;
2103	}
2104	if (hcint & HCINTMSK_AHBERR) {
2105		dwc2_hc_ahberr_intr(hsotg, chan, chnum, qtd);
2106		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2107			goto exit;
2108	}
2109	if (hcint & HCINTMSK_STALL) {
2110		dwc2_hc_stall_intr(hsotg, chan, chnum, qtd);
2111		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2112			goto exit;
2113	}
2114	if (hcint & HCINTMSK_NAK) {
2115		dwc2_hc_nak_intr(hsotg, chan, chnum, qtd);
2116		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2117			goto exit;
2118	}
2119	if (hcint & HCINTMSK_ACK) {
2120		dwc2_hc_ack_intr(hsotg, chan, chnum, qtd);
2121		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2122			goto exit;
2123	}
2124	if (hcint & HCINTMSK_NYET) {
2125		dwc2_hc_nyet_intr(hsotg, chan, chnum, qtd);
2126		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2127			goto exit;
2128	}
2129	if (hcint & HCINTMSK_XACTERR) {
2130		dwc2_hc_xacterr_intr(hsotg, chan, chnum, qtd);
2131		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2132			goto exit;
2133	}
2134	if (hcint & HCINTMSK_BBLERR) {
2135		dwc2_hc_babble_intr(hsotg, chan, chnum, qtd);
2136		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2137			goto exit;
2138	}
2139	if (hcint & HCINTMSK_FRMOVRUN) {
2140		dwc2_hc_frmovrun_intr(hsotg, chan, chnum, qtd);
2141		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2142			goto exit;
2143	}
2144	if (hcint & HCINTMSK_DATATGLERR) {
2145		dwc2_hc_datatglerr_intr(hsotg, chan, chnum, qtd);
2146		if (!dwc2_check_qtd_still_ok(qtd, chan->qh))
2147			goto exit;
2148	}
2149
2150exit:
2151	chan->hcint = 0;
2152}
2153
2154/*
2155 * This interrupt indicates that one or more host channels has a pending
2156 * interrupt. There are multiple conditions that can cause each host channel
2157 * interrupt. This function determines which conditions have occurred for each
2158 * host channel interrupt and handles them appropriately.
2159 */
2160static void dwc2_hc_intr(struct dwc2_hsotg *hsotg)
2161{
2162	u32 haint;
2163	int i;
2164	struct dwc2_host_chan *chan, *chan_tmp;
2165
2166	haint = dwc2_readl(hsotg->regs + HAINT);
2167	if (dbg_perio()) {
2168		dev_vdbg(hsotg->dev, "%s()\n", __func__);
2169
2170		dev_vdbg(hsotg->dev, "HAINT=%08x\n", haint);
2171	}
2172
2173	/*
2174	 * According to USB 2.0 spec section 11.18.8, a host must
2175	 * issue complete-split transactions in a microframe for a
2176	 * set of full-/low-speed endpoints in the same relative
2177	 * order as the start-splits were issued in a microframe for.
2178	 */
2179	list_for_each_entry_safe(chan, chan_tmp, &hsotg->split_order,
2180				 split_order_list_entry) {
2181		int hc_num = chan->hc_num;
2182
2183		if (haint & (1 << hc_num)) {
2184			dwc2_hc_n_intr(hsotg, hc_num);
2185			haint &= ~(1 << hc_num);
2186		}
2187	}
2188
2189	for (i = 0; i < hsotg->params.host_channels; i++) {
2190		if (haint & (1 << i))
2191			dwc2_hc_n_intr(hsotg, i);
2192	}
2193}
2194
2195/* This function handles interrupts for the HCD */
2196irqreturn_t dwc2_handle_hcd_intr(struct dwc2_hsotg *hsotg)
2197{
2198	u32 gintsts, dbg_gintsts;
2199	irqreturn_t retval = IRQ_NONE;
2200
2201	if (!dwc2_is_controller_alive(hsotg)) {
2202		dev_warn(hsotg->dev, "Controller is dead\n");
2203		return retval;
2204	}
2205
2206	spin_lock(&hsotg->lock);
2207
2208	/* Check if HOST Mode */
2209	if (dwc2_is_host_mode(hsotg)) {
2210		gintsts = dwc2_read_core_intr(hsotg);
2211		if (!gintsts) {
2212			spin_unlock(&hsotg->lock);
2213			return retval;
2214		}
2215
2216		retval = IRQ_HANDLED;
2217
2218		dbg_gintsts = gintsts;
2219#ifndef DEBUG_SOF
2220		dbg_gintsts &= ~GINTSTS_SOF;
2221#endif
2222		if (!dbg_perio())
2223			dbg_gintsts &= ~(GINTSTS_HCHINT | GINTSTS_RXFLVL |
2224					 GINTSTS_PTXFEMP);
2225
2226		/* Only print if there are any non-suppressed interrupts left */
2227		if (dbg_gintsts)
2228			dev_vdbg(hsotg->dev,
2229				 "DWC OTG HCD Interrupt Detected gintsts&gintmsk=0x%08x\n",
2230				 gintsts);
2231
2232		if (gintsts & GINTSTS_SOF)
2233			dwc2_sof_intr(hsotg);
2234		if (gintsts & GINTSTS_RXFLVL)
2235			dwc2_rx_fifo_level_intr(hsotg);
2236		if (gintsts & GINTSTS_NPTXFEMP)
2237			dwc2_np_tx_fifo_empty_intr(hsotg);
2238		if (gintsts & GINTSTS_PRTINT)
2239			dwc2_port_intr(hsotg);
2240		if (gintsts & GINTSTS_HCHINT)
2241			dwc2_hc_intr(hsotg);
2242		if (gintsts & GINTSTS_PTXFEMP)
2243			dwc2_perio_tx_fifo_empty_intr(hsotg);
2244
2245		if (dbg_gintsts) {
2246			dev_vdbg(hsotg->dev,
2247				 "DWC OTG HCD Finished Servicing Interrupts\n");
2248			dev_vdbg(hsotg->dev,
2249				 "DWC OTG HCD gintsts=0x%08x gintmsk=0x%08x\n",
2250				 dwc2_readl(hsotg->regs + GINTSTS),
2251				 dwc2_readl(hsotg->regs + GINTMSK));
2252		}
2253	}
2254
2255	spin_unlock(&hsotg->lock);
2256
2257	return retval;
2258}