1/*
   2 * MUSB OTG driver host support
   3 *
   4 * Copyright 2005 Mentor Graphics Corporation
   5 * Copyright (C) 2005-2006 by Texas Instruments
   6 * Copyright (C) 2006-2007 Nokia Corporation
   7 * Copyright (C) 2008-2009 MontaVista Software, Inc. <source@mvista.com>
   8 *
   9 * This program is free software; you can redistribute it and/or
  10 * modify it under the terms of the GNU General Public License
  11 * version 2 as published by the Free Software Foundation.
  12 *
  13 * This program is distributed in the hope that it will be useful, but
  14 * WITHOUT ANY WARRANTY; without even the implied warranty of
  15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 * General Public License for more details.
  17 *
  18 * You should have received a copy of the GNU General Public License
  19 * along with this program; if not, write to the Free Software
  20 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA
  21 * 02110-1301 USA
  22 *
  23 * THIS SOFTWARE IS PROVIDED "AS IS" AND ANY EXPRESS OR IMPLIED
  24 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
  25 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN
  26 * NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  27 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
  28 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
  29 * USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
  30 * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  31 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
  32 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  33 *
  34 */
  35
  36#include <linux/module.h>
  37#include <linux/kernel.h>
  38#include <linux/delay.h>
  39#include <linux/sched.h>
  40#include <linux/slab.h>
  41#include <linux/errno.h>
  42#include <linux/init.h>
  43#include <linux/list.h>
  44#include <linux/dma-mapping.h>
  45
  46#include "musb_core.h"
  47#include "musb_host.h"
  48
  49
  50/* MUSB HOST status 22-mar-2006
  51 *
  52 * - There's still lots of partial code duplication for fault paths, so
  53 *   they aren't handled as consistently as they need to be.
  54 *
  55 * - PIO mostly behaved when last tested.
  56 *     + including ep0, with all usbtest cases 9, 10
  57 *     + usbtest 14 (ep0out) doesn't seem to run at all
  58 *     + double buffered OUT/TX endpoints saw stalls(!) with certain usbtest
  59 *       configurations, but otherwise double buffering passes basic tests.
  60 *     + for 2.6.N, for N > ~10, needs API changes for hcd framework.
  61 *
  62 * - DMA (CPPI) ... partially behaves, not currently recommended
  63 *     + about 1/15 the speed of typical EHCI implementations (PCI)
  64 *     + RX, all too often reqpkt seems to misbehave after tx
  65 *     + TX, no known issues (other than evident silicon issue)
  66 *
  67 * - DMA (Mentor/OMAP) ...has at least toggle update problems
  68 *
  69 * - [23-feb-2009] minimal traffic scheduling to avoid bulk RX packet
  70 *   starvation ... nothing yet for TX, interrupt, or bulk.
  71 *
  72 * - Not tested with HNP, but some SRP paths seem to behave.
  73 *
  74 * NOTE 24-August-2006:
  75 *
  76 * - Bulk traffic finally uses both sides of hardware ep1, freeing up an
  77 *   extra endpoint for periodic use enabling hub + keybd + mouse.  That
  78 *   mostly works, except that with "usbnet" it's easy to trigger cases
  79 *   with "ping" where RX loses.  (a) ping to davinci, even "ping -f",
  80 *   fine; but (b) ping _from_ davinci, even "ping -c 1", ICMP RX loses
  81 *   although ARP RX wins.  (That test was done with a full speed link.)
  82 */
  83
  84
  85/*
  86 * NOTE on endpoint usage:
  87 *
  88 * CONTROL transfers all go through ep0.  BULK ones go through dedicated IN
  89 * and OUT endpoints ... hardware is dedicated for those "async" queue(s).
  90 * (Yes, bulk _could_ use more of the endpoints than that, and would even
  91 * benefit from it.)
  92 *
  93 * INTERUPPT and ISOCHRONOUS transfers are scheduled to the other endpoints.
  94 * So far that scheduling is both dumb and optimistic:  the endpoint will be
  95 * "claimed" until its software queue is no longer refilled.  No multiplexing
  96 * of transfers between endpoints, or anything clever.
  97 */
  98
  99
 100static void musb_ep_program(struct musb *musb, u8 epnum,
 101			struct urb *urb, int is_out,
 102			u8 *buf, u32 offset, u32 len);
 103
 104/*
 105 * Clear TX fifo. Needed to avoid BABBLE errors.
 106 */
 107static void musb_h_tx_flush_fifo(struct musb_hw_ep *ep)
 108{
 109	struct musb	*musb = ep->musb;
 110	void __iomem	*epio = ep->regs;
 111	u16		csr;
 112	u16		lastcsr = 0;
 113	int		retries = 1000;
 114
 115	csr = musb_readw(epio, MUSB_TXCSR);
 116	while (csr & MUSB_TXCSR_FIFONOTEMPTY) {
 117		if (csr != lastcsr)
 118			dev_dbg(musb->controller, "Host TX FIFONOTEMPTY csr: %02x\n", csr);
 119		lastcsr = csr;
 120		csr |= MUSB_TXCSR_FLUSHFIFO;
 121		musb_writew(epio, MUSB_TXCSR, csr);
 122		csr = musb_readw(epio, MUSB_TXCSR);
 123		if (WARN(retries-- < 1,
 124				"Could not flush host TX%d fifo: csr: %04x\n",
 125				ep->epnum, csr))
 126			return;
 127		mdelay(1);
 128	}
 129}
 130
 131static void musb_h_ep0_flush_fifo(struct musb_hw_ep *ep)
 132{
 133	void __iomem	*epio = ep->regs;
 134	u16		csr;
 135	int		retries = 5;
 136
 137	/* scrub any data left in the fifo */
 138	do {
 139		csr = musb_readw(epio, MUSB_TXCSR);
 140		if (!(csr & (MUSB_CSR0_TXPKTRDY | MUSB_CSR0_RXPKTRDY)))
 141			break;
 142		musb_writew(epio, MUSB_TXCSR, MUSB_CSR0_FLUSHFIFO);
 143		csr = musb_readw(epio, MUSB_TXCSR);
 144		udelay(10);
 145	} while (--retries);
 146
 147	WARN(!retries, "Could not flush host TX%d fifo: csr: %04x\n",
 148			ep->epnum, csr);
 149
 150	/* and reset for the next transfer */
 151	musb_writew(epio, MUSB_TXCSR, 0);
 152}
 153
 154/*
 155 * Start transmit. Caller is responsible for locking shared resources.
 156 * musb must be locked.
 157 */
 158static inline void musb_h_tx_start(struct musb_hw_ep *ep)
 159{
 160	u16	txcsr;
 161
 162	/* NOTE: no locks here; caller should lock and select EP */
 163	if (ep->epnum) {
 164		txcsr = musb_readw(ep->regs, MUSB_TXCSR);
 165		txcsr |= MUSB_TXCSR_TXPKTRDY | MUSB_TXCSR_H_WZC_BITS;
 166		musb_writew(ep->regs, MUSB_TXCSR, txcsr);
 167	} else {
 168		txcsr = MUSB_CSR0_H_SETUPPKT | MUSB_CSR0_TXPKTRDY;
 169		musb_writew(ep->regs, MUSB_CSR0, txcsr);
 170	}
 171
 172}
 173
 174static inline void musb_h_tx_dma_start(struct musb_hw_ep *ep)
 175{
 176	u16	txcsr;
 177
 178	/* NOTE: no locks here; caller should lock and select EP */
 179	txcsr = musb_readw(ep->regs, MUSB_TXCSR);
 180	txcsr |= MUSB_TXCSR_DMAENAB | MUSB_TXCSR_H_WZC_BITS;
 181	if (is_cppi_enabled())
 182		txcsr |= MUSB_TXCSR_DMAMODE;
 183	musb_writew(ep->regs, MUSB_TXCSR, txcsr);
 184}
 185
 186static void musb_ep_set_qh(struct musb_hw_ep *ep, int is_in, struct musb_qh *qh)
 187{
 188	if (is_in != 0 || ep->is_shared_fifo)
 189		ep->in_qh  = qh;
 190	if (is_in == 0 || ep->is_shared_fifo)
 191		ep->out_qh = qh;
 192}
 193
 194static struct musb_qh *musb_ep_get_qh(struct musb_hw_ep *ep, int is_in)
 195{
 196	return is_in ? ep->in_qh : ep->out_qh;
 197}
 198
 199/*
 200 * Start the URB at the front of an endpoint's queue
 201 * end must be claimed from the caller.
 202 *
 203 * Context: controller locked, irqs blocked
 204 */
 205static void
 206musb_start_urb(struct musb *musb, int is_in, struct musb_qh *qh)
 207{
 208	u16			frame;
 209	u32			len;
 210	void __iomem		*mbase =  musb->mregs;
 211	struct urb		*urb = next_urb(qh);
 212	void			*buf = urb->transfer_buffer;
 213	u32			offset = 0;
 214	struct musb_hw_ep	*hw_ep = qh->hw_ep;
 215	unsigned		pipe = urb->pipe;
 216	u8			address = usb_pipedevice(pipe);
 217	int			epnum = hw_ep->epnum;
 218
 219	/* initialize software qh state */
 220	qh->offset = 0;
 221	qh->segsize = 0;
 222
 223	/* gather right source of data */
 224	switch (qh->type) {
 225	case USB_ENDPOINT_XFER_CONTROL:
 226		/* control transfers always start with SETUP */
 227		is_in = 0;
 228		musb->ep0_stage = MUSB_EP0_START;
 229		buf = urb->setup_packet;
 230		len = 8;
 231		break;
 232	case USB_ENDPOINT_XFER_ISOC:
 233		qh->iso_idx = 0;
 234		qh->frame = 0;
 235		offset = urb->iso_frame_desc[0].offset;
 236		len = urb->iso_frame_desc[0].length;
 237		break;
 238	default:		/* bulk, interrupt */
 239		/* actual_length may be nonzero on retry paths */
 240		buf = urb->transfer_buffer + urb->actual_length;
 241		len = urb->transfer_buffer_length - urb->actual_length;
 242	}
 243
 244	dev_dbg(musb->controller, "qh %p urb %p dev%d ep%d%s%s, hw_ep %d, %p/%d\n",
 245			qh, urb, address, qh->epnum,
 246			is_in ? "in" : "out",
 247			({char *s; switch (qh->type) {
 248			case USB_ENDPOINT_XFER_CONTROL:	s = ""; break;
 249			case USB_ENDPOINT_XFER_BULK:	s = "-bulk"; break;
 250			case USB_ENDPOINT_XFER_ISOC:	s = "-iso"; break;
 251			default:			s = "-intr"; break;
 252			}; s; }),
 253			epnum, buf + offset, len);
 254
 255	/* Configure endpoint */
 256	musb_ep_set_qh(hw_ep, is_in, qh);
 257	musb_ep_program(musb, epnum, urb, !is_in, buf, offset, len);
 258
 259	/* transmit may have more work: start it when it is time */
 260	if (is_in)
 261		return;
 262
 263	/* determine if the time is right for a periodic transfer */
 264	switch (qh->type) {
 265	case USB_ENDPOINT_XFER_ISOC:
 266	case USB_ENDPOINT_XFER_INT:
 267		dev_dbg(musb->controller, "check whether there's still time for periodic Tx\n");
 268		frame = musb_readw(mbase, MUSB_FRAME);
 269		/* FIXME this doesn't implement that scheduling policy ...
 270		 * or handle framecounter wrapping
 271		 */
 272		if ((urb->transfer_flags & URB_ISO_ASAP)
 273				|| (frame >= urb->start_frame)) {
 274			/* REVISIT the SOF irq handler shouldn't duplicate
 275			 * this code; and we don't init urb->start_frame...
 276			 */
 277			qh->frame = 0;
 278			goto start;
 279		} else {
 280			qh->frame = urb->start_frame;
 281			/* enable SOF interrupt so we can count down */
 282			dev_dbg(musb->controller, "SOF for %d\n", epnum);
 283#if 1 /* ifndef	CONFIG_ARCH_DAVINCI */
 284			musb_writeb(mbase, MUSB_INTRUSBE, 0xff);
 285#endif
 286		}
 287		break;
 288	default:
 289start:
 290		dev_dbg(musb->controller, "Start TX%d %s\n", epnum,
 291			hw_ep->tx_channel ? "dma" : "pio");
 292
 293		if (!hw_ep->tx_channel)
 294			musb_h_tx_start(hw_ep);
 295		else if (is_cppi_enabled() || tusb_dma_omap())
 296			musb_h_tx_dma_start(hw_ep);
 297	}
 298}
 299
 300/* Context: caller owns controller lock, IRQs are blocked */
 301static void musb_giveback(struct musb *musb, struct urb *urb, int status)
 302__releases(musb->lock)
 303__acquires(musb->lock)
 304{
 305	dev_dbg(musb->controller,
 306			"complete %p %pF (%d), dev%d ep%d%s, %d/%d\n",
 307			urb, urb->complete, status,
 308			usb_pipedevice(urb->pipe),
 309			usb_pipeendpoint(urb->pipe),
 310			usb_pipein(urb->pipe) ? "in" : "out",
 311			urb->actual_length, urb->transfer_buffer_length
 312			);
 313
 314	usb_hcd_unlink_urb_from_ep(musb_to_hcd(musb), urb);
 315	spin_unlock(&musb->lock);
 316	usb_hcd_giveback_urb(musb_to_hcd(musb), urb, status);
 317	spin_lock(&musb->lock);
 318}
 319
 320/* For bulk/interrupt endpoints only */
 321static inline void musb_save_toggle(struct musb_qh *qh, int is_in,
 322				    struct urb *urb)
 323{
 324	void __iomem		*epio = qh->hw_ep->regs;
 325	u16			csr;
 326
 327	/*
 328	 * FIXME: the current Mentor DMA code seems to have
 329	 * problems getting toggle correct.
 330	 */
 331
 332	if (is_in)
 333		csr = musb_readw(epio, MUSB_RXCSR) & MUSB_RXCSR_H_DATATOGGLE;
 334	else
 335		csr = musb_readw(epio, MUSB_TXCSR) & MUSB_TXCSR_H_DATATOGGLE;
 336
 337	usb_settoggle(urb->dev, qh->epnum, !is_in, csr ? 1 : 0);
 338}
 339
 340/*
 341 * Advance this hardware endpoint's queue, completing the specified URB and
 342 * advancing to either the next URB queued to that qh, or else invalidating
 343 * that qh and advancing to the next qh scheduled after the current one.
 344 *
 345 * Context: caller owns controller lock, IRQs are blocked
 346 */
 347static void musb_advance_schedule(struct musb *musb, struct urb *urb,
 348				  struct musb_hw_ep *hw_ep, int is_in)
 349{
 350	struct musb_qh		*qh = musb_ep_get_qh(hw_ep, is_in);
 351	struct musb_hw_ep	*ep = qh->hw_ep;
 352	int			ready = qh->is_ready;
 353	int			status;
 354
 355	status = (urb->status == -EINPROGRESS) ? 0 : urb->status;
 356
 357	/* save toggle eagerly, for paranoia */
 358	switch (qh->type) {
 359	case USB_ENDPOINT_XFER_BULK:
 360	case USB_ENDPOINT_XFER_INT:
 361		musb_save_toggle(qh, is_in, urb);
 362		break;
 363	case USB_ENDPOINT_XFER_ISOC:
 364		if (status == 0 && urb->error_count)
 365			status = -EXDEV;
 366		break;
 367	}
 368
 369	qh->is_ready = 0;
 370	musb_giveback(musb, urb, status);
 371	qh->is_ready = ready;
 372
 373	/* reclaim resources (and bandwidth) ASAP; deschedule it, and
 374	 * invalidate qh as soon as list_empty(&hep->urb_list)
 375	 */
 376	if (list_empty(&qh->hep->urb_list)) {
 377		struct list_head	*head;
 
 378
 379		if (is_in)
 380			ep->rx_reinit = 1;
 381		else
 
 
 
 
 382			ep->tx_reinit = 1;
 
 
 
 
 
 383
 384		/* Clobber old pointers to this qh */
 385		musb_ep_set_qh(ep, is_in, NULL);
 386		qh->hep->hcpriv = NULL;
 387
 388		switch (qh->type) {
 389
 390		case USB_ENDPOINT_XFER_CONTROL:
 391		case USB_ENDPOINT_XFER_BULK:
 392			/* fifo policy for these lists, except that NAKing
 393			 * should rotate a qh to the end (for fairness).
 394			 */
 395			if (qh->mux == 1) {
 396				head = qh->ring.prev;
 397				list_del(&qh->ring);
 398				kfree(qh);
 399				qh = first_qh(head);
 400				break;
 401			}
 402
 403		case USB_ENDPOINT_XFER_ISOC:
 404		case USB_ENDPOINT_XFER_INT:
 405			/* this is where periodic bandwidth should be
 406			 * de-allocated if it's tracked and allocated;
 407			 * and where we'd update the schedule tree...
 408			 */
 409			kfree(qh);
 410			qh = NULL;
 411			break;
 412		}
 413	}
 414
 415	if (qh != NULL && qh->is_ready) {
 416		dev_dbg(musb->controller, "... next ep%d %cX urb %p\n",
 417		    hw_ep->epnum, is_in ? 'R' : 'T', next_urb(qh));
 418		musb_start_urb(musb, is_in, qh);
 419	}
 420}
 421
 422static u16 musb_h_flush_rxfifo(struct musb_hw_ep *hw_ep, u16 csr)
 423{
 424	/* we don't want fifo to fill itself again;
 425	 * ignore dma (various models),
 426	 * leave toggle alone (may not have been saved yet)
 427	 */
 428	csr |= MUSB_RXCSR_FLUSHFIFO | MUSB_RXCSR_RXPKTRDY;
 429	csr &= ~(MUSB_RXCSR_H_REQPKT
 430		| MUSB_RXCSR_H_AUTOREQ
 431		| MUSB_RXCSR_AUTOCLEAR);
 432
 433	/* write 2x to allow double buffering */
 434	musb_writew(hw_ep->regs, MUSB_RXCSR, csr);
 435	musb_writew(hw_ep->regs, MUSB_RXCSR, csr);
 436
 437	/* flush writebuffer */
 438	return musb_readw(hw_ep->regs, MUSB_RXCSR);
 439}
 440
 441/*
 442 * PIO RX for a packet (or part of it).
 443 */
 444static bool
 445musb_host_packet_rx(struct musb *musb, struct urb *urb, u8 epnum, u8 iso_err)
 446{
 447	u16			rx_count;
 448	u8			*buf;
 449	u16			csr;
 450	bool			done = false;
 451	u32			length;
 452	int			do_flush = 0;
 453	struct musb_hw_ep	*hw_ep = musb->endpoints + epnum;
 454	void __iomem		*epio = hw_ep->regs;
 455	struct musb_qh		*qh = hw_ep->in_qh;
 456	int			pipe = urb->pipe;
 457	void			*buffer = urb->transfer_buffer;
 458
 459	/* musb_ep_select(mbase, epnum); */
 460	rx_count = musb_readw(epio, MUSB_RXCOUNT);
 461	dev_dbg(musb->controller, "RX%d count %d, buffer %p len %d/%d\n", epnum, rx_count,
 462			urb->transfer_buffer, qh->offset,
 463			urb->transfer_buffer_length);
 464
 465	/* unload FIFO */
 466	if (usb_pipeisoc(pipe)) {
 467		int					status = 0;
 468		struct usb_iso_packet_descriptor	*d;
 469
 470		if (iso_err) {
 471			status = -EILSEQ;
 472			urb->error_count++;
 473		}
 474
 475		d = urb->iso_frame_desc + qh->iso_idx;
 476		buf = buffer + d->offset;
 477		length = d->length;
 478		if (rx_count > length) {
 479			if (status == 0) {
 480				status = -EOVERFLOW;
 481				urb->error_count++;
 482			}
 483			dev_dbg(musb->controller, "** OVERFLOW %d into %d\n", rx_count, length);
 484			do_flush = 1;
 485		} else
 486			length = rx_count;
 487		urb->actual_length += length;
 488		d->actual_length = length;
 489
 490		d->status = status;
 491
 492		/* see if we are done */
 493		done = (++qh->iso_idx >= urb->number_of_packets);
 494	} else {
 495		/* non-isoch */
 496		buf = buffer + qh->offset;
 497		length = urb->transfer_buffer_length - qh->offset;
 498		if (rx_count > length) {
 499			if (urb->status == -EINPROGRESS)
 500				urb->status = -EOVERFLOW;
 501			dev_dbg(musb->controller, "** OVERFLOW %d into %d\n", rx_count, length);
 502			do_flush = 1;
 503		} else
 504			length = rx_count;
 505		urb->actual_length += length;
 506		qh->offset += length;
 507
 508		/* see if we are done */
 509		done = (urb->actual_length == urb->transfer_buffer_length)
 510			|| (rx_count < qh->maxpacket)
 511			|| (urb->status != -EINPROGRESS);
 512		if (done
 513				&& (urb->status == -EINPROGRESS)
 514				&& (urb->transfer_flags & URB_SHORT_NOT_OK)
 515				&& (urb->actual_length
 516					< urb->transfer_buffer_length))
 517			urb->status = -EREMOTEIO;
 518	}
 519
 520	musb_read_fifo(hw_ep, length, buf);
 521
 522	csr = musb_readw(epio, MUSB_RXCSR);
 523	csr |= MUSB_RXCSR_H_WZC_BITS;
 524	if (unlikely(do_flush))
 525		musb_h_flush_rxfifo(hw_ep, csr);
 526	else {
 527		/* REVISIT this assumes AUTOCLEAR is never set */
 528		csr &= ~(MUSB_RXCSR_RXPKTRDY | MUSB_RXCSR_H_REQPKT);
 529		if (!done)
 530			csr |= MUSB_RXCSR_H_REQPKT;
 531		musb_writew(epio, MUSB_RXCSR, csr);
 532	}
 533
 534	return done;
 535}
 536
 537/* we don't always need to reinit a given side of an endpoint...
 538 * when we do, use tx/rx reinit routine and then construct a new CSR
 539 * to address data toggle, NYET, and DMA or PIO.
 540 *
 541 * it's possible that driver bugs (especially for DMA) or aborting a
 542 * transfer might have left the endpoint busier than it should be.
 543 * the busy/not-empty tests are basically paranoia.
 544 */
 545static void
 546musb_rx_reinit(struct musb *musb, struct musb_qh *qh, struct musb_hw_ep *ep)
 547{
 548	u16	csr;
 549
 550	/* NOTE:  we know the "rx" fifo reinit never triggers for ep0.
 551	 * That always uses tx_reinit since ep0 repurposes TX register
 552	 * offsets; the initial SETUP packet is also a kind of OUT.
 553	 */
 554
 555	/* if programmed for Tx, put it in RX mode */
 556	if (ep->is_shared_fifo) {
 557		csr = musb_readw(ep->regs, MUSB_TXCSR);
 558		if (csr & MUSB_TXCSR_MODE) {
 559			musb_h_tx_flush_fifo(ep);
 560			csr = musb_readw(ep->regs, MUSB_TXCSR);
 561			musb_writew(ep->regs, MUSB_TXCSR,
 562				    csr | MUSB_TXCSR_FRCDATATOG);
 563		}
 564
 565		/*
 566		 * Clear the MODE bit (and everything else) to enable Rx.
 567		 * NOTE: we mustn't clear the DMAMODE bit before DMAENAB.
 568		 */
 569		if (csr & MUSB_TXCSR_DMAMODE)
 570			musb_writew(ep->regs, MUSB_TXCSR, MUSB_TXCSR_DMAMODE);
 571		musb_writew(ep->regs, MUSB_TXCSR, 0);
 572
 573	/* scrub all previous state, clearing toggle */
 574	} else {
 575		csr = musb_readw(ep->regs, MUSB_RXCSR);
 576		if (csr & MUSB_RXCSR_RXPKTRDY)
 577			WARNING("rx%d, packet/%d ready?\n", ep->epnum,
 578				musb_readw(ep->regs, MUSB_RXCOUNT));
 579
 580		musb_h_flush_rxfifo(ep, MUSB_RXCSR_CLRDATATOG);
 581	}
 582
 583	/* target addr and (for multipoint) hub addr/port */
 584	if (musb->is_multipoint) {
 585		musb_write_rxfunaddr(ep->target_regs, qh->addr_reg);
 586		musb_write_rxhubaddr(ep->target_regs, qh->h_addr_reg);
 587		musb_write_rxhubport(ep->target_regs, qh->h_port_reg);
 588
 589	} else
 590		musb_writeb(musb->mregs, MUSB_FADDR, qh->addr_reg);
 591
 592	/* protocol/endpoint, interval/NAKlimit, i/o size */
 593	musb_writeb(ep->regs, MUSB_RXTYPE, qh->type_reg);
 594	musb_writeb(ep->regs, MUSB_RXINTERVAL, qh->intv_reg);
 595	/* NOTE: bulk combining rewrites high bits of maxpacket */
 596	/* Set RXMAXP with the FIFO size of the endpoint
 597	 * to disable double buffer mode.
 598	 */
 599	if (musb->double_buffer_not_ok)
 600		musb_writew(ep->regs, MUSB_RXMAXP, ep->max_packet_sz_rx);
 601	else
 602		musb_writew(ep->regs, MUSB_RXMAXP,
 603				qh->maxpacket | ((qh->hb_mult - 1) << 11));
 604
 605	ep->rx_reinit = 0;
 606}
 607
 608static bool musb_tx_dma_program(struct dma_controller *dma,
 609		struct musb_hw_ep *hw_ep, struct musb_qh *qh,
 610		struct urb *urb, u32 offset, u32 length)
 611{
 612	struct dma_channel	*channel = hw_ep->tx_channel;
 613	void __iomem		*epio = hw_ep->regs;
 614	u16			pkt_size = qh->maxpacket;
 615	u16			csr;
 616	u8			mode;
 617
 618#ifdef	CONFIG_USB_INVENTRA_DMA
 619	if (length > channel->max_len)
 620		length = channel->max_len;
 621
 622	csr = musb_readw(epio, MUSB_TXCSR);
 623	if (length > pkt_size) {
 624		mode = 1;
 625		csr |= MUSB_TXCSR_DMAMODE | MUSB_TXCSR_DMAENAB;
 626		/* autoset shouldn't be set in high bandwidth */
 627		if (qh->hb_mult == 1)
 628			csr |= MUSB_TXCSR_AUTOSET;
 629	} else {
 630		mode = 0;
 631		csr &= ~(MUSB_TXCSR_AUTOSET | MUSB_TXCSR_DMAMODE);
 632		csr |= MUSB_TXCSR_DMAENAB; /* against programmer's guide */
 633	}
 634	channel->desired_mode = mode;
 635	musb_writew(epio, MUSB_TXCSR, csr);
 636#else
 637	if (!is_cppi_enabled() && !tusb_dma_omap())
 638		return false;
 639
 640	channel->actual_len = 0;
 641
 642	/*
 643	 * TX uses "RNDIS" mode automatically but needs help
 644	 * to identify the zero-length-final-packet case.
 645	 */
 646	mode = (urb->transfer_flags & URB_ZERO_PACKET) ? 1 : 0;
 647#endif
 648
 649	qh->segsize = length;
 650
 651	/*
 652	 * Ensure the data reaches to main memory before starting
 653	 * DMA transfer
 654	 */
 655	wmb();
 656
 657	if (!dma->channel_program(channel, pkt_size, mode,
 658			urb->transfer_dma + offset, length)) {
 659		dma->channel_release(channel);
 660		hw_ep->tx_channel = NULL;
 661
 662		csr = musb_readw(epio, MUSB_TXCSR);
 663		csr &= ~(MUSB_TXCSR_AUTOSET | MUSB_TXCSR_DMAENAB);
 664		musb_writew(epio, MUSB_TXCSR, csr | MUSB_TXCSR_H_WZC_BITS);
 665		return false;
 666	}
 667	return true;
 668}
 669
 670/*
 671 * Program an HDRC endpoint as per the given URB
 672 * Context: irqs blocked, controller lock held
 673 */
 674static void musb_ep_program(struct musb *musb, u8 epnum,
 675			struct urb *urb, int is_out,
 676			u8 *buf, u32 offset, u32 len)
 677{
 678	struct dma_controller	*dma_controller;
 679	struct dma_channel	*dma_channel;
 680	u8			dma_ok;
 681	void __iomem		*mbase = musb->mregs;
 682	struct musb_hw_ep	*hw_ep = musb->endpoints + epnum;
 683	void __iomem		*epio = hw_ep->regs;
 684	struct musb_qh		*qh = musb_ep_get_qh(hw_ep, !is_out);
 685	u16			packet_sz = qh->maxpacket;
 686
 687	dev_dbg(musb->controller, "%s hw%d urb %p spd%d dev%d ep%d%s "
 688				"h_addr%02x h_port%02x bytes %d\n",
 689			is_out ? "-->" : "<--",
 690			epnum, urb, urb->dev->speed,
 691			qh->addr_reg, qh->epnum, is_out ? "out" : "in",
 692			qh->h_addr_reg, qh->h_port_reg,
 693			len);
 694
 695	musb_ep_select(mbase, epnum);
 696
 697	/* candidate for DMA? */
 698	dma_controller = musb->dma_controller;
 699	if (is_dma_capable() && epnum && dma_controller) {
 700		dma_channel = is_out ? hw_ep->tx_channel : hw_ep->rx_channel;
 701		if (!dma_channel) {
 702			dma_channel = dma_controller->channel_alloc(
 703					dma_controller, hw_ep, is_out);
 704			if (is_out)
 705				hw_ep->tx_channel = dma_channel;
 706			else
 707				hw_ep->rx_channel = dma_channel;
 708		}
 709	} else
 710		dma_channel = NULL;
 711
 712	/* make sure we clear DMAEnab, autoSet bits from previous run */
 713
 714	/* OUT/transmit/EP0 or IN/receive? */
 715	if (is_out) {
 716		u16	csr;
 717		u16	int_txe;
 718		u16	load_count;
 719
 720		csr = musb_readw(epio, MUSB_TXCSR);
 721
 722		/* disable interrupt in case we flush */
 723		int_txe = musb_readw(mbase, MUSB_INTRTXE);
 724		musb_writew(mbase, MUSB_INTRTXE, int_txe & ~(1 << epnum));
 725
 726		/* general endpoint setup */
 727		if (epnum) {
 728			/* flush all old state, set default */
 729			musb_h_tx_flush_fifo(hw_ep);
 730
 731			/*
 732			 * We must not clear the DMAMODE bit before or in
 733			 * the same cycle with the DMAENAB bit, so we clear
 734			 * the latter first...
 735			 */
 736			csr &= ~(MUSB_TXCSR_H_NAKTIMEOUT
 737					| MUSB_TXCSR_AUTOSET
 738					| MUSB_TXCSR_DMAENAB
 739					| MUSB_TXCSR_FRCDATATOG
 740					| MUSB_TXCSR_H_RXSTALL
 741					| MUSB_TXCSR_H_ERROR
 742					| MUSB_TXCSR_TXPKTRDY
 743					);
 744			csr |= MUSB_TXCSR_MODE;
 745
 746			if (usb_gettoggle(urb->dev, qh->epnum, 1))
 747				csr |= MUSB_TXCSR_H_WR_DATATOGGLE
 748					| MUSB_TXCSR_H_DATATOGGLE;
 749			else
 750				csr |= MUSB_TXCSR_CLRDATATOG;
 751
 752			musb_writew(epio, MUSB_TXCSR, csr);
 753			/* REVISIT may need to clear FLUSHFIFO ... */
 754			csr &= ~MUSB_TXCSR_DMAMODE;
 755			musb_writew(epio, MUSB_TXCSR, csr);
 756			csr = musb_readw(epio, MUSB_TXCSR);
 757		} else {
 758			/* endpoint 0: just flush */
 759			musb_h_ep0_flush_fifo(hw_ep);
 760		}
 761
 762		/* target addr and (for multipoint) hub addr/port */
 763		if (musb->is_multipoint) {
 764			musb_write_txfunaddr(mbase, epnum, qh->addr_reg);
 765			musb_write_txhubaddr(mbase, epnum, qh->h_addr_reg);
 766			musb_write_txhubport(mbase, epnum, qh->h_port_reg);
 767/* FIXME if !epnum, do the same for RX ... */
 768		} else
 769			musb_writeb(mbase, MUSB_FADDR, qh->addr_reg);
 770
 771		/* protocol/endpoint/interval/NAKlimit */
 772		if (epnum) {
 773			musb_writeb(epio, MUSB_TXTYPE, qh->type_reg);
 774			if (musb->double_buffer_not_ok)
 775				musb_writew(epio, MUSB_TXMAXP,
 776						hw_ep->max_packet_sz_tx);
 
 
 
 
 777			else
 778				musb_writew(epio, MUSB_TXMAXP,
 779						qh->maxpacket |
 780						((qh->hb_mult - 1) << 11));
 781			musb_writeb(epio, MUSB_TXINTERVAL, qh->intv_reg);
 782		} else {
 783			musb_writeb(epio, MUSB_NAKLIMIT0, qh->intv_reg);
 784			if (musb->is_multipoint)
 785				musb_writeb(epio, MUSB_TYPE0,
 786						qh->type_reg);
 787		}
 788
 789		if (can_bulk_split(musb, qh->type))
 790			load_count = min((u32) hw_ep->max_packet_sz_tx,
 791						len);
 792		else
 793			load_count = min((u32) packet_sz, len);
 794
 795		if (dma_channel && musb_tx_dma_program(dma_controller,
 796					hw_ep, qh, urb, offset, len))
 797			load_count = 0;
 798
 799		if (load_count) {
 800			/* PIO to load FIFO */
 801			qh->segsize = load_count;
 802			musb_write_fifo(hw_ep, load_count, buf);
 803		}
 804
 805		/* re-enable interrupt */
 806		musb_writew(mbase, MUSB_INTRTXE, int_txe);
 807
 808	/* IN/receive */
 809	} else {
 810		u16	csr;
 811
 812		if (hw_ep->rx_reinit) {
 813			musb_rx_reinit(musb, qh, hw_ep);
 814
 815			/* init new state: toggle and NYET, maybe DMA later */
 816			if (usb_gettoggle(urb->dev, qh->epnum, 0))
 817				csr = MUSB_RXCSR_H_WR_DATATOGGLE
 818					| MUSB_RXCSR_H_DATATOGGLE;
 819			else
 820				csr = 0;
 821			if (qh->type == USB_ENDPOINT_XFER_INT)
 822				csr |= MUSB_RXCSR_DISNYET;
 823
 824		} else {
 825			csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
 826
 827			if (csr & (MUSB_RXCSR_RXPKTRDY
 828					| MUSB_RXCSR_DMAENAB
 829					| MUSB_RXCSR_H_REQPKT))
 830				ERR("broken !rx_reinit, ep%d csr %04x\n",
 831						hw_ep->epnum, csr);
 832
 833			/* scrub any stale state, leaving toggle alone */
 834			csr &= MUSB_RXCSR_DISNYET;
 835		}
 836
 837		/* kick things off */
 838
 839		if ((is_cppi_enabled() || tusb_dma_omap()) && dma_channel) {
 840			/* Candidate for DMA */
 841			dma_channel->actual_len = 0L;
 842			qh->segsize = len;
 843
 844			/* AUTOREQ is in a DMA register */
 845			musb_writew(hw_ep->regs, MUSB_RXCSR, csr);
 846			csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
 847
 848			/*
 849			 * Unless caller treats short RX transfers as
 850			 * errors, we dare not queue multiple transfers.
 851			 */
 852			dma_ok = dma_controller->channel_program(dma_channel,
 853					packet_sz, !(urb->transfer_flags &
 854						     URB_SHORT_NOT_OK),
 855					urb->transfer_dma + offset,
 856					qh->segsize);
 857			if (!dma_ok) {
 858				dma_controller->channel_release(dma_channel);
 859				hw_ep->rx_channel = dma_channel = NULL;
 860			} else
 861				csr |= MUSB_RXCSR_DMAENAB;
 862		}
 863
 864		csr |= MUSB_RXCSR_H_REQPKT;
 865		dev_dbg(musb->controller, "RXCSR%d := %04x\n", epnum, csr);
 866		musb_writew(hw_ep->regs, MUSB_RXCSR, csr);
 867		csr = musb_readw(hw_ep->regs, MUSB_RXCSR);
 868	}
 869}
 870
 871
 872/*
 873 * Service the default endpoint (ep0) as host.
 874 * Return true until it's time to start the status stage.
 875 */
 876static bool musb_h_ep0_continue(struct musb *musb, u16 len, struct urb *urb)
 877{
 878	bool			 more = false;
 879	u8			*fifo_dest = NULL;
 880	u16			fifo_count = 0;
 881	struct musb_hw_ep	*hw_ep = musb->control_ep;
 882	struct musb_qh		*qh = hw_ep->in_qh;
 883	struct usb_ctrlrequest	*request;
 884
 885	switch (musb->ep0_stage) {
 886	case MUSB_EP0_IN:
 887		fifo_dest = urb->transfer_buffer + urb->actual_length;
 888		fifo_count = min_t(size_t, len, urb->transfer_buffer_length -
 889				   urb->actual_length);
 890		if (fifo_count < len)
 891			urb->status = -EOVERFLOW;
 892
 893		musb_read_fifo(hw_ep, fifo_count, fifo_dest);
 894
 895		urb->actual_length += fifo_count;
 896		if (len < qh->maxpacket) {
 897			/* always terminate on short read; it's
 898			 * rarely reported as an error.
 899			 */
 900		} else if (urb->actual_length <
 901				urb->transfer_buffer_length)
 902			more = true;
 903		break;
 904	case MUSB_EP0_START:
 905		request = (struct usb_ctrlrequest *) urb->setup_packet;
 906
 907		if (!request->wLength) {
 908			dev_dbg(musb->controller, "start no-DATA\n");
 909			break;
 910		} else if (request->bRequestType & USB_DIR_IN) {
 911			dev_dbg(musb->controller, "start IN-DATA\n");
 912			musb->ep0_stage = MUSB_EP0_IN;
 913			more = true;
 914			break;
 915		} else {
 916			dev_dbg(musb->controller, "start OUT-DATA\n");
 917			musb->ep0_stage = MUSB_EP0_OUT;
 918			more = true;
 919		}
 920		/* FALLTHROUGH */
 921	case MUSB_EP0_OUT:
 922		fifo_count = min_t(size_t, qh->maxpacket,
 923				   urb->transfer_buffer_length -
 924				   urb->actual_length);
 925		if (fifo_count) {
 926			fifo_dest = (u8 *) (urb->transfer_buffer
 927					+ urb->actual_length);
 928			dev_dbg(musb->controller, "Sending %d byte%s to ep0 fifo %p\n",
 929					fifo_count,
 930					(fifo_count == 1) ? "" : "s",
 931					fifo_dest);
 932			musb_write_fifo(hw_ep, fifo_count, fifo_dest);
 933
 934			urb->actual_length += fifo_count;
 935			more = true;
 936		}
 937		break;
 938	default:
 939		ERR("bogus ep0 stage %d\n", musb->ep0_stage);
 940		break;
 941	}
 942
 943	return more;
 944}
 945
 946/*
 947 * Handle default endpoint interrupt as host. Only called in IRQ time
 948 * from musb_interrupt().
 949 *
 950 * called with controller irqlocked
 951 */
 952irqreturn_t musb_h_ep0_irq(struct musb *musb)
 953{
 954	struct urb		*urb;
 955	u16			csr, len;
 956	int			status = 0;
 957	void __iomem		*mbase = musb->mregs;
 958	struct musb_hw_ep	*hw_ep = musb->control_ep;
 959	void __iomem		*epio = hw_ep->regs;
 960	struct musb_qh		*qh = hw_ep->in_qh;
 961	bool			complete = false;
 962	irqreturn_t		retval = IRQ_NONE;
 963
 964	/* ep0 only has one queue, "in" */
 965	urb = next_urb(qh);
 966
 967	musb_ep_select(mbase, 0);
 968	csr = musb_readw(epio, MUSB_CSR0);
 969	len = (csr & MUSB_CSR0_RXPKTRDY)
 970			? musb_readb(epio, MUSB_COUNT0)
 971			: 0;
 972
 973	dev_dbg(musb->controller, "<== csr0 %04x, qh %p, count %d, urb %p, stage %d\n",
 974		csr, qh, len, urb, musb->ep0_stage);
 975
 976	/* if we just did status stage, we are done */
 977	if (MUSB_EP0_STATUS == musb->ep0_stage) {
 978		retval = IRQ_HANDLED;
 979		complete = true;
 980	}
 981
 982	/* prepare status */
 983	if (csr & MUSB_CSR0_H_RXSTALL) {
 984		dev_dbg(musb->controller, "STALLING ENDPOINT\n");
 985		status = -EPIPE;
 986
 987	} else if (csr & MUSB_CSR0_H_ERROR) {
 988		dev_dbg(musb->controller, "no response, csr0 %04x\n", csr);
 989		status = -EPROTO;
 990
 991	} else if (csr & MUSB_CSR0_H_NAKTIMEOUT) {
 992		dev_dbg(musb->controller, "control NAK timeout\n");
 993
 994		/* NOTE:  this code path would be a good place to PAUSE a
 995		 * control transfer, if another one is queued, so that
 996		 * ep0 is more likely to stay busy.  That's already done
 997		 * for bulk RX transfers.
 998		 *
 999		 * if (qh->ring.next != &musb->control), then
1000		 * we have a candidate... NAKing is *NOT* an error
1001		 */
1002		musb_writew(epio, MUSB_CSR0, 0);
1003		retval = IRQ_HANDLED;
1004	}
1005
1006	if (status) {
1007		dev_dbg(musb->controller, "aborting\n");
1008		retval = IRQ_HANDLED;
1009		if (urb)
1010			urb->status = status;
1011		complete = true;
1012
1013		/* use the proper sequence to abort the transfer */
1014		if (csr & MUSB_CSR0_H_REQPKT) {
1015			csr &= ~MUSB_CSR0_H_REQPKT;
1016			musb_writew(epio, MUSB_CSR0, csr);
1017			csr &= ~MUSB_CSR0_H_NAKTIMEOUT;
1018			musb_writew(epio, MUSB_CSR0, csr);
1019		} else {
1020			musb_h_ep0_flush_fifo(hw_ep);
1021		}
1022
1023		musb_writeb(epio, MUSB_NAKLIMIT0, 0);
1024
1025		/* clear it */
1026		musb_writew(epio, MUSB_CSR0, 0);
1027	}
1028
1029	if (unlikely(!urb)) {
1030		/* stop endpoint since we have no place for its data, this
1031		 * SHOULD NEVER HAPPEN! */
1032		ERR("no URB for end 0\n");
1033
1034		musb_h_ep0_flush_fifo(hw_ep);
1035		goto done;
1036	}
1037
1038	if (!complete) {
1039		/* call common logic and prepare response */
1040		if (musb_h_ep0_continue(musb, len, urb)) {
1041			/* more packets required */
1042			csr = (MUSB_EP0_IN == musb->ep0_stage)
1043				?  MUSB_CSR0_H_REQPKT : MUSB_CSR0_TXPKTRDY;
1044		} else {
1045			/* data transfer complete; perform status phase */
1046			if (usb_pipeout(urb->pipe)
1047					|| !urb->transfer_buffer_length)
1048				csr = MUSB_CSR0_H_STATUSPKT
1049					| MUSB_CSR0_H_REQPKT;
1050			else
1051				csr = MUSB_CSR0_H_STATUSPKT
1052					| MUSB_CSR0_TXPKTRDY;
1053
1054			/* flag status stage */
1055			musb->ep0_stage = MUSB_EP0_STATUS;
1056
1057			dev_dbg(musb->controller, "ep0 STATUS, csr %04x\n", csr);
1058
1059		}
1060		musb_writew(epio, MUSB_CSR0, csr);
1061		retval = IRQ_HANDLED;
1062	} else
1063		musb->ep0_stage = MUSB_EP0_IDLE;
1064
1065	/* call completion handler if done */
1066	if (complete)
1067		musb_advance_schedule(musb, urb, hw_ep, 1);
1068done:
1069	return retval;
1070}
1071
1072
1073#ifdef CONFIG_USB_INVENTRA_DMA
1074
1075/* Host side TX (OUT) using Mentor DMA works as follows:
1076	submit_urb ->
1077		- if queue was empty, Program Endpoint
1078		- ... which starts DMA to fifo in mode 1 or 0
1079
1080	DMA Isr (transfer complete) -> TxAvail()
1081		- Stop DMA (~DmaEnab)	(<--- Alert ... currently happens
1082					only in musb_cleanup_urb)
1083		- TxPktRdy has to be set in mode 0 or for
1084			short packets in mode 1.
1085*/
1086
1087#endif
1088
1089/* Service a Tx-Available or dma completion irq for the endpoint */
1090void musb_host_tx(struct musb *musb, u8 epnum)
1091{
1092	int			pipe;
1093	bool			done = false;
1094	u16			tx_csr;
1095	size_t			length = 0;
1096	size_t			offset = 0;
1097	struct musb_hw_ep	*hw_ep = musb->endpoints + epnum;
1098	void __iomem		*epio = hw_ep->regs;
1099	struct musb_qh		*qh = hw_ep->out_qh;
1100	struct urb		*urb = next_urb(qh);
1101	u32			status = 0;
1102	void __iomem		*mbase = musb->mregs;
1103	struct dma_channel	*dma;
1104	bool			transfer_pending = false;
1105
1106	musb_ep_select(mbase, epnum);
1107	tx_csr = musb_readw(epio, MUSB_TXCSR);
1108
1109	/* with CPPI, DMA sometimes triggers "extra" irqs */
1110	if (!urb) {
1111		dev_dbg(musb->controller, "extra TX%d ready, csr %04x\n", epnum, tx_csr);
1112		return;
1113	}
1114
1115	pipe = urb->pipe;
1116	dma = is_dma_capable() ? hw_ep->tx_channel : NULL;
1117	dev_dbg(musb->controller, "OUT/TX%d end, csr %04x%s\n", epnum, tx_csr,
1118			dma ? ", dma" : "");
1119
1120	/* check for errors */
1121	if (tx_csr & MUSB_TXCSR_H_RXSTALL) {
1122		/* dma was disabled, fifo flushed */
1123		dev_dbg(musb->controller, "TX end %d stall\n", epnum);
1124
1125		/* stall; record URB status */
1126		status = -EPIPE;
1127
1128	} else if (tx_csr & MUSB_TXCSR_H_ERROR) {
1129		/* (NON-ISO) dma was disabled, fifo flushed */
1130		dev_dbg(musb->controller, "TX 3strikes on ep=%d\n", epnum);
1131
1132		status = -ETIMEDOUT;
1133
1134	} else if (tx_csr & MUSB_TXCSR_H_NAKTIMEOUT) {
1135		dev_dbg(musb->controller, "TX end=%d device not responding\n", epnum);
1136
1137		/* NOTE:  this code path would be a good place to PAUSE a
1138		 * transfer, if there's some other (nonperiodic) tx urb
1139		 * that could use this fifo.  (dma complicates it...)
1140		 * That's already done for bulk RX transfers.
1141		 *
1142		 * if (bulk && qh->ring.next != &musb->out_bulk), then
1143		 * we have a candidate... NAKing is *NOT* an error
1144		 */
1145		musb_ep_select(mbase, epnum);
1146		musb_writew(epio, MUSB_TXCSR,
1147				MUSB_TXCSR_H_WZC_BITS
1148				| MUSB_TXCSR_TXPKTRDY);
1149		return;
1150	}
1151
1152	if (status) {
1153		if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
1154			dma->status = MUSB_DMA_STATUS_CORE_ABORT;
1155			(void) musb->dma_controller->channel_abort(dma);
1156		}
1157
1158		/* do the proper sequence to abort the transfer in the
1159		 * usb core; the dma engine should already be stopped.
1160		 */
1161		musb_h_tx_flush_fifo(hw_ep);
1162		tx_csr &= ~(MUSB_TXCSR_AUTOSET
1163				| MUSB_TXCSR_DMAENAB
1164				| MUSB_TXCSR_H_ERROR
1165				| MUSB_TXCSR_H_RXSTALL
1166				| MUSB_TXCSR_H_NAKTIMEOUT
1167				);
1168
1169		musb_ep_select(mbase, epnum);
1170		musb_writew(epio, MUSB_TXCSR, tx_csr);
1171		/* REVISIT may need to clear FLUSHFIFO ... */
1172		musb_writew(epio, MUSB_TXCSR, tx_csr);
1173		musb_writeb(epio, MUSB_TXINTERVAL, 0);
1174
1175		done = true;
1176	}
1177
1178	/* second cppi case */
1179	if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
1180		dev_dbg(musb->controller, "extra TX%d ready, csr %04x\n", epnum, tx_csr);
1181		return;
1182	}
1183
1184	if (is_dma_capable() && dma && !status) {
1185		/*
1186		 * DMA has completed.  But if we're using DMA mode 1 (multi
1187		 * packet DMA), we need a terminal TXPKTRDY interrupt before
1188		 * we can consider this transfer completed, lest we trash
1189		 * its last packet when writing the next URB's data.  So we
1190		 * switch back to mode 0 to get that interrupt; we'll come
1191		 * back here once it happens.
1192		 */
1193		if (tx_csr & MUSB_TXCSR_DMAMODE) {
1194			/*
1195			 * We shouldn't clear DMAMODE with DMAENAB set; so
1196			 * clear them in a safe order.  That should be OK
1197			 * once TXPKTRDY has been set (and I've never seen
1198			 * it being 0 at this moment -- DMA interrupt latency
1199			 * is significant) but if it hasn't been then we have
1200			 * no choice but to stop being polite and ignore the
1201			 * programmer's guide... :-)
1202			 *
1203			 * Note that we must write TXCSR with TXPKTRDY cleared
1204			 * in order not to re-trigger the packet send (this bit
1205			 * can't be cleared by CPU), and there's another caveat:
1206			 * TXPKTRDY may be set shortly and then cleared in the
1207			 * double-buffered FIFO mode, so we do an extra TXCSR
1208			 * read for debouncing...
1209			 */
1210			tx_csr &= musb_readw(epio, MUSB_TXCSR);
1211			if (tx_csr & MUSB_TXCSR_TXPKTRDY) {
1212				tx_csr &= ~(MUSB_TXCSR_DMAENAB |
1213					    MUSB_TXCSR_TXPKTRDY);
1214				musb_writew(epio, MUSB_TXCSR,
1215					    tx_csr | MUSB_TXCSR_H_WZC_BITS);
1216			}
1217			tx_csr &= ~(MUSB_TXCSR_DMAMODE |
1218				    MUSB_TXCSR_TXPKTRDY);
1219			musb_writew(epio, MUSB_TXCSR,
1220				    tx_csr | MUSB_TXCSR_H_WZC_BITS);
1221
1222			/*
1223			 * There is no guarantee that we'll get an interrupt
1224			 * after clearing DMAMODE as we might have done this
1225			 * too late (after TXPKTRDY was cleared by controller).
1226			 * Re-read TXCSR as we have spoiled its previous value.
1227			 */
1228			tx_csr = musb_readw(epio, MUSB_TXCSR);
1229		}
1230
1231		/*
1232		 * We may get here from a DMA completion or TXPKTRDY interrupt.
1233		 * In any case, we must check the FIFO status here and bail out
1234		 * only if the FIFO still has data -- that should prevent the
1235		 * "missed" TXPKTRDY interrupts and deal with double-buffered
1236		 * FIFO mode too...
1237		 */
1238		if (tx_csr & (MUSB_TXCSR_FIFONOTEMPTY | MUSB_TXCSR_TXPKTRDY)) {
1239			dev_dbg(musb->controller, "DMA complete but packet still in FIFO, "
1240			    "CSR %04x\n", tx_csr);
1241			return;
1242		}
1243	}
1244
1245	if (!status || dma || usb_pipeisoc(pipe)) {
1246		if (dma)
1247			length = dma->actual_len;
1248		else
1249			length = qh->segsize;
1250		qh->offset += length;
1251
1252		if (usb_pipeisoc(pipe)) {
1253			struct usb_iso_packet_descriptor	*d;
1254
1255			d = urb->iso_frame_desc + qh->iso_idx;
1256			d->actual_length = length;
1257			d->status = status;
1258			if (++qh->iso_idx >= urb->number_of_packets) {
1259				done = true;
1260			} else {
1261				d++;
1262				offset = d->offset;
1263				length = d->length;
1264			}
1265		} else if (dma && urb->transfer_buffer_length == qh->offset) {
1266			done = true;
1267		} else {
1268			/* see if we need to send more data, or ZLP */
1269			if (qh->segsize < qh->maxpacket)
1270				done = true;
1271			else if (qh->offset == urb->transfer_buffer_length
1272					&& !(urb->transfer_flags
1273						& URB_ZERO_PACKET))
1274				done = true;
1275			if (!done) {
1276				offset = qh->offset;
1277				length = urb->transfer_buffer_length - offset;
1278				transfer_pending = true;
1279			}
1280		}
1281	}
1282
1283	/* urb->status != -EINPROGRESS means request has been faulted,
1284	 * so we must abort this transfer after cleanup
1285	 */
1286	if (urb->status != -EINPROGRESS) {
1287		done = true;
1288		if (status == 0)
1289			status = urb->status;
1290	}
1291
1292	if (done) {
1293		/* set status */
1294		urb->status = status;
1295		urb->actual_length = qh->offset;
1296		musb_advance_schedule(musb, urb, hw_ep, USB_DIR_OUT);
1297		return;
1298	} else if ((usb_pipeisoc(pipe) || transfer_pending) && dma) {
1299		if (musb_tx_dma_program(musb->dma_controller, hw_ep, qh, urb,
1300				offset, length)) {
1301			if (is_cppi_enabled() || tusb_dma_omap())
1302				musb_h_tx_dma_start(hw_ep);
1303			return;
1304		}
1305	} else	if (tx_csr & MUSB_TXCSR_DMAENAB) {
1306		dev_dbg(musb->controller, "not complete, but DMA enabled?\n");
1307		return;
1308	}
1309
1310	/*
1311	 * PIO: start next packet in this URB.
1312	 *
1313	 * REVISIT: some docs say that when hw_ep->tx_double_buffered,
1314	 * (and presumably, FIFO is not half-full) we should write *two*
1315	 * packets before updating TXCSR; other docs disagree...
1316	 */
1317	if (length > qh->maxpacket)
1318		length = qh->maxpacket;
1319	/* Unmap the buffer so that CPU can use it */
1320	usb_hcd_unmap_urb_for_dma(musb_to_hcd(musb), urb);
1321	musb_write_fifo(hw_ep, length, urb->transfer_buffer + offset);
1322	qh->segsize = length;
1323
1324	musb_ep_select(mbase, epnum);
1325	musb_writew(epio, MUSB_TXCSR,
1326			MUSB_TXCSR_H_WZC_BITS | MUSB_TXCSR_TXPKTRDY);
1327}
1328
1329
1330#ifdef CONFIG_USB_INVENTRA_DMA
1331
1332/* Host side RX (IN) using Mentor DMA works as follows:
1333	submit_urb ->
1334		- if queue was empty, ProgramEndpoint
1335		- first IN token is sent out (by setting ReqPkt)
1336	LinuxIsr -> RxReady()
1337	/\	=> first packet is received
1338	|	- Set in mode 0 (DmaEnab, ~ReqPkt)
1339	|		-> DMA Isr (transfer complete) -> RxReady()
1340	|		    - Ack receive (~RxPktRdy), turn off DMA (~DmaEnab)
1341	|		    - if urb not complete, send next IN token (ReqPkt)
1342	|			   |		else complete urb.
1343	|			   |
1344	---------------------------
1345 *
1346 * Nuances of mode 1:
1347 *	For short packets, no ack (+RxPktRdy) is sent automatically
1348 *	(even if AutoClear is ON)
1349 *	For full packets, ack (~RxPktRdy) and next IN token (+ReqPkt) is sent
1350 *	automatically => major problem, as collecting the next packet becomes
1351 *	difficult. Hence mode 1 is not used.
1352 *
1353 * REVISIT
1354 *	All we care about at this driver level is that
1355 *       (a) all URBs terminate with REQPKT cleared and fifo(s) empty;
1356 *       (b) termination conditions are: short RX, or buffer full;
1357 *       (c) fault modes include
1358 *           - iff URB_SHORT_NOT_OK, short RX status is -EREMOTEIO.
1359 *             (and that endpoint's dma queue stops immediately)
1360 *           - overflow (full, PLUS more bytes in the terminal packet)
1361 *
1362 *	So for example, usb-storage sets URB_SHORT_NOT_OK, and would
1363 *	thus be a great candidate for using mode 1 ... for all but the
1364 *	last packet of one URB's transfer.
1365 */
1366
1367#endif
1368
1369/* Schedule next QH from musb->in_bulk and move the current qh to
1370 * the end; avoids starvation for other endpoints.
1371 */
1372static void musb_bulk_rx_nak_timeout(struct musb *musb, struct musb_hw_ep *ep)
1373{
1374	struct dma_channel	*dma;
1375	struct urb		*urb;
1376	void __iomem		*mbase = musb->mregs;
1377	void __iomem		*epio = ep->regs;
1378	struct musb_qh		*cur_qh, *next_qh;
1379	u16			rx_csr;
1380
1381	musb_ep_select(mbase, ep->epnum);
1382	dma = is_dma_capable() ? ep->rx_channel : NULL;
1383
1384	/* clear nak timeout bit */
1385	rx_csr = musb_readw(epio, MUSB_RXCSR);
1386	rx_csr |= MUSB_RXCSR_H_WZC_BITS;
1387	rx_csr &= ~MUSB_RXCSR_DATAERROR;
1388	musb_writew(epio, MUSB_RXCSR, rx_csr);
1389
1390	cur_qh = first_qh(&musb->in_bulk);
1391	if (cur_qh) {
1392		urb = next_urb(cur_qh);
1393		if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
1394			dma->status = MUSB_DMA_STATUS_CORE_ABORT;
1395			musb->dma_controller->channel_abort(dma);
1396			urb->actual_length += dma->actual_len;
1397			dma->actual_len = 0L;
1398		}
1399		musb_save_toggle(cur_qh, 1, urb);
1400
1401		/* move cur_qh to end of queue */
1402		list_move_tail(&cur_qh->ring, &musb->in_bulk);
1403
1404		/* get the next qh from musb->in_bulk */
1405		next_qh = first_qh(&musb->in_bulk);
1406
1407		/* set rx_reinit and schedule the next qh */
1408		ep->rx_reinit = 1;
1409		musb_start_urb(musb, 1, next_qh);
1410	}
1411}
1412
1413/*
1414 * Service an RX interrupt for the given IN endpoint; docs cover bulk, iso,
1415 * and high-bandwidth IN transfer cases.
1416 */
1417void musb_host_rx(struct musb *musb, u8 epnum)
1418{
1419	struct urb		*urb;
1420	struct musb_hw_ep	*hw_ep = musb->endpoints + epnum;
1421	void __iomem		*epio = hw_ep->regs;
1422	struct musb_qh		*qh = hw_ep->in_qh;
1423	size_t			xfer_len;
1424	void __iomem		*mbase = musb->mregs;
1425	int			pipe;
1426	u16			rx_csr, val;
1427	bool			iso_err = false;
1428	bool			done = false;
1429	u32			status;
1430	struct dma_channel	*dma;
1431
1432	musb_ep_select(mbase, epnum);
1433
1434	urb = next_urb(qh);
1435	dma = is_dma_capable() ? hw_ep->rx_channel : NULL;
1436	status = 0;
1437	xfer_len = 0;
1438
1439	rx_csr = musb_readw(epio, MUSB_RXCSR);
1440	val = rx_csr;
1441
1442	if (unlikely(!urb)) {
1443		/* REVISIT -- THIS SHOULD NEVER HAPPEN ... but, at least
1444		 * usbtest #11 (unlinks) triggers it regularly, sometimes
1445		 * with fifo full.  (Only with DMA??)
1446		 */
1447		dev_dbg(musb->controller, "BOGUS RX%d ready, csr %04x, count %d\n", epnum, val,
1448			musb_readw(epio, MUSB_RXCOUNT));
1449		musb_h_flush_rxfifo(hw_ep, MUSB_RXCSR_CLRDATATOG);
1450		return;
1451	}
1452
1453	pipe = urb->pipe;
1454
1455	dev_dbg(musb->controller, "<== hw %d rxcsr %04x, urb actual %d (+dma %zu)\n",
1456		epnum, rx_csr, urb->actual_length,
1457		dma ? dma->actual_len : 0);
1458
1459	/* check for errors, concurrent stall & unlink is not really
1460	 * handled yet! */
1461	if (rx_csr & MUSB_RXCSR_H_RXSTALL) {
1462		dev_dbg(musb->controller, "RX end %d STALL\n", epnum);
1463
1464		/* stall; record URB status */
1465		status = -EPIPE;
1466
1467	} else if (rx_csr & MUSB_RXCSR_H_ERROR) {
1468		dev_dbg(musb->controller, "end %d RX proto error\n", epnum);
1469
1470		status = -EPROTO;
1471		musb_writeb(epio, MUSB_RXINTERVAL, 0);
1472
1473	} else if (rx_csr & MUSB_RXCSR_DATAERROR) {
1474
1475		if (USB_ENDPOINT_XFER_ISOC != qh->type) {
1476			dev_dbg(musb->controller, "RX end %d NAK timeout\n", epnum);
1477
1478			/* NOTE: NAKing is *NOT* an error, so we want to
1479			 * continue.  Except ... if there's a request for
1480			 * another QH, use that instead of starving it.
1481			 *
1482			 * Devices like Ethernet and serial adapters keep
1483			 * reads posted at all times, which will starve
1484			 * other devices without this logic.
1485			 */
1486			if (usb_pipebulk(urb->pipe)
1487					&& qh->mux == 1
1488					&& !list_is_singular(&musb->in_bulk)) {
1489				musb_bulk_rx_nak_timeout(musb, hw_ep);
1490				return;
1491			}
1492			musb_ep_select(mbase, epnum);
1493			rx_csr |= MUSB_RXCSR_H_WZC_BITS;
1494			rx_csr &= ~MUSB_RXCSR_DATAERROR;
1495			musb_writew(epio, MUSB_RXCSR, rx_csr);
1496
1497			goto finish;
1498		} else {
1499			dev_dbg(musb->controller, "RX end %d ISO data error\n", epnum);
1500			/* packet error reported later */
1501			iso_err = true;
1502		}
1503	} else if (rx_csr & MUSB_RXCSR_INCOMPRX) {
1504		dev_dbg(musb->controller, "end %d high bandwidth incomplete ISO packet RX\n",
1505				epnum);
1506		status = -EPROTO;
1507	}
1508
1509	/* faults abort the transfer */
1510	if (status) {
1511		/* clean up dma and collect transfer count */
1512		if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
1513			dma->status = MUSB_DMA_STATUS_CORE_ABORT;
1514			(void) musb->dma_controller->channel_abort(dma);
1515			xfer_len = dma->actual_len;
1516		}
1517		musb_h_flush_rxfifo(hw_ep, MUSB_RXCSR_CLRDATATOG);
1518		musb_writeb(epio, MUSB_RXINTERVAL, 0);
1519		done = true;
1520		goto finish;
1521	}
1522
1523	if (unlikely(dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY)) {
1524		/* SHOULD NEVER HAPPEN ... but at least DaVinci has done it */
1525		ERR("RX%d dma busy, csr %04x\n", epnum, rx_csr);
1526		goto finish;
1527	}
1528
1529	/* thorough shutdown for now ... given more precise fault handling
1530	 * and better queueing support, we might keep a DMA pipeline going
1531	 * while processing this irq for earlier completions.
1532	 */
1533
1534	/* FIXME this is _way_ too much in-line logic for Mentor DMA */
1535
1536#ifndef CONFIG_USB_INVENTRA_DMA
1537	if (rx_csr & MUSB_RXCSR_H_REQPKT)  {
1538		/* REVISIT this happened for a while on some short reads...
1539		 * the cleanup still needs investigation... looks bad...
1540		 * and also duplicates dma cleanup code above ... plus,
1541		 * shouldn't this be the "half full" double buffer case?
1542		 */
1543		if (dma_channel_status(dma) == MUSB_DMA_STATUS_BUSY) {
1544			dma->status = MUSB_DMA_STATUS_CORE_ABORT;
1545			(void) musb->dma_controller->channel_abort(dma);
1546			xfer_len = dma->actual_len;
1547			done = true;
1548		}
1549
1550		dev_dbg(musb->controller, "RXCSR%d %04x, reqpkt, len %zu%s\n", epnum, rx_csr,
1551				xfer_len, dma ? ", dma" : "");
1552		rx_csr &= ~MUSB_RXCSR_H_REQPKT;
1553
1554		musb_ep_select(mbase, epnum);
1555		musb_writew(epio, MUSB_RXCSR,
1556				MUSB_RXCSR_H_WZC_BITS | rx_csr);
1557	}
1558#endif
1559	if (dma && (rx_csr & MUSB_RXCSR_DMAENAB)) {
1560		xfer_len = dma->actual_len;
1561
1562		val &= ~(MUSB_RXCSR_DMAENAB
1563			| MUSB_RXCSR_H_AUTOREQ
1564			| MUSB_RXCSR_AUTOCLEAR
1565			| MUSB_RXCSR_RXPKTRDY);
1566		musb_writew(hw_ep->regs, MUSB_RXCSR, val);
1567
1568#ifdef CONFIG_USB_INVENTRA_DMA
1569		if (usb_pipeisoc(pipe)) {
1570			struct usb_iso_packet_descriptor *d;
1571
1572			d = urb->iso_frame_desc + qh->iso_idx;
1573			d->actual_length = xfer_len;
1574
1575			/* even if there was an error, we did the dma
1576			 * for iso_frame_desc->length
1577			 */
1578			if (d->status != -EILSEQ && d->status != -EOVERFLOW)
1579				d->status = 0;
1580
1581			if (++qh->iso_idx >= urb->number_of_packets)
1582				done = true;
1583			else
1584				done = false;
1585
1586		} else  {
1587		/* done if urb buffer is full or short packet is recd */
1588		done = (urb->actual_length + xfer_len >=
1589				urb->transfer_buffer_length
1590			|| dma->actual_len < qh->maxpacket);
1591		}
1592
1593		/* send IN token for next packet, without AUTOREQ */
1594		if (!done) {
1595			val |= MUSB_RXCSR_H_REQPKT;
1596			musb_writew(epio, MUSB_RXCSR,
1597				MUSB_RXCSR_H_WZC_BITS | val);
1598		}
1599
1600		dev_dbg(musb->controller, "ep %d dma %s, rxcsr %04x, rxcount %d\n", epnum,
1601			done ? "off" : "reset",
1602			musb_readw(epio, MUSB_RXCSR),
1603			musb_readw(epio, MUSB_RXCOUNT));
1604#else
1605		done = true;
1606#endif
1607	} else if (urb->status == -EINPROGRESS) {
1608		/* if no errors, be sure a packet is ready for unloading */
1609		if (unlikely(!(rx_csr & MUSB_RXCSR_RXPKTRDY))) {
1610			status = -EPROTO;
1611			ERR("Rx interrupt with no errors or packet!\n");
1612
1613			/* FIXME this is another "SHOULD NEVER HAPPEN" */
1614
1615/* SCRUB (RX) */
1616			/* do the proper sequence to abort the transfer */
1617			musb_ep_select(mbase, epnum);
1618			val &= ~MUSB_RXCSR_H_REQPKT;
1619			musb_writew(epio, MUSB_RXCSR, val);
1620			goto finish;
1621		}
1622
1623		/* we are expecting IN packets */
1624#ifdef CONFIG_USB_INVENTRA_DMA
1625		if (dma) {
1626			struct dma_controller	*c;
1627			u16			rx_count;
1628			int			ret, length;
1629			dma_addr_t		buf;
1630
1631			rx_count = musb_readw(epio, MUSB_RXCOUNT);
1632
1633			dev_dbg(musb->controller, "RX%d count %d, buffer 0x%x len %d/%d\n",
1634					epnum, rx_count,
1635					urb->transfer_dma
1636						+ urb->actual_length,
1637					qh->offset,
1638					urb->transfer_buffer_length);
1639
1640			c = musb->dma_controller;
1641
1642			if (usb_pipeisoc(pipe)) {
1643				int d_status = 0;
1644				struct usb_iso_packet_descriptor *d;
1645
1646				d = urb->iso_frame_desc + qh->iso_idx;
1647
1648				if (iso_err) {
1649					d_status = -EILSEQ;
1650					urb->error_count++;
1651				}
1652				if (rx_count > d->length) {
1653					if (d_status == 0) {
1654						d_status = -EOVERFLOW;
1655						urb->error_count++;
1656					}
1657					dev_dbg(musb->controller, "** OVERFLOW %d into %d\n",\
1658					    rx_count, d->length);
1659
1660					length = d->length;
1661				} else
1662					length = rx_count;
1663				d->status = d_status;
1664				buf = urb->transfer_dma + d->offset;
1665			} else {
1666				length = rx_count;
1667				buf = urb->transfer_dma +
1668						urb->actual_length;
1669			}
1670
1671			dma->desired_mode = 0;
1672#ifdef USE_MODE1
1673			/* because of the issue below, mode 1 will
1674			 * only rarely behave with correct semantics.
1675			 */
1676			if ((urb->transfer_flags &
1677						URB_SHORT_NOT_OK)
1678				&& (urb->transfer_buffer_length -
1679						urb->actual_length)
1680					> qh->maxpacket)
1681				dma->desired_mode = 1;
1682			if (rx_count < hw_ep->max_packet_sz_rx) {
1683				length = rx_count;
1684				dma->desired_mode = 0;
1685			} else {
1686				length = urb->transfer_buffer_length;
1687			}
1688#endif
1689
1690/* Disadvantage of using mode 1:
1691 *	It's basically usable only for mass storage class; essentially all
1692 *	other protocols also terminate transfers on short packets.
1693 *
1694 * Details:
1695 *	An extra IN token is sent at the end of the transfer (due to AUTOREQ)
1696 *	If you try to use mode 1 for (transfer_buffer_length - 512), and try
1697 *	to use the extra IN token to grab the last packet using mode 0, then
1698 *	the problem is that you cannot be sure when the device will send the
1699 *	last packet and RxPktRdy set. Sometimes the packet is recd too soon
1700 *	such that it gets lost when RxCSR is re-set at the end of the mode 1
1701 *	transfer, while sometimes it is recd just a little late so that if you
1702 *	try to configure for mode 0 soon after the mode 1 transfer is
1703 *	completed, you will find rxcount 0. Okay, so you might think why not
1704 *	wait for an interrupt when the pkt is recd. Well, you won't get any!
1705 */
1706
1707			val = musb_readw(epio, MUSB_RXCSR);
1708			val &= ~MUSB_RXCSR_H_REQPKT;
1709
1710			if (dma->desired_mode == 0)
1711				val &= ~MUSB_RXCSR_H_AUTOREQ;
1712			else
1713				val |= MUSB_RXCSR_H_AUTOREQ;
1714			val |= MUSB_RXCSR_DMAENAB;
1715
1716			/* autoclear shouldn't be set in high bandwidth */
1717			if (qh->hb_mult == 1)
1718				val |= MUSB_RXCSR_AUTOCLEAR;
1719
1720			musb_writew(epio, MUSB_RXCSR,
1721				MUSB_RXCSR_H_WZC_BITS | val);
1722
1723			/* REVISIT if when actual_length != 0,
1724			 * transfer_buffer_length needs to be
1725			 * adjusted first...
1726			 */
1727			ret = c->channel_program(
1728				dma, qh->maxpacket,
1729				dma->desired_mode, buf, length);
1730
1731			if (!ret) {
1732				c->channel_release(dma);
1733				hw_ep->rx_channel = NULL;
1734				dma = NULL;
1735				/* REVISIT reset CSR */
1736			}
1737		}
1738#endif	/* Mentor DMA */
1739
1740		if (!dma) {
1741			/* Unmap the buffer so that CPU can use it */
1742			usb_hcd_unmap_urb_for_dma(musb_to_hcd(musb), urb);
1743			done = musb_host_packet_rx(musb, urb,
1744					epnum, iso_err);
1745			dev_dbg(musb->controller, "read %spacket\n", done ? "last " : "");
1746		}
1747	}
1748
1749finish:
1750	urb->actual_length += xfer_len;
1751	qh->offset += xfer_len;
1752	if (done) {
1753		if (urb->status == -EINPROGRESS)
1754			urb->status = status;
1755		musb_advance_schedule(musb, urb, hw_ep, USB_DIR_IN);
1756	}
1757}
1758
1759/* schedule nodes correspond to peripheral endpoints, like an OHCI QH.
1760 * the software schedule associates multiple such nodes with a given
1761 * host side hardware endpoint + direction; scheduling may activate
1762 * that hardware endpoint.
1763 */
1764static int musb_schedule(
1765	struct musb		*musb,
1766	struct musb_qh		*qh,
1767	int			is_in)
1768{
1769	int			idle;
1770	int			best_diff;
1771	int			best_end, epnum;
1772	struct musb_hw_ep	*hw_ep = NULL;
1773	struct list_head	*head = NULL;
1774	u8			toggle;
1775	u8			txtype;
1776	struct urb		*urb = next_urb(qh);
1777
1778	/* use fixed hardware for control and bulk */
1779	if (qh->type == USB_ENDPOINT_XFER_CONTROL) {
1780		head = &musb->control;
1781		hw_ep = musb->control_ep;
1782		goto success;
1783	}
1784
1785	/* else, periodic transfers get muxed to other endpoints */
1786
1787	/*
1788	 * We know this qh hasn't been scheduled, so all we need to do
1789	 * is choose which hardware endpoint to put it on ...
1790	 *
1791	 * REVISIT what we really want here is a regular schedule tree
1792	 * like e.g. OHCI uses.
1793	 */
1794	best_diff = 4096;
1795	best_end = -1;
1796
1797	for (epnum = 1, hw_ep = musb->endpoints + 1;
1798			epnum < musb->nr_endpoints;
1799			epnum++, hw_ep++) {
1800		int	diff;
1801
1802		if (musb_ep_get_qh(hw_ep, is_in) != NULL)
1803			continue;
1804
1805		if (hw_ep == musb->bulk_ep)
1806			continue;
1807
1808		if (is_in)
1809			diff = hw_ep->max_packet_sz_rx;
1810		else
1811			diff = hw_ep->max_packet_sz_tx;
1812		diff -= (qh->maxpacket * qh->hb_mult);
1813
1814		if (diff >= 0 && best_diff > diff) {
1815
1816			/*
1817			 * Mentor controller has a bug in that if we schedule
1818			 * a BULK Tx transfer on an endpoint that had earlier
1819			 * handled ISOC then the BULK transfer has to start on
1820			 * a zero toggle.  If the BULK transfer starts on a 1
1821			 * toggle then this transfer will fail as the mentor
1822			 * controller starts the Bulk transfer on a 0 toggle
1823			 * irrespective of the programming of the toggle bits
1824			 * in the TXCSR register.  Check for this condition
1825			 * while allocating the EP for a Tx Bulk transfer.  If
1826			 * so skip this EP.
1827			 */
1828			hw_ep = musb->endpoints + epnum;
1829			toggle = usb_gettoggle(urb->dev, qh->epnum, !is_in);
1830			txtype = (musb_readb(hw_ep->regs, MUSB_TXTYPE)
1831					>> 4) & 0x3;
1832			if (!is_in && (qh->type == USB_ENDPOINT_XFER_BULK) &&
1833				toggle && (txtype == USB_ENDPOINT_XFER_ISOC))
1834				continue;
1835
1836			best_diff = diff;
1837			best_end = epnum;
1838		}
1839	}
1840	/* use bulk reserved ep1 if no other ep is free */
1841	if (best_end < 0 && qh->type == USB_ENDPOINT_XFER_BULK) {
1842		hw_ep = musb->bulk_ep;
1843		if (is_in)
1844			head = &musb->in_bulk;
1845		else
1846			head = &musb->out_bulk;
1847
1848		/* Enable bulk RX NAK timeout scheme when bulk requests are
1849		 * multiplexed.  This scheme doen't work in high speed to full
1850		 * speed scenario as NAK interrupts are not coming from a
1851		 * full speed device connected to a high speed device.
1852		 * NAK timeout interval is 8 (128 uframe or 16ms) for HS and
1853		 * 4 (8 frame or 8ms) for FS device.
1854		 */
1855		if (is_in && qh->dev)
1856			qh->intv_reg =
1857				(USB_SPEED_HIGH == qh->dev->speed) ? 8 : 4;
1858		goto success;
1859	} else if (best_end < 0) {
1860		return -ENOSPC;
1861	}
1862
1863	idle = 1;
1864	qh->mux = 0;
1865	hw_ep = musb->endpoints + best_end;
1866	dev_dbg(musb->controller, "qh %p periodic slot %d\n", qh, best_end);
1867success:
1868	if (head) {
1869		idle = list_empty(head);
1870		list_add_tail(&qh->ring, head);
1871		qh->mux = 1;
1872	}
1873	qh->hw_ep = hw_ep;
1874	qh->hep->hcpriv = qh;
1875	if (idle)
1876		musb_start_urb(musb, is_in, qh);
1877	return 0;
1878}
1879
1880static int musb_urb_enqueue(
1881	struct usb_hcd			*hcd,
1882	struct urb			*urb,
1883	gfp_t				mem_flags)
1884{
1885	unsigned long			flags;
1886	struct musb			*musb = hcd_to_musb(hcd);
1887	struct usb_host_endpoint	*hep = urb->ep;
1888	struct musb_qh			*qh;
1889	struct usb_endpoint_descriptor	*epd = &hep->desc;
1890	int				ret;
1891	unsigned			type_reg;
1892	unsigned			interval;
1893
1894	/* host role must be active */
1895	if (!is_host_active(musb) || !musb->is_active)
1896		return -ENODEV;
1897
1898	spin_lock_irqsave(&musb->lock, flags);
1899	ret = usb_hcd_link_urb_to_ep(hcd, urb);
1900	qh = ret ? NULL : hep->hcpriv;
1901	if (qh)
1902		urb->hcpriv = qh;
1903	spin_unlock_irqrestore(&musb->lock, flags);
1904
1905	/* DMA mapping was already done, if needed, and this urb is on
1906	 * hep->urb_list now ... so we're done, unless hep wasn't yet
1907	 * scheduled onto a live qh.
1908	 *
1909	 * REVISIT best to keep hep->hcpriv valid until the endpoint gets
1910	 * disabled, testing for empty qh->ring and avoiding qh setup costs
1911	 * except for the first urb queued after a config change.
1912	 */
1913	if (qh || ret)
1914		return ret;
1915
1916	/* Allocate and initialize qh, minimizing the work done each time
1917	 * hw_ep gets reprogrammed, or with irqs blocked.  Then schedule it.
1918	 *
1919	 * REVISIT consider a dedicated qh kmem_cache, so it's harder
1920	 * for bugs in other kernel code to break this driver...
1921	 */
1922	qh = kzalloc(sizeof *qh, mem_flags);
1923	if (!qh) {
1924		spin_lock_irqsave(&musb->lock, flags);
1925		usb_hcd_unlink_urb_from_ep(hcd, urb);
1926		spin_unlock_irqrestore(&musb->lock, flags);
1927		return -ENOMEM;
1928	}
1929
1930	qh->hep = hep;
1931	qh->dev = urb->dev;
1932	INIT_LIST_HEAD(&qh->ring);
1933	qh->is_ready = 1;
1934
1935	qh->maxpacket = le16_to_cpu(epd->wMaxPacketSize);
1936	qh->type = usb_endpoint_type(epd);
1937
1938	/* Bits 11 & 12 of wMaxPacketSize encode high bandwidth multiplier.
1939	 * Some musb cores don't support high bandwidth ISO transfers; and
1940	 * we don't (yet!) support high bandwidth interrupt transfers.
1941	 */
1942	qh->hb_mult = 1 + ((qh->maxpacket >> 11) & 0x03);
1943	if (qh->hb_mult > 1) {
1944		int ok = (qh->type == USB_ENDPOINT_XFER_ISOC);
1945
1946		if (ok)
1947			ok = (usb_pipein(urb->pipe) && musb->hb_iso_rx)
1948				|| (usb_pipeout(urb->pipe) && musb->hb_iso_tx);
1949		if (!ok) {
1950			ret = -EMSGSIZE;
1951			goto done;
1952		}
1953		qh->maxpacket &= 0x7ff;
1954	}
1955
1956	qh->epnum = usb_endpoint_num(epd);
1957
1958	/* NOTE: urb->dev->devnum is wrong during SET_ADDRESS */
1959	qh->addr_reg = (u8) usb_pipedevice(urb->pipe);
1960
1961	/* precompute rxtype/txtype/type0 register */
1962	type_reg = (qh->type << 4) | qh->epnum;
1963	switch (urb->dev->speed) {
1964	case USB_SPEED_LOW:
1965		type_reg |= 0xc0;
1966		break;
1967	case USB_SPEED_FULL:
1968		type_reg |= 0x80;
1969		break;
1970	default:
1971		type_reg |= 0x40;
1972	}
1973	qh->type_reg = type_reg;
1974
1975	/* Precompute RXINTERVAL/TXINTERVAL register */
1976	switch (qh->type) {
1977	case USB_ENDPOINT_XFER_INT:
1978		/*
1979		 * Full/low speeds use the  linear encoding,
1980		 * high speed uses the logarithmic encoding.
1981		 */
1982		if (urb->dev->speed <= USB_SPEED_FULL) {
1983			interval = max_t(u8, epd->bInterval, 1);
1984			break;
1985		}
1986		/* FALLTHROUGH */
1987	case USB_ENDPOINT_XFER_ISOC:
1988		/* ISO always uses logarithmic encoding */
1989		interval = min_t(u8, epd->bInterval, 16);
1990		break;
1991	default:
1992		/* REVISIT we actually want to use NAK limits, hinting to the
1993		 * transfer scheduling logic to try some other qh, e.g. try
1994		 * for 2 msec first:
1995		 *
1996		 * interval = (USB_SPEED_HIGH == urb->dev->speed) ? 16 : 2;
1997		 *
1998		 * The downside of disabling this is that transfer scheduling
1999		 * gets VERY unfair for nonperiodic transfers; a misbehaving
2000		 * peripheral could make that hurt.  That's perfectly normal
2001		 * for reads from network or serial adapters ... so we have
2002		 * partial NAKlimit support for bulk RX.
2003		 *
2004		 * The upside of disabling it is simpler transfer scheduling.
2005		 */
2006		interval = 0;
2007	}
2008	qh->intv_reg = interval;
2009
2010	/* precompute addressing for external hub/tt ports */
2011	if (musb->is_multipoint) {
2012		struct usb_device	*parent = urb->dev->parent;
2013
2014		if (parent != hcd->self.root_hub) {
2015			qh->h_addr_reg = (u8) parent->devnum;
2016
2017			/* set up tt info if needed */
2018			if (urb->dev->tt) {
2019				qh->h_port_reg = (u8) urb->dev->ttport;
2020				if (urb->dev->tt->hub)
2021					qh->h_addr_reg =
2022						(u8) urb->dev->tt->hub->devnum;
2023				if (urb->dev->tt->multi)
2024					qh->h_addr_reg |= 0x80;
2025			}
2026		}
2027	}
2028
2029	/* invariant: hep->hcpriv is null OR the qh that's already scheduled.
2030	 * until we get real dma queues (with an entry for each urb/buffer),
2031	 * we only have work to do in the former case.
2032	 */
2033	spin_lock_irqsave(&musb->lock, flags);
2034	if (hep->hcpriv) {
2035		/* some concurrent activity submitted another urb to hep...
2036		 * odd, rare, error prone, but legal.
2037		 */
2038		kfree(qh);
2039		qh = NULL;
2040		ret = 0;
2041	} else
2042		ret = musb_schedule(musb, qh,
2043				epd->bEndpointAddress & USB_ENDPOINT_DIR_MASK);
2044
2045	if (ret == 0) {
2046		urb->hcpriv = qh;
2047		/* FIXME set urb->start_frame for iso/intr, it's tested in
2048		 * musb_start_urb(), but otherwise only konicawc cares ...
2049		 */
2050	}
2051	spin_unlock_irqrestore(&musb->lock, flags);
2052
2053done:
2054	if (ret != 0) {
2055		spin_lock_irqsave(&musb->lock, flags);
2056		usb_hcd_unlink_urb_from_ep(hcd, urb);
2057		spin_unlock_irqrestore(&musb->lock, flags);
2058		kfree(qh);
2059	}
2060	return ret;
2061}
2062
2063
2064/*
2065 * abort a transfer that's at the head of a hardware queue.
2066 * called with controller locked, irqs blocked
2067 * that hardware queue advances to the next transfer, unless prevented
2068 */
2069static int musb_cleanup_urb(struct urb *urb, struct musb_qh *qh)
2070{
2071	struct musb_hw_ep	*ep = qh->hw_ep;
2072	struct musb		*musb = ep->musb;
2073	void __iomem		*epio = ep->regs;
2074	unsigned		hw_end = ep->epnum;
2075	void __iomem		*regs = ep->musb->mregs;
2076	int			is_in = usb_pipein(urb->pipe);
2077	int			status = 0;
2078	u16			csr;
2079
2080	musb_ep_select(regs, hw_end);
2081
2082	if (is_dma_capable()) {
2083		struct dma_channel	*dma;
2084
2085		dma = is_in ? ep->rx_channel : ep->tx_channel;
2086		if (dma) {
2087			status = ep->musb->dma_controller->channel_abort(dma);
2088			dev_dbg(musb->controller,
2089				"abort %cX%d DMA for urb %p --> %d\n",
2090				is_in ? 'R' : 'T', ep->epnum,
2091				urb, status);
2092			urb->actual_length += dma->actual_len;
2093		}
2094	}
2095
2096	/* turn off DMA requests, discard state, stop polling ... */
2097	if (is_in) {
2098		/* giveback saves bulk toggle */
2099		csr = musb_h_flush_rxfifo(ep, 0);
2100
2101		/* REVISIT we still get an irq; should likely clear the
2102		 * endpoint's irq status here to avoid bogus irqs.
2103		 * clearing that status is platform-specific...
2104		 */
2105	} else if (ep->epnum) {
2106		musb_h_tx_flush_fifo(ep);
2107		csr = musb_readw(epio, MUSB_TXCSR);
2108		csr &= ~(MUSB_TXCSR_AUTOSET
2109			| MUSB_TXCSR_DMAENAB
2110			| MUSB_TXCSR_H_RXSTALL
2111			| MUSB_TXCSR_H_NAKTIMEOUT
2112			| MUSB_TXCSR_H_ERROR
2113			| MUSB_TXCSR_TXPKTRDY);
2114		musb_writew(epio, MUSB_TXCSR, csr);
2115		/* REVISIT may need to clear FLUSHFIFO ... */
2116		musb_writew(epio, MUSB_TXCSR, csr);
2117		/* flush cpu writebuffer */
2118		csr = musb_readw(epio, MUSB_TXCSR);
2119	} else  {
2120		musb_h_ep0_flush_fifo(ep);
2121	}
2122	if (status == 0)
2123		musb_advance_schedule(ep->musb, urb, ep, is_in);
2124	return status;
2125}
2126
2127static int musb_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
2128{
2129	struct musb		*musb = hcd_to_musb(hcd);
2130	struct musb_qh		*qh;
2131	unsigned long		flags;
2132	int			is_in  = usb_pipein(urb->pipe);
2133	int			ret;
2134
2135	dev_dbg(musb->controller, "urb=%p, dev%d ep%d%s\n", urb,
2136			usb_pipedevice(urb->pipe),
2137			usb_pipeendpoint(urb->pipe),
2138			is_in ? "in" : "out");
2139
2140	spin_lock_irqsave(&musb->lock, flags);
2141	ret = usb_hcd_check_unlink_urb(hcd, urb, status);
2142	if (ret)
2143		goto done;
2144
2145	qh = urb->hcpriv;
2146	if (!qh)
2147		goto done;
2148
2149	/*
2150	 * Any URB not actively programmed into endpoint hardware can be
2151	 * immediately given back; that's any URB not at the head of an
2152	 * endpoint queue, unless someday we get real DMA queues.  And even
2153	 * if it's at the head, it might not be known to the hardware...
2154	 *
2155	 * Otherwise abort current transfer, pending DMA, etc.; urb->status
2156	 * has already been updated.  This is a synchronous abort; it'd be
2157	 * OK to hold off until after some IRQ, though.
2158	 *
2159	 * NOTE: qh is invalid unless !list_empty(&hep->urb_list)
2160	 */
2161	if (!qh->is_ready
2162			|| urb->urb_list.prev != &qh->hep->urb_list
2163			|| musb_ep_get_qh(qh->hw_ep, is_in) != qh) {
2164		int	ready = qh->is_ready;
2165
2166		qh->is_ready = 0;
2167		musb_giveback(musb, urb, 0);
2168		qh->is_ready = ready;
2169
2170		/* If nothing else (usually musb_giveback) is using it
2171		 * and its URB list has emptied, recycle this qh.
2172		 */
2173		if (ready && list_empty(&qh->hep->urb_list)) {
2174			qh->hep->hcpriv = NULL;
2175			list_del(&qh->ring);
2176			kfree(qh);
2177		}
2178	} else
2179		ret = musb_cleanup_urb(urb, qh);
2180done:
2181	spin_unlock_irqrestore(&musb->lock, flags);
2182	return ret;
2183}
2184
2185/* disable an endpoint */
2186static void
2187musb_h_disable(struct usb_hcd *hcd, struct usb_host_endpoint *hep)
2188{
2189	u8			is_in = hep->desc.bEndpointAddress & USB_DIR_IN;
2190	unsigned long		flags;
2191	struct musb		*musb = hcd_to_musb(hcd);
2192	struct musb_qh		*qh;
2193	struct urb		*urb;
2194
2195	spin_lock_irqsave(&musb->lock, flags);
2196
2197	qh = hep->hcpriv;
2198	if (qh == NULL)
2199		goto exit;
2200
2201	/* NOTE: qh is invalid unless !list_empty(&hep->urb_list) */
2202
2203	/* Kick the first URB off the hardware, if needed */
2204	qh->is_ready = 0;
2205	if (musb_ep_get_qh(qh->hw_ep, is_in) == qh) {
2206		urb = next_urb(qh);
2207
2208		/* make software (then hardware) stop ASAP */
2209		if (!urb->unlinked)
2210			urb->status = -ESHUTDOWN;
2211
2212		/* cleanup */
2213		musb_cleanup_urb(urb, qh);
2214
2215		/* Then nuke all the others ... and advance the
2216		 * queue on hw_ep (e.g. bulk ring) when we're done.
2217		 */
2218		while (!list_empty(&hep->urb_list)) {
2219			urb = next_urb(qh);
2220			urb->status = -ESHUTDOWN;
2221			musb_advance_schedule(musb, urb, qh->hw_ep, is_in);
2222		}
2223	} else {
2224		/* Just empty the queue; the hardware is busy with
2225		 * other transfers, and since !qh->is_ready nothing
2226		 * will activate any of these as it advances.
2227		 */
2228		while (!list_empty(&hep->urb_list))
2229			musb_giveback(musb, next_urb(qh), -ESHUTDOWN);
2230
2231		hep->hcpriv = NULL;
2232		list_del(&qh->ring);
2233		kfree(qh);
2234	}
2235exit:
2236	spin_unlock_irqrestore(&musb->lock, flags);
2237}
2238
2239static int musb_h_get_frame_number(struct usb_hcd *hcd)
2240{
2241	struct musb	*musb = hcd_to_musb(hcd);
2242
2243	return musb_readw(musb->mregs, MUSB_FRAME);
2244}
2245
2246static int musb_h_start(struct usb_hcd *hcd)
2247{
2248	struct musb	*musb = hcd_to_musb(hcd);
2249
2250	/* NOTE: musb_start() is called when the hub driver turns
2251	 * on port power, or when (OTG) peripheral starts.
2252	 */
2253	hcd->state = HC_STATE_RUNNING;
2254	musb->port1_status = 0;
2255	return 0;
2256}
2257
2258static void musb_h_stop(struct usb_hcd *hcd)
2259{
2260	musb_stop(hcd_to_musb(hcd));
2261	hcd->state = HC_STATE_HALT;
2262}
2263
2264static int musb_bus_suspend(struct usb_hcd *hcd)
2265{
2266	struct musb	*musb = hcd_to_musb(hcd);
2267	u8		devctl;
2268
2269	if (!is_host_active(musb))
2270		return 0;
2271
2272	switch (musb->xceiv->state) {
2273	case OTG_STATE_A_SUSPEND:
2274		return 0;
2275	case OTG_STATE_A_WAIT_VRISE:
2276		/* ID could be grounded even if there's no device
2277		 * on the other end of the cable.  NOTE that the
2278		 * A_WAIT_VRISE timers are messy with MUSB...
2279		 */
2280		devctl = musb_readb(musb->mregs, MUSB_DEVCTL);
2281		if ((devctl & MUSB_DEVCTL_VBUS) == MUSB_DEVCTL_VBUS)
2282			musb->xceiv->state = OTG_STATE_A_WAIT_BCON;
2283		break;
2284	default:
2285		break;
2286	}
2287
2288	if (musb->is_active) {
2289		WARNING("trying to suspend as %s while active\n",
2290				otg_state_string(musb->xceiv->state));
2291		return -EBUSY;
2292	} else
2293		return 0;
2294}
2295
2296static int musb_bus_resume(struct usb_hcd *hcd)
2297{
2298	/* resuming child port does the work */
2299	return 0;
2300}
2301
2302const struct hc_driver musb_hc_driver = {
2303	.description		= "musb-hcd",
2304	.product_desc		= "MUSB HDRC host driver",
2305	.hcd_priv_size		= sizeof(struct musb),
2306	.flags			= HCD_USB2 | HCD_MEMORY,
2307
2308	/* not using irq handler or reset hooks from usbcore, since
2309	 * those must be shared with peripheral code for OTG configs
2310	 */
2311
2312	.start			= musb_h_start,
2313	.stop			= musb_h_stop,
2314
2315	.get_frame_number	= musb_h_get_frame_number,
2316
2317	.urb_enqueue		= musb_urb_enqueue,
2318	.urb_dequeue		= musb_urb_dequeue,
2319	.endpoint_disable	= musb_h_disable,
2320
2321	.hub_status_data	= musb_hub_status_data,
2322	.hub_control		= musb_hub_control,
2323	.bus_suspend		= musb_bus_suspend,
2324	.bus_resume		= musb_bus_resume,
2325	/* .start_port_reset	= NULL, */
2326	/* .hub_irq_enable	= NULL, */
2327};