1// SPDX-License-Identifier: GPL-2.0+
   2/*
   3 * Copyright (C) 2011 Marvell International Ltd. All rights reserved.
   4 * Author: Chao Xie <chao.xie@marvell.com>
   5 *	   Neil Zhang <zhangwm@marvell.com>
   6 */
   7
   8#include <linux/module.h>
   9#include <linux/pci.h>
  10#include <linux/dma-mapping.h>
  11#include <linux/dmapool.h>
  12#include <linux/kernel.h>
  13#include <linux/delay.h>
  14#include <linux/ioport.h>
  15#include <linux/sched.h>
  16#include <linux/slab.h>
  17#include <linux/errno.h>
  18#include <linux/err.h>
  19#include <linux/timer.h>
  20#include <linux/list.h>
  21#include <linux/interrupt.h>
  22#include <linux/moduleparam.h>
  23#include <linux/device.h>
  24#include <linux/usb/ch9.h>
  25#include <linux/usb/gadget.h>
  26#include <linux/usb/otg.h>
  27#include <linux/pm.h>
  28#include <linux/io.h>
  29#include <linux/irq.h>
  30#include <linux/platform_device.h>
  31#include <linux/clk.h>
  32#include <linux/platform_data/mv_usb.h>
  33#include <asm/unaligned.h>
  34
  35#include "mv_udc.h"
  36
  37#define DRIVER_DESC		"Marvell PXA USB Device Controller driver"
  38
  39#define ep_dir(ep)	(((ep)->ep_num == 0) ? \
  40				((ep)->udc->ep0_dir) : ((ep)->direction))
  41
  42/* timeout value -- usec */
  43#define RESET_TIMEOUT		10000
  44#define FLUSH_TIMEOUT		10000
  45#define EPSTATUS_TIMEOUT	10000
  46#define PRIME_TIMEOUT		10000
  47#define READSAFE_TIMEOUT	1000
  48
  49#define LOOPS_USEC_SHIFT	1
  50#define LOOPS_USEC		(1 << LOOPS_USEC_SHIFT)
  51#define LOOPS(timeout)		((timeout) >> LOOPS_USEC_SHIFT)
  52
  53static DECLARE_COMPLETION(release_done);
  54
  55static const char driver_name[] = "mv_udc";
  56
  57static void nuke(struct mv_ep *ep, int status);
  58static void stop_activity(struct mv_udc *udc, struct usb_gadget_driver *driver);
  59
  60/* for endpoint 0 operations */
  61static const struct usb_endpoint_descriptor mv_ep0_desc = {
  62	.bLength =		USB_DT_ENDPOINT_SIZE,
  63	.bDescriptorType =	USB_DT_ENDPOINT,
  64	.bEndpointAddress =	0,
  65	.bmAttributes =		USB_ENDPOINT_XFER_CONTROL,
  66	.wMaxPacketSize =	EP0_MAX_PKT_SIZE,
  67};
  68
  69static void ep0_reset(struct mv_udc *udc)
  70{
  71	struct mv_ep *ep;
  72	u32 epctrlx;
  73	int i = 0;
  74
  75	/* ep0 in and out */
  76	for (i = 0; i < 2; i++) {
  77		ep = &udc->eps[i];
  78		ep->udc = udc;
  79
  80		/* ep0 dQH */
  81		ep->dqh = &udc->ep_dqh[i];
  82
  83		/* configure ep0 endpoint capabilities in dQH */
  84		ep->dqh->max_packet_length =
  85			(EP0_MAX_PKT_SIZE << EP_QUEUE_HEAD_MAX_PKT_LEN_POS)
  86			| EP_QUEUE_HEAD_IOS;
  87
  88		ep->dqh->next_dtd_ptr = EP_QUEUE_HEAD_NEXT_TERMINATE;
  89
  90		epctrlx = readl(&udc->op_regs->epctrlx[0]);
  91		if (i) {	/* TX */
  92			epctrlx |= EPCTRL_TX_ENABLE
  93				| (USB_ENDPOINT_XFER_CONTROL
  94					<< EPCTRL_TX_EP_TYPE_SHIFT);
  95
  96		} else {	/* RX */
  97			epctrlx |= EPCTRL_RX_ENABLE
  98				| (USB_ENDPOINT_XFER_CONTROL
  99					<< EPCTRL_RX_EP_TYPE_SHIFT);
 100		}
 101
 102		writel(epctrlx, &udc->op_regs->epctrlx[0]);
 103	}
 104}
 105
 106/* protocol ep0 stall, will automatically be cleared on new transaction */
 107static void ep0_stall(struct mv_udc *udc)
 108{
 109	u32	epctrlx;
 110
 111	/* set TX and RX to stall */
 112	epctrlx = readl(&udc->op_regs->epctrlx[0]);
 113	epctrlx |= EPCTRL_RX_EP_STALL | EPCTRL_TX_EP_STALL;
 114	writel(epctrlx, &udc->op_regs->epctrlx[0]);
 115
 116	/* update ep0 state */
 117	udc->ep0_state = WAIT_FOR_SETUP;
 118	udc->ep0_dir = EP_DIR_OUT;
 119}
 120
 121static int process_ep_req(struct mv_udc *udc, int index,
 122	struct mv_req *curr_req)
 123{
 124	struct mv_dtd	*curr_dtd;
 125	struct mv_dqh	*curr_dqh;
 126	int actual, remaining_length;
 127	int i, direction;
 128	int retval = 0;
 129	u32 errors;
 130	u32 bit_pos;
 131
 132	curr_dqh = &udc->ep_dqh[index];
 133	direction = index % 2;
 134
 135	curr_dtd = curr_req->head;
 136	actual = curr_req->req.length;
 137
 138	for (i = 0; i < curr_req->dtd_count; i++) {
 139		if (curr_dtd->size_ioc_sts & DTD_STATUS_ACTIVE) {
 140			dev_dbg(&udc->dev->dev, "%s, dTD not completed\n",
 141				udc->eps[index].name);
 142			return 1;
 143		}
 144
 145		errors = curr_dtd->size_ioc_sts & DTD_ERROR_MASK;
 146		if (!errors) {
 147			remaining_length =
 148				(curr_dtd->size_ioc_sts	& DTD_PACKET_SIZE)
 149					>> DTD_LENGTH_BIT_POS;
 150			actual -= remaining_length;
 151
 152			if (remaining_length) {
 153				if (direction) {
 154					dev_dbg(&udc->dev->dev,
 155						"TX dTD remains data\n");
 156					retval = -EPROTO;
 157					break;
 158				} else
 159					break;
 160			}
 161		} else {
 162			dev_info(&udc->dev->dev,
 163				"complete_tr error: ep=%d %s: error = 0x%x\n",
 164				index >> 1, direction ? "SEND" : "RECV",
 165				errors);
 166			if (errors & DTD_STATUS_HALTED) {
 167				/* Clear the errors and Halt condition */
 168				curr_dqh->size_ioc_int_sts &= ~errors;
 169				retval = -EPIPE;
 170			} else if (errors & DTD_STATUS_DATA_BUFF_ERR) {
 171				retval = -EPROTO;
 172			} else if (errors & DTD_STATUS_TRANSACTION_ERR) {
 173				retval = -EILSEQ;
 174			}
 175		}
 176		if (i != curr_req->dtd_count - 1)
 177			curr_dtd = (struct mv_dtd *)curr_dtd->next_dtd_virt;
 178	}
 179	if (retval)
 180		return retval;
 181
 182	if (direction == EP_DIR_OUT)
 183		bit_pos = 1 << curr_req->ep->ep_num;
 184	else
 185		bit_pos = 1 << (16 + curr_req->ep->ep_num);
 186
 187	while (curr_dqh->curr_dtd_ptr == curr_dtd->td_dma) {
 188		if (curr_dtd->dtd_next == EP_QUEUE_HEAD_NEXT_TERMINATE) {
 189			while (readl(&udc->op_regs->epstatus) & bit_pos)
 190				udelay(1);
 191			break;
 192		}
 193		udelay(1);
 194	}
 195
 196	curr_req->req.actual = actual;
 197
 198	return 0;
 199}
 200
 201/*
 202 * done() - retire a request; caller blocked irqs
 203 * @status : request status to be set, only works when
 204 * request is still in progress.
 205 */
 206static void done(struct mv_ep *ep, struct mv_req *req, int status)
 207	__releases(&ep->udc->lock)
 208	__acquires(&ep->udc->lock)
 209{
 210	struct mv_udc *udc = NULL;
 211	unsigned char stopped = ep->stopped;
 212	struct mv_dtd *curr_td, *next_td;
 213	int j;
 214
 215	udc = (struct mv_udc *)ep->udc;
 216	/* Removed the req from fsl_ep->queue */
 217	list_del_init(&req->queue);
 218
 219	/* req.status should be set as -EINPROGRESS in ep_queue() */
 220	if (req->req.status == -EINPROGRESS)
 221		req->req.status = status;
 222	else
 223		status = req->req.status;
 224
 225	/* Free dtd for the request */
 226	next_td = req->head;
 227	for (j = 0; j < req->dtd_count; j++) {
 228		curr_td = next_td;
 229		if (j != req->dtd_count - 1)
 230			next_td = curr_td->next_dtd_virt;
 231		dma_pool_free(udc->dtd_pool, curr_td, curr_td->td_dma);
 232	}
 233
 234	usb_gadget_unmap_request(&udc->gadget, &req->req, ep_dir(ep));
 235
 236	if (status && (status != -ESHUTDOWN))
 237		dev_info(&udc->dev->dev, "complete %s req %p stat %d len %u/%u",
 238			ep->ep.name, &req->req, status,
 239			req->req.actual, req->req.length);
 240
 241	ep->stopped = 1;
 242
 243	spin_unlock(&ep->udc->lock);
 244
 245	usb_gadget_giveback_request(&ep->ep, &req->req);
 246
 247	spin_lock(&ep->udc->lock);
 248	ep->stopped = stopped;
 249}
 250
 251static int queue_dtd(struct mv_ep *ep, struct mv_req *req)
 252{
 253	struct mv_udc *udc;
 254	struct mv_dqh *dqh;
 255	u32 bit_pos, direction;
 256	u32 usbcmd, epstatus;
 257	unsigned int loops;
 258	int retval = 0;
 259
 260	udc = ep->udc;
 261	direction = ep_dir(ep);
 262	dqh = &(udc->ep_dqh[ep->ep_num * 2 + direction]);
 263	bit_pos = 1 << (((direction == EP_DIR_OUT) ? 0 : 16) + ep->ep_num);
 264
 265	/* check if the pipe is empty */
 266	if (!(list_empty(&ep->queue))) {
 267		struct mv_req *lastreq;
 268		lastreq = list_entry(ep->queue.prev, struct mv_req, queue);
 269		lastreq->tail->dtd_next =
 270			req->head->td_dma & EP_QUEUE_HEAD_NEXT_POINTER_MASK;
 271
 272		wmb();
 273
 274		if (readl(&udc->op_regs->epprime) & bit_pos)
 275			goto done;
 276
 277		loops = LOOPS(READSAFE_TIMEOUT);
 278		while (1) {
 279			/* start with setting the semaphores */
 280			usbcmd = readl(&udc->op_regs->usbcmd);
 281			usbcmd |= USBCMD_ATDTW_TRIPWIRE_SET;
 282			writel(usbcmd, &udc->op_regs->usbcmd);
 283
 284			/* read the endpoint status */
 285			epstatus = readl(&udc->op_regs->epstatus) & bit_pos;
 286
 287			/*
 288			 * Reread the ATDTW semaphore bit to check if it is
 289			 * cleared. When hardware see a hazard, it will clear
 290			 * the bit or else we remain set to 1 and we can
 291			 * proceed with priming of endpoint if not already
 292			 * primed.
 293			 */
 294			if (readl(&udc->op_regs->usbcmd)
 295				& USBCMD_ATDTW_TRIPWIRE_SET)
 296				break;
 297
 298			loops--;
 299			if (loops == 0) {
 300				dev_err(&udc->dev->dev,
 301					"Timeout for ATDTW_TRIPWIRE...\n");
 302				retval = -ETIME;
 303				goto done;
 304			}
 305			udelay(LOOPS_USEC);
 306		}
 307
 308		/* Clear the semaphore */
 309		usbcmd = readl(&udc->op_regs->usbcmd);
 310		usbcmd &= USBCMD_ATDTW_TRIPWIRE_CLEAR;
 311		writel(usbcmd, &udc->op_regs->usbcmd);
 312
 313		if (epstatus)
 314			goto done;
 315	}
 316
 317	/* Write dQH next pointer and terminate bit to 0 */
 318	dqh->next_dtd_ptr = req->head->td_dma
 319				& EP_QUEUE_HEAD_NEXT_POINTER_MASK;
 320
 321	/* clear active and halt bit, in case set from a previous error */
 322	dqh->size_ioc_int_sts &= ~(DTD_STATUS_ACTIVE | DTD_STATUS_HALTED);
 323
 324	/* Ensure that updates to the QH will occur before priming. */
 325	wmb();
 326
 327	/* Prime the Endpoint */
 328	writel(bit_pos, &udc->op_regs->epprime);
 329
 330done:
 331	return retval;
 332}
 333
 334static struct mv_dtd *build_dtd(struct mv_req *req, unsigned *length,
 335		dma_addr_t *dma, int *is_last)
 336{
 337	struct mv_dtd *dtd;
 338	struct mv_udc *udc;
 339	struct mv_dqh *dqh;
 340	u32 temp, mult = 0;
 341
 342	/* how big will this transfer be? */
 343	if (usb_endpoint_xfer_isoc(req->ep->ep.desc)) {
 344		dqh = req->ep->dqh;
 345		mult = (dqh->max_packet_length >> EP_QUEUE_HEAD_MULT_POS)
 346				& 0x3;
 347		*length = min(req->req.length - req->req.actual,
 348				(unsigned)(mult * req->ep->ep.maxpacket));
 349	} else
 350		*length = min(req->req.length - req->req.actual,
 351				(unsigned)EP_MAX_LENGTH_TRANSFER);
 352
 353	udc = req->ep->udc;
 354
 355	/*
 356	 * Be careful that no _GFP_HIGHMEM is set,
 357	 * or we can not use dma_to_virt
 358	 */
 359	dtd = dma_pool_alloc(udc->dtd_pool, GFP_ATOMIC, dma);
 360	if (dtd == NULL)
 361		return dtd;
 362
 363	dtd->td_dma = *dma;
 364	/* initialize buffer page pointers */
 365	temp = (u32)(req->req.dma + req->req.actual);
 366	dtd->buff_ptr0 = cpu_to_le32(temp);
 367	temp &= ~0xFFF;
 368	dtd->buff_ptr1 = cpu_to_le32(temp + 0x1000);
 369	dtd->buff_ptr2 = cpu_to_le32(temp + 0x2000);
 370	dtd->buff_ptr3 = cpu_to_le32(temp + 0x3000);
 371	dtd->buff_ptr4 = cpu_to_le32(temp + 0x4000);
 372
 373	req->req.actual += *length;
 374
 375	/* zlp is needed if req->req.zero is set */
 376	if (req->req.zero) {
 377		if (*length == 0 || (*length % req->ep->ep.maxpacket) != 0)
 378			*is_last = 1;
 379		else
 380			*is_last = 0;
 381	} else if (req->req.length == req->req.actual)
 382		*is_last = 1;
 383	else
 384		*is_last = 0;
 385
 386	/* Fill in the transfer size; set active bit */
 387	temp = ((*length << DTD_LENGTH_BIT_POS) | DTD_STATUS_ACTIVE);
 388
 389	/* Enable interrupt for the last dtd of a request */
 390	if (*is_last && !req->req.no_interrupt)
 391		temp |= DTD_IOC;
 392
 393	temp |= mult << 10;
 394
 395	dtd->size_ioc_sts = temp;
 396
 397	mb();
 398
 399	return dtd;
 400}
 401
 402/* generate dTD linked list for a request */
 403static int req_to_dtd(struct mv_req *req)
 404{
 405	unsigned count;
 406	int is_last, is_first = 1;
 407	struct mv_dtd *dtd, *last_dtd = NULL;
 408	dma_addr_t dma;
 409
 410	do {
 411		dtd = build_dtd(req, &count, &dma, &is_last);
 412		if (dtd == NULL)
 413			return -ENOMEM;
 414
 415		if (is_first) {
 416			is_first = 0;
 417			req->head = dtd;
 418		} else {
 419			last_dtd->dtd_next = dma;
 420			last_dtd->next_dtd_virt = dtd;
 421		}
 422		last_dtd = dtd;
 423		req->dtd_count++;
 424	} while (!is_last);
 425
 426	/* set terminate bit to 1 for the last dTD */
 427	dtd->dtd_next = DTD_NEXT_TERMINATE;
 428
 429	req->tail = dtd;
 430
 431	return 0;
 432}
 433
 434static int mv_ep_enable(struct usb_ep *_ep,
 435		const struct usb_endpoint_descriptor *desc)
 436{
 437	struct mv_udc *udc;
 438	struct mv_ep *ep;
 439	struct mv_dqh *dqh;
 440	u16 max = 0;
 441	u32 bit_pos, epctrlx, direction;
 442	const unsigned char zlt = 1;
 443	unsigned char ios, mult;
 444	unsigned long flags;
 445
 446	ep = container_of(_ep, struct mv_ep, ep);
 447	udc = ep->udc;
 448
 449	if (!_ep || !desc
 450			|| desc->bDescriptorType != USB_DT_ENDPOINT)
 451		return -EINVAL;
 452
 453	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN)
 454		return -ESHUTDOWN;
 455
 456	direction = ep_dir(ep);
 457	max = usb_endpoint_maxp(desc);
 458
 459	/*
 460	 * disable HW zero length termination select
 461	 * driver handles zero length packet through req->req.zero
 462	 */
 463	bit_pos = 1 << ((direction == EP_DIR_OUT ? 0 : 16) + ep->ep_num);
 464
 465	/* Check if the Endpoint is Primed */
 466	if ((readl(&udc->op_regs->epprime) & bit_pos)
 467		|| (readl(&udc->op_regs->epstatus) & bit_pos)) {
 468		dev_info(&udc->dev->dev,
 469			"ep=%d %s: Init ERROR: ENDPTPRIME=0x%x,"
 470			" ENDPTSTATUS=0x%x, bit_pos=0x%x\n",
 471			(unsigned)ep->ep_num, direction ? "SEND" : "RECV",
 472			(unsigned)readl(&udc->op_regs->epprime),
 473			(unsigned)readl(&udc->op_regs->epstatus),
 474			(unsigned)bit_pos);
 475		goto en_done;
 476	}
 477
 478	/* Set the max packet length, interrupt on Setup and Mult fields */
 479	ios = 0;
 480	mult = 0;
 481	switch (desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK) {
 482	case USB_ENDPOINT_XFER_BULK:
 483	case USB_ENDPOINT_XFER_INT:
 484		break;
 485	case USB_ENDPOINT_XFER_CONTROL:
 486		ios = 1;
 487		break;
 488	case USB_ENDPOINT_XFER_ISOC:
 489		/* Calculate transactions needed for high bandwidth iso */
 490		mult = usb_endpoint_maxp_mult(desc);
 491		/* 3 transactions at most */
 492		if (mult > 3)
 493			goto en_done;
 494		break;
 495	default:
 496		goto en_done;
 497	}
 498
 499	spin_lock_irqsave(&udc->lock, flags);
 500	/* Get the endpoint queue head address */
 501	dqh = ep->dqh;
 502	dqh->max_packet_length = (max << EP_QUEUE_HEAD_MAX_PKT_LEN_POS)
 503		| (mult << EP_QUEUE_HEAD_MULT_POS)
 504		| (zlt ? EP_QUEUE_HEAD_ZLT_SEL : 0)
 505		| (ios ? EP_QUEUE_HEAD_IOS : 0);
 506	dqh->next_dtd_ptr = 1;
 507	dqh->size_ioc_int_sts = 0;
 508
 509	ep->ep.maxpacket = max;
 510	ep->ep.desc = desc;
 511	ep->stopped = 0;
 512
 513	/* Enable the endpoint for Rx or Tx and set the endpoint type */
 514	epctrlx = readl(&udc->op_regs->epctrlx[ep->ep_num]);
 515	if (direction == EP_DIR_IN) {
 516		epctrlx &= ~EPCTRL_TX_ALL_MASK;
 517		epctrlx |= EPCTRL_TX_ENABLE | EPCTRL_TX_DATA_TOGGLE_RST
 518			| ((desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK)
 519				<< EPCTRL_TX_EP_TYPE_SHIFT);
 520	} else {
 521		epctrlx &= ~EPCTRL_RX_ALL_MASK;
 522		epctrlx |= EPCTRL_RX_ENABLE | EPCTRL_RX_DATA_TOGGLE_RST
 523			| ((desc->bmAttributes & USB_ENDPOINT_XFERTYPE_MASK)
 524				<< EPCTRL_RX_EP_TYPE_SHIFT);
 525	}
 526	writel(epctrlx, &udc->op_regs->epctrlx[ep->ep_num]);
 527
 528	/*
 529	 * Implement Guideline (GL# USB-7) The unused endpoint type must
 530	 * be programmed to bulk.
 531	 */
 532	epctrlx = readl(&udc->op_regs->epctrlx[ep->ep_num]);
 533	if ((epctrlx & EPCTRL_RX_ENABLE) == 0) {
 534		epctrlx |= (USB_ENDPOINT_XFER_BULK
 535				<< EPCTRL_RX_EP_TYPE_SHIFT);
 536		writel(epctrlx, &udc->op_regs->epctrlx[ep->ep_num]);
 537	}
 538
 539	epctrlx = readl(&udc->op_regs->epctrlx[ep->ep_num]);
 540	if ((epctrlx & EPCTRL_TX_ENABLE) == 0) {
 541		epctrlx |= (USB_ENDPOINT_XFER_BULK
 542				<< EPCTRL_TX_EP_TYPE_SHIFT);
 543		writel(epctrlx, &udc->op_regs->epctrlx[ep->ep_num]);
 544	}
 545
 546	spin_unlock_irqrestore(&udc->lock, flags);
 547
 548	return 0;
 549en_done:
 550	return -EINVAL;
 551}
 552
 553static int  mv_ep_disable(struct usb_ep *_ep)
 554{
 555	struct mv_udc *udc;
 556	struct mv_ep *ep;
 557	struct mv_dqh *dqh;
 558	u32 epctrlx, direction;
 559	unsigned long flags;
 560
 561	ep = container_of(_ep, struct mv_ep, ep);
 562	if ((_ep == NULL) || !ep->ep.desc)
 563		return -EINVAL;
 564
 565	udc = ep->udc;
 566
 567	/* Get the endpoint queue head address */
 568	dqh = ep->dqh;
 569
 570	spin_lock_irqsave(&udc->lock, flags);
 571
 572	direction = ep_dir(ep);
 573
 574	/* Reset the max packet length and the interrupt on Setup */
 575	dqh->max_packet_length = 0;
 576
 577	/* Disable the endpoint for Rx or Tx and reset the endpoint type */
 578	epctrlx = readl(&udc->op_regs->epctrlx[ep->ep_num]);
 579	epctrlx &= ~((direction == EP_DIR_IN)
 580			? (EPCTRL_TX_ENABLE | EPCTRL_TX_TYPE)
 581			: (EPCTRL_RX_ENABLE | EPCTRL_RX_TYPE));
 582	writel(epctrlx, &udc->op_regs->epctrlx[ep->ep_num]);
 583
 584	/* nuke all pending requests (does flush) */
 585	nuke(ep, -ESHUTDOWN);
 586
 587	ep->ep.desc = NULL;
 588	ep->stopped = 1;
 589
 590	spin_unlock_irqrestore(&udc->lock, flags);
 591
 592	return 0;
 593}
 594
 595static struct usb_request *
 596mv_alloc_request(struct usb_ep *_ep, gfp_t gfp_flags)
 597{
 598	struct mv_req *req;
 599
 600	req = kzalloc(sizeof *req, gfp_flags);
 601	if (!req)
 602		return NULL;
 603
 604	req->req.dma = DMA_ADDR_INVALID;
 605	INIT_LIST_HEAD(&req->queue);
 606
 607	return &req->req;
 608}
 609
 610static void mv_free_request(struct usb_ep *_ep, struct usb_request *_req)
 611{
 612	struct mv_req *req = NULL;
 613
 614	req = container_of(_req, struct mv_req, req);
 615
 616	if (_req)
 617		kfree(req);
 618}
 619
 620static void mv_ep_fifo_flush(struct usb_ep *_ep)
 621{
 622	struct mv_udc *udc;
 623	u32 bit_pos, direction;
 624	struct mv_ep *ep;
 625	unsigned int loops;
 626
 627	if (!_ep)
 628		return;
 629
 630	ep = container_of(_ep, struct mv_ep, ep);
 631	if (!ep->ep.desc)
 632		return;
 633
 634	udc = ep->udc;
 635	direction = ep_dir(ep);
 636
 637	if (ep->ep_num == 0)
 638		bit_pos = (1 << 16) | 1;
 639	else if (direction == EP_DIR_OUT)
 640		bit_pos = 1 << ep->ep_num;
 641	else
 642		bit_pos = 1 << (16 + ep->ep_num);
 643
 644	loops = LOOPS(EPSTATUS_TIMEOUT);
 645	do {
 646		unsigned int inter_loops;
 647
 648		if (loops == 0) {
 649			dev_err(&udc->dev->dev,
 650				"TIMEOUT for ENDPTSTATUS=0x%x, bit_pos=0x%x\n",
 651				(unsigned)readl(&udc->op_regs->epstatus),
 652				(unsigned)bit_pos);
 653			return;
 654		}
 655		/* Write 1 to the Flush register */
 656		writel(bit_pos, &udc->op_regs->epflush);
 657
 658		/* Wait until flushing completed */
 659		inter_loops = LOOPS(FLUSH_TIMEOUT);
 660		while (readl(&udc->op_regs->epflush)) {
 661			/*
 662			 * ENDPTFLUSH bit should be cleared to indicate this
 663			 * operation is complete
 664			 */
 665			if (inter_loops == 0) {
 666				dev_err(&udc->dev->dev,
 667					"TIMEOUT for ENDPTFLUSH=0x%x,"
 668					"bit_pos=0x%x\n",
 669					(unsigned)readl(&udc->op_regs->epflush),
 670					(unsigned)bit_pos);
 671				return;
 672			}
 673			inter_loops--;
 674			udelay(LOOPS_USEC);
 675		}
 676		loops--;
 677	} while (readl(&udc->op_regs->epstatus) & bit_pos);
 678}
 679
 680/* queues (submits) an I/O request to an endpoint */
 681static int
 682mv_ep_queue(struct usb_ep *_ep, struct usb_request *_req, gfp_t gfp_flags)
 683{
 684	struct mv_ep *ep = container_of(_ep, struct mv_ep, ep);
 685	struct mv_req *req = container_of(_req, struct mv_req, req);
 686	struct mv_udc *udc = ep->udc;
 687	unsigned long flags;
 688	int retval;
 689
 690	/* catch various bogus parameters */
 691	if (!_req || !req->req.complete || !req->req.buf
 692			|| !list_empty(&req->queue)) {
 693		dev_err(&udc->dev->dev, "%s, bad params", __func__);
 694		return -EINVAL;
 695	}
 696	if (unlikely(!_ep || !ep->ep.desc)) {
 697		dev_err(&udc->dev->dev, "%s, bad ep", __func__);
 698		return -EINVAL;
 699	}
 700
 701	udc = ep->udc;
 702	if (!udc->driver || udc->gadget.speed == USB_SPEED_UNKNOWN)
 703		return -ESHUTDOWN;
 704
 705	req->ep = ep;
 706
 707	/* map virtual address to hardware */
 708	retval = usb_gadget_map_request(&udc->gadget, _req, ep_dir(ep));
 709	if (retval)
 710		return retval;
 711
 712	req->req.status = -EINPROGRESS;
 713	req->req.actual = 0;
 714	req->dtd_count = 0;
 715
 716	spin_lock_irqsave(&udc->lock, flags);
 717
 718	/* build dtds and push them to device queue */
 719	if (!req_to_dtd(req)) {
 720		retval = queue_dtd(ep, req);
 721		if (retval) {
 722			spin_unlock_irqrestore(&udc->lock, flags);
 723			dev_err(&udc->dev->dev, "Failed to queue dtd\n");
 724			goto err_unmap_dma;
 725		}
 726	} else {
 727		spin_unlock_irqrestore(&udc->lock, flags);
 728		dev_err(&udc->dev->dev, "Failed to dma_pool_alloc\n");
 729		retval = -ENOMEM;
 730		goto err_unmap_dma;
 731	}
 732
 733	/* Update ep0 state */
 734	if (ep->ep_num == 0)
 735		udc->ep0_state = DATA_STATE_XMIT;
 736
 737	/* irq handler advances the queue */
 738	list_add_tail(&req->queue, &ep->queue);
 739	spin_unlock_irqrestore(&udc->lock, flags);
 740
 741	return 0;
 742
 743err_unmap_dma:
 744	usb_gadget_unmap_request(&udc->gadget, _req, ep_dir(ep));
 745
 746	return retval;
 747}
 748
 749static void mv_prime_ep(struct mv_ep *ep, struct mv_req *req)
 750{
 751	struct mv_dqh *dqh = ep->dqh;
 752	u32 bit_pos;
 753
 754	/* Write dQH next pointer and terminate bit to 0 */
 755	dqh->next_dtd_ptr = req->head->td_dma
 756		& EP_QUEUE_HEAD_NEXT_POINTER_MASK;
 757
 758	/* clear active and halt bit, in case set from a previous error */
 759	dqh->size_ioc_int_sts &= ~(DTD_STATUS_ACTIVE | DTD_STATUS_HALTED);
 760
 761	/* Ensure that updates to the QH will occure before priming. */
 762	wmb();
 763
 764	bit_pos = 1 << (((ep_dir(ep) == EP_DIR_OUT) ? 0 : 16) + ep->ep_num);
 765
 766	/* Prime the Endpoint */
 767	writel(bit_pos, &ep->udc->op_regs->epprime);
 768}
 769
 770/* dequeues (cancels, unlinks) an I/O request from an endpoint */
 771static int mv_ep_dequeue(struct usb_ep *_ep, struct usb_request *_req)
 772{
 773	struct mv_ep *ep = container_of(_ep, struct mv_ep, ep);
 774	struct mv_req *req = NULL, *iter;
 775	struct mv_udc *udc = ep->udc;
 776	unsigned long flags;
 777	int stopped, ret = 0;
 778	u32 epctrlx;
 779
 780	if (!_ep || !_req)
 781		return -EINVAL;
 782
 783	spin_lock_irqsave(&ep->udc->lock, flags);
 784	stopped = ep->stopped;
 785
 786	/* Stop the ep before we deal with the queue */
 787	ep->stopped = 1;
 788	epctrlx = readl(&udc->op_regs->epctrlx[ep->ep_num]);
 789	if (ep_dir(ep) == EP_DIR_IN)
 790		epctrlx &= ~EPCTRL_TX_ENABLE;
 791	else
 792		epctrlx &= ~EPCTRL_RX_ENABLE;
 793	writel(epctrlx, &udc->op_regs->epctrlx[ep->ep_num]);
 794
 795	/* make sure it's actually queued on this endpoint */
 796	list_for_each_entry(iter, &ep->queue, queue) {
 797		if (&iter->req != _req)
 798			continue;
 799		req = iter;
 800		break;
 801	}
 802	if (!req) {
 803		ret = -EINVAL;
 804		goto out;
 805	}
 806
 807	/* The request is in progress, or completed but not dequeued */
 808	if (ep->queue.next == &req->queue) {
 809		_req->status = -ECONNRESET;
 810		mv_ep_fifo_flush(_ep);	/* flush current transfer */
 811
 812		/* The request isn't the last request in this ep queue */
 813		if (req->queue.next != &ep->queue) {
 814			struct mv_req *next_req;
 815
 816			next_req = list_entry(req->queue.next,
 817				struct mv_req, queue);
 818
 819			/* Point the QH to the first TD of next request */
 820			mv_prime_ep(ep, next_req);
 821		} else {
 822			struct mv_dqh *qh;
 823
 824			qh = ep->dqh;
 825			qh->next_dtd_ptr = 1;
 826			qh->size_ioc_int_sts = 0;
 827		}
 828
 829		/* The request hasn't been processed, patch up the TD chain */
 830	} else {
 831		struct mv_req *prev_req;
 832
 833		prev_req = list_entry(req->queue.prev, struct mv_req, queue);
 834		writel(readl(&req->tail->dtd_next),
 835				&prev_req->tail->dtd_next);
 836
 837	}
 838
 839	done(ep, req, -ECONNRESET);
 840
 841	/* Enable EP */
 842out:
 843	epctrlx = readl(&udc->op_regs->epctrlx[ep->ep_num]);
 844	if (ep_dir(ep) == EP_DIR_IN)
 845		epctrlx |= EPCTRL_TX_ENABLE;
 846	else
 847		epctrlx |= EPCTRL_RX_ENABLE;
 848	writel(epctrlx, &udc->op_regs->epctrlx[ep->ep_num]);
 849	ep->stopped = stopped;
 850
 851	spin_unlock_irqrestore(&ep->udc->lock, flags);
 852	return ret;
 853}
 854
 855static void ep_set_stall(struct mv_udc *udc, u8 ep_num, u8 direction, int stall)
 856{
 857	u32 epctrlx;
 858
 859	epctrlx = readl(&udc->op_regs->epctrlx[ep_num]);
 860
 861	if (stall) {
 862		if (direction == EP_DIR_IN)
 863			epctrlx |= EPCTRL_TX_EP_STALL;
 864		else
 865			epctrlx |= EPCTRL_RX_EP_STALL;
 866	} else {
 867		if (direction == EP_DIR_IN) {
 868			epctrlx &= ~EPCTRL_TX_EP_STALL;
 869			epctrlx |= EPCTRL_TX_DATA_TOGGLE_RST;
 870		} else {
 871			epctrlx &= ~EPCTRL_RX_EP_STALL;
 872			epctrlx |= EPCTRL_RX_DATA_TOGGLE_RST;
 873		}
 874	}
 875	writel(epctrlx, &udc->op_regs->epctrlx[ep_num]);
 876}
 877
 878static int ep_is_stall(struct mv_udc *udc, u8 ep_num, u8 direction)
 879{
 880	u32 epctrlx;
 881
 882	epctrlx = readl(&udc->op_regs->epctrlx[ep_num]);
 883
 884	if (direction == EP_DIR_OUT)
 885		return (epctrlx & EPCTRL_RX_EP_STALL) ? 1 : 0;
 886	else
 887		return (epctrlx & EPCTRL_TX_EP_STALL) ? 1 : 0;
 888}
 889
 890static int mv_ep_set_halt_wedge(struct usb_ep *_ep, int halt, int wedge)
 891{
 892	struct mv_ep *ep;
 893	unsigned long flags;
 894	int status = 0;
 895	struct mv_udc *udc;
 896
 897	ep = container_of(_ep, struct mv_ep, ep);
 898	udc = ep->udc;
 899	if (!_ep || !ep->ep.desc) {
 900		status = -EINVAL;
 901		goto out;
 902	}
 903
 904	if (ep->ep.desc->bmAttributes == USB_ENDPOINT_XFER_ISOC) {
 905		status = -EOPNOTSUPP;
 906		goto out;
 907	}
 908
 909	/*
 910	 * Attempt to halt IN ep will fail if any transfer requests
 911	 * are still queue
 912	 */
 913	if (halt && (ep_dir(ep) == EP_DIR_IN) && !list_empty(&ep->queue)) {
 914		status = -EAGAIN;
 915		goto out;
 916	}
 917
 918	spin_lock_irqsave(&ep->udc->lock, flags);
 919	ep_set_stall(udc, ep->ep_num, ep_dir(ep), halt);
 920	if (halt && wedge)
 921		ep->wedge = 1;
 922	else if (!halt)
 923		ep->wedge = 0;
 924	spin_unlock_irqrestore(&ep->udc->lock, flags);
 925
 926	if (ep->ep_num == 0) {
 927		udc->ep0_state = WAIT_FOR_SETUP;
 928		udc->ep0_dir = EP_DIR_OUT;
 929	}
 930out:
 931	return status;
 932}
 933
 934static int mv_ep_set_halt(struct usb_ep *_ep, int halt)
 935{
 936	return mv_ep_set_halt_wedge(_ep, halt, 0);
 937}
 938
 939static int mv_ep_set_wedge(struct usb_ep *_ep)
 940{
 941	return mv_ep_set_halt_wedge(_ep, 1, 1);
 942}
 943
 944static const struct usb_ep_ops mv_ep_ops = {
 945	.enable		= mv_ep_enable,
 946	.disable	= mv_ep_disable,
 947
 948	.alloc_request	= mv_alloc_request,
 949	.free_request	= mv_free_request,
 950
 951	.queue		= mv_ep_queue,
 952	.dequeue	= mv_ep_dequeue,
 953
 954	.set_wedge	= mv_ep_set_wedge,
 955	.set_halt	= mv_ep_set_halt,
 956	.fifo_flush	= mv_ep_fifo_flush,	/* flush fifo */
 957};
 958
 959static int udc_clock_enable(struct mv_udc *udc)
 960{
 961	return clk_prepare_enable(udc->clk);
 962}
 963
 964static void udc_clock_disable(struct mv_udc *udc)
 965{
 966	clk_disable_unprepare(udc->clk);
 967}
 968
 969static void udc_stop(struct mv_udc *udc)
 970{
 971	u32 tmp;
 972
 973	/* Disable interrupts */
 974	tmp = readl(&udc->op_regs->usbintr);
 975	tmp &= ~(USBINTR_INT_EN | USBINTR_ERR_INT_EN |
 976		USBINTR_PORT_CHANGE_DETECT_EN | USBINTR_RESET_EN);
 977	writel(tmp, &udc->op_regs->usbintr);
 978
 979	udc->stopped = 1;
 980
 981	/* Reset the Run the bit in the command register to stop VUSB */
 982	tmp = readl(&udc->op_regs->usbcmd);
 983	tmp &= ~USBCMD_RUN_STOP;
 984	writel(tmp, &udc->op_regs->usbcmd);
 985}
 986
 987static void udc_start(struct mv_udc *udc)
 988{
 989	u32 usbintr;
 990
 991	usbintr = USBINTR_INT_EN | USBINTR_ERR_INT_EN
 992		| USBINTR_PORT_CHANGE_DETECT_EN
 993		| USBINTR_RESET_EN | USBINTR_DEVICE_SUSPEND;
 994	/* Enable interrupts */
 995	writel(usbintr, &udc->op_regs->usbintr);
 996
 997	udc->stopped = 0;
 998
 999	/* Set the Run bit in the command register */
1000	writel(USBCMD_RUN_STOP, &udc->op_regs->usbcmd);
1001}
1002
1003static int udc_reset(struct mv_udc *udc)
1004{
1005	unsigned int loops;
1006	u32 tmp, portsc;
1007
1008	/* Stop the controller */
1009	tmp = readl(&udc->op_regs->usbcmd);
1010	tmp &= ~USBCMD_RUN_STOP;
1011	writel(tmp, &udc->op_regs->usbcmd);
1012
1013	/* Reset the controller to get default values */
1014	writel(USBCMD_CTRL_RESET, &udc->op_regs->usbcmd);
1015
1016	/* wait for reset to complete */
1017	loops = LOOPS(RESET_TIMEOUT);
1018	while (readl(&udc->op_regs->usbcmd) & USBCMD_CTRL_RESET) {
1019		if (loops == 0) {
1020			dev_err(&udc->dev->dev,
1021				"Wait for RESET completed TIMEOUT\n");
1022			return -ETIMEDOUT;
1023		}
1024		loops--;
1025		udelay(LOOPS_USEC);
1026	}
1027
1028	/* set controller to device mode */
1029	tmp = readl(&udc->op_regs->usbmode);
1030	tmp |= USBMODE_CTRL_MODE_DEVICE;
1031
1032	/* turn setup lockout off, require setup tripwire in usbcmd */
1033	tmp |= USBMODE_SETUP_LOCK_OFF;
1034
1035	writel(tmp, &udc->op_regs->usbmode);
1036
1037	writel(0x0, &udc->op_regs->epsetupstat);
1038
1039	/* Configure the Endpoint List Address */
1040	writel(udc->ep_dqh_dma & USB_EP_LIST_ADDRESS_MASK,
1041		&udc->op_regs->eplistaddr);
1042
1043	portsc = readl(&udc->op_regs->portsc[0]);
1044	if (readl(&udc->cap_regs->hcsparams) & HCSPARAMS_PPC)
1045		portsc &= (~PORTSCX_W1C_BITS | ~PORTSCX_PORT_POWER);
1046
1047	if (udc->force_fs)
1048		portsc |= PORTSCX_FORCE_FULL_SPEED_CONNECT;
1049	else
1050		portsc &= (~PORTSCX_FORCE_FULL_SPEED_CONNECT);
1051
1052	writel(portsc, &udc->op_regs->portsc[0]);
1053
1054	tmp = readl(&udc->op_regs->epctrlx[0]);
1055	tmp &= ~(EPCTRL_TX_EP_STALL | EPCTRL_RX_EP_STALL);
1056	writel(tmp, &udc->op_regs->epctrlx[0]);
1057
1058	return 0;
1059}
1060
1061static int mv_udc_enable_internal(struct mv_udc *udc)
1062{
1063	int retval;
1064
1065	if (udc->active)
1066		return 0;
1067
1068	dev_dbg(&udc->dev->dev, "enable udc\n");
1069	retval = udc_clock_enable(udc);
1070	if (retval)
1071		return retval;
1072
1073	if (udc->pdata->phy_init) {
1074		retval = udc->pdata->phy_init(udc->phy_regs);
1075		if (retval) {
1076			dev_err(&udc->dev->dev,
1077				"init phy error %d\n", retval);
1078			udc_clock_disable(udc);
1079			return retval;
1080		}
1081	}
1082	udc->active = 1;
1083
1084	return 0;
1085}
1086
1087static int mv_udc_enable(struct mv_udc *udc)
1088{
1089	if (udc->clock_gating)
1090		return mv_udc_enable_internal(udc);
1091
1092	return 0;
1093}
1094
1095static void mv_udc_disable_internal(struct mv_udc *udc)
1096{
1097	if (udc->active) {
1098		dev_dbg(&udc->dev->dev, "disable udc\n");
1099		if (udc->pdata->phy_deinit)
1100			udc->pdata->phy_deinit(udc->phy_regs);
1101		udc_clock_disable(udc);
1102		udc->active = 0;
1103	}
1104}
1105
1106static void mv_udc_disable(struct mv_udc *udc)
1107{
1108	if (udc->clock_gating)
1109		mv_udc_disable_internal(udc);
1110}
1111
1112static int mv_udc_get_frame(struct usb_gadget *gadget)
1113{
1114	struct mv_udc *udc;
1115	u16	retval;
1116
1117	if (!gadget)
1118		return -ENODEV;
1119
1120	udc = container_of(gadget, struct mv_udc, gadget);
1121
1122	retval = readl(&udc->op_regs->frindex) & USB_FRINDEX_MASKS;
1123
1124	return retval;
1125}
1126
1127/* Tries to wake up the host connected to this gadget */
1128static int mv_udc_wakeup(struct usb_gadget *gadget)
1129{
1130	struct mv_udc *udc = container_of(gadget, struct mv_udc, gadget);
1131	u32 portsc;
1132
1133	/* Remote wakeup feature not enabled by host */
1134	if (!udc->remote_wakeup)
1135		return -ENOTSUPP;
1136
1137	portsc = readl(&udc->op_regs->portsc);
1138	/* not suspended? */
1139	if (!(portsc & PORTSCX_PORT_SUSPEND))
1140		return 0;
1141	/* trigger force resume */
1142	portsc |= PORTSCX_PORT_FORCE_RESUME;
1143	writel(portsc, &udc->op_regs->portsc[0]);
1144	return 0;
1145}
1146
1147static int mv_udc_vbus_session(struct usb_gadget *gadget, int is_active)
1148{
1149	struct mv_udc *udc;
1150	unsigned long flags;
1151	int retval = 0;
1152
1153	udc = container_of(gadget, struct mv_udc, gadget);
1154	spin_lock_irqsave(&udc->lock, flags);
1155
1156	udc->vbus_active = (is_active != 0);
1157
1158	dev_dbg(&udc->dev->dev, "%s: softconnect %d, vbus_active %d\n",
1159		__func__, udc->softconnect, udc->vbus_active);
1160
1161	if (udc->driver && udc->softconnect && udc->vbus_active) {
1162		retval = mv_udc_enable(udc);
1163		if (retval == 0) {
1164			/* Clock is disabled, need re-init registers */
1165			udc_reset(udc);
1166			ep0_reset(udc);
1167			udc_start(udc);
1168		}
1169	} else if (udc->driver && udc->softconnect) {
1170		if (!udc->active)
1171			goto out;
1172
1173		/* stop all the transfer in queue*/
1174		stop_activity(udc, udc->driver);
1175		udc_stop(udc);
1176		mv_udc_disable(udc);
1177	}
1178
1179out:
1180	spin_unlock_irqrestore(&udc->lock, flags);
1181	return retval;
1182}
1183
1184static int mv_udc_pullup(struct usb_gadget *gadget, int is_on)
1185{
1186	struct mv_udc *udc;
1187	unsigned long flags;
1188	int retval = 0;
1189
1190	udc = container_of(gadget, struct mv_udc, gadget);
1191	spin_lock_irqsave(&udc->lock, flags);
1192
1193	udc->softconnect = (is_on != 0);
1194
1195	dev_dbg(&udc->dev->dev, "%s: softconnect %d, vbus_active %d\n",
1196			__func__, udc->softconnect, udc->vbus_active);
1197
1198	if (udc->driver && udc->softconnect && udc->vbus_active) {
1199		retval = mv_udc_enable(udc);
1200		if (retval == 0) {
1201			/* Clock is disabled, need re-init registers */
1202			udc_reset(udc);
1203			ep0_reset(udc);
1204			udc_start(udc);
1205		}
1206	} else if (udc->driver && udc->vbus_active) {
1207		/* stop all the transfer in queue*/
1208		stop_activity(udc, udc->driver);
1209		udc_stop(udc);
1210		mv_udc_disable(udc);
1211	}
1212
1213	spin_unlock_irqrestore(&udc->lock, flags);
1214	return retval;
1215}
1216
1217static int mv_udc_start(struct usb_gadget *, struct usb_gadget_driver *);
1218static int mv_udc_stop(struct usb_gadget *);
1219/* device controller usb_gadget_ops structure */
1220static const struct usb_gadget_ops mv_ops = {
1221
1222	/* returns the current frame number */
1223	.get_frame	= mv_udc_get_frame,
1224
1225	/* tries to wake up the host connected to this gadget */
1226	.wakeup		= mv_udc_wakeup,
1227
1228	/* notify controller that VBUS is powered or not */
1229	.vbus_session	= mv_udc_vbus_session,
1230
1231	/* D+ pullup, software-controlled connect/disconnect to USB host */
1232	.pullup		= mv_udc_pullup,
1233	.udc_start	= mv_udc_start,
1234	.udc_stop	= mv_udc_stop,
1235};
1236
1237static int eps_init(struct mv_udc *udc)
1238{
1239	struct mv_ep	*ep;
1240	char name[14];
1241	int i;
1242
1243	/* initialize ep0 */
1244	ep = &udc->eps[0];
1245	ep->udc = udc;
1246	strncpy(ep->name, "ep0", sizeof(ep->name));
1247	ep->ep.name = ep->name;
1248	ep->ep.ops = &mv_ep_ops;
1249	ep->wedge = 0;
1250	ep->stopped = 0;
1251	usb_ep_set_maxpacket_limit(&ep->ep, EP0_MAX_PKT_SIZE);
1252	ep->ep.caps.type_control = true;
1253	ep->ep.caps.dir_in = true;
1254	ep->ep.caps.dir_out = true;
1255	ep->ep_num = 0;
1256	ep->ep.desc = &mv_ep0_desc;
1257	INIT_LIST_HEAD(&ep->queue);
1258
1259	ep->ep_type = USB_ENDPOINT_XFER_CONTROL;
1260
1261	/* initialize other endpoints */
1262	for (i = 2; i < udc->max_eps * 2; i++) {
1263		ep = &udc->eps[i];
1264		if (i % 2) {
1265			snprintf(name, sizeof(name), "ep%din", i / 2);
1266			ep->direction = EP_DIR_IN;
1267			ep->ep.caps.dir_in = true;
1268		} else {
1269			snprintf(name, sizeof(name), "ep%dout", i / 2);
1270			ep->direction = EP_DIR_OUT;
1271			ep->ep.caps.dir_out = true;
1272		}
1273		ep->udc = udc;
1274		strncpy(ep->name, name, sizeof(ep->name));
1275		ep->ep.name = ep->name;
1276
1277		ep->ep.caps.type_iso = true;
1278		ep->ep.caps.type_bulk = true;
1279		ep->ep.caps.type_int = true;
1280
1281		ep->ep.ops = &mv_ep_ops;
1282		ep->stopped = 0;
1283		usb_ep_set_maxpacket_limit(&ep->ep, (unsigned short) ~0);
1284		ep->ep_num = i / 2;
1285
1286		INIT_LIST_HEAD(&ep->queue);
1287		list_add_tail(&ep->ep.ep_list, &udc->gadget.ep_list);
1288
1289		ep->dqh = &udc->ep_dqh[i];
1290	}
1291
1292	return 0;
1293}
1294
1295/* delete all endpoint requests, called with spinlock held */
1296static void nuke(struct mv_ep *ep, int status)
1297{
1298	/* called with spinlock held */
1299	ep->stopped = 1;
1300
1301	/* endpoint fifo flush */
1302	mv_ep_fifo_flush(&ep->ep);
1303
1304	while (!list_empty(&ep->queue)) {
1305		struct mv_req *req = NULL;
1306		req = list_entry(ep->queue.next, struct mv_req, queue);
1307		done(ep, req, status);
1308	}
1309}
1310
1311static void gadget_reset(struct mv_udc *udc, struct usb_gadget_driver *driver)
1312{
1313	struct mv_ep	*ep;
1314
1315	nuke(&udc->eps[0], -ESHUTDOWN);
1316
1317	list_for_each_entry(ep, &udc->gadget.ep_list, ep.ep_list) {
1318		nuke(ep, -ESHUTDOWN);
1319	}
1320
1321	/* report reset; the driver is already quiesced */
1322	if (driver) {
1323		spin_unlock(&udc->lock);
1324		usb_gadget_udc_reset(&udc->gadget, driver);
1325		spin_lock(&udc->lock);
1326	}
1327}
1328/* stop all USB activities */
1329static void stop_activity(struct mv_udc *udc, struct usb_gadget_driver *driver)
1330{
1331	struct mv_ep	*ep;
1332
1333	nuke(&udc->eps[0], -ESHUTDOWN);
1334
1335	list_for_each_entry(ep, &udc->gadget.ep_list, ep.ep_list) {
1336		nuke(ep, -ESHUTDOWN);
1337	}
1338
1339	/* report disconnect; the driver is already quiesced */
1340	if (driver) {
1341		spin_unlock(&udc->lock);
1342		driver->disconnect(&udc->gadget);
1343		spin_lock(&udc->lock);
1344	}
1345}
1346
1347static int mv_udc_start(struct usb_gadget *gadget,
1348		struct usb_gadget_driver *driver)
1349{
1350	struct mv_udc *udc;
1351	int retval = 0;
1352	unsigned long flags;
1353
1354	udc = container_of(gadget, struct mv_udc, gadget);
1355
1356	if (udc->driver)
1357		return -EBUSY;
1358
1359	spin_lock_irqsave(&udc->lock, flags);
1360
1361	/* hook up the driver ... */
1362	udc->driver = driver;
1363
1364	udc->usb_state = USB_STATE_ATTACHED;
1365	udc->ep0_state = WAIT_FOR_SETUP;
1366	udc->ep0_dir = EP_DIR_OUT;
1367
1368	spin_unlock_irqrestore(&udc->lock, flags);
1369
1370	if (udc->transceiver) {
1371		retval = otg_set_peripheral(udc->transceiver->otg,
1372					&udc->gadget);
1373		if (retval) {
1374			dev_err(&udc->dev->dev,
1375				"unable to register peripheral to otg\n");
1376			udc->driver = NULL;
1377			return retval;
1378		}
1379	}
1380
1381	/* When boot with cable attached, there will be no vbus irq occurred */
1382	if (udc->qwork)
1383		queue_work(udc->qwork, &udc->vbus_work);
1384
1385	return 0;
1386}
1387
1388static int mv_udc_stop(struct usb_gadget *gadget)
1389{
1390	struct mv_udc *udc;
1391	unsigned long flags;
1392
1393	udc = container_of(gadget, struct mv_udc, gadget);
1394
1395	spin_lock_irqsave(&udc->lock, flags);
1396
1397	mv_udc_enable(udc);
1398	udc_stop(udc);
1399
1400	/* stop all usb activities */
1401	udc->gadget.speed = USB_SPEED_UNKNOWN;
1402	stop_activity(udc, NULL);
1403	mv_udc_disable(udc);
1404
1405	spin_unlock_irqrestore(&udc->lock, flags);
1406
1407	/* unbind gadget driver */
1408	udc->driver = NULL;
1409
1410	return 0;
1411}
1412
1413static void mv_set_ptc(struct mv_udc *udc, u32 mode)
1414{
1415	u32 portsc;
1416
1417	portsc = readl(&udc->op_regs->portsc[0]);
1418	portsc |= mode << 16;
1419	writel(portsc, &udc->op_regs->portsc[0]);
1420}
1421
1422static void prime_status_complete(struct usb_ep *ep, struct usb_request *_req)
1423{
1424	struct mv_ep *mvep = container_of(ep, struct mv_ep, ep);
1425	struct mv_req *req = container_of(_req, struct mv_req, req);
1426	struct mv_udc *udc;
1427	unsigned long flags;
1428
1429	udc = mvep->udc;
1430
1431	dev_info(&udc->dev->dev, "switch to test mode %d\n", req->test_mode);
1432
1433	spin_lock_irqsave(&udc->lock, flags);
1434	if (req->test_mode) {
1435		mv_set_ptc(udc, req->test_mode);
1436		req->test_mode = 0;
1437	}
1438	spin_unlock_irqrestore(&udc->lock, flags);
1439}
1440
1441static int
1442udc_prime_status(struct mv_udc *udc, u8 direction, u16 status, bool empty)
1443{
1444	int retval = 0;
1445	struct mv_req *req;
1446	struct mv_ep *ep;
1447
1448	ep = &udc->eps[0];
1449	udc->ep0_dir = direction;
1450	udc->ep0_state = WAIT_FOR_OUT_STATUS;
1451
1452	req = udc->status_req;
1453
1454	/* fill in the request structure */
1455	if (empty == false) {
1456		*((u16 *) req->req.buf) = cpu_to_le16(status);
1457		req->req.length = 2;
1458	} else
1459		req->req.length = 0;
1460
1461	req->ep = ep;
1462	req->req.status = -EINPROGRESS;
1463	req->req.actual = 0;
1464	if (udc->test_mode) {
1465		req->req.complete = prime_status_complete;
1466		req->test_mode = udc->test_mode;
1467		udc->test_mode = 0;
1468	} else
1469		req->req.complete = NULL;
1470	req->dtd_count = 0;
1471
1472	if (req->req.dma == DMA_ADDR_INVALID) {
1473		req->req.dma = dma_map_single(ep->udc->gadget.dev.parent,
1474				req->req.buf, req->req.length,
1475				ep_dir(ep) ? DMA_TO_DEVICE : DMA_FROM_DEVICE);
1476		req->mapped = 1;
1477	}
1478
1479	/* prime the data phase */
1480	if (!req_to_dtd(req)) {
1481		retval = queue_dtd(ep, req);
1482		if (retval) {
1483			dev_err(&udc->dev->dev,
1484				"Failed to queue dtd when prime status\n");
1485			goto out;
1486		}
1487	} else{	/* no mem */
1488		retval = -ENOMEM;
1489		dev_err(&udc->dev->dev,
1490			"Failed to dma_pool_alloc when prime status\n");
1491		goto out;
1492	}
1493
1494	list_add_tail(&req->queue, &ep->queue);
1495
1496	return 0;
1497out:
1498	usb_gadget_unmap_request(&udc->gadget, &req->req, ep_dir(ep));
1499
1500	return retval;
1501}
1502
1503static void mv_udc_testmode(struct mv_udc *udc, u16 index)
1504{
1505	if (index <= USB_TEST_FORCE_ENABLE) {
1506		udc->test_mode = index;
1507		if (udc_prime_status(udc, EP_DIR_IN, 0, true))
1508			ep0_stall(udc);
1509	} else
1510		dev_err(&udc->dev->dev,
1511			"This test mode(%d) is not supported\n", index);
1512}
1513
1514static void ch9setaddress(struct mv_udc *udc, struct usb_ctrlrequest *setup)
1515{
1516	udc->dev_addr = (u8)setup->wValue;
1517
1518	/* update usb state */
1519	udc->usb_state = USB_STATE_ADDRESS;
1520
1521	if (udc_prime_status(udc, EP_DIR_IN, 0, true))
1522		ep0_stall(udc);
1523}
1524
1525static void ch9getstatus(struct mv_udc *udc, u8 ep_num,
1526	struct usb_ctrlrequest *setup)
1527{
1528	u16 status = 0;
1529	int retval;
1530
1531	if ((setup->bRequestType & (USB_DIR_IN | USB_TYPE_MASK))
1532		!= (USB_DIR_IN | USB_TYPE_STANDARD))
1533		return;
1534
1535	if ((setup->bRequestType & USB_RECIP_MASK) == USB_RECIP_DEVICE) {
1536		status = 1 << USB_DEVICE_SELF_POWERED;
1537		status |= udc->remote_wakeup << USB_DEVICE_REMOTE_WAKEUP;
1538	} else if ((setup->bRequestType & USB_RECIP_MASK)
1539			== USB_RECIP_INTERFACE) {
1540		/* get interface status */
1541		status = 0;
1542	} else if ((setup->bRequestType & USB_RECIP_MASK)
1543			== USB_RECIP_ENDPOINT) {
1544		u8 ep_num, direction;
1545
1546		ep_num = setup->wIndex & USB_ENDPOINT_NUMBER_MASK;
1547		direction = (setup->wIndex & USB_ENDPOINT_DIR_MASK)
1548				? EP_DIR_IN : EP_DIR_OUT;
1549		status = ep_is_stall(udc, ep_num, direction)
1550				<< USB_ENDPOINT_HALT;
1551	}
1552
1553	retval = udc_prime_status(udc, EP_DIR_IN, status, false);
1554	if (retval)
1555		ep0_stall(udc);
1556	else
1557		udc->ep0_state = DATA_STATE_XMIT;
1558}
1559
1560static void ch9clearfeature(struct mv_udc *udc, struct usb_ctrlrequest *setup)
1561{
1562	u8 ep_num;
1563	u8 direction;
1564	struct mv_ep *ep;
1565
1566	if ((setup->bRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))
1567		== ((USB_TYPE_STANDARD | USB_RECIP_DEVICE))) {
1568		switch (setup->wValue) {
1569		case USB_DEVICE_REMOTE_WAKEUP:
1570			udc->remote_wakeup = 0;
1571			break;
1572		default:
1573			goto out;
1574		}
1575	} else if ((setup->bRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))
1576		== ((USB_TYPE_STANDARD | USB_RECIP_ENDPOINT))) {
1577		switch (setup->wValue) {
1578		case USB_ENDPOINT_HALT:
1579			ep_num = setup->wIndex & USB_ENDPOINT_NUMBER_MASK;
1580			direction = (setup->wIndex & USB_ENDPOINT_DIR_MASK)
1581				? EP_DIR_IN : EP_DIR_OUT;
1582			if (setup->wValue != 0 || setup->wLength != 0
1583				|| ep_num > udc->max_eps)
1584				goto out;
1585			ep = &udc->eps[ep_num * 2 + direction];
1586			if (ep->wedge == 1)
1587				break;
1588			spin_unlock(&udc->lock);
1589			ep_set_stall(udc, ep_num, direction, 0);
1590			spin_lock(&udc->lock);
1591			break;
1592		default:
1593			goto out;
1594		}
1595	} else
1596		goto out;
1597
1598	if (udc_prime_status(udc, EP_DIR_IN, 0, true))
1599		ep0_stall(udc);
1600out:
1601	return;
1602}
1603
1604static void ch9setfeature(struct mv_udc *udc, struct usb_ctrlrequest *setup)
1605{
1606	u8 ep_num;
1607	u8 direction;
1608
1609	if ((setup->bRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))
1610		== ((USB_TYPE_STANDARD | USB_RECIP_DEVICE))) {
1611		switch (setup->wValue) {
1612		case USB_DEVICE_REMOTE_WAKEUP:
1613			udc->remote_wakeup = 1;
1614			break;
1615		case USB_DEVICE_TEST_MODE:
1616			if (setup->wIndex & 0xFF
1617				||  udc->gadget.speed != USB_SPEED_HIGH)
1618				ep0_stall(udc);
1619
1620			if (udc->usb_state != USB_STATE_CONFIGURED
1621				&& udc->usb_state != USB_STATE_ADDRESS
1622				&& udc->usb_state != USB_STATE_DEFAULT)
1623				ep0_stall(udc);
1624
1625			mv_udc_testmode(udc, (setup->wIndex >> 8));
1626			goto out;
1627		default:
1628			goto out;
1629		}
1630	} else if ((setup->bRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))
1631		== ((USB_TYPE_STANDARD | USB_RECIP_ENDPOINT))) {
1632		switch (setup->wValue) {
1633		case USB_ENDPOINT_HALT:
1634			ep_num = setup->wIndex & USB_ENDPOINT_NUMBER_MASK;
1635			direction = (setup->wIndex & USB_ENDPOINT_DIR_MASK)
1636				? EP_DIR_IN : EP_DIR_OUT;
1637			if (setup->wValue != 0 || setup->wLength != 0
1638				|| ep_num > udc->max_eps)
1639				goto out;
1640			spin_unlock(&udc->lock);
1641			ep_set_stall(udc, ep_num, direction, 1);
1642			spin_lock(&udc->lock);
1643			break;
1644		default:
1645			goto out;
1646		}
1647	} else
1648		goto out;
1649
1650	if (udc_prime_status(udc, EP_DIR_IN, 0, true))
1651		ep0_stall(udc);
1652out:
1653	return;
1654}
1655
1656static void handle_setup_packet(struct mv_udc *udc, u8 ep_num,
1657	struct usb_ctrlrequest *setup)
1658	__releases(&ep->udc->lock)
1659	__acquires(&ep->udc->lock)
1660{
1661	bool delegate = false;
1662
1663	nuke(&udc->eps[ep_num * 2 + EP_DIR_OUT], -ESHUTDOWN);
1664
1665	dev_dbg(&udc->dev->dev, "SETUP %02x.%02x v%04x i%04x l%04x\n",
1666			setup->bRequestType, setup->bRequest,
1667			setup->wValue, setup->wIndex, setup->wLength);
1668	/* We process some standard setup requests here */
1669	if ((setup->bRequestType & USB_TYPE_MASK) == USB_TYPE_STANDARD) {
1670		switch (setup->bRequest) {
1671		case USB_REQ_GET_STATUS:
1672			ch9getstatus(udc, ep_num, setup);
1673			break;
1674
1675		case USB_REQ_SET_ADDRESS:
1676			ch9setaddress(udc, setup);
1677			break;
1678
1679		case USB_REQ_CLEAR_FEATURE:
1680			ch9clearfeature(udc, setup);
1681			break;
1682
1683		case USB_REQ_SET_FEATURE:
1684			ch9setfeature(udc, setup);
1685			break;
1686
1687		default:
1688			delegate = true;
1689		}
1690	} else
1691		delegate = true;
1692
1693	/* delegate USB standard requests to the gadget driver */
1694	if (delegate == true) {
1695		/* USB requests handled by gadget */
1696		if (setup->wLength) {
1697			/* DATA phase from gadget, STATUS phase from udc */
1698			udc->ep0_dir = (setup->bRequestType & USB_DIR_IN)
1699					?  EP_DIR_IN : EP_DIR_OUT;
1700			spin_unlock(&udc->lock);
1701			if (udc->driver->setup(&udc->gadget,
1702				&udc->local_setup_buff) < 0)
1703				ep0_stall(udc);
1704			spin_lock(&udc->lock);
1705			udc->ep0_state = (setup->bRequestType & USB_DIR_IN)
1706					?  DATA_STATE_XMIT : DATA_STATE_RECV;
1707		} else {
1708			/* no DATA phase, IN STATUS phase from gadget */
1709			udc->ep0_dir = EP_DIR_IN;
1710			spin_unlock(&udc->lock);
1711			if (udc->driver->setup(&udc->gadget,
1712				&udc->local_setup_buff) < 0)
1713				ep0_stall(udc);
1714			spin_lock(&udc->lock);
1715			udc->ep0_state = WAIT_FOR_OUT_STATUS;
1716		}
1717	}
1718}
1719
1720/* complete DATA or STATUS phase of ep0 prime status phase if needed */
1721static void ep0_req_complete(struct mv_udc *udc,
1722	struct mv_ep *ep0, struct mv_req *req)
1723{
1724	u32 new_addr;
1725
1726	if (udc->usb_state == USB_STATE_ADDRESS) {
1727		/* set the new address */
1728		new_addr = (u32)udc->dev_addr;
1729		writel(new_addr << USB_DEVICE_ADDRESS_BIT_SHIFT,
1730			&udc->op_regs->deviceaddr);
1731	}
1732
1733	done(ep0, req, 0);
1734
1735	switch (udc->ep0_state) {
1736	case DATA_STATE_XMIT:
1737		/* receive status phase */
1738		if (udc_prime_status(udc, EP_DIR_OUT, 0, true))
1739			ep0_stall(udc);
1740		break;
1741	case DATA_STATE_RECV:
1742		/* send status phase */
1743		if (udc_prime_status(udc, EP_DIR_IN, 0 , true))
1744			ep0_stall(udc);
1745		break;
1746	case WAIT_FOR_OUT_STATUS:
1747		udc->ep0_state = WAIT_FOR_SETUP;
1748		break;
1749	case WAIT_FOR_SETUP:
1750		dev_err(&udc->dev->dev, "unexpect ep0 packets\n");
1751		break;
1752	default:
1753		ep0_stall(udc);
1754		break;
1755	}
1756}
1757
1758static void get_setup_data(struct mv_udc *udc, u8 ep_num, u8 *buffer_ptr)
1759{
1760	u32 temp;
1761	struct mv_dqh *dqh;
1762
1763	dqh = &udc->ep_dqh[ep_num * 2 + EP_DIR_OUT];
1764
1765	/* Clear bit in ENDPTSETUPSTAT */
1766	writel((1 << ep_num), &udc->op_regs->epsetupstat);
1767
1768	/* while a hazard exists when setup package arrives */
1769	do {
1770		/* Set Setup Tripwire */
1771		temp = readl(&udc->op_regs->usbcmd);
1772		writel(temp | USBCMD_SETUP_TRIPWIRE_SET, &udc->op_regs->usbcmd);
1773
1774		/* Copy the setup packet to local buffer */
1775		memcpy(buffer_ptr, (u8 *) dqh->setup_buffer, 8);
1776	} while (!(readl(&udc->op_regs->usbcmd) & USBCMD_SETUP_TRIPWIRE_SET));
1777
1778	/* Clear Setup Tripwire */
1779	temp = readl(&udc->op_regs->usbcmd);
1780	writel(temp & ~USBCMD_SETUP_TRIPWIRE_SET, &udc->op_regs->usbcmd);
1781}
1782
1783static void irq_process_tr_complete(struct mv_udc *udc)
1784{
1785	u32 tmp, bit_pos;
1786	int i, ep_num = 0, direction = 0;
1787	struct mv_ep	*curr_ep;
1788	struct mv_req *curr_req, *temp_req;
1789	int status;
1790
1791	/*
1792	 * We use separate loops for ENDPTSETUPSTAT and ENDPTCOMPLETE
1793	 * because the setup packets are to be read ASAP
1794	 */
1795
1796	/* Process all Setup packet received interrupts */
1797	tmp = readl(&udc->op_regs->epsetupstat);
1798
1799	if (tmp) {
1800		for (i = 0; i < udc->max_eps; i++) {
1801			if (tmp & (1 << i)) {
1802				get_setup_data(udc, i,
1803					(u8 *)(&udc->local_setup_buff));
1804				handle_setup_packet(udc, i,
1805					&udc->local_setup_buff);
1806			}
1807		}
1808	}
1809
1810	/* Don't clear the endpoint setup status register here.
1811	 * It is cleared as a setup packet is read out of the buffer
1812	 */
1813
1814	/* Process non-setup transaction complete interrupts */
1815	tmp = readl(&udc->op_regs->epcomplete);
1816
1817	if (!tmp)
1818		return;
1819
1820	writel(tmp, &udc->op_regs->epcomplete);
1821
1822	for (i = 0; i < udc->max_eps * 2; i++) {
1823		ep_num = i >> 1;
1824		direction = i % 2;
1825
1826		bit_pos = 1 << (ep_num + 16 * direction);
1827
1828		if (!(bit_pos & tmp))
1829			continue;
1830
1831		if (i == 1)
1832			curr_ep = &udc->eps[0];
1833		else
1834			curr_ep = &udc->eps[i];
1835		/* process the req queue until an uncomplete request */
1836		list_for_each_entry_safe(curr_req, temp_req,
1837			&curr_ep->queue, queue) {
1838			status = process_ep_req(udc, i, curr_req);
1839			if (status)
1840				break;
1841
1842			/* write back status to req */
1843			curr_req->req.status = status;
1844
1845			/* ep0 request completion */
1846			if (ep_num == 0) {
1847				ep0_req_complete(udc, curr_ep, curr_req);
1848				break;
1849			} else {
1850				done(curr_ep, curr_req, status);
1851			}
1852		}
1853	}
1854}
1855
1856static void irq_process_reset(struct mv_udc *udc)
1857{
1858	u32 tmp;
1859	unsigned int loops;
1860
1861	udc->ep0_dir = EP_DIR_OUT;
1862	udc->ep0_state = WAIT_FOR_SETUP;
1863	udc->remote_wakeup = 0;		/* default to 0 on reset */
1864
1865	/* The address bits are past bit 25-31. Set the address */
1866	tmp = readl(&udc->op_regs->deviceaddr);
1867	tmp &= ~(USB_DEVICE_ADDRESS_MASK);
1868	writel(tmp, &udc->op_regs->deviceaddr);
1869
1870	/* Clear all the setup token semaphores */
1871	tmp = readl(&udc->op_regs->epsetupstat);
1872	writel(tmp, &udc->op_regs->epsetupstat);
1873
1874	/* Clear all the endpoint complete status bits */
1875	tmp = readl(&udc->op_regs->epcomplete);
1876	writel(tmp, &udc->op_regs->epcomplete);
1877
1878	/* wait until all endptprime bits cleared */
1879	loops = LOOPS(PRIME_TIMEOUT);
1880	while (readl(&udc->op_regs->epprime) & 0xFFFFFFFF) {
1881		if (loops == 0) {
1882			dev_err(&udc->dev->dev,
1883				"Timeout for ENDPTPRIME = 0x%x\n",
1884				readl(&udc->op_regs->epprime));
1885			break;
1886		}
1887		loops--;
1888		udelay(LOOPS_USEC);
1889	}
1890
1891	/* Write 1s to the Flush register */
1892	writel((u32)~0, &udc->op_regs->epflush);
1893
1894	if (readl(&udc->op_regs->portsc[0]) & PORTSCX_PORT_RESET) {
1895		dev_info(&udc->dev->dev, "usb bus reset\n");
1896		udc->usb_state = USB_STATE_DEFAULT;
1897		/* reset all the queues, stop all USB activities */
1898		gadget_reset(udc, udc->driver);
1899	} else {
1900		dev_info(&udc->dev->dev, "USB reset portsc 0x%x\n",
1901			readl(&udc->op_regs->portsc));
1902
1903		/*
1904		 * re-initialize
1905		 * controller reset
1906		 */
1907		udc_reset(udc);
1908
1909		/* reset all the queues, stop all USB activities */
1910		stop_activity(udc, udc->driver);
1911
1912		/* reset ep0 dQH and endptctrl */
1913		ep0_reset(udc);
1914
1915		/* enable interrupt and set controller to run state */
1916		udc_start(udc);
1917
1918		udc->usb_state = USB_STATE_ATTACHED;
1919	}
1920}
1921
1922static void handle_bus_resume(struct mv_udc *udc)
1923{
1924	udc->usb_state = udc->resume_state;
1925	udc->resume_state = 0;
1926
1927	/* report resume to the driver */
1928	if (udc->driver) {
1929		if (udc->driver->resume) {
1930			spin_unlock(&udc->lock);
1931			udc->driver->resume(&udc->gadget);
1932			spin_lock(&udc->lock);
1933		}
1934	}
1935}
1936
1937static void irq_process_suspend(struct mv_udc *udc)
1938{
1939	udc->resume_state = udc->usb_state;
1940	udc->usb_state = USB_STATE_SUSPENDED;
1941
1942	if (udc->driver->suspend) {
1943		spin_unlock(&udc->lock);
1944		udc->driver->suspend(&udc->gadget);
1945		spin_lock(&udc->lock);
1946	}
1947}
1948
1949static void irq_process_port_change(struct mv_udc *udc)
1950{
1951	u32 portsc;
1952
1953	portsc = readl(&udc->op_regs->portsc[0]);
1954	if (!(portsc & PORTSCX_PORT_RESET)) {
1955		/* Get the speed */
1956		u32 speed = portsc & PORTSCX_PORT_SPEED_MASK;
1957		switch (speed) {
1958		case PORTSCX_PORT_SPEED_HIGH:
1959			udc->gadget.speed = USB_SPEED_HIGH;
1960			break;
1961		case PORTSCX_PORT_SPEED_FULL:
1962			udc->gadget.speed = USB_SPEED_FULL;
1963			break;
1964		case PORTSCX_PORT_SPEED_LOW:
1965			udc->gadget.speed = USB_SPEED_LOW;
1966			break;
1967		default:
1968			udc->gadget.speed = USB_SPEED_UNKNOWN;
1969			break;
1970		}
1971	}
1972
1973	if (portsc & PORTSCX_PORT_SUSPEND) {
1974		udc->resume_state = udc->usb_state;
1975		udc->usb_state = USB_STATE_SUSPENDED;
1976		if (udc->driver->suspend) {
1977			spin_unlock(&udc->lock);
1978			udc->driver->suspend(&udc->gadget);
1979			spin_lock(&udc->lock);
1980		}
1981	}
1982
1983	if (!(portsc & PORTSCX_PORT_SUSPEND)
1984		&& udc->usb_state == USB_STATE_SUSPENDED) {
1985		handle_bus_resume(udc);
1986	}
1987
1988	if (!udc->resume_state)
1989		udc->usb_state = USB_STATE_DEFAULT;
1990}
1991
1992static void irq_process_error(struct mv_udc *udc)
1993{
1994	/* Increment the error count */
1995	udc->errors++;
1996}
1997
1998static irqreturn_t mv_udc_irq(int irq, void *dev)
1999{
2000	struct mv_udc *udc = (struct mv_udc *)dev;
2001	u32 status, intr;
2002
2003	/* Disable ISR when stopped bit is set */
2004	if (udc->stopped)
2005		return IRQ_NONE;
2006
2007	spin_lock(&udc->lock);
2008
2009	status = readl(&udc->op_regs->usbsts);
2010	intr = readl(&udc->op_regs->usbintr);
2011	status &= intr;
2012
2013	if (status == 0) {
2014		spin_unlock(&udc->lock);
2015		return IRQ_NONE;
2016	}
2017
2018	/* Clear all the interrupts occurred */
2019	writel(status, &udc->op_regs->usbsts);
2020
2021	if (status & USBSTS_ERR)
2022		irq_process_error(udc);
2023
2024	if (status & USBSTS_RESET)
2025		irq_process_reset(udc);
2026
2027	if (status & USBSTS_PORT_CHANGE)
2028		irq_process_port_change(udc);
2029
2030	if (status & USBSTS_INT)
2031		irq_process_tr_complete(udc);
2032
2033	if (status & USBSTS_SUSPEND)
2034		irq_process_suspend(udc);
2035
2036	spin_unlock(&udc->lock);
2037
2038	return IRQ_HANDLED;
2039}
2040
2041static irqreturn_t mv_udc_vbus_irq(int irq, void *dev)
2042{
2043	struct mv_udc *udc = (struct mv_udc *)dev;
2044
2045	/* polling VBUS and init phy may cause too much time*/
2046	if (udc->qwork)
2047		queue_work(udc->qwork, &udc->vbus_work);
2048
2049	return IRQ_HANDLED;
2050}
2051
2052static void mv_udc_vbus_work(struct work_struct *work)
2053{
2054	struct mv_udc *udc;
2055	unsigned int vbus;
2056
2057	udc = container_of(work, struct mv_udc, vbus_work);
2058	if (!udc->pdata->vbus)
2059		return;
2060
2061	vbus = udc->pdata->vbus->poll();
2062	dev_info(&udc->dev->dev, "vbus is %d\n", vbus);
2063
2064	if (vbus == VBUS_HIGH)
2065		mv_udc_vbus_session(&udc->gadget, 1);
2066	else if (vbus == VBUS_LOW)
2067		mv_udc_vbus_session(&udc->gadget, 0);
2068}
2069
2070/* release device structure */
2071static void gadget_release(struct device *_dev)
2072{
2073	struct mv_udc *udc;
2074
2075	udc = dev_get_drvdata(_dev);
2076
2077	complete(udc->done);
2078}
2079
2080static void mv_udc_remove(struct platform_device *pdev)
2081{
2082	struct mv_udc *udc;
2083
2084	udc = platform_get_drvdata(pdev);
2085
2086	usb_del_gadget_udc(&udc->gadget);
2087
2088	if (udc->qwork)
2089		destroy_workqueue(udc->qwork);
2090
2091	/* free memory allocated in probe */
2092	dma_pool_destroy(udc->dtd_pool);
2093
2094	if (udc->ep_dqh)
2095		dma_free_coherent(&pdev->dev, udc->ep_dqh_size,
2096			udc->ep_dqh, udc->ep_dqh_dma);
2097
2098	mv_udc_disable(udc);
2099
2100	/* free dev, wait for the release() finished */
2101	wait_for_completion(udc->done);
 
 
2102}
2103
2104static int mv_udc_probe(struct platform_device *pdev)
2105{
2106	struct mv_usb_platform_data *pdata = dev_get_platdata(&pdev->dev);
2107	struct mv_udc *udc;
2108	int retval = 0;
2109	struct resource *r;
2110	size_t size;
2111
2112	if (pdata == NULL) {
2113		dev_err(&pdev->dev, "missing platform_data\n");
2114		return -ENODEV;
2115	}
2116
2117	udc = devm_kzalloc(&pdev->dev, sizeof(*udc), GFP_KERNEL);
2118	if (udc == NULL)
2119		return -ENOMEM;
2120
2121	udc->done = &release_done;
2122	udc->pdata = dev_get_platdata(&pdev->dev);
2123	spin_lock_init(&udc->lock);
2124
2125	udc->dev = pdev;
2126
2127	if (pdata->mode == MV_USB_MODE_OTG) {
2128		udc->transceiver = devm_usb_get_phy(&pdev->dev,
2129					USB_PHY_TYPE_USB2);
2130		if (IS_ERR(udc->transceiver)) {
2131			retval = PTR_ERR(udc->transceiver);
2132
2133			if (retval == -ENXIO)
2134				return retval;
2135
2136			udc->transceiver = NULL;
2137			return -EPROBE_DEFER;
2138		}
2139	}
2140
2141	/* udc only have one sysclk. */
2142	udc->clk = devm_clk_get(&pdev->dev, NULL);
2143	if (IS_ERR(udc->clk))
2144		return PTR_ERR(udc->clk);
2145
2146	r = platform_get_resource_byname(udc->dev, IORESOURCE_MEM, "capregs");
2147	if (r == NULL) {
2148		dev_err(&pdev->dev, "no I/O memory resource defined\n");
2149		return -ENODEV;
2150	}
2151
2152	udc->cap_regs = (struct mv_cap_regs __iomem *)
2153		devm_ioremap(&pdev->dev, r->start, resource_size(r));
2154	if (udc->cap_regs == NULL) {
2155		dev_err(&pdev->dev, "failed to map I/O memory\n");
2156		return -EBUSY;
2157	}
2158
2159	r = platform_get_resource_byname(udc->dev, IORESOURCE_MEM, "phyregs");
2160	if (r == NULL) {
2161		dev_err(&pdev->dev, "no phy I/O memory resource defined\n");
2162		return -ENODEV;
2163	}
2164
2165	udc->phy_regs = devm_ioremap(&pdev->dev, r->start, resource_size(r));
2166	if (udc->phy_regs == NULL) {
2167		dev_err(&pdev->dev, "failed to map phy I/O memory\n");
2168		return -EBUSY;
2169	}
2170
2171	/* we will acces controller register, so enable the clk */
2172	retval = mv_udc_enable_internal(udc);
2173	if (retval)
2174		return retval;
2175
2176	udc->op_regs =
2177		(struct mv_op_regs __iomem *)((unsigned long)udc->cap_regs
2178		+ (readl(&udc->cap_regs->caplength_hciversion)
2179			& CAPLENGTH_MASK));
2180	udc->max_eps = readl(&udc->cap_regs->dccparams) & DCCPARAMS_DEN_MASK;
2181
2182	/*
2183	 * some platform will use usb to download image, it may not disconnect
2184	 * usb gadget before loading kernel. So first stop udc here.
2185	 */
2186	udc_stop(udc);
2187	writel(0xFFFFFFFF, &udc->op_regs->usbsts);
2188
2189	size = udc->max_eps * sizeof(struct mv_dqh) *2;
2190	size = (size + DQH_ALIGNMENT - 1) & ~(DQH_ALIGNMENT - 1);
2191	udc->ep_dqh = dma_alloc_coherent(&pdev->dev, size,
2192					&udc->ep_dqh_dma, GFP_KERNEL);
2193
2194	if (udc->ep_dqh == NULL) {
2195		dev_err(&pdev->dev, "allocate dQH memory failed\n");
2196		retval = -ENOMEM;
2197		goto err_disable_clock;
2198	}
2199	udc->ep_dqh_size = size;
2200
2201	/* create dTD dma_pool resource */
2202	udc->dtd_pool = dma_pool_create("mv_dtd",
2203			&pdev->dev,
2204			sizeof(struct mv_dtd),
2205			DTD_ALIGNMENT,
2206			DMA_BOUNDARY);
2207
2208	if (!udc->dtd_pool) {
2209		retval = -ENOMEM;
2210		goto err_free_dma;
2211	}
2212
2213	size = udc->max_eps * sizeof(struct mv_ep) *2;
2214	udc->eps = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
2215	if (udc->eps == NULL) {
2216		retval = -ENOMEM;
2217		goto err_destroy_dma;
2218	}
2219
2220	/* initialize ep0 status request structure */
2221	udc->status_req = devm_kzalloc(&pdev->dev, sizeof(struct mv_req),
2222					GFP_KERNEL);
2223	if (!udc->status_req) {
2224		retval = -ENOMEM;
2225		goto err_destroy_dma;
2226	}
2227	INIT_LIST_HEAD(&udc->status_req->queue);
2228
2229	/* allocate a small amount of memory to get valid address */
2230	udc->status_req->req.buf = devm_kzalloc(&pdev->dev, 8, GFP_KERNEL);
2231	if (!udc->status_req->req.buf) {
2232		retval = -ENOMEM;
2233		goto err_destroy_dma;
2234	}
2235	udc->status_req->req.dma = DMA_ADDR_INVALID;
2236
2237	udc->resume_state = USB_STATE_NOTATTACHED;
2238	udc->usb_state = USB_STATE_POWERED;
2239	udc->ep0_dir = EP_DIR_OUT;
2240	udc->remote_wakeup = 0;
2241
2242	r = platform_get_resource(udc->dev, IORESOURCE_IRQ, 0);
2243	if (r == NULL) {
2244		dev_err(&pdev->dev, "no IRQ resource defined\n");
2245		retval = -ENODEV;
2246		goto err_destroy_dma;
2247	}
2248	udc->irq = r->start;
2249	if (devm_request_irq(&pdev->dev, udc->irq, mv_udc_irq,
2250		IRQF_SHARED, driver_name, udc)) {
2251		dev_err(&pdev->dev, "Request irq %d for UDC failed\n",
2252			udc->irq);
2253		retval = -ENODEV;
2254		goto err_destroy_dma;
2255	}
2256
2257	/* initialize gadget structure */
2258	udc->gadget.ops = &mv_ops;	/* usb_gadget_ops */
2259	udc->gadget.ep0 = &udc->eps[0].ep;	/* gadget ep0 */
2260	INIT_LIST_HEAD(&udc->gadget.ep_list);	/* ep_list */
2261	udc->gadget.speed = USB_SPEED_UNKNOWN;	/* speed */
2262	udc->gadget.max_speed = USB_SPEED_HIGH;	/* support dual speed */
2263
2264	/* the "gadget" abstracts/virtualizes the controller */
2265	udc->gadget.name = driver_name;		/* gadget name */
2266
2267	eps_init(udc);
2268
2269	/* VBUS detect: we can disable/enable clock on demand.*/
2270	if (udc->transceiver)
2271		udc->clock_gating = 1;
2272	else if (pdata->vbus) {
2273		udc->clock_gating = 1;
2274		retval = devm_request_threaded_irq(&pdev->dev,
2275				pdata->vbus->irq, NULL,
2276				mv_udc_vbus_irq, IRQF_ONESHOT, "vbus", udc);
2277		if (retval) {
2278			dev_info(&pdev->dev,
2279				"Can not request irq for VBUS, "
2280				"disable clock gating\n");
2281			udc->clock_gating = 0;
2282		}
2283
2284		udc->qwork = create_singlethread_workqueue("mv_udc_queue");
2285		if (!udc->qwork) {
2286			dev_err(&pdev->dev, "cannot create workqueue\n");
2287			retval = -ENOMEM;
2288			goto err_destroy_dma;
2289		}
2290
2291		INIT_WORK(&udc->vbus_work, mv_udc_vbus_work);
2292	}
2293
2294	/*
2295	 * When clock gating is supported, we can disable clk and phy.
2296	 * If not, it means that VBUS detection is not supported, we
2297	 * have to enable vbus active all the time to let controller work.
2298	 */
2299	if (udc->clock_gating)
2300		mv_udc_disable_internal(udc);
2301	else
2302		udc->vbus_active = 1;
2303
2304	retval = usb_add_gadget_udc_release(&pdev->dev, &udc->gadget,
2305			gadget_release);
2306	if (retval)
2307		goto err_create_workqueue;
2308
2309	platform_set_drvdata(pdev, udc);
2310	dev_info(&pdev->dev, "successful probe UDC device %s clock gating.\n",
2311		udc->clock_gating ? "with" : "without");
2312
2313	return 0;
2314
2315err_create_workqueue:
2316	if (udc->qwork)
2317		destroy_workqueue(udc->qwork);
2318err_destroy_dma:
2319	dma_pool_destroy(udc->dtd_pool);
2320err_free_dma:
2321	dma_free_coherent(&pdev->dev, udc->ep_dqh_size,
2322			udc->ep_dqh, udc->ep_dqh_dma);
2323err_disable_clock:
2324	mv_udc_disable_internal(udc);
2325
2326	return retval;
2327}
2328
2329#ifdef CONFIG_PM
2330static int mv_udc_suspend(struct device *dev)
2331{
2332	struct mv_udc *udc;
2333
2334	udc = dev_get_drvdata(dev);
2335
2336	/* if OTG is enabled, the following will be done in OTG driver*/
2337	if (udc->transceiver)
2338		return 0;
2339
2340	if (udc->pdata->vbus && udc->pdata->vbus->poll)
2341		if (udc->pdata->vbus->poll() == VBUS_HIGH) {
2342			dev_info(&udc->dev->dev, "USB cable is connected!\n");
2343			return -EAGAIN;
2344		}
2345
2346	/*
2347	 * only cable is unplugged, udc can suspend.
2348	 * So do not care about clock_gating == 1.
2349	 */
2350	if (!udc->clock_gating) {
2351		udc_stop(udc);
2352
2353		spin_lock_irq(&udc->lock);
2354		/* stop all usb activities */
2355		stop_activity(udc, udc->driver);
2356		spin_unlock_irq(&udc->lock);
2357
2358		mv_udc_disable_internal(udc);
2359	}
2360
2361	return 0;
2362}
2363
2364static int mv_udc_resume(struct device *dev)
2365{
2366	struct mv_udc *udc;
2367	int retval;
2368
2369	udc = dev_get_drvdata(dev);
2370
2371	/* if OTG is enabled, the following will be done in OTG driver*/
2372	if (udc->transceiver)
2373		return 0;
2374
2375	if (!udc->clock_gating) {
2376		retval = mv_udc_enable_internal(udc);
2377		if (retval)
2378			return retval;
2379
2380		if (udc->driver && udc->softconnect) {
2381			udc_reset(udc);
2382			ep0_reset(udc);
2383			udc_start(udc);
2384		}
2385	}
2386
2387	return 0;
2388}
2389
2390static const struct dev_pm_ops mv_udc_pm_ops = {
2391	.suspend	= mv_udc_suspend,
2392	.resume		= mv_udc_resume,
2393};
2394#endif
2395
2396static void mv_udc_shutdown(struct platform_device *pdev)
2397{
2398	struct mv_udc *udc;
2399	u32 mode;
2400
2401	udc = platform_get_drvdata(pdev);
2402	/* reset controller mode to IDLE */
2403	mv_udc_enable(udc);
2404	mode = readl(&udc->op_regs->usbmode);
2405	mode &= ~3;
2406	writel(mode, &udc->op_regs->usbmode);
2407	mv_udc_disable(udc);
2408}
2409
2410static struct platform_driver udc_driver = {
2411	.probe		= mv_udc_probe,
2412	.remove_new	= mv_udc_remove,
2413	.shutdown	= mv_udc_shutdown,
2414	.driver		= {
2415		.name	= "mv-udc",
2416#ifdef CONFIG_PM
2417		.pm	= &mv_udc_pm_ops,
2418#endif
2419	},
2420};
2421
2422module_platform_driver(udc_driver);
2423MODULE_ALIAS("platform:mv-udc");
2424MODULE_DESCRIPTION(DRIVER_DESC);
2425MODULE_AUTHOR("Chao Xie <chao.xie@marvell.com>");
2426MODULE_LICENSE("GPL");