1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Cadence USBSS DRD Driver - gadget side.
   4 *
   5 * Copyright (C) 2018-2019 Cadence Design Systems.
   6 * Copyright (C) 2017-2018 NXP
   7 *
   8 * Authors: Pawel Jez <pjez@cadence.com>,
   9 *          Pawel Laszczak <pawell@cadence.com>
  10 *          Peter Chen <peter.chen@nxp.com>
  11 */
  12
  13/*
  14 * Work around 1:
  15 * At some situations, the controller may get stale data address in TRB
  16 * at below sequences:
  17 * 1. Controller read TRB includes data address
  18 * 2. Software updates TRBs includes data address and Cycle bit
  19 * 3. Controller read TRB which includes Cycle bit
  20 * 4. DMA run with stale data address
  21 *
  22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23 * After preparing all TRBs driver needs to check the position of DMA and
  24 * if the DMA point to the first just added TRB and doorbell is 1,
  25 * then driver must defer making this TRB as valid. This TRB will be make
  26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27 * interrupt.
  28 *
  29 * Issue has been fixed in DEV_VER_V3 version of controller.
  30 *
  31 * Work around 2:
  32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
  33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  34 * in correct order. If the first packet in the buffer will not be handled,
  35 * then the following packets directed for other endpoints and  functions
  36 * will be blocked.
  37 * Additionally the packets directed to one endpoint can block entire on-chip
  38 * buffers. In this case transfer to other endpoints also will blocked.
  39 *
  40 * To resolve this issue after raising the descriptor missing interrupt
  41 * driver prepares internal usb_request object and use it to arm DMA transfer.
  42 *
  43 * The problematic situation was observed in case when endpoint has been enabled
  44 * but no usb_request were queued. Driver try detects such endpoints and will
  45 * use this workaround only for these endpoint.
  46 *
  47 * Driver use limited number of buffer. This number can be set by macro
  48 * CDNS3_WA2_NUM_BUFFERS.
  49 *
  50 * Such blocking situation was observed on ACM gadget. For this function
  51 * host send OUT data packet but ACM function is not prepared for this packet.
  52 * It's cause that buffer placed in on chip memory block transfer to other
  53 * endpoints.
  54 *
  55 * Issue has been fixed in DEV_VER_V2 version of controller.
  56 *
  57 */
  58
  59#include <linux/dma-mapping.h>
  60#include <linux/usb/gadget.h>
  61#include <linux/module.h>
  62#include <linux/dmapool.h>
  63#include <linux/iopoll.h>
  64
  65#include "core.h"
  66#include "gadget-export.h"
  67#include "cdns3-gadget.h"
  68#include "cdns3-trace.h"
  69#include "drd.h"
  70
  71static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  72				   struct usb_request *request,
  73				   gfp_t gfp_flags);
  74
  75static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
  76				 struct usb_request *request);
  77
  78static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
  79					struct usb_request *request);
  80
  81/**
  82 * cdns3_clear_register_bit - clear bit in given register.
  83 * @ptr: address of device controller register to be read and changed
  84 * @mask: bits requested to clar
  85 */
  86static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
  87{
  88	mask = readl(ptr) & ~mask;
  89	writel(mask, ptr);
  90}
  91
  92/**
  93 * cdns3_set_register_bit - set bit in given register.
  94 * @ptr: address of device controller register to be read and changed
  95 * @mask: bits requested to set
  96 */
  97void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  98{
  99	mask = readl(ptr) | mask;
 100	writel(mask, ptr);
 101}
 102
 103/**
 104 * cdns3_ep_addr_to_index - Macro converts endpoint address to
 105 * index of endpoint object in cdns3_device.eps[] container
 106 * @ep_addr: endpoint address for which endpoint object is required
 107 *
 108 */
 109u8 cdns3_ep_addr_to_index(u8 ep_addr)
 110{
 111	return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
 112}
 113
 114static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
 115			     struct cdns3_endpoint *priv_ep)
 116{
 117	int dma_index;
 118
 119	dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
 120
 121	return dma_index / TRB_SIZE;
 122}
 123
 124/**
 125 * cdns3_next_request - returns next request from list
 126 * @list: list containing requests
 127 *
 128 * Returns request or NULL if no requests in list
 129 */
 130struct usb_request *cdns3_next_request(struct list_head *list)
 131{
 132	return list_first_entry_or_null(list, struct usb_request, list);
 133}
 134
 135/**
 136 * cdns3_next_align_buf - returns next buffer from list
 137 * @list: list containing buffers
 138 *
 139 * Returns buffer or NULL if no buffers in list
 140 */
 141static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
 142{
 143	return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 144}
 145
 146/**
 147 * cdns3_next_priv_request - returns next request from list
 148 * @list: list containing requests
 149 *
 150 * Returns request or NULL if no requests in list
 151 */
 152static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
 153{
 154	return list_first_entry_or_null(list, struct cdns3_request, list);
 155}
 156
 157/**
 158 * cdns3_select_ep - selects endpoint
 159 * @priv_dev:  extended gadget object
 160 * @ep: endpoint address
 161 */
 162void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 163{
 164	if (priv_dev->selected_ep == ep)
 165		return;
 166
 167	priv_dev->selected_ep = ep;
 168	writel(ep, &priv_dev->regs->ep_sel);
 169}
 170
 171/**
 172 * cdns3_get_tdl - gets current tdl for selected endpoint.
 173 * @priv_dev:  extended gadget object
 174 *
 175 * Before calling this function the appropriate endpoint must
 176 * be selected by means of cdns3_select_ep function.
 177 */
 178static int cdns3_get_tdl(struct cdns3_device *priv_dev)
 179{
 180	if (priv_dev->dev_ver < DEV_VER_V3)
 181		return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 182	else
 183		return readl(&priv_dev->regs->ep_tdl);
 184}
 185
 186dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 187				 struct cdns3_trb *trb)
 188{
 189	u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 190
 191	return priv_ep->trb_pool_dma + offset;
 192}
 193
 194static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 195{
 196	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 197
 198	if (priv_ep->trb_pool) {
 199		dma_pool_free(priv_dev->eps_dma_pool,
 200			      priv_ep->trb_pool, priv_ep->trb_pool_dma);
 201		priv_ep->trb_pool = NULL;
 202	}
 203}
 204
 205/**
 206 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 207 * @priv_ep:  endpoint object
 208 *
 209 * Function will return 0 on success or -ENOMEM on allocation error
 210 */
 211int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 212{
 213	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 214	int ring_size = TRB_RING_SIZE;
 215	int num_trbs = ring_size / TRB_SIZE;
 216	struct cdns3_trb *link_trb;
 217
 218	if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
 219		cdns3_free_trb_pool(priv_ep);
 220
 221	if (!priv_ep->trb_pool) {
 222		priv_ep->trb_pool = dma_pool_alloc(priv_dev->eps_dma_pool,
 223						   GFP_ATOMIC,
 224						   &priv_ep->trb_pool_dma);
 225
 226		if (!priv_ep->trb_pool)
 227			return -ENOMEM;
 228
 229		priv_ep->alloc_ring_size = ring_size;
 230	}
 231
 232	memset(priv_ep->trb_pool, 0, ring_size);
 233
 234	priv_ep->num_trbs = num_trbs;
 235
 236	if (!priv_ep->num)
 237		return 0;
 238
 239	/* Initialize the last TRB as Link TRB */
 240	link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 241
 242	if (priv_ep->use_streams) {
 243		/*
 244		 * For stream capable endpoints driver use single correct TRB.
 245		 * The last trb has zeroed cycle bit
 246		 */
 247		link_trb->control = 0;
 248	} else {
 249		link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma));
 250		link_trb->control = cpu_to_le32(TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE);
 251	}
 252	return 0;
 253}
 254
 255/**
 256 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 257 * @priv_ep: endpoint object
 258 *
 259 * Endpoint must be selected before call to this function
 260 */
 261static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 262{
 263	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 264	int val;
 265
 266	trace_cdns3_halt(priv_ep, 1, 1);
 267
 268	writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 269	       &priv_dev->regs->ep_cmd);
 270
 271	/* wait for DFLUSH cleared */
 272	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 273				  !(val & EP_CMD_DFLUSH), 1, 1000);
 274	priv_ep->flags |= EP_STALLED;
 275	priv_ep->flags &= ~EP_STALL_PENDING;
 276}
 277
 278/**
 279 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 280 * @priv_dev: extended gadget object
 281 */
 282void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 283{
 284	int i;
 285
 286	writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 287
 288	cdns3_allow_enable_l1(priv_dev, 0);
 289	priv_dev->hw_configured_flag = 0;
 290	priv_dev->onchip_used_size = 0;
 291	priv_dev->out_mem_is_allocated = 0;
 292	priv_dev->wait_for_setup = 0;
 293	priv_dev->using_streams = 0;
 294
 295	for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
 296		if (priv_dev->eps[i])
 297			priv_dev->eps[i]->flags &= ~EP_CONFIGURED;
 298}
 299
 300/**
 301 * cdns3_ep_inc_trb - increment a trb index.
 302 * @index: Pointer to the TRB index to increment.
 303 * @cs: Cycle state
 304 * @trb_in_seg: number of TRBs in segment
 305 *
 306 * The index should never point to the link TRB. After incrementing,
 307 * if it is point to the link TRB, wrap around to the beginning and revert
 308 * cycle state bit The
 309 * link TRB is always at the last TRB entry.
 310 */
 311static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 312{
 313	(*index)++;
 314	if (*index == (trb_in_seg - 1)) {
 315		*index = 0;
 316		*cs ^=  1;
 317	}
 318}
 319
 320/**
 321 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 322 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 323 */
 324static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 325{
 326	priv_ep->free_trbs--;
 327	cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 328}
 329
 330/**
 331 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 332 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 333 */
 334static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 335{
 336	priv_ep->free_trbs++;
 337	cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 338}
 339
 340/**
 341 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 342 * @priv_dev: Extended gadget object
 343 * @enable: Enable/disable permit to transition to L1.
 344 *
 345 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 346 * then controller answer with ACK handshake.
 347 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 348 * then controller answer with NYET handshake.
 349 */
 350void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 351{
 352	if (enable)
 353		writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 354	else
 355		writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 356}
 357
 358enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 359{
 360	u32 reg;
 361
 362	reg = readl(&priv_dev->regs->usb_sts);
 363
 364	if (DEV_SUPERSPEED(reg))
 365		return USB_SPEED_SUPER;
 366	else if (DEV_HIGHSPEED(reg))
 367		return USB_SPEED_HIGH;
 368	else if (DEV_FULLSPEED(reg))
 369		return USB_SPEED_FULL;
 370	else if (DEV_LOWSPEED(reg))
 371		return USB_SPEED_LOW;
 372	return USB_SPEED_UNKNOWN;
 373}
 374
 375/**
 376 * cdns3_start_all_request - add to ring all request not started
 377 * @priv_dev: Extended gadget object
 378 * @priv_ep: The endpoint for whom request will be started.
 379 *
 380 * Returns return ENOMEM if transfer ring i not enough TRBs to start
 381 *         all requests.
 382 */
 383static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 384				   struct cdns3_endpoint *priv_ep)
 385{
 386	struct usb_request *request;
 387	int ret = 0;
 388	u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 389
 390	/*
 391	 * If the last pending transfer is INTERNAL
 392	 * OR streams are enabled for this endpoint
 393	 * do NOT start new transfer till the last one is pending
 394	 */
 395	if (!pending_empty) {
 396		struct cdns3_request *priv_req;
 397
 398		request = cdns3_next_request(&priv_ep->pending_req_list);
 399		priv_req = to_cdns3_request(request);
 400		if ((priv_req->flags & REQUEST_INTERNAL) ||
 401		    (priv_ep->flags & EP_TDLCHK_EN) ||
 402			priv_ep->use_streams) {
 403			dev_dbg(priv_dev->dev, "Blocking external request\n");
 404			return ret;
 405		}
 406	}
 407
 408	while (!list_empty(&priv_ep->deferred_req_list)) {
 409		request = cdns3_next_request(&priv_ep->deferred_req_list);
 410
 411		if (!priv_ep->use_streams) {
 412			ret = cdns3_ep_run_transfer(priv_ep, request);
 413		} else {
 414			priv_ep->stream_sg_idx = 0;
 415			ret = cdns3_ep_run_stream_transfer(priv_ep, request);
 416		}
 417		if (ret)
 418			return ret;
 419
 420		list_move_tail(&request->list, &priv_ep->pending_req_list);
 421		if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
 422			break;
 423	}
 424
 425	priv_ep->flags &= ~EP_RING_FULL;
 426	return ret;
 427}
 428
 429/*
 430 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
 431 * driver try to detect whether endpoint need additional internal
 432 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
 433 * if before first DESCMISS interrupt the DMA will be armed.
 434 */
 435#define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
 436	if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
 437		priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
 438		(reg) |= EP_STS_EN_DESCMISEN; \
 439	} } while (0)
 440
 441static void __cdns3_descmiss_copy_data(struct usb_request *request,
 442	struct usb_request *descmiss_req)
 443{
 444	int length = request->actual + descmiss_req->actual;
 445	struct scatterlist *s = request->sg;
 446
 447	if (!s) {
 448		if (length <= request->length) {
 449			memcpy(&((u8 *)request->buf)[request->actual],
 450			       descmiss_req->buf,
 451			       descmiss_req->actual);
 452			request->actual = length;
 453		} else {
 454			/* It should never occures */
 455			request->status = -ENOMEM;
 456		}
 457	} else {
 458		if (length <= sg_dma_len(s)) {
 459			void *p = phys_to_virt(sg_dma_address(s));
 460
 461			memcpy(&((u8 *)p)[request->actual],
 462				descmiss_req->buf,
 463				descmiss_req->actual);
 464			request->actual = length;
 465		} else {
 466			request->status = -ENOMEM;
 467		}
 468	}
 469}
 470
 471/**
 472 * cdns3_wa2_descmiss_copy_data - copy data from internal requests to
 473 * request queued by class driver.
 474 * @priv_ep: extended endpoint object
 475 * @request: request object
 476 */
 477static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 478					 struct usb_request *request)
 479{
 480	struct usb_request *descmiss_req;
 481	struct cdns3_request *descmiss_priv_req;
 482
 483	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 484		int chunk_end;
 485
 486		descmiss_priv_req =
 487			cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 488		descmiss_req = &descmiss_priv_req->request;
 489
 490		/* driver can't touch pending request */
 491		if (descmiss_priv_req->flags & REQUEST_PENDING)
 492			break;
 493
 494		chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
 495		request->status = descmiss_req->status;
 496		__cdns3_descmiss_copy_data(request, descmiss_req);
 497		list_del_init(&descmiss_priv_req->list);
 498		kfree(descmiss_req->buf);
 499		cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
 500		--priv_ep->wa2_counter;
 501
 502		if (!chunk_end)
 503			break;
 504	}
 505}
 506
 507static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
 508						     struct cdns3_endpoint *priv_ep,
 509						     struct cdns3_request *priv_req)
 510{
 511	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 512	    priv_req->flags & REQUEST_INTERNAL) {
 513		struct usb_request *req;
 514
 515		req = cdns3_next_request(&priv_ep->deferred_req_list);
 516
 517		priv_ep->descmis_req = NULL;
 518
 519		if (!req)
 520			return NULL;
 521
 522		/* unmap the gadget request before copying data */
 523		usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
 524						priv_ep->dir);
 525
 526		cdns3_wa2_descmiss_copy_data(priv_ep, req);
 527		if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
 528		    req->length != req->actual) {
 529			/* wait for next part of transfer */
 530			/* re-map the gadget request buffer*/
 531			usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
 532				usb_endpoint_dir_in(priv_ep->endpoint.desc));
 533			return NULL;
 534		}
 535
 536		if (req->status == -EINPROGRESS)
 537			req->status = 0;
 538
 539		list_del_init(&req->list);
 540		cdns3_start_all_request(priv_dev, priv_ep);
 541		return req;
 542	}
 543
 544	return &priv_req->request;
 545}
 546
 547static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
 548				     struct cdns3_endpoint *priv_ep,
 549				     struct cdns3_request *priv_req)
 550{
 551	int deferred = 0;
 552
 553	/*
 554	 * If transfer was queued before DESCMISS appear than we
 555	 * can disable handling of DESCMISS interrupt. Driver assumes that it
 556	 * can disable special treatment for this endpoint.
 557	 */
 558	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 559		u32 reg;
 560
 561		cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
 562		priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 563		reg = readl(&priv_dev->regs->ep_sts_en);
 564		reg &= ~EP_STS_EN_DESCMISEN;
 565		trace_cdns3_wa2(priv_ep, "workaround disabled\n");
 566		writel(reg, &priv_dev->regs->ep_sts_en);
 567	}
 568
 569	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
 570		u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 571		u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
 572
 573		/*
 574		 *  DESCMISS transfer has been finished, so data will be
 575		 *  directly copied from internal allocated usb_request
 576		 *  objects.
 577		 */
 578		if (pending_empty && !descmiss_empty &&
 579		    !(priv_req->flags & REQUEST_INTERNAL)) {
 580			cdns3_wa2_descmiss_copy_data(priv_ep,
 581						     &priv_req->request);
 582
 583			trace_cdns3_wa2(priv_ep, "get internal stored data");
 584
 585			list_add_tail(&priv_req->request.list,
 586				      &priv_ep->pending_req_list);
 587			cdns3_gadget_giveback(priv_ep, priv_req,
 588					      priv_req->request.status);
 589
 590			/*
 591			 * Intentionally driver returns positive value as
 592			 * correct value. It informs that transfer has
 593			 * been finished.
 594			 */
 595			return EINPROGRESS;
 596		}
 597
 598		/*
 599		 * Driver will wait for completion DESCMISS transfer,
 600		 * before starts new, not DESCMISS transfer.
 601		 */
 602		if (!pending_empty && !descmiss_empty) {
 603			trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
 604			deferred = 1;
 605		}
 606
 607		if (priv_req->flags & REQUEST_INTERNAL)
 608			list_add_tail(&priv_req->list,
 609				      &priv_ep->wa2_descmiss_req_list);
 610	}
 611
 612	return deferred;
 613}
 614
 615static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
 616{
 617	struct cdns3_request *priv_req;
 618
 619	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 620		u8 chain;
 621
 622		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 623		chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
 624
 625		trace_cdns3_wa2(priv_ep, "removes eldest request");
 626
 627		kfree(priv_req->request.buf);
 628		list_del_init(&priv_req->list);
 629		cdns3_gadget_ep_free_request(&priv_ep->endpoint,
 630					     &priv_req->request);
 631		--priv_ep->wa2_counter;
 632
 633		if (!chain)
 634			break;
 635	}
 636}
 637
 638/**
 639 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
 640 * @priv_ep: extended gadget object
 641 *
 642 * This function is used only for WA2. For more information see Work around 2
 643 * description.
 644 */
 645static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
 646{
 647	struct cdns3_request *priv_req;
 648	struct usb_request *request;
 649	u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 650
 651	/* check for pending transfer */
 652	if (!pending_empty) {
 653		trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
 654		return;
 655	}
 656
 657	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 658		priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 659		priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
 660	}
 661
 662	trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
 663
 664	if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
 665		trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
 666		cdns3_wa2_remove_old_request(priv_ep);
 667	}
 668
 669	request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
 670						GFP_ATOMIC);
 671	if (!request)
 672		goto err;
 673
 674	priv_req = to_cdns3_request(request);
 675	priv_req->flags |= REQUEST_INTERNAL;
 676
 677	/* if this field is still assigned it indicate that transfer related
 678	 * with this request has not been finished yet. Driver in this
 679	 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
 680	 * flag to previous one. It will indicate that current request is
 681	 * part of the previous one.
 682	 */
 683	if (priv_ep->descmis_req)
 684		priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
 685
 686	priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
 687					GFP_ATOMIC);
 688	priv_ep->wa2_counter++;
 689
 690	if (!priv_req->request.buf) {
 691		cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 692		goto err;
 693	}
 694
 695	priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
 696	priv_ep->descmis_req = priv_req;
 697
 698	__cdns3_gadget_ep_queue(&priv_ep->endpoint,
 699				&priv_ep->descmis_req->request,
 700				GFP_ATOMIC);
 701
 702	return;
 703
 704err:
 705	dev_err(priv_ep->cdns3_dev->dev,
 706		"Failed: No sufficient memory for DESCMIS\n");
 707}
 708
 709static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
 710{
 711	u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 712
 713	if (tdl) {
 714		u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
 715
 716		writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
 717		       &priv_dev->regs->ep_cmd);
 718	}
 719}
 720
 721static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
 722{
 723	u32 ep_sts_reg;
 724
 725	/* select EP0-out */
 726	cdns3_select_ep(priv_dev, 0);
 727
 728	ep_sts_reg = readl(&priv_dev->regs->ep_sts);
 729
 730	if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
 731		u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
 732		struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
 733
 734		if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
 735		    outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
 736			u8 pending_empty = list_empty(&outq_ep->pending_req_list);
 737
 738			if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
 739			    (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
 740			    !pending_empty) {
 741			} else {
 742				u32 ep_sts_en_reg;
 743				u32 ep_cmd_reg;
 744
 745				cdns3_select_ep(priv_dev, outq_ep->num |
 746						outq_ep->dir);
 747				ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
 748				ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
 749
 750				outq_ep->flags |= EP_TDLCHK_EN;
 751				cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 752						       EP_CFG_TDL_CHK);
 753
 754				cdns3_wa2_enable_detection(priv_dev, outq_ep,
 755							   ep_sts_en_reg);
 756				writel(ep_sts_en_reg,
 757				       &priv_dev->regs->ep_sts_en);
 758				/* reset tdl value to zero */
 759				cdns3_wa2_reset_tdl(priv_dev);
 760				/*
 761				 * Memory barrier - Reset tdl before ringing the
 762				 * doorbell.
 763				 */
 764				wmb();
 765				if (EP_CMD_DRDY & ep_cmd_reg) {
 766					trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
 767
 768				} else {
 769					trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
 770					/*
 771					 * ring doorbell to generate DESCMIS irq
 772					 */
 773					writel(EP_CMD_DRDY,
 774					       &priv_dev->regs->ep_cmd);
 775				}
 776			}
 777		}
 778	}
 779}
 780
 781/**
 782 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 783 * @priv_ep: The endpoint to whom the request belongs to
 784 * @priv_req: The request we're giving back
 785 * @status: completion code for the request
 786 *
 787 * Must be called with controller's lock held and interrupts disabled. This
 788 * function will unmap @req and call its ->complete() callback to notify upper
 789 * layers that it has completed.
 790 */
 791void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 792			   struct cdns3_request *priv_req,
 793			   int status)
 794{
 795	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 796	struct usb_request *request = &priv_req->request;
 797
 798	list_del_init(&request->list);
 799
 800	if (request->status == -EINPROGRESS)
 801		request->status = status;
 802
 803	usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
 804					priv_ep->dir);
 805
 806	if ((priv_req->flags & REQUEST_UNALIGNED) &&
 807	    priv_ep->dir == USB_DIR_OUT && !request->status) {
 808		/* Make DMA buffer CPU accessible */
 809		dma_sync_single_for_cpu(priv_dev->sysdev,
 810			priv_req->aligned_buf->dma,
 811			priv_req->aligned_buf->size,
 812			priv_req->aligned_buf->dir);
 813		memcpy(request->buf, priv_req->aligned_buf->buf,
 814		       request->length);
 815	}
 816
 817	priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 818	/* All TRBs have finished, clear the counter */
 819	priv_req->finished_trb = 0;
 820	trace_cdns3_gadget_giveback(priv_req);
 821
 822	if (priv_dev->dev_ver < DEV_VER_V2) {
 823		request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
 824						    priv_req);
 825		if (!request)
 826			return;
 827	}
 828
 829	if (request->complete) {
 830		spin_unlock(&priv_dev->lock);
 831		usb_gadget_giveback_request(&priv_ep->endpoint,
 832					    request);
 833		spin_lock(&priv_dev->lock);
 834	}
 835
 836	if (request->buf == priv_dev->zlp_buf)
 837		cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 838}
 839
 840static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 841{
 842	/* Work around for stale data address in TRB*/
 843	if (priv_ep->wa1_set) {
 844		trace_cdns3_wa1(priv_ep, "restore cycle bit");
 845
 846		priv_ep->wa1_set = 0;
 847		priv_ep->wa1_trb_index = 0xFFFF;
 848		if (priv_ep->wa1_cycle_bit) {
 849			priv_ep->wa1_trb->control =
 850				priv_ep->wa1_trb->control | cpu_to_le32(0x1);
 851		} else {
 852			priv_ep->wa1_trb->control =
 853				priv_ep->wa1_trb->control & cpu_to_le32(~0x1);
 854		}
 855	}
 856}
 857
 858static void cdns3_free_aligned_request_buf(struct work_struct *work)
 859{
 860	struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
 861					aligned_buf_wq);
 862	struct cdns3_aligned_buf *buf, *tmp;
 863	unsigned long flags;
 864
 865	spin_lock_irqsave(&priv_dev->lock, flags);
 866
 867	list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 868		if (!buf->in_use) {
 869			list_del(&buf->list);
 870
 871			/*
 872			 * Re-enable interrupts to free DMA capable memory.
 873			 * Driver can't free this memory with disabled
 874			 * interrupts.
 875			 */
 876			spin_unlock_irqrestore(&priv_dev->lock, flags);
 877			dma_free_noncoherent(priv_dev->sysdev, buf->size,
 878					  buf->buf, buf->dma, buf->dir);
 879			kfree(buf);
 880			spin_lock_irqsave(&priv_dev->lock, flags);
 881		}
 882	}
 883
 884	spin_unlock_irqrestore(&priv_dev->lock, flags);
 885}
 886
 887static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 888{
 889	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 890	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 891	struct cdns3_aligned_buf *buf;
 892
 893	/* check if buffer is aligned to 8. */
 894	if (!((uintptr_t)priv_req->request.buf & 0x7))
 895		return 0;
 896
 897	buf = priv_req->aligned_buf;
 898
 899	if (!buf || priv_req->request.length > buf->size) {
 900		buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 901		if (!buf)
 902			return -ENOMEM;
 903
 904		buf->size = priv_req->request.length;
 905		buf->dir = usb_endpoint_dir_in(priv_ep->endpoint.desc) ?
 906			DMA_TO_DEVICE : DMA_FROM_DEVICE;
 907
 908		buf->buf = dma_alloc_noncoherent(priv_dev->sysdev,
 909					      buf->size,
 910					      &buf->dma,
 911					      buf->dir,
 912					      GFP_ATOMIC);
 913		if (!buf->buf) {
 914			kfree(buf);
 915			return -ENOMEM;
 916		}
 917
 918		if (priv_req->aligned_buf) {
 919			trace_cdns3_free_aligned_request(priv_req);
 920			priv_req->aligned_buf->in_use = 0;
 921			queue_work(system_freezable_wq,
 922				   &priv_dev->aligned_buf_wq);
 923		}
 924
 925		buf->in_use = 1;
 926		priv_req->aligned_buf = buf;
 927
 928		list_add_tail(&buf->list,
 929			      &priv_dev->aligned_buf_list);
 930	}
 931
 932	if (priv_ep->dir == USB_DIR_IN) {
 933		/* Make DMA buffer CPU accessible */
 934		dma_sync_single_for_cpu(priv_dev->sysdev,
 935			buf->dma, buf->size, buf->dir);
 936		memcpy(buf->buf, priv_req->request.buf,
 937		       priv_req->request.length);
 938	}
 939
 940	/* Transfer DMA buffer ownership back to device */
 941	dma_sync_single_for_device(priv_dev->sysdev,
 942			buf->dma, buf->size, buf->dir);
 943
 944	priv_req->flags |= REQUEST_UNALIGNED;
 945	trace_cdns3_prepare_aligned_request(priv_req);
 946
 947	return 0;
 948}
 949
 950static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 951				  struct cdns3_trb *trb)
 952{
 953	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 954
 955	if (!priv_ep->wa1_set) {
 956		u32 doorbell;
 957
 958		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 959
 960		if (doorbell) {
 961			priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 962			priv_ep->wa1_set = 1;
 963			priv_ep->wa1_trb = trb;
 964			priv_ep->wa1_trb_index = priv_ep->enqueue;
 965			trace_cdns3_wa1(priv_ep, "set guard");
 966			return 0;
 967		}
 968	}
 969	return 1;
 970}
 971
 972static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 973					     struct cdns3_endpoint *priv_ep)
 974{
 975	int dma_index;
 976	u32 doorbell;
 977
 978	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 979	dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 980
 981	if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 982		cdns3_wa1_restore_cycle_bit(priv_ep);
 983}
 984
 985static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
 986					struct usb_request *request)
 987{
 988	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 989	struct cdns3_request *priv_req;
 990	struct cdns3_trb *trb;
 991	dma_addr_t trb_dma;
 992	int address;
 993	u32 control;
 994	u32 length;
 995	u32 tdl;
 996	unsigned int sg_idx = priv_ep->stream_sg_idx;
 997
 998	priv_req = to_cdns3_request(request);
 999	address = priv_ep->endpoint.desc->bEndpointAddress;
1000
1001	priv_ep->flags |= EP_PENDING_REQUEST;
1002
1003	/* must allocate buffer aligned to 8 */
1004	if (priv_req->flags & REQUEST_UNALIGNED)
1005		trb_dma = priv_req->aligned_buf->dma;
1006	else
1007		trb_dma = request->dma;
1008
1009	/*  For stream capable endpoints driver use only single TD. */
1010	trb = priv_ep->trb_pool + priv_ep->enqueue;
1011	priv_req->start_trb = priv_ep->enqueue;
1012	priv_req->end_trb = priv_req->start_trb;
1013	priv_req->trb = trb;
1014
1015	cdns3_select_ep(priv_ep->cdns3_dev, address);
1016
1017	control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1018		  TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1019
1020	if (!request->num_sgs) {
1021		trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1022		length = request->length;
1023	} else {
1024		trb->buffer = cpu_to_le32(TRB_BUFFER(request->sg[sg_idx].dma_address));
1025		length = request->sg[sg_idx].length;
1026	}
1027
1028	tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1029
1030	trb->length = cpu_to_le32(TRB_BURST_LEN(16) | TRB_LEN(length));
1031
1032	/*
1033	 * For DEV_VER_V2 controller version we have enabled
1034	 * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1035	 * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1036	 */
1037	if (priv_dev->dev_ver >= DEV_VER_V2) {
1038		if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1039			trb->length |= cpu_to_le32(TRB_TDL_SS_SIZE(tdl));
1040	}
1041	priv_req->flags |= REQUEST_PENDING;
1042
1043	trb->control = cpu_to_le32(control);
1044
1045	trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1046
1047	/*
1048	 * Memory barrier - Cycle Bit must be set before trb->length  and
1049	 * trb->buffer fields.
1050	 */
1051	wmb();
1052
1053	/* always first element */
1054	writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1055	       &priv_dev->regs->ep_traddr);
1056
1057	if (!(priv_ep->flags & EP_STALLED)) {
1058		trace_cdns3_ring(priv_ep);
1059		/*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1060		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1061
1062		priv_ep->prime_flag = false;
1063
1064		/*
1065		 * Controller version DEV_VER_V2 tdl calculation
1066		 * is based on TRB
1067		 */
1068
1069		if (priv_dev->dev_ver < DEV_VER_V2)
1070			writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1071			       &priv_dev->regs->ep_cmd);
1072		else if (priv_dev->dev_ver > DEV_VER_V2)
1073			writel(tdl, &priv_dev->regs->ep_tdl);
1074
1075		priv_ep->last_stream_id = priv_req->request.stream_id;
1076		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1077		writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1078		       EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1079
1080		trace_cdns3_doorbell_epx(priv_ep->name,
1081					 readl(&priv_dev->regs->ep_traddr));
1082	}
1083
1084	/* WORKAROUND for transition to L0 */
1085	__cdns3_gadget_wakeup(priv_dev);
1086
1087	return 0;
1088}
1089
1090static void cdns3_rearm_drdy_if_needed(struct cdns3_endpoint *priv_ep)
1091{
1092	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1093
1094	if (priv_dev->dev_ver < DEV_VER_V3)
1095		return;
1096
1097	if (readl(&priv_dev->regs->ep_sts) & EP_STS_TRBERR) {
1098		writel(EP_STS_TRBERR, &priv_dev->regs->ep_sts);
1099		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1100	}
1101}
1102
1103/**
1104 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1105 * @priv_ep: endpoint object
1106 * @request: request object
1107 *
1108 * Returns zero on success or negative value on failure
1109 */
1110static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1111				 struct usb_request *request)
1112{
1113	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1114	struct cdns3_request *priv_req;
1115	struct cdns3_trb *trb;
1116	struct cdns3_trb *link_trb = NULL;
1117	dma_addr_t trb_dma;
1118	u32 togle_pcs = 1;
1119	int sg_iter = 0;
1120	int num_trb;
1121	int address;
1122	u32 control;
1123	int pcs;
1124	u16 total_tdl = 0;
1125	struct scatterlist *s = NULL;
1126	bool sg_supported = !!(request->num_mapped_sgs);
1127
1128	if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1129		num_trb = priv_ep->interval;
1130	else
1131		num_trb = sg_supported ? request->num_mapped_sgs : 1;
1132
1133	if (num_trb > priv_ep->free_trbs) {
1134		priv_ep->flags |= EP_RING_FULL;
1135		return -ENOBUFS;
1136	}
1137
1138	priv_req = to_cdns3_request(request);
1139	address = priv_ep->endpoint.desc->bEndpointAddress;
1140
1141	priv_ep->flags |= EP_PENDING_REQUEST;
1142
1143	/* must allocate buffer aligned to 8 */
1144	if (priv_req->flags & REQUEST_UNALIGNED)
1145		trb_dma = priv_req->aligned_buf->dma;
1146	else
1147		trb_dma = request->dma;
1148
1149	trb = priv_ep->trb_pool + priv_ep->enqueue;
1150	priv_req->start_trb = priv_ep->enqueue;
1151	priv_req->trb = trb;
1152
1153	cdns3_select_ep(priv_ep->cdns3_dev, address);
1154
1155	/* prepare ring */
1156	if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
1157		int doorbell, dma_index;
1158		u32 ch_bit = 0;
1159
1160		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1161		dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1162
1163		/* Driver can't update LINK TRB if it is current processed. */
1164		if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1165			priv_ep->flags |= EP_DEFERRED_DRDY;
1166			return -ENOBUFS;
1167		}
1168
1169		/*updating C bt in  Link TRB before starting DMA*/
1170		link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1171		/*
1172		 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1173		 * that DMA stuck at the LINK TRB.
1174		 * On the other hand, removing TRB_CHAIN for longer TRs for
1175		 * epXout cause that DMA stuck after handling LINK TRB.
1176		 * To eliminate this strange behavioral driver set TRB_CHAIN
1177		 * bit only for TR size > 2.
1178		 */
1179		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1180		    TRBS_PER_SEGMENT > 2)
1181			ch_bit = TRB_CHAIN;
1182
1183		link_trb->control = cpu_to_le32(((priv_ep->pcs) ? TRB_CYCLE : 0) |
1184				    TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit);
1185	}
1186
1187	if (priv_dev->dev_ver <= DEV_VER_V2)
1188		togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1189
1190	if (sg_supported)
1191		s = request->sg;
1192
1193	/* set incorrect Cycle Bit for first trb*/
1194	control = priv_ep->pcs ? 0 : TRB_CYCLE;
1195	trb->length = 0;
1196	if (priv_dev->dev_ver >= DEV_VER_V2) {
1197		u16 td_size;
1198
1199		td_size = DIV_ROUND_UP(request->length,
1200				       priv_ep->endpoint.maxpacket);
1201		if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1202			trb->length = cpu_to_le32(TRB_TDL_SS_SIZE(td_size));
1203		else
1204			control |= TRB_TDL_HS_SIZE(td_size);
1205	}
1206
1207	do {
1208		u32 length;
1209
1210		/* fill TRB */
1211		control |= TRB_TYPE(TRB_NORMAL);
1212		if (sg_supported) {
1213			trb->buffer = cpu_to_le32(TRB_BUFFER(sg_dma_address(s)));
1214			length = sg_dma_len(s);
1215		} else {
1216			trb->buffer = cpu_to_le32(TRB_BUFFER(trb_dma));
1217			length = request->length;
1218		}
1219
1220		if (priv_ep->flags & EP_TDLCHK_EN)
1221			total_tdl += DIV_ROUND_UP(length,
1222					       priv_ep->endpoint.maxpacket);
1223
1224		trb->length |= cpu_to_le32(TRB_BURST_LEN(priv_ep->trb_burst_size) |
1225					TRB_LEN(length));
1226		pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1227
1228		/*
1229		 * first trb should be prepared as last to avoid processing
1230		 *  transfer to early
1231		 */
1232		if (sg_iter != 0)
1233			control |= pcs;
1234
1235		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
1236			control |= TRB_IOC | TRB_ISP;
1237		} else {
1238			/* for last element in TD or in SG list */
1239			if (sg_iter == (num_trb - 1) && sg_iter != 0)
1240				control |= pcs | TRB_IOC | TRB_ISP;
1241		}
1242
1243		if (sg_iter)
1244			trb->control = cpu_to_le32(control);
1245		else
1246			priv_req->trb->control = cpu_to_le32(control);
1247
1248		if (sg_supported) {
1249			trb->control |= cpu_to_le32(TRB_ISP);
1250			/* Don't set chain bit for last TRB */
1251			if (sg_iter < num_trb - 1)
1252				trb->control |= cpu_to_le32(TRB_CHAIN);
1253
1254			s = sg_next(s);
1255		}
1256
1257		control = 0;
1258		++sg_iter;
1259		priv_req->end_trb = priv_ep->enqueue;
1260		cdns3_ep_inc_enq(priv_ep);
1261		trb = priv_ep->trb_pool + priv_ep->enqueue;
1262		trb->length = 0;
1263	} while (sg_iter < num_trb);
1264
1265	trb = priv_req->trb;
1266
1267	priv_req->flags |= REQUEST_PENDING;
1268	priv_req->num_of_trb = num_trb;
1269
1270	if (sg_iter == 1)
1271		trb->control |= cpu_to_le32(TRB_IOC | TRB_ISP);
1272
1273	if (priv_dev->dev_ver < DEV_VER_V2 &&
1274	    (priv_ep->flags & EP_TDLCHK_EN)) {
1275		u16 tdl = total_tdl;
1276		u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1277
1278		if (tdl > EP_CMD_TDL_MAX) {
1279			tdl = EP_CMD_TDL_MAX;
1280			priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1281		}
1282
1283		if (old_tdl < tdl) {
1284			tdl -= old_tdl;
1285			writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1286			       &priv_dev->regs->ep_cmd);
1287		}
1288	}
1289
1290	/*
1291	 * Memory barrier - cycle bit must be set before other filds in trb.
1292	 */
1293	wmb();
1294
1295	/* give the TD to the consumer*/
1296	if (togle_pcs)
1297		trb->control = trb->control ^ cpu_to_le32(1);
1298
1299	if (priv_dev->dev_ver <= DEV_VER_V2)
1300		cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1301
1302	if (num_trb > 1) {
1303		int i = 0;
1304
1305		while (i < num_trb) {
1306			trace_cdns3_prepare_trb(priv_ep, trb + i);
1307			if (trb + i == link_trb) {
1308				trb = priv_ep->trb_pool;
1309				num_trb = num_trb - i;
1310				i = 0;
1311			} else {
1312				i++;
1313			}
1314		}
1315	} else {
1316		trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1317	}
1318
1319	/*
1320	 * Memory barrier - Cycle Bit must be set before trb->length  and
1321	 * trb->buffer fields.
1322	 */
1323	wmb();
1324
1325	/*
1326	 * For DMULT mode we can set address to transfer ring only once after
1327	 * enabling endpoint.
1328	 */
1329	if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1330		/*
1331		 * Until SW is not ready to handle the OUT transfer the ISO OUT
1332		 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1333		 * EP_CFG_ENABLE must be set before updating ep_traddr.
1334		 */
1335		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
1336		    !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1337			priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1338			cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1339					       EP_CFG_ENABLE);
1340		}
1341
1342		writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1343					priv_req->start_trb * TRB_SIZE),
1344					&priv_dev->regs->ep_traddr);
1345
1346		priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1347	}
1348
1349	if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1350		trace_cdns3_ring(priv_ep);
1351		/*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1352		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1353		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1354		cdns3_rearm_drdy_if_needed(priv_ep);
1355		trace_cdns3_doorbell_epx(priv_ep->name,
1356					 readl(&priv_dev->regs->ep_traddr));
1357	}
1358
1359	/* WORKAROUND for transition to L0 */
1360	__cdns3_gadget_wakeup(priv_dev);
1361
1362	return 0;
1363}
1364
1365void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1366{
1367	struct cdns3_endpoint *priv_ep;
1368	struct usb_ep *ep;
1369
1370	if (priv_dev->hw_configured_flag)
1371		return;
1372
1373	writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
1374
1375	cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1376			       USB_CONF_U1EN | USB_CONF_U2EN);
1377
1378	priv_dev->hw_configured_flag = 1;
1379
1380	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1381		if (ep->enabled) {
1382			priv_ep = ep_to_cdns3_ep(ep);
1383			cdns3_start_all_request(priv_dev, priv_ep);
1384		}
1385	}
1386
1387	cdns3_allow_enable_l1(priv_dev, 1);
1388}
1389
1390/**
1391 * cdns3_trb_handled - check whether trb has been handled by DMA
1392 *
1393 * @priv_ep: extended endpoint object.
1394 * @priv_req: request object for checking
1395 *
1396 * Endpoint must be selected before invoking this function.
1397 *
1398 * Returns false if request has not been handled by DMA, else returns true.
1399 *
1400 * SR - start ring
1401 * ER -  end ring
1402 * DQ = priv_ep->dequeue - dequeue position
1403 * EQ = priv_ep->enqueue -  enqueue position
1404 * ST = priv_req->start_trb - index of first TRB in transfer ring
1405 * ET = priv_req->end_trb - index of last TRB in transfer ring
1406 * CI = current_index - index of processed TRB by DMA.
1407 *
1408 * As first step, we check if the TRB between the ST and ET.
1409 * Then, we check if cycle bit for index priv_ep->dequeue
1410 * is correct.
1411 *
1412 * some rules:
1413 * 1. priv_ep->dequeue never equals to current_index.
1414 * 2  priv_ep->enqueue never exceed priv_ep->dequeue
1415 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1416 *    and priv_ep->free_trbs is zero.
1417 *    This case indicate that TR is full.
1418 *
1419 * At below two cases, the request have been handled.
1420 * Case 1 - priv_ep->dequeue < current_index
1421 *      SR ... EQ ... DQ ... CI ... ER
1422 *      SR ... DQ ... CI ... EQ ... ER
1423 *
1424 * Case 2 - priv_ep->dequeue > current_index
1425 * This situation takes place when CI go through the LINK TRB at the end of
1426 * transfer ring.
1427 *      SR ... CI ... EQ ... DQ ... ER
1428 */
1429static bool cdns3_trb_handled(struct cdns3_endpoint *priv_ep,
1430				  struct cdns3_request *priv_req)
1431{
1432	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1433	struct cdns3_trb *trb;
1434	int current_index = 0;
1435	int handled = 0;
1436	int doorbell;
1437
1438	current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1439	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1440
1441	/* current trb doesn't belong to this request */
1442	if (priv_req->start_trb < priv_req->end_trb) {
1443		if (priv_ep->dequeue > priv_req->end_trb)
1444			goto finish;
1445
1446		if (priv_ep->dequeue < priv_req->start_trb)
1447			goto finish;
1448	}
1449
1450	if ((priv_req->start_trb > priv_req->end_trb) &&
1451		(priv_ep->dequeue > priv_req->end_trb) &&
1452		(priv_ep->dequeue < priv_req->start_trb))
1453		goto finish;
1454
1455	if ((priv_req->start_trb == priv_req->end_trb) &&
1456		(priv_ep->dequeue != priv_req->end_trb))
1457		goto finish;
1458
1459	trb = &priv_ep->trb_pool[priv_ep->dequeue];
1460
1461	if ((le32_to_cpu(trb->control) & TRB_CYCLE) != priv_ep->ccs)
1462		goto finish;
1463
1464	if (doorbell == 1 && current_index == priv_ep->dequeue)
1465		goto finish;
1466
1467	/* The corner case for TRBS_PER_SEGMENT equal 2). */
1468	if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1469		handled = 1;
1470		goto finish;
1471	}
1472
1473	if (priv_ep->enqueue == priv_ep->dequeue &&
1474	    priv_ep->free_trbs == 0) {
1475		handled = 1;
1476	} else if (priv_ep->dequeue < current_index) {
1477		if ((current_index == (priv_ep->num_trbs - 1)) &&
1478		    !priv_ep->dequeue)
1479			goto finish;
1480
1481		handled = 1;
1482	} else if (priv_ep->dequeue  > current_index) {
1483			handled = 1;
1484	}
1485
1486finish:
1487	trace_cdns3_request_handled(priv_req, current_index, handled);
1488
1489	return handled;
1490}
1491
1492static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1493				     struct cdns3_endpoint *priv_ep)
1494{
1495	struct cdns3_request *priv_req;
1496	struct usb_request *request;
1497	struct cdns3_trb *trb;
1498	bool request_handled = false;
1499	bool transfer_end = false;
1500
1501	while (!list_empty(&priv_ep->pending_req_list)) {
1502		request = cdns3_next_request(&priv_ep->pending_req_list);
1503		priv_req = to_cdns3_request(request);
1504
1505		trb = priv_ep->trb_pool + priv_ep->dequeue;
1506
1507		/* The TRB was changed as link TRB, and the request was handled at ep_dequeue */
1508		while (TRB_FIELD_TO_TYPE(le32_to_cpu(trb->control)) == TRB_LINK) {
1509			trace_cdns3_complete_trb(priv_ep, trb);
1510			cdns3_ep_inc_deq(priv_ep);
1511			trb = priv_ep->trb_pool + priv_ep->dequeue;
1512		}
1513
1514		if (!request->stream_id) {
1515			/* Re-select endpoint. It could be changed by other CPU
1516			 * during handling usb_gadget_giveback_request.
1517			 */
1518			cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1519
1520			while (cdns3_trb_handled(priv_ep, priv_req)) {
1521				priv_req->finished_trb++;
1522				if (priv_req->finished_trb >= priv_req->num_of_trb)
1523					request_handled = true;
1524
1525				trb = priv_ep->trb_pool + priv_ep->dequeue;
1526				trace_cdns3_complete_trb(priv_ep, trb);
1527
1528				if (!transfer_end)
1529					request->actual +=
1530						TRB_LEN(le32_to_cpu(trb->length));
1531
1532				if (priv_req->num_of_trb > 1 &&
1533					le32_to_cpu(trb->control) & TRB_SMM &&
1534					le32_to_cpu(trb->control) & TRB_CHAIN)
1535					transfer_end = true;
1536
1537				cdns3_ep_inc_deq(priv_ep);
1538			}
1539
1540			if (request_handled) {
1541				cdns3_gadget_giveback(priv_ep, priv_req, 0);
1542				request_handled = false;
1543				transfer_end = false;
1544			} else {
1545				goto prepare_next_td;
1546			}
1547
1548			if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1549			    TRBS_PER_SEGMENT == 2)
1550				break;
1551		} else {
1552			/* Re-select endpoint. It could be changed by other CPU
1553			 * during handling usb_gadget_giveback_request.
1554			 */
1555			cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1556
1557			trb = priv_ep->trb_pool;
1558			trace_cdns3_complete_trb(priv_ep, trb);
1559
1560			if (trb != priv_req->trb)
1561				dev_warn(priv_dev->dev,
1562					 "request_trb=0x%p, queue_trb=0x%p\n",
1563					 priv_req->trb, trb);
1564
1565			request->actual += TRB_LEN(le32_to_cpu(trb->length));
1566
1567			if (!request->num_sgs ||
1568			    (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1569				priv_ep->stream_sg_idx = 0;
1570				cdns3_gadget_giveback(priv_ep, priv_req, 0);
1571			} else {
1572				priv_ep->stream_sg_idx++;
1573				cdns3_ep_run_stream_transfer(priv_ep, request);
1574			}
1575			break;
1576		}
1577	}
1578	priv_ep->flags &= ~EP_PENDING_REQUEST;
1579
1580prepare_next_td:
1581	if (!(priv_ep->flags & EP_STALLED) &&
1582	    !(priv_ep->flags & EP_STALL_PENDING))
1583		cdns3_start_all_request(priv_dev, priv_ep);
1584}
1585
1586void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1587{
1588	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1589
1590	cdns3_wa1_restore_cycle_bit(priv_ep);
1591
1592	if (rearm) {
1593		trace_cdns3_ring(priv_ep);
1594
1595		/* Cycle Bit must be updated before arming DMA. */
1596		wmb();
1597		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1598
1599		__cdns3_gadget_wakeup(priv_dev);
1600
1601		trace_cdns3_doorbell_epx(priv_ep->name,
1602					 readl(&priv_dev->regs->ep_traddr));
1603	}
1604}
1605
1606static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1607{
1608	u16 tdl = priv_ep->pending_tdl;
1609	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1610
1611	if (tdl > EP_CMD_TDL_MAX) {
1612		tdl = EP_CMD_TDL_MAX;
1613		priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1614	} else {
1615		priv_ep->pending_tdl = 0;
1616	}
1617
1618	writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1619}
1620
1621/**
1622 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1623 * @priv_ep: endpoint object
1624 *
1625 * Returns 0
1626 */
1627static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1628{
1629	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1630	u32 ep_sts_reg;
1631	struct usb_request *deferred_request;
1632	struct usb_request *pending_request;
1633	u32 tdl = 0;
1634
1635	cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1636
1637	trace_cdns3_epx_irq(priv_dev, priv_ep);
1638
1639	ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1640	writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1641
1642	if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1643		bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1644
1645		tdl = cdns3_get_tdl(priv_dev);
1646
1647		/*
1648		 * Continue the previous transfer:
1649		 * There is some racing between ERDY and PRIME. The device send
1650		 * ERDY and almost in the same time Host send PRIME. It cause
1651		 * that host ignore the ERDY packet and driver has to send it
1652		 * again.
1653		 */
1654		if (tdl && (dbusy || !EP_STS_BUFFEMPTY(ep_sts_reg) ||
1655		    EP_STS_HOSTPP(ep_sts_reg))) {
1656			writel(EP_CMD_ERDY |
1657			       EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1658			       &priv_dev->regs->ep_cmd);
1659			ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1660		} else {
1661			priv_ep->prime_flag = true;
1662
1663			pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1664			deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1665
1666			if (deferred_request && !pending_request) {
1667				cdns3_start_all_request(priv_dev, priv_ep);
1668			}
1669		}
1670	}
1671
1672	if (ep_sts_reg & EP_STS_TRBERR) {
1673		if (priv_ep->flags & EP_STALL_PENDING &&
1674		    !(ep_sts_reg & EP_STS_DESCMIS &&
1675		    priv_dev->dev_ver < DEV_VER_V2)) {
1676			cdns3_ep_stall_flush(priv_ep);
1677		}
1678
1679		/*
1680		 * For isochronous transfer driver completes request on
1681		 * IOC or on TRBERR. IOC appears only when device receive
1682		 * OUT data packet. If host disable stream or lost some packet
1683		 * then the only way to finish all queued transfer is to do it
1684		 * on TRBERR event.
1685		 */
1686		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1687		    !priv_ep->wa1_set) {
1688			if (!priv_ep->dir) {
1689				u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1690
1691				ep_cfg &= ~EP_CFG_ENABLE;
1692				writel(ep_cfg, &priv_dev->regs->ep_cfg);
1693				priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1694				priv_ep->flags |= EP_UPDATE_EP_TRBADDR;
1695			}
1696			cdns3_transfer_completed(priv_dev, priv_ep);
1697		} else if (!(priv_ep->flags & EP_STALLED) &&
1698			  !(priv_ep->flags & EP_STALL_PENDING)) {
1699			if (priv_ep->flags & EP_DEFERRED_DRDY) {
1700				priv_ep->flags &= ~EP_DEFERRED_DRDY;
1701				cdns3_start_all_request(priv_dev, priv_ep);
1702			} else {
1703				cdns3_rearm_transfer(priv_ep,
1704						     priv_ep->wa1_set);
1705			}
1706		}
1707	}
1708
1709	if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1710	    (ep_sts_reg & EP_STS_IOT)) {
1711		if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1712			if (ep_sts_reg & EP_STS_ISP)
1713				priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1714			else
1715				priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1716		}
1717
1718		if (!priv_ep->use_streams) {
1719			if ((ep_sts_reg & EP_STS_IOC) ||
1720			    (ep_sts_reg & EP_STS_ISP)) {
1721				cdns3_transfer_completed(priv_dev, priv_ep);
1722			} else if ((priv_ep->flags & EP_TDLCHK_EN) &
1723				   priv_ep->pending_tdl) {
1724				/* handle IOT with pending tdl */
1725				cdns3_reprogram_tdl(priv_ep);
1726			}
1727		} else if (priv_ep->dir == USB_DIR_OUT) {
1728			priv_ep->ep_sts_pending |= ep_sts_reg;
1729		} else if (ep_sts_reg & EP_STS_IOT) {
1730			cdns3_transfer_completed(priv_dev, priv_ep);
1731		}
1732	}
1733
1734	/*
1735	 * MD_EXIT interrupt sets when stream capable endpoint exits
1736	 * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1737	 */
1738	if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1739	    (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1740		priv_ep->ep_sts_pending = 0;
1741		cdns3_transfer_completed(priv_dev, priv_ep);
1742	}
1743
1744	/*
1745	 * WA2: this condition should only be meet when
1746	 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1747	 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1748	 * In other cases this interrupt will be disabled.
1749	 */
1750	if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1751	    !(priv_ep->flags & EP_STALLED))
1752		cdns3_wa2_descmissing_packet(priv_ep);
1753
1754	return 0;
1755}
1756
1757static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1758{
1759	if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect)
1760		priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1761}
1762
1763/**
1764 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1765 * @priv_dev: extended gadget object
1766 * @usb_ists: bitmap representation of device's reported interrupts
1767 * (usb_ists register value)
1768 */
1769static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1770					      u32 usb_ists)
1771__must_hold(&priv_dev->lock)
1772{
1773	int speed = 0;
1774
1775	trace_cdns3_usb_irq(priv_dev, usb_ists);
1776	if (usb_ists & USB_ISTS_L1ENTI) {
1777		/*
1778		 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1779		 * from L1. To fix it, if any DMA transfer is pending driver
1780		 * must starts driving resume signal immediately.
1781		 */
1782		if (readl(&priv_dev->regs->drbl))
1783			__cdns3_gadget_wakeup(priv_dev);
1784	}
1785
1786	/* Connection detected */
1787	if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1788		speed = cdns3_get_speed(priv_dev);
1789		priv_dev->gadget.speed = speed;
1790		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1791		cdns3_ep0_config(priv_dev);
1792	}
1793
1794	/* Disconnection detected */
1795	if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1796		spin_unlock(&priv_dev->lock);
1797		cdns3_disconnect_gadget(priv_dev);
1798		spin_lock(&priv_dev->lock);
1799		priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1800		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1801		cdns3_hw_reset_eps_config(priv_dev);
1802	}
1803
1804	if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1805		if (priv_dev->gadget_driver &&
1806		    priv_dev->gadget_driver->suspend) {
1807			spin_unlock(&priv_dev->lock);
1808			priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1809			spin_lock(&priv_dev->lock);
1810		}
1811	}
1812
1813	if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1814		if (priv_dev->gadget_driver &&
1815		    priv_dev->gadget_driver->resume) {
1816			spin_unlock(&priv_dev->lock);
1817			priv_dev->gadget_driver->resume(&priv_dev->gadget);
1818			spin_lock(&priv_dev->lock);
1819		}
1820	}
1821
1822	/* reset*/
1823	if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1824		if (priv_dev->gadget_driver) {
1825			spin_unlock(&priv_dev->lock);
1826			usb_gadget_udc_reset(&priv_dev->gadget,
1827					     priv_dev->gadget_driver);
1828			spin_lock(&priv_dev->lock);
1829
1830			/*read again to check the actual speed*/
1831			speed = cdns3_get_speed(priv_dev);
1832			priv_dev->gadget.speed = speed;
1833			cdns3_hw_reset_eps_config(priv_dev);
1834			cdns3_ep0_config(priv_dev);
1835		}
1836	}
1837}
1838
1839/**
1840 * cdns3_device_irq_handler - interrupt handler for device part of controller
1841 *
1842 * @irq: irq number for cdns3 core device
1843 * @data: structure of cdns3
1844 *
1845 * Returns IRQ_HANDLED or IRQ_NONE
1846 */
1847static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1848{
1849	struct cdns3_device *priv_dev = data;
1850	struct cdns *cdns = dev_get_drvdata(priv_dev->dev);
1851	irqreturn_t ret = IRQ_NONE;
1852	u32 reg;
1853
1854	if (cdns->in_lpm)
1855		return ret;
1856
1857	/* check USB device interrupt */
1858	reg = readl(&priv_dev->regs->usb_ists);
1859	if (reg) {
1860		/* After masking interrupts the new interrupts won't be
1861		 * reported in usb_ists/ep_ists. In order to not lose some
1862		 * of them driver disables only detected interrupts.
1863		 * They will be enabled ASAP after clearing source of
1864		 * interrupt. This an unusual behavior only applies to
1865		 * usb_ists register.
1866		 */
1867		reg = ~reg & readl(&priv_dev->regs->usb_ien);
1868		/* mask deferred interrupt. */
1869		writel(reg, &priv_dev->regs->usb_ien);
1870		ret = IRQ_WAKE_THREAD;
1871	}
1872
1873	/* check endpoint interrupt */
1874	reg = readl(&priv_dev->regs->ep_ists);
1875	if (reg) {
1876		writel(0, &priv_dev->regs->ep_ien);
1877		ret = IRQ_WAKE_THREAD;
1878	}
1879
1880	return ret;
1881}
1882
1883/**
1884 * cdns3_device_thread_irq_handler - interrupt handler for device part
1885 * of controller
1886 *
1887 * @irq: irq number for cdns3 core device
1888 * @data: structure of cdns3
1889 *
1890 * Returns IRQ_HANDLED or IRQ_NONE
1891 */
1892static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1893{
1894	struct cdns3_device *priv_dev = data;
1895	irqreturn_t ret = IRQ_NONE;
1896	unsigned long flags;
1897	unsigned int bit;
1898	unsigned long reg;
1899
1900	spin_lock_irqsave(&priv_dev->lock, flags);
1901
1902	reg = readl(&priv_dev->regs->usb_ists);
1903	if (reg) {
1904		writel(reg, &priv_dev->regs->usb_ists);
1905		writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1906		cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1907		ret = IRQ_HANDLED;
1908	}
1909
1910	reg = readl(&priv_dev->regs->ep_ists);
1911
1912	/* handle default endpoint OUT */
1913	if (reg & EP_ISTS_EP_OUT0) {
1914		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1915		ret = IRQ_HANDLED;
1916	}
1917
1918	/* handle default endpoint IN */
1919	if (reg & EP_ISTS_EP_IN0) {
1920		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1921		ret = IRQ_HANDLED;
1922	}
1923
1924	/* check if interrupt from non default endpoint, if no exit */
1925	reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1926	if (!reg)
1927		goto irqend;
1928
1929	for_each_set_bit(bit, &reg,
1930			 sizeof(u32) * BITS_PER_BYTE) {
1931		cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1932		ret = IRQ_HANDLED;
1933	}
1934
1935	if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1936		cdns3_wa2_check_outq_status(priv_dev);
1937
1938irqend:
1939	writel(~0, &priv_dev->regs->ep_ien);
1940	spin_unlock_irqrestore(&priv_dev->lock, flags);
1941
1942	return ret;
1943}
1944
1945/**
1946 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1947 *
1948 * The real reservation will occur during write to EP_CFG register,
1949 * this function is used to check if the 'size' reservation is allowed.
1950 *
1951 * @priv_dev: extended gadget object
1952 * @size: the size (KB) for EP would like to allocate
1953 * @is_in: endpoint direction
1954 *
1955 * Return 0 if the required size can met or negative value on failure
1956 */
1957static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1958					  int size, int is_in)
1959{
1960	int remained;
1961
1962	/* 2KB are reserved for EP0*/
1963	remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1964
1965	if (is_in) {
1966		if (remained < size)
1967			return -EPERM;
1968
1969		priv_dev->onchip_used_size += size;
1970	} else {
1971		int required;
1972
1973		/**
1974		 *  ALL OUT EPs are shared the same chunk onchip memory, so
1975		 * driver checks if it already has assigned enough buffers
1976		 */
1977		if (priv_dev->out_mem_is_allocated >= size)
1978			return 0;
1979
1980		required = size - priv_dev->out_mem_is_allocated;
1981
1982		if (required > remained)
1983			return -EPERM;
1984
1985		priv_dev->out_mem_is_allocated += required;
1986		priv_dev->onchip_used_size += required;
1987	}
1988
1989	return 0;
1990}
1991
1992static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1993				  struct cdns3_endpoint *priv_ep)
1994{
1995	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1996
1997	/* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1998	if (priv_dev->dev_ver <= DEV_VER_V2)
1999		writel(USB_CONF_DMULT, &regs->usb_conf);
2000
2001	if (priv_dev->dev_ver == DEV_VER_V2)
2002		writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
2003
2004	if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
2005		u32 mask;
2006
2007		if (priv_ep->dir)
2008			mask = BIT(priv_ep->num + 16);
2009		else
2010			mask = BIT(priv_ep->num);
2011
2012		if (priv_ep->type != USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
2013			cdns3_set_register_bit(&regs->tdl_from_trb, mask);
2014			cdns3_set_register_bit(&regs->tdl_beh, mask);
2015			cdns3_set_register_bit(&regs->tdl_beh2, mask);
2016			cdns3_set_register_bit(&regs->dma_adv_td, mask);
2017		}
2018
2019		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
2020			cdns3_set_register_bit(&regs->tdl_from_trb, mask);
2021
2022		cdns3_set_register_bit(&regs->dtrans, mask);
2023	}
2024}
2025
2026/**
2027 * cdns3_ep_config - Configure hardware endpoint
2028 * @priv_ep: extended endpoint object
2029 * @enable: set EP_CFG_ENABLE bit in ep_cfg register.
2030 */
2031int cdns3_ep_config(struct cdns3_endpoint *priv_ep, bool enable)
2032{
2033	bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
2034	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2035	u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
2036	u32 max_packet_size = 0;
2037	u8 maxburst = 0;
2038	u32 ep_cfg = 0;
2039	u8 buffering;
2040	u8 mult = 0;
2041	int ret;
2042
2043	buffering = priv_dev->ep_buf_size - 1;
2044
2045	cdns3_configure_dmult(priv_dev, priv_ep);
2046
2047	switch (priv_ep->type) {
2048	case USB_ENDPOINT_XFER_INT:
2049		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
2050
2051		if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2052			ep_cfg |= EP_CFG_TDL_CHK;
2053		break;
2054	case USB_ENDPOINT_XFER_BULK:
2055		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
2056
2057		if (priv_dev->dev_ver >= DEV_VER_V2 && !priv_ep->dir)
2058			ep_cfg |= EP_CFG_TDL_CHK;
2059		break;
2060	default:
2061		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2062		mult = priv_dev->ep_iso_burst - 1;
2063		buffering = mult + 1;
2064	}
2065
2066	switch (priv_dev->gadget.speed) {
2067	case USB_SPEED_FULL:
2068		max_packet_size = is_iso_ep ? 1023 : 64;
2069		break;
2070	case USB_SPEED_HIGH:
2071		max_packet_size = is_iso_ep ? 1024 : 512;
2072		break;
2073	case USB_SPEED_SUPER:
2074		/* It's limitation that driver assumes in driver. */
2075		mult = 0;
2076		max_packet_size = 1024;
2077		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2078			maxburst = priv_dev->ep_iso_burst - 1;
2079			buffering = (mult + 1) *
2080				    (maxburst + 1);
2081
2082			if (priv_ep->interval > 1)
2083				buffering++;
2084		} else {
2085			maxburst = priv_dev->ep_buf_size - 1;
2086		}
2087		break;
2088	default:
2089		/* all other speed are not supported */
2090		return -EINVAL;
2091	}
2092
2093	if (max_packet_size == 1024)
2094		priv_ep->trb_burst_size = 128;
2095	else if (max_packet_size >= 512)
2096		priv_ep->trb_burst_size = 64;
2097	else
2098		priv_ep->trb_burst_size = 16;
2099
2100	mult = min_t(u8, mult, EP_CFG_MULT_MAX);
2101	buffering = min_t(u8, buffering, EP_CFG_BUFFERING_MAX);
2102	maxburst = min_t(u8, maxburst, EP_CFG_MAXBURST_MAX);
2103
2104	/* onchip buffer is only allocated before configuration */
2105	if (!priv_dev->hw_configured_flag) {
2106		ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2107						     !!priv_ep->dir);
2108		if (ret) {
2109			dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2110			return ret;
2111		}
2112	}
2113
2114	if (enable)
2115		ep_cfg |= EP_CFG_ENABLE;
2116
2117	if (priv_ep->use_streams && priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2118		if (priv_dev->dev_ver >= DEV_VER_V3) {
2119			u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
2120
2121			/*
2122			 * Stream capable endpoints are handled by using ep_tdl
2123			 * register. Other endpoints use TDL from TRB feature.
2124			 */
2125			cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb,
2126						 mask);
2127		}
2128
2129		/*  Enable Stream Bit TDL chk and SID chk */
2130		ep_cfg |=  EP_CFG_STREAM_EN | EP_CFG_TDL_CHK | EP_CFG_SID_CHK;
2131	}
2132
2133	ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2134		  EP_CFG_MULT(mult) |
2135		  EP_CFG_BUFFERING(buffering) |
2136		  EP_CFG_MAXBURST(maxburst);
2137
2138	cdns3_select_ep(priv_dev, bEndpointAddress);
2139	writel(ep_cfg, &priv_dev->regs->ep_cfg);
2140	priv_ep->flags |= EP_CONFIGURED;
2141
2142	dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2143		priv_ep->name, ep_cfg);
2144
2145	return 0;
2146}
2147
2148/* Find correct direction for HW endpoint according to description */
2149static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2150				   struct cdns3_endpoint *priv_ep)
2151{
2152	return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2153	       (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2154}
2155
2156static struct
2157cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2158					struct usb_endpoint_descriptor *desc)
2159{
2160	struct usb_ep *ep;
2161	struct cdns3_endpoint *priv_ep;
2162
2163	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2164		unsigned long num;
2165		int ret;
2166		/* ep name pattern likes epXin or epXout */
2167		char c[2] = {ep->name[2], '\0'};
2168
2169		ret = kstrtoul(c, 10, &num);
2170		if (ret)
2171			return ERR_PTR(ret);
2172
2173		priv_ep = ep_to_cdns3_ep(ep);
2174		if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2175			if (!(priv_ep->flags & EP_CLAIMED)) {
2176				priv_ep->num  = num;
2177				return priv_ep;
2178			}
2179		}
2180	}
2181
2182	return ERR_PTR(-ENOENT);
2183}
2184
2185/*
2186 *  Cadence IP has one limitation that all endpoints must be configured
2187 * (Type & MaxPacketSize) before setting configuration through hardware
2188 * register, it means we can't change endpoints configuration after
2189 * set_configuration.
2190 *
2191 * This function set EP_CLAIMED flag which is added when the gadget driver
2192 * uses usb_ep_autoconfig to configure specific endpoint;
2193 * When the udc driver receives set_configurion request,
2194 * it goes through all claimed endpoints, and configure all endpoints
2195 * accordingly.
2196 *
2197 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2198 * ep_cfg register which can be changed after set_configuration, and do
2199 * some software operation accordingly.
2200 */
2201static struct
2202usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2203			      struct usb_endpoint_descriptor *desc,
2204			      struct usb_ss_ep_comp_descriptor *comp_desc)
2205{
2206	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2207	struct cdns3_endpoint *priv_ep;
2208	unsigned long flags;
2209
2210	priv_ep = cdns3_find_available_ep(priv_dev, desc);
2211	if (IS_ERR(priv_ep)) {
2212		dev_err(priv_dev->dev, "no available ep\n");
2213		return NULL;
2214	}
2215
2216	dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2217
2218	spin_lock_irqsave(&priv_dev->lock, flags);
2219	priv_ep->endpoint.desc = desc;
2220	priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2221	priv_ep->type = usb_endpoint_type(desc);
2222	priv_ep->flags |= EP_CLAIMED;
2223	priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2224
2225	spin_unlock_irqrestore(&priv_dev->lock, flags);
2226	return &priv_ep->endpoint;
2227}
2228
2229/**
2230 * cdns3_gadget_ep_alloc_request - Allocates request
2231 * @ep: endpoint object associated with request
2232 * @gfp_flags: gfp flags
2233 *
2234 * Returns allocated request address, NULL on allocation error
2235 */
2236struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2237						  gfp_t gfp_flags)
2238{
2239	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2240	struct cdns3_request *priv_req;
2241
2242	priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2243	if (!priv_req)
2244		return NULL;
2245
2246	priv_req->priv_ep = priv_ep;
2247
2248	trace_cdns3_alloc_request(priv_req);
2249	return &priv_req->request;
2250}
2251
2252/**
2253 * cdns3_gadget_ep_free_request - Free memory occupied by request
2254 * @ep: endpoint object associated with request
2255 * @request: request to free memory
2256 */
2257void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2258				  struct usb_request *request)
2259{
2260	struct cdns3_request *priv_req = to_cdns3_request(request);
2261
2262	if (priv_req->aligned_buf)
2263		priv_req->aligned_buf->in_use = 0;
2264
2265	trace_cdns3_free_request(priv_req);
2266	kfree(priv_req);
2267}
2268
2269/**
2270 * cdns3_gadget_ep_enable - Enable endpoint
2271 * @ep: endpoint object
2272 * @desc: endpoint descriptor
2273 *
2274 * Returns 0 on success, error code elsewhere
2275 */
2276static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2277				  const struct usb_endpoint_descriptor *desc)
2278{
2279	struct cdns3_endpoint *priv_ep;
2280	struct cdns3_device *priv_dev;
2281	const struct usb_ss_ep_comp_descriptor *comp_desc;
2282	u32 reg = EP_STS_EN_TRBERREN;
2283	u32 bEndpointAddress;
2284	unsigned long flags;
2285	int enable = 1;
2286	int ret = 0;
2287	int val;
2288
2289	if (!ep) {
2290		pr_debug("usbss: ep not configured?\n");
2291		return -EINVAL;
2292	}
2293
2294	priv_ep = ep_to_cdns3_ep(ep);
2295	priv_dev = priv_ep->cdns3_dev;
2296	comp_desc = priv_ep->endpoint.comp_desc;
2297
2298	if (!desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2299		dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2300		return -EINVAL;
2301	}
2302
2303	if (!desc->wMaxPacketSize) {
2304		dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2305		return -EINVAL;
2306	}
2307
2308	if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2309			  "%s is already enabled\n", priv_ep->name))
2310		return 0;
2311
2312	spin_lock_irqsave(&priv_dev->lock, flags);
2313
2314	priv_ep->endpoint.desc = desc;
2315	priv_ep->type = usb_endpoint_type(desc);
2316	priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2317
2318	if (priv_ep->interval > ISO_MAX_INTERVAL &&
2319	    priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2320		dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2321			ISO_MAX_INTERVAL);
2322
2323		ret =  -EINVAL;
2324		goto exit;
2325	}
2326
2327	bEndpointAddress = priv_ep->num | priv_ep->dir;
2328	cdns3_select_ep(priv_dev, bEndpointAddress);
2329
2330	/*
2331	 * For some versions of controller at some point during ISO OUT traffic
2332	 * DMA reads Transfer Ring for the EP which has never got doorbell.
2333	 * This issue was detected only on simulation, but to avoid this issue
2334	 * driver add protection against it. To fix it driver enable ISO OUT
2335	 * endpoint before setting DRBL. This special treatment of ISO OUT
2336	 * endpoints are recommended by controller specification.
2337	 */
2338	if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
2339		enable = 0;
2340
2341	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2342		/*
2343		 * Enable stream support (SS mode) related interrupts
2344		 * in EP_STS_EN Register
2345		 */
2346		if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2347			reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2348				EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2349				EP_STS_EN_STREAMREN;
2350			priv_ep->use_streams = true;
2351			ret = cdns3_ep_config(priv_ep, enable);
2352			priv_dev->using_streams |= true;
2353		}
2354	} else {
2355		ret = cdns3_ep_config(priv_ep, enable);
2356	}
2357
2358	if (ret)
2359		goto exit;
2360
2361	ret = cdns3_allocate_trb_pool(priv_ep);
2362	if (ret)
2363		goto exit;
2364
2365	bEndpointAddress = priv_ep->num | priv_ep->dir;
2366	cdns3_select_ep(priv_dev, bEndpointAddress);
2367
2368	trace_cdns3_gadget_ep_enable(priv_ep);
2369
2370	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2371
2372	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2373					!(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2374					1, 1000);
2375
2376	if (unlikely(ret)) {
2377		cdns3_free_trb_pool(priv_ep);
2378		ret =  -EINVAL;
2379		goto exit;
2380	}
2381
2382	/* enable interrupt for selected endpoint */
2383	cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2384			       BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2385
2386	if (priv_dev->dev_ver < DEV_VER_V2)
2387		cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2388
2389	writel(reg, &priv_dev->regs->ep_sts_en);
2390
2391	ep->desc = desc;
2392	priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2393			    EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2394	priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2395	priv_ep->wa1_set = 0;
2396	priv_ep->enqueue = 0;
2397	priv_ep->dequeue = 0;
2398	reg = readl(&priv_dev->regs->ep_sts);
2399	priv_ep->pcs = !!EP_STS_CCS(reg);
2400	priv_ep->ccs = !!EP_STS_CCS(reg);
2401	/* one TRB is reserved for link TRB used in DMULT mode*/
2402	priv_ep->free_trbs = priv_ep->num_trbs - 1;
2403exit:
2404	spin_unlock_irqrestore(&priv_dev->lock, flags);
2405
2406	return ret;
2407}
2408
2409/**
2410 * cdns3_gadget_ep_disable - Disable endpoint
2411 * @ep: endpoint object
2412 *
2413 * Returns 0 on success, error code elsewhere
2414 */
2415static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2416{
2417	struct cdns3_endpoint *priv_ep;
2418	struct cdns3_request *priv_req;
2419	struct cdns3_device *priv_dev;
2420	struct usb_request *request;
2421	unsigned long flags;
2422	int ret = 0;
2423	u32 ep_cfg;
2424	int val;
2425
2426	if (!ep) {
2427		pr_err("usbss: invalid parameters\n");
2428		return -EINVAL;
2429	}
2430
2431	priv_ep = ep_to_cdns3_ep(ep);
2432	priv_dev = priv_ep->cdns3_dev;
2433
2434	if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2435			  "%s is already disabled\n", priv_ep->name))
2436		return 0;
2437
2438	spin_lock_irqsave(&priv_dev->lock, flags);
2439
2440	trace_cdns3_gadget_ep_disable(priv_ep);
2441
2442	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2443
2444	ep_cfg = readl(&priv_dev->regs->ep_cfg);
2445	ep_cfg &= ~EP_CFG_ENABLE;
2446	writel(ep_cfg, &priv_dev->regs->ep_cfg);
2447
2448	/**
2449	 * Driver needs some time before resetting endpoint.
2450	 * It need waits for clearing DBUSY bit or for timeout expired.
2451	 * 10us is enough time for controller to stop transfer.
2452	 */
2453	readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2454				  !(val & EP_STS_DBUSY), 1, 10);
2455	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2456
2457	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2458				  !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2459				  1, 1000);
2460	if (unlikely(ret))
2461		dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2462			priv_ep->name);
2463
2464	while (!list_empty(&priv_ep->pending_req_list)) {
2465		request = cdns3_next_request(&priv_ep->pending_req_list);
2466
2467		cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2468				      -ESHUTDOWN);
2469	}
2470
2471	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2472		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2473
2474		kfree(priv_req->request.buf);
2475		cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2476					     &priv_req->request);
2477		list_del_init(&priv_req->list);
2478		--priv_ep->wa2_counter;
2479	}
2480
2481	while (!list_empty(&priv_ep->deferred_req_list)) {
2482		request = cdns3_next_request(&priv_ep->deferred_req_list);
2483
2484		cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2485				      -ESHUTDOWN);
2486	}
2487
2488	priv_ep->descmis_req = NULL;
2489
2490	ep->desc = NULL;
2491	priv_ep->flags &= ~EP_ENABLED;
2492	priv_ep->use_streams = false;
2493
2494	spin_unlock_irqrestore(&priv_dev->lock, flags);
2495
2496	return ret;
2497}
2498
2499/**
2500 * __cdns3_gadget_ep_queue - Transfer data on endpoint
2501 * @ep: endpoint object
2502 * @request: request object
2503 * @gfp_flags: gfp flags
2504 *
2505 * Returns 0 on success, error code elsewhere
2506 */
2507static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2508				   struct usb_request *request,
2509				   gfp_t gfp_flags)
2510{
2511	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2512	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2513	struct cdns3_request *priv_req;
2514	int ret = 0;
2515
2516	request->actual = 0;
2517	request->status = -EINPROGRESS;
2518	priv_req = to_cdns3_request(request);
2519	trace_cdns3_ep_queue(priv_req);
2520
2521	if (priv_dev->dev_ver < DEV_VER_V2) {
2522		ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2523						priv_req);
2524
2525		if (ret == EINPROGRESS)
2526			return 0;
2527	}
2528
2529	ret = cdns3_prepare_aligned_request_buf(priv_req);
2530	if (ret < 0)
2531		return ret;
2532
2533	ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2534					    usb_endpoint_dir_in(ep->desc));
2535	if (ret)
2536		return ret;
2537
2538	list_add_tail(&request->list, &priv_ep->deferred_req_list);
2539
2540	/*
2541	 * For stream capable endpoint if prime irq flag is set then only start
2542	 * request.
2543	 * If hardware endpoint configuration has not been set yet then
2544	 * just queue request in deferred list. Transfer will be started in
2545	 * cdns3_set_hw_configuration.
2546	 */
2547	if (!request->stream_id) {
2548		if (priv_dev->hw_configured_flag &&
2549		    !(priv_ep->flags & EP_STALLED) &&
2550		    !(priv_ep->flags & EP_STALL_PENDING))
2551			cdns3_start_all_request(priv_dev, priv_ep);
2552	} else {
2553		if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2554			cdns3_start_all_request(priv_dev, priv_ep);
2555	}
2556
2557	return 0;
2558}
2559
2560static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2561				 gfp_t gfp_flags)
2562{
2563	struct usb_request *zlp_request;
2564	struct cdns3_endpoint *priv_ep;
2565	struct cdns3_device *priv_dev;
2566	unsigned long flags;
2567	int ret;
2568
2569	if (!request || !ep)
2570		return -EINVAL;
2571
2572	priv_ep = ep_to_cdns3_ep(ep);
2573	priv_dev = priv_ep->cdns3_dev;
2574
2575	spin_lock_irqsave(&priv_dev->lock, flags);
2576
2577	ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2578
2579	if (ret == 0 && request->zero && request->length &&
2580	    (request->length % ep->maxpacket == 0)) {
2581		struct cdns3_request *priv_req;
2582
2583		zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2584		zlp_request->buf = priv_dev->zlp_buf;
2585		zlp_request->length = 0;
2586
2587		priv_req = to_cdns3_request(zlp_request);
2588		priv_req->flags |= REQUEST_ZLP;
2589
2590		dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2591			priv_ep->name);
2592		ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2593	}
2594
2595	spin_unlock_irqrestore(&priv_dev->lock, flags);
2596	return ret;
2597}
2598
2599/**
2600 * cdns3_gadget_ep_dequeue - Remove request from transfer queue
2601 * @ep: endpoint object associated with request
2602 * @request: request object
2603 *
2604 * Returns 0 on success, error code elsewhere
2605 */
2606int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2607			    struct usb_request *request)
2608{
2609	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2610	struct cdns3_device *priv_dev;
2611	struct usb_request *req, *req_temp;
2612	struct cdns3_request *priv_req;
2613	struct cdns3_trb *link_trb;
2614	u8 req_on_hw_ring = 0;
2615	unsigned long flags;
2616	int ret = 0;
2617	int val;
2618
2619	if (!ep || !request || !ep->desc)
2620		return -EINVAL;
2621
2622	priv_dev = priv_ep->cdns3_dev;
2623
2624	spin_lock_irqsave(&priv_dev->lock, flags);
2625
2626	priv_req = to_cdns3_request(request);
2627
2628	trace_cdns3_ep_dequeue(priv_req);
2629
2630	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2631
2632	list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2633				 list) {
2634		if (request == req) {
2635			req_on_hw_ring = 1;
2636			goto found;
2637		}
2638	}
2639
2640	list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2641				 list) {
2642		if (request == req)
2643			goto found;
2644	}
2645
2646	goto not_found;
2647
2648found:
2649	link_trb = priv_req->trb;
2650
2651	/* Update ring only if removed request is on pending_req_list list */
2652	if (req_on_hw_ring && link_trb) {
2653		/* Stop DMA */
2654		writel(EP_CMD_DFLUSH, &priv_dev->regs->ep_cmd);
2655
2656		/* wait for DFLUSH cleared */
2657		readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2658					  !(val & EP_CMD_DFLUSH), 1, 1000);
2659
2660		link_trb->buffer = cpu_to_le32(TRB_BUFFER(priv_ep->trb_pool_dma +
2661			((priv_req->end_trb + 1) * TRB_SIZE)));
2662		link_trb->control = cpu_to_le32((le32_to_cpu(link_trb->control) & TRB_CYCLE) |
2663				    TRB_TYPE(TRB_LINK) | TRB_CHAIN);
2664
2665		if (priv_ep->wa1_trb == priv_req->trb)
2666			cdns3_wa1_restore_cycle_bit(priv_ep);
2667	}
2668
2669	cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2670
2671	req = cdns3_next_request(&priv_ep->pending_req_list);
2672	if (req)
2673		cdns3_rearm_transfer(priv_ep, 1);
2674
2675not_found:
2676	spin_unlock_irqrestore(&priv_dev->lock, flags);
2677	return ret;
2678}
2679
2680/**
2681 * __cdns3_gadget_ep_set_halt - Sets stall on selected endpoint
2682 * Should be called after acquiring spin_lock and selecting ep
2683 * @priv_ep: endpoint object to set stall on.
2684 */
2685void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2686{
2687	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2688
2689	trace_cdns3_halt(priv_ep, 1, 0);
2690
2691	if (!(priv_ep->flags & EP_STALLED)) {
2692		u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2693
2694		if (!(ep_sts_reg & EP_STS_DBUSY))
2695			cdns3_ep_stall_flush(priv_ep);
2696		else
2697			priv_ep->flags |= EP_STALL_PENDING;
2698	}
2699}
2700
2701/**
2702 * __cdns3_gadget_ep_clear_halt - Clears stall on selected endpoint
2703 * Should be called after acquiring spin_lock and selecting ep
2704 * @priv_ep: endpoint object to clear stall on
2705 */
2706int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2707{
2708	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2709	struct usb_request *request;
2710	struct cdns3_request *priv_req;
2711	struct cdns3_trb *trb = NULL;
2712	struct cdns3_trb trb_tmp;
2713	int ret;
2714	int val;
2715
2716	trace_cdns3_halt(priv_ep, 0, 0);
2717
2718	request = cdns3_next_request(&priv_ep->pending_req_list);
2719	if (request) {
2720		priv_req = to_cdns3_request(request);
2721		trb = priv_req->trb;
2722		if (trb) {
2723			trb_tmp = *trb;
2724			trb->control = trb->control ^ cpu_to_le32(TRB_CYCLE);
2725		}
2726	}
2727
2728	writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2729
2730	/* wait for EPRST cleared */
2731	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2732					!(val & EP_CMD_EPRST), 1, 100);
2733	if (ret)
2734		return -EINVAL;
2735
2736	priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2737
2738	if (request) {
2739		if (trb)
2740			*trb = trb_tmp;
2741
2742		cdns3_rearm_transfer(priv_ep, 1);
2743	}
2744
2745	cdns3_start_all_request(priv_dev, priv_ep);
2746	return ret;
2747}
2748
2749/**
2750 * cdns3_gadget_ep_set_halt - Sets/clears stall on selected endpoint
2751 * @ep: endpoint object to set/clear stall on
2752 * @value: 1 for set stall, 0 for clear stall
2753 *
2754 * Returns 0 on success, error code elsewhere
2755 */
2756int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2757{
2758	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2759	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2760	unsigned long flags;
2761	int ret = 0;
2762
2763	if (!(priv_ep->flags & EP_ENABLED))
2764		return -EPERM;
2765
2766	spin_lock_irqsave(&priv_dev->lock, flags);
2767
2768	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2769
2770	if (!value) {
2771		priv_ep->flags &= ~EP_WEDGE;
2772		ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2773	} else {
2774		__cdns3_gadget_ep_set_halt(priv_ep);
2775	}
2776
2777	spin_unlock_irqrestore(&priv_dev->lock, flags);
2778
2779	return ret;
2780}
2781
2782extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2783
2784static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2785	.enable = cdns3_gadget_ep_enable,
2786	.disable = cdns3_gadget_ep_disable,
2787	.alloc_request = cdns3_gadget_ep_alloc_request,
2788	.free_request = cdns3_gadget_ep_free_request,
2789	.queue = cdns3_gadget_ep_queue,
2790	.dequeue = cdns3_gadget_ep_dequeue,
2791	.set_halt = cdns3_gadget_ep_set_halt,
2792	.set_wedge = cdns3_gadget_ep_set_wedge,
2793};
2794
2795/**
2796 * cdns3_gadget_get_frame - Returns number of actual ITP frame
2797 * @gadget: gadget object
2798 *
2799 * Returns number of actual ITP frame
2800 */
2801static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2802{
2803	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2804
2805	return readl(&priv_dev->regs->usb_itpn);
2806}
2807
2808int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2809{
2810	enum usb_device_speed speed;
2811
2812	speed = cdns3_get_speed(priv_dev);
2813
2814	if (speed >= USB_SPEED_SUPER)
2815		return 0;
2816
2817	/* Start driving resume signaling to indicate remote wakeup. */
2818	writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2819
2820	return 0;
2821}
2822
2823static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2824{
2825	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2826	unsigned long flags;
2827	int ret = 0;
2828
2829	spin_lock_irqsave(&priv_dev->lock, flags);
2830	ret = __cdns3_gadget_wakeup(priv_dev);
2831	spin_unlock_irqrestore(&priv_dev->lock, flags);
2832	return ret;
2833}
2834
2835static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2836					int is_selfpowered)
2837{
2838	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2839	unsigned long flags;
2840
2841	spin_lock_irqsave(&priv_dev->lock, flags);
2842	priv_dev->is_selfpowered = !!is_selfpowered;
2843	spin_unlock_irqrestore(&priv_dev->lock, flags);
2844	return 0;
2845}
2846
2847static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2848{
2849	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2850
2851	if (is_on) {
2852		writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2853	} else {
2854		writel(~0, &priv_dev->regs->ep_ists);
2855		writel(~0, &priv_dev->regs->usb_ists);
2856		writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2857	}
2858
2859	return 0;
2860}
2861
2862static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2863{
2864	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2865	u32 reg;
2866
2867	cdns3_ep0_config(priv_dev);
2868
2869	/* enable interrupts for endpoint 0 (in and out) */
2870	writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2871
2872	/*
2873	 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2874	 * revision of controller.
2875	 */
2876	if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2877		reg = readl(&regs->dbg_link1);
2878
2879		reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2880		reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2881		       DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2882		writel(reg, &regs->dbg_link1);
2883	}
2884
2885	/*
2886	 * By default some platforms has set protected access to memory.
2887	 * This cause problem with cache, so driver restore non-secure
2888	 * access to memory.
2889	 */
2890	reg = readl(&regs->dma_axi_ctrl);
2891	reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2892	       DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2893	writel(reg, &regs->dma_axi_ctrl);
2894
2895	/* enable generic interrupt*/
2896	writel(USB_IEN_INIT, &regs->usb_ien);
2897	writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2898	/*  keep Fast Access bit */
2899	writel(PUSB_PWR_FST_REG_ACCESS, &priv_dev->regs->usb_pwr);
2900
2901	cdns3_configure_dmult(priv_dev, NULL);
2902}
2903
2904/**
2905 * cdns3_gadget_udc_start - Gadget start
2906 * @gadget: gadget object
2907 * @driver: driver which operates on this gadget
2908 *
2909 * Returns 0 on success, error code elsewhere
2910 */
2911static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2912				  struct usb_gadget_driver *driver)
2913{
2914	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2915	unsigned long flags;
2916	enum usb_device_speed max_speed = driver->max_speed;
2917
2918	spin_lock_irqsave(&priv_dev->lock, flags);
2919	priv_dev->gadget_driver = driver;
2920
2921	/* limit speed if necessary */
2922	max_speed = min(driver->max_speed, gadget->max_speed);
2923
2924	switch (max_speed) {
2925	case USB_SPEED_FULL:
2926		writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2927		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2928		break;
2929	case USB_SPEED_HIGH:
2930		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2931		break;
2932	case USB_SPEED_SUPER:
2933		break;
2934	default:
2935		dev_err(priv_dev->dev,
2936			"invalid maximum_speed parameter %d\n",
2937			max_speed);
2938		fallthrough;
2939	case USB_SPEED_UNKNOWN:
2940		/* default to superspeed */
2941		max_speed = USB_SPEED_SUPER;
2942		break;
2943	}
2944
2945	cdns3_gadget_config(priv_dev);
2946	spin_unlock_irqrestore(&priv_dev->lock, flags);
2947	return 0;
2948}
2949
2950/**
2951 * cdns3_gadget_udc_stop - Stops gadget
2952 * @gadget: gadget object
2953 *
2954 * Returns 0
2955 */
2956static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2957{
2958	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2959	struct cdns3_endpoint *priv_ep;
2960	u32 bEndpointAddress;
2961	struct usb_ep *ep;
2962	int val;
2963
2964	priv_dev->gadget_driver = NULL;
2965
2966	priv_dev->onchip_used_size = 0;
2967	priv_dev->out_mem_is_allocated = 0;
2968	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2969
2970	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2971		priv_ep = ep_to_cdns3_ep(ep);
2972		bEndpointAddress = priv_ep->num | priv_ep->dir;
2973		cdns3_select_ep(priv_dev, bEndpointAddress);
2974		writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2975		readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2976					  !(val & EP_CMD_EPRST), 1, 100);
2977
2978		priv_ep->flags &= ~EP_CLAIMED;
2979	}
2980
2981	/* disable interrupt for device */
2982	writel(0, &priv_dev->regs->usb_ien);
2983	writel(0, &priv_dev->regs->usb_pwr);
2984	writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2985
2986	return 0;
2987}
2988
2989/**
2990 * cdns3_gadget_check_config - ensure cdns3 can support the USB configuration
2991 * @gadget: pointer to the USB gadget
2992 *
2993 * Used to record the maximum number of endpoints being used in a USB composite
2994 * device. (across all configurations)  This is to be used in the calculation
2995 * of the TXFIFO sizes when resizing internal memory for individual endpoints.
2996 * It will help ensured that the resizing logic reserves enough space for at
2997 * least one max packet.
2998 */
2999static int cdns3_gadget_check_config(struct usb_gadget *gadget)
3000{
3001	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
3002	struct usb_ep *ep;
3003	int n_in = 0;
3004	int total;
3005
3006	list_for_each_entry(ep, &gadget->ep_list, ep_list) {
3007		if (ep->claimed && (ep->address & USB_DIR_IN))
3008			n_in++;
3009	}
3010
3011	/* 2KB are reserved for EP0, 1KB for out*/
3012	total = 2 + n_in + 1;
3013
3014	if (total > priv_dev->onchip_buffers)
3015		return -ENOMEM;
3016
3017	priv_dev->ep_buf_size = priv_dev->ep_iso_burst =
3018			(priv_dev->onchip_buffers - 2) / (n_in + 1);
3019
3020	return 0;
3021}
3022
3023static const struct usb_gadget_ops cdns3_gadget_ops = {
3024	.get_frame = cdns3_gadget_get_frame,
3025	.wakeup = cdns3_gadget_wakeup,
3026	.set_selfpowered = cdns3_gadget_set_selfpowered,
3027	.pullup = cdns3_gadget_pullup,
3028	.udc_start = cdns3_gadget_udc_start,
3029	.udc_stop = cdns3_gadget_udc_stop,
3030	.match_ep = cdns3_gadget_match_ep,
3031	.check_config = cdns3_gadget_check_config,
3032};
3033
3034static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
3035{
3036	int i;
3037
3038	/* ep0 OUT point to ep0 IN. */
3039	priv_dev->eps[16] = NULL;
3040
3041	for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
3042		if (priv_dev->eps[i]) {
3043			cdns3_free_trb_pool(priv_dev->eps[i]);
3044			devm_kfree(priv_dev->dev, priv_dev->eps[i]);
3045		}
3046}
3047
3048/**
3049 * cdns3_init_eps - Initializes software endpoints of gadget
3050 * @priv_dev: extended gadget object
3051 *
3052 * Returns 0 on success, error code elsewhere
3053 */
3054static int cdns3_init_eps(struct cdns3_device *priv_dev)
3055{
3056	u32 ep_enabled_reg, iso_ep_reg;
3057	struct cdns3_endpoint *priv_ep;
3058	int ep_dir, ep_number;
3059	u32 ep_mask;
3060	int ret = 0;
3061	int i;
3062
3063	/* Read it from USB_CAP3 to USB_CAP5 */
3064	ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
3065	iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
3066
3067	dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
3068
3069	for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
3070		ep_dir = i >> 4;	/* i div 16 */
3071		ep_number = i & 0xF;	/* i % 16 */
3072		ep_mask = BIT(i);
3073
3074		if (!(ep_enabled_reg & ep_mask))
3075			continue;
3076
3077		if (ep_dir && !ep_number) {
3078			priv_dev->eps[i] = priv_dev->eps[0];
3079			continue;
3080		}
3081
3082		priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
3083				       GFP_KERNEL);
3084		if (!priv_ep)
3085			goto err;
3086
3087		/* set parent of endpoint object */
3088		priv_ep->cdns3_dev = priv_dev;
3089		priv_dev->eps[i] = priv_ep;
3090		priv_ep->num = ep_number;
3091		priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
3092
3093		if (!ep_number) {
3094			ret = cdns3_init_ep0(priv_dev, priv_ep);
3095			if (ret) {
3096				dev_err(priv_dev->dev, "Failed to init ep0\n");
3097				goto err;
3098			}
3099		} else {
3100			snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
3101				 ep_number, !!ep_dir ? "in" : "out");
3102			priv_ep->endpoint.name = priv_ep->name;
3103
3104			usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
3105						   CDNS3_EP_MAX_PACKET_LIMIT);
3106			priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
3107			priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
3108			if (ep_dir)
3109				priv_ep->endpoint.caps.dir_in = 1;
3110			else
3111				priv_ep->endpoint.caps.dir_out = 1;
3112
3113			if (iso_ep_reg & ep_mask)
3114				priv_ep->endpoint.caps.type_iso = 1;
3115
3116			priv_ep->endpoint.caps.type_bulk = 1;
3117			priv_ep->endpoint.caps.type_int = 1;
3118
3119			list_add_tail(&priv_ep->endpoint.ep_list,
3120				      &priv_dev->gadget.ep_list);
3121		}
3122
3123		priv_ep->flags = 0;
3124
3125		dev_dbg(priv_dev->dev, "Initialized  %s support: %s %s\n",
3126			 priv_ep->name,
3127			 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
3128			 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
3129
3130		INIT_LIST_HEAD(&priv_ep->pending_req_list);
3131		INIT_LIST_HEAD(&priv_ep->deferred_req_list);
3132		INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
3133	}
3134
3135	return 0;
3136err:
3137	cdns3_free_all_eps(priv_dev);
3138	return -ENOMEM;
3139}
3140
3141static void cdns3_gadget_release(struct device *dev)
3142{
3143	struct cdns3_device *priv_dev = container_of(dev,
3144			struct cdns3_device, gadget.dev);
3145
3146	kfree(priv_dev);
3147}
3148
3149static void cdns3_gadget_exit(struct cdns *cdns)
3150{
3151	struct cdns3_device *priv_dev;
3152
3153	priv_dev = cdns->gadget_dev;
3154
3155
3156	pm_runtime_mark_last_busy(cdns->dev);
3157	pm_runtime_put_autosuspend(cdns->dev);
3158
3159	usb_del_gadget(&priv_dev->gadget);
3160	devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
3161
3162	cdns3_free_all_eps(priv_dev);
3163
3164	while (!list_empty(&priv_dev->aligned_buf_list)) {
3165		struct cdns3_aligned_buf *buf;
3166
3167		buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3168		dma_free_noncoherent(priv_dev->sysdev, buf->size,
3169				  buf->buf,
3170				  buf->dma,
3171				  buf->dir);
3172
3173		list_del(&buf->list);
3174		kfree(buf);
3175	}
3176
3177	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3178			  priv_dev->setup_dma);
3179	dma_pool_destroy(priv_dev->eps_dma_pool);
3180
3181	kfree(priv_dev->zlp_buf);
3182	usb_put_gadget(&priv_dev->gadget);
3183	cdns->gadget_dev = NULL;
3184	cdns_drd_gadget_off(cdns);
3185}
3186
3187static int cdns3_gadget_start(struct cdns *cdns)
3188{
3189	struct cdns3_device *priv_dev;
3190	u32 max_speed;
3191	int ret;
3192
3193	priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3194	if (!priv_dev)
3195		return -ENOMEM;
3196
3197	usb_initialize_gadget(cdns->dev, &priv_dev->gadget,
3198			cdns3_gadget_release);
3199	cdns->gadget_dev = priv_dev;
3200	priv_dev->sysdev = cdns->dev;
3201	priv_dev->dev = cdns->dev;
3202	priv_dev->regs = cdns->dev_regs;
3203
3204	device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3205				 &priv_dev->onchip_buffers);
3206
3207	if (priv_dev->onchip_buffers <=  0) {
3208		u32 reg = readl(&priv_dev->regs->usb_cap2);
3209
3210		priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3211	}
3212
3213	if (!priv_dev->onchip_buffers)
3214		priv_dev->onchip_buffers = 256;
3215
3216	max_speed = usb_get_maximum_speed(cdns->dev);
3217
3218	/* Check the maximum_speed parameter */
3219	switch (max_speed) {
3220	case USB_SPEED_FULL:
3221	case USB_SPEED_HIGH:
3222	case USB_SPEED_SUPER:
3223		break;
3224	default:
3225		dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3226			max_speed);
3227		fallthrough;
3228	case USB_SPEED_UNKNOWN:
3229		/* default to superspeed */
3230		max_speed = USB_SPEED_SUPER;
3231		break;
3232	}
3233
3234	/* fill gadget fields */
3235	priv_dev->gadget.max_speed = max_speed;
3236	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3237	priv_dev->gadget.ops = &cdns3_gadget_ops;
3238	priv_dev->gadget.name = "usb-ss-gadget";
3239	priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3240	priv_dev->gadget.irq = cdns->dev_irq;
3241
3242	spin_lock_init(&priv_dev->lock);
3243	INIT_WORK(&priv_dev->pending_status_wq,
3244		  cdns3_pending_setup_status_handler);
3245
3246	INIT_WORK(&priv_dev->aligned_buf_wq,
3247		  cdns3_free_aligned_request_buf);
3248
3249	/* initialize endpoint container */
3250	INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3251	INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3252	priv_dev->eps_dma_pool = dma_pool_create("cdns3_eps_dma_pool",
3253						 priv_dev->sysdev,
3254						 TRB_RING_SIZE, 8, 0);
3255	if (!priv_dev->eps_dma_pool) {
3256		dev_err(priv_dev->dev, "Failed to create TRB dma pool\n");
3257		ret = -ENOMEM;
3258		goto err1;
3259	}
3260
3261	ret = cdns3_init_eps(priv_dev);
3262	if (ret) {
3263		dev_err(priv_dev->dev, "Failed to create endpoints\n");
3264		goto err1;
3265	}
3266
3267	/* allocate memory for setup packet buffer */
3268	priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3269						 &priv_dev->setup_dma, GFP_DMA);
3270	if (!priv_dev->setup_buf) {
3271		ret = -ENOMEM;
3272		goto err2;
3273	}
3274
3275	priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3276
3277	dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3278		readl(&priv_dev->regs->usb_cap6));
3279	dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3280		readl(&priv_dev->regs->usb_cap1));
3281	dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3282		readl(&priv_dev->regs->usb_cap2));
3283
3284	priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3285	if (priv_dev->dev_ver >= DEV_VER_V2)
3286		priv_dev->gadget.sg_supported = 1;
3287
3288	priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3289	if (!priv_dev->zlp_buf) {
3290		ret = -ENOMEM;
3291		goto err3;
3292	}
3293
3294	/* add USB gadget device */
3295	ret = usb_add_gadget(&priv_dev->gadget);
3296	if (ret < 0) {
3297		dev_err(priv_dev->dev, "Failed to add gadget\n");
3298		goto err4;
3299	}
3300
3301	return 0;
3302err4:
3303	kfree(priv_dev->zlp_buf);
3304err3:
3305	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3306			  priv_dev->setup_dma);
3307err2:
3308	cdns3_free_all_eps(priv_dev);
3309err1:
3310	dma_pool_destroy(priv_dev->eps_dma_pool);
3311
3312	usb_put_gadget(&priv_dev->gadget);
3313	cdns->gadget_dev = NULL;
3314	return ret;
3315}
3316
3317static int __cdns3_gadget_init(struct cdns *cdns)
3318{
3319	int ret = 0;
3320
3321	/* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3322	ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3323	if (ret) {
3324		dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3325		return ret;
3326	}
3327
3328	cdns_drd_gadget_on(cdns);
3329	pm_runtime_get_sync(cdns->dev);
3330
3331	ret = cdns3_gadget_start(cdns);
3332	if (ret) {
3333		pm_runtime_put_sync(cdns->dev);
3334		return ret;
3335	}
3336
3337	/*
3338	 * Because interrupt line can be shared with other components in
3339	 * driver it can't use IRQF_ONESHOT flag here.
3340	 */
3341	ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3342					cdns3_device_irq_handler,
3343					cdns3_device_thread_irq_handler,
3344					IRQF_SHARED, dev_name(cdns->dev),
3345					cdns->gadget_dev);
3346
3347	if (ret)
3348		goto err0;
3349
3350	return 0;
3351err0:
3352	cdns3_gadget_exit(cdns);
3353	return ret;
3354}
3355
3356static int cdns3_gadget_suspend(struct cdns *cdns, bool do_wakeup)
3357__must_hold(&cdns->lock)
3358{
3359	struct cdns3_device *priv_dev = cdns->gadget_dev;
3360
3361	spin_unlock(&cdns->lock);
3362	cdns3_disconnect_gadget(priv_dev);
3363	spin_lock(&cdns->lock);
3364
3365	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3366	usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3367	cdns3_hw_reset_eps_config(priv_dev);
3368
3369	/* disable interrupt for device */
3370	writel(0, &priv_dev->regs->usb_ien);
3371
3372	return 0;
3373}
3374
3375static int cdns3_gadget_resume(struct cdns *cdns, bool hibernated)
3376{
3377	struct cdns3_device *priv_dev = cdns->gadget_dev;
3378
3379	if (!priv_dev->gadget_driver)
3380		return 0;
3381
3382	cdns3_gadget_config(priv_dev);
3383	if (hibernated)
3384		writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
3385
3386	return 0;
3387}
3388
3389/**
3390 * cdns3_gadget_init - initialize device structure
3391 *
3392 * @cdns: cdns instance
3393 *
3394 * This function initializes the gadget.
3395 */
3396int cdns3_gadget_init(struct cdns *cdns)
3397{
3398	struct cdns_role_driver *rdrv;
3399
3400	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3401	if (!rdrv)
3402		return -ENOMEM;
3403
3404	rdrv->start	= __cdns3_gadget_init;
3405	rdrv->stop	= cdns3_gadget_exit;
3406	rdrv->suspend	= cdns3_gadget_suspend;
3407	rdrv->resume	= cdns3_gadget_resume;
3408	rdrv->state	= CDNS_ROLE_STATE_INACTIVE;
3409	rdrv->name	= "gadget";
3410	cdns->roles[USB_ROLE_DEVICE] = rdrv;
3411
3412	return 0;
3413}