1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Cadence USBSS DRD Driver - gadget side.
   4 *
   5 * Copyright (C) 2018-2019 Cadence Design Systems.
   6 * Copyright (C) 2017-2018 NXP
   7 *
   8 * Authors: Pawel Jez <pjez@cadence.com>,
   9 *          Pawel Laszczak <pawell@cadence.com>
  10 *          Peter Chen <peter.chen@nxp.com>
  11 */
  12
  13/*
  14 * Work around 1:
  15 * At some situations, the controller may get stale data address in TRB
  16 * at below sequences:
  17 * 1. Controller read TRB includes data address
  18 * 2. Software updates TRBs includes data address and Cycle bit
  19 * 3. Controller read TRB which includes Cycle bit
  20 * 4. DMA run with stale data address
  21 *
  22 * To fix this problem, driver needs to make the first TRB in TD as invalid.
  23 * After preparing all TRBs driver needs to check the position of DMA and
  24 * if the DMA point to the first just added TRB and doorbell is 1,
  25 * then driver must defer making this TRB as valid. This TRB will be make
  26 * as valid during adding next TRB only if DMA is stopped or at TRBERR
  27 * interrupt.
  28 *
  29 * Issue has been fixed in DEV_VER_V3 version of controller.
  30 *
  31 * Work around 2:
  32 * Controller for OUT endpoints has shared on-chip buffers for all incoming
  33 * packets, including ep0out. It's FIFO buffer, so packets must be handle by DMA
  34 * in correct order. If the first packet in the buffer will not be handled,
  35 * then the following packets directed for other endpoints and  functions
  36 * will be blocked.
  37 * Additionally the packets directed to one endpoint can block entire on-chip
  38 * buffers. In this case transfer to other endpoints also will blocked.
  39 *
  40 * To resolve this issue after raising the descriptor missing interrupt
  41 * driver prepares internal usb_request object and use it to arm DMA transfer.
  42 *
  43 * The problematic situation was observed in case when endpoint has been enabled
  44 * but no usb_request were queued. Driver try detects such endpoints and will
  45 * use this workaround only for these endpoint.
  46 *
  47 * Driver use limited number of buffer. This number can be set by macro
  48 * CDNS3_WA2_NUM_BUFFERS.
  49 *
  50 * Such blocking situation was observed on ACM gadget. For this function
  51 * host send OUT data packet but ACM function is not prepared for this packet.
  52 * It's cause that buffer placed in on chip memory block transfer to other
  53 * endpoints.
  54 *
  55 * Issue has been fixed in DEV_VER_V2 version of controller.
  56 *
  57 */
  58
  59#include <linux/dma-mapping.h>
  60#include <linux/usb/gadget.h>
  61#include <linux/module.h>
  62#include <linux/iopoll.h>
  63
  64#include "core.h"
  65#include "gadget-export.h"
  66#include "gadget.h"
  67#include "trace.h"
  68#include "drd.h"
  69
  70static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
  71				   struct usb_request *request,
  72				   gfp_t gfp_flags);
  73
  74static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
  75				 struct usb_request *request);
  76
  77static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
  78					struct usb_request *request);
  79
  80/**
  81 * cdns3_clear_register_bit - clear bit in given register.
  82 * @ptr: address of device controller register to be read and changed
  83 * @mask: bits requested to clar
  84 */
  85static void cdns3_clear_register_bit(void __iomem *ptr, u32 mask)
  86{
  87	mask = readl(ptr) & ~mask;
  88	writel(mask, ptr);
  89}
  90
  91/**
  92 * cdns3_set_register_bit - set bit in given register.
  93 * @ptr: address of device controller register to be read and changed
  94 * @mask: bits requested to set
  95 */
  96void cdns3_set_register_bit(void __iomem *ptr, u32 mask)
  97{
  98	mask = readl(ptr) | mask;
  99	writel(mask, ptr);
 100}
 101
 102/**
 103 * cdns3_ep_addr_to_index - Macro converts endpoint address to
 104 * index of endpoint object in cdns3_device.eps[] container
 105 * @ep_addr: endpoint address for which endpoint object is required
 106 *
 107 */
 108u8 cdns3_ep_addr_to_index(u8 ep_addr)
 109{
 110	return (((ep_addr & 0x7F)) + ((ep_addr & USB_DIR_IN) ? 16 : 0));
 111}
 112
 113static int cdns3_get_dma_pos(struct cdns3_device *priv_dev,
 114			     struct cdns3_endpoint *priv_ep)
 115{
 116	int dma_index;
 117
 118	dma_index = readl(&priv_dev->regs->ep_traddr) - priv_ep->trb_pool_dma;
 119
 120	return dma_index / TRB_SIZE;
 121}
 122
 123/**
 124 * cdns3_next_request - returns next request from list
 125 * @list: list containing requests
 126 *
 127 * Returns request or NULL if no requests in list
 128 */
 129struct usb_request *cdns3_next_request(struct list_head *list)
 130{
 131	return list_first_entry_or_null(list, struct usb_request, list);
 132}
 133
 134/**
 135 * cdns3_next_align_buf - returns next buffer from list
 136 * @list: list containing buffers
 137 *
 138 * Returns buffer or NULL if no buffers in list
 139 */
 140static struct cdns3_aligned_buf *cdns3_next_align_buf(struct list_head *list)
 141{
 142	return list_first_entry_or_null(list, struct cdns3_aligned_buf, list);
 143}
 144
 145/**
 146 * cdns3_next_priv_request - returns next request from list
 147 * @list: list containing requests
 148 *
 149 * Returns request or NULL if no requests in list
 150 */
 151static struct cdns3_request *cdns3_next_priv_request(struct list_head *list)
 152{
 153	return list_first_entry_or_null(list, struct cdns3_request, list);
 154}
 155
 156/**
 157 * select_ep - selects endpoint
 158 * @priv_dev:  extended gadget object
 159 * @ep: endpoint address
 160 */
 161void cdns3_select_ep(struct cdns3_device *priv_dev, u32 ep)
 162{
 163	if (priv_dev->selected_ep == ep)
 164		return;
 165
 166	priv_dev->selected_ep = ep;
 167	writel(ep, &priv_dev->regs->ep_sel);
 168}
 169
 170/**
 171 * cdns3_get_tdl - gets current tdl for selected endpoint.
 172 * @priv_dev:  extended gadget object
 173 *
 174 * Before calling this function the appropriate endpoint must
 175 * be selected by means of cdns3_select_ep function.
 176 */
 177static int cdns3_get_tdl(struct cdns3_device *priv_dev)
 178{
 179	if (priv_dev->dev_ver < DEV_VER_V3)
 180		return EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 181	else
 182		return readl(&priv_dev->regs->ep_tdl);
 183}
 184
 185dma_addr_t cdns3_trb_virt_to_dma(struct cdns3_endpoint *priv_ep,
 186				 struct cdns3_trb *trb)
 187{
 188	u32 offset = (char *)trb - (char *)priv_ep->trb_pool;
 189
 190	return priv_ep->trb_pool_dma + offset;
 191}
 192
 193static int cdns3_ring_size(struct cdns3_endpoint *priv_ep)
 194{
 195	switch (priv_ep->type) {
 196	case USB_ENDPOINT_XFER_ISOC:
 197		return TRB_ISO_RING_SIZE;
 198	case USB_ENDPOINT_XFER_CONTROL:
 199		return TRB_CTRL_RING_SIZE;
 200	default:
 201		if (priv_ep->use_streams)
 202			return TRB_STREAM_RING_SIZE;
 203		else
 204			return TRB_RING_SIZE;
 205	}
 206}
 207
 208static void cdns3_free_trb_pool(struct cdns3_endpoint *priv_ep)
 209{
 210	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 211
 212	if (priv_ep->trb_pool) {
 213		dma_free_coherent(priv_dev->sysdev,
 214				  cdns3_ring_size(priv_ep),
 215				  priv_ep->trb_pool, priv_ep->trb_pool_dma);
 216		priv_ep->trb_pool = NULL;
 217	}
 218}
 219
 220/**
 221 * cdns3_allocate_trb_pool - Allocates TRB's pool for selected endpoint
 222 * @priv_ep:  endpoint object
 223 *
 224 * Function will return 0 on success or -ENOMEM on allocation error
 225 */
 226int cdns3_allocate_trb_pool(struct cdns3_endpoint *priv_ep)
 227{
 228	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 229	int ring_size = cdns3_ring_size(priv_ep);
 230	int num_trbs = ring_size / TRB_SIZE;
 231	struct cdns3_trb *link_trb;
 232
 233	if (priv_ep->trb_pool && priv_ep->alloc_ring_size < ring_size)
 234		cdns3_free_trb_pool(priv_ep);
 235
 236	if (!priv_ep->trb_pool) {
 237		priv_ep->trb_pool = dma_alloc_coherent(priv_dev->sysdev,
 238						       ring_size,
 239						       &priv_ep->trb_pool_dma,
 240						       GFP_DMA32 | GFP_ATOMIC);
 241		if (!priv_ep->trb_pool)
 242			return -ENOMEM;
 243
 244		priv_ep->alloc_ring_size = ring_size;
 245	}
 246
 247	memset(priv_ep->trb_pool, 0, ring_size);
 248
 249	priv_ep->num_trbs = num_trbs;
 250
 251	if (!priv_ep->num)
 252		return 0;
 253
 254	/* Initialize the last TRB as Link TRB */
 
 255	link_trb = (priv_ep->trb_pool + (priv_ep->num_trbs - 1));
 
 
 256
 257	if (priv_ep->use_streams) {
 258		/*
 259		 * For stream capable endpoints driver use single correct TRB.
 260		 * The last trb has zeroed cycle bit
 261		 */
 262		link_trb->control = 0;
 263	} else {
 264		link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma);
 265		link_trb->control = TRB_CYCLE | TRB_TYPE(TRB_LINK) | TRB_TOGGLE;
 266	}
 267	return 0;
 268}
 269
 
 
 
 
 
 
 
 
 
 
 
 
 270/**
 271 * cdns3_ep_stall_flush - Stalls and flushes selected endpoint
 272 * @priv_ep: endpoint object
 273 *
 274 * Endpoint must be selected before call to this function
 275 */
 276static void cdns3_ep_stall_flush(struct cdns3_endpoint *priv_ep)
 277{
 278	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 279	int val;
 280
 281	trace_cdns3_halt(priv_ep, 1, 1);
 282
 283	writel(EP_CMD_DFLUSH | EP_CMD_ERDY | EP_CMD_SSTALL,
 284	       &priv_dev->regs->ep_cmd);
 285
 286	/* wait for DFLUSH cleared */
 287	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
 288				  !(val & EP_CMD_DFLUSH), 1, 1000);
 289	priv_ep->flags |= EP_STALLED;
 290	priv_ep->flags &= ~EP_STALL_PENDING;
 291}
 292
 293/**
 294 * cdns3_hw_reset_eps_config - reset endpoints configuration kept by controller.
 295 * @priv_dev: extended gadget object
 296 */
 297void cdns3_hw_reset_eps_config(struct cdns3_device *priv_dev)
 298{
 299	writel(USB_CONF_CFGRST, &priv_dev->regs->usb_conf);
 300
 301	cdns3_allow_enable_l1(priv_dev, 0);
 302	priv_dev->hw_configured_flag = 0;
 303	priv_dev->onchip_used_size = 0;
 304	priv_dev->out_mem_is_allocated = 0;
 305	priv_dev->wait_for_setup = 0;
 306	priv_dev->using_streams = 0;
 307}
 308
 309/**
 310 * cdns3_ep_inc_trb - increment a trb index.
 311 * @index: Pointer to the TRB index to increment.
 312 * @cs: Cycle state
 313 * @trb_in_seg: number of TRBs in segment
 314 *
 315 * The index should never point to the link TRB. After incrementing,
 316 * if it is point to the link TRB, wrap around to the beginning and revert
 317 * cycle state bit The
 318 * link TRB is always at the last TRB entry.
 319 */
 320static void cdns3_ep_inc_trb(int *index, u8 *cs, int trb_in_seg)
 321{
 322	(*index)++;
 323	if (*index == (trb_in_seg - 1)) {
 324		*index = 0;
 325		*cs ^=  1;
 326	}
 327}
 328
 329/**
 330 * cdns3_ep_inc_enq - increment endpoint's enqueue pointer
 331 * @priv_ep: The endpoint whose enqueue pointer we're incrementing
 332 */
 333static void cdns3_ep_inc_enq(struct cdns3_endpoint *priv_ep)
 334{
 335	priv_ep->free_trbs--;
 336	cdns3_ep_inc_trb(&priv_ep->enqueue, &priv_ep->pcs, priv_ep->num_trbs);
 337}
 338
 339/**
 340 * cdns3_ep_inc_deq - increment endpoint's dequeue pointer
 341 * @priv_ep: The endpoint whose dequeue pointer we're incrementing
 342 */
 343static void cdns3_ep_inc_deq(struct cdns3_endpoint *priv_ep)
 344{
 345	priv_ep->free_trbs++;
 346	cdns3_ep_inc_trb(&priv_ep->dequeue, &priv_ep->ccs, priv_ep->num_trbs);
 347}
 348
 349static void cdns3_move_deq_to_next_trb(struct cdns3_request *priv_req)
 350{
 351	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 352	int current_trb = priv_req->start_trb;
 353
 354	while (current_trb != priv_req->end_trb) {
 355		cdns3_ep_inc_deq(priv_ep);
 356		current_trb = priv_ep->dequeue;
 357	}
 358
 359	cdns3_ep_inc_deq(priv_ep);
 360}
 361
 362/**
 363 * cdns3_allow_enable_l1 - enable/disable permits to transition to L1.
 364 * @priv_dev: Extended gadget object
 365 * @enable: Enable/disable permit to transition to L1.
 366 *
 367 * If bit USB_CONF_L1EN is set and device receive Extended Token packet,
 368 * then controller answer with ACK handshake.
 369 * If bit USB_CONF_L1DS is set and device receive Extended Token packet,
 370 * then controller answer with NYET handshake.
 371 */
 372void cdns3_allow_enable_l1(struct cdns3_device *priv_dev, int enable)
 373{
 374	if (enable)
 375		writel(USB_CONF_L1EN, &priv_dev->regs->usb_conf);
 376	else
 377		writel(USB_CONF_L1DS, &priv_dev->regs->usb_conf);
 378}
 379
 380enum usb_device_speed cdns3_get_speed(struct cdns3_device *priv_dev)
 381{
 382	u32 reg;
 383
 384	reg = readl(&priv_dev->regs->usb_sts);
 385
 386	if (DEV_SUPERSPEED(reg))
 387		return USB_SPEED_SUPER;
 388	else if (DEV_HIGHSPEED(reg))
 389		return USB_SPEED_HIGH;
 390	else if (DEV_FULLSPEED(reg))
 391		return USB_SPEED_FULL;
 392	else if (DEV_LOWSPEED(reg))
 393		return USB_SPEED_LOW;
 394	return USB_SPEED_UNKNOWN;
 395}
 396
 397/**
 398 * cdns3_start_all_request - add to ring all request not started
 399 * @priv_dev: Extended gadget object
 400 * @priv_ep: The endpoint for whom request will be started.
 401 *
 402 * Returns return ENOMEM if transfer ring i not enough TRBs to start
 403 *         all requests.
 404 */
 405static int cdns3_start_all_request(struct cdns3_device *priv_dev,
 406				   struct cdns3_endpoint *priv_ep)
 407{
 408	struct usb_request *request;
 409	int ret = 0;
 410	u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 411
 412	/*
 413	 * If the last pending transfer is INTERNAL
 414	 * OR streams are enabled for this endpoint
 415	 * do NOT start new transfer till the last one is pending
 416	 */
 417	if (!pending_empty) {
 418		struct cdns3_request *priv_req;
 419
 420		request = cdns3_next_request(&priv_ep->pending_req_list);
 421		priv_req = to_cdns3_request(request);
 422		if ((priv_req->flags & REQUEST_INTERNAL) ||
 423		    (priv_ep->flags & EP_TDLCHK_EN) ||
 424			priv_ep->use_streams) {
 425			dev_dbg(priv_dev->dev, "Blocking external request\n");
 426			return ret;
 427		}
 428	}
 429
 430	while (!list_empty(&priv_ep->deferred_req_list)) {
 431		request = cdns3_next_request(&priv_ep->deferred_req_list);
 432
 433		if (!priv_ep->use_streams) {
 434			ret = cdns3_ep_run_transfer(priv_ep, request);
 435		} else {
 436			priv_ep->stream_sg_idx = 0;
 437			ret = cdns3_ep_run_stream_transfer(priv_ep, request);
 438		}
 439		if (ret)
 440			return ret;
 441
 442		list_del(&request->list);
 443		list_add_tail(&request->list,
 444			      &priv_ep->pending_req_list);
 445		if (request->stream_id != 0 || (priv_ep->flags & EP_TDLCHK_EN))
 446			break;
 447	}
 448
 449	priv_ep->flags &= ~EP_RING_FULL;
 450	return ret;
 451}
 452
 453/*
 454 * WA2: Set flag for all not ISOC OUT endpoints. If this flag is set
 455 * driver try to detect whether endpoint need additional internal
 456 * buffer for unblocking on-chip FIFO buffer. This flag will be cleared
 457 * if before first DESCMISS interrupt the DMA will be armed.
 458 */
 459#define cdns3_wa2_enable_detection(priv_dev, priv_ep, reg) do { \
 460	if (!priv_ep->dir && priv_ep->type != USB_ENDPOINT_XFER_ISOC) { \
 461		priv_ep->flags |= EP_QUIRK_EXTRA_BUF_DET; \
 462		(reg) |= EP_STS_EN_DESCMISEN; \
 463	} } while (0)
 464
 465/**
 466 * cdns3_wa2_descmiss_copy_data copy data from internal requests to
 467 * request queued by class driver.
 468 * @priv_ep: extended endpoint object
 469 * @request: request object
 470 */
 471static void cdns3_wa2_descmiss_copy_data(struct cdns3_endpoint *priv_ep,
 472					 struct usb_request *request)
 473{
 474	struct usb_request *descmiss_req;
 475	struct cdns3_request *descmiss_priv_req;
 476
 477	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 478		int chunk_end;
 479		int length;
 480
 481		descmiss_priv_req =
 482			cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 483		descmiss_req = &descmiss_priv_req->request;
 484
 485		/* driver can't touch pending request */
 486		if (descmiss_priv_req->flags & REQUEST_PENDING)
 487			break;
 488
 489		chunk_end = descmiss_priv_req->flags & REQUEST_INTERNAL_CH;
 490		length = request->actual + descmiss_req->actual;
 491
 492		request->status = descmiss_req->status;
 493
 494		if (length <= request->length) {
 495			memcpy(&((u8 *)request->buf)[request->actual],
 496			       descmiss_req->buf,
 497			       descmiss_req->actual);
 498			request->actual = length;
 499		} else {
 500			/* It should never occures */
 501			request->status = -ENOMEM;
 502		}
 503
 504		list_del_init(&descmiss_priv_req->list);
 505
 506		kfree(descmiss_req->buf);
 507		cdns3_gadget_ep_free_request(&priv_ep->endpoint, descmiss_req);
 508		--priv_ep->wa2_counter;
 509
 510		if (!chunk_end)
 511			break;
 512	}
 513}
 514
 515static struct usb_request *cdns3_wa2_gadget_giveback(struct cdns3_device *priv_dev,
 516						     struct cdns3_endpoint *priv_ep,
 517						     struct cdns3_request *priv_req)
 518{
 519	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN &&
 520	    priv_req->flags & REQUEST_INTERNAL) {
 521		struct usb_request *req;
 522
 523		req = cdns3_next_request(&priv_ep->deferred_req_list);
 524
 525		priv_ep->descmis_req = NULL;
 526
 527		if (!req)
 528			return NULL;
 529
 530		/* unmap the gadget request before copying data */
 531		usb_gadget_unmap_request_by_dev(priv_dev->sysdev, req,
 532						priv_ep->dir);
 533
 534		cdns3_wa2_descmiss_copy_data(priv_ep, req);
 535		if (!(priv_ep->flags & EP_QUIRK_END_TRANSFER) &&
 536		    req->length != req->actual) {
 537			/* wait for next part of transfer */
 538			/* re-map the gadget request buffer*/
 539			usb_gadget_map_request_by_dev(priv_dev->sysdev, req,
 540				usb_endpoint_dir_in(priv_ep->endpoint.desc));
 541			return NULL;
 542		}
 543
 544		if (req->status == -EINPROGRESS)
 545			req->status = 0;
 546
 547		list_del_init(&req->list);
 548		cdns3_start_all_request(priv_dev, priv_ep);
 549		return req;
 550	}
 551
 552	return &priv_req->request;
 553}
 554
 555static int cdns3_wa2_gadget_ep_queue(struct cdns3_device *priv_dev,
 556				     struct cdns3_endpoint *priv_ep,
 557				     struct cdns3_request *priv_req)
 558{
 559	int deferred = 0;
 560
 561	/*
 562	 * If transfer was queued before DESCMISS appear than we
 563	 * can disable handling of DESCMISS interrupt. Driver assumes that it
 564	 * can disable special treatment for this endpoint.
 565	 */
 566	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 567		u32 reg;
 568
 569		cdns3_select_ep(priv_dev, priv_ep->num | priv_ep->dir);
 570		priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 571		reg = readl(&priv_dev->regs->ep_sts_en);
 572		reg &= ~EP_STS_EN_DESCMISEN;
 573		trace_cdns3_wa2(priv_ep, "workaround disabled\n");
 574		writel(reg, &priv_dev->regs->ep_sts_en);
 575	}
 576
 577	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
 578		u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 579		u8 descmiss_empty = list_empty(&priv_ep->wa2_descmiss_req_list);
 580
 581		/*
 582		 *  DESCMISS transfer has been finished, so data will be
 583		 *  directly copied from internal allocated usb_request
 584		 *  objects.
 585		 */
 586		if (pending_empty && !descmiss_empty &&
 587		    !(priv_req->flags & REQUEST_INTERNAL)) {
 588			cdns3_wa2_descmiss_copy_data(priv_ep,
 589						     &priv_req->request);
 590
 591			trace_cdns3_wa2(priv_ep, "get internal stored data");
 592
 593			list_add_tail(&priv_req->request.list,
 594				      &priv_ep->pending_req_list);
 595			cdns3_gadget_giveback(priv_ep, priv_req,
 596					      priv_req->request.status);
 597
 598			/*
 599			 * Intentionally driver returns positive value as
 600			 * correct value. It informs that transfer has
 601			 * been finished.
 602			 */
 603			return EINPROGRESS;
 604		}
 605
 606		/*
 607		 * Driver will wait for completion DESCMISS transfer,
 608		 * before starts new, not DESCMISS transfer.
 609		 */
 610		if (!pending_empty && !descmiss_empty) {
 611			trace_cdns3_wa2(priv_ep, "wait for pending transfer\n");
 612			deferred = 1;
 613		}
 614
 615		if (priv_req->flags & REQUEST_INTERNAL)
 616			list_add_tail(&priv_req->list,
 617				      &priv_ep->wa2_descmiss_req_list);
 618	}
 619
 620	return deferred;
 621}
 622
 623static void cdns3_wa2_remove_old_request(struct cdns3_endpoint *priv_ep)
 624{
 625	struct cdns3_request *priv_req;
 626
 627	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
 628		u8 chain;
 629
 630		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
 631		chain = !!(priv_req->flags & REQUEST_INTERNAL_CH);
 632
 633		trace_cdns3_wa2(priv_ep, "removes eldest request");
 634
 635		kfree(priv_req->request.buf);
 636		cdns3_gadget_ep_free_request(&priv_ep->endpoint,
 637					     &priv_req->request);
 638		list_del_init(&priv_req->list);
 639		--priv_ep->wa2_counter;
 640
 641		if (!chain)
 642			break;
 643	}
 644}
 645
 646/**
 647 * cdns3_wa2_descmissing_packet - handles descriptor missing event.
 648 * @priv_ep: extended gadget object
 649 *
 650 * This function is used only for WA2. For more information see Work around 2
 651 * description.
 652 */
 653static void cdns3_wa2_descmissing_packet(struct cdns3_endpoint *priv_ep)
 654{
 655	struct cdns3_request *priv_req;
 656	struct usb_request *request;
 657	u8 pending_empty = list_empty(&priv_ep->pending_req_list);
 658
 659	/* check for pending transfer */
 660	if (!pending_empty) {
 661		trace_cdns3_wa2(priv_ep, "Ignoring Descriptor missing IRQ\n");
 662		return;
 663	}
 664
 665	if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET) {
 666		priv_ep->flags &= ~EP_QUIRK_EXTRA_BUF_DET;
 667		priv_ep->flags |= EP_QUIRK_EXTRA_BUF_EN;
 668	}
 669
 670	trace_cdns3_wa2(priv_ep, "Description Missing detected\n");
 671
 672	if (priv_ep->wa2_counter >= CDNS3_WA2_NUM_BUFFERS) {
 673		trace_cdns3_wa2(priv_ep, "WA2 overflow\n");
 674		cdns3_wa2_remove_old_request(priv_ep);
 675	}
 676
 677	request = cdns3_gadget_ep_alloc_request(&priv_ep->endpoint,
 678						GFP_ATOMIC);
 679	if (!request)
 680		goto err;
 681
 682	priv_req = to_cdns3_request(request);
 683	priv_req->flags |= REQUEST_INTERNAL;
 684
 685	/* if this field is still assigned it indicate that transfer related
 686	 * with this request has not been finished yet. Driver in this
 687	 * case simply allocate next request and assign flag REQUEST_INTERNAL_CH
 688	 * flag to previous one. It will indicate that current request is
 689	 * part of the previous one.
 690	 */
 691	if (priv_ep->descmis_req)
 692		priv_ep->descmis_req->flags |= REQUEST_INTERNAL_CH;
 693
 694	priv_req->request.buf = kzalloc(CDNS3_DESCMIS_BUF_SIZE,
 695					GFP_ATOMIC);
 696	priv_ep->wa2_counter++;
 697
 698	if (!priv_req->request.buf) {
 699		cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 700		goto err;
 701	}
 702
 703	priv_req->request.length = CDNS3_DESCMIS_BUF_SIZE;
 704	priv_ep->descmis_req = priv_req;
 705
 706	__cdns3_gadget_ep_queue(&priv_ep->endpoint,
 707				&priv_ep->descmis_req->request,
 708				GFP_ATOMIC);
 709
 710	return;
 711
 712err:
 713	dev_err(priv_ep->cdns3_dev->dev,
 714		"Failed: No sufficient memory for DESCMIS\n");
 715}
 716
 717static void cdns3_wa2_reset_tdl(struct cdns3_device *priv_dev)
 718{
 719	u16 tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
 720
 721	if (tdl) {
 722		u16 reset_val = EP_CMD_TDL_MAX + 1 - tdl;
 723
 724		writel(EP_CMD_TDL_SET(reset_val) | EP_CMD_STDL,
 725		       &priv_dev->regs->ep_cmd);
 726	}
 727}
 728
 729static void cdns3_wa2_check_outq_status(struct cdns3_device *priv_dev)
 730{
 731	u32 ep_sts_reg;
 732
 733	/* select EP0-out */
 734	cdns3_select_ep(priv_dev, 0);
 735
 736	ep_sts_reg = readl(&priv_dev->regs->ep_sts);
 737
 738	if (EP_STS_OUTQ_VAL(ep_sts_reg)) {
 739		u32 outq_ep_num = EP_STS_OUTQ_NO(ep_sts_reg);
 740		struct cdns3_endpoint *outq_ep = priv_dev->eps[outq_ep_num];
 741
 742		if ((outq_ep->flags & EP_ENABLED) && !(outq_ep->use_streams) &&
 743		    outq_ep->type != USB_ENDPOINT_XFER_ISOC && outq_ep_num) {
 744			u8 pending_empty = list_empty(&outq_ep->pending_req_list);
 745
 746			if ((outq_ep->flags & EP_QUIRK_EXTRA_BUF_DET) ||
 747			    (outq_ep->flags & EP_QUIRK_EXTRA_BUF_EN) ||
 748			    !pending_empty) {
 749			} else {
 750				u32 ep_sts_en_reg;
 751				u32 ep_cmd_reg;
 752
 753				cdns3_select_ep(priv_dev, outq_ep->num |
 754						outq_ep->dir);
 755				ep_sts_en_reg = readl(&priv_dev->regs->ep_sts_en);
 756				ep_cmd_reg = readl(&priv_dev->regs->ep_cmd);
 757
 758				outq_ep->flags |= EP_TDLCHK_EN;
 759				cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
 760						       EP_CFG_TDL_CHK);
 761
 762				cdns3_wa2_enable_detection(priv_dev, outq_ep,
 763							   ep_sts_en_reg);
 764				writel(ep_sts_en_reg,
 765				       &priv_dev->regs->ep_sts_en);
 766				/* reset tdl value to zero */
 767				cdns3_wa2_reset_tdl(priv_dev);
 768				/*
 769				 * Memory barrier - Reset tdl before ringing the
 770				 * doorbell.
 771				 */
 772				wmb();
 773				if (EP_CMD_DRDY & ep_cmd_reg) {
 774					trace_cdns3_wa2(outq_ep, "Enabling WA2 skipping doorbell\n");
 775
 776				} else {
 777					trace_cdns3_wa2(outq_ep, "Enabling WA2 ringing doorbell\n");
 778					/*
 779					 * ring doorbell to generate DESCMIS irq
 780					 */
 781					writel(EP_CMD_DRDY,
 782					       &priv_dev->regs->ep_cmd);
 783				}
 784			}
 785		}
 786	}
 787}
 788
 789/**
 790 * cdns3_gadget_giveback - call struct usb_request's ->complete callback
 791 * @priv_ep: The endpoint to whom the request belongs to
 792 * @priv_req: The request we're giving back
 793 * @status: completion code for the request
 794 *
 795 * Must be called with controller's lock held and interrupts disabled. This
 796 * function will unmap @req and call its ->complete() callback to notify upper
 797 * layers that it has completed.
 798 */
 799void cdns3_gadget_giveback(struct cdns3_endpoint *priv_ep,
 800			   struct cdns3_request *priv_req,
 801			   int status)
 802{
 803	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 804	struct usb_request *request = &priv_req->request;
 805
 806	list_del_init(&request->list);
 807
 808	if (request->status == -EINPROGRESS)
 809		request->status = status;
 810
 811	usb_gadget_unmap_request_by_dev(priv_dev->sysdev, request,
 812					priv_ep->dir);
 813
 814	if ((priv_req->flags & REQUEST_UNALIGNED) &&
 815	    priv_ep->dir == USB_DIR_OUT && !request->status)
 816		memcpy(request->buf, priv_req->aligned_buf->buf,
 817		       request->length);
 818
 819	priv_req->flags &= ~(REQUEST_PENDING | REQUEST_UNALIGNED);
 820	trace_cdns3_gadget_giveback(priv_req);
 821
 822	if (priv_dev->dev_ver < DEV_VER_V2) {
 823		request = cdns3_wa2_gadget_giveback(priv_dev, priv_ep,
 824						    priv_req);
 825		if (!request)
 826			return;
 827	}
 828
 829	if (request->complete) {
 830		spin_unlock(&priv_dev->lock);
 831		usb_gadget_giveback_request(&priv_ep->endpoint,
 832					    request);
 833		spin_lock(&priv_dev->lock);
 834	}
 835
 836	if (request->buf == priv_dev->zlp_buf)
 837		cdns3_gadget_ep_free_request(&priv_ep->endpoint, request);
 838}
 839
 840static void cdns3_wa1_restore_cycle_bit(struct cdns3_endpoint *priv_ep)
 841{
 842	/* Work around for stale data address in TRB*/
 843	if (priv_ep->wa1_set) {
 844		trace_cdns3_wa1(priv_ep, "restore cycle bit");
 845
 846		priv_ep->wa1_set = 0;
 847		priv_ep->wa1_trb_index = 0xFFFF;
 848		if (priv_ep->wa1_cycle_bit) {
 849			priv_ep->wa1_trb->control =
 850				priv_ep->wa1_trb->control | 0x1;
 851		} else {
 852			priv_ep->wa1_trb->control =
 853				priv_ep->wa1_trb->control & ~0x1;
 854		}
 855	}
 856}
 857
 858static void cdns3_free_aligned_request_buf(struct work_struct *work)
 859{
 860	struct cdns3_device *priv_dev = container_of(work, struct cdns3_device,
 861					aligned_buf_wq);
 862	struct cdns3_aligned_buf *buf, *tmp;
 863	unsigned long flags;
 864
 865	spin_lock_irqsave(&priv_dev->lock, flags);
 866
 867	list_for_each_entry_safe(buf, tmp, &priv_dev->aligned_buf_list, list) {
 868		if (!buf->in_use) {
 869			list_del(&buf->list);
 870
 871			/*
 872			 * Re-enable interrupts to free DMA capable memory.
 873			 * Driver can't free this memory with disabled
 874			 * interrupts.
 875			 */
 876			spin_unlock_irqrestore(&priv_dev->lock, flags);
 877			dma_free_coherent(priv_dev->sysdev, buf->size,
 878					  buf->buf, buf->dma);
 879			kfree(buf);
 880			spin_lock_irqsave(&priv_dev->lock, flags);
 881		}
 882	}
 883
 884	spin_unlock_irqrestore(&priv_dev->lock, flags);
 885}
 886
 887static int cdns3_prepare_aligned_request_buf(struct cdns3_request *priv_req)
 888{
 889	struct cdns3_endpoint *priv_ep = priv_req->priv_ep;
 890	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 891	struct cdns3_aligned_buf *buf;
 892
 893	/* check if buffer is aligned to 8. */
 894	if (!((uintptr_t)priv_req->request.buf & 0x7))
 895		return 0;
 896
 897	buf = priv_req->aligned_buf;
 898
 899	if (!buf || priv_req->request.length > buf->size) {
 900		buf = kzalloc(sizeof(*buf), GFP_ATOMIC);
 901		if (!buf)
 902			return -ENOMEM;
 903
 904		buf->size = priv_req->request.length;
 905
 906		buf->buf = dma_alloc_coherent(priv_dev->sysdev,
 907					      buf->size,
 908					      &buf->dma,
 909					      GFP_ATOMIC);
 910		if (!buf->buf) {
 911			kfree(buf);
 912			return -ENOMEM;
 913		}
 914
 915		if (priv_req->aligned_buf) {
 916			trace_cdns3_free_aligned_request(priv_req);
 917			priv_req->aligned_buf->in_use = 0;
 918			queue_work(system_freezable_wq,
 919				   &priv_dev->aligned_buf_wq);
 920		}
 921
 922		buf->in_use = 1;
 923		priv_req->aligned_buf = buf;
 924
 925		list_add_tail(&buf->list,
 926			      &priv_dev->aligned_buf_list);
 927	}
 928
 929	if (priv_ep->dir == USB_DIR_IN) {
 930		memcpy(buf->buf, priv_req->request.buf,
 931		       priv_req->request.length);
 932	}
 933
 934	priv_req->flags |= REQUEST_UNALIGNED;
 935	trace_cdns3_prepare_aligned_request(priv_req);
 936
 937	return 0;
 938}
 939
 940static int cdns3_wa1_update_guard(struct cdns3_endpoint *priv_ep,
 941				  struct cdns3_trb *trb)
 942{
 943	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 944
 945	if (!priv_ep->wa1_set) {
 946		u32 doorbell;
 947
 948		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 949
 950		if (doorbell) {
 951			priv_ep->wa1_cycle_bit = priv_ep->pcs ? TRB_CYCLE : 0;
 952			priv_ep->wa1_set = 1;
 953			priv_ep->wa1_trb = trb;
 954			priv_ep->wa1_trb_index = priv_ep->enqueue;
 955			trace_cdns3_wa1(priv_ep, "set guard");
 956			return 0;
 957		}
 958	}
 959	return 1;
 960}
 961
 962static void cdns3_wa1_tray_restore_cycle_bit(struct cdns3_device *priv_dev,
 963					     struct cdns3_endpoint *priv_ep)
 964{
 965	int dma_index;
 966	u32 doorbell;
 967
 968	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
 969	dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
 970
 971	if (!doorbell || dma_index != priv_ep->wa1_trb_index)
 972		cdns3_wa1_restore_cycle_bit(priv_ep);
 973}
 974
 975static int cdns3_ep_run_stream_transfer(struct cdns3_endpoint *priv_ep,
 976					struct usb_request *request)
 977{
 978	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
 979	struct cdns3_request *priv_req;
 980	struct cdns3_trb *trb;
 981	dma_addr_t trb_dma;
 982	int address;
 983	u32 control;
 984	u32 length;
 985	u32 tdl;
 986	unsigned int sg_idx = priv_ep->stream_sg_idx;
 987
 988	priv_req = to_cdns3_request(request);
 989	address = priv_ep->endpoint.desc->bEndpointAddress;
 990
 991	priv_ep->flags |= EP_PENDING_REQUEST;
 992
 993	/* must allocate buffer aligned to 8 */
 994	if (priv_req->flags & REQUEST_UNALIGNED)
 995		trb_dma = priv_req->aligned_buf->dma;
 996	else
 997		trb_dma = request->dma;
 998
 999	/*  For stream capable endpoints driver use only single TD. */
1000	trb = priv_ep->trb_pool + priv_ep->enqueue;
1001	priv_req->start_trb = priv_ep->enqueue;
1002	priv_req->end_trb = priv_req->start_trb;
1003	priv_req->trb = trb;
1004
1005	cdns3_select_ep(priv_ep->cdns3_dev, address);
1006
1007	control = TRB_TYPE(TRB_NORMAL) | TRB_CYCLE |
1008		  TRB_STREAM_ID(priv_req->request.stream_id) | TRB_ISP;
1009
1010	if (!request->num_sgs) {
1011		trb->buffer = TRB_BUFFER(trb_dma);
1012		length = request->length;
1013	} else {
1014		trb->buffer = TRB_BUFFER(request->sg[sg_idx].dma_address);
1015		length = request->sg[sg_idx].length;
1016	}
1017
1018	tdl = DIV_ROUND_UP(length, priv_ep->endpoint.maxpacket);
1019
1020	trb->length = TRB_BURST_LEN(16 /*priv_ep->trb_burst_size*/) |
1021				  TRB_LEN(length);
1022
1023	/*
1024	 * For DEV_VER_V2 controller version we have enabled
1025	 * USB_CONF2_EN_TDL_TRB in DMULT configuration.
1026	 * This enables TDL calculation based on TRB, hence setting TDL in TRB.
1027	 */
1028	if (priv_dev->dev_ver >= DEV_VER_V2) {
1029		if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1030			trb->length |= TRB_TDL_SS_SIZE(tdl);
1031	}
1032	priv_req->flags |= REQUEST_PENDING;
1033
1034	trb->control = control;
1035
1036	trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1037
1038	/*
1039	 * Memory barrier - Cycle Bit must be set before trb->length  and
1040	 * trb->buffer fields.
1041	 */
1042	wmb();
1043
1044	/* always first element */
1045	writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma),
1046	       &priv_dev->regs->ep_traddr);
1047
1048	if (!(priv_ep->flags & EP_STALLED)) {
1049		trace_cdns3_ring(priv_ep);
1050		/*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1051		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1052
1053		priv_ep->prime_flag = false;
1054
1055		/*
1056		 * Controller version DEV_VER_V2 tdl calculation
1057		 * is based on TRB
1058		 */
1059
1060		if (priv_dev->dev_ver < DEV_VER_V2)
1061			writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1062			       &priv_dev->regs->ep_cmd);
1063		else if (priv_dev->dev_ver > DEV_VER_V2)
1064			writel(tdl, &priv_dev->regs->ep_tdl);
1065
1066		priv_ep->last_stream_id = priv_req->request.stream_id;
1067		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1068		writel(EP_CMD_ERDY_SID(priv_req->request.stream_id) |
1069		       EP_CMD_ERDY, &priv_dev->regs->ep_cmd);
1070
1071		trace_cdns3_doorbell_epx(priv_ep->name,
1072					 readl(&priv_dev->regs->ep_traddr));
1073	}
1074
1075	/* WORKAROUND for transition to L0 */
1076	__cdns3_gadget_wakeup(priv_dev);
1077
1078	return 0;
1079}
1080
1081/**
1082 * cdns3_ep_run_transfer - start transfer on no-default endpoint hardware
1083 * @priv_ep: endpoint object
1084 * @request: request object
1085 *
1086 * Returns zero on success or negative value on failure
1087 */
1088static int cdns3_ep_run_transfer(struct cdns3_endpoint *priv_ep,
1089				 struct usb_request *request)
1090{
1091	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1092	struct cdns3_request *priv_req;
1093	struct cdns3_trb *trb;
1094	dma_addr_t trb_dma;
1095	u32 togle_pcs = 1;
1096	int sg_iter = 0;
1097	int num_trb;
1098	int address;
1099	u32 control;
1100	int pcs;
1101	u16 total_tdl = 0;
1102
1103	if (priv_ep->type == USB_ENDPOINT_XFER_ISOC)
1104		num_trb = priv_ep->interval;
1105	else
1106		num_trb = request->num_sgs ? request->num_sgs : 1;
1107
1108	if (num_trb > priv_ep->free_trbs) {
1109		priv_ep->flags |= EP_RING_FULL;
1110		return -ENOBUFS;
1111	}
1112
1113	priv_req = to_cdns3_request(request);
1114	address = priv_ep->endpoint.desc->bEndpointAddress;
1115
1116	priv_ep->flags |= EP_PENDING_REQUEST;
1117
1118	/* must allocate buffer aligned to 8 */
1119	if (priv_req->flags & REQUEST_UNALIGNED)
1120		trb_dma = priv_req->aligned_buf->dma;
1121	else
1122		trb_dma = request->dma;
1123
1124	trb = priv_ep->trb_pool + priv_ep->enqueue;
1125	priv_req->start_trb = priv_ep->enqueue;
1126	priv_req->trb = trb;
1127
1128	cdns3_select_ep(priv_ep->cdns3_dev, address);
1129
1130	/* prepare ring */
1131	if ((priv_ep->enqueue + num_trb)  >= (priv_ep->num_trbs - 1)) {
1132		struct cdns3_trb *link_trb;
1133		int doorbell, dma_index;
1134		u32 ch_bit = 0;
1135
1136		doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1137		dma_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1138
1139		/* Driver can't update LINK TRB if it is current processed. */
1140		if (doorbell && dma_index == priv_ep->num_trbs - 1) {
1141			priv_ep->flags |= EP_DEFERRED_DRDY;
1142			return -ENOBUFS;
1143		}
1144
1145		/*updating C bt in  Link TRB before starting DMA*/
1146		link_trb = priv_ep->trb_pool + (priv_ep->num_trbs - 1);
1147		/*
1148		 * For TRs size equal 2 enabling TRB_CHAIN for epXin causes
1149		 * that DMA stuck at the LINK TRB.
1150		 * On the other hand, removing TRB_CHAIN for longer TRs for
1151		 * epXout cause that DMA stuck after handling LINK TRB.
1152		 * To eliminate this strange behavioral driver set TRB_CHAIN
1153		 * bit only for TR size > 2.
1154		 */
1155		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC ||
1156		    TRBS_PER_SEGMENT > 2)
1157			ch_bit = TRB_CHAIN;
1158
1159		link_trb->control = ((priv_ep->pcs) ? TRB_CYCLE : 0) |
1160				    TRB_TYPE(TRB_LINK) | TRB_TOGGLE | ch_bit;
1161	}
1162
1163	if (priv_dev->dev_ver <= DEV_VER_V2)
1164		togle_pcs = cdns3_wa1_update_guard(priv_ep, trb);
1165
1166	/* set incorrect Cycle Bit for first trb*/
1167	control = priv_ep->pcs ? 0 : TRB_CYCLE;
1168
1169	do {
1170		u32 length;
1171		u16 td_size = 0;
1172
1173		/* fill TRB */
1174		control |= TRB_TYPE(TRB_NORMAL);
1175		trb->buffer = TRB_BUFFER(request->num_sgs == 0
1176				? trb_dma : request->sg[sg_iter].dma_address);
1177
1178		if (likely(!request->num_sgs))
1179			length = request->length;
1180		else
1181			length = request->sg[sg_iter].length;
1182
1183		if (likely(priv_dev->dev_ver >= DEV_VER_V2))
1184			td_size = DIV_ROUND_UP(length,
1185					       priv_ep->endpoint.maxpacket);
1186		else if (priv_ep->flags & EP_TDLCHK_EN)
1187			total_tdl += DIV_ROUND_UP(length,
1188					       priv_ep->endpoint.maxpacket);
1189
1190		trb->length = TRB_BURST_LEN(priv_ep->trb_burst_size) |
1191					TRB_LEN(length);
1192		if (priv_dev->gadget.speed == USB_SPEED_SUPER)
1193			trb->length |= TRB_TDL_SS_SIZE(td_size);
1194		else
1195			control |= TRB_TDL_HS_SIZE(td_size);
1196
1197		pcs = priv_ep->pcs ? TRB_CYCLE : 0;
1198
1199		/*
1200		 * first trb should be prepared as last to avoid processing
1201		 *  transfer to early
1202		 */
1203		if (sg_iter != 0)
1204			control |= pcs;
1205
1206		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir) {
1207			control |= TRB_IOC | TRB_ISP;
1208		} else {
1209			/* for last element in TD or in SG list */
1210			if (sg_iter == (num_trb - 1) && sg_iter != 0)
1211				control |= pcs | TRB_IOC | TRB_ISP;
1212		}
1213
1214		if (sg_iter)
1215			trb->control = control;
1216		else
1217			priv_req->trb->control = control;
1218
1219		control = 0;
1220		++sg_iter;
1221		priv_req->end_trb = priv_ep->enqueue;
1222		cdns3_ep_inc_enq(priv_ep);
1223		trb = priv_ep->trb_pool + priv_ep->enqueue;
1224	} while (sg_iter < num_trb);
1225
1226	trb = priv_req->trb;
1227
1228	priv_req->flags |= REQUEST_PENDING;
1229
1230	if (sg_iter == 1)
1231		trb->control |= TRB_IOC | TRB_ISP;
1232
1233	if (priv_dev->dev_ver < DEV_VER_V2 &&
1234	    (priv_ep->flags & EP_TDLCHK_EN)) {
1235		u16 tdl = total_tdl;
1236		u16 old_tdl = EP_CMD_TDL_GET(readl(&priv_dev->regs->ep_cmd));
1237
1238		if (tdl > EP_CMD_TDL_MAX) {
1239			tdl = EP_CMD_TDL_MAX;
1240			priv_ep->pending_tdl = total_tdl - EP_CMD_TDL_MAX;
1241		}
1242
1243		if (old_tdl < tdl) {
1244			tdl -= old_tdl;
1245			writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL,
1246			       &priv_dev->regs->ep_cmd);
1247		}
1248	}
1249
1250	/*
1251	 * Memory barrier - cycle bit must be set before other filds in trb.
1252	 */
1253	wmb();
1254
1255	/* give the TD to the consumer*/
1256	if (togle_pcs)
1257		trb->control =  trb->control ^ 1;
1258
1259	if (priv_dev->dev_ver <= DEV_VER_V2)
1260		cdns3_wa1_tray_restore_cycle_bit(priv_dev, priv_ep);
1261
1262	trace_cdns3_prepare_trb(priv_ep, priv_req->trb);
1263
1264	/*
1265	 * Memory barrier - Cycle Bit must be set before trb->length  and
1266	 * trb->buffer fields.
1267	 */
1268	wmb();
1269
1270	/*
1271	 * For DMULT mode we can set address to transfer ring only once after
1272	 * enabling endpoint.
1273	 */
1274	if (priv_ep->flags & EP_UPDATE_EP_TRBADDR) {
1275		/*
1276		 * Until SW is not ready to handle the OUT transfer the ISO OUT
1277		 * Endpoint should be disabled (EP_CFG.ENABLE = 0).
1278		 * EP_CFG_ENABLE must be set before updating ep_traddr.
1279		 */
1280		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir &&
1281		    !(priv_ep->flags & EP_QUIRK_ISO_OUT_EN)) {
1282			priv_ep->flags |= EP_QUIRK_ISO_OUT_EN;
1283			cdns3_set_register_bit(&priv_dev->regs->ep_cfg,
1284					       EP_CFG_ENABLE);
1285		}
1286
1287		writel(EP_TRADDR_TRADDR(priv_ep->trb_pool_dma +
1288					priv_req->start_trb * TRB_SIZE),
1289					&priv_dev->regs->ep_traddr);
1290
1291		priv_ep->flags &= ~EP_UPDATE_EP_TRBADDR;
1292	}
1293
1294	if (!priv_ep->wa1_set && !(priv_ep->flags & EP_STALLED)) {
1295		trace_cdns3_ring(priv_ep);
1296		/*clearing TRBERR and EP_STS_DESCMIS before seting DRDY*/
1297		writel(EP_STS_TRBERR | EP_STS_DESCMIS, &priv_dev->regs->ep_sts);
1298		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1299		trace_cdns3_doorbell_epx(priv_ep->name,
1300					 readl(&priv_dev->regs->ep_traddr));
1301	}
1302
1303	/* WORKAROUND for transition to L0 */
1304	__cdns3_gadget_wakeup(priv_dev);
1305
1306	return 0;
1307}
1308
1309void cdns3_set_hw_configuration(struct cdns3_device *priv_dev)
1310{
1311	struct cdns3_endpoint *priv_ep;
1312	struct usb_ep *ep;
1313	int val;
1314
1315	if (priv_dev->hw_configured_flag)
1316		return;
1317
1318	writel(USB_CONF_CFGSET, &priv_dev->regs->usb_conf);
 
1319
1320	cdns3_set_register_bit(&priv_dev->regs->usb_conf,
1321			       USB_CONF_U1EN | USB_CONF_U2EN);
1322
1323	/* wait until configuration set */
1324	readl_poll_timeout_atomic(&priv_dev->regs->usb_sts, val,
1325				  val & USB_STS_CFGSTS_MASK, 1, 100);
1326
1327	priv_dev->hw_configured_flag = 1;
1328
1329	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
1330		if (ep->enabled) {
1331			priv_ep = ep_to_cdns3_ep(ep);
1332			cdns3_start_all_request(priv_dev, priv_ep);
1333		}
1334	}
1335
1336	cdns3_allow_enable_l1(priv_dev, 1);
1337}
1338
1339/**
1340 * cdns3_request_handled - check whether request has been handled by DMA
1341 *
1342 * @priv_ep: extended endpoint object.
1343 * @priv_req: request object for checking
1344 *
1345 * Endpoint must be selected before invoking this function.
1346 *
1347 * Returns false if request has not been handled by DMA, else returns true.
1348 *
1349 * SR - start ring
1350 * ER -  end ring
1351 * DQ = priv_ep->dequeue - dequeue position
1352 * EQ = priv_ep->enqueue -  enqueue position
1353 * ST = priv_req->start_trb - index of first TRB in transfer ring
1354 * ET = priv_req->end_trb - index of last TRB in transfer ring
1355 * CI = current_index - index of processed TRB by DMA.
1356 *
1357 * As first step, function checks if cycle bit for priv_req->start_trb is
1358 * correct.
1359 *
1360 * some rules:
1361 * 1. priv_ep->dequeue never exceed current_index.
1362 * 2  priv_ep->enqueue never exceed priv_ep->dequeue
1363 * 3. exception: priv_ep->enqueue == priv_ep->dequeue
1364 *    and priv_ep->free_trbs is zero.
1365 *    This case indicate that TR is full.
1366 *
1367 * Then We can split recognition into two parts:
1368 * Case 1 - priv_ep->dequeue < current_index
1369 *      SR ... EQ ... DQ ... CI ... ER
1370 *      SR ... DQ ... CI ... EQ ... ER
1371 *
1372 *      Request has been handled by DMA if ST and ET is between DQ and CI.
1373 *
1374 * Case 2 - priv_ep->dequeue > current_index
1375 * This situation take place when CI go through the LINK TRB at the end of
1376 * transfer ring.
1377 *      SR ... CI ... EQ ... DQ ... ER
1378 *
1379 *      Request has been handled by DMA if ET is less then CI or
1380 *      ET is greater or equal DQ.
1381 */
1382static bool cdns3_request_handled(struct cdns3_endpoint *priv_ep,
1383				  struct cdns3_request *priv_req)
1384{
1385	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1386	struct cdns3_trb *trb;
1387	int current_index = 0;
1388	int handled = 0;
1389	int doorbell;
1390
1391	current_index = cdns3_get_dma_pos(priv_dev, priv_ep);
1392	doorbell = !!(readl(&priv_dev->regs->ep_cmd) & EP_CMD_DRDY);
1393
1394	trb = &priv_ep->trb_pool[priv_req->start_trb];
1395
1396	if ((trb->control  & TRB_CYCLE) != priv_ep->ccs)
1397		goto finish;
1398
1399	if (doorbell == 1 && current_index == priv_ep->dequeue)
1400		goto finish;
1401
1402	/* The corner case for TRBS_PER_SEGMENT equal 2). */
1403	if (TRBS_PER_SEGMENT == 2 && priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1404		handled = 1;
1405		goto finish;
1406	}
1407
1408	if (priv_ep->enqueue == priv_ep->dequeue &&
1409	    priv_ep->free_trbs == 0) {
1410		handled = 1;
1411	} else if (priv_ep->dequeue < current_index) {
1412		if ((current_index == (priv_ep->num_trbs - 1)) &&
1413		    !priv_ep->dequeue)
1414			goto finish;
1415
1416		if (priv_req->end_trb >= priv_ep->dequeue &&
1417		    priv_req->end_trb < current_index)
1418			handled = 1;
1419	} else if (priv_ep->dequeue  > current_index) {
1420		if (priv_req->end_trb  < current_index ||
1421		    priv_req->end_trb >= priv_ep->dequeue)
1422			handled = 1;
1423	}
1424
1425finish:
1426	trace_cdns3_request_handled(priv_req, current_index, handled);
1427
1428	return handled;
1429}
1430
1431static void cdns3_transfer_completed(struct cdns3_device *priv_dev,
1432				     struct cdns3_endpoint *priv_ep)
1433{
1434	struct cdns3_request *priv_req;
1435	struct usb_request *request;
1436	struct cdns3_trb *trb;
1437
1438	while (!list_empty(&priv_ep->pending_req_list)) {
1439		request = cdns3_next_request(&priv_ep->pending_req_list);
1440		priv_req = to_cdns3_request(request);
1441
1442		trb = priv_ep->trb_pool + priv_ep->dequeue;
1443
1444		/* Request was dequeued and TRB was changed to TRB_LINK. */
1445		if (TRB_FIELD_TO_TYPE(trb->control) == TRB_LINK) {
1446			trace_cdns3_complete_trb(priv_ep, trb);
1447			cdns3_move_deq_to_next_trb(priv_req);
1448		}
1449
1450		if (!request->stream_id) {
1451			/* Re-select endpoint. It could be changed by other CPU
1452			 * during handling usb_gadget_giveback_request.
1453			 */
1454			cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1455
1456			if (!cdns3_request_handled(priv_ep, priv_req))
1457				goto prepare_next_td;
1458
1459			trb = priv_ep->trb_pool + priv_ep->dequeue;
1460			trace_cdns3_complete_trb(priv_ep, trb);
1461
1462			if (trb != priv_req->trb)
1463				dev_warn(priv_dev->dev,
1464					 "request_trb=0x%p, queue_trb=0x%p\n",
1465					 priv_req->trb, trb);
1466
1467			request->actual = TRB_LEN(le32_to_cpu(trb->length));
1468			cdns3_move_deq_to_next_trb(priv_req);
1469			cdns3_gadget_giveback(priv_ep, priv_req, 0);
1470
1471			if (priv_ep->type != USB_ENDPOINT_XFER_ISOC &&
1472			    TRBS_PER_SEGMENT == 2)
1473				break;
1474		} else {
1475			/* Re-select endpoint. It could be changed by other CPU
1476			 * during handling usb_gadget_giveback_request.
1477			 */
1478			cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1479
1480			trb = priv_ep->trb_pool;
1481			trace_cdns3_complete_trb(priv_ep, trb);
 
 
 
 
 
 
1482
1483			if (trb != priv_req->trb)
1484				dev_warn(priv_dev->dev,
1485					 "request_trb=0x%p, queue_trb=0x%p\n",
1486					 priv_req->trb, trb);
1487
1488			request->actual += TRB_LEN(le32_to_cpu(trb->length));
1489
1490			if (!request->num_sgs ||
1491			    (request->num_sgs == (priv_ep->stream_sg_idx + 1))) {
1492				priv_ep->stream_sg_idx = 0;
1493				cdns3_gadget_giveback(priv_ep, priv_req, 0);
1494			} else {
1495				priv_ep->stream_sg_idx++;
1496				cdns3_ep_run_stream_transfer(priv_ep, request);
1497			}
1498			break;
1499		}
1500	}
1501	priv_ep->flags &= ~EP_PENDING_REQUEST;
1502
1503prepare_next_td:
1504	if (!(priv_ep->flags & EP_STALLED) &&
1505	    !(priv_ep->flags & EP_STALL_PENDING))
1506		cdns3_start_all_request(priv_dev, priv_ep);
1507}
1508
1509void cdns3_rearm_transfer(struct cdns3_endpoint *priv_ep, u8 rearm)
1510{
1511	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1512
1513	cdns3_wa1_restore_cycle_bit(priv_ep);
1514
1515	if (rearm) {
1516		trace_cdns3_ring(priv_ep);
1517
1518		/* Cycle Bit must be updated before arming DMA. */
1519		wmb();
1520		writel(EP_CMD_DRDY, &priv_dev->regs->ep_cmd);
1521
1522		__cdns3_gadget_wakeup(priv_dev);
1523
1524		trace_cdns3_doorbell_epx(priv_ep->name,
1525					 readl(&priv_dev->regs->ep_traddr));
1526	}
1527}
1528
1529static void cdns3_reprogram_tdl(struct cdns3_endpoint *priv_ep)
1530{
1531	u16 tdl = priv_ep->pending_tdl;
1532	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1533
1534	if (tdl > EP_CMD_TDL_MAX) {
1535		tdl = EP_CMD_TDL_MAX;
1536		priv_ep->pending_tdl -= EP_CMD_TDL_MAX;
1537	} else {
1538		priv_ep->pending_tdl = 0;
1539	}
1540
1541	writel(EP_CMD_TDL_SET(tdl) | EP_CMD_STDL, &priv_dev->regs->ep_cmd);
1542}
1543
1544/**
1545 * cdns3_check_ep_interrupt_proceed - Processes interrupt related to endpoint
1546 * @priv_ep: endpoint object
1547 *
1548 * Returns 0
1549 */
1550static int cdns3_check_ep_interrupt_proceed(struct cdns3_endpoint *priv_ep)
1551{
1552	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1553	u32 ep_sts_reg;
1554	struct usb_request *deferred_request;
1555	struct usb_request *pending_request;
1556	u32 tdl = 0;
1557
1558	cdns3_select_ep(priv_dev, priv_ep->endpoint.address);
1559
1560	trace_cdns3_epx_irq(priv_dev, priv_ep);
1561
1562	ep_sts_reg = readl(&priv_dev->regs->ep_sts);
1563	writel(ep_sts_reg, &priv_dev->regs->ep_sts);
1564
1565	if ((ep_sts_reg & EP_STS_PRIME) && priv_ep->use_streams) {
1566		bool dbusy = !!(ep_sts_reg & EP_STS_DBUSY);
1567
1568		tdl = cdns3_get_tdl(priv_dev);
1569
1570		/*
1571		 * Continue the previous transfer:
1572		 * There is some racing between ERDY and PRIME. The device send
1573		 * ERDY and almost in the same time Host send PRIME. It cause
1574		 * that host ignore the ERDY packet and driver has to send it
1575		 * again.
1576		 */
1577		if (tdl && (dbusy | !EP_STS_BUFFEMPTY(ep_sts_reg) |
1578		    EP_STS_HOSTPP(ep_sts_reg))) {
1579			writel(EP_CMD_ERDY |
1580			       EP_CMD_ERDY_SID(priv_ep->last_stream_id),
1581			       &priv_dev->regs->ep_cmd);
1582			ep_sts_reg &= ~(EP_STS_MD_EXIT | EP_STS_IOC);
1583		} else {
1584			priv_ep->prime_flag = true;
1585
1586			pending_request = cdns3_next_request(&priv_ep->pending_req_list);
1587			deferred_request = cdns3_next_request(&priv_ep->deferred_req_list);
1588
1589			if (deferred_request && !pending_request) {
1590				cdns3_start_all_request(priv_dev, priv_ep);
1591			}
1592		}
1593	}
1594
1595	if (ep_sts_reg & EP_STS_TRBERR) {
1596		if (priv_ep->flags & EP_STALL_PENDING &&
1597		    !(ep_sts_reg & EP_STS_DESCMIS &&
1598		    priv_dev->dev_ver < DEV_VER_V2)) {
1599			cdns3_ep_stall_flush(priv_ep);
1600		}
1601
1602		/*
1603		 * For isochronous transfer driver completes request on
1604		 * IOC or on TRBERR. IOC appears only when device receive
1605		 * OUT data packet. If host disable stream or lost some packet
1606		 * then the only way to finish all queued transfer is to do it
1607		 * on TRBERR event.
1608		 */
1609		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC &&
1610		    !priv_ep->wa1_set) {
1611			if (!priv_ep->dir) {
1612				u32 ep_cfg = readl(&priv_dev->regs->ep_cfg);
1613
1614				ep_cfg &= ~EP_CFG_ENABLE;
1615				writel(ep_cfg, &priv_dev->regs->ep_cfg);
1616				priv_ep->flags &= ~EP_QUIRK_ISO_OUT_EN;
1617			}
1618			cdns3_transfer_completed(priv_dev, priv_ep);
1619		} else if (!(priv_ep->flags & EP_STALLED) &&
1620			  !(priv_ep->flags & EP_STALL_PENDING)) {
1621			if (priv_ep->flags & EP_DEFERRED_DRDY) {
1622				priv_ep->flags &= ~EP_DEFERRED_DRDY;
1623				cdns3_start_all_request(priv_dev, priv_ep);
1624			} else {
1625				cdns3_rearm_transfer(priv_ep,
1626						     priv_ep->wa1_set);
1627			}
1628		}
1629	}
1630
1631	if ((ep_sts_reg & EP_STS_IOC) || (ep_sts_reg & EP_STS_ISP) ||
1632	    (ep_sts_reg & EP_STS_IOT)) {
1633		if (priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN) {
1634			if (ep_sts_reg & EP_STS_ISP)
1635				priv_ep->flags |= EP_QUIRK_END_TRANSFER;
1636			else
1637				priv_ep->flags &= ~EP_QUIRK_END_TRANSFER;
1638		}
1639
1640		if (!priv_ep->use_streams) {
1641			if ((ep_sts_reg & EP_STS_IOC) ||
1642			    (ep_sts_reg & EP_STS_ISP)) {
1643				cdns3_transfer_completed(priv_dev, priv_ep);
1644			} else if ((priv_ep->flags & EP_TDLCHK_EN) &
1645				   priv_ep->pending_tdl) {
1646				/* handle IOT with pending tdl */
1647				cdns3_reprogram_tdl(priv_ep);
1648			}
1649		} else if (priv_ep->dir == USB_DIR_OUT) {
1650			priv_ep->ep_sts_pending |= ep_sts_reg;
1651		} else if (ep_sts_reg & EP_STS_IOT) {
1652			cdns3_transfer_completed(priv_dev, priv_ep);
1653		}
1654	}
1655
1656	/*
1657	 * MD_EXIT interrupt sets when stream capable endpoint exits
1658	 * from MOVE DATA state of Bulk IN/OUT stream protocol state machine
1659	 */
1660	if (priv_ep->dir == USB_DIR_OUT && (ep_sts_reg & EP_STS_MD_EXIT) &&
1661	    (priv_ep->ep_sts_pending & EP_STS_IOT) && priv_ep->use_streams) {
1662		priv_ep->ep_sts_pending = 0;
1663		cdns3_transfer_completed(priv_dev, priv_ep);
1664	}
1665
1666	/*
1667	 * WA2: this condition should only be meet when
1668	 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_DET or
1669	 * priv_ep->flags & EP_QUIRK_EXTRA_BUF_EN.
1670	 * In other cases this interrupt will be disabled.
1671	 */
1672	if (ep_sts_reg & EP_STS_DESCMIS && priv_dev->dev_ver < DEV_VER_V2 &&
1673	    !(priv_ep->flags & EP_STALLED))
1674		cdns3_wa2_descmissing_packet(priv_ep);
1675
1676	return 0;
1677}
1678
1679static void cdns3_disconnect_gadget(struct cdns3_device *priv_dev)
1680{
1681	if (priv_dev->gadget_driver && priv_dev->gadget_driver->disconnect) {
1682		spin_unlock(&priv_dev->lock);
1683		priv_dev->gadget_driver->disconnect(&priv_dev->gadget);
1684		spin_lock(&priv_dev->lock);
1685	}
1686}
1687
1688/**
1689 * cdns3_check_usb_interrupt_proceed - Processes interrupt related to device
1690 * @priv_dev: extended gadget object
1691 * @usb_ists: bitmap representation of device's reported interrupts
1692 * (usb_ists register value)
1693 */
1694static void cdns3_check_usb_interrupt_proceed(struct cdns3_device *priv_dev,
1695					      u32 usb_ists)
1696{
1697	int speed = 0;
1698
1699	trace_cdns3_usb_irq(priv_dev, usb_ists);
1700	if (usb_ists & USB_ISTS_L1ENTI) {
1701		/*
1702		 * WORKAROUND: CDNS3 controller has issue with hardware resuming
1703		 * from L1. To fix it, if any DMA transfer is pending driver
1704		 * must starts driving resume signal immediately.
1705		 */
1706		if (readl(&priv_dev->regs->drbl))
1707			__cdns3_gadget_wakeup(priv_dev);
1708	}
1709
1710	/* Connection detected */
1711	if (usb_ists & (USB_ISTS_CON2I | USB_ISTS_CONI)) {
1712		speed = cdns3_get_speed(priv_dev);
1713		priv_dev->gadget.speed = speed;
1714		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_POWERED);
1715		cdns3_ep0_config(priv_dev);
1716	}
1717
1718	/* Disconnection detected */
1719	if (usb_ists & (USB_ISTS_DIS2I | USB_ISTS_DISI)) {
1720		cdns3_disconnect_gadget(priv_dev);
1721		priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
1722		usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
1723		cdns3_hw_reset_eps_config(priv_dev);
1724	}
1725
1726	if (usb_ists & (USB_ISTS_L2ENTI | USB_ISTS_U3ENTI)) {
1727		if (priv_dev->gadget_driver &&
1728		    priv_dev->gadget_driver->suspend) {
1729			spin_unlock(&priv_dev->lock);
1730			priv_dev->gadget_driver->suspend(&priv_dev->gadget);
1731			spin_lock(&priv_dev->lock);
1732		}
1733	}
1734
1735	if (usb_ists & (USB_ISTS_L2EXTI | USB_ISTS_U3EXTI)) {
1736		if (priv_dev->gadget_driver &&
1737		    priv_dev->gadget_driver->resume) {
1738			spin_unlock(&priv_dev->lock);
1739			priv_dev->gadget_driver->resume(&priv_dev->gadget);
1740			spin_lock(&priv_dev->lock);
1741		}
1742	}
1743
1744	/* reset*/
1745	if (usb_ists & (USB_ISTS_UWRESI | USB_ISTS_UHRESI | USB_ISTS_U2RESI)) {
1746		if (priv_dev->gadget_driver) {
1747			spin_unlock(&priv_dev->lock);
1748			usb_gadget_udc_reset(&priv_dev->gadget,
1749					     priv_dev->gadget_driver);
1750			spin_lock(&priv_dev->lock);
1751
1752			/*read again to check the actual speed*/
1753			speed = cdns3_get_speed(priv_dev);
1754			priv_dev->gadget.speed = speed;
1755			cdns3_hw_reset_eps_config(priv_dev);
1756			cdns3_ep0_config(priv_dev);
1757		}
1758	}
1759}
1760
1761/**
1762 * cdns3_device_irq_handler- interrupt handler for device part of controller
1763 *
1764 * @irq: irq number for cdns3 core device
1765 * @data: structure of cdns3
1766 *
1767 * Returns IRQ_HANDLED or IRQ_NONE
1768 */
1769static irqreturn_t cdns3_device_irq_handler(int irq, void *data)
1770{
1771	struct cdns3_device *priv_dev = data;
 
1772	irqreturn_t ret = IRQ_NONE;
1773	u32 reg;
1774
 
 
1775	/* check USB device interrupt */
1776	reg = readl(&priv_dev->regs->usb_ists);
1777	if (reg) {
1778		/* After masking interrupts the new interrupts won't be
1779		 * reported in usb_ists/ep_ists. In order to not lose some
1780		 * of them driver disables only detected interrupts.
1781		 * They will be enabled ASAP after clearing source of
1782		 * interrupt. This an unusual behavior only applies to
1783		 * usb_ists register.
1784		 */
1785		reg = ~reg & readl(&priv_dev->regs->usb_ien);
1786		/* mask deferred interrupt. */
1787		writel(reg, &priv_dev->regs->usb_ien);
1788		ret = IRQ_WAKE_THREAD;
1789	}
1790
1791	/* check endpoint interrupt */
1792	reg = readl(&priv_dev->regs->ep_ists);
1793	if (reg) {
1794		writel(0, &priv_dev->regs->ep_ien);
1795		ret = IRQ_WAKE_THREAD;
1796	}
1797
1798	return ret;
1799}
1800
1801/**
1802 * cdns3_device_thread_irq_handler- interrupt handler for device part
1803 * of controller
1804 *
1805 * @irq: irq number for cdns3 core device
1806 * @data: structure of cdns3
1807 *
1808 * Returns IRQ_HANDLED or IRQ_NONE
1809 */
1810static irqreturn_t cdns3_device_thread_irq_handler(int irq, void *data)
1811{
1812	struct cdns3_device *priv_dev = data;
 
1813	irqreturn_t ret = IRQ_NONE;
1814	unsigned long flags;
1815	unsigned int bit;
1816	unsigned long reg;
1817
 
1818	spin_lock_irqsave(&priv_dev->lock, flags);
1819
1820	reg = readl(&priv_dev->regs->usb_ists);
1821	if (reg) {
1822		writel(reg, &priv_dev->regs->usb_ists);
1823		writel(USB_IEN_INIT, &priv_dev->regs->usb_ien);
1824		cdns3_check_usb_interrupt_proceed(priv_dev, reg);
1825		ret = IRQ_HANDLED;
1826	}
1827
1828	reg = readl(&priv_dev->regs->ep_ists);
1829
1830	/* handle default endpoint OUT */
1831	if (reg & EP_ISTS_EP_OUT0) {
1832		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_OUT);
1833		ret = IRQ_HANDLED;
1834	}
1835
1836	/* handle default endpoint IN */
1837	if (reg & EP_ISTS_EP_IN0) {
1838		cdns3_check_ep0_interrupt_proceed(priv_dev, USB_DIR_IN);
1839		ret = IRQ_HANDLED;
1840	}
1841
1842	/* check if interrupt from non default endpoint, if no exit */
1843	reg &= ~(EP_ISTS_EP_OUT0 | EP_ISTS_EP_IN0);
1844	if (!reg)
1845		goto irqend;
1846
1847	for_each_set_bit(bit, &reg,
1848			 sizeof(u32) * BITS_PER_BYTE) {
1849		cdns3_check_ep_interrupt_proceed(priv_dev->eps[bit]);
1850		ret = IRQ_HANDLED;
1851	}
1852
1853	if (priv_dev->dev_ver < DEV_VER_V2 && priv_dev->using_streams)
1854		cdns3_wa2_check_outq_status(priv_dev);
1855
1856irqend:
1857	writel(~0, &priv_dev->regs->ep_ien);
1858	spin_unlock_irqrestore(&priv_dev->lock, flags);
1859
1860	return ret;
1861}
1862
1863/**
1864 * cdns3_ep_onchip_buffer_reserve - Try to reserve onchip buf for EP
1865 *
1866 * The real reservation will occur during write to EP_CFG register,
1867 * this function is used to check if the 'size' reservation is allowed.
1868 *
1869 * @priv_dev: extended gadget object
1870 * @size: the size (KB) for EP would like to allocate
1871 * @is_in: endpoint direction
1872 *
1873 * Return 0 if the required size can met or negative value on failure
1874 */
1875static int cdns3_ep_onchip_buffer_reserve(struct cdns3_device *priv_dev,
1876					  int size, int is_in)
1877{
1878	int remained;
1879
1880	/* 2KB are reserved for EP0*/
1881	remained = priv_dev->onchip_buffers - priv_dev->onchip_used_size - 2;
1882
1883	if (is_in) {
1884		if (remained < size)
1885			return -EPERM;
1886
1887		priv_dev->onchip_used_size += size;
1888	} else {
1889		int required;
1890
1891		/**
1892		 *  ALL OUT EPs are shared the same chunk onchip memory, so
1893		 * driver checks if it already has assigned enough buffers
1894		 */
1895		if (priv_dev->out_mem_is_allocated >= size)
1896			return 0;
1897
1898		required = size - priv_dev->out_mem_is_allocated;
1899
1900		if (required > remained)
1901			return -EPERM;
1902
1903		priv_dev->out_mem_is_allocated += required;
1904		priv_dev->onchip_used_size += required;
1905	}
1906
1907	return 0;
1908}
1909
1910static void cdns3_stream_ep_reconfig(struct cdns3_device *priv_dev,
1911				     struct cdns3_endpoint *priv_ep)
1912{
1913	if (!priv_ep->use_streams || priv_dev->gadget.speed < USB_SPEED_SUPER)
1914		return;
1915
1916	if (priv_dev->dev_ver >= DEV_VER_V3) {
1917		u32 mask = BIT(priv_ep->num + (priv_ep->dir ? 16 : 0));
1918
1919		/*
1920		 * Stream capable endpoints are handled by using ep_tdl
1921		 * register. Other endpoints use TDL from TRB feature.
1922		 */
1923		cdns3_clear_register_bit(&priv_dev->regs->tdl_from_trb, mask);
1924	}
1925
1926	/*  Enable Stream Bit TDL chk and SID chk */
1927	cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_STREAM_EN |
1928			       EP_CFG_TDL_CHK | EP_CFG_SID_CHK);
1929}
1930
1931static void cdns3_configure_dmult(struct cdns3_device *priv_dev,
1932				  struct cdns3_endpoint *priv_ep)
1933{
1934	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
1935
1936	/* For dev_ver > DEV_VER_V2 DMULT is configured per endpoint */
1937	if (priv_dev->dev_ver <= DEV_VER_V2)
1938		writel(USB_CONF_DMULT, &regs->usb_conf);
1939
1940	if (priv_dev->dev_ver == DEV_VER_V2)
1941		writel(USB_CONF2_EN_TDL_TRB, &regs->usb_conf2);
1942
1943	if (priv_dev->dev_ver >= DEV_VER_V3 && priv_ep) {
1944		u32 mask;
1945
1946		if (priv_ep->dir)
1947			mask = BIT(priv_ep->num + 16);
1948		else
1949			mask = BIT(priv_ep->num);
1950
1951		if (priv_ep->type != USB_ENDPOINT_XFER_ISOC) {
1952			cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1953			cdns3_set_register_bit(&regs->tdl_beh, mask);
1954			cdns3_set_register_bit(&regs->tdl_beh2, mask);
1955			cdns3_set_register_bit(&regs->dma_adv_td, mask);
1956		}
1957
1958		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC && !priv_ep->dir)
1959			cdns3_set_register_bit(&regs->tdl_from_trb, mask);
1960
1961		cdns3_set_register_bit(&regs->dtrans, mask);
1962	}
1963}
1964
1965/**
1966 * cdns3_ep_config Configure hardware endpoint
1967 * @priv_ep: extended endpoint object
1968 */
1969void cdns3_ep_config(struct cdns3_endpoint *priv_ep)
1970{
1971	bool is_iso_ep = (priv_ep->type == USB_ENDPOINT_XFER_ISOC);
1972	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
1973	u32 bEndpointAddress = priv_ep->num | priv_ep->dir;
1974	u32 max_packet_size = 0;
1975	u8 maxburst = 0;
1976	u32 ep_cfg = 0;
1977	u8 buffering;
1978	u8 mult = 0;
1979	int ret;
1980
1981	buffering = CDNS3_EP_BUF_SIZE - 1;
1982
1983	cdns3_configure_dmult(priv_dev, priv_ep);
1984
1985	switch (priv_ep->type) {
1986	case USB_ENDPOINT_XFER_INT:
1987		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_INT);
1988
1989		if ((priv_dev->dev_ver == DEV_VER_V2 && !priv_ep->dir) ||
1990		    priv_dev->dev_ver > DEV_VER_V2)
1991			ep_cfg |= EP_CFG_TDL_CHK;
1992		break;
1993	case USB_ENDPOINT_XFER_BULK:
1994		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_BULK);
1995
1996		if ((priv_dev->dev_ver == DEV_VER_V2  && !priv_ep->dir) ||
1997		    priv_dev->dev_ver > DEV_VER_V2)
1998			ep_cfg |= EP_CFG_TDL_CHK;
1999		break;
2000	default:
2001		ep_cfg = EP_CFG_EPTYPE(USB_ENDPOINT_XFER_ISOC);
2002		mult = CDNS3_EP_ISO_HS_MULT - 1;
2003		buffering = mult + 1;
2004	}
2005
2006	switch (priv_dev->gadget.speed) {
2007	case USB_SPEED_FULL:
2008		max_packet_size = is_iso_ep ? 1023 : 64;
2009		break;
2010	case USB_SPEED_HIGH:
2011		max_packet_size = is_iso_ep ? 1024 : 512;
2012		break;
2013	case USB_SPEED_SUPER:
2014		/* It's limitation that driver assumes in driver. */
2015		mult = 0;
2016		max_packet_size = 1024;
2017		if (priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2018			maxburst = CDNS3_EP_ISO_SS_BURST - 1;
2019			buffering = (mult + 1) *
2020				    (maxburst + 1);
2021
2022			if (priv_ep->interval > 1)
2023				buffering++;
2024		} else {
2025			maxburst = CDNS3_EP_BUF_SIZE - 1;
2026		}
2027		break;
2028	default:
2029		/* all other speed are not supported */
2030		return;
2031	}
2032
2033	if (max_packet_size == 1024)
2034		priv_ep->trb_burst_size = 128;
2035	else if (max_packet_size >= 512)
2036		priv_ep->trb_burst_size = 64;
2037	else
2038		priv_ep->trb_burst_size = 16;
2039
2040	ret = cdns3_ep_onchip_buffer_reserve(priv_dev, buffering + 1,
2041					     !!priv_ep->dir);
2042	if (ret) {
2043		dev_err(priv_dev->dev, "onchip mem is full, ep is invalid\n");
2044		return;
2045	}
2046
2047	ep_cfg |= EP_CFG_MAXPKTSIZE(max_packet_size) |
2048		  EP_CFG_MULT(mult) |
2049		  EP_CFG_BUFFERING(buffering) |
2050		  EP_CFG_MAXBURST(maxburst);
2051
2052	cdns3_select_ep(priv_dev, bEndpointAddress);
2053	writel(ep_cfg, &priv_dev->regs->ep_cfg);
2054
2055	dev_dbg(priv_dev->dev, "Configure %s: with val %08x\n",
2056		priv_ep->name, ep_cfg);
2057}
2058
2059/* Find correct direction for HW endpoint according to description */
2060static int cdns3_ep_dir_is_correct(struct usb_endpoint_descriptor *desc,
2061				   struct cdns3_endpoint *priv_ep)
2062{
2063	return (priv_ep->endpoint.caps.dir_in && usb_endpoint_dir_in(desc)) ||
2064	       (priv_ep->endpoint.caps.dir_out && usb_endpoint_dir_out(desc));
2065}
2066
2067static struct
2068cdns3_endpoint *cdns3_find_available_ep(struct cdns3_device *priv_dev,
2069					struct usb_endpoint_descriptor *desc)
2070{
2071	struct usb_ep *ep;
2072	struct cdns3_endpoint *priv_ep;
2073
2074	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2075		unsigned long num;
2076		int ret;
2077		/* ep name pattern likes epXin or epXout */
2078		char c[2] = {ep->name[2], '\0'};
2079
2080		ret = kstrtoul(c, 10, &num);
2081		if (ret)
2082			return ERR_PTR(ret);
2083
2084		priv_ep = ep_to_cdns3_ep(ep);
2085		if (cdns3_ep_dir_is_correct(desc, priv_ep)) {
2086			if (!(priv_ep->flags & EP_CLAIMED)) {
2087				priv_ep->num  = num;
2088				return priv_ep;
2089			}
2090		}
2091	}
2092
2093	return ERR_PTR(-ENOENT);
2094}
2095
2096/*
2097 *  Cadence IP has one limitation that all endpoints must be configured
2098 * (Type & MaxPacketSize) before setting configuration through hardware
2099 * register, it means we can't change endpoints configuration after
2100 * set_configuration.
2101 *
2102 * This function set EP_CLAIMED flag which is added when the gadget driver
2103 * uses usb_ep_autoconfig to configure specific endpoint;
2104 * When the udc driver receives set_configurion request,
2105 * it goes through all claimed endpoints, and configure all endpoints
2106 * accordingly.
2107 *
2108 * At usb_ep_ops.enable/disable, we only enable and disable endpoint through
2109 * ep_cfg register which can be changed after set_configuration, and do
2110 * some software operation accordingly.
2111 */
2112static struct
2113usb_ep *cdns3_gadget_match_ep(struct usb_gadget *gadget,
2114			      struct usb_endpoint_descriptor *desc,
2115			      struct usb_ss_ep_comp_descriptor *comp_desc)
2116{
2117	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2118	struct cdns3_endpoint *priv_ep;
2119	unsigned long flags;
2120
2121	priv_ep = cdns3_find_available_ep(priv_dev, desc);
2122	if (IS_ERR(priv_ep)) {
2123		dev_err(priv_dev->dev, "no available ep\n");
2124		return NULL;
2125	}
2126
2127	dev_dbg(priv_dev->dev, "match endpoint: %s\n", priv_ep->name);
2128
2129	spin_lock_irqsave(&priv_dev->lock, flags);
2130	priv_ep->endpoint.desc = desc;
2131	priv_ep->dir  = usb_endpoint_dir_in(desc) ? USB_DIR_IN : USB_DIR_OUT;
2132	priv_ep->type = usb_endpoint_type(desc);
2133	priv_ep->flags |= EP_CLAIMED;
2134	priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2135
2136	spin_unlock_irqrestore(&priv_dev->lock, flags);
2137	return &priv_ep->endpoint;
2138}
2139
2140/**
2141 * cdns3_gadget_ep_alloc_request Allocates request
2142 * @ep: endpoint object associated with request
2143 * @gfp_flags: gfp flags
2144 *
2145 * Returns allocated request address, NULL on allocation error
2146 */
2147struct usb_request *cdns3_gadget_ep_alloc_request(struct usb_ep *ep,
2148						  gfp_t gfp_flags)
2149{
2150	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2151	struct cdns3_request *priv_req;
2152
2153	priv_req = kzalloc(sizeof(*priv_req), gfp_flags);
2154	if (!priv_req)
2155		return NULL;
2156
2157	priv_req->priv_ep = priv_ep;
2158
2159	trace_cdns3_alloc_request(priv_req);
2160	return &priv_req->request;
2161}
2162
2163/**
2164 * cdns3_gadget_ep_free_request Free memory occupied by request
2165 * @ep: endpoint object associated with request
2166 * @request: request to free memory
2167 */
2168void cdns3_gadget_ep_free_request(struct usb_ep *ep,
2169				  struct usb_request *request)
2170{
2171	struct cdns3_request *priv_req = to_cdns3_request(request);
2172
2173	if (priv_req->aligned_buf)
2174		priv_req->aligned_buf->in_use = 0;
2175
2176	trace_cdns3_free_request(priv_req);
2177	kfree(priv_req);
2178}
2179
2180/**
2181 * cdns3_gadget_ep_enable Enable endpoint
2182 * @ep: endpoint object
2183 * @desc: endpoint descriptor
2184 *
2185 * Returns 0 on success, error code elsewhere
2186 */
2187static int cdns3_gadget_ep_enable(struct usb_ep *ep,
2188				  const struct usb_endpoint_descriptor *desc)
2189{
2190	struct cdns3_endpoint *priv_ep;
2191	struct cdns3_device *priv_dev;
2192	const struct usb_ss_ep_comp_descriptor *comp_desc;
2193	u32 reg = EP_STS_EN_TRBERREN;
2194	u32 bEndpointAddress;
2195	unsigned long flags;
2196	int enable = 1;
2197	int ret;
2198	int val;
2199
2200	priv_ep = ep_to_cdns3_ep(ep);
2201	priv_dev = priv_ep->cdns3_dev;
2202	comp_desc = priv_ep->endpoint.comp_desc;
2203
2204	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
2205		dev_dbg(priv_dev->dev, "usbss: invalid parameters\n");
2206		return -EINVAL;
2207	}
2208
2209	if (!desc->wMaxPacketSize) {
2210		dev_err(priv_dev->dev, "usbss: missing wMaxPacketSize\n");
2211		return -EINVAL;
2212	}
2213
2214	if (dev_WARN_ONCE(priv_dev->dev, priv_ep->flags & EP_ENABLED,
2215			  "%s is already enabled\n", priv_ep->name))
2216		return 0;
2217
2218	spin_lock_irqsave(&priv_dev->lock, flags);
2219
2220	priv_ep->endpoint.desc = desc;
2221	priv_ep->type = usb_endpoint_type(desc);
2222	priv_ep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
2223
2224	if (priv_ep->interval > ISO_MAX_INTERVAL &&
2225	    priv_ep->type == USB_ENDPOINT_XFER_ISOC) {
2226		dev_err(priv_dev->dev, "Driver is limited to %d period\n",
2227			ISO_MAX_INTERVAL);
2228
2229		ret =  -EINVAL;
2230		goto exit;
2231	}
2232
2233	bEndpointAddress = priv_ep->num | priv_ep->dir;
2234	cdns3_select_ep(priv_dev, bEndpointAddress);
2235
2236	if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
2237		/*
2238		 * Enable stream support (SS mode) related interrupts
2239		 * in EP_STS_EN Register
2240		 */
2241		if (priv_dev->gadget.speed >= USB_SPEED_SUPER) {
2242			reg |= EP_STS_EN_IOTEN | EP_STS_EN_PRIMEEEN |
2243				EP_STS_EN_SIDERREN | EP_STS_EN_MD_EXITEN |
2244				EP_STS_EN_STREAMREN;
2245			priv_ep->use_streams = true;
2246			cdns3_stream_ep_reconfig(priv_dev, priv_ep);
2247			priv_dev->using_streams |= true;
2248		}
2249	}
2250
2251	ret = cdns3_allocate_trb_pool(priv_ep);
2252
2253	if (ret)
2254		goto exit;
2255
2256	bEndpointAddress = priv_ep->num | priv_ep->dir;
2257	cdns3_select_ep(priv_dev, bEndpointAddress);
2258
2259	trace_cdns3_gadget_ep_enable(priv_ep);
2260
2261	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2262
2263	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2264					!(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2265					1, 1000);
2266
2267	if (unlikely(ret)) {
2268		cdns3_free_trb_pool(priv_ep);
2269		ret =  -EINVAL;
2270		goto exit;
2271	}
2272
2273	/* enable interrupt for selected endpoint */
2274	cdns3_set_register_bit(&priv_dev->regs->ep_ien,
2275			       BIT(cdns3_ep_addr_to_index(bEndpointAddress)));
2276
2277	if (priv_dev->dev_ver < DEV_VER_V2)
2278		cdns3_wa2_enable_detection(priv_dev, priv_ep, reg);
2279
2280	writel(reg, &priv_dev->regs->ep_sts_en);
2281
2282	/*
2283	 * For some versions of controller at some point during ISO OUT traffic
2284	 * DMA reads Transfer Ring for the EP which has never got doorbell.
2285	 * This issue was detected only on simulation, but to avoid this issue
2286	 * driver add protection against it. To fix it driver enable ISO OUT
2287	 * endpoint before setting DRBL. This special treatment of ISO OUT
2288	 * endpoints are recommended by controller specification.
2289	 */
2290	if (priv_ep->type == USB_ENDPOINT_XFER_ISOC  && !priv_ep->dir)
2291		enable = 0;
2292
2293	if (enable)
2294		cdns3_set_register_bit(&priv_dev->regs->ep_cfg, EP_CFG_ENABLE);
2295
2296	ep->desc = desc;
2297	priv_ep->flags &= ~(EP_PENDING_REQUEST | EP_STALLED | EP_STALL_PENDING |
2298			    EP_QUIRK_ISO_OUT_EN | EP_QUIRK_EXTRA_BUF_EN);
2299	priv_ep->flags |= EP_ENABLED | EP_UPDATE_EP_TRBADDR;
2300	priv_ep->wa1_set = 0;
2301	priv_ep->enqueue = 0;
2302	priv_ep->dequeue = 0;
2303	reg = readl(&priv_dev->regs->ep_sts);
2304	priv_ep->pcs = !!EP_STS_CCS(reg);
2305	priv_ep->ccs = !!EP_STS_CCS(reg);
2306	/* one TRB is reserved for link TRB used in DMULT mode*/
2307	priv_ep->free_trbs = priv_ep->num_trbs - 1;
2308exit:
2309	spin_unlock_irqrestore(&priv_dev->lock, flags);
2310
2311	return ret;
2312}
2313
2314/**
2315 * cdns3_gadget_ep_disable Disable endpoint
2316 * @ep: endpoint object
2317 *
2318 * Returns 0 on success, error code elsewhere
2319 */
2320static int cdns3_gadget_ep_disable(struct usb_ep *ep)
2321{
2322	struct cdns3_endpoint *priv_ep;
2323	struct cdns3_request *priv_req;
2324	struct cdns3_device *priv_dev;
2325	struct usb_request *request;
2326	unsigned long flags;
2327	int ret = 0;
2328	u32 ep_cfg;
2329	int val;
2330
2331	if (!ep) {
2332		pr_err("usbss: invalid parameters\n");
2333		return -EINVAL;
2334	}
2335
2336	priv_ep = ep_to_cdns3_ep(ep);
2337	priv_dev = priv_ep->cdns3_dev;
2338
2339	if (dev_WARN_ONCE(priv_dev->dev, !(priv_ep->flags & EP_ENABLED),
2340			  "%s is already disabled\n", priv_ep->name))
2341		return 0;
2342
2343	spin_lock_irqsave(&priv_dev->lock, flags);
2344
2345	trace_cdns3_gadget_ep_disable(priv_ep);
2346
2347	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2348
2349	ep_cfg = readl(&priv_dev->regs->ep_cfg);
2350	ep_cfg &= ~EP_CFG_ENABLE;
2351	writel(ep_cfg, &priv_dev->regs->ep_cfg);
2352
2353	/**
2354	 * Driver needs some time before resetting endpoint.
2355	 * It need waits for clearing DBUSY bit or for timeout expired.
2356	 * 10us is enough time for controller to stop transfer.
2357	 */
2358	readl_poll_timeout_atomic(&priv_dev->regs->ep_sts, val,
2359				  !(val & EP_STS_DBUSY), 1, 10);
2360	writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2361
2362	readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2363				  !(val & (EP_CMD_CSTALL | EP_CMD_EPRST)),
2364				  1, 1000);
2365	if (unlikely(ret))
2366		dev_err(priv_dev->dev, "Timeout: %s resetting failed.\n",
2367			priv_ep->name);
2368
2369	while (!list_empty(&priv_ep->pending_req_list)) {
2370		request = cdns3_next_request(&priv_ep->pending_req_list);
2371
2372		cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2373				      -ESHUTDOWN);
2374	}
2375
2376	while (!list_empty(&priv_ep->wa2_descmiss_req_list)) {
2377		priv_req = cdns3_next_priv_request(&priv_ep->wa2_descmiss_req_list);
2378
2379		kfree(priv_req->request.buf);
2380		cdns3_gadget_ep_free_request(&priv_ep->endpoint,
2381					     &priv_req->request);
2382		list_del_init(&priv_req->list);
2383		--priv_ep->wa2_counter;
2384	}
2385
2386	while (!list_empty(&priv_ep->deferred_req_list)) {
2387		request = cdns3_next_request(&priv_ep->deferred_req_list);
2388
2389		cdns3_gadget_giveback(priv_ep, to_cdns3_request(request),
2390				      -ESHUTDOWN);
2391	}
2392
2393	priv_ep->descmis_req = NULL;
2394
2395	ep->desc = NULL;
2396	priv_ep->flags &= ~EP_ENABLED;
2397	priv_ep->use_streams = false;
2398
2399	spin_unlock_irqrestore(&priv_dev->lock, flags);
2400
2401	return ret;
2402}
2403
2404/**
2405 * cdns3_gadget_ep_queue Transfer data on endpoint
2406 * @ep: endpoint object
2407 * @request: request object
2408 * @gfp_flags: gfp flags
2409 *
2410 * Returns 0 on success, error code elsewhere
2411 */
2412static int __cdns3_gadget_ep_queue(struct usb_ep *ep,
2413				   struct usb_request *request,
2414				   gfp_t gfp_flags)
2415{
2416	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2417	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2418	struct cdns3_request *priv_req;
2419	int ret = 0;
2420
2421	request->actual = 0;
2422	request->status = -EINPROGRESS;
2423	priv_req = to_cdns3_request(request);
2424	trace_cdns3_ep_queue(priv_req);
2425
2426	if (priv_dev->dev_ver < DEV_VER_V2) {
2427		ret = cdns3_wa2_gadget_ep_queue(priv_dev, priv_ep,
2428						priv_req);
2429
2430		if (ret == EINPROGRESS)
2431			return 0;
2432	}
2433
2434	ret = cdns3_prepare_aligned_request_buf(priv_req);
2435	if (ret < 0)
2436		return ret;
2437
2438	ret = usb_gadget_map_request_by_dev(priv_dev->sysdev, request,
2439					    usb_endpoint_dir_in(ep->desc));
2440	if (ret)
2441		return ret;
2442
2443	list_add_tail(&request->list, &priv_ep->deferred_req_list);
2444
2445	/*
2446	 * For stream capable endpoint if prime irq flag is set then only start
2447	 * request.
2448	 * If hardware endpoint configuration has not been set yet then
2449	 * just queue request in deferred list. Transfer will be started in
2450	 * cdns3_set_hw_configuration.
2451	 */
2452	if (!request->stream_id) {
2453		if (priv_dev->hw_configured_flag &&
2454		    !(priv_ep->flags & EP_STALLED) &&
2455		    !(priv_ep->flags & EP_STALL_PENDING))
2456			cdns3_start_all_request(priv_dev, priv_ep);
2457	} else {
2458		if (priv_dev->hw_configured_flag && priv_ep->prime_flag)
2459			cdns3_start_all_request(priv_dev, priv_ep);
2460	}
2461
2462	return 0;
2463}
2464
2465static int cdns3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
2466				 gfp_t gfp_flags)
2467{
2468	struct usb_request *zlp_request;
2469	struct cdns3_endpoint *priv_ep;
2470	struct cdns3_device *priv_dev;
2471	unsigned long flags;
2472	int ret;
2473
2474	if (!request || !ep)
2475		return -EINVAL;
2476
2477	priv_ep = ep_to_cdns3_ep(ep);
2478	priv_dev = priv_ep->cdns3_dev;
2479
2480	spin_lock_irqsave(&priv_dev->lock, flags);
2481
2482	ret = __cdns3_gadget_ep_queue(ep, request, gfp_flags);
2483
2484	if (ret == 0 && request->zero && request->length &&
2485	    (request->length % ep->maxpacket == 0)) {
2486		struct cdns3_request *priv_req;
2487
2488		zlp_request = cdns3_gadget_ep_alloc_request(ep, GFP_ATOMIC);
2489		zlp_request->buf = priv_dev->zlp_buf;
2490		zlp_request->length = 0;
2491
2492		priv_req = to_cdns3_request(zlp_request);
2493		priv_req->flags |= REQUEST_ZLP;
2494
2495		dev_dbg(priv_dev->dev, "Queuing ZLP for endpoint: %s\n",
2496			priv_ep->name);
2497		ret = __cdns3_gadget_ep_queue(ep, zlp_request, gfp_flags);
2498	}
2499
2500	spin_unlock_irqrestore(&priv_dev->lock, flags);
2501	return ret;
2502}
2503
2504/**
2505 * cdns3_gadget_ep_dequeue Remove request from transfer queue
2506 * @ep: endpoint object associated with request
2507 * @request: request object
2508 *
2509 * Returns 0 on success, error code elsewhere
2510 */
2511int cdns3_gadget_ep_dequeue(struct usb_ep *ep,
2512			    struct usb_request *request)
2513{
2514	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2515	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2516	struct usb_request *req, *req_temp;
2517	struct cdns3_request *priv_req;
2518	struct cdns3_trb *link_trb;
2519	u8 req_on_hw_ring = 0;
2520	unsigned long flags;
2521	int ret = 0;
2522
2523	if (!ep || !request || !ep->desc)
2524		return -EINVAL;
2525
2526	spin_lock_irqsave(&priv_dev->lock, flags);
2527
2528	priv_req = to_cdns3_request(request);
2529
2530	trace_cdns3_ep_dequeue(priv_req);
2531
2532	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2533
2534	list_for_each_entry_safe(req, req_temp, &priv_ep->pending_req_list,
2535				 list) {
2536		if (request == req) {
2537			req_on_hw_ring = 1;
2538			goto found;
2539		}
2540	}
2541
2542	list_for_each_entry_safe(req, req_temp, &priv_ep->deferred_req_list,
2543				 list) {
2544		if (request == req)
2545			goto found;
2546	}
2547
2548	goto not_found;
2549
2550found:
2551	link_trb = priv_req->trb;
2552
2553	/* Update ring only if removed request is on pending_req_list list */
2554	if (req_on_hw_ring && link_trb) {
2555		link_trb->buffer = TRB_BUFFER(priv_ep->trb_pool_dma +
2556			((priv_req->end_trb + 1) * TRB_SIZE));
2557		link_trb->control = (link_trb->control & TRB_CYCLE) |
2558				    TRB_TYPE(TRB_LINK) | TRB_CHAIN;
2559
2560		if (priv_ep->wa1_trb == priv_req->trb)
2561			cdns3_wa1_restore_cycle_bit(priv_ep);
2562	}
2563
2564	cdns3_gadget_giveback(priv_ep, priv_req, -ECONNRESET);
2565
2566not_found:
2567	spin_unlock_irqrestore(&priv_dev->lock, flags);
2568	return ret;
2569}
2570
2571/**
2572 * __cdns3_gadget_ep_set_halt Sets stall on selected endpoint
2573 * Should be called after acquiring spin_lock and selecting ep
2574 * @priv_ep: endpoint object to set stall on.
2575 */
2576void __cdns3_gadget_ep_set_halt(struct cdns3_endpoint *priv_ep)
2577{
2578	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2579
2580	trace_cdns3_halt(priv_ep, 1, 0);
2581
2582	if (!(priv_ep->flags & EP_STALLED)) {
2583		u32 ep_sts_reg = readl(&priv_dev->regs->ep_sts);
2584
2585		if (!(ep_sts_reg & EP_STS_DBUSY))
2586			cdns3_ep_stall_flush(priv_ep);
2587		else
2588			priv_ep->flags |= EP_STALL_PENDING;
2589	}
2590}
2591
2592/**
2593 * __cdns3_gadget_ep_clear_halt Clears stall on selected endpoint
2594 * Should be called after acquiring spin_lock and selecting ep
2595 * @priv_ep: endpoint object to clear stall on
2596 */
2597int __cdns3_gadget_ep_clear_halt(struct cdns3_endpoint *priv_ep)
2598{
2599	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2600	struct usb_request *request;
2601	struct cdns3_request *priv_req;
2602	struct cdns3_trb *trb = NULL;
2603	int ret;
2604	int val;
2605
2606	trace_cdns3_halt(priv_ep, 0, 0);
2607
2608	request = cdns3_next_request(&priv_ep->pending_req_list);
2609	if (request) {
2610		priv_req = to_cdns3_request(request);
2611		trb = priv_req->trb;
2612		if (trb)
2613			trb->control = trb->control ^ TRB_CYCLE;
2614	}
2615
2616	writel(EP_CMD_CSTALL | EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2617
2618	/* wait for EPRST cleared */
2619	ret = readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2620					!(val & EP_CMD_EPRST), 1, 100);
2621	if (ret)
2622		return -EINVAL;
2623
2624	priv_ep->flags &= ~(EP_STALLED | EP_STALL_PENDING);
2625
2626	if (request) {
2627		if (trb)
2628			trb->control = trb->control ^ TRB_CYCLE;
2629		cdns3_rearm_transfer(priv_ep, 1);
2630	}
2631
2632	cdns3_start_all_request(priv_dev, priv_ep);
2633	return ret;
2634}
2635
2636/**
2637 * cdns3_gadget_ep_set_halt Sets/clears stall on selected endpoint
2638 * @ep: endpoint object to set/clear stall on
2639 * @value: 1 for set stall, 0 for clear stall
2640 *
2641 * Returns 0 on success, error code elsewhere
2642 */
2643int cdns3_gadget_ep_set_halt(struct usb_ep *ep, int value)
2644{
2645	struct cdns3_endpoint *priv_ep = ep_to_cdns3_ep(ep);
2646	struct cdns3_device *priv_dev = priv_ep->cdns3_dev;
2647	unsigned long flags;
2648	int ret = 0;
2649
2650	if (!(priv_ep->flags & EP_ENABLED))
2651		return -EPERM;
2652
2653	spin_lock_irqsave(&priv_dev->lock, flags);
2654
2655	cdns3_select_ep(priv_dev, ep->desc->bEndpointAddress);
2656
2657	if (!value) {
2658		priv_ep->flags &= ~EP_WEDGE;
2659		ret = __cdns3_gadget_ep_clear_halt(priv_ep);
2660	} else {
2661		__cdns3_gadget_ep_set_halt(priv_ep);
2662	}
2663
2664	spin_unlock_irqrestore(&priv_dev->lock, flags);
2665
2666	return ret;
2667}
2668
2669extern const struct usb_ep_ops cdns3_gadget_ep0_ops;
2670
2671static const struct usb_ep_ops cdns3_gadget_ep_ops = {
2672	.enable = cdns3_gadget_ep_enable,
2673	.disable = cdns3_gadget_ep_disable,
2674	.alloc_request = cdns3_gadget_ep_alloc_request,
2675	.free_request = cdns3_gadget_ep_free_request,
2676	.queue = cdns3_gadget_ep_queue,
2677	.dequeue = cdns3_gadget_ep_dequeue,
2678	.set_halt = cdns3_gadget_ep_set_halt,
2679	.set_wedge = cdns3_gadget_ep_set_wedge,
2680};
2681
2682/**
2683 * cdns3_gadget_get_frame Returns number of actual ITP frame
2684 * @gadget: gadget object
2685 *
2686 * Returns number of actual ITP frame
2687 */
2688static int cdns3_gadget_get_frame(struct usb_gadget *gadget)
2689{
2690	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2691
2692	return readl(&priv_dev->regs->usb_itpn);
2693}
2694
2695int __cdns3_gadget_wakeup(struct cdns3_device *priv_dev)
2696{
2697	enum usb_device_speed speed;
2698
2699	speed = cdns3_get_speed(priv_dev);
2700
2701	if (speed >= USB_SPEED_SUPER)
2702		return 0;
2703
2704	/* Start driving resume signaling to indicate remote wakeup. */
2705	writel(USB_CONF_LGO_L0, &priv_dev->regs->usb_conf);
2706
2707	return 0;
2708}
2709
2710static int cdns3_gadget_wakeup(struct usb_gadget *gadget)
2711{
2712	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2713	unsigned long flags;
2714	int ret = 0;
2715
2716	spin_lock_irqsave(&priv_dev->lock, flags);
2717	ret = __cdns3_gadget_wakeup(priv_dev);
2718	spin_unlock_irqrestore(&priv_dev->lock, flags);
2719	return ret;
2720}
2721
2722static int cdns3_gadget_set_selfpowered(struct usb_gadget *gadget,
2723					int is_selfpowered)
2724{
2725	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2726	unsigned long flags;
2727
2728	spin_lock_irqsave(&priv_dev->lock, flags);
2729	priv_dev->is_selfpowered = !!is_selfpowered;
2730	spin_unlock_irqrestore(&priv_dev->lock, flags);
2731	return 0;
2732}
2733
2734static int cdns3_gadget_pullup(struct usb_gadget *gadget, int is_on)
2735{
2736	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2737
2738	if (is_on)
2739		writel(USB_CONF_DEVEN, &priv_dev->regs->usb_conf);
2740	else
2741		writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2742
2743	return 0;
2744}
2745
2746static void cdns3_gadget_config(struct cdns3_device *priv_dev)
2747{
2748	struct cdns3_usb_regs __iomem *regs = priv_dev->regs;
2749	u32 reg;
2750
2751	cdns3_ep0_config(priv_dev);
2752
2753	/* enable interrupts for endpoint 0 (in and out) */
2754	writel(EP_IEN_EP_OUT0 | EP_IEN_EP_IN0, &regs->ep_ien);
2755
2756	/*
2757	 * Driver needs to modify LFPS minimal U1 Exit time for DEV_VER_TI_V1
2758	 * revision of controller.
2759	 */
2760	if (priv_dev->dev_ver == DEV_VER_TI_V1) {
2761		reg = readl(&regs->dbg_link1);
2762
2763		reg &= ~DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_MASK;
2764		reg |= DBG_LINK1_LFPS_MIN_GEN_U1_EXIT(0x55) |
2765		       DBG_LINK1_LFPS_MIN_GEN_U1_EXIT_SET;
2766		writel(reg, &regs->dbg_link1);
2767	}
2768
2769	/*
2770	 * By default some platforms has set protected access to memory.
2771	 * This cause problem with cache, so driver restore non-secure
2772	 * access to memory.
2773	 */
2774	reg = readl(&regs->dma_axi_ctrl);
2775	reg |= DMA_AXI_CTRL_MARPROT(DMA_AXI_CTRL_NON_SECURE) |
2776	       DMA_AXI_CTRL_MAWPROT(DMA_AXI_CTRL_NON_SECURE);
2777	writel(reg, &regs->dma_axi_ctrl);
2778
2779	/* enable generic interrupt*/
2780	writel(USB_IEN_INIT, &regs->usb_ien);
2781	writel(USB_CONF_CLK2OFFDS | USB_CONF_L1DS, &regs->usb_conf);
2782
2783	cdns3_configure_dmult(priv_dev, NULL);
2784}
2785
2786/**
2787 * cdns3_gadget_udc_start Gadget start
2788 * @gadget: gadget object
2789 * @driver: driver which operates on this gadget
2790 *
2791 * Returns 0 on success, error code elsewhere
2792 */
2793static int cdns3_gadget_udc_start(struct usb_gadget *gadget,
2794				  struct usb_gadget_driver *driver)
2795{
2796	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2797	unsigned long flags;
2798	enum usb_device_speed max_speed = driver->max_speed;
2799
2800	spin_lock_irqsave(&priv_dev->lock, flags);
2801	priv_dev->gadget_driver = driver;
2802
2803	/* limit speed if necessary */
2804	max_speed = min(driver->max_speed, gadget->max_speed);
2805
2806	switch (max_speed) {
2807	case USB_SPEED_FULL:
2808		writel(USB_CONF_SFORCE_FS, &priv_dev->regs->usb_conf);
2809		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2810		break;
2811	case USB_SPEED_HIGH:
2812		writel(USB_CONF_USB3DIS, &priv_dev->regs->usb_conf);
2813		break;
2814	case USB_SPEED_SUPER:
2815		break;
2816	default:
2817		dev_err(priv_dev->dev,
2818			"invalid maximum_speed parameter %d\n",
2819			max_speed);
2820		fallthrough;
2821	case USB_SPEED_UNKNOWN:
2822		/* default to superspeed */
2823		max_speed = USB_SPEED_SUPER;
2824		break;
2825	}
2826
2827	cdns3_gadget_config(priv_dev);
2828	spin_unlock_irqrestore(&priv_dev->lock, flags);
2829	return 0;
2830}
2831
2832/**
2833 * cdns3_gadget_udc_stop Stops gadget
2834 * @gadget: gadget object
2835 *
2836 * Returns 0
2837 */
2838static int cdns3_gadget_udc_stop(struct usb_gadget *gadget)
2839{
2840	struct cdns3_device *priv_dev = gadget_to_cdns3_device(gadget);
2841	struct cdns3_endpoint *priv_ep;
2842	u32 bEndpointAddress;
2843	struct usb_ep *ep;
 
2844	int val;
2845
2846	priv_dev->gadget_driver = NULL;
2847
2848	priv_dev->onchip_used_size = 0;
2849	priv_dev->out_mem_is_allocated = 0;
2850	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
2851
2852	list_for_each_entry(ep, &priv_dev->gadget.ep_list, ep_list) {
2853		priv_ep = ep_to_cdns3_ep(ep);
2854		bEndpointAddress = priv_ep->num | priv_ep->dir;
2855		cdns3_select_ep(priv_dev, bEndpointAddress);
2856		writel(EP_CMD_EPRST, &priv_dev->regs->ep_cmd);
2857		readl_poll_timeout_atomic(&priv_dev->regs->ep_cmd, val,
2858					  !(val & EP_CMD_EPRST), 1, 100);
2859
2860		priv_ep->flags &= ~EP_CLAIMED;
2861	}
2862
2863	/* disable interrupt for device */
2864	writel(0, &priv_dev->regs->usb_ien);
2865	writel(USB_CONF_DEVDS, &priv_dev->regs->usb_conf);
2866
2867	return 0;
2868}
2869
2870static const struct usb_gadget_ops cdns3_gadget_ops = {
2871	.get_frame = cdns3_gadget_get_frame,
2872	.wakeup = cdns3_gadget_wakeup,
2873	.set_selfpowered = cdns3_gadget_set_selfpowered,
2874	.pullup = cdns3_gadget_pullup,
2875	.udc_start = cdns3_gadget_udc_start,
2876	.udc_stop = cdns3_gadget_udc_stop,
2877	.match_ep = cdns3_gadget_match_ep,
2878};
2879
2880static void cdns3_free_all_eps(struct cdns3_device *priv_dev)
2881{
2882	int i;
2883
2884	/* ep0 OUT point to ep0 IN. */
2885	priv_dev->eps[16] = NULL;
2886
2887	for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++)
2888		if (priv_dev->eps[i]) {
2889			cdns3_free_trb_pool(priv_dev->eps[i]);
2890			devm_kfree(priv_dev->dev, priv_dev->eps[i]);
2891		}
2892}
2893
2894/**
2895 * cdns3_init_eps Initializes software endpoints of gadget
2896 * @priv_dev: extended gadget object
2897 *
2898 * Returns 0 on success, error code elsewhere
2899 */
2900static int cdns3_init_eps(struct cdns3_device *priv_dev)
2901{
2902	u32 ep_enabled_reg, iso_ep_reg;
2903	struct cdns3_endpoint *priv_ep;
2904	int ep_dir, ep_number;
2905	u32 ep_mask;
2906	int ret = 0;
2907	int i;
2908
2909	/* Read it from USB_CAP3 to USB_CAP5 */
2910	ep_enabled_reg = readl(&priv_dev->regs->usb_cap3);
2911	iso_ep_reg = readl(&priv_dev->regs->usb_cap4);
2912
2913	dev_dbg(priv_dev->dev, "Initializing non-zero endpoints\n");
2914
2915	for (i = 0; i < CDNS3_ENDPOINTS_MAX_COUNT; i++) {
2916		ep_dir = i >> 4;	/* i div 16 */
2917		ep_number = i & 0xF;	/* i % 16 */
2918		ep_mask = BIT(i);
2919
2920		if (!(ep_enabled_reg & ep_mask))
2921			continue;
2922
2923		if (ep_dir && !ep_number) {
2924			priv_dev->eps[i] = priv_dev->eps[0];
2925			continue;
2926		}
2927
2928		priv_ep = devm_kzalloc(priv_dev->dev, sizeof(*priv_ep),
2929				       GFP_KERNEL);
2930		if (!priv_ep)
2931			goto err;
2932
2933		/* set parent of endpoint object */
2934		priv_ep->cdns3_dev = priv_dev;
2935		priv_dev->eps[i] = priv_ep;
2936		priv_ep->num = ep_number;
2937		priv_ep->dir = ep_dir ? USB_DIR_IN : USB_DIR_OUT;
2938
2939		if (!ep_number) {
2940			ret = cdns3_init_ep0(priv_dev, priv_ep);
2941			if (ret) {
2942				dev_err(priv_dev->dev, "Failed to init ep0\n");
2943				goto err;
2944			}
2945		} else {
2946			snprintf(priv_ep->name, sizeof(priv_ep->name), "ep%d%s",
2947				 ep_number, !!ep_dir ? "in" : "out");
2948			priv_ep->endpoint.name = priv_ep->name;
2949
2950			usb_ep_set_maxpacket_limit(&priv_ep->endpoint,
2951						   CDNS3_EP_MAX_PACKET_LIMIT);
2952			priv_ep->endpoint.max_streams = CDNS3_EP_MAX_STREAMS;
2953			priv_ep->endpoint.ops = &cdns3_gadget_ep_ops;
2954			if (ep_dir)
2955				priv_ep->endpoint.caps.dir_in = 1;
2956			else
2957				priv_ep->endpoint.caps.dir_out = 1;
2958
2959			if (iso_ep_reg & ep_mask)
2960				priv_ep->endpoint.caps.type_iso = 1;
2961
2962			priv_ep->endpoint.caps.type_bulk = 1;
2963			priv_ep->endpoint.caps.type_int = 1;
2964
2965			list_add_tail(&priv_ep->endpoint.ep_list,
2966				      &priv_dev->gadget.ep_list);
2967		}
2968
2969		priv_ep->flags = 0;
2970
2971		dev_dbg(priv_dev->dev, "Initialized  %s support: %s %s\n",
2972			 priv_ep->name,
2973			 priv_ep->endpoint.caps.type_bulk ? "BULK, INT" : "",
2974			 priv_ep->endpoint.caps.type_iso ? "ISO" : "");
2975
2976		INIT_LIST_HEAD(&priv_ep->pending_req_list);
2977		INIT_LIST_HEAD(&priv_ep->deferred_req_list);
2978		INIT_LIST_HEAD(&priv_ep->wa2_descmiss_req_list);
2979	}
2980
2981	return 0;
2982err:
2983	cdns3_free_all_eps(priv_dev);
2984	return -ENOMEM;
2985}
2986
2987void cdns3_gadget_exit(struct cdns3 *cdns)
2988{
2989	struct cdns3_device *priv_dev;
2990
2991	priv_dev = cdns->gadget_dev;
2992
2993	devm_free_irq(cdns->dev, cdns->dev_irq, priv_dev);
2994
2995	pm_runtime_mark_last_busy(cdns->dev);
2996	pm_runtime_put_autosuspend(cdns->dev);
2997
2998	usb_del_gadget_udc(&priv_dev->gadget);
2999
3000	cdns3_free_all_eps(priv_dev);
3001
3002	while (!list_empty(&priv_dev->aligned_buf_list)) {
3003		struct cdns3_aligned_buf *buf;
3004
3005		buf = cdns3_next_align_buf(&priv_dev->aligned_buf_list);
3006		dma_free_coherent(priv_dev->sysdev, buf->size,
3007				  buf->buf,
3008				  buf->dma);
3009
3010		list_del(&buf->list);
3011		kfree(buf);
3012	}
3013
3014	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3015			  priv_dev->setup_dma);
3016
3017	kfree(priv_dev->zlp_buf);
3018	kfree(priv_dev);
3019	cdns->gadget_dev = NULL;
3020	cdns3_drd_gadget_off(cdns);
3021}
3022
3023static int cdns3_gadget_start(struct cdns3 *cdns)
3024{
3025	struct cdns3_device *priv_dev;
3026	u32 max_speed;
3027	int ret;
3028
3029	priv_dev = kzalloc(sizeof(*priv_dev), GFP_KERNEL);
3030	if (!priv_dev)
3031		return -ENOMEM;
3032
3033	cdns->gadget_dev = priv_dev;
3034	priv_dev->sysdev = cdns->dev;
3035	priv_dev->dev = cdns->dev;
3036	priv_dev->regs = cdns->dev_regs;
3037
3038	device_property_read_u16(priv_dev->dev, "cdns,on-chip-buff-size",
3039				 &priv_dev->onchip_buffers);
3040
3041	if (priv_dev->onchip_buffers <=  0) {
3042		u32 reg = readl(&priv_dev->regs->usb_cap2);
3043
3044		priv_dev->onchip_buffers = USB_CAP2_ACTUAL_MEM_SIZE(reg);
3045	}
3046
3047	if (!priv_dev->onchip_buffers)
3048		priv_dev->onchip_buffers = 256;
3049
3050	max_speed = usb_get_maximum_speed(cdns->dev);
3051
3052	/* Check the maximum_speed parameter */
3053	switch (max_speed) {
3054	case USB_SPEED_FULL:
3055	case USB_SPEED_HIGH:
3056	case USB_SPEED_SUPER:
3057		break;
3058	default:
3059		dev_err(cdns->dev, "invalid maximum_speed parameter %d\n",
3060			max_speed);
3061		fallthrough;
3062	case USB_SPEED_UNKNOWN:
3063		/* default to superspeed */
3064		max_speed = USB_SPEED_SUPER;
3065		break;
3066	}
3067
3068	/* fill gadget fields */
3069	priv_dev->gadget.max_speed = max_speed;
3070	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3071	priv_dev->gadget.ops = &cdns3_gadget_ops;
3072	priv_dev->gadget.name = "usb-ss-gadget";
3073	priv_dev->gadget.sg_supported = 1;
3074	priv_dev->gadget.quirk_avoids_skb_reserve = 1;
3075	priv_dev->gadget.irq = cdns->dev_irq;
3076
3077	spin_lock_init(&priv_dev->lock);
3078	INIT_WORK(&priv_dev->pending_status_wq,
3079		  cdns3_pending_setup_status_handler);
3080
3081	INIT_WORK(&priv_dev->aligned_buf_wq,
3082		  cdns3_free_aligned_request_buf);
3083
3084	/* initialize endpoint container */
3085	INIT_LIST_HEAD(&priv_dev->gadget.ep_list);
3086	INIT_LIST_HEAD(&priv_dev->aligned_buf_list);
3087
3088	ret = cdns3_init_eps(priv_dev);
3089	if (ret) {
3090		dev_err(priv_dev->dev, "Failed to create endpoints\n");
3091		goto err1;
3092	}
3093
3094	/* allocate memory for setup packet buffer */
3095	priv_dev->setup_buf = dma_alloc_coherent(priv_dev->sysdev, 8,
3096						 &priv_dev->setup_dma, GFP_DMA);
3097	if (!priv_dev->setup_buf) {
3098		ret = -ENOMEM;
3099		goto err2;
3100	}
3101
3102	priv_dev->dev_ver = readl(&priv_dev->regs->usb_cap6);
3103
3104	dev_dbg(priv_dev->dev, "Device Controller version: %08x\n",
3105		readl(&priv_dev->regs->usb_cap6));
3106	dev_dbg(priv_dev->dev, "USB Capabilities:: %08x\n",
3107		readl(&priv_dev->regs->usb_cap1));
3108	dev_dbg(priv_dev->dev, "On-Chip memory configuration: %08x\n",
3109		readl(&priv_dev->regs->usb_cap2));
3110
3111	priv_dev->dev_ver = GET_DEV_BASE_VERSION(priv_dev->dev_ver);
3112
3113	priv_dev->zlp_buf = kzalloc(CDNS3_EP_ZLP_BUF_SIZE, GFP_KERNEL);
3114	if (!priv_dev->zlp_buf) {
3115		ret = -ENOMEM;
3116		goto err3;
3117	}
3118
3119	/* add USB gadget device */
3120	ret = usb_add_gadget_udc(priv_dev->dev, &priv_dev->gadget);
3121	if (ret < 0) {
3122		dev_err(priv_dev->dev,
3123			"Failed to register USB device controller\n");
3124		goto err4;
3125	}
3126
3127	return 0;
3128err4:
3129	kfree(priv_dev->zlp_buf);
3130err3:
3131	dma_free_coherent(priv_dev->sysdev, 8, priv_dev->setup_buf,
3132			  priv_dev->setup_dma);
3133err2:
3134	cdns3_free_all_eps(priv_dev);
3135err1:
3136	cdns->gadget_dev = NULL;
3137	return ret;
3138}
3139
3140static int __cdns3_gadget_init(struct cdns3 *cdns)
3141{
3142	int ret = 0;
3143
3144	/* Ensure 32-bit DMA Mask in case we switched back from Host mode */
3145	ret = dma_set_mask_and_coherent(cdns->dev, DMA_BIT_MASK(32));
3146	if (ret) {
3147		dev_err(cdns->dev, "Failed to set dma mask: %d\n", ret);
3148		return ret;
3149	}
3150
3151	cdns3_drd_gadget_on(cdns);
3152	pm_runtime_get_sync(cdns->dev);
3153
3154	ret = cdns3_gadget_start(cdns);
3155	if (ret)
3156		return ret;
3157
3158	/*
3159	 * Because interrupt line can be shared with other components in
3160	 * driver it can't use IRQF_ONESHOT flag here.
3161	 */
3162	ret = devm_request_threaded_irq(cdns->dev, cdns->dev_irq,
3163					cdns3_device_irq_handler,
3164					cdns3_device_thread_irq_handler,
3165					IRQF_SHARED, dev_name(cdns->dev),
3166					cdns->gadget_dev);
3167
3168	if (ret)
3169		goto err0;
3170
3171	return 0;
3172err0:
3173	cdns3_gadget_exit(cdns);
3174	return ret;
3175}
3176
3177static int cdns3_gadget_suspend(struct cdns3 *cdns, bool do_wakeup)
3178{
3179	struct cdns3_device *priv_dev = cdns->gadget_dev;
3180
3181	cdns3_disconnect_gadget(priv_dev);
3182
3183	priv_dev->gadget.speed = USB_SPEED_UNKNOWN;
3184	usb_gadget_set_state(&priv_dev->gadget, USB_STATE_NOTATTACHED);
3185	cdns3_hw_reset_eps_config(priv_dev);
3186
3187	/* disable interrupt for device */
3188	writel(0, &priv_dev->regs->usb_ien);
3189
3190	return 0;
3191}
3192
3193static int cdns3_gadget_resume(struct cdns3 *cdns, bool hibernated)
3194{
3195	struct cdns3_device *priv_dev = cdns->gadget_dev;
3196
3197	if (!priv_dev->gadget_driver)
3198		return 0;
3199
3200	cdns3_gadget_config(priv_dev);
3201
3202	return 0;
3203}
3204
3205/**
3206 * cdns3_gadget_init - initialize device structure
3207 *
3208 * @cdns: cdns3 instance
3209 *
3210 * This function initializes the gadget.
3211 */
3212int cdns3_gadget_init(struct cdns3 *cdns)
3213{
3214	struct cdns3_role_driver *rdrv;
3215
3216	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
3217	if (!rdrv)
3218		return -ENOMEM;
3219
3220	rdrv->start	= __cdns3_gadget_init;
3221	rdrv->stop	= cdns3_gadget_exit;
3222	rdrv->suspend	= cdns3_gadget_suspend;
3223	rdrv->resume	= cdns3_gadget_resume;
3224	rdrv->state	= CDNS3_ROLE_STATE_INACTIVE;
3225	rdrv->name	= "gadget";
3226	cdns->roles[USB_ROLE_DEVICE] = rdrv;
3227
3228	return 0;
3229}