1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Cadence CDNSP DRD Driver.
   4 *
   5 * Copyright (C) 2020 Cadence.
   6 *
   7 * Author: Pawel Laszczak <pawell@cadence.com>
   8 *
   9 */
  10
  11#include <linux/moduleparam.h>
  12#include <linux/dma-mapping.h>
  13#include <linux/module.h>
  14#include <linux/iopoll.h>
  15#include <linux/delay.h>
  16#include <linux/log2.h>
  17#include <linux/slab.h>
  18#include <linux/pci.h>
  19#include <linux/irq.h>
  20#include <linux/dmi.h>
  21
  22#include "core.h"
  23#include "gadget-export.h"
  24#include "drd.h"
  25#include "cdnsp-gadget.h"
  26#include "cdnsp-trace.h"
  27
  28unsigned int cdnsp_port_speed(unsigned int port_status)
  29{
  30	/*Detect gadget speed based on PORTSC register*/
  31	if (DEV_SUPERSPEEDPLUS(port_status))
  32		return USB_SPEED_SUPER_PLUS;
  33	else if (DEV_SUPERSPEED(port_status))
  34		return USB_SPEED_SUPER;
  35	else if (DEV_HIGHSPEED(port_status))
  36		return USB_SPEED_HIGH;
  37	else if (DEV_FULLSPEED(port_status))
  38		return USB_SPEED_FULL;
  39
  40	/* If device is detached then speed will be USB_SPEED_UNKNOWN.*/
  41	return USB_SPEED_UNKNOWN;
  42}
  43
  44/*
  45 * Given a port state, this function returns a value that would result in the
  46 * port being in the same state, if the value was written to the port status
  47 * control register.
  48 * Save Read Only (RO) bits and save read/write bits where
  49 * writing a 0 clears the bit and writing a 1 sets the bit (RWS).
  50 * For all other types (RW1S, RW1CS, RW, and RZ), writing a '0' has no effect.
  51 */
  52u32 cdnsp_port_state_to_neutral(u32 state)
  53{
  54	/* Save read-only status and port state. */
  55	return (state & CDNSP_PORT_RO) | (state & CDNSP_PORT_RWS);
  56}
  57
  58/**
  59 * cdnsp_find_next_ext_cap - Find the offset of the extended capabilities
  60 *                           with capability ID id.
  61 * @base: PCI MMIO registers base address.
  62 * @start: Address at which to start looking, (0 or HCC_PARAMS to start at
  63 *         beginning of list)
  64 * @id: Extended capability ID to search for.
  65 *
  66 * Returns the offset of the next matching extended capability structure.
  67 * Some capabilities can occur several times,
  68 * e.g., the EXT_CAPS_PROTOCOL, and this provides a way to find them all.
  69 */
  70int cdnsp_find_next_ext_cap(void __iomem *base, u32 start, int id)
  71{
  72	u32 offset = start;
  73	u32 next;
  74	u32 val;
  75
  76	if (!start || start == HCC_PARAMS_OFFSET) {
  77		val = readl(base + HCC_PARAMS_OFFSET);
  78		if (val == ~0)
  79			return 0;
  80
  81		offset = HCC_EXT_CAPS(val) << 2;
  82		if (!offset)
  83			return 0;
  84	}
  85
  86	do {
  87		val = readl(base + offset);
  88		if (val == ~0)
  89			return 0;
  90
  91		if (EXT_CAPS_ID(val) == id && offset != start)
  92			return offset;
  93
  94		next = EXT_CAPS_NEXT(val);
  95		offset += next << 2;
  96	} while (next);
  97
  98	return 0;
  99}
 100
 101void cdnsp_set_link_state(struct cdnsp_device *pdev,
 102			  __le32 __iomem *port_regs,
 103			  u32 link_state)
 104{
 105	int port_num = 0xFF;
 106	u32 temp;
 107
 108	temp = readl(port_regs);
 109	temp = cdnsp_port_state_to_neutral(temp);
 110	temp |= PORT_WKCONN_E | PORT_WKDISC_E;
 111	writel(temp, port_regs);
 112
 113	temp &= ~PORT_PLS_MASK;
 114	temp |= PORT_LINK_STROBE | link_state;
 115
 116	if (pdev->active_port)
 117		port_num = pdev->active_port->port_num;
 118
 119	trace_cdnsp_handle_port_status(port_num, readl(port_regs));
 120	writel(temp, port_regs);
 121	trace_cdnsp_link_state_changed(port_num, readl(port_regs));
 122}
 123
 124static void cdnsp_disable_port(struct cdnsp_device *pdev,
 125			       __le32 __iomem *port_regs)
 126{
 127	u32 temp = cdnsp_port_state_to_neutral(readl(port_regs));
 128
 129	writel(temp | PORT_PED, port_regs);
 130}
 131
 132static void cdnsp_clear_port_change_bit(struct cdnsp_device *pdev,
 133					__le32 __iomem *port_regs)
 134{
 135	u32 portsc = readl(port_regs);
 136
 137	writel(cdnsp_port_state_to_neutral(portsc) |
 138	       (portsc & PORT_CHANGE_BITS), port_regs);
 139}
 140
 141static void cdnsp_set_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
 142{
 143	__le32 __iomem *reg;
 144	void __iomem *base;
 145	u32 offset = 0;
 146
 147	base = &pdev->cap_regs->hc_capbase;
 148	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 149	reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
 150
 151	bit = readl(reg) | bit;
 152	writel(bit, reg);
 153}
 154
 155static void cdnsp_clear_chicken_bits_2(struct cdnsp_device *pdev, u32 bit)
 156{
 157	__le32 __iomem *reg;
 158	void __iomem *base;
 159	u32 offset = 0;
 160
 161	base = &pdev->cap_regs->hc_capbase;
 162	offset = cdnsp_find_next_ext_cap(base, offset, D_XEC_PRE_REGS_CAP);
 163	reg = base + offset + REG_CHICKEN_BITS_2_OFFSET;
 164
 165	bit = readl(reg) & ~bit;
 166	writel(bit, reg);
 167}
 168
 169/*
 170 * Disable interrupts and begin the controller halting process.
 171 */
 172static void cdnsp_quiesce(struct cdnsp_device *pdev)
 173{
 174	u32 halted;
 175	u32 mask;
 176	u32 cmd;
 177
 178	mask = ~(u32)(CDNSP_IRQS);
 179
 180	halted = readl(&pdev->op_regs->status) & STS_HALT;
 181	if (!halted)
 182		mask &= ~(CMD_R_S | CMD_DEVEN);
 183
 184	cmd = readl(&pdev->op_regs->command);
 185	cmd &= mask;
 186	writel(cmd, &pdev->op_regs->command);
 187}
 188
 189/*
 190 * Force controller into halt state.
 191 *
 192 * Disable any IRQs and clear the run/stop bit.
 193 * Controller will complete any current and actively pipelined transactions, and
 194 * should halt within 16 ms of the run/stop bit being cleared.
 195 * Read controller Halted bit in the status register to see when the
 196 * controller is finished.
 197 */
 198int cdnsp_halt(struct cdnsp_device *pdev)
 199{
 200	int ret;
 201	u32 val;
 202
 203	cdnsp_quiesce(pdev);
 204
 205	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, val,
 206					val & STS_HALT, 1,
 207					CDNSP_MAX_HALT_USEC);
 208	if (ret) {
 209		dev_err(pdev->dev, "ERROR: Device halt failed\n");
 210		return ret;
 211	}
 212
 213	pdev->cdnsp_state |= CDNSP_STATE_HALTED;
 214
 215	return 0;
 216}
 217
 218/*
 219 * device controller died, register read returns 0xffffffff, or command never
 220 * ends.
 221 */
 222void cdnsp_died(struct cdnsp_device *pdev)
 223{
 224	dev_err(pdev->dev, "ERROR: CDNSP controller not responding\n");
 225	pdev->cdnsp_state |= CDNSP_STATE_DYING;
 226	cdnsp_halt(pdev);
 227}
 228
 229/*
 230 * Set the run bit and wait for the device to be running.
 231 */
 232static int cdnsp_start(struct cdnsp_device *pdev)
 233{
 234	u32 temp;
 235	int ret;
 236
 237	temp = readl(&pdev->op_regs->command);
 238	temp |= (CMD_R_S | CMD_DEVEN);
 239	writel(temp, &pdev->op_regs->command);
 240
 241	pdev->cdnsp_state = 0;
 242
 243	/*
 244	 * Wait for the STS_HALT Status bit to be 0 to indicate the device is
 245	 * running.
 246	 */
 247	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
 248					!(temp & STS_HALT), 1,
 249					CDNSP_MAX_HALT_USEC);
 250	if (ret) {
 251		pdev->cdnsp_state = CDNSP_STATE_DYING;
 252		dev_err(pdev->dev, "ERROR: Controller run failed\n");
 253	}
 254
 255	return ret;
 256}
 257
 258/*
 259 * Reset a halted controller.
 260 *
 261 * This resets pipelines, timers, counters, state machines, etc.
 262 * Transactions will be terminated immediately, and operational registers
 263 * will be set to their defaults.
 264 */
 265int cdnsp_reset(struct cdnsp_device *pdev)
 266{
 267	u32 command;
 268	u32 temp;
 269	int ret;
 270
 271	temp = readl(&pdev->op_regs->status);
 272
 273	if (temp == ~(u32)0) {
 274		dev_err(pdev->dev, "Device not accessible, reset failed.\n");
 275		return -ENODEV;
 276	}
 277
 278	if ((temp & STS_HALT) == 0) {
 279		dev_err(pdev->dev, "Controller not halted, aborting reset.\n");
 280		return -EINVAL;
 281	}
 282
 283	command = readl(&pdev->op_regs->command);
 284	command |= CMD_RESET;
 285	writel(command, &pdev->op_regs->command);
 286
 287	ret = readl_poll_timeout_atomic(&pdev->op_regs->command, temp,
 288					!(temp & CMD_RESET), 1,
 289					10 * 1000);
 290	if (ret) {
 291		dev_err(pdev->dev, "ERROR: Controller reset failed\n");
 292		return ret;
 293	}
 294
 295	/*
 296	 * CDNSP cannot write any doorbells or operational registers other
 297	 * than status until the "Controller Not Ready" flag is cleared.
 298	 */
 299	ret = readl_poll_timeout_atomic(&pdev->op_regs->status, temp,
 300					!(temp & STS_CNR), 1,
 301					10 * 1000);
 302
 303	if (ret) {
 304		dev_err(pdev->dev, "ERROR: Controller not ready to work\n");
 305		return ret;
 306	}
 307
 308	dev_dbg(pdev->dev, "Controller ready to work");
 309
 310	return ret;
 311}
 312
 313/*
 314 * cdnsp_get_endpoint_index - Find the index for an endpoint given its
 315 * descriptor.Use the return value to right shift 1 for the bitmask.
 316 *
 317 * Index = (epnum * 2) + direction - 1,
 318 * where direction = 0 for OUT, 1 for IN.
 319 * For control endpoints, the IN index is used (OUT index is unused), so
 320 * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
 321 */
 322static unsigned int
 323	cdnsp_get_endpoint_index(const struct usb_endpoint_descriptor *desc)
 324{
 325	unsigned int index = (unsigned int)usb_endpoint_num(desc);
 326
 327	if (usb_endpoint_xfer_control(desc))
 328		return index * 2;
 329
 330	return (index * 2) + (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
 331}
 332
 333/*
 334 * Find the flag for this endpoint (for use in the control context). Use the
 335 * endpoint index to create a bitmask. The slot context is bit 0, endpoint 0 is
 336 * bit 1, etc.
 337 */
 338static unsigned int
 339	cdnsp_get_endpoint_flag(const struct usb_endpoint_descriptor *desc)
 340{
 341	return 1 << (cdnsp_get_endpoint_index(desc) + 1);
 342}
 343
 344int cdnsp_ep_enqueue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
 345{
 346	struct cdnsp_device *pdev = pep->pdev;
 347	struct usb_request *request;
 348	int ret;
 349
 350	if (preq->epnum == 0 && !list_empty(&pep->pending_list)) {
 351		trace_cdnsp_request_enqueue_busy(preq);
 352		return -EBUSY;
 353	}
 354
 355	request = &preq->request;
 356	request->actual = 0;
 357	request->status = -EINPROGRESS;
 358	preq->direction = pep->direction;
 359	preq->epnum = pep->number;
 360	preq->td.drbl = 0;
 361
 362	ret = usb_gadget_map_request_by_dev(pdev->dev, request, pep->direction);
 363	if (ret) {
 364		trace_cdnsp_request_enqueue_error(preq);
 365		return ret;
 366	}
 367
 368	list_add_tail(&preq->list, &pep->pending_list);
 369
 370	trace_cdnsp_request_enqueue(preq);
 371
 372	switch (usb_endpoint_type(pep->endpoint.desc)) {
 373	case USB_ENDPOINT_XFER_CONTROL:
 374		ret = cdnsp_queue_ctrl_tx(pdev, preq);
 375		break;
 376	case USB_ENDPOINT_XFER_BULK:
 377	case USB_ENDPOINT_XFER_INT:
 378		ret = cdnsp_queue_bulk_tx(pdev, preq);
 379		break;
 380	case USB_ENDPOINT_XFER_ISOC:
 381		ret = cdnsp_queue_isoc_tx(pdev, preq);
 382	}
 383
 384	if (ret)
 385		goto unmap;
 386
 387	return 0;
 388
 389unmap:
 390	usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
 391					pep->direction);
 392	list_del(&preq->list);
 393	trace_cdnsp_request_enqueue_error(preq);
 394
 395	return ret;
 396}
 397
 398/*
 399 * Remove the request's TD from the endpoint ring. This may cause the
 400 * controller to stop USB transfers, potentially stopping in the middle of a
 401 * TRB buffer. The controller should pick up where it left off in the TD,
 402 * unless a Set Transfer Ring Dequeue Pointer is issued.
 403 *
 404 * The TRBs that make up the buffers for the canceled request will be "removed"
 405 * from the ring. Since the ring is a contiguous structure, they can't be
 406 * physically removed. Instead, there are two options:
 407 *
 408 *  1) If the controller is in the middle of processing the request to be
 409 *     canceled, we simply move the ring's dequeue pointer past those TRBs
 410 *     using the Set Transfer Ring Dequeue Pointer command. This will be
 411 *     the common case, when drivers timeout on the last submitted request
 412 *     and attempt to cancel.
 413 *
 414 *  2) If the controller is in the middle of a different TD, we turn the TRBs
 415 *     into a series of 1-TRB transfer no-op TDs. No-ops shouldn't be chained.
 416 *     The controller will need to invalidate the any TRBs it has cached after
 417 *     the stop endpoint command.
 418 *
 419 *  3) The TD may have completed by the time the Stop Endpoint Command
 420 *     completes, so software needs to handle that case too.
 421 *
 422 */
 423int cdnsp_ep_dequeue(struct cdnsp_ep *pep, struct cdnsp_request *preq)
 424{
 425	struct cdnsp_device *pdev = pep->pdev;
 426	int ret_stop = 0;
 427	int ret_rem;
 428
 429	trace_cdnsp_request_dequeue(preq);
 430
 431	if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_RUNNING)
 432		ret_stop = cdnsp_cmd_stop_ep(pdev, pep);
 433
 434	ret_rem = cdnsp_remove_request(pdev, preq, pep);
 435
 436	return ret_rem ? ret_rem : ret_stop;
 437}
 438
 439static void cdnsp_zero_in_ctx(struct cdnsp_device *pdev)
 440{
 441	struct cdnsp_input_control_ctx *ctrl_ctx;
 442	struct cdnsp_slot_ctx *slot_ctx;
 443	struct cdnsp_ep_ctx *ep_ctx;
 444	int i;
 445
 446	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 447
 448	/*
 449	 * When a device's add flag and drop flag are zero, any subsequent
 450	 * configure endpoint command will leave that endpoint's state
 451	 * untouched. Make sure we don't leave any old state in the input
 452	 * endpoint contexts.
 453	 */
 454	ctrl_ctx->drop_flags = 0;
 455	ctrl_ctx->add_flags = 0;
 456	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 457	slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
 458
 459	/* Endpoint 0 is always valid */
 460	slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
 461	for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i) {
 462		ep_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, i);
 463		ep_ctx->ep_info = 0;
 464		ep_ctx->ep_info2 = 0;
 465		ep_ctx->deq = 0;
 466		ep_ctx->tx_info = 0;
 467	}
 468}
 469
 470/* Issue a configure endpoint command and wait for it to finish. */
 471static int cdnsp_configure_endpoint(struct cdnsp_device *pdev)
 472{
 473	int ret;
 474
 475	cdnsp_queue_configure_endpoint(pdev, pdev->cmd.in_ctx->dma);
 476	cdnsp_ring_cmd_db(pdev);
 477	ret = cdnsp_wait_for_cmd_compl(pdev);
 478	if (ret) {
 479		dev_err(pdev->dev,
 480			"ERR: unexpected command completion code 0x%x.\n", ret);
 481		return -EINVAL;
 482	}
 483
 484	return ret;
 485}
 486
 487static void cdnsp_invalidate_ep_events(struct cdnsp_device *pdev,
 488				       struct cdnsp_ep *pep)
 489{
 490	struct cdnsp_segment *segment;
 491	union cdnsp_trb *event;
 492	u32 cycle_state;
 493	u32  data;
 494
 495	event = pdev->event_ring->dequeue;
 496	segment = pdev->event_ring->deq_seg;
 497	cycle_state = pdev->event_ring->cycle_state;
 498
 499	while (1) {
 500		data = le32_to_cpu(event->trans_event.flags);
 501
 502		/* Check the owner of the TRB. */
 503		if ((data & TRB_CYCLE) != cycle_state)
 504			break;
 505
 506		if (TRB_FIELD_TO_TYPE(data) == TRB_TRANSFER &&
 507		    TRB_TO_EP_ID(data) == (pep->idx + 1)) {
 508			data |= TRB_EVENT_INVALIDATE;
 509			event->trans_event.flags = cpu_to_le32(data);
 510		}
 511
 512		if (cdnsp_last_trb_on_seg(segment, event)) {
 513			cycle_state ^= 1;
 514			segment = pdev->event_ring->deq_seg->next;
 515			event = segment->trbs;
 516		} else {
 517			event++;
 518		}
 519	}
 520}
 521
 522int cdnsp_wait_for_cmd_compl(struct cdnsp_device *pdev)
 523{
 524	struct cdnsp_segment *event_deq_seg;
 525	union cdnsp_trb *cmd_trb;
 526	dma_addr_t cmd_deq_dma;
 527	union cdnsp_trb *event;
 528	u32 cycle_state;
 529	int ret, val;
 530	u64 cmd_dma;
 531	u32  flags;
 532
 533	cmd_trb = pdev->cmd.command_trb;
 534	pdev->cmd.status = 0;
 535
 536	trace_cdnsp_cmd_wait_for_compl(pdev->cmd_ring, &cmd_trb->generic);
 537
 538	ret = readl_poll_timeout_atomic(&pdev->op_regs->cmd_ring, val,
 539					!CMD_RING_BUSY(val), 1,
 540					CDNSP_CMD_TIMEOUT);
 541	if (ret) {
 542		dev_err(pdev->dev, "ERR: Timeout while waiting for command\n");
 543		trace_cdnsp_cmd_timeout(pdev->cmd_ring, &cmd_trb->generic);
 544		pdev->cdnsp_state = CDNSP_STATE_DYING;
 545		return -ETIMEDOUT;
 546	}
 547
 548	event = pdev->event_ring->dequeue;
 549	event_deq_seg = pdev->event_ring->deq_seg;
 550	cycle_state = pdev->event_ring->cycle_state;
 551
 552	cmd_deq_dma = cdnsp_trb_virt_to_dma(pdev->cmd_ring->deq_seg, cmd_trb);
 553	if (!cmd_deq_dma)
 554		return -EINVAL;
 555
 556	while (1) {
 557		flags = le32_to_cpu(event->event_cmd.flags);
 558
 559		/* Check the owner of the TRB. */
 560		if ((flags & TRB_CYCLE) != cycle_state)
 561			return -EINVAL;
 562
 563		cmd_dma = le64_to_cpu(event->event_cmd.cmd_trb);
 564
 565		/*
 566		 * Check whether the completion event is for last queued
 567		 * command.
 568		 */
 569		if (TRB_FIELD_TO_TYPE(flags) != TRB_COMPLETION ||
 570		    cmd_dma != (u64)cmd_deq_dma) {
 571			if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
 572				event++;
 573				continue;
 574			}
 575
 576			if (cdnsp_last_trb_on_ring(pdev->event_ring,
 577						   event_deq_seg, event))
 578				cycle_state ^= 1;
 579
 580			event_deq_seg = event_deq_seg->next;
 581			event = event_deq_seg->trbs;
 582			continue;
 583		}
 584
 585		trace_cdnsp_handle_command(pdev->cmd_ring, &cmd_trb->generic);
 586
 587		pdev->cmd.status = GET_COMP_CODE(le32_to_cpu(event->event_cmd.status));
 588		if (pdev->cmd.status == COMP_SUCCESS)
 589			return 0;
 590
 591		return -pdev->cmd.status;
 592	}
 593}
 594
 595int cdnsp_halt_endpoint(struct cdnsp_device *pdev,
 596			struct cdnsp_ep *pep,
 597			int value)
 598{
 599	int ret;
 600
 601	trace_cdnsp_ep_halt(value ? "Set" : "Clear");
 602
 603	ret = cdnsp_cmd_stop_ep(pdev, pep);
 604	if (ret)
 605		return ret;
 606
 607	if (value) {
 608		if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_STOPPED) {
 609			cdnsp_queue_halt_endpoint(pdev, pep->idx);
 610			cdnsp_ring_cmd_db(pdev);
 611			ret = cdnsp_wait_for_cmd_compl(pdev);
 612		}
 613
 614		pep->ep_state |= EP_HALTED;
 615	} else {
 616		cdnsp_queue_reset_ep(pdev, pep->idx);
 617		cdnsp_ring_cmd_db(pdev);
 618		ret = cdnsp_wait_for_cmd_compl(pdev);
 619		trace_cdnsp_handle_cmd_reset_ep(pep->out_ctx);
 620
 621		if (ret)
 622			return ret;
 623
 624		pep->ep_state &= ~EP_HALTED;
 625
 626		if (pep->idx != 0 && !(pep->ep_state & EP_WEDGE))
 627			cdnsp_ring_doorbell_for_active_rings(pdev, pep);
 628
 629		pep->ep_state &= ~EP_WEDGE;
 630	}
 631
 632	return 0;
 633}
 634
 635static int cdnsp_update_eps_configuration(struct cdnsp_device *pdev,
 636					  struct cdnsp_ep *pep)
 637{
 638	struct cdnsp_input_control_ctx *ctrl_ctx;
 639	struct cdnsp_slot_ctx *slot_ctx;
 640	int ret = 0;
 641	u32 ep_sts;
 642	int i;
 643
 644	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 645
 646	/* Don't issue the command if there's no endpoints to update. */
 647	if (ctrl_ctx->add_flags == 0 && ctrl_ctx->drop_flags == 0)
 648		return 0;
 649
 650	ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
 651	ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
 652	ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
 653
 654	/* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
 655	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 656	for (i = CDNSP_ENDPOINTS_NUM; i >= 1; i--) {
 657		__le32 le32 = cpu_to_le32(BIT(i));
 658
 659		if ((pdev->eps[i - 1].ring && !(ctrl_ctx->drop_flags & le32)) ||
 660		    (ctrl_ctx->add_flags & le32) || i == 1) {
 661			slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
 662			slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
 663			break;
 664		}
 665	}
 666
 667	ep_sts = GET_EP_CTX_STATE(pep->out_ctx);
 668
 669	if ((ctrl_ctx->add_flags != cpu_to_le32(SLOT_FLAG) &&
 670	     ep_sts == EP_STATE_DISABLED) ||
 671	    (ep_sts != EP_STATE_DISABLED && ctrl_ctx->drop_flags))
 672		ret = cdnsp_configure_endpoint(pdev);
 673
 674	trace_cdnsp_configure_endpoint(cdnsp_get_slot_ctx(&pdev->out_ctx));
 675	trace_cdnsp_handle_cmd_config_ep(pep->out_ctx);
 676
 677	cdnsp_zero_in_ctx(pdev);
 678
 679	return ret;
 680}
 681
 682/*
 683 * This submits a Reset Device Command, which will set the device state to 0,
 684 * set the device address to 0, and disable all the endpoints except the default
 685 * control endpoint. The USB core should come back and call
 686 * cdnsp_setup_device(), and then re-set up the configuration.
 687 */
 688int cdnsp_reset_device(struct cdnsp_device *pdev)
 689{
 690	struct cdnsp_slot_ctx *slot_ctx;
 691	int slot_state;
 692	int ret, i;
 693
 694	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 695	slot_ctx->dev_info = 0;
 696	pdev->device_address = 0;
 697
 698	/* If device is not setup, there is no point in resetting it. */
 699	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 700	slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 701	trace_cdnsp_reset_device(slot_ctx);
 702
 703	if (slot_state <= SLOT_STATE_DEFAULT &&
 704	    pdev->eps[0].ep_state & EP_HALTED) {
 705		cdnsp_halt_endpoint(pdev, &pdev->eps[0], 0);
 706	}
 707
 708	/*
 709	 * During Reset Device command controller shall transition the
 710	 * endpoint ep0 to the Running State.
 711	 */
 712	pdev->eps[0].ep_state &= ~(EP_STOPPED | EP_HALTED);
 713	pdev->eps[0].ep_state |= EP_ENABLED;
 714
 715	if (slot_state <= SLOT_STATE_DEFAULT)
 716		return 0;
 717
 718	cdnsp_queue_reset_device(pdev);
 719	cdnsp_ring_cmd_db(pdev);
 720	ret = cdnsp_wait_for_cmd_compl(pdev);
 721
 722	/*
 723	 * After Reset Device command all not default endpoints
 724	 * are in Disabled state.
 725	 */
 726	for (i = 1; i < CDNSP_ENDPOINTS_NUM; ++i)
 727		pdev->eps[i].ep_state |= EP_STOPPED | EP_UNCONFIGURED;
 728
 729	trace_cdnsp_handle_cmd_reset_dev(slot_ctx);
 730
 731	if (ret)
 732		dev_err(pdev->dev, "Reset device failed with error code %d",
 733			ret);
 734
 735	return ret;
 736}
 737
 738/*
 739 * Sets the MaxPStreams field and the Linear Stream Array field.
 740 * Sets the dequeue pointer to the stream context array.
 741 */
 742static void cdnsp_setup_streams_ep_input_ctx(struct cdnsp_device *pdev,
 743					     struct cdnsp_ep_ctx *ep_ctx,
 744					     struct cdnsp_stream_info *stream_info)
 745{
 746	u32 max_primary_streams;
 747
 748	/* MaxPStreams is the number of stream context array entries, not the
 749	 * number we're actually using. Must be in 2^(MaxPstreams + 1) format.
 750	 * fls(0) = 0, fls(0x1) = 1, fls(0x10) = 2, fls(0x100) = 3, etc.
 751	 */
 752	max_primary_streams = fls(stream_info->num_stream_ctxs) - 2;
 753	ep_ctx->ep_info &= cpu_to_le32(~EP_MAXPSTREAMS_MASK);
 754	ep_ctx->ep_info |= cpu_to_le32(EP_MAXPSTREAMS(max_primary_streams)
 755				       | EP_HAS_LSA);
 756	ep_ctx->deq  = cpu_to_le64(stream_info->ctx_array_dma);
 757}
 758
 759/*
 760 * The drivers use this function to prepare a bulk endpoints to use streams.
 761 *
 762 * Don't allow the call to succeed if endpoint only supports one stream
 763 * (which means it doesn't support streams at all).
 764 */
 765int cdnsp_alloc_streams(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
 766{
 767	unsigned int num_streams = usb_ss_max_streams(pep->endpoint.comp_desc);
 768	unsigned int num_stream_ctxs;
 769	int ret;
 770
 771	if (num_streams ==  0)
 772		return 0;
 773
 774	if (num_streams > STREAM_NUM_STREAMS)
 775		return -EINVAL;
 776
 777	/*
 778	 * Add two to the number of streams requested to account for
 779	 * stream 0 that is reserved for controller usage and one additional
 780	 * for TASK SET FULL response.
 781	 */
 782	num_streams += 2;
 783
 784	/* The stream context array size must be a power of two */
 785	num_stream_ctxs = roundup_pow_of_two(num_streams);
 786
 787	trace_cdnsp_stream_number(pep, num_stream_ctxs, num_streams);
 788
 789	ret = cdnsp_alloc_stream_info(pdev, pep, num_stream_ctxs, num_streams);
 790	if (ret)
 791		return ret;
 792
 793	cdnsp_setup_streams_ep_input_ctx(pdev, pep->in_ctx, &pep->stream_info);
 794
 795	pep->ep_state |= EP_HAS_STREAMS;
 796	pep->stream_info.td_count = 0;
 797	pep->stream_info.first_prime_det = 0;
 798
 799	/* Subtract 1 for stream 0, which drivers can't use. */
 800	return num_streams - 1;
 801}
 802
 803int cdnsp_disable_slot(struct cdnsp_device *pdev)
 804{
 805	int ret;
 806
 807	cdnsp_queue_slot_control(pdev, TRB_DISABLE_SLOT);
 808	cdnsp_ring_cmd_db(pdev);
 809	ret = cdnsp_wait_for_cmd_compl(pdev);
 810
 811	pdev->slot_id = 0;
 812	pdev->active_port = NULL;
 813
 814	trace_cdnsp_handle_cmd_disable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
 815
 816	memset(pdev->in_ctx.bytes, 0, CDNSP_CTX_SIZE);
 817	memset(pdev->out_ctx.bytes, 0, CDNSP_CTX_SIZE);
 818
 819	return ret;
 820}
 821
 822int cdnsp_enable_slot(struct cdnsp_device *pdev)
 823{
 824	struct cdnsp_slot_ctx *slot_ctx;
 825	int slot_state;
 826	int ret;
 827
 828	/* If device is not setup, there is no point in resetting it */
 829	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 830	slot_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 831
 832	if (slot_state != SLOT_STATE_DISABLED)
 833		return 0;
 834
 835	cdnsp_queue_slot_control(pdev, TRB_ENABLE_SLOT);
 836	cdnsp_ring_cmd_db(pdev);
 837	ret = cdnsp_wait_for_cmd_compl(pdev);
 838	if (ret)
 839		goto show_trace;
 840
 841	pdev->slot_id = 1;
 842
 843show_trace:
 844	trace_cdnsp_handle_cmd_enable_slot(cdnsp_get_slot_ctx(&pdev->out_ctx));
 845
 846	return ret;
 847}
 848
 849/*
 850 * Issue an Address Device command with BSR=0 if setup is SETUP_CONTEXT_ONLY
 851 * or with BSR = 1 if set_address is SETUP_CONTEXT_ADDRESS.
 852 */
 853int cdnsp_setup_device(struct cdnsp_device *pdev, enum cdnsp_setup_dev setup)
 854{
 855	struct cdnsp_input_control_ctx *ctrl_ctx;
 856	struct cdnsp_slot_ctx *slot_ctx;
 857	int dev_state = 0;
 858	int ret;
 859
 860	if (!pdev->slot_id) {
 861		trace_cdnsp_slot_id("incorrect");
 862		return -EINVAL;
 863	}
 864
 865	if (!pdev->active_port->port_num)
 866		return -EINVAL;
 867
 868	slot_ctx = cdnsp_get_slot_ctx(&pdev->out_ctx);
 869	dev_state = GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state));
 870
 871	if (setup == SETUP_CONTEXT_ONLY && dev_state == SLOT_STATE_DEFAULT) {
 872		trace_cdnsp_slot_already_in_default(slot_ctx);
 873		return 0;
 874	}
 875
 876	slot_ctx = cdnsp_get_slot_ctx(&pdev->in_ctx);
 877	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 878
 879	if (!slot_ctx->dev_info || dev_state == SLOT_STATE_DEFAULT) {
 880		ret = cdnsp_setup_addressable_priv_dev(pdev);
 881		if (ret)
 882			return ret;
 883	}
 884
 885	cdnsp_copy_ep0_dequeue_into_input_ctx(pdev);
 886
 887	ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
 888	ctrl_ctx->drop_flags = 0;
 889
 890	trace_cdnsp_setup_device_slot(slot_ctx);
 891
 892	cdnsp_queue_address_device(pdev, pdev->in_ctx.dma, setup);
 893	cdnsp_ring_cmd_db(pdev);
 894	ret = cdnsp_wait_for_cmd_compl(pdev);
 895
 896	trace_cdnsp_handle_cmd_addr_dev(cdnsp_get_slot_ctx(&pdev->out_ctx));
 897
 898	/* Zero the input context control for later use. */
 899	ctrl_ctx->add_flags = 0;
 900	ctrl_ctx->drop_flags = 0;
 901
 902	return ret;
 903}
 904
 905void cdnsp_set_usb2_hardware_lpm(struct cdnsp_device *pdev,
 906				 struct usb_request *req,
 907				 int enable)
 908{
 909	if (pdev->active_port != &pdev->usb2_port || !pdev->gadget.lpm_capable)
 910		return;
 911
 912	trace_cdnsp_lpm(enable);
 913
 914	if (enable)
 915		writel(PORT_BESL(CDNSP_DEFAULT_BESL) | PORT_L1S_NYET | PORT_HLE,
 916		       &pdev->active_port->regs->portpmsc);
 917	else
 918		writel(PORT_L1S_NYET, &pdev->active_port->regs->portpmsc);
 919}
 920
 921static int cdnsp_get_frame(struct cdnsp_device *pdev)
 922{
 923	return readl(&pdev->run_regs->microframe_index) >> 3;
 924}
 925
 926static int cdnsp_gadget_ep_enable(struct usb_ep *ep,
 927				  const struct usb_endpoint_descriptor *desc)
 928{
 929	struct cdnsp_input_control_ctx *ctrl_ctx;
 930	struct cdnsp_device *pdev;
 931	struct cdnsp_ep *pep;
 932	unsigned long flags;
 933	u32 added_ctxs;
 934	int ret;
 935
 936	if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT ||
 937	    !desc->wMaxPacketSize)
 938		return -EINVAL;
 939
 940	pep = to_cdnsp_ep(ep);
 941	pdev = pep->pdev;
 942	pep->ep_state &= ~EP_UNCONFIGURED;
 943
 944	if (dev_WARN_ONCE(pdev->dev, pep->ep_state & EP_ENABLED,
 945			  "%s is already enabled\n", pep->name))
 946		return 0;
 947
 948	spin_lock_irqsave(&pdev->lock, flags);
 949
 950	added_ctxs = cdnsp_get_endpoint_flag(desc);
 951	if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
 952		dev_err(pdev->dev, "ERROR: Bad endpoint number\n");
 953		ret = -EINVAL;
 954		goto unlock;
 955	}
 956
 957	pep->interval = desc->bInterval ? BIT(desc->bInterval - 1) : 0;
 958
 959	if (pdev->gadget.speed == USB_SPEED_FULL) {
 960		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_INT)
 961			pep->interval = desc->bInterval << 3;
 962		if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC)
 963			pep->interval = BIT(desc->bInterval - 1) << 3;
 964	}
 965
 966	if (usb_endpoint_type(desc) == USB_ENDPOINT_XFER_ISOC) {
 967		if (pep->interval > BIT(12)) {
 968			dev_err(pdev->dev, "bInterval %d not supported\n",
 969				desc->bInterval);
 970			ret = -EINVAL;
 971			goto unlock;
 972		}
 973		cdnsp_set_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
 974	}
 975
 976	ret = cdnsp_endpoint_init(pdev, pep, GFP_ATOMIC);
 977	if (ret)
 978		goto unlock;
 979
 980	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
 981	ctrl_ctx->add_flags = cpu_to_le32(added_ctxs);
 982	ctrl_ctx->drop_flags = 0;
 983
 984	ret = cdnsp_update_eps_configuration(pdev, pep);
 985	if (ret) {
 986		cdnsp_free_endpoint_rings(pdev, pep);
 987		goto unlock;
 988	}
 989
 990	pep->ep_state |= EP_ENABLED;
 991	pep->ep_state &= ~EP_STOPPED;
 992
 993unlock:
 994	trace_cdnsp_ep_enable_end(pep, 0);
 995	spin_unlock_irqrestore(&pdev->lock, flags);
 996
 997	return ret;
 998}
 999
1000static int cdnsp_gadget_ep_disable(struct usb_ep *ep)
1001{
1002	struct cdnsp_input_control_ctx *ctrl_ctx;
1003	struct cdnsp_request *preq;
1004	struct cdnsp_device *pdev;
1005	struct cdnsp_ep *pep;
1006	unsigned long flags;
1007	u32 drop_flag;
1008	int ret = 0;
1009
1010	if (!ep)
1011		return -EINVAL;
1012
1013	pep = to_cdnsp_ep(ep);
1014	pdev = pep->pdev;
1015
1016	spin_lock_irqsave(&pdev->lock, flags);
1017
1018	if (!(pep->ep_state & EP_ENABLED)) {
1019		dev_err(pdev->dev, "%s is already disabled\n", pep->name);
1020		ret = -EINVAL;
1021		goto finish;
1022	}
1023
1024	pep->ep_state |= EP_DIS_IN_RROGRESS;
1025
1026	/* Endpoint was unconfigured by Reset Device command. */
1027	if (!(pep->ep_state & EP_UNCONFIGURED))
1028		cdnsp_cmd_stop_ep(pdev, pep);
1029
1030	/* Remove all queued USB requests. */
1031	while (!list_empty(&pep->pending_list)) {
1032		preq = next_request(&pep->pending_list);
1033		cdnsp_ep_dequeue(pep, preq);
1034	}
1035
1036	cdnsp_invalidate_ep_events(pdev, pep);
1037
1038	pep->ep_state &= ~EP_DIS_IN_RROGRESS;
1039	drop_flag = cdnsp_get_endpoint_flag(pep->endpoint.desc);
1040	ctrl_ctx = cdnsp_get_input_control_ctx(&pdev->in_ctx);
1041	ctrl_ctx->drop_flags = cpu_to_le32(drop_flag);
1042	ctrl_ctx->add_flags = 0;
1043
1044	cdnsp_endpoint_zero(pdev, pep);
1045
1046	if (!(pep->ep_state & EP_UNCONFIGURED))
1047		ret = cdnsp_update_eps_configuration(pdev, pep);
1048
1049	cdnsp_free_endpoint_rings(pdev, pep);
1050
1051	pep->ep_state &= ~(EP_ENABLED | EP_UNCONFIGURED);
1052	pep->ep_state |= EP_STOPPED;
1053
1054finish:
1055	trace_cdnsp_ep_disable_end(pep, 0);
1056	spin_unlock_irqrestore(&pdev->lock, flags);
1057
1058	return ret;
1059}
1060
1061static struct usb_request *cdnsp_gadget_ep_alloc_request(struct usb_ep *ep,
1062							 gfp_t gfp_flags)
1063{
1064	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1065	struct cdnsp_request *preq;
1066
1067	preq = kzalloc(sizeof(*preq), gfp_flags);
1068	if (!preq)
1069		return NULL;
1070
1071	preq->epnum = pep->number;
1072	preq->pep = pep;
1073
1074	trace_cdnsp_alloc_request(preq);
1075
1076	return &preq->request;
1077}
1078
1079static void cdnsp_gadget_ep_free_request(struct usb_ep *ep,
1080					 struct usb_request *request)
1081{
1082	struct cdnsp_request *preq = to_cdnsp_request(request);
1083
1084	trace_cdnsp_free_request(preq);
1085	kfree(preq);
1086}
1087
1088static int cdnsp_gadget_ep_queue(struct usb_ep *ep,
1089				 struct usb_request *request,
1090				 gfp_t gfp_flags)
1091{
1092	struct cdnsp_request *preq;
1093	struct cdnsp_device *pdev;
1094	struct cdnsp_ep *pep;
1095	unsigned long flags;
1096	int ret;
1097
1098	if (!request || !ep)
1099		return -EINVAL;
1100
1101	pep = to_cdnsp_ep(ep);
1102	pdev = pep->pdev;
1103
1104	if (!(pep->ep_state & EP_ENABLED)) {
1105		dev_err(pdev->dev, "%s: can't queue to disabled endpoint\n",
1106			pep->name);
1107		return -EINVAL;
1108	}
1109
1110	preq = to_cdnsp_request(request);
1111	spin_lock_irqsave(&pdev->lock, flags);
1112	ret = cdnsp_ep_enqueue(pep, preq);
1113	spin_unlock_irqrestore(&pdev->lock, flags);
1114
1115	return ret;
1116}
1117
1118static int cdnsp_gadget_ep_dequeue(struct usb_ep *ep,
1119				   struct usb_request *request)
1120{
1121	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1122	struct cdnsp_device *pdev = pep->pdev;
1123	unsigned long flags;
1124	int ret;
1125
1126	if (request->status != -EINPROGRESS)
1127		return 0;
1128
1129	if (!pep->endpoint.desc) {
1130		dev_err(pdev->dev,
1131			"%s: can't dequeue to disabled endpoint\n",
1132			pep->name);
1133		return -ESHUTDOWN;
1134	}
1135
1136	/* Requests has been dequeued during disabling endpoint. */
1137	if (!(pep->ep_state & EP_ENABLED))
1138		return 0;
1139
1140	spin_lock_irqsave(&pdev->lock, flags);
1141	ret = cdnsp_ep_dequeue(pep, to_cdnsp_request(request));
1142	spin_unlock_irqrestore(&pdev->lock, flags);
1143
1144	return ret;
1145}
1146
1147static int cdnsp_gadget_ep_set_halt(struct usb_ep *ep, int value)
1148{
1149	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1150	struct cdnsp_device *pdev = pep->pdev;
1151	struct cdnsp_request *preq;
1152	unsigned long flags;
1153	int ret;
1154
1155	spin_lock_irqsave(&pdev->lock, flags);
1156
1157	preq = next_request(&pep->pending_list);
1158	if (value) {
1159		if (preq) {
1160			trace_cdnsp_ep_busy_try_halt_again(pep, 0);
1161			ret = -EAGAIN;
1162			goto done;
1163		}
1164	}
1165
1166	ret = cdnsp_halt_endpoint(pdev, pep, value);
1167
1168done:
1169	spin_unlock_irqrestore(&pdev->lock, flags);
1170	return ret;
1171}
1172
1173static int cdnsp_gadget_ep_set_wedge(struct usb_ep *ep)
1174{
1175	struct cdnsp_ep *pep = to_cdnsp_ep(ep);
1176	struct cdnsp_device *pdev = pep->pdev;
1177	unsigned long flags;
1178	int ret;
1179
1180	spin_lock_irqsave(&pdev->lock, flags);
1181	pep->ep_state |= EP_WEDGE;
1182	ret = cdnsp_halt_endpoint(pdev, pep, 1);
1183	spin_unlock_irqrestore(&pdev->lock, flags);
1184
1185	return ret;
1186}
1187
1188static const struct usb_ep_ops cdnsp_gadget_ep0_ops = {
1189	.enable		= cdnsp_gadget_ep_enable,
1190	.disable	= cdnsp_gadget_ep_disable,
1191	.alloc_request	= cdnsp_gadget_ep_alloc_request,
1192	.free_request	= cdnsp_gadget_ep_free_request,
1193	.queue		= cdnsp_gadget_ep_queue,
1194	.dequeue	= cdnsp_gadget_ep_dequeue,
1195	.set_halt	= cdnsp_gadget_ep_set_halt,
1196	.set_wedge	= cdnsp_gadget_ep_set_wedge,
1197};
1198
1199static const struct usb_ep_ops cdnsp_gadget_ep_ops = {
1200	.enable		= cdnsp_gadget_ep_enable,
1201	.disable	= cdnsp_gadget_ep_disable,
1202	.alloc_request	= cdnsp_gadget_ep_alloc_request,
1203	.free_request	= cdnsp_gadget_ep_free_request,
1204	.queue		= cdnsp_gadget_ep_queue,
1205	.dequeue	= cdnsp_gadget_ep_dequeue,
1206	.set_halt	= cdnsp_gadget_ep_set_halt,
1207	.set_wedge	= cdnsp_gadget_ep_set_wedge,
1208};
1209
1210void cdnsp_gadget_giveback(struct cdnsp_ep *pep,
1211			   struct cdnsp_request *preq,
1212			   int status)
1213{
1214	struct cdnsp_device *pdev = pep->pdev;
1215
1216	list_del(&preq->list);
1217
1218	if (preq->request.status == -EINPROGRESS)
1219		preq->request.status = status;
1220
1221	usb_gadget_unmap_request_by_dev(pdev->dev, &preq->request,
1222					preq->direction);
1223
1224	trace_cdnsp_request_giveback(preq);
1225
1226	if (preq != &pdev->ep0_preq) {
1227		spin_unlock(&pdev->lock);
1228		usb_gadget_giveback_request(&pep->endpoint, &preq->request);
1229		spin_lock(&pdev->lock);
1230	}
1231}
1232
1233static struct usb_endpoint_descriptor cdnsp_gadget_ep0_desc = {
1234	.bLength =		USB_DT_ENDPOINT_SIZE,
1235	.bDescriptorType =	USB_DT_ENDPOINT,
1236	.bmAttributes =		USB_ENDPOINT_XFER_CONTROL,
1237};
1238
1239static int cdnsp_run(struct cdnsp_device *pdev,
1240		     enum usb_device_speed speed)
1241{
1242	u32 fs_speed = 0;
1243	u32 temp;
1244	int ret;
1245
1246	temp = readl(&pdev->ir_set->irq_control);
1247	temp &= ~IMOD_INTERVAL_MASK;
1248	temp |= ((IMOD_DEFAULT_INTERVAL / 250) & IMOD_INTERVAL_MASK);
1249	writel(temp, &pdev->ir_set->irq_control);
1250
1251	temp = readl(&pdev->port3x_regs->mode_addr);
1252
1253	switch (speed) {
1254	case USB_SPEED_SUPER_PLUS:
1255		temp |= CFG_3XPORT_SSP_SUPPORT;
1256		break;
1257	case USB_SPEED_SUPER:
1258		temp &= ~CFG_3XPORT_SSP_SUPPORT;
1259		break;
1260	case USB_SPEED_HIGH:
1261		break;
1262	case USB_SPEED_FULL:
1263		fs_speed = PORT_REG6_FORCE_FS;
1264		break;
1265	default:
1266		dev_err(pdev->dev, "invalid maximum_speed parameter %d\n",
1267			speed);
1268		fallthrough;
1269	case USB_SPEED_UNKNOWN:
1270		/* Default to superspeed. */
1271		speed = USB_SPEED_SUPER;
1272		break;
1273	}
1274
1275	if (speed >= USB_SPEED_SUPER) {
1276		writel(temp, &pdev->port3x_regs->mode_addr);
1277		cdnsp_set_link_state(pdev, &pdev->usb3_port.regs->portsc,
1278				     XDEV_RXDETECT);
1279	} else {
1280		cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1281	}
1282
1283	cdnsp_set_link_state(pdev, &pdev->usb2_port.regs->portsc,
1284			     XDEV_RXDETECT);
1285
1286	cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1287
1288	writel(PORT_REG6_L1_L0_HW_EN | fs_speed, &pdev->port20_regs->port_reg6);
1289
1290	ret = cdnsp_start(pdev);
1291	if (ret) {
1292		ret = -ENODEV;
1293		goto err;
1294	}
1295
1296	temp = readl(&pdev->op_regs->command);
1297	temp |= (CMD_INTE);
1298	writel(temp, &pdev->op_regs->command);
1299
1300	temp = readl(&pdev->ir_set->irq_pending);
1301	writel(IMAN_IE_SET(temp), &pdev->ir_set->irq_pending);
1302
1303	trace_cdnsp_init("Controller ready to work");
1304	return 0;
1305err:
1306	cdnsp_halt(pdev);
1307	return ret;
1308}
1309
1310static int cdnsp_gadget_udc_start(struct usb_gadget *g,
1311				  struct usb_gadget_driver *driver)
1312{
1313	enum usb_device_speed max_speed = driver->max_speed;
1314	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1315	unsigned long flags;
1316	int ret;
1317
1318	spin_lock_irqsave(&pdev->lock, flags);
1319	pdev->gadget_driver = driver;
1320
1321	/* limit speed if necessary */
1322	max_speed = min(driver->max_speed, g->max_speed);
1323	ret = cdnsp_run(pdev, max_speed);
1324
1325	spin_unlock_irqrestore(&pdev->lock, flags);
1326
1327	return ret;
1328}
1329
1330/*
1331 * Update Event Ring Dequeue Pointer:
1332 * - When all events have finished
1333 * - To avoid "Event Ring Full Error" condition
1334 */
1335void cdnsp_update_erst_dequeue(struct cdnsp_device *pdev,
1336			       union cdnsp_trb *event_ring_deq,
1337			       u8 clear_ehb)
1338{
1339	u64 temp_64;
1340	dma_addr_t deq;
1341
1342	temp_64 = cdnsp_read_64(&pdev->ir_set->erst_dequeue);
1343
1344	/* If necessary, update the HW's version of the event ring deq ptr. */
1345	if (event_ring_deq != pdev->event_ring->dequeue) {
1346		deq = cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1347					    pdev->event_ring->dequeue);
1348		temp_64 &= ERST_PTR_MASK;
1349		temp_64 |= ((u64)deq & (u64)~ERST_PTR_MASK);
1350	}
1351
1352	/* Clear the event handler busy flag (RW1C). */
1353	if (clear_ehb)
1354		temp_64 |= ERST_EHB;
1355	else
1356		temp_64 &= ~ERST_EHB;
1357
1358	cdnsp_write_64(temp_64, &pdev->ir_set->erst_dequeue);
1359}
1360
1361static void cdnsp_clear_cmd_ring(struct cdnsp_device *pdev)
1362{
1363	struct cdnsp_segment *seg;
1364	u64 val_64;
1365	int i;
1366
1367	cdnsp_initialize_ring_info(pdev->cmd_ring);
1368
1369	seg = pdev->cmd_ring->first_seg;
1370	for (i = 0; i < pdev->cmd_ring->num_segs; i++) {
1371		memset(seg->trbs, 0,
1372		       sizeof(union cdnsp_trb) * (TRBS_PER_SEGMENT - 1));
1373		seg = seg->next;
1374	}
1375
1376	/* Set the address in the Command Ring Control register. */
1377	val_64 = cdnsp_read_64(&pdev->op_regs->cmd_ring);
1378	val_64 = (val_64 & (u64)CMD_RING_RSVD_BITS) |
1379		 (pdev->cmd_ring->first_seg->dma & (u64)~CMD_RING_RSVD_BITS) |
1380		 pdev->cmd_ring->cycle_state;
1381	cdnsp_write_64(val_64, &pdev->op_regs->cmd_ring);
1382}
1383
1384static void cdnsp_consume_all_events(struct cdnsp_device *pdev)
1385{
1386	struct cdnsp_segment *event_deq_seg;
1387	union cdnsp_trb *event_ring_deq;
1388	union cdnsp_trb *event;
1389	u32 cycle_bit;
1390
1391	event_ring_deq = pdev->event_ring->dequeue;
1392	event_deq_seg = pdev->event_ring->deq_seg;
1393	event = pdev->event_ring->dequeue;
1394
1395	/* Update ring dequeue pointer. */
1396	while (1) {
1397		cycle_bit = (le32_to_cpu(event->event_cmd.flags) & TRB_CYCLE);
1398
1399		/* Does the controller or driver own the TRB? */
1400		if (cycle_bit != pdev->event_ring->cycle_state)
1401			break;
1402
1403		cdnsp_inc_deq(pdev, pdev->event_ring);
1404
1405		if (!cdnsp_last_trb_on_seg(event_deq_seg, event)) {
1406			event++;
1407			continue;
1408		}
1409
1410		if (cdnsp_last_trb_on_ring(pdev->event_ring, event_deq_seg,
1411					   event))
1412			cycle_bit ^= 1;
1413
1414		event_deq_seg = event_deq_seg->next;
1415		event = event_deq_seg->trbs;
1416	}
1417
1418	cdnsp_update_erst_dequeue(pdev,  event_ring_deq, 1);
1419}
1420
1421static void cdnsp_stop(struct cdnsp_device *pdev)
1422{
1423	u32 temp;
1424
1425	/* Remove internally queued request for ep0. */
1426	if (!list_empty(&pdev->eps[0].pending_list)) {
1427		struct cdnsp_request *req;
1428
1429		req = next_request(&pdev->eps[0].pending_list);
1430		if (req == &pdev->ep0_preq)
1431			cdnsp_ep_dequeue(&pdev->eps[0], req);
1432	}
1433
1434	cdnsp_disable_port(pdev, &pdev->usb2_port.regs->portsc);
1435	cdnsp_disable_port(pdev, &pdev->usb3_port.regs->portsc);
1436	cdnsp_disable_slot(pdev);
1437	cdnsp_halt(pdev);
1438
1439	temp = readl(&pdev->op_regs->status);
1440	writel((temp & ~0x1fff) | STS_EINT, &pdev->op_regs->status);
1441	temp = readl(&pdev->ir_set->irq_pending);
1442	writel(IMAN_IE_CLEAR(temp), &pdev->ir_set->irq_pending);
1443
1444	cdnsp_clear_port_change_bit(pdev, &pdev->usb2_port.regs->portsc);
1445	cdnsp_clear_port_change_bit(pdev, &pdev->usb3_port.regs->portsc);
1446
1447	/* Clear interrupt line */
1448	temp = readl(&pdev->ir_set->irq_pending);
1449	temp |= IMAN_IP;
1450	writel(temp, &pdev->ir_set->irq_pending);
1451
1452	cdnsp_consume_all_events(pdev);
1453	cdnsp_clear_cmd_ring(pdev);
1454
1455	trace_cdnsp_exit("Controller stopped.");
1456}
1457
1458/*
1459 * Stop controller.
1460 * This function is called by the gadget core when the driver is removed.
1461 * Disable slot, disable IRQs, and quiesce the controller.
1462 */
1463static int cdnsp_gadget_udc_stop(struct usb_gadget *g)
1464{
1465	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1466	unsigned long flags;
1467
1468	spin_lock_irqsave(&pdev->lock, flags);
1469	cdnsp_stop(pdev);
1470	pdev->gadget_driver = NULL;
1471	spin_unlock_irqrestore(&pdev->lock, flags);
1472
1473	return 0;
1474}
1475
1476static int cdnsp_gadget_get_frame(struct usb_gadget *g)
1477{
1478	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1479
1480	return cdnsp_get_frame(pdev);
1481}
1482
1483static void __cdnsp_gadget_wakeup(struct cdnsp_device *pdev)
1484{
1485	struct cdnsp_port_regs __iomem *port_regs;
1486	u32 portpm, portsc;
1487
1488	port_regs = pdev->active_port->regs;
1489	portsc = readl(&port_regs->portsc) & PORT_PLS_MASK;
1490
1491	/* Remote wakeup feature is not enabled by host. */
1492	if (pdev->gadget.speed < USB_SPEED_SUPER && portsc == XDEV_U2) {
1493		portpm = readl(&port_regs->portpmsc);
1494
1495		if (!(portpm & PORT_RWE))
1496			return;
1497	}
1498
1499	if (portsc == XDEV_U3 && !pdev->may_wakeup)
1500		return;
1501
1502	cdnsp_set_link_state(pdev, &port_regs->portsc, XDEV_U0);
1503
1504	pdev->cdnsp_state |= CDNSP_WAKEUP_PENDING;
1505}
1506
1507static int cdnsp_gadget_wakeup(struct usb_gadget *g)
1508{
1509	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1510	unsigned long flags;
1511
1512	spin_lock_irqsave(&pdev->lock, flags);
1513	__cdnsp_gadget_wakeup(pdev);
1514	spin_unlock_irqrestore(&pdev->lock, flags);
1515
1516	return 0;
1517}
1518
1519static int cdnsp_gadget_set_selfpowered(struct usb_gadget *g,
1520					int is_selfpowered)
1521{
1522	struct cdnsp_device *pdev = gadget_to_cdnsp(g);
1523	unsigned long flags;
1524
1525	spin_lock_irqsave(&pdev->lock, flags);
1526	g->is_selfpowered = !!is_selfpowered;
1527	spin_unlock_irqrestore(&pdev->lock, flags);
1528
1529	return 0;
1530}
1531
1532static int cdnsp_gadget_pullup(struct usb_gadget *gadget, int is_on)
1533{
1534	struct cdnsp_device *pdev = gadget_to_cdnsp(gadget);
1535	struct cdns *cdns = dev_get_drvdata(pdev->dev);
1536	unsigned long flags;
1537
1538	trace_cdnsp_pullup(is_on);
1539
1540	/*
1541	 * Disable events handling while controller is being
1542	 * enabled/disabled.
1543	 */
1544	disable_irq(cdns->dev_irq);
1545	spin_lock_irqsave(&pdev->lock, flags);
1546
1547	if (!is_on) {
1548		cdnsp_reset_device(pdev);
1549		cdns_clear_vbus(cdns);
1550	} else {
1551		cdns_set_vbus(cdns);
1552	}
1553
1554	spin_unlock_irqrestore(&pdev->lock, flags);
1555	enable_irq(cdns->dev_irq);
1556
1557	return 0;
1558}
1559
1560static const struct usb_gadget_ops cdnsp_gadget_ops = {
1561	.get_frame		= cdnsp_gadget_get_frame,
1562	.wakeup			= cdnsp_gadget_wakeup,
1563	.set_selfpowered	= cdnsp_gadget_set_selfpowered,
1564	.pullup			= cdnsp_gadget_pullup,
1565	.udc_start		= cdnsp_gadget_udc_start,
1566	.udc_stop		= cdnsp_gadget_udc_stop,
1567};
1568
1569static void cdnsp_get_ep_buffering(struct cdnsp_device *pdev,
1570				   struct cdnsp_ep *pep)
1571{
1572	void __iomem *reg = &pdev->cap_regs->hc_capbase;
1573	int endpoints;
1574
1575	reg += cdnsp_find_next_ext_cap(reg, 0, XBUF_CAP_ID);
1576
1577	if (!pep->direction) {
1578		pep->buffering = readl(reg + XBUF_RX_TAG_MASK_0_OFFSET);
1579		pep->buffering_period = readl(reg + XBUF_RX_TAG_MASK_1_OFFSET);
1580		pep->buffering = (pep->buffering + 1) / 2;
1581		pep->buffering_period = (pep->buffering_period + 1) / 2;
1582		return;
1583	}
1584
1585	endpoints = HCS_ENDPOINTS(pdev->hcs_params1) / 2;
1586
1587	/* Set to XBUF_TX_TAG_MASK_0 register. */
1588	reg += XBUF_TX_CMD_OFFSET + (endpoints * 2 + 2) * sizeof(u32);
1589	/* Set reg to XBUF_TX_TAG_MASK_N related with this endpoint. */
1590	reg += pep->number * sizeof(u32) * 2;
1591
1592	pep->buffering = (readl(reg) + 1) / 2;
1593	pep->buffering_period = pep->buffering;
1594}
1595
1596static int cdnsp_gadget_init_endpoints(struct cdnsp_device *pdev)
1597{
1598	int max_streams = HCC_MAX_PSA(pdev->hcc_params);
1599	struct cdnsp_ep *pep;
1600	int i;
1601
1602	INIT_LIST_HEAD(&pdev->gadget.ep_list);
1603
1604	if (max_streams < STREAM_LOG_STREAMS) {
1605		dev_err(pdev->dev, "Stream size %d not supported\n",
1606			max_streams);
1607		return -EINVAL;
1608	}
1609
1610	max_streams = STREAM_LOG_STREAMS;
1611
1612	for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1613		bool direction = !(i & 1); /* Start from OUT endpoint. */
1614		u8 epnum = ((i + 1) >> 1);
1615
1616		if (!CDNSP_IF_EP_EXIST(pdev, epnum, direction))
1617			continue;
1618
1619		pep = &pdev->eps[i];
1620		pep->pdev = pdev;
1621		pep->number = epnum;
1622		pep->direction = direction; /* 0 for OUT, 1 for IN. */
1623
1624		/*
1625		 * Ep0 is bidirectional, so ep0in and ep0out are represented by
1626		 * pdev->eps[0]
1627		 */
1628		if (epnum == 0) {
1629			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1630				 epnum, "BiDir");
1631
1632			pep->idx = 0;
1633			usb_ep_set_maxpacket_limit(&pep->endpoint, 512);
1634			pep->endpoint.maxburst = 1;
1635			pep->endpoint.ops = &cdnsp_gadget_ep0_ops;
1636			pep->endpoint.desc = &cdnsp_gadget_ep0_desc;
1637			pep->endpoint.comp_desc = NULL;
1638			pep->endpoint.caps.type_control = true;
1639			pep->endpoint.caps.dir_in = true;
1640			pep->endpoint.caps.dir_out = true;
1641
1642			pdev->ep0_preq.epnum = pep->number;
1643			pdev->ep0_preq.pep = pep;
1644			pdev->gadget.ep0 = &pep->endpoint;
1645		} else {
1646			snprintf(pep->name, sizeof(pep->name), "ep%d%s",
1647				 epnum, (pep->direction) ? "in" : "out");
1648
1649			pep->idx =  (epnum * 2 + (direction ? 1 : 0)) - 1;
1650			usb_ep_set_maxpacket_limit(&pep->endpoint, 1024);
1651
1652			pep->endpoint.max_streams = max_streams;
1653			pep->endpoint.ops = &cdnsp_gadget_ep_ops;
1654			list_add_tail(&pep->endpoint.ep_list,
1655				      &pdev->gadget.ep_list);
1656
1657			pep->endpoint.caps.type_iso = true;
1658			pep->endpoint.caps.type_bulk = true;
1659			pep->endpoint.caps.type_int = true;
1660
1661			pep->endpoint.caps.dir_in = direction;
1662			pep->endpoint.caps.dir_out = !direction;
1663		}
1664
1665		pep->endpoint.name = pep->name;
1666		pep->in_ctx = cdnsp_get_ep_ctx(&pdev->in_ctx, pep->idx);
1667		pep->out_ctx = cdnsp_get_ep_ctx(&pdev->out_ctx, pep->idx);
1668		cdnsp_get_ep_buffering(pdev, pep);
1669
1670		dev_dbg(pdev->dev, "Init %s, MPS: %04x SupType: "
1671			"CTRL: %s, INT: %s, BULK: %s, ISOC %s, "
1672			"SupDir IN: %s, OUT: %s\n",
1673			pep->name, 1024,
1674			(pep->endpoint.caps.type_control) ? "yes" : "no",
1675			(pep->endpoint.caps.type_int) ? "yes" : "no",
1676			(pep->endpoint.caps.type_bulk) ? "yes" : "no",
1677			(pep->endpoint.caps.type_iso) ? "yes" : "no",
1678			(pep->endpoint.caps.dir_in) ? "yes" : "no",
1679			(pep->endpoint.caps.dir_out) ? "yes" : "no");
1680
1681		INIT_LIST_HEAD(&pep->pending_list);
1682	}
1683
1684	return 0;
1685}
1686
1687static void cdnsp_gadget_free_endpoints(struct cdnsp_device *pdev)
1688{
1689	struct cdnsp_ep *pep;
1690	int i;
1691
1692	for (i = 0; i < CDNSP_ENDPOINTS_NUM; i++) {
1693		pep = &pdev->eps[i];
1694		if (pep->number != 0 && pep->out_ctx)
1695			list_del(&pep->endpoint.ep_list);
1696	}
1697}
1698
1699void cdnsp_disconnect_gadget(struct cdnsp_device *pdev)
1700{
1701	pdev->cdnsp_state |= CDNSP_STATE_DISCONNECT_PENDING;
1702
1703	if (pdev->gadget_driver && pdev->gadget_driver->disconnect) {
1704		spin_unlock(&pdev->lock);
1705		pdev->gadget_driver->disconnect(&pdev->gadget);
1706		spin_lock(&pdev->lock);
1707	}
1708
1709	pdev->gadget.speed = USB_SPEED_UNKNOWN;
1710	usb_gadget_set_state(&pdev->gadget, USB_STATE_NOTATTACHED);
1711
1712	pdev->cdnsp_state &= ~CDNSP_STATE_DISCONNECT_PENDING;
1713}
1714
1715void cdnsp_suspend_gadget(struct cdnsp_device *pdev)
1716{
1717	if (pdev->gadget_driver && pdev->gadget_driver->suspend) {
1718		spin_unlock(&pdev->lock);
1719		pdev->gadget_driver->suspend(&pdev->gadget);
1720		spin_lock(&pdev->lock);
1721	}
1722}
1723
1724void cdnsp_resume_gadget(struct cdnsp_device *pdev)
1725{
1726	if (pdev->gadget_driver && pdev->gadget_driver->resume) {
1727		spin_unlock(&pdev->lock);
1728		pdev->gadget_driver->resume(&pdev->gadget);
1729		spin_lock(&pdev->lock);
1730	}
1731}
1732
1733void cdnsp_irq_reset(struct cdnsp_device *pdev)
1734{
1735	struct cdnsp_port_regs __iomem *port_regs;
1736
1737	cdnsp_reset_device(pdev);
1738
1739	port_regs = pdev->active_port->regs;
1740	pdev->gadget.speed = cdnsp_port_speed(readl(port_regs));
1741
1742	spin_unlock(&pdev->lock);
1743	usb_gadget_udc_reset(&pdev->gadget, pdev->gadget_driver);
1744	spin_lock(&pdev->lock);
1745
1746	switch (pdev->gadget.speed) {
1747	case USB_SPEED_SUPER_PLUS:
1748	case USB_SPEED_SUPER:
1749		cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1750		pdev->gadget.ep0->maxpacket = 512;
1751		break;
1752	case USB_SPEED_HIGH:
1753	case USB_SPEED_FULL:
1754		cdnsp_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
1755		pdev->gadget.ep0->maxpacket = 64;
1756		break;
1757	default:
1758		/* Low speed is not supported. */
1759		dev_err(pdev->dev, "Unknown device speed\n");
1760		break;
1761	}
1762
1763	cdnsp_clear_chicken_bits_2(pdev, CHICKEN_XDMA_2_TP_CACHE_DIS);
1764	cdnsp_setup_device(pdev, SETUP_CONTEXT_ONLY);
1765	usb_gadget_set_state(&pdev->gadget, USB_STATE_DEFAULT);
1766}
1767
1768static void cdnsp_get_rev_cap(struct cdnsp_device *pdev)
1769{
1770	void __iomem *reg = &pdev->cap_regs->hc_capbase;
1771
1772	reg += cdnsp_find_next_ext_cap(reg, 0, RTL_REV_CAP);
1773	pdev->rev_cap  = reg;
1774
1775	dev_info(pdev->dev, "Rev: %08x/%08x, eps: %08x, buff: %08x/%08x\n",
1776		 readl(&pdev->rev_cap->ctrl_revision),
1777		 readl(&pdev->rev_cap->rtl_revision),
1778		 readl(&pdev->rev_cap->ep_supported),
1779		 readl(&pdev->rev_cap->rx_buff_size),
1780		 readl(&pdev->rev_cap->tx_buff_size));
1781}
1782
1783static int cdnsp_gen_setup(struct cdnsp_device *pdev)
1784{
1785	int ret;
1786	u32 reg;
1787
1788	pdev->cap_regs = pdev->regs;
1789	pdev->op_regs = pdev->regs +
1790		HC_LENGTH(readl(&pdev->cap_regs->hc_capbase));
1791	pdev->run_regs = pdev->regs +
1792		(readl(&pdev->cap_regs->run_regs_off) & RTSOFF_MASK);
1793
1794	/* Cache read-only capability registers */
1795	pdev->hcs_params1 = readl(&pdev->cap_regs->hcs_params1);
1796	pdev->hcc_params = readl(&pdev->cap_regs->hc_capbase);
1797	pdev->hci_version = HC_VERSION(pdev->hcc_params);
1798	pdev->hcc_params = readl(&pdev->cap_regs->hcc_params);
1799
1800	cdnsp_get_rev_cap(pdev);
1801
1802	/* Make sure the Device Controller is halted. */
1803	ret = cdnsp_halt(pdev);
1804	if (ret)
1805		return ret;
1806
1807	/* Reset the internal controller memory state and registers. */
1808	ret = cdnsp_reset(pdev);
1809	if (ret)
1810		return ret;
1811
1812	/*
1813	 * Set dma_mask and coherent_dma_mask to 64-bits,
1814	 * if controller supports 64-bit addressing.
1815	 */
1816	if (HCC_64BIT_ADDR(pdev->hcc_params) &&
1817	    !dma_set_mask(pdev->dev, DMA_BIT_MASK(64))) {
1818		dev_dbg(pdev->dev, "Enabling 64-bit DMA addresses.\n");
1819		dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(64));
1820	} else {
1821		/*
1822		 * This is to avoid error in cases where a 32-bit USB
1823		 * controller is used on a 64-bit capable system.
1824		 */
1825		ret = dma_set_mask(pdev->dev, DMA_BIT_MASK(32));
1826		if (ret)
1827			return ret;
1828
1829		dev_dbg(pdev->dev, "Enabling 32-bit DMA addresses.\n");
1830		dma_set_coherent_mask(pdev->dev, DMA_BIT_MASK(32));
1831	}
1832
1833	spin_lock_init(&pdev->lock);
1834
1835	ret = cdnsp_mem_init(pdev);
1836	if (ret)
1837		return ret;
1838
1839	/*
1840	 * Software workaround for U1: after transition
1841	 * to U1 the controller starts gating clock, and in some cases,
1842	 * it causes that controller stack.
1843	 */
1844	reg = readl(&pdev->port3x_regs->mode_2);
1845	reg &= ~CFG_3XPORT_U1_PIPE_CLK_GATE_EN;
1846	writel(reg, &pdev->port3x_regs->mode_2);
1847
1848	return 0;
1849}
1850
1851static int __cdnsp_gadget_init(struct cdns *cdns)
1852{
1853	struct cdnsp_device *pdev;
1854	u32 max_speed;
1855	int ret = -ENOMEM;
1856
1857	cdns_drd_gadget_on(cdns);
1858
1859	pdev = kzalloc(sizeof(*pdev), GFP_KERNEL);
1860	if (!pdev)
1861		return -ENOMEM;
1862
1863	pm_runtime_get_sync(cdns->dev);
1864
1865	cdns->gadget_dev = pdev;
1866	pdev->dev = cdns->dev;
1867	pdev->regs = cdns->dev_regs;
1868	max_speed = usb_get_maximum_speed(cdns->dev);
1869
1870	switch (max_speed) {
1871	case USB_SPEED_FULL:
1872	case USB_SPEED_HIGH:
1873	case USB_SPEED_SUPER:
1874	case USB_SPEED_SUPER_PLUS:
1875		break;
1876	default:
1877		dev_err(cdns->dev, "invalid speed parameter %d\n", max_speed);
1878		fallthrough;
1879	case USB_SPEED_UNKNOWN:
1880		/* Default to SSP */
1881		max_speed = USB_SPEED_SUPER_PLUS;
1882		break;
1883	}
1884
1885	pdev->gadget.ops = &cdnsp_gadget_ops;
1886	pdev->gadget.name = "cdnsp-gadget";
1887	pdev->gadget.speed = USB_SPEED_UNKNOWN;
1888	pdev->gadget.sg_supported = 1;
1889	pdev->gadget.max_speed = max_speed;
1890	pdev->gadget.lpm_capable = 1;
1891
1892	pdev->setup_buf = kzalloc(CDNSP_EP0_SETUP_SIZE, GFP_KERNEL);
1893	if (!pdev->setup_buf)
1894		goto free_pdev;
1895
1896	/*
1897	 * Controller supports not aligned buffer but it should improve
1898	 * performance.
1899	 */
1900	pdev->gadget.quirk_ep_out_aligned_size = true;
1901
1902	ret = cdnsp_gen_setup(pdev);
1903	if (ret) {
1904		dev_err(pdev->dev, "Generic initialization failed %d\n", ret);
1905		goto free_setup;
1906	}
1907
1908	ret = cdnsp_gadget_init_endpoints(pdev);
1909	if (ret) {
1910		dev_err(pdev->dev, "failed to initialize endpoints\n");
1911		goto halt_pdev;
1912	}
1913
1914	ret = usb_add_gadget_udc(pdev->dev, &pdev->gadget);
1915	if (ret) {
1916		dev_err(pdev->dev, "failed to register udc\n");
1917		goto free_endpoints;
1918	}
1919
1920	ret = devm_request_threaded_irq(pdev->dev, cdns->dev_irq,
1921					cdnsp_irq_handler,
1922					cdnsp_thread_irq_handler, IRQF_SHARED,
1923					dev_name(pdev->dev), pdev);
1924	if (ret)
1925		goto del_gadget;
1926
1927	return 0;
1928
1929del_gadget:
1930	usb_del_gadget_udc(&pdev->gadget);
1931free_endpoints:
1932	cdnsp_gadget_free_endpoints(pdev);
1933halt_pdev:
1934	cdnsp_halt(pdev);
1935	cdnsp_reset(pdev);
1936	cdnsp_mem_cleanup(pdev);
1937free_setup:
1938	kfree(pdev->setup_buf);
1939free_pdev:
1940	kfree(pdev);
1941
1942	return ret;
1943}
1944
1945static void cdnsp_gadget_exit(struct cdns *cdns)
1946{
1947	struct cdnsp_device *pdev = cdns->gadget_dev;
1948
1949	devm_free_irq(pdev->dev, cdns->dev_irq, pdev);
1950	pm_runtime_mark_last_busy(cdns->dev);
1951	pm_runtime_put_autosuspend(cdns->dev);
1952	usb_del_gadget_udc(&pdev->gadget);
1953	cdnsp_gadget_free_endpoints(pdev);
1954	cdnsp_mem_cleanup(pdev);
1955	kfree(pdev);
1956	cdns->gadget_dev = NULL;
1957	cdns_drd_gadget_off(cdns);
1958}
1959
1960static int cdnsp_gadget_suspend(struct cdns *cdns, bool do_wakeup)
1961{
1962	struct cdnsp_device *pdev = cdns->gadget_dev;
1963	unsigned long flags;
1964
1965	if (pdev->link_state == XDEV_U3)
1966		return 0;
1967
1968	spin_lock_irqsave(&pdev->lock, flags);
1969	cdnsp_disconnect_gadget(pdev);
1970	cdnsp_stop(pdev);
1971	spin_unlock_irqrestore(&pdev->lock, flags);
1972
1973	return 0;
1974}
1975
1976static int cdnsp_gadget_resume(struct cdns *cdns, bool hibernated)
1977{
1978	struct cdnsp_device *pdev = cdns->gadget_dev;
1979	enum usb_device_speed max_speed;
1980	unsigned long flags;
1981	int ret;
1982
1983	if (!pdev->gadget_driver)
1984		return 0;
1985
1986	spin_lock_irqsave(&pdev->lock, flags);
1987	max_speed = pdev->gadget_driver->max_speed;
1988
1989	/* Limit speed if necessary. */
1990	max_speed = min(max_speed, pdev->gadget.max_speed);
1991
1992	ret = cdnsp_run(pdev, max_speed);
1993
1994	if (pdev->link_state == XDEV_U3)
1995		__cdnsp_gadget_wakeup(pdev);
1996
1997	spin_unlock_irqrestore(&pdev->lock, flags);
1998
1999	return ret;
2000}
2001
2002/**
2003 * cdnsp_gadget_init - initialize device structure
2004 * @cdns: cdnsp instance
2005 *
2006 * This function initializes the gadget.
2007 */
2008int cdnsp_gadget_init(struct cdns *cdns)
2009{
2010	struct cdns_role_driver *rdrv;
2011
2012	rdrv = devm_kzalloc(cdns->dev, sizeof(*rdrv), GFP_KERNEL);
2013	if (!rdrv)
2014		return -ENOMEM;
2015
2016	rdrv->start	= __cdnsp_gadget_init;
2017	rdrv->stop	= cdnsp_gadget_exit;
2018	rdrv->suspend	= cdnsp_gadget_suspend;
2019	rdrv->resume	= cdnsp_gadget_resume;
2020	rdrv->state	= CDNS_ROLE_STATE_INACTIVE;
2021	rdrv->name	= "gadget";
2022	cdns->roles[USB_ROLE_DEVICE] = rdrv;
2023
2024	return 0;
2025}