1// SPDX-License-Identifier: GPL-2.0
   2/*
   3 * Cadence CDNSP DRD Driver.
   4 *
   5 * Copyright (C) 2020 Cadence.
   6 *
   7 * Author: Pawel Laszczak <pawell@cadence.com>
   8 *
   9 * Code based on Linux XHCI driver.
  10 * Origin: Copyright (C) 2008 Intel Corp
  11 */
  12
  13/*
  14 * Ring initialization rules:
  15 * 1. Each segment is initialized to zero, except for link TRBs.
  16 * 2. Ring cycle state = 0. This represents Producer Cycle State (PCS) or
  17 *    Consumer Cycle State (CCS), depending on ring function.
  18 * 3. Enqueue pointer = dequeue pointer = address of first TRB in the segment.
  19 *
  20 * Ring behavior rules:
  21 * 1. A ring is empty if enqueue == dequeue. This means there will always be at
  22 *    least one free TRB in the ring. This is useful if you want to turn that
  23 *    into a link TRB and expand the ring.
  24 * 2. When incrementing an enqueue or dequeue pointer, if the next TRB is a
  25 *    link TRB, then load the pointer with the address in the link TRB. If the
  26 *    link TRB had its toggle bit set, you may need to update the ring cycle
  27 *    state (see cycle bit rules). You may have to do this multiple times
  28 *    until you reach a non-link TRB.
  29 * 3. A ring is full if enqueue++ (for the definition of increment above)
  30 *    equals the dequeue pointer.
  31 *
  32 * Cycle bit rules:
  33 * 1. When a consumer increments a dequeue pointer and encounters a toggle bit
  34 *    in a link TRB, it must toggle the ring cycle state.
  35 * 2. When a producer increments an enqueue pointer and encounters a toggle bit
  36 *    in a link TRB, it must toggle the ring cycle state.
  37 *
  38 * Producer rules:
  39 * 1. Check if ring is full before you enqueue.
  40 * 2. Write the ring cycle state to the cycle bit in the TRB you're enqueuing.
  41 *    Update enqueue pointer between each write (which may update the ring
  42 *    cycle state).
  43 * 3. Notify consumer. If SW is producer, it rings the doorbell for command
  44 *    and endpoint rings. If controller is the producer for the event ring,
  45 *    and it generates an interrupt according to interrupt modulation rules.
  46 *
  47 * Consumer rules:
  48 * 1. Check if TRB belongs to you. If the cycle bit == your ring cycle state,
  49 *    the TRB is owned by the consumer.
  50 * 2. Update dequeue pointer (which may update the ring cycle state) and
  51 *    continue processing TRBs until you reach a TRB which is not owned by you.
  52 * 3. Notify the producer. SW is the consumer for the event ring, and it
  53 *    updates event ring dequeue pointer. Controller is the consumer for the
  54 *    command and endpoint rings; it generates events on the event ring
  55 *    for these.
  56 */
  57
  58#include <linux/scatterlist.h>
  59#include <linux/dma-mapping.h>
  60#include <linux/delay.h>
  61#include <linux/slab.h>
  62#include <linux/irq.h>
  63
  64#include "cdnsp-trace.h"
  65#include "cdnsp-gadget.h"
  66
  67/*
  68 * Returns zero if the TRB isn't in this segment, otherwise it returns the DMA
  69 * address of the TRB.
  70 */
  71dma_addr_t cdnsp_trb_virt_to_dma(struct cdnsp_segment *seg,
  72				 union cdnsp_trb *trb)
  73{
  74	unsigned long segment_offset = trb - seg->trbs;
  75
  76	if (trb < seg->trbs || segment_offset >= TRBS_PER_SEGMENT)
  77		return 0;
  78
  79	return seg->dma + (segment_offset * sizeof(*trb));
  80}
  81
  82static bool cdnsp_trb_is_noop(union cdnsp_trb *trb)
  83{
  84	return TRB_TYPE_NOOP_LE32(trb->generic.field[3]);
  85}
  86
  87static bool cdnsp_trb_is_link(union cdnsp_trb *trb)
  88{
  89	return TRB_TYPE_LINK_LE32(trb->link.control);
  90}
  91
  92bool cdnsp_last_trb_on_seg(struct cdnsp_segment *seg, union cdnsp_trb *trb)
  93{
  94	return trb == &seg->trbs[TRBS_PER_SEGMENT - 1];
  95}
  96
  97bool cdnsp_last_trb_on_ring(struct cdnsp_ring *ring,
  98			    struct cdnsp_segment *seg,
  99			    union cdnsp_trb *trb)
 100{
 101	return cdnsp_last_trb_on_seg(seg, trb) && (seg->next == ring->first_seg);
 102}
 103
 104static bool cdnsp_link_trb_toggles_cycle(union cdnsp_trb *trb)
 105{
 106	return le32_to_cpu(trb->link.control) & LINK_TOGGLE;
 107}
 108
 109static void cdnsp_trb_to_noop(union cdnsp_trb *trb, u32 noop_type)
 110{
 111	if (cdnsp_trb_is_link(trb)) {
 112		/* Unchain chained link TRBs. */
 113		trb->link.control &= cpu_to_le32(~TRB_CHAIN);
 114	} else {
 115		trb->generic.field[0] = 0;
 116		trb->generic.field[1] = 0;
 117		trb->generic.field[2] = 0;
 118		/* Preserve only the cycle bit of this TRB. */
 119		trb->generic.field[3] &= cpu_to_le32(TRB_CYCLE);
 120		trb->generic.field[3] |= cpu_to_le32(TRB_TYPE(noop_type));
 121	}
 122}
 123
 124/*
 125 * Updates trb to point to the next TRB in the ring, and updates seg if the next
 126 * TRB is in a new segment. This does not skip over link TRBs, and it does not
 127 * effect the ring dequeue or enqueue pointers.
 128 */
 129static void cdnsp_next_trb(struct cdnsp_device *pdev,
 130			   struct cdnsp_ring *ring,
 131			   struct cdnsp_segment **seg,
 132			   union cdnsp_trb **trb)
 133{
 134	if (cdnsp_trb_is_link(*trb)) {
 135		*seg = (*seg)->next;
 136		*trb = ((*seg)->trbs);
 137	} else {
 138		(*trb)++;
 139	}
 140}
 141
 142/*
 143 * See Cycle bit rules. SW is the consumer for the event ring only.
 144 * Don't make a ring full of link TRBs. That would be dumb and this would loop.
 145 */
 146void cdnsp_inc_deq(struct cdnsp_device *pdev, struct cdnsp_ring *ring)
 147{
 148	/* event ring doesn't have link trbs, check for last trb. */
 149	if (ring->type == TYPE_EVENT) {
 150		if (!cdnsp_last_trb_on_seg(ring->deq_seg, ring->dequeue)) {
 151			ring->dequeue++;
 152			goto out;
 153		}
 154
 155		if (cdnsp_last_trb_on_ring(ring, ring->deq_seg, ring->dequeue))
 156			ring->cycle_state ^= 1;
 157
 158		ring->deq_seg = ring->deq_seg->next;
 159		ring->dequeue = ring->deq_seg->trbs;
 160		goto out;
 161	}
 162
 163	/* All other rings have link trbs. */
 164	if (!cdnsp_trb_is_link(ring->dequeue)) {
 165		ring->dequeue++;
 166		ring->num_trbs_free++;
 167	}
 168	while (cdnsp_trb_is_link(ring->dequeue)) {
 169		ring->deq_seg = ring->deq_seg->next;
 170		ring->dequeue = ring->deq_seg->trbs;
 171	}
 172out:
 173	trace_cdnsp_inc_deq(ring);
 174}
 175
 176/*
 177 * See Cycle bit rules. SW is the consumer for the event ring only.
 178 * Don't make a ring full of link TRBs. That would be dumb and this would loop.
 179 *
 180 * If we've just enqueued a TRB that is in the middle of a TD (meaning the
 181 * chain bit is set), then set the chain bit in all the following link TRBs.
 182 * If we've enqueued the last TRB in a TD, make sure the following link TRBs
 183 * have their chain bit cleared (so that each Link TRB is a separate TD).
 184 *
 185 * @more_trbs_coming:	Will you enqueue more TRBs before ringing the doorbell.
 186 */
 187static void cdnsp_inc_enq(struct cdnsp_device *pdev,
 188			  struct cdnsp_ring *ring,
 189			  bool more_trbs_coming)
 190{
 191	union cdnsp_trb *next;
 192	u32 chain;
 193
 194	chain = le32_to_cpu(ring->enqueue->generic.field[3]) & TRB_CHAIN;
 195
 196	/* If this is not event ring, there is one less usable TRB. */
 197	if (!cdnsp_trb_is_link(ring->enqueue))
 198		ring->num_trbs_free--;
 199	next = ++(ring->enqueue);
 200
 201	/* Update the dequeue pointer further if that was a link TRB */
 202	while (cdnsp_trb_is_link(next)) {
 203		/*
 204		 * If the caller doesn't plan on enqueuing more TDs before
 205		 * ringing the doorbell, then we don't want to give the link TRB
 206		 * to the hardware just yet. We'll give the link TRB back in
 207		 * cdnsp_prepare_ring() just before we enqueue the TD at the
 208		 * top of the ring.
 209		 */
 210		if (!chain && !more_trbs_coming)
 211			break;
 212
 213		next->link.control &= cpu_to_le32(~TRB_CHAIN);
 214		next->link.control |= cpu_to_le32(chain);
 215
 216		/* Give this link TRB to the hardware */
 217		wmb();
 218		next->link.control ^= cpu_to_le32(TRB_CYCLE);
 219
 220		/* Toggle the cycle bit after the last ring segment. */
 221		if (cdnsp_link_trb_toggles_cycle(next))
 222			ring->cycle_state ^= 1;
 223
 224		ring->enq_seg = ring->enq_seg->next;
 225		ring->enqueue = ring->enq_seg->trbs;
 226		next = ring->enqueue;
 227	}
 228
 229	trace_cdnsp_inc_enq(ring);
 230}
 231
 232/*
 233 * Check to see if there's room to enqueue num_trbs on the ring and make sure
 234 * enqueue pointer will not advance into dequeue segment.
 235 */
 236static bool cdnsp_room_on_ring(struct cdnsp_device *pdev,
 237			       struct cdnsp_ring *ring,
 238			       unsigned int num_trbs)
 239{
 240	int num_trbs_in_deq_seg;
 241
 242	if (ring->num_trbs_free < num_trbs)
 243		return false;
 244
 245	if (ring->type != TYPE_COMMAND && ring->type != TYPE_EVENT) {
 246		num_trbs_in_deq_seg = ring->dequeue - ring->deq_seg->trbs;
 247
 248		if (ring->num_trbs_free < num_trbs + num_trbs_in_deq_seg)
 249			return false;
 250	}
 251
 252	return true;
 253}
 254
 255/*
 256 * Workaround for L1: controller has issue with resuming from L1 after
 257 * setting doorbell for endpoint during L1 state. This function forces
 258 * resume signal in such case.
 259 */
 260static void cdnsp_force_l0_go(struct cdnsp_device *pdev)
 261{
 262	if (pdev->active_port == &pdev->usb2_port && pdev->gadget.lpm_capable)
 263		cdnsp_set_link_state(pdev, &pdev->active_port->regs->portsc, XDEV_U0);
 264}
 265
 266/* Ring the doorbell after placing a command on the ring. */
 267void cdnsp_ring_cmd_db(struct cdnsp_device *pdev)
 268{
 269	writel(DB_VALUE_CMD, &pdev->dba->cmd_db);
 270}
 271
 272/*
 273 * Ring the doorbell after placing a transfer on the ring.
 274 * Returns true if doorbell was set, otherwise false.
 275 */
 276static bool cdnsp_ring_ep_doorbell(struct cdnsp_device *pdev,
 277				   struct cdnsp_ep *pep,
 278				   unsigned int stream_id)
 279{
 280	__le32 __iomem *reg_addr = &pdev->dba->ep_db;
 281	unsigned int ep_state = pep->ep_state;
 282	unsigned int db_value;
 283
 284	/*
 285	 * Don't ring the doorbell for this endpoint if endpoint is halted or
 286	 * disabled.
 287	 */
 288	if (ep_state & EP_HALTED || !(ep_state & EP_ENABLED))
 289		return false;
 290
 291	/* For stream capable endpoints driver can ring doorbell only twice. */
 292	if (pep->ep_state & EP_HAS_STREAMS) {
 293		if (pep->stream_info.drbls_count >= 2)
 294			return false;
 295
 296		pep->stream_info.drbls_count++;
 297	}
 298
 299	pep->ep_state &= ~EP_STOPPED;
 300
 301	if (pep->idx == 0 && pdev->ep0_stage == CDNSP_DATA_STAGE &&
 302	    !pdev->ep0_expect_in)
 303		db_value = DB_VALUE_EP0_OUT(pep->idx, stream_id);
 304	else
 305		db_value = DB_VALUE(pep->idx, stream_id);
 306
 307	trace_cdnsp_tr_drbl(pep, stream_id);
 308
 309	writel(db_value, reg_addr);
 310
 311	cdnsp_force_l0_go(pdev);
 312
 313	/* Doorbell was set. */
 314	return true;
 315}
 316
 317/*
 318 * Get the right ring for the given pep and stream_id.
 319 * If the endpoint supports streams, boundary check the USB request's stream ID.
 320 * If the endpoint doesn't support streams, return the singular endpoint ring.
 321 */
 322static struct cdnsp_ring *cdnsp_get_transfer_ring(struct cdnsp_device *pdev,
 323						  struct cdnsp_ep *pep,
 324						  unsigned int stream_id)
 325{
 326	if (!(pep->ep_state & EP_HAS_STREAMS))
 327		return pep->ring;
 328
 329	if (stream_id == 0 || stream_id >= pep->stream_info.num_streams) {
 330		dev_err(pdev->dev, "ERR: %s ring doesn't exist for SID: %d.\n",
 331			pep->name, stream_id);
 332		return NULL;
 333	}
 334
 335	return pep->stream_info.stream_rings[stream_id];
 336}
 337
 338static struct cdnsp_ring *
 339	cdnsp_request_to_transfer_ring(struct cdnsp_device *pdev,
 340				       struct cdnsp_request *preq)
 341{
 342	return cdnsp_get_transfer_ring(pdev, preq->pep,
 343				       preq->request.stream_id);
 344}
 345
 346/* Ring the doorbell for any rings with pending requests. */
 347void cdnsp_ring_doorbell_for_active_rings(struct cdnsp_device *pdev,
 348					  struct cdnsp_ep *pep)
 349{
 350	struct cdnsp_stream_info *stream_info;
 351	unsigned int stream_id;
 352	int ret;
 353
 354	if (pep->ep_state & EP_DIS_IN_RROGRESS)
 355		return;
 356
 357	/* A ring has pending Request if its TD list is not empty. */
 358	if (!(pep->ep_state & EP_HAS_STREAMS) && pep->number) {
 359		if (pep->ring && !list_empty(&pep->ring->td_list))
 360			cdnsp_ring_ep_doorbell(pdev, pep, 0);
 361		return;
 362	}
 363
 364	stream_info = &pep->stream_info;
 365
 366	for (stream_id = 1; stream_id < stream_info->num_streams; stream_id++) {
 367		struct cdnsp_td *td, *td_temp;
 368		struct cdnsp_ring *ep_ring;
 369
 370		if (stream_info->drbls_count >= 2)
 371			return;
 372
 373		ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);
 374		if (!ep_ring)
 375			continue;
 376
 377		if (!ep_ring->stream_active || ep_ring->stream_rejected)
 378			continue;
 379
 380		list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,
 381					 td_list) {
 382			if (td->drbl)
 383				continue;
 384
 385			ret = cdnsp_ring_ep_doorbell(pdev, pep, stream_id);
 386			if (ret)
 387				td->drbl = 1;
 388		}
 389	}
 390}
 391
 392/*
 393 * Get the hw dequeue pointer controller stopped on, either directly from the
 394 * endpoint context, or if streams are in use from the stream context.
 395 * The returned hw_dequeue contains the lowest four bits with cycle state
 396 * and possible stream context type.
 397 */
 398static u64 cdnsp_get_hw_deq(struct cdnsp_device *pdev,
 399			    unsigned int ep_index,
 400			    unsigned int stream_id)
 401{
 402	struct cdnsp_stream_ctx *st_ctx;
 403	struct cdnsp_ep *pep;
 404
 405	pep = &pdev->eps[stream_id];
 406
 407	if (pep->ep_state & EP_HAS_STREAMS) {
 408		st_ctx = &pep->stream_info.stream_ctx_array[stream_id];
 409		return le64_to_cpu(st_ctx->stream_ring);
 410	}
 411
 412	return le64_to_cpu(pep->out_ctx->deq);
 413}
 414
 415/*
 416 * Move the controller endpoint ring dequeue pointer past cur_td.
 417 * Record the new state of the controller endpoint ring dequeue segment,
 418 * dequeue pointer, and new consumer cycle state in state.
 419 * Update internal representation of the ring's dequeue pointer.
 420 *
 421 * We do this in three jumps:
 422 *  - First we update our new ring state to be the same as when the
 423 *    controller stopped.
 424 *  - Then we traverse the ring to find the segment that contains
 425 *    the last TRB in the TD. We toggle the controller new cycle state
 426 *    when we pass any link TRBs with the toggle cycle bit set.
 427 *  - Finally we move the dequeue state one TRB further, toggling the cycle bit
 428 *    if we've moved it past a link TRB with the toggle cycle bit set.
 429 */
 430static void cdnsp_find_new_dequeue_state(struct cdnsp_device *pdev,
 431					 struct cdnsp_ep *pep,
 432					 unsigned int stream_id,
 433					 struct cdnsp_td *cur_td,
 434					 struct cdnsp_dequeue_state *state)
 435{
 436	bool td_last_trb_found = false;
 437	struct cdnsp_segment *new_seg;
 438	struct cdnsp_ring *ep_ring;
 439	union cdnsp_trb *new_deq;
 440	bool cycle_found = false;
 441	u64 hw_dequeue;
 442
 443	ep_ring = cdnsp_get_transfer_ring(pdev, pep, stream_id);
 444	if (!ep_ring)
 445		return;
 446
 447	/*
 448	 * Dig out the cycle state saved by the controller during the
 449	 * stop endpoint command.
 450	 */
 451	hw_dequeue = cdnsp_get_hw_deq(pdev, pep->idx, stream_id);
 452	new_seg = ep_ring->deq_seg;
 453	new_deq = ep_ring->dequeue;
 454	state->new_cycle_state = hw_dequeue & 0x1;
 455	state->stream_id = stream_id;
 456
 457	/*
 458	 * We want to find the pointer, segment and cycle state of the new trb
 459	 * (the one after current TD's last_trb). We know the cycle state at
 460	 * hw_dequeue, so walk the ring until both hw_dequeue and last_trb are
 461	 * found.
 462	 */
 463	do {
 464		if (!cycle_found && cdnsp_trb_virt_to_dma(new_seg, new_deq)
 465		    == (dma_addr_t)(hw_dequeue & ~0xf)) {
 466			cycle_found = true;
 467
 468			if (td_last_trb_found)
 469				break;
 470		}
 471
 472		if (new_deq == cur_td->last_trb)
 473			td_last_trb_found = true;
 474
 475		if (cycle_found && cdnsp_trb_is_link(new_deq) &&
 476		    cdnsp_link_trb_toggles_cycle(new_deq))
 477			state->new_cycle_state ^= 0x1;
 478
 479		cdnsp_next_trb(pdev, ep_ring, &new_seg, &new_deq);
 480
 481		/* Search wrapped around, bail out. */
 482		if (new_deq == pep->ring->dequeue) {
 483			dev_err(pdev->dev,
 484				"Error: Failed finding new dequeue state\n");
 485			state->new_deq_seg = NULL;
 486			state->new_deq_ptr = NULL;
 487			return;
 488		}
 489
 490	} while (!cycle_found || !td_last_trb_found);
 491
 492	state->new_deq_seg = new_seg;
 493	state->new_deq_ptr = new_deq;
 494
 495	trace_cdnsp_new_deq_state(state);
 496}
 497
 498/*
 499 * flip_cycle means flip the cycle bit of all but the first and last TRB.
 500 * (The last TRB actually points to the ring enqueue pointer, which is not part
 501 * of this TD.) This is used to remove partially enqueued isoc TDs from a ring.
 502 */
 503static void cdnsp_td_to_noop(struct cdnsp_device *pdev,
 504			     struct cdnsp_ring *ep_ring,
 505			     struct cdnsp_td *td,
 506			     bool flip_cycle)
 507{
 508	struct cdnsp_segment *seg = td->start_seg;
 509	union cdnsp_trb *trb = td->first_trb;
 510
 511	while (1) {
 512		cdnsp_trb_to_noop(trb, TRB_TR_NOOP);
 513
 514		/* flip cycle if asked to */
 515		if (flip_cycle && trb != td->first_trb && trb != td->last_trb)
 516			trb->generic.field[3] ^= cpu_to_le32(TRB_CYCLE);
 517
 518		if (trb == td->last_trb)
 519			break;
 520
 521		cdnsp_next_trb(pdev, ep_ring, &seg, &trb);
 522	}
 523}
 524
 525/*
 526 * This TD is defined by the TRBs starting at start_trb in start_seg and ending
 527 * at end_trb, which may be in another segment. If the suspect DMA address is a
 528 * TRB in this TD, this function returns that TRB's segment. Otherwise it
 529 * returns 0.
 530 */
 531static struct cdnsp_segment *cdnsp_trb_in_td(struct cdnsp_device *pdev,
 532					     struct cdnsp_segment *start_seg,
 533					     union cdnsp_trb *start_trb,
 534					     union cdnsp_trb *end_trb,
 535					     dma_addr_t suspect_dma)
 536{
 537	struct cdnsp_segment *cur_seg;
 538	union cdnsp_trb *temp_trb;
 539	dma_addr_t end_seg_dma;
 540	dma_addr_t end_trb_dma;
 541	dma_addr_t start_dma;
 542
 543	start_dma = cdnsp_trb_virt_to_dma(start_seg, start_trb);
 544	cur_seg = start_seg;
 545
 546	do {
 547		if (start_dma == 0)
 548			return NULL;
 549
 550		temp_trb = &cur_seg->trbs[TRBS_PER_SEGMENT - 1];
 551		/* We may get an event for a Link TRB in the middle of a TD */
 552		end_seg_dma = cdnsp_trb_virt_to_dma(cur_seg, temp_trb);
 553		/* If the end TRB isn't in this segment, this is set to 0 */
 554		end_trb_dma = cdnsp_trb_virt_to_dma(cur_seg, end_trb);
 555
 556		trace_cdnsp_looking_trb_in_td(suspect_dma, start_dma,
 557					      end_trb_dma, cur_seg->dma,
 558					      end_seg_dma);
 559
 560		if (end_trb_dma > 0) {
 561			/*
 562			 * The end TRB is in this segment, so suspect should
 563			 * be here
 564			 */
 565			if (start_dma <= end_trb_dma) {
 566				if (suspect_dma >= start_dma &&
 567				    suspect_dma <= end_trb_dma) {
 568					return cur_seg;
 569				}
 570			} else {
 571				/*
 572				 * Case for one segment with a
 573				 * TD wrapped around to the top
 574				 */
 575				if ((suspect_dma >= start_dma &&
 576				     suspect_dma <= end_seg_dma) ||
 577				    (suspect_dma >= cur_seg->dma &&
 578				     suspect_dma <= end_trb_dma)) {
 579					return cur_seg;
 580				}
 581			}
 582
 583			return NULL;
 584		}
 585
 586		/* Might still be somewhere in this segment */
 587		if (suspect_dma >= start_dma && suspect_dma <= end_seg_dma)
 588			return cur_seg;
 589
 590		cur_seg = cur_seg->next;
 591		start_dma = cdnsp_trb_virt_to_dma(cur_seg, &cur_seg->trbs[0]);
 592	} while (cur_seg != start_seg);
 593
 594	return NULL;
 595}
 596
 597static void cdnsp_unmap_td_bounce_buffer(struct cdnsp_device *pdev,
 598					 struct cdnsp_ring *ring,
 599					 struct cdnsp_td *td)
 600{
 601	struct cdnsp_segment *seg = td->bounce_seg;
 602	struct cdnsp_request *preq;
 603	size_t len;
 604
 605	if (!seg)
 606		return;
 607
 608	preq = td->preq;
 609
 610	trace_cdnsp_bounce_unmap(td->preq, seg->bounce_len, seg->bounce_offs,
 611				 seg->bounce_dma, 0);
 612
 613	if (!preq->direction) {
 614		dma_unmap_single(pdev->dev, seg->bounce_dma,
 615				 ring->bounce_buf_len,  DMA_TO_DEVICE);
 616		return;
 617	}
 618
 619	dma_unmap_single(pdev->dev, seg->bounce_dma, ring->bounce_buf_len,
 620			 DMA_FROM_DEVICE);
 621
 622	/* For in transfers we need to copy the data from bounce to sg */
 623	len = sg_pcopy_from_buffer(preq->request.sg, preq->request.num_sgs,
 624				   seg->bounce_buf, seg->bounce_len,
 625				   seg->bounce_offs);
 626	if (len != seg->bounce_len)
 627		dev_warn(pdev->dev, "WARN Wrong bounce buffer read length: %zu != %d\n",
 628			 len, seg->bounce_len);
 629
 630	seg->bounce_len = 0;
 631	seg->bounce_offs = 0;
 632}
 633
 634static int cdnsp_cmd_set_deq(struct cdnsp_device *pdev,
 635			     struct cdnsp_ep *pep,
 636			     struct cdnsp_dequeue_state *deq_state)
 637{
 638	struct cdnsp_ring *ep_ring;
 639	int ret;
 640
 641	if (!deq_state->new_deq_ptr || !deq_state->new_deq_seg) {
 642		cdnsp_ring_doorbell_for_active_rings(pdev, pep);
 643		return 0;
 644	}
 645
 646	cdnsp_queue_new_dequeue_state(pdev, pep, deq_state);
 647	cdnsp_ring_cmd_db(pdev);
 648	ret = cdnsp_wait_for_cmd_compl(pdev);
 649
 650	trace_cdnsp_handle_cmd_set_deq(cdnsp_get_slot_ctx(&pdev->out_ctx));
 651	trace_cdnsp_handle_cmd_set_deq_ep(pep->out_ctx);
 652
 653	/*
 654	 * Update the ring's dequeue segment and dequeue pointer
 655	 * to reflect the new position.
 656	 */
 657	ep_ring = cdnsp_get_transfer_ring(pdev, pep, deq_state->stream_id);
 658
 659	if (cdnsp_trb_is_link(ep_ring->dequeue)) {
 660		ep_ring->deq_seg = ep_ring->deq_seg->next;
 661		ep_ring->dequeue = ep_ring->deq_seg->trbs;
 662	}
 663
 664	while (ep_ring->dequeue != deq_state->new_deq_ptr) {
 665		ep_ring->num_trbs_free++;
 666		ep_ring->dequeue++;
 667
 668		if (cdnsp_trb_is_link(ep_ring->dequeue)) {
 669			if (ep_ring->dequeue == deq_state->new_deq_ptr)
 670				break;
 671
 672			ep_ring->deq_seg = ep_ring->deq_seg->next;
 673			ep_ring->dequeue = ep_ring->deq_seg->trbs;
 674		}
 675	}
 676
 677	/*
 678	 * Probably there was TIMEOUT during handling Set Dequeue Pointer
 679	 * command. It's critical error and controller will be stopped.
 680	 */
 681	if (ret)
 682		return -ESHUTDOWN;
 683
 684	/* Restart any rings with pending requests */
 685	cdnsp_ring_doorbell_for_active_rings(pdev, pep);
 686
 687	return 0;
 688}
 689
 690int cdnsp_remove_request(struct cdnsp_device *pdev,
 691			 struct cdnsp_request *preq,
 692			 struct cdnsp_ep *pep)
 693{
 694	struct cdnsp_dequeue_state deq_state;
 695	struct cdnsp_td *cur_td = NULL;
 696	struct cdnsp_ring *ep_ring;
 697	struct cdnsp_segment *seg;
 698	int status = -ECONNRESET;
 699	int ret = 0;
 700	u64 hw_deq;
 701
 702	memset(&deq_state, 0, sizeof(deq_state));
 703
 704	trace_cdnsp_remove_request(pep->out_ctx);
 705	trace_cdnsp_remove_request_td(preq);
 706
 707	cur_td = &preq->td;
 708	ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);
 709
 710	/*
 711	 * If we stopped on the TD we need to cancel, then we have to
 712	 * move the controller endpoint ring dequeue pointer past
 713	 * this TD.
 714	 */
 715	hw_deq = cdnsp_get_hw_deq(pdev, pep->idx, preq->request.stream_id);
 716	hw_deq &= ~0xf;
 717
 718	seg = cdnsp_trb_in_td(pdev, cur_td->start_seg, cur_td->first_trb,
 719			      cur_td->last_trb, hw_deq);
 720
 721	if (seg && (pep->ep_state & EP_ENABLED))
 
 722		cdnsp_find_new_dequeue_state(pdev, pep, preq->request.stream_id,
 723					     cur_td, &deq_state);
 724	else
 725		cdnsp_td_to_noop(pdev, ep_ring, cur_td, false);
 726
 727	/*
 728	 * The event handler won't see a completion for this TD anymore,
 729	 * so remove it from the endpoint ring's TD list.
 730	 */
 731	list_del_init(&cur_td->td_list);
 732	ep_ring->num_tds--;
 733	pep->stream_info.td_count--;
 734
 735	/*
 736	 * During disconnecting all endpoint will be disabled so we don't
 737	 * have to worry about updating dequeue pointer.
 738	 */
 739	if (pdev->cdnsp_state & CDNSP_STATE_DISCONNECT_PENDING) {
 
 740		status = -ESHUTDOWN;
 741		ret = cdnsp_cmd_set_deq(pdev, pep, &deq_state);
 742	}
 743
 744	cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, cur_td);
 745	cdnsp_gadget_giveback(pep, cur_td->preq, status);
 746
 747	return ret;
 748}
 749
 750static int cdnsp_update_port_id(struct cdnsp_device *pdev, u32 port_id)
 751{
 752	struct cdnsp_port *port = pdev->active_port;
 753	u8 old_port = 0;
 754
 755	if (port && port->port_num == port_id)
 756		return 0;
 757
 758	if (port)
 759		old_port = port->port_num;
 760
 761	if (port_id == pdev->usb2_port.port_num) {
 762		port = &pdev->usb2_port;
 763	} else if (port_id == pdev->usb3_port.port_num) {
 764		port  = &pdev->usb3_port;
 765	} else {
 766		dev_err(pdev->dev, "Port event with invalid port ID %d\n",
 767			port_id);
 768		return -EINVAL;
 769	}
 770
 771	if (port_id != old_port) {
 772		cdnsp_disable_slot(pdev);
 773		pdev->active_port = port;
 774		cdnsp_enable_slot(pdev);
 775	}
 776
 777	if (port_id == pdev->usb2_port.port_num)
 778		cdnsp_set_usb2_hardware_lpm(pdev, NULL, 1);
 779	else
 780		writel(PORT_U1_TIMEOUT(1) | PORT_U2_TIMEOUT(1),
 781		       &pdev->usb3_port.regs->portpmsc);
 782
 783	return 0;
 784}
 785
 786static void cdnsp_handle_port_status(struct cdnsp_device *pdev,
 787				     union cdnsp_trb *event)
 788{
 789	struct cdnsp_port_regs __iomem *port_regs;
 790	u32 portsc, cmd_regs;
 791	bool port2 = false;
 792	u32 link_state;
 793	u32 port_id;
 794
 795	/* Port status change events always have a successful completion code */
 796	if (GET_COMP_CODE(le32_to_cpu(event->generic.field[2])) != COMP_SUCCESS)
 797		dev_err(pdev->dev, "ERR: incorrect PSC event\n");
 798
 799	port_id = GET_PORT_ID(le32_to_cpu(event->generic.field[0]));
 800
 801	if (cdnsp_update_port_id(pdev, port_id))
 802		goto cleanup;
 803
 804	port_regs = pdev->active_port->regs;
 805
 806	if (port_id == pdev->usb2_port.port_num)
 807		port2 = true;
 808
 809new_event:
 810	portsc = readl(&port_regs->portsc);
 811	writel(cdnsp_port_state_to_neutral(portsc) |
 812	       (portsc & PORT_CHANGE_BITS), &port_regs->portsc);
 813
 814	trace_cdnsp_handle_port_status(pdev->active_port->port_num, portsc);
 815
 816	pdev->gadget.speed = cdnsp_port_speed(portsc);
 817	link_state = portsc & PORT_PLS_MASK;
 818
 819	/* Port Link State change detected. */
 820	if ((portsc & PORT_PLC)) {
 821		if (!(pdev->cdnsp_state & CDNSP_WAKEUP_PENDING)  &&
 822		    link_state == XDEV_RESUME) {
 823			cmd_regs = readl(&pdev->op_regs->command);
 824			if (!(cmd_regs & CMD_R_S))
 825				goto cleanup;
 826
 827			if (DEV_SUPERSPEED_ANY(portsc)) {
 828				cdnsp_set_link_state(pdev, &port_regs->portsc,
 829						     XDEV_U0);
 830
 831				cdnsp_resume_gadget(pdev);
 832			}
 833		}
 834
 835		if ((pdev->cdnsp_state & CDNSP_WAKEUP_PENDING) &&
 836		    link_state == XDEV_U0) {
 837			pdev->cdnsp_state &= ~CDNSP_WAKEUP_PENDING;
 838
 839			cdnsp_force_header_wakeup(pdev, 1);
 840			cdnsp_ring_cmd_db(pdev);
 841			cdnsp_wait_for_cmd_compl(pdev);
 842		}
 843
 844		if (link_state == XDEV_U0 && pdev->link_state == XDEV_U3 &&
 845		    !DEV_SUPERSPEED_ANY(portsc))
 846			cdnsp_resume_gadget(pdev);
 847
 848		if (link_state == XDEV_U3 &&  pdev->link_state != XDEV_U3)
 849			cdnsp_suspend_gadget(pdev);
 850
 851		pdev->link_state = link_state;
 852	}
 853
 854	if (portsc & PORT_CSC) {
 855		/* Detach device. */
 856		if (pdev->gadget.connected && !(portsc & PORT_CONNECT))
 857			cdnsp_disconnect_gadget(pdev);
 858
 859		/* Attach device. */
 860		if (portsc & PORT_CONNECT) {
 861			if (!port2)
 862				cdnsp_irq_reset(pdev);
 863
 864			usb_gadget_set_state(&pdev->gadget, USB_STATE_ATTACHED);
 865		}
 866	}
 867
 868	/* Port reset. */
 869	if ((portsc & (PORT_RC | PORT_WRC)) && (portsc & PORT_CONNECT)) {
 870		cdnsp_irq_reset(pdev);
 871		pdev->u1_allowed = 0;
 872		pdev->u2_allowed = 0;
 873		pdev->may_wakeup = 0;
 874	}
 875
 876	if (portsc & PORT_CEC)
 877		dev_err(pdev->dev, "Port Over Current detected\n");
 878
 879	if (portsc & PORT_CEC)
 880		dev_err(pdev->dev, "Port Configure Error detected\n");
 881
 882	if (readl(&port_regs->portsc) & PORT_CHANGE_BITS)
 883		goto new_event;
 884
 885cleanup:
 886	cdnsp_inc_deq(pdev, pdev->event_ring);
 887}
 888
 889static void cdnsp_td_cleanup(struct cdnsp_device *pdev,
 890			     struct cdnsp_td *td,
 891			     struct cdnsp_ring *ep_ring,
 892			     int *status)
 893{
 894	struct cdnsp_request *preq = td->preq;
 895
 896	/* if a bounce buffer was used to align this td then unmap it */
 897	cdnsp_unmap_td_bounce_buffer(pdev, ep_ring, td);
 898
 899	/*
 900	 * If the controller said we transferred more data than the buffer
 901	 * length, Play it safe and say we didn't transfer anything.
 902	 */
 903	if (preq->request.actual > preq->request.length) {
 904		preq->request.actual = 0;
 905		*status = 0;
 906	}
 907
 908	list_del_init(&td->td_list);
 909	ep_ring->num_tds--;
 910	preq->pep->stream_info.td_count--;
 911
 912	cdnsp_gadget_giveback(preq->pep, preq, *status);
 913}
 914
 915static void cdnsp_finish_td(struct cdnsp_device *pdev,
 916			    struct cdnsp_td *td,
 917			    struct cdnsp_transfer_event *event,
 918			    struct cdnsp_ep *ep,
 919			    int *status)
 920{
 921	struct cdnsp_ring *ep_ring;
 922	u32 trb_comp_code;
 923
 924	ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
 925	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
 926
 927	if (trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
 928	    trb_comp_code == COMP_STOPPED ||
 929	    trb_comp_code == COMP_STOPPED_SHORT_PACKET) {
 930		/*
 931		 * The Endpoint Stop Command completion will take care of any
 932		 * stopped TDs. A stopped TD may be restarted, so don't update
 933		 * the ring dequeue pointer or take this TD off any lists yet.
 934		 */
 935		return;
 936	}
 937
 938	/* Update ring dequeue pointer */
 939	while (ep_ring->dequeue != td->last_trb)
 940		cdnsp_inc_deq(pdev, ep_ring);
 941
 942	cdnsp_inc_deq(pdev, ep_ring);
 943
 944	cdnsp_td_cleanup(pdev, td, ep_ring, status);
 945}
 946
 947/* sum trb lengths from ring dequeue up to stop_trb, _excluding_ stop_trb */
 948static int cdnsp_sum_trb_lengths(struct cdnsp_device *pdev,
 949				 struct cdnsp_ring *ring,
 950				 union cdnsp_trb *stop_trb)
 951{
 952	struct cdnsp_segment *seg = ring->deq_seg;
 953	union cdnsp_trb *trb = ring->dequeue;
 954	u32 sum;
 955
 956	for (sum = 0; trb != stop_trb; cdnsp_next_trb(pdev, ring, &seg, &trb)) {
 957		if (!cdnsp_trb_is_noop(trb) && !cdnsp_trb_is_link(trb))
 958			sum += TRB_LEN(le32_to_cpu(trb->generic.field[2]));
 959	}
 960	return sum;
 961}
 962
 963static int cdnsp_giveback_first_trb(struct cdnsp_device *pdev,
 964				    struct cdnsp_ep *pep,
 965				    unsigned int stream_id,
 966				    int start_cycle,
 967				    struct cdnsp_generic_trb *start_trb)
 968{
 969	/*
 970	 * Pass all the TRBs to the hardware at once and make sure this write
 971	 * isn't reordered.
 972	 */
 973	wmb();
 974
 975	if (start_cycle)
 976		start_trb->field[3] |= cpu_to_le32(start_cycle);
 977	else
 978		start_trb->field[3] &= cpu_to_le32(~TRB_CYCLE);
 979
 980	if ((pep->ep_state & EP_HAS_STREAMS) &&
 981	    !pep->stream_info.first_prime_det) {
 982		trace_cdnsp_wait_for_prime(pep, stream_id);
 983		return 0;
 984	}
 985
 986	return cdnsp_ring_ep_doorbell(pdev, pep, stream_id);
 987}
 988
 989/*
 990 * Process control tds, update USB request status and actual_length.
 991 */
 992static void cdnsp_process_ctrl_td(struct cdnsp_device *pdev,
 993				  struct cdnsp_td *td,
 994				  union cdnsp_trb *event_trb,
 995				  struct cdnsp_transfer_event *event,
 996				  struct cdnsp_ep *pep,
 997				  int *status)
 998{
 999	struct cdnsp_ring *ep_ring;
1000	u32 remaining;
1001	u32 trb_type;
1002
1003	trb_type = TRB_FIELD_TO_TYPE(le32_to_cpu(event_trb->generic.field[3]));
1004	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1005	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1006
1007	/*
1008	 * if on data stage then update the actual_length of the USB
1009	 * request and flag it as set, so it won't be overwritten in the event
1010	 * for the last TRB.
1011	 */
1012	if (trb_type == TRB_DATA) {
1013		td->request_length_set = true;
1014		td->preq->request.actual = td->preq->request.length - remaining;
1015	}
1016
1017	/* at status stage */
1018	if (!td->request_length_set)
1019		td->preq->request.actual = td->preq->request.length;
1020
1021	if (pdev->ep0_stage == CDNSP_DATA_STAGE && pep->number == 0 &&
1022	    pdev->three_stage_setup) {
1023		td = list_entry(ep_ring->td_list.next, struct cdnsp_td,
1024				td_list);
1025		pdev->ep0_stage = CDNSP_STATUS_STAGE;
1026
1027		cdnsp_giveback_first_trb(pdev, pep, 0, ep_ring->cycle_state,
1028					 &td->last_trb->generic);
1029		return;
1030	}
1031
1032	*status = 0;
1033
1034	cdnsp_finish_td(pdev, td, event, pep, status);
1035}
1036
1037/*
1038 * Process isochronous tds, update usb request status and actual_length.
1039 */
1040static void cdnsp_process_isoc_td(struct cdnsp_device *pdev,
1041				  struct cdnsp_td *td,
1042				  union cdnsp_trb *ep_trb,
1043				  struct cdnsp_transfer_event *event,
1044				  struct cdnsp_ep *pep,
1045				  int status)
1046{
1047	struct cdnsp_request *preq = td->preq;
1048	u32 remaining, requested, ep_trb_len;
1049	bool sum_trbs_for_length = false;
1050	struct cdnsp_ring *ep_ring;
1051	u32 trb_comp_code;
1052	u32 td_length;
1053
1054	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1055	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1056	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1057	ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
1058
1059	requested = preq->request.length;
1060
1061	/* handle completion code */
1062	switch (trb_comp_code) {
1063	case COMP_SUCCESS:
1064		preq->request.status = 0;
1065		break;
1066	case COMP_SHORT_PACKET:
1067		preq->request.status = 0;
1068		sum_trbs_for_length = true;
1069		break;
1070	case COMP_ISOCH_BUFFER_OVERRUN:
1071	case COMP_BABBLE_DETECTED_ERROR:
1072		preq->request.status = -EOVERFLOW;
1073		break;
1074	case COMP_STOPPED:
1075		sum_trbs_for_length = true;
1076		break;
1077	case COMP_STOPPED_SHORT_PACKET:
1078		/* field normally containing residue now contains transferred */
1079		preq->request.status  = 0;
1080		requested = remaining;
1081		break;
1082	case COMP_STOPPED_LENGTH_INVALID:
1083		requested = 0;
1084		remaining = 0;
1085		break;
1086	default:
1087		sum_trbs_for_length = true;
1088		preq->request.status = -1;
1089		break;
1090	}
1091
1092	if (sum_trbs_for_length) {
1093		td_length = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb);
1094		td_length += ep_trb_len - remaining;
1095	} else {
1096		td_length = requested;
1097	}
1098
1099	td->preq->request.actual += td_length;
1100
1101	cdnsp_finish_td(pdev, td, event, pep, &status);
1102}
1103
1104static void cdnsp_skip_isoc_td(struct cdnsp_device *pdev,
1105			       struct cdnsp_td *td,
1106			       struct cdnsp_transfer_event *event,
1107			       struct cdnsp_ep *pep,
1108			       int status)
1109{
1110	struct cdnsp_ring *ep_ring;
1111
1112	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1113	td->preq->request.status = -EXDEV;
1114	td->preq->request.actual = 0;
1115
1116	/* Update ring dequeue pointer */
1117	while (ep_ring->dequeue != td->last_trb)
1118		cdnsp_inc_deq(pdev, ep_ring);
1119
1120	cdnsp_inc_deq(pdev, ep_ring);
1121
1122	cdnsp_td_cleanup(pdev, td, ep_ring, &status);
1123}
1124
1125/*
1126 * Process bulk and interrupt tds, update usb request status and actual_length.
1127 */
1128static void cdnsp_process_bulk_intr_td(struct cdnsp_device *pdev,
1129				       struct cdnsp_td *td,
1130				       union cdnsp_trb *ep_trb,
1131				       struct cdnsp_transfer_event *event,
1132				       struct cdnsp_ep *ep,
1133				       int *status)
1134{
1135	u32 remaining, requested, ep_trb_len;
1136	struct cdnsp_ring *ep_ring;
1137	u32 trb_comp_code;
1138
1139	ep_ring = cdnsp_dma_to_transfer_ring(ep, le64_to_cpu(event->buffer));
1140	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1141	remaining = EVENT_TRB_LEN(le32_to_cpu(event->transfer_len));
1142	ep_trb_len = TRB_LEN(le32_to_cpu(ep_trb->generic.field[2]));
1143	requested = td->preq->request.length;
1144
1145	switch (trb_comp_code) {
1146	case COMP_SUCCESS:
1147	case COMP_SHORT_PACKET:
1148		*status = 0;
1149		break;
1150	case COMP_STOPPED_SHORT_PACKET:
1151		td->preq->request.actual = remaining;
1152		goto finish_td;
1153	case COMP_STOPPED_LENGTH_INVALID:
1154		/* Stopped on ep trb with invalid length, exclude it. */
1155		ep_trb_len = 0;
1156		remaining = 0;
1157		break;
1158	}
1159
1160	if (ep_trb == td->last_trb)
1161		ep_trb_len = requested - remaining;
1162	else
1163		ep_trb_len = cdnsp_sum_trb_lengths(pdev, ep_ring, ep_trb) +
1164						   ep_trb_len - remaining;
1165	td->preq->request.actual = ep_trb_len;
1166
1167finish_td:
1168	ep->stream_info.drbls_count--;
1169
1170	cdnsp_finish_td(pdev, td, event, ep, status);
1171}
1172
1173static void cdnsp_handle_tx_nrdy(struct cdnsp_device *pdev,
1174				 struct cdnsp_transfer_event *event)
1175{
1176	struct cdnsp_generic_trb *generic;
1177	struct cdnsp_ring *ep_ring;
1178	struct cdnsp_ep *pep;
1179	int cur_stream;
1180	int ep_index;
1181	int host_sid;
1182	int dev_sid;
1183
1184	generic = (struct cdnsp_generic_trb *)event;
1185	ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1186	dev_sid = TRB_TO_DEV_STREAM(le32_to_cpu(generic->field[0]));
1187	host_sid = TRB_TO_HOST_STREAM(le32_to_cpu(generic->field[2]));
1188
1189	pep = &pdev->eps[ep_index];
1190
1191	if (!(pep->ep_state & EP_HAS_STREAMS))
1192		return;
1193
1194	if (host_sid == STREAM_PRIME_ACK) {
1195		pep->stream_info.first_prime_det = 1;
1196		for (cur_stream = 1; cur_stream < pep->stream_info.num_streams;
1197		    cur_stream++) {
1198			ep_ring = pep->stream_info.stream_rings[cur_stream];
1199			ep_ring->stream_active = 1;
1200			ep_ring->stream_rejected = 0;
1201		}
1202	}
1203
1204	if (host_sid == STREAM_REJECTED) {
1205		struct cdnsp_td *td, *td_temp;
1206
1207		pep->stream_info.drbls_count--;
1208		ep_ring = pep->stream_info.stream_rings[dev_sid];
1209		ep_ring->stream_active = 0;
1210		ep_ring->stream_rejected = 1;
1211
1212		list_for_each_entry_safe(td, td_temp, &ep_ring->td_list,
1213					 td_list) {
1214			td->drbl = 0;
1215		}
1216	}
1217
1218	cdnsp_ring_doorbell_for_active_rings(pdev, pep);
1219}
1220
1221/*
1222 * If this function returns an error condition, it means it got a Transfer
1223 * event with a corrupted TRB DMA address or endpoint is disabled.
1224 */
1225static int cdnsp_handle_tx_event(struct cdnsp_device *pdev,
1226				 struct cdnsp_transfer_event *event)
1227{
1228	const struct usb_endpoint_descriptor *desc;
1229	bool handling_skipped_tds = false;
1230	struct cdnsp_segment *ep_seg;
1231	struct cdnsp_ring *ep_ring;
1232	int status = -EINPROGRESS;
1233	union cdnsp_trb *ep_trb;
1234	dma_addr_t ep_trb_dma;
1235	struct cdnsp_ep *pep;
1236	struct cdnsp_td *td;
1237	u32 trb_comp_code;
1238	int invalidate;
1239	int ep_index;
1240
1241	invalidate = le32_to_cpu(event->flags) & TRB_EVENT_INVALIDATE;
1242	ep_index = TRB_TO_EP_ID(le32_to_cpu(event->flags)) - 1;
1243	trb_comp_code = GET_COMP_CODE(le32_to_cpu(event->transfer_len));
1244	ep_trb_dma = le64_to_cpu(event->buffer);
1245
1246	pep = &pdev->eps[ep_index];
1247	ep_ring = cdnsp_dma_to_transfer_ring(pep, le64_to_cpu(event->buffer));
1248
1249	/*
1250	 * If device is disconnect then all requests will be dequeued
1251	 * by upper layers as part of disconnect sequence.
1252	 * We don't want handle such event to avoid racing.
1253	 */
1254	if (invalidate || !pdev->gadget.connected)
1255		goto cleanup;
1256
1257	if (GET_EP_CTX_STATE(pep->out_ctx) == EP_STATE_DISABLED) {
1258		trace_cdnsp_ep_disabled(pep->out_ctx);
1259		goto err_out;
1260	}
1261
1262	/* Some transfer events don't always point to a trb*/
1263	if (!ep_ring) {
1264		switch (trb_comp_code) {
1265		case COMP_INVALID_STREAM_TYPE_ERROR:
1266		case COMP_INVALID_STREAM_ID_ERROR:
1267		case COMP_RING_UNDERRUN:
1268		case COMP_RING_OVERRUN:
1269			goto cleanup;
1270		default:
1271			dev_err(pdev->dev, "ERROR: %s event for unknown ring\n",
1272				pep->name);
1273			goto err_out;
1274		}
1275	}
1276
1277	/* Look for some error cases that need special treatment. */
1278	switch (trb_comp_code) {
1279	case COMP_BABBLE_DETECTED_ERROR:
1280		status = -EOVERFLOW;
1281		break;
1282	case COMP_RING_UNDERRUN:
1283	case COMP_RING_OVERRUN:
1284		/*
1285		 * When the Isoch ring is empty, the controller will generate
1286		 * a Ring Overrun Event for IN Isoch endpoint or Ring
1287		 * Underrun Event for OUT Isoch endpoint.
1288		 */
1289		goto cleanup;
1290	case COMP_MISSED_SERVICE_ERROR:
1291		/*
1292		 * When encounter missed service error, one or more isoc tds
1293		 * may be missed by controller.
1294		 * Set skip flag of the ep_ring; Complete the missed tds as
1295		 * short transfer when process the ep_ring next time.
1296		 */
1297		pep->skip = true;
1298		break;
1299	}
1300
1301	do {
1302		/*
1303		 * This TRB should be in the TD at the head of this ring's TD
1304		 * list.
1305		 */
1306		if (list_empty(&ep_ring->td_list)) {
1307			/*
1308			 * Don't print warnings if it's due to a stopped
1309			 * endpoint generating an extra completion event, or
1310			 * a event for the last TRB of a short TD we already
1311			 * got a short event for.
1312			 * The short TD is already removed from the TD list.
1313			 */
1314			if (!(trb_comp_code == COMP_STOPPED ||
1315			      trb_comp_code == COMP_STOPPED_LENGTH_INVALID ||
1316			      ep_ring->last_td_was_short))
1317				trace_cdnsp_trb_without_td(ep_ring,
1318					(struct cdnsp_generic_trb *)event);
1319
1320			if (pep->skip) {
1321				pep->skip = false;
1322				trace_cdnsp_ep_list_empty_with_skip(pep, 0);
1323			}
1324
1325			goto cleanup;
1326		}
1327
1328		td = list_entry(ep_ring->td_list.next, struct cdnsp_td,
1329				td_list);
1330
1331		/* Is this a TRB in the currently executing TD? */
1332		ep_seg = cdnsp_trb_in_td(pdev, ep_ring->deq_seg,
1333					 ep_ring->dequeue, td->last_trb,
1334					 ep_trb_dma);
1335
1336		desc = td->preq->pep->endpoint.desc;
1337
1338		if (ep_seg) {
1339			ep_trb = &ep_seg->trbs[(ep_trb_dma - ep_seg->dma)
1340					       / sizeof(*ep_trb)];
1341
1342			trace_cdnsp_handle_transfer(ep_ring,
1343					(struct cdnsp_generic_trb *)ep_trb);
1344
1345			if (pep->skip && usb_endpoint_xfer_isoc(desc) &&
1346			    td->last_trb != ep_trb)
1347				return -EAGAIN;
1348		}
1349
1350		/*
1351		 * Skip the Force Stopped Event. The event_trb(ep_trb_dma)
1352		 * of FSE is not in the current TD pointed by ep_ring->dequeue
1353		 * because that the hardware dequeue pointer still at the
1354		 * previous TRB of the current TD. The previous TRB maybe a
1355		 * Link TD or the last TRB of the previous TD. The command
1356		 * completion handle will take care the rest.
1357		 */
1358		if (!ep_seg && (trb_comp_code == COMP_STOPPED ||
1359				trb_comp_code == COMP_STOPPED_LENGTH_INVALID)) {
1360			pep->skip = false;
1361			goto cleanup;
1362		}
1363
1364		if (!ep_seg) {
1365			if (!pep->skip || !usb_endpoint_xfer_isoc(desc)) {
1366				/* Something is busted, give up! */
1367				dev_err(pdev->dev,
1368					"ERROR Transfer event TRB DMA ptr not "
1369					"part of current TD ep_index %d "
1370					"comp_code %u\n", ep_index,
1371					trb_comp_code);
1372				return -EINVAL;
1373			}
1374
1375			cdnsp_skip_isoc_td(pdev, td, event, pep, status);
1376			goto cleanup;
1377		}
1378
1379		if (trb_comp_code == COMP_SHORT_PACKET)
1380			ep_ring->last_td_was_short = true;
1381		else
1382			ep_ring->last_td_was_short = false;
1383
1384		if (pep->skip) {
1385			pep->skip = false;
1386			cdnsp_skip_isoc_td(pdev, td, event, pep, status);
1387			goto cleanup;
1388		}
1389
1390		if (cdnsp_trb_is_noop(ep_trb))
1391			goto cleanup;
1392
1393		if (usb_endpoint_xfer_control(desc))
1394			cdnsp_process_ctrl_td(pdev, td, ep_trb, event, pep,
1395					      &status);
1396		else if (usb_endpoint_xfer_isoc(desc))
1397			cdnsp_process_isoc_td(pdev, td, ep_trb, event, pep,
1398					      status);
1399		else
1400			cdnsp_process_bulk_intr_td(pdev, td, ep_trb, event, pep,
1401						   &status);
1402cleanup:
1403		handling_skipped_tds = pep->skip;
1404
1405		/*
1406		 * Do not update event ring dequeue pointer if we're in a loop
1407		 * processing missed tds.
1408		 */
1409		if (!handling_skipped_tds)
1410			cdnsp_inc_deq(pdev, pdev->event_ring);
1411
1412	/*
1413	 * If ep->skip is set, it means there are missed tds on the
1414	 * endpoint ring need to take care of.
1415	 * Process them as short transfer until reach the td pointed by
1416	 * the event.
1417	 */
1418	} while (handling_skipped_tds);
1419	return 0;
1420
1421err_out:
1422	dev_err(pdev->dev, "@%016llx %08x %08x %08x %08x\n",
1423		(unsigned long long)
1424		cdnsp_trb_virt_to_dma(pdev->event_ring->deq_seg,
1425				      pdev->event_ring->dequeue),
1426		 lower_32_bits(le64_to_cpu(event->buffer)),
1427		 upper_32_bits(le64_to_cpu(event->buffer)),
1428		 le32_to_cpu(event->transfer_len),
1429		 le32_to_cpu(event->flags));
1430	return -EINVAL;
1431}
1432
1433/*
1434 * This function handles all events on the event ring.
1435 * Returns true for "possibly more events to process" (caller should call
1436 * again), otherwise false if done.
1437 */
1438static bool cdnsp_handle_event(struct cdnsp_device *pdev)
1439{
1440	unsigned int comp_code;
1441	union cdnsp_trb *event;
1442	bool update_ptrs = true;
1443	u32 cycle_bit;
1444	int ret = 0;
1445	u32 flags;
1446
1447	event = pdev->event_ring->dequeue;
1448	flags = le32_to_cpu(event->event_cmd.flags);
1449	cycle_bit = (flags & TRB_CYCLE);
1450
1451	/* Does the controller or driver own the TRB? */
1452	if (cycle_bit != pdev->event_ring->cycle_state)
1453		return false;
1454
1455	trace_cdnsp_handle_event(pdev->event_ring, &event->generic);
1456
1457	/*
1458	 * Barrier between reading the TRB_CYCLE (valid) flag above and any
1459	 * reads of the event's flags/data below.
1460	 */
1461	rmb();
1462
1463	switch (flags & TRB_TYPE_BITMASK) {
1464	case TRB_TYPE(TRB_COMPLETION):
1465		/*
1466		 * Command can't be handled in interrupt context so just
1467		 * increment command ring dequeue pointer.
1468		 */
1469		cdnsp_inc_deq(pdev, pdev->cmd_ring);
1470		break;
1471	case TRB_TYPE(TRB_PORT_STATUS):
1472		cdnsp_handle_port_status(pdev, event);
1473		update_ptrs = false;
1474		break;
1475	case TRB_TYPE(TRB_TRANSFER):
1476		ret = cdnsp_handle_tx_event(pdev, &event->trans_event);
1477		if (ret >= 0)
1478			update_ptrs = false;
1479		break;
1480	case TRB_TYPE(TRB_SETUP):
1481		pdev->ep0_stage = CDNSP_SETUP_STAGE;
1482		pdev->setup_id = TRB_SETUPID_TO_TYPE(flags);
1483		pdev->setup_speed = TRB_SETUP_SPEEDID(flags);
1484		pdev->setup = *((struct usb_ctrlrequest *)
1485				&event->trans_event.buffer);
1486
1487		cdnsp_setup_analyze(pdev);
1488		break;
1489	case TRB_TYPE(TRB_ENDPOINT_NRDY):
1490		cdnsp_handle_tx_nrdy(pdev, &event->trans_event);
1491		break;
1492	case TRB_TYPE(TRB_HC_EVENT): {
1493		comp_code = GET_COMP_CODE(le32_to_cpu(event->generic.field[2]));
1494
1495		switch (comp_code) {
1496		case COMP_EVENT_RING_FULL_ERROR:
1497			dev_err(pdev->dev, "Event Ring Full\n");
1498			break;
1499		default:
1500			dev_err(pdev->dev, "Controller error code 0x%02x\n",
1501				comp_code);
1502		}
1503
1504		break;
1505	}
1506	case TRB_TYPE(TRB_MFINDEX_WRAP):
1507	case TRB_TYPE(TRB_DRB_OVERFLOW):
1508		break;
1509	default:
1510		dev_warn(pdev->dev, "ERROR unknown event type %ld\n",
1511			 TRB_FIELD_TO_TYPE(flags));
1512	}
1513
1514	if (update_ptrs)
1515		/* Update SW event ring dequeue pointer. */
1516		cdnsp_inc_deq(pdev, pdev->event_ring);
1517
1518	/*
1519	 * Caller will call us again to check if there are more items
1520	 * on the event ring.
1521	 */
1522	return true;
1523}
1524
1525irqreturn_t cdnsp_thread_irq_handler(int irq, void *data)
1526{
1527	struct cdnsp_device *pdev = (struct cdnsp_device *)data;
1528	union cdnsp_trb *event_ring_deq;
1529	unsigned long flags;
1530	int counter = 0;
1531
1532	local_bh_disable();
1533	spin_lock_irqsave(&pdev->lock, flags);
1534
1535	if (pdev->cdnsp_state & (CDNSP_STATE_HALTED | CDNSP_STATE_DYING)) {
1536		/*
1537		 * While removing or stopping driver there may still be deferred
1538		 * not handled interrupt which should not be treated as error.
1539		 * Driver should simply ignore it.
1540		 */
1541		if (pdev->gadget_driver)
1542			cdnsp_died(pdev);
1543
1544		spin_unlock_irqrestore(&pdev->lock, flags);
1545		local_bh_enable();
1546		return IRQ_HANDLED;
1547	}
1548
1549	event_ring_deq = pdev->event_ring->dequeue;
1550
1551	while (cdnsp_handle_event(pdev)) {
1552		if (++counter >= TRBS_PER_EV_DEQ_UPDATE) {
1553			cdnsp_update_erst_dequeue(pdev, event_ring_deq, 0);
1554			event_ring_deq = pdev->event_ring->dequeue;
1555			counter = 0;
1556		}
1557	}
1558
1559	cdnsp_update_erst_dequeue(pdev, event_ring_deq, 1);
1560
1561	spin_unlock_irqrestore(&pdev->lock, flags);
1562	local_bh_enable();
1563
1564	return IRQ_HANDLED;
1565}
1566
1567irqreturn_t cdnsp_irq_handler(int irq, void *priv)
1568{
1569	struct cdnsp_device *pdev = (struct cdnsp_device *)priv;
1570	u32 irq_pending;
1571	u32 status;
1572
1573	status = readl(&pdev->op_regs->status);
1574
1575	if (status == ~(u32)0) {
1576		cdnsp_died(pdev);
1577		return IRQ_HANDLED;
1578	}
1579
1580	if (!(status & STS_EINT))
1581		return IRQ_NONE;
1582
1583	writel(status | STS_EINT, &pdev->op_regs->status);
1584	irq_pending = readl(&pdev->ir_set->irq_pending);
1585	irq_pending |= IMAN_IP;
1586	writel(irq_pending, &pdev->ir_set->irq_pending);
1587
1588	if (status & STS_FATAL) {
1589		cdnsp_died(pdev);
1590		return IRQ_HANDLED;
1591	}
1592
1593	return IRQ_WAKE_THREAD;
1594}
1595
1596/*
1597 * Generic function for queuing a TRB on a ring.
1598 * The caller must have checked to make sure there's room on the ring.
1599 *
1600 * @more_trbs_coming:	Will you enqueue more TRBs before setting doorbell?
1601 */
1602static void cdnsp_queue_trb(struct cdnsp_device *pdev, struct cdnsp_ring *ring,
1603			    bool more_trbs_coming, u32 field1, u32 field2,
1604			    u32 field3, u32 field4)
1605{
1606	struct cdnsp_generic_trb *trb;
1607
1608	trb = &ring->enqueue->generic;
1609
1610	trb->field[0] = cpu_to_le32(field1);
1611	trb->field[1] = cpu_to_le32(field2);
1612	trb->field[2] = cpu_to_le32(field3);
1613	trb->field[3] = cpu_to_le32(field4);
1614
1615	trace_cdnsp_queue_trb(ring, trb);
1616	cdnsp_inc_enq(pdev, ring, more_trbs_coming);
1617}
1618
1619/*
1620 * Does various checks on the endpoint ring, and makes it ready to
1621 * queue num_trbs.
1622 */
1623static int cdnsp_prepare_ring(struct cdnsp_device *pdev,
1624			      struct cdnsp_ring *ep_ring,
1625			      u32 ep_state, unsigned
1626			      int num_trbs,
1627			      gfp_t mem_flags)
1628{
1629	unsigned int num_trbs_needed;
1630
1631	/* Make sure the endpoint has been added to controller schedule. */
1632	switch (ep_state) {
1633	case EP_STATE_STOPPED:
1634	case EP_STATE_RUNNING:
1635	case EP_STATE_HALTED:
1636		break;
1637	default:
1638		dev_err(pdev->dev, "ERROR: incorrect endpoint state\n");
1639		return -EINVAL;
1640	}
1641
1642	while (1) {
1643		if (cdnsp_room_on_ring(pdev, ep_ring, num_trbs))
1644			break;
1645
1646		trace_cdnsp_no_room_on_ring("try ring expansion");
1647
1648		num_trbs_needed = num_trbs - ep_ring->num_trbs_free;
1649		if (cdnsp_ring_expansion(pdev, ep_ring, num_trbs_needed,
1650					 mem_flags)) {
1651			dev_err(pdev->dev, "Ring expansion failed\n");
1652			return -ENOMEM;
1653		}
1654	}
1655
1656	while (cdnsp_trb_is_link(ep_ring->enqueue)) {
1657		ep_ring->enqueue->link.control |= cpu_to_le32(TRB_CHAIN);
1658		/* The cycle bit must be set as the last operation. */
1659		wmb();
1660		ep_ring->enqueue->link.control ^= cpu_to_le32(TRB_CYCLE);
1661
1662		/* Toggle the cycle bit after the last ring segment. */
1663		if (cdnsp_link_trb_toggles_cycle(ep_ring->enqueue))
1664			ep_ring->cycle_state ^= 1;
1665		ep_ring->enq_seg = ep_ring->enq_seg->next;
1666		ep_ring->enqueue = ep_ring->enq_seg->trbs;
1667	}
1668	return 0;
1669}
1670
1671static int cdnsp_prepare_transfer(struct cdnsp_device *pdev,
1672				  struct cdnsp_request *preq,
1673				  unsigned int num_trbs)
1674{
1675	struct cdnsp_ring *ep_ring;
1676	int ret;
1677
1678	ep_ring = cdnsp_get_transfer_ring(pdev, preq->pep,
1679					  preq->request.stream_id);
1680	if (!ep_ring)
1681		return -EINVAL;
1682
1683	ret = cdnsp_prepare_ring(pdev, ep_ring,
1684				 GET_EP_CTX_STATE(preq->pep->out_ctx),
1685				 num_trbs, GFP_ATOMIC);
1686	if (ret)
1687		return ret;
1688
1689	INIT_LIST_HEAD(&preq->td.td_list);
1690	preq->td.preq = preq;
1691
1692	/* Add this TD to the tail of the endpoint ring's TD list. */
1693	list_add_tail(&preq->td.td_list, &ep_ring->td_list);
1694	ep_ring->num_tds++;
1695	preq->pep->stream_info.td_count++;
1696
1697	preq->td.start_seg = ep_ring->enq_seg;
1698	preq->td.first_trb = ep_ring->enqueue;
1699
1700	return 0;
1701}
1702
1703static unsigned int cdnsp_count_trbs(u64 addr, u64 len)
1704{
1705	unsigned int num_trbs;
1706
1707	num_trbs = DIV_ROUND_UP(len + (addr & (TRB_MAX_BUFF_SIZE - 1)),
1708				TRB_MAX_BUFF_SIZE);
1709	if (num_trbs == 0)
1710		num_trbs++;
1711
1712	return num_trbs;
1713}
1714
1715static unsigned int count_trbs_needed(struct cdnsp_request *preq)
1716{
1717	return cdnsp_count_trbs(preq->request.dma, preq->request.length);
1718}
1719
1720static unsigned int count_sg_trbs_needed(struct cdnsp_request *preq)
1721{
1722	unsigned int i, len, full_len, num_trbs = 0;
1723	struct scatterlist *sg;
1724
1725	full_len = preq->request.length;
1726
1727	for_each_sg(preq->request.sg, sg, preq->request.num_sgs, i) {
1728		len = sg_dma_len(sg);
1729		num_trbs += cdnsp_count_trbs(sg_dma_address(sg), len);
1730		len = min(len, full_len);
1731		full_len -= len;
1732		if (full_len == 0)
1733			break;
1734	}
1735
1736	return num_trbs;
1737}
1738
1739static void cdnsp_check_trb_math(struct cdnsp_request *preq, int running_total)
1740{
1741	if (running_total != preq->request.length)
1742		dev_err(preq->pep->pdev->dev,
1743			"%s - Miscalculated tx length, "
1744			"queued %#x, asked for %#x (%d)\n",
1745			preq->pep->name, running_total,
1746			preq->request.length, preq->request.actual);
1747}
1748
1749/*
1750 * TD size is the number of max packet sized packets remaining in the TD
1751 * (*not* including this TRB).
1752 *
1753 * Total TD packet count = total_packet_count =
1754 *     DIV_ROUND_UP(TD size in bytes / wMaxPacketSize)
1755 *
1756 * Packets transferred up to and including this TRB = packets_transferred =
1757 *     rounddown(total bytes transferred including this TRB / wMaxPacketSize)
1758 *
1759 * TD size = total_packet_count - packets_transferred
1760 *
1761 * It must fit in bits 21:17, so it can't be bigger than 31.
1762 * This is taken care of in the TRB_TD_SIZE() macro
1763 *
1764 * The last TRB in a TD must have the TD size set to zero.
1765 */
1766static u32 cdnsp_td_remainder(struct cdnsp_device *pdev,
1767			      int transferred,
1768			      int trb_buff_len,
1769			      unsigned int td_total_len,
1770			      struct cdnsp_request *preq,
1771			      bool more_trbs_coming,
1772			      bool zlp)
1773{
1774	u32 maxp, total_packet_count;
1775
1776	/* Before ZLP driver needs set TD_SIZE = 1. */
1777	if (zlp)
1778		return 1;
1779
1780	/* One TRB with a zero-length data packet. */
1781	if (!more_trbs_coming || (transferred == 0 && trb_buff_len == 0) ||
1782	    trb_buff_len == td_total_len)
1783		return 0;
1784
1785	maxp = usb_endpoint_maxp(preq->pep->endpoint.desc);
1786	total_packet_count = DIV_ROUND_UP(td_total_len, maxp);
1787
1788	/* Queuing functions don't count the current TRB into transferred. */
1789	return (total_packet_count - ((transferred + trb_buff_len) / maxp));
1790}
1791
1792static int cdnsp_align_td(struct cdnsp_device *pdev,
1793			  struct cdnsp_request *preq, u32 enqd_len,
1794			  u32 *trb_buff_len, struct cdnsp_segment *seg)
1795{
1796	struct device *dev = pdev->dev;
1797	unsigned int unalign;
1798	unsigned int max_pkt;
1799	u32 new_buff_len;
1800
1801	max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc);
1802	unalign = (enqd_len + *trb_buff_len) % max_pkt;
1803
1804	/* We got lucky, last normal TRB data on segment is packet aligned. */
1805	if (unalign == 0)
1806		return 0;
1807
1808	/* Is the last nornal TRB alignable by splitting it. */
1809	if (*trb_buff_len > unalign) {
1810		*trb_buff_len -= unalign;
1811		trace_cdnsp_bounce_align_td_split(preq, *trb_buff_len,
1812						  enqd_len, 0, unalign);
1813		return 0;
1814	}
1815
1816	/*
1817	 * We want enqd_len + trb_buff_len to sum up to a number aligned to
1818	 * number which is divisible by the endpoint's wMaxPacketSize. IOW:
1819	 * (size of currently enqueued TRBs + remainder) % wMaxPacketSize == 0.
1820	 */
1821	new_buff_len = max_pkt - (enqd_len % max_pkt);
1822
1823	if (new_buff_len > (preq->request.length - enqd_len))
1824		new_buff_len = (preq->request.length - enqd_len);
1825
1826	/* Create a max max_pkt sized bounce buffer pointed to by last trb. */
1827	if (preq->direction) {
1828		sg_pcopy_to_buffer(preq->request.sg,
1829				   preq->request.num_mapped_sgs,
1830				   seg->bounce_buf, new_buff_len, enqd_len);
1831		seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
1832						 max_pkt, DMA_TO_DEVICE);
1833	} else {
1834		seg->bounce_dma = dma_map_single(dev, seg->bounce_buf,
1835						 max_pkt, DMA_FROM_DEVICE);
1836	}
1837
1838	if (dma_mapping_error(dev, seg->bounce_dma)) {
1839		/* Try without aligning.*/
1840		dev_warn(pdev->dev,
1841			 "Failed mapping bounce buffer, not aligning\n");
1842		return 0;
1843	}
1844
1845	*trb_buff_len = new_buff_len;
1846	seg->bounce_len = new_buff_len;
1847	seg->bounce_offs = enqd_len;
1848
1849	trace_cdnsp_bounce_map(preq, new_buff_len, enqd_len, seg->bounce_dma,
1850			       unalign);
1851
1852	/*
1853	 * Bounce buffer successful aligned and seg->bounce_dma will be used
1854	 * in transfer TRB as new transfer buffer address.
1855	 */
1856	return 1;
1857}
1858
1859int cdnsp_queue_bulk_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq)
1860{
1861	unsigned int enqd_len, block_len, trb_buff_len, full_len;
1862	unsigned int start_cycle, num_sgs = 0;
1863	struct cdnsp_generic_trb *start_trb;
1864	u32 field, length_field, remainder;
1865	struct scatterlist *sg = NULL;
1866	bool more_trbs_coming = true;
1867	bool need_zero_pkt = false;
1868	bool zero_len_trb = false;
1869	struct cdnsp_ring *ring;
1870	bool first_trb = true;
1871	unsigned int num_trbs;
1872	struct cdnsp_ep *pep;
1873	u64 addr, send_addr;
1874	int sent_len, ret;
1875
1876	ring = cdnsp_request_to_transfer_ring(pdev, preq);
1877	if (!ring)
1878		return -EINVAL;
1879
1880	full_len = preq->request.length;
1881
1882	if (preq->request.num_sgs) {
1883		num_sgs = preq->request.num_sgs;
1884		sg = preq->request.sg;
1885		addr = (u64)sg_dma_address(sg);
1886		block_len = sg_dma_len(sg);
1887		num_trbs = count_sg_trbs_needed(preq);
1888	} else {
1889		num_trbs = count_trbs_needed(preq);
1890		addr = (u64)preq->request.dma;
1891		block_len = full_len;
1892	}
1893
1894	pep = preq->pep;
1895
1896	/* Deal with request.zero - need one more td/trb. */
1897	if (preq->request.zero && preq->request.length &&
1898	    IS_ALIGNED(full_len, usb_endpoint_maxp(pep->endpoint.desc))) {
1899		need_zero_pkt = true;
1900		num_trbs++;
1901	}
1902
1903	ret = cdnsp_prepare_transfer(pdev, preq, num_trbs);
1904	if (ret)
1905		return ret;
1906
1907	/*
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1908	 * Don't give the first TRB to the hardware (by toggling the cycle bit)
1909	 * until we've finished creating all the other TRBs. The ring's cycle
1910	 * state may change as we enqueue the other TRBs, so save it too.
1911	 */
1912	start_trb = &ring->enqueue->generic;
1913	start_cycle = ring->cycle_state;
1914	send_addr = addr;
1915
1916	/* Queue the TRBs, even if they are zero-length */
1917	for (enqd_len = 0; zero_len_trb || first_trb || enqd_len < full_len;
1918	     enqd_len += trb_buff_len) {
1919		field = TRB_TYPE(TRB_NORMAL);
1920
1921		/* TRB buffer should not cross 64KB boundaries */
1922		trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
1923		trb_buff_len = min(trb_buff_len, block_len);
1924		if (enqd_len + trb_buff_len > full_len)
1925			trb_buff_len = full_len - enqd_len;
1926
1927		/* Don't change the cycle bit of the first TRB until later */
1928		if (first_trb) {
1929			first_trb = false;
1930			if (start_cycle == 0)
1931				field |= TRB_CYCLE;
1932		} else {
1933			field |= ring->cycle_state;
1934		}
1935
1936		/*
1937		 * Chain all the TRBs together; clear the chain bit in the last
1938		 * TRB to indicate it's the last TRB in the chain.
1939		 */
1940		if (enqd_len + trb_buff_len < full_len || need_zero_pkt) {
1941			field |= TRB_CHAIN;
1942			if (cdnsp_trb_is_link(ring->enqueue + 1)) {
1943				if (cdnsp_align_td(pdev, preq, enqd_len,
1944						   &trb_buff_len,
1945						   ring->enq_seg)) {
1946					send_addr = ring->enq_seg->bounce_dma;
1947					/* Assuming TD won't span 2 segs */
1948					preq->td.bounce_seg = ring->enq_seg;
1949				}
1950			}
1951		}
1952
1953		if (enqd_len + trb_buff_len >= full_len) {
1954			if (need_zero_pkt && !zero_len_trb) {
1955				zero_len_trb = true;
1956			} else {
1957				zero_len_trb = false;
1958				field &= ~TRB_CHAIN;
1959				field |= TRB_IOC;
1960				more_trbs_coming = false;
1961				need_zero_pkt = false;
1962				preq->td.last_trb = ring->enqueue;
1963			}
1964		}
1965
1966		/* Only set interrupt on short packet for OUT endpoints. */
1967		if (!preq->direction)
1968			field |= TRB_ISP;
1969
1970		/* Set the TRB length, TD size, and interrupter fields. */
1971		remainder = cdnsp_td_remainder(pdev, enqd_len, trb_buff_len,
1972					       full_len, preq,
1973					       more_trbs_coming,
1974					       zero_len_trb);
1975
1976		length_field = TRB_LEN(trb_buff_len) | TRB_TD_SIZE(remainder) |
1977			TRB_INTR_TARGET(0);
1978
1979		cdnsp_queue_trb(pdev, ring, more_trbs_coming,
1980				lower_32_bits(send_addr),
1981				upper_32_bits(send_addr),
1982				length_field,
1983				field);
1984
1985		addr += trb_buff_len;
1986		sent_len = trb_buff_len;
1987		while (sg && sent_len >= block_len) {
1988			/* New sg entry */
1989			--num_sgs;
1990			sent_len -= block_len;
1991			if (num_sgs != 0) {
1992				sg = sg_next(sg);
1993				block_len = sg_dma_len(sg);
1994				addr = (u64)sg_dma_address(sg);
1995				addr += sent_len;
1996			}
1997		}
1998		block_len -= sent_len;
1999		send_addr = addr;
 
 
 
 
 
 
 
 
 
 
 
2000	}
2001
2002	cdnsp_check_trb_math(preq, enqd_len);
2003	ret = cdnsp_giveback_first_trb(pdev, pep, preq->request.stream_id,
2004				       start_cycle, start_trb);
2005
2006	if (ret)
2007		preq->td.drbl = 1;
2008
2009	return 0;
2010}
2011
2012int cdnsp_queue_ctrl_tx(struct cdnsp_device *pdev, struct cdnsp_request *preq)
2013{
2014	u32 field, length_field, zlp = 0;
2015	struct cdnsp_ep *pep = preq->pep;
2016	struct cdnsp_ring *ep_ring;
2017	int num_trbs;
2018	u32 maxp;
2019	int ret;
2020
2021	ep_ring = cdnsp_request_to_transfer_ring(pdev, preq);
2022	if (!ep_ring)
2023		return -EINVAL;
2024
2025	/* 1 TRB for data, 1 for status */
2026	num_trbs = (pdev->three_stage_setup) ? 2 : 1;
2027
2028	maxp = usb_endpoint_maxp(pep->endpoint.desc);
2029
2030	if (preq->request.zero && preq->request.length &&
2031	    (preq->request.length % maxp == 0)) {
2032		num_trbs++;
2033		zlp = 1;
2034	}
2035
2036	ret = cdnsp_prepare_transfer(pdev, preq, num_trbs);
2037	if (ret)
2038		return ret;
2039
2040	/* If there's data, queue data TRBs */
2041	if (preq->request.length > 0) {
2042		field = TRB_TYPE(TRB_DATA);
2043
2044		if (zlp)
2045			field |= TRB_CHAIN;
2046		else
2047			field |= TRB_IOC | (pdev->ep0_expect_in ? 0 : TRB_ISP);
2048
2049		if (pdev->ep0_expect_in)
2050			field |= TRB_DIR_IN;
2051
2052		length_field = TRB_LEN(preq->request.length) |
2053			       TRB_TD_SIZE(zlp) | TRB_INTR_TARGET(0);
2054
2055		cdnsp_queue_trb(pdev, ep_ring, true,
2056				lower_32_bits(preq->request.dma),
2057				upper_32_bits(preq->request.dma), length_field,
2058				field | ep_ring->cycle_state |
2059				TRB_SETUPID(pdev->setup_id) |
2060				pdev->setup_speed);
2061
2062		if (zlp) {
2063			field = TRB_TYPE(TRB_NORMAL) | TRB_IOC;
2064
2065			if (!pdev->ep0_expect_in)
2066				field = TRB_ISP;
2067
2068			cdnsp_queue_trb(pdev, ep_ring, true,
2069					lower_32_bits(preq->request.dma),
2070					upper_32_bits(preq->request.dma), 0,
2071					field | ep_ring->cycle_state |
2072					TRB_SETUPID(pdev->setup_id) |
2073					pdev->setup_speed);
2074		}
2075
2076		pdev->ep0_stage = CDNSP_DATA_STAGE;
2077	}
2078
2079	/* Save the DMA address of the last TRB in the TD. */
2080	preq->td.last_trb = ep_ring->enqueue;
2081
2082	/* Queue status TRB. */
2083	if (preq->request.length == 0)
2084		field = ep_ring->cycle_state;
2085	else
2086		field = (ep_ring->cycle_state ^ 1);
2087
2088	if (preq->request.length > 0 && pdev->ep0_expect_in)
2089		field |= TRB_DIR_IN;
2090
2091	if (pep->ep_state & EP0_HALTED_STATUS) {
2092		pep->ep_state &= ~EP0_HALTED_STATUS;
2093		field |= TRB_SETUPSTAT(TRB_SETUPSTAT_STALL);
2094	} else {
2095		field |= TRB_SETUPSTAT(TRB_SETUPSTAT_ACK);
2096	}
2097
2098	cdnsp_queue_trb(pdev, ep_ring, false, 0, 0, TRB_INTR_TARGET(0),
2099			field | TRB_IOC | TRB_SETUPID(pdev->setup_id) |
2100			TRB_TYPE(TRB_STATUS) | pdev->setup_speed);
2101
2102	cdnsp_ring_ep_doorbell(pdev, pep, preq->request.stream_id);
2103
2104	return 0;
2105}
2106
2107int cdnsp_cmd_stop_ep(struct cdnsp_device *pdev, struct cdnsp_ep *pep)
2108{
2109	u32 ep_state = GET_EP_CTX_STATE(pep->out_ctx);
2110	int ret = 0;
2111
2112	if (ep_state == EP_STATE_STOPPED || ep_state == EP_STATE_DISABLED ||
2113	    ep_state == EP_STATE_HALTED) {
2114		trace_cdnsp_ep_stopped_or_disabled(pep->out_ctx);
2115		goto ep_stopped;
2116	}
2117
2118	cdnsp_queue_stop_endpoint(pdev, pep->idx);
2119	cdnsp_ring_cmd_db(pdev);
2120	ret = cdnsp_wait_for_cmd_compl(pdev);
2121
2122	trace_cdnsp_handle_cmd_stop_ep(pep->out_ctx);
2123
2124ep_stopped:
2125	pep->ep_state |= EP_STOPPED;
2126	return ret;
2127}
2128
2129/*
2130 * The transfer burst count field of the isochronous TRB defines the number of
2131 * bursts that are required to move all packets in this TD. Only SuperSpeed
2132 * devices can burst up to bMaxBurst number of packets per service interval.
2133 * This field is zero based, meaning a value of zero in the field means one
2134 * burst. Basically, for everything but SuperSpeed devices, this field will be
2135 * zero.
2136 */
2137static unsigned int cdnsp_get_burst_count(struct cdnsp_device *pdev,
2138					  struct cdnsp_request *preq,
2139					  unsigned int total_packet_count)
2140{
2141	unsigned int max_burst;
2142
2143	if (pdev->gadget.speed < USB_SPEED_SUPER)
2144		return 0;
2145
2146	max_burst = preq->pep->endpoint.comp_desc->bMaxBurst;
2147	return DIV_ROUND_UP(total_packet_count, max_burst + 1) - 1;
2148}
2149
2150/*
2151 * Returns the number of packets in the last "burst" of packets. This field is
2152 * valid for all speeds of devices. USB 2.0 devices can only do one "burst", so
2153 * the last burst packet count is equal to the total number of packets in the
2154 * TD. SuperSpeed endpoints can have up to 3 bursts. All but the last burst
2155 * must contain (bMaxBurst + 1) number of packets, but the last burst can
2156 * contain 1 to (bMaxBurst + 1) packets.
2157 */
2158static unsigned int
2159	cdnsp_get_last_burst_packet_count(struct cdnsp_device *pdev,
2160					  struct cdnsp_request *preq,
2161					  unsigned int total_packet_count)
2162{
2163	unsigned int max_burst;
2164	unsigned int residue;
2165
2166	if (pdev->gadget.speed >= USB_SPEED_SUPER) {
2167		/* bMaxBurst is zero based: 0 means 1 packet per burst. */
2168		max_burst = preq->pep->endpoint.comp_desc->bMaxBurst;
2169		residue = total_packet_count % (max_burst + 1);
2170
2171		/*
2172		 * If residue is zero, the last burst contains (max_burst + 1)
2173		 * number of packets, but the TLBPC field is zero-based.
2174		 */
2175		if (residue == 0)
2176			return max_burst;
2177
2178		return residue - 1;
2179	}
2180	if (total_packet_count == 0)
2181		return 0;
2182
2183	return total_packet_count - 1;
2184}
2185
2186/* Queue function isoc transfer */
2187int cdnsp_queue_isoc_tx(struct cdnsp_device *pdev,
2188			struct cdnsp_request *preq)
2189{
2190	unsigned int trb_buff_len, td_len, td_remain_len, block_len;
2191	unsigned int burst_count, last_burst_pkt;
2192	unsigned int total_pkt_count, max_pkt;
2193	struct cdnsp_generic_trb *start_trb;
2194	struct scatterlist *sg = NULL;
2195	bool more_trbs_coming = true;
2196	struct cdnsp_ring *ep_ring;
2197	unsigned int num_sgs = 0;
2198	int running_total = 0;
2199	u32 field, length_field;
2200	u64 addr, send_addr;
2201	int start_cycle;
2202	int trbs_per_td;
2203	int i, sent_len, ret;
2204
2205	ep_ring = preq->pep->ring;
2206
2207	td_len = preq->request.length;
2208
2209	if (preq->request.num_sgs) {
2210		num_sgs = preq->request.num_sgs;
2211		sg = preq->request.sg;
2212		addr = (u64)sg_dma_address(sg);
2213		block_len = sg_dma_len(sg);
2214		trbs_per_td = count_sg_trbs_needed(preq);
2215	} else {
2216		addr = (u64)preq->request.dma;
2217		block_len = td_len;
2218		trbs_per_td = count_trbs_needed(preq);
2219	}
2220
2221	ret = cdnsp_prepare_transfer(pdev, preq, trbs_per_td);
2222	if (ret)
2223		return ret;
2224
2225	start_trb = &ep_ring->enqueue->generic;
2226	start_cycle = ep_ring->cycle_state;
2227	td_remain_len = td_len;
2228	send_addr = addr;
2229
2230	max_pkt = usb_endpoint_maxp(preq->pep->endpoint.desc);
2231	total_pkt_count = DIV_ROUND_UP(td_len, max_pkt);
2232
2233	/* A zero-length transfer still involves at least one packet. */
2234	if (total_pkt_count == 0)
2235		total_pkt_count++;
2236
2237	burst_count = cdnsp_get_burst_count(pdev, preq, total_pkt_count);
2238	last_burst_pkt = cdnsp_get_last_burst_packet_count(pdev, preq,
2239							   total_pkt_count);
2240
2241	/*
2242	 * Set isoc specific data for the first TRB in a TD.
2243	 * Prevent HW from getting the TRBs by keeping the cycle state
2244	 * inverted in the first TDs isoc TRB.
2245	 */
2246	field = TRB_TYPE(TRB_ISOC) | TRB_TLBPC(last_burst_pkt) |
2247		TRB_SIA | TRB_TBC(burst_count);
2248
2249	if (!start_cycle)
2250		field |= TRB_CYCLE;
2251
2252	/* Fill the rest of the TRB fields, and remaining normal TRBs. */
2253	for (i = 0; i < trbs_per_td; i++) {
2254		u32 remainder;
2255
2256		/* Calculate TRB length. */
2257		trb_buff_len = TRB_BUFF_LEN_UP_TO_BOUNDARY(addr);
2258		trb_buff_len = min(trb_buff_len, block_len);
2259		if (trb_buff_len > td_remain_len)
2260			trb_buff_len = td_remain_len;
2261
2262		/* Set the TRB length, TD size, & interrupter fields. */
2263		remainder = cdnsp_td_remainder(pdev, running_total,
2264					       trb_buff_len, td_len, preq,
2265					       more_trbs_coming, 0);
2266
2267		length_field = TRB_LEN(trb_buff_len) | TRB_TD_SIZE(remainder) |
2268			TRB_INTR_TARGET(0);
2269
2270		/* Only first TRB is isoc, overwrite otherwise. */
2271		if (i) {
2272			field = TRB_TYPE(TRB_NORMAL) | ep_ring->cycle_state;
2273			length_field |= TRB_TD_SIZE(remainder);
2274		} else {
2275			length_field |= TRB_TD_SIZE_TBC(burst_count);
2276		}
2277
2278		/* Only set interrupt on short packet for OUT EPs. */
2279		if (usb_endpoint_dir_out(preq->pep->endpoint.desc))
2280			field |= TRB_ISP;
2281
2282		/* Set the chain bit for all except the last TRB. */
2283		if (i < trbs_per_td - 1) {
2284			more_trbs_coming = true;
2285			field |= TRB_CHAIN;
2286		} else {
2287			more_trbs_coming = false;
2288			preq->td.last_trb = ep_ring->enqueue;
2289			field |= TRB_IOC;
2290		}
2291
2292		cdnsp_queue_trb(pdev, ep_ring, more_trbs_coming,
2293				lower_32_bits(send_addr), upper_32_bits(send_addr),
2294				length_field, field);
2295
2296		running_total += trb_buff_len;
2297		addr += trb_buff_len;
2298		td_remain_len -= trb_buff_len;
2299
2300		sent_len = trb_buff_len;
2301		while (sg && sent_len >= block_len) {
2302			/* New sg entry */
2303			--num_sgs;
2304			sent_len -= block_len;
2305			if (num_sgs != 0) {
2306				sg = sg_next(sg);
2307				block_len = sg_dma_len(sg);
2308				addr = (u64)sg_dma_address(sg);
2309				addr += sent_len;
2310			}
2311		}
2312		block_len -= sent_len;
2313		send_addr = addr;
2314	}
2315
2316	/* Check TD length */
2317	if (running_total != td_len) {
2318		dev_err(pdev->dev, "ISOC TD length unmatch\n");
2319		ret = -EINVAL;
2320		goto cleanup;
2321	}
2322
2323	cdnsp_giveback_first_trb(pdev, preq->pep, preq->request.stream_id,
2324				 start_cycle, start_trb);
2325
2326	return 0;
2327
2328cleanup:
2329	/* Clean up a partially enqueued isoc transfer. */
2330	list_del_init(&preq->td.td_list);
2331	ep_ring->num_tds--;
2332
2333	/*
2334	 * Use the first TD as a temporary variable to turn the TDs we've
2335	 * queued into No-ops with a software-owned cycle bit.
2336	 * That way the hardware won't accidentally start executing bogus TDs
2337	 * when we partially overwrite them.
2338	 * td->first_trb and td->start_seg are already set.
2339	 */
2340	preq->td.last_trb = ep_ring->enqueue;
2341	/* Every TRB except the first & last will have its cycle bit flipped. */
2342	cdnsp_td_to_noop(pdev, ep_ring, &preq->td, true);
2343
2344	/* Reset the ring enqueue back to the first TRB and its cycle bit. */
2345	ep_ring->enqueue = preq->td.first_trb;
2346	ep_ring->enq_seg = preq->td.start_seg;
2347	ep_ring->cycle_state = start_cycle;
2348	return ret;
2349}
2350
2351/****		Command Ring Operations		****/
2352/*
2353 * Generic function for queuing a command TRB on the command ring.
2354 * Driver queue only one command to ring in the moment.
2355 */
2356static void cdnsp_queue_command(struct cdnsp_device *pdev,
2357				u32 field1,
2358				u32 field2,
2359				u32 field3,
2360				u32 field4)
2361{
2362	cdnsp_prepare_ring(pdev, pdev->cmd_ring, EP_STATE_RUNNING, 1,
2363			   GFP_ATOMIC);
2364
2365	pdev->cmd.command_trb = pdev->cmd_ring->enqueue;
2366
2367	cdnsp_queue_trb(pdev, pdev->cmd_ring, false, field1, field2,
2368			field3, field4 | pdev->cmd_ring->cycle_state);
2369}
2370
2371/* Queue a slot enable or disable request on the command ring */
2372void cdnsp_queue_slot_control(struct cdnsp_device *pdev, u32 trb_type)
2373{
2374	cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(trb_type) |
2375			    SLOT_ID_FOR_TRB(pdev->slot_id));
2376}
2377
2378/* Queue an address device command TRB */
2379void cdnsp_queue_address_device(struct cdnsp_device *pdev,
2380				dma_addr_t in_ctx_ptr,
2381				enum cdnsp_setup_dev setup)
2382{
2383	cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr),
2384			    upper_32_bits(in_ctx_ptr), 0,
2385			    TRB_TYPE(TRB_ADDR_DEV) |
2386			    SLOT_ID_FOR_TRB(pdev->slot_id) |
2387			    (setup == SETUP_CONTEXT_ONLY ? TRB_BSR : 0));
2388}
2389
2390/* Queue a reset device command TRB */
2391void cdnsp_queue_reset_device(struct cdnsp_device *pdev)
2392{
2393	cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_RESET_DEV) |
2394			    SLOT_ID_FOR_TRB(pdev->slot_id));
2395}
2396
2397/* Queue a configure endpoint command TRB */
2398void cdnsp_queue_configure_endpoint(struct cdnsp_device *pdev,
2399				    dma_addr_t in_ctx_ptr)
2400{
2401	cdnsp_queue_command(pdev, lower_32_bits(in_ctx_ptr),
2402			    upper_32_bits(in_ctx_ptr), 0,
2403			    TRB_TYPE(TRB_CONFIG_EP) |
2404			    SLOT_ID_FOR_TRB(pdev->slot_id));
2405}
2406
2407/*
2408 * Suspend is set to indicate "Stop Endpoint Command" is being issued to stop
2409 * activity on an endpoint that is about to be suspended.
2410 */
2411void cdnsp_queue_stop_endpoint(struct cdnsp_device *pdev, unsigned int ep_index)
2412{
2413	cdnsp_queue_command(pdev, 0, 0, 0, SLOT_ID_FOR_TRB(pdev->slot_id) |
2414			    EP_ID_FOR_TRB(ep_index) | TRB_TYPE(TRB_STOP_RING));
2415}
2416
2417/* Set Transfer Ring Dequeue Pointer command. */
2418void cdnsp_queue_new_dequeue_state(struct cdnsp_device *pdev,
2419				   struct cdnsp_ep *pep,
2420				   struct cdnsp_dequeue_state *deq_state)
2421{
2422	u32 trb_stream_id = STREAM_ID_FOR_TRB(deq_state->stream_id);
2423	u32 trb_slot_id = SLOT_ID_FOR_TRB(pdev->slot_id);
2424	u32 type = TRB_TYPE(TRB_SET_DEQ);
2425	u32 trb_sct = 0;
2426	dma_addr_t addr;
2427
2428	addr = cdnsp_trb_virt_to_dma(deq_state->new_deq_seg,
2429				     deq_state->new_deq_ptr);
2430
2431	if (deq_state->stream_id)
2432		trb_sct = SCT_FOR_TRB(SCT_PRI_TR);
2433
2434	cdnsp_queue_command(pdev, lower_32_bits(addr) | trb_sct |
2435			    deq_state->new_cycle_state, upper_32_bits(addr),
2436			    trb_stream_id, trb_slot_id |
2437			    EP_ID_FOR_TRB(pep->idx) | type);
2438}
2439
2440void cdnsp_queue_reset_ep(struct cdnsp_device *pdev, unsigned int ep_index)
2441{
2442	return cdnsp_queue_command(pdev, 0, 0, 0,
2443				   SLOT_ID_FOR_TRB(pdev->slot_id) |
2444				   EP_ID_FOR_TRB(ep_index) |
2445				   TRB_TYPE(TRB_RESET_EP));
2446}
2447
2448/*
2449 * Queue a halt endpoint request on the command ring.
2450 */
2451void cdnsp_queue_halt_endpoint(struct cdnsp_device *pdev, unsigned int ep_index)
2452{
2453	cdnsp_queue_command(pdev, 0, 0, 0, TRB_TYPE(TRB_HALT_ENDPOINT) |
2454			    SLOT_ID_FOR_TRB(pdev->slot_id) |
2455			    EP_ID_FOR_TRB(ep_index));
2456}
2457
2458void cdnsp_force_header_wakeup(struct cdnsp_device *pdev, int intf_num)
2459{
2460	u32 lo, mid;
2461
2462	lo = TRB_FH_TO_PACKET_TYPE(TRB_FH_TR_PACKET) |
2463	     TRB_FH_TO_DEVICE_ADDRESS(pdev->device_address);
2464	mid = TRB_FH_TR_PACKET_DEV_NOT |
2465	      TRB_FH_TO_NOT_TYPE(TRB_FH_TR_PACKET_FUNCTION_WAKE) |
2466	      TRB_FH_TO_INTERFACE(intf_num);
2467
2468	cdnsp_queue_command(pdev, lo, mid, 0,
2469			    TRB_TYPE(TRB_FORCE_HEADER) | SET_PORT_ID(2));
2470}