1/*
   2 * Copyright (C) 2001-2004 by David Brownell
   3 *
   4 * This program is free software; you can redistribute it and/or modify it
   5 * under the terms of the GNU General Public License as published by the
   6 * Free Software Foundation; either version 2 of the License, or (at your
   7 * option) any later version.
   8 *
   9 * This program is distributed in the hope that it will be useful, but
  10 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  11 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  12 * for more details.
  13 *
  14 * You should have received a copy of the GNU General Public License
  15 * along with this program; if not, write to the Free Software Foundation,
  16 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  17 */
  18
  19/* this file is part of ehci-hcd.c */
  20
  21/*-------------------------------------------------------------------------*/
  22
  23/*
  24 * EHCI hardware queue manipulation ... the core.  QH/QTD manipulation.
  25 *
  26 * Control, bulk, and interrupt traffic all use "qh" lists.  They list "qtd"
  27 * entries describing USB transactions, max 16-20kB/entry (with 4kB-aligned
  28 * buffers needed for the larger number).  We use one QH per endpoint, queue
  29 * multiple urbs (all three types) per endpoint.  URBs may need several qtds.
  30 *
  31 * ISO traffic uses "ISO TD" (itd, and sitd) records, and (along with
  32 * interrupts) needs careful scheduling.  Performance improvements can be
  33 * an ongoing challenge.  That's in "ehci-sched.c".
  34 *
  35 * USB 1.1 devices are handled (a) by "companion" OHCI or UHCI root hubs,
  36 * or otherwise through transaction translators (TTs) in USB 2.0 hubs using
  37 * (b) special fields in qh entries or (c) split iso entries.  TTs will
  38 * buffer low/full speed data so the host collects it at high speed.
  39 */
  40
  41/*-------------------------------------------------------------------------*/
  42
  43/* fill a qtd, returning how much of the buffer we were able to queue up */
  44
  45static int
  46qtd_fill(struct ehci_hcd *ehci, struct ehci_qtd *qtd, dma_addr_t buf,
  47		  size_t len, int token, int maxpacket)
  48{
  49	int	i, count;
  50	u64	addr = buf;
  51
  52	/* one buffer entry per 4K ... first might be short or unaligned */
  53	qtd->hw_buf[0] = cpu_to_hc32(ehci, (u32)addr);
  54	qtd->hw_buf_hi[0] = cpu_to_hc32(ehci, (u32)(addr >> 32));
  55	count = 0x1000 - (buf & 0x0fff);	/* rest of that page */
  56	if (likely (len < count))		/* ... iff needed */
  57		count = len;
  58	else {
  59		buf +=  0x1000;
  60		buf &= ~0x0fff;
  61
  62		/* per-qtd limit: from 16K to 20K (best alignment) */
  63		for (i = 1; count < len && i < 5; i++) {
  64			addr = buf;
  65			qtd->hw_buf[i] = cpu_to_hc32(ehci, (u32)addr);
  66			qtd->hw_buf_hi[i] = cpu_to_hc32(ehci,
  67					(u32)(addr >> 32));
  68			buf += 0x1000;
  69			if ((count + 0x1000) < len)
  70				count += 0x1000;
  71			else
  72				count = len;
  73		}
  74
  75		/* short packets may only terminate transfers */
  76		if (count != len)
  77			count -= (count % maxpacket);
  78	}
  79	qtd->hw_token = cpu_to_hc32(ehci, (count << 16) | token);
  80	qtd->length = count;
  81
  82	return count;
  83}
  84
  85/*-------------------------------------------------------------------------*/
  86
  87static inline void
  88qh_update (struct ehci_hcd *ehci, struct ehci_qh *qh, struct ehci_qtd *qtd)
  89{
  90	struct ehci_qh_hw *hw = qh->hw;
  91
  92	/* writes to an active overlay are unsafe */
  93	BUG_ON(qh->qh_state != QH_STATE_IDLE);
  94
  95	hw->hw_qtd_next = QTD_NEXT(ehci, qtd->qtd_dma);
  96	hw->hw_alt_next = EHCI_LIST_END(ehci);
  97
  98	/* Except for control endpoints, we make hardware maintain data
  99	 * toggle (like OHCI) ... here (re)initialize the toggle in the QH,
 100	 * and set the pseudo-toggle in udev. Only usb_clear_halt() will
 101	 * ever clear it.
 102	 */
 103	if (!(hw->hw_info1 & cpu_to_hc32(ehci, 1 << 14))) {
 104		unsigned	is_out, epnum;
 105
 106		is_out = qh->is_out;
 107		epnum = (hc32_to_cpup(ehci, &hw->hw_info1) >> 8) & 0x0f;
 108		if (unlikely (!usb_gettoggle (qh->dev, epnum, is_out))) {
 109			hw->hw_token &= ~cpu_to_hc32(ehci, QTD_TOGGLE);
 110			usb_settoggle (qh->dev, epnum, is_out, 1);
 111		}
 112	}
 113
 114	hw->hw_token &= cpu_to_hc32(ehci, QTD_TOGGLE | QTD_STS_PING);
 115}
 116
 117/* if it weren't for a common silicon quirk (writing the dummy into the qh
 118 * overlay, so qh->hw_token wrongly becomes inactive/halted), only fault
 119 * recovery (including urb dequeue) would need software changes to a QH...
 120 */
 121static void
 122qh_refresh (struct ehci_hcd *ehci, struct ehci_qh *qh)
 123{
 124	struct ehci_qtd *qtd;
 125
 126	if (list_empty (&qh->qtd_list))
 127		qtd = qh->dummy;
 128	else {
 129		qtd = list_entry (qh->qtd_list.next,
 130				struct ehci_qtd, qtd_list);
 131		/*
 132		 * first qtd may already be partially processed.
 133		 * If we come here during unlink, the QH overlay region
 134		 * might have reference to the just unlinked qtd. The
 135		 * qtd is updated in qh_completions(). Update the QH
 136		 * overlay here.
 137		 */
 138		if (cpu_to_hc32(ehci, qtd->qtd_dma) == qh->hw->hw_current) {
 139			qh->hw->hw_qtd_next = qtd->hw_next;
 140			qtd = NULL;
 141		}
 142	}
 143
 144	if (qtd)
 145		qh_update (ehci, qh, qtd);
 
 
 
 
 
 
 
 
 
 146}
 147
 148/*-------------------------------------------------------------------------*/
 149
 150static void qh_link_async(struct ehci_hcd *ehci, struct ehci_qh *qh);
 151
 152static void ehci_clear_tt_buffer_complete(struct usb_hcd *hcd,
 153		struct usb_host_endpoint *ep)
 154{
 155	struct ehci_hcd		*ehci = hcd_to_ehci(hcd);
 156	struct ehci_qh		*qh = ep->hcpriv;
 157	unsigned long		flags;
 158
 159	spin_lock_irqsave(&ehci->lock, flags);
 160	qh->clearing_tt = 0;
 161	if (qh->qh_state == QH_STATE_IDLE && !list_empty(&qh->qtd_list)
 162			&& ehci->rh_state == EHCI_RH_RUNNING)
 163		qh_link_async(ehci, qh);
 164	spin_unlock_irqrestore(&ehci->lock, flags);
 165}
 166
 167static void ehci_clear_tt_buffer(struct ehci_hcd *ehci, struct ehci_qh *qh,
 168		struct urb *urb, u32 token)
 169{
 170
 171	/* If an async split transaction gets an error or is unlinked,
 172	 * the TT buffer may be left in an indeterminate state.  We
 173	 * have to clear the TT buffer.
 174	 *
 175	 * Note: this routine is never called for Isochronous transfers.
 176	 */
 177	if (urb->dev->tt && !usb_pipeint(urb->pipe) && !qh->clearing_tt) {
 178#ifdef DEBUG
 179		struct usb_device *tt = urb->dev->tt->hub;
 180		dev_dbg(&tt->dev,
 181			"clear tt buffer port %d, a%d ep%d t%08x\n",
 182			urb->dev->ttport, urb->dev->devnum,
 183			usb_pipeendpoint(urb->pipe), token);
 184#endif /* DEBUG */
 185		if (!ehci_is_TDI(ehci)
 186				|| urb->dev->tt->hub !=
 187				   ehci_to_hcd(ehci)->self.root_hub) {
 188			if (usb_hub_clear_tt_buffer(urb) == 0)
 189				qh->clearing_tt = 1;
 190		} else {
 191
 192			/* REVISIT ARC-derived cores don't clear the root
 193			 * hub TT buffer in this way...
 194			 */
 195		}
 196	}
 197}
 198
 199static int qtd_copy_status (
 200	struct ehci_hcd *ehci,
 201	struct urb *urb,
 202	size_t length,
 203	u32 token
 204)
 205{
 206	int	status = -EINPROGRESS;
 207
 208	/* count IN/OUT bytes, not SETUP (even short packets) */
 209	if (likely (QTD_PID (token) != 2))
 210		urb->actual_length += length - QTD_LENGTH (token);
 211
 212	/* don't modify error codes */
 213	if (unlikely(urb->unlinked))
 214		return status;
 215
 216	/* force cleanup after short read; not always an error */
 217	if (unlikely (IS_SHORT_READ (token)))
 218		status = -EREMOTEIO;
 219
 220	/* serious "can't proceed" faults reported by the hardware */
 221	if (token & QTD_STS_HALT) {
 222		if (token & QTD_STS_BABBLE) {
 223			/* FIXME "must" disable babbling device's port too */
 224			status = -EOVERFLOW;
 225		/* CERR nonzero + halt --> stall */
 226		} else if (QTD_CERR(token)) {
 227			status = -EPIPE;
 228
 229		/* In theory, more than one of the following bits can be set
 230		 * since they are sticky and the transaction is retried.
 231		 * Which to test first is rather arbitrary.
 232		 */
 233		} else if (token & QTD_STS_MMF) {
 234			/* fs/ls interrupt xfer missed the complete-split */
 235			status = -EPROTO;
 236		} else if (token & QTD_STS_DBE) {
 237			status = (QTD_PID (token) == 1) /* IN ? */
 238				? -ENOSR  /* hc couldn't read data */
 239				: -ECOMM; /* hc couldn't write data */
 240		} else if (token & QTD_STS_XACT) {
 241			/* timeout, bad CRC, wrong PID, etc */
 242			ehci_dbg(ehci, "devpath %s ep%d%s 3strikes\n",
 243				urb->dev->devpath,
 244				usb_pipeendpoint(urb->pipe),
 245				usb_pipein(urb->pipe) ? "in" : "out");
 246			status = -EPROTO;
 247		} else {	/* unknown */
 248			status = -EPROTO;
 249		}
 250
 251		ehci_vdbg (ehci,
 252			"dev%d ep%d%s qtd token %08x --> status %d\n",
 253			usb_pipedevice (urb->pipe),
 254			usb_pipeendpoint (urb->pipe),
 255			usb_pipein (urb->pipe) ? "in" : "out",
 256			token, status);
 257	}
 258
 259	return status;
 260}
 261
 262static void
 263ehci_urb_done(struct ehci_hcd *ehci, struct urb *urb, int status)
 264__releases(ehci->lock)
 265__acquires(ehci->lock)
 266{
 267	if (likely (urb->hcpriv != NULL)) {
 268		struct ehci_qh	*qh = (struct ehci_qh *) urb->hcpriv;
 269
 270		/* S-mask in a QH means it's an interrupt urb */
 271		if ((qh->hw->hw_info2 & cpu_to_hc32(ehci, QH_SMASK)) != 0) {
 272
 273			/* ... update hc-wide periodic stats (for usbfs) */
 274			ehci_to_hcd(ehci)->self.bandwidth_int_reqs--;
 275		}
 276		qh_put (qh);
 277	}
 278
 279	if (unlikely(urb->unlinked)) {
 280		COUNT(ehci->stats.unlink);
 281	} else {
 282		/* report non-error and short read status as zero */
 283		if (status == -EINPROGRESS || status == -EREMOTEIO)
 284			status = 0;
 285		COUNT(ehci->stats.complete);
 286	}
 287
 288#ifdef EHCI_URB_TRACE
 289	ehci_dbg (ehci,
 290		"%s %s urb %p ep%d%s status %d len %d/%d\n",
 291		__func__, urb->dev->devpath, urb,
 292		usb_pipeendpoint (urb->pipe),
 293		usb_pipein (urb->pipe) ? "in" : "out",
 294		status,
 295		urb->actual_length, urb->transfer_buffer_length);
 296#endif
 297
 298	/* complete() can reenter this HCD */
 299	usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
 300	spin_unlock (&ehci->lock);
 301	usb_hcd_giveback_urb(ehci_to_hcd(ehci), urb, status);
 302	spin_lock (&ehci->lock);
 303}
 304
 305static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);
 306static void unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh);
 307
 308static int qh_schedule (struct ehci_hcd *ehci, struct ehci_qh *qh);
 309
 310/*
 311 * Process and free completed qtds for a qh, returning URBs to drivers.
 312 * Chases up to qh->hw_current.  Returns number of completions called,
 313 * indicating how much "real" work we did.
 314 */
 315static unsigned
 316qh_completions (struct ehci_hcd *ehci, struct ehci_qh *qh)
 317{
 318	struct ehci_qtd		*last, *end = qh->dummy;
 319	struct list_head	*entry, *tmp;
 320	int			last_status;
 321	int			stopped;
 322	unsigned		count = 0;
 323	u8			state;
 324	struct ehci_qh_hw	*hw = qh->hw;
 325
 326	if (unlikely (list_empty (&qh->qtd_list)))
 327		return count;
 328
 329	/* completions (or tasks on other cpus) must never clobber HALT
 330	 * till we've gone through and cleaned everything up, even when
 331	 * they add urbs to this qh's queue or mark them for unlinking.
 332	 *
 333	 * NOTE:  unlinking expects to be done in queue order.
 334	 *
 335	 * It's a bug for qh->qh_state to be anything other than
 336	 * QH_STATE_IDLE, unless our caller is scan_async() or
 337	 * scan_periodic().
 338	 */
 339	state = qh->qh_state;
 340	qh->qh_state = QH_STATE_COMPLETING;
 341	stopped = (state == QH_STATE_IDLE);
 342
 343 rescan:
 344	last = NULL;
 345	last_status = -EINPROGRESS;
 346	qh->needs_rescan = 0;
 347
 348	/* remove de-activated QTDs from front of queue.
 349	 * after faults (including short reads), cleanup this urb
 350	 * then let the queue advance.
 351	 * if queue is stopped, handles unlinks.
 352	 */
 353	list_for_each_safe (entry, tmp, &qh->qtd_list) {
 354		struct ehci_qtd	*qtd;
 355		struct urb	*urb;
 356		u32		token = 0;
 357
 358		qtd = list_entry (entry, struct ehci_qtd, qtd_list);
 359		urb = qtd->urb;
 360
 361		/* clean up any state from previous QTD ...*/
 362		if (last) {
 363			if (likely (last->urb != urb)) {
 364				ehci_urb_done(ehci, last->urb, last_status);
 365				count++;
 366				last_status = -EINPROGRESS;
 367			}
 368			ehci_qtd_free (ehci, last);
 369			last = NULL;
 370		}
 371
 372		/* ignore urbs submitted during completions we reported */
 373		if (qtd == end)
 374			break;
 375
 376		/* hardware copies qtd out of qh overlay */
 377		rmb ();
 378		token = hc32_to_cpu(ehci, qtd->hw_token);
 379
 380		/* always clean up qtds the hc de-activated */
 381 retry_xacterr:
 382		if ((token & QTD_STS_ACTIVE) == 0) {
 383
 384			/* Report Data Buffer Error: non-fatal but useful */
 385			if (token & QTD_STS_DBE)
 386				ehci_dbg(ehci,
 387					"detected DataBufferErr for urb %p ep%d%s len %d, qtd %p [qh %p]\n",
 388					urb,
 389					usb_endpoint_num(&urb->ep->desc),
 390					usb_endpoint_dir_in(&urb->ep->desc) ? "in" : "out",
 391					urb->transfer_buffer_length,
 392					qtd,
 393					qh);
 394
 395			/* on STALL, error, and short reads this urb must
 396			 * complete and all its qtds must be recycled.
 397			 */
 398			if ((token & QTD_STS_HALT) != 0) {
 399
 400				/* retry transaction errors until we
 401				 * reach the software xacterr limit
 402				 */
 403				if ((token & QTD_STS_XACT) &&
 404						QTD_CERR(token) == 0 &&
 405						++qh->xacterrs < QH_XACTERR_MAX &&
 406						!urb->unlinked) {
 407					ehci_dbg(ehci,
 408	"detected XactErr len %zu/%zu retry %d\n",
 409	qtd->length - QTD_LENGTH(token), qtd->length, qh->xacterrs);
 410
 411					/* reset the token in the qtd and the
 412					 * qh overlay (which still contains
 413					 * the qtd) so that we pick up from
 414					 * where we left off
 415					 */
 416					token &= ~QTD_STS_HALT;
 417					token |= QTD_STS_ACTIVE |
 418							(EHCI_TUNE_CERR << 10);
 419					qtd->hw_token = cpu_to_hc32(ehci,
 420							token);
 421					wmb();
 422					hw->hw_token = cpu_to_hc32(ehci,
 423							token);
 424					goto retry_xacterr;
 425				}
 426				stopped = 1;
 427
 428			/* magic dummy for some short reads; qh won't advance.
 429			 * that silicon quirk can kick in with this dummy too.
 430			 *
 431			 * other short reads won't stop the queue, including
 432			 * control transfers (status stage handles that) or
 433			 * most other single-qtd reads ... the queue stops if
 434			 * URB_SHORT_NOT_OK was set so the driver submitting
 435			 * the urbs could clean it up.
 436			 */
 437			} else if (IS_SHORT_READ (token)
 438					&& !(qtd->hw_alt_next
 439						& EHCI_LIST_END(ehci))) {
 440				stopped = 1;
 441			}
 442
 443		/* stop scanning when we reach qtds the hc is using */
 444		} else if (likely (!stopped
 445				&& ehci->rh_state == EHCI_RH_RUNNING)) {
 446			break;
 447
 448		/* scan the whole queue for unlinks whenever it stops */
 449		} else {
 450			stopped = 1;
 451
 452			/* cancel everything if we halt, suspend, etc */
 453			if (ehci->rh_state != EHCI_RH_RUNNING)
 454				last_status = -ESHUTDOWN;
 455
 456			/* this qtd is active; skip it unless a previous qtd
 457			 * for its urb faulted, or its urb was canceled.
 458			 */
 459			else if (last_status == -EINPROGRESS && !urb->unlinked)
 460				continue;
 461
 462			/* qh unlinked; token in overlay may be most current */
 463			if (state == QH_STATE_IDLE
 464					&& cpu_to_hc32(ehci, qtd->qtd_dma)
 465						== hw->hw_current) {
 
 
 
 
 
 
 
 466				token = hc32_to_cpu(ehci, hw->hw_token);
 
 467
 468				/* An unlink may leave an incomplete
 469				 * async transaction in the TT buffer.
 470				 * We have to clear it.
 471				 */
 472				ehci_clear_tt_buffer(ehci, qh, urb, token);
 473			}
 474		}
 475
 476		/* unless we already know the urb's status, collect qtd status
 477		 * and update count of bytes transferred.  in common short read
 478		 * cases with only one data qtd (including control transfers),
 479		 * queue processing won't halt.  but with two or more qtds (for
 480		 * example, with a 32 KB transfer), when the first qtd gets a
 481		 * short read the second must be removed by hand.
 482		 */
 483		if (last_status == -EINPROGRESS) {
 484			last_status = qtd_copy_status(ehci, urb,
 485					qtd->length, token);
 486			if (last_status == -EREMOTEIO
 487					&& (qtd->hw_alt_next
 488						& EHCI_LIST_END(ehci)))
 489				last_status = -EINPROGRESS;
 490
 491			/* As part of low/full-speed endpoint-halt processing
 492			 * we must clear the TT buffer (11.17.5).
 493			 */
 494			if (unlikely(last_status != -EINPROGRESS &&
 495					last_status != -EREMOTEIO)) {
 496				/* The TT's in some hubs malfunction when they
 497				 * receive this request following a STALL (they
 498				 * stop sending isochronous packets).  Since a
 499				 * STALL can't leave the TT buffer in a busy
 500				 * state (if you believe Figures 11-48 - 11-51
 501				 * in the USB 2.0 spec), we won't clear the TT
 502				 * buffer in this case.  Strictly speaking this
 503				 * is a violation of the spec.
 504				 */
 505				if (last_status != -EPIPE)
 506					ehci_clear_tt_buffer(ehci, qh, urb,
 507							token);
 508			}
 509		}
 510
 511		/* if we're removing something not at the queue head,
 512		 * patch the hardware queue pointer.
 513		 */
 514		if (stopped && qtd->qtd_list.prev != &qh->qtd_list) {
 515			last = list_entry (qtd->qtd_list.prev,
 516					struct ehci_qtd, qtd_list);
 517			last->hw_next = qtd->hw_next;
 518		}
 519
 520		/* remove qtd; it's recycled after possible urb completion */
 521		list_del (&qtd->qtd_list);
 522		last = qtd;
 523
 524		/* reinit the xacterr counter for the next qtd */
 525		qh->xacterrs = 0;
 526	}
 527
 528	/* last urb's completion might still need calling */
 529	if (likely (last != NULL)) {
 530		ehci_urb_done(ehci, last->urb, last_status);
 531		count++;
 532		ehci_qtd_free (ehci, last);
 533	}
 534
 535	/* Do we need to rescan for URBs dequeued during a giveback? */
 536	if (unlikely(qh->needs_rescan)) {
 537		/* If the QH is already unlinked, do the rescan now. */
 538		if (state == QH_STATE_IDLE)
 539			goto rescan;
 540
 541		/* Otherwise we have to wait until the QH is fully unlinked.
 542		 * Our caller will start an unlink if qh->needs_rescan is
 543		 * set.  But if an unlink has already started, nothing needs
 544		 * to be done.
 545		 */
 546		if (state != QH_STATE_LINKED)
 547			qh->needs_rescan = 0;
 548	}
 549
 550	/* restore original state; caller must unlink or relink */
 551	qh->qh_state = state;
 552
 553	/* be sure the hardware's done with the qh before refreshing
 554	 * it after fault cleanup, or recovering from silicon wrongly
 555	 * overlaying the dummy qtd (which reduces DMA chatter).
 
 
 
 
 
 
 
 
 
 
 
 556	 */
 557	if (stopped != 0 || hw->hw_qtd_next == EHCI_LIST_END(ehci)) {
 558		switch (state) {
 559		case QH_STATE_IDLE:
 560			qh_refresh(ehci, qh);
 561			break;
 562		case QH_STATE_LINKED:
 563			/* We won't refresh a QH that's linked (after the HC
 564			 * stopped the queue).  That avoids a race:
 565			 *  - HC reads first part of QH;
 566			 *  - CPU updates that first part and the token;
 567			 *  - HC reads rest of that QH, including token
 568			 * Result:  HC gets an inconsistent image, and then
 569			 * DMAs to/from the wrong memory (corrupting it).
 570			 *
 571			 * That should be rare for interrupt transfers,
 572			 * except maybe high bandwidth ...
 573			 */
 574
 575			/* Tell the caller to start an unlink */
 576			qh->needs_rescan = 1;
 577			break;
 578		/* otherwise, unlink already started */
 579		}
 580	}
 581
 582	return count;
 583}
 584
 585/*-------------------------------------------------------------------------*/
 586
 587// high bandwidth multiplier, as encoded in highspeed endpoint descriptors
 588#define hb_mult(wMaxPacketSize) (1 + (((wMaxPacketSize) >> 11) & 0x03))
 589// ... and packet size, for any kind of endpoint descriptor
 590#define max_packet(wMaxPacketSize) ((wMaxPacketSize) & 0x07ff)
 591
 592/*
 593 * reverse of qh_urb_transaction:  free a list of TDs.
 594 * used for cleanup after errors, before HC sees an URB's TDs.
 595 */
 596static void qtd_list_free (
 597	struct ehci_hcd		*ehci,
 598	struct urb		*urb,
 599	struct list_head	*qtd_list
 600) {
 601	struct list_head	*entry, *temp;
 602
 603	list_for_each_safe (entry, temp, qtd_list) {
 604		struct ehci_qtd	*qtd;
 605
 606		qtd = list_entry (entry, struct ehci_qtd, qtd_list);
 607		list_del (&qtd->qtd_list);
 608		ehci_qtd_free (ehci, qtd);
 609	}
 610}
 611
 612/*
 613 * create a list of filled qtds for this URB; won't link into qh.
 614 */
 615static struct list_head *
 616qh_urb_transaction (
 617	struct ehci_hcd		*ehci,
 618	struct urb		*urb,
 619	struct list_head	*head,
 620	gfp_t			flags
 621) {
 622	struct ehci_qtd		*qtd, *qtd_prev;
 623	dma_addr_t		buf;
 624	int			len, this_sg_len, maxpacket;
 625	int			is_input;
 626	u32			token;
 627	int			i;
 628	struct scatterlist	*sg;
 629
 630	/*
 631	 * URBs map to sequences of QTDs:  one logical transaction
 632	 */
 633	qtd = ehci_qtd_alloc (ehci, flags);
 634	if (unlikely (!qtd))
 635		return NULL;
 636	list_add_tail (&qtd->qtd_list, head);
 637	qtd->urb = urb;
 638
 639	token = QTD_STS_ACTIVE;
 640	token |= (EHCI_TUNE_CERR << 10);
 641	/* for split transactions, SplitXState initialized to zero */
 642
 643	len = urb->transfer_buffer_length;
 644	is_input = usb_pipein (urb->pipe);
 645	if (usb_pipecontrol (urb->pipe)) {
 646		/* SETUP pid */
 647		qtd_fill(ehci, qtd, urb->setup_dma,
 648				sizeof (struct usb_ctrlrequest),
 649				token | (2 /* "setup" */ << 8), 8);
 650
 651		/* ... and always at least one more pid */
 652		token ^= QTD_TOGGLE;
 653		qtd_prev = qtd;
 654		qtd = ehci_qtd_alloc (ehci, flags);
 655		if (unlikely (!qtd))
 656			goto cleanup;
 657		qtd->urb = urb;
 658		qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
 659		list_add_tail (&qtd->qtd_list, head);
 660
 661		/* for zero length DATA stages, STATUS is always IN */
 662		if (len == 0)
 663			token |= (1 /* "in" */ << 8);
 664	}
 665
 666	/*
 667	 * data transfer stage:  buffer setup
 668	 */
 669	i = urb->num_mapped_sgs;
 670	if (len > 0 && i > 0) {
 671		sg = urb->sg;
 672		buf = sg_dma_address(sg);
 673
 674		/* urb->transfer_buffer_length may be smaller than the
 675		 * size of the scatterlist (or vice versa)
 676		 */
 677		this_sg_len = min_t(int, sg_dma_len(sg), len);
 678	} else {
 679		sg = NULL;
 680		buf = urb->transfer_dma;
 681		this_sg_len = len;
 682	}
 683
 684	if (is_input)
 685		token |= (1 /* "in" */ << 8);
 686	/* else it's already initted to "out" pid (0 << 8) */
 687
 688	maxpacket = max_packet(usb_maxpacket(urb->dev, urb->pipe, !is_input));
 689
 690	/*
 691	 * buffer gets wrapped in one or more qtds;
 692	 * last one may be "short" (including zero len)
 693	 * and may serve as a control status ack
 694	 */
 695	for (;;) {
 696		int this_qtd_len;
 697
 698		this_qtd_len = qtd_fill(ehci, qtd, buf, this_sg_len, token,
 699				maxpacket);
 700		this_sg_len -= this_qtd_len;
 701		len -= this_qtd_len;
 702		buf += this_qtd_len;
 703
 704		/*
 705		 * short reads advance to a "magic" dummy instead of the next
 706		 * qtd ... that forces the queue to stop, for manual cleanup.
 707		 * (this will usually be overridden later.)
 708		 */
 709		if (is_input)
 710			qtd->hw_alt_next = ehci->async->hw->hw_alt_next;
 711
 712		/* qh makes control packets use qtd toggle; maybe switch it */
 713		if ((maxpacket & (this_qtd_len + (maxpacket - 1))) == 0)
 714			token ^= QTD_TOGGLE;
 715
 716		if (likely(this_sg_len <= 0)) {
 717			if (--i <= 0 || len <= 0)
 718				break;
 719			sg = sg_next(sg);
 720			buf = sg_dma_address(sg);
 721			this_sg_len = min_t(int, sg_dma_len(sg), len);
 722		}
 723
 724		qtd_prev = qtd;
 725		qtd = ehci_qtd_alloc (ehci, flags);
 726		if (unlikely (!qtd))
 727			goto cleanup;
 728		qtd->urb = urb;
 729		qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
 730		list_add_tail (&qtd->qtd_list, head);
 731	}
 732
 733	/*
 734	 * unless the caller requires manual cleanup after short reads,
 735	 * have the alt_next mechanism keep the queue running after the
 736	 * last data qtd (the only one, for control and most other cases).
 737	 */
 738	if (likely ((urb->transfer_flags & URB_SHORT_NOT_OK) == 0
 739				|| usb_pipecontrol (urb->pipe)))
 740		qtd->hw_alt_next = EHCI_LIST_END(ehci);
 741
 742	/*
 743	 * control requests may need a terminating data "status" ack;
 744	 * other OUT ones may need a terminating short packet
 745	 * (zero length).
 746	 */
 747	if (likely (urb->transfer_buffer_length != 0)) {
 748		int	one_more = 0;
 749
 750		if (usb_pipecontrol (urb->pipe)) {
 751			one_more = 1;
 752			token ^= 0x0100;	/* "in" <--> "out"  */
 753			token |= QTD_TOGGLE;	/* force DATA1 */
 754		} else if (usb_pipeout(urb->pipe)
 755				&& (urb->transfer_flags & URB_ZERO_PACKET)
 756				&& !(urb->transfer_buffer_length % maxpacket)) {
 757			one_more = 1;
 758		}
 759		if (one_more) {
 760			qtd_prev = qtd;
 761			qtd = ehci_qtd_alloc (ehci, flags);
 762			if (unlikely (!qtd))
 763				goto cleanup;
 764			qtd->urb = urb;
 765			qtd_prev->hw_next = QTD_NEXT(ehci, qtd->qtd_dma);
 766			list_add_tail (&qtd->qtd_list, head);
 767
 768			/* never any data in such packets */
 769			qtd_fill(ehci, qtd, 0, 0, token, 0);
 770		}
 771	}
 772
 773	/* by default, enable interrupt on urb completion */
 774	if (likely (!(urb->transfer_flags & URB_NO_INTERRUPT)))
 775		qtd->hw_token |= cpu_to_hc32(ehci, QTD_IOC);
 776	return head;
 777
 778cleanup:
 779	qtd_list_free (ehci, urb, head);
 780	return NULL;
 781}
 782
 783/*-------------------------------------------------------------------------*/
 784
 785// Would be best to create all qh's from config descriptors,
 786// when each interface/altsetting is established.  Unlink
 787// any previous qh and cancel its urbs first; endpoints are
 788// implicitly reset then (data toggle too).
 789// That'd mean updating how usbcore talks to HCDs. (2.7?)
 790
 791
 792/*
 793 * Each QH holds a qtd list; a QH is used for everything except iso.
 794 *
 795 * For interrupt urbs, the scheduler must set the microframe scheduling
 796 * mask(s) each time the QH gets scheduled.  For highspeed, that's
 797 * just one microframe in the s-mask.  For split interrupt transactions
 798 * there are additional complications: c-mask, maybe FSTNs.
 799 */
 800static struct ehci_qh *
 801qh_make (
 802	struct ehci_hcd		*ehci,
 803	struct urb		*urb,
 804	gfp_t			flags
 805) {
 806	struct ehci_qh		*qh = ehci_qh_alloc (ehci, flags);
 807	u32			info1 = 0, info2 = 0;
 808	int			is_input, type;
 809	int			maxp = 0;
 810	struct usb_tt		*tt = urb->dev->tt;
 811	struct ehci_qh_hw	*hw;
 812
 813	if (!qh)
 814		return qh;
 815
 816	/*
 817	 * init endpoint/device data for this QH
 818	 */
 819	info1 |= usb_pipeendpoint (urb->pipe) << 8;
 820	info1 |= usb_pipedevice (urb->pipe) << 0;
 821
 822	is_input = usb_pipein (urb->pipe);
 823	type = usb_pipetype (urb->pipe);
 824	maxp = usb_maxpacket (urb->dev, urb->pipe, !is_input);
 825
 826	/* 1024 byte maxpacket is a hardware ceiling.  High bandwidth
 827	 * acts like up to 3KB, but is built from smaller packets.
 828	 */
 829	if (max_packet(maxp) > 1024) {
 830		ehci_dbg(ehci, "bogus qh maxpacket %d\n", max_packet(maxp));
 831		goto done;
 832	}
 833
 834	/* Compute interrupt scheduling parameters just once, and save.
 835	 * - allowing for high bandwidth, how many nsec/uframe are used?
 836	 * - split transactions need a second CSPLIT uframe; same question
 837	 * - splits also need a schedule gap (for full/low speed I/O)
 838	 * - qh has a polling interval
 839	 *
 840	 * For control/bulk requests, the HC or TT handles these.
 841	 */
 842	if (type == PIPE_INTERRUPT) {
 843		qh->usecs = NS_TO_US(usb_calc_bus_time(USB_SPEED_HIGH,
 
 
 844				is_input, 0,
 845				hb_mult(maxp) * max_packet(maxp)));
 846		qh->start = NO_FRAME;
 847		qh->stamp = ehci->periodic_stamp;
 848
 849		if (urb->dev->speed == USB_SPEED_HIGH) {
 850			qh->c_usecs = 0;
 851			qh->gap_uf = 0;
 852
 853			qh->period = urb->interval >> 3;
 854			if (qh->period == 0 && urb->interval != 1) {
 855				/* NOTE interval 2 or 4 uframes could work.
 856				 * But interval 1 scheduling is simpler, and
 857				 * includes high bandwidth.
 858				 */
 859				urb->interval = 1;
 860			} else if (qh->period > ehci->periodic_size) {
 861				qh->period = ehci->periodic_size;
 862				urb->interval = qh->period << 3;
 863			}
 
 
 
 
 
 
 
 
 
 864		} else {
 865			int		think_time;
 866
 867			/* gap is f(FS/LS transfer times) */
 868			qh->gap_uf = 1 + usb_calc_bus_time (urb->dev->speed,
 869					is_input, 0, maxp) / (125 * 1000);
 870
 871			/* FIXME this just approximates SPLIT/CSPLIT times */
 872			if (is_input) {		// SPLIT, gap, CSPLIT+DATA
 873				qh->c_usecs = qh->usecs + HS_USECS (0);
 874				qh->usecs = HS_USECS (1);
 875			} else {		// SPLIT+DATA, gap, CSPLIT
 876				qh->usecs += HS_USECS (1);
 877				qh->c_usecs = HS_USECS (0);
 878			}
 879
 880			think_time = tt ? tt->think_time : 0;
 881			qh->tt_usecs = NS_TO_US (think_time +
 882					usb_calc_bus_time (urb->dev->speed,
 883					is_input, 0, max_packet (maxp)));
 884			qh->period = urb->interval;
 885			if (qh->period > ehci->periodic_size) {
 886				qh->period = ehci->periodic_size;
 887				urb->interval = qh->period;
 888			}
 
 
 
 
 
 
 
 889		}
 890	}
 891
 892	/* support for tt scheduling, and access to toggles */
 893	qh->dev = urb->dev;
 
 894
 895	/* using TT? */
 896	switch (urb->dev->speed) {
 897	case USB_SPEED_LOW:
 898		info1 |= (1 << 12);	/* EPS "low" */
 899		/* FALL THROUGH */
 900
 901	case USB_SPEED_FULL:
 902		/* EPS 0 means "full" */
 903		if (type != PIPE_INTERRUPT)
 904			info1 |= (EHCI_TUNE_RL_TT << 28);
 905		if (type == PIPE_CONTROL) {
 906			info1 |= (1 << 27);	/* for TT */
 907			info1 |= 1 << 14;	/* toggle from qtd */
 908		}
 909		info1 |= maxp << 16;
 910
 911		info2 |= (EHCI_TUNE_MULT_TT << 30);
 912
 913		/* Some Freescale processors have an erratum in which the
 914		 * port number in the queue head was 0..N-1 instead of 1..N.
 915		 */
 916		if (ehci_has_fsl_portno_bug(ehci))
 917			info2 |= (urb->dev->ttport-1) << 23;
 918		else
 919			info2 |= urb->dev->ttport << 23;
 920
 921		/* set the address of the TT; for TDI's integrated
 922		 * root hub tt, leave it zeroed.
 923		 */
 924		if (tt && tt->hub != ehci_to_hcd(ehci)->self.root_hub)
 925			info2 |= tt->hub->devnum << 16;
 926
 927		/* NOTE:  if (PIPE_INTERRUPT) { scheduler sets c-mask } */
 928
 929		break;
 930
 931	case USB_SPEED_HIGH:		/* no TT involved */
 932		info1 |= (2 << 12);	/* EPS "high" */
 933		if (type == PIPE_CONTROL) {
 934			info1 |= (EHCI_TUNE_RL_HS << 28);
 935			info1 |= 64 << 16;	/* usb2 fixed maxpacket */
 936			info1 |= 1 << 14;	/* toggle from qtd */
 937			info2 |= (EHCI_TUNE_MULT_HS << 30);
 938		} else if (type == PIPE_BULK) {
 939			info1 |= (EHCI_TUNE_RL_HS << 28);
 940			/* The USB spec says that high speed bulk endpoints
 941			 * always use 512 byte maxpacket.  But some device
 942			 * vendors decided to ignore that, and MSFT is happy
 943			 * to help them do so.  So now people expect to use
 944			 * such nonconformant devices with Linux too; sigh.
 945			 */
 946			info1 |= max_packet(maxp) << 16;
 947			info2 |= (EHCI_TUNE_MULT_HS << 30);
 948		} else {		/* PIPE_INTERRUPT */
 949			info1 |= max_packet (maxp) << 16;
 950			info2 |= hb_mult (maxp) << 30;
 951		}
 952		break;
 953	default:
 954		ehci_dbg(ehci, "bogus dev %p speed %d\n", urb->dev,
 955			urb->dev->speed);
 956done:
 957		qh_put (qh);
 958		return NULL;
 959	}
 960
 961	/* NOTE:  if (PIPE_INTERRUPT) { scheduler sets s-mask } */
 962
 963	/* init as live, toggle clear, advance to dummy */
 964	qh->qh_state = QH_STATE_IDLE;
 965	hw = qh->hw;
 966	hw->hw_info1 = cpu_to_hc32(ehci, info1);
 967	hw->hw_info2 = cpu_to_hc32(ehci, info2);
 968	qh->is_out = !is_input;
 969	usb_settoggle (urb->dev, usb_pipeendpoint (urb->pipe), !is_input, 1);
 970	qh_refresh (ehci, qh);
 971	return qh;
 972}
 973
 974/*-------------------------------------------------------------------------*/
 975
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 976/* move qh (and its qtds) onto async queue; maybe enable queue.  */
 977
 978static void qh_link_async (struct ehci_hcd *ehci, struct ehci_qh *qh)
 979{
 980	__hc32		dma = QH_NEXT(ehci, qh->qh_dma);
 981	struct ehci_qh	*head;
 982
 983	/* Don't link a QH if there's a Clear-TT-Buffer pending */
 984	if (unlikely(qh->clearing_tt))
 985		return;
 986
 987	WARN_ON(qh->qh_state != QH_STATE_IDLE);
 988
 989	/* (re)start the async schedule? */
 990	head = ehci->async;
 991	timer_action_done (ehci, TIMER_ASYNC_OFF);
 992	if (!head->qh_next.qh) {
 993		if (!(ehci->command & CMD_ASE)) {
 994			/* in case a clear of CMD_ASE didn't take yet */
 995			(void)handshake(ehci, &ehci->regs->status,
 996					STS_ASS, 0, 150);
 997			ehci->command |= CMD_ASE;
 998			ehci_writel(ehci, ehci->command, &ehci->regs->command);
 999			/* posted write need not be known to HC yet ... */
1000		}
1001	}
1002
1003	/* clear halt and/or toggle; and maybe recover from silicon quirk */
1004	qh_refresh(ehci, qh);
1005
1006	/* splice right after start */
 
1007	qh->qh_next = head->qh_next;
1008	qh->hw->hw_next = head->hw->hw_next;
1009	wmb ();
1010
1011	head->qh_next.qh = qh;
1012	head->hw->hw_next = dma;
1013
1014	qh_get(qh);
1015	qh->xacterrs = 0;
1016	qh->qh_state = QH_STATE_LINKED;
 
 
1017	/* qtd completions reported later by interrupt */
 
 
1018}
1019
1020/*-------------------------------------------------------------------------*/
1021
1022/*
1023 * For control/bulk/interrupt, return QH with these TDs appended.
1024 * Allocates and initializes the QH if necessary.
1025 * Returns null if it can't allocate a QH it needs to.
1026 * If the QH has TDs (urbs) already, that's great.
1027 */
1028static struct ehci_qh *qh_append_tds (
1029	struct ehci_hcd		*ehci,
1030	struct urb		*urb,
1031	struct list_head	*qtd_list,
1032	int			epnum,
1033	void			**ptr
1034)
1035{
1036	struct ehci_qh		*qh = NULL;
1037	__hc32			qh_addr_mask = cpu_to_hc32(ehci, 0x7f);
1038
1039	qh = (struct ehci_qh *) *ptr;
1040	if (unlikely (qh == NULL)) {
1041		/* can't sleep here, we have ehci->lock... */
1042		qh = qh_make (ehci, urb, GFP_ATOMIC);
1043		*ptr = qh;
1044	}
1045	if (likely (qh != NULL)) {
1046		struct ehci_qtd	*qtd;
1047
1048		if (unlikely (list_empty (qtd_list)))
1049			qtd = NULL;
1050		else
1051			qtd = list_entry (qtd_list->next, struct ehci_qtd,
1052					qtd_list);
1053
1054		/* control qh may need patching ... */
1055		if (unlikely (epnum == 0)) {
1056
1057                        /* usb_reset_device() briefly reverts to address 0 */
1058                        if (usb_pipedevice (urb->pipe) == 0)
1059				qh->hw->hw_info1 &= ~qh_addr_mask;
1060		}
1061
1062		/* just one way to queue requests: swap with the dummy qtd.
1063		 * only hc or qh_refresh() ever modify the overlay.
1064		 */
1065		if (likely (qtd != NULL)) {
1066			struct ehci_qtd		*dummy;
1067			dma_addr_t		dma;
1068			__hc32			token;
1069
1070			/* to avoid racing the HC, use the dummy td instead of
1071			 * the first td of our list (becomes new dummy).  both
1072			 * tds stay deactivated until we're done, when the
1073			 * HC is allowed to fetch the old dummy (4.10.2).
1074			 */
1075			token = qtd->hw_token;
1076			qtd->hw_token = HALT_BIT(ehci);
1077
1078			dummy = qh->dummy;
1079
1080			dma = dummy->qtd_dma;
1081			*dummy = *qtd;
1082			dummy->qtd_dma = dma;
1083
1084			list_del (&qtd->qtd_list);
1085			list_add (&dummy->qtd_list, qtd_list);
1086			list_splice_tail(qtd_list, &qh->qtd_list);
1087
1088			ehci_qtd_init(ehci, qtd, qtd->qtd_dma);
1089			qh->dummy = qtd;
1090
1091			/* hc must see the new dummy at list end */
1092			dma = qtd->qtd_dma;
1093			qtd = list_entry (qh->qtd_list.prev,
1094					struct ehci_qtd, qtd_list);
1095			qtd->hw_next = QTD_NEXT(ehci, dma);
1096
1097			/* let the hc process these next qtds */
1098			wmb ();
1099			dummy->hw_token = token;
1100
1101			urb->hcpriv = qh_get (qh);
1102		}
1103	}
1104	return qh;
1105}
1106
1107/*-------------------------------------------------------------------------*/
1108
1109static int
1110submit_async (
1111	struct ehci_hcd		*ehci,
1112	struct urb		*urb,
1113	struct list_head	*qtd_list,
1114	gfp_t			mem_flags
1115) {
1116	int			epnum;
1117	unsigned long		flags;
1118	struct ehci_qh		*qh = NULL;
1119	int			rc;
1120
1121	epnum = urb->ep->desc.bEndpointAddress;
1122
1123#ifdef EHCI_URB_TRACE
1124	{
1125		struct ehci_qtd *qtd;
1126		qtd = list_entry(qtd_list->next, struct ehci_qtd, qtd_list);
1127		ehci_dbg(ehci,
1128			 "%s %s urb %p ep%d%s len %d, qtd %p [qh %p]\n",
1129			 __func__, urb->dev->devpath, urb,
1130			 epnum & 0x0f, (epnum & USB_DIR_IN) ? "in" : "out",
1131			 urb->transfer_buffer_length,
1132			 qtd, urb->ep->hcpriv);
1133	}
1134#endif
1135
1136	spin_lock_irqsave (&ehci->lock, flags);
1137	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1138		rc = -ESHUTDOWN;
1139		goto done;
1140	}
1141	rc = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1142	if (unlikely(rc))
1143		goto done;
1144
1145	qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
1146	if (unlikely(qh == NULL)) {
1147		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1148		rc = -ENOMEM;
1149		goto done;
1150	}
1151
1152	/* Control/bulk operations through TTs don't need scheduling,
1153	 * the HC and TT handle it when the TT has a buffer ready.
1154	 */
1155	if (likely (qh->qh_state == QH_STATE_IDLE))
1156		qh_link_async(ehci, qh);
1157 done:
1158	spin_unlock_irqrestore (&ehci->lock, flags);
1159	if (unlikely (qh == NULL))
1160		qtd_list_free (ehci, urb, qtd_list);
1161	return rc;
1162}
1163
1164/*-------------------------------------------------------------------------*/
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1165
1166/* the async qh for the qtds being reclaimed are now unlinked from the HC */
1167
1168static void end_unlink_async (struct ehci_hcd *ehci)
1169{
1170	struct ehci_qh		*qh = ehci->reclaim;
1171	struct ehci_qh		*next;
1172
1173	iaa_watchdog_done(ehci);
 
1174
1175	// qh->hw_next = cpu_to_hc32(qh->qh_dma);
1176	qh->qh_state = QH_STATE_IDLE;
1177	qh->qh_next.qh = NULL;
1178	qh_put (qh);			// refcount from reclaim
 
 
1179
1180	/* other unlink(s) may be pending (in QH_STATE_UNLINK_WAIT) */
1181	next = qh->reclaim;
1182	ehci->reclaim = next;
1183	qh->reclaim = NULL;
1184
1185	qh_completions (ehci, qh);
 
 
 
 
 
 
 
 
 
 
1186
1187	if (!list_empty(&qh->qtd_list) && ehci->rh_state == EHCI_RH_RUNNING) {
1188		qh_link_async (ehci, qh);
1189	} else {
1190		/* it's not free to turn the async schedule on/off; leave it
1191		 * active but idle for a while once it empties.
1192		 */
1193		if (ehci->rh_state == EHCI_RH_RUNNING
1194				&& ehci->async->qh_next.qh == NULL)
1195			timer_action (ehci, TIMER_ASYNC_OFF);
1196	}
1197	qh_put(qh);			/* refcount from async list */
1198
1199	if (next) {
1200		ehci->reclaim = NULL;
1201		start_unlink_async (ehci, next);
1202	}
1203
1204	if (ehci->has_synopsys_hc_bug)
1205		ehci_writel(ehci, (u32) ehci->async->qh_dma,
1206			    &ehci->regs->async_next);
1207}
 
 
1208
1209/* makes sure the async qh will become idle */
1210/* caller must own ehci->lock */
1211
1212static void start_unlink_async (struct ehci_hcd *ehci, struct ehci_qh *qh)
1213{
1214	struct ehci_qh	*prev;
1215
1216#ifdef DEBUG
1217	assert_spin_locked(&ehci->lock);
1218	if (ehci->reclaim
1219			|| (qh->qh_state != QH_STATE_LINKED
1220				&& qh->qh_state != QH_STATE_UNLINK_WAIT)
1221			)
1222		BUG ();
1223#endif
1224
1225	/* stop async schedule right now? */
1226	if (unlikely (qh == ehci->async)) {
1227		/* can't get here without STS_ASS set */
1228		if (ehci->rh_state != EHCI_RH_HALTED
1229				&& !ehci->reclaim) {
1230			/* ... and CMD_IAAD clear */
1231			ehci->command &= ~CMD_ASE;
1232			ehci_writel(ehci, ehci->command, &ehci->regs->command);
1233			wmb ();
1234			// handshake later, if we need to
1235			timer_action_done (ehci, TIMER_ASYNC_OFF);
1236		}
1237		return;
1238	}
1239
1240	qh->qh_state = QH_STATE_UNLINK;
1241	ehci->reclaim = qh = qh_get (qh);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1242
 
 
 
 
 
1243	prev = ehci->async;
1244	while (prev->qh_next.qh != qh)
1245		prev = prev->qh_next.qh;
1246
1247	prev->hw->hw_next = qh->hw->hw_next;
1248	prev->qh_next = qh->qh_next;
1249	if (ehci->qh_scan_next == qh)
1250		ehci->qh_scan_next = qh->qh_next.qh;
1251	wmb ();
 
 
 
 
 
 
 
1252
1253	/* If the controller isn't running, we don't have to wait for it */
1254	if (unlikely(ehci->rh_state != EHCI_RH_RUNNING)) {
1255		/* if (unlikely (qh->reclaim != 0))
1256		 *	this will recurse, probably not much
1257		 */
1258		end_unlink_async (ehci);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1259		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1260	}
1261
1262	ehci_writel(ehci, ehci->command | CMD_IAAD, &ehci->regs->command);
1263	(void)ehci_readl(ehci, &ehci->regs->command);
1264	iaa_watchdog_start(ehci);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1265}
1266
1267/*-------------------------------------------------------------------------*/
1268
1269static void scan_async (struct ehci_hcd *ehci)
1270{
1271	bool			stopped;
1272	struct ehci_qh		*qh;
1273	enum ehci_timer_action	action = TIMER_IO_WATCHDOG;
1274
1275	timer_action_done (ehci, TIMER_ASYNC_SHRINK);
1276	stopped = (ehci->rh_state != EHCI_RH_RUNNING);
1277
1278	ehci->qh_scan_next = ehci->async->qh_next.qh;
1279	while (ehci->qh_scan_next) {
1280		qh = ehci->qh_scan_next;
1281		ehci->qh_scan_next = qh->qh_next.qh;
1282 rescan:
1283		/* clean any finished work for this qh */
1284		if (!list_empty(&qh->qtd_list)) {
1285			int temp;
1286
1287			/*
1288			 * Unlinks could happen here; completion reporting
1289			 * drops the lock.  That's why ehci->qh_scan_next
1290			 * always holds the next qh to scan; if the next qh
1291			 * gets unlinked then ehci->qh_scan_next is adjusted
1292			 * in start_unlink_async().
1293			 */
1294			qh = qh_get(qh);
1295			temp = qh_completions(ehci, qh);
1296			if (qh->needs_rescan)
1297				unlink_async(ehci, qh);
1298			qh->unlink_time = jiffies + EHCI_SHRINK_JIFFIES;
1299			qh_put(qh);
1300			if (temp != 0)
1301				goto rescan;
1302		}
1303
1304		/* unlink idle entries, reducing DMA usage as well
1305		 * as HCD schedule-scanning costs.  delay for any qh
1306		 * we just scanned, there's a not-unusual case that it
1307		 * doesn't stay idle for long.
1308		 * (plus, avoids some kind of re-activation race.)
1309		 */
1310		if (list_empty(&qh->qtd_list)
1311				&& qh->qh_state == QH_STATE_LINKED) {
1312			if (!ehci->reclaim && (stopped ||
1313					time_after_eq(jiffies, qh->unlink_time)))
1314				start_unlink_async(ehci, qh);
1315			else
1316				action = TIMER_ASYNC_SHRINK;
 
 
 
1317		}
1318	}
1319	if (action == TIMER_ASYNC_SHRINK)
1320		timer_action (ehci, TIMER_ASYNC_SHRINK);
 
 
 
 
 
 
 
 
 
 
 
1321}