1/*
   2 * Copyright (c) 2001-2004 by David Brownell
   3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms of the GNU General Public License as published by the
   7 * Free Software Foundation; either version 2 of the License, or (at your
   8 * option) any later version.
   9 *
  10 * This program is distributed in the hope that it will be useful, but
  11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 * for more details.
  14 *
  15 * You should have received a copy of the GNU General Public License
  16 * along with this program; if not, write to the Free Software Foundation,
  17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20/* this file is part of ehci-hcd.c */
  21
  22/*-------------------------------------------------------------------------*/
  23
  24/*
  25 * EHCI scheduled transaction support:  interrupt, iso, split iso
  26 * These are called "periodic" transactions in the EHCI spec.
  27 *
  28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
  29 * with the "asynchronous" transaction support (control/bulk transfers).
  30 * The only real difference is in how interrupt transfers are scheduled.
  31 *
  32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
  33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
  34 * pre-calculated schedule data to make appending to the queue be quick.
  35 */
  36
  37static int ehci_get_frame (struct usb_hcd *hcd);
  38
  39#ifdef CONFIG_PCI
  40
  41static unsigned ehci_read_frame_index(struct ehci_hcd *ehci)
  42{
  43	unsigned uf;
  44
  45	/*
  46	 * The MosChip MCS9990 controller updates its microframe counter
  47	 * a little before the frame counter, and occasionally we will read
  48	 * the invalid intermediate value.  Avoid problems by checking the
  49	 * microframe number (the low-order 3 bits); if they are 0 then
  50	 * re-read the register to get the correct value.
  51	 */
  52	uf = ehci_readl(ehci, &ehci->regs->frame_index);
  53	if (unlikely(ehci->frame_index_bug && ((uf & 7) == 0)))
  54		uf = ehci_readl(ehci, &ehci->regs->frame_index);
  55	return uf;
  56}
  57
  58#endif
  59
  60/*-------------------------------------------------------------------------*/
  61
  62/*
  63 * periodic_next_shadow - return "next" pointer on shadow list
  64 * @periodic: host pointer to qh/itd/sitd
  65 * @tag: hardware tag for type of this record
  66 */
  67static union ehci_shadow *
  68periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  69		__hc32 tag)
  70{
  71	switch (hc32_to_cpu(ehci, tag)) {
  72	case Q_TYPE_QH:
  73		return &periodic->qh->qh_next;
  74	case Q_TYPE_FSTN:
  75		return &periodic->fstn->fstn_next;
  76	case Q_TYPE_ITD:
  77		return &periodic->itd->itd_next;
  78	// case Q_TYPE_SITD:
  79	default:
  80		return &periodic->sitd->sitd_next;
  81	}
  82}
  83
  84static __hc32 *
  85shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  86		__hc32 tag)
  87{
  88	switch (hc32_to_cpu(ehci, tag)) {
  89	/* our ehci_shadow.qh is actually software part */
  90	case Q_TYPE_QH:
  91		return &periodic->qh->hw->hw_next;
  92	/* others are hw parts */
  93	default:
  94		return periodic->hw_next;
  95	}
  96}
  97
  98/* caller must hold ehci->lock */
  99static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
 100{
 101	union ehci_shadow	*prev_p = &ehci->pshadow[frame];
 102	__hc32			*hw_p = &ehci->periodic[frame];
 103	union ehci_shadow	here = *prev_p;
 104
 105	/* find predecessor of "ptr"; hw and shadow lists are in sync */
 106	while (here.ptr && here.ptr != ptr) {
 107		prev_p = periodic_next_shadow(ehci, prev_p,
 108				Q_NEXT_TYPE(ehci, *hw_p));
 109		hw_p = shadow_next_periodic(ehci, &here,
 110				Q_NEXT_TYPE(ehci, *hw_p));
 111		here = *prev_p;
 112	}
 113	/* an interrupt entry (at list end) could have been shared */
 114	if (!here.ptr)
 115		return;
 116
 117	/* update shadow and hardware lists ... the old "next" pointers
 118	 * from ptr may still be in use, the caller updates them.
 119	 */
 120	*prev_p = *periodic_next_shadow(ehci, &here,
 121			Q_NEXT_TYPE(ehci, *hw_p));
 122
 123	if (!ehci->use_dummy_qh ||
 124	    *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p))
 125			!= EHCI_LIST_END(ehci))
 126		*hw_p = *shadow_next_periodic(ehci, &here,
 127				Q_NEXT_TYPE(ehci, *hw_p));
 128	else
 129		*hw_p = ehci->dummy->qh_dma;
 130}
 131
 132/* how many of the uframe's 125 usecs are allocated? */
 133static unsigned short
 134periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
 135{
 136	__hc32			*hw_p = &ehci->periodic [frame];
 137	union ehci_shadow	*q = &ehci->pshadow [frame];
 138	unsigned		usecs = 0;
 139	struct ehci_qh_hw	*hw;
 140
 141	while (q->ptr) {
 142		switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
 143		case Q_TYPE_QH:
 144			hw = q->qh->hw;
 145			/* is it in the S-mask? */
 146			if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe))
 147				usecs += q->qh->usecs;
 148			/* ... or C-mask? */
 149			if (hw->hw_info2 & cpu_to_hc32(ehci,
 150					1 << (8 + uframe)))
 151				usecs += q->qh->c_usecs;
 152			hw_p = &hw->hw_next;
 153			q = &q->qh->qh_next;
 154			break;
 155		// case Q_TYPE_FSTN:
 156		default:
 157			/* for "save place" FSTNs, count the relevant INTR
 158			 * bandwidth from the previous frame
 159			 */
 160			if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) {
 161				ehci_dbg (ehci, "ignoring FSTN cost ...\n");
 162			}
 163			hw_p = &q->fstn->hw_next;
 164			q = &q->fstn->fstn_next;
 165			break;
 166		case Q_TYPE_ITD:
 167			if (q->itd->hw_transaction[uframe])
 168				usecs += q->itd->stream->usecs;
 169			hw_p = &q->itd->hw_next;
 170			q = &q->itd->itd_next;
 171			break;
 172		case Q_TYPE_SITD:
 173			/* is it in the S-mask?  (count SPLIT, DATA) */
 174			if (q->sitd->hw_uframe & cpu_to_hc32(ehci,
 175					1 << uframe)) {
 176				if (q->sitd->hw_fullspeed_ep &
 177						cpu_to_hc32(ehci, 1<<31))
 178					usecs += q->sitd->stream->usecs;
 179				else	/* worst case for OUT start-split */
 180					usecs += HS_USECS_ISO (188);
 181			}
 182
 183			/* ... C-mask?  (count CSPLIT, DATA) */
 184			if (q->sitd->hw_uframe &
 185					cpu_to_hc32(ehci, 1 << (8 + uframe))) {
 186				/* worst case for IN complete-split */
 187				usecs += q->sitd->stream->c_usecs;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 188			}
 
 
 
 
 
 
 
 
 189
 190			hw_p = &q->sitd->hw_next;
 191			q = &q->sitd->sitd_next;
 192			break;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 193		}
 194	}
 195#ifdef	DEBUG
 196	if (usecs > ehci->uframe_periodic_max)
 197		ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
 198			frame * 8 + uframe, usecs);
 199#endif
 200	return usecs;
 
 
 
 
 
 
 
 201}
 202
 203/*-------------------------------------------------------------------------*/
 204
 205static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 206{
 207	if (!dev1->tt || !dev2->tt)
 208		return 0;
 209	if (dev1->tt != dev2->tt)
 210		return 0;
 211	if (dev1->tt->multi)
 212		return dev1->ttport == dev2->ttport;
 213	else
 214		return 1;
 215}
 216
 217#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
 218
 219/* Which uframe does the low/fullspeed transfer start in?
 220 *
 221 * The parameter is the mask of ssplits in "H-frame" terms
 222 * and this returns the transfer start uframe in "B-frame" terms,
 223 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
 224 * will cause a transfer in "B-frame" uframe 0.  "B-frames" lag
 225 * "H-frames" by 1 uframe.  See the EHCI spec sec 4.5 and figure 4.7.
 226 */
 227static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
 228{
 229	unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask);
 230	if (!smask) {
 231		ehci_err(ehci, "invalid empty smask!\n");
 232		/* uframe 7 can't have bw so this will indicate failure */
 233		return 7;
 234	}
 235	return ffs(smask) - 1;
 236}
 237
 238static const unsigned char
 239max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
 240
 241/* carryover low/fullspeed bandwidth that crosses uframe boundries */
 242static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
 243{
 244	int i;
 245	for (i=0; i<7; i++) {
 246		if (max_tt_usecs[i] < tt_usecs[i]) {
 247			tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
 248			tt_usecs[i] = max_tt_usecs[i];
 249		}
 250	}
 251}
 252
 253/* How many of the tt's periodic downstream 1000 usecs are allocated?
 254 *
 255 * While this measures the bandwidth in terms of usecs/uframe,
 256 * the low/fullspeed bus has no notion of uframes, so any particular
 257 * low/fullspeed transfer can "carry over" from one uframe to the next,
 258 * since the TT just performs downstream transfers in sequence.
 259 *
 260 * For example two separate 100 usec transfers can start in the same uframe,
 261 * and the second one would "carry over" 75 usecs into the next uframe.
 262 */
 263static void
 264periodic_tt_usecs (
 265	struct ehci_hcd *ehci,
 266	struct usb_device *dev,
 267	unsigned frame,
 268	unsigned short tt_usecs[8]
 269)
 270{
 271	__hc32			*hw_p = &ehci->periodic [frame];
 272	union ehci_shadow	*q = &ehci->pshadow [frame];
 273	unsigned char		uf;
 274
 275	memset(tt_usecs, 0, 16);
 276
 277	while (q->ptr) {
 278		switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
 279		case Q_TYPE_ITD:
 280			hw_p = &q->itd->hw_next;
 281			q = &q->itd->itd_next;
 282			continue;
 283		case Q_TYPE_QH:
 284			if (same_tt(dev, q->qh->dev)) {
 285				uf = tt_start_uframe(ehci, q->qh->hw->hw_info2);
 286				tt_usecs[uf] += q->qh->tt_usecs;
 287			}
 288			hw_p = &q->qh->hw->hw_next;
 289			q = &q->qh->qh_next;
 290			continue;
 291		case Q_TYPE_SITD:
 292			if (same_tt(dev, q->sitd->urb->dev)) {
 293				uf = tt_start_uframe(ehci, q->sitd->hw_uframe);
 294				tt_usecs[uf] += q->sitd->stream->tt_usecs;
 295			}
 296			hw_p = &q->sitd->hw_next;
 297			q = &q->sitd->sitd_next;
 298			continue;
 299		// case Q_TYPE_FSTN:
 300		default:
 301			ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n",
 302					frame);
 303			hw_p = &q->fstn->hw_next;
 304			q = &q->fstn->fstn_next;
 305		}
 306	}
 307
 308	carryover_tt_bandwidth(tt_usecs);
 309
 310	if (max_tt_usecs[7] < tt_usecs[7])
 311		ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n",
 312			frame, tt_usecs[7] - max_tt_usecs[7]);
 313}
 314
 315/*
 316 * Return true if the device's tt's downstream bus is available for a
 317 * periodic transfer of the specified length (usecs), starting at the
 318 * specified frame/uframe.  Note that (as summarized in section 11.19
 319 * of the usb 2.0 spec) TTs can buffer multiple transactions for each
 320 * uframe.
 321 *
 322 * The uframe parameter is when the fullspeed/lowspeed transfer
 323 * should be executed in "B-frame" terms, which is the same as the
 324 * highspeed ssplit's uframe (which is in "H-frame" terms).  For example
 325 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
 326 * See the EHCI spec sec 4.5 and fig 4.7.
 327 *
 328 * This checks if the full/lowspeed bus, at the specified starting uframe,
 329 * has the specified bandwidth available, according to rules listed
 330 * in USB 2.0 spec section 11.18.1 fig 11-60.
 331 *
 332 * This does not check if the transfer would exceed the max ssplit
 333 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
 334 * since proper scheduling limits ssplits to less than 16 per uframe.
 335 */
 336static int tt_available (
 337	struct ehci_hcd		*ehci,
 338	unsigned		period,
 339	struct usb_device	*dev,
 340	unsigned		frame,
 341	unsigned		uframe,
 342	u16			usecs
 343)
 344{
 
 
 
 345	if ((period == 0) || (uframe >= 7))	/* error */
 346		return 0;
 347
 348	for (; frame < ehci->periodic_size; frame += period) {
 349		unsigned short tt_usecs[8];
 
 
 
 
 
 350
 351		periodic_tt_usecs (ehci, dev, frame, tt_usecs);
 
 
 352
 353		ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in"
 354			" schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
 355			frame, usecs, uframe,
 356			tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3],
 357			tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]);
 358
 359		if (max_tt_usecs[uframe] <= tt_usecs[uframe]) {
 360			ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n",
 361				frame, uframe);
 362			return 0;
 363		}
 364
 365		/* special case for isoc transfers larger than 125us:
 366		 * the first and each subsequent fully used uframe
 367		 * must be empty, so as to not illegally delay
 368		 * already scheduled transactions
 369		 */
 370		if (125 < usecs) {
 371			int ufs = (usecs / 125);
 372			int i;
 373			for (i = uframe; i < (uframe + ufs) && i < 8; i++)
 374				if (0 < tt_usecs[i]) {
 375					ehci_vdbg(ehci,
 376						"multi-uframe xfer can't fit "
 377						"in frame %d uframe %d\n",
 378						frame, i);
 379					return 0;
 380				}
 381		}
 382
 383		tt_usecs[uframe] += usecs;
 384
 385		carryover_tt_bandwidth(tt_usecs);
 386
 387		/* fail if the carryover pushed bw past the last uframe's limit */
 388		if (max_tt_usecs[7] < tt_usecs[7]) {
 389			ehci_vdbg(ehci,
 390				"tt unavailable usecs %d frame %d uframe %d\n",
 391				usecs, frame, uframe);
 392			return 0;
 393		}
 394	}
 395
 396	return 1;
 397}
 398
 399#else
 400
 401/* return true iff the device's transaction translator is available
 402 * for a periodic transfer starting at the specified frame, using
 403 * all the uframes in the mask.
 404 */
 405static int tt_no_collision (
 406	struct ehci_hcd		*ehci,
 407	unsigned		period,
 408	struct usb_device	*dev,
 409	unsigned		frame,
 410	u32			uf_mask
 411)
 412{
 413	if (period == 0)	/* error */
 414		return 0;
 415
 416	/* note bandwidth wastage:  split never follows csplit
 417	 * (different dev or endpoint) until the next uframe.
 418	 * calling convention doesn't make that distinction.
 419	 */
 420	for (; frame < ehci->periodic_size; frame += period) {
 421		union ehci_shadow	here;
 422		__hc32			type;
 423		struct ehci_qh_hw	*hw;
 424
 425		here = ehci->pshadow [frame];
 426		type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]);
 427		while (here.ptr) {
 428			switch (hc32_to_cpu(ehci, type)) {
 429			case Q_TYPE_ITD:
 430				type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
 431				here = here.itd->itd_next;
 432				continue;
 433			case Q_TYPE_QH:
 434				hw = here.qh->hw;
 435				if (same_tt (dev, here.qh->dev)) {
 436					u32		mask;
 437
 438					mask = hc32_to_cpu(ehci,
 439							hw->hw_info2);
 440					/* "knows" no gap is needed */
 441					mask |= mask >> 8;
 442					if (mask & uf_mask)
 443						break;
 444				}
 445				type = Q_NEXT_TYPE(ehci, hw->hw_next);
 446				here = here.qh->qh_next;
 447				continue;
 448			case Q_TYPE_SITD:
 449				if (same_tt (dev, here.sitd->urb->dev)) {
 450					u16		mask;
 451
 452					mask = hc32_to_cpu(ehci, here.sitd
 453								->hw_uframe);
 454					/* FIXME assumes no gap for IN! */
 455					mask |= mask >> 8;
 456					if (mask & uf_mask)
 457						break;
 458				}
 459				type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
 460				here = here.sitd->sitd_next;
 461				continue;
 462			// case Q_TYPE_FSTN:
 463			default:
 464				ehci_dbg (ehci,
 465					"periodic frame %d bogus type %d\n",
 466					frame, type);
 467			}
 468
 469			/* collision or error */
 470			return 0;
 471		}
 472	}
 473
 474	/* no collision */
 475	return 1;
 476}
 477
 478#endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
 479
 480/*-------------------------------------------------------------------------*/
 481
 482static int enable_periodic (struct ehci_hcd *ehci)
 483{
 484	int	status;
 
 485
 486	if (ehci->periodic_sched++)
 487		return 0;
 488
 489	/* did clearing PSE did take effect yet?
 490	 * takes effect only at frame boundaries...
 491	 */
 492	status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
 493					     STS_PSS, 0, 9 * 125);
 494	if (status) {
 495		usb_hc_died(ehci_to_hcd(ehci));
 496		return status;
 497	}
 498
 499	ehci->command |= CMD_PSE;
 500	ehci_writel(ehci, ehci->command, &ehci->regs->command);
 501	/* posted write ... PSS happens later */
 502
 503	/* make sure ehci_work scans these */
 504	ehci->next_uframe = ehci_read_frame_index(ehci)
 505		% (ehci->periodic_size << 3);
 506	if (unlikely(ehci->broken_periodic))
 507		ehci->last_periodic_enable = ktime_get_real();
 508	return 0;
 509}
 510
 511static int disable_periodic (struct ehci_hcd *ehci)
 512{
 513	int	status;
 514
 515	if (--ehci->periodic_sched)
 516		return 0;
 517
 518	if (unlikely(ehci->broken_periodic)) {
 519		/* delay experimentally determined */
 520		ktime_t safe = ktime_add_us(ehci->last_periodic_enable, 1000);
 521		ktime_t now = ktime_get_real();
 522		s64 delay = ktime_us_delta(safe, now);
 523
 524		if (unlikely(delay > 0))
 525			udelay(delay);
 526	}
 527
 528	/* did setting PSE not take effect yet?
 529	 * takes effect only at frame boundaries...
 530	 */
 531	status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
 532					     STS_PSS, STS_PSS, 9 * 125);
 533	if (status) {
 534		usb_hc_died(ehci_to_hcd(ehci));
 535		return status;
 536	}
 537
 538	ehci->command &= ~CMD_PSE;
 539	ehci_writel(ehci, ehci->command, &ehci->regs->command);
 540	/* posted write ... */
 541
 542	free_cached_lists(ehci);
 543
 544	ehci->next_uframe = -1;
 545	return 0;
 546}
 547
 548/*-------------------------------------------------------------------------*/
 549
 550/* periodic schedule slots have iso tds (normal or split) first, then a
 551 * sparse tree for active interrupt transfers.
 552 *
 553 * this just links in a qh; caller guarantees uframe masks are set right.
 554 * no FSTN support (yet; ehci 0.96+)
 555 */
 556static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
 557{
 558	unsigned	i;
 559	unsigned	period = qh->period;
 560
 561	dev_dbg (&qh->dev->dev,
 562		"link qh%d-%04x/%p start %d [%d/%d us]\n",
 563		period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
 564			& (QH_CMASK | QH_SMASK),
 565		qh, qh->start, qh->usecs, qh->c_usecs);
 566
 567	/* high bandwidth, or otherwise every microframe */
 568	if (period == 0)
 569		period = 1;
 570
 571	for (i = qh->start; i < ehci->periodic_size; i += period) {
 572		union ehci_shadow	*prev = &ehci->pshadow[i];
 573		__hc32			*hw_p = &ehci->periodic[i];
 574		union ehci_shadow	here = *prev;
 575		__hc32			type = 0;
 576
 577		/* skip the iso nodes at list head */
 578		while (here.ptr) {
 579			type = Q_NEXT_TYPE(ehci, *hw_p);
 580			if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
 581				break;
 582			prev = periodic_next_shadow(ehci, prev, type);
 583			hw_p = shadow_next_periodic(ehci, &here, type);
 584			here = *prev;
 585		}
 586
 587		/* sorting each branch by period (slow-->fast)
 588		 * enables sharing interior tree nodes
 589		 */
 590		while (here.ptr && qh != here.qh) {
 591			if (qh->period > here.qh->period)
 592				break;
 593			prev = &here.qh->qh_next;
 594			hw_p = &here.qh->hw->hw_next;
 595			here = *prev;
 596		}
 597		/* link in this qh, unless some earlier pass did that */
 598		if (qh != here.qh) {
 599			qh->qh_next = here;
 600			if (here.qh)
 601				qh->hw->hw_next = *hw_p;
 602			wmb ();
 603			prev->qh = qh;
 604			*hw_p = QH_NEXT (ehci, qh->qh_dma);
 605		}
 606	}
 607	qh->qh_state = QH_STATE_LINKED;
 608	qh->xacterrs = 0;
 609	qh_get (qh);
 
 
 
 
 
 610
 611	/* update per-qh bandwidth for usbfs */
 612	ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
 613		? ((qh->usecs + qh->c_usecs) / qh->period)
 614		: (qh->usecs * 8);
 615
 616	/* maybe enable periodic schedule processing */
 617	return enable_periodic(ehci);
 
 618}
 619
 620static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
 621{
 622	unsigned	i;
 623	unsigned	period;
 624
 625	// FIXME:
 626	// IF this isn't high speed
 627	//   and this qh is active in the current uframe
 628	//   (and overlay token SplitXstate is false?)
 629	// THEN
 630	//   qh->hw_info1 |= cpu_to_hc32(1 << 7 /* "ignore" */);
 
 
 
 
 
 
 
 
 631
 632	/* high bandwidth, or otherwise part of every microframe */
 633	if ((period = qh->period) == 0)
 634		period = 1;
 635
 636	for (i = qh->start; i < ehci->periodic_size; i += period)
 637		periodic_unlink (ehci, i, qh);
 638
 639	/* update per-qh bandwidth for usbfs */
 640	ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
 641		? ((qh->usecs + qh->c_usecs) / qh->period)
 642		: (qh->usecs * 8);
 643
 644	dev_dbg (&qh->dev->dev,
 645		"unlink qh%d-%04x/%p start %d [%d/%d us]\n",
 646		qh->period,
 647		hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
 648		qh, qh->start, qh->usecs, qh->c_usecs);
 649
 650	/* qh->qh_next still "live" to HC */
 651	qh->qh_state = QH_STATE_UNLINK;
 652	qh->qh_next.ptr = NULL;
 653	qh_put (qh);
 654
 655	/* maybe turn off periodic schedule */
 656	return disable_periodic(ehci);
 
 
 657}
 658
 659static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
 660{
 661	unsigned		wait;
 662	struct ehci_qh_hw	*hw = qh->hw;
 663	int			rc;
 664
 665	/* If the QH isn't linked then there's nothing we can do
 666	 * unless we were called during a giveback, in which case
 667	 * qh_completions() has to deal with it.
 
 
 668	 */
 669	if (qh->qh_state != QH_STATE_LINKED) {
 670		if (qh->qh_state == QH_STATE_COMPLETING)
 671			qh->needs_rescan = 1;
 
 
 
 672		return;
 673	}
 
 
 674
 675	qh_unlink_periodic (ehci, qh);
 676
 677	/* simple/paranoid:  always delay, expecting the HC needs to read
 678	 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
 679	 * expect khubd to clean up after any CSPLITs we won't issue.
 680	 * active high speed queues may need bigger delays...
 681	 */
 682	if (list_empty (&qh->qtd_list)
 683			|| (cpu_to_hc32(ehci, QH_CMASK)
 684					& hw->hw_info2) != 0)
 685		wait = 2;
 686	else
 687		wait = 55;	/* worst case: 3 * 1024 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 688
 689	udelay (wait);
 690	qh->qh_state = QH_STATE_IDLE;
 691	hw->hw_next = EHCI_LIST_END(ehci);
 692	wmb ();
 693
 694	qh_completions(ehci, qh);
 
 695
 696	/* reschedule QH iff another request is queued */
 697	if (!list_empty(&qh->qtd_list) &&
 698			ehci->rh_state == EHCI_RH_RUNNING) {
 699		rc = qh_schedule(ehci, qh);
 
 
 
 
 700
 701		/* An error here likely indicates handshake failure
 702		 * or no space left in the schedule.  Neither fault
 703		 * should happen often ...
 704		 *
 705		 * FIXME kill the now-dysfunctional queued urbs
 706		 */
 707		if (rc != 0)
 708			ehci_err(ehci, "can't reschedule qh %p, err %d\n",
 709					qh, rc);
 
 710	}
 
 
 
 
 711}
 712
 713/*-------------------------------------------------------------------------*/
 714
 715static int check_period (
 716	struct ehci_hcd *ehci,
 717	unsigned	frame,
 718	unsigned	uframe,
 719	unsigned	period,
 720	unsigned	usecs
 721) {
 722	int		claimed;
 723
 724	/* complete split running into next frame?
 725	 * given FSTN support, we could sometimes check...
 726	 */
 727	if (uframe >= 8)
 728		return 0;
 729
 730	/* convert "usecs we need" to "max already claimed" */
 731	usecs = ehci->uframe_periodic_max - usecs;
 732
 733	/* we "know" 2 and 4 uframe intervals were rejected; so
 734	 * for period 0, check _every_ microframe in the schedule.
 735	 */
 736	if (unlikely (period == 0)) {
 737		do {
 738			for (uframe = 0; uframe < 7; uframe++) {
 739				claimed = periodic_usecs (ehci, frame, uframe);
 740				if (claimed > usecs)
 741					return 0;
 742			}
 743		} while ((frame += 1) < ehci->periodic_size);
 744
 745	/* just check the specified uframe, at that period */
 746	} else {
 747		do {
 748			claimed = periodic_usecs (ehci, frame, uframe);
 749			if (claimed > usecs)
 750				return 0;
 751		} while ((frame += period) < ehci->periodic_size);
 752	}
 753
 754	// success!
 755	return 1;
 756}
 757
 758static int check_intr_schedule (
 759	struct ehci_hcd		*ehci,
 760	unsigned		frame,
 761	unsigned		uframe,
 762	const struct ehci_qh	*qh,
 763	__hc32			*c_maskp
 
 764)
 765{
 766	int		retval = -ENOSPC;
 767	u8		mask = 0;
 768
 769	if (qh->c_usecs && uframe >= 6)		/* FSTN territory? */
 770		goto done;
 771
 772	if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
 773		goto done;
 774	if (!qh->c_usecs) {
 775		retval = 0;
 776		*c_maskp = 0;
 777		goto done;
 778	}
 779
 780#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
 781	if (tt_available (ehci, qh->period, qh->dev, frame, uframe,
 782				qh->tt_usecs)) {
 783		unsigned i;
 784
 785		/* TODO : this may need FSTN for SSPLIT in uframe 5. */
 786		for (i=uframe+1; i<8 && i<uframe+4; i++)
 787			if (!check_period (ehci, frame, i,
 788						qh->period, qh->c_usecs))
 789				goto done;
 790			else
 791				mask |= 1 << i;
 792
 793		retval = 0;
 794
 795		*c_maskp = cpu_to_hc32(ehci, mask << 8);
 796	}
 797#else
 798	/* Make sure this tt's buffer is also available for CSPLITs.
 799	 * We pessimize a bit; probably the typical full speed case
 800	 * doesn't need the second CSPLIT.
 801	 *
 802	 * NOTE:  both SPLIT and CSPLIT could be checked in just
 803	 * one smart pass...
 804	 */
 805	mask = 0x03 << (uframe + qh->gap_uf);
 806	*c_maskp = cpu_to_hc32(ehci, mask << 8);
 807
 808	mask |= 1 << uframe;
 809	if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
 810		if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
 811					qh->period, qh->c_usecs))
 812			goto done;
 813		if (!check_period (ehci, frame, uframe + qh->gap_uf,
 814					qh->period, qh->c_usecs))
 815			goto done;
 816		retval = 0;
 817	}
 818#endif
 819done:
 820	return retval;
 821}
 822
 823/* "first fit" scheduling policy used the first time through,
 824 * or when the previous schedule slot can't be re-used.
 825 */
 826static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
 827{
 828	int		status;
 829	unsigned	uframe;
 830	__hc32		c_mask;
 831	unsigned	frame;		/* 0..(qh->period - 1), or NO_FRAME */
 832	struct ehci_qh_hw	*hw = qh->hw;
 
 833
 834	qh_refresh(ehci, qh);
 835	hw->hw_next = EHCI_LIST_END(ehci);
 836	frame = qh->start;
 837
 838	/* reuse the previous schedule slots, if we can */
 839	if (frame < qh->period) {
 840		uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK);
 841		status = check_intr_schedule (ehci, frame, --uframe,
 842				qh, &c_mask);
 843	} else {
 844		uframe = 0;
 845		c_mask = 0;
 846		status = -ENOSPC;
 
 
 
 847	}
 
 848
 849	/* else scan the schedule to find a group of slots such that all
 850	 * uframes have enough periodic bandwidth available.
 851	 */
 852	if (status) {
 853		/* "normal" case, uframing flexible except with splits */
 854		if (qh->period) {
 855			int		i;
 856
 857			for (i = qh->period; status && i > 0; --i) {
 858				frame = ++ehci->random_frame % qh->period;
 859				for (uframe = 0; uframe < 8; uframe++) {
 860					status = check_intr_schedule (ehci,
 861							frame, uframe, qh,
 862							&c_mask);
 863					if (status == 0)
 864						break;
 865				}
 866			}
 867
 868		/* qh->period == 0 means every uframe */
 869		} else {
 870			frame = 0;
 871			status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
 872		}
 873		if (status)
 874			goto done;
 875		qh->start = frame;
 876
 877		/* reset S-frame and (maybe) C-frame masks */
 878		hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
 879		hw->hw_info2 |= qh->period
 880			? cpu_to_hc32(ehci, 1 << uframe)
 881			: cpu_to_hc32(ehci, QH_SMASK);
 882		hw->hw_info2 |= c_mask;
 883	} else
 884		ehci_dbg (ehci, "reused qh %p schedule\n", qh);
 
 
 
 
 
 
 
 
 
 
 
 
 885
 886	/* stuff into the periodic schedule */
 887	status = qh_link_periodic (ehci, qh);
 888done:
 889	return status;
 890}
 891
 892static int intr_submit (
 893	struct ehci_hcd		*ehci,
 894	struct urb		*urb,
 895	struct list_head	*qtd_list,
 896	gfp_t			mem_flags
 897) {
 898	unsigned		epnum;
 899	unsigned long		flags;
 900	struct ehci_qh		*qh;
 901	int			status;
 902	struct list_head	empty;
 903
 904	/* get endpoint and transfer/schedule data */
 905	epnum = urb->ep->desc.bEndpointAddress;
 906
 907	spin_lock_irqsave (&ehci->lock, flags);
 908
 909	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
 910		status = -ESHUTDOWN;
 911		goto done_not_linked;
 912	}
 913	status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
 914	if (unlikely(status))
 915		goto done_not_linked;
 916
 917	/* get qh and force any scheduling errors */
 918	INIT_LIST_HEAD (&empty);
 919	qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
 920	if (qh == NULL) {
 921		status = -ENOMEM;
 922		goto done;
 923	}
 924	if (qh->qh_state == QH_STATE_IDLE) {
 925		if ((status = qh_schedule (ehci, qh)) != 0)
 926			goto done;
 927	}
 928
 929	/* then queue the urb's tds to the qh */
 930	qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
 931	BUG_ON (qh == NULL);
 932
 
 
 
 
 
 
 
 
 
 933	/* ... update usbfs periodic stats */
 934	ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
 935
 936done:
 937	if (unlikely(status))
 938		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
 939done_not_linked:
 940	spin_unlock_irqrestore (&ehci->lock, flags);
 941	if (status)
 942		qtd_list_free (ehci, urb, qtd_list);
 943
 944	return status;
 945}
 946
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 947/*-------------------------------------------------------------------------*/
 948
 949/* ehci_iso_stream ops work with both ITD and SITD */
 950
 951static struct ehci_iso_stream *
 952iso_stream_alloc (gfp_t mem_flags)
 953{
 954	struct ehci_iso_stream *stream;
 955
 956	stream = kzalloc(sizeof *stream, mem_flags);
 957	if (likely (stream != NULL)) {
 958		INIT_LIST_HEAD(&stream->td_list);
 959		INIT_LIST_HEAD(&stream->free_list);
 960		stream->next_uframe = -1;
 961		stream->refcount = 1;
 962	}
 963	return stream;
 964}
 965
 966static void
 967iso_stream_init (
 968	struct ehci_hcd		*ehci,
 969	struct ehci_iso_stream	*stream,
 970	struct usb_device	*dev,
 971	int			pipe,
 972	unsigned		interval
 973)
 974{
 975	static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
 976
 
 977	u32			buf1;
 978	unsigned		epnum, maxp;
 979	int			is_input;
 980	long			bandwidth;
 981
 982	/*
 983	 * this might be a "high bandwidth" highspeed endpoint,
 984	 * as encoded in the ep descriptor's wMaxPacket field
 985	 */
 986	epnum = usb_pipeendpoint (pipe);
 987	is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
 988	maxp = usb_maxpacket(dev, pipe, !is_input);
 989	if (is_input) {
 990		buf1 = (1 << 11);
 991	} else {
 992		buf1 = 0;
 993	}
 994
 995	/* knows about ITD vs SITD */
 996	if (dev->speed == USB_SPEED_HIGH) {
 997		unsigned multi = hb_mult(maxp);
 998
 999		stream->highspeed = 1;
1000
1001		maxp = max_packet(maxp);
1002		buf1 |= maxp;
1003		maxp *= multi;
1004
1005		stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
1006		stream->buf1 = cpu_to_hc32(ehci, buf1);
1007		stream->buf2 = cpu_to_hc32(ehci, multi);
1008
1009		/* usbfs wants to report the average usecs per frame tied up
1010		 * when transfers on this endpoint are scheduled ...
1011		 */
1012		stream->usecs = HS_USECS_ISO (maxp);
1013		bandwidth = stream->usecs * 8;
1014		bandwidth /= interval;
 
 
 
 
 
 
 
 
 
 
1015
1016	} else {
1017		u32		addr;
1018		int		think_time;
1019		int		hs_transfers;
1020
1021		addr = dev->ttport << 24;
1022		if (!ehci_is_TDI(ehci)
1023				|| (dev->tt->hub !=
1024					ehci_to_hcd(ehci)->self.root_hub))
1025			addr |= dev->tt->hub->devnum << 16;
1026		addr |= epnum << 8;
1027		addr |= dev->devnum;
1028		stream->usecs = HS_USECS_ISO (maxp);
1029		think_time = dev->tt ? dev->tt->think_time : 0;
1030		stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
1031				dev->speed, is_input, 1, maxp));
1032		hs_transfers = max (1u, (maxp + 187) / 188);
1033		if (is_input) {
1034			u32	tmp;
1035
1036			addr |= 1 << 31;
1037			stream->c_usecs = stream->usecs;
1038			stream->usecs = HS_USECS_ISO (1);
1039			stream->raw_mask = 1;
1040
1041			/* c-mask as specified in USB 2.0 11.18.4 3.c */
1042			tmp = (1 << (hs_transfers + 2)) - 1;
1043			stream->raw_mask |= tmp << (8 + 2);
1044		} else
1045			stream->raw_mask = smask_out [hs_transfers - 1];
1046		bandwidth = stream->usecs + stream->c_usecs;
1047		bandwidth /= interval << 3;
1048
1049		/* stream->splits gets created from raw_mask later */
 
 
 
 
 
 
 
 
 
 
 
 
 
1050		stream->address = cpu_to_hc32(ehci, addr);
1051	}
1052	stream->bandwidth = bandwidth;
1053
1054	stream->udev = dev;
 
1055
1056	stream->bEndpointAddress = is_input | epnum;
1057	stream->interval = interval;
1058	stream->maxp = maxp;
1059}
1060
1061static void
1062iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
1063{
1064	stream->refcount--;
1065
1066	/* free whenever just a dev->ep reference remains.
1067	 * not like a QH -- no persistent state (toggle, halt)
1068	 */
1069	if (stream->refcount == 1) {
1070		// BUG_ON (!list_empty(&stream->td_list));
1071
1072		while (!list_empty (&stream->free_list)) {
1073			struct list_head	*entry;
1074
1075			entry = stream->free_list.next;
1076			list_del (entry);
1077
1078			/* knows about ITD vs SITD */
1079			if (stream->highspeed) {
1080				struct ehci_itd		*itd;
1081
1082				itd = list_entry (entry, struct ehci_itd,
1083						itd_list);
1084				dma_pool_free (ehci->itd_pool, itd,
1085						itd->itd_dma);
1086			} else {
1087				struct ehci_sitd	*sitd;
1088
1089				sitd = list_entry (entry, struct ehci_sitd,
1090						sitd_list);
1091				dma_pool_free (ehci->sitd_pool, sitd,
1092						sitd->sitd_dma);
1093			}
1094		}
1095
1096		stream->bEndpointAddress &= 0x0f;
1097		if (stream->ep)
1098			stream->ep->hcpriv = NULL;
1099
1100		kfree(stream);
1101	}
1102}
1103
1104static inline struct ehci_iso_stream *
1105iso_stream_get (struct ehci_iso_stream *stream)
1106{
1107	if (likely (stream != NULL))
1108		stream->refcount++;
1109	return stream;
1110}
1111
1112static struct ehci_iso_stream *
1113iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
1114{
1115	unsigned		epnum;
1116	struct ehci_iso_stream	*stream;
1117	struct usb_host_endpoint *ep;
1118	unsigned long		flags;
1119
1120	epnum = usb_pipeendpoint (urb->pipe);
1121	if (usb_pipein(urb->pipe))
1122		ep = urb->dev->ep_in[epnum];
1123	else
1124		ep = urb->dev->ep_out[epnum];
1125
1126	spin_lock_irqsave (&ehci->lock, flags);
1127	stream = ep->hcpriv;
1128
1129	if (unlikely (stream == NULL)) {
1130		stream = iso_stream_alloc(GFP_ATOMIC);
1131		if (likely (stream != NULL)) {
1132			/* dev->ep owns the initial refcount */
1133			ep->hcpriv = stream;
1134			stream->ep = ep;
1135			iso_stream_init(ehci, stream, urb->dev, urb->pipe,
1136					urb->interval);
1137		}
1138
1139	/* if dev->ep [epnum] is a QH, hw is set */
1140	} else if (unlikely (stream->hw != NULL)) {
1141		ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
1142			urb->dev->devpath, epnum,
1143			usb_pipein(urb->pipe) ? "in" : "out");
1144		stream = NULL;
1145	}
1146
1147	/* caller guarantees an eventual matching iso_stream_put */
1148	stream = iso_stream_get (stream);
1149
1150	spin_unlock_irqrestore (&ehci->lock, flags);
1151	return stream;
1152}
1153
1154/*-------------------------------------------------------------------------*/
1155
1156/* ehci_iso_sched ops can be ITD-only or SITD-only */
1157
1158static struct ehci_iso_sched *
1159iso_sched_alloc (unsigned packets, gfp_t mem_flags)
1160{
1161	struct ehci_iso_sched	*iso_sched;
1162	int			size = sizeof *iso_sched;
1163
1164	size += packets * sizeof (struct ehci_iso_packet);
1165	iso_sched = kzalloc(size, mem_flags);
1166	if (likely (iso_sched != NULL)) {
1167		INIT_LIST_HEAD (&iso_sched->td_list);
1168	}
1169	return iso_sched;
1170}
1171
1172static inline void
1173itd_sched_init(
1174	struct ehci_hcd		*ehci,
1175	struct ehci_iso_sched	*iso_sched,
1176	struct ehci_iso_stream	*stream,
1177	struct urb		*urb
1178)
1179{
1180	unsigned	i;
1181	dma_addr_t	dma = urb->transfer_dma;
1182
1183	/* how many uframes are needed for these transfers */
1184	iso_sched->span = urb->number_of_packets * stream->interval;
1185
1186	/* figure out per-uframe itd fields that we'll need later
1187	 * when we fit new itds into the schedule.
1188	 */
1189	for (i = 0; i < urb->number_of_packets; i++) {
1190		struct ehci_iso_packet	*uframe = &iso_sched->packet [i];
1191		unsigned		length;
1192		dma_addr_t		buf;
1193		u32			trans;
1194
1195		length = urb->iso_frame_desc [i].length;
1196		buf = dma + urb->iso_frame_desc [i].offset;
1197
1198		trans = EHCI_ISOC_ACTIVE;
1199		trans |= buf & 0x0fff;
1200		if (unlikely (((i + 1) == urb->number_of_packets))
1201				&& !(urb->transfer_flags & URB_NO_INTERRUPT))
1202			trans |= EHCI_ITD_IOC;
1203		trans |= length << 16;
1204		uframe->transaction = cpu_to_hc32(ehci, trans);
1205
1206		/* might need to cross a buffer page within a uframe */
1207		uframe->bufp = (buf & ~(u64)0x0fff);
1208		buf += length;
1209		if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
1210			uframe->cross = 1;
1211	}
1212}
1213
1214static void
1215iso_sched_free (
1216	struct ehci_iso_stream	*stream,
1217	struct ehci_iso_sched	*iso_sched
1218)
1219{
1220	if (!iso_sched)
1221		return;
1222	// caller must hold ehci->lock!
1223	list_splice (&iso_sched->td_list, &stream->free_list);
1224	kfree (iso_sched);
1225}
1226
1227static int
1228itd_urb_transaction (
1229	struct ehci_iso_stream	*stream,
1230	struct ehci_hcd		*ehci,
1231	struct urb		*urb,
1232	gfp_t			mem_flags
1233)
1234{
1235	struct ehci_itd		*itd;
1236	dma_addr_t		itd_dma;
1237	int			i;
1238	unsigned		num_itds;
1239	struct ehci_iso_sched	*sched;
1240	unsigned long		flags;
1241
1242	sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1243	if (unlikely (sched == NULL))
1244		return -ENOMEM;
1245
1246	itd_sched_init(ehci, sched, stream, urb);
1247
1248	if (urb->interval < 8)
1249		num_itds = 1 + (sched->span + 7) / 8;
1250	else
1251		num_itds = urb->number_of_packets;
1252
1253	/* allocate/init ITDs */
1254	spin_lock_irqsave (&ehci->lock, flags);
1255	for (i = 0; i < num_itds; i++) {
1256
1257		/* free_list.next might be cache-hot ... but maybe
1258		 * the HC caches it too. avoid that issue for now.
 
1259		 */
1260
1261		/* prefer previously-allocated itds */
1262		if (likely (!list_empty(&stream->free_list))) {
1263			itd = list_entry (stream->free_list.prev,
1264					struct ehci_itd, itd_list);
 
 
1265			list_del (&itd->itd_list);
1266			itd_dma = itd->itd_dma;
1267		} else {
 
1268			spin_unlock_irqrestore (&ehci->lock, flags);
1269			itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
1270					&itd_dma);
1271			spin_lock_irqsave (&ehci->lock, flags);
1272			if (!itd) {
1273				iso_sched_free(stream, sched);
1274				spin_unlock_irqrestore(&ehci->lock, flags);
1275				return -ENOMEM;
1276			}
1277		}
1278
1279		memset (itd, 0, sizeof *itd);
1280		itd->itd_dma = itd_dma;
 
1281		list_add (&itd->itd_list, &sched->td_list);
1282	}
1283	spin_unlock_irqrestore (&ehci->lock, flags);
1284
1285	/* temporarily store schedule info in hcpriv */
1286	urb->hcpriv = sched;
1287	urb->error_count = 0;
1288	return 0;
1289}
1290
1291/*-------------------------------------------------------------------------*/
1292
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1293static inline int
1294itd_slot_ok (
1295	struct ehci_hcd		*ehci,
1296	u32			mod,
1297	u32			uframe,
1298	u8			usecs,
1299	u32			period
1300)
1301{
1302	uframe %= period;
1303	do {
1304		/* can't commit more than uframe_periodic_max usec */
1305		if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
1306				> (ehci->uframe_periodic_max - usecs))
1307			return 0;
1308
1309		/* we know urb->interval is 2^N uframes */
1310		uframe += period;
1311	} while (uframe < mod);
 
 
 
 
 
1312	return 1;
1313}
1314
1315static inline int
1316sitd_slot_ok (
1317	struct ehci_hcd		*ehci,
1318	u32			mod,
1319	struct ehci_iso_stream	*stream,
1320	u32			uframe,
1321	struct ehci_iso_sched	*sched,
1322	u32			period_uframes
1323)
1324{
1325	u32			mask, tmp;
1326	u32			frame, uf;
 
 
1327
1328	mask = stream->raw_mask << (uframe & 7);
 
 
1329
1330	/* for IN, don't wrap CSPLIT into the next frame */
1331	if (mask & ~0xffff)
1332		return 0;
1333
1334	/* check bandwidth */
1335	uframe %= period_uframes;
1336	frame = uframe >> 3;
1337
1338#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1339	/* The tt's fullspeed bus bandwidth must be available.
1340	 * tt_available scheduling guarantees 10+% for control/bulk.
1341	 */
1342	uf = uframe & 7;
1343	if (!tt_available(ehci, period_uframes >> 3,
1344			stream->udev, frame, uf, stream->tt_usecs))
1345		return 0;
1346#else
1347	/* tt must be idle for start(s), any gap, and csplit.
1348	 * assume scheduling slop leaves 10+% for control/bulk.
1349	 */
1350	if (!tt_no_collision(ehci, period_uframes >> 3,
1351			stream->udev, frame, mask))
1352		return 0;
1353#endif
1354
1355	/* this multi-pass logic is simple, but performance may
1356	 * suffer when the schedule data isn't cached.
1357	 */
1358	do {
1359		u32		max_used;
1360
1361		frame = uframe >> 3;
1362		uf = uframe & 7;
1363
1364		/* check starts (OUT uses more than one) */
1365		max_used = ehci->uframe_periodic_max - stream->usecs;
1366		for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1367			if (periodic_usecs (ehci, frame, uf) > max_used)
 
1368				return 0;
1369		}
1370
1371		/* for IN, check CSPLIT */
1372		if (stream->c_usecs) {
1373			uf = uframe & 7;
1374			max_used = ehci->uframe_periodic_max - stream->c_usecs;
1375			do {
1376				tmp = 1 << uf;
1377				tmp <<= 8;
1378				if ((stream->raw_mask & tmp) == 0)
1379					continue;
1380				if (periodic_usecs (ehci, frame, uf)
1381						> max_used)
1382					return 0;
1383			} while (++uf < 8);
1384		}
1385
1386		/* we know urb->interval is 2^N uframes */
1387		uframe += period_uframes;
1388	} while (uframe < mod);
1389
1390	stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7));
 
1391	return 1;
1392}
1393
1394/*
1395 * This scheduler plans almost as far into the future as it has actual
1396 * periodic schedule slots.  (Affected by TUNE_FLS, which defaults to
1397 * "as small as possible" to be cache-friendlier.)  That limits the size
1398 * transfers you can stream reliably; avoid more than 64 msec per urb.
1399 * Also avoid queue depths of less than ehci's worst irq latency (affected
1400 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1401 * and other factors); or more than about 230 msec total (for portability,
1402 * given EHCI_TUNE_FLS and the slop).  Or, write a smarter scheduler!
1403 */
1404
1405#define SCHEDULE_SLOP	80	/* microframes */
1406
1407static int
1408iso_stream_schedule (
1409	struct ehci_hcd		*ehci,
1410	struct urb		*urb,
1411	struct ehci_iso_stream	*stream
1412)
1413{
1414	u32			now, next, start, period, span;
1415	int			status;
 
1416	unsigned		mod = ehci->periodic_size << 3;
1417	struct ehci_iso_sched	*sched = urb->hcpriv;
 
 
1418
1419	period = urb->interval;
1420	span = sched->span;
1421	if (!stream->highspeed) {
1422		period <<= 3;
1423		span <<= 3;
1424	}
1425
1426	if (span > mod - SCHEDULE_SLOP) {
1427		ehci_dbg (ehci, "iso request %p too long\n", urb);
1428		status = -EFBIG;
1429		goto fail;
1430	}
 
 
 
 
 
 
 
 
 
1431
1432	now = ehci_read_frame_index(ehci) & (mod - 1);
1433
1434	/* Typical case: reuse current schedule, stream is still active.
1435	 * Hopefully there are no gaps from the host falling behind
1436	 * (irq delays etc), but if there are we'll take the next
1437	 * slot in the schedule, implicitly assuming URB_ISO_ASAP.
1438	 */
1439	if (likely (!list_empty (&stream->td_list))) {
1440		u32	excess;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1441
1442		/* For high speed devices, allow scheduling within the
1443		 * isochronous scheduling threshold.  For full speed devices
1444		 * and Intel PCI-based controllers, don't (work around for
1445		 * Intel ICH9 bug).
1446		 */
1447		if (!stream->highspeed && ehci->fs_i_thresh)
1448			next = now + ehci->i_thresh;
1449		else
1450			next = now;
 
 
 
 
1451
1452		/* Fell behind (by up to twice the slop amount)?
1453		 * We decide based on the time of the last currently-scheduled
1454		 * slot, not the time of the next available slot.
1455		 */
1456		excess = (stream->next_uframe - period - next) & (mod - 1);
1457		if (excess >= mod - 2 * SCHEDULE_SLOP)
1458			start = next + excess - mod + period *
1459					DIV_ROUND_UP(mod - excess, period);
1460		else
1461			start = next + excess + period;
1462		if (start - now >= mod) {
1463			ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n",
1464					urb, start - now - period, period,
1465					mod);
1466			status = -EFBIG;
1467			goto fail;
1468		}
 
 
 
1469	}
1470
1471	/* need to schedule; when's the next (u)frame we could start?
1472	 * this is bigger than ehci->i_thresh allows; scheduling itself
1473	 * isn't free, the slop should handle reasonably slow cpus.  it
1474	 * can also help high bandwidth if the dma and irq loads don't
1475	 * jump until after the queue is primed.
 
 
 
 
 
 
1476	 */
1477	else {
1478		int done = 0;
1479		start = SCHEDULE_SLOP + (now & ~0x07);
1480
1481		/* NOTE:  assumes URB_ISO_ASAP, to limit complexity/bugs */
1482
1483		/* find a uframe slot with enough bandwidth.
1484		 * Early uframes are more precious because full-speed
1485		 * iso IN transfers can't use late uframes,
1486		 * and therefore they should be allocated last.
1487		 */
1488		next = start;
1489		start += period;
1490		do {
1491			start--;
1492			/* check schedule: enough space? */
1493			if (stream->highspeed) {
1494				if (itd_slot_ok(ehci, mod, start,
1495						stream->usecs, period))
1496					done = 1;
1497			} else {
1498				if ((start % 8) >= 6)
1499					continue;
1500				if (sitd_slot_ok(ehci, mod, stream,
1501						start, sched, period))
1502					done = 1;
1503			}
1504		} while (start > next && !done);
1505
1506		/* no room in the schedule */
1507		if (!done) {
1508			ehci_dbg(ehci, "iso resched full %p (now %d max %d)\n",
1509				urb, now, now + mod);
1510			status = -ENOSPC;
1511			goto fail;
1512		}
 
 
 
 
 
 
 
 
 
 
1513	}
1514
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1515	/* Tried to schedule too far into the future? */
1516	if (unlikely(start - now + span - period
1517				>= mod - 2 * SCHEDULE_SLOP)) {
1518		ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n",
1519				urb, start - now, span - period,
1520				mod - 2 * SCHEDULE_SLOP);
1521		status = -EFBIG;
1522		goto fail;
1523	}
1524
1525	stream->next_uframe = start & (mod - 1);
 
1526
1527	/* report high speed start in uframes; full speed, in frames */
1528	urb->start_frame = stream->next_uframe;
1529	if (!stream->highspeed)
1530		urb->start_frame >>= 3;
1531	return 0;
 
 
 
 
1532
1533 fail:
1534	iso_sched_free(stream, sched);
1535	urb->hcpriv = NULL;
1536	return status;
1537}
1538
1539/*-------------------------------------------------------------------------*/
1540
1541static inline void
1542itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1543		struct ehci_itd *itd)
1544{
1545	int i;
1546
1547	/* it's been recently zeroed */
1548	itd->hw_next = EHCI_LIST_END(ehci);
1549	itd->hw_bufp [0] = stream->buf0;
1550	itd->hw_bufp [1] = stream->buf1;
1551	itd->hw_bufp [2] = stream->buf2;
1552
1553	for (i = 0; i < 8; i++)
1554		itd->index[i] = -1;
1555
1556	/* All other fields are filled when scheduling */
1557}
1558
1559static inline void
1560itd_patch(
1561	struct ehci_hcd		*ehci,
1562	struct ehci_itd		*itd,
1563	struct ehci_iso_sched	*iso_sched,
1564	unsigned		index,
1565	u16			uframe
1566)
1567{
1568	struct ehci_iso_packet	*uf = &iso_sched->packet [index];
1569	unsigned		pg = itd->pg;
1570
1571	// BUG_ON (pg == 6 && uf->cross);
1572
1573	uframe &= 0x07;
1574	itd->index [uframe] = index;
1575
1576	itd->hw_transaction[uframe] = uf->transaction;
1577	itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1578	itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1579	itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1580
1581	/* iso_frame_desc[].offset must be strictly increasing */
1582	if (unlikely (uf->cross)) {
1583		u64	bufp = uf->bufp + 4096;
1584
1585		itd->pg = ++pg;
1586		itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1587		itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1588	}
1589}
1590
1591static inline void
1592itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1593{
1594	union ehci_shadow	*prev = &ehci->pshadow[frame];
1595	__hc32			*hw_p = &ehci->periodic[frame];
1596	union ehci_shadow	here = *prev;
1597	__hc32			type = 0;
1598
1599	/* skip any iso nodes which might belong to previous microframes */
1600	while (here.ptr) {
1601		type = Q_NEXT_TYPE(ehci, *hw_p);
1602		if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
1603			break;
1604		prev = periodic_next_shadow(ehci, prev, type);
1605		hw_p = shadow_next_periodic(ehci, &here, type);
1606		here = *prev;
1607	}
1608
1609	itd->itd_next = here;
1610	itd->hw_next = *hw_p;
1611	prev->itd = itd;
1612	itd->frame = frame;
1613	wmb ();
1614	*hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1615}
1616
1617/* fit urb's itds into the selected schedule slot; activate as needed */
1618static int
1619itd_link_urb (
1620	struct ehci_hcd		*ehci,
1621	struct urb		*urb,
1622	unsigned		mod,
1623	struct ehci_iso_stream	*stream
1624)
1625{
1626	int			packet;
1627	unsigned		next_uframe, uframe, frame;
1628	struct ehci_iso_sched	*iso_sched = urb->hcpriv;
1629	struct ehci_itd		*itd;
1630
1631	next_uframe = stream->next_uframe & (mod - 1);
1632
1633	if (unlikely (list_empty(&stream->td_list))) {
1634		ehci_to_hcd(ehci)->self.bandwidth_allocated
1635				+= stream->bandwidth;
1636		ehci_vdbg (ehci,
1637			"schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1638			urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1639			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1640			urb->interval,
1641			next_uframe >> 3, next_uframe & 0x7);
1642	}
1643
1644	if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1645		if (ehci->amd_pll_fix == 1)
1646			usb_amd_quirk_pll_disable();
1647	}
1648
1649	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1650
1651	/* fill iTDs uframe by uframe */
1652	for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
 
1653		if (itd == NULL) {
1654			/* ASSERT:  we have all necessary itds */
1655			// BUG_ON (list_empty (&iso_sched->td_list));
1656
1657			/* ASSERT:  no itds for this endpoint in this uframe */
1658
1659			itd = list_entry (iso_sched->td_list.next,
1660					struct ehci_itd, itd_list);
1661			list_move_tail (&itd->itd_list, &stream->td_list);
1662			itd->stream = iso_stream_get (stream);
1663			itd->urb = urb;
1664			itd_init (ehci, stream, itd);
1665		}
1666
1667		uframe = next_uframe & 0x07;
1668		frame = next_uframe >> 3;
1669
1670		itd_patch(ehci, itd, iso_sched, packet, uframe);
1671
1672		next_uframe += stream->interval;
1673		next_uframe &= mod - 1;
1674		packet++;
1675
1676		/* link completed itds into the schedule */
1677		if (((next_uframe >> 3) != frame)
1678				|| packet == urb->number_of_packets) {
1679			itd_link(ehci, frame & (ehci->periodic_size - 1), itd);
1680			itd = NULL;
1681		}
1682	}
1683	stream->next_uframe = next_uframe;
1684
1685	/* don't need that schedule data any more */
1686	iso_sched_free (stream, iso_sched);
1687	urb->hcpriv = NULL;
1688
1689	timer_action (ehci, TIMER_IO_WATCHDOG);
1690	return enable_periodic(ehci);
1691}
1692
1693#define	ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1694
1695/* Process and recycle a completed ITD.  Return true iff its urb completed,
1696 * and hence its completion callback probably added things to the hardware
1697 * schedule.
1698 *
1699 * Note that we carefully avoid recycling this descriptor until after any
1700 * completion callback runs, so that it won't be reused quickly.  That is,
1701 * assuming (a) no more than two urbs per frame on this endpoint, and also
1702 * (b) only this endpoint's completions submit URBs.  It seems some silicon
1703 * corrupts things if you reuse completed descriptors very quickly...
1704 */
1705static unsigned
1706itd_complete (
1707	struct ehci_hcd	*ehci,
1708	struct ehci_itd	*itd
1709) {
1710	struct urb				*urb = itd->urb;
1711	struct usb_iso_packet_descriptor	*desc;
1712	u32					t;
1713	unsigned				uframe;
1714	int					urb_index = -1;
1715	struct ehci_iso_stream			*stream = itd->stream;
1716	struct usb_device			*dev;
1717	unsigned				retval = false;
1718
1719	/* for each uframe with a packet */
1720	for (uframe = 0; uframe < 8; uframe++) {
1721		if (likely (itd->index[uframe] == -1))
1722			continue;
1723		urb_index = itd->index[uframe];
1724		desc = &urb->iso_frame_desc [urb_index];
1725
1726		t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]);
1727		itd->hw_transaction [uframe] = 0;
1728
1729		/* report transfer status */
1730		if (unlikely (t & ISO_ERRS)) {
1731			urb->error_count++;
1732			if (t & EHCI_ISOC_BUF_ERR)
1733				desc->status = usb_pipein (urb->pipe)
1734					? -ENOSR  /* hc couldn't read */
1735					: -ECOMM; /* hc couldn't write */
1736			else if (t & EHCI_ISOC_BABBLE)
1737				desc->status = -EOVERFLOW;
1738			else /* (t & EHCI_ISOC_XACTERR) */
1739				desc->status = -EPROTO;
1740
1741			/* HC need not update length with this error */
1742			if (!(t & EHCI_ISOC_BABBLE)) {
1743				desc->actual_length = EHCI_ITD_LENGTH(t);
1744				urb->actual_length += desc->actual_length;
1745			}
1746		} else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1747			desc->status = 0;
1748			desc->actual_length = EHCI_ITD_LENGTH(t);
1749			urb->actual_length += desc->actual_length;
1750		} else {
1751			/* URB was too late */
1752			desc->status = -EXDEV;
1753		}
1754	}
1755
1756	/* handle completion now? */
1757	if (likely ((urb_index + 1) != urb->number_of_packets))
1758		goto done;
1759
1760	/* ASSERT: it's really the last itd for this urb
1761	list_for_each_entry (itd, &stream->td_list, itd_list)
1762		BUG_ON (itd->urb == urb);
1763	 */
1764
1765	/* give urb back to the driver; completion often (re)submits */
1766	dev = urb->dev;
1767	ehci_urb_done(ehci, urb, 0);
1768	retval = true;
1769	urb = NULL;
1770	(void) disable_periodic(ehci);
1771	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1772
 
 
 
 
1773	if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1774		if (ehci->amd_pll_fix == 1)
1775			usb_amd_quirk_pll_enable();
1776	}
1777
1778	if (unlikely(list_is_singular(&stream->td_list))) {
1779		ehci_to_hcd(ehci)->self.bandwidth_allocated
1780				-= stream->bandwidth;
1781		ehci_vdbg (ehci,
1782			"deschedule devp %s ep%d%s-iso\n",
1783			dev->devpath, stream->bEndpointAddress & 0x0f,
1784			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1785	}
1786	iso_stream_put (ehci, stream);
1787
1788done:
1789	itd->urb = NULL;
1790	if (ehci->clock_frame != itd->frame || itd->index[7] != -1) {
1791		/* OK to recycle this ITD now. */
1792		itd->stream = NULL;
1793		list_move(&itd->itd_list, &stream->free_list);
1794		iso_stream_put(ehci, stream);
1795	} else {
1796		/* HW might remember this ITD, so we can't recycle it yet.
1797		 * Move it to a safe place until a new frame starts.
1798		 */
1799		list_move(&itd->itd_list, &ehci->cached_itd_list);
1800		if (stream->refcount == 2) {
1801			/* If iso_stream_put() were called here, stream
1802			 * would be freed.  Instead, just prevent reuse.
1803			 */
1804			stream->ep->hcpriv = NULL;
1805			stream->ep = NULL;
1806		}
1807	}
 
1808	return retval;
1809}
1810
1811/*-------------------------------------------------------------------------*/
1812
1813static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
1814	gfp_t mem_flags)
1815{
1816	int			status = -EINVAL;
1817	unsigned long		flags;
1818	struct ehci_iso_stream	*stream;
1819
1820	/* Get iso_stream head */
1821	stream = iso_stream_find (ehci, urb);
1822	if (unlikely (stream == NULL)) {
1823		ehci_dbg (ehci, "can't get iso stream\n");
1824		return -ENOMEM;
1825	}
1826	if (unlikely (urb->interval != stream->interval)) {
1827		ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1828			stream->interval, urb->interval);
1829		goto done;
1830	}
1831
1832#ifdef EHCI_URB_TRACE
1833	ehci_dbg (ehci,
1834		"%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1835		__func__, urb->dev->devpath, urb,
1836		usb_pipeendpoint (urb->pipe),
1837		usb_pipein (urb->pipe) ? "in" : "out",
1838		urb->transfer_buffer_length,
1839		urb->number_of_packets, urb->interval,
1840		stream);
1841#endif
1842
1843	/* allocate ITDs w/o locking anything */
1844	status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1845	if (unlikely (status < 0)) {
1846		ehci_dbg (ehci, "can't init itds\n");
1847		goto done;
1848	}
1849
1850	/* schedule ... need to lock */
1851	spin_lock_irqsave (&ehci->lock, flags);
1852	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1853		status = -ESHUTDOWN;
1854		goto done_not_linked;
1855	}
1856	status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1857	if (unlikely(status))
1858		goto done_not_linked;
1859	status = iso_stream_schedule(ehci, urb, stream);
1860	if (likely (status == 0))
1861		itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1862	else
 
 
 
1863		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1864done_not_linked:
 
1865	spin_unlock_irqrestore (&ehci->lock, flags);
1866
1867done:
1868	if (unlikely (status < 0))
1869		iso_stream_put (ehci, stream);
1870	return status;
1871}
1872
1873/*-------------------------------------------------------------------------*/
1874
1875/*
1876 * "Split ISO TDs" ... used for USB 1.1 devices going through the
1877 * TTs in USB 2.0 hubs.  These need microframe scheduling.
1878 */
1879
1880static inline void
1881sitd_sched_init(
1882	struct ehci_hcd		*ehci,
1883	struct ehci_iso_sched	*iso_sched,
1884	struct ehci_iso_stream	*stream,
1885	struct urb		*urb
1886)
1887{
1888	unsigned	i;
1889	dma_addr_t	dma = urb->transfer_dma;
1890
1891	/* how many frames are needed for these transfers */
1892	iso_sched->span = urb->number_of_packets * stream->interval;
1893
1894	/* figure out per-frame sitd fields that we'll need later
1895	 * when we fit new sitds into the schedule.
1896	 */
1897	for (i = 0; i < urb->number_of_packets; i++) {
1898		struct ehci_iso_packet	*packet = &iso_sched->packet [i];
1899		unsigned		length;
1900		dma_addr_t		buf;
1901		u32			trans;
1902
1903		length = urb->iso_frame_desc [i].length & 0x03ff;
1904		buf = dma + urb->iso_frame_desc [i].offset;
1905
1906		trans = SITD_STS_ACTIVE;
1907		if (((i + 1) == urb->number_of_packets)
1908				&& !(urb->transfer_flags & URB_NO_INTERRUPT))
1909			trans |= SITD_IOC;
1910		trans |= length << 16;
1911		packet->transaction = cpu_to_hc32(ehci, trans);
1912
1913		/* might need to cross a buffer page within a td */
1914		packet->bufp = buf;
1915		packet->buf1 = (buf + length) & ~0x0fff;
1916		if (packet->buf1 != (buf & ~(u64)0x0fff))
1917			packet->cross = 1;
1918
1919		/* OUT uses multiple start-splits */
1920		if (stream->bEndpointAddress & USB_DIR_IN)
1921			continue;
1922		length = (length + 187) / 188;
1923		if (length > 1) /* BEGIN vs ALL */
1924			length |= 1 << 3;
1925		packet->buf1 |= length;
1926	}
1927}
1928
1929static int
1930sitd_urb_transaction (
1931	struct ehci_iso_stream	*stream,
1932	struct ehci_hcd		*ehci,
1933	struct urb		*urb,
1934	gfp_t			mem_flags
1935)
1936{
1937	struct ehci_sitd	*sitd;
1938	dma_addr_t		sitd_dma;
1939	int			i;
1940	struct ehci_iso_sched	*iso_sched;
1941	unsigned long		flags;
1942
1943	iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1944	if (iso_sched == NULL)
1945		return -ENOMEM;
1946
1947	sitd_sched_init(ehci, iso_sched, stream, urb);
1948
1949	/* allocate/init sITDs */
1950	spin_lock_irqsave (&ehci->lock, flags);
1951	for (i = 0; i < urb->number_of_packets; i++) {
1952
1953		/* NOTE:  for now, we don't try to handle wraparound cases
1954		 * for IN (using sitd->hw_backpointer, like a FSTN), which
1955		 * means we never need two sitds for full speed packets.
1956		 */
1957
1958		/* free_list.next might be cache-hot ... but maybe
1959		 * the HC caches it too. avoid that issue for now.
 
1960		 */
1961
1962		/* prefer previously-allocated sitds */
1963		if (!list_empty(&stream->free_list)) {
1964			sitd = list_entry (stream->free_list.prev,
1965					 struct ehci_sitd, sitd_list);
 
 
1966			list_del (&sitd->sitd_list);
1967			sitd_dma = sitd->sitd_dma;
1968		} else {
 
1969			spin_unlock_irqrestore (&ehci->lock, flags);
1970			sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1971					&sitd_dma);
1972			spin_lock_irqsave (&ehci->lock, flags);
1973			if (!sitd) {
1974				iso_sched_free(stream, iso_sched);
1975				spin_unlock_irqrestore(&ehci->lock, flags);
1976				return -ENOMEM;
1977			}
1978		}
1979
1980		memset (sitd, 0, sizeof *sitd);
1981		sitd->sitd_dma = sitd_dma;
 
1982		list_add (&sitd->sitd_list, &iso_sched->td_list);
1983	}
1984
1985	/* temporarily store schedule info in hcpriv */
1986	urb->hcpriv = iso_sched;
1987	urb->error_count = 0;
1988
1989	spin_unlock_irqrestore (&ehci->lock, flags);
1990	return 0;
1991}
1992
1993/*-------------------------------------------------------------------------*/
1994
1995static inline void
1996sitd_patch(
1997	struct ehci_hcd		*ehci,
1998	struct ehci_iso_stream	*stream,
1999	struct ehci_sitd	*sitd,
2000	struct ehci_iso_sched	*iso_sched,
2001	unsigned		index
2002)
2003{
2004	struct ehci_iso_packet	*uf = &iso_sched->packet [index];
2005	u64			bufp = uf->bufp;
2006
2007	sitd->hw_next = EHCI_LIST_END(ehci);
2008	sitd->hw_fullspeed_ep = stream->address;
2009	sitd->hw_uframe = stream->splits;
2010	sitd->hw_results = uf->transaction;
2011	sitd->hw_backpointer = EHCI_LIST_END(ehci);
2012
2013	bufp = uf->bufp;
2014	sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
2015	sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
2016
2017	sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
2018	if (uf->cross)
2019		bufp += 4096;
2020	sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
2021	sitd->index = index;
2022}
2023
2024static inline void
2025sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
2026{
2027	/* note: sitd ordering could matter (CSPLIT then SSPLIT) */
2028	sitd->sitd_next = ehci->pshadow [frame];
2029	sitd->hw_next = ehci->periodic [frame];
2030	ehci->pshadow [frame].sitd = sitd;
2031	sitd->frame = frame;
2032	wmb ();
2033	ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
2034}
2035
2036/* fit urb's sitds into the selected schedule slot; activate as needed */
2037static int
2038sitd_link_urb (
2039	struct ehci_hcd		*ehci,
2040	struct urb		*urb,
2041	unsigned		mod,
2042	struct ehci_iso_stream	*stream
2043)
2044{
2045	int			packet;
2046	unsigned		next_uframe;
2047	struct ehci_iso_sched	*sched = urb->hcpriv;
2048	struct ehci_sitd	*sitd;
2049
2050	next_uframe = stream->next_uframe;
2051
2052	if (list_empty(&stream->td_list)) {
2053		/* usbfs ignores TT bandwidth */
2054		ehci_to_hcd(ehci)->self.bandwidth_allocated
2055				+= stream->bandwidth;
2056		ehci_vdbg (ehci,
2057			"sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
2058			urb->dev->devpath, stream->bEndpointAddress & 0x0f,
2059			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
2060			(next_uframe >> 3) & (ehci->periodic_size - 1),
2061			stream->interval, hc32_to_cpu(ehci, stream->splits));
2062	}
2063
2064	if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2065		if (ehci->amd_pll_fix == 1)
2066			usb_amd_quirk_pll_disable();
2067	}
2068
2069	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2070
2071	/* fill sITDs frame by frame */
2072	for (packet = 0, sitd = NULL;
2073			packet < urb->number_of_packets;
2074			packet++) {
2075
2076		/* ASSERT:  we have all necessary sitds */
2077		BUG_ON (list_empty (&sched->td_list));
2078
2079		/* ASSERT:  no itds for this endpoint in this frame */
2080
2081		sitd = list_entry (sched->td_list.next,
2082				struct ehci_sitd, sitd_list);
2083		list_move_tail (&sitd->sitd_list, &stream->td_list);
2084		sitd->stream = iso_stream_get (stream);
2085		sitd->urb = urb;
2086
2087		sitd_patch(ehci, stream, sitd, sched, packet);
2088		sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1),
2089				sitd);
2090
2091		next_uframe += stream->interval << 3;
2092	}
2093	stream->next_uframe = next_uframe & (mod - 1);
2094
2095	/* don't need that schedule data any more */
2096	iso_sched_free (stream, sched);
2097	urb->hcpriv = NULL;
2098
2099	timer_action (ehci, TIMER_IO_WATCHDOG);
2100	return enable_periodic(ehci);
2101}
2102
2103/*-------------------------------------------------------------------------*/
2104
2105#define	SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2106				| SITD_STS_XACT | SITD_STS_MMF)
2107
2108/* Process and recycle a completed SITD.  Return true iff its urb completed,
2109 * and hence its completion callback probably added things to the hardware
2110 * schedule.
2111 *
2112 * Note that we carefully avoid recycling this descriptor until after any
2113 * completion callback runs, so that it won't be reused quickly.  That is,
2114 * assuming (a) no more than two urbs per frame on this endpoint, and also
2115 * (b) only this endpoint's completions submit URBs.  It seems some silicon
2116 * corrupts things if you reuse completed descriptors very quickly...
2117 */
2118static unsigned
2119sitd_complete (
2120	struct ehci_hcd		*ehci,
2121	struct ehci_sitd	*sitd
2122) {
2123	struct urb				*urb = sitd->urb;
2124	struct usb_iso_packet_descriptor	*desc;
2125	u32					t;
2126	int					urb_index = -1;
2127	struct ehci_iso_stream			*stream = sitd->stream;
2128	struct usb_device			*dev;
2129	unsigned				retval = false;
2130
2131	urb_index = sitd->index;
2132	desc = &urb->iso_frame_desc [urb_index];
2133	t = hc32_to_cpup(ehci, &sitd->hw_results);
2134
2135	/* report transfer status */
2136	if (t & SITD_ERRS) {
2137		urb->error_count++;
2138		if (t & SITD_STS_DBE)
2139			desc->status = usb_pipein (urb->pipe)
2140				? -ENOSR  /* hc couldn't read */
2141				: -ECOMM; /* hc couldn't write */
2142		else if (t & SITD_STS_BABBLE)
2143			desc->status = -EOVERFLOW;
2144		else /* XACT, MMF, etc */
2145			desc->status = -EPROTO;
 
 
 
2146	} else {
2147		desc->status = 0;
2148		desc->actual_length = desc->length - SITD_LENGTH(t);
2149		urb->actual_length += desc->actual_length;
2150	}
2151
2152	/* handle completion now? */
2153	if ((urb_index + 1) != urb->number_of_packets)
2154		goto done;
2155
2156	/* ASSERT: it's really the last sitd for this urb
2157	list_for_each_entry (sitd, &stream->td_list, sitd_list)
2158		BUG_ON (sitd->urb == urb);
2159	 */
2160
2161	/* give urb back to the driver; completion often (re)submits */
2162	dev = urb->dev;
2163	ehci_urb_done(ehci, urb, 0);
2164	retval = true;
2165	urb = NULL;
2166	(void) disable_periodic(ehci);
2167	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2168
 
 
 
 
2169	if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2170		if (ehci->amd_pll_fix == 1)
2171			usb_amd_quirk_pll_enable();
2172	}
2173
2174	if (list_is_singular(&stream->td_list)) {
2175		ehci_to_hcd(ehci)->self.bandwidth_allocated
2176				-= stream->bandwidth;
2177		ehci_vdbg (ehci,
2178			"deschedule devp %s ep%d%s-iso\n",
2179			dev->devpath, stream->bEndpointAddress & 0x0f,
2180			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
2181	}
2182	iso_stream_put (ehci, stream);
2183
2184done:
2185	sitd->urb = NULL;
2186	if (ehci->clock_frame != sitd->frame) {
2187		/* OK to recycle this SITD now. */
2188		sitd->stream = NULL;
2189		list_move(&sitd->sitd_list, &stream->free_list);
2190		iso_stream_put(ehci, stream);
2191	} else {
2192		/* HW might remember this SITD, so we can't recycle it yet.
2193		 * Move it to a safe place until a new frame starts.
2194		 */
2195		list_move(&sitd->sitd_list, &ehci->cached_sitd_list);
2196		if (stream->refcount == 2) {
2197			/* If iso_stream_put() were called here, stream
2198			 * would be freed.  Instead, just prevent reuse.
2199			 */
2200			stream->ep->hcpriv = NULL;
2201			stream->ep = NULL;
2202		}
2203	}
 
2204	return retval;
2205}
2206
2207
2208static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
2209	gfp_t mem_flags)
2210{
2211	int			status = -EINVAL;
2212	unsigned long		flags;
2213	struct ehci_iso_stream	*stream;
2214
2215	/* Get iso_stream head */
2216	stream = iso_stream_find (ehci, urb);
2217	if (stream == NULL) {
2218		ehci_dbg (ehci, "can't get iso stream\n");
2219		return -ENOMEM;
2220	}
2221	if (urb->interval != stream->interval) {
2222		ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
2223			stream->interval, urb->interval);
2224		goto done;
2225	}
2226
2227#ifdef EHCI_URB_TRACE
2228	ehci_dbg (ehci,
2229		"submit %p dev%s ep%d%s-iso len %d\n",
2230		urb, urb->dev->devpath,
2231		usb_pipeendpoint (urb->pipe),
2232		usb_pipein (urb->pipe) ? "in" : "out",
2233		urb->transfer_buffer_length);
2234#endif
2235
2236	/* allocate SITDs */
2237	status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
2238	if (status < 0) {
2239		ehci_dbg (ehci, "can't init sitds\n");
2240		goto done;
2241	}
2242
2243	/* schedule ... need to lock */
2244	spin_lock_irqsave (&ehci->lock, flags);
2245	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
2246		status = -ESHUTDOWN;
2247		goto done_not_linked;
2248	}
2249	status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2250	if (unlikely(status))
2251		goto done_not_linked;
2252	status = iso_stream_schedule(ehci, urb, stream);
2253	if (status == 0)
2254		sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
2255	else
 
 
 
2256		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2257done_not_linked:
 
2258	spin_unlock_irqrestore (&ehci->lock, flags);
2259
2260done:
2261	if (status < 0)
2262		iso_stream_put (ehci, stream);
2263	return status;
2264}
2265
2266/*-------------------------------------------------------------------------*/
2267
2268static void free_cached_lists(struct ehci_hcd *ehci)
2269{
2270	struct ehci_itd *itd, *n;
2271	struct ehci_sitd *sitd, *sn;
2272
2273	list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) {
2274		struct ehci_iso_stream	*stream = itd->stream;
2275		itd->stream = NULL;
2276		list_move(&itd->itd_list, &stream->free_list);
2277		iso_stream_put(ehci, stream);
2278	}
2279
2280	list_for_each_entry_safe(sitd, sn, &ehci->cached_sitd_list, sitd_list) {
2281		struct ehci_iso_stream	*stream = sitd->stream;
2282		sitd->stream = NULL;
2283		list_move(&sitd->sitd_list, &stream->free_list);
2284		iso_stream_put(ehci, stream);
2285	}
2286}
2287
2288/*-------------------------------------------------------------------------*/
2289
2290static void
2291scan_periodic (struct ehci_hcd *ehci)
2292{
2293	unsigned	now_uframe, frame, clock, clock_frame, mod;
2294	unsigned	modified;
2295
2296	mod = ehci->periodic_size << 3;
2297
2298	/*
2299	 * When running, scan from last scan point up to "now"
2300	 * else clean up by scanning everything that's left.
2301	 * Touches as few pages as possible:  cache-friendly.
2302	 */
2303	now_uframe = ehci->next_uframe;
2304	if (ehci->rh_state == EHCI_RH_RUNNING) {
2305		clock = ehci_read_frame_index(ehci);
2306		clock_frame = (clock >> 3) & (ehci->periodic_size - 1);
2307	} else  {
2308		clock = now_uframe + mod - 1;
2309		clock_frame = -1;
2310	}
2311	if (ehci->clock_frame != clock_frame) {
2312		free_cached_lists(ehci);
2313		ehci->clock_frame = clock_frame;
2314	}
2315	clock &= mod - 1;
2316	clock_frame = clock >> 3;
2317	++ehci->periodic_stamp;
2318
 
2319	for (;;) {
2320		union ehci_shadow	q, *q_p;
2321		__hc32			type, *hw_p;
2322		unsigned		incomplete = false;
2323
2324		frame = now_uframe >> 3;
2325
2326restart:
2327		/* scan each element in frame's queue for completions */
2328		q_p = &ehci->pshadow [frame];
2329		hw_p = &ehci->periodic [frame];
2330		q.ptr = q_p->ptr;
2331		type = Q_NEXT_TYPE(ehci, *hw_p);
2332		modified = 0;
2333
2334		while (q.ptr != NULL) {
2335			unsigned		uf;
2336			union ehci_shadow	temp;
2337			int			live;
2338
2339			live = (ehci->rh_state == EHCI_RH_RUNNING);
2340			switch (hc32_to_cpu(ehci, type)) {
2341			case Q_TYPE_QH:
2342				/* handle any completions */
2343				temp.qh = qh_get (q.qh);
2344				type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next);
2345				q = q.qh->qh_next;
2346				if (temp.qh->stamp != ehci->periodic_stamp) {
2347					modified = qh_completions(ehci, temp.qh);
2348					if (!modified)
2349						temp.qh->stamp = ehci->periodic_stamp;
2350					if (unlikely(list_empty(&temp.qh->qtd_list) ||
2351							temp.qh->needs_rescan))
2352						intr_deschedule(ehci, temp.qh);
2353				}
2354				qh_put (temp.qh);
2355				break;
2356			case Q_TYPE_FSTN:
2357				/* for "save place" FSTNs, look at QH entries
2358				 * in the previous frame for completions.
2359				 */
2360				if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) {
2361					ehci_dbg(ehci,
2362						"ignoring completions from FSTNs\n");
2363				}
2364				type = Q_NEXT_TYPE(ehci, q.fstn->hw_next);
2365				q = q.fstn->fstn_next;
2366				break;
2367			case Q_TYPE_ITD:
2368				/* If this ITD is still active, leave it for
2369				 * later processing ... check the next entry.
2370				 * No need to check for activity unless the
2371				 * frame is current.
2372				 */
2373				if (frame == clock_frame && live) {
2374					rmb();
2375					for (uf = 0; uf < 8; uf++) {
2376						if (q.itd->hw_transaction[uf] &
2377							    ITD_ACTIVE(ehci))
2378							break;
2379					}
2380					if (uf < 8) {
2381						incomplete = true;
2382						q_p = &q.itd->itd_next;
2383						hw_p = &q.itd->hw_next;
2384						type = Q_NEXT_TYPE(ehci,
2385							q.itd->hw_next);
2386						q = *q_p;
2387						break;
2388					}
2389				}
2390
2391				/* Take finished ITDs out of the schedule
2392				 * and process them:  recycle, maybe report
2393				 * URB completion.  HC won't cache the
2394				 * pointer for much longer, if at all.
2395				 */
2396				*q_p = q.itd->itd_next;
2397				if (!ehci->use_dummy_qh ||
2398				    q.itd->hw_next != EHCI_LIST_END(ehci))
2399					*hw_p = q.itd->hw_next;
2400				else
2401					*hw_p = ehci->dummy->qh_dma;
 
2402				type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2403				wmb();
2404				modified = itd_complete (ehci, q.itd);
2405				q = *q_p;
2406				break;
2407			case Q_TYPE_SITD:
2408				/* If this SITD is still active, leave it for
2409				 * later processing ... check the next entry.
2410				 * No need to check for activity unless the
2411				 * frame is current.
2412				 */
2413				if (((frame == clock_frame) ||
2414				     (((frame + 1) & (ehci->periodic_size - 1))
2415				      == clock_frame))
2416				    && live
2417				    && (q.sitd->hw_results &
2418					SITD_ACTIVE(ehci))) {
2419
2420					incomplete = true;
2421					q_p = &q.sitd->sitd_next;
2422					hw_p = &q.sitd->hw_next;
2423					type = Q_NEXT_TYPE(ehci,
2424							q.sitd->hw_next);
2425					q = *q_p;
2426					break;
2427				}
2428
2429				/* Take finished SITDs out of the schedule
2430				 * and process them:  recycle, maybe report
2431				 * URB completion.
2432				 */
2433				*q_p = q.sitd->sitd_next;
2434				if (!ehci->use_dummy_qh ||
2435				    q.sitd->hw_next != EHCI_LIST_END(ehci))
2436					*hw_p = q.sitd->hw_next;
2437				else
2438					*hw_p = ehci->dummy->qh_dma;
 
2439				type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2440				wmb();
2441				modified = sitd_complete (ehci, q.sitd);
2442				q = *q_p;
2443				break;
2444			default:
2445				ehci_dbg(ehci, "corrupt type %d frame %d shadow %p\n",
2446					type, frame, q.ptr);
2447				// BUG ();
 
 
 
 
2448				q.ptr = NULL;
 
2449			}
2450
2451			/* assume completion callbacks modify the queue */
2452			if (unlikely (modified)) {
2453				if (likely(ehci->periodic_sched > 0))
2454					goto restart;
2455				/* short-circuit this scan */
2456				now_uframe = clock;
2457				break;
2458			}
2459		}
2460
2461		/* If we can tell we caught up to the hardware, stop now.
2462		 * We can't advance our scan without collecting the ISO
2463		 * transfers that are still pending in this frame.
2464		 */
2465		if (incomplete && ehci->rh_state == EHCI_RH_RUNNING) {
2466			ehci->next_uframe = now_uframe;
2467			break;
2468		}
2469
2470		// FIXME:  this assumes we won't get lapped when
2471		// latencies climb; that should be rare, but...
2472		// detect it, and just go all the way around.
2473		// FLR might help detect this case, so long as latencies
2474		// don't exceed periodic_size msec (default 1.024 sec).
2475
2476		// FIXME:  likewise assumes HC doesn't halt mid-scan
2477
2478		if (now_uframe == clock) {
2479			unsigned	now;
2480
2481			if (ehci->rh_state != EHCI_RH_RUNNING
2482					|| ehci->periodic_sched == 0)
2483				break;
2484			ehci->next_uframe = now_uframe;
2485			now = ehci_read_frame_index(ehci) & (mod - 1);
2486			if (now_uframe == now)
2487				break;
2488
2489			/* rescan the rest of this frame, then ... */
2490			clock = now;
2491			clock_frame = clock >> 3;
2492			if (ehci->clock_frame != clock_frame) {
2493				free_cached_lists(ehci);
2494				ehci->clock_frame = clock_frame;
2495				++ehci->periodic_stamp;
2496			}
2497		} else {
2498			now_uframe++;
2499			now_uframe &= mod - 1;
2500		}
2501	}
2502}