1/*
   2 * Copyright (c) 2001-2004 by David Brownell
   3 * Copyright (c) 2003 Michal Sojka, for high-speed iso transfers
   4 *
   5 * This program is free software; you can redistribute it and/or modify it
   6 * under the terms of the GNU General Public License as published by the
   7 * Free Software Foundation; either version 2 of the License, or (at your
   8 * option) any later version.
   9 *
  10 * This program is distributed in the hope that it will be useful, but
  11 * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  12 * or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
  13 * for more details.
  14 *
  15 * You should have received a copy of the GNU General Public License
  16 * along with this program; if not, write to the Free Software Foundation,
  17 * Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
  18 */
  19
  20/* this file is part of ehci-hcd.c */
  21
  22/*-------------------------------------------------------------------------*/
  23
  24/*
  25 * EHCI scheduled transaction support:  interrupt, iso, split iso
  26 * These are called "periodic" transactions in the EHCI spec.
  27 *
  28 * Note that for interrupt transfers, the QH/QTD manipulation is shared
  29 * with the "asynchronous" transaction support (control/bulk transfers).
  30 * The only real difference is in how interrupt transfers are scheduled.
  31 *
  32 * For ISO, we make an "iso_stream" head to serve the same role as a QH.
  33 * It keeps track of every ITD (or SITD) that's linked, and holds enough
  34 * pre-calculated schedule data to make appending to the queue be quick.
  35 */
  36
  37static int ehci_get_frame (struct usb_hcd *hcd);
  38
  39/*-------------------------------------------------------------------------*/
  40
  41/*
  42 * periodic_next_shadow - return "next" pointer on shadow list
  43 * @periodic: host pointer to qh/itd/sitd
  44 * @tag: hardware tag for type of this record
  45 */
  46static union ehci_shadow *
  47periodic_next_shadow(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  48		__hc32 tag)
  49{
  50	switch (hc32_to_cpu(ehci, tag)) {
  51	case Q_TYPE_QH:
  52		return &periodic->qh->qh_next;
  53	case Q_TYPE_FSTN:
  54		return &periodic->fstn->fstn_next;
  55	case Q_TYPE_ITD:
  56		return &periodic->itd->itd_next;
  57	// case Q_TYPE_SITD:
  58	default:
  59		return &periodic->sitd->sitd_next;
  60	}
  61}
  62
  63static __hc32 *
  64shadow_next_periodic(struct ehci_hcd *ehci, union ehci_shadow *periodic,
  65		__hc32 tag)
  66{
  67	switch (hc32_to_cpu(ehci, tag)) {
  68	/* our ehci_shadow.qh is actually software part */
  69	case Q_TYPE_QH:
  70		return &periodic->qh->hw->hw_next;
  71	/* others are hw parts */
  72	default:
  73		return periodic->hw_next;
  74	}
  75}
  76
  77/* caller must hold ehci->lock */
  78static void periodic_unlink (struct ehci_hcd *ehci, unsigned frame, void *ptr)
  79{
  80	union ehci_shadow	*prev_p = &ehci->pshadow[frame];
  81	__hc32			*hw_p = &ehci->periodic[frame];
  82	union ehci_shadow	here = *prev_p;
  83
  84	/* find predecessor of "ptr"; hw and shadow lists are in sync */
  85	while (here.ptr && here.ptr != ptr) {
  86		prev_p = periodic_next_shadow(ehci, prev_p,
  87				Q_NEXT_TYPE(ehci, *hw_p));
  88		hw_p = shadow_next_periodic(ehci, &here,
  89				Q_NEXT_TYPE(ehci, *hw_p));
  90		here = *prev_p;
  91	}
  92	/* an interrupt entry (at list end) could have been shared */
  93	if (!here.ptr)
  94		return;
  95
  96	/* update shadow and hardware lists ... the old "next" pointers
  97	 * from ptr may still be in use, the caller updates them.
  98	 */
  99	*prev_p = *periodic_next_shadow(ehci, &here,
 100			Q_NEXT_TYPE(ehci, *hw_p));
 101
 102	if (!ehci->use_dummy_qh ||
 103	    *shadow_next_periodic(ehci, &here, Q_NEXT_TYPE(ehci, *hw_p))
 104			!= EHCI_LIST_END(ehci))
 105		*hw_p = *shadow_next_periodic(ehci, &here,
 106				Q_NEXT_TYPE(ehci, *hw_p));
 107	else
 108		*hw_p = ehci->dummy->qh_dma;
 109}
 110
 111/* how many of the uframe's 125 usecs are allocated? */
 112static unsigned short
 113periodic_usecs (struct ehci_hcd *ehci, unsigned frame, unsigned uframe)
 114{
 115	__hc32			*hw_p = &ehci->periodic [frame];
 116	union ehci_shadow	*q = &ehci->pshadow [frame];
 117	unsigned		usecs = 0;
 118	struct ehci_qh_hw	*hw;
 119
 120	while (q->ptr) {
 121		switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
 122		case Q_TYPE_QH:
 123			hw = q->qh->hw;
 124			/* is it in the S-mask? */
 125			if (hw->hw_info2 & cpu_to_hc32(ehci, 1 << uframe))
 126				usecs += q->qh->usecs;
 127			/* ... or C-mask? */
 128			if (hw->hw_info2 & cpu_to_hc32(ehci,
 129					1 << (8 + uframe)))
 130				usecs += q->qh->c_usecs;
 131			hw_p = &hw->hw_next;
 132			q = &q->qh->qh_next;
 133			break;
 134		// case Q_TYPE_FSTN:
 135		default:
 136			/* for "save place" FSTNs, count the relevant INTR
 137			 * bandwidth from the previous frame
 138			 */
 139			if (q->fstn->hw_prev != EHCI_LIST_END(ehci)) {
 140				ehci_dbg (ehci, "ignoring FSTN cost ...\n");
 141			}
 142			hw_p = &q->fstn->hw_next;
 143			q = &q->fstn->fstn_next;
 144			break;
 145		case Q_TYPE_ITD:
 146			if (q->itd->hw_transaction[uframe])
 147				usecs += q->itd->stream->usecs;
 148			hw_p = &q->itd->hw_next;
 149			q = &q->itd->itd_next;
 150			break;
 151		case Q_TYPE_SITD:
 152			/* is it in the S-mask?  (count SPLIT, DATA) */
 153			if (q->sitd->hw_uframe & cpu_to_hc32(ehci,
 154					1 << uframe)) {
 155				if (q->sitd->hw_fullspeed_ep &
 156						cpu_to_hc32(ehci, 1<<31))
 157					usecs += q->sitd->stream->usecs;
 158				else	/* worst case for OUT start-split */
 159					usecs += HS_USECS_ISO (188);
 160			}
 161
 162			/* ... C-mask?  (count CSPLIT, DATA) */
 163			if (q->sitd->hw_uframe &
 164					cpu_to_hc32(ehci, 1 << (8 + uframe))) {
 165				/* worst case for IN complete-split */
 166				usecs += q->sitd->stream->c_usecs;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 167			}
 
 
 
 
 
 
 
 
 168
 169			hw_p = &q->sitd->hw_next;
 170			q = &q->sitd->sitd_next;
 171			break;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 172		}
 173	}
 174#ifdef	DEBUG
 175	if (usecs > ehci->uframe_periodic_max)
 176		ehci_err (ehci, "uframe %d sched overrun: %d usecs\n",
 177			frame * 8 + uframe, usecs);
 178#endif
 179	return usecs;
 
 
 
 
 
 
 
 180}
 181
 182/*-------------------------------------------------------------------------*/
 183
 184static int same_tt (struct usb_device *dev1, struct usb_device *dev2)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 185{
 186	if (!dev1->tt || !dev2->tt)
 187		return 0;
 188	if (dev1->tt != dev2->tt)
 189		return 0;
 190	if (dev1->tt->multi)
 191		return dev1->ttport == dev2->ttport;
 192	else
 193		return 1;
 194}
 195
 196#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
 197
 198/* Which uframe does the low/fullspeed transfer start in?
 199 *
 200 * The parameter is the mask of ssplits in "H-frame" terms
 201 * and this returns the transfer start uframe in "B-frame" terms,
 202 * which allows both to match, e.g. a ssplit in "H-frame" uframe 0
 203 * will cause a transfer in "B-frame" uframe 0.  "B-frames" lag
 204 * "H-frames" by 1 uframe.  See the EHCI spec sec 4.5 and figure 4.7.
 205 */
 206static inline unsigned char tt_start_uframe(struct ehci_hcd *ehci, __hc32 mask)
 207{
 208	unsigned char smask = QH_SMASK & hc32_to_cpu(ehci, mask);
 209	if (!smask) {
 210		ehci_err(ehci, "invalid empty smask!\n");
 211		/* uframe 7 can't have bw so this will indicate failure */
 212		return 7;
 213	}
 214	return ffs(smask) - 1;
 215}
 216
 217static const unsigned char
 218max_tt_usecs[] = { 125, 125, 125, 125, 125, 125, 30, 0 };
 219
 220/* carryover low/fullspeed bandwidth that crosses uframe boundries */
 221static inline void carryover_tt_bandwidth(unsigned short tt_usecs[8])
 222{
 223	int i;
 224	for (i=0; i<7; i++) {
 225		if (max_tt_usecs[i] < tt_usecs[i]) {
 226			tt_usecs[i+1] += tt_usecs[i] - max_tt_usecs[i];
 227			tt_usecs[i] = max_tt_usecs[i];
 228		}
 229	}
 230}
 231
 232/* How many of the tt's periodic downstream 1000 usecs are allocated?
 233 *
 234 * While this measures the bandwidth in terms of usecs/uframe,
 235 * the low/fullspeed bus has no notion of uframes, so any particular
 236 * low/fullspeed transfer can "carry over" from one uframe to the next,
 237 * since the TT just performs downstream transfers in sequence.
 238 *
 239 * For example two separate 100 usec transfers can start in the same uframe,
 240 * and the second one would "carry over" 75 usecs into the next uframe.
 241 */
 242static void
 243periodic_tt_usecs (
 244	struct ehci_hcd *ehci,
 245	struct usb_device *dev,
 246	unsigned frame,
 247	unsigned short tt_usecs[8]
 248)
 249{
 250	__hc32			*hw_p = &ehci->periodic [frame];
 251	union ehci_shadow	*q = &ehci->pshadow [frame];
 252	unsigned char		uf;
 253
 254	memset(tt_usecs, 0, 16);
 255
 256	while (q->ptr) {
 257		switch (hc32_to_cpu(ehci, Q_NEXT_TYPE(ehci, *hw_p))) {
 258		case Q_TYPE_ITD:
 259			hw_p = &q->itd->hw_next;
 260			q = &q->itd->itd_next;
 261			continue;
 262		case Q_TYPE_QH:
 263			if (same_tt(dev, q->qh->dev)) {
 264				uf = tt_start_uframe(ehci, q->qh->hw->hw_info2);
 265				tt_usecs[uf] += q->qh->tt_usecs;
 266			}
 267			hw_p = &q->qh->hw->hw_next;
 268			q = &q->qh->qh_next;
 269			continue;
 270		case Q_TYPE_SITD:
 271			if (same_tt(dev, q->sitd->urb->dev)) {
 272				uf = tt_start_uframe(ehci, q->sitd->hw_uframe);
 273				tt_usecs[uf] += q->sitd->stream->tt_usecs;
 274			}
 275			hw_p = &q->sitd->hw_next;
 276			q = &q->sitd->sitd_next;
 277			continue;
 278		// case Q_TYPE_FSTN:
 279		default:
 280			ehci_dbg(ehci, "ignoring periodic frame %d FSTN\n",
 281					frame);
 282			hw_p = &q->fstn->hw_next;
 283			q = &q->fstn->fstn_next;
 284		}
 285	}
 286
 287	carryover_tt_bandwidth(tt_usecs);
 288
 289	if (max_tt_usecs[7] < tt_usecs[7])
 290		ehci_err(ehci, "frame %d tt sched overrun: %d usecs\n",
 291			frame, tt_usecs[7] - max_tt_usecs[7]);
 292}
 293
 294/*
 295 * Return true if the device's tt's downstream bus is available for a
 296 * periodic transfer of the specified length (usecs), starting at the
 297 * specified frame/uframe.  Note that (as summarized in section 11.19
 298 * of the usb 2.0 spec) TTs can buffer multiple transactions for each
 299 * uframe.
 300 *
 301 * The uframe parameter is when the fullspeed/lowspeed transfer
 302 * should be executed in "B-frame" terms, which is the same as the
 303 * highspeed ssplit's uframe (which is in "H-frame" terms).  For example
 304 * a ssplit in "H-frame" 0 causes a transfer in "B-frame" 0.
 305 * See the EHCI spec sec 4.5 and fig 4.7.
 306 *
 307 * This checks if the full/lowspeed bus, at the specified starting uframe,
 308 * has the specified bandwidth available, according to rules listed
 309 * in USB 2.0 spec section 11.18.1 fig 11-60.
 310 *
 311 * This does not check if the transfer would exceed the max ssplit
 312 * limit of 16, specified in USB 2.0 spec section 11.18.4 requirement #4,
 313 * since proper scheduling limits ssplits to less than 16 per uframe.
 314 */
 315static int tt_available (
 316	struct ehci_hcd		*ehci,
 317	unsigned		period,
 318	struct usb_device	*dev,
 319	unsigned		frame,
 320	unsigned		uframe,
 321	u16			usecs
 322)
 323{
 
 
 
 324	if ((period == 0) || (uframe >= 7))	/* error */
 325		return 0;
 326
 327	for (; frame < ehci->periodic_size; frame += period) {
 328		unsigned short tt_usecs[8];
 
 
 329
 330		periodic_tt_usecs (ehci, dev, frame, tt_usecs);
 
 
 
 
 
 331
 332		ehci_vdbg(ehci, "tt frame %d check %d usecs start uframe %d in"
 333			" schedule %d/%d/%d/%d/%d/%d/%d/%d\n",
 334			frame, usecs, uframe,
 335			tt_usecs[0], tt_usecs[1], tt_usecs[2], tt_usecs[3],
 336			tt_usecs[4], tt_usecs[5], tt_usecs[6], tt_usecs[7]);
 337
 338		if (max_tt_usecs[uframe] <= tt_usecs[uframe]) {
 339			ehci_vdbg(ehci, "frame %d uframe %d fully scheduled\n",
 340				frame, uframe);
 341			return 0;
 342		}
 343
 344		/* special case for isoc transfers larger than 125us:
 345		 * the first and each subsequent fully used uframe
 346		 * must be empty, so as to not illegally delay
 347		 * already scheduled transactions
 348		 */
 349		if (125 < usecs) {
 350			int ufs = (usecs / 125);
 351			int i;
 352			for (i = uframe; i < (uframe + ufs) && i < 8; i++)
 353				if (0 < tt_usecs[i]) {
 354					ehci_vdbg(ehci,
 355						"multi-uframe xfer can't fit "
 356						"in frame %d uframe %d\n",
 357						frame, i);
 358					return 0;
 359				}
 360		}
 361
 362		tt_usecs[uframe] += usecs;
 363
 364		carryover_tt_bandwidth(tt_usecs);
 365
 366		/* fail if the carryover pushed bw past the last uframe's limit */
 367		if (max_tt_usecs[7] < tt_usecs[7]) {
 368			ehci_vdbg(ehci,
 369				"tt unavailable usecs %d frame %d uframe %d\n",
 370				usecs, frame, uframe);
 371			return 0;
 372		}
 373	}
 374
 375	return 1;
 376}
 377
 378#else
 379
 380/* return true iff the device's transaction translator is available
 381 * for a periodic transfer starting at the specified frame, using
 382 * all the uframes in the mask.
 383 */
 384static int tt_no_collision (
 385	struct ehci_hcd		*ehci,
 386	unsigned		period,
 387	struct usb_device	*dev,
 388	unsigned		frame,
 389	u32			uf_mask
 390)
 391{
 392	if (period == 0)	/* error */
 393		return 0;
 394
 395	/* note bandwidth wastage:  split never follows csplit
 396	 * (different dev or endpoint) until the next uframe.
 397	 * calling convention doesn't make that distinction.
 398	 */
 399	for (; frame < ehci->periodic_size; frame += period) {
 400		union ehci_shadow	here;
 401		__hc32			type;
 402		struct ehci_qh_hw	*hw;
 403
 404		here = ehci->pshadow [frame];
 405		type = Q_NEXT_TYPE(ehci, ehci->periodic [frame]);
 406		while (here.ptr) {
 407			switch (hc32_to_cpu(ehci, type)) {
 408			case Q_TYPE_ITD:
 409				type = Q_NEXT_TYPE(ehci, here.itd->hw_next);
 410				here = here.itd->itd_next;
 411				continue;
 412			case Q_TYPE_QH:
 413				hw = here.qh->hw;
 414				if (same_tt (dev, here.qh->dev)) {
 415					u32		mask;
 416
 417					mask = hc32_to_cpu(ehci,
 418							hw->hw_info2);
 419					/* "knows" no gap is needed */
 420					mask |= mask >> 8;
 421					if (mask & uf_mask)
 422						break;
 423				}
 424				type = Q_NEXT_TYPE(ehci, hw->hw_next);
 425				here = here.qh->qh_next;
 426				continue;
 427			case Q_TYPE_SITD:
 428				if (same_tt (dev, here.sitd->urb->dev)) {
 429					u16		mask;
 430
 431					mask = hc32_to_cpu(ehci, here.sitd
 432								->hw_uframe);
 433					/* FIXME assumes no gap for IN! */
 434					mask |= mask >> 8;
 435					if (mask & uf_mask)
 436						break;
 437				}
 438				type = Q_NEXT_TYPE(ehci, here.sitd->hw_next);
 439				here = here.sitd->sitd_next;
 440				continue;
 441			// case Q_TYPE_FSTN:
 442			default:
 443				ehci_dbg (ehci,
 444					"periodic frame %d bogus type %d\n",
 445					frame, type);
 446			}
 447
 448			/* collision or error */
 449			return 0;
 450		}
 451	}
 452
 453	/* no collision */
 454	return 1;
 455}
 456
 457#endif /* CONFIG_USB_EHCI_TT_NEWSCHED */
 458
 459/*-------------------------------------------------------------------------*/
 460
 461static int enable_periodic (struct ehci_hcd *ehci)
 462{
 463	u32	cmd;
 464	int	status;
 465
 466	if (ehci->periodic_sched++)
 467		return 0;
 468
 469	/* did clearing PSE did take effect yet?
 470	 * takes effect only at frame boundaries...
 471	 */
 472	status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
 473					     STS_PSS, 0, 9 * 125);
 474	if (status) {
 475		usb_hc_died(ehci_to_hcd(ehci));
 476		return status;
 477	}
 478
 479	cmd = ehci_readl(ehci, &ehci->regs->command) | CMD_PSE;
 480	ehci_writel(ehci, cmd, &ehci->regs->command);
 481	/* posted write ... PSS happens later */
 482	ehci_to_hcd(ehci)->state = HC_STATE_RUNNING;
 483
 484	/* make sure ehci_work scans these */
 485	ehci->next_uframe = ehci_readl(ehci, &ehci->regs->frame_index)
 486		% (ehci->periodic_size << 3);
 487	if (unlikely(ehci->broken_periodic))
 488		ehci->last_periodic_enable = ktime_get_real();
 489	return 0;
 490}
 491
 492static int disable_periodic (struct ehci_hcd *ehci)
 493{
 494	u32	cmd;
 495	int	status;
 496
 497	if (--ehci->periodic_sched)
 498		return 0;
 499
 500	if (unlikely(ehci->broken_periodic)) {
 501		/* delay experimentally determined */
 502		ktime_t safe = ktime_add_us(ehci->last_periodic_enable, 1000);
 503		ktime_t now = ktime_get_real();
 504		s64 delay = ktime_us_delta(safe, now);
 505
 506		if (unlikely(delay > 0))
 507			udelay(delay);
 508	}
 509
 510	/* did setting PSE not take effect yet?
 511	 * takes effect only at frame boundaries...
 512	 */
 513	status = handshake_on_error_set_halt(ehci, &ehci->regs->status,
 514					     STS_PSS, STS_PSS, 9 * 125);
 515	if (status) {
 516		usb_hc_died(ehci_to_hcd(ehci));
 517		return status;
 518	}
 519
 520	cmd = ehci_readl(ehci, &ehci->regs->command) & ~CMD_PSE;
 521	ehci_writel(ehci, cmd, &ehci->regs->command);
 522	/* posted write ... */
 523
 524	free_cached_lists(ehci);
 525
 526	ehci->next_uframe = -1;
 527	return 0;
 528}
 529
 530/*-------------------------------------------------------------------------*/
 531
 532/* periodic schedule slots have iso tds (normal or split) first, then a
 533 * sparse tree for active interrupt transfers.
 534 *
 535 * this just links in a qh; caller guarantees uframe masks are set right.
 536 * no FSTN support (yet; ehci 0.96+)
 537 */
 538static int qh_link_periodic (struct ehci_hcd *ehci, struct ehci_qh *qh)
 539{
 540	unsigned	i;
 541	unsigned	period = qh->period;
 542
 543	dev_dbg (&qh->dev->dev,
 544		"link qh%d-%04x/%p start %d [%d/%d us]\n",
 545		period, hc32_to_cpup(ehci, &qh->hw->hw_info2)
 546			& (QH_CMASK | QH_SMASK),
 547		qh, qh->start, qh->usecs, qh->c_usecs);
 548
 549	/* high bandwidth, or otherwise every microframe */
 550	if (period == 0)
 551		period = 1;
 552
 553	for (i = qh->start; i < ehci->periodic_size; i += period) {
 554		union ehci_shadow	*prev = &ehci->pshadow[i];
 555		__hc32			*hw_p = &ehci->periodic[i];
 556		union ehci_shadow	here = *prev;
 557		__hc32			type = 0;
 558
 559		/* skip the iso nodes at list head */
 560		while (here.ptr) {
 561			type = Q_NEXT_TYPE(ehci, *hw_p);
 562			if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
 563				break;
 564			prev = periodic_next_shadow(ehci, prev, type);
 565			hw_p = shadow_next_periodic(ehci, &here, type);
 566			here = *prev;
 567		}
 568
 569		/* sorting each branch by period (slow-->fast)
 570		 * enables sharing interior tree nodes
 571		 */
 572		while (here.ptr && qh != here.qh) {
 573			if (qh->period > here.qh->period)
 574				break;
 575			prev = &here.qh->qh_next;
 576			hw_p = &here.qh->hw->hw_next;
 577			here = *prev;
 578		}
 579		/* link in this qh, unless some earlier pass did that */
 580		if (qh != here.qh) {
 581			qh->qh_next = here;
 582			if (here.qh)
 583				qh->hw->hw_next = *hw_p;
 584			wmb ();
 585			prev->qh = qh;
 586			*hw_p = QH_NEXT (ehci, qh->qh_dma);
 587		}
 588	}
 589	qh->qh_state = QH_STATE_LINKED;
 590	qh->xacterrs = 0;
 591	qh_get (qh);
 
 
 
 
 
 592
 593	/* update per-qh bandwidth for usbfs */
 594	ehci_to_hcd(ehci)->self.bandwidth_allocated += qh->period
 595		? ((qh->usecs + qh->c_usecs) / qh->period)
 596		: (qh->usecs * 8);
 597
 598	/* maybe enable periodic schedule processing */
 599	return enable_periodic(ehci);
 
 600}
 601
 602static int qh_unlink_periodic(struct ehci_hcd *ehci, struct ehci_qh *qh)
 603{
 604	unsigned	i;
 605	unsigned	period;
 606
 607	// FIXME:
 608	// IF this isn't high speed
 609	//   and this qh is active in the current uframe
 610	//   (and overlay token SplitXstate is false?)
 611	// THEN
 612	//   qh->hw_info1 |= cpu_to_hc32(1 << 7 /* "ignore" */);
 
 
 
 
 
 
 
 
 613
 614	/* high bandwidth, or otherwise part of every microframe */
 615	if ((period = qh->period) == 0)
 616		period = 1;
 617
 618	for (i = qh->start; i < ehci->periodic_size; i += period)
 619		periodic_unlink (ehci, i, qh);
 620
 621	/* update per-qh bandwidth for usbfs */
 622	ehci_to_hcd(ehci)->self.bandwidth_allocated -= qh->period
 623		? ((qh->usecs + qh->c_usecs) / qh->period)
 624		: (qh->usecs * 8);
 625
 626	dev_dbg (&qh->dev->dev,
 627		"unlink qh%d-%04x/%p start %d [%d/%d us]\n",
 628		qh->period,
 629		hc32_to_cpup(ehci, &qh->hw->hw_info2) & (QH_CMASK | QH_SMASK),
 630		qh, qh->start, qh->usecs, qh->c_usecs);
 631
 632	/* qh->qh_next still "live" to HC */
 633	qh->qh_state = QH_STATE_UNLINK;
 634	qh->qh_next.ptr = NULL;
 635	qh_put (qh);
 636
 637	/* maybe turn off periodic schedule */
 638	return disable_periodic(ehci);
 
 
 639}
 640
 641static void intr_deschedule (struct ehci_hcd *ehci, struct ehci_qh *qh)
 642{
 643	unsigned		wait;
 644	struct ehci_qh_hw	*hw = qh->hw;
 645	int			rc;
 646
 647	/* If the QH isn't linked then there's nothing we can do
 648	 * unless we were called during a giveback, in which case
 649	 * qh_completions() has to deal with it.
 
 
 650	 */
 651	if (qh->qh_state != QH_STATE_LINKED) {
 652		if (qh->qh_state == QH_STATE_COMPLETING)
 653			qh->needs_rescan = 1;
 
 
 
 654		return;
 655	}
 
 
 656
 657	qh_unlink_periodic (ehci, qh);
 658
 659	/* simple/paranoid:  always delay, expecting the HC needs to read
 660	 * qh->hw_next or finish a writeback after SPLIT/CSPLIT ... and
 661	 * expect khubd to clean up after any CSPLITs we won't issue.
 662	 * active high speed queues may need bigger delays...
 
 
 
 663	 */
 664	if (list_empty (&qh->qtd_list)
 665			|| (cpu_to_hc32(ehci, QH_CMASK)
 666					& hw->hw_info2) != 0)
 667		wait = 2;
 668	else
 669		wait = 55;	/* worst case: 3 * 1024 */
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 670
 671	udelay (wait);
 672	qh->qh_state = QH_STATE_IDLE;
 673	hw->hw_next = EHCI_LIST_END(ehci);
 674	wmb ();
 675
 676	qh_completions(ehci, qh);
 
 677
 678	/* reschedule QH iff another request is queued */
 679	if (!list_empty(&qh->qtd_list) &&
 680			HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
 681		rc = qh_schedule(ehci, qh);
 
 
 
 
 682
 683		/* An error here likely indicates handshake failure
 684		 * or no space left in the schedule.  Neither fault
 685		 * should happen often ...
 686		 *
 687		 * FIXME kill the now-dysfunctional queued urbs
 688		 */
 689		if (rc != 0)
 690			ehci_err(ehci, "can't reschedule qh %p, err %d\n",
 691					qh, rc);
 
 692	}
 
 
 
 
 693}
 694
 695/*-------------------------------------------------------------------------*/
 696
 697static int check_period (
 698	struct ehci_hcd *ehci,
 699	unsigned	frame,
 700	unsigned	uframe,
 701	unsigned	period,
 702	unsigned	usecs
 703) {
 704	int		claimed;
 705
 706	/* complete split running into next frame?
 707	 * given FSTN support, we could sometimes check...
 708	 */
 709	if (uframe >= 8)
 710		return 0;
 711
 712	/* convert "usecs we need" to "max already claimed" */
 713	usecs = ehci->uframe_periodic_max - usecs;
 714
 715	/* we "know" 2 and 4 uframe intervals were rejected; so
 716	 * for period 0, check _every_ microframe in the schedule.
 717	 */
 718	if (unlikely (period == 0)) {
 719		do {
 720			for (uframe = 0; uframe < 7; uframe++) {
 721				claimed = periodic_usecs (ehci, frame, uframe);
 722				if (claimed > usecs)
 723					return 0;
 724			}
 725		} while ((frame += 1) < ehci->periodic_size);
 726
 727	/* just check the specified uframe, at that period */
 728	} else {
 729		do {
 730			claimed = periodic_usecs (ehci, frame, uframe);
 731			if (claimed > usecs)
 732				return 0;
 733		} while ((frame += period) < ehci->periodic_size);
 734	}
 735
 736	// success!
 737	return 1;
 738}
 739
 740static int check_intr_schedule (
 741	struct ehci_hcd		*ehci,
 742	unsigned		frame,
 743	unsigned		uframe,
 744	const struct ehci_qh	*qh,
 745	__hc32			*c_maskp
 
 746)
 747{
 748	int		retval = -ENOSPC;
 749	u8		mask = 0;
 750
 751	if (qh->c_usecs && uframe >= 6)		/* FSTN territory? */
 752		goto done;
 753
 754	if (!check_period (ehci, frame, uframe, qh->period, qh->usecs))
 755		goto done;
 756	if (!qh->c_usecs) {
 757		retval = 0;
 758		*c_maskp = 0;
 759		goto done;
 760	}
 761
 762#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
 763	if (tt_available (ehci, qh->period, qh->dev, frame, uframe,
 764				qh->tt_usecs)) {
 765		unsigned i;
 766
 767		/* TODO : this may need FSTN for SSPLIT in uframe 5. */
 768		for (i=uframe+1; i<8 && i<uframe+4; i++)
 769			if (!check_period (ehci, frame, i,
 770						qh->period, qh->c_usecs))
 771				goto done;
 772			else
 773				mask |= 1 << i;
 774
 775		retval = 0;
 776
 777		*c_maskp = cpu_to_hc32(ehci, mask << 8);
 778	}
 779#else
 780	/* Make sure this tt's buffer is also available for CSPLITs.
 781	 * We pessimize a bit; probably the typical full speed case
 782	 * doesn't need the second CSPLIT.
 783	 *
 784	 * NOTE:  both SPLIT and CSPLIT could be checked in just
 785	 * one smart pass...
 786	 */
 787	mask = 0x03 << (uframe + qh->gap_uf);
 788	*c_maskp = cpu_to_hc32(ehci, mask << 8);
 789
 790	mask |= 1 << uframe;
 791	if (tt_no_collision (ehci, qh->period, qh->dev, frame, mask)) {
 792		if (!check_period (ehci, frame, uframe + qh->gap_uf + 1,
 793					qh->period, qh->c_usecs))
 794			goto done;
 795		if (!check_period (ehci, frame, uframe + qh->gap_uf,
 796					qh->period, qh->c_usecs))
 797			goto done;
 798		retval = 0;
 799	}
 800#endif
 801done:
 802	return retval;
 803}
 804
 805/* "first fit" scheduling policy used the first time through,
 806 * or when the previous schedule slot can't be re-used.
 807 */
 808static int qh_schedule(struct ehci_hcd *ehci, struct ehci_qh *qh)
 809{
 810	int		status;
 811	unsigned	uframe;
 812	__hc32		c_mask;
 813	unsigned	frame;		/* 0..(qh->period - 1), or NO_FRAME */
 814	struct ehci_qh_hw	*hw = qh->hw;
 
 815
 816	qh_refresh(ehci, qh);
 817	hw->hw_next = EHCI_LIST_END(ehci);
 818	frame = qh->start;
 819
 820	/* reuse the previous schedule slots, if we can */
 821	if (frame < qh->period) {
 822		uframe = ffs(hc32_to_cpup(ehci, &hw->hw_info2) & QH_SMASK);
 823		status = check_intr_schedule (ehci, frame, --uframe,
 824				qh, &c_mask);
 825	} else {
 826		uframe = 0;
 827		c_mask = 0;
 828		status = -ENOSPC;
 
 
 
 829	}
 
 830
 831	/* else scan the schedule to find a group of slots such that all
 832	 * uframes have enough periodic bandwidth available.
 833	 */
 834	if (status) {
 835		/* "normal" case, uframing flexible except with splits */
 836		if (qh->period) {
 837			int		i;
 838
 839			for (i = qh->period; status && i > 0; --i) {
 840				frame = ++ehci->random_frame % qh->period;
 841				for (uframe = 0; uframe < 8; uframe++) {
 842					status = check_intr_schedule (ehci,
 843							frame, uframe, qh,
 844							&c_mask);
 845					if (status == 0)
 846						break;
 847				}
 848			}
 849
 850		/* qh->period == 0 means every uframe */
 851		} else {
 852			frame = 0;
 853			status = check_intr_schedule (ehci, 0, 0, qh, &c_mask);
 854		}
 855		if (status)
 856			goto done;
 857		qh->start = frame;
 858
 859		/* reset S-frame and (maybe) C-frame masks */
 860		hw->hw_info2 &= cpu_to_hc32(ehci, ~(QH_CMASK | QH_SMASK));
 861		hw->hw_info2 |= qh->period
 862			? cpu_to_hc32(ehci, 1 << uframe)
 863			: cpu_to_hc32(ehci, QH_SMASK);
 864		hw->hw_info2 |= c_mask;
 865	} else
 866		ehci_dbg (ehci, "reused qh %p schedule\n", qh);
 
 
 
 
 
 
 
 
 
 
 
 
 867
 868	/* stuff into the periodic schedule */
 869	status = qh_link_periodic (ehci, qh);
 870done:
 871	return status;
 872}
 873
 874static int intr_submit (
 875	struct ehci_hcd		*ehci,
 876	struct urb		*urb,
 877	struct list_head	*qtd_list,
 878	gfp_t			mem_flags
 879) {
 880	unsigned		epnum;
 881	unsigned long		flags;
 882	struct ehci_qh		*qh;
 883	int			status;
 884	struct list_head	empty;
 885
 886	/* get endpoint and transfer/schedule data */
 887	epnum = urb->ep->desc.bEndpointAddress;
 888
 889	spin_lock_irqsave (&ehci->lock, flags);
 890
 891	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
 892		status = -ESHUTDOWN;
 893		goto done_not_linked;
 894	}
 895	status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
 896	if (unlikely(status))
 897		goto done_not_linked;
 898
 899	/* get qh and force any scheduling errors */
 900	INIT_LIST_HEAD (&empty);
 901	qh = qh_append_tds(ehci, urb, &empty, epnum, &urb->ep->hcpriv);
 902	if (qh == NULL) {
 903		status = -ENOMEM;
 904		goto done;
 905	}
 906	if (qh->qh_state == QH_STATE_IDLE) {
 907		if ((status = qh_schedule (ehci, qh)) != 0)
 908			goto done;
 909	}
 910
 911	/* then queue the urb's tds to the qh */
 912	qh = qh_append_tds(ehci, urb, qtd_list, epnum, &urb->ep->hcpriv);
 913	BUG_ON (qh == NULL);
 914
 
 
 
 
 
 
 
 
 
 915	/* ... update usbfs periodic stats */
 916	ehci_to_hcd(ehci)->self.bandwidth_int_reqs++;
 917
 918done:
 919	if (unlikely(status))
 920		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
 921done_not_linked:
 922	spin_unlock_irqrestore (&ehci->lock, flags);
 923	if (status)
 924		qtd_list_free (ehci, urb, qtd_list);
 925
 926	return status;
 927}
 928
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 929/*-------------------------------------------------------------------------*/
 930
 931/* ehci_iso_stream ops work with both ITD and SITD */
 932
 933static struct ehci_iso_stream *
 934iso_stream_alloc (gfp_t mem_flags)
 935{
 936	struct ehci_iso_stream *stream;
 937
 938	stream = kzalloc(sizeof *stream, mem_flags);
 939	if (likely (stream != NULL)) {
 940		INIT_LIST_HEAD(&stream->td_list);
 941		INIT_LIST_HEAD(&stream->free_list);
 942		stream->next_uframe = -1;
 943		stream->refcount = 1;
 944	}
 945	return stream;
 946}
 947
 948static void
 949iso_stream_init (
 950	struct ehci_hcd		*ehci,
 951	struct ehci_iso_stream	*stream,
 952	struct usb_device	*dev,
 953	int			pipe,
 954	unsigned		interval
 955)
 956{
 957	static const u8 smask_out [] = { 0x01, 0x03, 0x07, 0x0f, 0x1f, 0x3f };
 958
 
 959	u32			buf1;
 960	unsigned		epnum, maxp;
 961	int			is_input;
 962	long			bandwidth;
 963
 964	/*
 965	 * this might be a "high bandwidth" highspeed endpoint,
 966	 * as encoded in the ep descriptor's wMaxPacket field
 967	 */
 968	epnum = usb_pipeendpoint (pipe);
 969	is_input = usb_pipein (pipe) ? USB_DIR_IN : 0;
 970	maxp = usb_maxpacket(dev, pipe, !is_input);
 971	if (is_input) {
 972		buf1 = (1 << 11);
 973	} else {
 974		buf1 = 0;
 975	}
 976
 977	/* knows about ITD vs SITD */
 978	if (dev->speed == USB_SPEED_HIGH) {
 979		unsigned multi = hb_mult(maxp);
 980
 981		stream->highspeed = 1;
 982
 983		maxp = max_packet(maxp);
 984		buf1 |= maxp;
 985		maxp *= multi;
 986
 987		stream->buf0 = cpu_to_hc32(ehci, (epnum << 8) | dev->devnum);
 988		stream->buf1 = cpu_to_hc32(ehci, buf1);
 989		stream->buf2 = cpu_to_hc32(ehci, multi);
 990
 991		/* usbfs wants to report the average usecs per frame tied up
 992		 * when transfers on this endpoint are scheduled ...
 993		 */
 994		stream->usecs = HS_USECS_ISO (maxp);
 995		bandwidth = stream->usecs * 8;
 996		bandwidth /= interval;
 
 
 
 
 
 
 
 
 
 
 997
 998	} else {
 999		u32		addr;
1000		int		think_time;
1001		int		hs_transfers;
1002
1003		addr = dev->ttport << 24;
1004		if (!ehci_is_TDI(ehci)
1005				|| (dev->tt->hub !=
1006					ehci_to_hcd(ehci)->self.root_hub))
1007			addr |= dev->tt->hub->devnum << 16;
1008		addr |= epnum << 8;
1009		addr |= dev->devnum;
1010		stream->usecs = HS_USECS_ISO (maxp);
1011		think_time = dev->tt ? dev->tt->think_time : 0;
1012		stream->tt_usecs = NS_TO_US (think_time + usb_calc_bus_time (
1013				dev->speed, is_input, 1, maxp));
1014		hs_transfers = max (1u, (maxp + 187) / 188);
1015		if (is_input) {
1016			u32	tmp;
1017
1018			addr |= 1 << 31;
1019			stream->c_usecs = stream->usecs;
1020			stream->usecs = HS_USECS_ISO (1);
1021			stream->raw_mask = 1;
1022
1023			/* c-mask as specified in USB 2.0 11.18.4 3.c */
1024			tmp = (1 << (hs_transfers + 2)) - 1;
1025			stream->raw_mask |= tmp << (8 + 2);
1026		} else
1027			stream->raw_mask = smask_out [hs_transfers - 1];
1028		bandwidth = stream->usecs + stream->c_usecs;
1029		bandwidth /= interval << 3;
1030
1031		/* stream->splits gets created from raw_mask later */
 
 
 
 
 
 
 
 
 
 
 
 
 
1032		stream->address = cpu_to_hc32(ehci, addr);
1033	}
1034	stream->bandwidth = bandwidth;
1035
1036	stream->udev = dev;
 
1037
1038	stream->bEndpointAddress = is_input | epnum;
1039	stream->interval = interval;
1040	stream->maxp = maxp;
1041}
1042
1043static void
1044iso_stream_put(struct ehci_hcd *ehci, struct ehci_iso_stream *stream)
1045{
1046	stream->refcount--;
1047
1048	/* free whenever just a dev->ep reference remains.
1049	 * not like a QH -- no persistent state (toggle, halt)
1050	 */
1051	if (stream->refcount == 1) {
1052		// BUG_ON (!list_empty(&stream->td_list));
1053
1054		while (!list_empty (&stream->free_list)) {
1055			struct list_head	*entry;
1056
1057			entry = stream->free_list.next;
1058			list_del (entry);
1059
1060			/* knows about ITD vs SITD */
1061			if (stream->highspeed) {
1062				struct ehci_itd		*itd;
1063
1064				itd = list_entry (entry, struct ehci_itd,
1065						itd_list);
1066				dma_pool_free (ehci->itd_pool, itd,
1067						itd->itd_dma);
1068			} else {
1069				struct ehci_sitd	*sitd;
1070
1071				sitd = list_entry (entry, struct ehci_sitd,
1072						sitd_list);
1073				dma_pool_free (ehci->sitd_pool, sitd,
1074						sitd->sitd_dma);
1075			}
1076		}
1077
1078		stream->bEndpointAddress &= 0x0f;
1079		if (stream->ep)
1080			stream->ep->hcpriv = NULL;
1081
1082		kfree(stream);
1083	}
1084}
1085
1086static inline struct ehci_iso_stream *
1087iso_stream_get (struct ehci_iso_stream *stream)
1088{
1089	if (likely (stream != NULL))
1090		stream->refcount++;
1091	return stream;
1092}
1093
1094static struct ehci_iso_stream *
1095iso_stream_find (struct ehci_hcd *ehci, struct urb *urb)
1096{
1097	unsigned		epnum;
1098	struct ehci_iso_stream	*stream;
1099	struct usb_host_endpoint *ep;
1100	unsigned long		flags;
1101
1102	epnum = usb_pipeendpoint (urb->pipe);
1103	if (usb_pipein(urb->pipe))
1104		ep = urb->dev->ep_in[epnum];
1105	else
1106		ep = urb->dev->ep_out[epnum];
1107
1108	spin_lock_irqsave (&ehci->lock, flags);
1109	stream = ep->hcpriv;
1110
1111	if (unlikely (stream == NULL)) {
1112		stream = iso_stream_alloc(GFP_ATOMIC);
1113		if (likely (stream != NULL)) {
1114			/* dev->ep owns the initial refcount */
1115			ep->hcpriv = stream;
1116			stream->ep = ep;
1117			iso_stream_init(ehci, stream, urb->dev, urb->pipe,
1118					urb->interval);
1119		}
1120
1121	/* if dev->ep [epnum] is a QH, hw is set */
1122	} else if (unlikely (stream->hw != NULL)) {
1123		ehci_dbg (ehci, "dev %s ep%d%s, not iso??\n",
1124			urb->dev->devpath, epnum,
1125			usb_pipein(urb->pipe) ? "in" : "out");
1126		stream = NULL;
1127	}
1128
1129	/* caller guarantees an eventual matching iso_stream_put */
1130	stream = iso_stream_get (stream);
1131
1132	spin_unlock_irqrestore (&ehci->lock, flags);
1133	return stream;
1134}
1135
1136/*-------------------------------------------------------------------------*/
1137
1138/* ehci_iso_sched ops can be ITD-only or SITD-only */
1139
1140static struct ehci_iso_sched *
1141iso_sched_alloc (unsigned packets, gfp_t mem_flags)
1142{
1143	struct ehci_iso_sched	*iso_sched;
1144	int			size = sizeof *iso_sched;
1145
1146	size += packets * sizeof (struct ehci_iso_packet);
1147	iso_sched = kzalloc(size, mem_flags);
1148	if (likely (iso_sched != NULL)) {
1149		INIT_LIST_HEAD (&iso_sched->td_list);
1150	}
1151	return iso_sched;
1152}
1153
1154static inline void
1155itd_sched_init(
1156	struct ehci_hcd		*ehci,
1157	struct ehci_iso_sched	*iso_sched,
1158	struct ehci_iso_stream	*stream,
1159	struct urb		*urb
1160)
1161{
1162	unsigned	i;
1163	dma_addr_t	dma = urb->transfer_dma;
1164
1165	/* how many uframes are needed for these transfers */
1166	iso_sched->span = urb->number_of_packets * stream->interval;
1167
1168	/* figure out per-uframe itd fields that we'll need later
1169	 * when we fit new itds into the schedule.
1170	 */
1171	for (i = 0; i < urb->number_of_packets; i++) {
1172		struct ehci_iso_packet	*uframe = &iso_sched->packet [i];
1173		unsigned		length;
1174		dma_addr_t		buf;
1175		u32			trans;
1176
1177		length = urb->iso_frame_desc [i].length;
1178		buf = dma + urb->iso_frame_desc [i].offset;
1179
1180		trans = EHCI_ISOC_ACTIVE;
1181		trans |= buf & 0x0fff;
1182		if (unlikely (((i + 1) == urb->number_of_packets))
1183				&& !(urb->transfer_flags & URB_NO_INTERRUPT))
1184			trans |= EHCI_ITD_IOC;
1185		trans |= length << 16;
1186		uframe->transaction = cpu_to_hc32(ehci, trans);
1187
1188		/* might need to cross a buffer page within a uframe */
1189		uframe->bufp = (buf & ~(u64)0x0fff);
1190		buf += length;
1191		if (unlikely ((uframe->bufp != (buf & ~(u64)0x0fff))))
1192			uframe->cross = 1;
1193	}
1194}
1195
1196static void
1197iso_sched_free (
1198	struct ehci_iso_stream	*stream,
1199	struct ehci_iso_sched	*iso_sched
1200)
1201{
1202	if (!iso_sched)
1203		return;
1204	// caller must hold ehci->lock!
1205	list_splice (&iso_sched->td_list, &stream->free_list);
1206	kfree (iso_sched);
1207}
1208
1209static int
1210itd_urb_transaction (
1211	struct ehci_iso_stream	*stream,
1212	struct ehci_hcd		*ehci,
1213	struct urb		*urb,
1214	gfp_t			mem_flags
1215)
1216{
1217	struct ehci_itd		*itd;
1218	dma_addr_t		itd_dma;
1219	int			i;
1220	unsigned		num_itds;
1221	struct ehci_iso_sched	*sched;
1222	unsigned long		flags;
1223
1224	sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1225	if (unlikely (sched == NULL))
1226		return -ENOMEM;
1227
1228	itd_sched_init(ehci, sched, stream, urb);
1229
1230	if (urb->interval < 8)
1231		num_itds = 1 + (sched->span + 7) / 8;
1232	else
1233		num_itds = urb->number_of_packets;
1234
1235	/* allocate/init ITDs */
1236	spin_lock_irqsave (&ehci->lock, flags);
1237	for (i = 0; i < num_itds; i++) {
1238
1239		/* free_list.next might be cache-hot ... but maybe
1240		 * the HC caches it too. avoid that issue for now.
 
1241		 */
1242
1243		/* prefer previously-allocated itds */
1244		if (likely (!list_empty(&stream->free_list))) {
1245			itd = list_entry (stream->free_list.prev,
1246					struct ehci_itd, itd_list);
 
 
1247			list_del (&itd->itd_list);
1248			itd_dma = itd->itd_dma;
1249		} else {
 
1250			spin_unlock_irqrestore (&ehci->lock, flags);
1251			itd = dma_pool_alloc (ehci->itd_pool, mem_flags,
1252					&itd_dma);
1253			spin_lock_irqsave (&ehci->lock, flags);
1254			if (!itd) {
1255				iso_sched_free(stream, sched);
1256				spin_unlock_irqrestore(&ehci->lock, flags);
1257				return -ENOMEM;
1258			}
1259		}
1260
1261		memset (itd, 0, sizeof *itd);
1262		itd->itd_dma = itd_dma;
 
1263		list_add (&itd->itd_list, &sched->td_list);
1264	}
1265	spin_unlock_irqrestore (&ehci->lock, flags);
1266
1267	/* temporarily store schedule info in hcpriv */
1268	urb->hcpriv = sched;
1269	urb->error_count = 0;
1270	return 0;
1271}
1272
1273/*-------------------------------------------------------------------------*/
1274
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1275static inline int
1276itd_slot_ok (
1277	struct ehci_hcd		*ehci,
1278	u32			mod,
1279	u32			uframe,
1280	u8			usecs,
1281	u32			period
1282)
1283{
1284	uframe %= period;
1285	do {
1286		/* can't commit more than uframe_periodic_max usec */
1287		if (periodic_usecs (ehci, uframe >> 3, uframe & 0x7)
1288				> (ehci->uframe_periodic_max - usecs))
1289			return 0;
1290
1291		/* we know urb->interval is 2^N uframes */
1292		uframe += period;
1293	} while (uframe < mod);
 
 
1294	return 1;
1295}
1296
1297static inline int
1298sitd_slot_ok (
1299	struct ehci_hcd		*ehci,
1300	u32			mod,
1301	struct ehci_iso_stream	*stream,
1302	u32			uframe,
1303	struct ehci_iso_sched	*sched,
1304	u32			period_uframes
1305)
1306{
1307	u32			mask, tmp;
1308	u32			frame, uf;
1309
1310	mask = stream->raw_mask << (uframe & 7);
 
 
 
 
1311
1312	/* for IN, don't wrap CSPLIT into the next frame */
1313	if (mask & ~0xffff)
1314		return 0;
1315
1316	/* this multi-pass logic is simple, but performance may
1317	 * suffer when the schedule data isn't cached.
1318	 */
1319
1320	/* check bandwidth */
1321	uframe %= period_uframes;
1322	do {
1323		u32		max_used;
1324
1325		frame = uframe >> 3;
1326		uf = uframe & 7;
1327
1328#ifdef CONFIG_USB_EHCI_TT_NEWSCHED
1329		/* The tt's fullspeed bus bandwidth must be available.
1330		 * tt_available scheduling guarantees 10+% for control/bulk.
1331		 */
1332		if (!tt_available (ehci, period_uframes << 3,
1333				stream->udev, frame, uf, stream->tt_usecs))
1334			return 0;
1335#else
1336		/* tt must be idle for start(s), any gap, and csplit.
1337		 * assume scheduling slop leaves 10+% for control/bulk.
1338		 */
1339		if (!tt_no_collision (ehci, period_uframes << 3,
1340				stream->udev, frame, mask))
1341			return 0;
1342#endif
1343
 
 
 
 
1344		/* check starts (OUT uses more than one) */
1345		max_used = ehci->uframe_periodic_max - stream->usecs;
1346		for (tmp = stream->raw_mask & 0xff; tmp; tmp >>= 1, uf++) {
1347			if (periodic_usecs (ehci, frame, uf) > max_used)
 
1348				return 0;
1349		}
1350
1351		/* for IN, check CSPLIT */
1352		if (stream->c_usecs) {
1353			uf = uframe & 7;
1354			max_used = ehci->uframe_periodic_max - stream->c_usecs;
1355			do {
1356				tmp = 1 << uf;
1357				tmp <<= 8;
1358				if ((stream->raw_mask & tmp) == 0)
1359					continue;
1360				if (periodic_usecs (ehci, frame, uf)
1361						> max_used)
1362					return 0;
1363			} while (++uf < 8);
1364		}
1365
1366		/* we know urb->interval is 2^N uframes */
1367		uframe += period_uframes;
1368	} while (uframe < mod);
1369
1370	stream->splits = cpu_to_hc32(ehci, stream->raw_mask << (uframe & 7));
 
1371	return 1;
1372}
1373
1374/*
1375 * This scheduler plans almost as far into the future as it has actual
1376 * periodic schedule slots.  (Affected by TUNE_FLS, which defaults to
1377 * "as small as possible" to be cache-friendlier.)  That limits the size
1378 * transfers you can stream reliably; avoid more than 64 msec per urb.
1379 * Also avoid queue depths of less than ehci's worst irq latency (affected
1380 * by the per-urb URB_NO_INTERRUPT hint, the log2_irq_thresh module parameter,
1381 * and other factors); or more than about 230 msec total (for portability,
1382 * given EHCI_TUNE_FLS and the slop).  Or, write a smarter scheduler!
1383 */
1384
1385#define SCHEDULE_SLOP	80	/* microframes */
1386
1387static int
1388iso_stream_schedule (
1389	struct ehci_hcd		*ehci,
1390	struct urb		*urb,
1391	struct ehci_iso_stream	*stream
1392)
1393{
1394	u32			now, next, start, period, span;
1395	int			status;
 
1396	unsigned		mod = ehci->periodic_size << 3;
1397	struct ehci_iso_sched	*sched = urb->hcpriv;
 
 
1398
1399	period = urb->interval;
1400	span = sched->span;
1401	if (!stream->highspeed) {
1402		period <<= 3;
1403		span <<= 3;
1404	}
1405
1406	if (span > mod - SCHEDULE_SLOP) {
1407		ehci_dbg (ehci, "iso request %p too long\n", urb);
1408		status = -EFBIG;
1409		goto fail;
1410	}
 
 
 
 
 
 
 
 
 
1411
1412	now = ehci_readl(ehci, &ehci->regs->frame_index) & (mod - 1);
1413
1414	/* Typical case: reuse current schedule, stream is still active.
1415	 * Hopefully there are no gaps from the host falling behind
1416	 * (irq delays etc), but if there are we'll take the next
1417	 * slot in the schedule, implicitly assuming URB_ISO_ASAP.
1418	 */
1419	if (likely (!list_empty (&stream->td_list))) {
1420		u32	excess;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1421
1422		/* For high speed devices, allow scheduling within the
1423		 * isochronous scheduling threshold.  For full speed devices
1424		 * and Intel PCI-based controllers, don't (work around for
1425		 * Intel ICH9 bug).
1426		 */
1427		if (!stream->highspeed && ehci->fs_i_thresh)
1428			next = now + ehci->i_thresh;
1429		else
1430			next = now;
 
 
 
 
1431
1432		/* Fell behind (by up to twice the slop amount)?
1433		 * We decide based on the time of the last currently-scheduled
1434		 * slot, not the time of the next available slot.
1435		 */
1436		excess = (stream->next_uframe - period - next) & (mod - 1);
1437		if (excess >= mod - 2 * SCHEDULE_SLOP)
1438			start = next + excess - mod + period *
1439					DIV_ROUND_UP(mod - excess, period);
1440		else
1441			start = next + excess + period;
1442		if (start - now >= mod) {
1443			ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n",
1444					urb, start - now - period, period,
1445					mod);
1446			status = -EFBIG;
1447			goto fail;
1448		}
 
 
 
1449	}
1450
1451	/* need to schedule; when's the next (u)frame we could start?
1452	 * this is bigger than ehci->i_thresh allows; scheduling itself
1453	 * isn't free, the slop should handle reasonably slow cpus.  it
1454	 * can also help high bandwidth if the dma and irq loads don't
1455	 * jump until after the queue is primed.
 
 
 
 
 
 
1456	 */
1457	else {
1458		start = SCHEDULE_SLOP + (now & ~0x07);
 
1459
1460		/* NOTE:  assumes URB_ISO_ASAP, to limit complexity/bugs */
 
1461
1462		/* find a uframe slot with enough bandwidth */
1463		next = start + period;
1464		for (; start < next; start++) {
1465
1466			/* check schedule: enough space? */
1467			if (stream->highspeed) {
1468				if (itd_slot_ok(ehci, mod, start,
1469						stream->usecs, period))
1470					break;
1471			} else {
1472				if ((start % 8) >= 6)
1473					continue;
1474				if (sitd_slot_ok(ehci, mod, stream,
1475						start, sched, period))
1476					break;
1477			}
1478		}
1479
1480		/* no room in the schedule */
1481		if (start == next) {
1482			ehci_dbg(ehci, "iso resched full %p (now %d max %d)\n",
1483				urb, now, now + mod);
1484			status = -ENOSPC;
1485			goto fail;
1486		}
1487	}
1488
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1489	/* Tried to schedule too far into the future? */
1490	if (unlikely(start - now + span - period
1491				>= mod - 2 * SCHEDULE_SLOP)) {
1492		ehci_dbg(ehci, "request %p would overflow (%d+%d >= %d)\n",
1493				urb, start - now, span - period,
1494				mod - 2 * SCHEDULE_SLOP);
1495		status = -EFBIG;
1496		goto fail;
1497	}
1498
1499	stream->next_uframe = start & (mod - 1);
 
1500
1501	/* report high speed start in uframes; full speed, in frames */
1502	urb->start_frame = stream->next_uframe;
1503	if (!stream->highspeed)
1504		urb->start_frame >>= 3;
1505	return 0;
 
 
 
 
1506
1507 fail:
1508	iso_sched_free(stream, sched);
1509	urb->hcpriv = NULL;
1510	return status;
1511}
1512
1513/*-------------------------------------------------------------------------*/
1514
1515static inline void
1516itd_init(struct ehci_hcd *ehci, struct ehci_iso_stream *stream,
1517		struct ehci_itd *itd)
1518{
1519	int i;
1520
1521	/* it's been recently zeroed */
1522	itd->hw_next = EHCI_LIST_END(ehci);
1523	itd->hw_bufp [0] = stream->buf0;
1524	itd->hw_bufp [1] = stream->buf1;
1525	itd->hw_bufp [2] = stream->buf2;
1526
1527	for (i = 0; i < 8; i++)
1528		itd->index[i] = -1;
1529
1530	/* All other fields are filled when scheduling */
1531}
1532
1533static inline void
1534itd_patch(
1535	struct ehci_hcd		*ehci,
1536	struct ehci_itd		*itd,
1537	struct ehci_iso_sched	*iso_sched,
1538	unsigned		index,
1539	u16			uframe
1540)
1541{
1542	struct ehci_iso_packet	*uf = &iso_sched->packet [index];
1543	unsigned		pg = itd->pg;
1544
1545	// BUG_ON (pg == 6 && uf->cross);
1546
1547	uframe &= 0x07;
1548	itd->index [uframe] = index;
1549
1550	itd->hw_transaction[uframe] = uf->transaction;
1551	itd->hw_transaction[uframe] |= cpu_to_hc32(ehci, pg << 12);
1552	itd->hw_bufp[pg] |= cpu_to_hc32(ehci, uf->bufp & ~(u32)0);
1553	itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(uf->bufp >> 32));
1554
1555	/* iso_frame_desc[].offset must be strictly increasing */
1556	if (unlikely (uf->cross)) {
1557		u64	bufp = uf->bufp + 4096;
1558
1559		itd->pg = ++pg;
1560		itd->hw_bufp[pg] |= cpu_to_hc32(ehci, bufp & ~(u32)0);
1561		itd->hw_bufp_hi[pg] |= cpu_to_hc32(ehci, (u32)(bufp >> 32));
1562	}
1563}
1564
1565static inline void
1566itd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_itd *itd)
1567{
1568	union ehci_shadow	*prev = &ehci->pshadow[frame];
1569	__hc32			*hw_p = &ehci->periodic[frame];
1570	union ehci_shadow	here = *prev;
1571	__hc32			type = 0;
1572
1573	/* skip any iso nodes which might belong to previous microframes */
1574	while (here.ptr) {
1575		type = Q_NEXT_TYPE(ehci, *hw_p);
1576		if (type == cpu_to_hc32(ehci, Q_TYPE_QH))
1577			break;
1578		prev = periodic_next_shadow(ehci, prev, type);
1579		hw_p = shadow_next_periodic(ehci, &here, type);
1580		here = *prev;
1581	}
1582
1583	itd->itd_next = here;
1584	itd->hw_next = *hw_p;
1585	prev->itd = itd;
1586	itd->frame = frame;
1587	wmb ();
1588	*hw_p = cpu_to_hc32(ehci, itd->itd_dma | Q_TYPE_ITD);
1589}
1590
1591/* fit urb's itds into the selected schedule slot; activate as needed */
1592static int
1593itd_link_urb (
1594	struct ehci_hcd		*ehci,
1595	struct urb		*urb,
1596	unsigned		mod,
1597	struct ehci_iso_stream	*stream
1598)
1599{
1600	int			packet;
1601	unsigned		next_uframe, uframe, frame;
1602	struct ehci_iso_sched	*iso_sched = urb->hcpriv;
1603	struct ehci_itd		*itd;
1604
1605	next_uframe = stream->next_uframe & (mod - 1);
1606
1607	if (unlikely (list_empty(&stream->td_list))) {
1608		ehci_to_hcd(ehci)->self.bandwidth_allocated
1609				+= stream->bandwidth;
1610		ehci_vdbg (ehci,
1611			"schedule devp %s ep%d%s-iso period %d start %d.%d\n",
1612			urb->dev->devpath, stream->bEndpointAddress & 0x0f,
1613			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
1614			urb->interval,
1615			next_uframe >> 3, next_uframe & 0x7);
1616	}
1617
1618	if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1619		if (ehci->amd_pll_fix == 1)
1620			usb_amd_quirk_pll_disable();
1621	}
1622
1623	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
1624
1625	/* fill iTDs uframe by uframe */
1626	for (packet = 0, itd = NULL; packet < urb->number_of_packets; ) {
 
1627		if (itd == NULL) {
1628			/* ASSERT:  we have all necessary itds */
1629			// BUG_ON (list_empty (&iso_sched->td_list));
1630
1631			/* ASSERT:  no itds for this endpoint in this uframe */
1632
1633			itd = list_entry (iso_sched->td_list.next,
1634					struct ehci_itd, itd_list);
1635			list_move_tail (&itd->itd_list, &stream->td_list);
1636			itd->stream = iso_stream_get (stream);
1637			itd->urb = urb;
1638			itd_init (ehci, stream, itd);
1639		}
1640
1641		uframe = next_uframe & 0x07;
1642		frame = next_uframe >> 3;
1643
1644		itd_patch(ehci, itd, iso_sched, packet, uframe);
1645
1646		next_uframe += stream->interval;
1647		next_uframe &= mod - 1;
1648		packet++;
1649
1650		/* link completed itds into the schedule */
1651		if (((next_uframe >> 3) != frame)
1652				|| packet == urb->number_of_packets) {
1653			itd_link(ehci, frame & (ehci->periodic_size - 1), itd);
1654			itd = NULL;
1655		}
1656	}
1657	stream->next_uframe = next_uframe;
1658
1659	/* don't need that schedule data any more */
1660	iso_sched_free (stream, iso_sched);
1661	urb->hcpriv = NULL;
1662
1663	timer_action (ehci, TIMER_IO_WATCHDOG);
1664	return enable_periodic(ehci);
1665}
1666
1667#define	ISO_ERRS (EHCI_ISOC_BUF_ERR | EHCI_ISOC_BABBLE | EHCI_ISOC_XACTERR)
1668
1669/* Process and recycle a completed ITD.  Return true iff its urb completed,
1670 * and hence its completion callback probably added things to the hardware
1671 * schedule.
1672 *
1673 * Note that we carefully avoid recycling this descriptor until after any
1674 * completion callback runs, so that it won't be reused quickly.  That is,
1675 * assuming (a) no more than two urbs per frame on this endpoint, and also
1676 * (b) only this endpoint's completions submit URBs.  It seems some silicon
1677 * corrupts things if you reuse completed descriptors very quickly...
1678 */
1679static unsigned
1680itd_complete (
1681	struct ehci_hcd	*ehci,
1682	struct ehci_itd	*itd
1683) {
1684	struct urb				*urb = itd->urb;
1685	struct usb_iso_packet_descriptor	*desc;
1686	u32					t;
1687	unsigned				uframe;
1688	int					urb_index = -1;
1689	struct ehci_iso_stream			*stream = itd->stream;
1690	struct usb_device			*dev;
1691	unsigned				retval = false;
1692
1693	/* for each uframe with a packet */
1694	for (uframe = 0; uframe < 8; uframe++) {
1695		if (likely (itd->index[uframe] == -1))
1696			continue;
1697		urb_index = itd->index[uframe];
1698		desc = &urb->iso_frame_desc [urb_index];
1699
1700		t = hc32_to_cpup(ehci, &itd->hw_transaction [uframe]);
1701		itd->hw_transaction [uframe] = 0;
1702
1703		/* report transfer status */
1704		if (unlikely (t & ISO_ERRS)) {
1705			urb->error_count++;
1706			if (t & EHCI_ISOC_BUF_ERR)
1707				desc->status = usb_pipein (urb->pipe)
1708					? -ENOSR  /* hc couldn't read */
1709					: -ECOMM; /* hc couldn't write */
1710			else if (t & EHCI_ISOC_BABBLE)
1711				desc->status = -EOVERFLOW;
1712			else /* (t & EHCI_ISOC_XACTERR) */
1713				desc->status = -EPROTO;
1714
1715			/* HC need not update length with this error */
1716			if (!(t & EHCI_ISOC_BABBLE)) {
1717				desc->actual_length = EHCI_ITD_LENGTH(t);
1718				urb->actual_length += desc->actual_length;
1719			}
1720		} else if (likely ((t & EHCI_ISOC_ACTIVE) == 0)) {
1721			desc->status = 0;
1722			desc->actual_length = EHCI_ITD_LENGTH(t);
1723			urb->actual_length += desc->actual_length;
1724		} else {
1725			/* URB was too late */
1726			desc->status = -EXDEV;
1727		}
1728	}
1729
1730	/* handle completion now? */
1731	if (likely ((urb_index + 1) != urb->number_of_packets))
1732		goto done;
1733
1734	/* ASSERT: it's really the last itd for this urb
1735	list_for_each_entry (itd, &stream->td_list, itd_list)
1736		BUG_ON (itd->urb == urb);
1737	 */
1738
1739	/* give urb back to the driver; completion often (re)submits */
1740	dev = urb->dev;
1741	ehci_urb_done(ehci, urb, 0);
1742	retval = true;
1743	urb = NULL;
1744	(void) disable_periodic(ehci);
1745	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
1746
 
 
 
 
1747	if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
1748		if (ehci->amd_pll_fix == 1)
1749			usb_amd_quirk_pll_enable();
1750	}
1751
1752	if (unlikely(list_is_singular(&stream->td_list))) {
1753		ehci_to_hcd(ehci)->self.bandwidth_allocated
1754				-= stream->bandwidth;
1755		ehci_vdbg (ehci,
1756			"deschedule devp %s ep%d%s-iso\n",
1757			dev->devpath, stream->bEndpointAddress & 0x0f,
1758			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
1759	}
1760	iso_stream_put (ehci, stream);
1761
1762done:
1763	itd->urb = NULL;
1764	if (ehci->clock_frame != itd->frame || itd->index[7] != -1) {
1765		/* OK to recycle this ITD now. */
1766		itd->stream = NULL;
1767		list_move(&itd->itd_list, &stream->free_list);
1768		iso_stream_put(ehci, stream);
1769	} else {
1770		/* HW might remember this ITD, so we can't recycle it yet.
1771		 * Move it to a safe place until a new frame starts.
1772		 */
1773		list_move(&itd->itd_list, &ehci->cached_itd_list);
1774		if (stream->refcount == 2) {
1775			/* If iso_stream_put() were called here, stream
1776			 * would be freed.  Instead, just prevent reuse.
1777			 */
1778			stream->ep->hcpriv = NULL;
1779			stream->ep = NULL;
1780		}
1781	}
 
1782	return retval;
1783}
1784
1785/*-------------------------------------------------------------------------*/
1786
1787static int itd_submit (struct ehci_hcd *ehci, struct urb *urb,
1788	gfp_t mem_flags)
1789{
1790	int			status = -EINVAL;
1791	unsigned long		flags;
1792	struct ehci_iso_stream	*stream;
1793
1794	/* Get iso_stream head */
1795	stream = iso_stream_find (ehci, urb);
1796	if (unlikely (stream == NULL)) {
1797		ehci_dbg (ehci, "can't get iso stream\n");
1798		return -ENOMEM;
1799	}
1800	if (unlikely (urb->interval != stream->interval)) {
1801		ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
1802			stream->interval, urb->interval);
1803		goto done;
1804	}
1805
1806#ifdef EHCI_URB_TRACE
1807	ehci_dbg (ehci,
1808		"%s %s urb %p ep%d%s len %d, %d pkts %d uframes [%p]\n",
1809		__func__, urb->dev->devpath, urb,
1810		usb_pipeendpoint (urb->pipe),
1811		usb_pipein (urb->pipe) ? "in" : "out",
1812		urb->transfer_buffer_length,
1813		urb->number_of_packets, urb->interval,
1814		stream);
1815#endif
1816
1817	/* allocate ITDs w/o locking anything */
1818	status = itd_urb_transaction (stream, ehci, urb, mem_flags);
1819	if (unlikely (status < 0)) {
1820		ehci_dbg (ehci, "can't init itds\n");
1821		goto done;
1822	}
1823
1824	/* schedule ... need to lock */
1825	spin_lock_irqsave (&ehci->lock, flags);
1826	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
1827		status = -ESHUTDOWN;
1828		goto done_not_linked;
1829	}
1830	status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
1831	if (unlikely(status))
1832		goto done_not_linked;
1833	status = iso_stream_schedule(ehci, urb, stream);
1834	if (likely (status == 0))
1835		itd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
1836	else
 
 
 
1837		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
1838done_not_linked:
 
1839	spin_unlock_irqrestore (&ehci->lock, flags);
1840
1841done:
1842	if (unlikely (status < 0))
1843		iso_stream_put (ehci, stream);
1844	return status;
1845}
1846
1847/*-------------------------------------------------------------------------*/
1848
1849/*
1850 * "Split ISO TDs" ... used for USB 1.1 devices going through the
1851 * TTs in USB 2.0 hubs.  These need microframe scheduling.
1852 */
1853
1854static inline void
1855sitd_sched_init(
1856	struct ehci_hcd		*ehci,
1857	struct ehci_iso_sched	*iso_sched,
1858	struct ehci_iso_stream	*stream,
1859	struct urb		*urb
1860)
1861{
1862	unsigned	i;
1863	dma_addr_t	dma = urb->transfer_dma;
1864
1865	/* how many frames are needed for these transfers */
1866	iso_sched->span = urb->number_of_packets * stream->interval;
1867
1868	/* figure out per-frame sitd fields that we'll need later
1869	 * when we fit new sitds into the schedule.
1870	 */
1871	for (i = 0; i < urb->number_of_packets; i++) {
1872		struct ehci_iso_packet	*packet = &iso_sched->packet [i];
1873		unsigned		length;
1874		dma_addr_t		buf;
1875		u32			trans;
1876
1877		length = urb->iso_frame_desc [i].length & 0x03ff;
1878		buf = dma + urb->iso_frame_desc [i].offset;
1879
1880		trans = SITD_STS_ACTIVE;
1881		if (((i + 1) == urb->number_of_packets)
1882				&& !(urb->transfer_flags & URB_NO_INTERRUPT))
1883			trans |= SITD_IOC;
1884		trans |= length << 16;
1885		packet->transaction = cpu_to_hc32(ehci, trans);
1886
1887		/* might need to cross a buffer page within a td */
1888		packet->bufp = buf;
1889		packet->buf1 = (buf + length) & ~0x0fff;
1890		if (packet->buf1 != (buf & ~(u64)0x0fff))
1891			packet->cross = 1;
1892
1893		/* OUT uses multiple start-splits */
1894		if (stream->bEndpointAddress & USB_DIR_IN)
1895			continue;
1896		length = (length + 187) / 188;
1897		if (length > 1) /* BEGIN vs ALL */
1898			length |= 1 << 3;
1899		packet->buf1 |= length;
1900	}
1901}
1902
1903static int
1904sitd_urb_transaction (
1905	struct ehci_iso_stream	*stream,
1906	struct ehci_hcd		*ehci,
1907	struct urb		*urb,
1908	gfp_t			mem_flags
1909)
1910{
1911	struct ehci_sitd	*sitd;
1912	dma_addr_t		sitd_dma;
1913	int			i;
1914	struct ehci_iso_sched	*iso_sched;
1915	unsigned long		flags;
1916
1917	iso_sched = iso_sched_alloc (urb->number_of_packets, mem_flags);
1918	if (iso_sched == NULL)
1919		return -ENOMEM;
1920
1921	sitd_sched_init(ehci, iso_sched, stream, urb);
1922
1923	/* allocate/init sITDs */
1924	spin_lock_irqsave (&ehci->lock, flags);
1925	for (i = 0; i < urb->number_of_packets; i++) {
1926
1927		/* NOTE:  for now, we don't try to handle wraparound cases
1928		 * for IN (using sitd->hw_backpointer, like a FSTN), which
1929		 * means we never need two sitds for full speed packets.
1930		 */
1931
1932		/* free_list.next might be cache-hot ... but maybe
1933		 * the HC caches it too. avoid that issue for now.
 
1934		 */
1935
1936		/* prefer previously-allocated sitds */
1937		if (!list_empty(&stream->free_list)) {
1938			sitd = list_entry (stream->free_list.prev,
1939					 struct ehci_sitd, sitd_list);
 
 
1940			list_del (&sitd->sitd_list);
1941			sitd_dma = sitd->sitd_dma;
1942		} else {
 
1943			spin_unlock_irqrestore (&ehci->lock, flags);
1944			sitd = dma_pool_alloc (ehci->sitd_pool, mem_flags,
1945					&sitd_dma);
1946			spin_lock_irqsave (&ehci->lock, flags);
1947			if (!sitd) {
1948				iso_sched_free(stream, iso_sched);
1949				spin_unlock_irqrestore(&ehci->lock, flags);
1950				return -ENOMEM;
1951			}
1952		}
1953
1954		memset (sitd, 0, sizeof *sitd);
1955		sitd->sitd_dma = sitd_dma;
 
1956		list_add (&sitd->sitd_list, &iso_sched->td_list);
1957	}
1958
1959	/* temporarily store schedule info in hcpriv */
1960	urb->hcpriv = iso_sched;
1961	urb->error_count = 0;
1962
1963	spin_unlock_irqrestore (&ehci->lock, flags);
1964	return 0;
1965}
1966
1967/*-------------------------------------------------------------------------*/
1968
1969static inline void
1970sitd_patch(
1971	struct ehci_hcd		*ehci,
1972	struct ehci_iso_stream	*stream,
1973	struct ehci_sitd	*sitd,
1974	struct ehci_iso_sched	*iso_sched,
1975	unsigned		index
1976)
1977{
1978	struct ehci_iso_packet	*uf = &iso_sched->packet [index];
1979	u64			bufp = uf->bufp;
1980
1981	sitd->hw_next = EHCI_LIST_END(ehci);
1982	sitd->hw_fullspeed_ep = stream->address;
1983	sitd->hw_uframe = stream->splits;
1984	sitd->hw_results = uf->transaction;
1985	sitd->hw_backpointer = EHCI_LIST_END(ehci);
1986
1987	bufp = uf->bufp;
1988	sitd->hw_buf[0] = cpu_to_hc32(ehci, bufp);
1989	sitd->hw_buf_hi[0] = cpu_to_hc32(ehci, bufp >> 32);
1990
1991	sitd->hw_buf[1] = cpu_to_hc32(ehci, uf->buf1);
1992	if (uf->cross)
1993		bufp += 4096;
1994	sitd->hw_buf_hi[1] = cpu_to_hc32(ehci, bufp >> 32);
1995	sitd->index = index;
1996}
1997
1998static inline void
1999sitd_link (struct ehci_hcd *ehci, unsigned frame, struct ehci_sitd *sitd)
2000{
2001	/* note: sitd ordering could matter (CSPLIT then SSPLIT) */
2002	sitd->sitd_next = ehci->pshadow [frame];
2003	sitd->hw_next = ehci->periodic [frame];
2004	ehci->pshadow [frame].sitd = sitd;
2005	sitd->frame = frame;
2006	wmb ();
2007	ehci->periodic[frame] = cpu_to_hc32(ehci, sitd->sitd_dma | Q_TYPE_SITD);
2008}
2009
2010/* fit urb's sitds into the selected schedule slot; activate as needed */
2011static int
2012sitd_link_urb (
2013	struct ehci_hcd		*ehci,
2014	struct urb		*urb,
2015	unsigned		mod,
2016	struct ehci_iso_stream	*stream
2017)
2018{
2019	int			packet;
2020	unsigned		next_uframe;
2021	struct ehci_iso_sched	*sched = urb->hcpriv;
2022	struct ehci_sitd	*sitd;
2023
2024	next_uframe = stream->next_uframe;
2025
2026	if (list_empty(&stream->td_list)) {
2027		/* usbfs ignores TT bandwidth */
2028		ehci_to_hcd(ehci)->self.bandwidth_allocated
2029				+= stream->bandwidth;
2030		ehci_vdbg (ehci,
2031			"sched devp %s ep%d%s-iso [%d] %dms/%04x\n",
2032			urb->dev->devpath, stream->bEndpointAddress & 0x0f,
2033			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out",
2034			(next_uframe >> 3) & (ehci->periodic_size - 1),
2035			stream->interval, hc32_to_cpu(ehci, stream->splits));
2036	}
2037
2038	if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2039		if (ehci->amd_pll_fix == 1)
2040			usb_amd_quirk_pll_disable();
2041	}
2042
2043	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs++;
2044
2045	/* fill sITDs frame by frame */
2046	for (packet = 0, sitd = NULL;
2047			packet < urb->number_of_packets;
2048			packet++) {
2049
2050		/* ASSERT:  we have all necessary sitds */
2051		BUG_ON (list_empty (&sched->td_list));
2052
2053		/* ASSERT:  no itds for this endpoint in this frame */
2054
2055		sitd = list_entry (sched->td_list.next,
2056				struct ehci_sitd, sitd_list);
2057		list_move_tail (&sitd->sitd_list, &stream->td_list);
2058		sitd->stream = iso_stream_get (stream);
2059		sitd->urb = urb;
2060
2061		sitd_patch(ehci, stream, sitd, sched, packet);
2062		sitd_link(ehci, (next_uframe >> 3) & (ehci->periodic_size - 1),
2063				sitd);
2064
2065		next_uframe += stream->interval << 3;
2066	}
2067	stream->next_uframe = next_uframe & (mod - 1);
2068
2069	/* don't need that schedule data any more */
2070	iso_sched_free (stream, sched);
2071	urb->hcpriv = NULL;
2072
2073	timer_action (ehci, TIMER_IO_WATCHDOG);
2074	return enable_periodic(ehci);
2075}
2076
2077/*-------------------------------------------------------------------------*/
2078
2079#define	SITD_ERRS (SITD_STS_ERR | SITD_STS_DBE | SITD_STS_BABBLE \
2080				| SITD_STS_XACT | SITD_STS_MMF)
2081
2082/* Process and recycle a completed SITD.  Return true iff its urb completed,
2083 * and hence its completion callback probably added things to the hardware
2084 * schedule.
2085 *
2086 * Note that we carefully avoid recycling this descriptor until after any
2087 * completion callback runs, so that it won't be reused quickly.  That is,
2088 * assuming (a) no more than two urbs per frame on this endpoint, and also
2089 * (b) only this endpoint's completions submit URBs.  It seems some silicon
2090 * corrupts things if you reuse completed descriptors very quickly...
2091 */
2092static unsigned
2093sitd_complete (
2094	struct ehci_hcd		*ehci,
2095	struct ehci_sitd	*sitd
2096) {
2097	struct urb				*urb = sitd->urb;
2098	struct usb_iso_packet_descriptor	*desc;
2099	u32					t;
2100	int					urb_index = -1;
2101	struct ehci_iso_stream			*stream = sitd->stream;
2102	struct usb_device			*dev;
2103	unsigned				retval = false;
2104
2105	urb_index = sitd->index;
2106	desc = &urb->iso_frame_desc [urb_index];
2107	t = hc32_to_cpup(ehci, &sitd->hw_results);
2108
2109	/* report transfer status */
2110	if (t & SITD_ERRS) {
2111		urb->error_count++;
2112		if (t & SITD_STS_DBE)
2113			desc->status = usb_pipein (urb->pipe)
2114				? -ENOSR  /* hc couldn't read */
2115				: -ECOMM; /* hc couldn't write */
2116		else if (t & SITD_STS_BABBLE)
2117			desc->status = -EOVERFLOW;
2118		else /* XACT, MMF, etc */
2119			desc->status = -EPROTO;
 
 
 
2120	} else {
2121		desc->status = 0;
2122		desc->actual_length = desc->length - SITD_LENGTH(t);
2123		urb->actual_length += desc->actual_length;
2124	}
2125
2126	/* handle completion now? */
2127	if ((urb_index + 1) != urb->number_of_packets)
2128		goto done;
2129
2130	/* ASSERT: it's really the last sitd for this urb
2131	list_for_each_entry (sitd, &stream->td_list, sitd_list)
2132		BUG_ON (sitd->urb == urb);
2133	 */
2134
2135	/* give urb back to the driver; completion often (re)submits */
2136	dev = urb->dev;
2137	ehci_urb_done(ehci, urb, 0);
2138	retval = true;
2139	urb = NULL;
2140	(void) disable_periodic(ehci);
2141	ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs--;
2142
 
 
 
 
2143	if (ehci_to_hcd(ehci)->self.bandwidth_isoc_reqs == 0) {
2144		if (ehci->amd_pll_fix == 1)
2145			usb_amd_quirk_pll_enable();
2146	}
2147
2148	if (list_is_singular(&stream->td_list)) {
2149		ehci_to_hcd(ehci)->self.bandwidth_allocated
2150				-= stream->bandwidth;
2151		ehci_vdbg (ehci,
2152			"deschedule devp %s ep%d%s-iso\n",
2153			dev->devpath, stream->bEndpointAddress & 0x0f,
2154			(stream->bEndpointAddress & USB_DIR_IN) ? "in" : "out");
2155	}
2156	iso_stream_put (ehci, stream);
2157
2158done:
2159	sitd->urb = NULL;
2160	if (ehci->clock_frame != sitd->frame) {
2161		/* OK to recycle this SITD now. */
2162		sitd->stream = NULL;
2163		list_move(&sitd->sitd_list, &stream->free_list);
2164		iso_stream_put(ehci, stream);
2165	} else {
2166		/* HW might remember this SITD, so we can't recycle it yet.
2167		 * Move it to a safe place until a new frame starts.
2168		 */
2169		list_move(&sitd->sitd_list, &ehci->cached_sitd_list);
2170		if (stream->refcount == 2) {
2171			/* If iso_stream_put() were called here, stream
2172			 * would be freed.  Instead, just prevent reuse.
2173			 */
2174			stream->ep->hcpriv = NULL;
2175			stream->ep = NULL;
2176		}
2177	}
 
2178	return retval;
2179}
2180
2181
2182static int sitd_submit (struct ehci_hcd *ehci, struct urb *urb,
2183	gfp_t mem_flags)
2184{
2185	int			status = -EINVAL;
2186	unsigned long		flags;
2187	struct ehci_iso_stream	*stream;
2188
2189	/* Get iso_stream head */
2190	stream = iso_stream_find (ehci, urb);
2191	if (stream == NULL) {
2192		ehci_dbg (ehci, "can't get iso stream\n");
2193		return -ENOMEM;
2194	}
2195	if (urb->interval != stream->interval) {
2196		ehci_dbg (ehci, "can't change iso interval %d --> %d\n",
2197			stream->interval, urb->interval);
2198		goto done;
2199	}
2200
2201#ifdef EHCI_URB_TRACE
2202	ehci_dbg (ehci,
2203		"submit %p dev%s ep%d%s-iso len %d\n",
2204		urb, urb->dev->devpath,
2205		usb_pipeendpoint (urb->pipe),
2206		usb_pipein (urb->pipe) ? "in" : "out",
2207		urb->transfer_buffer_length);
2208#endif
2209
2210	/* allocate SITDs */
2211	status = sitd_urb_transaction (stream, ehci, urb, mem_flags);
2212	if (status < 0) {
2213		ehci_dbg (ehci, "can't init sitds\n");
2214		goto done;
2215	}
2216
2217	/* schedule ... need to lock */
2218	spin_lock_irqsave (&ehci->lock, flags);
2219	if (unlikely(!HCD_HW_ACCESSIBLE(ehci_to_hcd(ehci)))) {
2220		status = -ESHUTDOWN;
2221		goto done_not_linked;
2222	}
2223	status = usb_hcd_link_urb_to_ep(ehci_to_hcd(ehci), urb);
2224	if (unlikely(status))
2225		goto done_not_linked;
2226	status = iso_stream_schedule(ehci, urb, stream);
2227	if (status == 0)
2228		sitd_link_urb (ehci, urb, ehci->periodic_size << 3, stream);
2229	else
 
 
 
2230		usb_hcd_unlink_urb_from_ep(ehci_to_hcd(ehci), urb);
2231done_not_linked:
 
2232	spin_unlock_irqrestore (&ehci->lock, flags);
2233
2234done:
2235	if (status < 0)
2236		iso_stream_put (ehci, stream);
2237	return status;
2238}
2239
2240/*-------------------------------------------------------------------------*/
2241
2242static void free_cached_lists(struct ehci_hcd *ehci)
2243{
2244	struct ehci_itd *itd, *n;
2245	struct ehci_sitd *sitd, *sn;
2246
2247	list_for_each_entry_safe(itd, n, &ehci->cached_itd_list, itd_list) {
2248		struct ehci_iso_stream	*stream = itd->stream;
2249		itd->stream = NULL;
2250		list_move(&itd->itd_list, &stream->free_list);
2251		iso_stream_put(ehci, stream);
2252	}
2253
2254	list_for_each_entry_safe(sitd, sn, &ehci->cached_sitd_list, sitd_list) {
2255		struct ehci_iso_stream	*stream = sitd->stream;
2256		sitd->stream = NULL;
2257		list_move(&sitd->sitd_list, &stream->free_list);
2258		iso_stream_put(ehci, stream);
2259	}
2260}
2261
2262/*-------------------------------------------------------------------------*/
2263
2264static void
2265scan_periodic (struct ehci_hcd *ehci)
2266{
2267	unsigned	now_uframe, frame, clock, clock_frame, mod;
2268	unsigned	modified;
2269
2270	mod = ehci->periodic_size << 3;
2271
2272	/*
2273	 * When running, scan from last scan point up to "now"
2274	 * else clean up by scanning everything that's left.
2275	 * Touches as few pages as possible:  cache-friendly.
2276	 */
2277	now_uframe = ehci->next_uframe;
2278	if (HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2279		clock = ehci_readl(ehci, &ehci->regs->frame_index);
2280		clock_frame = (clock >> 3) & (ehci->periodic_size - 1);
2281	} else  {
2282		clock = now_uframe + mod - 1;
2283		clock_frame = -1;
2284	}
2285	if (ehci->clock_frame != clock_frame) {
2286		free_cached_lists(ehci);
2287		ehci->clock_frame = clock_frame;
2288	}
2289	clock &= mod - 1;
2290	clock_frame = clock >> 3;
2291	++ehci->periodic_stamp;
2292
 
2293	for (;;) {
2294		union ehci_shadow	q, *q_p;
2295		__hc32			type, *hw_p;
2296		unsigned		incomplete = false;
2297
2298		frame = now_uframe >> 3;
2299
2300restart:
2301		/* scan each element in frame's queue for completions */
2302		q_p = &ehci->pshadow [frame];
2303		hw_p = &ehci->periodic [frame];
2304		q.ptr = q_p->ptr;
2305		type = Q_NEXT_TYPE(ehci, *hw_p);
2306		modified = 0;
2307
2308		while (q.ptr != NULL) {
2309			unsigned		uf;
2310			union ehci_shadow	temp;
2311			int			live;
2312
2313			live = HC_IS_RUNNING (ehci_to_hcd(ehci)->state);
2314			switch (hc32_to_cpu(ehci, type)) {
2315			case Q_TYPE_QH:
2316				/* handle any completions */
2317				temp.qh = qh_get (q.qh);
2318				type = Q_NEXT_TYPE(ehci, q.qh->hw->hw_next);
2319				q = q.qh->qh_next;
2320				if (temp.qh->stamp != ehci->periodic_stamp) {
2321					modified = qh_completions(ehci, temp.qh);
2322					if (!modified)
2323						temp.qh->stamp = ehci->periodic_stamp;
2324					if (unlikely(list_empty(&temp.qh->qtd_list) ||
2325							temp.qh->needs_rescan))
2326						intr_deschedule(ehci, temp.qh);
2327				}
2328				qh_put (temp.qh);
2329				break;
2330			case Q_TYPE_FSTN:
2331				/* for "save place" FSTNs, look at QH entries
2332				 * in the previous frame for completions.
2333				 */
2334				if (q.fstn->hw_prev != EHCI_LIST_END(ehci)) {
2335					dbg ("ignoring completions from FSTNs");
2336				}
2337				type = Q_NEXT_TYPE(ehci, q.fstn->hw_next);
2338				q = q.fstn->fstn_next;
2339				break;
2340			case Q_TYPE_ITD:
2341				/* If this ITD is still active, leave it for
2342				 * later processing ... check the next entry.
2343				 * No need to check for activity unless the
2344				 * frame is current.
2345				 */
2346				if (frame == clock_frame && live) {
2347					rmb();
2348					for (uf = 0; uf < 8; uf++) {
2349						if (q.itd->hw_transaction[uf] &
2350							    ITD_ACTIVE(ehci))
2351							break;
2352					}
2353					if (uf < 8) {
2354						incomplete = true;
2355						q_p = &q.itd->itd_next;
2356						hw_p = &q.itd->hw_next;
2357						type = Q_NEXT_TYPE(ehci,
2358							q.itd->hw_next);
2359						q = *q_p;
2360						break;
2361					}
2362				}
2363
2364				/* Take finished ITDs out of the schedule
2365				 * and process them:  recycle, maybe report
2366				 * URB completion.  HC won't cache the
2367				 * pointer for much longer, if at all.
2368				 */
2369				*q_p = q.itd->itd_next;
2370				if (!ehci->use_dummy_qh ||
2371				    q.itd->hw_next != EHCI_LIST_END(ehci))
2372					*hw_p = q.itd->hw_next;
2373				else
2374					*hw_p = ehci->dummy->qh_dma;
 
2375				type = Q_NEXT_TYPE(ehci, q.itd->hw_next);
2376				wmb();
2377				modified = itd_complete (ehci, q.itd);
2378				q = *q_p;
2379				break;
2380			case Q_TYPE_SITD:
2381				/* If this SITD is still active, leave it for
2382				 * later processing ... check the next entry.
2383				 * No need to check for activity unless the
2384				 * frame is current.
2385				 */
2386				if (((frame == clock_frame) ||
2387				     (((frame + 1) & (ehci->periodic_size - 1))
2388				      == clock_frame))
2389				    && live
2390				    && (q.sitd->hw_results &
2391					SITD_ACTIVE(ehci))) {
2392
2393					incomplete = true;
2394					q_p = &q.sitd->sitd_next;
2395					hw_p = &q.sitd->hw_next;
2396					type = Q_NEXT_TYPE(ehci,
2397							q.sitd->hw_next);
2398					q = *q_p;
2399					break;
2400				}
2401
2402				/* Take finished SITDs out of the schedule
2403				 * and process them:  recycle, maybe report
2404				 * URB completion.
2405				 */
2406				*q_p = q.sitd->sitd_next;
2407				if (!ehci->use_dummy_qh ||
2408				    q.sitd->hw_next != EHCI_LIST_END(ehci))
2409					*hw_p = q.sitd->hw_next;
2410				else
2411					*hw_p = ehci->dummy->qh_dma;
 
2412				type = Q_NEXT_TYPE(ehci, q.sitd->hw_next);
2413				wmb();
2414				modified = sitd_complete (ehci, q.sitd);
2415				q = *q_p;
2416				break;
2417			default:
2418				dbg ("corrupt type %d frame %d shadow %p",
2419					type, frame, q.ptr);
2420				// BUG ();
 
 
 
 
2421				q.ptr = NULL;
 
2422			}
2423
2424			/* assume completion callbacks modify the queue */
2425			if (unlikely (modified)) {
2426				if (likely(ehci->periodic_sched > 0))
2427					goto restart;
2428				/* short-circuit this scan */
2429				now_uframe = clock;
2430				break;
2431			}
2432		}
2433
2434		/* If we can tell we caught up to the hardware, stop now.
2435		 * We can't advance our scan without collecting the ISO
2436		 * transfers that are still pending in this frame.
2437		 */
2438		if (incomplete && HC_IS_RUNNING(ehci_to_hcd(ehci)->state)) {
2439			ehci->next_uframe = now_uframe;
2440			break;
2441		}
2442
2443		// FIXME:  this assumes we won't get lapped when
2444		// latencies climb; that should be rare, but...
2445		// detect it, and just go all the way around.
2446		// FLR might help detect this case, so long as latencies
2447		// don't exceed periodic_size msec (default 1.024 sec).
2448
2449		// FIXME:  likewise assumes HC doesn't halt mid-scan
2450
2451		if (now_uframe == clock) {
2452			unsigned	now;
2453
2454			if (!HC_IS_RUNNING (ehci_to_hcd(ehci)->state)
2455					|| ehci->periodic_sched == 0)
2456				break;
2457			ehci->next_uframe = now_uframe;
2458			now = ehci_readl(ehci, &ehci->regs->frame_index) &
2459					(mod - 1);
2460			if (now_uframe == now)
2461				break;
2462
2463			/* rescan the rest of this frame, then ... */
2464			clock = now;
2465			clock_frame = clock >> 3;
2466			if (ehci->clock_frame != clock_frame) {
2467				free_cached_lists(ehci);
2468				ehci->clock_frame = clock_frame;
2469				++ehci->periodic_stamp;
2470			}
2471		} else {
2472			now_uframe++;
2473			now_uframe &= mod - 1;
2474		}
2475	}
2476}