1/*
   2 * Copyright (C) 2005-2006 by Texas Instruments
   3 *
   4 * This file implements a DMA  interface using TI's CPPI DMA.
   5 * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
   6 * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci.
   7 */
   8
 
   9#include <linux/platform_device.h>
  10#include <linux/slab.h>
  11#include <linux/usb.h>
  12
  13#include "musb_core.h"
  14#include "musb_debug.h"
  15#include "cppi_dma.h"
  16
  17
  18/* CPPI DMA status 7-mar-2006:
  19 *
  20 * - See musb_{host,gadget}.c for more info
  21 *
  22 * - Correct RX DMA generally forces the engine into irq-per-packet mode,
  23 *   which can easily saturate the CPU under non-mass-storage loads.
  24 *
  25 * NOTES 24-aug-2006 (2.6.18-rc4):
  26 *
  27 * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
  28 *   evidently after the 1 byte packet was received and acked, the queue
  29 *   of BDs got garbaged so it wouldn't empty the fifo.  (rxcsr 0x2003,
  30 *   and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
  31 *   004001ff 00000001 .. 8feff860)  Host was just getting NAKed on tx
  32 *   of its next (512 byte) packet.  IRQ issues?
  33 *
  34 * REVISIT:  the "transfer DMA" glue between CPPI and USB fifos will
  35 * evidently also directly update the RX and TX CSRs ... so audit all
  36 * host and peripheral side DMA code to avoid CSR access after DMA has
  37 * been started.
  38 */
  39
  40/* REVISIT now we can avoid preallocating these descriptors; or
  41 * more simply, switch to a global freelist not per-channel ones.
  42 * Note: at full speed, 64 descriptors == 4K bulk data.
  43 */
  44#define NUM_TXCHAN_BD       64
  45#define NUM_RXCHAN_BD       64
  46
  47static inline void cpu_drain_writebuffer(void)
  48{
  49	wmb();
  50#ifdef	CONFIG_CPU_ARM926T
  51	/* REVISIT this "should not be needed",
  52	 * but lack of it sure seemed to hurt ...
  53	 */
  54	asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
  55#endif
  56}
  57
  58static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c)
  59{
  60	struct cppi_descriptor	*bd = c->freelist;
  61
  62	if (bd)
  63		c->freelist = bd->next;
  64	return bd;
  65}
  66
  67static inline void
  68cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd)
  69{
  70	if (!bd)
  71		return;
  72	bd->next = c->freelist;
  73	c->freelist = bd;
  74}
  75
  76/*
  77 *  Start DMA controller
  78 *
  79 *  Initialize the DMA controller as necessary.
  80 */
  81
  82/* zero out entire rx state RAM entry for the channel */
  83static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx)
  84{
  85	musb_writel(&rx->rx_skipbytes, 0, 0);
  86	musb_writel(&rx->rx_head, 0, 0);
  87	musb_writel(&rx->rx_sop, 0, 0);
  88	musb_writel(&rx->rx_current, 0, 0);
  89	musb_writel(&rx->rx_buf_current, 0, 0);
  90	musb_writel(&rx->rx_len_len, 0, 0);
  91	musb_writel(&rx->rx_cnt_cnt, 0, 0);
  92}
  93
  94/* zero out entire tx state RAM entry for the channel */
  95static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr)
  96{
  97	musb_writel(&tx->tx_head, 0, 0);
  98	musb_writel(&tx->tx_buf, 0, 0);
  99	musb_writel(&tx->tx_current, 0, 0);
 100	musb_writel(&tx->tx_buf_current, 0, 0);
 101	musb_writel(&tx->tx_info, 0, 0);
 102	musb_writel(&tx->tx_rem_len, 0, 0);
 103	/* musb_writel(&tx->tx_dummy, 0, 0); */
 104	musb_writel(&tx->tx_complete, 0, ptr);
 105}
 106
 107static void __init cppi_pool_init(struct cppi *cppi, struct cppi_channel *c)
 108{
 109	int	j;
 110
 111	/* initialize channel fields */
 112	c->head = NULL;
 113	c->tail = NULL;
 114	c->last_processed = NULL;
 115	c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
 116	c->controller = cppi;
 117	c->is_rndis = 0;
 118	c->freelist = NULL;
 119
 120	/* build the BD Free list for the channel */
 121	for (j = 0; j < NUM_TXCHAN_BD + 1; j++) {
 122		struct cppi_descriptor	*bd;
 123		dma_addr_t		dma;
 124
 125		bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma);
 126		bd->dma = dma;
 127		cppi_bd_free(c, bd);
 128	}
 129}
 130
 131static int cppi_channel_abort(struct dma_channel *);
 132
 133static void cppi_pool_free(struct cppi_channel *c)
 134{
 135	struct cppi		*cppi = c->controller;
 136	struct cppi_descriptor	*bd;
 137
 138	(void) cppi_channel_abort(&c->channel);
 139	c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
 140	c->controller = NULL;
 141
 142	/* free all its bds */
 143	bd = c->last_processed;
 144	do {
 145		if (bd)
 146			dma_pool_free(cppi->pool, bd, bd->dma);
 147		bd = cppi_bd_alloc(c);
 148	} while (bd);
 149	c->last_processed = NULL;
 150}
 151
 152static int __init cppi_controller_start(struct dma_controller *c)
 153{
 154	struct cppi	*controller;
 155	void __iomem	*tibase;
 156	int		i;
 157
 158	controller = container_of(c, struct cppi, controller);
 159
 160	/* do whatever is necessary to start controller */
 161	for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
 162		controller->tx[i].transmit = true;
 163		controller->tx[i].index = i;
 164	}
 165	for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
 166		controller->rx[i].transmit = false;
 167		controller->rx[i].index = i;
 168	}
 169
 170	/* setup BD list on a per channel basis */
 171	for (i = 0; i < ARRAY_SIZE(controller->tx); i++)
 172		cppi_pool_init(controller, controller->tx + i);
 173	for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
 174		cppi_pool_init(controller, controller->rx + i);
 175
 176	tibase =  controller->tibase;
 177	INIT_LIST_HEAD(&controller->tx_complete);
 178
 179	/* initialise tx/rx channel head pointers to zero */
 180	for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
 181		struct cppi_channel	*tx_ch = controller->tx + i;
 182		struct cppi_tx_stateram __iomem *tx;
 183
 184		INIT_LIST_HEAD(&tx_ch->tx_complete);
 185
 186		tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i);
 187		tx_ch->state_ram = tx;
 188		cppi_reset_tx(tx, 0);
 189	}
 190	for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
 191		struct cppi_channel	*rx_ch = controller->rx + i;
 192		struct cppi_rx_stateram __iomem *rx;
 193
 194		INIT_LIST_HEAD(&rx_ch->tx_complete);
 195
 196		rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i);
 197		rx_ch->state_ram = rx;
 198		cppi_reset_rx(rx);
 199	}
 200
 201	/* enable individual cppi channels */
 202	musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG,
 203			DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 204	musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG,
 205			DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 206
 207	/* enable tx/rx CPPI control */
 208	musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
 209	musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
 210
 211	/* disable RNDIS mode, also host rx RNDIS autorequest */
 212	musb_writel(tibase, DAVINCI_RNDIS_REG, 0);
 213	musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0);
 214
 215	return 0;
 216}
 217
 218/*
 219 *  Stop DMA controller
 220 *
 221 *  De-Init the DMA controller as necessary.
 222 */
 223
 224static int cppi_controller_stop(struct dma_controller *c)
 225{
 226	struct cppi		*controller;
 227	void __iomem		*tibase;
 228	int			i;
 229	struct musb		*musb;
 230
 231	controller = container_of(c, struct cppi, controller);
 232	musb = controller->musb;
 233
 234	tibase = controller->tibase;
 235	/* DISABLE INDIVIDUAL CHANNEL Interrupts */
 236	musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG,
 237			DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 238	musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG,
 239			DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 240
 241	dev_dbg(musb->controller, "Tearing down RX and TX Channels\n");
 242	for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
 243		/* FIXME restructure of txdma to use bds like rxdma */
 244		controller->tx[i].last_processed = NULL;
 245		cppi_pool_free(controller->tx + i);
 246	}
 247	for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
 248		cppi_pool_free(controller->rx + i);
 249
 250	/* in Tx Case proper teardown is supported. We resort to disabling
 251	 * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is
 252	 * complete TX CPPI cannot be disabled.
 253	 */
 254	/*disable tx/rx cppi */
 255	musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
 256	musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
 257
 258	return 0;
 259}
 260
 261/* While dma channel is allocated, we only want the core irqs active
 262 * for fault reports, otherwise we'd get irqs that we don't care about.
 263 * Except for TX irqs, where dma done != fifo empty and reusable ...
 264 *
 265 * NOTE: docs don't say either way, but irq masking **enables** irqs.
 266 *
 267 * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
 268 */
 269static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum)
 270{
 271	musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8));
 272}
 273
 274static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum)
 275{
 276	musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8));
 277}
 278
 279
 280/*
 281 * Allocate a CPPI Channel for DMA.  With CPPI, channels are bound to
 282 * each transfer direction of a non-control endpoint, so allocating
 283 * (and deallocating) is mostly a way to notice bad housekeeping on
 284 * the software side.  We assume the irqs are always active.
 285 */
 286static struct dma_channel *
 287cppi_channel_allocate(struct dma_controller *c,
 288		struct musb_hw_ep *ep, u8 transmit)
 289{
 290	struct cppi		*controller;
 291	u8			index;
 292	struct cppi_channel	*cppi_ch;
 293	void __iomem		*tibase;
 294	struct musb		*musb;
 295
 296	controller = container_of(c, struct cppi, controller);
 297	tibase = controller->tibase;
 298	musb = controller->musb;
 299
 300	/* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */
 301	index = ep->epnum - 1;
 302
 303	/* return the corresponding CPPI Channel Handle, and
 304	 * probably disable the non-CPPI irq until we need it.
 305	 */
 306	if (transmit) {
 307		if (index >= ARRAY_SIZE(controller->tx)) {
 308			dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'T', index);
 309			return NULL;
 310		}
 311		cppi_ch = controller->tx + index;
 312	} else {
 313		if (index >= ARRAY_SIZE(controller->rx)) {
 314			dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'R', index);
 315			return NULL;
 316		}
 317		cppi_ch = controller->rx + index;
 318		core_rxirq_disable(tibase, ep->epnum);
 319	}
 320
 321	/* REVISIT make this an error later once the same driver code works
 322	 * with the other DMA engine too
 323	 */
 324	if (cppi_ch->hw_ep)
 325		dev_dbg(musb->controller, "re-allocating DMA%d %cX channel %p\n",
 326				index, transmit ? 'T' : 'R', cppi_ch);
 327	cppi_ch->hw_ep = ep;
 328	cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
 329	cppi_ch->channel.max_len = 0x7fffffff;
 330
 331	dev_dbg(musb->controller, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R');
 332	return &cppi_ch->channel;
 333}
 334
 335/* Release a CPPI Channel.  */
 336static void cppi_channel_release(struct dma_channel *channel)
 337{
 338	struct cppi_channel	*c;
 339	void __iomem		*tibase;
 340
 341	/* REVISIT:  for paranoia, check state and abort if needed... */
 342
 343	c = container_of(channel, struct cppi_channel, channel);
 344	tibase = c->controller->tibase;
 345	if (!c->hw_ep)
 346		dev_dbg(c->controller->musb->controller,
 347			"releasing idle DMA channel %p\n", c);
 348	else if (!c->transmit)
 349		core_rxirq_enable(tibase, c->index + 1);
 350
 351	/* for now, leave its cppi IRQ enabled (we won't trigger it) */
 352	c->hw_ep = NULL;
 353	channel->status = MUSB_DMA_STATUS_UNKNOWN;
 354}
 355
 356/* Context: controller irqlocked */
 357static void
 358cppi_dump_rx(int level, struct cppi_channel *c, const char *tag)
 359{
 360	void __iomem			*base = c->controller->mregs;
 361	struct cppi_rx_stateram __iomem	*rx = c->state_ram;
 362
 363	musb_ep_select(base, c->index + 1);
 364
 365	dev_dbg(c->controller->musb->controller,
 366		"RX DMA%d%s: %d left, csr %04x, "
 367		"%08x H%08x S%08x C%08x, "
 368		"B%08x L%08x %08x .. %08x"
 369		"\n",
 370		c->index, tag,
 371		musb_readl(c->controller->tibase,
 372			DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index),
 373		musb_readw(c->hw_ep->regs, MUSB_RXCSR),
 374
 375		musb_readl(&rx->rx_skipbytes, 0),
 376		musb_readl(&rx->rx_head, 0),
 377		musb_readl(&rx->rx_sop, 0),
 378		musb_readl(&rx->rx_current, 0),
 379
 380		musb_readl(&rx->rx_buf_current, 0),
 381		musb_readl(&rx->rx_len_len, 0),
 382		musb_readl(&rx->rx_cnt_cnt, 0),
 383		musb_readl(&rx->rx_complete, 0)
 384		);
 385}
 386
 387/* Context: controller irqlocked */
 388static void
 389cppi_dump_tx(int level, struct cppi_channel *c, const char *tag)
 390{
 391	void __iomem			*base = c->controller->mregs;
 392	struct cppi_tx_stateram __iomem	*tx = c->state_ram;
 393
 394	musb_ep_select(base, c->index + 1);
 395
 396	dev_dbg(c->controller->musb->controller,
 397		"TX DMA%d%s: csr %04x, "
 398		"H%08x S%08x C%08x %08x, "
 399		"F%08x L%08x .. %08x"
 400		"\n",
 401		c->index, tag,
 402		musb_readw(c->hw_ep->regs, MUSB_TXCSR),
 403
 404		musb_readl(&tx->tx_head, 0),
 405		musb_readl(&tx->tx_buf, 0),
 406		musb_readl(&tx->tx_current, 0),
 407		musb_readl(&tx->tx_buf_current, 0),
 408
 409		musb_readl(&tx->tx_info, 0),
 410		musb_readl(&tx->tx_rem_len, 0),
 411		/* dummy/unused word 6 */
 412		musb_readl(&tx->tx_complete, 0)
 413		);
 414}
 415
 416/* Context: controller irqlocked */
 417static inline void
 418cppi_rndis_update(struct cppi_channel *c, int is_rx,
 419		void __iomem *tibase, int is_rndis)
 420{
 421	/* we may need to change the rndis flag for this cppi channel */
 422	if (c->is_rndis != is_rndis) {
 423		u32	value = musb_readl(tibase, DAVINCI_RNDIS_REG);
 424		u32	temp = 1 << (c->index);
 425
 426		if (is_rx)
 427			temp <<= 16;
 428		if (is_rndis)
 429			value |= temp;
 430		else
 431			value &= ~temp;
 432		musb_writel(tibase, DAVINCI_RNDIS_REG, value);
 433		c->is_rndis = is_rndis;
 434	}
 435}
 436
 437#ifdef CONFIG_USB_MUSB_DEBUG
 438static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd)
 439{
 440	pr_debug("RXBD/%s %08x: "
 441			"nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
 442			tag, bd->dma,
 443			bd->hw_next, bd->hw_bufp, bd->hw_off_len,
 444			bd->hw_options);
 445}
 446#endif
 447
 448static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
 449{
 450#ifdef CONFIG_USB_MUSB_DEBUG
 451	struct cppi_descriptor	*bd;
 452
 453	if (!_dbg_level(level))
 454		return;
 455	cppi_dump_rx(level, rx, tag);
 456	if (rx->last_processed)
 457		cppi_dump_rxbd("last", rx->last_processed);
 458	for (bd = rx->head; bd; bd = bd->next)
 459		cppi_dump_rxbd("active", bd);
 460#endif
 461}
 462
 463
 464/* NOTE:  DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
 465 * so we won't ever use it (see "CPPI RX Woes" below).
 466 */
 467static inline int cppi_autoreq_update(struct cppi_channel *rx,
 468		void __iomem *tibase, int onepacket, unsigned n_bds)
 469{
 470	u32	val;
 471
 472#ifdef	RNDIS_RX_IS_USABLE
 473	u32	tmp;
 474	/* assert(is_host_active(musb)) */
 475
 476	/* start from "AutoReq never" */
 477	tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
 478	val = tmp & ~((0x3) << (rx->index * 2));
 479
 480	/* HCD arranged reqpkt for packet #1.  we arrange int
 481	 * for all but the last one, maybe in two segments.
 482	 */
 483	if (!onepacket) {
 484#if 0
 485		/* use two segments, autoreq "all" then the last "never" */
 486		val |= ((0x3) << (rx->index * 2));
 487		n_bds--;
 488#else
 489		/* one segment, autoreq "all-but-last" */
 490		val |= ((0x1) << (rx->index * 2));
 491#endif
 492	}
 493
 494	if (val != tmp) {
 495		int n = 100;
 496
 497		/* make sure that autoreq is updated before continuing */
 498		musb_writel(tibase, DAVINCI_AUTOREQ_REG, val);
 499		do {
 500			tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
 501			if (tmp == val)
 502				break;
 503			cpu_relax();
 504		} while (n-- > 0);
 505	}
 506#endif
 507
 508	/* REQPKT is turned off after each segment */
 509	if (n_bds && rx->channel.actual_len) {
 510		void __iomem	*regs = rx->hw_ep->regs;
 511
 512		val = musb_readw(regs, MUSB_RXCSR);
 513		if (!(val & MUSB_RXCSR_H_REQPKT)) {
 514			val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
 515			musb_writew(regs, MUSB_RXCSR, val);
 516			/* flush writebufer */
 517			val = musb_readw(regs, MUSB_RXCSR);
 518		}
 519	}
 520	return n_bds;
 521}
 522
 523
 524/* Buffer enqueuing Logic:
 525 *
 526 *  - RX builds new queues each time, to help handle routine "early
 527 *    termination" cases (faults, including errors and short reads)
 528 *    more correctly.
 529 *
 530 *  - for now, TX reuses the same queue of BDs every time
 531 *
 532 * REVISIT long term, we want a normal dynamic model.
 533 * ... the goal will be to append to the
 534 * existing queue, processing completed "dma buffers" (segments) on the fly.
 535 *
 536 * Otherwise we force an IRQ latency between requests, which slows us a lot
 537 * (especially in "transparent" dma).  Unfortunately that model seems to be
 538 * inherent in the DMA model from the Mentor code, except in the rare case
 539 * of transfers big enough (~128+ KB) that we could append "middle" segments
 540 * in the TX paths.  (RX can't do this, see below.)
 541 *
 542 * That's true even in the CPPI- friendly iso case, where most urbs have
 543 * several small segments provided in a group and where the "packet at a time"
 544 * "transparent" DMA model is always correct, even on the RX side.
 545 */
 546
 547/*
 548 * CPPI TX:
 549 * ========
 550 * TX is a lot more reasonable than RX; it doesn't need to run in
 551 * irq-per-packet mode very often.  RNDIS mode seems to behave too
 552 * (except how it handles the exactly-N-packets case).  Building a
 553 * txdma queue with multiple requests (urb or usb_request) looks
 554 * like it would work ... but fault handling would need much testing.
 555 *
 556 * The main issue with TX mode RNDIS relates to transfer lengths that
 557 * are an exact multiple of the packet length.  It appears that there's
 558 * a hiccup in that case (maybe the DMA completes before the ZLP gets
 559 * written?) boiling down to not being able to rely on CPPI writing any
 560 * terminating zero length packet before the next transfer is written.
 561 * So that's punted to PIO; better yet, gadget drivers can avoid it.
 562 *
 563 * Plus, there's allegedly an undocumented constraint that rndis transfer
 564 * length be a multiple of 64 bytes ... but the chip doesn't act that
 565 * way, and we really don't _want_ that behavior anyway.
 566 *
 567 * On TX, "transparent" mode works ... although experiments have shown
 568 * problems trying to use the SOP/EOP bits in different USB packets.
 569 *
 570 * REVISIT try to handle terminating zero length packets using CPPI
 571 * instead of doing it by PIO after an IRQ.  (Meanwhile, make Ethernet
 572 * links avoid that issue by forcing them to avoid zlps.)
 573 */
 574static void
 575cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx)
 576{
 577	unsigned		maxpacket = tx->maxpacket;
 578	dma_addr_t		addr = tx->buf_dma + tx->offset;
 579	size_t			length = tx->buf_len - tx->offset;
 580	struct cppi_descriptor	*bd;
 581	unsigned		n_bds;
 582	unsigned		i;
 583	struct cppi_tx_stateram	__iomem *tx_ram = tx->state_ram;
 584	int			rndis;
 585
 586	/* TX can use the CPPI "rndis" mode, where we can probably fit this
 587	 * transfer in one BD and one IRQ.  The only time we would NOT want
 588	 * to use it is when hardware constraints prevent it, or if we'd
 589	 * trigger the "send a ZLP?" confusion.
 590	 */
 591	rndis = (maxpacket & 0x3f) == 0
 592		&& length > maxpacket
 593		&& length < 0xffff
 594		&& (length % maxpacket) != 0;
 595
 596	if (rndis) {
 597		maxpacket = length;
 598		n_bds = 1;
 599	} else {
 600		n_bds = length / maxpacket;
 601		if (!length || (length % maxpacket))
 602			n_bds++;
 603		n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD);
 604		length = min(n_bds * maxpacket, length);
 605	}
 606
 607	dev_dbg(musb->controller, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n",
 608			tx->index,
 609			maxpacket,
 610			rndis ? "rndis" : "transparent",
 611			n_bds,
 612			(unsigned long long)addr, length);
 613
 614	cppi_rndis_update(tx, 0, musb->ctrl_base, rndis);
 615
 616	/* assuming here that channel_program is called during
 617	 * transfer initiation ... current code maintains state
 618	 * for one outstanding request only (no queues, not even
 619	 * the implicit ones of an iso urb).
 620	 */
 621
 622	bd = tx->freelist;
 623	tx->head = bd;
 624	tx->last_processed = NULL;
 625
 626	/* FIXME use BD pool like RX side does, and just queue
 627	 * the minimum number for this request.
 628	 */
 629
 630	/* Prepare queue of BDs first, then hand it to hardware.
 631	 * All BDs except maybe the last should be of full packet
 632	 * size; for RNDIS there _is_ only that last packet.
 633	 */
 634	for (i = 0; i < n_bds; ) {
 635		if (++i < n_bds && bd->next)
 636			bd->hw_next = bd->next->dma;
 637		else
 638			bd->hw_next = 0;
 639
 640		bd->hw_bufp = tx->buf_dma + tx->offset;
 641
 642		/* FIXME set EOP only on the last packet,
 643		 * SOP only on the first ... avoid IRQs
 644		 */
 645		if ((tx->offset + maxpacket) <= tx->buf_len) {
 646			tx->offset += maxpacket;
 647			bd->hw_off_len = maxpacket;
 648			bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
 649				| CPPI_OWN_SET | maxpacket;
 650		} else {
 651			/* only this one may be a partial USB Packet */
 652			u32		partial_len;
 653
 654			partial_len = tx->buf_len - tx->offset;
 655			tx->offset = tx->buf_len;
 656			bd->hw_off_len = partial_len;
 657
 658			bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
 659				| CPPI_OWN_SET | partial_len;
 660			if (partial_len == 0)
 661				bd->hw_options |= CPPI_ZERO_SET;
 662		}
 663
 664		dev_dbg(musb->controller, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n",
 665				bd, bd->hw_next, bd->hw_bufp,
 666				bd->hw_off_len, bd->hw_options);
 667
 668		/* update the last BD enqueued to the list */
 669		tx->tail = bd;
 670		bd = bd->next;
 671	}
 672
 673	/* BDs live in DMA-coherent memory, but writes might be pending */
 674	cpu_drain_writebuffer();
 675
 676	/* Write to the HeadPtr in state RAM to trigger */
 677	musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma);
 678
 679	cppi_dump_tx(5, tx, "/S");
 680}
 681
 682/*
 683 * CPPI RX Woes:
 684 * =============
 685 * Consider a 1KB bulk RX buffer in two scenarios:  (a) it's fed two 300 byte
 686 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
 687 * (Full speed transfers have similar scenarios.)
 688 *
 689 * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
 690 * and the next packet goes into a buffer that's queued later; while (b) fills
 691 * the buffer with 1024 bytes.  How to do that with CPPI?
 692 *
 693 * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
 694 *   (b) loses **BADLY** because nothing (!) happens when that second packet
 695 *   fills the buffer, much less when a third one arrives.  (Which makes this
 696 *   not a "true" RNDIS mode.  In the RNDIS protocol short-packet termination
 697 *   is optional, and it's fine if peripherals -- not hosts! -- pad messages
 698 *   out to end-of-buffer.  Standard PCI host controller DMA descriptors
 699 *   implement that mode by default ... which is no accident.)
 700 *
 701 * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
 702 *   converse problems:  (b) is handled right, but (a) loses badly.  CPPI RX
 703 *   ignores SOP/EOP markings and processes both of those BDs; so both packets
 704 *   are loaded into the buffer (with a 212 byte gap between them), and the next
 705 *   buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP
 706 *   are intended as outputs for RX queues, not inputs...)
 707 *
 708 * - A variant of "transparent" mode -- one BD at a time -- is the only way to
 709 *   reliably make both cases work, with software handling both cases correctly
 710 *   and at the significant penalty of needing an IRQ per packet.  (The lack of
 711 *   I/O overlap can be slightly ameliorated by enabling double buffering.)
 712 *
 713 * So how to get rid of IRQ-per-packet?  The transparent multi-BD case could
 714 * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK
 715 * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors
 716 * with guaranteed driver level fault recovery and scrubbing out what's left
 717 * of that garbaged datastream.
 718 *
 719 * But there seems to be no way to identify the cases where CPPI RNDIS mode
 720 * is appropriate -- which do NOT include RNDIS host drivers, but do include
 721 * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
 722 * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
 723 * that applies best on the peripheral side (and which could fail rudely).
 724 *
 725 * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
 726 * cases other than mass storage class.  Otherwise we're correct but slow,
 727 * since CPPI penalizes our need for a "true RNDIS" default mode.
 728 */
 729
 730
 731/* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY
 732 *
 733 * IFF
 734 *  (a)	peripheral mode ... since rndis peripherals could pad their
 735 *	writes to hosts, causing i/o failure; or we'd have to cope with
 736 *	a largely unknowable variety of host side protocol variants
 737 *  (b)	and short reads are NOT errors ... since full reads would
 738 *	cause those same i/o failures
 739 *  (c)	and read length is
 740 *	- less than 64KB (max per cppi descriptor)
 741 *	- not a multiple of 4096 (g_zero default, full reads typical)
 742 *	- N (>1) packets long, ditto (full reads not EXPECTED)
 743 * THEN
 744 *   try rx rndis mode
 745 *
 746 * Cost of heuristic failing:  RXDMA wedges at the end of transfers that
 747 * fill out the whole buffer.  Buggy host side usb network drivers could
 748 * trigger that, but "in the field" such bugs seem to be all but unknown.
 749 *
 750 * So this module parameter lets the heuristic be disabled.  When using
 751 * gadgetfs, the heuristic will probably need to be disabled.
 752 */
 753static int cppi_rx_rndis = 1;
 754
 755module_param(cppi_rx_rndis, bool, 0);
 756MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
 757
 758
 759/**
 760 * cppi_next_rx_segment - dma read for the next chunk of a buffer
 761 * @musb: the controller
 762 * @rx: dma channel
 763 * @onepacket: true unless caller treats short reads as errors, and
 764 *	performs fault recovery above usbcore.
 765 * Context: controller irqlocked
 766 *
 767 * See above notes about why we can't use multi-BD RX queues except in
 768 * rare cases (mass storage class), and can never use the hardware "rndis"
 769 * mode (since it's not a "true" RNDIS mode) with complete safety..
 770 *
 771 * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
 772 * code to recover from corrupted datastreams after each short transfer.
 773 */
 774static void
 775cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
 776{
 777	unsigned		maxpacket = rx->maxpacket;
 778	dma_addr_t		addr = rx->buf_dma + rx->offset;
 779	size_t			length = rx->buf_len - rx->offset;
 780	struct cppi_descriptor	*bd, *tail;
 781	unsigned		n_bds;
 782	unsigned		i;
 783	void __iomem		*tibase = musb->ctrl_base;
 784	int			is_rndis = 0;
 785	struct cppi_rx_stateram	__iomem *rx_ram = rx->state_ram;
 786
 787	if (onepacket) {
 788		/* almost every USB driver, host or peripheral side */
 789		n_bds = 1;
 790
 791		/* maybe apply the heuristic above */
 792		if (cppi_rx_rndis
 793				&& is_peripheral_active(musb)
 794				&& length > maxpacket
 795				&& (length & ~0xffff) == 0
 796				&& (length & 0x0fff) != 0
 797				&& (length & (maxpacket - 1)) == 0) {
 798			maxpacket = length;
 799			is_rndis = 1;
 800		}
 801	} else {
 802		/* virtually nothing except mass storage class */
 803		if (length > 0xffff) {
 804			n_bds = 0xffff / maxpacket;
 805			length = n_bds * maxpacket;
 806		} else {
 807			n_bds = length / maxpacket;
 808			if (length % maxpacket)
 809				n_bds++;
 810		}
 811		if (n_bds == 1)
 812			onepacket = 1;
 813		else
 814			n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
 815	}
 816
 817	/* In host mode, autorequest logic can generate some IN tokens; it's
 818	 * tricky since we can't leave REQPKT set in RXCSR after the transfer
 819	 * finishes. So:  multipacket transfers involve two or more segments.
 820	 * And always at least two IRQs ... RNDIS mode is not an option.
 821	 */
 822	if (is_host_active(musb))
 823		n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
 824
 825	cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
 826
 827	length = min(n_bds * maxpacket, length);
 828
 829	dev_dbg(musb->controller, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
 830			"dma 0x%llx len %u %u/%u\n",
 831			rx->index, maxpacket,
 832			onepacket
 833				? (is_rndis ? "rndis" : "onepacket")
 834				: "multipacket",
 835			n_bds,
 836			musb_readl(tibase,
 837				DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
 838					& 0xffff,
 839			(unsigned long long)addr, length,
 840			rx->channel.actual_len, rx->buf_len);
 841
 842	/* only queue one segment at a time, since the hardware prevents
 843	 * correct queue shutdown after unexpected short packets
 844	 */
 845	bd = cppi_bd_alloc(rx);
 846	rx->head = bd;
 847
 848	/* Build BDs for all packets in this segment */
 849	for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
 850		u32	bd_len;
 851
 852		if (i) {
 853			bd = cppi_bd_alloc(rx);
 854			if (!bd)
 855				break;
 856			tail->next = bd;
 857			tail->hw_next = bd->dma;
 858		}
 859		bd->hw_next = 0;
 860
 861		/* all but the last packet will be maxpacket size */
 862		if (maxpacket < length)
 863			bd_len = maxpacket;
 864		else
 865			bd_len = length;
 866
 867		bd->hw_bufp = addr;
 868		addr += bd_len;
 869		rx->offset += bd_len;
 870
 871		bd->hw_off_len = (0 /*offset*/ << 16) + bd_len;
 872		bd->buflen = bd_len;
 873
 874		bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
 875		length -= bd_len;
 876	}
 877
 878	/* we always expect at least one reusable BD! */
 879	if (!tail) {
 880		WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
 881		return;
 882	} else if (i < n_bds)
 883		WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
 884
 885	tail->next = NULL;
 886	tail->hw_next = 0;
 887
 888	bd = rx->head;
 889	rx->tail = tail;
 890
 891	/* short reads and other faults should terminate this entire
 892	 * dma segment.  we want one "dma packet" per dma segment, not
 893	 * one per USB packet, terminating the whole queue at once...
 894	 * NOTE that current hardware seems to ignore SOP and EOP.
 895	 */
 896	bd->hw_options |= CPPI_SOP_SET;
 897	tail->hw_options |= CPPI_EOP_SET;
 898
 899#ifdef CONFIG_USB_MUSB_DEBUG
 900	if (_dbg_level(5)) {
 901		struct cppi_descriptor	*d;
 902
 903		for (d = rx->head; d; d = d->next)
 904			cppi_dump_rxbd("S", d);
 905	}
 906#endif
 907
 908	/* in case the preceding transfer left some state... */
 909	tail = rx->last_processed;
 910	if (tail) {
 911		tail->next = bd;
 912		tail->hw_next = bd->dma;
 913	}
 914
 915	core_rxirq_enable(tibase, rx->index + 1);
 916
 917	/* BDs live in DMA-coherent memory, but writes might be pending */
 918	cpu_drain_writebuffer();
 919
 920	/* REVISIT specs say to write this AFTER the BUFCNT register
 921	 * below ... but that loses badly.
 922	 */
 923	musb_writel(&rx_ram->rx_head, 0, bd->dma);
 924
 925	/* bufferCount must be at least 3, and zeroes on completion
 926	 * unless it underflows below zero, or stops at two, or keeps
 927	 * growing ... grr.
 928	 */
 929	i = musb_readl(tibase,
 930			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
 931			& 0xffff;
 932
 933	if (!i)
 934		musb_writel(tibase,
 935			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
 936			n_bds + 2);
 937	else if (n_bds > (i - 3))
 938		musb_writel(tibase,
 939			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
 940			n_bds - (i - 3));
 941
 942	i = musb_readl(tibase,
 943			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
 944			& 0xffff;
 945	if (i < (2 + n_bds)) {
 946		dev_dbg(musb->controller, "bufcnt%d underrun - %d (for %d)\n",
 947					rx->index, i, n_bds);
 948		musb_writel(tibase,
 949			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
 950			n_bds + 2);
 951	}
 952
 953	cppi_dump_rx(4, rx, "/S");
 954}
 955
 956/**
 957 * cppi_channel_program - program channel for data transfer
 958 * @ch: the channel
 959 * @maxpacket: max packet size
 960 * @mode: For RX, 1 unless the usb protocol driver promised to treat
 961 *	all short reads as errors and kick in high level fault recovery.
 962 *	For TX, ignored because of RNDIS mode races/glitches.
 963 * @dma_addr: dma address of buffer
 964 * @len: length of buffer
 965 * Context: controller irqlocked
 966 */
 967static int cppi_channel_program(struct dma_channel *ch,
 968		u16 maxpacket, u8 mode,
 969		dma_addr_t dma_addr, u32 len)
 970{
 971	struct cppi_channel	*cppi_ch;
 972	struct cppi		*controller;
 973	struct musb		*musb;
 974
 975	cppi_ch = container_of(ch, struct cppi_channel, channel);
 976	controller = cppi_ch->controller;
 977	musb = controller->musb;
 978
 979	switch (ch->status) {
 980	case MUSB_DMA_STATUS_BUS_ABORT:
 981	case MUSB_DMA_STATUS_CORE_ABORT:
 982		/* fault irq handler should have handled cleanup */
 983		WARNING("%cX DMA%d not cleaned up after abort!\n",
 984				cppi_ch->transmit ? 'T' : 'R',
 985				cppi_ch->index);
 986		/* WARN_ON(1); */
 987		break;
 988	case MUSB_DMA_STATUS_BUSY:
 989		WARNING("program active channel?  %cX DMA%d\n",
 990				cppi_ch->transmit ? 'T' : 'R',
 991				cppi_ch->index);
 992		/* WARN_ON(1); */
 993		break;
 994	case MUSB_DMA_STATUS_UNKNOWN:
 995		dev_dbg(musb->controller, "%cX DMA%d not allocated!\n",
 996				cppi_ch->transmit ? 'T' : 'R',
 997				cppi_ch->index);
 998		/* FALLTHROUGH */
 999	case MUSB_DMA_STATUS_FREE:
1000		break;
1001	}
1002
1003	ch->status = MUSB_DMA_STATUS_BUSY;
1004
1005	/* set transfer parameters, then queue up its first segment */
1006	cppi_ch->buf_dma = dma_addr;
1007	cppi_ch->offset = 0;
1008	cppi_ch->maxpacket = maxpacket;
1009	cppi_ch->buf_len = len;
1010	cppi_ch->channel.actual_len = 0;
1011
1012	/* TX channel? or RX? */
1013	if (cppi_ch->transmit)
1014		cppi_next_tx_segment(musb, cppi_ch);
1015	else
1016		cppi_next_rx_segment(musb, cppi_ch, mode);
1017
1018	return true;
1019}
1020
1021static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
1022{
1023	struct cppi_channel		*rx = &cppi->rx[ch];
1024	struct cppi_rx_stateram __iomem	*state = rx->state_ram;
1025	struct cppi_descriptor		*bd;
1026	struct cppi_descriptor		*last = rx->last_processed;
1027	bool				completed = false;
1028	bool				acked = false;
1029	int				i;
1030	dma_addr_t			safe2ack;
1031	void __iomem			*regs = rx->hw_ep->regs;
1032	struct musb			*musb = cppi->musb;
1033
1034	cppi_dump_rx(6, rx, "/K");
1035
1036	bd = last ? last->next : rx->head;
1037	if (!bd)
1038		return false;
1039
1040	/* run through all completed BDs */
1041	for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
1042			(safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
1043			i++, bd = bd->next) {
1044		u16	len;
1045
1046		/* catch latest BD writes from CPPI */
1047		rmb();
1048		if (!completed && (bd->hw_options & CPPI_OWN_SET))
1049			break;
1050
1051		dev_dbg(musb->controller, "C/RXBD %llx: nxt %08x buf %08x "
1052			"off.len %08x opt.len %08x (%d)\n",
1053			(unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
1054			bd->hw_off_len, bd->hw_options,
1055			rx->channel.actual_len);
1056
1057		/* actual packet received length */
1058		if ((bd->hw_options & CPPI_SOP_SET) && !completed)
1059			len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
1060		else
1061			len = 0;
1062
1063		if (bd->hw_options & CPPI_EOQ_MASK)
1064			completed = true;
1065
1066		if (!completed && len < bd->buflen) {
1067			/* NOTE:  when we get a short packet, RXCSR_H_REQPKT
1068			 * must have been cleared, and no more DMA packets may
1069			 * active be in the queue... TI docs didn't say, but
1070			 * CPPI ignores those BDs even though OWN is still set.
1071			 */
1072			completed = true;
1073			dev_dbg(musb->controller, "rx short %d/%d (%d)\n",
1074					len, bd->buflen,
1075					rx->channel.actual_len);
1076		}
1077
1078		/* If we got here, we expect to ack at least one BD; meanwhile
1079		 * CPPI may completing other BDs while we scan this list...
1080		 *
1081		 * RACE: we can notice OWN cleared before CPPI raises the
1082		 * matching irq by writing that BD as the completion pointer.
1083		 * In such cases, stop scanning and wait for the irq, avoiding
1084		 * lost acks and states where BD ownership is unclear.
1085		 */
1086		if (bd->dma == safe2ack) {
1087			musb_writel(&state->rx_complete, 0, safe2ack);
1088			safe2ack = musb_readl(&state->rx_complete, 0);
1089			acked = true;
1090			if (bd->dma == safe2ack)
1091				safe2ack = 0;
1092		}
1093
1094		rx->channel.actual_len += len;
1095
1096		cppi_bd_free(rx, last);
1097		last = bd;
1098
1099		/* stop scanning on end-of-segment */
1100		if (bd->hw_next == 0)
1101			completed = true;
1102	}
1103	rx->last_processed = last;
1104
1105	/* dma abort, lost ack, or ... */
1106	if (!acked && last) {
1107		int	csr;
1108
1109		if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
1110			musb_writel(&state->rx_complete, 0, safe2ack);
1111		if (safe2ack == 0) {
1112			cppi_bd_free(rx, last);
1113			rx->last_processed = NULL;
1114
1115			/* if we land here on the host side, H_REQPKT will
1116			 * be clear and we need to restart the queue...
1117			 */
1118			WARN_ON(rx->head);
1119		}
1120		musb_ep_select(cppi->mregs, rx->index + 1);
1121		csr = musb_readw(regs, MUSB_RXCSR);
1122		if (csr & MUSB_RXCSR_DMAENAB) {
1123			dev_dbg(musb->controller, "list%d %p/%p, last %llx%s, csr %04x\n",
1124				rx->index,
1125				rx->head, rx->tail,
1126				rx->last_processed
1127					? (unsigned long long)
1128						rx->last_processed->dma
1129					: 0,
1130				completed ? ", completed" : "",
1131				csr);
1132			cppi_dump_rxq(4, "/what?", rx);
1133		}
1134	}
1135	if (!completed) {
1136		int	csr;
1137
1138		rx->head = bd;
1139
1140		/* REVISIT seems like "autoreq all but EOP" doesn't...
1141		 * setting it here "should" be racey, but seems to work
1142		 */
1143		csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1144		if (is_host_active(cppi->musb)
1145				&& bd
1146				&& !(csr & MUSB_RXCSR_H_REQPKT)) {
1147			csr |= MUSB_RXCSR_H_REQPKT;
1148			musb_writew(regs, MUSB_RXCSR,
1149					MUSB_RXCSR_H_WZC_BITS | csr);
1150			csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1151		}
1152	} else {
1153		rx->head = NULL;
1154		rx->tail = NULL;
1155	}
1156
1157	cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
1158	return completed;
1159}
1160
1161irqreturn_t cppi_interrupt(int irq, void *dev_id)
1162{
1163	struct musb		*musb = dev_id;
1164	struct cppi		*cppi;
1165	void __iomem		*tibase;
1166	struct musb_hw_ep	*hw_ep = NULL;
1167	u32			rx, tx;
1168	int			i, index;
1169	unsigned long		uninitialized_var(flags);
1170
1171	cppi = container_of(musb->dma_controller, struct cppi, controller);
1172	if (cppi->irq)
1173		spin_lock_irqsave(&musb->lock, flags);
1174
1175	tibase = musb->ctrl_base;
1176
1177	tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG);
1178	rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG);
1179
1180	if (!tx && !rx) {
1181		if (cppi->irq)
1182			spin_unlock_irqrestore(&musb->lock, flags);
1183		return IRQ_NONE;
1184	}
1185
1186	dev_dbg(musb->controller, "CPPI IRQ Tx%x Rx%x\n", tx, rx);
1187
1188	/* process TX channels */
1189	for (index = 0; tx; tx = tx >> 1, index++) {
1190		struct cppi_channel		*tx_ch;
1191		struct cppi_tx_stateram __iomem	*tx_ram;
1192		bool				completed = false;
1193		struct cppi_descriptor		*bd;
1194
1195		if (!(tx & 1))
1196			continue;
1197
1198		tx_ch = cppi->tx + index;
1199		tx_ram = tx_ch->state_ram;
1200
1201		/* FIXME  need a cppi_tx_scan() routine, which
1202		 * can also be called from abort code
1203		 */
1204
1205		cppi_dump_tx(5, tx_ch, "/E");
1206
1207		bd = tx_ch->head;
1208
1209		/*
1210		 * If Head is null then this could mean that a abort interrupt
1211		 * that needs to be acknowledged.
1212		 */
1213		if (NULL == bd) {
1214			dev_dbg(musb->controller, "null BD\n");
1215			musb_writel(&tx_ram->tx_complete, 0, 0);
1216			continue;
1217		}
1218
1219		/* run through all completed BDs */
1220		for (i = 0; !completed && bd && i < NUM_TXCHAN_BD;
1221				i++, bd = bd->next) {
1222			u16	len;
1223
1224			/* catch latest BD writes from CPPI */
1225			rmb();
1226			if (bd->hw_options & CPPI_OWN_SET)
1227				break;
1228
1229			dev_dbg(musb->controller, "C/TXBD %p n %x b %x off %x opt %x\n",
1230					bd, bd->hw_next, bd->hw_bufp,
1231					bd->hw_off_len, bd->hw_options);
1232
1233			len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK;
1234			tx_ch->channel.actual_len += len;
1235
1236			tx_ch->last_processed = bd;
1237
1238			/* write completion register to acknowledge
1239			 * processing of completed BDs, and possibly
1240			 * release the IRQ; EOQ might not be set ...
1241			 *
1242			 * REVISIT use the same ack strategy as rx
1243			 *
1244			 * REVISIT have observed bit 18 set; huh??
1245			 */
1246			/* if ((bd->hw_options & CPPI_EOQ_MASK)) */
1247				musb_writel(&tx_ram->tx_complete, 0, bd->dma);
1248
1249			/* stop scanning on end-of-segment */
1250			if (bd->hw_next == 0)
1251				completed = true;
1252		}
1253
1254		/* on end of segment, maybe go to next one */
1255		if (completed) {
1256			/* cppi_dump_tx(4, tx_ch, "/complete"); */
1257
1258			/* transfer more, or report completion */
1259			if (tx_ch->offset >= tx_ch->buf_len) {
1260				tx_ch->head = NULL;
1261				tx_ch->tail = NULL;
1262				tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1263
1264				hw_ep = tx_ch->hw_ep;
1265
1266				musb_dma_completion(musb, index + 1, 1);
1267
1268			} else {
1269				/* Bigger transfer than we could fit in
1270				 * that first batch of descriptors...
1271				 */
1272				cppi_next_tx_segment(musb, tx_ch);
1273			}
1274		} else
1275			tx_ch->head = bd;
1276	}
1277
1278	/* Start processing the RX block */
1279	for (index = 0; rx; rx = rx >> 1, index++) {
1280
1281		if (rx & 1) {
1282			struct cppi_channel		*rx_ch;
1283
1284			rx_ch = cppi->rx + index;
1285
1286			/* let incomplete dma segments finish */
1287			if (!cppi_rx_scan(cppi, index))
1288				continue;
1289
1290			/* start another dma segment if needed */
1291			if (rx_ch->channel.actual_len != rx_ch->buf_len
1292					&& rx_ch->channel.actual_len
1293						== rx_ch->offset) {
1294				cppi_next_rx_segment(musb, rx_ch, 1);
1295				continue;
1296			}
1297
1298			/* all segments completed! */
1299			rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1300
1301			hw_ep = rx_ch->hw_ep;
1302
1303			core_rxirq_disable(tibase, index + 1);
1304			musb_dma_completion(musb, index + 1, 0);
1305		}
1306	}
1307
1308	/* write to CPPI EOI register to re-enable interrupts */
1309	musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0);
1310
1311	if (cppi->irq)
1312		spin_unlock_irqrestore(&musb->lock, flags);
1313
1314	return IRQ_HANDLED;
1315}
1316
1317/* Instantiate a software object representing a DMA controller. */
1318struct dma_controller *__init
1319dma_controller_create(struct musb *musb, void __iomem *mregs)
1320{
1321	struct cppi		*controller;
1322	struct device		*dev = musb->controller;
1323	struct platform_device	*pdev = to_platform_device(dev);
1324	int			irq = platform_get_irq_byname(pdev, "dma");
1325
1326	controller = kzalloc(sizeof *controller, GFP_KERNEL);
1327	if (!controller)
1328		return NULL;
1329
1330	controller->mregs = mregs;
1331	controller->tibase = mregs - DAVINCI_BASE_OFFSET;
1332
1333	controller->musb = musb;
1334	controller->controller.start = cppi_controller_start;
1335	controller->controller.stop = cppi_controller_stop;
1336	controller->controller.channel_alloc = cppi_channel_allocate;
1337	controller->controller.channel_release = cppi_channel_release;
1338	controller->controller.channel_program = cppi_channel_program;
1339	controller->controller.channel_abort = cppi_channel_abort;
1340
1341	/* NOTE: allocating from on-chip SRAM would give the least
1342	 * contention for memory access, if that ever matters here.
1343	 */
1344
1345	/* setup BufferPool */
1346	controller->pool = dma_pool_create("cppi",
1347			controller->musb->controller,
1348			sizeof(struct cppi_descriptor),
1349			CPPI_DESCRIPTOR_ALIGN, 0);
1350	if (!controller->pool) {
1351		kfree(controller);
1352		return NULL;
1353	}
1354
1355	if (irq > 0) {
1356		if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) {
1357			dev_err(dev, "request_irq %d failed!\n", irq);
1358			dma_controller_destroy(&controller->controller);
1359			return NULL;
1360		}
1361		controller->irq = irq;
1362	}
1363
1364	return &controller->controller;
1365}
1366
1367/*
1368 *  Destroy a previously-instantiated DMA controller.
1369 */
1370void dma_controller_destroy(struct dma_controller *c)
1371{
1372	struct cppi	*cppi;
1373
1374	cppi = container_of(c, struct cppi, controller);
1375
1376	if (cppi->irq)
1377		free_irq(cppi->irq, cppi->musb);
1378
1379	/* assert:  caller stopped the controller first */
1380	dma_pool_destroy(cppi->pool);
1381
1382	kfree(cppi);
1383}
1384
1385/*
1386 * Context: controller irqlocked, endpoint selected
1387 */
1388static int cppi_channel_abort(struct dma_channel *channel)
1389{
1390	struct cppi_channel	*cppi_ch;
1391	struct cppi		*controller;
1392	void __iomem		*mbase;
1393	void __iomem		*tibase;
1394	void __iomem		*regs;
1395	u32			value;
1396	struct cppi_descriptor	*queue;
1397
1398	cppi_ch = container_of(channel, struct cppi_channel, channel);
1399
1400	controller = cppi_ch->controller;
1401
1402	switch (channel->status) {
1403	case MUSB_DMA_STATUS_BUS_ABORT:
1404	case MUSB_DMA_STATUS_CORE_ABORT:
1405		/* from RX or TX fault irq handler */
1406	case MUSB_DMA_STATUS_BUSY:
1407		/* the hardware needs shutting down */
1408		regs = cppi_ch->hw_ep->regs;
1409		break;
1410	case MUSB_DMA_STATUS_UNKNOWN:
1411	case MUSB_DMA_STATUS_FREE:
1412		return 0;
1413	default:
1414		return -EINVAL;
1415	}
1416
1417	if (!cppi_ch->transmit && cppi_ch->head)
1418		cppi_dump_rxq(3, "/abort", cppi_ch);
1419
1420	mbase = controller->mregs;
1421	tibase = controller->tibase;
1422
1423	queue = cppi_ch->head;
1424	cppi_ch->head = NULL;
1425	cppi_ch->tail = NULL;
1426
1427	/* REVISIT should rely on caller having done this,
1428	 * and caller should rely on us not changing it.
1429	 * peripheral code is safe ... check host too.
1430	 */
1431	musb_ep_select(mbase, cppi_ch->index + 1);
1432
1433	if (cppi_ch->transmit) {
1434		struct cppi_tx_stateram __iomem *tx_ram;
1435		/* REVISIT put timeouts on these controller handshakes */
1436
1437		cppi_dump_tx(6, cppi_ch, " (teardown)");
1438
1439		/* teardown DMA engine then usb core */
1440		do {
1441			value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
1442		} while (!(value & CPPI_TEAR_READY));
1443		musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
1444
1445		tx_ram = cppi_ch->state_ram;
1446		do {
1447			value = musb_readl(&tx_ram->tx_complete, 0);
1448		} while (0xFFFFFFFC != value);
1449
1450		/* FIXME clean up the transfer state ... here?
1451		 * the completion routine should get called with
1452		 * an appropriate status code.
1453		 */
1454
1455		value = musb_readw(regs, MUSB_TXCSR);
1456		value &= ~MUSB_TXCSR_DMAENAB;
1457		value |= MUSB_TXCSR_FLUSHFIFO;
1458		musb_writew(regs, MUSB_TXCSR, value);
1459		musb_writew(regs, MUSB_TXCSR, value);
1460
1461		/*
1462		 * 1. Write to completion Ptr value 0x1(bit 0 set)
1463		 *    (write back mode)
1464		 * 2. Wait for abort interrupt and then put the channel in
1465		 *    compare mode by writing 1 to the tx_complete register.
1466		 */
1467		cppi_reset_tx(tx_ram, 1);
1468		cppi_ch->head = NULL;
1469		musb_writel(&tx_ram->tx_complete, 0, 1);
1470		cppi_dump_tx(5, cppi_ch, " (done teardown)");
1471
1472		/* REVISIT tx side _should_ clean up the same way
1473		 * as the RX side ... this does no cleanup at all!
1474		 */
1475
1476	} else /* RX */ {
1477		u16			csr;
1478
1479		/* NOTE: docs don't guarantee any of this works ...  we
1480		 * expect that if the usb core stops telling the cppi core
1481		 * to pull more data from it, then it'll be safe to flush
1482		 * current RX DMA state iff any pending fifo transfer is done.
1483		 */
1484
1485		core_rxirq_disable(tibase, cppi_ch->index + 1);
1486
1487		/* for host, ensure ReqPkt is never set again */
1488		if (is_host_active(cppi_ch->controller->musb)) {
1489			value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
1490			value &= ~((0x3) << (cppi_ch->index * 2));
1491			musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
1492		}
1493
1494		csr = musb_readw(regs, MUSB_RXCSR);
1495
1496		/* for host, clear (just) ReqPkt at end of current packet(s) */
1497		if (is_host_active(cppi_ch->controller->musb)) {
1498			csr |= MUSB_RXCSR_H_WZC_BITS;
1499			csr &= ~MUSB_RXCSR_H_REQPKT;
1500		} else
1501			csr |= MUSB_RXCSR_P_WZC_BITS;
1502
1503		/* clear dma enable */
1504		csr &= ~(MUSB_RXCSR_DMAENAB);
1505		musb_writew(regs, MUSB_RXCSR, csr);
1506		csr = musb_readw(regs, MUSB_RXCSR);
1507
1508		/* Quiesce: wait for current dma to finish (if not cleanup).
1509		 * We can't use bit zero of stateram->rx_sop, since that
1510		 * refers to an entire "DMA packet" not just emptying the
1511		 * current fifo.  Most segments need multiple usb packets.
1512		 */
1513		if (channel->status == MUSB_DMA_STATUS_BUSY)
1514			udelay(50);
1515
1516		/* scan the current list, reporting any data that was
1517		 * transferred and acking any IRQ
1518		 */
1519		cppi_rx_scan(controller, cppi_ch->index);
1520
1521		/* clobber the existing state once it's idle
1522		 *
1523		 * NOTE:  arguably, we should also wait for all the other
1524		 * RX channels to quiesce (how??) and then temporarily
1525		 * disable RXCPPI_CTRL_REG ... but it seems that we can
1526		 * rely on the controller restarting from state ram, with
1527		 * only RXCPPI_BUFCNT state being bogus.  BUFCNT will
1528		 * correct itself after the next DMA transfer though.
1529		 *
1530		 * REVISIT does using rndis mode change that?
1531		 */
1532		cppi_reset_rx(cppi_ch->state_ram);
1533
1534		/* next DMA request _should_ load cppi head ptr */
1535
1536		/* ... we don't "free" that list, only mutate it in place.  */
1537		cppi_dump_rx(5, cppi_ch, " (done abort)");
1538
1539		/* clean up previously pending bds */
1540		cppi_bd_free(cppi_ch, cppi_ch->last_processed);
1541		cppi_ch->last_processed = NULL;
1542
1543		while (queue) {
1544			struct cppi_descriptor	*tmp = queue->next;
1545
1546			cppi_bd_free(cppi_ch, queue);
1547			queue = tmp;
1548		}
1549	}
1550
1551	channel->status = MUSB_DMA_STATUS_FREE;
1552	cppi_ch->buf_dma = 0;
1553	cppi_ch->offset = 0;
1554	cppi_ch->buf_len = 0;
1555	cppi_ch->maxpacket = 0;
1556	return 0;
1557}
1558
1559/* TBD Queries:
1560 *
1561 * Power Management ... probably turn off cppi during suspend, restart;
1562 * check state ram?  Clocking is presumably shared with usb core.
1563 */