1/*
   2 * Copyright (C) 2005-2006 by Texas Instruments
   3 *
   4 * This file implements a DMA  interface using TI's CPPI DMA.
   5 * For now it's DaVinci-only, but CPPI isn't specific to DaVinci or USB.
   6 * The TUSB6020, using VLYNQ, has CPPI that looks much like DaVinci.
   7 */
   8
   9#include <linux/module.h>
  10#include <linux/platform_device.h>
  11#include <linux/slab.h>
  12#include <linux/usb.h>
  13
  14#include "musb_core.h"
  15#include "musb_debug.h"
  16#include "cppi_dma.h"
  17
  18
  19/* CPPI DMA status 7-mar-2006:
  20 *
  21 * - See musb_{host,gadget}.c for more info
  22 *
  23 * - Correct RX DMA generally forces the engine into irq-per-packet mode,
  24 *   which can easily saturate the CPU under non-mass-storage loads.
  25 *
  26 * NOTES 24-aug-2006 (2.6.18-rc4):
  27 *
  28 * - peripheral RXDMA wedged in a test with packets of length 512/512/1.
  29 *   evidently after the 1 byte packet was received and acked, the queue
  30 *   of BDs got garbaged so it wouldn't empty the fifo.  (rxcsr 0x2003,
  31 *   and RX DMA0: 4 left, 80000000 8feff880, 8feff860 8feff860; 8f321401
  32 *   004001ff 00000001 .. 8feff860)  Host was just getting NAKed on tx
  33 *   of its next (512 byte) packet.  IRQ issues?
  34 *
  35 * REVISIT:  the "transfer DMA" glue between CPPI and USB fifos will
  36 * evidently also directly update the RX and TX CSRs ... so audit all
  37 * host and peripheral side DMA code to avoid CSR access after DMA has
  38 * been started.
  39 */
  40
  41/* REVISIT now we can avoid preallocating these descriptors; or
  42 * more simply, switch to a global freelist not per-channel ones.
  43 * Note: at full speed, 64 descriptors == 4K bulk data.
  44 */
  45#define NUM_TXCHAN_BD       64
  46#define NUM_RXCHAN_BD       64
  47
  48static inline void cpu_drain_writebuffer(void)
  49{
  50	wmb();
  51#ifdef	CONFIG_CPU_ARM926T
  52	/* REVISIT this "should not be needed",
  53	 * but lack of it sure seemed to hurt ...
  54	 */
  55	asm("mcr p15, 0, r0, c7, c10, 4 @ drain write buffer\n");
  56#endif
  57}
  58
  59static inline struct cppi_descriptor *cppi_bd_alloc(struct cppi_channel *c)
  60{
  61	struct cppi_descriptor	*bd = c->freelist;
  62
  63	if (bd)
  64		c->freelist = bd->next;
  65	return bd;
  66}
  67
  68static inline void
  69cppi_bd_free(struct cppi_channel *c, struct cppi_descriptor *bd)
  70{
  71	if (!bd)
  72		return;
  73	bd->next = c->freelist;
  74	c->freelist = bd;
  75}
  76
  77/*
  78 *  Start DMA controller
  79 *
  80 *  Initialize the DMA controller as necessary.
  81 */
  82
  83/* zero out entire rx state RAM entry for the channel */
  84static void cppi_reset_rx(struct cppi_rx_stateram __iomem *rx)
  85{
  86	musb_writel(&rx->rx_skipbytes, 0, 0);
  87	musb_writel(&rx->rx_head, 0, 0);
  88	musb_writel(&rx->rx_sop, 0, 0);
  89	musb_writel(&rx->rx_current, 0, 0);
  90	musb_writel(&rx->rx_buf_current, 0, 0);
  91	musb_writel(&rx->rx_len_len, 0, 0);
  92	musb_writel(&rx->rx_cnt_cnt, 0, 0);
  93}
  94
  95/* zero out entire tx state RAM entry for the channel */
  96static void cppi_reset_tx(struct cppi_tx_stateram __iomem *tx, u32 ptr)
  97{
  98	musb_writel(&tx->tx_head, 0, 0);
  99	musb_writel(&tx->tx_buf, 0, 0);
 100	musb_writel(&tx->tx_current, 0, 0);
 101	musb_writel(&tx->tx_buf_current, 0, 0);
 102	musb_writel(&tx->tx_info, 0, 0);
 103	musb_writel(&tx->tx_rem_len, 0, 0);
 104	/* musb_writel(&tx->tx_dummy, 0, 0); */
 105	musb_writel(&tx->tx_complete, 0, ptr);
 106}
 107
 108static void cppi_pool_init(struct cppi *cppi, struct cppi_channel *c)
 109{
 110	int	j;
 111
 112	/* initialize channel fields */
 113	c->head = NULL;
 114	c->tail = NULL;
 115	c->last_processed = NULL;
 116	c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
 117	c->controller = cppi;
 118	c->is_rndis = 0;
 119	c->freelist = NULL;
 120
 121	/* build the BD Free list for the channel */
 122	for (j = 0; j < NUM_TXCHAN_BD + 1; j++) {
 123		struct cppi_descriptor	*bd;
 124		dma_addr_t		dma;
 125
 126		bd = dma_pool_alloc(cppi->pool, GFP_KERNEL, &dma);
 127		bd->dma = dma;
 128		cppi_bd_free(c, bd);
 129	}
 130}
 131
 132static int cppi_channel_abort(struct dma_channel *);
 133
 134static void cppi_pool_free(struct cppi_channel *c)
 135{
 136	struct cppi		*cppi = c->controller;
 137	struct cppi_descriptor	*bd;
 138
 139	(void) cppi_channel_abort(&c->channel);
 140	c->channel.status = MUSB_DMA_STATUS_UNKNOWN;
 141	c->controller = NULL;
 142
 143	/* free all its bds */
 144	bd = c->last_processed;
 145	do {
 146		if (bd)
 147			dma_pool_free(cppi->pool, bd, bd->dma);
 148		bd = cppi_bd_alloc(c);
 149	} while (bd);
 150	c->last_processed = NULL;
 151}
 152
 153static void cppi_controller_start(struct cppi *controller)
 154{
 
 155	void __iomem	*tibase;
 156	int		i;
 157
 
 
 158	/* do whatever is necessary to start controller */
 159	for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
 160		controller->tx[i].transmit = true;
 161		controller->tx[i].index = i;
 162	}
 163	for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
 164		controller->rx[i].transmit = false;
 165		controller->rx[i].index = i;
 166	}
 167
 168	/* setup BD list on a per channel basis */
 169	for (i = 0; i < ARRAY_SIZE(controller->tx); i++)
 170		cppi_pool_init(controller, controller->tx + i);
 171	for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
 172		cppi_pool_init(controller, controller->rx + i);
 173
 174	tibase =  controller->tibase;
 175	INIT_LIST_HEAD(&controller->tx_complete);
 176
 177	/* initialise tx/rx channel head pointers to zero */
 178	for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
 179		struct cppi_channel	*tx_ch = controller->tx + i;
 180		struct cppi_tx_stateram __iomem *tx;
 181
 182		INIT_LIST_HEAD(&tx_ch->tx_complete);
 183
 184		tx = tibase + DAVINCI_TXCPPI_STATERAM_OFFSET(i);
 185		tx_ch->state_ram = tx;
 186		cppi_reset_tx(tx, 0);
 187	}
 188	for (i = 0; i < ARRAY_SIZE(controller->rx); i++) {
 189		struct cppi_channel	*rx_ch = controller->rx + i;
 190		struct cppi_rx_stateram __iomem *rx;
 191
 192		INIT_LIST_HEAD(&rx_ch->tx_complete);
 193
 194		rx = tibase + DAVINCI_RXCPPI_STATERAM_OFFSET(i);
 195		rx_ch->state_ram = rx;
 196		cppi_reset_rx(rx);
 197	}
 198
 199	/* enable individual cppi channels */
 200	musb_writel(tibase, DAVINCI_TXCPPI_INTENAB_REG,
 201			DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 202	musb_writel(tibase, DAVINCI_RXCPPI_INTENAB_REG,
 203			DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 204
 205	/* enable tx/rx CPPI control */
 206	musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
 207	musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_ENABLE);
 208
 209	/* disable RNDIS mode, also host rx RNDIS autorequest */
 210	musb_writel(tibase, DAVINCI_RNDIS_REG, 0);
 211	musb_writel(tibase, DAVINCI_AUTOREQ_REG, 0);
 
 
 212}
 213
 214/*
 215 *  Stop DMA controller
 216 *
 217 *  De-Init the DMA controller as necessary.
 218 */
 219
 220static void cppi_controller_stop(struct cppi *controller)
 221{
 
 222	void __iomem		*tibase;
 223	int			i;
 224	struct musb		*musb;
 225
 
 226	musb = controller->musb;
 227
 228	tibase = controller->tibase;
 229	/* DISABLE INDIVIDUAL CHANNEL Interrupts */
 230	musb_writel(tibase, DAVINCI_TXCPPI_INTCLR_REG,
 231			DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 232	musb_writel(tibase, DAVINCI_RXCPPI_INTCLR_REG,
 233			DAVINCI_DMA_ALL_CHANNELS_ENABLE);
 234
 235	dev_dbg(musb->controller, "Tearing down RX and TX Channels\n");
 236	for (i = 0; i < ARRAY_SIZE(controller->tx); i++) {
 237		/* FIXME restructure of txdma to use bds like rxdma */
 238		controller->tx[i].last_processed = NULL;
 239		cppi_pool_free(controller->tx + i);
 240	}
 241	for (i = 0; i < ARRAY_SIZE(controller->rx); i++)
 242		cppi_pool_free(controller->rx + i);
 243
 244	/* in Tx Case proper teardown is supported. We resort to disabling
 245	 * Tx/Rx CPPI after cleanup of Tx channels. Before TX teardown is
 246	 * complete TX CPPI cannot be disabled.
 247	 */
 248	/*disable tx/rx cppi */
 249	musb_writel(tibase, DAVINCI_TXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
 250	musb_writel(tibase, DAVINCI_RXCPPI_CTRL_REG, DAVINCI_DMA_CTRL_DISABLE);
 
 
 251}
 252
 253/* While dma channel is allocated, we only want the core irqs active
 254 * for fault reports, otherwise we'd get irqs that we don't care about.
 255 * Except for TX irqs, where dma done != fifo empty and reusable ...
 256 *
 257 * NOTE: docs don't say either way, but irq masking **enables** irqs.
 258 *
 259 * REVISIT same issue applies to pure PIO usage too, and non-cppi dma...
 260 */
 261static inline void core_rxirq_disable(void __iomem *tibase, unsigned epnum)
 262{
 263	musb_writel(tibase, DAVINCI_USB_INT_MASK_CLR_REG, 1 << (epnum + 8));
 264}
 265
 266static inline void core_rxirq_enable(void __iomem *tibase, unsigned epnum)
 267{
 268	musb_writel(tibase, DAVINCI_USB_INT_MASK_SET_REG, 1 << (epnum + 8));
 269}
 270
 271
 272/*
 273 * Allocate a CPPI Channel for DMA.  With CPPI, channels are bound to
 274 * each transfer direction of a non-control endpoint, so allocating
 275 * (and deallocating) is mostly a way to notice bad housekeeping on
 276 * the software side.  We assume the irqs are always active.
 277 */
 278static struct dma_channel *
 279cppi_channel_allocate(struct dma_controller *c,
 280		struct musb_hw_ep *ep, u8 transmit)
 281{
 282	struct cppi		*controller;
 283	u8			index;
 284	struct cppi_channel	*cppi_ch;
 285	void __iomem		*tibase;
 286	struct musb		*musb;
 287
 288	controller = container_of(c, struct cppi, controller);
 289	tibase = controller->tibase;
 290	musb = controller->musb;
 291
 292	/* ep0 doesn't use DMA; remember cppi indices are 0..N-1 */
 293	index = ep->epnum - 1;
 294
 295	/* return the corresponding CPPI Channel Handle, and
 296	 * probably disable the non-CPPI irq until we need it.
 297	 */
 298	if (transmit) {
 299		if (index >= ARRAY_SIZE(controller->tx)) {
 300			dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'T', index);
 301			return NULL;
 302		}
 303		cppi_ch = controller->tx + index;
 304	} else {
 305		if (index >= ARRAY_SIZE(controller->rx)) {
 306			dev_dbg(musb->controller, "no %cX%d CPPI channel\n", 'R', index);
 307			return NULL;
 308		}
 309		cppi_ch = controller->rx + index;
 310		core_rxirq_disable(tibase, ep->epnum);
 311	}
 312
 313	/* REVISIT make this an error later once the same driver code works
 314	 * with the other DMA engine too
 315	 */
 316	if (cppi_ch->hw_ep)
 317		dev_dbg(musb->controller, "re-allocating DMA%d %cX channel %p\n",
 318				index, transmit ? 'T' : 'R', cppi_ch);
 319	cppi_ch->hw_ep = ep;
 320	cppi_ch->channel.status = MUSB_DMA_STATUS_FREE;
 321	cppi_ch->channel.max_len = 0x7fffffff;
 322
 323	dev_dbg(musb->controller, "Allocate CPPI%d %cX\n", index, transmit ? 'T' : 'R');
 324	return &cppi_ch->channel;
 325}
 326
 327/* Release a CPPI Channel.  */
 328static void cppi_channel_release(struct dma_channel *channel)
 329{
 330	struct cppi_channel	*c;
 331	void __iomem		*tibase;
 332
 333	/* REVISIT:  for paranoia, check state and abort if needed... */
 334
 335	c = container_of(channel, struct cppi_channel, channel);
 336	tibase = c->controller->tibase;
 337	if (!c->hw_ep)
 338		dev_dbg(c->controller->musb->controller,
 339			"releasing idle DMA channel %p\n", c);
 340	else if (!c->transmit)
 341		core_rxirq_enable(tibase, c->index + 1);
 342
 343	/* for now, leave its cppi IRQ enabled (we won't trigger it) */
 344	c->hw_ep = NULL;
 345	channel->status = MUSB_DMA_STATUS_UNKNOWN;
 346}
 347
 348/* Context: controller irqlocked */
 349static void
 350cppi_dump_rx(int level, struct cppi_channel *c, const char *tag)
 351{
 352	void __iomem			*base = c->controller->mregs;
 353	struct cppi_rx_stateram __iomem	*rx = c->state_ram;
 354
 355	musb_ep_select(base, c->index + 1);
 356
 357	dev_dbg(c->controller->musb->controller,
 358		"RX DMA%d%s: %d left, csr %04x, "
 359		"%08x H%08x S%08x C%08x, "
 360		"B%08x L%08x %08x .. %08x"
 361		"\n",
 362		c->index, tag,
 363		musb_readl(c->controller->tibase,
 364			DAVINCI_RXCPPI_BUFCNT0_REG + 4 * c->index),
 365		musb_readw(c->hw_ep->regs, MUSB_RXCSR),
 366
 367		musb_readl(&rx->rx_skipbytes, 0),
 368		musb_readl(&rx->rx_head, 0),
 369		musb_readl(&rx->rx_sop, 0),
 370		musb_readl(&rx->rx_current, 0),
 371
 372		musb_readl(&rx->rx_buf_current, 0),
 373		musb_readl(&rx->rx_len_len, 0),
 374		musb_readl(&rx->rx_cnt_cnt, 0),
 375		musb_readl(&rx->rx_complete, 0)
 376		);
 377}
 378
 379/* Context: controller irqlocked */
 380static void
 381cppi_dump_tx(int level, struct cppi_channel *c, const char *tag)
 382{
 383	void __iomem			*base = c->controller->mregs;
 384	struct cppi_tx_stateram __iomem	*tx = c->state_ram;
 385
 386	musb_ep_select(base, c->index + 1);
 387
 388	dev_dbg(c->controller->musb->controller,
 389		"TX DMA%d%s: csr %04x, "
 390		"H%08x S%08x C%08x %08x, "
 391		"F%08x L%08x .. %08x"
 392		"\n",
 393		c->index, tag,
 394		musb_readw(c->hw_ep->regs, MUSB_TXCSR),
 395
 396		musb_readl(&tx->tx_head, 0),
 397		musb_readl(&tx->tx_buf, 0),
 398		musb_readl(&tx->tx_current, 0),
 399		musb_readl(&tx->tx_buf_current, 0),
 400
 401		musb_readl(&tx->tx_info, 0),
 402		musb_readl(&tx->tx_rem_len, 0),
 403		/* dummy/unused word 6 */
 404		musb_readl(&tx->tx_complete, 0)
 405		);
 406}
 407
 408/* Context: controller irqlocked */
 409static inline void
 410cppi_rndis_update(struct cppi_channel *c, int is_rx,
 411		void __iomem *tibase, int is_rndis)
 412{
 413	/* we may need to change the rndis flag for this cppi channel */
 414	if (c->is_rndis != is_rndis) {
 415		u32	value = musb_readl(tibase, DAVINCI_RNDIS_REG);
 416		u32	temp = 1 << (c->index);
 417
 418		if (is_rx)
 419			temp <<= 16;
 420		if (is_rndis)
 421			value |= temp;
 422		else
 423			value &= ~temp;
 424		musb_writel(tibase, DAVINCI_RNDIS_REG, value);
 425		c->is_rndis = is_rndis;
 426	}
 427}
 428
 
 429static void cppi_dump_rxbd(const char *tag, struct cppi_descriptor *bd)
 430{
 431	pr_debug("RXBD/%s %08x: "
 432			"nxt %08x buf %08x off.blen %08x opt.plen %08x\n",
 433			tag, bd->dma,
 434			bd->hw_next, bd->hw_bufp, bd->hw_off_len,
 435			bd->hw_options);
 436}
 
 437
 438static void cppi_dump_rxq(int level, const char *tag, struct cppi_channel *rx)
 439{
 
 440	struct cppi_descriptor	*bd;
 441
 
 
 442	cppi_dump_rx(level, rx, tag);
 443	if (rx->last_processed)
 444		cppi_dump_rxbd("last", rx->last_processed);
 445	for (bd = rx->head; bd; bd = bd->next)
 446		cppi_dump_rxbd("active", bd);
 
 447}
 448
 449
 450/* NOTE:  DaVinci autoreq is ignored except for host side "RNDIS" mode RX;
 451 * so we won't ever use it (see "CPPI RX Woes" below).
 452 */
 453static inline int cppi_autoreq_update(struct cppi_channel *rx,
 454		void __iomem *tibase, int onepacket, unsigned n_bds)
 455{
 456	u32	val;
 457
 458#ifdef	RNDIS_RX_IS_USABLE
 459	u32	tmp;
 460	/* assert(is_host_active(musb)) */
 461
 462	/* start from "AutoReq never" */
 463	tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
 464	val = tmp & ~((0x3) << (rx->index * 2));
 465
 466	/* HCD arranged reqpkt for packet #1.  we arrange int
 467	 * for all but the last one, maybe in two segments.
 468	 */
 469	if (!onepacket) {
 470#if 0
 471		/* use two segments, autoreq "all" then the last "never" */
 472		val |= ((0x3) << (rx->index * 2));
 473		n_bds--;
 474#else
 475		/* one segment, autoreq "all-but-last" */
 476		val |= ((0x1) << (rx->index * 2));
 477#endif
 478	}
 479
 480	if (val != tmp) {
 481		int n = 100;
 482
 483		/* make sure that autoreq is updated before continuing */
 484		musb_writel(tibase, DAVINCI_AUTOREQ_REG, val);
 485		do {
 486			tmp = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
 487			if (tmp == val)
 488				break;
 489			cpu_relax();
 490		} while (n-- > 0);
 491	}
 492#endif
 493
 494	/* REQPKT is turned off after each segment */
 495	if (n_bds && rx->channel.actual_len) {
 496		void __iomem	*regs = rx->hw_ep->regs;
 497
 498		val = musb_readw(regs, MUSB_RXCSR);
 499		if (!(val & MUSB_RXCSR_H_REQPKT)) {
 500			val |= MUSB_RXCSR_H_REQPKT | MUSB_RXCSR_H_WZC_BITS;
 501			musb_writew(regs, MUSB_RXCSR, val);
 502			/* flush writebuffer */
 503			val = musb_readw(regs, MUSB_RXCSR);
 504		}
 505	}
 506	return n_bds;
 507}
 508
 509
 510/* Buffer enqueuing Logic:
 511 *
 512 *  - RX builds new queues each time, to help handle routine "early
 513 *    termination" cases (faults, including errors and short reads)
 514 *    more correctly.
 515 *
 516 *  - for now, TX reuses the same queue of BDs every time
 517 *
 518 * REVISIT long term, we want a normal dynamic model.
 519 * ... the goal will be to append to the
 520 * existing queue, processing completed "dma buffers" (segments) on the fly.
 521 *
 522 * Otherwise we force an IRQ latency between requests, which slows us a lot
 523 * (especially in "transparent" dma).  Unfortunately that model seems to be
 524 * inherent in the DMA model from the Mentor code, except in the rare case
 525 * of transfers big enough (~128+ KB) that we could append "middle" segments
 526 * in the TX paths.  (RX can't do this, see below.)
 527 *
 528 * That's true even in the CPPI- friendly iso case, where most urbs have
 529 * several small segments provided in a group and where the "packet at a time"
 530 * "transparent" DMA model is always correct, even on the RX side.
 531 */
 532
 533/*
 534 * CPPI TX:
 535 * ========
 536 * TX is a lot more reasonable than RX; it doesn't need to run in
 537 * irq-per-packet mode very often.  RNDIS mode seems to behave too
 538 * (except how it handles the exactly-N-packets case).  Building a
 539 * txdma queue with multiple requests (urb or usb_request) looks
 540 * like it would work ... but fault handling would need much testing.
 541 *
 542 * The main issue with TX mode RNDIS relates to transfer lengths that
 543 * are an exact multiple of the packet length.  It appears that there's
 544 * a hiccup in that case (maybe the DMA completes before the ZLP gets
 545 * written?) boiling down to not being able to rely on CPPI writing any
 546 * terminating zero length packet before the next transfer is written.
 547 * So that's punted to PIO; better yet, gadget drivers can avoid it.
 548 *
 549 * Plus, there's allegedly an undocumented constraint that rndis transfer
 550 * length be a multiple of 64 bytes ... but the chip doesn't act that
 551 * way, and we really don't _want_ that behavior anyway.
 552 *
 553 * On TX, "transparent" mode works ... although experiments have shown
 554 * problems trying to use the SOP/EOP bits in different USB packets.
 555 *
 556 * REVISIT try to handle terminating zero length packets using CPPI
 557 * instead of doing it by PIO after an IRQ.  (Meanwhile, make Ethernet
 558 * links avoid that issue by forcing them to avoid zlps.)
 559 */
 560static void
 561cppi_next_tx_segment(struct musb *musb, struct cppi_channel *tx)
 562{
 563	unsigned		maxpacket = tx->maxpacket;
 564	dma_addr_t		addr = tx->buf_dma + tx->offset;
 565	size_t			length = tx->buf_len - tx->offset;
 566	struct cppi_descriptor	*bd;
 567	unsigned		n_bds;
 568	unsigned		i;
 569	struct cppi_tx_stateram	__iomem *tx_ram = tx->state_ram;
 570	int			rndis;
 571
 572	/* TX can use the CPPI "rndis" mode, where we can probably fit this
 573	 * transfer in one BD and one IRQ.  The only time we would NOT want
 574	 * to use it is when hardware constraints prevent it, or if we'd
 575	 * trigger the "send a ZLP?" confusion.
 576	 */
 577	rndis = (maxpacket & 0x3f) == 0
 578		&& length > maxpacket
 579		&& length < 0xffff
 580		&& (length % maxpacket) != 0;
 581
 582	if (rndis) {
 583		maxpacket = length;
 584		n_bds = 1;
 585	} else {
 586		n_bds = length / maxpacket;
 587		if (!length || (length % maxpacket))
 588			n_bds++;
 589		n_bds = min(n_bds, (unsigned) NUM_TXCHAN_BD);
 590		length = min(n_bds * maxpacket, length);
 591	}
 592
 593	dev_dbg(musb->controller, "TX DMA%d, pktSz %d %s bds %d dma 0x%llx len %u\n",
 594			tx->index,
 595			maxpacket,
 596			rndis ? "rndis" : "transparent",
 597			n_bds,
 598			(unsigned long long)addr, length);
 599
 600	cppi_rndis_update(tx, 0, musb->ctrl_base, rndis);
 601
 602	/* assuming here that channel_program is called during
 603	 * transfer initiation ... current code maintains state
 604	 * for one outstanding request only (no queues, not even
 605	 * the implicit ones of an iso urb).
 606	 */
 607
 608	bd = tx->freelist;
 609	tx->head = bd;
 610	tx->last_processed = NULL;
 611
 612	/* FIXME use BD pool like RX side does, and just queue
 613	 * the minimum number for this request.
 614	 */
 615
 616	/* Prepare queue of BDs first, then hand it to hardware.
 617	 * All BDs except maybe the last should be of full packet
 618	 * size; for RNDIS there _is_ only that last packet.
 619	 */
 620	for (i = 0; i < n_bds; ) {
 621		if (++i < n_bds && bd->next)
 622			bd->hw_next = bd->next->dma;
 623		else
 624			bd->hw_next = 0;
 625
 626		bd->hw_bufp = tx->buf_dma + tx->offset;
 627
 628		/* FIXME set EOP only on the last packet,
 629		 * SOP only on the first ... avoid IRQs
 630		 */
 631		if ((tx->offset + maxpacket) <= tx->buf_len) {
 632			tx->offset += maxpacket;
 633			bd->hw_off_len = maxpacket;
 634			bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
 635				| CPPI_OWN_SET | maxpacket;
 636		} else {
 637			/* only this one may be a partial USB Packet */
 638			u32		partial_len;
 639
 640			partial_len = tx->buf_len - tx->offset;
 641			tx->offset = tx->buf_len;
 642			bd->hw_off_len = partial_len;
 643
 644			bd->hw_options = CPPI_SOP_SET | CPPI_EOP_SET
 645				| CPPI_OWN_SET | partial_len;
 646			if (partial_len == 0)
 647				bd->hw_options |= CPPI_ZERO_SET;
 648		}
 649
 650		dev_dbg(musb->controller, "TXBD %p: nxt %08x buf %08x len %04x opt %08x\n",
 651				bd, bd->hw_next, bd->hw_bufp,
 652				bd->hw_off_len, bd->hw_options);
 653
 654		/* update the last BD enqueued to the list */
 655		tx->tail = bd;
 656		bd = bd->next;
 657	}
 658
 659	/* BDs live in DMA-coherent memory, but writes might be pending */
 660	cpu_drain_writebuffer();
 661
 662	/* Write to the HeadPtr in state RAM to trigger */
 663	musb_writel(&tx_ram->tx_head, 0, (u32)tx->freelist->dma);
 664
 665	cppi_dump_tx(5, tx, "/S");
 666}
 667
 668/*
 669 * CPPI RX Woes:
 670 * =============
 671 * Consider a 1KB bulk RX buffer in two scenarios:  (a) it's fed two 300 byte
 672 * packets back-to-back, and (b) it's fed two 512 byte packets back-to-back.
 673 * (Full speed transfers have similar scenarios.)
 674 *
 675 * The correct behavior for Linux is that (a) fills the buffer with 300 bytes,
 676 * and the next packet goes into a buffer that's queued later; while (b) fills
 677 * the buffer with 1024 bytes.  How to do that with CPPI?
 678 *
 679 * - RX queues in "rndis" mode -- one single BD -- handle (a) correctly, but
 680 *   (b) loses **BADLY** because nothing (!) happens when that second packet
 681 *   fills the buffer, much less when a third one arrives.  (Which makes this
 682 *   not a "true" RNDIS mode.  In the RNDIS protocol short-packet termination
 683 *   is optional, and it's fine if peripherals -- not hosts! -- pad messages
 684 *   out to end-of-buffer.  Standard PCI host controller DMA descriptors
 685 *   implement that mode by default ... which is no accident.)
 686 *
 687 * - RX queues in "transparent" mode -- two BDs with 512 bytes each -- have
 688 *   converse problems:  (b) is handled right, but (a) loses badly.  CPPI RX
 689 *   ignores SOP/EOP markings and processes both of those BDs; so both packets
 690 *   are loaded into the buffer (with a 212 byte gap between them), and the next
 691 *   buffer queued will NOT get its 300 bytes of data. (It seems like SOP/EOP
 692 *   are intended as outputs for RX queues, not inputs...)
 693 *
 694 * - A variant of "transparent" mode -- one BD at a time -- is the only way to
 695 *   reliably make both cases work, with software handling both cases correctly
 696 *   and at the significant penalty of needing an IRQ per packet.  (The lack of
 697 *   I/O overlap can be slightly ameliorated by enabling double buffering.)
 698 *
 699 * So how to get rid of IRQ-per-packet?  The transparent multi-BD case could
 700 * be used in special cases like mass storage, which sets URB_SHORT_NOT_OK
 701 * (or maybe its peripheral side counterpart) to flag (a) scenarios as errors
 702 * with guaranteed driver level fault recovery and scrubbing out what's left
 703 * of that garbaged datastream.
 704 *
 705 * But there seems to be no way to identify the cases where CPPI RNDIS mode
 706 * is appropriate -- which do NOT include RNDIS host drivers, but do include
 707 * the CDC Ethernet driver! -- and the documentation is incomplete/wrong.
 708 * So we can't _ever_ use RX RNDIS mode ... except by using a heuristic
 709 * that applies best on the peripheral side (and which could fail rudely).
 710 *
 711 * Leaving only "transparent" mode; we avoid multi-bd modes in almost all
 712 * cases other than mass storage class.  Otherwise we're correct but slow,
 713 * since CPPI penalizes our need for a "true RNDIS" default mode.
 714 */
 715
 716
 717/* Heuristic, intended to kick in for ethernet/rndis peripheral ONLY
 718 *
 719 * IFF
 720 *  (a)	peripheral mode ... since rndis peripherals could pad their
 721 *	writes to hosts, causing i/o failure; or we'd have to cope with
 722 *	a largely unknowable variety of host side protocol variants
 723 *  (b)	and short reads are NOT errors ... since full reads would
 724 *	cause those same i/o failures
 725 *  (c)	and read length is
 726 *	- less than 64KB (max per cppi descriptor)
 727 *	- not a multiple of 4096 (g_zero default, full reads typical)
 728 *	- N (>1) packets long, ditto (full reads not EXPECTED)
 729 * THEN
 730 *   try rx rndis mode
 731 *
 732 * Cost of heuristic failing:  RXDMA wedges at the end of transfers that
 733 * fill out the whole buffer.  Buggy host side usb network drivers could
 734 * trigger that, but "in the field" such bugs seem to be all but unknown.
 735 *
 736 * So this module parameter lets the heuristic be disabled.  When using
 737 * gadgetfs, the heuristic will probably need to be disabled.
 738 */
 739static bool cppi_rx_rndis = 1;
 740
 741module_param(cppi_rx_rndis, bool, 0);
 742MODULE_PARM_DESC(cppi_rx_rndis, "enable/disable RX RNDIS heuristic");
 743
 744
 745/**
 746 * cppi_next_rx_segment - dma read for the next chunk of a buffer
 747 * @musb: the controller
 748 * @rx: dma channel
 749 * @onepacket: true unless caller treats short reads as errors, and
 750 *	performs fault recovery above usbcore.
 751 * Context: controller irqlocked
 752 *
 753 * See above notes about why we can't use multi-BD RX queues except in
 754 * rare cases (mass storage class), and can never use the hardware "rndis"
 755 * mode (since it's not a "true" RNDIS mode) with complete safety..
 756 *
 757 * It's ESSENTIAL that callers specify "onepacket" mode unless they kick in
 758 * code to recover from corrupted datastreams after each short transfer.
 759 */
 760static void
 761cppi_next_rx_segment(struct musb *musb, struct cppi_channel *rx, int onepacket)
 762{
 763	unsigned		maxpacket = rx->maxpacket;
 764	dma_addr_t		addr = rx->buf_dma + rx->offset;
 765	size_t			length = rx->buf_len - rx->offset;
 766	struct cppi_descriptor	*bd, *tail;
 767	unsigned		n_bds;
 768	unsigned		i;
 769	void __iomem		*tibase = musb->ctrl_base;
 770	int			is_rndis = 0;
 771	struct cppi_rx_stateram	__iomem *rx_ram = rx->state_ram;
 772	struct cppi_descriptor	*d;
 773
 774	if (onepacket) {
 775		/* almost every USB driver, host or peripheral side */
 776		n_bds = 1;
 777
 778		/* maybe apply the heuristic above */
 779		if (cppi_rx_rndis
 780				&& is_peripheral_active(musb)
 781				&& length > maxpacket
 782				&& (length & ~0xffff) == 0
 783				&& (length & 0x0fff) != 0
 784				&& (length & (maxpacket - 1)) == 0) {
 785			maxpacket = length;
 786			is_rndis = 1;
 787		}
 788	} else {
 789		/* virtually nothing except mass storage class */
 790		if (length > 0xffff) {
 791			n_bds = 0xffff / maxpacket;
 792			length = n_bds * maxpacket;
 793		} else {
 794			n_bds = length / maxpacket;
 795			if (length % maxpacket)
 796				n_bds++;
 797		}
 798		if (n_bds == 1)
 799			onepacket = 1;
 800		else
 801			n_bds = min(n_bds, (unsigned) NUM_RXCHAN_BD);
 802	}
 803
 804	/* In host mode, autorequest logic can generate some IN tokens; it's
 805	 * tricky since we can't leave REQPKT set in RXCSR after the transfer
 806	 * finishes. So:  multipacket transfers involve two or more segments.
 807	 * And always at least two IRQs ... RNDIS mode is not an option.
 808	 */
 809	if (is_host_active(musb))
 810		n_bds = cppi_autoreq_update(rx, tibase, onepacket, n_bds);
 811
 812	cppi_rndis_update(rx, 1, musb->ctrl_base, is_rndis);
 813
 814	length = min(n_bds * maxpacket, length);
 815
 816	dev_dbg(musb->controller, "RX DMA%d seg, maxp %d %s bds %d (cnt %d) "
 817			"dma 0x%llx len %u %u/%u\n",
 818			rx->index, maxpacket,
 819			onepacket
 820				? (is_rndis ? "rndis" : "onepacket")
 821				: "multipacket",
 822			n_bds,
 823			musb_readl(tibase,
 824				DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
 825					& 0xffff,
 826			(unsigned long long)addr, length,
 827			rx->channel.actual_len, rx->buf_len);
 828
 829	/* only queue one segment at a time, since the hardware prevents
 830	 * correct queue shutdown after unexpected short packets
 831	 */
 832	bd = cppi_bd_alloc(rx);
 833	rx->head = bd;
 834
 835	/* Build BDs for all packets in this segment */
 836	for (i = 0, tail = NULL; bd && i < n_bds; i++, tail = bd) {
 837		u32	bd_len;
 838
 839		if (i) {
 840			bd = cppi_bd_alloc(rx);
 841			if (!bd)
 842				break;
 843			tail->next = bd;
 844			tail->hw_next = bd->dma;
 845		}
 846		bd->hw_next = 0;
 847
 848		/* all but the last packet will be maxpacket size */
 849		if (maxpacket < length)
 850			bd_len = maxpacket;
 851		else
 852			bd_len = length;
 853
 854		bd->hw_bufp = addr;
 855		addr += bd_len;
 856		rx->offset += bd_len;
 857
 858		bd->hw_off_len = (0 /*offset*/ << 16) + bd_len;
 859		bd->buflen = bd_len;
 860
 861		bd->hw_options = CPPI_OWN_SET | (i == 0 ? length : 0);
 862		length -= bd_len;
 863	}
 864
 865	/* we always expect at least one reusable BD! */
 866	if (!tail) {
 867		WARNING("rx dma%d -- no BDs? need %d\n", rx->index, n_bds);
 868		return;
 869	} else if (i < n_bds)
 870		WARNING("rx dma%d -- only %d of %d BDs\n", rx->index, i, n_bds);
 871
 872	tail->next = NULL;
 873	tail->hw_next = 0;
 874
 875	bd = rx->head;
 876	rx->tail = tail;
 877
 878	/* short reads and other faults should terminate this entire
 879	 * dma segment.  we want one "dma packet" per dma segment, not
 880	 * one per USB packet, terminating the whole queue at once...
 881	 * NOTE that current hardware seems to ignore SOP and EOP.
 882	 */
 883	bd->hw_options |= CPPI_SOP_SET;
 884	tail->hw_options |= CPPI_EOP_SET;
 885
 886	for (d = rx->head; d; d = d->next)
 887		cppi_dump_rxbd("S", d);
 
 
 
 
 
 
 888
 889	/* in case the preceding transfer left some state... */
 890	tail = rx->last_processed;
 891	if (tail) {
 892		tail->next = bd;
 893		tail->hw_next = bd->dma;
 894	}
 895
 896	core_rxirq_enable(tibase, rx->index + 1);
 897
 898	/* BDs live in DMA-coherent memory, but writes might be pending */
 899	cpu_drain_writebuffer();
 900
 901	/* REVISIT specs say to write this AFTER the BUFCNT register
 902	 * below ... but that loses badly.
 903	 */
 904	musb_writel(&rx_ram->rx_head, 0, bd->dma);
 905
 906	/* bufferCount must be at least 3, and zeroes on completion
 907	 * unless it underflows below zero, or stops at two, or keeps
 908	 * growing ... grr.
 909	 */
 910	i = musb_readl(tibase,
 911			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
 912			& 0xffff;
 913
 914	if (!i)
 915		musb_writel(tibase,
 916			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
 917			n_bds + 2);
 918	else if (n_bds > (i - 3))
 919		musb_writel(tibase,
 920			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
 921			n_bds - (i - 3));
 922
 923	i = musb_readl(tibase,
 924			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4))
 925			& 0xffff;
 926	if (i < (2 + n_bds)) {
 927		dev_dbg(musb->controller, "bufcnt%d underrun - %d (for %d)\n",
 928					rx->index, i, n_bds);
 929		musb_writel(tibase,
 930			DAVINCI_RXCPPI_BUFCNT0_REG + (rx->index * 4),
 931			n_bds + 2);
 932	}
 933
 934	cppi_dump_rx(4, rx, "/S");
 935}
 936
 937/**
 938 * cppi_channel_program - program channel for data transfer
 939 * @ch: the channel
 940 * @maxpacket: max packet size
 941 * @mode: For RX, 1 unless the usb protocol driver promised to treat
 942 *	all short reads as errors and kick in high level fault recovery.
 943 *	For TX, ignored because of RNDIS mode races/glitches.
 944 * @dma_addr: dma address of buffer
 945 * @len: length of buffer
 946 * Context: controller irqlocked
 947 */
 948static int cppi_channel_program(struct dma_channel *ch,
 949		u16 maxpacket, u8 mode,
 950		dma_addr_t dma_addr, u32 len)
 951{
 952	struct cppi_channel	*cppi_ch;
 953	struct cppi		*controller;
 954	struct musb		*musb;
 955
 956	cppi_ch = container_of(ch, struct cppi_channel, channel);
 957	controller = cppi_ch->controller;
 958	musb = controller->musb;
 959
 960	switch (ch->status) {
 961	case MUSB_DMA_STATUS_BUS_ABORT:
 962	case MUSB_DMA_STATUS_CORE_ABORT:
 963		/* fault irq handler should have handled cleanup */
 964		WARNING("%cX DMA%d not cleaned up after abort!\n",
 965				cppi_ch->transmit ? 'T' : 'R',
 966				cppi_ch->index);
 967		/* WARN_ON(1); */
 968		break;
 969	case MUSB_DMA_STATUS_BUSY:
 970		WARNING("program active channel?  %cX DMA%d\n",
 971				cppi_ch->transmit ? 'T' : 'R',
 972				cppi_ch->index);
 973		/* WARN_ON(1); */
 974		break;
 975	case MUSB_DMA_STATUS_UNKNOWN:
 976		dev_dbg(musb->controller, "%cX DMA%d not allocated!\n",
 977				cppi_ch->transmit ? 'T' : 'R',
 978				cppi_ch->index);
 979		/* FALLTHROUGH */
 980	case MUSB_DMA_STATUS_FREE:
 981		break;
 982	}
 983
 984	ch->status = MUSB_DMA_STATUS_BUSY;
 985
 986	/* set transfer parameters, then queue up its first segment */
 987	cppi_ch->buf_dma = dma_addr;
 988	cppi_ch->offset = 0;
 989	cppi_ch->maxpacket = maxpacket;
 990	cppi_ch->buf_len = len;
 991	cppi_ch->channel.actual_len = 0;
 992
 993	/* TX channel? or RX? */
 994	if (cppi_ch->transmit)
 995		cppi_next_tx_segment(musb, cppi_ch);
 996	else
 997		cppi_next_rx_segment(musb, cppi_ch, mode);
 998
 999	return true;
1000}
1001
1002static bool cppi_rx_scan(struct cppi *cppi, unsigned ch)
1003{
1004	struct cppi_channel		*rx = &cppi->rx[ch];
1005	struct cppi_rx_stateram __iomem	*state = rx->state_ram;
1006	struct cppi_descriptor		*bd;
1007	struct cppi_descriptor		*last = rx->last_processed;
1008	bool				completed = false;
1009	bool				acked = false;
1010	int				i;
1011	dma_addr_t			safe2ack;
1012	void __iomem			*regs = rx->hw_ep->regs;
1013	struct musb			*musb = cppi->musb;
1014
1015	cppi_dump_rx(6, rx, "/K");
1016
1017	bd = last ? last->next : rx->head;
1018	if (!bd)
1019		return false;
1020
1021	/* run through all completed BDs */
1022	for (i = 0, safe2ack = musb_readl(&state->rx_complete, 0);
1023			(safe2ack || completed) && bd && i < NUM_RXCHAN_BD;
1024			i++, bd = bd->next) {
1025		u16	len;
1026
1027		/* catch latest BD writes from CPPI */
1028		rmb();
1029		if (!completed && (bd->hw_options & CPPI_OWN_SET))
1030			break;
1031
1032		dev_dbg(musb->controller, "C/RXBD %llx: nxt %08x buf %08x "
1033			"off.len %08x opt.len %08x (%d)\n",
1034			(unsigned long long)bd->dma, bd->hw_next, bd->hw_bufp,
1035			bd->hw_off_len, bd->hw_options,
1036			rx->channel.actual_len);
1037
1038		/* actual packet received length */
1039		if ((bd->hw_options & CPPI_SOP_SET) && !completed)
1040			len = bd->hw_off_len & CPPI_RECV_PKTLEN_MASK;
1041		else
1042			len = 0;
1043
1044		if (bd->hw_options & CPPI_EOQ_MASK)
1045			completed = true;
1046
1047		if (!completed && len < bd->buflen) {
1048			/* NOTE:  when we get a short packet, RXCSR_H_REQPKT
1049			 * must have been cleared, and no more DMA packets may
1050			 * active be in the queue... TI docs didn't say, but
1051			 * CPPI ignores those BDs even though OWN is still set.
1052			 */
1053			completed = true;
1054			dev_dbg(musb->controller, "rx short %d/%d (%d)\n",
1055					len, bd->buflen,
1056					rx->channel.actual_len);
1057		}
1058
1059		/* If we got here, we expect to ack at least one BD; meanwhile
1060		 * CPPI may completing other BDs while we scan this list...
1061		 *
1062		 * RACE: we can notice OWN cleared before CPPI raises the
1063		 * matching irq by writing that BD as the completion pointer.
1064		 * In such cases, stop scanning and wait for the irq, avoiding
1065		 * lost acks and states where BD ownership is unclear.
1066		 */
1067		if (bd->dma == safe2ack) {
1068			musb_writel(&state->rx_complete, 0, safe2ack);
1069			safe2ack = musb_readl(&state->rx_complete, 0);
1070			acked = true;
1071			if (bd->dma == safe2ack)
1072				safe2ack = 0;
1073		}
1074
1075		rx->channel.actual_len += len;
1076
1077		cppi_bd_free(rx, last);
1078		last = bd;
1079
1080		/* stop scanning on end-of-segment */
1081		if (bd->hw_next == 0)
1082			completed = true;
1083	}
1084	rx->last_processed = last;
1085
1086	/* dma abort, lost ack, or ... */
1087	if (!acked && last) {
1088		int	csr;
1089
1090		if (safe2ack == 0 || safe2ack == rx->last_processed->dma)
1091			musb_writel(&state->rx_complete, 0, safe2ack);
1092		if (safe2ack == 0) {
1093			cppi_bd_free(rx, last);
1094			rx->last_processed = NULL;
1095
1096			/* if we land here on the host side, H_REQPKT will
1097			 * be clear and we need to restart the queue...
1098			 */
1099			WARN_ON(rx->head);
1100		}
1101		musb_ep_select(cppi->mregs, rx->index + 1);
1102		csr = musb_readw(regs, MUSB_RXCSR);
1103		if (csr & MUSB_RXCSR_DMAENAB) {
1104			dev_dbg(musb->controller, "list%d %p/%p, last %llx%s, csr %04x\n",
1105				rx->index,
1106				rx->head, rx->tail,
1107				rx->last_processed
1108					? (unsigned long long)
1109						rx->last_processed->dma
1110					: 0,
1111				completed ? ", completed" : "",
1112				csr);
1113			cppi_dump_rxq(4, "/what?", rx);
1114		}
1115	}
1116	if (!completed) {
1117		int	csr;
1118
1119		rx->head = bd;
1120
1121		/* REVISIT seems like "autoreq all but EOP" doesn't...
1122		 * setting it here "should" be racey, but seems to work
1123		 */
1124		csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1125		if (is_host_active(cppi->musb)
1126				&& bd
1127				&& !(csr & MUSB_RXCSR_H_REQPKT)) {
1128			csr |= MUSB_RXCSR_H_REQPKT;
1129			musb_writew(regs, MUSB_RXCSR,
1130					MUSB_RXCSR_H_WZC_BITS | csr);
1131			csr = musb_readw(rx->hw_ep->regs, MUSB_RXCSR);
1132		}
1133	} else {
1134		rx->head = NULL;
1135		rx->tail = NULL;
1136	}
1137
1138	cppi_dump_rx(6, rx, completed ? "/completed" : "/cleaned");
1139	return completed;
1140}
1141
1142irqreturn_t cppi_interrupt(int irq, void *dev_id)
1143{
1144	struct musb		*musb = dev_id;
1145	struct cppi		*cppi;
1146	void __iomem		*tibase;
1147	struct musb_hw_ep	*hw_ep = NULL;
1148	u32			rx, tx;
1149	int			i, index;
1150	unsigned long		uninitialized_var(flags);
1151
1152	cppi = container_of(musb->dma_controller, struct cppi, controller);
1153	if (cppi->irq)
1154		spin_lock_irqsave(&musb->lock, flags);
1155
1156	tibase = musb->ctrl_base;
1157
1158	tx = musb_readl(tibase, DAVINCI_TXCPPI_MASKED_REG);
1159	rx = musb_readl(tibase, DAVINCI_RXCPPI_MASKED_REG);
1160
1161	if (!tx && !rx) {
1162		if (cppi->irq)
1163			spin_unlock_irqrestore(&musb->lock, flags);
1164		return IRQ_NONE;
1165	}
1166
1167	dev_dbg(musb->controller, "CPPI IRQ Tx%x Rx%x\n", tx, rx);
1168
1169	/* process TX channels */
1170	for (index = 0; tx; tx = tx >> 1, index++) {
1171		struct cppi_channel		*tx_ch;
1172		struct cppi_tx_stateram __iomem	*tx_ram;
1173		bool				completed = false;
1174		struct cppi_descriptor		*bd;
1175
1176		if (!(tx & 1))
1177			continue;
1178
1179		tx_ch = cppi->tx + index;
1180		tx_ram = tx_ch->state_ram;
1181
1182		/* FIXME  need a cppi_tx_scan() routine, which
1183		 * can also be called from abort code
1184		 */
1185
1186		cppi_dump_tx(5, tx_ch, "/E");
1187
1188		bd = tx_ch->head;
1189
1190		/*
1191		 * If Head is null then this could mean that a abort interrupt
1192		 * that needs to be acknowledged.
1193		 */
1194		if (NULL == bd) {
1195			dev_dbg(musb->controller, "null BD\n");
1196			musb_writel(&tx_ram->tx_complete, 0, 0);
1197			continue;
1198		}
1199
1200		/* run through all completed BDs */
1201		for (i = 0; !completed && bd && i < NUM_TXCHAN_BD;
1202				i++, bd = bd->next) {
1203			u16	len;
1204
1205			/* catch latest BD writes from CPPI */
1206			rmb();
1207			if (bd->hw_options & CPPI_OWN_SET)
1208				break;
1209
1210			dev_dbg(musb->controller, "C/TXBD %p n %x b %x off %x opt %x\n",
1211					bd, bd->hw_next, bd->hw_bufp,
1212					bd->hw_off_len, bd->hw_options);
1213
1214			len = bd->hw_off_len & CPPI_BUFFER_LEN_MASK;
1215			tx_ch->channel.actual_len += len;
1216
1217			tx_ch->last_processed = bd;
1218
1219			/* write completion register to acknowledge
1220			 * processing of completed BDs, and possibly
1221			 * release the IRQ; EOQ might not be set ...
1222			 *
1223			 * REVISIT use the same ack strategy as rx
1224			 *
1225			 * REVISIT have observed bit 18 set; huh??
1226			 */
1227			/* if ((bd->hw_options & CPPI_EOQ_MASK)) */
1228				musb_writel(&tx_ram->tx_complete, 0, bd->dma);
1229
1230			/* stop scanning on end-of-segment */
1231			if (bd->hw_next == 0)
1232				completed = true;
1233		}
1234
1235		/* on end of segment, maybe go to next one */
1236		if (completed) {
1237			/* cppi_dump_tx(4, tx_ch, "/complete"); */
1238
1239			/* transfer more, or report completion */
1240			if (tx_ch->offset >= tx_ch->buf_len) {
1241				tx_ch->head = NULL;
1242				tx_ch->tail = NULL;
1243				tx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1244
1245				hw_ep = tx_ch->hw_ep;
1246
1247				musb_dma_completion(musb, index + 1, 1);
1248
1249			} else {
1250				/* Bigger transfer than we could fit in
1251				 * that first batch of descriptors...
1252				 */
1253				cppi_next_tx_segment(musb, tx_ch);
1254			}
1255		} else
1256			tx_ch->head = bd;
1257	}
1258
1259	/* Start processing the RX block */
1260	for (index = 0; rx; rx = rx >> 1, index++) {
1261
1262		if (rx & 1) {
1263			struct cppi_channel		*rx_ch;
1264
1265			rx_ch = cppi->rx + index;
1266
1267			/* let incomplete dma segments finish */
1268			if (!cppi_rx_scan(cppi, index))
1269				continue;
1270
1271			/* start another dma segment if needed */
1272			if (rx_ch->channel.actual_len != rx_ch->buf_len
1273					&& rx_ch->channel.actual_len
1274						== rx_ch->offset) {
1275				cppi_next_rx_segment(musb, rx_ch, 1);
1276				continue;
1277			}
1278
1279			/* all segments completed! */
1280			rx_ch->channel.status = MUSB_DMA_STATUS_FREE;
1281
1282			hw_ep = rx_ch->hw_ep;
1283
1284			core_rxirq_disable(tibase, index + 1);
1285			musb_dma_completion(musb, index + 1, 0);
1286		}
1287	}
1288
1289	/* write to CPPI EOI register to re-enable interrupts */
1290	musb_writel(tibase, DAVINCI_CPPI_EOI_REG, 0);
1291
1292	if (cppi->irq)
1293		spin_unlock_irqrestore(&musb->lock, flags);
1294
1295	return IRQ_HANDLED;
1296}
1297EXPORT_SYMBOL_GPL(cppi_interrupt);
1298
1299/* Instantiate a software object representing a DMA controller. */
1300struct dma_controller *
1301cppi_dma_controller_create(struct musb *musb, void __iomem *mregs)
1302{
1303	struct cppi		*controller;
1304	struct device		*dev = musb->controller;
1305	struct platform_device	*pdev = to_platform_device(dev);
1306	int			irq = platform_get_irq_byname(pdev, "dma");
1307
1308	controller = kzalloc(sizeof *controller, GFP_KERNEL);
1309	if (!controller)
1310		return NULL;
1311
1312	controller->mregs = mregs;
1313	controller->tibase = mregs - DAVINCI_BASE_OFFSET;
1314
1315	controller->musb = musb;
 
 
1316	controller->controller.channel_alloc = cppi_channel_allocate;
1317	controller->controller.channel_release = cppi_channel_release;
1318	controller->controller.channel_program = cppi_channel_program;
1319	controller->controller.channel_abort = cppi_channel_abort;
1320
1321	/* NOTE: allocating from on-chip SRAM would give the least
1322	 * contention for memory access, if that ever matters here.
1323	 */
1324
1325	/* setup BufferPool */
1326	controller->pool = dma_pool_create("cppi",
1327			controller->musb->controller,
1328			sizeof(struct cppi_descriptor),
1329			CPPI_DESCRIPTOR_ALIGN, 0);
1330	if (!controller->pool) {
1331		kfree(controller);
1332		return NULL;
1333	}
1334
1335	if (irq > 0) {
1336		if (request_irq(irq, cppi_interrupt, 0, "cppi-dma", musb)) {
1337			dev_err(dev, "request_irq %d failed!\n", irq);
1338			musb_dma_controller_destroy(&controller->controller);
1339			return NULL;
1340		}
1341		controller->irq = irq;
1342	}
1343
1344	cppi_controller_start(controller);
1345	return &controller->controller;
1346}
1347EXPORT_SYMBOL_GPL(cppi_dma_controller_create);
1348
1349/*
1350 *  Destroy a previously-instantiated DMA controller.
1351 */
1352void cppi_dma_controller_destroy(struct dma_controller *c)
1353{
1354	struct cppi	*cppi;
1355
1356	cppi = container_of(c, struct cppi, controller);
1357
1358	cppi_controller_stop(cppi);
1359
1360	if (cppi->irq)
1361		free_irq(cppi->irq, cppi->musb);
1362
1363	/* assert:  caller stopped the controller first */
1364	dma_pool_destroy(cppi->pool);
1365
1366	kfree(cppi);
1367}
1368EXPORT_SYMBOL_GPL(cppi_dma_controller_destroy);
1369
1370/*
1371 * Context: controller irqlocked, endpoint selected
1372 */
1373static int cppi_channel_abort(struct dma_channel *channel)
1374{
1375	struct cppi_channel	*cppi_ch;
1376	struct cppi		*controller;
1377	void __iomem		*mbase;
1378	void __iomem		*tibase;
1379	void __iomem		*regs;
1380	u32			value;
1381	struct cppi_descriptor	*queue;
1382
1383	cppi_ch = container_of(channel, struct cppi_channel, channel);
1384
1385	controller = cppi_ch->controller;
1386
1387	switch (channel->status) {
1388	case MUSB_DMA_STATUS_BUS_ABORT:
1389	case MUSB_DMA_STATUS_CORE_ABORT:
1390		/* from RX or TX fault irq handler */
1391	case MUSB_DMA_STATUS_BUSY:
1392		/* the hardware needs shutting down */
1393		regs = cppi_ch->hw_ep->regs;
1394		break;
1395	case MUSB_DMA_STATUS_UNKNOWN:
1396	case MUSB_DMA_STATUS_FREE:
1397		return 0;
1398	default:
1399		return -EINVAL;
1400	}
1401
1402	if (!cppi_ch->transmit && cppi_ch->head)
1403		cppi_dump_rxq(3, "/abort", cppi_ch);
1404
1405	mbase = controller->mregs;
1406	tibase = controller->tibase;
1407
1408	queue = cppi_ch->head;
1409	cppi_ch->head = NULL;
1410	cppi_ch->tail = NULL;
1411
1412	/* REVISIT should rely on caller having done this,
1413	 * and caller should rely on us not changing it.
1414	 * peripheral code is safe ... check host too.
1415	 */
1416	musb_ep_select(mbase, cppi_ch->index + 1);
1417
1418	if (cppi_ch->transmit) {
1419		struct cppi_tx_stateram __iomem *tx_ram;
1420		/* REVISIT put timeouts on these controller handshakes */
1421
1422		cppi_dump_tx(6, cppi_ch, " (teardown)");
1423
1424		/* teardown DMA engine then usb core */
1425		do {
1426			value = musb_readl(tibase, DAVINCI_TXCPPI_TEAR_REG);
1427		} while (!(value & CPPI_TEAR_READY));
1428		musb_writel(tibase, DAVINCI_TXCPPI_TEAR_REG, cppi_ch->index);
1429
1430		tx_ram = cppi_ch->state_ram;
1431		do {
1432			value = musb_readl(&tx_ram->tx_complete, 0);
1433		} while (0xFFFFFFFC != value);
1434
1435		/* FIXME clean up the transfer state ... here?
1436		 * the completion routine should get called with
1437		 * an appropriate status code.
1438		 */
1439
1440		value = musb_readw(regs, MUSB_TXCSR);
1441		value &= ~MUSB_TXCSR_DMAENAB;
1442		value |= MUSB_TXCSR_FLUSHFIFO;
1443		musb_writew(regs, MUSB_TXCSR, value);
1444		musb_writew(regs, MUSB_TXCSR, value);
1445
1446		/*
1447		 * 1. Write to completion Ptr value 0x1(bit 0 set)
1448		 *    (write back mode)
1449		 * 2. Wait for abort interrupt and then put the channel in
1450		 *    compare mode by writing 1 to the tx_complete register.
1451		 */
1452		cppi_reset_tx(tx_ram, 1);
1453		cppi_ch->head = NULL;
1454		musb_writel(&tx_ram->tx_complete, 0, 1);
1455		cppi_dump_tx(5, cppi_ch, " (done teardown)");
1456
1457		/* REVISIT tx side _should_ clean up the same way
1458		 * as the RX side ... this does no cleanup at all!
1459		 */
1460
1461	} else /* RX */ {
1462		u16			csr;
1463
1464		/* NOTE: docs don't guarantee any of this works ...  we
1465		 * expect that if the usb core stops telling the cppi core
1466		 * to pull more data from it, then it'll be safe to flush
1467		 * current RX DMA state iff any pending fifo transfer is done.
1468		 */
1469
1470		core_rxirq_disable(tibase, cppi_ch->index + 1);
1471
1472		/* for host, ensure ReqPkt is never set again */
1473		if (is_host_active(cppi_ch->controller->musb)) {
1474			value = musb_readl(tibase, DAVINCI_AUTOREQ_REG);
1475			value &= ~((0x3) << (cppi_ch->index * 2));
1476			musb_writel(tibase, DAVINCI_AUTOREQ_REG, value);
1477		}
1478
1479		csr = musb_readw(regs, MUSB_RXCSR);
1480
1481		/* for host, clear (just) ReqPkt at end of current packet(s) */
1482		if (is_host_active(cppi_ch->controller->musb)) {
1483			csr |= MUSB_RXCSR_H_WZC_BITS;
1484			csr &= ~MUSB_RXCSR_H_REQPKT;
1485		} else
1486			csr |= MUSB_RXCSR_P_WZC_BITS;
1487
1488		/* clear dma enable */
1489		csr &= ~(MUSB_RXCSR_DMAENAB);
1490		musb_writew(regs, MUSB_RXCSR, csr);
1491		csr = musb_readw(regs, MUSB_RXCSR);
1492
1493		/* Quiesce: wait for current dma to finish (if not cleanup).
1494		 * We can't use bit zero of stateram->rx_sop, since that
1495		 * refers to an entire "DMA packet" not just emptying the
1496		 * current fifo.  Most segments need multiple usb packets.
1497		 */
1498		if (channel->status == MUSB_DMA_STATUS_BUSY)
1499			udelay(50);
1500
1501		/* scan the current list, reporting any data that was
1502		 * transferred and acking any IRQ
1503		 */
1504		cppi_rx_scan(controller, cppi_ch->index);
1505
1506		/* clobber the existing state once it's idle
1507		 *
1508		 * NOTE:  arguably, we should also wait for all the other
1509		 * RX channels to quiesce (how??) and then temporarily
1510		 * disable RXCPPI_CTRL_REG ... but it seems that we can
1511		 * rely on the controller restarting from state ram, with
1512		 * only RXCPPI_BUFCNT state being bogus.  BUFCNT will
1513		 * correct itself after the next DMA transfer though.
1514		 *
1515		 * REVISIT does using rndis mode change that?
1516		 */
1517		cppi_reset_rx(cppi_ch->state_ram);
1518
1519		/* next DMA request _should_ load cppi head ptr */
1520
1521		/* ... we don't "free" that list, only mutate it in place.  */
1522		cppi_dump_rx(5, cppi_ch, " (done abort)");
1523
1524		/* clean up previously pending bds */
1525		cppi_bd_free(cppi_ch, cppi_ch->last_processed);
1526		cppi_ch->last_processed = NULL;
1527
1528		while (queue) {
1529			struct cppi_descriptor	*tmp = queue->next;
1530
1531			cppi_bd_free(cppi_ch, queue);
1532			queue = tmp;
1533		}
1534	}
1535
1536	channel->status = MUSB_DMA_STATUS_FREE;
1537	cppi_ch->buf_dma = 0;
1538	cppi_ch->offset = 0;
1539	cppi_ch->buf_len = 0;
1540	cppi_ch->maxpacket = 0;
1541	return 0;
1542}
1543
1544/* TBD Queries:
1545 *
1546 * Power Management ... probably turn off cppi during suspend, restart;
1547 * check state ram?  Clocking is presumably shared with usb core.
1548 */