Linux Audio

Check our new training course

Linux kernel drivers training

May 6-19, 2025
Register
Loading...
v6.8
   1// SPDX-License-Identifier: GPL-2.0-or-later
   2/*
   3 * Driver for the Cirrus Logic EP93xx DMA Controller
   4 *
   5 * Copyright (C) 2011 Mika Westerberg
   6 *
   7 * DMA M2P implementation is based on the original
   8 * arch/arm/mach-ep93xx/dma-m2p.c which has following copyrights:
   9 *
  10 *   Copyright (C) 2006 Lennert Buytenhek <buytenh@wantstofly.org>
  11 *   Copyright (C) 2006 Applied Data Systems
  12 *   Copyright (C) 2009 Ryan Mallon <rmallon@gmail.com>
  13 *
  14 * This driver is based on dw_dmac and amba-pl08x drivers.
 
 
 
 
 
  15 */
  16
  17#include <linux/clk.h>
  18#include <linux/init.h>
  19#include <linux/interrupt.h>
  20#include <linux/dmaengine.h>
  21#include <linux/module.h>
  22#include <linux/mod_devicetable.h>
  23#include <linux/platform_device.h>
  24#include <linux/slab.h>
  25
  26#include <linux/platform_data/dma-ep93xx.h>
  27
  28#include "dmaengine.h"
  29
  30/* M2P registers */
  31#define M2P_CONTROL			0x0000
  32#define M2P_CONTROL_STALLINT		BIT(0)
  33#define M2P_CONTROL_NFBINT		BIT(1)
  34#define M2P_CONTROL_CH_ERROR_INT	BIT(3)
  35#define M2P_CONTROL_ENABLE		BIT(4)
  36#define M2P_CONTROL_ICE			BIT(6)
  37
  38#define M2P_INTERRUPT			0x0004
  39#define M2P_INTERRUPT_STALL		BIT(0)
  40#define M2P_INTERRUPT_NFB		BIT(1)
  41#define M2P_INTERRUPT_ERROR		BIT(3)
  42
  43#define M2P_PPALLOC			0x0008
  44#define M2P_STATUS			0x000c
  45
  46#define M2P_MAXCNT0			0x0020
  47#define M2P_BASE0			0x0024
  48#define M2P_MAXCNT1			0x0030
  49#define M2P_BASE1			0x0034
  50
  51#define M2P_STATE_IDLE			0
  52#define M2P_STATE_STALL			1
  53#define M2P_STATE_ON			2
  54#define M2P_STATE_NEXT			3
  55
  56/* M2M registers */
  57#define M2M_CONTROL			0x0000
  58#define M2M_CONTROL_DONEINT		BIT(2)
  59#define M2M_CONTROL_ENABLE		BIT(3)
  60#define M2M_CONTROL_START		BIT(4)
  61#define M2M_CONTROL_DAH			BIT(11)
  62#define M2M_CONTROL_SAH			BIT(12)
  63#define M2M_CONTROL_PW_SHIFT		9
  64#define M2M_CONTROL_PW_8		(0 << M2M_CONTROL_PW_SHIFT)
  65#define M2M_CONTROL_PW_16		(1 << M2M_CONTROL_PW_SHIFT)
  66#define M2M_CONTROL_PW_32		(2 << M2M_CONTROL_PW_SHIFT)
  67#define M2M_CONTROL_PW_MASK		(3 << M2M_CONTROL_PW_SHIFT)
  68#define M2M_CONTROL_TM_SHIFT		13
  69#define M2M_CONTROL_TM_TX		(1 << M2M_CONTROL_TM_SHIFT)
  70#define M2M_CONTROL_TM_RX		(2 << M2M_CONTROL_TM_SHIFT)
  71#define M2M_CONTROL_NFBINT		BIT(21)
  72#define M2M_CONTROL_RSS_SHIFT		22
  73#define M2M_CONTROL_RSS_SSPRX		(1 << M2M_CONTROL_RSS_SHIFT)
  74#define M2M_CONTROL_RSS_SSPTX		(2 << M2M_CONTROL_RSS_SHIFT)
  75#define M2M_CONTROL_RSS_IDE		(3 << M2M_CONTROL_RSS_SHIFT)
  76#define M2M_CONTROL_NO_HDSK		BIT(24)
  77#define M2M_CONTROL_PWSC_SHIFT		25
  78
  79#define M2M_INTERRUPT			0x0004
  80#define M2M_INTERRUPT_MASK		6
  81
  82#define M2M_STATUS			0x000c
  83#define M2M_STATUS_CTL_SHIFT		1
  84#define M2M_STATUS_CTL_IDLE		(0 << M2M_STATUS_CTL_SHIFT)
  85#define M2M_STATUS_CTL_STALL		(1 << M2M_STATUS_CTL_SHIFT)
  86#define M2M_STATUS_CTL_MEMRD		(2 << M2M_STATUS_CTL_SHIFT)
  87#define M2M_STATUS_CTL_MEMWR		(3 << M2M_STATUS_CTL_SHIFT)
  88#define M2M_STATUS_CTL_BWCWAIT		(4 << M2M_STATUS_CTL_SHIFT)
  89#define M2M_STATUS_CTL_MASK		(7 << M2M_STATUS_CTL_SHIFT)
  90#define M2M_STATUS_BUF_SHIFT		4
  91#define M2M_STATUS_BUF_NO		(0 << M2M_STATUS_BUF_SHIFT)
  92#define M2M_STATUS_BUF_ON		(1 << M2M_STATUS_BUF_SHIFT)
  93#define M2M_STATUS_BUF_NEXT		(2 << M2M_STATUS_BUF_SHIFT)
  94#define M2M_STATUS_BUF_MASK		(3 << M2M_STATUS_BUF_SHIFT)
  95#define M2M_STATUS_DONE			BIT(6)
  96
  97#define M2M_BCR0			0x0010
  98#define M2M_BCR1			0x0014
  99#define M2M_SAR_BASE0			0x0018
 100#define M2M_SAR_BASE1			0x001c
 101#define M2M_DAR_BASE0			0x002c
 102#define M2M_DAR_BASE1			0x0030
 103
 104#define DMA_MAX_CHAN_BYTES		0xffff
 105#define DMA_MAX_CHAN_DESCRIPTORS	32
 106
 107struct ep93xx_dma_engine;
 108static int ep93xx_dma_slave_config_write(struct dma_chan *chan,
 109					 enum dma_transfer_direction dir,
 110					 struct dma_slave_config *config);
 111
 112/**
 113 * struct ep93xx_dma_desc - EP93xx specific transaction descriptor
 114 * @src_addr: source address of the transaction
 115 * @dst_addr: destination address of the transaction
 116 * @size: size of the transaction (in bytes)
 117 * @complete: this descriptor is completed
 118 * @txd: dmaengine API descriptor
 119 * @tx_list: list of linked descriptors
 120 * @node: link used for putting this into a channel queue
 121 */
 122struct ep93xx_dma_desc {
 123	u32				src_addr;
 124	u32				dst_addr;
 125	size_t				size;
 126	bool				complete;
 127	struct dma_async_tx_descriptor	txd;
 128	struct list_head		tx_list;
 129	struct list_head		node;
 130};
 131
 132/**
 133 * struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel
 134 * @chan: dmaengine API channel
 135 * @edma: pointer to the engine device
 136 * @regs: memory mapped registers
 137 * @irq: interrupt number of the channel
 138 * @clk: clock used by this channel
 139 * @tasklet: channel specific tasklet used for callbacks
 140 * @lock: lock protecting the fields following
 141 * @flags: flags for the channel
 142 * @buffer: which buffer to use next (0/1)
 143 * @active: flattened chain of descriptors currently being processed
 144 * @queue: pending descriptors which are handled next
 145 * @free_list: list of free descriptors which can be used
 146 * @runtime_addr: physical address currently used as dest/src (M2M only). This
 147 *                is set via .device_config before slave operation is
 148 *                prepared
 149 * @runtime_ctrl: M2M runtime values for the control register.
 150 * @slave_config: slave configuration
 151 *
 152 * As EP93xx DMA controller doesn't support real chained DMA descriptors we
 153 * will have slightly different scheme here: @active points to a head of
 154 * flattened DMA descriptor chain.
 155 *
 156 * @queue holds pending transactions. These are linked through the first
 157 * descriptor in the chain. When a descriptor is moved to the @active queue,
 158 * the first and chained descriptors are flattened into a single list.
 159 *
 160 * @chan.private holds pointer to &struct ep93xx_dma_data which contains
 161 * necessary channel configuration information. For memcpy channels this must
 162 * be %NULL.
 163 */
 164struct ep93xx_dma_chan {
 165	struct dma_chan			chan;
 166	const struct ep93xx_dma_engine	*edma;
 167	void __iomem			*regs;
 168	int				irq;
 169	struct clk			*clk;
 170	struct tasklet_struct		tasklet;
 171	/* protects the fields following */
 172	spinlock_t			lock;
 173	unsigned long			flags;
 174/* Channel is configured for cyclic transfers */
 175#define EP93XX_DMA_IS_CYCLIC		0
 176
 177	int				buffer;
 178	struct list_head		active;
 179	struct list_head		queue;
 180	struct list_head		free_list;
 181	u32				runtime_addr;
 182	u32				runtime_ctrl;
 183	struct dma_slave_config		slave_config;
 184};
 185
 186/**
 187 * struct ep93xx_dma_engine - the EP93xx DMA engine instance
 188 * @dma_dev: holds the dmaengine device
 189 * @m2m: is this an M2M or M2P device
 190 * @hw_setup: method which sets the channel up for operation
 191 * @hw_synchronize: synchronizes DMA channel termination to current context
 192 * @hw_shutdown: shuts the channel down and flushes whatever is left
 193 * @hw_submit: pushes active descriptor(s) to the hardware
 194 * @hw_interrupt: handle the interrupt
 195 * @num_channels: number of channels for this instance
 196 * @channels: array of channels
 197 *
 198 * There is one instance of this struct for the M2P channels and one for the
 199 * M2M channels. hw_xxx() methods are used to perform operations which are
 200 * different on M2M and M2P channels. These methods are called with channel
 201 * lock held and interrupts disabled so they cannot sleep.
 202 */
 203struct ep93xx_dma_engine {
 204	struct dma_device	dma_dev;
 205	bool			m2m;
 206	int			(*hw_setup)(struct ep93xx_dma_chan *);
 207	void			(*hw_synchronize)(struct ep93xx_dma_chan *);
 208	void			(*hw_shutdown)(struct ep93xx_dma_chan *);
 209	void			(*hw_submit)(struct ep93xx_dma_chan *);
 210	int			(*hw_interrupt)(struct ep93xx_dma_chan *);
 211#define INTERRUPT_UNKNOWN	0
 212#define INTERRUPT_DONE		1
 213#define INTERRUPT_NEXT_BUFFER	2
 214
 215	size_t			num_channels;
 216	struct ep93xx_dma_chan	channels[] __counted_by(num_channels);
 217};
 218
 219static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac)
 220{
 221	return &edmac->chan.dev->device;
 222}
 223
 224static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan)
 225{
 226	return container_of(chan, struct ep93xx_dma_chan, chan);
 227}
 228
 229/**
 230 * ep93xx_dma_set_active - set new active descriptor chain
 231 * @edmac: channel
 232 * @desc: head of the new active descriptor chain
 233 *
 234 * Sets @desc to be the head of the new active descriptor chain. This is the
 235 * chain which is processed next. The active list must be empty before calling
 236 * this function.
 237 *
 238 * Called with @edmac->lock held and interrupts disabled.
 239 */
 240static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac,
 241				  struct ep93xx_dma_desc *desc)
 242{
 243	BUG_ON(!list_empty(&edmac->active));
 244
 245	list_add_tail(&desc->node, &edmac->active);
 246
 247	/* Flatten the @desc->tx_list chain into @edmac->active list */
 248	while (!list_empty(&desc->tx_list)) {
 249		struct ep93xx_dma_desc *d = list_first_entry(&desc->tx_list,
 250			struct ep93xx_dma_desc, node);
 251
 252		/*
 253		 * We copy the callback parameters from the first descriptor
 254		 * to all the chained descriptors. This way we can call the
 255		 * callback without having to find out the first descriptor in
 256		 * the chain. Useful for cyclic transfers.
 257		 */
 258		d->txd.callback = desc->txd.callback;
 259		d->txd.callback_param = desc->txd.callback_param;
 260
 261		list_move_tail(&d->node, &edmac->active);
 262	}
 263}
 264
 265/* Called with @edmac->lock held and interrupts disabled */
 266static struct ep93xx_dma_desc *
 267ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac)
 268{
 269	return list_first_entry_or_null(&edmac->active,
 270					struct ep93xx_dma_desc, node);
 
 
 271}
 272
 273/**
 274 * ep93xx_dma_advance_active - advances to the next active descriptor
 275 * @edmac: channel
 276 *
 277 * Function advances active descriptor to the next in the @edmac->active and
 278 * returns %true if we still have descriptors in the chain to process.
 279 * Otherwise returns %false.
 280 *
 281 * When the channel is in cyclic mode always returns %true.
 282 *
 283 * Called with @edmac->lock held and interrupts disabled.
 284 */
 285static bool ep93xx_dma_advance_active(struct ep93xx_dma_chan *edmac)
 286{
 287	struct ep93xx_dma_desc *desc;
 288
 289	list_rotate_left(&edmac->active);
 290
 291	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
 292		return true;
 293
 294	desc = ep93xx_dma_get_active(edmac);
 295	if (!desc)
 296		return false;
 297
 298	/*
 299	 * If txd.cookie is set it means that we are back in the first
 300	 * descriptor in the chain and hence done with it.
 301	 */
 302	return !desc->txd.cookie;
 303}
 304
 305/*
 306 * M2P DMA implementation
 307 */
 308
 309static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control)
 310{
 311	writel(control, edmac->regs + M2P_CONTROL);
 312	/*
 313	 * EP93xx User's Guide states that we must perform a dummy read after
 314	 * write to the control register.
 315	 */
 316	readl(edmac->regs + M2P_CONTROL);
 317}
 318
 319static int m2p_hw_setup(struct ep93xx_dma_chan *edmac)
 320{
 321	struct ep93xx_dma_data *data = edmac->chan.private;
 322	u32 control;
 323
 324	writel(data->port & 0xf, edmac->regs + M2P_PPALLOC);
 325
 326	control = M2P_CONTROL_CH_ERROR_INT | M2P_CONTROL_ICE
 327		| M2P_CONTROL_ENABLE;
 328	m2p_set_control(edmac, control);
 329
 330	edmac->buffer = 0;
 331
 332	return 0;
 333}
 334
 335static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac)
 336{
 337	return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3;
 338}
 339
 340static void m2p_hw_synchronize(struct ep93xx_dma_chan *edmac)
 341{
 342	unsigned long flags;
 343	u32 control;
 344
 345	spin_lock_irqsave(&edmac->lock, flags);
 346	control = readl(edmac->regs + M2P_CONTROL);
 347	control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
 348	m2p_set_control(edmac, control);
 349	spin_unlock_irqrestore(&edmac->lock, flags);
 350
 351	while (m2p_channel_state(edmac) >= M2P_STATE_ON)
 352		schedule();
 353}
 354
 355static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac)
 356{
 357	m2p_set_control(edmac, 0);
 358
 359	while (m2p_channel_state(edmac) != M2P_STATE_IDLE)
 360		dev_warn(chan2dev(edmac), "M2P: Not yet IDLE\n");
 361}
 362
 363static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)
 364{
 365	struct ep93xx_dma_desc *desc;
 366	u32 bus_addr;
 367
 368	desc = ep93xx_dma_get_active(edmac);
 369	if (!desc) {
 370		dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n");
 371		return;
 372	}
 373
 374	if (ep93xx_dma_chan_direction(&edmac->chan) == DMA_MEM_TO_DEV)
 375		bus_addr = desc->src_addr;
 376	else
 377		bus_addr = desc->dst_addr;
 378
 379	if (edmac->buffer == 0) {
 380		writel(desc->size, edmac->regs + M2P_MAXCNT0);
 381		writel(bus_addr, edmac->regs + M2P_BASE0);
 382	} else {
 383		writel(desc->size, edmac->regs + M2P_MAXCNT1);
 384		writel(bus_addr, edmac->regs + M2P_BASE1);
 385	}
 386
 387	edmac->buffer ^= 1;
 388}
 389
 390static void m2p_hw_submit(struct ep93xx_dma_chan *edmac)
 391{
 392	u32 control = readl(edmac->regs + M2P_CONTROL);
 393
 394	m2p_fill_desc(edmac);
 395	control |= M2P_CONTROL_STALLINT;
 396
 397	if (ep93xx_dma_advance_active(edmac)) {
 398		m2p_fill_desc(edmac);
 399		control |= M2P_CONTROL_NFBINT;
 400	}
 401
 402	m2p_set_control(edmac, control);
 403}
 404
 405static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac)
 406{
 407	u32 irq_status = readl(edmac->regs + M2P_INTERRUPT);
 408	u32 control;
 409
 410	if (irq_status & M2P_INTERRUPT_ERROR) {
 411		struct ep93xx_dma_desc *desc = ep93xx_dma_get_active(edmac);
 412
 413		/* Clear the error interrupt */
 414		writel(1, edmac->regs + M2P_INTERRUPT);
 415
 416		/*
 417		 * It seems that there is no easy way of reporting errors back
 418		 * to client so we just report the error here and continue as
 419		 * usual.
 420		 *
 421		 * Revisit this when there is a mechanism to report back the
 422		 * errors.
 423		 */
 424		dev_err(chan2dev(edmac),
 425			"DMA transfer failed! Details:\n"
 426			"\tcookie	: %d\n"
 427			"\tsrc_addr	: 0x%08x\n"
 428			"\tdst_addr	: 0x%08x\n"
 429			"\tsize		: %zu\n",
 430			desc->txd.cookie, desc->src_addr, desc->dst_addr,
 431			desc->size);
 432	}
 433
 434	/*
 435	 * Even latest E2 silicon revision sometimes assert STALL interrupt
 436	 * instead of NFB. Therefore we treat them equally, basing on the
 437	 * amount of data we still have to transfer.
 438	 */
 439	if (!(irq_status & (M2P_INTERRUPT_STALL | M2P_INTERRUPT_NFB)))
 440		return INTERRUPT_UNKNOWN;
 
 
 
 
 
 441
 442	if (ep93xx_dma_advance_active(edmac)) {
 443		m2p_fill_desc(edmac);
 444		return INTERRUPT_NEXT_BUFFER;
 445	}
 446
 447	/* Disable interrupts */
 448	control = readl(edmac->regs + M2P_CONTROL);
 449	control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
 450	m2p_set_control(edmac, control);
 451
 452	return INTERRUPT_DONE;
 453}
 454
 455/*
 456 * M2M DMA implementation
 457 */
 458
 459static int m2m_hw_setup(struct ep93xx_dma_chan *edmac)
 460{
 461	const struct ep93xx_dma_data *data = edmac->chan.private;
 462	u32 control = 0;
 463
 464	if (!data) {
 465		/* This is memcpy channel, nothing to configure */
 466		writel(control, edmac->regs + M2M_CONTROL);
 467		return 0;
 468	}
 469
 470	switch (data->port) {
 471	case EP93XX_DMA_SSP:
 472		/*
 473		 * This was found via experimenting - anything less than 5
 474		 * causes the channel to perform only a partial transfer which
 475		 * leads to problems since we don't get DONE interrupt then.
 476		 */
 477		control = (5 << M2M_CONTROL_PWSC_SHIFT);
 478		control |= M2M_CONTROL_NO_HDSK;
 479
 480		if (data->direction == DMA_MEM_TO_DEV) {
 481			control |= M2M_CONTROL_DAH;
 482			control |= M2M_CONTROL_TM_TX;
 483			control |= M2M_CONTROL_RSS_SSPTX;
 484		} else {
 485			control |= M2M_CONTROL_SAH;
 486			control |= M2M_CONTROL_TM_RX;
 487			control |= M2M_CONTROL_RSS_SSPRX;
 488		}
 489		break;
 490
 491	case EP93XX_DMA_IDE:
 492		/*
 493		 * This IDE part is totally untested. Values below are taken
 494		 * from the EP93xx Users's Guide and might not be correct.
 495		 */
 496		if (data->direction == DMA_MEM_TO_DEV) {
 497			/* Worst case from the UG */
 498			control = (3 << M2M_CONTROL_PWSC_SHIFT);
 499			control |= M2M_CONTROL_DAH;
 500			control |= M2M_CONTROL_TM_TX;
 501		} else {
 502			control = (2 << M2M_CONTROL_PWSC_SHIFT);
 503			control |= M2M_CONTROL_SAH;
 504			control |= M2M_CONTROL_TM_RX;
 505		}
 506
 507		control |= M2M_CONTROL_NO_HDSK;
 508		control |= M2M_CONTROL_RSS_IDE;
 509		control |= M2M_CONTROL_PW_16;
 510		break;
 511
 512	default:
 513		return -EINVAL;
 514	}
 515
 516	writel(control, edmac->regs + M2M_CONTROL);
 517	return 0;
 518}
 519
 520static void m2m_hw_shutdown(struct ep93xx_dma_chan *edmac)
 521{
 522	/* Just disable the channel */
 523	writel(0, edmac->regs + M2M_CONTROL);
 524}
 525
 526static void m2m_fill_desc(struct ep93xx_dma_chan *edmac)
 527{
 528	struct ep93xx_dma_desc *desc;
 529
 530	desc = ep93xx_dma_get_active(edmac);
 531	if (!desc) {
 532		dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n");
 533		return;
 534	}
 535
 536	if (edmac->buffer == 0) {
 537		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE0);
 538		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE0);
 539		writel(desc->size, edmac->regs + M2M_BCR0);
 540	} else {
 541		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE1);
 542		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE1);
 543		writel(desc->size, edmac->regs + M2M_BCR1);
 544	}
 545
 546	edmac->buffer ^= 1;
 547}
 548
 549static void m2m_hw_submit(struct ep93xx_dma_chan *edmac)
 550{
 551	struct ep93xx_dma_data *data = edmac->chan.private;
 552	u32 control = readl(edmac->regs + M2M_CONTROL);
 553
 554	/*
 555	 * Since we allow clients to configure PW (peripheral width) we always
 556	 * clear PW bits here and then set them according what is given in
 557	 * the runtime configuration.
 558	 */
 559	control &= ~M2M_CONTROL_PW_MASK;
 560	control |= edmac->runtime_ctrl;
 561
 562	m2m_fill_desc(edmac);
 563	control |= M2M_CONTROL_DONEINT;
 564
 565	if (ep93xx_dma_advance_active(edmac)) {
 566		m2m_fill_desc(edmac);
 567		control |= M2M_CONTROL_NFBINT;
 568	}
 569
 570	/*
 571	 * Now we can finally enable the channel. For M2M channel this must be
 572	 * done _after_ the BCRx registers are programmed.
 573	 */
 574	control |= M2M_CONTROL_ENABLE;
 575	writel(control, edmac->regs + M2M_CONTROL);
 576
 577	if (!data) {
 578		/*
 579		 * For memcpy channels the software trigger must be asserted
 580		 * in order to start the memcpy operation.
 581		 */
 582		control |= M2M_CONTROL_START;
 583		writel(control, edmac->regs + M2M_CONTROL);
 584	}
 585}
 586
 587/*
 588 * According to EP93xx User's Guide, we should receive DONE interrupt when all
 589 * M2M DMA controller transactions complete normally. This is not always the
 590 * case - sometimes EP93xx M2M DMA asserts DONE interrupt when the DMA channel
 591 * is still running (channel Buffer FSM in DMA_BUF_ON state, and channel
 592 * Control FSM in DMA_MEM_RD state, observed at least in IDE-DMA operation).
 593 * In effect, disabling the channel when only DONE bit is set could stop
 594 * currently running DMA transfer. To avoid this, we use Buffer FSM and
 595 * Control FSM to check current state of DMA channel.
 596 */
 597static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac)
 598{
 599	u32 status = readl(edmac->regs + M2M_STATUS);
 600	u32 ctl_fsm = status & M2M_STATUS_CTL_MASK;
 601	u32 buf_fsm = status & M2M_STATUS_BUF_MASK;
 602	bool done = status & M2M_STATUS_DONE;
 603	bool last_done;
 604	u32 control;
 605	struct ep93xx_dma_desc *desc;
 606
 607	/* Accept only DONE and NFB interrupts */
 608	if (!(readl(edmac->regs + M2M_INTERRUPT) & M2M_INTERRUPT_MASK))
 609		return INTERRUPT_UNKNOWN;
 610
 611	if (done) {
 612		/* Clear the DONE bit */
 613		writel(0, edmac->regs + M2M_INTERRUPT);
 614	}
 615
 616	/*
 617	 * Check whether we are done with descriptors or not. This, together
 618	 * with DMA channel state, determines action to take in interrupt.
 619	 */
 620	desc = ep93xx_dma_get_active(edmac);
 621	last_done = !desc || desc->txd.cookie;
 622
 623	/*
 624	 * Use M2M DMA Buffer FSM and Control FSM to check current state of
 625	 * DMA channel. Using DONE and NFB bits from channel status register
 626	 * or bits from channel interrupt register is not reliable.
 627	 */
 628	if (!last_done &&
 629	    (buf_fsm == M2M_STATUS_BUF_NO ||
 630	     buf_fsm == M2M_STATUS_BUF_ON)) {
 631		/*
 632		 * Two buffers are ready for update when Buffer FSM is in
 633		 * DMA_NO_BUF state. Only one buffer can be prepared without
 634		 * disabling the channel or polling the DONE bit.
 635		 * To simplify things, always prepare only one buffer.
 636		 */
 637		if (ep93xx_dma_advance_active(edmac)) {
 638			m2m_fill_desc(edmac);
 639			if (done && !edmac->chan.private) {
 640				/* Software trigger for memcpy channel */
 641				control = readl(edmac->regs + M2M_CONTROL);
 642				control |= M2M_CONTROL_START;
 643				writel(control, edmac->regs + M2M_CONTROL);
 644			}
 645			return INTERRUPT_NEXT_BUFFER;
 646		} else {
 647			last_done = true;
 648		}
 649	}
 650
 651	/*
 652	 * Disable the channel only when Buffer FSM is in DMA_NO_BUF state
 653	 * and Control FSM is in DMA_STALL state.
 654	 */
 655	if (last_done &&
 656	    buf_fsm == M2M_STATUS_BUF_NO &&
 657	    ctl_fsm == M2M_STATUS_CTL_STALL) {
 658		/* Disable interrupts and the channel */
 659		control = readl(edmac->regs + M2M_CONTROL);
 660		control &= ~(M2M_CONTROL_DONEINT | M2M_CONTROL_NFBINT
 661			    | M2M_CONTROL_ENABLE);
 662		writel(control, edmac->regs + M2M_CONTROL);
 663		return INTERRUPT_DONE;
 664	}
 665
 666	/*
 667	 * Nothing to do this time.
 668	 */
 669	return INTERRUPT_NEXT_BUFFER;
 670}
 671
 672/*
 673 * DMA engine API implementation
 674 */
 675
 676static struct ep93xx_dma_desc *
 677ep93xx_dma_desc_get(struct ep93xx_dma_chan *edmac)
 678{
 679	struct ep93xx_dma_desc *desc, *_desc;
 680	struct ep93xx_dma_desc *ret = NULL;
 681	unsigned long flags;
 682
 683	spin_lock_irqsave(&edmac->lock, flags);
 684	list_for_each_entry_safe(desc, _desc, &edmac->free_list, node) {
 685		if (async_tx_test_ack(&desc->txd)) {
 686			list_del_init(&desc->node);
 687
 688			/* Re-initialize the descriptor */
 689			desc->src_addr = 0;
 690			desc->dst_addr = 0;
 691			desc->size = 0;
 692			desc->complete = false;
 693			desc->txd.cookie = 0;
 694			desc->txd.callback = NULL;
 695			desc->txd.callback_param = NULL;
 696
 697			ret = desc;
 698			break;
 699		}
 700	}
 701	spin_unlock_irqrestore(&edmac->lock, flags);
 702	return ret;
 703}
 704
 705static void ep93xx_dma_desc_put(struct ep93xx_dma_chan *edmac,
 706				struct ep93xx_dma_desc *desc)
 707{
 708	if (desc) {
 709		unsigned long flags;
 710
 711		spin_lock_irqsave(&edmac->lock, flags);
 712		list_splice_init(&desc->tx_list, &edmac->free_list);
 713		list_add(&desc->node, &edmac->free_list);
 714		spin_unlock_irqrestore(&edmac->lock, flags);
 715	}
 716}
 717
 718/**
 719 * ep93xx_dma_advance_work - start processing the next pending transaction
 720 * @edmac: channel
 721 *
 722 * If we have pending transactions queued and we are currently idling, this
 723 * function takes the next queued transaction from the @edmac->queue and
 724 * pushes it to the hardware for execution.
 725 */
 726static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)
 727{
 728	struct ep93xx_dma_desc *new;
 729	unsigned long flags;
 730
 731	spin_lock_irqsave(&edmac->lock, flags);
 732	if (!list_empty(&edmac->active) || list_empty(&edmac->queue)) {
 733		spin_unlock_irqrestore(&edmac->lock, flags);
 734		return;
 735	}
 736
 737	/* Take the next descriptor from the pending queue */
 738	new = list_first_entry(&edmac->queue, struct ep93xx_dma_desc, node);
 739	list_del_init(&new->node);
 740
 741	ep93xx_dma_set_active(edmac, new);
 742
 743	/* Push it to the hardware */
 744	edmac->edma->hw_submit(edmac);
 745	spin_unlock_irqrestore(&edmac->lock, flags);
 746}
 747
 748static void ep93xx_dma_tasklet(struct tasklet_struct *t)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 749{
 750	struct ep93xx_dma_chan *edmac = from_tasklet(edmac, t, tasklet);
 751	struct ep93xx_dma_desc *desc, *d;
 752	struct dmaengine_desc_callback cb;
 
 753	LIST_HEAD(list);
 754
 755	memset(&cb, 0, sizeof(cb));
 756	spin_lock_irq(&edmac->lock);
 757	/*
 758	 * If dma_terminate_all() was called before we get to run, the active
 759	 * list has become empty. If that happens we aren't supposed to do
 760	 * anything more than call ep93xx_dma_advance_work().
 761	 */
 762	desc = ep93xx_dma_get_active(edmac);
 763	if (desc) {
 764		if (desc->complete) {
 765			/* mark descriptor complete for non cyclic case only */
 766			if (!test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
 767				dma_cookie_complete(&desc->txd);
 768			list_splice_init(&edmac->active, &list);
 769		}
 770		dmaengine_desc_get_callback(&desc->txd, &cb);
 
 771	}
 772	spin_unlock_irq(&edmac->lock);
 773
 774	/* Pick up the next descriptor from the queue */
 775	ep93xx_dma_advance_work(edmac);
 776
 777	/* Now we can release all the chained descriptors */
 778	list_for_each_entry_safe(desc, d, &list, node) {
 779		dma_descriptor_unmap(&desc->txd);
 
 
 
 
 
 
 780		ep93xx_dma_desc_put(edmac, desc);
 781	}
 782
 783	dmaengine_desc_callback_invoke(&cb, NULL);
 
 784}
 785
 786static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id)
 787{
 788	struct ep93xx_dma_chan *edmac = dev_id;
 789	struct ep93xx_dma_desc *desc;
 790	irqreturn_t ret = IRQ_HANDLED;
 791
 792	spin_lock(&edmac->lock);
 793
 794	desc = ep93xx_dma_get_active(edmac);
 795	if (!desc) {
 796		dev_warn(chan2dev(edmac),
 797			 "got interrupt while active list is empty\n");
 798		spin_unlock(&edmac->lock);
 799		return IRQ_NONE;
 800	}
 801
 802	switch (edmac->edma->hw_interrupt(edmac)) {
 803	case INTERRUPT_DONE:
 804		desc->complete = true;
 805		tasklet_schedule(&edmac->tasklet);
 806		break;
 807
 808	case INTERRUPT_NEXT_BUFFER:
 809		if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
 810			tasklet_schedule(&edmac->tasklet);
 811		break;
 812
 813	default:
 814		dev_warn(chan2dev(edmac), "unknown interrupt!\n");
 815		ret = IRQ_NONE;
 816		break;
 817	}
 818
 819	spin_unlock(&edmac->lock);
 820	return ret;
 821}
 822
 823/**
 824 * ep93xx_dma_tx_submit - set the prepared descriptor(s) to be executed
 825 * @tx: descriptor to be executed
 826 *
 827 * Function will execute given descriptor on the hardware or if the hardware
 828 * is busy, queue the descriptor to be executed later on. Returns cookie which
 829 * can be used to poll the status of the descriptor.
 830 */
 831static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 832{
 833	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(tx->chan);
 834	struct ep93xx_dma_desc *desc;
 835	dma_cookie_t cookie;
 836	unsigned long flags;
 837
 838	spin_lock_irqsave(&edmac->lock, flags);
 839	cookie = dma_cookie_assign(tx);
 840
 841	desc = container_of(tx, struct ep93xx_dma_desc, txd);
 842
 843	/*
 844	 * If nothing is currently prosessed, we push this descriptor
 845	 * directly to the hardware. Otherwise we put the descriptor
 846	 * to the pending queue.
 847	 */
 848	if (list_empty(&edmac->active)) {
 849		ep93xx_dma_set_active(edmac, desc);
 850		edmac->edma->hw_submit(edmac);
 851	} else {
 852		list_add_tail(&desc->node, &edmac->queue);
 853	}
 854
 855	spin_unlock_irqrestore(&edmac->lock, flags);
 856	return cookie;
 857}
 858
 859/**
 860 * ep93xx_dma_alloc_chan_resources - allocate resources for the channel
 861 * @chan: channel to allocate resources
 862 *
 863 * Function allocates necessary resources for the given DMA channel and
 864 * returns number of allocated descriptors for the channel. Negative errno
 865 * is returned in case of failure.
 866 */
 867static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan)
 868{
 869	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
 870	struct ep93xx_dma_data *data = chan->private;
 871	const char *name = dma_chan_name(chan);
 872	int ret, i;
 873
 874	/* Sanity check the channel parameters */
 875	if (!edmac->edma->m2m) {
 876		if (!data)
 877			return -EINVAL;
 878		if (data->port < EP93XX_DMA_I2S1 ||
 879		    data->port > EP93XX_DMA_IRDA)
 880			return -EINVAL;
 881		if (data->direction != ep93xx_dma_chan_direction(chan))
 882			return -EINVAL;
 883	} else {
 884		if (data) {
 885			switch (data->port) {
 886			case EP93XX_DMA_SSP:
 887			case EP93XX_DMA_IDE:
 888				if (!is_slave_direction(data->direction))
 
 889					return -EINVAL;
 890				break;
 891			default:
 892				return -EINVAL;
 893			}
 894		}
 895	}
 896
 897	if (data && data->name)
 898		name = data->name;
 899
 900	ret = clk_prepare_enable(edmac->clk);
 901	if (ret)
 902		return ret;
 903
 904	ret = request_irq(edmac->irq, ep93xx_dma_interrupt, 0, name, edmac);
 905	if (ret)
 906		goto fail_clk_disable;
 907
 908	spin_lock_irq(&edmac->lock);
 909	dma_cookie_init(&edmac->chan);
 910	ret = edmac->edma->hw_setup(edmac);
 911	spin_unlock_irq(&edmac->lock);
 912
 913	if (ret)
 914		goto fail_free_irq;
 915
 916	for (i = 0; i < DMA_MAX_CHAN_DESCRIPTORS; i++) {
 917		struct ep93xx_dma_desc *desc;
 918
 919		desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 920		if (!desc) {
 921			dev_warn(chan2dev(edmac), "not enough descriptors\n");
 922			break;
 923		}
 924
 925		INIT_LIST_HEAD(&desc->tx_list);
 926
 927		dma_async_tx_descriptor_init(&desc->txd, chan);
 928		desc->txd.flags = DMA_CTRL_ACK;
 929		desc->txd.tx_submit = ep93xx_dma_tx_submit;
 930
 931		ep93xx_dma_desc_put(edmac, desc);
 932	}
 933
 934	return i;
 935
 936fail_free_irq:
 937	free_irq(edmac->irq, edmac);
 938fail_clk_disable:
 939	clk_disable_unprepare(edmac->clk);
 940
 941	return ret;
 942}
 943
 944/**
 945 * ep93xx_dma_free_chan_resources - release resources for the channel
 946 * @chan: channel
 947 *
 948 * Function releases all the resources allocated for the given channel.
 949 * The channel must be idle when this is called.
 950 */
 951static void ep93xx_dma_free_chan_resources(struct dma_chan *chan)
 952{
 953	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
 954	struct ep93xx_dma_desc *desc, *d;
 955	unsigned long flags;
 956	LIST_HEAD(list);
 957
 958	BUG_ON(!list_empty(&edmac->active));
 959	BUG_ON(!list_empty(&edmac->queue));
 960
 961	spin_lock_irqsave(&edmac->lock, flags);
 962	edmac->edma->hw_shutdown(edmac);
 963	edmac->runtime_addr = 0;
 964	edmac->runtime_ctrl = 0;
 965	edmac->buffer = 0;
 966	list_splice_init(&edmac->free_list, &list);
 967	spin_unlock_irqrestore(&edmac->lock, flags);
 968
 969	list_for_each_entry_safe(desc, d, &list, node)
 970		kfree(desc);
 971
 972	clk_disable_unprepare(edmac->clk);
 973	free_irq(edmac->irq, edmac);
 974}
 975
 976/**
 977 * ep93xx_dma_prep_dma_memcpy - prepare a memcpy DMA operation
 978 * @chan: channel
 979 * @dest: destination bus address
 980 * @src: source bus address
 981 * @len: size of the transaction
 982 * @flags: flags for the descriptor
 983 *
 984 * Returns a valid DMA descriptor or %NULL in case of failure.
 985 */
 986static struct dma_async_tx_descriptor *
 987ep93xx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
 988			   dma_addr_t src, size_t len, unsigned long flags)
 989{
 990	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
 991	struct ep93xx_dma_desc *desc, *first;
 992	size_t bytes, offset;
 993
 994	first = NULL;
 995	for (offset = 0; offset < len; offset += bytes) {
 996		desc = ep93xx_dma_desc_get(edmac);
 997		if (!desc) {
 998			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
 999			goto fail;
1000		}
1001
1002		bytes = min_t(size_t, len - offset, DMA_MAX_CHAN_BYTES);
1003
1004		desc->src_addr = src + offset;
1005		desc->dst_addr = dest + offset;
1006		desc->size = bytes;
1007
1008		if (!first)
1009			first = desc;
1010		else
1011			list_add_tail(&desc->node, &first->tx_list);
1012	}
1013
1014	first->txd.cookie = -EBUSY;
1015	first->txd.flags = flags;
1016
1017	return &first->txd;
1018fail:
1019	ep93xx_dma_desc_put(edmac, first);
1020	return NULL;
1021}
1022
1023/**
1024 * ep93xx_dma_prep_slave_sg - prepare a slave DMA operation
1025 * @chan: channel
1026 * @sgl: list of buffers to transfer
1027 * @sg_len: number of entries in @sgl
1028 * @dir: direction of tha DMA transfer
1029 * @flags: flags for the descriptor
1030 * @context: operation context (ignored)
1031 *
1032 * Returns a valid DMA descriptor or %NULL in case of failure.
1033 */
1034static struct dma_async_tx_descriptor *
1035ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1036			 unsigned int sg_len, enum dma_transfer_direction dir,
1037			 unsigned long flags, void *context)
1038{
1039	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1040	struct ep93xx_dma_desc *desc, *first;
1041	struct scatterlist *sg;
1042	int i;
1043
1044	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
1045		dev_warn(chan2dev(edmac),
1046			 "channel was configured with different direction\n");
1047		return NULL;
1048	}
1049
1050	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
1051		dev_warn(chan2dev(edmac),
1052			 "channel is already used for cyclic transfers\n");
1053		return NULL;
1054	}
1055
1056	ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config);
1057
1058	first = NULL;
1059	for_each_sg(sgl, sg, sg_len, i) {
1060		size_t len = sg_dma_len(sg);
1061
1062		if (len > DMA_MAX_CHAN_BYTES) {
1063			dev_warn(chan2dev(edmac), "too big transfer size %zu\n",
1064				 len);
1065			goto fail;
1066		}
1067
1068		desc = ep93xx_dma_desc_get(edmac);
1069		if (!desc) {
1070			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
1071			goto fail;
1072		}
1073
1074		if (dir == DMA_MEM_TO_DEV) {
1075			desc->src_addr = sg_dma_address(sg);
1076			desc->dst_addr = edmac->runtime_addr;
1077		} else {
1078			desc->src_addr = edmac->runtime_addr;
1079			desc->dst_addr = sg_dma_address(sg);
1080		}
1081		desc->size = len;
1082
1083		if (!first)
1084			first = desc;
1085		else
1086			list_add_tail(&desc->node, &first->tx_list);
1087	}
1088
1089	first->txd.cookie = -EBUSY;
1090	first->txd.flags = flags;
1091
1092	return &first->txd;
1093
1094fail:
1095	ep93xx_dma_desc_put(edmac, first);
1096	return NULL;
1097}
1098
1099/**
1100 * ep93xx_dma_prep_dma_cyclic - prepare a cyclic DMA operation
1101 * @chan: channel
1102 * @dma_addr: DMA mapped address of the buffer
1103 * @buf_len: length of the buffer (in bytes)
1104 * @period_len: length of a single period
1105 * @dir: direction of the operation
1106 * @flags: tx descriptor status flags
1107 *
1108 * Prepares a descriptor for cyclic DMA operation. This means that once the
1109 * descriptor is submitted, we will be submitting in a @period_len sized
1110 * buffers and calling callback once the period has been elapsed. Transfer
1111 * terminates only when client calls dmaengine_terminate_all() for this
1112 * channel.
1113 *
1114 * Returns a valid DMA descriptor or %NULL in case of failure.
1115 */
1116static struct dma_async_tx_descriptor *
1117ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
1118			   size_t buf_len, size_t period_len,
1119			   enum dma_transfer_direction dir, unsigned long flags)
1120{
1121	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1122	struct ep93xx_dma_desc *desc, *first;
1123	size_t offset = 0;
1124
1125	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
1126		dev_warn(chan2dev(edmac),
1127			 "channel was configured with different direction\n");
1128		return NULL;
1129	}
1130
1131	if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
1132		dev_warn(chan2dev(edmac),
1133			 "channel is already used for cyclic transfers\n");
1134		return NULL;
1135	}
1136
1137	if (period_len > DMA_MAX_CHAN_BYTES) {
1138		dev_warn(chan2dev(edmac), "too big period length %zu\n",
1139			 period_len);
1140		return NULL;
1141	}
1142
1143	ep93xx_dma_slave_config_write(chan, dir, &edmac->slave_config);
1144
1145	/* Split the buffer into period size chunks */
1146	first = NULL;
1147	for (offset = 0; offset < buf_len; offset += period_len) {
1148		desc = ep93xx_dma_desc_get(edmac);
1149		if (!desc) {
1150			dev_warn(chan2dev(edmac), "couldn't get descriptor\n");
1151			goto fail;
1152		}
1153
1154		if (dir == DMA_MEM_TO_DEV) {
1155			desc->src_addr = dma_addr + offset;
1156			desc->dst_addr = edmac->runtime_addr;
1157		} else {
1158			desc->src_addr = edmac->runtime_addr;
1159			desc->dst_addr = dma_addr + offset;
1160		}
1161
1162		desc->size = period_len;
1163
1164		if (!first)
1165			first = desc;
1166		else
1167			list_add_tail(&desc->node, &first->tx_list);
1168	}
1169
1170	first->txd.cookie = -EBUSY;
1171
1172	return &first->txd;
1173
1174fail:
1175	ep93xx_dma_desc_put(edmac, first);
1176	return NULL;
1177}
1178
1179/**
1180 * ep93xx_dma_synchronize - Synchronizes the termination of transfers to the
1181 * current context.
1182 * @chan: channel
1183 *
1184 * Synchronizes the DMA channel termination to the current context. When this
1185 * function returns it is guaranteed that all transfers for previously issued
1186 * descriptors have stopped and it is safe to free the memory associated
1187 * with them. Furthermore it is guaranteed that all complete callback functions
1188 * for a previously submitted descriptor have finished running and it is safe to
1189 * free resources accessed from within the complete callbacks.
1190 */
1191static void ep93xx_dma_synchronize(struct dma_chan *chan)
1192{
1193	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1194
1195	if (edmac->edma->hw_synchronize)
1196		edmac->edma->hw_synchronize(edmac);
1197}
1198
1199/**
1200 * ep93xx_dma_terminate_all - terminate all transactions
1201 * @chan: channel
1202 *
1203 * Stops all DMA transactions. All descriptors are put back to the
1204 * @edmac->free_list and callbacks are _not_ called.
1205 */
1206static int ep93xx_dma_terminate_all(struct dma_chan *chan)
1207{
1208	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1209	struct ep93xx_dma_desc *desc, *_d;
1210	unsigned long flags;
1211	LIST_HEAD(list);
1212
1213	spin_lock_irqsave(&edmac->lock, flags);
1214	/* First we disable and flush the DMA channel */
1215	edmac->edma->hw_shutdown(edmac);
1216	clear_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags);
1217	list_splice_init(&edmac->active, &list);
1218	list_splice_init(&edmac->queue, &list);
1219	/*
1220	 * We then re-enable the channel. This way we can continue submitting
1221	 * the descriptors by just calling ->hw_submit() again.
1222	 */
1223	edmac->edma->hw_setup(edmac);
1224	spin_unlock_irqrestore(&edmac->lock, flags);
1225
1226	list_for_each_entry_safe(desc, _d, &list, node)
1227		ep93xx_dma_desc_put(edmac, desc);
1228
1229	return 0;
1230}
1231
1232static int ep93xx_dma_slave_config(struct dma_chan *chan,
1233				   struct dma_slave_config *config)
1234{
1235	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1236
1237	memcpy(&edmac->slave_config, config, sizeof(*config));
1238
1239	return 0;
1240}
1241
1242static int ep93xx_dma_slave_config_write(struct dma_chan *chan,
1243					 enum dma_transfer_direction dir,
1244					 struct dma_slave_config *config)
1245{
1246	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1247	enum dma_slave_buswidth width;
1248	unsigned long flags;
1249	u32 addr, ctrl;
1250
1251	if (!edmac->edma->m2m)
1252		return -EINVAL;
1253
1254	switch (dir) {
1255	case DMA_DEV_TO_MEM:
1256		width = config->src_addr_width;
1257		addr = config->src_addr;
1258		break;
1259
1260	case DMA_MEM_TO_DEV:
1261		width = config->dst_addr_width;
1262		addr = config->dst_addr;
1263		break;
1264
1265	default:
1266		return -EINVAL;
1267	}
1268
1269	switch (width) {
1270	case DMA_SLAVE_BUSWIDTH_1_BYTE:
1271		ctrl = 0;
1272		break;
1273	case DMA_SLAVE_BUSWIDTH_2_BYTES:
1274		ctrl = M2M_CONTROL_PW_16;
1275		break;
1276	case DMA_SLAVE_BUSWIDTH_4_BYTES:
1277		ctrl = M2M_CONTROL_PW_32;
1278		break;
1279	default:
1280		return -EINVAL;
1281	}
1282
1283	spin_lock_irqsave(&edmac->lock, flags);
1284	edmac->runtime_addr = addr;
1285	edmac->runtime_ctrl = ctrl;
1286	spin_unlock_irqrestore(&edmac->lock, flags);
1287
1288	return 0;
1289}
1290
1291/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1292 * ep93xx_dma_tx_status - check if a transaction is completed
1293 * @chan: channel
1294 * @cookie: transaction specific cookie
1295 * @state: state of the transaction is stored here if given
1296 *
1297 * This function can be used to query state of a given transaction.
1298 */
1299static enum dma_status ep93xx_dma_tx_status(struct dma_chan *chan,
1300					    dma_cookie_t cookie,
1301					    struct dma_tx_state *state)
1302{
1303	return dma_cookie_status(chan, cookie, state);
 
 
 
 
 
 
 
 
1304}
1305
1306/**
1307 * ep93xx_dma_issue_pending - push pending transactions to the hardware
1308 * @chan: channel
1309 *
1310 * When this function is called, all pending transactions are pushed to the
1311 * hardware and executed.
1312 */
1313static void ep93xx_dma_issue_pending(struct dma_chan *chan)
1314{
1315	ep93xx_dma_advance_work(to_ep93xx_dma_chan(chan));
1316}
1317
1318static int __init ep93xx_dma_probe(struct platform_device *pdev)
1319{
1320	struct ep93xx_dma_platform_data *pdata = dev_get_platdata(&pdev->dev);
1321	struct ep93xx_dma_engine *edma;
1322	struct dma_device *dma_dev;
 
1323	int ret, i;
1324
1325	edma = kzalloc(struct_size(edma, channels, pdata->num_channels), GFP_KERNEL);
 
1326	if (!edma)
1327		return -ENOMEM;
1328
1329	dma_dev = &edma->dma_dev;
1330	edma->m2m = platform_get_device_id(pdev)->driver_data;
1331	edma->num_channels = pdata->num_channels;
1332
1333	INIT_LIST_HEAD(&dma_dev->channels);
1334	for (i = 0; i < pdata->num_channels; i++) {
1335		const struct ep93xx_dma_chan_data *cdata = &pdata->channels[i];
1336		struct ep93xx_dma_chan *edmac = &edma->channels[i];
1337
1338		edmac->chan.device = dma_dev;
1339		edmac->regs = cdata->base;
1340		edmac->irq = cdata->irq;
1341		edmac->edma = edma;
1342
1343		edmac->clk = clk_get(NULL, cdata->name);
1344		if (IS_ERR(edmac->clk)) {
1345			dev_warn(&pdev->dev, "failed to get clock for %s\n",
1346				 cdata->name);
1347			continue;
1348		}
1349
1350		spin_lock_init(&edmac->lock);
1351		INIT_LIST_HEAD(&edmac->active);
1352		INIT_LIST_HEAD(&edmac->queue);
1353		INIT_LIST_HEAD(&edmac->free_list);
1354		tasklet_setup(&edmac->tasklet, ep93xx_dma_tasklet);
 
1355
1356		list_add_tail(&edmac->chan.device_node,
1357			      &dma_dev->channels);
1358	}
1359
1360	dma_cap_zero(dma_dev->cap_mask);
1361	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1362	dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1363
1364	dma_dev->dev = &pdev->dev;
1365	dma_dev->device_alloc_chan_resources = ep93xx_dma_alloc_chan_resources;
1366	dma_dev->device_free_chan_resources = ep93xx_dma_free_chan_resources;
1367	dma_dev->device_prep_slave_sg = ep93xx_dma_prep_slave_sg;
1368	dma_dev->device_prep_dma_cyclic = ep93xx_dma_prep_dma_cyclic;
1369	dma_dev->device_config = ep93xx_dma_slave_config;
1370	dma_dev->device_synchronize = ep93xx_dma_synchronize;
1371	dma_dev->device_terminate_all = ep93xx_dma_terminate_all;
1372	dma_dev->device_issue_pending = ep93xx_dma_issue_pending;
1373	dma_dev->device_tx_status = ep93xx_dma_tx_status;
1374
1375	dma_set_max_seg_size(dma_dev->dev, DMA_MAX_CHAN_BYTES);
1376
1377	if (edma->m2m) {
1378		dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1379		dma_dev->device_prep_dma_memcpy = ep93xx_dma_prep_dma_memcpy;
1380
1381		edma->hw_setup = m2m_hw_setup;
1382		edma->hw_shutdown = m2m_hw_shutdown;
1383		edma->hw_submit = m2m_hw_submit;
1384		edma->hw_interrupt = m2m_hw_interrupt;
1385	} else {
1386		dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
1387
1388		edma->hw_synchronize = m2p_hw_synchronize;
1389		edma->hw_setup = m2p_hw_setup;
1390		edma->hw_shutdown = m2p_hw_shutdown;
1391		edma->hw_submit = m2p_hw_submit;
1392		edma->hw_interrupt = m2p_hw_interrupt;
1393	}
1394
1395	ret = dma_async_device_register(dma_dev);
1396	if (unlikely(ret)) {
1397		for (i = 0; i < edma->num_channels; i++) {
1398			struct ep93xx_dma_chan *edmac = &edma->channels[i];
1399			if (!IS_ERR_OR_NULL(edmac->clk))
1400				clk_put(edmac->clk);
1401		}
1402		kfree(edma);
1403	} else {
1404		dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n",
1405			 edma->m2m ? "M" : "P");
1406	}
1407
1408	return ret;
1409}
1410
1411static const struct platform_device_id ep93xx_dma_driver_ids[] = {
1412	{ "ep93xx-dma-m2p", 0 },
1413	{ "ep93xx-dma-m2m", 1 },
1414	{ },
1415};
1416
1417static struct platform_driver ep93xx_dma_driver = {
1418	.driver		= {
1419		.name	= "ep93xx-dma",
1420	},
1421	.id_table	= ep93xx_dma_driver_ids,
1422};
1423
1424static int __init ep93xx_dma_module_init(void)
1425{
1426	return platform_driver_probe(&ep93xx_dma_driver, ep93xx_dma_probe);
1427}
1428subsys_initcall(ep93xx_dma_module_init);
1429
1430MODULE_AUTHOR("Mika Westerberg <mika.westerberg@iki.fi>");
1431MODULE_DESCRIPTION("EP93xx DMA driver");
v3.5.6
 
   1/*
   2 * Driver for the Cirrus Logic EP93xx DMA Controller
   3 *
   4 * Copyright (C) 2011 Mika Westerberg
   5 *
   6 * DMA M2P implementation is based on the original
   7 * arch/arm/mach-ep93xx/dma-m2p.c which has following copyrights:
   8 *
   9 *   Copyright (C) 2006 Lennert Buytenhek <buytenh@wantstofly.org>
  10 *   Copyright (C) 2006 Applied Data Systems
  11 *   Copyright (C) 2009 Ryan Mallon <rmallon@gmail.com>
  12 *
  13 * This driver is based on dw_dmac and amba-pl08x drivers.
  14 *
  15 * This program is free software; you can redistribute it and/or modify
  16 * it under the terms of the GNU General Public License as published by
  17 * the Free Software Foundation; either version 2 of the License, or
  18 * (at your option) any later version.
  19 */
  20
  21#include <linux/clk.h>
  22#include <linux/init.h>
  23#include <linux/interrupt.h>
  24#include <linux/dmaengine.h>
  25#include <linux/module.h>
 
  26#include <linux/platform_device.h>
  27#include <linux/slab.h>
  28
  29#include <mach/dma.h>
  30
  31#include "dmaengine.h"
  32
  33/* M2P registers */
  34#define M2P_CONTROL			0x0000
  35#define M2P_CONTROL_STALLINT		BIT(0)
  36#define M2P_CONTROL_NFBINT		BIT(1)
  37#define M2P_CONTROL_CH_ERROR_INT	BIT(3)
  38#define M2P_CONTROL_ENABLE		BIT(4)
  39#define M2P_CONTROL_ICE			BIT(6)
  40
  41#define M2P_INTERRUPT			0x0004
  42#define M2P_INTERRUPT_STALL		BIT(0)
  43#define M2P_INTERRUPT_NFB		BIT(1)
  44#define M2P_INTERRUPT_ERROR		BIT(3)
  45
  46#define M2P_PPALLOC			0x0008
  47#define M2P_STATUS			0x000c
  48
  49#define M2P_MAXCNT0			0x0020
  50#define M2P_BASE0			0x0024
  51#define M2P_MAXCNT1			0x0030
  52#define M2P_BASE1			0x0034
  53
  54#define M2P_STATE_IDLE			0
  55#define M2P_STATE_STALL			1
  56#define M2P_STATE_ON			2
  57#define M2P_STATE_NEXT			3
  58
  59/* M2M registers */
  60#define M2M_CONTROL			0x0000
  61#define M2M_CONTROL_DONEINT		BIT(2)
  62#define M2M_CONTROL_ENABLE		BIT(3)
  63#define M2M_CONTROL_START		BIT(4)
  64#define M2M_CONTROL_DAH			BIT(11)
  65#define M2M_CONTROL_SAH			BIT(12)
  66#define M2M_CONTROL_PW_SHIFT		9
  67#define M2M_CONTROL_PW_8		(0 << M2M_CONTROL_PW_SHIFT)
  68#define M2M_CONTROL_PW_16		(1 << M2M_CONTROL_PW_SHIFT)
  69#define M2M_CONTROL_PW_32		(2 << M2M_CONTROL_PW_SHIFT)
  70#define M2M_CONTROL_PW_MASK		(3 << M2M_CONTROL_PW_SHIFT)
  71#define M2M_CONTROL_TM_SHIFT		13
  72#define M2M_CONTROL_TM_TX		(1 << M2M_CONTROL_TM_SHIFT)
  73#define M2M_CONTROL_TM_RX		(2 << M2M_CONTROL_TM_SHIFT)
  74#define M2M_CONTROL_NFBINT		BIT(21)
  75#define M2M_CONTROL_RSS_SHIFT		22
  76#define M2M_CONTROL_RSS_SSPRX		(1 << M2M_CONTROL_RSS_SHIFT)
  77#define M2M_CONTROL_RSS_SSPTX		(2 << M2M_CONTROL_RSS_SHIFT)
  78#define M2M_CONTROL_RSS_IDE		(3 << M2M_CONTROL_RSS_SHIFT)
  79#define M2M_CONTROL_NO_HDSK		BIT(24)
  80#define M2M_CONTROL_PWSC_SHIFT		25
  81
  82#define M2M_INTERRUPT			0x0004
  83#define M2M_INTERRUPT_MASK		6
  84
  85#define M2M_STATUS			0x000c
  86#define M2M_STATUS_CTL_SHIFT		1
  87#define M2M_STATUS_CTL_IDLE		(0 << M2M_STATUS_CTL_SHIFT)
  88#define M2M_STATUS_CTL_STALL		(1 << M2M_STATUS_CTL_SHIFT)
  89#define M2M_STATUS_CTL_MEMRD		(2 << M2M_STATUS_CTL_SHIFT)
  90#define M2M_STATUS_CTL_MEMWR		(3 << M2M_STATUS_CTL_SHIFT)
  91#define M2M_STATUS_CTL_BWCWAIT		(4 << M2M_STATUS_CTL_SHIFT)
  92#define M2M_STATUS_CTL_MASK		(7 << M2M_STATUS_CTL_SHIFT)
  93#define M2M_STATUS_BUF_SHIFT		4
  94#define M2M_STATUS_BUF_NO		(0 << M2M_STATUS_BUF_SHIFT)
  95#define M2M_STATUS_BUF_ON		(1 << M2M_STATUS_BUF_SHIFT)
  96#define M2M_STATUS_BUF_NEXT		(2 << M2M_STATUS_BUF_SHIFT)
  97#define M2M_STATUS_BUF_MASK		(3 << M2M_STATUS_BUF_SHIFT)
  98#define M2M_STATUS_DONE			BIT(6)
  99
 100#define M2M_BCR0			0x0010
 101#define M2M_BCR1			0x0014
 102#define M2M_SAR_BASE0			0x0018
 103#define M2M_SAR_BASE1			0x001c
 104#define M2M_DAR_BASE0			0x002c
 105#define M2M_DAR_BASE1			0x0030
 106
 107#define DMA_MAX_CHAN_BYTES		0xffff
 108#define DMA_MAX_CHAN_DESCRIPTORS	32
 109
 110struct ep93xx_dma_engine;
 
 
 
 111
 112/**
 113 * struct ep93xx_dma_desc - EP93xx specific transaction descriptor
 114 * @src_addr: source address of the transaction
 115 * @dst_addr: destination address of the transaction
 116 * @size: size of the transaction (in bytes)
 117 * @complete: this descriptor is completed
 118 * @txd: dmaengine API descriptor
 119 * @tx_list: list of linked descriptors
 120 * @node: link used for putting this into a channel queue
 121 */
 122struct ep93xx_dma_desc {
 123	u32				src_addr;
 124	u32				dst_addr;
 125	size_t				size;
 126	bool				complete;
 127	struct dma_async_tx_descriptor	txd;
 128	struct list_head		tx_list;
 129	struct list_head		node;
 130};
 131
 132/**
 133 * struct ep93xx_dma_chan - an EP93xx DMA M2P/M2M channel
 134 * @chan: dmaengine API channel
 135 * @edma: pointer to to the engine device
 136 * @regs: memory mapped registers
 137 * @irq: interrupt number of the channel
 138 * @clk: clock used by this channel
 139 * @tasklet: channel specific tasklet used for callbacks
 140 * @lock: lock protecting the fields following
 141 * @flags: flags for the channel
 142 * @buffer: which buffer to use next (0/1)
 143 * @active: flattened chain of descriptors currently being processed
 144 * @queue: pending descriptors which are handled next
 145 * @free_list: list of free descriptors which can be used
 146 * @runtime_addr: physical address currently used as dest/src (M2M only). This
 147 *                is set via %DMA_SLAVE_CONFIG before slave operation is
 148 *                prepared
 149 * @runtime_ctrl: M2M runtime values for the control register.
 
 150 *
 151 * As EP93xx DMA controller doesn't support real chained DMA descriptors we
 152 * will have slightly different scheme here: @active points to a head of
 153 * flattened DMA descriptor chain.
 154 *
 155 * @queue holds pending transactions. These are linked through the first
 156 * descriptor in the chain. When a descriptor is moved to the @active queue,
 157 * the first and chained descriptors are flattened into a single list.
 158 *
 159 * @chan.private holds pointer to &struct ep93xx_dma_data which contains
 160 * necessary channel configuration information. For memcpy channels this must
 161 * be %NULL.
 162 */
 163struct ep93xx_dma_chan {
 164	struct dma_chan			chan;
 165	const struct ep93xx_dma_engine	*edma;
 166	void __iomem			*regs;
 167	int				irq;
 168	struct clk			*clk;
 169	struct tasklet_struct		tasklet;
 170	/* protects the fields following */
 171	spinlock_t			lock;
 172	unsigned long			flags;
 173/* Channel is configured for cyclic transfers */
 174#define EP93XX_DMA_IS_CYCLIC		0
 175
 176	int				buffer;
 177	struct list_head		active;
 178	struct list_head		queue;
 179	struct list_head		free_list;
 180	u32				runtime_addr;
 181	u32				runtime_ctrl;
 
 182};
 183
 184/**
 185 * struct ep93xx_dma_engine - the EP93xx DMA engine instance
 186 * @dma_dev: holds the dmaengine device
 187 * @m2m: is this an M2M or M2P device
 188 * @hw_setup: method which sets the channel up for operation
 
 189 * @hw_shutdown: shuts the channel down and flushes whatever is left
 190 * @hw_submit: pushes active descriptor(s) to the hardware
 191 * @hw_interrupt: handle the interrupt
 192 * @num_channels: number of channels for this instance
 193 * @channels: array of channels
 194 *
 195 * There is one instance of this struct for the M2P channels and one for the
 196 * M2M channels. hw_xxx() methods are used to perform operations which are
 197 * different on M2M and M2P channels. These methods are called with channel
 198 * lock held and interrupts disabled so they cannot sleep.
 199 */
 200struct ep93xx_dma_engine {
 201	struct dma_device	dma_dev;
 202	bool			m2m;
 203	int			(*hw_setup)(struct ep93xx_dma_chan *);
 
 204	void			(*hw_shutdown)(struct ep93xx_dma_chan *);
 205	void			(*hw_submit)(struct ep93xx_dma_chan *);
 206	int			(*hw_interrupt)(struct ep93xx_dma_chan *);
 207#define INTERRUPT_UNKNOWN	0
 208#define INTERRUPT_DONE		1
 209#define INTERRUPT_NEXT_BUFFER	2
 210
 211	size_t			num_channels;
 212	struct ep93xx_dma_chan	channels[];
 213};
 214
 215static inline struct device *chan2dev(struct ep93xx_dma_chan *edmac)
 216{
 217	return &edmac->chan.dev->device;
 218}
 219
 220static struct ep93xx_dma_chan *to_ep93xx_dma_chan(struct dma_chan *chan)
 221{
 222	return container_of(chan, struct ep93xx_dma_chan, chan);
 223}
 224
 225/**
 226 * ep93xx_dma_set_active - set new active descriptor chain
 227 * @edmac: channel
 228 * @desc: head of the new active descriptor chain
 229 *
 230 * Sets @desc to be the head of the new active descriptor chain. This is the
 231 * chain which is processed next. The active list must be empty before calling
 232 * this function.
 233 *
 234 * Called with @edmac->lock held and interrupts disabled.
 235 */
 236static void ep93xx_dma_set_active(struct ep93xx_dma_chan *edmac,
 237				  struct ep93xx_dma_desc *desc)
 238{
 239	BUG_ON(!list_empty(&edmac->active));
 240
 241	list_add_tail(&desc->node, &edmac->active);
 242
 243	/* Flatten the @desc->tx_list chain into @edmac->active list */
 244	while (!list_empty(&desc->tx_list)) {
 245		struct ep93xx_dma_desc *d = list_first_entry(&desc->tx_list,
 246			struct ep93xx_dma_desc, node);
 247
 248		/*
 249		 * We copy the callback parameters from the first descriptor
 250		 * to all the chained descriptors. This way we can call the
 251		 * callback without having to find out the first descriptor in
 252		 * the chain. Useful for cyclic transfers.
 253		 */
 254		d->txd.callback = desc->txd.callback;
 255		d->txd.callback_param = desc->txd.callback_param;
 256
 257		list_move_tail(&d->node, &edmac->active);
 258	}
 259}
 260
 261/* Called with @edmac->lock held and interrupts disabled */
 262static struct ep93xx_dma_desc *
 263ep93xx_dma_get_active(struct ep93xx_dma_chan *edmac)
 264{
 265	if (list_empty(&edmac->active))
 266		return NULL;
 267
 268	return list_first_entry(&edmac->active, struct ep93xx_dma_desc, node);
 269}
 270
 271/**
 272 * ep93xx_dma_advance_active - advances to the next active descriptor
 273 * @edmac: channel
 274 *
 275 * Function advances active descriptor to the next in the @edmac->active and
 276 * returns %true if we still have descriptors in the chain to process.
 277 * Otherwise returns %false.
 278 *
 279 * When the channel is in cyclic mode always returns %true.
 280 *
 281 * Called with @edmac->lock held and interrupts disabled.
 282 */
 283static bool ep93xx_dma_advance_active(struct ep93xx_dma_chan *edmac)
 284{
 285	struct ep93xx_dma_desc *desc;
 286
 287	list_rotate_left(&edmac->active);
 288
 289	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
 290		return true;
 291
 292	desc = ep93xx_dma_get_active(edmac);
 293	if (!desc)
 294		return false;
 295
 296	/*
 297	 * If txd.cookie is set it means that we are back in the first
 298	 * descriptor in the chain and hence done with it.
 299	 */
 300	return !desc->txd.cookie;
 301}
 302
 303/*
 304 * M2P DMA implementation
 305 */
 306
 307static void m2p_set_control(struct ep93xx_dma_chan *edmac, u32 control)
 308{
 309	writel(control, edmac->regs + M2P_CONTROL);
 310	/*
 311	 * EP93xx User's Guide states that we must perform a dummy read after
 312	 * write to the control register.
 313	 */
 314	readl(edmac->regs + M2P_CONTROL);
 315}
 316
 317static int m2p_hw_setup(struct ep93xx_dma_chan *edmac)
 318{
 319	struct ep93xx_dma_data *data = edmac->chan.private;
 320	u32 control;
 321
 322	writel(data->port & 0xf, edmac->regs + M2P_PPALLOC);
 323
 324	control = M2P_CONTROL_CH_ERROR_INT | M2P_CONTROL_ICE
 325		| M2P_CONTROL_ENABLE;
 326	m2p_set_control(edmac, control);
 327
 
 
 328	return 0;
 329}
 330
 331static inline u32 m2p_channel_state(struct ep93xx_dma_chan *edmac)
 332{
 333	return (readl(edmac->regs + M2P_STATUS) >> 4) & 0x3;
 334}
 335
 336static void m2p_hw_shutdown(struct ep93xx_dma_chan *edmac)
 337{
 
 338	u32 control;
 339
 
 340	control = readl(edmac->regs + M2P_CONTROL);
 341	control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
 342	m2p_set_control(edmac, control);
 
 343
 344	while (m2p_channel_state(edmac) >= M2P_STATE_ON)
 345		cpu_relax();
 
 346
 
 
 347	m2p_set_control(edmac, 0);
 348
 349	while (m2p_channel_state(edmac) == M2P_STATE_STALL)
 350		cpu_relax();
 351}
 352
 353static void m2p_fill_desc(struct ep93xx_dma_chan *edmac)
 354{
 355	struct ep93xx_dma_desc *desc;
 356	u32 bus_addr;
 357
 358	desc = ep93xx_dma_get_active(edmac);
 359	if (!desc) {
 360		dev_warn(chan2dev(edmac), "M2P: empty descriptor list\n");
 361		return;
 362	}
 363
 364	if (ep93xx_dma_chan_direction(&edmac->chan) == DMA_MEM_TO_DEV)
 365		bus_addr = desc->src_addr;
 366	else
 367		bus_addr = desc->dst_addr;
 368
 369	if (edmac->buffer == 0) {
 370		writel(desc->size, edmac->regs + M2P_MAXCNT0);
 371		writel(bus_addr, edmac->regs + M2P_BASE0);
 372	} else {
 373		writel(desc->size, edmac->regs + M2P_MAXCNT1);
 374		writel(bus_addr, edmac->regs + M2P_BASE1);
 375	}
 376
 377	edmac->buffer ^= 1;
 378}
 379
 380static void m2p_hw_submit(struct ep93xx_dma_chan *edmac)
 381{
 382	u32 control = readl(edmac->regs + M2P_CONTROL);
 383
 384	m2p_fill_desc(edmac);
 385	control |= M2P_CONTROL_STALLINT;
 386
 387	if (ep93xx_dma_advance_active(edmac)) {
 388		m2p_fill_desc(edmac);
 389		control |= M2P_CONTROL_NFBINT;
 390	}
 391
 392	m2p_set_control(edmac, control);
 393}
 394
 395static int m2p_hw_interrupt(struct ep93xx_dma_chan *edmac)
 396{
 397	u32 irq_status = readl(edmac->regs + M2P_INTERRUPT);
 398	u32 control;
 399
 400	if (irq_status & M2P_INTERRUPT_ERROR) {
 401		struct ep93xx_dma_desc *desc = ep93xx_dma_get_active(edmac);
 402
 403		/* Clear the error interrupt */
 404		writel(1, edmac->regs + M2P_INTERRUPT);
 405
 406		/*
 407		 * It seems that there is no easy way of reporting errors back
 408		 * to client so we just report the error here and continue as
 409		 * usual.
 410		 *
 411		 * Revisit this when there is a mechanism to report back the
 412		 * errors.
 413		 */
 414		dev_err(chan2dev(edmac),
 415			"DMA transfer failed! Details:\n"
 416			"\tcookie	: %d\n"
 417			"\tsrc_addr	: 0x%08x\n"
 418			"\tdst_addr	: 0x%08x\n"
 419			"\tsize		: %zu\n",
 420			desc->txd.cookie, desc->src_addr, desc->dst_addr,
 421			desc->size);
 422	}
 423
 424	switch (irq_status & (M2P_INTERRUPT_STALL | M2P_INTERRUPT_NFB)) {
 425	case M2P_INTERRUPT_STALL:
 426		/* Disable interrupts */
 427		control = readl(edmac->regs + M2P_CONTROL);
 428		control &= ~(M2P_CONTROL_STALLINT | M2P_CONTROL_NFBINT);
 429		m2p_set_control(edmac, control);
 430
 431		return INTERRUPT_DONE;
 432
 433	case M2P_INTERRUPT_NFB:
 434		if (ep93xx_dma_advance_active(edmac))
 435			m2p_fill_desc(edmac);
 436
 
 
 437		return INTERRUPT_NEXT_BUFFER;
 438	}
 439
 440	return INTERRUPT_UNKNOWN;
 
 
 
 
 
 441}
 442
 443/*
 444 * M2M DMA implementation
 445 */
 446
 447static int m2m_hw_setup(struct ep93xx_dma_chan *edmac)
 448{
 449	const struct ep93xx_dma_data *data = edmac->chan.private;
 450	u32 control = 0;
 451
 452	if (!data) {
 453		/* This is memcpy channel, nothing to configure */
 454		writel(control, edmac->regs + M2M_CONTROL);
 455		return 0;
 456	}
 457
 458	switch (data->port) {
 459	case EP93XX_DMA_SSP:
 460		/*
 461		 * This was found via experimenting - anything less than 5
 462		 * causes the channel to perform only a partial transfer which
 463		 * leads to problems since we don't get DONE interrupt then.
 464		 */
 465		control = (5 << M2M_CONTROL_PWSC_SHIFT);
 466		control |= M2M_CONTROL_NO_HDSK;
 467
 468		if (data->direction == DMA_MEM_TO_DEV) {
 469			control |= M2M_CONTROL_DAH;
 470			control |= M2M_CONTROL_TM_TX;
 471			control |= M2M_CONTROL_RSS_SSPTX;
 472		} else {
 473			control |= M2M_CONTROL_SAH;
 474			control |= M2M_CONTROL_TM_RX;
 475			control |= M2M_CONTROL_RSS_SSPRX;
 476		}
 477		break;
 478
 479	case EP93XX_DMA_IDE:
 480		/*
 481		 * This IDE part is totally untested. Values below are taken
 482		 * from the EP93xx Users's Guide and might not be correct.
 483		 */
 484		if (data->direction == DMA_MEM_TO_DEV) {
 485			/* Worst case from the UG */
 486			control = (3 << M2M_CONTROL_PWSC_SHIFT);
 487			control |= M2M_CONTROL_DAH;
 488			control |= M2M_CONTROL_TM_TX;
 489		} else {
 490			control = (2 << M2M_CONTROL_PWSC_SHIFT);
 491			control |= M2M_CONTROL_SAH;
 492			control |= M2M_CONTROL_TM_RX;
 493		}
 494
 495		control |= M2M_CONTROL_NO_HDSK;
 496		control |= M2M_CONTROL_RSS_IDE;
 497		control |= M2M_CONTROL_PW_16;
 498		break;
 499
 500	default:
 501		return -EINVAL;
 502	}
 503
 504	writel(control, edmac->regs + M2M_CONTROL);
 505	return 0;
 506}
 507
 508static void m2m_hw_shutdown(struct ep93xx_dma_chan *edmac)
 509{
 510	/* Just disable the channel */
 511	writel(0, edmac->regs + M2M_CONTROL);
 512}
 513
 514static void m2m_fill_desc(struct ep93xx_dma_chan *edmac)
 515{
 516	struct ep93xx_dma_desc *desc;
 517
 518	desc = ep93xx_dma_get_active(edmac);
 519	if (!desc) {
 520		dev_warn(chan2dev(edmac), "M2M: empty descriptor list\n");
 521		return;
 522	}
 523
 524	if (edmac->buffer == 0) {
 525		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE0);
 526		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE0);
 527		writel(desc->size, edmac->regs + M2M_BCR0);
 528	} else {
 529		writel(desc->src_addr, edmac->regs + M2M_SAR_BASE1);
 530		writel(desc->dst_addr, edmac->regs + M2M_DAR_BASE1);
 531		writel(desc->size, edmac->regs + M2M_BCR1);
 532	}
 533
 534	edmac->buffer ^= 1;
 535}
 536
 537static void m2m_hw_submit(struct ep93xx_dma_chan *edmac)
 538{
 539	struct ep93xx_dma_data *data = edmac->chan.private;
 540	u32 control = readl(edmac->regs + M2M_CONTROL);
 541
 542	/*
 543	 * Since we allow clients to configure PW (peripheral width) we always
 544	 * clear PW bits here and then set them according what is given in
 545	 * the runtime configuration.
 546	 */
 547	control &= ~M2M_CONTROL_PW_MASK;
 548	control |= edmac->runtime_ctrl;
 549
 550	m2m_fill_desc(edmac);
 551	control |= M2M_CONTROL_DONEINT;
 552
 553	if (ep93xx_dma_advance_active(edmac)) {
 554		m2m_fill_desc(edmac);
 555		control |= M2M_CONTROL_NFBINT;
 556	}
 557
 558	/*
 559	 * Now we can finally enable the channel. For M2M channel this must be
 560	 * done _after_ the BCRx registers are programmed.
 561	 */
 562	control |= M2M_CONTROL_ENABLE;
 563	writel(control, edmac->regs + M2M_CONTROL);
 564
 565	if (!data) {
 566		/*
 567		 * For memcpy channels the software trigger must be asserted
 568		 * in order to start the memcpy operation.
 569		 */
 570		control |= M2M_CONTROL_START;
 571		writel(control, edmac->regs + M2M_CONTROL);
 572	}
 573}
 574
 575/*
 576 * According to EP93xx User's Guide, we should receive DONE interrupt when all
 577 * M2M DMA controller transactions complete normally. This is not always the
 578 * case - sometimes EP93xx M2M DMA asserts DONE interrupt when the DMA channel
 579 * is still running (channel Buffer FSM in DMA_BUF_ON state, and channel
 580 * Control FSM in DMA_MEM_RD state, observed at least in IDE-DMA operation).
 581 * In effect, disabling the channel when only DONE bit is set could stop
 582 * currently running DMA transfer. To avoid this, we use Buffer FSM and
 583 * Control FSM to check current state of DMA channel.
 584 */
 585static int m2m_hw_interrupt(struct ep93xx_dma_chan *edmac)
 586{
 587	u32 status = readl(edmac->regs + M2M_STATUS);
 588	u32 ctl_fsm = status & M2M_STATUS_CTL_MASK;
 589	u32 buf_fsm = status & M2M_STATUS_BUF_MASK;
 590	bool done = status & M2M_STATUS_DONE;
 591	bool last_done;
 592	u32 control;
 593	struct ep93xx_dma_desc *desc;
 594
 595	/* Accept only DONE and NFB interrupts */
 596	if (!(readl(edmac->regs + M2M_INTERRUPT) & M2M_INTERRUPT_MASK))
 597		return INTERRUPT_UNKNOWN;
 598
 599	if (done) {
 600		/* Clear the DONE bit */
 601		writel(0, edmac->regs + M2M_INTERRUPT);
 602	}
 603
 604	/*
 605	 * Check whether we are done with descriptors or not. This, together
 606	 * with DMA channel state, determines action to take in interrupt.
 607	 */
 608	desc = ep93xx_dma_get_active(edmac);
 609	last_done = !desc || desc->txd.cookie;
 610
 611	/*
 612	 * Use M2M DMA Buffer FSM and Control FSM to check current state of
 613	 * DMA channel. Using DONE and NFB bits from channel status register
 614	 * or bits from channel interrupt register is not reliable.
 615	 */
 616	if (!last_done &&
 617	    (buf_fsm == M2M_STATUS_BUF_NO ||
 618	     buf_fsm == M2M_STATUS_BUF_ON)) {
 619		/*
 620		 * Two buffers are ready for update when Buffer FSM is in
 621		 * DMA_NO_BUF state. Only one buffer can be prepared without
 622		 * disabling the channel or polling the DONE bit.
 623		 * To simplify things, always prepare only one buffer.
 624		 */
 625		if (ep93xx_dma_advance_active(edmac)) {
 626			m2m_fill_desc(edmac);
 627			if (done && !edmac->chan.private) {
 628				/* Software trigger for memcpy channel */
 629				control = readl(edmac->regs + M2M_CONTROL);
 630				control |= M2M_CONTROL_START;
 631				writel(control, edmac->regs + M2M_CONTROL);
 632			}
 633			return INTERRUPT_NEXT_BUFFER;
 634		} else {
 635			last_done = true;
 636		}
 637	}
 638
 639	/*
 640	 * Disable the channel only when Buffer FSM is in DMA_NO_BUF state
 641	 * and Control FSM is in DMA_STALL state.
 642	 */
 643	if (last_done &&
 644	    buf_fsm == M2M_STATUS_BUF_NO &&
 645	    ctl_fsm == M2M_STATUS_CTL_STALL) {
 646		/* Disable interrupts and the channel */
 647		control = readl(edmac->regs + M2M_CONTROL);
 648		control &= ~(M2M_CONTROL_DONEINT | M2M_CONTROL_NFBINT
 649			    | M2M_CONTROL_ENABLE);
 650		writel(control, edmac->regs + M2M_CONTROL);
 651		return INTERRUPT_DONE;
 652	}
 653
 654	/*
 655	 * Nothing to do this time.
 656	 */
 657	return INTERRUPT_NEXT_BUFFER;
 658}
 659
 660/*
 661 * DMA engine API implementation
 662 */
 663
 664static struct ep93xx_dma_desc *
 665ep93xx_dma_desc_get(struct ep93xx_dma_chan *edmac)
 666{
 667	struct ep93xx_dma_desc *desc, *_desc;
 668	struct ep93xx_dma_desc *ret = NULL;
 669	unsigned long flags;
 670
 671	spin_lock_irqsave(&edmac->lock, flags);
 672	list_for_each_entry_safe(desc, _desc, &edmac->free_list, node) {
 673		if (async_tx_test_ack(&desc->txd)) {
 674			list_del_init(&desc->node);
 675
 676			/* Re-initialize the descriptor */
 677			desc->src_addr = 0;
 678			desc->dst_addr = 0;
 679			desc->size = 0;
 680			desc->complete = false;
 681			desc->txd.cookie = 0;
 682			desc->txd.callback = NULL;
 683			desc->txd.callback_param = NULL;
 684
 685			ret = desc;
 686			break;
 687		}
 688	}
 689	spin_unlock_irqrestore(&edmac->lock, flags);
 690	return ret;
 691}
 692
 693static void ep93xx_dma_desc_put(struct ep93xx_dma_chan *edmac,
 694				struct ep93xx_dma_desc *desc)
 695{
 696	if (desc) {
 697		unsigned long flags;
 698
 699		spin_lock_irqsave(&edmac->lock, flags);
 700		list_splice_init(&desc->tx_list, &edmac->free_list);
 701		list_add(&desc->node, &edmac->free_list);
 702		spin_unlock_irqrestore(&edmac->lock, flags);
 703	}
 704}
 705
 706/**
 707 * ep93xx_dma_advance_work - start processing the next pending transaction
 708 * @edmac: channel
 709 *
 710 * If we have pending transactions queued and we are currently idling, this
 711 * function takes the next queued transaction from the @edmac->queue and
 712 * pushes it to the hardware for execution.
 713 */
 714static void ep93xx_dma_advance_work(struct ep93xx_dma_chan *edmac)
 715{
 716	struct ep93xx_dma_desc *new;
 717	unsigned long flags;
 718
 719	spin_lock_irqsave(&edmac->lock, flags);
 720	if (!list_empty(&edmac->active) || list_empty(&edmac->queue)) {
 721		spin_unlock_irqrestore(&edmac->lock, flags);
 722		return;
 723	}
 724
 725	/* Take the next descriptor from the pending queue */
 726	new = list_first_entry(&edmac->queue, struct ep93xx_dma_desc, node);
 727	list_del_init(&new->node);
 728
 729	ep93xx_dma_set_active(edmac, new);
 730
 731	/* Push it to the hardware */
 732	edmac->edma->hw_submit(edmac);
 733	spin_unlock_irqrestore(&edmac->lock, flags);
 734}
 735
 736static void ep93xx_dma_unmap_buffers(struct ep93xx_dma_desc *desc)
 737{
 738	struct device *dev = desc->txd.chan->device->dev;
 739
 740	if (!(desc->txd.flags & DMA_COMPL_SKIP_SRC_UNMAP)) {
 741		if (desc->txd.flags & DMA_COMPL_SRC_UNMAP_SINGLE)
 742			dma_unmap_single(dev, desc->src_addr, desc->size,
 743					 DMA_TO_DEVICE);
 744		else
 745			dma_unmap_page(dev, desc->src_addr, desc->size,
 746				       DMA_TO_DEVICE);
 747	}
 748	if (!(desc->txd.flags & DMA_COMPL_SKIP_DEST_UNMAP)) {
 749		if (desc->txd.flags & DMA_COMPL_DEST_UNMAP_SINGLE)
 750			dma_unmap_single(dev, desc->dst_addr, desc->size,
 751					 DMA_FROM_DEVICE);
 752		else
 753			dma_unmap_page(dev, desc->dst_addr, desc->size,
 754				       DMA_FROM_DEVICE);
 755	}
 756}
 757
 758static void ep93xx_dma_tasklet(unsigned long data)
 759{
 760	struct ep93xx_dma_chan *edmac = (struct ep93xx_dma_chan *)data;
 761	struct ep93xx_dma_desc *desc, *d;
 762	dma_async_tx_callback callback = NULL;
 763	void *callback_param = NULL;
 764	LIST_HEAD(list);
 765
 
 766	spin_lock_irq(&edmac->lock);
 767	/*
 768	 * If dma_terminate_all() was called before we get to run, the active
 769	 * list has become empty. If that happens we aren't supposed to do
 770	 * anything more than call ep93xx_dma_advance_work().
 771	 */
 772	desc = ep93xx_dma_get_active(edmac);
 773	if (desc) {
 774		if (desc->complete) {
 775			/* mark descriptor complete for non cyclic case only */
 776			if (!test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
 777				dma_cookie_complete(&desc->txd);
 778			list_splice_init(&edmac->active, &list);
 779		}
 780		callback = desc->txd.callback;
 781		callback_param = desc->txd.callback_param;
 782	}
 783	spin_unlock_irq(&edmac->lock);
 784
 785	/* Pick up the next descriptor from the queue */
 786	ep93xx_dma_advance_work(edmac);
 787
 788	/* Now we can release all the chained descriptors */
 789	list_for_each_entry_safe(desc, d, &list, node) {
 790		/*
 791		 * For the memcpy channels the API requires us to unmap the
 792		 * buffers unless requested otherwise.
 793		 */
 794		if (!edmac->chan.private)
 795			ep93xx_dma_unmap_buffers(desc);
 796
 797		ep93xx_dma_desc_put(edmac, desc);
 798	}
 799
 800	if (callback)
 801		callback(callback_param);
 802}
 803
 804static irqreturn_t ep93xx_dma_interrupt(int irq, void *dev_id)
 805{
 806	struct ep93xx_dma_chan *edmac = dev_id;
 807	struct ep93xx_dma_desc *desc;
 808	irqreturn_t ret = IRQ_HANDLED;
 809
 810	spin_lock(&edmac->lock);
 811
 812	desc = ep93xx_dma_get_active(edmac);
 813	if (!desc) {
 814		dev_warn(chan2dev(edmac),
 815			 "got interrupt while active list is empty\n");
 816		spin_unlock(&edmac->lock);
 817		return IRQ_NONE;
 818	}
 819
 820	switch (edmac->edma->hw_interrupt(edmac)) {
 821	case INTERRUPT_DONE:
 822		desc->complete = true;
 823		tasklet_schedule(&edmac->tasklet);
 824		break;
 825
 826	case INTERRUPT_NEXT_BUFFER:
 827		if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags))
 828			tasklet_schedule(&edmac->tasklet);
 829		break;
 830
 831	default:
 832		dev_warn(chan2dev(edmac), "unknown interrupt!\n");
 833		ret = IRQ_NONE;
 834		break;
 835	}
 836
 837	spin_unlock(&edmac->lock);
 838	return ret;
 839}
 840
 841/**
 842 * ep93xx_dma_tx_submit - set the prepared descriptor(s) to be executed
 843 * @tx: descriptor to be executed
 844 *
 845 * Function will execute given descriptor on the hardware or if the hardware
 846 * is busy, queue the descriptor to be executed later on. Returns cookie which
 847 * can be used to poll the status of the descriptor.
 848 */
 849static dma_cookie_t ep93xx_dma_tx_submit(struct dma_async_tx_descriptor *tx)
 850{
 851	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(tx->chan);
 852	struct ep93xx_dma_desc *desc;
 853	dma_cookie_t cookie;
 854	unsigned long flags;
 855
 856	spin_lock_irqsave(&edmac->lock, flags);
 857	cookie = dma_cookie_assign(tx);
 858
 859	desc = container_of(tx, struct ep93xx_dma_desc, txd);
 860
 861	/*
 862	 * If nothing is currently prosessed, we push this descriptor
 863	 * directly to the hardware. Otherwise we put the descriptor
 864	 * to the pending queue.
 865	 */
 866	if (list_empty(&edmac->active)) {
 867		ep93xx_dma_set_active(edmac, desc);
 868		edmac->edma->hw_submit(edmac);
 869	} else {
 870		list_add_tail(&desc->node, &edmac->queue);
 871	}
 872
 873	spin_unlock_irqrestore(&edmac->lock, flags);
 874	return cookie;
 875}
 876
 877/**
 878 * ep93xx_dma_alloc_chan_resources - allocate resources for the channel
 879 * @chan: channel to allocate resources
 880 *
 881 * Function allocates necessary resources for the given DMA channel and
 882 * returns number of allocated descriptors for the channel. Negative errno
 883 * is returned in case of failure.
 884 */
 885static int ep93xx_dma_alloc_chan_resources(struct dma_chan *chan)
 886{
 887	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
 888	struct ep93xx_dma_data *data = chan->private;
 889	const char *name = dma_chan_name(chan);
 890	int ret, i;
 891
 892	/* Sanity check the channel parameters */
 893	if (!edmac->edma->m2m) {
 894		if (!data)
 895			return -EINVAL;
 896		if (data->port < EP93XX_DMA_I2S1 ||
 897		    data->port > EP93XX_DMA_IRDA)
 898			return -EINVAL;
 899		if (data->direction != ep93xx_dma_chan_direction(chan))
 900			return -EINVAL;
 901	} else {
 902		if (data) {
 903			switch (data->port) {
 904			case EP93XX_DMA_SSP:
 905			case EP93XX_DMA_IDE:
 906				if (data->direction != DMA_MEM_TO_DEV &&
 907				    data->direction != DMA_DEV_TO_MEM)
 908					return -EINVAL;
 909				break;
 910			default:
 911				return -EINVAL;
 912			}
 913		}
 914	}
 915
 916	if (data && data->name)
 917		name = data->name;
 918
 919	ret = clk_enable(edmac->clk);
 920	if (ret)
 921		return ret;
 922
 923	ret = request_irq(edmac->irq, ep93xx_dma_interrupt, 0, name, edmac);
 924	if (ret)
 925		goto fail_clk_disable;
 926
 927	spin_lock_irq(&edmac->lock);
 928	dma_cookie_init(&edmac->chan);
 929	ret = edmac->edma->hw_setup(edmac);
 930	spin_unlock_irq(&edmac->lock);
 931
 932	if (ret)
 933		goto fail_free_irq;
 934
 935	for (i = 0; i < DMA_MAX_CHAN_DESCRIPTORS; i++) {
 936		struct ep93xx_dma_desc *desc;
 937
 938		desc = kzalloc(sizeof(*desc), GFP_KERNEL);
 939		if (!desc) {
 940			dev_warn(chan2dev(edmac), "not enough descriptors\n");
 941			break;
 942		}
 943
 944		INIT_LIST_HEAD(&desc->tx_list);
 945
 946		dma_async_tx_descriptor_init(&desc->txd, chan);
 947		desc->txd.flags = DMA_CTRL_ACK;
 948		desc->txd.tx_submit = ep93xx_dma_tx_submit;
 949
 950		ep93xx_dma_desc_put(edmac, desc);
 951	}
 952
 953	return i;
 954
 955fail_free_irq:
 956	free_irq(edmac->irq, edmac);
 957fail_clk_disable:
 958	clk_disable(edmac->clk);
 959
 960	return ret;
 961}
 962
 963/**
 964 * ep93xx_dma_free_chan_resources - release resources for the channel
 965 * @chan: channel
 966 *
 967 * Function releases all the resources allocated for the given channel.
 968 * The channel must be idle when this is called.
 969 */
 970static void ep93xx_dma_free_chan_resources(struct dma_chan *chan)
 971{
 972	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
 973	struct ep93xx_dma_desc *desc, *d;
 974	unsigned long flags;
 975	LIST_HEAD(list);
 976
 977	BUG_ON(!list_empty(&edmac->active));
 978	BUG_ON(!list_empty(&edmac->queue));
 979
 980	spin_lock_irqsave(&edmac->lock, flags);
 981	edmac->edma->hw_shutdown(edmac);
 982	edmac->runtime_addr = 0;
 983	edmac->runtime_ctrl = 0;
 984	edmac->buffer = 0;
 985	list_splice_init(&edmac->free_list, &list);
 986	spin_unlock_irqrestore(&edmac->lock, flags);
 987
 988	list_for_each_entry_safe(desc, d, &list, node)
 989		kfree(desc);
 990
 991	clk_disable(edmac->clk);
 992	free_irq(edmac->irq, edmac);
 993}
 994
 995/**
 996 * ep93xx_dma_prep_dma_memcpy - prepare a memcpy DMA operation
 997 * @chan: channel
 998 * @dest: destination bus address
 999 * @src: source bus address
1000 * @len: size of the transaction
1001 * @flags: flags for the descriptor
1002 *
1003 * Returns a valid DMA descriptor or %NULL in case of failure.
1004 */
1005static struct dma_async_tx_descriptor *
1006ep93xx_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
1007			   dma_addr_t src, size_t len, unsigned long flags)
1008{
1009	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1010	struct ep93xx_dma_desc *desc, *first;
1011	size_t bytes, offset;
1012
1013	first = NULL;
1014	for (offset = 0; offset < len; offset += bytes) {
1015		desc = ep93xx_dma_desc_get(edmac);
1016		if (!desc) {
1017			dev_warn(chan2dev(edmac), "couln't get descriptor\n");
1018			goto fail;
1019		}
1020
1021		bytes = min_t(size_t, len - offset, DMA_MAX_CHAN_BYTES);
1022
1023		desc->src_addr = src + offset;
1024		desc->dst_addr = dest + offset;
1025		desc->size = bytes;
1026
1027		if (!first)
1028			first = desc;
1029		else
1030			list_add_tail(&desc->node, &first->tx_list);
1031	}
1032
1033	first->txd.cookie = -EBUSY;
1034	first->txd.flags = flags;
1035
1036	return &first->txd;
1037fail:
1038	ep93xx_dma_desc_put(edmac, first);
1039	return NULL;
1040}
1041
1042/**
1043 * ep93xx_dma_prep_slave_sg - prepare a slave DMA operation
1044 * @chan: channel
1045 * @sgl: list of buffers to transfer
1046 * @sg_len: number of entries in @sgl
1047 * @dir: direction of tha DMA transfer
1048 * @flags: flags for the descriptor
1049 * @context: operation context (ignored)
1050 *
1051 * Returns a valid DMA descriptor or %NULL in case of failure.
1052 */
1053static struct dma_async_tx_descriptor *
1054ep93xx_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1055			 unsigned int sg_len, enum dma_transfer_direction dir,
1056			 unsigned long flags, void *context)
1057{
1058	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1059	struct ep93xx_dma_desc *desc, *first;
1060	struct scatterlist *sg;
1061	int i;
1062
1063	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
1064		dev_warn(chan2dev(edmac),
1065			 "channel was configured with different direction\n");
1066		return NULL;
1067	}
1068
1069	if (test_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
1070		dev_warn(chan2dev(edmac),
1071			 "channel is already used for cyclic transfers\n");
1072		return NULL;
1073	}
1074
 
 
1075	first = NULL;
1076	for_each_sg(sgl, sg, sg_len, i) {
1077		size_t sg_len = sg_dma_len(sg);
1078
1079		if (sg_len > DMA_MAX_CHAN_BYTES) {
1080			dev_warn(chan2dev(edmac), "too big transfer size %d\n",
1081				 sg_len);
1082			goto fail;
1083		}
1084
1085		desc = ep93xx_dma_desc_get(edmac);
1086		if (!desc) {
1087			dev_warn(chan2dev(edmac), "couln't get descriptor\n");
1088			goto fail;
1089		}
1090
1091		if (dir == DMA_MEM_TO_DEV) {
1092			desc->src_addr = sg_dma_address(sg);
1093			desc->dst_addr = edmac->runtime_addr;
1094		} else {
1095			desc->src_addr = edmac->runtime_addr;
1096			desc->dst_addr = sg_dma_address(sg);
1097		}
1098		desc->size = sg_len;
1099
1100		if (!first)
1101			first = desc;
1102		else
1103			list_add_tail(&desc->node, &first->tx_list);
1104	}
1105
1106	first->txd.cookie = -EBUSY;
1107	first->txd.flags = flags;
1108
1109	return &first->txd;
1110
1111fail:
1112	ep93xx_dma_desc_put(edmac, first);
1113	return NULL;
1114}
1115
1116/**
1117 * ep93xx_dma_prep_dma_cyclic - prepare a cyclic DMA operation
1118 * @chan: channel
1119 * @dma_addr: DMA mapped address of the buffer
1120 * @buf_len: length of the buffer (in bytes)
1121 * @period_len: lenght of a single period
1122 * @dir: direction of the operation
1123 * @context: operation context (ignored)
1124 *
1125 * Prepares a descriptor for cyclic DMA operation. This means that once the
1126 * descriptor is submitted, we will be submitting in a @period_len sized
1127 * buffers and calling callback once the period has been elapsed. Transfer
1128 * terminates only when client calls dmaengine_terminate_all() for this
1129 * channel.
1130 *
1131 * Returns a valid DMA descriptor or %NULL in case of failure.
1132 */
1133static struct dma_async_tx_descriptor *
1134ep93xx_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t dma_addr,
1135			   size_t buf_len, size_t period_len,
1136			   enum dma_transfer_direction dir, void *context)
1137{
1138	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1139	struct ep93xx_dma_desc *desc, *first;
1140	size_t offset = 0;
1141
1142	if (!edmac->edma->m2m && dir != ep93xx_dma_chan_direction(chan)) {
1143		dev_warn(chan2dev(edmac),
1144			 "channel was configured with different direction\n");
1145		return NULL;
1146	}
1147
1148	if (test_and_set_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags)) {
1149		dev_warn(chan2dev(edmac),
1150			 "channel is already used for cyclic transfers\n");
1151		return NULL;
1152	}
1153
1154	if (period_len > DMA_MAX_CHAN_BYTES) {
1155		dev_warn(chan2dev(edmac), "too big period length %d\n",
1156			 period_len);
1157		return NULL;
1158	}
1159
 
 
1160	/* Split the buffer into period size chunks */
1161	first = NULL;
1162	for (offset = 0; offset < buf_len; offset += period_len) {
1163		desc = ep93xx_dma_desc_get(edmac);
1164		if (!desc) {
1165			dev_warn(chan2dev(edmac), "couln't get descriptor\n");
1166			goto fail;
1167		}
1168
1169		if (dir == DMA_MEM_TO_DEV) {
1170			desc->src_addr = dma_addr + offset;
1171			desc->dst_addr = edmac->runtime_addr;
1172		} else {
1173			desc->src_addr = edmac->runtime_addr;
1174			desc->dst_addr = dma_addr + offset;
1175		}
1176
1177		desc->size = period_len;
1178
1179		if (!first)
1180			first = desc;
1181		else
1182			list_add_tail(&desc->node, &first->tx_list);
1183	}
1184
1185	first->txd.cookie = -EBUSY;
1186
1187	return &first->txd;
1188
1189fail:
1190	ep93xx_dma_desc_put(edmac, first);
1191	return NULL;
1192}
1193
1194/**
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1195 * ep93xx_dma_terminate_all - terminate all transactions
1196 * @edmac: channel
1197 *
1198 * Stops all DMA transactions. All descriptors are put back to the
1199 * @edmac->free_list and callbacks are _not_ called.
1200 */
1201static int ep93xx_dma_terminate_all(struct ep93xx_dma_chan *edmac)
1202{
 
1203	struct ep93xx_dma_desc *desc, *_d;
1204	unsigned long flags;
1205	LIST_HEAD(list);
1206
1207	spin_lock_irqsave(&edmac->lock, flags);
1208	/* First we disable and flush the DMA channel */
1209	edmac->edma->hw_shutdown(edmac);
1210	clear_bit(EP93XX_DMA_IS_CYCLIC, &edmac->flags);
1211	list_splice_init(&edmac->active, &list);
1212	list_splice_init(&edmac->queue, &list);
1213	/*
1214	 * We then re-enable the channel. This way we can continue submitting
1215	 * the descriptors by just calling ->hw_submit() again.
1216	 */
1217	edmac->edma->hw_setup(edmac);
1218	spin_unlock_irqrestore(&edmac->lock, flags);
1219
1220	list_for_each_entry_safe(desc, _d, &list, node)
1221		ep93xx_dma_desc_put(edmac, desc);
1222
1223	return 0;
1224}
1225
1226static int ep93xx_dma_slave_config(struct ep93xx_dma_chan *edmac,
1227				   struct dma_slave_config *config)
1228{
 
 
 
 
 
 
 
 
 
 
 
 
1229	enum dma_slave_buswidth width;
1230	unsigned long flags;
1231	u32 addr, ctrl;
1232
1233	if (!edmac->edma->m2m)
1234		return -EINVAL;
1235
1236	switch (config->direction) {
1237	case DMA_DEV_TO_MEM:
1238		width = config->src_addr_width;
1239		addr = config->src_addr;
1240		break;
1241
1242	case DMA_MEM_TO_DEV:
1243		width = config->dst_addr_width;
1244		addr = config->dst_addr;
1245		break;
1246
1247	default:
1248		return -EINVAL;
1249	}
1250
1251	switch (width) {
1252	case DMA_SLAVE_BUSWIDTH_1_BYTE:
1253		ctrl = 0;
1254		break;
1255	case DMA_SLAVE_BUSWIDTH_2_BYTES:
1256		ctrl = M2M_CONTROL_PW_16;
1257		break;
1258	case DMA_SLAVE_BUSWIDTH_4_BYTES:
1259		ctrl = M2M_CONTROL_PW_32;
1260		break;
1261	default:
1262		return -EINVAL;
1263	}
1264
1265	spin_lock_irqsave(&edmac->lock, flags);
1266	edmac->runtime_addr = addr;
1267	edmac->runtime_ctrl = ctrl;
1268	spin_unlock_irqrestore(&edmac->lock, flags);
1269
1270	return 0;
1271}
1272
1273/**
1274 * ep93xx_dma_control - manipulate all pending operations on a channel
1275 * @chan: channel
1276 * @cmd: control command to perform
1277 * @arg: optional argument
1278 *
1279 * Controls the channel. Function returns %0 in case of success or negative
1280 * error in case of failure.
1281 */
1282static int ep93xx_dma_control(struct dma_chan *chan, enum dma_ctrl_cmd cmd,
1283			      unsigned long arg)
1284{
1285	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1286	struct dma_slave_config *config;
1287
1288	switch (cmd) {
1289	case DMA_TERMINATE_ALL:
1290		return ep93xx_dma_terminate_all(edmac);
1291
1292	case DMA_SLAVE_CONFIG:
1293		config = (struct dma_slave_config *)arg;
1294		return ep93xx_dma_slave_config(edmac, config);
1295
1296	default:
1297		break;
1298	}
1299
1300	return -ENOSYS;
1301}
1302
1303/**
1304 * ep93xx_dma_tx_status - check if a transaction is completed
1305 * @chan: channel
1306 * @cookie: transaction specific cookie
1307 * @state: state of the transaction is stored here if given
1308 *
1309 * This function can be used to query state of a given transaction.
1310 */
1311static enum dma_status ep93xx_dma_tx_status(struct dma_chan *chan,
1312					    dma_cookie_t cookie,
1313					    struct dma_tx_state *state)
1314{
1315	struct ep93xx_dma_chan *edmac = to_ep93xx_dma_chan(chan);
1316	enum dma_status ret;
1317	unsigned long flags;
1318
1319	spin_lock_irqsave(&edmac->lock, flags);
1320	ret = dma_cookie_status(chan, cookie, state);
1321	spin_unlock_irqrestore(&edmac->lock, flags);
1322
1323	return ret;
1324}
1325
1326/**
1327 * ep93xx_dma_issue_pending - push pending transactions to the hardware
1328 * @chan: channel
1329 *
1330 * When this function is called, all pending transactions are pushed to the
1331 * hardware and executed.
1332 */
1333static void ep93xx_dma_issue_pending(struct dma_chan *chan)
1334{
1335	ep93xx_dma_advance_work(to_ep93xx_dma_chan(chan));
1336}
1337
1338static int __init ep93xx_dma_probe(struct platform_device *pdev)
1339{
1340	struct ep93xx_dma_platform_data *pdata = dev_get_platdata(&pdev->dev);
1341	struct ep93xx_dma_engine *edma;
1342	struct dma_device *dma_dev;
1343	size_t edma_size;
1344	int ret, i;
1345
1346	edma_size = pdata->num_channels * sizeof(struct ep93xx_dma_chan);
1347	edma = kzalloc(sizeof(*edma) + edma_size, GFP_KERNEL);
1348	if (!edma)
1349		return -ENOMEM;
1350
1351	dma_dev = &edma->dma_dev;
1352	edma->m2m = platform_get_device_id(pdev)->driver_data;
1353	edma->num_channels = pdata->num_channels;
1354
1355	INIT_LIST_HEAD(&dma_dev->channels);
1356	for (i = 0; i < pdata->num_channels; i++) {
1357		const struct ep93xx_dma_chan_data *cdata = &pdata->channels[i];
1358		struct ep93xx_dma_chan *edmac = &edma->channels[i];
1359
1360		edmac->chan.device = dma_dev;
1361		edmac->regs = cdata->base;
1362		edmac->irq = cdata->irq;
1363		edmac->edma = edma;
1364
1365		edmac->clk = clk_get(NULL, cdata->name);
1366		if (IS_ERR(edmac->clk)) {
1367			dev_warn(&pdev->dev, "failed to get clock for %s\n",
1368				 cdata->name);
1369			continue;
1370		}
1371
1372		spin_lock_init(&edmac->lock);
1373		INIT_LIST_HEAD(&edmac->active);
1374		INIT_LIST_HEAD(&edmac->queue);
1375		INIT_LIST_HEAD(&edmac->free_list);
1376		tasklet_init(&edmac->tasklet, ep93xx_dma_tasklet,
1377			     (unsigned long)edmac);
1378
1379		list_add_tail(&edmac->chan.device_node,
1380			      &dma_dev->channels);
1381	}
1382
1383	dma_cap_zero(dma_dev->cap_mask);
1384	dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
1385	dma_cap_set(DMA_CYCLIC, dma_dev->cap_mask);
1386
1387	dma_dev->dev = &pdev->dev;
1388	dma_dev->device_alloc_chan_resources = ep93xx_dma_alloc_chan_resources;
1389	dma_dev->device_free_chan_resources = ep93xx_dma_free_chan_resources;
1390	dma_dev->device_prep_slave_sg = ep93xx_dma_prep_slave_sg;
1391	dma_dev->device_prep_dma_cyclic = ep93xx_dma_prep_dma_cyclic;
1392	dma_dev->device_control = ep93xx_dma_control;
 
 
1393	dma_dev->device_issue_pending = ep93xx_dma_issue_pending;
1394	dma_dev->device_tx_status = ep93xx_dma_tx_status;
1395
1396	dma_set_max_seg_size(dma_dev->dev, DMA_MAX_CHAN_BYTES);
1397
1398	if (edma->m2m) {
1399		dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
1400		dma_dev->device_prep_dma_memcpy = ep93xx_dma_prep_dma_memcpy;
1401
1402		edma->hw_setup = m2m_hw_setup;
1403		edma->hw_shutdown = m2m_hw_shutdown;
1404		edma->hw_submit = m2m_hw_submit;
1405		edma->hw_interrupt = m2m_hw_interrupt;
1406	} else {
1407		dma_cap_set(DMA_PRIVATE, dma_dev->cap_mask);
1408
 
1409		edma->hw_setup = m2p_hw_setup;
1410		edma->hw_shutdown = m2p_hw_shutdown;
1411		edma->hw_submit = m2p_hw_submit;
1412		edma->hw_interrupt = m2p_hw_interrupt;
1413	}
1414
1415	ret = dma_async_device_register(dma_dev);
1416	if (unlikely(ret)) {
1417		for (i = 0; i < edma->num_channels; i++) {
1418			struct ep93xx_dma_chan *edmac = &edma->channels[i];
1419			if (!IS_ERR_OR_NULL(edmac->clk))
1420				clk_put(edmac->clk);
1421		}
1422		kfree(edma);
1423	} else {
1424		dev_info(dma_dev->dev, "EP93xx M2%s DMA ready\n",
1425			 edma->m2m ? "M" : "P");
1426	}
1427
1428	return ret;
1429}
1430
1431static struct platform_device_id ep93xx_dma_driver_ids[] = {
1432	{ "ep93xx-dma-m2p", 0 },
1433	{ "ep93xx-dma-m2m", 1 },
1434	{ },
1435};
1436
1437static struct platform_driver ep93xx_dma_driver = {
1438	.driver		= {
1439		.name	= "ep93xx-dma",
1440	},
1441	.id_table	= ep93xx_dma_driver_ids,
1442};
1443
1444static int __init ep93xx_dma_module_init(void)
1445{
1446	return platform_driver_probe(&ep93xx_dma_driver, ep93xx_dma_probe);
1447}
1448subsys_initcall(ep93xx_dma_module_init);
1449
1450MODULE_AUTHOR("Mika Westerberg <mika.westerberg@iki.fi>");
1451MODULE_DESCRIPTION("EP93xx DMA driver");
1452MODULE_LICENSE("GPL");