1/*
   2 * Driver for the Atmel AHB DMA Controller (aka HDMA or DMAC on AT91 systems)
   3 *
   4 * Copyright (C) 2008 Atmel Corporation
   5 *
   6 * This program is free software; you can redistribute it and/or modify
   7 * it under the terms of the GNU General Public License as published by
   8 * the Free Software Foundation; either version 2 of the License, or
   9 * (at your option) any later version.
  10 *
  11 *
  12 * This supports the Atmel AHB DMA Controller found in several Atmel SoCs.
  13 * The only Atmel DMA Controller that is not covered by this driver is the one
  14 * found on AT91SAM9263.
  15 */
  16
  17#include <dt-bindings/dma/at91.h>
  18#include <linux/clk.h>
  19#include <linux/dmaengine.h>
  20#include <linux/dma-mapping.h>
  21#include <linux/dmapool.h>
  22#include <linux/interrupt.h>
  23#include <linux/module.h>
  24#include <linux/platform_device.h>
  25#include <linux/slab.h>
  26#include <linux/of.h>
  27#include <linux/of_device.h>
  28#include <linux/of_dma.h>
  29
  30#include "at_hdmac_regs.h"
  31#include "dmaengine.h"
  32
  33/*
  34 * Glossary
  35 * --------
  36 *
  37 * at_hdmac		: Name of the ATmel AHB DMA Controller
  38 * at_dma_ / atdma	: ATmel DMA controller entity related
  39 * atc_	/ atchan	: ATmel DMA Channel entity related
  40 */
  41
  42#define	ATC_DEFAULT_CFG		(ATC_FIFOCFG_HALFFIFO)
  43#define	ATC_DEFAULT_CTRLB	(ATC_SIF(AT_DMA_MEM_IF) \
  44				|ATC_DIF(AT_DMA_MEM_IF))
  45#define ATC_DMA_BUSWIDTHS\
  46	(BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\
  47	BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\
  48	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |\
  49	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
  50
  51#define ATC_MAX_DSCR_TRIALS	10
  52
  53/*
  54 * Initial number of descriptors to allocate for each channel. This could
  55 * be increased during dma usage.
  56 */
  57static unsigned int init_nr_desc_per_channel = 64;
  58module_param(init_nr_desc_per_channel, uint, 0644);
  59MODULE_PARM_DESC(init_nr_desc_per_channel,
  60		 "initial descriptors per channel (default: 64)");
  61
  62
  63/* prototypes */
  64static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx);
  65static void atc_issue_pending(struct dma_chan *chan);
  66
  67
  68/*----------------------------------------------------------------------*/
  69
  70static inline unsigned int atc_get_xfer_width(dma_addr_t src, dma_addr_t dst,
  71						size_t len)
  72{
  73	unsigned int width;
  74
  75	if (!((src | dst  | len) & 3))
  76		width = 2;
  77	else if (!((src | dst | len) & 1))
  78		width = 1;
  79	else
  80		width = 0;
  81
  82	return width;
  83}
  84
  85static struct at_desc *atc_first_active(struct at_dma_chan *atchan)
  86{
  87	return list_first_entry(&atchan->active_list,
  88				struct at_desc, desc_node);
  89}
  90
  91static struct at_desc *atc_first_queued(struct at_dma_chan *atchan)
  92{
  93	return list_first_entry(&atchan->queue,
  94				struct at_desc, desc_node);
  95}
  96
  97/**
  98 * atc_alloc_descriptor - allocate and return an initialized descriptor
  99 * @chan: the channel to allocate descriptors for
 100 * @gfp_flags: GFP allocation flags
 101 *
 102 * Note: The ack-bit is positioned in the descriptor flag at creation time
 103 *       to make initial allocation more convenient. This bit will be cleared
 104 *       and control will be given to client at usage time (during
 105 *       preparation functions).
 106 */
 107static struct at_desc *atc_alloc_descriptor(struct dma_chan *chan,
 108					    gfp_t gfp_flags)
 109{
 110	struct at_desc	*desc = NULL;
 111	struct at_dma	*atdma = to_at_dma(chan->device);
 112	dma_addr_t phys;
 113
 114	desc = dma_pool_alloc(atdma->dma_desc_pool, gfp_flags, &phys);
 115	if (desc) {
 116		memset(desc, 0, sizeof(struct at_desc));
 117		INIT_LIST_HEAD(&desc->tx_list);
 118		dma_async_tx_descriptor_init(&desc->txd, chan);
 119		/* txd.flags will be overwritten in prep functions */
 120		desc->txd.flags = DMA_CTRL_ACK;
 121		desc->txd.tx_submit = atc_tx_submit;
 122		desc->txd.phys = phys;
 123	}
 124
 125	return desc;
 126}
 127
 128/**
 129 * atc_desc_get - get an unused descriptor from free_list
 130 * @atchan: channel we want a new descriptor for
 131 */
 132static struct at_desc *atc_desc_get(struct at_dma_chan *atchan)
 133{
 134	struct at_desc *desc, *_desc;
 135	struct at_desc *ret = NULL;
 136	unsigned long flags;
 137	unsigned int i = 0;
 138	LIST_HEAD(tmp_list);
 139
 140	spin_lock_irqsave(&atchan->lock, flags);
 141	list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
 142		i++;
 143		if (async_tx_test_ack(&desc->txd)) {
 144			list_del(&desc->desc_node);
 145			ret = desc;
 146			break;
 147		}
 148		dev_dbg(chan2dev(&atchan->chan_common),
 149				"desc %p not ACKed\n", desc);
 150	}
 151	spin_unlock_irqrestore(&atchan->lock, flags);
 152	dev_vdbg(chan2dev(&atchan->chan_common),
 153		"scanned %u descriptors on freelist\n", i);
 154
 155	/* no more descriptor available in initial pool: create one more */
 156	if (!ret) {
 157		ret = atc_alloc_descriptor(&atchan->chan_common, GFP_ATOMIC);
 158		if (ret) {
 159			spin_lock_irqsave(&atchan->lock, flags);
 160			atchan->descs_allocated++;
 161			spin_unlock_irqrestore(&atchan->lock, flags);
 162		} else {
 163			dev_err(chan2dev(&atchan->chan_common),
 164					"not enough descriptors available\n");
 165		}
 166	}
 167
 168	return ret;
 169}
 170
 171/**
 172 * atc_desc_put - move a descriptor, including any children, to the free list
 173 * @atchan: channel we work on
 174 * @desc: descriptor, at the head of a chain, to move to free list
 175 */
 176static void atc_desc_put(struct at_dma_chan *atchan, struct at_desc *desc)
 177{
 178	if (desc) {
 179		struct at_desc *child;
 180		unsigned long flags;
 181
 182		spin_lock_irqsave(&atchan->lock, flags);
 183		list_for_each_entry(child, &desc->tx_list, desc_node)
 184			dev_vdbg(chan2dev(&atchan->chan_common),
 185					"moving child desc %p to freelist\n",
 186					child);
 187		list_splice_init(&desc->tx_list, &atchan->free_list);
 188		dev_vdbg(chan2dev(&atchan->chan_common),
 189			 "moving desc %p to freelist\n", desc);
 190		list_add(&desc->desc_node, &atchan->free_list);
 191		spin_unlock_irqrestore(&atchan->lock, flags);
 192	}
 193}
 194
 195/**
 196 * atc_desc_chain - build chain adding a descriptor
 197 * @first: address of first descriptor of the chain
 198 * @prev: address of previous descriptor of the chain
 199 * @desc: descriptor to queue
 200 *
 201 * Called from prep_* functions
 202 */
 203static void atc_desc_chain(struct at_desc **first, struct at_desc **prev,
 204			   struct at_desc *desc)
 205{
 206	if (!(*first)) {
 207		*first = desc;
 208	} else {
 209		/* inform the HW lli about chaining */
 210		(*prev)->lli.dscr = desc->txd.phys;
 211		/* insert the link descriptor to the LD ring */
 212		list_add_tail(&desc->desc_node,
 213				&(*first)->tx_list);
 214	}
 215	*prev = desc;
 216}
 217
 218/**
 219 * atc_dostart - starts the DMA engine for real
 220 * @atchan: the channel we want to start
 221 * @first: first descriptor in the list we want to begin with
 222 *
 223 * Called with atchan->lock held and bh disabled
 224 */
 225static void atc_dostart(struct at_dma_chan *atchan, struct at_desc *first)
 226{
 227	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);
 228
 229	/* ASSERT:  channel is idle */
 230	if (atc_chan_is_enabled(atchan)) {
 231		dev_err(chan2dev(&atchan->chan_common),
 232			"BUG: Attempted to start non-idle channel\n");
 233		dev_err(chan2dev(&atchan->chan_common),
 234			"  channel: s0x%x d0x%x ctrl0x%x:0x%x l0x%x\n",
 235			channel_readl(atchan, SADDR),
 236			channel_readl(atchan, DADDR),
 237			channel_readl(atchan, CTRLA),
 238			channel_readl(atchan, CTRLB),
 239			channel_readl(atchan, DSCR));
 240
 241		/* The tasklet will hopefully advance the queue... */
 242		return;
 243	}
 244
 245	vdbg_dump_regs(atchan);
 246
 247	channel_writel(atchan, SADDR, 0);
 248	channel_writel(atchan, DADDR, 0);
 249	channel_writel(atchan, CTRLA, 0);
 250	channel_writel(atchan, CTRLB, 0);
 251	channel_writel(atchan, DSCR, first->txd.phys);
 252	channel_writel(atchan, SPIP, ATC_SPIP_HOLE(first->src_hole) |
 253		       ATC_SPIP_BOUNDARY(first->boundary));
 254	channel_writel(atchan, DPIP, ATC_DPIP_HOLE(first->dst_hole) |
 255		       ATC_DPIP_BOUNDARY(first->boundary));
 256	dma_writel(atdma, CHER, atchan->mask);
 257
 258	vdbg_dump_regs(atchan);
 259}
 260
 261/*
 262 * atc_get_desc_by_cookie - get the descriptor of a cookie
 263 * @atchan: the DMA channel
 264 * @cookie: the cookie to get the descriptor for
 265 */
 266static struct at_desc *atc_get_desc_by_cookie(struct at_dma_chan *atchan,
 267						dma_cookie_t cookie)
 268{
 269	struct at_desc *desc, *_desc;
 270
 271	list_for_each_entry_safe(desc, _desc, &atchan->queue, desc_node) {
 272		if (desc->txd.cookie == cookie)
 273			return desc;
 274	}
 275
 276	list_for_each_entry_safe(desc, _desc, &atchan->active_list, desc_node) {
 277		if (desc->txd.cookie == cookie)
 278			return desc;
 279	}
 280
 281	return NULL;
 282}
 283
 284/**
 285 * atc_calc_bytes_left - calculates the number of bytes left according to the
 286 * value read from CTRLA.
 287 *
 288 * @current_len: the number of bytes left before reading CTRLA
 289 * @ctrla: the value of CTRLA
 290 */
 291static inline int atc_calc_bytes_left(int current_len, u32 ctrla)
 292{
 293	u32 btsize = (ctrla & ATC_BTSIZE_MAX);
 294	u32 src_width = ATC_REG_TO_SRC_WIDTH(ctrla);
 295
 296	/*
 297	 * According to the datasheet, when reading the Control A Register
 298	 * (ctrla), the Buffer Transfer Size (btsize) bitfield refers to the
 299	 * number of transfers completed on the Source Interface.
 300	 * So btsize is always a number of source width transfers.
 301	 */
 302	return current_len - (btsize << src_width);
 303}
 304
 305/**
 306 * atc_get_bytes_left - get the number of bytes residue for a cookie
 307 * @chan: DMA channel
 308 * @cookie: transaction identifier to check status of
 309 */
 310static int atc_get_bytes_left(struct dma_chan *chan, dma_cookie_t cookie)
 311{
 312	struct at_dma_chan      *atchan = to_at_dma_chan(chan);
 313	struct at_desc *desc_first = atc_first_active(atchan);
 314	struct at_desc *desc;
 315	int ret;
 316	u32 ctrla, dscr, trials;
 317
 318	/*
 319	 * If the cookie doesn't match to the currently running transfer then
 320	 * we can return the total length of the associated DMA transfer,
 321	 * because it is still queued.
 322	 */
 323	desc = atc_get_desc_by_cookie(atchan, cookie);
 324	if (desc == NULL)
 325		return -EINVAL;
 326	else if (desc != desc_first)
 327		return desc->total_len;
 328
 329	/* cookie matches to the currently running transfer */
 330	ret = desc_first->total_len;
 331
 332	if (desc_first->lli.dscr) {
 333		/* hardware linked list transfer */
 334
 335		/*
 336		 * Calculate the residue by removing the length of the child
 337		 * descriptors already transferred from the total length.
 338		 * To get the current child descriptor we can use the value of
 339		 * the channel's DSCR register and compare it against the value
 340		 * of the hardware linked list structure of each child
 341		 * descriptor.
 342		 *
 343		 * The CTRLA register provides us with the amount of data
 344		 * already read from the source for the current child
 345		 * descriptor. So we can compute a more accurate residue by also
 346		 * removing the number of bytes corresponding to this amount of
 347		 * data.
 348		 *
 349		 * However, the DSCR and CTRLA registers cannot be read both
 350		 * atomically. Hence a race condition may occur: the first read
 351		 * register may refer to one child descriptor whereas the second
 352		 * read may refer to a later child descriptor in the list
 353		 * because of the DMA transfer progression inbetween the two
 354		 * reads.
 355		 *
 356		 * One solution could have been to pause the DMA transfer, read
 357		 * the DSCR and CTRLA then resume the DMA transfer. Nonetheless,
 358		 * this approach presents some drawbacks:
 359		 * - If the DMA transfer is paused, RX overruns or TX underruns
 360		 *   are more likey to occur depending on the system latency.
 361		 *   Taking the USART driver as an example, it uses a cyclic DMA
 362		 *   transfer to read data from the Receive Holding Register
 363		 *   (RHR) to avoid RX overruns since the RHR is not protected
 364		 *   by any FIFO on most Atmel SoCs. So pausing the DMA transfer
 365		 *   to compute the residue would break the USART driver design.
 366		 * - The atc_pause() function masks interrupts but we'd rather
 367		 *   avoid to do so for system latency purpose.
 368		 *
 369		 * Then we'd rather use another solution: the DSCR is read a
 370		 * first time, the CTRLA is read in turn, next the DSCR is read
 371		 * a second time. If the two consecutive read values of the DSCR
 372		 * are the same then we assume both refers to the very same
 373		 * child descriptor as well as the CTRLA value read inbetween
 374		 * does. For cyclic tranfers, the assumption is that a full loop
 375		 * is "not so fast".
 376		 * If the two DSCR values are different, we read again the CTRLA
 377		 * then the DSCR till two consecutive read values from DSCR are
 378		 * equal or till the maxium trials is reach.
 379		 * This algorithm is very unlikely not to find a stable value for
 380		 * DSCR.
 381		 */
 382
 383		dscr = channel_readl(atchan, DSCR);
 384		rmb(); /* ensure DSCR is read before CTRLA */
 385		ctrla = channel_readl(atchan, CTRLA);
 386		for (trials = 0; trials < ATC_MAX_DSCR_TRIALS; ++trials) {
 387			u32 new_dscr;
 388
 389			rmb(); /* ensure DSCR is read after CTRLA */
 390			new_dscr = channel_readl(atchan, DSCR);
 391
 392			/*
 393			 * If the DSCR register value has not changed inside the
 394			 * DMA controller since the previous read, we assume
 395			 * that both the dscr and ctrla values refers to the
 396			 * very same descriptor.
 397			 */
 398			if (likely(new_dscr == dscr))
 399				break;
 400
 401			/*
 402			 * DSCR has changed inside the DMA controller, so the
 403			 * previouly read value of CTRLA may refer to an already
 404			 * processed descriptor hence could be outdated.
 405			 * We need to update ctrla to match the current
 406			 * descriptor.
 407			 */
 408			dscr = new_dscr;
 409			rmb(); /* ensure DSCR is read before CTRLA */
 410			ctrla = channel_readl(atchan, CTRLA);
 411		}
 412		if (unlikely(trials >= ATC_MAX_DSCR_TRIALS))
 413			return -ETIMEDOUT;
 414
 415		/* for the first descriptor we can be more accurate */
 416		if (desc_first->lli.dscr == dscr)
 417			return atc_calc_bytes_left(ret, ctrla);
 418
 419		ret -= desc_first->len;
 420		list_for_each_entry(desc, &desc_first->tx_list, desc_node) {
 421			if (desc->lli.dscr == dscr)
 422				break;
 423
 424			ret -= desc->len;
 425		}
 426
 427		/*
 428		 * For the current descriptor in the chain we can calculate
 429		 * the remaining bytes using the channel's register.
 430		 */
 431		ret = atc_calc_bytes_left(ret, ctrla);
 432	} else {
 433		/* single transfer */
 434		ctrla = channel_readl(atchan, CTRLA);
 435		ret = atc_calc_bytes_left(ret, ctrla);
 436	}
 437
 438	return ret;
 439}
 440
 441/**
 442 * atc_chain_complete - finish work for one transaction chain
 443 * @atchan: channel we work on
 444 * @desc: descriptor at the head of the chain we want do complete
 445 *
 446 * Called with atchan->lock held and bh disabled */
 447static void
 448atc_chain_complete(struct at_dma_chan *atchan, struct at_desc *desc)
 449{
 450	struct dma_async_tx_descriptor	*txd = &desc->txd;
 451	struct at_dma			*atdma = to_at_dma(atchan->chan_common.device);
 
 452
 453	dev_vdbg(chan2dev(&atchan->chan_common),
 454		"descriptor %u complete\n", txd->cookie);
 455
 
 
 456	/* mark the descriptor as complete for non cyclic cases only */
 457	if (!atc_chan_is_cyclic(atchan))
 458		dma_cookie_complete(txd);
 459
 460	/* If the transfer was a memset, free our temporary buffer */
 461	if (desc->memset_buffer) {
 462		dma_pool_free(atdma->memset_pool, desc->memset_vaddr,
 463			      desc->memset_paddr);
 464		desc->memset_buffer = false;
 465	}
 466
 467	/* move children to free_list */
 468	list_splice_init(&desc->tx_list, &atchan->free_list);
 469	/* move myself to free_list */
 470	list_move(&desc->desc_node, &atchan->free_list);
 471
 
 
 472	dma_descriptor_unmap(txd);
 473	/* for cyclic transfers,
 474	 * no need to replay callback function while stopping */
 475	if (!atc_chan_is_cyclic(atchan)) {
 476		dma_async_tx_callback	callback = txd->callback;
 477		void			*param = txd->callback_param;
 478
 479		/*
 480		 * The API requires that no submissions are done from a
 481		 * callback, so we don't need to drop the lock here
 482		 */
 483		if (callback)
 484			callback(param);
 485	}
 486
 487	dma_run_dependencies(txd);
 488}
 489
 490/**
 491 * atc_complete_all - finish work for all transactions
 492 * @atchan: channel to complete transactions for
 493 *
 494 * Eventually submit queued descriptors if any
 495 *
 496 * Assume channel is idle while calling this function
 497 * Called with atchan->lock held and bh disabled
 498 */
 499static void atc_complete_all(struct at_dma_chan *atchan)
 500{
 501	struct at_desc *desc, *_desc;
 502	LIST_HEAD(list);
 
 503
 504	dev_vdbg(chan2dev(&atchan->chan_common), "complete all\n");
 505
 
 
 506	/*
 507	 * Submit queued descriptors ASAP, i.e. before we go through
 508	 * the completed ones.
 509	 */
 510	if (!list_empty(&atchan->queue))
 511		atc_dostart(atchan, atc_first_queued(atchan));
 512	/* empty active_list now it is completed */
 513	list_splice_init(&atchan->active_list, &list);
 514	/* empty queue list by moving descriptors (if any) to active_list */
 515	list_splice_init(&atchan->queue, &atchan->active_list);
 516
 
 
 517	list_for_each_entry_safe(desc, _desc, &list, desc_node)
 518		atc_chain_complete(atchan, desc);
 519}
 520
 521/**
 522 * atc_advance_work - at the end of a transaction, move forward
 523 * @atchan: channel where the transaction ended
 524 *
 525 * Called with atchan->lock held and bh disabled
 526 */
 527static void atc_advance_work(struct at_dma_chan *atchan)
 528{
 
 
 
 529	dev_vdbg(chan2dev(&atchan->chan_common), "advance_work\n");
 530
 531	if (atc_chan_is_enabled(atchan))
 
 
 
 532		return;
 533
 534	if (list_empty(&atchan->active_list) ||
 535	    list_is_singular(&atchan->active_list)) {
 536		atc_complete_all(atchan);
 537	} else {
 538		atc_chain_complete(atchan, atc_first_active(atchan));
 539		/* advance work */
 540		atc_dostart(atchan, atc_first_active(atchan));
 541	}
 
 
 542}
 543
 544
 545/**
 546 * atc_handle_error - handle errors reported by DMA controller
 547 * @atchan: channel where error occurs
 548 *
 549 * Called with atchan->lock held and bh disabled
 550 */
 551static void atc_handle_error(struct at_dma_chan *atchan)
 552{
 553	struct at_desc *bad_desc;
 554	struct at_desc *child;
 
 555
 
 556	/*
 557	 * The descriptor currently at the head of the active list is
 558	 * broked. Since we don't have any way to report errors, we'll
 559	 * just have to scream loudly and try to carry on.
 560	 */
 561	bad_desc = atc_first_active(atchan);
 562	list_del_init(&bad_desc->desc_node);
 563
 564	/* As we are stopped, take advantage to push queued descriptors
 565	 * in active_list */
 566	list_splice_init(&atchan->queue, atchan->active_list.prev);
 567
 568	/* Try to restart the controller */
 569	if (!list_empty(&atchan->active_list))
 570		atc_dostart(atchan, atc_first_active(atchan));
 571
 572	/*
 573	 * KERN_CRITICAL may seem harsh, but since this only happens
 574	 * when someone submits a bad physical address in a
 575	 * descriptor, we should consider ourselves lucky that the
 576	 * controller flagged an error instead of scribbling over
 577	 * random memory locations.
 578	 */
 579	dev_crit(chan2dev(&atchan->chan_common),
 580			"Bad descriptor submitted for DMA!\n");
 581	dev_crit(chan2dev(&atchan->chan_common),
 582			"  cookie: %d\n", bad_desc->txd.cookie);
 583	atc_dump_lli(atchan, &bad_desc->lli);
 584	list_for_each_entry(child, &bad_desc->tx_list, desc_node)
 585		atc_dump_lli(atchan, &child->lli);
 586
 
 
 587	/* Pretend the descriptor completed successfully */
 588	atc_chain_complete(atchan, bad_desc);
 589}
 590
 591/**
 592 * atc_handle_cyclic - at the end of a period, run callback function
 593 * @atchan: channel used for cyclic operations
 594 *
 595 * Called with atchan->lock held and bh disabled
 596 */
 597static void atc_handle_cyclic(struct at_dma_chan *atchan)
 598{
 599	struct at_desc			*first = atc_first_active(atchan);
 600	struct dma_async_tx_descriptor	*txd = &first->txd;
 601	dma_async_tx_callback		callback = txd->callback;
 602	void				*param = txd->callback_param;
 603
 604	dev_vdbg(chan2dev(&atchan->chan_common),
 605			"new cyclic period llp 0x%08x\n",
 606			channel_readl(atchan, DSCR));
 607
 608	if (callback)
 609		callback(param);
 610}
 611
 612/*--  IRQ & Tasklet  ---------------------------------------------------*/
 613
 614static void atc_tasklet(unsigned long data)
 615{
 616	struct at_dma_chan *atchan = (struct at_dma_chan *)data;
 617	unsigned long flags;
 618
 619	spin_lock_irqsave(&atchan->lock, flags);
 620	if (test_and_clear_bit(ATC_IS_ERROR, &atchan->status))
 621		atc_handle_error(atchan);
 622	else if (atc_chan_is_cyclic(atchan))
 623		atc_handle_cyclic(atchan);
 624	else
 625		atc_advance_work(atchan);
 626
 627	spin_unlock_irqrestore(&atchan->lock, flags);
 
 
 
 628}
 629
 630static irqreturn_t at_dma_interrupt(int irq, void *dev_id)
 631{
 632	struct at_dma		*atdma = (struct at_dma *)dev_id;
 633	struct at_dma_chan	*atchan;
 634	int			i;
 635	u32			status, pending, imr;
 636	int			ret = IRQ_NONE;
 637
 638	do {
 639		imr = dma_readl(atdma, EBCIMR);
 640		status = dma_readl(atdma, EBCISR);
 641		pending = status & imr;
 642
 643		if (!pending)
 644			break;
 645
 646		dev_vdbg(atdma->dma_common.dev,
 647			"interrupt: status = 0x%08x, 0x%08x, 0x%08x\n",
 648			 status, imr, pending);
 649
 650		for (i = 0; i < atdma->dma_common.chancnt; i++) {
 651			atchan = &atdma->chan[i];
 652			if (pending & (AT_DMA_BTC(i) | AT_DMA_ERR(i))) {
 653				if (pending & AT_DMA_ERR(i)) {
 654					/* Disable channel on AHB error */
 655					dma_writel(atdma, CHDR,
 656						AT_DMA_RES(i) | atchan->mask);
 657					/* Give information to tasklet */
 658					set_bit(ATC_IS_ERROR, &atchan->status);
 659				}
 660				tasklet_schedule(&atchan->tasklet);
 661				ret = IRQ_HANDLED;
 662			}
 663		}
 664
 665	} while (pending);
 666
 667	return ret;
 668}
 669
 670
 671/*--  DMA Engine API  --------------------------------------------------*/
 672
 673/**
 674 * atc_tx_submit - set the prepared descriptor(s) to be executed by the engine
 675 * @desc: descriptor at the head of the transaction chain
 676 *
 677 * Queue chain if DMA engine is working already
 678 *
 679 * Cookie increment and adding to active_list or queue must be atomic
 680 */
 681static dma_cookie_t atc_tx_submit(struct dma_async_tx_descriptor *tx)
 682{
 683	struct at_desc		*desc = txd_to_at_desc(tx);
 684	struct at_dma_chan	*atchan = to_at_dma_chan(tx->chan);
 685	dma_cookie_t		cookie;
 686	unsigned long		flags;
 687
 688	spin_lock_irqsave(&atchan->lock, flags);
 689	cookie = dma_cookie_assign(tx);
 690
 691	if (list_empty(&atchan->active_list)) {
 692		dev_vdbg(chan2dev(tx->chan), "tx_submit: started %u\n",
 693				desc->txd.cookie);
 694		atc_dostart(atchan, desc);
 695		list_add_tail(&desc->desc_node, &atchan->active_list);
 696	} else {
 697		dev_vdbg(chan2dev(tx->chan), "tx_submit: queued %u\n",
 698				desc->txd.cookie);
 699		list_add_tail(&desc->desc_node, &atchan->queue);
 700	}
 701
 702	spin_unlock_irqrestore(&atchan->lock, flags);
 703
 704	return cookie;
 705}
 706
 707/**
 708 * atc_prep_dma_interleaved - prepare memory to memory interleaved operation
 709 * @chan: the channel to prepare operation on
 710 * @xt: Interleaved transfer template
 711 * @flags: tx descriptor status flags
 712 */
 713static struct dma_async_tx_descriptor *
 714atc_prep_dma_interleaved(struct dma_chan *chan,
 715			 struct dma_interleaved_template *xt,
 716			 unsigned long flags)
 717{
 718	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 719	struct data_chunk	*first = xt->sgl;
 720	struct at_desc		*desc = NULL;
 721	size_t			xfer_count;
 722	unsigned int		dwidth;
 723	u32			ctrla;
 724	u32			ctrlb;
 725	size_t			len = 0;
 726	int			i;
 727
 728	if (unlikely(!xt || xt->numf != 1 || !xt->frame_size))
 729		return NULL;
 730
 
 
 731	dev_info(chan2dev(chan),
 732		 "%s: src=%pad, dest=%pad, numf=%d, frame_size=%d, flags=0x%lx\n",
 733		__func__, &xt->src_start, &xt->dst_start, xt->numf,
 734		xt->frame_size, flags);
 735
 736	/*
 737	 * The controller can only "skip" X bytes every Y bytes, so we
 738	 * need to make sure we are given a template that fit that
 739	 * description, ie a template with chunks that always have the
 740	 * same size, with the same ICGs.
 741	 */
 742	for (i = 0; i < xt->frame_size; i++) {
 743		struct data_chunk *chunk = xt->sgl + i;
 744
 745		if ((chunk->size != xt->sgl->size) ||
 746		    (dmaengine_get_dst_icg(xt, chunk) != dmaengine_get_dst_icg(xt, first)) ||
 747		    (dmaengine_get_src_icg(xt, chunk) != dmaengine_get_src_icg(xt, first))) {
 748			dev_err(chan2dev(chan),
 749				"%s: the controller can transfer only identical chunks\n",
 750				__func__);
 751			return NULL;
 752		}
 753
 754		len += chunk->size;
 755	}
 756
 757	dwidth = atc_get_xfer_width(xt->src_start,
 758				    xt->dst_start, len);
 759
 760	xfer_count = len >> dwidth;
 761	if (xfer_count > ATC_BTSIZE_MAX) {
 762		dev_err(chan2dev(chan), "%s: buffer is too big\n", __func__);
 763		return NULL;
 764	}
 765
 766	ctrla = ATC_SRC_WIDTH(dwidth) |
 767		ATC_DST_WIDTH(dwidth);
 768
 769	ctrlb =   ATC_DEFAULT_CTRLB | ATC_IEN
 770		| ATC_SRC_ADDR_MODE_INCR
 771		| ATC_DST_ADDR_MODE_INCR
 772		| ATC_SRC_PIP
 773		| ATC_DST_PIP
 774		| ATC_FC_MEM2MEM;
 775
 776	/* create the transfer */
 777	desc = atc_desc_get(atchan);
 778	if (!desc) {
 779		dev_err(chan2dev(chan),
 780			"%s: couldn't allocate our descriptor\n", __func__);
 781		return NULL;
 782	}
 783
 784	desc->lli.saddr = xt->src_start;
 785	desc->lli.daddr = xt->dst_start;
 786	desc->lli.ctrla = ctrla | xfer_count;
 787	desc->lli.ctrlb = ctrlb;
 788
 789	desc->boundary = first->size >> dwidth;
 790	desc->dst_hole = (dmaengine_get_dst_icg(xt, first) >> dwidth) + 1;
 791	desc->src_hole = (dmaengine_get_src_icg(xt, first) >> dwidth) + 1;
 792
 793	desc->txd.cookie = -EBUSY;
 794	desc->total_len = desc->len = len;
 795
 796	/* set end-of-link to the last link descriptor of list*/
 797	set_desc_eol(desc);
 798
 799	desc->txd.flags = flags; /* client is in control of this ack */
 800
 801	return &desc->txd;
 802}
 803
 804/**
 805 * atc_prep_dma_memcpy - prepare a memcpy operation
 806 * @chan: the channel to prepare operation on
 807 * @dest: operation virtual destination address
 808 * @src: operation virtual source address
 809 * @len: operation length
 810 * @flags: tx descriptor status flags
 811 */
 812static struct dma_async_tx_descriptor *
 813atc_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 814		size_t len, unsigned long flags)
 815{
 816	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 817	struct at_desc		*desc = NULL;
 818	struct at_desc		*first = NULL;
 819	struct at_desc		*prev = NULL;
 820	size_t			xfer_count;
 821	size_t			offset;
 822	unsigned int		src_width;
 823	unsigned int		dst_width;
 824	u32			ctrla;
 825	u32			ctrlb;
 826
 827	dev_vdbg(chan2dev(chan), "prep_dma_memcpy: d%pad s%pad l0x%zx f0x%lx\n",
 828			&dest, &src, len, flags);
 829
 830	if (unlikely(!len)) {
 831		dev_dbg(chan2dev(chan), "prep_dma_memcpy: length is zero!\n");
 832		return NULL;
 833	}
 834
 835	ctrlb =   ATC_DEFAULT_CTRLB | ATC_IEN
 836		| ATC_SRC_ADDR_MODE_INCR
 837		| ATC_DST_ADDR_MODE_INCR
 838		| ATC_FC_MEM2MEM;
 839
 840	/*
 841	 * We can be a lot more clever here, but this should take care
 842	 * of the most common optimization.
 843	 */
 844	src_width = dst_width = atc_get_xfer_width(src, dest, len);
 845
 846	ctrla = ATC_SRC_WIDTH(src_width) |
 847		ATC_DST_WIDTH(dst_width);
 848
 849	for (offset = 0; offset < len; offset += xfer_count << src_width) {
 850		xfer_count = min_t(size_t, (len - offset) >> src_width,
 851				ATC_BTSIZE_MAX);
 852
 853		desc = atc_desc_get(atchan);
 854		if (!desc)
 855			goto err_desc_get;
 856
 857		desc->lli.saddr = src + offset;
 858		desc->lli.daddr = dest + offset;
 859		desc->lli.ctrla = ctrla | xfer_count;
 860		desc->lli.ctrlb = ctrlb;
 861
 862		desc->txd.cookie = 0;
 863		desc->len = xfer_count << src_width;
 864
 865		atc_desc_chain(&first, &prev, desc);
 866	}
 867
 868	/* First descriptor of the chain embedds additional information */
 869	first->txd.cookie = -EBUSY;
 870	first->total_len = len;
 871
 872	/* set end-of-link to the last link descriptor of list*/
 873	set_desc_eol(desc);
 874
 875	first->txd.flags = flags; /* client is in control of this ack */
 876
 877	return &first->txd;
 878
 879err_desc_get:
 880	atc_desc_put(atchan, first);
 881	return NULL;
 882}
 883
 884static struct at_desc *atc_create_memset_desc(struct dma_chan *chan,
 885					      dma_addr_t psrc,
 886					      dma_addr_t pdst,
 887					      size_t len)
 888{
 889	struct at_dma_chan *atchan = to_at_dma_chan(chan);
 890	struct at_desc *desc;
 891	size_t xfer_count;
 892
 893	u32 ctrla = ATC_SRC_WIDTH(2) | ATC_DST_WIDTH(2);
 894	u32 ctrlb = ATC_DEFAULT_CTRLB | ATC_IEN |
 895		ATC_SRC_ADDR_MODE_FIXED |
 896		ATC_DST_ADDR_MODE_INCR |
 897		ATC_FC_MEM2MEM;
 898
 899	xfer_count = len >> 2;
 900	if (xfer_count > ATC_BTSIZE_MAX) {
 901		dev_err(chan2dev(chan), "%s: buffer is too big\n",
 902			__func__);
 903		return NULL;
 904	}
 905
 906	desc = atc_desc_get(atchan);
 907	if (!desc) {
 908		dev_err(chan2dev(chan), "%s: can't get a descriptor\n",
 909			__func__);
 910		return NULL;
 911	}
 912
 913	desc->lli.saddr = psrc;
 914	desc->lli.daddr = pdst;
 915	desc->lli.ctrla = ctrla | xfer_count;
 916	desc->lli.ctrlb = ctrlb;
 917
 918	desc->txd.cookie = 0;
 919	desc->len = len;
 920
 921	return desc;
 922}
 923
 924/**
 925 * atc_prep_dma_memset - prepare a memcpy operation
 926 * @chan: the channel to prepare operation on
 927 * @dest: operation virtual destination address
 928 * @value: value to set memory buffer to
 929 * @len: operation length
 930 * @flags: tx descriptor status flags
 931 */
 932static struct dma_async_tx_descriptor *
 933atc_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
 934		    size_t len, unsigned long flags)
 935{
 936	struct at_dma		*atdma = to_at_dma(chan->device);
 937	struct at_desc		*desc;
 938	void __iomem		*vaddr;
 939	dma_addr_t		paddr;
 940
 941	dev_vdbg(chan2dev(chan), "%s: d%pad v0x%x l0x%zx f0x%lx\n", __func__,
 942		&dest, value, len, flags);
 943
 944	if (unlikely(!len)) {
 945		dev_dbg(chan2dev(chan), "%s: length is zero!\n", __func__);
 946		return NULL;
 947	}
 948
 949	if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {
 950		dev_dbg(chan2dev(chan), "%s: buffer is not aligned\n",
 951			__func__);
 952		return NULL;
 953	}
 954
 955	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr);
 956	if (!vaddr) {
 957		dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",
 958			__func__);
 959		return NULL;
 960	}
 961	*(u32*)vaddr = value;
 962
 963	desc = atc_create_memset_desc(chan, paddr, dest, len);
 964	if (!desc) {
 965		dev_err(chan2dev(chan), "%s: couldn't get a descriptor\n",
 966			__func__);
 967		goto err_free_buffer;
 968	}
 969
 970	desc->memset_paddr = paddr;
 971	desc->memset_vaddr = vaddr;
 972	desc->memset_buffer = true;
 973
 974	desc->txd.cookie = -EBUSY;
 975	desc->total_len = len;
 976
 977	/* set end-of-link on the descriptor */
 978	set_desc_eol(desc);
 979
 980	desc->txd.flags = flags;
 981
 982	return &desc->txd;
 983
 984err_free_buffer:
 985	dma_pool_free(atdma->memset_pool, vaddr, paddr);
 986	return NULL;
 987}
 988
 989static struct dma_async_tx_descriptor *
 990atc_prep_dma_memset_sg(struct dma_chan *chan,
 991		       struct scatterlist *sgl,
 992		       unsigned int sg_len, int value,
 993		       unsigned long flags)
 994{
 995	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
 996	struct at_dma		*atdma = to_at_dma(chan->device);
 997	struct at_desc		*desc = NULL, *first = NULL, *prev = NULL;
 998	struct scatterlist	*sg;
 999	void __iomem		*vaddr;
1000	dma_addr_t		paddr;
1001	size_t			total_len = 0;
1002	int			i;
1003
1004	dev_vdbg(chan2dev(chan), "%s: v0x%x l0x%zx f0x%lx\n", __func__,
1005		 value, sg_len, flags);
1006
1007	if (unlikely(!sgl || !sg_len)) {
1008		dev_dbg(chan2dev(chan), "%s: scatterlist is empty!\n",
1009			__func__);
1010		return NULL;
1011	}
1012
1013	vaddr = dma_pool_alloc(atdma->memset_pool, GFP_ATOMIC, &paddr);
1014	if (!vaddr) {
1015		dev_err(chan2dev(chan), "%s: couldn't allocate buffer\n",
1016			__func__);
1017		return NULL;
1018	}
1019	*(u32*)vaddr = value;
1020
1021	for_each_sg(sgl, sg, sg_len, i) {
1022		dma_addr_t dest = sg_dma_address(sg);
1023		size_t len = sg_dma_len(sg);
1024
1025		dev_vdbg(chan2dev(chan), "%s: d%pad, l0x%zx\n",
1026			 __func__, &dest, len);
1027
1028		if (!is_dma_fill_aligned(chan->device, dest, 0, len)) {
1029			dev_err(chan2dev(chan), "%s: buffer is not aligned\n",
1030				__func__);
1031			goto err_put_desc;
1032		}
1033
1034		desc = atc_create_memset_desc(chan, paddr, dest, len);
1035		if (!desc)
1036			goto err_put_desc;
1037
1038		atc_desc_chain(&first, &prev, desc);
1039
1040		total_len += len;
1041	}
1042
1043	/*
1044	 * Only set the buffer pointers on the last descriptor to
1045	 * avoid free'ing while we have our transfer still going
1046	 */
1047	desc->memset_paddr = paddr;
1048	desc->memset_vaddr = vaddr;
1049	desc->memset_buffer = true;
1050
1051	first->txd.cookie = -EBUSY;
1052	first->total_len = total_len;
1053
1054	/* set end-of-link on the descriptor */
1055	set_desc_eol(desc);
1056
1057	first->txd.flags = flags;
1058
1059	return &first->txd;
1060
1061err_put_desc:
1062	atc_desc_put(atchan, first);
1063	return NULL;
1064}
1065
1066/**
1067 * atc_prep_slave_sg - prepare descriptors for a DMA_SLAVE transaction
1068 * @chan: DMA channel
1069 * @sgl: scatterlist to transfer to/from
1070 * @sg_len: number of entries in @scatterlist
1071 * @direction: DMA direction
1072 * @flags: tx descriptor status flags
1073 * @context: transaction context (ignored)
1074 */
1075static struct dma_async_tx_descriptor *
1076atc_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
1077		unsigned int sg_len, enum dma_transfer_direction direction,
1078		unsigned long flags, void *context)
1079{
1080	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1081	struct at_dma_slave	*atslave = chan->private;
1082	struct dma_slave_config	*sconfig = &atchan->dma_sconfig;
1083	struct at_desc		*first = NULL;
1084	struct at_desc		*prev = NULL;
1085	u32			ctrla;
1086	u32			ctrlb;
1087	dma_addr_t		reg;
1088	unsigned int		reg_width;
1089	unsigned int		mem_width;
1090	unsigned int		i;
1091	struct scatterlist	*sg;
1092	size_t			total_len = 0;
1093
1094	dev_vdbg(chan2dev(chan), "prep_slave_sg (%d): %s f0x%lx\n",
1095			sg_len,
1096			direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE",
1097			flags);
1098
1099	if (unlikely(!atslave || !sg_len)) {
1100		dev_dbg(chan2dev(chan), "prep_slave_sg: sg length is zero!\n");
1101		return NULL;
1102	}
1103
1104	ctrla =   ATC_SCSIZE(sconfig->src_maxburst)
1105		| ATC_DCSIZE(sconfig->dst_maxburst);
1106	ctrlb = ATC_IEN;
1107
1108	switch (direction) {
1109	case DMA_MEM_TO_DEV:
1110		reg_width = convert_buswidth(sconfig->dst_addr_width);
1111		ctrla |=  ATC_DST_WIDTH(reg_width);
1112		ctrlb |=  ATC_DST_ADDR_MODE_FIXED
1113			| ATC_SRC_ADDR_MODE_INCR
1114			| ATC_FC_MEM2PER
1115			| ATC_SIF(atchan->mem_if) | ATC_DIF(atchan->per_if);
1116		reg = sconfig->dst_addr;
1117		for_each_sg(sgl, sg, sg_len, i) {
1118			struct at_desc	*desc;
1119			u32		len;
1120			u32		mem;
1121
1122			desc = atc_desc_get(atchan);
1123			if (!desc)
1124				goto err_desc_get;
1125
1126			mem = sg_dma_address(sg);
1127			len = sg_dma_len(sg);
1128			if (unlikely(!len)) {
1129				dev_dbg(chan2dev(chan),
1130					"prep_slave_sg: sg(%d) data length is zero\n", i);
1131				goto err;
1132			}
1133			mem_width = 2;
1134			if (unlikely(mem & 3 || len & 3))
1135				mem_width = 0;
1136
1137			desc->lli.saddr = mem;
1138			desc->lli.daddr = reg;
1139			desc->lli.ctrla = ctrla
1140					| ATC_SRC_WIDTH(mem_width)
1141					| len >> mem_width;
1142			desc->lli.ctrlb = ctrlb;
1143			desc->len = len;
1144
1145			atc_desc_chain(&first, &prev, desc);
1146			total_len += len;
1147		}
1148		break;
1149	case DMA_DEV_TO_MEM:
1150		reg_width = convert_buswidth(sconfig->src_addr_width);
1151		ctrla |=  ATC_SRC_WIDTH(reg_width);
1152		ctrlb |=  ATC_DST_ADDR_MODE_INCR
1153			| ATC_SRC_ADDR_MODE_FIXED
1154			| ATC_FC_PER2MEM
1155			| ATC_SIF(atchan->per_if) | ATC_DIF(atchan->mem_if);
1156
1157		reg = sconfig->src_addr;
1158		for_each_sg(sgl, sg, sg_len, i) {
1159			struct at_desc	*desc;
1160			u32		len;
1161			u32		mem;
1162
1163			desc = atc_desc_get(atchan);
1164			if (!desc)
1165				goto err_desc_get;
1166
1167			mem = sg_dma_address(sg);
1168			len = sg_dma_len(sg);
1169			if (unlikely(!len)) {
1170				dev_dbg(chan2dev(chan),
1171					"prep_slave_sg: sg(%d) data length is zero\n", i);
1172				goto err;
1173			}
1174			mem_width = 2;
1175			if (unlikely(mem & 3 || len & 3))
1176				mem_width = 0;
1177
1178			desc->lli.saddr = reg;
1179			desc->lli.daddr = mem;
1180			desc->lli.ctrla = ctrla
1181					| ATC_DST_WIDTH(mem_width)
1182					| len >> reg_width;
1183			desc->lli.ctrlb = ctrlb;
1184			desc->len = len;
1185
1186			atc_desc_chain(&first, &prev, desc);
1187			total_len += len;
1188		}
1189		break;
1190	default:
1191		return NULL;
1192	}
1193
1194	/* set end-of-link to the last link descriptor of list*/
1195	set_desc_eol(prev);
1196
1197	/* First descriptor of the chain embedds additional information */
1198	first->txd.cookie = -EBUSY;
1199	first->total_len = total_len;
1200
1201	/* first link descriptor of list is responsible of flags */
1202	first->txd.flags = flags; /* client is in control of this ack */
1203
1204	return &first->txd;
1205
1206err_desc_get:
1207	dev_err(chan2dev(chan), "not enough descriptors available\n");
1208err:
1209	atc_desc_put(atchan, first);
1210	return NULL;
1211}
1212
1213/**
1214 * atc_prep_dma_sg - prepare memory to memory scather-gather operation
1215 * @chan: the channel to prepare operation on
1216 * @dst_sg: destination scatterlist
1217 * @dst_nents: number of destination scatterlist entries
1218 * @src_sg: source scatterlist
1219 * @src_nents: number of source scatterlist entries
1220 * @flags: tx descriptor status flags
1221 */
1222static struct dma_async_tx_descriptor *
1223atc_prep_dma_sg(struct dma_chan *chan,
1224		struct scatterlist *dst_sg, unsigned int dst_nents,
1225		struct scatterlist *src_sg, unsigned int src_nents,
1226		unsigned long flags)
1227{
1228	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1229	struct at_desc		*desc = NULL;
1230	struct at_desc		*first = NULL;
1231	struct at_desc		*prev = NULL;
1232	unsigned int		src_width;
1233	unsigned int		dst_width;
1234	size_t			xfer_count;
1235	u32			ctrla;
1236	u32			ctrlb;
1237	size_t			dst_len = 0, src_len = 0;
1238	dma_addr_t		dst = 0, src = 0;
1239	size_t			len = 0, total_len = 0;
1240
1241	if (unlikely(dst_nents == 0 || src_nents == 0))
1242		return NULL;
1243
1244	if (unlikely(dst_sg == NULL || src_sg == NULL))
1245		return NULL;
1246
1247	ctrlb =   ATC_DEFAULT_CTRLB | ATC_IEN
1248		| ATC_SRC_ADDR_MODE_INCR
1249		| ATC_DST_ADDR_MODE_INCR
1250		| ATC_FC_MEM2MEM;
1251
1252	/*
1253	 * loop until there is either no more source or no more destination
1254	 * scatterlist entry
1255	 */
1256	while (true) {
1257
1258		/* prepare the next transfer */
1259		if (dst_len == 0) {
1260
1261			/* no more destination scatterlist entries */
1262			if (!dst_sg || !dst_nents)
1263				break;
1264
1265			dst = sg_dma_address(dst_sg);
1266			dst_len = sg_dma_len(dst_sg);
1267
1268			dst_sg = sg_next(dst_sg);
1269			dst_nents--;
1270		}
1271
1272		if (src_len == 0) {
1273
1274			/* no more source scatterlist entries */
1275			if (!src_sg || !src_nents)
1276				break;
1277
1278			src = sg_dma_address(src_sg);
1279			src_len = sg_dma_len(src_sg);
1280
1281			src_sg = sg_next(src_sg);
1282			src_nents--;
1283		}
1284
1285		len = min_t(size_t, src_len, dst_len);
1286		if (len == 0)
1287			continue;
1288
1289		/* take care for the alignment */
1290		src_width = dst_width = atc_get_xfer_width(src, dst, len);
1291
1292		ctrla = ATC_SRC_WIDTH(src_width) |
1293			ATC_DST_WIDTH(dst_width);
1294
1295		/*
1296		 * The number of transfers to set up refer to the source width
1297		 * that depends on the alignment.
1298		 */
1299		xfer_count = len >> src_width;
1300		if (xfer_count > ATC_BTSIZE_MAX) {
1301			xfer_count = ATC_BTSIZE_MAX;
1302			len = ATC_BTSIZE_MAX << src_width;
1303		}
1304
1305		/* create the transfer */
1306		desc = atc_desc_get(atchan);
1307		if (!desc)
1308			goto err_desc_get;
1309
1310		desc->lli.saddr = src;
1311		desc->lli.daddr = dst;
1312		desc->lli.ctrla = ctrla | xfer_count;
1313		desc->lli.ctrlb = ctrlb;
1314
1315		desc->txd.cookie = 0;
1316		desc->len = len;
1317
1318		atc_desc_chain(&first, &prev, desc);
1319
1320		/* update the lengths and addresses for the next loop cycle */
1321		dst_len -= len;
1322		src_len -= len;
1323		dst += len;
1324		src += len;
1325
1326		total_len += len;
1327	}
1328
1329	/* First descriptor of the chain embedds additional information */
1330	first->txd.cookie = -EBUSY;
1331	first->total_len = total_len;
1332
1333	/* set end-of-link to the last link descriptor of list*/
1334	set_desc_eol(desc);
1335
1336	first->txd.flags = flags; /* client is in control of this ack */
1337
1338	return &first->txd;
1339
1340err_desc_get:
1341	atc_desc_put(atchan, first);
1342	return NULL;
1343}
1344
1345/**
1346 * atc_dma_cyclic_check_values
1347 * Check for too big/unaligned periods and unaligned DMA buffer
1348 */
1349static int
1350atc_dma_cyclic_check_values(unsigned int reg_width, dma_addr_t buf_addr,
1351		size_t period_len)
1352{
1353	if (period_len > (ATC_BTSIZE_MAX << reg_width))
1354		goto err_out;
1355	if (unlikely(period_len & ((1 << reg_width) - 1)))
1356		goto err_out;
1357	if (unlikely(buf_addr & ((1 << reg_width) - 1)))
1358		goto err_out;
1359
1360	return 0;
1361
1362err_out:
1363	return -EINVAL;
1364}
1365
1366/**
1367 * atc_dma_cyclic_fill_desc - Fill one period descriptor
1368 */
1369static int
1370atc_dma_cyclic_fill_desc(struct dma_chan *chan, struct at_desc *desc,
1371		unsigned int period_index, dma_addr_t buf_addr,
1372		unsigned int reg_width, size_t period_len,
1373		enum dma_transfer_direction direction)
1374{
1375	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1376	struct dma_slave_config	*sconfig = &atchan->dma_sconfig;
1377	u32			ctrla;
1378
1379	/* prepare common CRTLA value */
1380	ctrla =   ATC_SCSIZE(sconfig->src_maxburst)
1381		| ATC_DCSIZE(sconfig->dst_maxburst)
1382		| ATC_DST_WIDTH(reg_width)
1383		| ATC_SRC_WIDTH(reg_width)
1384		| period_len >> reg_width;
1385
1386	switch (direction) {
1387	case DMA_MEM_TO_DEV:
1388		desc->lli.saddr = buf_addr + (period_len * period_index);
1389		desc->lli.daddr = sconfig->dst_addr;
1390		desc->lli.ctrla = ctrla;
1391		desc->lli.ctrlb = ATC_DST_ADDR_MODE_FIXED
1392				| ATC_SRC_ADDR_MODE_INCR
1393				| ATC_FC_MEM2PER
1394				| ATC_SIF(atchan->mem_if)
1395				| ATC_DIF(atchan->per_if);
1396		desc->len = period_len;
1397		break;
1398
1399	case DMA_DEV_TO_MEM:
1400		desc->lli.saddr = sconfig->src_addr;
1401		desc->lli.daddr = buf_addr + (period_len * period_index);
1402		desc->lli.ctrla = ctrla;
1403		desc->lli.ctrlb = ATC_DST_ADDR_MODE_INCR
1404				| ATC_SRC_ADDR_MODE_FIXED
1405				| ATC_FC_PER2MEM
1406				| ATC_SIF(atchan->per_if)
1407				| ATC_DIF(atchan->mem_if);
1408		desc->len = period_len;
1409		break;
1410
1411	default:
1412		return -EINVAL;
1413	}
1414
1415	return 0;
1416}
1417
1418/**
1419 * atc_prep_dma_cyclic - prepare the cyclic DMA transfer
1420 * @chan: the DMA channel to prepare
1421 * @buf_addr: physical DMA address where the buffer starts
1422 * @buf_len: total number of bytes for the entire buffer
1423 * @period_len: number of bytes for each period
1424 * @direction: transfer direction, to or from device
1425 * @flags: tx descriptor status flags
1426 */
1427static struct dma_async_tx_descriptor *
1428atc_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr, size_t buf_len,
1429		size_t period_len, enum dma_transfer_direction direction,
1430		unsigned long flags)
1431{
1432	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1433	struct at_dma_slave	*atslave = chan->private;
1434	struct dma_slave_config	*sconfig = &atchan->dma_sconfig;
1435	struct at_desc		*first = NULL;
1436	struct at_desc		*prev = NULL;
1437	unsigned long		was_cyclic;
1438	unsigned int		reg_width;
1439	unsigned int		periods = buf_len / period_len;
1440	unsigned int		i;
1441
1442	dev_vdbg(chan2dev(chan), "prep_dma_cyclic: %s buf@%pad - %d (%d/%d)\n",
1443			direction == DMA_MEM_TO_DEV ? "TO DEVICE" : "FROM DEVICE",
1444			&buf_addr,
1445			periods, buf_len, period_len);
1446
1447	if (unlikely(!atslave || !buf_len || !period_len)) {
1448		dev_dbg(chan2dev(chan), "prep_dma_cyclic: length is zero!\n");
1449		return NULL;
1450	}
1451
1452	was_cyclic = test_and_set_bit(ATC_IS_CYCLIC, &atchan->status);
1453	if (was_cyclic) {
1454		dev_dbg(chan2dev(chan), "prep_dma_cyclic: channel in use!\n");
1455		return NULL;
1456	}
1457
1458	if (unlikely(!is_slave_direction(direction)))
1459		goto err_out;
1460
1461	if (sconfig->direction == DMA_MEM_TO_DEV)
1462		reg_width = convert_buswidth(sconfig->dst_addr_width);
1463	else
1464		reg_width = convert_buswidth(sconfig->src_addr_width);
1465
1466	/* Check for too big/unaligned periods and unaligned DMA buffer */
1467	if (atc_dma_cyclic_check_values(reg_width, buf_addr, period_len))
1468		goto err_out;
1469
1470	/* build cyclic linked list */
1471	for (i = 0; i < periods; i++) {
1472		struct at_desc	*desc;
1473
1474		desc = atc_desc_get(atchan);
1475		if (!desc)
1476			goto err_desc_get;
1477
1478		if (atc_dma_cyclic_fill_desc(chan, desc, i, buf_addr,
1479					     reg_width, period_len, direction))
1480			goto err_desc_get;
1481
1482		atc_desc_chain(&first, &prev, desc);
1483	}
1484
1485	/* lets make a cyclic list */
1486	prev->lli.dscr = first->txd.phys;
1487
1488	/* First descriptor of the chain embedds additional information */
1489	first->txd.cookie = -EBUSY;
1490	first->total_len = buf_len;
1491
1492	return &first->txd;
1493
1494err_desc_get:
1495	dev_err(chan2dev(chan), "not enough descriptors available\n");
1496	atc_desc_put(atchan, first);
1497err_out:
1498	clear_bit(ATC_IS_CYCLIC, &atchan->status);
1499	return NULL;
1500}
1501
1502static int atc_config(struct dma_chan *chan,
1503		      struct dma_slave_config *sconfig)
1504{
1505	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1506
1507	dev_vdbg(chan2dev(chan), "%s\n", __func__);
1508
1509	/* Check if it is chan is configured for slave transfers */
1510	if (!chan->private)
1511		return -EINVAL;
1512
1513	memcpy(&atchan->dma_sconfig, sconfig, sizeof(*sconfig));
1514
1515	convert_burst(&atchan->dma_sconfig.src_maxburst);
1516	convert_burst(&atchan->dma_sconfig.dst_maxburst);
1517
1518	return 0;
1519}
1520
1521static int atc_pause(struct dma_chan *chan)
1522{
1523	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1524	struct at_dma		*atdma = to_at_dma(chan->device);
1525	int			chan_id = atchan->chan_common.chan_id;
1526	unsigned long		flags;
1527
1528	LIST_HEAD(list);
1529
1530	dev_vdbg(chan2dev(chan), "%s\n", __func__);
1531
1532	spin_lock_irqsave(&atchan->lock, flags);
1533
1534	dma_writel(atdma, CHER, AT_DMA_SUSP(chan_id));
1535	set_bit(ATC_IS_PAUSED, &atchan->status);
1536
1537	spin_unlock_irqrestore(&atchan->lock, flags);
1538
1539	return 0;
1540}
1541
1542static int atc_resume(struct dma_chan *chan)
1543{
1544	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1545	struct at_dma		*atdma = to_at_dma(chan->device);
1546	int			chan_id = atchan->chan_common.chan_id;
1547	unsigned long		flags;
1548
1549	LIST_HEAD(list);
1550
1551	dev_vdbg(chan2dev(chan), "%s\n", __func__);
1552
1553	if (!atc_chan_is_paused(atchan))
1554		return 0;
1555
1556	spin_lock_irqsave(&atchan->lock, flags);
1557
1558	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id));
1559	clear_bit(ATC_IS_PAUSED, &atchan->status);
1560
1561	spin_unlock_irqrestore(&atchan->lock, flags);
1562
1563	return 0;
1564}
1565
1566static int atc_terminate_all(struct dma_chan *chan)
1567{
1568	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1569	struct at_dma		*atdma = to_at_dma(chan->device);
1570	int			chan_id = atchan->chan_common.chan_id;
1571	struct at_desc		*desc, *_desc;
1572	unsigned long		flags;
1573
1574	LIST_HEAD(list);
1575
1576	dev_vdbg(chan2dev(chan), "%s\n", __func__);
1577
1578	/*
1579	 * This is only called when something went wrong elsewhere, so
1580	 * we don't really care about the data. Just disable the
1581	 * channel. We still have to poll the channel enable bit due
1582	 * to AHB/HSB limitations.
1583	 */
1584	spin_lock_irqsave(&atchan->lock, flags);
1585
1586	/* disabling channel: must also remove suspend state */
1587	dma_writel(atdma, CHDR, AT_DMA_RES(chan_id) | atchan->mask);
1588
1589	/* confirm that this channel is disabled */
1590	while (dma_readl(atdma, CHSR) & atchan->mask)
1591		cpu_relax();
1592
1593	/* active_list entries will end up before queued entries */
1594	list_splice_init(&atchan->queue, &list);
1595	list_splice_init(&atchan->active_list, &list);
1596
 
 
1597	/* Flush all pending and queued descriptors */
1598	list_for_each_entry_safe(desc, _desc, &list, desc_node)
1599		atc_chain_complete(atchan, desc);
1600
1601	clear_bit(ATC_IS_PAUSED, &atchan->status);
1602	/* if channel dedicated to cyclic operations, free it */
1603	clear_bit(ATC_IS_CYCLIC, &atchan->status);
1604
1605	spin_unlock_irqrestore(&atchan->lock, flags);
1606
1607	return 0;
1608}
1609
1610/**
1611 * atc_tx_status - poll for transaction completion
1612 * @chan: DMA channel
1613 * @cookie: transaction identifier to check status of
1614 * @txstate: if not %NULL updated with transaction state
1615 *
1616 * If @txstate is passed in, upon return it reflect the driver
1617 * internal state and can be used with dma_async_is_complete() to check
1618 * the status of multiple cookies without re-checking hardware state.
1619 */
1620static enum dma_status
1621atc_tx_status(struct dma_chan *chan,
1622		dma_cookie_t cookie,
1623		struct dma_tx_state *txstate)
1624{
1625	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1626	unsigned long		flags;
1627	enum dma_status		ret;
1628	int bytes = 0;
1629
1630	ret = dma_cookie_status(chan, cookie, txstate);
1631	if (ret == DMA_COMPLETE)
1632		return ret;
1633	/*
1634	 * There's no point calculating the residue if there's
1635	 * no txstate to store the value.
1636	 */
1637	if (!txstate)
1638		return DMA_ERROR;
1639
1640	spin_lock_irqsave(&atchan->lock, flags);
1641
1642	/*  Get number of bytes left in the active transactions */
1643	bytes = atc_get_bytes_left(chan, cookie);
1644
1645	spin_unlock_irqrestore(&atchan->lock, flags);
1646
1647	if (unlikely(bytes < 0)) {
1648		dev_vdbg(chan2dev(chan), "get residual bytes error\n");
1649		return DMA_ERROR;
1650	} else {
1651		dma_set_residue(txstate, bytes);
1652	}
1653
1654	dev_vdbg(chan2dev(chan), "tx_status %d: cookie = %d residue = %d\n",
1655		 ret, cookie, bytes);
1656
1657	return ret;
1658}
1659
1660/**
1661 * atc_issue_pending - try to finish work
1662 * @chan: target DMA channel
1663 */
1664static void atc_issue_pending(struct dma_chan *chan)
1665{
1666	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1667	unsigned long		flags;
1668
1669	dev_vdbg(chan2dev(chan), "issue_pending\n");
1670
1671	/* Not needed for cyclic transfers */
1672	if (atc_chan_is_cyclic(atchan))
1673		return;
1674
1675	spin_lock_irqsave(&atchan->lock, flags);
1676	atc_advance_work(atchan);
1677	spin_unlock_irqrestore(&atchan->lock, flags);
1678}
1679
1680/**
1681 * atc_alloc_chan_resources - allocate resources for DMA channel
1682 * @chan: allocate descriptor resources for this channel
1683 * @client: current client requesting the channel be ready for requests
1684 *
1685 * return - the number of allocated descriptors
1686 */
1687static int atc_alloc_chan_resources(struct dma_chan *chan)
1688{
1689	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1690	struct at_dma		*atdma = to_at_dma(chan->device);
1691	struct at_desc		*desc;
1692	struct at_dma_slave	*atslave;
1693	unsigned long		flags;
1694	int			i;
1695	u32			cfg;
1696	LIST_HEAD(tmp_list);
1697
1698	dev_vdbg(chan2dev(chan), "alloc_chan_resources\n");
1699
1700	/* ASSERT:  channel is idle */
1701	if (atc_chan_is_enabled(atchan)) {
1702		dev_dbg(chan2dev(chan), "DMA channel not idle ?\n");
1703		return -EIO;
1704	}
1705
 
 
 
 
 
1706	cfg = ATC_DEFAULT_CFG;
1707
1708	atslave = chan->private;
1709	if (atslave) {
1710		/*
1711		 * We need controller-specific data to set up slave
1712		 * transfers.
1713		 */
1714		BUG_ON(!atslave->dma_dev || atslave->dma_dev != atdma->dma_common.dev);
1715
1716		/* if cfg configuration specified take it instead of default */
1717		if (atslave->cfg)
1718			cfg = atslave->cfg;
1719	}
1720
1721	/* have we already been set up?
1722	 * reconfigure channel but no need to reallocate descriptors */
1723	if (!list_empty(&atchan->free_list))
1724		return atchan->descs_allocated;
1725
1726	/* Allocate initial pool of descriptors */
1727	for (i = 0; i < init_nr_desc_per_channel; i++) {
1728		desc = atc_alloc_descriptor(chan, GFP_KERNEL);
1729		if (!desc) {
1730			dev_err(atdma->dma_common.dev,
1731				"Only %d initial descriptors\n", i);
1732			break;
1733		}
1734		list_add_tail(&desc->desc_node, &tmp_list);
1735	}
1736
1737	spin_lock_irqsave(&atchan->lock, flags);
1738	atchan->descs_allocated = i;
1739	list_splice(&tmp_list, &atchan->free_list);
1740	dma_cookie_init(chan);
1741	spin_unlock_irqrestore(&atchan->lock, flags);
1742
1743	/* channel parameters */
1744	channel_writel(atchan, CFG, cfg);
1745
1746	dev_dbg(chan2dev(chan),
1747		"alloc_chan_resources: allocated %d descriptors\n",
1748		atchan->descs_allocated);
1749
1750	return atchan->descs_allocated;
1751}
1752
1753/**
1754 * atc_free_chan_resources - free all channel resources
1755 * @chan: DMA channel
1756 */
1757static void atc_free_chan_resources(struct dma_chan *chan)
1758{
1759	struct at_dma_chan	*atchan = to_at_dma_chan(chan);
1760	struct at_dma		*atdma = to_at_dma(chan->device);
1761	struct at_desc		*desc, *_desc;
1762	LIST_HEAD(list);
1763
1764	dev_dbg(chan2dev(chan), "free_chan_resources: (descs allocated=%u)\n",
1765		atchan->descs_allocated);
1766
1767	/* ASSERT:  channel is idle */
1768	BUG_ON(!list_empty(&atchan->active_list));
1769	BUG_ON(!list_empty(&atchan->queue));
1770	BUG_ON(atc_chan_is_enabled(atchan));
1771
1772	list_for_each_entry_safe(desc, _desc, &atchan->free_list, desc_node) {
1773		dev_vdbg(chan2dev(chan), "  freeing descriptor %p\n", desc);
1774		list_del(&desc->desc_node);
1775		/* free link descriptor */
1776		dma_pool_free(atdma->dma_desc_pool, desc, desc->txd.phys);
1777	}
1778	list_splice_init(&atchan->free_list, &list);
1779	atchan->descs_allocated = 0;
1780	atchan->status = 0;
1781
 
 
 
 
 
 
1782	dev_vdbg(chan2dev(chan), "free_chan_resources: done\n");
1783}
1784
1785#ifdef CONFIG_OF
1786static bool at_dma_filter(struct dma_chan *chan, void *slave)
1787{
1788	struct at_dma_slave *atslave = slave;
1789
1790	if (atslave->dma_dev == chan->device->dev) {
1791		chan->private = atslave;
1792		return true;
1793	} else {
1794		return false;
1795	}
1796}
1797
1798static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec,
1799				     struct of_dma *of_dma)
1800{
1801	struct dma_chan *chan;
1802	struct at_dma_chan *atchan;
1803	struct at_dma_slave *atslave;
1804	dma_cap_mask_t mask;
1805	unsigned int per_id;
1806	struct platform_device *dmac_pdev;
1807
1808	if (dma_spec->args_count != 2)
1809		return NULL;
1810
1811	dmac_pdev = of_find_device_by_node(dma_spec->np);
 
 
1812
1813	dma_cap_zero(mask);
1814	dma_cap_set(DMA_SLAVE, mask);
1815
1816	atslave = devm_kzalloc(&dmac_pdev->dev, sizeof(*atslave), GFP_KERNEL);
1817	if (!atslave)
 
1818		return NULL;
 
1819
1820	atslave->cfg = ATC_DST_H2SEL_HW | ATC_SRC_H2SEL_HW;
1821	/*
1822	 * We can fill both SRC_PER and DST_PER, one of these fields will be
1823	 * ignored depending on DMA transfer direction.
1824	 */
1825	per_id = dma_spec->args[1] & AT91_DMA_CFG_PER_ID_MASK;
1826	atslave->cfg |= ATC_DST_PER_MSB(per_id) | ATC_DST_PER(per_id)
1827		     | ATC_SRC_PER_MSB(per_id) | ATC_SRC_PER(per_id);
1828	/*
1829	 * We have to translate the value we get from the device tree since
1830	 * the half FIFO configuration value had to be 0 to keep backward
1831	 * compatibility.
1832	 */
1833	switch (dma_spec->args[1] & AT91_DMA_CFG_FIFOCFG_MASK) {
1834	case AT91_DMA_CFG_FIFOCFG_ALAP:
1835		atslave->cfg |= ATC_FIFOCFG_LARGESTBURST;
1836		break;
1837	case AT91_DMA_CFG_FIFOCFG_ASAP:
1838		atslave->cfg |= ATC_FIFOCFG_ENOUGHSPACE;
1839		break;
1840	case AT91_DMA_CFG_FIFOCFG_HALF:
1841	default:
1842		atslave->cfg |= ATC_FIFOCFG_HALFFIFO;
1843	}
1844	atslave->dma_dev = &dmac_pdev->dev;
1845
1846	chan = dma_request_channel(mask, at_dma_filter, atslave);
1847	if (!chan)
 
 
1848		return NULL;
 
1849
1850	atchan = to_at_dma_chan(chan);
1851	atchan->per_if = dma_spec->args[0] & 0xff;
1852	atchan->mem_if = (dma_spec->args[0] >> 16) & 0xff;
1853
1854	return chan;
1855}
1856#else
1857static struct dma_chan *at_dma_xlate(struct of_phandle_args *dma_spec,
1858				     struct of_dma *of_dma)
1859{
1860	return NULL;
1861}
1862#endif
1863
1864/*--  Module Management  -----------------------------------------------*/
1865
1866/* cap_mask is a multi-u32 bitfield, fill it with proper C code. */
1867static struct at_dma_platform_data at91sam9rl_config = {
1868	.nr_channels = 2,
1869};
1870static struct at_dma_platform_data at91sam9g45_config = {
1871	.nr_channels = 8,
1872};
1873
1874#if defined(CONFIG_OF)
1875static const struct of_device_id atmel_dma_dt_ids[] = {
1876	{
1877		.compatible = "atmel,at91sam9rl-dma",
1878		.data = &at91sam9rl_config,
1879	}, {
1880		.compatible = "atmel,at91sam9g45-dma",
1881		.data = &at91sam9g45_config,
1882	}, {
1883		/* sentinel */
1884	}
1885};
1886
1887MODULE_DEVICE_TABLE(of, atmel_dma_dt_ids);
1888#endif
1889
1890static const struct platform_device_id atdma_devtypes[] = {
1891	{
1892		.name = "at91sam9rl_dma",
1893		.driver_data = (unsigned long) &at91sam9rl_config,
1894	}, {
1895		.name = "at91sam9g45_dma",
1896		.driver_data = (unsigned long) &at91sam9g45_config,
1897	}, {
1898		/* sentinel */
1899	}
1900};
1901
1902static inline const struct at_dma_platform_data * __init at_dma_get_driver_data(
1903						struct platform_device *pdev)
1904{
1905	if (pdev->dev.of_node) {
1906		const struct of_device_id *match;
1907		match = of_match_node(atmel_dma_dt_ids, pdev->dev.of_node);
1908		if (match == NULL)
1909			return NULL;
1910		return match->data;
1911	}
1912	return (struct at_dma_platform_data *)
1913			platform_get_device_id(pdev)->driver_data;
1914}
1915
1916/**
1917 * at_dma_off - disable DMA controller
1918 * @atdma: the Atmel HDAMC device
1919 */
1920static void at_dma_off(struct at_dma *atdma)
1921{
1922	dma_writel(atdma, EN, 0);
1923
1924	/* disable all interrupts */
1925	dma_writel(atdma, EBCIDR, -1L);
1926
1927	/* confirm that all channels are disabled */
1928	while (dma_readl(atdma, CHSR) & atdma->all_chan_mask)
1929		cpu_relax();
1930}
1931
1932static int __init at_dma_probe(struct platform_device *pdev)
1933{
1934	struct resource		*io;
1935	struct at_dma		*atdma;
1936	size_t			size;
1937	int			irq;
1938	int			err;
1939	int			i;
1940	const struct at_dma_platform_data *plat_dat;
1941
1942	/* setup platform data for each SoC */
1943	dma_cap_set(DMA_MEMCPY, at91sam9rl_config.cap_mask);
1944	dma_cap_set(DMA_SG, at91sam9rl_config.cap_mask);
1945	dma_cap_set(DMA_INTERLEAVE, at91sam9g45_config.cap_mask);
1946	dma_cap_set(DMA_MEMCPY, at91sam9g45_config.cap_mask);
1947	dma_cap_set(DMA_MEMSET, at91sam9g45_config.cap_mask);
1948	dma_cap_set(DMA_MEMSET_SG, at91sam9g45_config.cap_mask);
1949	dma_cap_set(DMA_PRIVATE, at91sam9g45_config.cap_mask);
1950	dma_cap_set(DMA_SLAVE, at91sam9g45_config.cap_mask);
1951	dma_cap_set(DMA_SG, at91sam9g45_config.cap_mask);
1952
1953	/* get DMA parameters from controller type */
1954	plat_dat = at_dma_get_driver_data(pdev);
1955	if (!plat_dat)
1956		return -ENODEV;
1957
1958	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1959	if (!io)
1960		return -EINVAL;
1961
1962	irq = platform_get_irq(pdev, 0);
1963	if (irq < 0)
1964		return irq;
1965
1966	size = sizeof(struct at_dma);
1967	size += plat_dat->nr_channels * sizeof(struct at_dma_chan);
1968	atdma = kzalloc(size, GFP_KERNEL);
1969	if (!atdma)
1970		return -ENOMEM;
1971
1972	/* discover transaction capabilities */
1973	atdma->dma_common.cap_mask = plat_dat->cap_mask;
1974	atdma->all_chan_mask = (1 << plat_dat->nr_channels) - 1;
1975
1976	size = resource_size(io);
1977	if (!request_mem_region(io->start, size, pdev->dev.driver->name)) {
1978		err = -EBUSY;
1979		goto err_kfree;
1980	}
1981
1982	atdma->regs = ioremap(io->start, size);
1983	if (!atdma->regs) {
1984		err = -ENOMEM;
1985		goto err_release_r;
1986	}
1987
1988	atdma->clk = clk_get(&pdev->dev, "dma_clk");
1989	if (IS_ERR(atdma->clk)) {
1990		err = PTR_ERR(atdma->clk);
1991		goto err_clk;
1992	}
1993	err = clk_prepare_enable(atdma->clk);
1994	if (err)
1995		goto err_clk_prepare;
1996
1997	/* force dma off, just in case */
1998	at_dma_off(atdma);
1999
2000	err = request_irq(irq, at_dma_interrupt, 0, "at_hdmac", atdma);
2001	if (err)
2002		goto err_irq;
2003
2004	platform_set_drvdata(pdev, atdma);
2005
2006	/* create a pool of consistent memory blocks for hardware descriptors */
2007	atdma->dma_desc_pool = dma_pool_create("at_hdmac_desc_pool",
2008			&pdev->dev, sizeof(struct at_desc),
2009			4 /* word alignment */, 0);
2010	if (!atdma->dma_desc_pool) {
2011		dev_err(&pdev->dev, "No memory for descriptors dma pool\n");
2012		err = -ENOMEM;
2013		goto err_desc_pool_create;
2014	}
2015
2016	/* create a pool of consistent memory blocks for memset blocks */
2017	atdma->memset_pool = dma_pool_create("at_hdmac_memset_pool",
2018					     &pdev->dev, sizeof(int), 4, 0);
2019	if (!atdma->memset_pool) {
2020		dev_err(&pdev->dev, "No memory for memset dma pool\n");
2021		err = -ENOMEM;
2022		goto err_memset_pool_create;
2023	}
2024
2025	/* clear any pending interrupt */
2026	while (dma_readl(atdma, EBCISR))
2027		cpu_relax();
2028
2029	/* initialize channels related values */
2030	INIT_LIST_HEAD(&atdma->dma_common.channels);
2031	for (i = 0; i < plat_dat->nr_channels; i++) {
2032		struct at_dma_chan	*atchan = &atdma->chan[i];
2033
2034		atchan->mem_if = AT_DMA_MEM_IF;
2035		atchan->per_if = AT_DMA_PER_IF;
2036		atchan->chan_common.device = &atdma->dma_common;
2037		dma_cookie_init(&atchan->chan_common);
2038		list_add_tail(&atchan->chan_common.device_node,
2039				&atdma->dma_common.channels);
2040
2041		atchan->ch_regs = atdma->regs + ch_regs(i);
2042		spin_lock_init(&atchan->lock);
2043		atchan->mask = 1 << i;
2044
2045		INIT_LIST_HEAD(&atchan->active_list);
2046		INIT_LIST_HEAD(&atchan->queue);
2047		INIT_LIST_HEAD(&atchan->free_list);
2048
2049		tasklet_init(&atchan->tasklet, atc_tasklet,
2050				(unsigned long)atchan);
2051		atc_enable_chan_irq(atdma, i);
2052	}
2053
2054	/* set base routines */
2055	atdma->dma_common.device_alloc_chan_resources = atc_alloc_chan_resources;
2056	atdma->dma_common.device_free_chan_resources = atc_free_chan_resources;
2057	atdma->dma_common.device_tx_status = atc_tx_status;
2058	atdma->dma_common.device_issue_pending = atc_issue_pending;
2059	atdma->dma_common.dev = &pdev->dev;
2060
2061	/* set prep routines based on capability */
2062	if (dma_has_cap(DMA_INTERLEAVE, atdma->dma_common.cap_mask))
2063		atdma->dma_common.device_prep_interleaved_dma = atc_prep_dma_interleaved;
2064
2065	if (dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask))
2066		atdma->dma_common.device_prep_dma_memcpy = atc_prep_dma_memcpy;
2067
2068	if (dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask)) {
2069		atdma->dma_common.device_prep_dma_memset = atc_prep_dma_memset;
2070		atdma->dma_common.device_prep_dma_memset_sg = atc_prep_dma_memset_sg;
2071		atdma->dma_common.fill_align = DMAENGINE_ALIGN_4_BYTES;
2072	}
2073
2074	if (dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)) {
2075		atdma->dma_common.device_prep_slave_sg = atc_prep_slave_sg;
2076		/* controller can do slave DMA: can trigger cyclic transfers */
2077		dma_cap_set(DMA_CYCLIC, atdma->dma_common.cap_mask);
2078		atdma->dma_common.device_prep_dma_cyclic = atc_prep_dma_cyclic;
2079		atdma->dma_common.device_config = atc_config;
2080		atdma->dma_common.device_pause = atc_pause;
2081		atdma->dma_common.device_resume = atc_resume;
2082		atdma->dma_common.device_terminate_all = atc_terminate_all;
2083		atdma->dma_common.src_addr_widths = ATC_DMA_BUSWIDTHS;
2084		atdma->dma_common.dst_addr_widths = ATC_DMA_BUSWIDTHS;
2085		atdma->dma_common.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2086		atdma->dma_common.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
2087	}
2088
2089	if (dma_has_cap(DMA_SG, atdma->dma_common.cap_mask))
2090		atdma->dma_common.device_prep_dma_sg = atc_prep_dma_sg;
2091
2092	dma_writel(atdma, EN, AT_DMA_ENABLE);
2093
2094	dev_info(&pdev->dev, "Atmel AHB DMA Controller ( %s%s%s%s), %d channels\n",
2095	  dma_has_cap(DMA_MEMCPY, atdma->dma_common.cap_mask) ? "cpy " : "",
2096	  dma_has_cap(DMA_MEMSET, atdma->dma_common.cap_mask) ? "set " : "",
2097	  dma_has_cap(DMA_SLAVE, atdma->dma_common.cap_mask)  ? "slave " : "",
2098	  dma_has_cap(DMA_SG, atdma->dma_common.cap_mask)  ? "sg-cpy " : "",
2099	  plat_dat->nr_channels);
2100
2101	dma_async_device_register(&atdma->dma_common);
2102
2103	/*
2104	 * Do not return an error if the dmac node is not present in order to
2105	 * not break the existing way of requesting channel with
2106	 * dma_request_channel().
2107	 */
2108	if (pdev->dev.of_node) {
2109		err = of_dma_controller_register(pdev->dev.of_node,
2110						 at_dma_xlate, atdma);
2111		if (err) {
2112			dev_err(&pdev->dev, "could not register of_dma_controller\n");
2113			goto err_of_dma_controller_register;
2114		}
2115	}
2116
2117	return 0;
2118
2119err_of_dma_controller_register:
2120	dma_async_device_unregister(&atdma->dma_common);
2121	dma_pool_destroy(atdma->memset_pool);
2122err_memset_pool_create:
2123	dma_pool_destroy(atdma->dma_desc_pool);
2124err_desc_pool_create:
2125	free_irq(platform_get_irq(pdev, 0), atdma);
2126err_irq:
2127	clk_disable_unprepare(atdma->clk);
2128err_clk_prepare:
2129	clk_put(atdma->clk);
2130err_clk:
2131	iounmap(atdma->regs);
2132	atdma->regs = NULL;
2133err_release_r:
2134	release_mem_region(io->start, size);
2135err_kfree:
2136	kfree(atdma);
2137	return err;
2138}
2139
2140static int at_dma_remove(struct platform_device *pdev)
2141{
2142	struct at_dma		*atdma = platform_get_drvdata(pdev);
2143	struct dma_chan		*chan, *_chan;
2144	struct resource		*io;
2145
2146	at_dma_off(atdma);
 
 
2147	dma_async_device_unregister(&atdma->dma_common);
2148
2149	dma_pool_destroy(atdma->memset_pool);
2150	dma_pool_destroy(atdma->dma_desc_pool);
2151	free_irq(platform_get_irq(pdev, 0), atdma);
2152
2153	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
2154			device_node) {
2155		struct at_dma_chan	*atchan = to_at_dma_chan(chan);
2156
2157		/* Disable interrupts */
2158		atc_disable_chan_irq(atdma, chan->chan_id);
2159
2160		tasklet_kill(&atchan->tasklet);
2161		list_del(&chan->device_node);
2162	}
2163
2164	clk_disable_unprepare(atdma->clk);
2165	clk_put(atdma->clk);
2166
2167	iounmap(atdma->regs);
2168	atdma->regs = NULL;
2169
2170	io = platform_get_resource(pdev, IORESOURCE_MEM, 0);
2171	release_mem_region(io->start, resource_size(io));
2172
2173	kfree(atdma);
2174
2175	return 0;
2176}
2177
2178static void at_dma_shutdown(struct platform_device *pdev)
2179{
2180	struct at_dma	*atdma = platform_get_drvdata(pdev);
2181
2182	at_dma_off(platform_get_drvdata(pdev));
2183	clk_disable_unprepare(atdma->clk);
2184}
2185
2186static int at_dma_prepare(struct device *dev)
2187{
2188	struct platform_device *pdev = to_platform_device(dev);
2189	struct at_dma *atdma = platform_get_drvdata(pdev);
2190	struct dma_chan *chan, *_chan;
2191
2192	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
2193			device_node) {
2194		struct at_dma_chan *atchan = to_at_dma_chan(chan);
2195		/* wait for transaction completion (except in cyclic case) */
2196		if (atc_chan_is_enabled(atchan) && !atc_chan_is_cyclic(atchan))
2197			return -EAGAIN;
2198	}
2199	return 0;
2200}
2201
2202static void atc_suspend_cyclic(struct at_dma_chan *atchan)
2203{
2204	struct dma_chan	*chan = &atchan->chan_common;
2205
2206	/* Channel should be paused by user
2207	 * do it anyway even if it is not done already */
2208	if (!atc_chan_is_paused(atchan)) {
2209		dev_warn(chan2dev(chan),
2210		"cyclic channel not paused, should be done by channel user\n");
2211		atc_pause(chan);
2212	}
2213
2214	/* now preserve additional data for cyclic operations */
2215	/* next descriptor address in the cyclic list */
2216	atchan->save_dscr = channel_readl(atchan, DSCR);
2217
2218	vdbg_dump_regs(atchan);
2219}
2220
2221static int at_dma_suspend_noirq(struct device *dev)
2222{
2223	struct platform_device *pdev = to_platform_device(dev);
2224	struct at_dma *atdma = platform_get_drvdata(pdev);
2225	struct dma_chan *chan, *_chan;
2226
2227	/* preserve data */
2228	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
2229			device_node) {
2230		struct at_dma_chan *atchan = to_at_dma_chan(chan);
2231
2232		if (atc_chan_is_cyclic(atchan))
2233			atc_suspend_cyclic(atchan);
2234		atchan->save_cfg = channel_readl(atchan, CFG);
2235	}
2236	atdma->save_imr = dma_readl(atdma, EBCIMR);
2237
2238	/* disable DMA controller */
2239	at_dma_off(atdma);
2240	clk_disable_unprepare(atdma->clk);
2241	return 0;
2242}
2243
2244static void atc_resume_cyclic(struct at_dma_chan *atchan)
2245{
2246	struct at_dma	*atdma = to_at_dma(atchan->chan_common.device);
2247
2248	/* restore channel status for cyclic descriptors list:
2249	 * next descriptor in the cyclic list at the time of suspend */
2250	channel_writel(atchan, SADDR, 0);
2251	channel_writel(atchan, DADDR, 0);
2252	channel_writel(atchan, CTRLA, 0);
2253	channel_writel(atchan, CTRLB, 0);
2254	channel_writel(atchan, DSCR, atchan->save_dscr);
2255	dma_writel(atdma, CHER, atchan->mask);
2256
2257	/* channel pause status should be removed by channel user
2258	 * We cannot take the initiative to do it here */
2259
2260	vdbg_dump_regs(atchan);
2261}
2262
2263static int at_dma_resume_noirq(struct device *dev)
2264{
2265	struct platform_device *pdev = to_platform_device(dev);
2266	struct at_dma *atdma = platform_get_drvdata(pdev);
2267	struct dma_chan *chan, *_chan;
2268
2269	/* bring back DMA controller */
2270	clk_prepare_enable(atdma->clk);
2271	dma_writel(atdma, EN, AT_DMA_ENABLE);
2272
2273	/* clear any pending interrupt */
2274	while (dma_readl(atdma, EBCISR))
2275		cpu_relax();
2276
2277	/* restore saved data */
2278	dma_writel(atdma, EBCIER, atdma->save_imr);
2279	list_for_each_entry_safe(chan, _chan, &atdma->dma_common.channels,
2280			device_node) {
2281		struct at_dma_chan *atchan = to_at_dma_chan(chan);
2282
2283		channel_writel(atchan, CFG, atchan->save_cfg);
2284		if (atc_chan_is_cyclic(atchan))
2285			atc_resume_cyclic(atchan);
2286	}
2287	return 0;
2288}
2289
2290static const struct dev_pm_ops at_dma_dev_pm_ops = {
2291	.prepare = at_dma_prepare,
2292	.suspend_noirq = at_dma_suspend_noirq,
2293	.resume_noirq = at_dma_resume_noirq,
2294};
2295
2296static struct platform_driver at_dma_driver = {
2297	.remove		= at_dma_remove,
2298	.shutdown	= at_dma_shutdown,
2299	.id_table	= atdma_devtypes,
2300	.driver = {
2301		.name	= "at_hdmac",
2302		.pm	= &at_dma_dev_pm_ops,
2303		.of_match_table	= of_match_ptr(atmel_dma_dt_ids),
2304	},
2305};
2306
2307static int __init at_dma_init(void)
2308{
2309	return platform_driver_probe(&at_dma_driver, at_dma_probe);
2310}
2311subsys_initcall(at_dma_init);
2312
2313static void __exit at_dma_exit(void)
2314{
2315	platform_driver_unregister(&at_dma_driver);
2316}
2317module_exit(at_dma_exit);
2318
2319MODULE_DESCRIPTION("Atmel AHB DMA Controller driver");
2320MODULE_AUTHOR("Nicolas Ferre <nicolas.ferre@atmel.com>");
2321MODULE_LICENSE("GPL");
2322MODULE_ALIAS("platform:at_hdmac");