1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Driver for the Atmel Extensible DMA Controller (aka XDMAC on AT91 systems)
   4 *
   5 * Copyright (C) 2014 Atmel Corporation
   6 *
   7 * Author: Ludovic Desroches <ludovic.desroches@atmel.com>
 
 
 
 
 
 
 
 
 
 
 
 
   8 */
   9
  10#include <asm/barrier.h>
  11#include <dt-bindings/dma/at91.h>
  12#include <linux/clk.h>
  13#include <linux/dmaengine.h>
  14#include <linux/dmapool.h>
  15#include <linux/interrupt.h>
  16#include <linux/irq.h>
  17#include <linux/kernel.h>
  18#include <linux/list.h>
  19#include <linux/module.h>
  20#include <linux/of_dma.h>
  21#include <linux/of_platform.h>
  22#include <linux/platform_device.h>
  23#include <linux/pm.h>
  24
  25#include "dmaengine.h"
  26
  27/* Global registers */
  28#define AT_XDMAC_GTYPE		0x00	/* Global Type Register */
  29#define		AT_XDMAC_NB_CH(i)	(((i) & 0x1F) + 1)		/* Number of Channels Minus One */
  30#define		AT_XDMAC_FIFO_SZ(i)	(((i) >> 5) & 0x7FF)		/* Number of Bytes */
  31#define		AT_XDMAC_NB_REQ(i)	((((i) >> 16) & 0x3F) + 1)	/* Number of Peripheral Requests Minus One */
  32#define AT_XDMAC_GCFG		0x04	/* Global Configuration Register */
  33#define AT_XDMAC_GWAC		0x08	/* Global Weighted Arbiter Configuration Register */
  34#define AT_XDMAC_GIE		0x0C	/* Global Interrupt Enable Register */
  35#define AT_XDMAC_GID		0x10	/* Global Interrupt Disable Register */
  36#define AT_XDMAC_GIM		0x14	/* Global Interrupt Mask Register */
  37#define AT_XDMAC_GIS		0x18	/* Global Interrupt Status Register */
  38#define AT_XDMAC_GE		0x1C	/* Global Channel Enable Register */
  39#define AT_XDMAC_GD		0x20	/* Global Channel Disable Register */
  40#define AT_XDMAC_GS		0x24	/* Global Channel Status Register */
  41#define AT_XDMAC_GRS		0x28	/* Global Channel Read Suspend Register */
  42#define AT_XDMAC_GWS		0x2C	/* Global Write Suspend Register */
  43#define AT_XDMAC_GRWS		0x30	/* Global Channel Read Write Suspend Register */
  44#define AT_XDMAC_GRWR		0x34	/* Global Channel Read Write Resume Register */
  45#define AT_XDMAC_GSWR		0x38	/* Global Channel Software Request Register */
  46#define AT_XDMAC_GSWS		0x3C	/* Global channel Software Request Status Register */
  47#define AT_XDMAC_GSWF		0x40	/* Global Channel Software Flush Request Register */
  48#define AT_XDMAC_VERSION	0xFFC	/* XDMAC Version Register */
  49
  50/* Channel relative registers offsets */
  51#define AT_XDMAC_CIE		0x00	/* Channel Interrupt Enable Register */
  52#define		AT_XDMAC_CIE_BIE	BIT(0)	/* End of Block Interrupt Enable Bit */
  53#define		AT_XDMAC_CIE_LIE	BIT(1)	/* End of Linked List Interrupt Enable Bit */
  54#define		AT_XDMAC_CIE_DIE	BIT(2)	/* End of Disable Interrupt Enable Bit */
  55#define		AT_XDMAC_CIE_FIE	BIT(3)	/* End of Flush Interrupt Enable Bit */
  56#define		AT_XDMAC_CIE_RBEIE	BIT(4)	/* Read Bus Error Interrupt Enable Bit */
  57#define		AT_XDMAC_CIE_WBEIE	BIT(5)	/* Write Bus Error Interrupt Enable Bit */
  58#define		AT_XDMAC_CIE_ROIE	BIT(6)	/* Request Overflow Interrupt Enable Bit */
  59#define AT_XDMAC_CID		0x04	/* Channel Interrupt Disable Register */
  60#define		AT_XDMAC_CID_BID	BIT(0)	/* End of Block Interrupt Disable Bit */
  61#define		AT_XDMAC_CID_LID	BIT(1)	/* End of Linked List Interrupt Disable Bit */
  62#define		AT_XDMAC_CID_DID	BIT(2)	/* End of Disable Interrupt Disable Bit */
  63#define		AT_XDMAC_CID_FID	BIT(3)	/* End of Flush Interrupt Disable Bit */
  64#define		AT_XDMAC_CID_RBEID	BIT(4)	/* Read Bus Error Interrupt Disable Bit */
  65#define		AT_XDMAC_CID_WBEID	BIT(5)	/* Write Bus Error Interrupt Disable Bit */
  66#define		AT_XDMAC_CID_ROID	BIT(6)	/* Request Overflow Interrupt Disable Bit */
  67#define AT_XDMAC_CIM		0x08	/* Channel Interrupt Mask Register */
  68#define		AT_XDMAC_CIM_BIM	BIT(0)	/* End of Block Interrupt Mask Bit */
  69#define		AT_XDMAC_CIM_LIM	BIT(1)	/* End of Linked List Interrupt Mask Bit */
  70#define		AT_XDMAC_CIM_DIM	BIT(2)	/* End of Disable Interrupt Mask Bit */
  71#define		AT_XDMAC_CIM_FIM	BIT(3)	/* End of Flush Interrupt Mask Bit */
  72#define		AT_XDMAC_CIM_RBEIM	BIT(4)	/* Read Bus Error Interrupt Mask Bit */
  73#define		AT_XDMAC_CIM_WBEIM	BIT(5)	/* Write Bus Error Interrupt Mask Bit */
  74#define		AT_XDMAC_CIM_ROIM	BIT(6)	/* Request Overflow Interrupt Mask Bit */
  75#define AT_XDMAC_CIS		0x0C	/* Channel Interrupt Status Register */
  76#define		AT_XDMAC_CIS_BIS	BIT(0)	/* End of Block Interrupt Status Bit */
  77#define		AT_XDMAC_CIS_LIS	BIT(1)	/* End of Linked List Interrupt Status Bit */
  78#define		AT_XDMAC_CIS_DIS	BIT(2)	/* End of Disable Interrupt Status Bit */
  79#define		AT_XDMAC_CIS_FIS	BIT(3)	/* End of Flush Interrupt Status Bit */
  80#define		AT_XDMAC_CIS_RBEIS	BIT(4)	/* Read Bus Error Interrupt Status Bit */
  81#define		AT_XDMAC_CIS_WBEIS	BIT(5)	/* Write Bus Error Interrupt Status Bit */
  82#define		AT_XDMAC_CIS_ROIS	BIT(6)	/* Request Overflow Interrupt Status Bit */
  83#define AT_XDMAC_CSA		0x10	/* Channel Source Address Register */
  84#define AT_XDMAC_CDA		0x14	/* Channel Destination Address Register */
  85#define AT_XDMAC_CNDA		0x18	/* Channel Next Descriptor Address Register */
  86#define		AT_XDMAC_CNDA_NDAIF(i)	((i) & 0x1)			/* Channel x Next Descriptor Interface */
  87#define		AT_XDMAC_CNDA_NDA(i)	((i) & 0xfffffffc)		/* Channel x Next Descriptor Address */
  88#define AT_XDMAC_CNDC		0x1C	/* Channel Next Descriptor Control Register */
  89#define		AT_XDMAC_CNDC_NDE		(0x1 << 0)		/* Channel x Next Descriptor Enable */
  90#define		AT_XDMAC_CNDC_NDSUP		(0x1 << 1)		/* Channel x Next Descriptor Source Update */
  91#define		AT_XDMAC_CNDC_NDDUP		(0x1 << 2)		/* Channel x Next Descriptor Destination Update */
  92#define		AT_XDMAC_CNDC_NDVIEW_NDV0	(0x0 << 3)		/* Channel x Next Descriptor View 0 */
  93#define		AT_XDMAC_CNDC_NDVIEW_NDV1	(0x1 << 3)		/* Channel x Next Descriptor View 1 */
  94#define		AT_XDMAC_CNDC_NDVIEW_NDV2	(0x2 << 3)		/* Channel x Next Descriptor View 2 */
  95#define		AT_XDMAC_CNDC_NDVIEW_NDV3	(0x3 << 3)		/* Channel x Next Descriptor View 3 */
  96#define AT_XDMAC_CUBC		0x20	/* Channel Microblock Control Register */
  97#define AT_XDMAC_CBC		0x24	/* Channel Block Control Register */
  98#define AT_XDMAC_CC		0x28	/* Channel Configuration Register */
  99#define		AT_XDMAC_CC_TYPE	(0x1 << 0)	/* Channel Transfer Type */
 100#define			AT_XDMAC_CC_TYPE_MEM_TRAN	(0x0 << 0)	/* Memory to Memory Transfer */
 101#define			AT_XDMAC_CC_TYPE_PER_TRAN	(0x1 << 0)	/* Peripheral to Memory or Memory to Peripheral Transfer */
 102#define		AT_XDMAC_CC_MBSIZE_MASK	(0x3 << 1)
 103#define			AT_XDMAC_CC_MBSIZE_SINGLE	(0x0 << 1)
 104#define			AT_XDMAC_CC_MBSIZE_FOUR		(0x1 << 1)
 105#define			AT_XDMAC_CC_MBSIZE_EIGHT	(0x2 << 1)
 106#define			AT_XDMAC_CC_MBSIZE_SIXTEEN	(0x3 << 1)
 107#define		AT_XDMAC_CC_DSYNC	(0x1 << 4)	/* Channel Synchronization */
 108#define			AT_XDMAC_CC_DSYNC_PER2MEM	(0x0 << 4)
 109#define			AT_XDMAC_CC_DSYNC_MEM2PER	(0x1 << 4)
 110#define		AT_XDMAC_CC_PROT	(0x1 << 5)	/* Channel Protection */
 111#define			AT_XDMAC_CC_PROT_SEC		(0x0 << 5)
 112#define			AT_XDMAC_CC_PROT_UNSEC		(0x1 << 5)
 113#define		AT_XDMAC_CC_SWREQ	(0x1 << 6)	/* Channel Software Request Trigger */
 114#define			AT_XDMAC_CC_SWREQ_HWR_CONNECTED	(0x0 << 6)
 115#define			AT_XDMAC_CC_SWREQ_SWR_CONNECTED	(0x1 << 6)
 116#define		AT_XDMAC_CC_MEMSET	(0x1 << 7)	/* Channel Fill Block of memory */
 117#define			AT_XDMAC_CC_MEMSET_NORMAL_MODE	(0x0 << 7)
 118#define			AT_XDMAC_CC_MEMSET_HW_MODE	(0x1 << 7)
 119#define		AT_XDMAC_CC_CSIZE(i)	((0x7 & (i)) << 8)	/* Channel Chunk Size */
 120#define		AT_XDMAC_CC_DWIDTH_OFFSET	11
 121#define		AT_XDMAC_CC_DWIDTH_MASK	(0x3 << AT_XDMAC_CC_DWIDTH_OFFSET)
 122#define		AT_XDMAC_CC_DWIDTH(i)	((0x3 & (i)) << AT_XDMAC_CC_DWIDTH_OFFSET)	/* Channel Data Width */
 123#define			AT_XDMAC_CC_DWIDTH_BYTE		0x0
 124#define			AT_XDMAC_CC_DWIDTH_HALFWORD	0x1
 125#define			AT_XDMAC_CC_DWIDTH_WORD		0x2
 126#define			AT_XDMAC_CC_DWIDTH_DWORD	0x3
 127#define		AT_XDMAC_CC_SIF(i)	((0x1 & (i)) << 13)	/* Channel Source Interface Identifier */
 128#define		AT_XDMAC_CC_DIF(i)	((0x1 & (i)) << 14)	/* Channel Destination Interface Identifier */
 129#define		AT_XDMAC_CC_SAM_MASK	(0x3 << 16)	/* Channel Source Addressing Mode */
 130#define			AT_XDMAC_CC_SAM_FIXED_AM	(0x0 << 16)
 131#define			AT_XDMAC_CC_SAM_INCREMENTED_AM	(0x1 << 16)
 132#define			AT_XDMAC_CC_SAM_UBS_AM		(0x2 << 16)
 133#define			AT_XDMAC_CC_SAM_UBS_DS_AM	(0x3 << 16)
 134#define		AT_XDMAC_CC_DAM_MASK	(0x3 << 18)	/* Channel Source Addressing Mode */
 135#define			AT_XDMAC_CC_DAM_FIXED_AM	(0x0 << 18)
 136#define			AT_XDMAC_CC_DAM_INCREMENTED_AM	(0x1 << 18)
 137#define			AT_XDMAC_CC_DAM_UBS_AM		(0x2 << 18)
 138#define			AT_XDMAC_CC_DAM_UBS_DS_AM	(0x3 << 18)
 139#define		AT_XDMAC_CC_INITD	(0x1 << 21)	/* Channel Initialization Terminated (read only) */
 140#define			AT_XDMAC_CC_INITD_TERMINATED	(0x0 << 21)
 141#define			AT_XDMAC_CC_INITD_IN_PROGRESS	(0x1 << 21)
 142#define		AT_XDMAC_CC_RDIP	(0x1 << 22)	/* Read in Progress (read only) */
 143#define			AT_XDMAC_CC_RDIP_DONE		(0x0 << 22)
 144#define			AT_XDMAC_CC_RDIP_IN_PROGRESS	(0x1 << 22)
 145#define		AT_XDMAC_CC_WRIP	(0x1 << 23)	/* Write in Progress (read only) */
 146#define			AT_XDMAC_CC_WRIP_DONE		(0x0 << 23)
 147#define			AT_XDMAC_CC_WRIP_IN_PROGRESS	(0x1 << 23)
 148#define		AT_XDMAC_CC_PERID(i)	(0x7f & (i) << 24)	/* Channel Peripheral Identifier */
 149#define AT_XDMAC_CDS_MSP	0x2C	/* Channel Data Stride Memory Set Pattern */
 150#define AT_XDMAC_CSUS		0x30	/* Channel Source Microblock Stride */
 151#define AT_XDMAC_CDUS		0x34	/* Channel Destination Microblock Stride */
 152
 153#define AT_XDMAC_CHAN_REG_BASE	0x50	/* Channel registers base address */
 154
 155/* Microblock control members */
 156#define AT_XDMAC_MBR_UBC_UBLEN_MAX	0xFFFFFFUL	/* Maximum Microblock Length */
 157#define AT_XDMAC_MBR_UBC_NDE		(0x1 << 24)	/* Next Descriptor Enable */
 158#define AT_XDMAC_MBR_UBC_NSEN		(0x1 << 25)	/* Next Descriptor Source Update */
 159#define AT_XDMAC_MBR_UBC_NDEN		(0x1 << 26)	/* Next Descriptor Destination Update */
 160#define AT_XDMAC_MBR_UBC_NDV0		(0x0 << 27)	/* Next Descriptor View 0 */
 161#define AT_XDMAC_MBR_UBC_NDV1		(0x1 << 27)	/* Next Descriptor View 1 */
 162#define AT_XDMAC_MBR_UBC_NDV2		(0x2 << 27)	/* Next Descriptor View 2 */
 163#define AT_XDMAC_MBR_UBC_NDV3		(0x3 << 27)	/* Next Descriptor View 3 */
 164
 165#define AT_XDMAC_MAX_CHAN	0x20
 166#define AT_XDMAC_MAX_CSIZE	16	/* 16 data */
 167#define AT_XDMAC_MAX_DWIDTH	8	/* 64 bits */
 168#define AT_XDMAC_RESIDUE_MAX_RETRIES	5
 169
 170#define AT_XDMAC_DMA_BUSWIDTHS\
 171	(BIT(DMA_SLAVE_BUSWIDTH_UNDEFINED) |\
 172	BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |\
 173	BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |\
 174	BIT(DMA_SLAVE_BUSWIDTH_4_BYTES) |\
 175	BIT(DMA_SLAVE_BUSWIDTH_8_BYTES))
 176
 177enum atc_status {
 178	AT_XDMAC_CHAN_IS_CYCLIC = 0,
 179	AT_XDMAC_CHAN_IS_PAUSED,
 180};
 181
 182/* ----- Channels ----- */
 183struct at_xdmac_chan {
 184	struct dma_chan			chan;
 185	void __iomem			*ch_regs;
 186	u32				mask;		/* Channel Mask */
 187	u32				cfg;		/* Channel Configuration Register */
 188	u8				perid;		/* Peripheral ID */
 189	u8				perif;		/* Peripheral Interface */
 190	u8				memif;		/* Memory Interface */
 191	u32				save_cc;
 192	u32				save_cim;
 193	u32				save_cnda;
 194	u32				save_cndc;
 195	u32				irq_status;
 196	unsigned long			status;
 197	struct tasklet_struct		tasklet;
 198	struct dma_slave_config		sconfig;
 199
 200	spinlock_t			lock;
 201
 202	struct list_head		xfers_list;
 203	struct list_head		free_descs_list;
 204};
 205
 206
 207/* ----- Controller ----- */
 208struct at_xdmac {
 209	struct dma_device	dma;
 210	void __iomem		*regs;
 211	int			irq;
 212	struct clk		*clk;
 213	u32			save_gim;
 214	struct dma_pool		*at_xdmac_desc_pool;
 215	struct at_xdmac_chan	chan[];
 216};
 217
 218
 219/* ----- Descriptors ----- */
 220
 221/* Linked List Descriptor */
 222struct at_xdmac_lld {
 223	dma_addr_t	mbr_nda;	/* Next Descriptor Member */
 224	u32		mbr_ubc;	/* Microblock Control Member */
 225	dma_addr_t	mbr_sa;		/* Source Address Member */
 226	dma_addr_t	mbr_da;		/* Destination Address Member */
 227	u32		mbr_cfg;	/* Configuration Register */
 228	u32		mbr_bc;		/* Block Control Register */
 229	u32		mbr_ds;		/* Data Stride Register */
 230	u32		mbr_sus;	/* Source Microblock Stride Register */
 231	u32		mbr_dus;	/* Destination Microblock Stride Register */
 232};
 233
 234/* 64-bit alignment needed to update CNDA and CUBC registers in an atomic way. */
 235struct at_xdmac_desc {
 236	struct at_xdmac_lld		lld;
 237	enum dma_transfer_direction	direction;
 238	struct dma_async_tx_descriptor	tx_dma_desc;
 239	struct list_head		desc_node;
 240	/* Following members are only used by the first descriptor */
 241	bool				active_xfer;
 242	unsigned int			xfer_size;
 243	struct list_head		descs_list;
 244	struct list_head		xfer_node;
 245} __aligned(sizeof(u64));
 246
 247static inline void __iomem *at_xdmac_chan_reg_base(struct at_xdmac *atxdmac, unsigned int chan_nb)
 248{
 249	return atxdmac->regs + (AT_XDMAC_CHAN_REG_BASE + chan_nb * 0x40);
 250}
 251
 252#define at_xdmac_read(atxdmac, reg) readl_relaxed((atxdmac)->regs + (reg))
 253#define at_xdmac_write(atxdmac, reg, value) \
 254	writel_relaxed((value), (atxdmac)->regs + (reg))
 255
 256#define at_xdmac_chan_read(atchan, reg) readl_relaxed((atchan)->ch_regs + (reg))
 257#define at_xdmac_chan_write(atchan, reg, value) writel_relaxed((value), (atchan)->ch_regs + (reg))
 258
 259static inline struct at_xdmac_chan *to_at_xdmac_chan(struct dma_chan *dchan)
 260{
 261	return container_of(dchan, struct at_xdmac_chan, chan);
 262}
 263
 264static struct device *chan2dev(struct dma_chan *chan)
 265{
 266	return &chan->dev->device;
 267}
 268
 269static inline struct at_xdmac *to_at_xdmac(struct dma_device *ddev)
 270{
 271	return container_of(ddev, struct at_xdmac, dma);
 272}
 273
 274static inline struct at_xdmac_desc *txd_to_at_desc(struct dma_async_tx_descriptor *txd)
 275{
 276	return container_of(txd, struct at_xdmac_desc, tx_dma_desc);
 277}
 278
 279static inline int at_xdmac_chan_is_cyclic(struct at_xdmac_chan *atchan)
 280{
 281	return test_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);
 282}
 283
 284static inline int at_xdmac_chan_is_paused(struct at_xdmac_chan *atchan)
 285{
 286	return test_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
 287}
 288
 289static inline int at_xdmac_csize(u32 maxburst)
 290{
 291	int csize;
 292
 293	csize = ffs(maxburst) - 1;
 294	if (csize > 4)
 295		csize = -EINVAL;
 296
 297	return csize;
 298};
 299
 300static inline bool at_xdmac_chan_is_peripheral_xfer(u32 cfg)
 301{
 302	return cfg & AT_XDMAC_CC_TYPE_PER_TRAN;
 303}
 304
 305static inline u8 at_xdmac_get_dwidth(u32 cfg)
 306{
 307	return (cfg & AT_XDMAC_CC_DWIDTH_MASK) >> AT_XDMAC_CC_DWIDTH_OFFSET;
 308};
 309
 310static unsigned int init_nr_desc_per_channel = 64;
 311module_param(init_nr_desc_per_channel, uint, 0644);
 312MODULE_PARM_DESC(init_nr_desc_per_channel,
 313		 "initial descriptors per channel (default: 64)");
 314
 315
 316static bool at_xdmac_chan_is_enabled(struct at_xdmac_chan *atchan)
 317{
 318	return at_xdmac_chan_read(atchan, AT_XDMAC_GS) & atchan->mask;
 319}
 320
 321static void at_xdmac_off(struct at_xdmac *atxdmac)
 322{
 323	at_xdmac_write(atxdmac, AT_XDMAC_GD, -1L);
 324
 325	/* Wait that all chans are disabled. */
 326	while (at_xdmac_read(atxdmac, AT_XDMAC_GS))
 327		cpu_relax();
 328
 329	at_xdmac_write(atxdmac, AT_XDMAC_GID, -1L);
 330}
 331
 332/* Call with lock hold. */
 333static void at_xdmac_start_xfer(struct at_xdmac_chan *atchan,
 334				struct at_xdmac_desc *first)
 335{
 336	struct at_xdmac	*atxdmac = to_at_xdmac(atchan->chan.device);
 337	u32		reg;
 338
 339	dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, first);
 340
 341	if (at_xdmac_chan_is_enabled(atchan))
 342		return;
 343
 344	/* Set transfer as active to not try to start it again. */
 345	first->active_xfer = true;
 346
 347	/* Tell xdmac where to get the first descriptor. */
 348	reg = AT_XDMAC_CNDA_NDA(first->tx_dma_desc.phys)
 349	      | AT_XDMAC_CNDA_NDAIF(atchan->memif);
 350	at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, reg);
 351
 352	/*
 353	 * When doing non cyclic transfer we need to use the next
 354	 * descriptor view 2 since some fields of the configuration register
 355	 * depend on transfer size and src/dest addresses.
 356	 */
 357	if (at_xdmac_chan_is_cyclic(atchan))
 358		reg = AT_XDMAC_CNDC_NDVIEW_NDV1;
 359	else if (first->lld.mbr_ubc & AT_XDMAC_MBR_UBC_NDV3)
 360		reg = AT_XDMAC_CNDC_NDVIEW_NDV3;
 361	else
 362		reg = AT_XDMAC_CNDC_NDVIEW_NDV2;
 363	/*
 364	 * Even if the register will be updated from the configuration in the
 365	 * descriptor when using view 2 or higher, the PROT bit won't be set
 366	 * properly. This bit can be modified only by using the channel
 367	 * configuration register.
 368	 */
 369	at_xdmac_chan_write(atchan, AT_XDMAC_CC, first->lld.mbr_cfg);
 370
 371	reg |= AT_XDMAC_CNDC_NDDUP
 372	       | AT_XDMAC_CNDC_NDSUP
 373	       | AT_XDMAC_CNDC_NDE;
 374	at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, reg);
 375
 376	dev_vdbg(chan2dev(&atchan->chan),
 377		 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
 378		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),
 379		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
 380		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),
 381		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),
 382		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),
 383		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
 384
 385	at_xdmac_chan_write(atchan, AT_XDMAC_CID, 0xffffffff);
 386	reg = AT_XDMAC_CIE_RBEIE | AT_XDMAC_CIE_WBEIE;
 387	/*
 388	 * Request Overflow Error is only for peripheral synchronized transfers
 389	 */
 390	if (at_xdmac_chan_is_peripheral_xfer(first->lld.mbr_cfg))
 391		reg |= AT_XDMAC_CIE_ROIE;
 392
 393	/*
 394	 * There is no end of list when doing cyclic dma, we need to get
 395	 * an interrupt after each periods.
 396	 */
 397	if (at_xdmac_chan_is_cyclic(atchan))
 398		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,
 399				    reg | AT_XDMAC_CIE_BIE);
 400	else
 401		at_xdmac_chan_write(atchan, AT_XDMAC_CIE,
 402				    reg | AT_XDMAC_CIE_LIE);
 403	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atchan->mask);
 404	dev_vdbg(chan2dev(&atchan->chan),
 405		 "%s: enable channel (0x%08x)\n", __func__, atchan->mask);
 406	wmb();
 407	at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);
 408
 409	dev_vdbg(chan2dev(&atchan->chan),
 410		 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
 411		 __func__, at_xdmac_chan_read(atchan, AT_XDMAC_CC),
 412		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
 413		 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),
 414		 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),
 415		 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),
 416		 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
 417
 418}
 419
 420static dma_cookie_t at_xdmac_tx_submit(struct dma_async_tx_descriptor *tx)
 421{
 422	struct at_xdmac_desc	*desc = txd_to_at_desc(tx);
 423	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(tx->chan);
 424	dma_cookie_t		cookie;
 425	unsigned long		irqflags;
 426
 427	spin_lock_irqsave(&atchan->lock, irqflags);
 428	cookie = dma_cookie_assign(tx);
 429
 430	dev_vdbg(chan2dev(tx->chan), "%s: atchan 0x%p, add desc 0x%p to xfers_list\n",
 431		 __func__, atchan, desc);
 432	list_add_tail(&desc->xfer_node, &atchan->xfers_list);
 433	if (list_is_singular(&atchan->xfers_list))
 434		at_xdmac_start_xfer(atchan, desc);
 435
 436	spin_unlock_irqrestore(&atchan->lock, irqflags);
 437	return cookie;
 438}
 439
 440static struct at_xdmac_desc *at_xdmac_alloc_desc(struct dma_chan *chan,
 441						 gfp_t gfp_flags)
 442{
 443	struct at_xdmac_desc	*desc;
 444	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);
 445	dma_addr_t		phys;
 446
 447	desc = dma_pool_zalloc(atxdmac->at_xdmac_desc_pool, gfp_flags, &phys);
 448	if (desc) {
 449		INIT_LIST_HEAD(&desc->descs_list);
 450		dma_async_tx_descriptor_init(&desc->tx_dma_desc, chan);
 451		desc->tx_dma_desc.tx_submit = at_xdmac_tx_submit;
 452		desc->tx_dma_desc.phys = phys;
 453	}
 454
 455	return desc;
 456}
 457
 458static void at_xdmac_init_used_desc(struct at_xdmac_desc *desc)
 459{
 460	memset(&desc->lld, 0, sizeof(desc->lld));
 461	INIT_LIST_HEAD(&desc->descs_list);
 462	desc->direction = DMA_TRANS_NONE;
 463	desc->xfer_size = 0;
 464	desc->active_xfer = false;
 465}
 466
 467/* Call must be protected by lock. */
 468static struct at_xdmac_desc *at_xdmac_get_desc(struct at_xdmac_chan *atchan)
 469{
 470	struct at_xdmac_desc *desc;
 471
 472	if (list_empty(&atchan->free_descs_list)) {
 473		desc = at_xdmac_alloc_desc(&atchan->chan, GFP_NOWAIT);
 474	} else {
 475		desc = list_first_entry(&atchan->free_descs_list,
 476					struct at_xdmac_desc, desc_node);
 477		list_del(&desc->desc_node);
 478		at_xdmac_init_used_desc(desc);
 479	}
 480
 481	return desc;
 482}
 483
 484static void at_xdmac_queue_desc(struct dma_chan *chan,
 485				struct at_xdmac_desc *prev,
 486				struct at_xdmac_desc *desc)
 487{
 488	if (!prev || !desc)
 489		return;
 490
 491	prev->lld.mbr_nda = desc->tx_dma_desc.phys;
 492	prev->lld.mbr_ubc |= AT_XDMAC_MBR_UBC_NDE;
 493
 494	dev_dbg(chan2dev(chan),	"%s: chain lld: prev=0x%p, mbr_nda=%pad\n",
 495		__func__, prev, &prev->lld.mbr_nda);
 496}
 497
 498static inline void at_xdmac_increment_block_count(struct dma_chan *chan,
 499						  struct at_xdmac_desc *desc)
 500{
 501	if (!desc)
 502		return;
 503
 504	desc->lld.mbr_bc++;
 505
 506	dev_dbg(chan2dev(chan),
 507		"%s: incrementing the block count of the desc 0x%p\n",
 508		__func__, desc);
 509}
 510
 511static struct dma_chan *at_xdmac_xlate(struct of_phandle_args *dma_spec,
 512				       struct of_dma *of_dma)
 513{
 514	struct at_xdmac		*atxdmac = of_dma->of_dma_data;
 515	struct at_xdmac_chan	*atchan;
 516	struct dma_chan		*chan;
 517	struct device		*dev = atxdmac->dma.dev;
 518
 519	if (dma_spec->args_count != 1) {
 520		dev_err(dev, "dma phandler args: bad number of args\n");
 521		return NULL;
 522	}
 523
 524	chan = dma_get_any_slave_channel(&atxdmac->dma);
 525	if (!chan) {
 526		dev_err(dev, "can't get a dma channel\n");
 527		return NULL;
 528	}
 529
 530	atchan = to_at_xdmac_chan(chan);
 531	atchan->memif = AT91_XDMAC_DT_GET_MEM_IF(dma_spec->args[0]);
 532	atchan->perif = AT91_XDMAC_DT_GET_PER_IF(dma_spec->args[0]);
 533	atchan->perid = AT91_XDMAC_DT_GET_PERID(dma_spec->args[0]);
 534	dev_dbg(dev, "chan dt cfg: memif=%u perif=%u perid=%u\n",
 535		 atchan->memif, atchan->perif, atchan->perid);
 536
 537	return chan;
 538}
 539
 540static int at_xdmac_compute_chan_conf(struct dma_chan *chan,
 541				      enum dma_transfer_direction direction)
 542{
 543	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
 544	int			csize, dwidth;
 545
 546	if (direction == DMA_DEV_TO_MEM) {
 547		atchan->cfg =
 548			AT91_XDMAC_DT_PERID(atchan->perid)
 549			| AT_XDMAC_CC_DAM_INCREMENTED_AM
 550			| AT_XDMAC_CC_SAM_FIXED_AM
 551			| AT_XDMAC_CC_DIF(atchan->memif)
 552			| AT_XDMAC_CC_SIF(atchan->perif)
 553			| AT_XDMAC_CC_SWREQ_HWR_CONNECTED
 554			| AT_XDMAC_CC_DSYNC_PER2MEM
 555			| AT_XDMAC_CC_MBSIZE_SIXTEEN
 556			| AT_XDMAC_CC_TYPE_PER_TRAN;
 557		csize = ffs(atchan->sconfig.src_maxburst) - 1;
 558		if (csize < 0) {
 559			dev_err(chan2dev(chan), "invalid src maxburst value\n");
 560			return -EINVAL;
 561		}
 562		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);
 563		dwidth = ffs(atchan->sconfig.src_addr_width) - 1;
 564		if (dwidth < 0) {
 565			dev_err(chan2dev(chan), "invalid src addr width value\n");
 566			return -EINVAL;
 567		}
 568		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);
 569	} else if (direction == DMA_MEM_TO_DEV) {
 570		atchan->cfg =
 571			AT91_XDMAC_DT_PERID(atchan->perid)
 572			| AT_XDMAC_CC_DAM_FIXED_AM
 573			| AT_XDMAC_CC_SAM_INCREMENTED_AM
 574			| AT_XDMAC_CC_DIF(atchan->perif)
 575			| AT_XDMAC_CC_SIF(atchan->memif)
 576			| AT_XDMAC_CC_SWREQ_HWR_CONNECTED
 577			| AT_XDMAC_CC_DSYNC_MEM2PER
 578			| AT_XDMAC_CC_MBSIZE_SIXTEEN
 579			| AT_XDMAC_CC_TYPE_PER_TRAN;
 580		csize = ffs(atchan->sconfig.dst_maxburst) - 1;
 581		if (csize < 0) {
 582			dev_err(chan2dev(chan), "invalid src maxburst value\n");
 583			return -EINVAL;
 584		}
 585		atchan->cfg |= AT_XDMAC_CC_CSIZE(csize);
 586		dwidth = ffs(atchan->sconfig.dst_addr_width) - 1;
 587		if (dwidth < 0) {
 588			dev_err(chan2dev(chan), "invalid dst addr width value\n");
 589			return -EINVAL;
 590		}
 591		atchan->cfg |= AT_XDMAC_CC_DWIDTH(dwidth);
 592	}
 593
 594	dev_dbg(chan2dev(chan),	"%s: cfg=0x%08x\n", __func__, atchan->cfg);
 595
 596	return 0;
 597}
 598
 599/*
 600 * Only check that maxburst and addr width values are supported by the
 601 * the controller but not that the configuration is good to perform the
 602 * transfer since we don't know the direction at this stage.
 603 */
 604static int at_xdmac_check_slave_config(struct dma_slave_config *sconfig)
 605{
 606	if ((sconfig->src_maxburst > AT_XDMAC_MAX_CSIZE)
 607	    || (sconfig->dst_maxburst > AT_XDMAC_MAX_CSIZE))
 608		return -EINVAL;
 609
 610	if ((sconfig->src_addr_width > AT_XDMAC_MAX_DWIDTH)
 611	    || (sconfig->dst_addr_width > AT_XDMAC_MAX_DWIDTH))
 612		return -EINVAL;
 613
 614	return 0;
 615}
 616
 617static int at_xdmac_set_slave_config(struct dma_chan *chan,
 618				      struct dma_slave_config *sconfig)
 619{
 620	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
 621
 622	if (at_xdmac_check_slave_config(sconfig)) {
 623		dev_err(chan2dev(chan), "invalid slave configuration\n");
 624		return -EINVAL;
 625	}
 626
 627	memcpy(&atchan->sconfig, sconfig, sizeof(atchan->sconfig));
 628
 629	return 0;
 630}
 631
 632static struct dma_async_tx_descriptor *
 633at_xdmac_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 634		       unsigned int sg_len, enum dma_transfer_direction direction,
 635		       unsigned long flags, void *context)
 636{
 637	struct at_xdmac_chan		*atchan = to_at_xdmac_chan(chan);
 638	struct at_xdmac_desc		*first = NULL, *prev = NULL;
 639	struct scatterlist		*sg;
 640	int				i;
 641	unsigned int			xfer_size = 0;
 642	unsigned long			irqflags;
 643	struct dma_async_tx_descriptor	*ret = NULL;
 644
 645	if (!sgl)
 646		return NULL;
 647
 648	if (!is_slave_direction(direction)) {
 649		dev_err(chan2dev(chan), "invalid DMA direction\n");
 650		return NULL;
 651	}
 652
 653	dev_dbg(chan2dev(chan), "%s: sg_len=%d, dir=%s, flags=0x%lx\n",
 654		 __func__, sg_len,
 655		 direction == DMA_MEM_TO_DEV ? "to device" : "from device",
 656		 flags);
 657
 658	/* Protect dma_sconfig field that can be modified by set_slave_conf. */
 659	spin_lock_irqsave(&atchan->lock, irqflags);
 660
 661	if (at_xdmac_compute_chan_conf(chan, direction))
 662		goto spin_unlock;
 663
 664	/* Prepare descriptors. */
 665	for_each_sg(sgl, sg, sg_len, i) {
 666		struct at_xdmac_desc	*desc = NULL;
 667		u32			len, mem, dwidth, fixed_dwidth;
 668
 669		len = sg_dma_len(sg);
 670		mem = sg_dma_address(sg);
 671		if (unlikely(!len)) {
 672			dev_err(chan2dev(chan), "sg data length is zero\n");
 673			goto spin_unlock;
 674		}
 675		dev_dbg(chan2dev(chan), "%s: * sg%d len=%u, mem=0x%08x\n",
 676			 __func__, i, len, mem);
 677
 678		desc = at_xdmac_get_desc(atchan);
 679		if (!desc) {
 680			dev_err(chan2dev(chan), "can't get descriptor\n");
 681			if (first)
 682				list_splice_init(&first->descs_list, &atchan->free_descs_list);
 683			goto spin_unlock;
 684		}
 685
 686		/* Linked list descriptor setup. */
 687		if (direction == DMA_DEV_TO_MEM) {
 688			desc->lld.mbr_sa = atchan->sconfig.src_addr;
 689			desc->lld.mbr_da = mem;
 690		} else {
 691			desc->lld.mbr_sa = mem;
 692			desc->lld.mbr_da = atchan->sconfig.dst_addr;
 693		}
 694		dwidth = at_xdmac_get_dwidth(atchan->cfg);
 695		fixed_dwidth = IS_ALIGNED(len, 1 << dwidth)
 696			       ? dwidth
 697			       : AT_XDMAC_CC_DWIDTH_BYTE;
 698		desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2			/* next descriptor view */
 699			| AT_XDMAC_MBR_UBC_NDEN					/* next descriptor dst parameter update */
 700			| AT_XDMAC_MBR_UBC_NSEN					/* next descriptor src parameter update */
 701			| (len >> fixed_dwidth);				/* microblock length */
 702		desc->lld.mbr_cfg = (atchan->cfg & ~AT_XDMAC_CC_DWIDTH_MASK) |
 703				    AT_XDMAC_CC_DWIDTH(fixed_dwidth);
 704		dev_dbg(chan2dev(chan),
 705			 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
 706			 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
 707
 708		/* Chain lld. */
 709		if (prev)
 710			at_xdmac_queue_desc(chan, prev, desc);
 711
 712		prev = desc;
 713		if (!first)
 714			first = desc;
 715
 716		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
 717			 __func__, desc, first);
 718		list_add_tail(&desc->desc_node, &first->descs_list);
 719		xfer_size += len;
 720	}
 721
 722
 723	first->tx_dma_desc.flags = flags;
 724	first->xfer_size = xfer_size;
 725	first->direction = direction;
 726	ret = &first->tx_dma_desc;
 727
 728spin_unlock:
 729	spin_unlock_irqrestore(&atchan->lock, irqflags);
 730	return ret;
 731}
 732
 733static struct dma_async_tx_descriptor *
 734at_xdmac_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf_addr,
 735			 size_t buf_len, size_t period_len,
 736			 enum dma_transfer_direction direction,
 737			 unsigned long flags)
 738{
 739	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
 740	struct at_xdmac_desc	*first = NULL, *prev = NULL;
 741	unsigned int		periods = buf_len / period_len;
 742	int			i;
 743	unsigned long		irqflags;
 744
 745	dev_dbg(chan2dev(chan), "%s: buf_addr=%pad, buf_len=%zd, period_len=%zd, dir=%s, flags=0x%lx\n",
 746		__func__, &buf_addr, buf_len, period_len,
 747		direction == DMA_MEM_TO_DEV ? "mem2per" : "per2mem", flags);
 748
 749	if (!is_slave_direction(direction)) {
 750		dev_err(chan2dev(chan), "invalid DMA direction\n");
 751		return NULL;
 752	}
 753
 754	if (test_and_set_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status)) {
 755		dev_err(chan2dev(chan), "channel currently used\n");
 756		return NULL;
 757	}
 758
 759	if (at_xdmac_compute_chan_conf(chan, direction))
 760		return NULL;
 761
 762	for (i = 0; i < periods; i++) {
 763		struct at_xdmac_desc	*desc = NULL;
 764
 765		spin_lock_irqsave(&atchan->lock, irqflags);
 766		desc = at_xdmac_get_desc(atchan);
 767		if (!desc) {
 768			dev_err(chan2dev(chan), "can't get descriptor\n");
 769			if (first)
 770				list_splice_init(&first->descs_list, &atchan->free_descs_list);
 771			spin_unlock_irqrestore(&atchan->lock, irqflags);
 772			return NULL;
 773		}
 774		spin_unlock_irqrestore(&atchan->lock, irqflags);
 775		dev_dbg(chan2dev(chan),
 776			"%s: desc=0x%p, tx_dma_desc.phys=%pad\n",
 777			__func__, desc, &desc->tx_dma_desc.phys);
 778
 779		if (direction == DMA_DEV_TO_MEM) {
 780			desc->lld.mbr_sa = atchan->sconfig.src_addr;
 781			desc->lld.mbr_da = buf_addr + i * period_len;
 782		} else {
 783			desc->lld.mbr_sa = buf_addr + i * period_len;
 784			desc->lld.mbr_da = atchan->sconfig.dst_addr;
 785		}
 786		desc->lld.mbr_cfg = atchan->cfg;
 787		desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV1
 788			| AT_XDMAC_MBR_UBC_NDEN
 789			| AT_XDMAC_MBR_UBC_NSEN
 790			| period_len >> at_xdmac_get_dwidth(desc->lld.mbr_cfg);
 791
 792		dev_dbg(chan2dev(chan),
 793			 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
 794			 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc);
 795
 796		/* Chain lld. */
 797		if (prev)
 798			at_xdmac_queue_desc(chan, prev, desc);
 799
 800		prev = desc;
 801		if (!first)
 802			first = desc;
 803
 804		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
 805			 __func__, desc, first);
 806		list_add_tail(&desc->desc_node, &first->descs_list);
 807	}
 808
 809	at_xdmac_queue_desc(chan, prev, first);
 810	first->tx_dma_desc.flags = flags;
 811	first->xfer_size = buf_len;
 812	first->direction = direction;
 813
 814	return &first->tx_dma_desc;
 815}
 816
 817static inline u32 at_xdmac_align_width(struct dma_chan *chan, dma_addr_t addr)
 818{
 819	u32 width;
 820
 821	/*
 822	 * Check address alignment to select the greater data width we
 823	 * can use.
 824	 *
 825	 * Some XDMAC implementations don't provide dword transfer, in
 826	 * this case selecting dword has the same behavior as
 827	 * selecting word transfers.
 828	 */
 829	if (!(addr & 7)) {
 830		width = AT_XDMAC_CC_DWIDTH_DWORD;
 831		dev_dbg(chan2dev(chan), "%s: dwidth: double word\n", __func__);
 832	} else if (!(addr & 3)) {
 833		width = AT_XDMAC_CC_DWIDTH_WORD;
 834		dev_dbg(chan2dev(chan), "%s: dwidth: word\n", __func__);
 835	} else if (!(addr & 1)) {
 836		width = AT_XDMAC_CC_DWIDTH_HALFWORD;
 837		dev_dbg(chan2dev(chan), "%s: dwidth: half word\n", __func__);
 838	} else {
 839		width = AT_XDMAC_CC_DWIDTH_BYTE;
 840		dev_dbg(chan2dev(chan), "%s: dwidth: byte\n", __func__);
 841	}
 842
 843	return width;
 844}
 845
 846static struct at_xdmac_desc *
 847at_xdmac_interleaved_queue_desc(struct dma_chan *chan,
 848				struct at_xdmac_chan *atchan,
 849				struct at_xdmac_desc *prev,
 850				dma_addr_t src, dma_addr_t dst,
 851				struct dma_interleaved_template *xt,
 852				struct data_chunk *chunk)
 853{
 854	struct at_xdmac_desc	*desc;
 855	u32			dwidth;
 856	unsigned long		flags;
 857	size_t			ublen;
 858	/*
 859	 * WARNING: The channel configuration is set here since there is no
 860	 * dmaengine_slave_config call in this case. Moreover we don't know the
 861	 * direction, it involves we can't dynamically set the source and dest
 862	 * interface so we have to use the same one. Only interface 0 allows EBI
 863	 * access. Hopefully we can access DDR through both ports (at least on
 864	 * SAMA5D4x), so we can use the same interface for source and dest,
 865	 * that solves the fact we don't know the direction.
 866	 * ERRATA: Even if useless for memory transfers, the PERID has to not
 867	 * match the one of another channel. If not, it could lead to spurious
 868	 * flag status.
 869	 */
 870	u32			chan_cc = AT_XDMAC_CC_PERID(0x3f)
 871					| AT_XDMAC_CC_DIF(0)
 872					| AT_XDMAC_CC_SIF(0)
 873					| AT_XDMAC_CC_MBSIZE_SIXTEEN
 874					| AT_XDMAC_CC_TYPE_MEM_TRAN;
 875
 876	dwidth = at_xdmac_align_width(chan, src | dst | chunk->size);
 877	if (chunk->size >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
 878		dev_dbg(chan2dev(chan),
 879			"%s: chunk too big (%zu, max size %lu)...\n",
 880			__func__, chunk->size,
 881			AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth);
 882		return NULL;
 883	}
 884
 885	if (prev)
 886		dev_dbg(chan2dev(chan),
 887			"Adding items at the end of desc 0x%p\n", prev);
 888
 889	if (xt->src_inc) {
 890		if (xt->src_sgl)
 891			chan_cc |=  AT_XDMAC_CC_SAM_UBS_AM;
 892		else
 893			chan_cc |=  AT_XDMAC_CC_SAM_INCREMENTED_AM;
 894	}
 895
 896	if (xt->dst_inc) {
 897		if (xt->dst_sgl)
 898			chan_cc |=  AT_XDMAC_CC_DAM_UBS_AM;
 899		else
 900			chan_cc |=  AT_XDMAC_CC_DAM_INCREMENTED_AM;
 901	}
 902
 903	spin_lock_irqsave(&atchan->lock, flags);
 904	desc = at_xdmac_get_desc(atchan);
 905	spin_unlock_irqrestore(&atchan->lock, flags);
 906	if (!desc) {
 907		dev_err(chan2dev(chan), "can't get descriptor\n");
 908		return NULL;
 909	}
 910
 911	chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
 912
 913	ublen = chunk->size >> dwidth;
 914
 915	desc->lld.mbr_sa = src;
 916	desc->lld.mbr_da = dst;
 917	desc->lld.mbr_sus = dmaengine_get_src_icg(xt, chunk);
 918	desc->lld.mbr_dus = dmaengine_get_dst_icg(xt, chunk);
 919
 920	desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3
 921		| AT_XDMAC_MBR_UBC_NDEN
 922		| AT_XDMAC_MBR_UBC_NSEN
 923		| ublen;
 924	desc->lld.mbr_cfg = chan_cc;
 925
 926	dev_dbg(chan2dev(chan),
 927		"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
 928		__func__, &desc->lld.mbr_sa, &desc->lld.mbr_da,
 929		desc->lld.mbr_ubc, desc->lld.mbr_cfg);
 930
 931	/* Chain lld. */
 932	if (prev)
 933		at_xdmac_queue_desc(chan, prev, desc);
 934
 935	return desc;
 936}
 937
 938static struct dma_async_tx_descriptor *
 939at_xdmac_prep_interleaved(struct dma_chan *chan,
 940			  struct dma_interleaved_template *xt,
 941			  unsigned long flags)
 942{
 943	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
 944	struct at_xdmac_desc	*prev = NULL, *first = NULL;
 945	dma_addr_t		dst_addr, src_addr;
 946	size_t			src_skip = 0, dst_skip = 0, len = 0;
 947	struct data_chunk	*chunk;
 948	int			i;
 949
 950	if (!xt || !xt->numf || (xt->dir != DMA_MEM_TO_MEM))
 951		return NULL;
 952
 953	/*
 954	 * TODO: Handle the case where we have to repeat a chain of
 955	 * descriptors...
 956	 */
 957	if ((xt->numf > 1) && (xt->frame_size > 1))
 958		return NULL;
 959
 960	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, numf=%zu, frame_size=%zu, flags=0x%lx\n",
 961		__func__, &xt->src_start, &xt->dst_start,	xt->numf,
 962		xt->frame_size, flags);
 963
 964	src_addr = xt->src_start;
 965	dst_addr = xt->dst_start;
 966
 967	if (xt->numf > 1) {
 968		first = at_xdmac_interleaved_queue_desc(chan, atchan,
 969							NULL,
 970							src_addr, dst_addr,
 971							xt, xt->sgl);
 972
 973		/* Length of the block is (BLEN+1) microblocks. */
 974		for (i = 0; i < xt->numf - 1; i++)
 975			at_xdmac_increment_block_count(chan, first);
 976
 977		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
 978			__func__, first, first);
 979		list_add_tail(&first->desc_node, &first->descs_list);
 980	} else {
 981		for (i = 0; i < xt->frame_size; i++) {
 982			size_t src_icg = 0, dst_icg = 0;
 983			struct at_xdmac_desc *desc;
 984
 985			chunk = xt->sgl + i;
 986
 987			dst_icg = dmaengine_get_dst_icg(xt, chunk);
 988			src_icg = dmaengine_get_src_icg(xt, chunk);
 989
 990			src_skip = chunk->size + src_icg;
 991			dst_skip = chunk->size + dst_icg;
 992
 993			dev_dbg(chan2dev(chan),
 994				"%s: chunk size=%zu, src icg=%zu, dst icg=%zu\n",
 995				__func__, chunk->size, src_icg, dst_icg);
 996
 997			desc = at_xdmac_interleaved_queue_desc(chan, atchan,
 998							       prev,
 999							       src_addr, dst_addr,
1000							       xt, chunk);
1001			if (!desc) {
1002				list_splice_init(&first->descs_list,
1003						 &atchan->free_descs_list);
1004				return NULL;
1005			}
1006
1007			if (!first)
1008				first = desc;
1009
1010			dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
1011				__func__, desc, first);
1012			list_add_tail(&desc->desc_node, &first->descs_list);
1013
1014			if (xt->src_sgl)
1015				src_addr += src_skip;
1016
1017			if (xt->dst_sgl)
1018				dst_addr += dst_skip;
1019
1020			len += chunk->size;
1021			prev = desc;
1022		}
1023	}
1024
1025	first->tx_dma_desc.cookie = -EBUSY;
1026	first->tx_dma_desc.flags = flags;
1027	first->xfer_size = len;
1028
1029	return &first->tx_dma_desc;
1030}
1031
1032static struct dma_async_tx_descriptor *
1033at_xdmac_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
1034			 size_t len, unsigned long flags)
1035{
1036	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1037	struct at_xdmac_desc	*first = NULL, *prev = NULL;
1038	size_t			remaining_size = len, xfer_size = 0, ublen;
1039	dma_addr_t		src_addr = src, dst_addr = dest;
1040	u32			dwidth;
1041	/*
1042	 * WARNING: We don't know the direction, it involves we can't
1043	 * dynamically set the source and dest interface so we have to use the
1044	 * same one. Only interface 0 allows EBI access. Hopefully we can
1045	 * access DDR through both ports (at least on SAMA5D4x), so we can use
1046	 * the same interface for source and dest, that solves the fact we
1047	 * don't know the direction.
1048	 * ERRATA: Even if useless for memory transfers, the PERID has to not
1049	 * match the one of another channel. If not, it could lead to spurious
1050	 * flag status.
1051	 */
1052	u32			chan_cc = AT_XDMAC_CC_PERID(0x3f)
1053					| AT_XDMAC_CC_DAM_INCREMENTED_AM
1054					| AT_XDMAC_CC_SAM_INCREMENTED_AM
1055					| AT_XDMAC_CC_DIF(0)
1056					| AT_XDMAC_CC_SIF(0)
1057					| AT_XDMAC_CC_MBSIZE_SIXTEEN
1058					| AT_XDMAC_CC_TYPE_MEM_TRAN;
1059	unsigned long		irqflags;
1060
1061	dev_dbg(chan2dev(chan), "%s: src=%pad, dest=%pad, len=%zd, flags=0x%lx\n",
1062		__func__, &src, &dest, len, flags);
1063
1064	if (unlikely(!len))
1065		return NULL;
1066
1067	dwidth = at_xdmac_align_width(chan, src_addr | dst_addr);
1068
1069	/* Prepare descriptors. */
1070	while (remaining_size) {
1071		struct at_xdmac_desc	*desc = NULL;
1072
1073		dev_dbg(chan2dev(chan), "%s: remaining_size=%zu\n", __func__, remaining_size);
1074
1075		spin_lock_irqsave(&atchan->lock, irqflags);
1076		desc = at_xdmac_get_desc(atchan);
1077		spin_unlock_irqrestore(&atchan->lock, irqflags);
1078		if (!desc) {
1079			dev_err(chan2dev(chan), "can't get descriptor\n");
1080			if (first)
1081				list_splice_init(&first->descs_list, &atchan->free_descs_list);
1082			return NULL;
1083		}
1084
1085		/* Update src and dest addresses. */
1086		src_addr += xfer_size;
1087		dst_addr += xfer_size;
1088
1089		if (remaining_size >= AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)
1090			xfer_size = AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth;
1091		else
1092			xfer_size = remaining_size;
1093
1094		dev_dbg(chan2dev(chan), "%s: xfer_size=%zu\n", __func__, xfer_size);
1095
1096		/* Check remaining length and change data width if needed. */
1097		dwidth = at_xdmac_align_width(chan,
1098					      src_addr | dst_addr | xfer_size);
1099		chan_cc &= ~AT_XDMAC_CC_DWIDTH_MASK;
1100		chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
1101
1102		ublen = xfer_size >> dwidth;
1103		remaining_size -= xfer_size;
1104
1105		desc->lld.mbr_sa = src_addr;
1106		desc->lld.mbr_da = dst_addr;
1107		desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV2
1108			| AT_XDMAC_MBR_UBC_NDEN
1109			| AT_XDMAC_MBR_UBC_NSEN
1110			| ublen;
1111		desc->lld.mbr_cfg = chan_cc;
1112
1113		dev_dbg(chan2dev(chan),
1114			 "%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
1115			 __func__, &desc->lld.mbr_sa, &desc->lld.mbr_da, desc->lld.mbr_ubc, desc->lld.mbr_cfg);
1116
1117		/* Chain lld. */
1118		if (prev)
1119			at_xdmac_queue_desc(chan, prev, desc);
1120
1121		prev = desc;
1122		if (!first)
1123			first = desc;
1124
1125		dev_dbg(chan2dev(chan), "%s: add desc 0x%p to descs_list 0x%p\n",
1126			 __func__, desc, first);
1127		list_add_tail(&desc->desc_node, &first->descs_list);
1128	}
1129
1130	first->tx_dma_desc.flags = flags;
1131	first->xfer_size = len;
1132
1133	return &first->tx_dma_desc;
1134}
1135
1136static struct at_xdmac_desc *at_xdmac_memset_create_desc(struct dma_chan *chan,
1137							 struct at_xdmac_chan *atchan,
1138							 dma_addr_t dst_addr,
1139							 size_t len,
1140							 int value)
1141{
1142	struct at_xdmac_desc	*desc;
1143	unsigned long		flags;
1144	size_t			ublen;
1145	u32			dwidth;
1146	/*
1147	 * WARNING: The channel configuration is set here since there is no
1148	 * dmaengine_slave_config call in this case. Moreover we don't know the
1149	 * direction, it involves we can't dynamically set the source and dest
1150	 * interface so we have to use the same one. Only interface 0 allows EBI
1151	 * access. Hopefully we can access DDR through both ports (at least on
1152	 * SAMA5D4x), so we can use the same interface for source and dest,
1153	 * that solves the fact we don't know the direction.
1154	 * ERRATA: Even if useless for memory transfers, the PERID has to not
1155	 * match the one of another channel. If not, it could lead to spurious
1156	 * flag status.
1157	 */
1158	u32			chan_cc = AT_XDMAC_CC_PERID(0x3f)
1159					| AT_XDMAC_CC_DAM_UBS_AM
1160					| AT_XDMAC_CC_SAM_INCREMENTED_AM
1161					| AT_XDMAC_CC_DIF(0)
1162					| AT_XDMAC_CC_SIF(0)
1163					| AT_XDMAC_CC_MBSIZE_SIXTEEN
1164					| AT_XDMAC_CC_MEMSET_HW_MODE
1165					| AT_XDMAC_CC_TYPE_MEM_TRAN;
1166
1167	dwidth = at_xdmac_align_width(chan, dst_addr);
1168
1169	if (len >= (AT_XDMAC_MBR_UBC_UBLEN_MAX << dwidth)) {
1170		dev_err(chan2dev(chan),
1171			"%s: Transfer too large, aborting...\n",
1172			__func__);
1173		return NULL;
1174	}
1175
1176	spin_lock_irqsave(&atchan->lock, flags);
1177	desc = at_xdmac_get_desc(atchan);
1178	spin_unlock_irqrestore(&atchan->lock, flags);
1179	if (!desc) {
1180		dev_err(chan2dev(chan), "can't get descriptor\n");
1181		return NULL;
1182	}
1183
1184	chan_cc |= AT_XDMAC_CC_DWIDTH(dwidth);
1185
1186	ublen = len >> dwidth;
1187
1188	desc->lld.mbr_da = dst_addr;
1189	desc->lld.mbr_ds = value;
1190	desc->lld.mbr_ubc = AT_XDMAC_MBR_UBC_NDV3
1191		| AT_XDMAC_MBR_UBC_NDEN
1192		| AT_XDMAC_MBR_UBC_NSEN
1193		| ublen;
1194	desc->lld.mbr_cfg = chan_cc;
1195
1196	dev_dbg(chan2dev(chan),
1197		"%s: lld: mbr_da=%pad, mbr_ds=0x%08x, mbr_ubc=0x%08x, mbr_cfg=0x%08x\n",
1198		__func__, &desc->lld.mbr_da, desc->lld.mbr_ds, desc->lld.mbr_ubc,
1199		desc->lld.mbr_cfg);
1200
1201	return desc;
1202}
1203
1204static struct dma_async_tx_descriptor *
1205at_xdmac_prep_dma_memset(struct dma_chan *chan, dma_addr_t dest, int value,
1206			 size_t len, unsigned long flags)
1207{
1208	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1209	struct at_xdmac_desc	*desc;
1210
1211	dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%zu, pattern=0x%x, flags=0x%lx\n",
1212		__func__, &dest, len, value, flags);
1213
1214	if (unlikely(!len))
1215		return NULL;
1216
1217	desc = at_xdmac_memset_create_desc(chan, atchan, dest, len, value);
1218	list_add_tail(&desc->desc_node, &desc->descs_list);
1219
1220	desc->tx_dma_desc.cookie = -EBUSY;
1221	desc->tx_dma_desc.flags = flags;
1222	desc->xfer_size = len;
1223
1224	return &desc->tx_dma_desc;
1225}
1226
1227static struct dma_async_tx_descriptor *
1228at_xdmac_prep_dma_memset_sg(struct dma_chan *chan, struct scatterlist *sgl,
1229			    unsigned int sg_len, int value,
1230			    unsigned long flags)
1231{
1232	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1233	struct at_xdmac_desc	*desc, *pdesc = NULL,
1234				*ppdesc = NULL, *first = NULL;
1235	struct scatterlist	*sg, *psg = NULL, *ppsg = NULL;
1236	size_t			stride = 0, pstride = 0, len = 0;
1237	int			i;
1238
1239	if (!sgl)
1240		return NULL;
1241
1242	dev_dbg(chan2dev(chan), "%s: sg_len=%d, value=0x%x, flags=0x%lx\n",
1243		__func__, sg_len, value, flags);
1244
1245	/* Prepare descriptors. */
1246	for_each_sg(sgl, sg, sg_len, i) {
1247		dev_dbg(chan2dev(chan), "%s: dest=%pad, len=%d, pattern=0x%x, flags=0x%lx\n",
1248			__func__, &sg_dma_address(sg), sg_dma_len(sg),
1249			value, flags);
1250		desc = at_xdmac_memset_create_desc(chan, atchan,
1251						   sg_dma_address(sg),
1252						   sg_dma_len(sg),
1253						   value);
1254		if (!desc && first)
1255			list_splice_init(&first->descs_list,
1256					 &atchan->free_descs_list);
1257
1258		if (!first)
1259			first = desc;
1260
1261		/* Update our strides */
1262		pstride = stride;
1263		if (psg)
1264			stride = sg_dma_address(sg) -
1265				(sg_dma_address(psg) + sg_dma_len(psg));
1266
1267		/*
1268		 * The scatterlist API gives us only the address and
1269		 * length of each elements.
1270		 *
1271		 * Unfortunately, we don't have the stride, which we
1272		 * will need to compute.
1273		 *
1274		 * That make us end up in a situation like this one:
1275		 *    len    stride    len    stride    len
1276		 * +-------+        +-------+        +-------+
1277		 * |  N-2  |        |  N-1  |        |   N   |
1278		 * +-------+        +-------+        +-------+
1279		 *
1280		 * We need all these three elements (N-2, N-1 and N)
1281		 * to actually take the decision on whether we need to
1282		 * queue N-1 or reuse N-2.
1283		 *
1284		 * We will only consider N if it is the last element.
1285		 */
1286		if (ppdesc && pdesc) {
1287			if ((stride == pstride) &&
1288			    (sg_dma_len(ppsg) == sg_dma_len(psg))) {
1289				dev_dbg(chan2dev(chan),
1290					"%s: desc 0x%p can be merged with desc 0x%p\n",
1291					__func__, pdesc, ppdesc);
1292
1293				/*
1294				 * Increment the block count of the
1295				 * N-2 descriptor
1296				 */
1297				at_xdmac_increment_block_count(chan, ppdesc);
1298				ppdesc->lld.mbr_dus = stride;
1299
1300				/*
1301				 * Put back the N-1 descriptor in the
1302				 * free descriptor list
1303				 */
1304				list_add_tail(&pdesc->desc_node,
1305					      &atchan->free_descs_list);
1306
1307				/*
1308				 * Make our N-1 descriptor pointer
1309				 * point to the N-2 since they were
1310				 * actually merged.
1311				 */
1312				pdesc = ppdesc;
1313
1314			/*
1315			 * Rule out the case where we don't have
1316			 * pstride computed yet (our second sg
1317			 * element)
1318			 *
1319			 * We also want to catch the case where there
1320			 * would be a negative stride,
1321			 */
1322			} else if (pstride ||
1323				   sg_dma_address(sg) < sg_dma_address(psg)) {
1324				/*
1325				 * Queue the N-1 descriptor after the
1326				 * N-2
1327				 */
1328				at_xdmac_queue_desc(chan, ppdesc, pdesc);
1329
1330				/*
1331				 * Add the N-1 descriptor to the list
1332				 * of the descriptors used for this
1333				 * transfer
1334				 */
1335				list_add_tail(&desc->desc_node,
1336					      &first->descs_list);
1337				dev_dbg(chan2dev(chan),
1338					"%s: add desc 0x%p to descs_list 0x%p\n",
1339					__func__, desc, first);
1340			}
1341		}
1342
1343		/*
1344		 * If we are the last element, just see if we have the
1345		 * same size than the previous element.
1346		 *
1347		 * If so, we can merge it with the previous descriptor
1348		 * since we don't care about the stride anymore.
1349		 */
1350		if ((i == (sg_len - 1)) &&
1351		    sg_dma_len(psg) == sg_dma_len(sg)) {
1352			dev_dbg(chan2dev(chan),
1353				"%s: desc 0x%p can be merged with desc 0x%p\n",
1354				__func__, desc, pdesc);
1355
1356			/*
1357			 * Increment the block count of the N-1
1358			 * descriptor
1359			 */
1360			at_xdmac_increment_block_count(chan, pdesc);
1361			pdesc->lld.mbr_dus = stride;
1362
1363			/*
1364			 * Put back the N descriptor in the free
1365			 * descriptor list
1366			 */
1367			list_add_tail(&desc->desc_node,
1368				      &atchan->free_descs_list);
1369		}
1370
1371		/* Update our descriptors */
1372		ppdesc = pdesc;
1373		pdesc = desc;
1374
1375		/* Update our scatter pointers */
1376		ppsg = psg;
1377		psg = sg;
1378
1379		len += sg_dma_len(sg);
1380	}
1381
1382	first->tx_dma_desc.cookie = -EBUSY;
1383	first->tx_dma_desc.flags = flags;
1384	first->xfer_size = len;
1385
1386	return &first->tx_dma_desc;
1387}
1388
1389static enum dma_status
1390at_xdmac_tx_status(struct dma_chan *chan, dma_cookie_t cookie,
1391		struct dma_tx_state *txstate)
1392{
1393	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1394	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);
1395	struct at_xdmac_desc	*desc, *_desc;
1396	struct list_head	*descs_list;
1397	enum dma_status		ret;
1398	int			residue, retry;
1399	u32			cur_nda, check_nda, cur_ubc, mask, value;
1400	u8			dwidth = 0;
1401	unsigned long		flags;
1402	bool			initd;
1403
1404	ret = dma_cookie_status(chan, cookie, txstate);
1405	if (ret == DMA_COMPLETE)
1406		return ret;
1407
1408	if (!txstate)
1409		return ret;
1410
1411	spin_lock_irqsave(&atchan->lock, flags);
1412
1413	desc = list_first_entry(&atchan->xfers_list, struct at_xdmac_desc, xfer_node);
1414
1415	/*
1416	 * If the transfer has not been started yet, don't need to compute the
1417	 * residue, it's the transfer length.
1418	 */
1419	if (!desc->active_xfer) {
1420		dma_set_residue(txstate, desc->xfer_size);
1421		goto spin_unlock;
1422	}
1423
1424	residue = desc->xfer_size;
1425	/*
1426	 * Flush FIFO: only relevant when the transfer is source peripheral
1427	 * synchronized. Flush is needed before reading CUBC because data in
1428	 * the FIFO are not reported by CUBC. Reporting a residue of the
1429	 * transfer length while we have data in FIFO can cause issue.
1430	 * Usecase: atmel USART has a timeout which means I have received
1431	 * characters but there is no more character received for a while. On
1432	 * timeout, it requests the residue. If the data are in the DMA FIFO,
1433	 * we will return a residue of the transfer length. It means no data
1434	 * received. If an application is waiting for these data, it will hang
1435	 * since we won't have another USART timeout without receiving new
1436	 * data.
1437	 */
1438	mask = AT_XDMAC_CC_TYPE | AT_XDMAC_CC_DSYNC;
1439	value = AT_XDMAC_CC_TYPE_PER_TRAN | AT_XDMAC_CC_DSYNC_PER2MEM;
1440	if ((desc->lld.mbr_cfg & mask) == value) {
1441		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);
1442		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))
1443			cpu_relax();
1444	}
1445
1446	/*
1447	 * The easiest way to compute the residue should be to pause the DMA
1448	 * but doing this can lead to miss some data as some devices don't
1449	 * have FIFO.
1450	 * We need to read several registers because:
1451	 * - DMA is running therefore a descriptor change is possible while
1452	 * reading these registers
1453	 * - When the block transfer is done, the value of the CUBC register
1454	 * is set to its initial value until the fetch of the next descriptor.
1455	 * This value will corrupt the residue calculation so we have to skip
1456	 * it.
1457	 *
1458	 * INITD --------                    ------------
1459	 *              |____________________|
1460	 *       _______________________  _______________
1461	 * NDA       @desc2             \/   @desc3
1462	 *       _______________________/\_______________
1463	 *       __________  ___________  _______________
1464	 * CUBC       0    \/ MAX desc1 \/  MAX desc2
1465	 *       __________/\___________/\_______________
1466	 *
1467	 * Since descriptors are aligned on 64 bits, we can assume that
1468	 * the update of NDA and CUBC is atomic.
1469	 * Memory barriers are used to ensure the read order of the registers.
1470	 * A max number of retries is set because unlikely it could never ends.
1471	 */
1472	for (retry = 0; retry < AT_XDMAC_RESIDUE_MAX_RETRIES; retry++) {
1473		check_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;
1474		rmb();
1475		cur_ubc = at_xdmac_chan_read(atchan, AT_XDMAC_CUBC);
1476		rmb();
1477		initd = !!(at_xdmac_chan_read(atchan, AT_XDMAC_CC) & AT_XDMAC_CC_INITD);
1478		rmb();
 
 
1479		cur_nda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA) & 0xfffffffc;
1480		rmb();
1481
1482		if ((check_nda == cur_nda) && initd)
1483			break;
1484	}
1485
1486	if (unlikely(retry >= AT_XDMAC_RESIDUE_MAX_RETRIES)) {
1487		ret = DMA_ERROR;
1488		goto spin_unlock;
1489	}
1490
1491	/*
1492	 * Flush FIFO: only relevant when the transfer is source peripheral
1493	 * synchronized. Another flush is needed here because CUBC is updated
1494	 * when the controller sends the data write command. It can lead to
1495	 * report data that are not written in the memory or the device. The
1496	 * FIFO flush ensures that data are really written.
1497	 */
1498	if ((desc->lld.mbr_cfg & mask) == value) {
1499		at_xdmac_write(atxdmac, AT_XDMAC_GSWF, atchan->mask);
1500		while (!(at_xdmac_chan_read(atchan, AT_XDMAC_CIS) & AT_XDMAC_CIS_FIS))
1501			cpu_relax();
1502	}
1503
1504	/*
1505	 * Remove size of all microblocks already transferred and the current
1506	 * one. Then add the remaining size to transfer of the current
1507	 * microblock.
1508	 */
1509	descs_list = &desc->descs_list;
1510	list_for_each_entry_safe(desc, _desc, descs_list, desc_node) {
1511		dwidth = at_xdmac_get_dwidth(desc->lld.mbr_cfg);
1512		residue -= (desc->lld.mbr_ubc & 0xffffff) << dwidth;
1513		if ((desc->lld.mbr_nda & 0xfffffffc) == cur_nda)
1514			break;
1515	}
1516	residue += cur_ubc << dwidth;
1517
1518	dma_set_residue(txstate, residue);
1519
1520	dev_dbg(chan2dev(chan),
1521		 "%s: desc=0x%p, tx_dma_desc.phys=%pad, tx_status=%d, cookie=%d, residue=%d\n",
1522		 __func__, desc, &desc->tx_dma_desc.phys, ret, cookie, residue);
1523
1524spin_unlock:
1525	spin_unlock_irqrestore(&atchan->lock, flags);
1526	return ret;
1527}
1528
1529/* Call must be protected by lock. */
1530static void at_xdmac_remove_xfer(struct at_xdmac_chan *atchan,
1531				    struct at_xdmac_desc *desc)
1532{
1533	dev_dbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
1534
1535	/*
1536	 * Remove the transfer from the transfer list then move the transfer
1537	 * descriptors into the free descriptors list.
1538	 */
1539	list_del(&desc->xfer_node);
1540	list_splice_init(&desc->descs_list, &atchan->free_descs_list);
1541}
1542
1543static void at_xdmac_advance_work(struct at_xdmac_chan *atchan)
1544{
1545	struct at_xdmac_desc	*desc;
 
 
 
1546
1547	/*
1548	 * If channel is enabled, do nothing, advance_work will be triggered
1549	 * after the interruption.
1550	 */
1551	if (!at_xdmac_chan_is_enabled(atchan) && !list_empty(&atchan->xfers_list)) {
1552		desc = list_first_entry(&atchan->xfers_list,
1553					struct at_xdmac_desc,
1554					xfer_node);
1555		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
1556		if (!desc->active_xfer)
1557			at_xdmac_start_xfer(atchan, desc);
1558	}
 
 
1559}
1560
1561static void at_xdmac_handle_cyclic(struct at_xdmac_chan *atchan)
1562{
1563	struct at_xdmac_desc		*desc;
1564	struct dma_async_tx_descriptor	*txd;
1565
1566	if (!list_empty(&atchan->xfers_list)) {
1567		desc = list_first_entry(&atchan->xfers_list,
1568					struct at_xdmac_desc, xfer_node);
1569		txd = &desc->tx_dma_desc;
1570
1571		if (txd->flags & DMA_PREP_INTERRUPT)
1572			dmaengine_desc_get_callback_invoke(txd, NULL);
1573	}
1574}
1575
1576static void at_xdmac_handle_error(struct at_xdmac_chan *atchan)
1577{
1578	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);
1579	struct at_xdmac_desc	*bad_desc;
1580
1581	/*
1582	 * The descriptor currently at the head of the active list is
1583	 * broken. Since we don't have any way to report errors, we'll
1584	 * just have to scream loudly and try to continue with other
1585	 * descriptors queued (if any).
1586	 */
1587	if (atchan->irq_status & AT_XDMAC_CIS_RBEIS)
1588		dev_err(chan2dev(&atchan->chan), "read bus error!!!");
1589	if (atchan->irq_status & AT_XDMAC_CIS_WBEIS)
1590		dev_err(chan2dev(&atchan->chan), "write bus error!!!");
1591	if (atchan->irq_status & AT_XDMAC_CIS_ROIS)
1592		dev_err(chan2dev(&atchan->chan), "request overflow error!!!");
1593
1594	spin_lock_irq(&atchan->lock);
1595
1596	/* Channel must be disabled first as it's not done automatically */
1597	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);
1598	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)
1599		cpu_relax();
1600
1601	bad_desc = list_first_entry(&atchan->xfers_list,
1602				    struct at_xdmac_desc,
1603				    xfer_node);
1604
1605	spin_unlock_irq(&atchan->lock);
1606
1607	/* Print bad descriptor's details if needed */
1608	dev_dbg(chan2dev(&atchan->chan),
1609		"%s: lld: mbr_sa=%pad, mbr_da=%pad, mbr_ubc=0x%08x\n",
1610		__func__, &bad_desc->lld.mbr_sa, &bad_desc->lld.mbr_da,
1611		bad_desc->lld.mbr_ubc);
1612
1613	/* Then continue with usual descriptor management */
 
1614}
1615
1616static void at_xdmac_tasklet(unsigned long data)
1617{
1618	struct at_xdmac_chan	*atchan = (struct at_xdmac_chan *)data;
1619	struct at_xdmac_desc	*desc;
1620	u32			error_mask;
1621
1622	dev_dbg(chan2dev(&atchan->chan), "%s: status=0x%08x\n",
1623		__func__, atchan->irq_status);
1624
1625	error_mask = AT_XDMAC_CIS_RBEIS
1626		     | AT_XDMAC_CIS_WBEIS
1627		     | AT_XDMAC_CIS_ROIS;
1628
1629	if (at_xdmac_chan_is_cyclic(atchan)) {
1630		at_xdmac_handle_cyclic(atchan);
1631	} else if ((atchan->irq_status & AT_XDMAC_CIS_LIS)
1632		   || (atchan->irq_status & error_mask)) {
1633		struct dma_async_tx_descriptor  *txd;
1634
1635		if (atchan->irq_status & error_mask)
1636			at_xdmac_handle_error(atchan);
 
 
 
 
1637
1638		spin_lock_irq(&atchan->lock);
1639		desc = list_first_entry(&atchan->xfers_list,
1640					struct at_xdmac_desc,
1641					xfer_node);
1642		dev_vdbg(chan2dev(&atchan->chan), "%s: desc 0x%p\n", __func__, desc);
1643		if (!desc->active_xfer) {
1644			dev_err(chan2dev(&atchan->chan), "Xfer not active: exiting");
1645			spin_unlock_irq(&atchan->lock);
1646			return;
1647		}
1648
1649		txd = &desc->tx_dma_desc;
1650
1651		at_xdmac_remove_xfer(atchan, desc);
1652		spin_unlock_irq(&atchan->lock);
1653
1654		dma_cookie_complete(txd);
1655		if (txd->flags & DMA_PREP_INTERRUPT)
1656			dmaengine_desc_get_callback_invoke(txd, NULL);
 
 
1657
1658		dma_run_dependencies(txd);
1659
1660		spin_lock_irq(&atchan->lock);
1661		at_xdmac_advance_work(atchan);
1662		spin_unlock_irq(&atchan->lock);
1663	}
1664}
1665
1666static irqreturn_t at_xdmac_interrupt(int irq, void *dev_id)
1667{
1668	struct at_xdmac		*atxdmac = (struct at_xdmac *)dev_id;
1669	struct at_xdmac_chan	*atchan;
1670	u32			imr, status, pending;
1671	u32			chan_imr, chan_status;
1672	int			i, ret = IRQ_NONE;
1673
1674	do {
1675		imr = at_xdmac_read(atxdmac, AT_XDMAC_GIM);
1676		status = at_xdmac_read(atxdmac, AT_XDMAC_GIS);
1677		pending = status & imr;
1678
1679		dev_vdbg(atxdmac->dma.dev,
1680			 "%s: status=0x%08x, imr=0x%08x, pending=0x%08x\n",
1681			 __func__, status, imr, pending);
1682
1683		if (!pending)
1684			break;
1685
1686		/* We have to find which channel has generated the interrupt. */
1687		for (i = 0; i < atxdmac->dma.chancnt; i++) {
1688			if (!((1 << i) & pending))
1689				continue;
1690
1691			atchan = &atxdmac->chan[i];
1692			chan_imr = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);
1693			chan_status = at_xdmac_chan_read(atchan, AT_XDMAC_CIS);
1694			atchan->irq_status = chan_status & chan_imr;
1695			dev_vdbg(atxdmac->dma.dev,
1696				 "%s: chan%d: imr=0x%x, status=0x%x\n",
1697				 __func__, i, chan_imr, chan_status);
1698			dev_vdbg(chan2dev(&atchan->chan),
1699				 "%s: CC=0x%08x CNDA=0x%08x, CNDC=0x%08x, CSA=0x%08x, CDA=0x%08x, CUBC=0x%08x\n",
1700				 __func__,
1701				 at_xdmac_chan_read(atchan, AT_XDMAC_CC),
1702				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDA),
1703				 at_xdmac_chan_read(atchan, AT_XDMAC_CNDC),
1704				 at_xdmac_chan_read(atchan, AT_XDMAC_CSA),
1705				 at_xdmac_chan_read(atchan, AT_XDMAC_CDA),
1706				 at_xdmac_chan_read(atchan, AT_XDMAC_CUBC));
1707
1708			if (atchan->irq_status & (AT_XDMAC_CIS_RBEIS | AT_XDMAC_CIS_WBEIS))
1709				at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);
1710
1711			tasklet_schedule(&atchan->tasklet);
1712			ret = IRQ_HANDLED;
1713		}
1714
1715	} while (pending);
1716
1717	return ret;
1718}
1719
1720static void at_xdmac_issue_pending(struct dma_chan *chan)
1721{
1722	struct at_xdmac_chan *atchan = to_at_xdmac_chan(chan);
1723	unsigned long flags;
1724
1725	dev_dbg(chan2dev(&atchan->chan), "%s\n", __func__);
1726
1727	if (!at_xdmac_chan_is_cyclic(atchan)) {
1728		spin_lock_irqsave(&atchan->lock, flags);
1729		at_xdmac_advance_work(atchan);
1730		spin_unlock_irqrestore(&atchan->lock, flags);
1731	}
1732
1733	return;
1734}
1735
1736static int at_xdmac_device_config(struct dma_chan *chan,
1737				  struct dma_slave_config *config)
1738{
1739	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1740	int ret;
1741	unsigned long		flags;
1742
1743	dev_dbg(chan2dev(chan), "%s\n", __func__);
1744
1745	spin_lock_irqsave(&atchan->lock, flags);
1746	ret = at_xdmac_set_slave_config(chan, config);
1747	spin_unlock_irqrestore(&atchan->lock, flags);
1748
1749	return ret;
1750}
1751
1752static int at_xdmac_device_pause(struct dma_chan *chan)
1753{
1754	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1755	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);
1756	unsigned long		flags;
1757
1758	dev_dbg(chan2dev(chan), "%s\n", __func__);
1759
1760	if (test_and_set_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status))
1761		return 0;
1762
1763	spin_lock_irqsave(&atchan->lock, flags);
1764	at_xdmac_write(atxdmac, AT_XDMAC_GRWS, atchan->mask);
1765	while (at_xdmac_chan_read(atchan, AT_XDMAC_CC)
1766	       & (AT_XDMAC_CC_WRIP | AT_XDMAC_CC_RDIP))
1767		cpu_relax();
1768	spin_unlock_irqrestore(&atchan->lock, flags);
1769
1770	return 0;
1771}
1772
1773static int at_xdmac_device_resume(struct dma_chan *chan)
1774{
1775	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1776	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);
1777	unsigned long		flags;
1778
1779	dev_dbg(chan2dev(chan), "%s\n", __func__);
1780
1781	spin_lock_irqsave(&atchan->lock, flags);
1782	if (!at_xdmac_chan_is_paused(atchan)) {
1783		spin_unlock_irqrestore(&atchan->lock, flags);
1784		return 0;
1785	}
1786
1787	at_xdmac_write(atxdmac, AT_XDMAC_GRWR, atchan->mask);
1788	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
1789	spin_unlock_irqrestore(&atchan->lock, flags);
1790
1791	return 0;
1792}
1793
1794static int at_xdmac_device_terminate_all(struct dma_chan *chan)
1795{
1796	struct at_xdmac_desc	*desc, *_desc;
1797	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1798	struct at_xdmac		*atxdmac = to_at_xdmac(atchan->chan.device);
1799	unsigned long		flags;
1800
1801	dev_dbg(chan2dev(chan), "%s\n", __func__);
1802
1803	spin_lock_irqsave(&atchan->lock, flags);
1804	at_xdmac_write(atxdmac, AT_XDMAC_GD, atchan->mask);
1805	while (at_xdmac_read(atxdmac, AT_XDMAC_GS) & atchan->mask)
1806		cpu_relax();
1807
1808	/* Cancel all pending transfers. */
1809	list_for_each_entry_safe(desc, _desc, &atchan->xfers_list, xfer_node)
1810		at_xdmac_remove_xfer(atchan, desc);
1811
1812	clear_bit(AT_XDMAC_CHAN_IS_PAUSED, &atchan->status);
1813	clear_bit(AT_XDMAC_CHAN_IS_CYCLIC, &atchan->status);
1814	spin_unlock_irqrestore(&atchan->lock, flags);
1815
1816	return 0;
1817}
1818
1819static int at_xdmac_alloc_chan_resources(struct dma_chan *chan)
1820{
1821	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1822	struct at_xdmac_desc	*desc;
1823	int			i;
 
 
 
1824
1825	if (at_xdmac_chan_is_enabled(atchan)) {
1826		dev_err(chan2dev(chan),
1827			"can't allocate channel resources (channel enabled)\n");
1828		return -EIO;
 
1829	}
1830
1831	if (!list_empty(&atchan->free_descs_list)) {
1832		dev_err(chan2dev(chan),
1833			"can't allocate channel resources (channel not free from a previous use)\n");
1834		return -EIO;
 
1835	}
1836
1837	for (i = 0; i < init_nr_desc_per_channel; i++) {
1838		desc = at_xdmac_alloc_desc(chan, GFP_KERNEL);
1839		if (!desc) {
1840			dev_warn(chan2dev(chan),
1841				"only %d descriptors have been allocated\n", i);
1842			break;
1843		}
1844		list_add_tail(&desc->desc_node, &atchan->free_descs_list);
1845	}
1846
1847	dma_cookie_init(chan);
1848
1849	dev_dbg(chan2dev(chan), "%s: allocated %d descriptors\n", __func__, i);
1850
 
 
1851	return i;
1852}
1853
1854static void at_xdmac_free_chan_resources(struct dma_chan *chan)
1855{
1856	struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1857	struct at_xdmac		*atxdmac = to_at_xdmac(chan->device);
1858	struct at_xdmac_desc	*desc, *_desc;
1859
1860	list_for_each_entry_safe(desc, _desc, &atchan->free_descs_list, desc_node) {
1861		dev_dbg(chan2dev(chan), "%s: freeing descriptor %p\n", __func__, desc);
1862		list_del(&desc->desc_node);
1863		dma_pool_free(atxdmac->at_xdmac_desc_pool, desc, desc->tx_dma_desc.phys);
1864	}
1865
1866	return;
1867}
1868
1869#ifdef CONFIG_PM
1870static int atmel_xdmac_prepare(struct device *dev)
1871{
1872	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);
 
1873	struct dma_chan		*chan, *_chan;
1874
1875	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
1876		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1877
1878		/* Wait for transfer completion, except in cyclic case. */
1879		if (at_xdmac_chan_is_enabled(atchan) && !at_xdmac_chan_is_cyclic(atchan))
1880			return -EAGAIN;
1881	}
1882	return 0;
1883}
1884#else
1885#	define atmel_xdmac_prepare NULL
1886#endif
1887
1888#ifdef CONFIG_PM_SLEEP
1889static int atmel_xdmac_suspend(struct device *dev)
1890{
1891	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);
 
1892	struct dma_chan		*chan, *_chan;
1893
1894	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
1895		struct at_xdmac_chan	*atchan = to_at_xdmac_chan(chan);
1896
1897		atchan->save_cc = at_xdmac_chan_read(atchan, AT_XDMAC_CC);
1898		if (at_xdmac_chan_is_cyclic(atchan)) {
1899			if (!at_xdmac_chan_is_paused(atchan))
1900				at_xdmac_device_pause(chan);
1901			atchan->save_cim = at_xdmac_chan_read(atchan, AT_XDMAC_CIM);
1902			atchan->save_cnda = at_xdmac_chan_read(atchan, AT_XDMAC_CNDA);
1903			atchan->save_cndc = at_xdmac_chan_read(atchan, AT_XDMAC_CNDC);
1904		}
1905	}
1906	atxdmac->save_gim = at_xdmac_read(atxdmac, AT_XDMAC_GIM);
1907
1908	at_xdmac_off(atxdmac);
1909	clk_disable_unprepare(atxdmac->clk);
1910	return 0;
1911}
1912
1913static int atmel_xdmac_resume(struct device *dev)
1914{
1915	struct at_xdmac		*atxdmac = dev_get_drvdata(dev);
 
1916	struct at_xdmac_chan	*atchan;
1917	struct dma_chan		*chan, *_chan;
1918	int			i;
1919	int ret;
1920
1921	ret = clk_prepare_enable(atxdmac->clk);
1922	if (ret)
1923		return ret;
1924
1925	/* Clear pending interrupts. */
1926	for (i = 0; i < atxdmac->dma.chancnt; i++) {
1927		atchan = &atxdmac->chan[i];
1928		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))
1929			cpu_relax();
1930	}
1931
1932	at_xdmac_write(atxdmac, AT_XDMAC_GIE, atxdmac->save_gim);
1933	list_for_each_entry_safe(chan, _chan, &atxdmac->dma.channels, device_node) {
1934		atchan = to_at_xdmac_chan(chan);
1935		at_xdmac_chan_write(atchan, AT_XDMAC_CC, atchan->save_cc);
1936		if (at_xdmac_chan_is_cyclic(atchan)) {
1937			if (at_xdmac_chan_is_paused(atchan))
1938				at_xdmac_device_resume(chan);
1939			at_xdmac_chan_write(atchan, AT_XDMAC_CNDA, atchan->save_cnda);
1940			at_xdmac_chan_write(atchan, AT_XDMAC_CNDC, atchan->save_cndc);
1941			at_xdmac_chan_write(atchan, AT_XDMAC_CIE, atchan->save_cim);
1942			wmb();
1943			at_xdmac_write(atxdmac, AT_XDMAC_GE, atchan->mask);
1944		}
1945	}
1946	return 0;
1947}
1948#endif /* CONFIG_PM_SLEEP */
1949
1950static int at_xdmac_probe(struct platform_device *pdev)
1951{
 
1952	struct at_xdmac	*atxdmac;
1953	int		irq, size, nr_channels, i, ret;
1954	void __iomem	*base;
1955	u32		reg;
1956
 
 
 
 
1957	irq = platform_get_irq(pdev, 0);
1958	if (irq < 0)
1959		return irq;
1960
1961	base = devm_platform_ioremap_resource(pdev, 0);
1962	if (IS_ERR(base))
1963		return PTR_ERR(base);
1964
1965	/*
1966	 * Read number of xdmac channels, read helper function can't be used
1967	 * since atxdmac is not yet allocated and we need to know the number
1968	 * of channels to do the allocation.
1969	 */
1970	reg = readl_relaxed(base + AT_XDMAC_GTYPE);
1971	nr_channels = AT_XDMAC_NB_CH(reg);
1972	if (nr_channels > AT_XDMAC_MAX_CHAN) {
1973		dev_err(&pdev->dev, "invalid number of channels (%u)\n",
1974			nr_channels);
1975		return -EINVAL;
1976	}
1977
1978	size = sizeof(*atxdmac);
1979	size += nr_channels * sizeof(struct at_xdmac_chan);
1980	atxdmac = devm_kzalloc(&pdev->dev, size, GFP_KERNEL);
1981	if (!atxdmac) {
1982		dev_err(&pdev->dev, "can't allocate at_xdmac structure\n");
1983		return -ENOMEM;
1984	}
1985
1986	atxdmac->regs = base;
1987	atxdmac->irq = irq;
1988
1989	atxdmac->clk = devm_clk_get(&pdev->dev, "dma_clk");
1990	if (IS_ERR(atxdmac->clk)) {
1991		dev_err(&pdev->dev, "can't get dma_clk\n");
1992		return PTR_ERR(atxdmac->clk);
1993	}
1994
1995	/* Do not use dev res to prevent races with tasklet */
1996	ret = request_irq(atxdmac->irq, at_xdmac_interrupt, 0, "at_xdmac", atxdmac);
1997	if (ret) {
1998		dev_err(&pdev->dev, "can't request irq\n");
1999		return ret;
2000	}
2001
2002	ret = clk_prepare_enable(atxdmac->clk);
2003	if (ret) {
2004		dev_err(&pdev->dev, "can't prepare or enable clock\n");
2005		goto err_free_irq;
2006	}
2007
2008	atxdmac->at_xdmac_desc_pool =
2009		dmam_pool_create(dev_name(&pdev->dev), &pdev->dev,
2010				sizeof(struct at_xdmac_desc), 4, 0);
2011	if (!atxdmac->at_xdmac_desc_pool) {
2012		dev_err(&pdev->dev, "no memory for descriptors dma pool\n");
2013		ret = -ENOMEM;
2014		goto err_clk_disable;
2015	}
2016
2017	dma_cap_set(DMA_CYCLIC, atxdmac->dma.cap_mask);
2018	dma_cap_set(DMA_INTERLEAVE, atxdmac->dma.cap_mask);
2019	dma_cap_set(DMA_MEMCPY, atxdmac->dma.cap_mask);
2020	dma_cap_set(DMA_MEMSET, atxdmac->dma.cap_mask);
2021	dma_cap_set(DMA_MEMSET_SG, atxdmac->dma.cap_mask);
2022	dma_cap_set(DMA_SLAVE, atxdmac->dma.cap_mask);
2023	/*
2024	 * Without DMA_PRIVATE the driver is not able to allocate more than
2025	 * one channel, second allocation fails in private_candidate.
2026	 */
2027	dma_cap_set(DMA_PRIVATE, atxdmac->dma.cap_mask);
2028	atxdmac->dma.dev				= &pdev->dev;
2029	atxdmac->dma.device_alloc_chan_resources	= at_xdmac_alloc_chan_resources;
2030	atxdmac->dma.device_free_chan_resources		= at_xdmac_free_chan_resources;
2031	atxdmac->dma.device_tx_status			= at_xdmac_tx_status;
2032	atxdmac->dma.device_issue_pending		= at_xdmac_issue_pending;
2033	atxdmac->dma.device_prep_dma_cyclic		= at_xdmac_prep_dma_cyclic;
2034	atxdmac->dma.device_prep_interleaved_dma	= at_xdmac_prep_interleaved;
2035	atxdmac->dma.device_prep_dma_memcpy		= at_xdmac_prep_dma_memcpy;
2036	atxdmac->dma.device_prep_dma_memset		= at_xdmac_prep_dma_memset;
2037	atxdmac->dma.device_prep_dma_memset_sg		= at_xdmac_prep_dma_memset_sg;
2038	atxdmac->dma.device_prep_slave_sg		= at_xdmac_prep_slave_sg;
2039	atxdmac->dma.device_config			= at_xdmac_device_config;
2040	atxdmac->dma.device_pause			= at_xdmac_device_pause;
2041	atxdmac->dma.device_resume			= at_xdmac_device_resume;
2042	atxdmac->dma.device_terminate_all		= at_xdmac_device_terminate_all;
2043	atxdmac->dma.src_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;
2044	atxdmac->dma.dst_addr_widths = AT_XDMAC_DMA_BUSWIDTHS;
2045	atxdmac->dma.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
2046	atxdmac->dma.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
2047
2048	/* Disable all chans and interrupts. */
2049	at_xdmac_off(atxdmac);
2050
2051	/* Init channels. */
2052	INIT_LIST_HEAD(&atxdmac->dma.channels);
2053	for (i = 0; i < nr_channels; i++) {
2054		struct at_xdmac_chan *atchan = &atxdmac->chan[i];
2055
2056		atchan->chan.device = &atxdmac->dma;
2057		list_add_tail(&atchan->chan.device_node,
2058			      &atxdmac->dma.channels);
2059
2060		atchan->ch_regs = at_xdmac_chan_reg_base(atxdmac, i);
2061		atchan->mask = 1 << i;
2062
2063		spin_lock_init(&atchan->lock);
2064		INIT_LIST_HEAD(&atchan->xfers_list);
2065		INIT_LIST_HEAD(&atchan->free_descs_list);
2066		tasklet_init(&atchan->tasklet, at_xdmac_tasklet,
2067			     (unsigned long)atchan);
2068
2069		/* Clear pending interrupts. */
2070		while (at_xdmac_chan_read(atchan, AT_XDMAC_CIS))
2071			cpu_relax();
2072	}
2073	platform_set_drvdata(pdev, atxdmac);
2074
2075	ret = dma_async_device_register(&atxdmac->dma);
2076	if (ret) {
2077		dev_err(&pdev->dev, "fail to register DMA engine device\n");
2078		goto err_clk_disable;
2079	}
2080
2081	ret = of_dma_controller_register(pdev->dev.of_node,
2082					 at_xdmac_xlate, atxdmac);
2083	if (ret) {
2084		dev_err(&pdev->dev, "could not register of dma controller\n");
2085		goto err_dma_unregister;
2086	}
2087
2088	dev_info(&pdev->dev, "%d channels, mapped at 0x%p\n",
2089		 nr_channels, atxdmac->regs);
2090
2091	return 0;
2092
2093err_dma_unregister:
2094	dma_async_device_unregister(&atxdmac->dma);
2095err_clk_disable:
2096	clk_disable_unprepare(atxdmac->clk);
2097err_free_irq:
2098	free_irq(atxdmac->irq, atxdmac);
2099	return ret;
2100}
2101
2102static int at_xdmac_remove(struct platform_device *pdev)
2103{
2104	struct at_xdmac	*atxdmac = (struct at_xdmac *)platform_get_drvdata(pdev);
2105	int		i;
2106
2107	at_xdmac_off(atxdmac);
2108	of_dma_controller_free(pdev->dev.of_node);
2109	dma_async_device_unregister(&atxdmac->dma);
2110	clk_disable_unprepare(atxdmac->clk);
2111
2112	free_irq(atxdmac->irq, atxdmac);
2113
2114	for (i = 0; i < atxdmac->dma.chancnt; i++) {
2115		struct at_xdmac_chan *atchan = &atxdmac->chan[i];
2116
2117		tasklet_kill(&atchan->tasklet);
2118		at_xdmac_free_chan_resources(&atchan->chan);
2119	}
2120
2121	return 0;
2122}
2123
2124static const struct dev_pm_ops atmel_xdmac_dev_pm_ops = {
2125	.prepare	= atmel_xdmac_prepare,
2126	SET_LATE_SYSTEM_SLEEP_PM_OPS(atmel_xdmac_suspend, atmel_xdmac_resume)
2127};
2128
2129static const struct of_device_id atmel_xdmac_dt_ids[] = {
2130	{
2131		.compatible = "atmel,sama5d4-dma",
2132	}, {
2133		/* sentinel */
2134	}
2135};
2136MODULE_DEVICE_TABLE(of, atmel_xdmac_dt_ids);
2137
2138static struct platform_driver at_xdmac_driver = {
2139	.probe		= at_xdmac_probe,
2140	.remove		= at_xdmac_remove,
2141	.driver = {
2142		.name		= "at_xdmac",
2143		.of_match_table	= of_match_ptr(atmel_xdmac_dt_ids),
2144		.pm		= &atmel_xdmac_dev_pm_ops,
2145	}
2146};
2147
2148static int __init at_xdmac_init(void)
2149{
2150	return platform_driver_probe(&at_xdmac_driver, at_xdmac_probe);
2151}
2152subsys_initcall(at_xdmac_init);
2153
2154MODULE_DESCRIPTION("Atmel Extended DMA Controller driver");
2155MODULE_AUTHOR("Ludovic Desroches <ludovic.desroches@atmel.com>");
2156MODULE_LICENSE("GPL");