1/*
   2 * Copyright (C) 2017 Spreadtrum Communications Inc.
   3 *
   4 * SPDX-License-Identifier: GPL-2.0
   5 */
   6
   7#include <linux/clk.h>
   8#include <linux/dma-mapping.h>
   9#include <linux/dma/sprd-dma.h>
  10#include <linux/errno.h>
  11#include <linux/init.h>
  12#include <linux/interrupt.h>
  13#include <linux/io.h>
  14#include <linux/kernel.h>
  15#include <linux/module.h>
  16#include <linux/of.h>
  17#include <linux/of_dma.h>
  18#include <linux/of_device.h>
  19#include <linux/pm_runtime.h>
  20#include <linux/slab.h>
  21
  22#include "virt-dma.h"
  23
  24#define SPRD_DMA_CHN_REG_OFFSET		0x1000
  25#define SPRD_DMA_CHN_REG_LENGTH		0x40
  26#define SPRD_DMA_MEMCPY_MIN_SIZE	64
  27
  28/* DMA global registers definition */
  29#define SPRD_DMA_GLB_PAUSE		0x0
  30#define SPRD_DMA_GLB_FRAG_WAIT		0x4
  31#define SPRD_DMA_GLB_REQ_PEND0_EN	0x8
  32#define SPRD_DMA_GLB_REQ_PEND1_EN	0xc
  33#define SPRD_DMA_GLB_INT_RAW_STS	0x10
  34#define SPRD_DMA_GLB_INT_MSK_STS	0x14
  35#define SPRD_DMA_GLB_REQ_STS		0x18
  36#define SPRD_DMA_GLB_CHN_EN_STS		0x1c
  37#define SPRD_DMA_GLB_DEBUG_STS		0x20
  38#define SPRD_DMA_GLB_ARB_SEL_STS	0x24
  39#define SPRD_DMA_GLB_2STAGE_GRP1	0x28
  40#define SPRD_DMA_GLB_2STAGE_GRP2	0x2c
  41#define SPRD_DMA_GLB_REQ_UID(uid)	(0x4 * ((uid) - 1))
  42#define SPRD_DMA_GLB_REQ_UID_OFFSET	0x2000
  43
  44/* DMA channel registers definition */
  45#define SPRD_DMA_CHN_PAUSE		0x0
  46#define SPRD_DMA_CHN_REQ		0x4
  47#define SPRD_DMA_CHN_CFG		0x8
  48#define SPRD_DMA_CHN_INTC		0xc
  49#define SPRD_DMA_CHN_SRC_ADDR		0x10
  50#define SPRD_DMA_CHN_DES_ADDR		0x14
  51#define SPRD_DMA_CHN_FRG_LEN		0x18
  52#define SPRD_DMA_CHN_BLK_LEN		0x1c
  53#define SPRD_DMA_CHN_TRSC_LEN		0x20
  54#define SPRD_DMA_CHN_TRSF_STEP		0x24
  55#define SPRD_DMA_CHN_WARP_PTR		0x28
  56#define SPRD_DMA_CHN_WARP_TO		0x2c
  57#define SPRD_DMA_CHN_LLIST_PTR		0x30
  58#define SPRD_DMA_CHN_FRAG_STEP		0x34
  59#define SPRD_DMA_CHN_SRC_BLK_STEP	0x38
  60#define SPRD_DMA_CHN_DES_BLK_STEP	0x3c
  61
  62/* SPRD_DMA_GLB_2STAGE_GRP register definition */
  63#define SPRD_DMA_GLB_2STAGE_EN		BIT(24)
  64#define SPRD_DMA_GLB_CHN_INT_MASK	GENMASK(23, 20)
  65#define SPRD_DMA_GLB_DEST_INT		BIT(22)
  66#define SPRD_DMA_GLB_SRC_INT		BIT(20)
  67#define SPRD_DMA_GLB_LIST_DONE_TRG	BIT(19)
  68#define SPRD_DMA_GLB_TRANS_DONE_TRG	BIT(18)
  69#define SPRD_DMA_GLB_BLOCK_DONE_TRG	BIT(17)
  70#define SPRD_DMA_GLB_FRAG_DONE_TRG	BIT(16)
  71#define SPRD_DMA_GLB_TRG_OFFSET		16
  72#define SPRD_DMA_GLB_DEST_CHN_MASK	GENMASK(13, 8)
  73#define SPRD_DMA_GLB_DEST_CHN_OFFSET	8
  74#define SPRD_DMA_GLB_SRC_CHN_MASK	GENMASK(5, 0)
  75
  76/* SPRD_DMA_CHN_INTC register definition */
  77#define SPRD_DMA_INT_MASK		GENMASK(4, 0)
  78#define SPRD_DMA_INT_CLR_OFFSET		24
  79#define SPRD_DMA_FRAG_INT_EN		BIT(0)
  80#define SPRD_DMA_BLK_INT_EN		BIT(1)
  81#define SPRD_DMA_TRANS_INT_EN		BIT(2)
  82#define SPRD_DMA_LIST_INT_EN		BIT(3)
  83#define SPRD_DMA_CFG_ERR_INT_EN		BIT(4)
  84
  85/* SPRD_DMA_CHN_CFG register definition */
  86#define SPRD_DMA_CHN_EN			BIT(0)
  87#define SPRD_DMA_LINKLIST_EN		BIT(4)
  88#define SPRD_DMA_WAIT_BDONE_OFFSET	24
  89#define SPRD_DMA_DONOT_WAIT_BDONE	1
  90
  91/* SPRD_DMA_CHN_REQ register definition */
  92#define SPRD_DMA_REQ_EN			BIT(0)
  93
  94/* SPRD_DMA_CHN_PAUSE register definition */
  95#define SPRD_DMA_PAUSE_EN		BIT(0)
  96#define SPRD_DMA_PAUSE_STS		BIT(2)
  97#define SPRD_DMA_PAUSE_CNT		0x2000
  98
  99/* DMA_CHN_WARP_* register definition */
 100#define SPRD_DMA_HIGH_ADDR_MASK		GENMASK(31, 28)
 101#define SPRD_DMA_LOW_ADDR_MASK		GENMASK(31, 0)
 102#define SPRD_DMA_WRAP_ADDR_MASK		GENMASK(27, 0)
 103#define SPRD_DMA_HIGH_ADDR_OFFSET	4
 104
 105/* SPRD_DMA_CHN_INTC register definition */
 106#define SPRD_DMA_FRAG_INT_STS		BIT(16)
 107#define SPRD_DMA_BLK_INT_STS		BIT(17)
 108#define SPRD_DMA_TRSC_INT_STS		BIT(18)
 109#define SPRD_DMA_LIST_INT_STS		BIT(19)
 110#define SPRD_DMA_CFGERR_INT_STS		BIT(20)
 111#define SPRD_DMA_CHN_INT_STS					\
 112	(SPRD_DMA_FRAG_INT_STS | SPRD_DMA_BLK_INT_STS |		\
 113	 SPRD_DMA_TRSC_INT_STS | SPRD_DMA_LIST_INT_STS |	\
 114	 SPRD_DMA_CFGERR_INT_STS)
 115
 116/* SPRD_DMA_CHN_FRG_LEN register definition */
 117#define SPRD_DMA_SRC_DATAWIDTH_OFFSET	30
 118#define SPRD_DMA_DES_DATAWIDTH_OFFSET	28
 119#define SPRD_DMA_SWT_MODE_OFFSET	26
 120#define SPRD_DMA_REQ_MODE_OFFSET	24
 121#define SPRD_DMA_REQ_MODE_MASK		GENMASK(1, 0)
 122#define SPRD_DMA_WRAP_SEL_DEST		BIT(23)
 123#define SPRD_DMA_WRAP_EN		BIT(22)
 124#define SPRD_DMA_FIX_SEL_OFFSET		21
 125#define SPRD_DMA_FIX_EN_OFFSET		20
 126#define SPRD_DMA_LLIST_END		BIT(19)
 127#define SPRD_DMA_FRG_LEN_MASK		GENMASK(16, 0)
 128
 129/* SPRD_DMA_CHN_BLK_LEN register definition */
 130#define SPRD_DMA_BLK_LEN_MASK		GENMASK(16, 0)
 131
 132/* SPRD_DMA_CHN_TRSC_LEN register definition */
 133#define SPRD_DMA_TRSC_LEN_MASK		GENMASK(27, 0)
 134
 135/* SPRD_DMA_CHN_TRSF_STEP register definition */
 136#define SPRD_DMA_DEST_TRSF_STEP_OFFSET	16
 137#define SPRD_DMA_SRC_TRSF_STEP_OFFSET	0
 138#define SPRD_DMA_TRSF_STEP_MASK		GENMASK(15, 0)
 139
 140/* SPRD DMA_SRC_BLK_STEP register definition */
 141#define SPRD_DMA_LLIST_HIGH_MASK	GENMASK(31, 28)
 142#define SPRD_DMA_LLIST_HIGH_SHIFT	28
 143
 144/* define DMA channel mode & trigger mode mask */
 145#define SPRD_DMA_CHN_MODE_MASK		GENMASK(7, 0)
 146#define SPRD_DMA_TRG_MODE_MASK		GENMASK(7, 0)
 147#define SPRD_DMA_INT_TYPE_MASK		GENMASK(7, 0)
 148
 149/* define the DMA transfer step type */
 150#define SPRD_DMA_NONE_STEP		0
 151#define SPRD_DMA_BYTE_STEP		1
 152#define SPRD_DMA_SHORT_STEP		2
 153#define SPRD_DMA_WORD_STEP		4
 154#define SPRD_DMA_DWORD_STEP		8
 155
 156#define SPRD_DMA_SOFTWARE_UID		0
 157
 158/* dma data width values */
 159enum sprd_dma_datawidth {
 160	SPRD_DMA_DATAWIDTH_1_BYTE,
 161	SPRD_DMA_DATAWIDTH_2_BYTES,
 162	SPRD_DMA_DATAWIDTH_4_BYTES,
 163	SPRD_DMA_DATAWIDTH_8_BYTES,
 164};
 165
 166/* dma channel hardware configuration */
 167struct sprd_dma_chn_hw {
 168	u32 pause;
 169	u32 req;
 170	u32 cfg;
 171	u32 intc;
 172	u32 src_addr;
 173	u32 des_addr;
 174	u32 frg_len;
 175	u32 blk_len;
 176	u32 trsc_len;
 177	u32 trsf_step;
 178	u32 wrap_ptr;
 179	u32 wrap_to;
 180	u32 llist_ptr;
 181	u32 frg_step;
 182	u32 src_blk_step;
 183	u32 des_blk_step;
 184};
 185
 186/* dma request description */
 187struct sprd_dma_desc {
 188	struct virt_dma_desc	vd;
 189	struct sprd_dma_chn_hw	chn_hw;
 190	enum dma_transfer_direction dir;
 191};
 192
 193/* dma channel description */
 194struct sprd_dma_chn {
 195	struct virt_dma_chan	vc;
 196	void __iomem		*chn_base;
 197	struct sprd_dma_linklist	linklist;
 198	struct dma_slave_config	slave_cfg;
 199	u32			chn_num;
 200	u32			dev_id;
 201	enum sprd_dma_chn_mode	chn_mode;
 202	enum sprd_dma_trg_mode	trg_mode;
 203	enum sprd_dma_int_type	int_type;
 204	struct sprd_dma_desc	*cur_desc;
 205};
 206
 207/* SPRD dma device */
 208struct sprd_dma_dev {
 209	struct dma_device	dma_dev;
 210	void __iomem		*glb_base;
 211	struct clk		*clk;
 212	struct clk		*ashb_clk;
 213	int			irq;
 214	u32			total_chns;
 215	struct sprd_dma_chn	channels[];
 216};
 217
 218static void sprd_dma_free_desc(struct virt_dma_desc *vd);
 219static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param);
 220static struct of_dma_filter_info sprd_dma_info = {
 221	.filter_fn = sprd_dma_filter_fn,
 222};
 223
 224static inline struct sprd_dma_chn *to_sprd_dma_chan(struct dma_chan *c)
 225{
 226	return container_of(c, struct sprd_dma_chn, vc.chan);
 227}
 228
 229static inline struct sprd_dma_dev *to_sprd_dma_dev(struct dma_chan *c)
 230{
 231	struct sprd_dma_chn *schan = to_sprd_dma_chan(c);
 232
 233	return container_of(schan, struct sprd_dma_dev, channels[c->chan_id]);
 234}
 235
 236static inline struct sprd_dma_desc *to_sprd_dma_desc(struct virt_dma_desc *vd)
 237{
 238	return container_of(vd, struct sprd_dma_desc, vd);
 239}
 240
 241static void sprd_dma_glb_update(struct sprd_dma_dev *sdev, u32 reg,
 242				u32 mask, u32 val)
 243{
 244	u32 orig = readl(sdev->glb_base + reg);
 245	u32 tmp;
 246
 247	tmp = (orig & ~mask) | val;
 248	writel(tmp, sdev->glb_base + reg);
 249}
 250
 251static void sprd_dma_chn_update(struct sprd_dma_chn *schan, u32 reg,
 252				u32 mask, u32 val)
 253{
 254	u32 orig = readl(schan->chn_base + reg);
 255	u32 tmp;
 256
 257	tmp = (orig & ~mask) | val;
 258	writel(tmp, schan->chn_base + reg);
 259}
 260
 261static int sprd_dma_enable(struct sprd_dma_dev *sdev)
 262{
 263	int ret;
 264
 265	ret = clk_prepare_enable(sdev->clk);
 266	if (ret)
 267		return ret;
 268
 269	/*
 270	 * The ashb_clk is optional and only for AGCP DMA controller, so we
 271	 * need add one condition to check if the ashb_clk need enable.
 272	 */
 273	if (!IS_ERR(sdev->ashb_clk))
 274		ret = clk_prepare_enable(sdev->ashb_clk);
 275
 276	return ret;
 277}
 278
 279static void sprd_dma_disable(struct sprd_dma_dev *sdev)
 280{
 281	clk_disable_unprepare(sdev->clk);
 282
 283	/*
 284	 * Need to check if we need disable the optional ashb_clk for AGCP DMA.
 285	 */
 286	if (!IS_ERR(sdev->ashb_clk))
 287		clk_disable_unprepare(sdev->ashb_clk);
 288}
 289
 290static void sprd_dma_set_uid(struct sprd_dma_chn *schan)
 291{
 292	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 293	u32 dev_id = schan->dev_id;
 294
 295	if (dev_id != SPRD_DMA_SOFTWARE_UID) {
 296		u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET +
 297				 SPRD_DMA_GLB_REQ_UID(dev_id);
 298
 299		writel(schan->chn_num + 1, sdev->glb_base + uid_offset);
 300	}
 301}
 302
 303static void sprd_dma_unset_uid(struct sprd_dma_chn *schan)
 304{
 305	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 306	u32 dev_id = schan->dev_id;
 307
 308	if (dev_id != SPRD_DMA_SOFTWARE_UID) {
 309		u32 uid_offset = SPRD_DMA_GLB_REQ_UID_OFFSET +
 310				 SPRD_DMA_GLB_REQ_UID(dev_id);
 311
 312		writel(0, sdev->glb_base + uid_offset);
 313	}
 314}
 315
 316static void sprd_dma_clear_int(struct sprd_dma_chn *schan)
 317{
 318	sprd_dma_chn_update(schan, SPRD_DMA_CHN_INTC,
 319			    SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET,
 320			    SPRD_DMA_INT_MASK << SPRD_DMA_INT_CLR_OFFSET);
 321}
 322
 323static void sprd_dma_enable_chn(struct sprd_dma_chn *schan)
 324{
 325	sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN,
 326			    SPRD_DMA_CHN_EN);
 327}
 328
 329static void sprd_dma_disable_chn(struct sprd_dma_chn *schan)
 330{
 331	sprd_dma_chn_update(schan, SPRD_DMA_CHN_CFG, SPRD_DMA_CHN_EN, 0);
 332}
 333
 334static void sprd_dma_soft_request(struct sprd_dma_chn *schan)
 335{
 336	sprd_dma_chn_update(schan, SPRD_DMA_CHN_REQ, SPRD_DMA_REQ_EN,
 337			    SPRD_DMA_REQ_EN);
 338}
 339
 340static void sprd_dma_pause_resume(struct sprd_dma_chn *schan, bool enable)
 341{
 342	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 343	u32 pause, timeout = SPRD_DMA_PAUSE_CNT;
 344
 345	if (enable) {
 346		sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,
 347				    SPRD_DMA_PAUSE_EN, SPRD_DMA_PAUSE_EN);
 348
 349		do {
 350			pause = readl(schan->chn_base + SPRD_DMA_CHN_PAUSE);
 351			if (pause & SPRD_DMA_PAUSE_STS)
 352				break;
 353
 354			cpu_relax();
 355		} while (--timeout > 0);
 356
 357		if (!timeout)
 358			dev_warn(sdev->dma_dev.dev,
 359				 "pause dma controller timeout\n");
 360	} else {
 361		sprd_dma_chn_update(schan, SPRD_DMA_CHN_PAUSE,
 362				    SPRD_DMA_PAUSE_EN, 0);
 363	}
 364}
 365
 366static void sprd_dma_stop_and_disable(struct sprd_dma_chn *schan)
 367{
 368	u32 cfg = readl(schan->chn_base + SPRD_DMA_CHN_CFG);
 369
 370	if (!(cfg & SPRD_DMA_CHN_EN))
 371		return;
 372
 373	sprd_dma_pause_resume(schan, true);
 374	sprd_dma_disable_chn(schan);
 375}
 376
 377static unsigned long sprd_dma_get_src_addr(struct sprd_dma_chn *schan)
 378{
 379	unsigned long addr, addr_high;
 380
 381	addr = readl(schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);
 382	addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_PTR) &
 383		    SPRD_DMA_HIGH_ADDR_MASK;
 384
 385	return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET);
 386}
 387
 388static unsigned long sprd_dma_get_dst_addr(struct sprd_dma_chn *schan)
 389{
 390	unsigned long addr, addr_high;
 391
 392	addr = readl(schan->chn_base + SPRD_DMA_CHN_DES_ADDR);
 393	addr_high = readl(schan->chn_base + SPRD_DMA_CHN_WARP_TO) &
 394		    SPRD_DMA_HIGH_ADDR_MASK;
 395
 396	return addr | (addr_high << SPRD_DMA_HIGH_ADDR_OFFSET);
 397}
 398
 399static enum sprd_dma_int_type sprd_dma_get_int_type(struct sprd_dma_chn *schan)
 400{
 401	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 402	u32 intc_sts = readl(schan->chn_base + SPRD_DMA_CHN_INTC) &
 403		       SPRD_DMA_CHN_INT_STS;
 404
 405	switch (intc_sts) {
 406	case SPRD_DMA_CFGERR_INT_STS:
 407		return SPRD_DMA_CFGERR_INT;
 408
 409	case SPRD_DMA_LIST_INT_STS:
 410		return SPRD_DMA_LIST_INT;
 411
 412	case SPRD_DMA_TRSC_INT_STS:
 413		return SPRD_DMA_TRANS_INT;
 414
 415	case SPRD_DMA_BLK_INT_STS:
 416		return SPRD_DMA_BLK_INT;
 417
 418	case SPRD_DMA_FRAG_INT_STS:
 419		return SPRD_DMA_FRAG_INT;
 420
 421	default:
 422		dev_warn(sdev->dma_dev.dev, "incorrect dma interrupt type\n");
 423		return SPRD_DMA_NO_INT;
 424	}
 425}
 426
 427static enum sprd_dma_req_mode sprd_dma_get_req_type(struct sprd_dma_chn *schan)
 428{
 429	u32 frag_reg = readl(schan->chn_base + SPRD_DMA_CHN_FRG_LEN);
 430
 431	return (frag_reg >> SPRD_DMA_REQ_MODE_OFFSET) & SPRD_DMA_REQ_MODE_MASK;
 432}
 433
 434static int sprd_dma_set_2stage_config(struct sprd_dma_chn *schan)
 435{
 436	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 437	u32 val, chn = schan->chn_num + 1;
 438
 439	switch (schan->chn_mode) {
 440	case SPRD_DMA_SRC_CHN0:
 441		val = chn & SPRD_DMA_GLB_SRC_CHN_MASK;
 442		val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET;
 443		val |= SPRD_DMA_GLB_2STAGE_EN;
 444		if (schan->int_type != SPRD_DMA_NO_INT)
 445			val |= SPRD_DMA_GLB_SRC_INT;
 446
 447		sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP1, val, val);
 448		break;
 449
 450	case SPRD_DMA_SRC_CHN1:
 451		val = chn & SPRD_DMA_GLB_SRC_CHN_MASK;
 452		val |= BIT(schan->trg_mode - 1) << SPRD_DMA_GLB_TRG_OFFSET;
 453		val |= SPRD_DMA_GLB_2STAGE_EN;
 454		if (schan->int_type != SPRD_DMA_NO_INT)
 455			val |= SPRD_DMA_GLB_SRC_INT;
 456
 457		sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP2, val, val);
 458		break;
 459
 460	case SPRD_DMA_DST_CHN0:
 461		val = (chn << SPRD_DMA_GLB_DEST_CHN_OFFSET) &
 462			SPRD_DMA_GLB_DEST_CHN_MASK;
 463		val |= SPRD_DMA_GLB_2STAGE_EN;
 464		if (schan->int_type != SPRD_DMA_NO_INT)
 465			val |= SPRD_DMA_GLB_DEST_INT;
 466
 467		sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP1, val, val);
 468		break;
 469
 470	case SPRD_DMA_DST_CHN1:
 471		val = (chn << SPRD_DMA_GLB_DEST_CHN_OFFSET) &
 472			SPRD_DMA_GLB_DEST_CHN_MASK;
 473		val |= SPRD_DMA_GLB_2STAGE_EN;
 474		if (schan->int_type != SPRD_DMA_NO_INT)
 475			val |= SPRD_DMA_GLB_DEST_INT;
 476
 477		sprd_dma_glb_update(sdev, SPRD_DMA_GLB_2STAGE_GRP2, val, val);
 478		break;
 479
 480	default:
 481		dev_err(sdev->dma_dev.dev, "invalid channel mode setting %d\n",
 482			schan->chn_mode);
 483		return -EINVAL;
 484	}
 485
 486	return 0;
 487}
 488
 489static void sprd_dma_set_pending(struct sprd_dma_chn *schan, bool enable)
 490{
 491	struct sprd_dma_dev *sdev = to_sprd_dma_dev(&schan->vc.chan);
 492	u32 reg, val, req_id;
 493
 494	if (schan->dev_id == SPRD_DMA_SOFTWARE_UID)
 495		return;
 496
 497	/* The DMA request id always starts from 0. */
 498	req_id = schan->dev_id - 1;
 499
 500	if (req_id < 32) {
 501		reg = SPRD_DMA_GLB_REQ_PEND0_EN;
 502		val = BIT(req_id);
 503	} else {
 504		reg = SPRD_DMA_GLB_REQ_PEND1_EN;
 505		val = BIT(req_id - 32);
 506	}
 507
 508	sprd_dma_glb_update(sdev, reg, val, enable ? val : 0);
 509}
 510
 511static void sprd_dma_set_chn_config(struct sprd_dma_chn *schan,
 512				    struct sprd_dma_desc *sdesc)
 513{
 514	struct sprd_dma_chn_hw *cfg = &sdesc->chn_hw;
 515
 516	writel(cfg->pause, schan->chn_base + SPRD_DMA_CHN_PAUSE);
 517	writel(cfg->cfg, schan->chn_base + SPRD_DMA_CHN_CFG);
 518	writel(cfg->intc, schan->chn_base + SPRD_DMA_CHN_INTC);
 519	writel(cfg->src_addr, schan->chn_base + SPRD_DMA_CHN_SRC_ADDR);
 520	writel(cfg->des_addr, schan->chn_base + SPRD_DMA_CHN_DES_ADDR);
 521	writel(cfg->frg_len, schan->chn_base + SPRD_DMA_CHN_FRG_LEN);
 522	writel(cfg->blk_len, schan->chn_base + SPRD_DMA_CHN_BLK_LEN);
 523	writel(cfg->trsc_len, schan->chn_base + SPRD_DMA_CHN_TRSC_LEN);
 524	writel(cfg->trsf_step, schan->chn_base + SPRD_DMA_CHN_TRSF_STEP);
 525	writel(cfg->wrap_ptr, schan->chn_base + SPRD_DMA_CHN_WARP_PTR);
 526	writel(cfg->wrap_to, schan->chn_base + SPRD_DMA_CHN_WARP_TO);
 527	writel(cfg->llist_ptr, schan->chn_base + SPRD_DMA_CHN_LLIST_PTR);
 528	writel(cfg->frg_step, schan->chn_base + SPRD_DMA_CHN_FRAG_STEP);
 529	writel(cfg->src_blk_step, schan->chn_base + SPRD_DMA_CHN_SRC_BLK_STEP);
 530	writel(cfg->des_blk_step, schan->chn_base + SPRD_DMA_CHN_DES_BLK_STEP);
 531	writel(cfg->req, schan->chn_base + SPRD_DMA_CHN_REQ);
 532}
 533
 534static void sprd_dma_start(struct sprd_dma_chn *schan)
 535{
 536	struct virt_dma_desc *vd = vchan_next_desc(&schan->vc);
 537
 538	if (!vd)
 539		return;
 540
 541	list_del(&vd->node);
 542	schan->cur_desc = to_sprd_dma_desc(vd);
 543
 544	/*
 545	 * Set 2-stage configuration if the channel starts one 2-stage
 546	 * transfer.
 547	 */
 548	if (schan->chn_mode && sprd_dma_set_2stage_config(schan))
 549		return;
 550
 551	/*
 552	 * Copy the DMA configuration from DMA descriptor to this hardware
 553	 * channel.
 554	 */
 555	sprd_dma_set_chn_config(schan, schan->cur_desc);
 556	sprd_dma_set_uid(schan);
 557	sprd_dma_set_pending(schan, true);
 558	sprd_dma_enable_chn(schan);
 559
 560	if (schan->dev_id == SPRD_DMA_SOFTWARE_UID &&
 561	    schan->chn_mode != SPRD_DMA_DST_CHN0 &&
 562	    schan->chn_mode != SPRD_DMA_DST_CHN1)
 563		sprd_dma_soft_request(schan);
 564}
 565
 566static void sprd_dma_stop(struct sprd_dma_chn *schan)
 567{
 568	sprd_dma_stop_and_disable(schan);
 569	sprd_dma_set_pending(schan, false);
 570	sprd_dma_unset_uid(schan);
 571	sprd_dma_clear_int(schan);
 572	schan->cur_desc = NULL;
 573}
 574
 575static bool sprd_dma_check_trans_done(struct sprd_dma_desc *sdesc,
 576				      enum sprd_dma_int_type int_type,
 577				      enum sprd_dma_req_mode req_mode)
 578{
 579	if (int_type == SPRD_DMA_NO_INT)
 580		return false;
 581
 582	if (int_type >= req_mode + 1)
 583		return true;
 584	else
 585		return false;
 586}
 587
 588static irqreturn_t dma_irq_handle(int irq, void *dev_id)
 589{
 590	struct sprd_dma_dev *sdev = (struct sprd_dma_dev *)dev_id;
 591	u32 irq_status = readl(sdev->glb_base + SPRD_DMA_GLB_INT_MSK_STS);
 592	struct sprd_dma_chn *schan;
 593	struct sprd_dma_desc *sdesc;
 594	enum sprd_dma_req_mode req_type;
 595	enum sprd_dma_int_type int_type;
 596	bool trans_done = false, cyclic = false;
 597	u32 i;
 598
 599	while (irq_status) {
 600		i = __ffs(irq_status);
 601		irq_status &= (irq_status - 1);
 602		schan = &sdev->channels[i];
 603
 604		spin_lock(&schan->vc.lock);
 605
 606		sdesc = schan->cur_desc;
 607		if (!sdesc) {
 608			spin_unlock(&schan->vc.lock);
 609			return IRQ_HANDLED;
 610		}
 611
 612		int_type = sprd_dma_get_int_type(schan);
 613		req_type = sprd_dma_get_req_type(schan);
 614		sprd_dma_clear_int(schan);
 615
 616		/* cyclic mode schedule callback */
 617		cyclic = schan->linklist.phy_addr ? true : false;
 618		if (cyclic == true) {
 619			vchan_cyclic_callback(&sdesc->vd);
 620		} else {
 621			/* Check if the dma request descriptor is done. */
 622			trans_done = sprd_dma_check_trans_done(sdesc, int_type,
 623							       req_type);
 624			if (trans_done == true) {
 625				vchan_cookie_complete(&sdesc->vd);
 626				schan->cur_desc = NULL;
 627				sprd_dma_start(schan);
 628			}
 629		}
 630		spin_unlock(&schan->vc.lock);
 631	}
 632
 633	return IRQ_HANDLED;
 634}
 635
 636static int sprd_dma_alloc_chan_resources(struct dma_chan *chan)
 637{
 638	return pm_runtime_get_sync(chan->device->dev);
 639}
 640
 641static void sprd_dma_free_chan_resources(struct dma_chan *chan)
 642{
 643	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 644	struct virt_dma_desc *cur_vd = NULL;
 645	unsigned long flags;
 646
 647	spin_lock_irqsave(&schan->vc.lock, flags);
 648	if (schan->cur_desc)
 649		cur_vd = &schan->cur_desc->vd;
 650
 651	sprd_dma_stop(schan);
 652	spin_unlock_irqrestore(&schan->vc.lock, flags);
 653
 654	if (cur_vd)
 655		sprd_dma_free_desc(cur_vd);
 656
 657	vchan_free_chan_resources(&schan->vc);
 658	pm_runtime_put(chan->device->dev);
 659}
 660
 661static enum dma_status sprd_dma_tx_status(struct dma_chan *chan,
 662					  dma_cookie_t cookie,
 663					  struct dma_tx_state *txstate)
 664{
 665	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 666	struct virt_dma_desc *vd;
 667	unsigned long flags;
 668	enum dma_status ret;
 669	u32 pos;
 670
 671	ret = dma_cookie_status(chan, cookie, txstate);
 672	if (ret == DMA_COMPLETE || !txstate)
 673		return ret;
 674
 675	spin_lock_irqsave(&schan->vc.lock, flags);
 676	vd = vchan_find_desc(&schan->vc, cookie);
 677	if (vd) {
 678		struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd);
 679		struct sprd_dma_chn_hw *hw = &sdesc->chn_hw;
 680
 681		if (hw->trsc_len > 0)
 682			pos = hw->trsc_len;
 683		else if (hw->blk_len > 0)
 684			pos = hw->blk_len;
 685		else if (hw->frg_len > 0)
 686			pos = hw->frg_len;
 687		else
 688			pos = 0;
 689	} else if (schan->cur_desc && schan->cur_desc->vd.tx.cookie == cookie) {
 690		struct sprd_dma_desc *sdesc = schan->cur_desc;
 691
 692		if (sdesc->dir == DMA_DEV_TO_MEM)
 693			pos = sprd_dma_get_dst_addr(schan);
 694		else
 695			pos = sprd_dma_get_src_addr(schan);
 696	} else {
 697		pos = 0;
 698	}
 699	spin_unlock_irqrestore(&schan->vc.lock, flags);
 700
 701	dma_set_residue(txstate, pos);
 702	return ret;
 703}
 704
 705static void sprd_dma_issue_pending(struct dma_chan *chan)
 706{
 707	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 708	unsigned long flags;
 709
 710	spin_lock_irqsave(&schan->vc.lock, flags);
 711	if (vchan_issue_pending(&schan->vc) && !schan->cur_desc)
 712		sprd_dma_start(schan);
 713	spin_unlock_irqrestore(&schan->vc.lock, flags);
 714}
 715
 716static int sprd_dma_get_datawidth(enum dma_slave_buswidth buswidth)
 717{
 718	switch (buswidth) {
 719	case DMA_SLAVE_BUSWIDTH_1_BYTE:
 720	case DMA_SLAVE_BUSWIDTH_2_BYTES:
 721	case DMA_SLAVE_BUSWIDTH_4_BYTES:
 722	case DMA_SLAVE_BUSWIDTH_8_BYTES:
 723		return ffs(buswidth) - 1;
 724
 725	default:
 726		return -EINVAL;
 727	}
 728}
 729
 730static int sprd_dma_get_step(enum dma_slave_buswidth buswidth)
 731{
 732	switch (buswidth) {
 733	case DMA_SLAVE_BUSWIDTH_1_BYTE:
 734	case DMA_SLAVE_BUSWIDTH_2_BYTES:
 735	case DMA_SLAVE_BUSWIDTH_4_BYTES:
 736	case DMA_SLAVE_BUSWIDTH_8_BYTES:
 737		return buswidth;
 738
 739	default:
 740		return -EINVAL;
 741	}
 742}
 743
 744static int sprd_dma_fill_desc(struct dma_chan *chan,
 745			      struct sprd_dma_chn_hw *hw,
 746			      unsigned int sglen, int sg_index,
 747			      dma_addr_t src, dma_addr_t dst, u32 len,
 748			      enum dma_transfer_direction dir,
 749			      unsigned long flags,
 750			      struct dma_slave_config *slave_cfg)
 751{
 752	struct sprd_dma_dev *sdev = to_sprd_dma_dev(chan);
 753	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 754	enum sprd_dma_chn_mode chn_mode = schan->chn_mode;
 755	u32 req_mode = (flags >> SPRD_DMA_REQ_SHIFT) & SPRD_DMA_REQ_MODE_MASK;
 756	u32 int_mode = flags & SPRD_DMA_INT_MASK;
 757	int src_datawidth, dst_datawidth, src_step, dst_step;
 758	u32 temp, fix_mode = 0, fix_en = 0;
 759	phys_addr_t llist_ptr;
 760
 761	if (dir == DMA_MEM_TO_DEV) {
 762		src_step = sprd_dma_get_step(slave_cfg->src_addr_width);
 763		if (src_step < 0) {
 764			dev_err(sdev->dma_dev.dev, "invalid source step\n");
 765			return src_step;
 766		}
 767
 768		/*
 769		 * For 2-stage transfer, destination channel step can not be 0,
 770		 * since destination device is AON IRAM.
 771		 */
 772		if (chn_mode == SPRD_DMA_DST_CHN0 ||
 773		    chn_mode == SPRD_DMA_DST_CHN1)
 774			dst_step = src_step;
 775		else
 776			dst_step = SPRD_DMA_NONE_STEP;
 777	} else {
 778		dst_step = sprd_dma_get_step(slave_cfg->dst_addr_width);
 779		if (dst_step < 0) {
 780			dev_err(sdev->dma_dev.dev, "invalid destination step\n");
 781			return dst_step;
 782		}
 783		src_step = SPRD_DMA_NONE_STEP;
 784	}
 785
 786	src_datawidth = sprd_dma_get_datawidth(slave_cfg->src_addr_width);
 787	if (src_datawidth < 0) {
 788		dev_err(sdev->dma_dev.dev, "invalid source datawidth\n");
 789		return src_datawidth;
 790	}
 791
 792	dst_datawidth = sprd_dma_get_datawidth(slave_cfg->dst_addr_width);
 793	if (dst_datawidth < 0) {
 794		dev_err(sdev->dma_dev.dev, "invalid destination datawidth\n");
 795		return dst_datawidth;
 796	}
 797
 798	if (slave_cfg->slave_id)
 799		schan->dev_id = slave_cfg->slave_id;
 800
 801	hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
 802
 803	/*
 804	 * wrap_ptr and wrap_to will save the high 4 bits source address and
 805	 * destination address.
 806	 */
 807	hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK;
 808	hw->wrap_to = (dst >> SPRD_DMA_HIGH_ADDR_OFFSET) & SPRD_DMA_HIGH_ADDR_MASK;
 809	hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK;
 810	hw->des_addr = dst & SPRD_DMA_LOW_ADDR_MASK;
 811
 812	/*
 813	 * If the src step and dst step both are 0 or both are not 0, that means
 814	 * we can not enable the fix mode. If one is 0 and another one is not,
 815	 * we can enable the fix mode.
 816	 */
 817	if ((src_step != 0 && dst_step != 0) || (src_step | dst_step) == 0) {
 818		fix_en = 0;
 819	} else {
 820		fix_en = 1;
 821		if (src_step)
 822			fix_mode = 1;
 823		else
 824			fix_mode = 0;
 825	}
 826
 827	hw->intc = int_mode | SPRD_DMA_CFG_ERR_INT_EN;
 828
 829	temp = src_datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET;
 830	temp |= dst_datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET;
 831	temp |= req_mode << SPRD_DMA_REQ_MODE_OFFSET;
 832	temp |= fix_mode << SPRD_DMA_FIX_SEL_OFFSET;
 833	temp |= fix_en << SPRD_DMA_FIX_EN_OFFSET;
 834	temp |= schan->linklist.wrap_addr ?
 835		SPRD_DMA_WRAP_EN | SPRD_DMA_WRAP_SEL_DEST : 0;
 836	temp |= slave_cfg->src_maxburst & SPRD_DMA_FRG_LEN_MASK;
 837	hw->frg_len = temp;
 838
 839	hw->blk_len = slave_cfg->src_maxburst & SPRD_DMA_BLK_LEN_MASK;
 840	hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK;
 841
 842	temp = (dst_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET;
 843	temp |= (src_step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET;
 844	hw->trsf_step = temp;
 845
 846	/* link-list configuration */
 847	if (schan->linklist.phy_addr) {
 848		hw->cfg |= SPRD_DMA_LINKLIST_EN;
 849
 850		/* link-list index */
 851		temp = sglen ? (sg_index + 1) % sglen : 0;
 852
 853		/* Next link-list configuration's physical address offset */
 854		temp = temp * sizeof(*hw) + SPRD_DMA_CHN_SRC_ADDR;
 855		/*
 856		 * Set the link-list pointer point to next link-list
 857		 * configuration's physical address.
 858		 */
 859		llist_ptr = schan->linklist.phy_addr + temp;
 860		hw->llist_ptr = lower_32_bits(llist_ptr);
 861		hw->src_blk_step = (upper_32_bits(llist_ptr) << SPRD_DMA_LLIST_HIGH_SHIFT) &
 862			SPRD_DMA_LLIST_HIGH_MASK;
 863
 864		if (schan->linklist.wrap_addr) {
 865			hw->wrap_ptr |= schan->linklist.wrap_addr &
 866				SPRD_DMA_WRAP_ADDR_MASK;
 867			hw->wrap_to |= dst & SPRD_DMA_WRAP_ADDR_MASK;
 868		}
 869	} else {
 870		hw->llist_ptr = 0;
 871		hw->src_blk_step = 0;
 872	}
 873
 874	hw->frg_step = 0;
 875	hw->des_blk_step = 0;
 876	return 0;
 877}
 878
 879static int sprd_dma_fill_linklist_desc(struct dma_chan *chan,
 880				       unsigned int sglen, int sg_index,
 881				       dma_addr_t src, dma_addr_t dst, u32 len,
 882				       enum dma_transfer_direction dir,
 883				       unsigned long flags,
 884				       struct dma_slave_config *slave_cfg)
 885{
 886	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 887	struct sprd_dma_chn_hw *hw;
 888
 889	if (!schan->linklist.virt_addr)
 890		return -EINVAL;
 891
 892	hw = (struct sprd_dma_chn_hw *)(schan->linklist.virt_addr +
 893					sg_index * sizeof(*hw));
 894
 895	return sprd_dma_fill_desc(chan, hw, sglen, sg_index, src, dst, len,
 896				  dir, flags, slave_cfg);
 897}
 898
 899static struct dma_async_tx_descriptor *
 900sprd_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
 901			 size_t len, unsigned long flags)
 902{
 903	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 904	struct sprd_dma_desc *sdesc;
 905	struct sprd_dma_chn_hw *hw;
 906	enum sprd_dma_datawidth datawidth;
 907	u32 step, temp;
 908
 909	sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
 910	if (!sdesc)
 911		return NULL;
 912
 913	hw = &sdesc->chn_hw;
 914
 915	hw->cfg = SPRD_DMA_DONOT_WAIT_BDONE << SPRD_DMA_WAIT_BDONE_OFFSET;
 916	hw->intc = SPRD_DMA_TRANS_INT | SPRD_DMA_CFG_ERR_INT_EN;
 917	hw->src_addr = src & SPRD_DMA_LOW_ADDR_MASK;
 918	hw->des_addr = dest & SPRD_DMA_LOW_ADDR_MASK;
 919	hw->wrap_ptr = (src >> SPRD_DMA_HIGH_ADDR_OFFSET) &
 920		SPRD_DMA_HIGH_ADDR_MASK;
 921	hw->wrap_to = (dest >> SPRD_DMA_HIGH_ADDR_OFFSET) &
 922		SPRD_DMA_HIGH_ADDR_MASK;
 923
 924	if (IS_ALIGNED(len, 8)) {
 925		datawidth = SPRD_DMA_DATAWIDTH_8_BYTES;
 926		step = SPRD_DMA_DWORD_STEP;
 927	} else if (IS_ALIGNED(len, 4)) {
 928		datawidth = SPRD_DMA_DATAWIDTH_4_BYTES;
 929		step = SPRD_DMA_WORD_STEP;
 930	} else if (IS_ALIGNED(len, 2)) {
 931		datawidth = SPRD_DMA_DATAWIDTH_2_BYTES;
 932		step = SPRD_DMA_SHORT_STEP;
 933	} else {
 934		datawidth = SPRD_DMA_DATAWIDTH_1_BYTE;
 935		step = SPRD_DMA_BYTE_STEP;
 936	}
 937
 938	temp = datawidth << SPRD_DMA_SRC_DATAWIDTH_OFFSET;
 939	temp |= datawidth << SPRD_DMA_DES_DATAWIDTH_OFFSET;
 940	temp |= SPRD_DMA_TRANS_REQ << SPRD_DMA_REQ_MODE_OFFSET;
 941	temp |= len & SPRD_DMA_FRG_LEN_MASK;
 942	hw->frg_len = temp;
 943
 944	hw->blk_len = len & SPRD_DMA_BLK_LEN_MASK;
 945	hw->trsc_len = len & SPRD_DMA_TRSC_LEN_MASK;
 946
 947	temp = (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_DEST_TRSF_STEP_OFFSET;
 948	temp |= (step & SPRD_DMA_TRSF_STEP_MASK) << SPRD_DMA_SRC_TRSF_STEP_OFFSET;
 949	hw->trsf_step = temp;
 950
 951	return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
 952}
 953
 954static struct dma_async_tx_descriptor *
 955sprd_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
 956		       unsigned int sglen, enum dma_transfer_direction dir,
 957		       unsigned long flags, void *context)
 958{
 959	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
 960	struct dma_slave_config *slave_cfg = &schan->slave_cfg;
 961	dma_addr_t src = 0, dst = 0;
 962	dma_addr_t start_src = 0, start_dst = 0;
 963	struct sprd_dma_desc *sdesc;
 964	struct scatterlist *sg;
 965	u32 len = 0;
 966	int ret, i;
 967
 968	if (!is_slave_direction(dir))
 969		return NULL;
 970
 971	if (context) {
 972		struct sprd_dma_linklist *ll_cfg =
 973			(struct sprd_dma_linklist *)context;
 974
 975		schan->linklist.phy_addr = ll_cfg->phy_addr;
 976		schan->linklist.virt_addr = ll_cfg->virt_addr;
 977		schan->linklist.wrap_addr = ll_cfg->wrap_addr;
 978	} else {
 979		schan->linklist.phy_addr = 0;
 980		schan->linklist.virt_addr = 0;
 981		schan->linklist.wrap_addr = 0;
 982	}
 983
 984	/*
 985	 * Set channel mode, interrupt mode and trigger mode for 2-stage
 986	 * transfer.
 987	 */
 988	schan->chn_mode =
 989		(flags >> SPRD_DMA_CHN_MODE_SHIFT) & SPRD_DMA_CHN_MODE_MASK;
 990	schan->trg_mode =
 991		(flags >> SPRD_DMA_TRG_MODE_SHIFT) & SPRD_DMA_TRG_MODE_MASK;
 992	schan->int_type = flags & SPRD_DMA_INT_TYPE_MASK;
 993
 994	sdesc = kzalloc(sizeof(*sdesc), GFP_NOWAIT);
 995	if (!sdesc)
 996		return NULL;
 997
 998	sdesc->dir = dir;
 999
1000	for_each_sg(sgl, sg, sglen, i) {
1001		len = sg_dma_len(sg);
1002
1003		if (dir == DMA_MEM_TO_DEV) {
1004			src = sg_dma_address(sg);
1005			dst = slave_cfg->dst_addr;
1006		} else {
1007			src = slave_cfg->src_addr;
1008			dst = sg_dma_address(sg);
1009		}
1010
1011		if (!i) {
1012			start_src = src;
1013			start_dst = dst;
1014		}
1015
1016		/*
1017		 * The link-list mode needs at least 2 link-list
1018		 * configurations. If there is only one sg, it doesn't
1019		 * need to fill the link-list configuration.
1020		 */
1021		if (sglen < 2)
1022			break;
1023
1024		ret = sprd_dma_fill_linklist_desc(chan, sglen, i, src, dst, len,
1025						  dir, flags, slave_cfg);
1026		if (ret) {
1027			kfree(sdesc);
1028			return NULL;
1029		}
1030	}
1031
1032	ret = sprd_dma_fill_desc(chan, &sdesc->chn_hw, 0, 0, start_src,
1033				 start_dst, len, dir, flags, slave_cfg);
1034	if (ret) {
1035		kfree(sdesc);
1036		return NULL;
1037	}
1038
1039	return vchan_tx_prep(&schan->vc, &sdesc->vd, flags);
1040}
1041
1042static int sprd_dma_slave_config(struct dma_chan *chan,
1043				 struct dma_slave_config *config)
1044{
1045	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1046	struct dma_slave_config *slave_cfg = &schan->slave_cfg;
1047
1048	memcpy(slave_cfg, config, sizeof(*config));
1049	return 0;
1050}
1051
1052static int sprd_dma_pause(struct dma_chan *chan)
1053{
1054	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1055	unsigned long flags;
1056
1057	spin_lock_irqsave(&schan->vc.lock, flags);
1058	sprd_dma_pause_resume(schan, true);
1059	spin_unlock_irqrestore(&schan->vc.lock, flags);
1060
1061	return 0;
1062}
1063
1064static int sprd_dma_resume(struct dma_chan *chan)
1065{
1066	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1067	unsigned long flags;
1068
1069	spin_lock_irqsave(&schan->vc.lock, flags);
1070	sprd_dma_pause_resume(schan, false);
1071	spin_unlock_irqrestore(&schan->vc.lock, flags);
1072
1073	return 0;
1074}
1075
1076static int sprd_dma_terminate_all(struct dma_chan *chan)
1077{
1078	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1079	struct virt_dma_desc *cur_vd = NULL;
1080	unsigned long flags;
1081	LIST_HEAD(head);
1082
1083	spin_lock_irqsave(&schan->vc.lock, flags);
1084	if (schan->cur_desc)
1085		cur_vd = &schan->cur_desc->vd;
1086
1087	sprd_dma_stop(schan);
1088
1089	vchan_get_all_descriptors(&schan->vc, &head);
1090	spin_unlock_irqrestore(&schan->vc.lock, flags);
1091
1092	if (cur_vd)
1093		sprd_dma_free_desc(cur_vd);
1094
1095	vchan_dma_desc_free_list(&schan->vc, &head);
1096	return 0;
1097}
1098
1099static void sprd_dma_free_desc(struct virt_dma_desc *vd)
1100{
1101	struct sprd_dma_desc *sdesc = to_sprd_dma_desc(vd);
1102
1103	kfree(sdesc);
1104}
1105
1106static bool sprd_dma_filter_fn(struct dma_chan *chan, void *param)
1107{
1108	struct sprd_dma_chn *schan = to_sprd_dma_chan(chan);
1109	u32 slave_id = *(u32 *)param;
1110
1111	schan->dev_id = slave_id;
1112	return true;
1113}
1114
1115static int sprd_dma_probe(struct platform_device *pdev)
1116{
1117	struct device_node *np = pdev->dev.of_node;
1118	struct sprd_dma_dev *sdev;
1119	struct sprd_dma_chn *dma_chn;
1120	u32 chn_count;
1121	int ret, i;
1122
1123	ret = device_property_read_u32(&pdev->dev, "#dma-channels", &chn_count);
1124	if (ret) {
1125		dev_err(&pdev->dev, "get dma channels count failed\n");
1126		return ret;
1127	}
1128
1129	sdev = devm_kzalloc(&pdev->dev,
1130			    struct_size(sdev, channels, chn_count),
1131			    GFP_KERNEL);
1132	if (!sdev)
1133		return -ENOMEM;
1134
1135	sdev->clk = devm_clk_get(&pdev->dev, "enable");
1136	if (IS_ERR(sdev->clk)) {
1137		dev_err(&pdev->dev, "get enable clock failed\n");
1138		return PTR_ERR(sdev->clk);
1139	}
1140
1141	/* ashb clock is optional for AGCP DMA */
1142	sdev->ashb_clk = devm_clk_get(&pdev->dev, "ashb_eb");
1143	if (IS_ERR(sdev->ashb_clk))
1144		dev_warn(&pdev->dev, "no optional ashb eb clock\n");
1145
1146	/*
1147	 * We have three DMA controllers: AP DMA, AON DMA and AGCP DMA. For AGCP
1148	 * DMA controller, it can or do not request the irq, which will save
1149	 * system power without resuming system by DMA interrupts if AGCP DMA
1150	 * does not request the irq. Thus the DMA interrupts property should
1151	 * be optional.
1152	 */
1153	sdev->irq = platform_get_irq(pdev, 0);
1154	if (sdev->irq > 0) {
1155		ret = devm_request_irq(&pdev->dev, sdev->irq, dma_irq_handle,
1156				       0, "sprd_dma", (void *)sdev);
1157		if (ret < 0) {
1158			dev_err(&pdev->dev, "request dma irq failed\n");
1159			return ret;
1160		}
1161	} else {
1162		dev_warn(&pdev->dev, "no interrupts for the dma controller\n");
1163	}
1164
1165	sdev->glb_base = devm_platform_ioremap_resource(pdev, 0);
1166	if (IS_ERR(sdev->glb_base))
1167		return PTR_ERR(sdev->glb_base);
1168
1169	dma_cap_set(DMA_MEMCPY, sdev->dma_dev.cap_mask);
1170	sdev->total_chns = chn_count;
1171	sdev->dma_dev.chancnt = chn_count;
1172	INIT_LIST_HEAD(&sdev->dma_dev.channels);
1173	INIT_LIST_HEAD(&sdev->dma_dev.global_node);
1174	sdev->dma_dev.dev = &pdev->dev;
1175	sdev->dma_dev.device_alloc_chan_resources = sprd_dma_alloc_chan_resources;
1176	sdev->dma_dev.device_free_chan_resources = sprd_dma_free_chan_resources;
1177	sdev->dma_dev.device_tx_status = sprd_dma_tx_status;
1178	sdev->dma_dev.device_issue_pending = sprd_dma_issue_pending;
1179	sdev->dma_dev.device_prep_dma_memcpy = sprd_dma_prep_dma_memcpy;
1180	sdev->dma_dev.device_prep_slave_sg = sprd_dma_prep_slave_sg;
1181	sdev->dma_dev.device_config = sprd_dma_slave_config;
1182	sdev->dma_dev.device_pause = sprd_dma_pause;
1183	sdev->dma_dev.device_resume = sprd_dma_resume;
1184	sdev->dma_dev.device_terminate_all = sprd_dma_terminate_all;
1185
1186	for (i = 0; i < chn_count; i++) {
1187		dma_chn = &sdev->channels[i];
1188		dma_chn->chn_num = i;
1189		dma_chn->cur_desc = NULL;
1190		/* get each channel's registers base address. */
1191		dma_chn->chn_base = sdev->glb_base + SPRD_DMA_CHN_REG_OFFSET +
1192				    SPRD_DMA_CHN_REG_LENGTH * i;
1193
1194		dma_chn->vc.desc_free = sprd_dma_free_desc;
1195		vchan_init(&dma_chn->vc, &sdev->dma_dev);
1196	}
1197
1198	platform_set_drvdata(pdev, sdev);
1199	ret = sprd_dma_enable(sdev);
1200	if (ret)
1201		return ret;
1202
1203	pm_runtime_set_active(&pdev->dev);
1204	pm_runtime_enable(&pdev->dev);
1205
1206	ret = pm_runtime_get_sync(&pdev->dev);
1207	if (ret < 0)
1208		goto err_rpm;
1209
1210	ret = dma_async_device_register(&sdev->dma_dev);
1211	if (ret < 0) {
1212		dev_err(&pdev->dev, "register dma device failed:%d\n", ret);
1213		goto err_register;
1214	}
1215
1216	sprd_dma_info.dma_cap = sdev->dma_dev.cap_mask;
1217	ret = of_dma_controller_register(np, of_dma_simple_xlate,
1218					 &sprd_dma_info);
1219	if (ret)
1220		goto err_of_register;
1221
1222	pm_runtime_put(&pdev->dev);
1223	return 0;
1224
1225err_of_register:
1226	dma_async_device_unregister(&sdev->dma_dev);
1227err_register:
1228	pm_runtime_put_noidle(&pdev->dev);
1229	pm_runtime_disable(&pdev->dev);
1230err_rpm:
1231	sprd_dma_disable(sdev);
1232	return ret;
1233}
1234
1235static int sprd_dma_remove(struct platform_device *pdev)
1236{
1237	struct sprd_dma_dev *sdev = platform_get_drvdata(pdev);
1238	struct sprd_dma_chn *c, *cn;
1239	int ret;
1240
1241	ret = pm_runtime_get_sync(&pdev->dev);
1242	if (ret < 0)
1243		return ret;
1244
1245	/* explicitly free the irq */
1246	if (sdev->irq > 0)
1247		devm_free_irq(&pdev->dev, sdev->irq, sdev);
1248
1249	list_for_each_entry_safe(c, cn, &sdev->dma_dev.channels,
1250				 vc.chan.device_node) {
1251		list_del(&c->vc.chan.device_node);
1252		tasklet_kill(&c->vc.task);
1253	}
1254
1255	of_dma_controller_free(pdev->dev.of_node);
1256	dma_async_device_unregister(&sdev->dma_dev);
1257	sprd_dma_disable(sdev);
1258
1259	pm_runtime_put_noidle(&pdev->dev);
1260	pm_runtime_disable(&pdev->dev);
1261	return 0;
1262}
1263
1264static const struct of_device_id sprd_dma_match[] = {
1265	{ .compatible = "sprd,sc9860-dma", },
1266	{},
1267};
 
1268
1269static int __maybe_unused sprd_dma_runtime_suspend(struct device *dev)
1270{
1271	struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
1272
1273	sprd_dma_disable(sdev);
1274	return 0;
1275}
1276
1277static int __maybe_unused sprd_dma_runtime_resume(struct device *dev)
1278{
1279	struct sprd_dma_dev *sdev = dev_get_drvdata(dev);
1280	int ret;
1281
1282	ret = sprd_dma_enable(sdev);
1283	if (ret)
1284		dev_err(sdev->dma_dev.dev, "enable dma failed\n");
1285
1286	return ret;
1287}
1288
1289static const struct dev_pm_ops sprd_dma_pm_ops = {
1290	SET_RUNTIME_PM_OPS(sprd_dma_runtime_suspend,
1291			   sprd_dma_runtime_resume,
1292			   NULL)
1293};
1294
1295static struct platform_driver sprd_dma_driver = {
1296	.probe = sprd_dma_probe,
1297	.remove = sprd_dma_remove,
1298	.driver = {
1299		.name = "sprd-dma",
1300		.of_match_table = sprd_dma_match,
1301		.pm = &sprd_dma_pm_ops,
1302	},
1303};
1304module_platform_driver(sprd_dma_driver);
1305
1306MODULE_LICENSE("GPL v2");
1307MODULE_DESCRIPTION("DMA driver for Spreadtrum");
1308MODULE_AUTHOR("Baolin Wang <baolin.wang@spreadtrum.com>");
1309MODULE_AUTHOR("Eric Long <eric.long@spreadtrum.com>");
1310MODULE_ALIAS("platform:sprd-dma");