1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * SA11x0 DMAengine support
   4 *
   5 * Copyright (C) 2012 Russell King
   6 *   Derived in part from arch/arm/mach-sa1100/dma.c,
   7 *   Copyright (C) 2000, 2001 by Nicolas Pitre
   8 */
   9#include <linux/sched.h>
  10#include <linux/device.h>
  11#include <linux/dmaengine.h>
  12#include <linux/init.h>
  13#include <linux/interrupt.h>
  14#include <linux/kernel.h>
  15#include <linux/module.h>
  16#include <linux/platform_device.h>
  17#include <linux/slab.h>
  18#include <linux/spinlock.h>
  19
  20#include "virt-dma.h"
  21
  22#define NR_PHY_CHAN	6
  23#define DMA_ALIGN	3
  24#define DMA_MAX_SIZE	0x1fff
  25#define DMA_CHUNK_SIZE	0x1000
  26
  27#define DMA_DDAR	0x00
  28#define DMA_DCSR_S	0x04
  29#define DMA_DCSR_C	0x08
  30#define DMA_DCSR_R	0x0c
  31#define DMA_DBSA	0x10
  32#define DMA_DBTA	0x14
  33#define DMA_DBSB	0x18
  34#define DMA_DBTB	0x1c
  35#define DMA_SIZE	0x20
  36
  37#define DCSR_RUN	(1 << 0)
  38#define DCSR_IE		(1 << 1)
  39#define DCSR_ERROR	(1 << 2)
  40#define DCSR_DONEA	(1 << 3)
  41#define DCSR_STRTA	(1 << 4)
  42#define DCSR_DONEB	(1 << 5)
  43#define DCSR_STRTB	(1 << 6)
  44#define DCSR_BIU	(1 << 7)
  45
  46#define DDAR_RW		(1 << 0)	/* 0 = W, 1 = R */
  47#define DDAR_E		(1 << 1)	/* 0 = LE, 1 = BE */
  48#define DDAR_BS		(1 << 2)	/* 0 = BS4, 1 = BS8 */
  49#define DDAR_DW		(1 << 3)	/* 0 = 8b, 1 = 16b */
  50#define DDAR_Ser0UDCTr	(0x0 << 4)
  51#define DDAR_Ser0UDCRc	(0x1 << 4)
  52#define DDAR_Ser1SDLCTr	(0x2 << 4)
  53#define DDAR_Ser1SDLCRc	(0x3 << 4)
  54#define DDAR_Ser1UARTTr	(0x4 << 4)
  55#define DDAR_Ser1UARTRc	(0x5 << 4)
  56#define DDAR_Ser2ICPTr	(0x6 << 4)
  57#define DDAR_Ser2ICPRc	(0x7 << 4)
  58#define DDAR_Ser3UARTTr	(0x8 << 4)
  59#define DDAR_Ser3UARTRc	(0x9 << 4)
  60#define DDAR_Ser4MCP0Tr	(0xa << 4)
  61#define DDAR_Ser4MCP0Rc	(0xb << 4)
  62#define DDAR_Ser4MCP1Tr	(0xc << 4)
  63#define DDAR_Ser4MCP1Rc	(0xd << 4)
  64#define DDAR_Ser4SSPTr	(0xe << 4)
  65#define DDAR_Ser4SSPRc	(0xf << 4)
  66
  67struct sa11x0_dma_sg {
  68	u32			addr;
  69	u32			len;
  70};
  71
  72struct sa11x0_dma_desc {
  73	struct virt_dma_desc	vd;
  74
  75	u32			ddar;
  76	size_t			size;
  77	unsigned		period;
  78	bool			cyclic;
  79
  80	unsigned		sglen;
  81	struct sa11x0_dma_sg	sg[] __counted_by(sglen);
  82};
  83
  84struct sa11x0_dma_phy;
  85
  86struct sa11x0_dma_chan {
  87	struct virt_dma_chan	vc;
  88
  89	/* protected by c->vc.lock */
  90	struct sa11x0_dma_phy	*phy;
  91	enum dma_status		status;
  92
  93	/* protected by d->lock */
  94	struct list_head	node;
  95
  96	u32			ddar;
  97	const char		*name;
  98};
  99
 100struct sa11x0_dma_phy {
 101	void __iomem		*base;
 102	struct sa11x0_dma_dev	*dev;
 103	unsigned		num;
 104
 105	struct sa11x0_dma_chan	*vchan;
 106
 107	/* Protected by c->vc.lock */
 108	unsigned		sg_load;
 109	struct sa11x0_dma_desc	*txd_load;
 110	unsigned		sg_done;
 111	struct sa11x0_dma_desc	*txd_done;
 112	u32			dbs[2];
 113	u32			dbt[2];
 114	u32			dcsr;
 115};
 116
 117struct sa11x0_dma_dev {
 118	struct dma_device	slave;
 119	void __iomem		*base;
 120	spinlock_t		lock;
 121	struct tasklet_struct	task;
 122	struct list_head	chan_pending;
 123	struct sa11x0_dma_phy	phy[NR_PHY_CHAN];
 124};
 125
 126static struct sa11x0_dma_chan *to_sa11x0_dma_chan(struct dma_chan *chan)
 127{
 128	return container_of(chan, struct sa11x0_dma_chan, vc.chan);
 129}
 130
 131static struct sa11x0_dma_dev *to_sa11x0_dma(struct dma_device *dmadev)
 132{
 133	return container_of(dmadev, struct sa11x0_dma_dev, slave);
 134}
 135
 136static struct sa11x0_dma_desc *sa11x0_dma_next_desc(struct sa11x0_dma_chan *c)
 137{
 138	struct virt_dma_desc *vd = vchan_next_desc(&c->vc);
 139
 140	return vd ? container_of(vd, struct sa11x0_dma_desc, vd) : NULL;
 141}
 142
 143static void sa11x0_dma_free_desc(struct virt_dma_desc *vd)
 144{
 145	kfree(container_of(vd, struct sa11x0_dma_desc, vd));
 146}
 147
 148static void sa11x0_dma_start_desc(struct sa11x0_dma_phy *p, struct sa11x0_dma_desc *txd)
 149{
 150	list_del(&txd->vd.node);
 151	p->txd_load = txd;
 152	p->sg_load = 0;
 153
 154	dev_vdbg(p->dev->slave.dev, "pchan %u: txd %p[%x]: starting: DDAR:%x\n",
 155		p->num, &txd->vd, txd->vd.tx.cookie, txd->ddar);
 156}
 157
 158static void noinline sa11x0_dma_start_sg(struct sa11x0_dma_phy *p,
 159	struct sa11x0_dma_chan *c)
 160{
 161	struct sa11x0_dma_desc *txd = p->txd_load;
 162	struct sa11x0_dma_sg *sg;
 163	void __iomem *base = p->base;
 164	unsigned dbsx, dbtx;
 165	u32 dcsr;
 166
 167	if (!txd)
 168		return;
 169
 170	dcsr = readl_relaxed(base + DMA_DCSR_R);
 171
 172	/* Don't try to load the next transfer if both buffers are started */
 173	if ((dcsr & (DCSR_STRTA | DCSR_STRTB)) == (DCSR_STRTA | DCSR_STRTB))
 174		return;
 175
 176	if (p->sg_load == txd->sglen) {
 177		if (!txd->cyclic) {
 178			struct sa11x0_dma_desc *txn = sa11x0_dma_next_desc(c);
 179
 180			/*
 181			 * We have reached the end of the current descriptor.
 182			 * Peek at the next descriptor, and if compatible with
 183			 * the current, start processing it.
 184			 */
 185			if (txn && txn->ddar == txd->ddar) {
 186				txd = txn;
 187				sa11x0_dma_start_desc(p, txn);
 188			} else {
 189				p->txd_load = NULL;
 190				return;
 191			}
 192		} else {
 193			/* Cyclic: reset back to beginning */
 194			p->sg_load = 0;
 195		}
 196	}
 197
 198	sg = &txd->sg[p->sg_load++];
 199
 200	/* Select buffer to load according to channel status */
 201	if (((dcsr & (DCSR_BIU | DCSR_STRTB)) == (DCSR_BIU | DCSR_STRTB)) ||
 202	    ((dcsr & (DCSR_BIU | DCSR_STRTA)) == 0)) {
 203		dbsx = DMA_DBSA;
 204		dbtx = DMA_DBTA;
 205		dcsr = DCSR_STRTA | DCSR_IE | DCSR_RUN;
 206	} else {
 207		dbsx = DMA_DBSB;
 208		dbtx = DMA_DBTB;
 209		dcsr = DCSR_STRTB | DCSR_IE | DCSR_RUN;
 210	}
 211
 212	writel_relaxed(sg->addr, base + dbsx);
 213	writel_relaxed(sg->len, base + dbtx);
 214	writel(dcsr, base + DMA_DCSR_S);
 215
 216	dev_dbg(p->dev->slave.dev, "pchan %u: load: DCSR:%02x DBS%c:%08x DBT%c:%08x\n",
 217		p->num, dcsr,
 218		'A' + (dbsx == DMA_DBSB), sg->addr,
 219		'A' + (dbtx == DMA_DBTB), sg->len);
 220}
 221
 222static void noinline sa11x0_dma_complete(struct sa11x0_dma_phy *p,
 223	struct sa11x0_dma_chan *c)
 224{
 225	struct sa11x0_dma_desc *txd = p->txd_done;
 226
 227	if (++p->sg_done == txd->sglen) {
 228		if (!txd->cyclic) {
 229			vchan_cookie_complete(&txd->vd);
 230
 231			p->sg_done = 0;
 232			p->txd_done = p->txd_load;
 233
 234			if (!p->txd_done)
 235				tasklet_schedule(&p->dev->task);
 236		} else {
 237			if ((p->sg_done % txd->period) == 0)
 238				vchan_cyclic_callback(&txd->vd);
 239
 240			/* Cyclic: reset back to beginning */
 241			p->sg_done = 0;
 242		}
 243	}
 244
 245	sa11x0_dma_start_sg(p, c);
 246}
 247
 248static irqreturn_t sa11x0_dma_irq(int irq, void *dev_id)
 249{
 250	struct sa11x0_dma_phy *p = dev_id;
 251	struct sa11x0_dma_dev *d = p->dev;
 252	struct sa11x0_dma_chan *c;
 253	u32 dcsr;
 254
 255	dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 256	if (!(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB)))
 257		return IRQ_NONE;
 258
 259	/* Clear reported status bits */
 260	writel_relaxed(dcsr & (DCSR_ERROR | DCSR_DONEA | DCSR_DONEB),
 261		p->base + DMA_DCSR_C);
 262
 263	dev_dbg(d->slave.dev, "pchan %u: irq: DCSR:%02x\n", p->num, dcsr);
 264
 265	if (dcsr & DCSR_ERROR) {
 266		dev_err(d->slave.dev, "pchan %u: error. DCSR:%02x DDAR:%08x DBSA:%08x DBTA:%08x DBSB:%08x DBTB:%08x\n",
 267			p->num, dcsr,
 268			readl_relaxed(p->base + DMA_DDAR),
 269			readl_relaxed(p->base + DMA_DBSA),
 270			readl_relaxed(p->base + DMA_DBTA),
 271			readl_relaxed(p->base + DMA_DBSB),
 272			readl_relaxed(p->base + DMA_DBTB));
 273	}
 274
 275	c = p->vchan;
 276	if (c) {
 277		unsigned long flags;
 278
 279		spin_lock_irqsave(&c->vc.lock, flags);
 280		/*
 281		 * Now that we're holding the lock, check that the vchan
 282		 * really is associated with this pchan before touching the
 283		 * hardware.  This should always succeed, because we won't
 284		 * change p->vchan or c->phy while the channel is actively
 285		 * transferring.
 286		 */
 287		if (c->phy == p) {
 288			if (dcsr & DCSR_DONEA)
 289				sa11x0_dma_complete(p, c);
 290			if (dcsr & DCSR_DONEB)
 291				sa11x0_dma_complete(p, c);
 292		}
 293		spin_unlock_irqrestore(&c->vc.lock, flags);
 294	}
 295
 296	return IRQ_HANDLED;
 297}
 298
 299static void sa11x0_dma_start_txd(struct sa11x0_dma_chan *c)
 300{
 301	struct sa11x0_dma_desc *txd = sa11x0_dma_next_desc(c);
 302
 303	/* If the issued list is empty, we have no further txds to process */
 304	if (txd) {
 305		struct sa11x0_dma_phy *p = c->phy;
 306
 307		sa11x0_dma_start_desc(p, txd);
 308		p->txd_done = txd;
 309		p->sg_done = 0;
 310
 311		/* The channel should not have any transfers started */
 312		WARN_ON(readl_relaxed(p->base + DMA_DCSR_R) &
 313				      (DCSR_STRTA | DCSR_STRTB));
 314
 315		/* Clear the run and start bits before changing DDAR */
 316		writel_relaxed(DCSR_RUN | DCSR_STRTA | DCSR_STRTB,
 317			       p->base + DMA_DCSR_C);
 318		writel_relaxed(txd->ddar, p->base + DMA_DDAR);
 319
 320		/* Try to start both buffers */
 321		sa11x0_dma_start_sg(p, c);
 322		sa11x0_dma_start_sg(p, c);
 323	}
 324}
 325
 326static void sa11x0_dma_tasklet(struct tasklet_struct *t)
 327{
 328	struct sa11x0_dma_dev *d = from_tasklet(d, t, task);
 329	struct sa11x0_dma_phy *p;
 330	struct sa11x0_dma_chan *c;
 331	unsigned pch, pch_alloc = 0;
 332
 333	dev_dbg(d->slave.dev, "tasklet enter\n");
 334
 335	list_for_each_entry(c, &d->slave.channels, vc.chan.device_node) {
 336		spin_lock_irq(&c->vc.lock);
 337		p = c->phy;
 338		if (p && !p->txd_done) {
 339			sa11x0_dma_start_txd(c);
 340			if (!p->txd_done) {
 341				/* No current txd associated with this channel */
 342				dev_dbg(d->slave.dev, "pchan %u: free\n", p->num);
 343
 344				/* Mark this channel free */
 345				c->phy = NULL;
 346				p->vchan = NULL;
 347			}
 348		}
 349		spin_unlock_irq(&c->vc.lock);
 350	}
 351
 352	spin_lock_irq(&d->lock);
 353	for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 354		p = &d->phy[pch];
 355
 356		if (p->vchan == NULL && !list_empty(&d->chan_pending)) {
 357			c = list_first_entry(&d->chan_pending,
 358				struct sa11x0_dma_chan, node);
 359			list_del_init(&c->node);
 360
 361			pch_alloc |= 1 << pch;
 362
 363			/* Mark this channel allocated */
 364			p->vchan = c;
 365
 366			dev_dbg(d->slave.dev, "pchan %u: alloc vchan %p\n", pch, &c->vc);
 367		}
 368	}
 369	spin_unlock_irq(&d->lock);
 370
 371	for (pch = 0; pch < NR_PHY_CHAN; pch++) {
 372		if (pch_alloc & (1 << pch)) {
 373			p = &d->phy[pch];
 374			c = p->vchan;
 375
 376			spin_lock_irq(&c->vc.lock);
 377			c->phy = p;
 378
 379			sa11x0_dma_start_txd(c);
 380			spin_unlock_irq(&c->vc.lock);
 381		}
 382	}
 383
 384	dev_dbg(d->slave.dev, "tasklet exit\n");
 385}
 386
 387
 388static void sa11x0_dma_free_chan_resources(struct dma_chan *chan)
 389{
 390	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 391	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 392	unsigned long flags;
 393
 394	spin_lock_irqsave(&d->lock, flags);
 395	list_del_init(&c->node);
 396	spin_unlock_irqrestore(&d->lock, flags);
 397
 398	vchan_free_chan_resources(&c->vc);
 399}
 400
 401static dma_addr_t sa11x0_dma_pos(struct sa11x0_dma_phy *p)
 402{
 403	unsigned reg;
 404	u32 dcsr;
 405
 406	dcsr = readl_relaxed(p->base + DMA_DCSR_R);
 407
 408	if ((dcsr & (DCSR_BIU | DCSR_STRTA)) == DCSR_STRTA ||
 409	    (dcsr & (DCSR_BIU | DCSR_STRTB)) == DCSR_BIU)
 410		reg = DMA_DBSA;
 411	else
 412		reg = DMA_DBSB;
 413
 414	return readl_relaxed(p->base + reg);
 415}
 416
 417static enum dma_status sa11x0_dma_tx_status(struct dma_chan *chan,
 418	dma_cookie_t cookie, struct dma_tx_state *state)
 419{
 420	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 421	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 422	struct sa11x0_dma_phy *p;
 423	struct virt_dma_desc *vd;
 424	unsigned long flags;
 425	enum dma_status ret;
 426
 427	ret = dma_cookie_status(&c->vc.chan, cookie, state);
 428	if (ret == DMA_COMPLETE)
 429		return ret;
 430
 431	if (!state)
 432		return c->status;
 433
 434	spin_lock_irqsave(&c->vc.lock, flags);
 435	p = c->phy;
 436
 437	/*
 438	 * If the cookie is on our issue queue, then the residue is
 439	 * its total size.
 440	 */
 441	vd = vchan_find_desc(&c->vc, cookie);
 442	if (vd) {
 443		state->residue = container_of(vd, struct sa11x0_dma_desc, vd)->size;
 444	} else if (!p) {
 445		state->residue = 0;
 446	} else {
 447		struct sa11x0_dma_desc *txd;
 448		size_t bytes = 0;
 449
 450		if (p->txd_done && p->txd_done->vd.tx.cookie == cookie)
 451			txd = p->txd_done;
 452		else if (p->txd_load && p->txd_load->vd.tx.cookie == cookie)
 453			txd = p->txd_load;
 454		else
 455			txd = NULL;
 456
 457		ret = c->status;
 458		if (txd) {
 459			dma_addr_t addr = sa11x0_dma_pos(p);
 460			unsigned i;
 461
 462			dev_vdbg(d->slave.dev, "tx_status: addr:%pad\n", &addr);
 463
 464			for (i = 0; i < txd->sglen; i++) {
 465				dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x\n",
 466					i, txd->sg[i].addr, txd->sg[i].len);
 467				if (addr >= txd->sg[i].addr &&
 468				    addr < txd->sg[i].addr + txd->sg[i].len) {
 469					unsigned len;
 470
 471					len = txd->sg[i].len -
 472						(addr - txd->sg[i].addr);
 473					dev_vdbg(d->slave.dev, "tx_status: [%u] +%x\n",
 474						i, len);
 475					bytes += len;
 476					i++;
 477					break;
 478				}
 479			}
 480			for (; i < txd->sglen; i++) {
 481				dev_vdbg(d->slave.dev, "tx_status: [%u] %x+%x ++\n",
 482					i, txd->sg[i].addr, txd->sg[i].len);
 483				bytes += txd->sg[i].len;
 484			}
 485		}
 486		state->residue = bytes;
 487	}
 488	spin_unlock_irqrestore(&c->vc.lock, flags);
 489
 490	dev_vdbg(d->slave.dev, "tx_status: bytes 0x%x\n", state->residue);
 491
 492	return ret;
 493}
 494
 495/*
 496 * Move pending txds to the issued list, and re-init pending list.
 497 * If not already pending, add this channel to the list of pending
 498 * channels and trigger the tasklet to run.
 499 */
 500static void sa11x0_dma_issue_pending(struct dma_chan *chan)
 501{
 502	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 503	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 504	unsigned long flags;
 505
 506	spin_lock_irqsave(&c->vc.lock, flags);
 507	if (vchan_issue_pending(&c->vc)) {
 508		if (!c->phy) {
 509			spin_lock(&d->lock);
 510			if (list_empty(&c->node)) {
 511				list_add_tail(&c->node, &d->chan_pending);
 512				tasklet_schedule(&d->task);
 513				dev_dbg(d->slave.dev, "vchan %p: issued\n", &c->vc);
 514			}
 515			spin_unlock(&d->lock);
 516		}
 517	} else
 518		dev_dbg(d->slave.dev, "vchan %p: nothing to issue\n", &c->vc);
 519	spin_unlock_irqrestore(&c->vc.lock, flags);
 520}
 521
 522static struct dma_async_tx_descriptor *sa11x0_dma_prep_slave_sg(
 523	struct dma_chan *chan, struct scatterlist *sg, unsigned int sglen,
 524	enum dma_transfer_direction dir, unsigned long flags, void *context)
 525{
 526	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 527	struct sa11x0_dma_desc *txd;
 528	struct scatterlist *sgent;
 529	unsigned i, j = sglen;
 530	size_t size = 0;
 531
 532	/* SA11x0 channels can only operate in their native direction */
 533	if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 534		dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 535			&c->vc, c->ddar, dir);
 536		return NULL;
 537	}
 538
 539	/* Do not allow zero-sized txds */
 540	if (sglen == 0)
 541		return NULL;
 542
 543	for_each_sg(sg, sgent, sglen, i) {
 544		dma_addr_t addr = sg_dma_address(sgent);
 545		unsigned int len = sg_dma_len(sgent);
 546
 547		if (len > DMA_MAX_SIZE)
 548			j += DIV_ROUND_UP(len, DMA_MAX_SIZE & ~DMA_ALIGN) - 1;
 549		if (addr & DMA_ALIGN) {
 550			dev_dbg(chan->device->dev, "vchan %p: bad buffer alignment: %pad\n",
 551				&c->vc, &addr);
 552			return NULL;
 553		}
 554	}
 555
 556	txd = kzalloc(struct_size(txd, sg, j), GFP_ATOMIC);
 557	if (!txd) {
 558		dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 559		return NULL;
 560	}
 561	txd->sglen = j;
 562
 563	j = 0;
 564	for_each_sg(sg, sgent, sglen, i) {
 565		dma_addr_t addr = sg_dma_address(sgent);
 566		unsigned len = sg_dma_len(sgent);
 567
 568		size += len;
 569
 570		do {
 571			unsigned tlen = len;
 572
 573			/*
 574			 * Check whether the transfer will fit.  If not, try
 575			 * to split the transfer up such that we end up with
 576			 * equal chunks - but make sure that we preserve the
 577			 * alignment.  This avoids small segments.
 578			 */
 579			if (tlen > DMA_MAX_SIZE) {
 580				unsigned mult = DIV_ROUND_UP(tlen,
 581					DMA_MAX_SIZE & ~DMA_ALIGN);
 582
 583				tlen = (tlen / mult) & ~DMA_ALIGN;
 584			}
 585
 586			txd->sg[j].addr = addr;
 587			txd->sg[j].len = tlen;
 588
 589			addr += tlen;
 590			len -= tlen;
 591			j++;
 592		} while (len);
 593	}
 594
 595	txd->ddar = c->ddar;
 596	txd->size = size;
 
 597
 598	dev_dbg(chan->device->dev, "vchan %p: txd %p: size %zu nr %u\n",
 599		&c->vc, &txd->vd, txd->size, txd->sglen);
 600
 601	return vchan_tx_prep(&c->vc, &txd->vd, flags);
 602}
 603
 604static struct dma_async_tx_descriptor *sa11x0_dma_prep_dma_cyclic(
 605	struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
 606	enum dma_transfer_direction dir, unsigned long flags)
 607{
 608	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 609	struct sa11x0_dma_desc *txd;
 610	unsigned i, j, k, sglen, sgperiod;
 611
 612	/* SA11x0 channels can only operate in their native direction */
 613	if (dir != (c->ddar & DDAR_RW ? DMA_DEV_TO_MEM : DMA_MEM_TO_DEV)) {
 614		dev_err(chan->device->dev, "vchan %p: bad DMA direction: DDAR:%08x dir:%u\n",
 615			&c->vc, c->ddar, dir);
 616		return NULL;
 617	}
 618
 619	sgperiod = DIV_ROUND_UP(period, DMA_MAX_SIZE & ~DMA_ALIGN);
 620	sglen = size * sgperiod / period;
 621
 622	/* Do not allow zero-sized txds */
 623	if (sglen == 0)
 624		return NULL;
 625
 626	txd = kzalloc(struct_size(txd, sg, sglen), GFP_ATOMIC);
 627	if (!txd) {
 628		dev_dbg(chan->device->dev, "vchan %p: kzalloc failed\n", &c->vc);
 629		return NULL;
 630	}
 631	txd->sglen = sglen;
 632
 633	for (i = k = 0; i < size / period; i++) {
 634		size_t tlen, len = period;
 635
 636		for (j = 0; j < sgperiod; j++, k++) {
 637			tlen = len;
 638
 639			if (tlen > DMA_MAX_SIZE) {
 640				unsigned mult = DIV_ROUND_UP(tlen, DMA_MAX_SIZE & ~DMA_ALIGN);
 641				tlen = (tlen / mult) & ~DMA_ALIGN;
 642			}
 643
 644			txd->sg[k].addr = addr;
 645			txd->sg[k].len = tlen;
 646			addr += tlen;
 647			len -= tlen;
 648		}
 649
 650		WARN_ON(len != 0);
 651	}
 652
 653	WARN_ON(k != sglen);
 654
 655	txd->ddar = c->ddar;
 656	txd->size = size;
 
 657	txd->cyclic = 1;
 658	txd->period = sgperiod;
 659
 660	return vchan_tx_prep(&c->vc, &txd->vd, DMA_PREP_INTERRUPT | DMA_CTRL_ACK);
 661}
 662
 663static int sa11x0_dma_device_config(struct dma_chan *chan,
 664				    struct dma_slave_config *cfg)
 665{
 666	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 667	u32 ddar = c->ddar & ((0xf << 4) | DDAR_RW);
 668	dma_addr_t addr;
 669	enum dma_slave_buswidth width;
 670	u32 maxburst;
 671
 672	if (ddar & DDAR_RW) {
 673		addr = cfg->src_addr;
 674		width = cfg->src_addr_width;
 675		maxburst = cfg->src_maxburst;
 676	} else {
 677		addr = cfg->dst_addr;
 678		width = cfg->dst_addr_width;
 679		maxburst = cfg->dst_maxburst;
 680	}
 681
 682	if ((width != DMA_SLAVE_BUSWIDTH_1_BYTE &&
 683	     width != DMA_SLAVE_BUSWIDTH_2_BYTES) ||
 684	    (maxburst != 4 && maxburst != 8))
 685		return -EINVAL;
 686
 687	if (width == DMA_SLAVE_BUSWIDTH_2_BYTES)
 688		ddar |= DDAR_DW;
 689	if (maxburst == 8)
 690		ddar |= DDAR_BS;
 691
 692	dev_dbg(c->vc.chan.device->dev, "vchan %p: dma_slave_config addr %pad width %u burst %u\n",
 693		&c->vc, &addr, width, maxburst);
 694
 695	c->ddar = ddar | (addr & 0xf0000000) | (addr & 0x003ffffc) << 6;
 696
 697	return 0;
 698}
 699
 700static int sa11x0_dma_device_pause(struct dma_chan *chan)
 701{
 702	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 703	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 704	struct sa11x0_dma_phy *p;
 705	unsigned long flags;
 706
 707	dev_dbg(d->slave.dev, "vchan %p: pause\n", &c->vc);
 708	spin_lock_irqsave(&c->vc.lock, flags);
 709	if (c->status == DMA_IN_PROGRESS) {
 710		c->status = DMA_PAUSED;
 711
 712		p = c->phy;
 713		if (p) {
 714			writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
 715		} else {
 716			spin_lock(&d->lock);
 717			list_del_init(&c->node);
 718			spin_unlock(&d->lock);
 719		}
 720	}
 721	spin_unlock_irqrestore(&c->vc.lock, flags);
 722
 723	return 0;
 724}
 725
 726static int sa11x0_dma_device_resume(struct dma_chan *chan)
 727{
 728	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 729	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 730	struct sa11x0_dma_phy *p;
 731	unsigned long flags;
 732
 733	dev_dbg(d->slave.dev, "vchan %p: resume\n", &c->vc);
 734	spin_lock_irqsave(&c->vc.lock, flags);
 735	if (c->status == DMA_PAUSED) {
 736		c->status = DMA_IN_PROGRESS;
 737
 738		p = c->phy;
 739		if (p) {
 740			writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_S);
 741		} else if (!list_empty(&c->vc.desc_issued)) {
 742			spin_lock(&d->lock);
 743			list_add_tail(&c->node, &d->chan_pending);
 744			spin_unlock(&d->lock);
 745		}
 746	}
 747	spin_unlock_irqrestore(&c->vc.lock, flags);
 748
 749	return 0;
 750}
 751
 752static int sa11x0_dma_device_terminate_all(struct dma_chan *chan)
 753{
 754	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 755	struct sa11x0_dma_dev *d = to_sa11x0_dma(chan->device);
 756	struct sa11x0_dma_phy *p;
 757	LIST_HEAD(head);
 758	unsigned long flags;
 759
 760	dev_dbg(d->slave.dev, "vchan %p: terminate all\n", &c->vc);
 761	/* Clear the tx descriptor lists */
 762	spin_lock_irqsave(&c->vc.lock, flags);
 763	vchan_get_all_descriptors(&c->vc, &head);
 764
 765	p = c->phy;
 766	if (p) {
 767		dev_dbg(d->slave.dev, "pchan %u: terminating\n", p->num);
 768		/* vchan is assigned to a pchan - stop the channel */
 769		writel(DCSR_RUN | DCSR_IE |
 770		       DCSR_STRTA | DCSR_DONEA |
 771		       DCSR_STRTB | DCSR_DONEB,
 772		       p->base + DMA_DCSR_C);
 773
 774		if (p->txd_load) {
 775			if (p->txd_load != p->txd_done)
 776				list_add_tail(&p->txd_load->vd.node, &head);
 777			p->txd_load = NULL;
 778		}
 779		if (p->txd_done) {
 780			list_add_tail(&p->txd_done->vd.node, &head);
 781			p->txd_done = NULL;
 782		}
 783		c->phy = NULL;
 784		spin_lock(&d->lock);
 785		p->vchan = NULL;
 786		spin_unlock(&d->lock);
 787		tasklet_schedule(&d->task);
 788	}
 789	spin_unlock_irqrestore(&c->vc.lock, flags);
 790	vchan_dma_desc_free_list(&c->vc, &head);
 791
 792	return 0;
 793}
 794
 795struct sa11x0_dma_channel_desc {
 796	u32 ddar;
 797	const char *name;
 798};
 799
 800#define CD(d1, d2) { .ddar = DDAR_##d1 | d2, .name = #d1 }
 801static const struct sa11x0_dma_channel_desc chan_desc[] = {
 802	CD(Ser0UDCTr, 0),
 803	CD(Ser0UDCRc, DDAR_RW),
 804	CD(Ser1SDLCTr, 0),
 805	CD(Ser1SDLCRc, DDAR_RW),
 806	CD(Ser1UARTTr, 0),
 807	CD(Ser1UARTRc, DDAR_RW),
 808	CD(Ser2ICPTr, 0),
 809	CD(Ser2ICPRc, DDAR_RW),
 810	CD(Ser3UARTTr, 0),
 811	CD(Ser3UARTRc, DDAR_RW),
 812	CD(Ser4MCP0Tr, 0),
 813	CD(Ser4MCP0Rc, DDAR_RW),
 814	CD(Ser4MCP1Tr, 0),
 815	CD(Ser4MCP1Rc, DDAR_RW),
 816	CD(Ser4SSPTr, 0),
 817	CD(Ser4SSPRc, DDAR_RW),
 818};
 819
 820static const struct dma_slave_map sa11x0_dma_map[] = {
 821	{ "sa11x0-ir", "tx", "Ser2ICPTr" },
 822	{ "sa11x0-ir", "rx", "Ser2ICPRc" },
 823	{ "sa11x0-ssp", "tx", "Ser4SSPTr" },
 824	{ "sa11x0-ssp", "rx", "Ser4SSPRc" },
 825};
 826
 827static bool sa11x0_dma_filter_fn(struct dma_chan *chan, void *param)
 828{
 829	struct sa11x0_dma_chan *c = to_sa11x0_dma_chan(chan);
 830	const char *p = param;
 831
 832	return !strcmp(c->name, p);
 833}
 834
 835static int sa11x0_dma_init_dmadev(struct dma_device *dmadev,
 836	struct device *dev)
 837{
 838	unsigned i;
 839
 840	INIT_LIST_HEAD(&dmadev->channels);
 841	dmadev->dev = dev;
 842	dmadev->device_free_chan_resources = sa11x0_dma_free_chan_resources;
 843	dmadev->device_config = sa11x0_dma_device_config;
 844	dmadev->device_pause = sa11x0_dma_device_pause;
 845	dmadev->device_resume = sa11x0_dma_device_resume;
 846	dmadev->device_terminate_all = sa11x0_dma_device_terminate_all;
 847	dmadev->device_tx_status = sa11x0_dma_tx_status;
 848	dmadev->device_issue_pending = sa11x0_dma_issue_pending;
 849
 850	for (i = 0; i < ARRAY_SIZE(chan_desc); i++) {
 851		struct sa11x0_dma_chan *c;
 852
 853		c = kzalloc(sizeof(*c), GFP_KERNEL);
 854		if (!c) {
 855			dev_err(dev, "no memory for channel %u\n", i);
 856			return -ENOMEM;
 857		}
 858
 859		c->status = DMA_IN_PROGRESS;
 860		c->ddar = chan_desc[i].ddar;
 861		c->name = chan_desc[i].name;
 862		INIT_LIST_HEAD(&c->node);
 863
 864		c->vc.desc_free = sa11x0_dma_free_desc;
 865		vchan_init(&c->vc, dmadev);
 866	}
 867
 868	return dma_async_device_register(dmadev);
 869}
 870
 871static int sa11x0_dma_request_irq(struct platform_device *pdev, int nr,
 872	void *data)
 873{
 874	int irq = platform_get_irq(pdev, nr);
 875
 876	if (irq <= 0)
 877		return -ENXIO;
 878
 879	return request_irq(irq, sa11x0_dma_irq, 0, dev_name(&pdev->dev), data);
 880}
 881
 882static void sa11x0_dma_free_irq(struct platform_device *pdev, int nr,
 883	void *data)
 884{
 885	int irq = platform_get_irq(pdev, nr);
 886	if (irq > 0)
 887		free_irq(irq, data);
 888}
 889
 890static void sa11x0_dma_free_channels(struct dma_device *dmadev)
 891{
 892	struct sa11x0_dma_chan *c, *cn;
 893
 894	list_for_each_entry_safe(c, cn, &dmadev->channels, vc.chan.device_node) {
 895		list_del(&c->vc.chan.device_node);
 896		tasklet_kill(&c->vc.task);
 897		kfree(c);
 898	}
 899}
 900
 901static int sa11x0_dma_probe(struct platform_device *pdev)
 902{
 903	struct sa11x0_dma_dev *d;
 904	struct resource *res;
 905	unsigned i;
 906	int ret;
 907
 908	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 909	if (!res)
 910		return -ENXIO;
 911
 912	d = kzalloc(sizeof(*d), GFP_KERNEL);
 913	if (!d) {
 914		ret = -ENOMEM;
 915		goto err_alloc;
 916	}
 917
 918	spin_lock_init(&d->lock);
 919	INIT_LIST_HEAD(&d->chan_pending);
 920
 921	d->slave.filter.fn = sa11x0_dma_filter_fn;
 922	d->slave.filter.mapcnt = ARRAY_SIZE(sa11x0_dma_map);
 923	d->slave.filter.map = sa11x0_dma_map;
 924
 925	d->base = ioremap(res->start, resource_size(res));
 926	if (!d->base) {
 927		ret = -ENOMEM;
 928		goto err_ioremap;
 929	}
 930
 931	tasklet_setup(&d->task, sa11x0_dma_tasklet);
 932
 933	for (i = 0; i < NR_PHY_CHAN; i++) {
 934		struct sa11x0_dma_phy *p = &d->phy[i];
 935
 936		p->dev = d;
 937		p->num = i;
 938		p->base = d->base + i * DMA_SIZE;
 939		writel_relaxed(DCSR_RUN | DCSR_IE | DCSR_ERROR |
 940			DCSR_DONEA | DCSR_STRTA | DCSR_DONEB | DCSR_STRTB,
 941			p->base + DMA_DCSR_C);
 942		writel_relaxed(0, p->base + DMA_DDAR);
 943
 944		ret = sa11x0_dma_request_irq(pdev, i, p);
 945		if (ret) {
 946			while (i) {
 947				i--;
 948				sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 949			}
 950			goto err_irq;
 951		}
 952	}
 953
 954	dma_cap_set(DMA_SLAVE, d->slave.cap_mask);
 955	dma_cap_set(DMA_CYCLIC, d->slave.cap_mask);
 956	d->slave.device_prep_slave_sg = sa11x0_dma_prep_slave_sg;
 957	d->slave.device_prep_dma_cyclic = sa11x0_dma_prep_dma_cyclic;
 958	d->slave.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
 959	d->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
 960	d->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
 961				   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
 962	d->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
 963				   BIT(DMA_SLAVE_BUSWIDTH_2_BYTES);
 964	ret = sa11x0_dma_init_dmadev(&d->slave, &pdev->dev);
 965	if (ret) {
 966		dev_warn(d->slave.dev, "failed to register slave async device: %d\n",
 967			ret);
 968		goto err_slave_reg;
 969	}
 970
 971	platform_set_drvdata(pdev, d);
 972	return 0;
 973
 974 err_slave_reg:
 975	sa11x0_dma_free_channels(&d->slave);
 976	for (i = 0; i < NR_PHY_CHAN; i++)
 977		sa11x0_dma_free_irq(pdev, i, &d->phy[i]);
 978 err_irq:
 979	tasklet_kill(&d->task);
 980	iounmap(d->base);
 981 err_ioremap:
 982	kfree(d);
 983 err_alloc:
 984	return ret;
 985}
 986
 987static void sa11x0_dma_remove(struct platform_device *pdev)
 988{
 989	struct sa11x0_dma_dev *d = platform_get_drvdata(pdev);
 990	unsigned pch;
 991
 992	dma_async_device_unregister(&d->slave);
 993
 994	sa11x0_dma_free_channels(&d->slave);
 995	for (pch = 0; pch < NR_PHY_CHAN; pch++)
 996		sa11x0_dma_free_irq(pdev, pch, &d->phy[pch]);
 997	tasklet_kill(&d->task);
 998	iounmap(d->base);
 999	kfree(d);
 
 
1000}
1001
1002static __maybe_unused int sa11x0_dma_suspend(struct device *dev)
1003{
1004	struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1005	unsigned pch;
1006
1007	for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1008		struct sa11x0_dma_phy *p = &d->phy[pch];
1009		u32 dcsr, saved_dcsr;
1010
1011		dcsr = saved_dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1012		if (dcsr & DCSR_RUN) {
1013			writel(DCSR_RUN | DCSR_IE, p->base + DMA_DCSR_C);
1014			dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1015		}
1016
1017		saved_dcsr &= DCSR_RUN | DCSR_IE;
1018		if (dcsr & DCSR_BIU) {
1019			p->dbs[0] = readl_relaxed(p->base + DMA_DBSB);
1020			p->dbt[0] = readl_relaxed(p->base + DMA_DBTB);
1021			p->dbs[1] = readl_relaxed(p->base + DMA_DBSA);
1022			p->dbt[1] = readl_relaxed(p->base + DMA_DBTA);
1023			saved_dcsr |= (dcsr & DCSR_STRTA ? DCSR_STRTB : 0) |
1024				      (dcsr & DCSR_STRTB ? DCSR_STRTA : 0);
1025		} else {
1026			p->dbs[0] = readl_relaxed(p->base + DMA_DBSA);
1027			p->dbt[0] = readl_relaxed(p->base + DMA_DBTA);
1028			p->dbs[1] = readl_relaxed(p->base + DMA_DBSB);
1029			p->dbt[1] = readl_relaxed(p->base + DMA_DBTB);
1030			saved_dcsr |= dcsr & (DCSR_STRTA | DCSR_STRTB);
1031		}
1032		p->dcsr = saved_dcsr;
1033
1034		writel(DCSR_STRTA | DCSR_STRTB, p->base + DMA_DCSR_C);
1035	}
1036
1037	return 0;
1038}
1039
1040static __maybe_unused int sa11x0_dma_resume(struct device *dev)
1041{
1042	struct sa11x0_dma_dev *d = dev_get_drvdata(dev);
1043	unsigned pch;
1044
1045	for (pch = 0; pch < NR_PHY_CHAN; pch++) {
1046		struct sa11x0_dma_phy *p = &d->phy[pch];
1047		struct sa11x0_dma_desc *txd = NULL;
1048		u32 dcsr = readl_relaxed(p->base + DMA_DCSR_R);
1049
1050		WARN_ON(dcsr & (DCSR_BIU | DCSR_STRTA | DCSR_STRTB | DCSR_RUN));
1051
1052		if (p->txd_done)
1053			txd = p->txd_done;
1054		else if (p->txd_load)
1055			txd = p->txd_load;
1056
1057		if (!txd)
1058			continue;
1059
1060		writel_relaxed(txd->ddar, p->base + DMA_DDAR);
1061
1062		writel_relaxed(p->dbs[0], p->base + DMA_DBSA);
1063		writel_relaxed(p->dbt[0], p->base + DMA_DBTA);
1064		writel_relaxed(p->dbs[1], p->base + DMA_DBSB);
1065		writel_relaxed(p->dbt[1], p->base + DMA_DBTB);
1066		writel_relaxed(p->dcsr, p->base + DMA_DCSR_S);
1067	}
1068
1069	return 0;
1070}
1071
1072static const struct dev_pm_ops sa11x0_dma_pm_ops = {
1073	SET_NOIRQ_SYSTEM_SLEEP_PM_OPS(sa11x0_dma_suspend, sa11x0_dma_resume)
 
 
 
 
 
1074};
1075
1076static struct platform_driver sa11x0_dma_driver = {
1077	.driver = {
1078		.name	= "sa11x0-dma",
1079		.pm	= &sa11x0_dma_pm_ops,
1080	},
1081	.probe		= sa11x0_dma_probe,
1082	.remove_new	= sa11x0_dma_remove,
1083};
1084
1085static int __init sa11x0_dma_init(void)
1086{
1087	return platform_driver_register(&sa11x0_dma_driver);
1088}
1089subsys_initcall(sa11x0_dma_init);
1090
1091static void __exit sa11x0_dma_exit(void)
1092{
1093	platform_driver_unregister(&sa11x0_dma_driver);
1094}
1095module_exit(sa11x0_dma_exit);
1096
1097MODULE_AUTHOR("Russell King");
1098MODULE_DESCRIPTION("SA-11x0 DMA driver");
1099MODULE_LICENSE("GPL v2");
1100MODULE_ALIAS("platform:sa11x0-dma");