Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * S3C24XX DMA handling
4 *
5 * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
6 *
7 * based on amba-pl08x.c
8 *
9 * Copyright (c) 2006 ARM Ltd.
10 * Copyright (c) 2010 ST-Ericsson SA
11 *
12 * Author: Peter Pearse <peter.pearse@arm.com>
13 * Author: Linus Walleij <linus.walleij@stericsson.com>
14 *
15 * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals
16 * that can be routed to any of the 4 to 8 hardware-channels.
17 *
18 * Therefore on these DMA controllers the number of channels
19 * and the number of incoming DMA signals are two totally different things.
20 * It is usually not possible to theoretically handle all physical signals,
21 * so a multiplexing scheme with possible denial of use is necessary.
22 *
23 * Open items:
24 * - bursts
25 */
26
27#include <linux/platform_device.h>
28#include <linux/types.h>
29#include <linux/dmaengine.h>
30#include <linux/dma-mapping.h>
31#include <linux/interrupt.h>
32#include <linux/clk.h>
33#include <linux/module.h>
34#include <linux/mod_devicetable.h>
35#include <linux/slab.h>
36#include <linux/platform_data/dma-s3c24xx.h>
37
38#include "dmaengine.h"
39#include "virt-dma.h"
40
41#define MAX_DMA_CHANNELS 8
42
43#define S3C24XX_DISRC 0x00
44#define S3C24XX_DISRCC 0x04
45#define S3C24XX_DISRCC_INC_INCREMENT 0
46#define S3C24XX_DISRCC_INC_FIXED BIT(0)
47#define S3C24XX_DISRCC_LOC_AHB 0
48#define S3C24XX_DISRCC_LOC_APB BIT(1)
49
50#define S3C24XX_DIDST 0x08
51#define S3C24XX_DIDSTC 0x0c
52#define S3C24XX_DIDSTC_INC_INCREMENT 0
53#define S3C24XX_DIDSTC_INC_FIXED BIT(0)
54#define S3C24XX_DIDSTC_LOC_AHB 0
55#define S3C24XX_DIDSTC_LOC_APB BIT(1)
56#define S3C24XX_DIDSTC_INT_TC0 0
57#define S3C24XX_DIDSTC_INT_RELOAD BIT(2)
58
59#define S3C24XX_DCON 0x10
60
61#define S3C24XX_DCON_TC_MASK 0xfffff
62#define S3C24XX_DCON_DSZ_BYTE (0 << 20)
63#define S3C24XX_DCON_DSZ_HALFWORD (1 << 20)
64#define S3C24XX_DCON_DSZ_WORD (2 << 20)
65#define S3C24XX_DCON_DSZ_MASK (3 << 20)
66#define S3C24XX_DCON_DSZ_SHIFT 20
67#define S3C24XX_DCON_AUTORELOAD 0
68#define S3C24XX_DCON_NORELOAD BIT(22)
69#define S3C24XX_DCON_HWTRIG BIT(23)
70#define S3C24XX_DCON_HWSRC_SHIFT 24
71#define S3C24XX_DCON_SERV_SINGLE 0
72#define S3C24XX_DCON_SERV_WHOLE BIT(27)
73#define S3C24XX_DCON_TSZ_UNIT 0
74#define S3C24XX_DCON_TSZ_BURST4 BIT(28)
75#define S3C24XX_DCON_INT BIT(29)
76#define S3C24XX_DCON_SYNC_PCLK 0
77#define S3C24XX_DCON_SYNC_HCLK BIT(30)
78#define S3C24XX_DCON_DEMAND 0
79#define S3C24XX_DCON_HANDSHAKE BIT(31)
80
81#define S3C24XX_DSTAT 0x14
82#define S3C24XX_DSTAT_STAT_BUSY BIT(20)
83#define S3C24XX_DSTAT_CURRTC_MASK 0xfffff
84
85#define S3C24XX_DMASKTRIG 0x20
86#define S3C24XX_DMASKTRIG_SWTRIG BIT(0)
87#define S3C24XX_DMASKTRIG_ON BIT(1)
88#define S3C24XX_DMASKTRIG_STOP BIT(2)
89
90#define S3C24XX_DMAREQSEL 0x24
91#define S3C24XX_DMAREQSEL_HW BIT(0)
92
93/*
94 * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel
95 * for a DMA source. Instead only specific channels are valid.
96 * All of these SoCs have 4 physical channels and the number of request
97 * source bits is 3. Additionally we also need 1 bit to mark the channel
98 * as valid.
99 * Therefore we separate the chansel element of the channel data into 4
100 * parts of 4 bits each, to hold the information if the channel is valid
101 * and the hw request source to use.
102 *
103 * Example:
104 * SDI is valid on channels 0, 2 and 3 - with varying hw request sources.
105 * For it the chansel field would look like
106 *
107 * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1
108 * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2
109 * ((BIT(3) | 2) << 0 * 4) // channel 0, with request source 2
110 */
111#define S3C24XX_CHANSEL_WIDTH 4
112#define S3C24XX_CHANSEL_VALID BIT(3)
113#define S3C24XX_CHANSEL_REQ_MASK 7
114
115/*
116 * struct soc_data - vendor-specific config parameters for individual SoCs
117 * @stride: spacing between the registers of each channel
118 * @has_reqsel: does the controller use the newer requestselection mechanism
119 * @has_clocks: are controllable dma-clocks present
120 */
121struct soc_data {
122 int stride;
123 bool has_reqsel;
124 bool has_clocks;
125};
126
127/*
128 * enum s3c24xx_dma_chan_state - holds the virtual channel states
129 * @S3C24XX_DMA_CHAN_IDLE: the channel is idle
130 * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport
131 * channel and is running a transfer on it
132 * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport
133 * channel to become available (only pertains to memcpy channels)
134 */
135enum s3c24xx_dma_chan_state {
136 S3C24XX_DMA_CHAN_IDLE,
137 S3C24XX_DMA_CHAN_RUNNING,
138 S3C24XX_DMA_CHAN_WAITING,
139};
140
141/*
142 * struct s3c24xx_sg - structure containing data per sg
143 * @src_addr: src address of sg
144 * @dst_addr: dst address of sg
145 * @len: transfer len in bytes
146 * @node: node for txd's dsg_list
147 */
148struct s3c24xx_sg {
149 dma_addr_t src_addr;
150 dma_addr_t dst_addr;
151 size_t len;
152 struct list_head node;
153};
154
155/*
156 * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor
157 * @vd: virtual DMA descriptor
158 * @dsg_list: list of children sg's
159 * @at: sg currently being transfered
160 * @width: transfer width
161 * @disrcc: value for source control register
162 * @didstc: value for destination control register
163 * @dcon: base value for dcon register
164 * @cyclic: indicate cyclic transfer
165 */
166struct s3c24xx_txd {
167 struct virt_dma_desc vd;
168 struct list_head dsg_list;
169 struct list_head *at;
170 u8 width;
171 u32 disrcc;
172 u32 didstc;
173 u32 dcon;
174 bool cyclic;
175};
176
177struct s3c24xx_dma_chan;
178
179/*
180 * struct s3c24xx_dma_phy - holder for the physical channels
181 * @id: physical index to this channel
182 * @valid: does the channel have all required elements
183 * @base: virtual memory base (remapped) for the this channel
184 * @irq: interrupt for this channel
185 * @clk: clock for this channel
186 * @lock: a lock to use when altering an instance of this struct
187 * @serving: virtual channel currently being served by this physicalchannel
188 * @host: a pointer to the host (internal use)
189 */
190struct s3c24xx_dma_phy {
191 unsigned int id;
192 bool valid;
193 void __iomem *base;
194 int irq;
195 struct clk *clk;
196 spinlock_t lock;
197 struct s3c24xx_dma_chan *serving;
198 struct s3c24xx_dma_engine *host;
199};
200
201/*
202 * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel
203 * @id: the id of the channel
204 * @name: name of the channel
205 * @vc: wrappped virtual channel
206 * @phy: the physical channel utilized by this channel, if there is one
207 * @runtime_addr: address for RX/TX according to the runtime config
208 * @at: active transaction on this channel
209 * @lock: a lock for this channel data
210 * @host: a pointer to the host (internal use)
211 * @state: whether the channel is idle, running etc
212 * @slave: whether this channel is a device (slave) or for memcpy
213 */
214struct s3c24xx_dma_chan {
215 int id;
216 const char *name;
217 struct virt_dma_chan vc;
218 struct s3c24xx_dma_phy *phy;
219 struct dma_slave_config cfg;
220 struct s3c24xx_txd *at;
221 struct s3c24xx_dma_engine *host;
222 enum s3c24xx_dma_chan_state state;
223 bool slave;
224};
225
226/*
227 * struct s3c24xx_dma_engine - the local state holder for the S3C24XX
228 * @pdev: the corresponding platform device
229 * @pdata: platform data passed in from the platform/machine
230 * @base: virtual memory base (remapped)
231 * @slave: slave engine for this instance
232 * @memcpy: memcpy engine for this instance
233 * @phy_chans: array of data for the physical channels
234 */
235struct s3c24xx_dma_engine {
236 struct platform_device *pdev;
237 const struct s3c24xx_dma_platdata *pdata;
238 struct soc_data *sdata;
239 void __iomem *base;
240 struct dma_device slave;
241 struct dma_device memcpy;
242 struct s3c24xx_dma_phy *phy_chans;
243};
244
245/*
246 * Physical channel handling
247 */
248
249/*
250 * Check whether a certain channel is busy or not.
251 */
252static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy)
253{
254 unsigned int val = readl(phy->base + S3C24XX_DSTAT);
255 return val & S3C24XX_DSTAT_STAT_BUSY;
256}
257
258static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan,
259 struct s3c24xx_dma_phy *phy)
260{
261 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
262 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
263 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
264 int phyvalid;
265
266 /* every phy is valid for memcopy channels */
267 if (!s3cchan->slave)
268 return true;
269
270 /* On newer variants all phys can be used for all virtual channels */
271 if (s3cdma->sdata->has_reqsel)
272 return true;
273
274 phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH));
275 return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false;
276}
277
278/*
279 * Allocate a physical channel for a virtual channel
280 *
281 * Try to locate a physical channel to be used for this transfer. If all
282 * are taken return NULL and the requester will have to cope by using
283 * some fallback PIO mode or retrying later.
284 */
285static
286struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)
287{
288 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
289 struct s3c24xx_dma_phy *phy = NULL;
290 unsigned long flags;
291 int i;
292 int ret;
293
294 for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
295 phy = &s3cdma->phy_chans[i];
296
297 if (!phy->valid)
298 continue;
299
300 if (!s3c24xx_dma_phy_valid(s3cchan, phy))
301 continue;
302
303 spin_lock_irqsave(&phy->lock, flags);
304
305 if (!phy->serving) {
306 phy->serving = s3cchan;
307 spin_unlock_irqrestore(&phy->lock, flags);
308 break;
309 }
310
311 spin_unlock_irqrestore(&phy->lock, flags);
312 }
313
314 /* No physical channel available, cope with it */
315 if (i == s3cdma->pdata->num_phy_channels) {
316 dev_warn(&s3cdma->pdev->dev, "no phy channel available\n");
317 return NULL;
318 }
319
320 /* start the phy clock */
321 if (s3cdma->sdata->has_clocks) {
322 ret = clk_enable(phy->clk);
323 if (ret) {
324 dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n",
325 phy->id, ret);
326 phy->serving = NULL;
327 return NULL;
328 }
329 }
330
331 return phy;
332}
333
334/*
335 * Mark the physical channel as free.
336 *
337 * This drops the link between the physical and virtual channel.
338 */
339static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy)
340{
341 struct s3c24xx_dma_engine *s3cdma = phy->host;
342
343 if (s3cdma->sdata->has_clocks)
344 clk_disable(phy->clk);
345
346 phy->serving = NULL;
347}
348
349/*
350 * Stops the channel by writing the stop bit.
351 * This should not be used for an on-going transfer, but as a method of
352 * shutting down a channel (eg, when it's no longer used) or terminating a
353 * transfer.
354 */
355static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy)
356{
357 writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG);
358}
359
360/*
361 * Virtual channel handling
362 */
363
364static inline
365struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan)
366{
367 return container_of(chan, struct s3c24xx_dma_chan, vc.chan);
368}
369
370static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan)
371{
372 struct s3c24xx_dma_phy *phy = s3cchan->phy;
373 struct s3c24xx_txd *txd = s3cchan->at;
374 u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK;
375
376 return tc * txd->width;
377}
378
379static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan,
380 struct dma_slave_config *config)
381{
382 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
383 unsigned long flags;
384 int ret = 0;
385
386 /* Reject definitely invalid configurations */
387 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
388 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
389 return -EINVAL;
390
391 spin_lock_irqsave(&s3cchan->vc.lock, flags);
392
393 if (!s3cchan->slave) {
394 ret = -EINVAL;
395 goto out;
396 }
397
398 s3cchan->cfg = *config;
399
400out:
401 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
402 return ret;
403}
404
405/*
406 * Transfer handling
407 */
408
409static inline
410struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx)
411{
412 return container_of(tx, struct s3c24xx_txd, vd.tx);
413}
414
415static struct s3c24xx_txd *s3c24xx_dma_get_txd(void)
416{
417 struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
418
419 if (txd) {
420 INIT_LIST_HEAD(&txd->dsg_list);
421 txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD;
422 }
423
424 return txd;
425}
426
427static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd)
428{
429 struct s3c24xx_sg *dsg, *_dsg;
430
431 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
432 list_del(&dsg->node);
433 kfree(dsg);
434 }
435
436 kfree(txd);
437}
438
439static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan,
440 struct s3c24xx_txd *txd)
441{
442 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
443 struct s3c24xx_dma_phy *phy = s3cchan->phy;
444 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
445 struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node);
446 u32 dcon = txd->dcon;
447 u32 val;
448
449 /* transfer-size and -count from len and width */
450 switch (txd->width) {
451 case 1:
452 dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len;
453 break;
454 case 2:
455 dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2);
456 break;
457 case 4:
458 dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4);
459 break;
460 }
461
462 if (s3cchan->slave) {
463 struct s3c24xx_dma_channel *cdata =
464 &pdata->channels[s3cchan->id];
465
466 if (s3cdma->sdata->has_reqsel) {
467 writel_relaxed((cdata->chansel << 1) |
468 S3C24XX_DMAREQSEL_HW,
469 phy->base + S3C24XX_DMAREQSEL);
470 } else {
471 int csel = cdata->chansel >> (phy->id *
472 S3C24XX_CHANSEL_WIDTH);
473
474 csel &= S3C24XX_CHANSEL_REQ_MASK;
475 dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT;
476 dcon |= S3C24XX_DCON_HWTRIG;
477 }
478 } else {
479 if (s3cdma->sdata->has_reqsel)
480 writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL);
481 }
482
483 writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC);
484 writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC);
485 writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST);
486 writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC);
487 writel_relaxed(dcon, phy->base + S3C24XX_DCON);
488
489 val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG);
490 val &= ~S3C24XX_DMASKTRIG_STOP;
491 val |= S3C24XX_DMASKTRIG_ON;
492
493 /* trigger the dma operation for memcpy transfers */
494 if (!s3cchan->slave)
495 val |= S3C24XX_DMASKTRIG_SWTRIG;
496
497 writel(val, phy->base + S3C24XX_DMASKTRIG);
498}
499
500/*
501 * Set the initial DMA register values and start first sg.
502 */
503static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)
504{
505 struct s3c24xx_dma_phy *phy = s3cchan->phy;
506 struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc);
507 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
508
509 list_del(&txd->vd.node);
510
511 s3cchan->at = txd;
512
513 /* Wait for channel inactive */
514 while (s3c24xx_dma_phy_busy(phy))
515 cpu_relax();
516
517 /* point to the first element of the sg list */
518 txd->at = txd->dsg_list.next;
519 s3c24xx_dma_start_next_sg(s3cchan, txd);
520}
521
522/*
523 * Try to allocate a physical channel. When successful, assign it to
524 * this virtual channel, and initiate the next descriptor. The
525 * virtual channel lock must be held at this point.
526 */
527static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan)
528{
529 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
530 struct s3c24xx_dma_phy *phy;
531
532 phy = s3c24xx_dma_get_phy(s3cchan);
533 if (!phy) {
534 dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n",
535 s3cchan->name);
536 s3cchan->state = S3C24XX_DMA_CHAN_WAITING;
537 return;
538 }
539
540 dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n",
541 phy->id, s3cchan->name);
542
543 s3cchan->phy = phy;
544 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
545
546 s3c24xx_dma_start_next_txd(s3cchan);
547}
548
549static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy,
550 struct s3c24xx_dma_chan *s3cchan)
551{
552 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
553
554 dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n",
555 phy->id, s3cchan->name);
556
557 /*
558 * We do this without taking the lock; we're really only concerned
559 * about whether this pointer is NULL or not, and we're guaranteed
560 * that this will only be called when it _already_ is non-NULL.
561 */
562 phy->serving = s3cchan;
563 s3cchan->phy = phy;
564 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
565 s3c24xx_dma_start_next_txd(s3cchan);
566}
567
568/*
569 * Free a physical DMA channel, potentially reallocating it to another
570 * virtual channel if we have any pending.
571 */
572static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan)
573{
574 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
575 struct s3c24xx_dma_chan *p, *next;
576
577retry:
578 next = NULL;
579
580 /* Find a waiting virtual channel for the next transfer. */
581 list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node)
582 if (p->state == S3C24XX_DMA_CHAN_WAITING) {
583 next = p;
584 break;
585 }
586
587 if (!next) {
588 list_for_each_entry(p, &s3cdma->slave.channels,
589 vc.chan.device_node)
590 if (p->state == S3C24XX_DMA_CHAN_WAITING &&
591 s3c24xx_dma_phy_valid(p, s3cchan->phy)) {
592 next = p;
593 break;
594 }
595 }
596
597 /* Ensure that the physical channel is stopped */
598 s3c24xx_dma_terminate_phy(s3cchan->phy);
599
600 if (next) {
601 bool success;
602
603 /*
604 * Eww. We know this isn't going to deadlock
605 * but lockdep probably doesn't.
606 */
607 spin_lock(&next->vc.lock);
608 /* Re-check the state now that we have the lock */
609 success = next->state == S3C24XX_DMA_CHAN_WAITING;
610 if (success)
611 s3c24xx_dma_phy_reassign_start(s3cchan->phy, next);
612 spin_unlock(&next->vc.lock);
613
614 /* If the state changed, try to find another channel */
615 if (!success)
616 goto retry;
617 } else {
618 /* No more jobs, so free up the physical channel */
619 s3c24xx_dma_put_phy(s3cchan->phy);
620 }
621
622 s3cchan->phy = NULL;
623 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
624}
625
626static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd)
627{
628 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
629 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan);
630
631 if (!s3cchan->slave)
632 dma_descriptor_unmap(&vd->tx);
633
634 s3c24xx_dma_free_txd(txd);
635}
636
637static irqreturn_t s3c24xx_dma_irq(int irq, void *data)
638{
639 struct s3c24xx_dma_phy *phy = data;
640 struct s3c24xx_dma_chan *s3cchan = phy->serving;
641 struct s3c24xx_txd *txd;
642
643 dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id);
644
645 /*
646 * Interrupts happen to notify the completion of a transfer and the
647 * channel should have moved into its stop state already on its own.
648 * Therefore interrupts on channels not bound to a virtual channel
649 * should never happen. Nevertheless send a terminate command to the
650 * channel if the unlikely case happens.
651 */
652 if (unlikely(!s3cchan)) {
653 dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n",
654 phy->id);
655
656 s3c24xx_dma_terminate_phy(phy);
657
658 return IRQ_HANDLED;
659 }
660
661 spin_lock(&s3cchan->vc.lock);
662 txd = s3cchan->at;
663 if (txd) {
664 /* when more sg's are in this txd, start the next one */
665 if (!list_is_last(txd->at, &txd->dsg_list)) {
666 txd->at = txd->at->next;
667 if (txd->cyclic)
668 vchan_cyclic_callback(&txd->vd);
669 s3c24xx_dma_start_next_sg(s3cchan, txd);
670 } else if (!txd->cyclic) {
671 s3cchan->at = NULL;
672 vchan_cookie_complete(&txd->vd);
673
674 /*
675 * And start the next descriptor (if any),
676 * otherwise free this channel.
677 */
678 if (vchan_next_desc(&s3cchan->vc))
679 s3c24xx_dma_start_next_txd(s3cchan);
680 else
681 s3c24xx_dma_phy_free(s3cchan);
682 } else {
683 vchan_cyclic_callback(&txd->vd);
684
685 /* Cyclic: reset at beginning */
686 txd->at = txd->dsg_list.next;
687 s3c24xx_dma_start_next_sg(s3cchan, txd);
688 }
689 }
690 spin_unlock(&s3cchan->vc.lock);
691
692 return IRQ_HANDLED;
693}
694
695/*
696 * The DMA ENGINE API
697 */
698
699static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
700{
701 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
702 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
703 LIST_HEAD(head);
704 unsigned long flags;
705 int ret;
706
707 spin_lock_irqsave(&s3cchan->vc.lock, flags);
708
709 if (!s3cchan->phy && !s3cchan->at) {
710 dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n",
711 s3cchan->id);
712 ret = -EINVAL;
713 goto unlock;
714 }
715
716 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
717
718 /* Mark physical channel as free */
719 if (s3cchan->phy)
720 s3c24xx_dma_phy_free(s3cchan);
721
722 /* Dequeue current job */
723 if (s3cchan->at) {
724 vchan_terminate_vdesc(&s3cchan->at->vd);
725 s3cchan->at = NULL;
726 }
727
728 /* Dequeue jobs not yet fired as well */
729
730 vchan_get_all_descriptors(&s3cchan->vc, &head);
731
732 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
733
734 vchan_dma_desc_free_list(&s3cchan->vc, &head);
735
736 return 0;
737
738unlock:
739 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
740
741 return ret;
742}
743
744static void s3c24xx_dma_synchronize(struct dma_chan *chan)
745{
746 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
747
748 vchan_synchronize(&s3cchan->vc);
749}
750
751static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan)
752{
753 /* Ensure all queued descriptors are freed */
754 vchan_free_chan_resources(to_virt_chan(chan));
755}
756
757static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan,
758 dma_cookie_t cookie, struct dma_tx_state *txstate)
759{
760 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
761 struct s3c24xx_txd *txd;
762 struct s3c24xx_sg *dsg;
763 struct virt_dma_desc *vd;
764 unsigned long flags;
765 enum dma_status ret;
766 size_t bytes = 0;
767
768 spin_lock_irqsave(&s3cchan->vc.lock, flags);
769 ret = dma_cookie_status(chan, cookie, txstate);
770
771 /*
772 * There's no point calculating the residue if there's
773 * no txstate to store the value.
774 */
775 if (ret == DMA_COMPLETE || !txstate) {
776 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
777 return ret;
778 }
779
780 vd = vchan_find_desc(&s3cchan->vc, cookie);
781 if (vd) {
782 /* On the issued list, so hasn't been processed yet */
783 txd = to_s3c24xx_txd(&vd->tx);
784
785 list_for_each_entry(dsg, &txd->dsg_list, node)
786 bytes += dsg->len;
787 } else {
788 /*
789 * Currently running, so sum over the pending sg's and
790 * the currently active one.
791 */
792 txd = s3cchan->at;
793
794 dsg = list_entry(txd->at, struct s3c24xx_sg, node);
795 list_for_each_entry_from(dsg, &txd->dsg_list, node)
796 bytes += dsg->len;
797
798 bytes += s3c24xx_dma_getbytes_chan(s3cchan);
799 }
800 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
801
802 /*
803 * This cookie not complete yet
804 * Get number of bytes left in the active transactions and queue
805 */
806 dma_set_residue(txstate, bytes);
807
808 /* Whether waiting or running, we're in progress */
809 return ret;
810}
811
812/*
813 * Initialize a descriptor to be used by memcpy submit
814 */
815static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy(
816 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
817 size_t len, unsigned long flags)
818{
819 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
820 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
821 struct s3c24xx_txd *txd;
822 struct s3c24xx_sg *dsg;
823 int src_mod, dest_mod;
824
825 dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %zu bytes from %s\n",
826 len, s3cchan->name);
827
828 if ((len & S3C24XX_DCON_TC_MASK) != len) {
829 dev_err(&s3cdma->pdev->dev, "memcpy size %zu to large\n", len);
830 return NULL;
831 }
832
833 txd = s3c24xx_dma_get_txd();
834 if (!txd)
835 return NULL;
836
837 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
838 if (!dsg) {
839 s3c24xx_dma_free_txd(txd);
840 return NULL;
841 }
842 list_add_tail(&dsg->node, &txd->dsg_list);
843
844 dsg->src_addr = src;
845 dsg->dst_addr = dest;
846 dsg->len = len;
847
848 /*
849 * Determine a suitable transfer width.
850 * The DMA controller cannot fetch/store information which is not
851 * naturally aligned on the bus, i.e., a 4 byte fetch must start at
852 * an address divisible by 4 - more generally addr % width must be 0.
853 */
854 src_mod = src % 4;
855 dest_mod = dest % 4;
856 switch (len % 4) {
857 case 0:
858 txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1;
859 break;
860 case 2:
861 txd->width = ((src_mod == 2 || src_mod == 0) &&
862 (dest_mod == 2 || dest_mod == 0)) ? 2 : 1;
863 break;
864 default:
865 txd->width = 1;
866 break;
867 }
868
869 txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT;
870 txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT;
871 txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK |
872 S3C24XX_DCON_SERV_WHOLE;
873
874 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
875}
876
877static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic(
878 struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
879 enum dma_transfer_direction direction, unsigned long flags)
880{
881 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
882 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
883 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
884 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
885 struct s3c24xx_txd *txd;
886 struct s3c24xx_sg *dsg;
887 unsigned sg_len;
888 dma_addr_t slave_addr;
889 u32 hwcfg = 0;
890 int i;
891
892 dev_dbg(&s3cdma->pdev->dev,
893 "prepare cyclic transaction of %zu bytes with period %zu from %s\n",
894 size, period, s3cchan->name);
895
896 if (!is_slave_direction(direction)) {
897 dev_err(&s3cdma->pdev->dev,
898 "direction %d unsupported\n", direction);
899 return NULL;
900 }
901
902 txd = s3c24xx_dma_get_txd();
903 if (!txd)
904 return NULL;
905
906 txd->cyclic = 1;
907
908 if (cdata->handshake)
909 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
910
911 switch (cdata->bus) {
912 case S3C24XX_DMA_APB:
913 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
914 hwcfg |= S3C24XX_DISRCC_LOC_APB;
915 break;
916 case S3C24XX_DMA_AHB:
917 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
918 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
919 break;
920 }
921
922 /*
923 * Always assume our peripheral desintation is a fixed
924 * address in memory.
925 */
926 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
927
928 /*
929 * Individual dma operations are requested by the slave,
930 * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
931 */
932 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
933
934 if (direction == DMA_MEM_TO_DEV) {
935 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
936 S3C24XX_DISRCC_INC_INCREMENT;
937 txd->didstc = hwcfg;
938 slave_addr = s3cchan->cfg.dst_addr;
939 txd->width = s3cchan->cfg.dst_addr_width;
940 } else {
941 txd->disrcc = hwcfg;
942 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
943 S3C24XX_DIDSTC_INC_INCREMENT;
944 slave_addr = s3cchan->cfg.src_addr;
945 txd->width = s3cchan->cfg.src_addr_width;
946 }
947
948 sg_len = size / period;
949
950 for (i = 0; i < sg_len; i++) {
951 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
952 if (!dsg) {
953 s3c24xx_dma_free_txd(txd);
954 return NULL;
955 }
956 list_add_tail(&dsg->node, &txd->dsg_list);
957
958 dsg->len = period;
959 /* Check last period length */
960 if (i == sg_len - 1)
961 dsg->len = size - period * i;
962 if (direction == DMA_MEM_TO_DEV) {
963 dsg->src_addr = addr + period * i;
964 dsg->dst_addr = slave_addr;
965 } else { /* DMA_DEV_TO_MEM */
966 dsg->src_addr = slave_addr;
967 dsg->dst_addr = addr + period * i;
968 }
969 }
970
971 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
972}
973
974static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg(
975 struct dma_chan *chan, struct scatterlist *sgl,
976 unsigned int sg_len, enum dma_transfer_direction direction,
977 unsigned long flags, void *context)
978{
979 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
980 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
981 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
982 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
983 struct s3c24xx_txd *txd;
984 struct s3c24xx_sg *dsg;
985 struct scatterlist *sg;
986 dma_addr_t slave_addr;
987 u32 hwcfg = 0;
988 int tmp;
989
990 dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n",
991 sg_dma_len(sgl), s3cchan->name);
992
993 txd = s3c24xx_dma_get_txd();
994 if (!txd)
995 return NULL;
996
997 if (cdata->handshake)
998 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
999
1000 switch (cdata->bus) {
1001 case S3C24XX_DMA_APB:
1002 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
1003 hwcfg |= S3C24XX_DISRCC_LOC_APB;
1004 break;
1005 case S3C24XX_DMA_AHB:
1006 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
1007 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
1008 break;
1009 }
1010
1011 /*
1012 * Always assume our peripheral desintation is a fixed
1013 * address in memory.
1014 */
1015 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
1016
1017 /*
1018 * Individual dma operations are requested by the slave,
1019 * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
1020 */
1021 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
1022
1023 if (direction == DMA_MEM_TO_DEV) {
1024 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
1025 S3C24XX_DISRCC_INC_INCREMENT;
1026 txd->didstc = hwcfg;
1027 slave_addr = s3cchan->cfg.dst_addr;
1028 txd->width = s3cchan->cfg.dst_addr_width;
1029 } else if (direction == DMA_DEV_TO_MEM) {
1030 txd->disrcc = hwcfg;
1031 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
1032 S3C24XX_DIDSTC_INC_INCREMENT;
1033 slave_addr = s3cchan->cfg.src_addr;
1034 txd->width = s3cchan->cfg.src_addr_width;
1035 } else {
1036 s3c24xx_dma_free_txd(txd);
1037 dev_err(&s3cdma->pdev->dev,
1038 "direction %d unsupported\n", direction);
1039 return NULL;
1040 }
1041
1042 for_each_sg(sgl, sg, sg_len, tmp) {
1043 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
1044 if (!dsg) {
1045 s3c24xx_dma_free_txd(txd);
1046 return NULL;
1047 }
1048 list_add_tail(&dsg->node, &txd->dsg_list);
1049
1050 dsg->len = sg_dma_len(sg);
1051 if (direction == DMA_MEM_TO_DEV) {
1052 dsg->src_addr = sg_dma_address(sg);
1053 dsg->dst_addr = slave_addr;
1054 } else { /* DMA_DEV_TO_MEM */
1055 dsg->src_addr = slave_addr;
1056 dsg->dst_addr = sg_dma_address(sg);
1057 }
1058 }
1059
1060 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
1061}
1062
1063/*
1064 * Slave transactions callback to the slave device to allow
1065 * synchronization of slave DMA signals with the DMAC enable
1066 */
1067static void s3c24xx_dma_issue_pending(struct dma_chan *chan)
1068{
1069 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
1070 unsigned long flags;
1071
1072 spin_lock_irqsave(&s3cchan->vc.lock, flags);
1073 if (vchan_issue_pending(&s3cchan->vc)) {
1074 if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING)
1075 s3c24xx_dma_phy_alloc_and_start(s3cchan);
1076 }
1077 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
1078}
1079
1080/*
1081 * Bringup and teardown
1082 */
1083
1084/*
1085 * Initialise the DMAC memcpy/slave channels.
1086 * Make a local wrapper to hold required data
1087 */
1088static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma,
1089 struct dma_device *dmadev, unsigned int channels, bool slave)
1090{
1091 struct s3c24xx_dma_chan *chan;
1092 int i;
1093
1094 INIT_LIST_HEAD(&dmadev->channels);
1095
1096 /*
1097 * Register as many many memcpy as we have physical channels,
1098 * we won't always be able to use all but the code will have
1099 * to cope with that situation.
1100 */
1101 for (i = 0; i < channels; i++) {
1102 chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL);
1103 if (!chan)
1104 return -ENOMEM;
1105
1106 chan->id = i;
1107 chan->host = s3cdma;
1108 chan->state = S3C24XX_DMA_CHAN_IDLE;
1109
1110 if (slave) {
1111 chan->slave = true;
1112 chan->name = kasprintf(GFP_KERNEL, "slave%d", i);
1113 if (!chan->name)
1114 return -ENOMEM;
1115 } else {
1116 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1117 if (!chan->name)
1118 return -ENOMEM;
1119 }
1120 dev_dbg(dmadev->dev,
1121 "initialize virtual channel \"%s\"\n",
1122 chan->name);
1123
1124 chan->vc.desc_free = s3c24xx_dma_desc_free;
1125 vchan_init(&chan->vc, dmadev);
1126 }
1127 dev_info(dmadev->dev, "initialized %d virtual %s channels\n",
1128 i, slave ? "slave" : "memcpy");
1129 return i;
1130}
1131
1132static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev)
1133{
1134 struct s3c24xx_dma_chan *chan = NULL;
1135 struct s3c24xx_dma_chan *next;
1136
1137 list_for_each_entry_safe(chan,
1138 next, &dmadev->channels, vc.chan.device_node) {
1139 list_del(&chan->vc.chan.device_node);
1140 tasklet_kill(&chan->vc.task);
1141 }
1142}
1143
1144/* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */
1145static struct soc_data soc_s3c2410 = {
1146 .stride = 0x40,
1147 .has_reqsel = false,
1148 .has_clocks = false,
1149};
1150
1151/* s3c2412 and s3c2413 have a 0x40 stride and dmareqsel mechanism */
1152static struct soc_data soc_s3c2412 = {
1153 .stride = 0x40,
1154 .has_reqsel = true,
1155 .has_clocks = true,
1156};
1157
1158/* s3c2443 and following have a 0x100 stride and dmareqsel mechanism */
1159static struct soc_data soc_s3c2443 = {
1160 .stride = 0x100,
1161 .has_reqsel = true,
1162 .has_clocks = true,
1163};
1164
1165static const struct platform_device_id s3c24xx_dma_driver_ids[] = {
1166 {
1167 .name = "s3c2410-dma",
1168 .driver_data = (kernel_ulong_t)&soc_s3c2410,
1169 }, {
1170 .name = "s3c2412-dma",
1171 .driver_data = (kernel_ulong_t)&soc_s3c2412,
1172 }, {
1173 .name = "s3c2443-dma",
1174 .driver_data = (kernel_ulong_t)&soc_s3c2443,
1175 },
1176 { },
1177};
1178
1179static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev)
1180{
1181 return (struct soc_data *)
1182 platform_get_device_id(pdev)->driver_data;
1183}
1184
1185static int s3c24xx_dma_probe(struct platform_device *pdev)
1186{
1187 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1188 struct s3c24xx_dma_engine *s3cdma;
1189 struct soc_data *sdata;
1190 struct resource *res;
1191 int ret;
1192 int i;
1193
1194 if (!pdata) {
1195 dev_err(&pdev->dev, "platform data missing\n");
1196 return -ENODEV;
1197 }
1198
1199 /* Basic sanity check */
1200 if (pdata->num_phy_channels > MAX_DMA_CHANNELS) {
1201 dev_err(&pdev->dev, "too many dma channels %d, max %d\n",
1202 pdata->num_phy_channels, MAX_DMA_CHANNELS);
1203 return -EINVAL;
1204 }
1205
1206 sdata = s3c24xx_dma_get_soc_data(pdev);
1207 if (!sdata)
1208 return -EINVAL;
1209
1210 s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL);
1211 if (!s3cdma)
1212 return -ENOMEM;
1213
1214 s3cdma->pdev = pdev;
1215 s3cdma->pdata = pdata;
1216 s3cdma->sdata = sdata;
1217
1218 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1219 s3cdma->base = devm_ioremap_resource(&pdev->dev, res);
1220 if (IS_ERR(s3cdma->base))
1221 return PTR_ERR(s3cdma->base);
1222
1223 s3cdma->phy_chans = devm_kcalloc(&pdev->dev,
1224 pdata->num_phy_channels,
1225 sizeof(struct s3c24xx_dma_phy),
1226 GFP_KERNEL);
1227 if (!s3cdma->phy_chans)
1228 return -ENOMEM;
1229
1230 /* acquire irqs and clocks for all physical channels */
1231 for (i = 0; i < pdata->num_phy_channels; i++) {
1232 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1233 char clk_name[6];
1234
1235 phy->id = i;
1236 phy->base = s3cdma->base + (i * sdata->stride);
1237 phy->host = s3cdma;
1238
1239 phy->irq = platform_get_irq(pdev, i);
1240 if (phy->irq < 0)
1241 continue;
1242
1243 ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq,
1244 0, pdev->name, phy);
1245 if (ret) {
1246 dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n",
1247 i, ret);
1248 continue;
1249 }
1250
1251 if (sdata->has_clocks) {
1252 sprintf(clk_name, "dma.%d", i);
1253 phy->clk = devm_clk_get(&pdev->dev, clk_name);
1254 if (IS_ERR(phy->clk) && sdata->has_clocks) {
1255 dev_err(&pdev->dev, "unable to acquire clock for channel %d, error %lu\n",
1256 i, PTR_ERR(phy->clk));
1257 continue;
1258 }
1259
1260 ret = clk_prepare(phy->clk);
1261 if (ret) {
1262 dev_err(&pdev->dev, "clock for phy %d failed, error %d\n",
1263 i, ret);
1264 continue;
1265 }
1266 }
1267
1268 spin_lock_init(&phy->lock);
1269 phy->valid = true;
1270
1271 dev_dbg(&pdev->dev, "physical channel %d is %s\n",
1272 i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE");
1273 }
1274
1275 /* Initialize memcpy engine */
1276 dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask);
1277 dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask);
1278 s3cdma->memcpy.dev = &pdev->dev;
1279 s3cdma->memcpy.device_free_chan_resources =
1280 s3c24xx_dma_free_chan_resources;
1281 s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy;
1282 s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status;
1283 s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending;
1284 s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config;
1285 s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all;
1286 s3cdma->memcpy.device_synchronize = s3c24xx_dma_synchronize;
1287
1288 /* Initialize slave engine for SoC internal dedicated peripherals */
1289 dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask);
1290 dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask);
1291 dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask);
1292 s3cdma->slave.dev = &pdev->dev;
1293 s3cdma->slave.device_free_chan_resources =
1294 s3c24xx_dma_free_chan_resources;
1295 s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status;
1296 s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending;
1297 s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg;
1298 s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
1299 s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
1300 s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
1301 s3cdma->slave.device_synchronize = s3c24xx_dma_synchronize;
1302 s3cdma->slave.filter.map = pdata->slave_map;
1303 s3cdma->slave.filter.mapcnt = pdata->slavecnt;
1304 s3cdma->slave.filter.fn = s3c24xx_dma_filter;
1305
1306 /* Register as many memcpy channels as there are physical channels */
1307 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy,
1308 pdata->num_phy_channels, false);
1309 if (ret <= 0) {
1310 dev_warn(&pdev->dev,
1311 "%s failed to enumerate memcpy channels - %d\n",
1312 __func__, ret);
1313 goto err_memcpy;
1314 }
1315
1316 /* Register slave channels */
1317 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave,
1318 pdata->num_channels, true);
1319 if (ret <= 0) {
1320 dev_warn(&pdev->dev,
1321 "%s failed to enumerate slave channels - %d\n",
1322 __func__, ret);
1323 goto err_slave;
1324 }
1325
1326 ret = dma_async_device_register(&s3cdma->memcpy);
1327 if (ret) {
1328 dev_warn(&pdev->dev,
1329 "%s failed to register memcpy as an async device - %d\n",
1330 __func__, ret);
1331 goto err_memcpy_reg;
1332 }
1333
1334 ret = dma_async_device_register(&s3cdma->slave);
1335 if (ret) {
1336 dev_warn(&pdev->dev,
1337 "%s failed to register slave as an async device - %d\n",
1338 __func__, ret);
1339 goto err_slave_reg;
1340 }
1341
1342 platform_set_drvdata(pdev, s3cdma);
1343 dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n",
1344 pdata->num_phy_channels);
1345
1346 return 0;
1347
1348err_slave_reg:
1349 dma_async_device_unregister(&s3cdma->memcpy);
1350err_memcpy_reg:
1351 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1352err_slave:
1353 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1354err_memcpy:
1355 if (sdata->has_clocks)
1356 for (i = 0; i < pdata->num_phy_channels; i++) {
1357 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1358 if (phy->valid)
1359 clk_unprepare(phy->clk);
1360 }
1361
1362 return ret;
1363}
1364
1365static void s3c24xx_dma_free_irq(struct platform_device *pdev,
1366 struct s3c24xx_dma_engine *s3cdma)
1367{
1368 int i;
1369
1370 for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
1371 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1372
1373 devm_free_irq(&pdev->dev, phy->irq, phy);
1374 }
1375}
1376
1377static int s3c24xx_dma_remove(struct platform_device *pdev)
1378{
1379 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1380 struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev);
1381 struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev);
1382 int i;
1383
1384 dma_async_device_unregister(&s3cdma->slave);
1385 dma_async_device_unregister(&s3cdma->memcpy);
1386
1387 s3c24xx_dma_free_irq(pdev, s3cdma);
1388
1389 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1390 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1391
1392 if (sdata->has_clocks)
1393 for (i = 0; i < pdata->num_phy_channels; i++) {
1394 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1395 if (phy->valid)
1396 clk_unprepare(phy->clk);
1397 }
1398
1399 return 0;
1400}
1401
1402static struct platform_driver s3c24xx_dma_driver = {
1403 .driver = {
1404 .name = "s3c24xx-dma",
1405 },
1406 .id_table = s3c24xx_dma_driver_ids,
1407 .probe = s3c24xx_dma_probe,
1408 .remove = s3c24xx_dma_remove,
1409};
1410
1411module_platform_driver(s3c24xx_dma_driver);
1412
1413bool s3c24xx_dma_filter(struct dma_chan *chan, void *param)
1414{
1415 struct s3c24xx_dma_chan *s3cchan;
1416
1417 if (chan->device->dev->driver != &s3c24xx_dma_driver.driver)
1418 return false;
1419
1420 s3cchan = to_s3c24xx_dma_chan(chan);
1421
1422 return s3cchan->id == (uintptr_t)param;
1423}
1424EXPORT_SYMBOL(s3c24xx_dma_filter);
1425
1426MODULE_DESCRIPTION("S3C24XX DMA Driver");
1427MODULE_AUTHOR("Heiko Stuebner");
1428MODULE_LICENSE("GPL v2");
1/*
2 * S3C24XX DMA handling
3 *
4 * Copyright (c) 2013 Heiko Stuebner <heiko@sntech.de>
5 *
6 * based on amba-pl08x.c
7 *
8 * Copyright (c) 2006 ARM Ltd.
9 * Copyright (c) 2010 ST-Ericsson SA
10 *
11 * Author: Peter Pearse <peter.pearse@arm.com>
12 * Author: Linus Walleij <linus.walleij@stericsson.com>
13 *
14 * This program is free software; you can redistribute it and/or modify it
15 * under the terms of the GNU General Public License as published by the Free
16 * Software Foundation; either version 2 of the License, or (at your option)
17 * any later version.
18 *
19 * The DMA controllers in S3C24XX SoCs have a varying number of DMA signals
20 * that can be routed to any of the 4 to 8 hardware-channels.
21 *
22 * Therefore on these DMA controllers the number of channels
23 * and the number of incoming DMA signals are two totally different things.
24 * It is usually not possible to theoretically handle all physical signals,
25 * so a multiplexing scheme with possible denial of use is necessary.
26 *
27 * Open items:
28 * - bursts
29 */
30
31#include <linux/platform_device.h>
32#include <linux/types.h>
33#include <linux/dmaengine.h>
34#include <linux/dma-mapping.h>
35#include <linux/interrupt.h>
36#include <linux/clk.h>
37#include <linux/module.h>
38#include <linux/slab.h>
39#include <linux/platform_data/dma-s3c24xx.h>
40
41#include "dmaengine.h"
42#include "virt-dma.h"
43
44#define MAX_DMA_CHANNELS 8
45
46#define S3C24XX_DISRC 0x00
47#define S3C24XX_DISRCC 0x04
48#define S3C24XX_DISRCC_INC_INCREMENT 0
49#define S3C24XX_DISRCC_INC_FIXED BIT(0)
50#define S3C24XX_DISRCC_LOC_AHB 0
51#define S3C24XX_DISRCC_LOC_APB BIT(1)
52
53#define S3C24XX_DIDST 0x08
54#define S3C24XX_DIDSTC 0x0c
55#define S3C24XX_DIDSTC_INC_INCREMENT 0
56#define S3C24XX_DIDSTC_INC_FIXED BIT(0)
57#define S3C24XX_DIDSTC_LOC_AHB 0
58#define S3C24XX_DIDSTC_LOC_APB BIT(1)
59#define S3C24XX_DIDSTC_INT_TC0 0
60#define S3C24XX_DIDSTC_INT_RELOAD BIT(2)
61
62#define S3C24XX_DCON 0x10
63
64#define S3C24XX_DCON_TC_MASK 0xfffff
65#define S3C24XX_DCON_DSZ_BYTE (0 << 20)
66#define S3C24XX_DCON_DSZ_HALFWORD (1 << 20)
67#define S3C24XX_DCON_DSZ_WORD (2 << 20)
68#define S3C24XX_DCON_DSZ_MASK (3 << 20)
69#define S3C24XX_DCON_DSZ_SHIFT 20
70#define S3C24XX_DCON_AUTORELOAD 0
71#define S3C24XX_DCON_NORELOAD BIT(22)
72#define S3C24XX_DCON_HWTRIG BIT(23)
73#define S3C24XX_DCON_HWSRC_SHIFT 24
74#define S3C24XX_DCON_SERV_SINGLE 0
75#define S3C24XX_DCON_SERV_WHOLE BIT(27)
76#define S3C24XX_DCON_TSZ_UNIT 0
77#define S3C24XX_DCON_TSZ_BURST4 BIT(28)
78#define S3C24XX_DCON_INT BIT(29)
79#define S3C24XX_DCON_SYNC_PCLK 0
80#define S3C24XX_DCON_SYNC_HCLK BIT(30)
81#define S3C24XX_DCON_DEMAND 0
82#define S3C24XX_DCON_HANDSHAKE BIT(31)
83
84#define S3C24XX_DSTAT 0x14
85#define S3C24XX_DSTAT_STAT_BUSY BIT(20)
86#define S3C24XX_DSTAT_CURRTC_MASK 0xfffff
87
88#define S3C24XX_DMASKTRIG 0x20
89#define S3C24XX_DMASKTRIG_SWTRIG BIT(0)
90#define S3C24XX_DMASKTRIG_ON BIT(1)
91#define S3C24XX_DMASKTRIG_STOP BIT(2)
92
93#define S3C24XX_DMAREQSEL 0x24
94#define S3C24XX_DMAREQSEL_HW BIT(0)
95
96/*
97 * S3C2410, S3C2440 and S3C2442 SoCs cannot select any physical channel
98 * for a DMA source. Instead only specific channels are valid.
99 * All of these SoCs have 4 physical channels and the number of request
100 * source bits is 3. Additionally we also need 1 bit to mark the channel
101 * as valid.
102 * Therefore we separate the chansel element of the channel data into 4
103 * parts of 4 bits each, to hold the information if the channel is valid
104 * and the hw request source to use.
105 *
106 * Example:
107 * SDI is valid on channels 0, 2 and 3 - with varying hw request sources.
108 * For it the chansel field would look like
109 *
110 * ((BIT(3) | 1) << 3 * 4) | // channel 3, with request source 1
111 * ((BIT(3) | 2) << 2 * 4) | // channel 2, with request source 2
112 * ((BIT(3) | 2) << 0 * 4) // channel 0, with request source 2
113 */
114#define S3C24XX_CHANSEL_WIDTH 4
115#define S3C24XX_CHANSEL_VALID BIT(3)
116#define S3C24XX_CHANSEL_REQ_MASK 7
117
118/*
119 * struct soc_data - vendor-specific config parameters for individual SoCs
120 * @stride: spacing between the registers of each channel
121 * @has_reqsel: does the controller use the newer requestselection mechanism
122 * @has_clocks: are controllable dma-clocks present
123 */
124struct soc_data {
125 int stride;
126 bool has_reqsel;
127 bool has_clocks;
128};
129
130/*
131 * enum s3c24xx_dma_chan_state - holds the virtual channel states
132 * @S3C24XX_DMA_CHAN_IDLE: the channel is idle
133 * @S3C24XX_DMA_CHAN_RUNNING: the channel has allocated a physical transport
134 * channel and is running a transfer on it
135 * @S3C24XX_DMA_CHAN_WAITING: the channel is waiting for a physical transport
136 * channel to become available (only pertains to memcpy channels)
137 */
138enum s3c24xx_dma_chan_state {
139 S3C24XX_DMA_CHAN_IDLE,
140 S3C24XX_DMA_CHAN_RUNNING,
141 S3C24XX_DMA_CHAN_WAITING,
142};
143
144/*
145 * struct s3c24xx_sg - structure containing data per sg
146 * @src_addr: src address of sg
147 * @dst_addr: dst address of sg
148 * @len: transfer len in bytes
149 * @node: node for txd's dsg_list
150 */
151struct s3c24xx_sg {
152 dma_addr_t src_addr;
153 dma_addr_t dst_addr;
154 size_t len;
155 struct list_head node;
156};
157
158/*
159 * struct s3c24xx_txd - wrapper for struct dma_async_tx_descriptor
160 * @vd: virtual DMA descriptor
161 * @dsg_list: list of children sg's
162 * @at: sg currently being transfered
163 * @width: transfer width
164 * @disrcc: value for source control register
165 * @didstc: value for destination control register
166 * @dcon: base value for dcon register
167 * @cyclic: indicate cyclic transfer
168 */
169struct s3c24xx_txd {
170 struct virt_dma_desc vd;
171 struct list_head dsg_list;
172 struct list_head *at;
173 u8 width;
174 u32 disrcc;
175 u32 didstc;
176 u32 dcon;
177 bool cyclic;
178};
179
180struct s3c24xx_dma_chan;
181
182/*
183 * struct s3c24xx_dma_phy - holder for the physical channels
184 * @id: physical index to this channel
185 * @valid: does the channel have all required elements
186 * @base: virtual memory base (remapped) for the this channel
187 * @irq: interrupt for this channel
188 * @clk: clock for this channel
189 * @lock: a lock to use when altering an instance of this struct
190 * @serving: virtual channel currently being served by this physicalchannel
191 * @host: a pointer to the host (internal use)
192 */
193struct s3c24xx_dma_phy {
194 unsigned int id;
195 bool valid;
196 void __iomem *base;
197 int irq;
198 struct clk *clk;
199 spinlock_t lock;
200 struct s3c24xx_dma_chan *serving;
201 struct s3c24xx_dma_engine *host;
202};
203
204/*
205 * struct s3c24xx_dma_chan - this structure wraps a DMA ENGINE channel
206 * @id: the id of the channel
207 * @name: name of the channel
208 * @vc: wrappped virtual channel
209 * @phy: the physical channel utilized by this channel, if there is one
210 * @runtime_addr: address for RX/TX according to the runtime config
211 * @at: active transaction on this channel
212 * @lock: a lock for this channel data
213 * @host: a pointer to the host (internal use)
214 * @state: whether the channel is idle, running etc
215 * @slave: whether this channel is a device (slave) or for memcpy
216 */
217struct s3c24xx_dma_chan {
218 int id;
219 const char *name;
220 struct virt_dma_chan vc;
221 struct s3c24xx_dma_phy *phy;
222 struct dma_slave_config cfg;
223 struct s3c24xx_txd *at;
224 struct s3c24xx_dma_engine *host;
225 enum s3c24xx_dma_chan_state state;
226 bool slave;
227};
228
229/*
230 * struct s3c24xx_dma_engine - the local state holder for the S3C24XX
231 * @pdev: the corresponding platform device
232 * @pdata: platform data passed in from the platform/machine
233 * @base: virtual memory base (remapped)
234 * @slave: slave engine for this instance
235 * @memcpy: memcpy engine for this instance
236 * @phy_chans: array of data for the physical channels
237 */
238struct s3c24xx_dma_engine {
239 struct platform_device *pdev;
240 const struct s3c24xx_dma_platdata *pdata;
241 struct soc_data *sdata;
242 void __iomem *base;
243 struct dma_device slave;
244 struct dma_device memcpy;
245 struct s3c24xx_dma_phy *phy_chans;
246};
247
248/*
249 * Physical channel handling
250 */
251
252/*
253 * Check whether a certain channel is busy or not.
254 */
255static int s3c24xx_dma_phy_busy(struct s3c24xx_dma_phy *phy)
256{
257 unsigned int val = readl(phy->base + S3C24XX_DSTAT);
258 return val & S3C24XX_DSTAT_STAT_BUSY;
259}
260
261static bool s3c24xx_dma_phy_valid(struct s3c24xx_dma_chan *s3cchan,
262 struct s3c24xx_dma_phy *phy)
263{
264 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
265 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
266 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
267 int phyvalid;
268
269 /* every phy is valid for memcopy channels */
270 if (!s3cchan->slave)
271 return true;
272
273 /* On newer variants all phys can be used for all virtual channels */
274 if (s3cdma->sdata->has_reqsel)
275 return true;
276
277 phyvalid = (cdata->chansel >> (phy->id * S3C24XX_CHANSEL_WIDTH));
278 return (phyvalid & S3C24XX_CHANSEL_VALID) ? true : false;
279}
280
281/*
282 * Allocate a physical channel for a virtual channel
283 *
284 * Try to locate a physical channel to be used for this transfer. If all
285 * are taken return NULL and the requester will have to cope by using
286 * some fallback PIO mode or retrying later.
287 */
288static
289struct s3c24xx_dma_phy *s3c24xx_dma_get_phy(struct s3c24xx_dma_chan *s3cchan)
290{
291 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
292 struct s3c24xx_dma_phy *phy = NULL;
293 unsigned long flags;
294 int i;
295 int ret;
296
297 for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
298 phy = &s3cdma->phy_chans[i];
299
300 if (!phy->valid)
301 continue;
302
303 if (!s3c24xx_dma_phy_valid(s3cchan, phy))
304 continue;
305
306 spin_lock_irqsave(&phy->lock, flags);
307
308 if (!phy->serving) {
309 phy->serving = s3cchan;
310 spin_unlock_irqrestore(&phy->lock, flags);
311 break;
312 }
313
314 spin_unlock_irqrestore(&phy->lock, flags);
315 }
316
317 /* No physical channel available, cope with it */
318 if (i == s3cdma->pdata->num_phy_channels) {
319 dev_warn(&s3cdma->pdev->dev, "no phy channel available\n");
320 return NULL;
321 }
322
323 /* start the phy clock */
324 if (s3cdma->sdata->has_clocks) {
325 ret = clk_enable(phy->clk);
326 if (ret) {
327 dev_err(&s3cdma->pdev->dev, "could not enable clock for channel %d, err %d\n",
328 phy->id, ret);
329 phy->serving = NULL;
330 return NULL;
331 }
332 }
333
334 return phy;
335}
336
337/*
338 * Mark the physical channel as free.
339 *
340 * This drops the link between the physical and virtual channel.
341 */
342static inline void s3c24xx_dma_put_phy(struct s3c24xx_dma_phy *phy)
343{
344 struct s3c24xx_dma_engine *s3cdma = phy->host;
345
346 if (s3cdma->sdata->has_clocks)
347 clk_disable(phy->clk);
348
349 phy->serving = NULL;
350}
351
352/*
353 * Stops the channel by writing the stop bit.
354 * This should not be used for an on-going transfer, but as a method of
355 * shutting down a channel (eg, when it's no longer used) or terminating a
356 * transfer.
357 */
358static void s3c24xx_dma_terminate_phy(struct s3c24xx_dma_phy *phy)
359{
360 writel(S3C24XX_DMASKTRIG_STOP, phy->base + S3C24XX_DMASKTRIG);
361}
362
363/*
364 * Virtual channel handling
365 */
366
367static inline
368struct s3c24xx_dma_chan *to_s3c24xx_dma_chan(struct dma_chan *chan)
369{
370 return container_of(chan, struct s3c24xx_dma_chan, vc.chan);
371}
372
373static u32 s3c24xx_dma_getbytes_chan(struct s3c24xx_dma_chan *s3cchan)
374{
375 struct s3c24xx_dma_phy *phy = s3cchan->phy;
376 struct s3c24xx_txd *txd = s3cchan->at;
377 u32 tc = readl(phy->base + S3C24XX_DSTAT) & S3C24XX_DSTAT_CURRTC_MASK;
378
379 return tc * txd->width;
380}
381
382static int s3c24xx_dma_set_runtime_config(struct dma_chan *chan,
383 struct dma_slave_config *config)
384{
385 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
386 unsigned long flags;
387 int ret = 0;
388
389 /* Reject definitely invalid configurations */
390 if (config->src_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES ||
391 config->dst_addr_width == DMA_SLAVE_BUSWIDTH_8_BYTES)
392 return -EINVAL;
393
394 spin_lock_irqsave(&s3cchan->vc.lock, flags);
395
396 if (!s3cchan->slave) {
397 ret = -EINVAL;
398 goto out;
399 }
400
401 s3cchan->cfg = *config;
402
403out:
404 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
405 return ret;
406}
407
408/*
409 * Transfer handling
410 */
411
412static inline
413struct s3c24xx_txd *to_s3c24xx_txd(struct dma_async_tx_descriptor *tx)
414{
415 return container_of(tx, struct s3c24xx_txd, vd.tx);
416}
417
418static struct s3c24xx_txd *s3c24xx_dma_get_txd(void)
419{
420 struct s3c24xx_txd *txd = kzalloc(sizeof(*txd), GFP_NOWAIT);
421
422 if (txd) {
423 INIT_LIST_HEAD(&txd->dsg_list);
424 txd->dcon = S3C24XX_DCON_INT | S3C24XX_DCON_NORELOAD;
425 }
426
427 return txd;
428}
429
430static void s3c24xx_dma_free_txd(struct s3c24xx_txd *txd)
431{
432 struct s3c24xx_sg *dsg, *_dsg;
433
434 list_for_each_entry_safe(dsg, _dsg, &txd->dsg_list, node) {
435 list_del(&dsg->node);
436 kfree(dsg);
437 }
438
439 kfree(txd);
440}
441
442static void s3c24xx_dma_start_next_sg(struct s3c24xx_dma_chan *s3cchan,
443 struct s3c24xx_txd *txd)
444{
445 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
446 struct s3c24xx_dma_phy *phy = s3cchan->phy;
447 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
448 struct s3c24xx_sg *dsg = list_entry(txd->at, struct s3c24xx_sg, node);
449 u32 dcon = txd->dcon;
450 u32 val;
451
452 /* transfer-size and -count from len and width */
453 switch (txd->width) {
454 case 1:
455 dcon |= S3C24XX_DCON_DSZ_BYTE | dsg->len;
456 break;
457 case 2:
458 dcon |= S3C24XX_DCON_DSZ_HALFWORD | (dsg->len / 2);
459 break;
460 case 4:
461 dcon |= S3C24XX_DCON_DSZ_WORD | (dsg->len / 4);
462 break;
463 }
464
465 if (s3cchan->slave) {
466 struct s3c24xx_dma_channel *cdata =
467 &pdata->channels[s3cchan->id];
468
469 if (s3cdma->sdata->has_reqsel) {
470 writel_relaxed((cdata->chansel << 1) |
471 S3C24XX_DMAREQSEL_HW,
472 phy->base + S3C24XX_DMAREQSEL);
473 } else {
474 int csel = cdata->chansel >> (phy->id *
475 S3C24XX_CHANSEL_WIDTH);
476
477 csel &= S3C24XX_CHANSEL_REQ_MASK;
478 dcon |= csel << S3C24XX_DCON_HWSRC_SHIFT;
479 dcon |= S3C24XX_DCON_HWTRIG;
480 }
481 } else {
482 if (s3cdma->sdata->has_reqsel)
483 writel_relaxed(0, phy->base + S3C24XX_DMAREQSEL);
484 }
485
486 writel_relaxed(dsg->src_addr, phy->base + S3C24XX_DISRC);
487 writel_relaxed(txd->disrcc, phy->base + S3C24XX_DISRCC);
488 writel_relaxed(dsg->dst_addr, phy->base + S3C24XX_DIDST);
489 writel_relaxed(txd->didstc, phy->base + S3C24XX_DIDSTC);
490 writel_relaxed(dcon, phy->base + S3C24XX_DCON);
491
492 val = readl_relaxed(phy->base + S3C24XX_DMASKTRIG);
493 val &= ~S3C24XX_DMASKTRIG_STOP;
494 val |= S3C24XX_DMASKTRIG_ON;
495
496 /* trigger the dma operation for memcpy transfers */
497 if (!s3cchan->slave)
498 val |= S3C24XX_DMASKTRIG_SWTRIG;
499
500 writel(val, phy->base + S3C24XX_DMASKTRIG);
501}
502
503/*
504 * Set the initial DMA register values and start first sg.
505 */
506static void s3c24xx_dma_start_next_txd(struct s3c24xx_dma_chan *s3cchan)
507{
508 struct s3c24xx_dma_phy *phy = s3cchan->phy;
509 struct virt_dma_desc *vd = vchan_next_desc(&s3cchan->vc);
510 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
511
512 list_del(&txd->vd.node);
513
514 s3cchan->at = txd;
515
516 /* Wait for channel inactive */
517 while (s3c24xx_dma_phy_busy(phy))
518 cpu_relax();
519
520 /* point to the first element of the sg list */
521 txd->at = txd->dsg_list.next;
522 s3c24xx_dma_start_next_sg(s3cchan, txd);
523}
524
525static void s3c24xx_dma_free_txd_list(struct s3c24xx_dma_engine *s3cdma,
526 struct s3c24xx_dma_chan *s3cchan)
527{
528 LIST_HEAD(head);
529
530 vchan_get_all_descriptors(&s3cchan->vc, &head);
531 vchan_dma_desc_free_list(&s3cchan->vc, &head);
532}
533
534/*
535 * Try to allocate a physical channel. When successful, assign it to
536 * this virtual channel, and initiate the next descriptor. The
537 * virtual channel lock must be held at this point.
538 */
539static void s3c24xx_dma_phy_alloc_and_start(struct s3c24xx_dma_chan *s3cchan)
540{
541 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
542 struct s3c24xx_dma_phy *phy;
543
544 phy = s3c24xx_dma_get_phy(s3cchan);
545 if (!phy) {
546 dev_dbg(&s3cdma->pdev->dev, "no physical channel available for xfer on %s\n",
547 s3cchan->name);
548 s3cchan->state = S3C24XX_DMA_CHAN_WAITING;
549 return;
550 }
551
552 dev_dbg(&s3cdma->pdev->dev, "allocated physical channel %d for xfer on %s\n",
553 phy->id, s3cchan->name);
554
555 s3cchan->phy = phy;
556 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
557
558 s3c24xx_dma_start_next_txd(s3cchan);
559}
560
561static void s3c24xx_dma_phy_reassign_start(struct s3c24xx_dma_phy *phy,
562 struct s3c24xx_dma_chan *s3cchan)
563{
564 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
565
566 dev_dbg(&s3cdma->pdev->dev, "reassigned physical channel %d for xfer on %s\n",
567 phy->id, s3cchan->name);
568
569 /*
570 * We do this without taking the lock; we're really only concerned
571 * about whether this pointer is NULL or not, and we're guaranteed
572 * that this will only be called when it _already_ is non-NULL.
573 */
574 phy->serving = s3cchan;
575 s3cchan->phy = phy;
576 s3cchan->state = S3C24XX_DMA_CHAN_RUNNING;
577 s3c24xx_dma_start_next_txd(s3cchan);
578}
579
580/*
581 * Free a physical DMA channel, potentially reallocating it to another
582 * virtual channel if we have any pending.
583 */
584static void s3c24xx_dma_phy_free(struct s3c24xx_dma_chan *s3cchan)
585{
586 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
587 struct s3c24xx_dma_chan *p, *next;
588
589retry:
590 next = NULL;
591
592 /* Find a waiting virtual channel for the next transfer. */
593 list_for_each_entry(p, &s3cdma->memcpy.channels, vc.chan.device_node)
594 if (p->state == S3C24XX_DMA_CHAN_WAITING) {
595 next = p;
596 break;
597 }
598
599 if (!next) {
600 list_for_each_entry(p, &s3cdma->slave.channels,
601 vc.chan.device_node)
602 if (p->state == S3C24XX_DMA_CHAN_WAITING &&
603 s3c24xx_dma_phy_valid(p, s3cchan->phy)) {
604 next = p;
605 break;
606 }
607 }
608
609 /* Ensure that the physical channel is stopped */
610 s3c24xx_dma_terminate_phy(s3cchan->phy);
611
612 if (next) {
613 bool success;
614
615 /*
616 * Eww. We know this isn't going to deadlock
617 * but lockdep probably doesn't.
618 */
619 spin_lock(&next->vc.lock);
620 /* Re-check the state now that we have the lock */
621 success = next->state == S3C24XX_DMA_CHAN_WAITING;
622 if (success)
623 s3c24xx_dma_phy_reassign_start(s3cchan->phy, next);
624 spin_unlock(&next->vc.lock);
625
626 /* If the state changed, try to find another channel */
627 if (!success)
628 goto retry;
629 } else {
630 /* No more jobs, so free up the physical channel */
631 s3c24xx_dma_put_phy(s3cchan->phy);
632 }
633
634 s3cchan->phy = NULL;
635 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
636}
637
638static void s3c24xx_dma_desc_free(struct virt_dma_desc *vd)
639{
640 struct s3c24xx_txd *txd = to_s3c24xx_txd(&vd->tx);
641 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(vd->tx.chan);
642
643 if (!s3cchan->slave)
644 dma_descriptor_unmap(&vd->tx);
645
646 s3c24xx_dma_free_txd(txd);
647}
648
649static irqreturn_t s3c24xx_dma_irq(int irq, void *data)
650{
651 struct s3c24xx_dma_phy *phy = data;
652 struct s3c24xx_dma_chan *s3cchan = phy->serving;
653 struct s3c24xx_txd *txd;
654
655 dev_dbg(&phy->host->pdev->dev, "interrupt on channel %d\n", phy->id);
656
657 /*
658 * Interrupts happen to notify the completion of a transfer and the
659 * channel should have moved into its stop state already on its own.
660 * Therefore interrupts on channels not bound to a virtual channel
661 * should never happen. Nevertheless send a terminate command to the
662 * channel if the unlikely case happens.
663 */
664 if (unlikely(!s3cchan)) {
665 dev_err(&phy->host->pdev->dev, "interrupt on unused channel %d\n",
666 phy->id);
667
668 s3c24xx_dma_terminate_phy(phy);
669
670 return IRQ_HANDLED;
671 }
672
673 spin_lock(&s3cchan->vc.lock);
674 txd = s3cchan->at;
675 if (txd) {
676 /* when more sg's are in this txd, start the next one */
677 if (!list_is_last(txd->at, &txd->dsg_list)) {
678 txd->at = txd->at->next;
679 if (txd->cyclic)
680 vchan_cyclic_callback(&txd->vd);
681 s3c24xx_dma_start_next_sg(s3cchan, txd);
682 } else if (!txd->cyclic) {
683 s3cchan->at = NULL;
684 vchan_cookie_complete(&txd->vd);
685
686 /*
687 * And start the next descriptor (if any),
688 * otherwise free this channel.
689 */
690 if (vchan_next_desc(&s3cchan->vc))
691 s3c24xx_dma_start_next_txd(s3cchan);
692 else
693 s3c24xx_dma_phy_free(s3cchan);
694 } else {
695 vchan_cyclic_callback(&txd->vd);
696
697 /* Cyclic: reset at beginning */
698 txd->at = txd->dsg_list.next;
699 s3c24xx_dma_start_next_sg(s3cchan, txd);
700 }
701 }
702 spin_unlock(&s3cchan->vc.lock);
703
704 return IRQ_HANDLED;
705}
706
707/*
708 * The DMA ENGINE API
709 */
710
711static int s3c24xx_dma_terminate_all(struct dma_chan *chan)
712{
713 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
714 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
715 unsigned long flags;
716 int ret = 0;
717
718 spin_lock_irqsave(&s3cchan->vc.lock, flags);
719
720 if (!s3cchan->phy && !s3cchan->at) {
721 dev_err(&s3cdma->pdev->dev, "trying to terminate already stopped channel %d\n",
722 s3cchan->id);
723 ret = -EINVAL;
724 goto unlock;
725 }
726
727 s3cchan->state = S3C24XX_DMA_CHAN_IDLE;
728
729 /* Mark physical channel as free */
730 if (s3cchan->phy)
731 s3c24xx_dma_phy_free(s3cchan);
732
733 /* Dequeue current job */
734 if (s3cchan->at) {
735 vchan_terminate_vdesc(&s3cchan->at->vd);
736 s3cchan->at = NULL;
737 }
738
739 /* Dequeue jobs not yet fired as well */
740 s3c24xx_dma_free_txd_list(s3cdma, s3cchan);
741unlock:
742 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
743
744 return ret;
745}
746
747static void s3c24xx_dma_synchronize(struct dma_chan *chan)
748{
749 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
750
751 vchan_synchronize(&s3cchan->vc);
752}
753
754static void s3c24xx_dma_free_chan_resources(struct dma_chan *chan)
755{
756 /* Ensure all queued descriptors are freed */
757 vchan_free_chan_resources(to_virt_chan(chan));
758}
759
760static enum dma_status s3c24xx_dma_tx_status(struct dma_chan *chan,
761 dma_cookie_t cookie, struct dma_tx_state *txstate)
762{
763 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
764 struct s3c24xx_txd *txd;
765 struct s3c24xx_sg *dsg;
766 struct virt_dma_desc *vd;
767 unsigned long flags;
768 enum dma_status ret;
769 size_t bytes = 0;
770
771 spin_lock_irqsave(&s3cchan->vc.lock, flags);
772 ret = dma_cookie_status(chan, cookie, txstate);
773
774 /*
775 * There's no point calculating the residue if there's
776 * no txstate to store the value.
777 */
778 if (ret == DMA_COMPLETE || !txstate) {
779 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
780 return ret;
781 }
782
783 vd = vchan_find_desc(&s3cchan->vc, cookie);
784 if (vd) {
785 /* On the issued list, so hasn't been processed yet */
786 txd = to_s3c24xx_txd(&vd->tx);
787
788 list_for_each_entry(dsg, &txd->dsg_list, node)
789 bytes += dsg->len;
790 } else {
791 /*
792 * Currently running, so sum over the pending sg's and
793 * the currently active one.
794 */
795 txd = s3cchan->at;
796
797 dsg = list_entry(txd->at, struct s3c24xx_sg, node);
798 list_for_each_entry_from(dsg, &txd->dsg_list, node)
799 bytes += dsg->len;
800
801 bytes += s3c24xx_dma_getbytes_chan(s3cchan);
802 }
803 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
804
805 /*
806 * This cookie not complete yet
807 * Get number of bytes left in the active transactions and queue
808 */
809 dma_set_residue(txstate, bytes);
810
811 /* Whether waiting or running, we're in progress */
812 return ret;
813}
814
815/*
816 * Initialize a descriptor to be used by memcpy submit
817 */
818static struct dma_async_tx_descriptor *s3c24xx_dma_prep_memcpy(
819 struct dma_chan *chan, dma_addr_t dest, dma_addr_t src,
820 size_t len, unsigned long flags)
821{
822 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
823 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
824 struct s3c24xx_txd *txd;
825 struct s3c24xx_sg *dsg;
826 int src_mod, dest_mod;
827
828 dev_dbg(&s3cdma->pdev->dev, "prepare memcpy of %zu bytes from %s\n",
829 len, s3cchan->name);
830
831 if ((len & S3C24XX_DCON_TC_MASK) != len) {
832 dev_err(&s3cdma->pdev->dev, "memcpy size %zu to large\n", len);
833 return NULL;
834 }
835
836 txd = s3c24xx_dma_get_txd();
837 if (!txd)
838 return NULL;
839
840 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
841 if (!dsg) {
842 s3c24xx_dma_free_txd(txd);
843 return NULL;
844 }
845 list_add_tail(&dsg->node, &txd->dsg_list);
846
847 dsg->src_addr = src;
848 dsg->dst_addr = dest;
849 dsg->len = len;
850
851 /*
852 * Determine a suitable transfer width.
853 * The DMA controller cannot fetch/store information which is not
854 * naturally aligned on the bus, i.e., a 4 byte fetch must start at
855 * an address divisible by 4 - more generally addr % width must be 0.
856 */
857 src_mod = src % 4;
858 dest_mod = dest % 4;
859 switch (len % 4) {
860 case 0:
861 txd->width = (src_mod == 0 && dest_mod == 0) ? 4 : 1;
862 break;
863 case 2:
864 txd->width = ((src_mod == 2 || src_mod == 0) &&
865 (dest_mod == 2 || dest_mod == 0)) ? 2 : 1;
866 break;
867 default:
868 txd->width = 1;
869 break;
870 }
871
872 txd->disrcc = S3C24XX_DISRCC_LOC_AHB | S3C24XX_DISRCC_INC_INCREMENT;
873 txd->didstc = S3C24XX_DIDSTC_LOC_AHB | S3C24XX_DIDSTC_INC_INCREMENT;
874 txd->dcon |= S3C24XX_DCON_DEMAND | S3C24XX_DCON_SYNC_HCLK |
875 S3C24XX_DCON_SERV_WHOLE;
876
877 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
878}
879
880static struct dma_async_tx_descriptor *s3c24xx_dma_prep_dma_cyclic(
881 struct dma_chan *chan, dma_addr_t addr, size_t size, size_t period,
882 enum dma_transfer_direction direction, unsigned long flags)
883{
884 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
885 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
886 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
887 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
888 struct s3c24xx_txd *txd;
889 struct s3c24xx_sg *dsg;
890 unsigned sg_len;
891 dma_addr_t slave_addr;
892 u32 hwcfg = 0;
893 int i;
894
895 dev_dbg(&s3cdma->pdev->dev,
896 "prepare cyclic transaction of %zu bytes with period %zu from %s\n",
897 size, period, s3cchan->name);
898
899 if (!is_slave_direction(direction)) {
900 dev_err(&s3cdma->pdev->dev,
901 "direction %d unsupported\n", direction);
902 return NULL;
903 }
904
905 txd = s3c24xx_dma_get_txd();
906 if (!txd)
907 return NULL;
908
909 txd->cyclic = 1;
910
911 if (cdata->handshake)
912 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
913
914 switch (cdata->bus) {
915 case S3C24XX_DMA_APB:
916 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
917 hwcfg |= S3C24XX_DISRCC_LOC_APB;
918 break;
919 case S3C24XX_DMA_AHB:
920 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
921 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
922 break;
923 }
924
925 /*
926 * Always assume our peripheral desintation is a fixed
927 * address in memory.
928 */
929 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
930
931 /*
932 * Individual dma operations are requested by the slave,
933 * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
934 */
935 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
936
937 if (direction == DMA_MEM_TO_DEV) {
938 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
939 S3C24XX_DISRCC_INC_INCREMENT;
940 txd->didstc = hwcfg;
941 slave_addr = s3cchan->cfg.dst_addr;
942 txd->width = s3cchan->cfg.dst_addr_width;
943 } else {
944 txd->disrcc = hwcfg;
945 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
946 S3C24XX_DIDSTC_INC_INCREMENT;
947 slave_addr = s3cchan->cfg.src_addr;
948 txd->width = s3cchan->cfg.src_addr_width;
949 }
950
951 sg_len = size / period;
952
953 for (i = 0; i < sg_len; i++) {
954 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
955 if (!dsg) {
956 s3c24xx_dma_free_txd(txd);
957 return NULL;
958 }
959 list_add_tail(&dsg->node, &txd->dsg_list);
960
961 dsg->len = period;
962 /* Check last period length */
963 if (i == sg_len - 1)
964 dsg->len = size - period * i;
965 if (direction == DMA_MEM_TO_DEV) {
966 dsg->src_addr = addr + period * i;
967 dsg->dst_addr = slave_addr;
968 } else { /* DMA_DEV_TO_MEM */
969 dsg->src_addr = slave_addr;
970 dsg->dst_addr = addr + period * i;
971 }
972 }
973
974 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
975}
976
977static struct dma_async_tx_descriptor *s3c24xx_dma_prep_slave_sg(
978 struct dma_chan *chan, struct scatterlist *sgl,
979 unsigned int sg_len, enum dma_transfer_direction direction,
980 unsigned long flags, void *context)
981{
982 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
983 struct s3c24xx_dma_engine *s3cdma = s3cchan->host;
984 const struct s3c24xx_dma_platdata *pdata = s3cdma->pdata;
985 struct s3c24xx_dma_channel *cdata = &pdata->channels[s3cchan->id];
986 struct s3c24xx_txd *txd;
987 struct s3c24xx_sg *dsg;
988 struct scatterlist *sg;
989 dma_addr_t slave_addr;
990 u32 hwcfg = 0;
991 int tmp;
992
993 dev_dbg(&s3cdma->pdev->dev, "prepare transaction of %d bytes from %s\n",
994 sg_dma_len(sgl), s3cchan->name);
995
996 txd = s3c24xx_dma_get_txd();
997 if (!txd)
998 return NULL;
999
1000 if (cdata->handshake)
1001 txd->dcon |= S3C24XX_DCON_HANDSHAKE;
1002
1003 switch (cdata->bus) {
1004 case S3C24XX_DMA_APB:
1005 txd->dcon |= S3C24XX_DCON_SYNC_PCLK;
1006 hwcfg |= S3C24XX_DISRCC_LOC_APB;
1007 break;
1008 case S3C24XX_DMA_AHB:
1009 txd->dcon |= S3C24XX_DCON_SYNC_HCLK;
1010 hwcfg |= S3C24XX_DISRCC_LOC_AHB;
1011 break;
1012 }
1013
1014 /*
1015 * Always assume our peripheral desintation is a fixed
1016 * address in memory.
1017 */
1018 hwcfg |= S3C24XX_DISRCC_INC_FIXED;
1019
1020 /*
1021 * Individual dma operations are requested by the slave,
1022 * so serve only single atomic operations (S3C24XX_DCON_SERV_SINGLE).
1023 */
1024 txd->dcon |= S3C24XX_DCON_SERV_SINGLE;
1025
1026 if (direction == DMA_MEM_TO_DEV) {
1027 txd->disrcc = S3C24XX_DISRCC_LOC_AHB |
1028 S3C24XX_DISRCC_INC_INCREMENT;
1029 txd->didstc = hwcfg;
1030 slave_addr = s3cchan->cfg.dst_addr;
1031 txd->width = s3cchan->cfg.dst_addr_width;
1032 } else if (direction == DMA_DEV_TO_MEM) {
1033 txd->disrcc = hwcfg;
1034 txd->didstc = S3C24XX_DIDSTC_LOC_AHB |
1035 S3C24XX_DIDSTC_INC_INCREMENT;
1036 slave_addr = s3cchan->cfg.src_addr;
1037 txd->width = s3cchan->cfg.src_addr_width;
1038 } else {
1039 s3c24xx_dma_free_txd(txd);
1040 dev_err(&s3cdma->pdev->dev,
1041 "direction %d unsupported\n", direction);
1042 return NULL;
1043 }
1044
1045 for_each_sg(sgl, sg, sg_len, tmp) {
1046 dsg = kzalloc(sizeof(*dsg), GFP_NOWAIT);
1047 if (!dsg) {
1048 s3c24xx_dma_free_txd(txd);
1049 return NULL;
1050 }
1051 list_add_tail(&dsg->node, &txd->dsg_list);
1052
1053 dsg->len = sg_dma_len(sg);
1054 if (direction == DMA_MEM_TO_DEV) {
1055 dsg->src_addr = sg_dma_address(sg);
1056 dsg->dst_addr = slave_addr;
1057 } else { /* DMA_DEV_TO_MEM */
1058 dsg->src_addr = slave_addr;
1059 dsg->dst_addr = sg_dma_address(sg);
1060 }
1061 }
1062
1063 return vchan_tx_prep(&s3cchan->vc, &txd->vd, flags);
1064}
1065
1066/*
1067 * Slave transactions callback to the slave device to allow
1068 * synchronization of slave DMA signals with the DMAC enable
1069 */
1070static void s3c24xx_dma_issue_pending(struct dma_chan *chan)
1071{
1072 struct s3c24xx_dma_chan *s3cchan = to_s3c24xx_dma_chan(chan);
1073 unsigned long flags;
1074
1075 spin_lock_irqsave(&s3cchan->vc.lock, flags);
1076 if (vchan_issue_pending(&s3cchan->vc)) {
1077 if (!s3cchan->phy && s3cchan->state != S3C24XX_DMA_CHAN_WAITING)
1078 s3c24xx_dma_phy_alloc_and_start(s3cchan);
1079 }
1080 spin_unlock_irqrestore(&s3cchan->vc.lock, flags);
1081}
1082
1083/*
1084 * Bringup and teardown
1085 */
1086
1087/*
1088 * Initialise the DMAC memcpy/slave channels.
1089 * Make a local wrapper to hold required data
1090 */
1091static int s3c24xx_dma_init_virtual_channels(struct s3c24xx_dma_engine *s3cdma,
1092 struct dma_device *dmadev, unsigned int channels, bool slave)
1093{
1094 struct s3c24xx_dma_chan *chan;
1095 int i;
1096
1097 INIT_LIST_HEAD(&dmadev->channels);
1098
1099 /*
1100 * Register as many many memcpy as we have physical channels,
1101 * we won't always be able to use all but the code will have
1102 * to cope with that situation.
1103 */
1104 for (i = 0; i < channels; i++) {
1105 chan = devm_kzalloc(dmadev->dev, sizeof(*chan), GFP_KERNEL);
1106 if (!chan)
1107 return -ENOMEM;
1108
1109 chan->id = i;
1110 chan->host = s3cdma;
1111 chan->state = S3C24XX_DMA_CHAN_IDLE;
1112
1113 if (slave) {
1114 chan->slave = true;
1115 chan->name = kasprintf(GFP_KERNEL, "slave%d", i);
1116 if (!chan->name)
1117 return -ENOMEM;
1118 } else {
1119 chan->name = kasprintf(GFP_KERNEL, "memcpy%d", i);
1120 if (!chan->name)
1121 return -ENOMEM;
1122 }
1123 dev_dbg(dmadev->dev,
1124 "initialize virtual channel \"%s\"\n",
1125 chan->name);
1126
1127 chan->vc.desc_free = s3c24xx_dma_desc_free;
1128 vchan_init(&chan->vc, dmadev);
1129 }
1130 dev_info(dmadev->dev, "initialized %d virtual %s channels\n",
1131 i, slave ? "slave" : "memcpy");
1132 return i;
1133}
1134
1135static void s3c24xx_dma_free_virtual_channels(struct dma_device *dmadev)
1136{
1137 struct s3c24xx_dma_chan *chan = NULL;
1138 struct s3c24xx_dma_chan *next;
1139
1140 list_for_each_entry_safe(chan,
1141 next, &dmadev->channels, vc.chan.device_node) {
1142 list_del(&chan->vc.chan.device_node);
1143 tasklet_kill(&chan->vc.task);
1144 }
1145}
1146
1147/* s3c2410, s3c2440 and s3c2442 have a 0x40 stride without separate clocks */
1148static struct soc_data soc_s3c2410 = {
1149 .stride = 0x40,
1150 .has_reqsel = false,
1151 .has_clocks = false,
1152};
1153
1154/* s3c2412 and s3c2413 have a 0x40 stride and dmareqsel mechanism */
1155static struct soc_data soc_s3c2412 = {
1156 .stride = 0x40,
1157 .has_reqsel = true,
1158 .has_clocks = true,
1159};
1160
1161/* s3c2443 and following have a 0x100 stride and dmareqsel mechanism */
1162static struct soc_data soc_s3c2443 = {
1163 .stride = 0x100,
1164 .has_reqsel = true,
1165 .has_clocks = true,
1166};
1167
1168static const struct platform_device_id s3c24xx_dma_driver_ids[] = {
1169 {
1170 .name = "s3c2410-dma",
1171 .driver_data = (kernel_ulong_t)&soc_s3c2410,
1172 }, {
1173 .name = "s3c2412-dma",
1174 .driver_data = (kernel_ulong_t)&soc_s3c2412,
1175 }, {
1176 .name = "s3c2443-dma",
1177 .driver_data = (kernel_ulong_t)&soc_s3c2443,
1178 },
1179 { },
1180};
1181
1182static struct soc_data *s3c24xx_dma_get_soc_data(struct platform_device *pdev)
1183{
1184 return (struct soc_data *)
1185 platform_get_device_id(pdev)->driver_data;
1186}
1187
1188static int s3c24xx_dma_probe(struct platform_device *pdev)
1189{
1190 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1191 struct s3c24xx_dma_engine *s3cdma;
1192 struct soc_data *sdata;
1193 struct resource *res;
1194 int ret;
1195 int i;
1196
1197 if (!pdata) {
1198 dev_err(&pdev->dev, "platform data missing\n");
1199 return -ENODEV;
1200 }
1201
1202 /* Basic sanity check */
1203 if (pdata->num_phy_channels > MAX_DMA_CHANNELS) {
1204 dev_err(&pdev->dev, "to many dma channels %d, max %d\n",
1205 pdata->num_phy_channels, MAX_DMA_CHANNELS);
1206 return -EINVAL;
1207 }
1208
1209 sdata = s3c24xx_dma_get_soc_data(pdev);
1210 if (!sdata)
1211 return -EINVAL;
1212
1213 s3cdma = devm_kzalloc(&pdev->dev, sizeof(*s3cdma), GFP_KERNEL);
1214 if (!s3cdma)
1215 return -ENOMEM;
1216
1217 s3cdma->pdev = pdev;
1218 s3cdma->pdata = pdata;
1219 s3cdma->sdata = sdata;
1220
1221 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1222 s3cdma->base = devm_ioremap_resource(&pdev->dev, res);
1223 if (IS_ERR(s3cdma->base))
1224 return PTR_ERR(s3cdma->base);
1225
1226 s3cdma->phy_chans = devm_kzalloc(&pdev->dev,
1227 sizeof(struct s3c24xx_dma_phy) *
1228 pdata->num_phy_channels,
1229 GFP_KERNEL);
1230 if (!s3cdma->phy_chans)
1231 return -ENOMEM;
1232
1233 /* acquire irqs and clocks for all physical channels */
1234 for (i = 0; i < pdata->num_phy_channels; i++) {
1235 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1236 char clk_name[6];
1237
1238 phy->id = i;
1239 phy->base = s3cdma->base + (i * sdata->stride);
1240 phy->host = s3cdma;
1241
1242 phy->irq = platform_get_irq(pdev, i);
1243 if (phy->irq < 0) {
1244 dev_err(&pdev->dev, "failed to get irq %d, err %d\n",
1245 i, phy->irq);
1246 continue;
1247 }
1248
1249 ret = devm_request_irq(&pdev->dev, phy->irq, s3c24xx_dma_irq,
1250 0, pdev->name, phy);
1251 if (ret) {
1252 dev_err(&pdev->dev, "Unable to request irq for channel %d, error %d\n",
1253 i, ret);
1254 continue;
1255 }
1256
1257 if (sdata->has_clocks) {
1258 sprintf(clk_name, "dma.%d", i);
1259 phy->clk = devm_clk_get(&pdev->dev, clk_name);
1260 if (IS_ERR(phy->clk) && sdata->has_clocks) {
1261 dev_err(&pdev->dev, "unable to acquire clock for channel %d, error %lu\n",
1262 i, PTR_ERR(phy->clk));
1263 continue;
1264 }
1265
1266 ret = clk_prepare(phy->clk);
1267 if (ret) {
1268 dev_err(&pdev->dev, "clock for phy %d failed, error %d\n",
1269 i, ret);
1270 continue;
1271 }
1272 }
1273
1274 spin_lock_init(&phy->lock);
1275 phy->valid = true;
1276
1277 dev_dbg(&pdev->dev, "physical channel %d is %s\n",
1278 i, s3c24xx_dma_phy_busy(phy) ? "BUSY" : "FREE");
1279 }
1280
1281 /* Initialize memcpy engine */
1282 dma_cap_set(DMA_MEMCPY, s3cdma->memcpy.cap_mask);
1283 dma_cap_set(DMA_PRIVATE, s3cdma->memcpy.cap_mask);
1284 s3cdma->memcpy.dev = &pdev->dev;
1285 s3cdma->memcpy.device_free_chan_resources =
1286 s3c24xx_dma_free_chan_resources;
1287 s3cdma->memcpy.device_prep_dma_memcpy = s3c24xx_dma_prep_memcpy;
1288 s3cdma->memcpy.device_tx_status = s3c24xx_dma_tx_status;
1289 s3cdma->memcpy.device_issue_pending = s3c24xx_dma_issue_pending;
1290 s3cdma->memcpy.device_config = s3c24xx_dma_set_runtime_config;
1291 s3cdma->memcpy.device_terminate_all = s3c24xx_dma_terminate_all;
1292 s3cdma->memcpy.device_synchronize = s3c24xx_dma_synchronize;
1293
1294 /* Initialize slave engine for SoC internal dedicated peripherals */
1295 dma_cap_set(DMA_SLAVE, s3cdma->slave.cap_mask);
1296 dma_cap_set(DMA_CYCLIC, s3cdma->slave.cap_mask);
1297 dma_cap_set(DMA_PRIVATE, s3cdma->slave.cap_mask);
1298 s3cdma->slave.dev = &pdev->dev;
1299 s3cdma->slave.device_free_chan_resources =
1300 s3c24xx_dma_free_chan_resources;
1301 s3cdma->slave.device_tx_status = s3c24xx_dma_tx_status;
1302 s3cdma->slave.device_issue_pending = s3c24xx_dma_issue_pending;
1303 s3cdma->slave.device_prep_slave_sg = s3c24xx_dma_prep_slave_sg;
1304 s3cdma->slave.device_prep_dma_cyclic = s3c24xx_dma_prep_dma_cyclic;
1305 s3cdma->slave.device_config = s3c24xx_dma_set_runtime_config;
1306 s3cdma->slave.device_terminate_all = s3c24xx_dma_terminate_all;
1307 s3cdma->slave.device_synchronize = s3c24xx_dma_synchronize;
1308 s3cdma->slave.filter.map = pdata->slave_map;
1309 s3cdma->slave.filter.mapcnt = pdata->slavecnt;
1310 s3cdma->slave.filter.fn = s3c24xx_dma_filter;
1311
1312 /* Register as many memcpy channels as there are physical channels */
1313 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->memcpy,
1314 pdata->num_phy_channels, false);
1315 if (ret <= 0) {
1316 dev_warn(&pdev->dev,
1317 "%s failed to enumerate memcpy channels - %d\n",
1318 __func__, ret);
1319 goto err_memcpy;
1320 }
1321
1322 /* Register slave channels */
1323 ret = s3c24xx_dma_init_virtual_channels(s3cdma, &s3cdma->slave,
1324 pdata->num_channels, true);
1325 if (ret <= 0) {
1326 dev_warn(&pdev->dev,
1327 "%s failed to enumerate slave channels - %d\n",
1328 __func__, ret);
1329 goto err_slave;
1330 }
1331
1332 ret = dma_async_device_register(&s3cdma->memcpy);
1333 if (ret) {
1334 dev_warn(&pdev->dev,
1335 "%s failed to register memcpy as an async device - %d\n",
1336 __func__, ret);
1337 goto err_memcpy_reg;
1338 }
1339
1340 ret = dma_async_device_register(&s3cdma->slave);
1341 if (ret) {
1342 dev_warn(&pdev->dev,
1343 "%s failed to register slave as an async device - %d\n",
1344 __func__, ret);
1345 goto err_slave_reg;
1346 }
1347
1348 platform_set_drvdata(pdev, s3cdma);
1349 dev_info(&pdev->dev, "Loaded dma driver with %d physical channels\n",
1350 pdata->num_phy_channels);
1351
1352 return 0;
1353
1354err_slave_reg:
1355 dma_async_device_unregister(&s3cdma->memcpy);
1356err_memcpy_reg:
1357 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1358err_slave:
1359 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1360err_memcpy:
1361 if (sdata->has_clocks)
1362 for (i = 0; i < pdata->num_phy_channels; i++) {
1363 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1364 if (phy->valid)
1365 clk_unprepare(phy->clk);
1366 }
1367
1368 return ret;
1369}
1370
1371static void s3c24xx_dma_free_irq(struct platform_device *pdev,
1372 struct s3c24xx_dma_engine *s3cdma)
1373{
1374 int i;
1375
1376 for (i = 0; i < s3cdma->pdata->num_phy_channels; i++) {
1377 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1378
1379 devm_free_irq(&pdev->dev, phy->irq, phy);
1380 }
1381}
1382
1383static int s3c24xx_dma_remove(struct platform_device *pdev)
1384{
1385 const struct s3c24xx_dma_platdata *pdata = dev_get_platdata(&pdev->dev);
1386 struct s3c24xx_dma_engine *s3cdma = platform_get_drvdata(pdev);
1387 struct soc_data *sdata = s3c24xx_dma_get_soc_data(pdev);
1388 int i;
1389
1390 dma_async_device_unregister(&s3cdma->slave);
1391 dma_async_device_unregister(&s3cdma->memcpy);
1392
1393 s3c24xx_dma_free_irq(pdev, s3cdma);
1394
1395 s3c24xx_dma_free_virtual_channels(&s3cdma->slave);
1396 s3c24xx_dma_free_virtual_channels(&s3cdma->memcpy);
1397
1398 if (sdata->has_clocks)
1399 for (i = 0; i < pdata->num_phy_channels; i++) {
1400 struct s3c24xx_dma_phy *phy = &s3cdma->phy_chans[i];
1401 if (phy->valid)
1402 clk_unprepare(phy->clk);
1403 }
1404
1405 return 0;
1406}
1407
1408static struct platform_driver s3c24xx_dma_driver = {
1409 .driver = {
1410 .name = "s3c24xx-dma",
1411 },
1412 .id_table = s3c24xx_dma_driver_ids,
1413 .probe = s3c24xx_dma_probe,
1414 .remove = s3c24xx_dma_remove,
1415};
1416
1417module_platform_driver(s3c24xx_dma_driver);
1418
1419bool s3c24xx_dma_filter(struct dma_chan *chan, void *param)
1420{
1421 struct s3c24xx_dma_chan *s3cchan;
1422
1423 if (chan->device->dev->driver != &s3c24xx_dma_driver.driver)
1424 return false;
1425
1426 s3cchan = to_s3c24xx_dma_chan(chan);
1427
1428 return s3cchan->id == (uintptr_t)param;
1429}
1430EXPORT_SYMBOL(s3c24xx_dma_filter);
1431
1432MODULE_DESCRIPTION("S3C24XX DMA Driver");
1433MODULE_AUTHOR("Heiko Stuebner");
1434MODULE_LICENSE("GPL v2");