Loading...
1/*
2 * Copyright (C) 2014 Emilio López
3 * Emilio López <emilio@elopez.com.ar>
4 *
5 * This program is free software; you can redistribute it and/or modify
6 * it under the terms of the GNU General Public License as published by
7 * the Free Software Foundation; either version 2 of the License, or
8 * (at your option) any later version.
9 */
10
11#include <linux/bitmap.h>
12#include <linux/bitops.h>
13#include <linux/clk.h>
14#include <linux/dmaengine.h>
15#include <linux/dmapool.h>
16#include <linux/interrupt.h>
17#include <linux/module.h>
18#include <linux/of_dma.h>
19#include <linux/platform_device.h>
20#include <linux/slab.h>
21#include <linux/spinlock.h>
22
23#include "virt-dma.h"
24
25/** Common macros to normal and dedicated DMA registers **/
26
27#define SUN4I_DMA_CFG_LOADING BIT(31)
28#define SUN4I_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 25)
29#define SUN4I_DMA_CFG_DST_BURST_LENGTH(len) ((len) << 23)
30#define SUN4I_DMA_CFG_DST_ADDR_MODE(mode) ((mode) << 21)
31#define SUN4I_DMA_CFG_DST_DRQ_TYPE(type) ((type) << 16)
32#define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 9)
33#define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len) ((len) << 7)
34#define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5)
35#define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type)
36
37/** Normal DMA register values **/
38
39/* Normal DMA source/destination data request type values */
40#define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16
41#define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1)
42
43/** Normal DMA register layout **/
44
45/* Dedicated DMA source/destination address mode values */
46#define SUN4I_NDMA_ADDR_MODE_LINEAR 0
47#define SUN4I_NDMA_ADDR_MODE_IO 1
48
49/* Normal DMA configuration register layout */
50#define SUN4I_NDMA_CFG_CONT_MODE BIT(30)
51#define SUN4I_NDMA_CFG_WAIT_STATE(n) ((n) << 27)
52#define SUN4I_NDMA_CFG_DST_NON_SECURE BIT(22)
53#define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
54#define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6)
55
56/** Dedicated DMA register values **/
57
58/* Dedicated DMA source/destination address mode values */
59#define SUN4I_DDMA_ADDR_MODE_LINEAR 0
60#define SUN4I_DDMA_ADDR_MODE_IO 1
61#define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE 2
62#define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE 3
63
64/* Dedicated DMA source/destination data request type values */
65#define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1
66#define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1)
67
68/** Dedicated DMA register layout **/
69
70/* Dedicated DMA configuration register layout */
71#define SUN4I_DDMA_CFG_BUSY BIT(30)
72#define SUN4I_DDMA_CFG_CONT_MODE BIT(29)
73#define SUN4I_DDMA_CFG_DST_NON_SECURE BIT(28)
74#define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
75#define SUN4I_DDMA_CFG_SRC_NON_SECURE BIT(12)
76
77/* Dedicated DMA parameter register layout */
78#define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n) (((n) - 1) << 24)
79#define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n) (((n) - 1) << 16)
80#define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n) (((n) - 1) << 8)
81#define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n) (((n) - 1) << 0)
82
83/** DMA register offsets **/
84
85/* General register offsets */
86#define SUN4I_DMA_IRQ_ENABLE_REG 0x0
87#define SUN4I_DMA_IRQ_PENDING_STATUS_REG 0x4
88
89/* Normal DMA register offsets */
90#define SUN4I_NDMA_CHANNEL_REG_BASE(n) (0x100 + (n) * 0x20)
91#define SUN4I_NDMA_CFG_REG 0x0
92#define SUN4I_NDMA_SRC_ADDR_REG 0x4
93#define SUN4I_NDMA_DST_ADDR_REG 0x8
94#define SUN4I_NDMA_BYTE_COUNT_REG 0xC
95
96/* Dedicated DMA register offsets */
97#define SUN4I_DDMA_CHANNEL_REG_BASE(n) (0x300 + (n) * 0x20)
98#define SUN4I_DDMA_CFG_REG 0x0
99#define SUN4I_DDMA_SRC_ADDR_REG 0x4
100#define SUN4I_DDMA_DST_ADDR_REG 0x8
101#define SUN4I_DDMA_BYTE_COUNT_REG 0xC
102#define SUN4I_DDMA_PARA_REG 0x18
103
104/** DMA Driver **/
105
106/*
107 * Normal DMA has 8 channels, and Dedicated DMA has another 8, so
108 * that's 16 channels. As for endpoints, there's 29 and 21
109 * respectively. Given that the Normal DMA endpoints (other than
110 * SDRAM) can be used as tx/rx, we need 78 vchans in total
111 */
112#define SUN4I_NDMA_NR_MAX_CHANNELS 8
113#define SUN4I_DDMA_NR_MAX_CHANNELS 8
114#define SUN4I_DMA_NR_MAX_CHANNELS \
115 (SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
116#define SUN4I_NDMA_NR_MAX_VCHANS (29 * 2 - 1)
117#define SUN4I_DDMA_NR_MAX_VCHANS 21
118#define SUN4I_DMA_NR_MAX_VCHANS \
119 (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
120
121/* This set of SUN4I_DDMA timing parameters were found experimentally while
122 * working with the SPI driver and seem to make it behave correctly */
123#define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
124 (SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) | \
125 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) | \
126 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) | \
127 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
128
129struct sun4i_dma_pchan {
130 /* Register base of channel */
131 void __iomem *base;
132 /* vchan currently being serviced */
133 struct sun4i_dma_vchan *vchan;
134 /* Is this a dedicated pchan? */
135 int is_dedicated;
136};
137
138struct sun4i_dma_vchan {
139 struct virt_dma_chan vc;
140 struct dma_slave_config cfg;
141 struct sun4i_dma_pchan *pchan;
142 struct sun4i_dma_promise *processing;
143 struct sun4i_dma_contract *contract;
144 u8 endpoint;
145 int is_dedicated;
146};
147
148struct sun4i_dma_promise {
149 u32 cfg;
150 u32 para;
151 dma_addr_t src;
152 dma_addr_t dst;
153 size_t len;
154 struct list_head list;
155};
156
157/* A contract is a set of promises */
158struct sun4i_dma_contract {
159 struct virt_dma_desc vd;
160 struct list_head demands;
161 struct list_head completed_demands;
162 int is_cyclic;
163};
164
165struct sun4i_dma_dev {
166 DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS);
167 struct dma_device slave;
168 struct sun4i_dma_pchan *pchans;
169 struct sun4i_dma_vchan *vchans;
170 void __iomem *base;
171 struct clk *clk;
172 int irq;
173 spinlock_t lock;
174};
175
176static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
177{
178 return container_of(dev, struct sun4i_dma_dev, slave);
179}
180
181static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
182{
183 return container_of(chan, struct sun4i_dma_vchan, vc.chan);
184}
185
186static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
187{
188 return container_of(vd, struct sun4i_dma_contract, vd);
189}
190
191static struct device *chan2dev(struct dma_chan *chan)
192{
193 return &chan->dev->device;
194}
195
196static int convert_burst(u32 maxburst)
197{
198 if (maxburst > 8)
199 return -EINVAL;
200
201 /* 1 -> 0, 4 -> 1, 8 -> 2 */
202 return (maxburst >> 2);
203}
204
205static int convert_buswidth(enum dma_slave_buswidth addr_width)
206{
207 if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
208 return -EINVAL;
209
210 /* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */
211 return (addr_width >> 1);
212}
213
214static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
215{
216 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
217
218 vchan_free_chan_resources(&vchan->vc);
219}
220
221static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
222 struct sun4i_dma_vchan *vchan)
223{
224 struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
225 unsigned long flags;
226 int i, max;
227
228 /*
229 * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and
230 * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones
231 */
232 if (vchan->is_dedicated) {
233 i = SUN4I_NDMA_NR_MAX_CHANNELS;
234 max = SUN4I_DMA_NR_MAX_CHANNELS;
235 } else {
236 i = 0;
237 max = SUN4I_NDMA_NR_MAX_CHANNELS;
238 }
239
240 spin_lock_irqsave(&priv->lock, flags);
241 for_each_clear_bit_from(i, &priv->pchans_used, max) {
242 pchan = &pchans[i];
243 pchan->vchan = vchan;
244 set_bit(i, priv->pchans_used);
245 break;
246 }
247 spin_unlock_irqrestore(&priv->lock, flags);
248
249 return pchan;
250}
251
252static void release_pchan(struct sun4i_dma_dev *priv,
253 struct sun4i_dma_pchan *pchan)
254{
255 unsigned long flags;
256 int nr = pchan - priv->pchans;
257
258 spin_lock_irqsave(&priv->lock, flags);
259
260 pchan->vchan = NULL;
261 clear_bit(nr, priv->pchans_used);
262
263 spin_unlock_irqrestore(&priv->lock, flags);
264}
265
266static void configure_pchan(struct sun4i_dma_pchan *pchan,
267 struct sun4i_dma_promise *d)
268{
269 /*
270 * Configure addresses and misc parameters depending on type
271 * SUN4I_DDMA has an extra field with timing parameters
272 */
273 if (pchan->is_dedicated) {
274 writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
275 writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
276 writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
277 writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
278 writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
279 } else {
280 writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
281 writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
282 writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
283 writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
284 }
285}
286
287static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
288 struct sun4i_dma_pchan *pchan,
289 int half, int end)
290{
291 u32 reg;
292 int pchan_number = pchan - priv->pchans;
293 unsigned long flags;
294
295 spin_lock_irqsave(&priv->lock, flags);
296
297 reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
298
299 if (half)
300 reg |= BIT(pchan_number * 2);
301 else
302 reg &= ~BIT(pchan_number * 2);
303
304 if (end)
305 reg |= BIT(pchan_number * 2 + 1);
306 else
307 reg &= ~BIT(pchan_number * 2 + 1);
308
309 writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
310
311 spin_unlock_irqrestore(&priv->lock, flags);
312}
313
314/**
315 * Execute pending operations on a vchan
316 *
317 * When given a vchan, this function will try to acquire a suitable
318 * pchan and, if successful, will configure it to fulfill a promise
319 * from the next pending contract.
320 *
321 * This function must be called with &vchan->vc.lock held.
322 */
323static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
324 struct sun4i_dma_vchan *vchan)
325{
326 struct sun4i_dma_promise *promise = NULL;
327 struct sun4i_dma_contract *contract = NULL;
328 struct sun4i_dma_pchan *pchan;
329 struct virt_dma_desc *vd;
330 int ret;
331
332 lockdep_assert_held(&vchan->vc.lock);
333
334 /* We need a pchan to do anything, so secure one if available */
335 pchan = find_and_use_pchan(priv, vchan);
336 if (!pchan)
337 return -EBUSY;
338
339 /*
340 * Channel endpoints must not be repeated, so if this vchan
341 * has already submitted some work, we can't do anything else
342 */
343 if (vchan->processing) {
344 dev_dbg(chan2dev(&vchan->vc.chan),
345 "processing something to this endpoint already\n");
346 ret = -EBUSY;
347 goto release_pchan;
348 }
349
350 do {
351 /* Figure out which contract we're working with today */
352 vd = vchan_next_desc(&vchan->vc);
353 if (!vd) {
354 dev_dbg(chan2dev(&vchan->vc.chan),
355 "No pending contract found");
356 ret = 0;
357 goto release_pchan;
358 }
359
360 contract = to_sun4i_dma_contract(vd);
361 if (list_empty(&contract->demands)) {
362 /* The contract has been completed so mark it as such */
363 list_del(&contract->vd.node);
364 vchan_cookie_complete(&contract->vd);
365 dev_dbg(chan2dev(&vchan->vc.chan),
366 "Empty contract found and marked complete");
367 }
368 } while (list_empty(&contract->demands));
369
370 /* Now find out what we need to do */
371 promise = list_first_entry(&contract->demands,
372 struct sun4i_dma_promise, list);
373 vchan->processing = promise;
374
375 /* ... and make it reality */
376 if (promise) {
377 vchan->contract = contract;
378 vchan->pchan = pchan;
379 set_pchan_interrupt(priv, pchan, contract->is_cyclic, 1);
380 configure_pchan(pchan, promise);
381 }
382
383 return 0;
384
385release_pchan:
386 release_pchan(priv, pchan);
387 return ret;
388}
389
390static int sanitize_config(struct dma_slave_config *sconfig,
391 enum dma_transfer_direction direction)
392{
393 switch (direction) {
394 case DMA_MEM_TO_DEV:
395 if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
396 !sconfig->dst_maxburst)
397 return -EINVAL;
398
399 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
400 sconfig->src_addr_width = sconfig->dst_addr_width;
401
402 if (!sconfig->src_maxburst)
403 sconfig->src_maxburst = sconfig->dst_maxburst;
404
405 break;
406
407 case DMA_DEV_TO_MEM:
408 if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
409 !sconfig->src_maxburst)
410 return -EINVAL;
411
412 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
413 sconfig->dst_addr_width = sconfig->src_addr_width;
414
415 if (!sconfig->dst_maxburst)
416 sconfig->dst_maxburst = sconfig->src_maxburst;
417
418 break;
419 default:
420 return 0;
421 }
422
423 return 0;
424}
425
426/**
427 * Generate a promise, to be used in a normal DMA contract.
428 *
429 * A NDMA promise contains all the information required to program the
430 * normal part of the DMA Engine and get data copied. A non-executed
431 * promise will live in the demands list on a contract. Once it has been
432 * completed, it will be moved to the completed demands list for later freeing.
433 * All linked promises will be freed when the corresponding contract is freed
434 */
435static struct sun4i_dma_promise *
436generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
437 size_t len, struct dma_slave_config *sconfig,
438 enum dma_transfer_direction direction)
439{
440 struct sun4i_dma_promise *promise;
441 int ret;
442
443 ret = sanitize_config(sconfig, direction);
444 if (ret)
445 return NULL;
446
447 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
448 if (!promise)
449 return NULL;
450
451 promise->src = src;
452 promise->dst = dest;
453 promise->len = len;
454 promise->cfg = SUN4I_DMA_CFG_LOADING |
455 SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
456
457 dev_dbg(chan2dev(chan),
458 "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
459 sconfig->src_maxburst, sconfig->dst_maxburst,
460 sconfig->src_addr_width, sconfig->dst_addr_width);
461
462 /* Source burst */
463 ret = convert_burst(sconfig->src_maxburst);
464 if (IS_ERR_VALUE(ret))
465 goto fail;
466 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
467
468 /* Destination burst */
469 ret = convert_burst(sconfig->dst_maxburst);
470 if (IS_ERR_VALUE(ret))
471 goto fail;
472 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
473
474 /* Source bus width */
475 ret = convert_buswidth(sconfig->src_addr_width);
476 if (IS_ERR_VALUE(ret))
477 goto fail;
478 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
479
480 /* Destination bus width */
481 ret = convert_buswidth(sconfig->dst_addr_width);
482 if (IS_ERR_VALUE(ret))
483 goto fail;
484 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
485
486 return promise;
487
488fail:
489 kfree(promise);
490 return NULL;
491}
492
493/**
494 * Generate a promise, to be used in a dedicated DMA contract.
495 *
496 * A DDMA promise contains all the information required to program the
497 * Dedicated part of the DMA Engine and get data copied. A non-executed
498 * promise will live in the demands list on a contract. Once it has been
499 * completed, it will be moved to the completed demands list for later freeing.
500 * All linked promises will be freed when the corresponding contract is freed
501 */
502static struct sun4i_dma_promise *
503generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
504 size_t len, struct dma_slave_config *sconfig)
505{
506 struct sun4i_dma_promise *promise;
507 int ret;
508
509 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
510 if (!promise)
511 return NULL;
512
513 promise->src = src;
514 promise->dst = dest;
515 promise->len = len;
516 promise->cfg = SUN4I_DMA_CFG_LOADING |
517 SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
518
519 /* Source burst */
520 ret = convert_burst(sconfig->src_maxburst);
521 if (IS_ERR_VALUE(ret))
522 goto fail;
523 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
524
525 /* Destination burst */
526 ret = convert_burst(sconfig->dst_maxburst);
527 if (IS_ERR_VALUE(ret))
528 goto fail;
529 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
530
531 /* Source bus width */
532 ret = convert_buswidth(sconfig->src_addr_width);
533 if (IS_ERR_VALUE(ret))
534 goto fail;
535 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
536
537 /* Destination bus width */
538 ret = convert_buswidth(sconfig->dst_addr_width);
539 if (IS_ERR_VALUE(ret))
540 goto fail;
541 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
542
543 return promise;
544
545fail:
546 kfree(promise);
547 return NULL;
548}
549
550/**
551 * Generate a contract
552 *
553 * Contracts function as DMA descriptors. As our hardware does not support
554 * linked lists, we need to implement SG via software. We use a contract
555 * to hold all the pieces of the request and process them serially one
556 * after another. Each piece is represented as a promise.
557 */
558static struct sun4i_dma_contract *generate_dma_contract(void)
559{
560 struct sun4i_dma_contract *contract;
561
562 contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
563 if (!contract)
564 return NULL;
565
566 INIT_LIST_HEAD(&contract->demands);
567 INIT_LIST_HEAD(&contract->completed_demands);
568
569 return contract;
570}
571
572/**
573 * Get next promise on a cyclic transfer
574 *
575 * Cyclic contracts contain a series of promises which are executed on a
576 * loop. This function returns the next promise from a cyclic contract,
577 * so it can be programmed into the hardware.
578 */
579static struct sun4i_dma_promise *
580get_next_cyclic_promise(struct sun4i_dma_contract *contract)
581{
582 struct sun4i_dma_promise *promise;
583
584 promise = list_first_entry_or_null(&contract->demands,
585 struct sun4i_dma_promise, list);
586 if (!promise) {
587 list_splice_init(&contract->completed_demands,
588 &contract->demands);
589 promise = list_first_entry(&contract->demands,
590 struct sun4i_dma_promise, list);
591 }
592
593 return promise;
594}
595
596/**
597 * Free a contract and all its associated promises
598 */
599static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
600{
601 struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
602 struct sun4i_dma_promise *promise, *tmp;
603
604 /* Free all the demands and completed demands */
605 list_for_each_entry_safe(promise, tmp, &contract->demands, list)
606 kfree(promise);
607
608 list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
609 kfree(promise);
610
611 kfree(contract);
612}
613
614static struct dma_async_tx_descriptor *
615sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
616 dma_addr_t src, size_t len, unsigned long flags)
617{
618 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
619 struct dma_slave_config *sconfig = &vchan->cfg;
620 struct sun4i_dma_promise *promise;
621 struct sun4i_dma_contract *contract;
622
623 contract = generate_dma_contract();
624 if (!contract)
625 return NULL;
626
627 /*
628 * We can only do the copy to bus aligned addresses, so
629 * choose the best one so we get decent performance. We also
630 * maximize the burst size for this same reason.
631 */
632 sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
633 sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
634 sconfig->src_maxburst = 8;
635 sconfig->dst_maxburst = 8;
636
637 if (vchan->is_dedicated)
638 promise = generate_ddma_promise(chan, src, dest, len, sconfig);
639 else
640 promise = generate_ndma_promise(chan, src, dest, len, sconfig,
641 DMA_MEM_TO_MEM);
642
643 if (!promise) {
644 kfree(contract);
645 return NULL;
646 }
647
648 /* Configure memcpy mode */
649 if (vchan->is_dedicated) {
650 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) |
651 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM);
652 } else {
653 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
654 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
655 }
656
657 /* Fill the contract with our only promise */
658 list_add_tail(&promise->list, &contract->demands);
659
660 /* And add it to the vchan */
661 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
662}
663
664static struct dma_async_tx_descriptor *
665sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
666 size_t period_len, enum dma_transfer_direction dir,
667 unsigned long flags)
668{
669 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
670 struct dma_slave_config *sconfig = &vchan->cfg;
671 struct sun4i_dma_promise *promise;
672 struct sun4i_dma_contract *contract;
673 dma_addr_t src, dest;
674 u32 endpoints;
675 int nr_periods, offset, plength, i;
676
677 if (!is_slave_direction(dir)) {
678 dev_err(chan2dev(chan), "Invalid DMA direction\n");
679 return NULL;
680 }
681
682 if (vchan->is_dedicated) {
683 /*
684 * As we are using this just for audio data, we need to use
685 * normal DMA. There is nothing stopping us from supporting
686 * dedicated DMA here as well, so if a client comes up and
687 * requires it, it will be simple to implement it.
688 */
689 dev_err(chan2dev(chan),
690 "Cyclic transfers are only supported on Normal DMA\n");
691 return NULL;
692 }
693
694 contract = generate_dma_contract();
695 if (!contract)
696 return NULL;
697
698 contract->is_cyclic = 1;
699
700 /* Figure out the endpoints and the address we need */
701 if (dir == DMA_MEM_TO_DEV) {
702 src = buf;
703 dest = sconfig->dst_addr;
704 endpoints = SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
705 SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
706 SUN4I_DMA_CFG_DST_ADDR_MODE(SUN4I_NDMA_ADDR_MODE_IO);
707 } else {
708 src = sconfig->src_addr;
709 dest = buf;
710 endpoints = SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
711 SUN4I_DMA_CFG_SRC_ADDR_MODE(SUN4I_NDMA_ADDR_MODE_IO) |
712 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
713 }
714
715 /*
716 * We will be using half done interrupts to make two periods
717 * out of a promise, so we need to program the DMA engine less
718 * often
719 */
720
721 /*
722 * The engine can interrupt on half-transfer, so we can use
723 * this feature to program the engine half as often as if we
724 * didn't use it (keep in mind the hardware doesn't support
725 * linked lists).
726 *
727 * Say you have a set of periods (| marks the start/end, I for
728 * interrupt, P for programming the engine to do a new
729 * transfer), the easy but slow way would be to do
730 *
731 * |---|---|---|---| (periods / promises)
732 * P I,P I,P I,P I
733 *
734 * Using half transfer interrupts you can do
735 *
736 * |-------|-------| (promises as configured on hw)
737 * |---|---|---|---| (periods)
738 * P I I,P I I
739 *
740 * Which requires half the engine programming for the same
741 * functionality.
742 */
743 nr_periods = DIV_ROUND_UP(len / period_len, 2);
744 for (i = 0; i < nr_periods; i++) {
745 /* Calculate the offset in the buffer and the length needed */
746 offset = i * period_len * 2;
747 plength = min((len - offset), (period_len * 2));
748 if (dir == DMA_MEM_TO_DEV)
749 src = buf + offset;
750 else
751 dest = buf + offset;
752
753 /* Make the promise */
754 promise = generate_ndma_promise(chan, src, dest,
755 plength, sconfig, dir);
756 if (!promise) {
757 /* TODO: should we free everything? */
758 return NULL;
759 }
760 promise->cfg |= endpoints;
761
762 /* Then add it to the contract */
763 list_add_tail(&promise->list, &contract->demands);
764 }
765
766 /* And add it to the vchan */
767 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
768}
769
770static struct dma_async_tx_descriptor *
771sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
772 unsigned int sg_len, enum dma_transfer_direction dir,
773 unsigned long flags, void *context)
774{
775 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
776 struct dma_slave_config *sconfig = &vchan->cfg;
777 struct sun4i_dma_promise *promise;
778 struct sun4i_dma_contract *contract;
779 u8 ram_type, io_mode, linear_mode;
780 struct scatterlist *sg;
781 dma_addr_t srcaddr, dstaddr;
782 u32 endpoints, para;
783 int i;
784
785 if (!sgl)
786 return NULL;
787
788 if (!is_slave_direction(dir)) {
789 dev_err(chan2dev(chan), "Invalid DMA direction\n");
790 return NULL;
791 }
792
793 contract = generate_dma_contract();
794 if (!contract)
795 return NULL;
796
797 if (vchan->is_dedicated) {
798 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
799 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
800 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
801 } else {
802 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
803 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
804 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
805 }
806
807 if (dir == DMA_MEM_TO_DEV)
808 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
809 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
810 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
811 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
812 else
813 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
814 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
815 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
816 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
817
818 for_each_sg(sgl, sg, sg_len, i) {
819 /* Figure out addresses */
820 if (dir == DMA_MEM_TO_DEV) {
821 srcaddr = sg_dma_address(sg);
822 dstaddr = sconfig->dst_addr;
823 } else {
824 srcaddr = sconfig->src_addr;
825 dstaddr = sg_dma_address(sg);
826 }
827
828 /*
829 * These are the magic DMA engine timings that keep SPI going.
830 * I haven't seen any interface on DMAEngine to configure
831 * timings, and so far they seem to work for everything we
832 * support, so I've kept them here. I don't know if other
833 * devices need different timings because, as usual, we only
834 * have the "para" bitfield meanings, but no comment on what
835 * the values should be when doing a certain operation :|
836 */
837 para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
838
839 /* And make a suitable promise */
840 if (vchan->is_dedicated)
841 promise = generate_ddma_promise(chan, srcaddr, dstaddr,
842 sg_dma_len(sg),
843 sconfig);
844 else
845 promise = generate_ndma_promise(chan, srcaddr, dstaddr,
846 sg_dma_len(sg),
847 sconfig, dir);
848
849 if (!promise)
850 return NULL; /* TODO: should we free everything? */
851
852 promise->cfg |= endpoints;
853 promise->para = para;
854
855 /* Then add it to the contract */
856 list_add_tail(&promise->list, &contract->demands);
857 }
858
859 /*
860 * Once we've got all the promises ready, add the contract
861 * to the pending list on the vchan
862 */
863 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
864}
865
866static int sun4i_dma_terminate_all(struct dma_chan *chan)
867{
868 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
869 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
870 struct sun4i_dma_pchan *pchan = vchan->pchan;
871 LIST_HEAD(head);
872 unsigned long flags;
873
874 spin_lock_irqsave(&vchan->vc.lock, flags);
875 vchan_get_all_descriptors(&vchan->vc, &head);
876 spin_unlock_irqrestore(&vchan->vc.lock, flags);
877
878 /*
879 * Clearing the configuration register will halt the pchan. Interrupts
880 * may still trigger, so don't forget to disable them.
881 */
882 if (pchan) {
883 if (pchan->is_dedicated)
884 writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
885 else
886 writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
887 set_pchan_interrupt(priv, pchan, 0, 0);
888 release_pchan(priv, pchan);
889 }
890
891 spin_lock_irqsave(&vchan->vc.lock, flags);
892 vchan_dma_desc_free_list(&vchan->vc, &head);
893 /* Clear these so the vchan is usable again */
894 vchan->processing = NULL;
895 vchan->pchan = NULL;
896 spin_unlock_irqrestore(&vchan->vc.lock, flags);
897
898 return 0;
899}
900
901static int sun4i_dma_config(struct dma_chan *chan,
902 struct dma_slave_config *config)
903{
904 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
905
906 memcpy(&vchan->cfg, config, sizeof(*config));
907
908 return 0;
909}
910
911static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
912 struct of_dma *ofdma)
913{
914 struct sun4i_dma_dev *priv = ofdma->of_dma_data;
915 struct sun4i_dma_vchan *vchan;
916 struct dma_chan *chan;
917 u8 is_dedicated = dma_spec->args[0];
918 u8 endpoint = dma_spec->args[1];
919
920 /* Check if type is Normal or Dedicated */
921 if (is_dedicated != 0 && is_dedicated != 1)
922 return NULL;
923
924 /* Make sure the endpoint looks sane */
925 if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
926 (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
927 return NULL;
928
929 chan = dma_get_any_slave_channel(&priv->slave);
930 if (!chan)
931 return NULL;
932
933 /* Assign the endpoint to the vchan */
934 vchan = to_sun4i_dma_vchan(chan);
935 vchan->is_dedicated = is_dedicated;
936 vchan->endpoint = endpoint;
937
938 return chan;
939}
940
941static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
942 dma_cookie_t cookie,
943 struct dma_tx_state *state)
944{
945 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
946 struct sun4i_dma_pchan *pchan = vchan->pchan;
947 struct sun4i_dma_contract *contract;
948 struct sun4i_dma_promise *promise;
949 struct virt_dma_desc *vd;
950 unsigned long flags;
951 enum dma_status ret;
952 size_t bytes = 0;
953
954 ret = dma_cookie_status(chan, cookie, state);
955 if (!state || (ret == DMA_COMPLETE))
956 return ret;
957
958 spin_lock_irqsave(&vchan->vc.lock, flags);
959 vd = vchan_find_desc(&vchan->vc, cookie);
960 if (!vd)
961 goto exit;
962 contract = to_sun4i_dma_contract(vd);
963
964 list_for_each_entry(promise, &contract->demands, list)
965 bytes += promise->len;
966
967 /*
968 * The hardware is configured to return the remaining byte
969 * quantity. If possible, replace the first listed element's
970 * full size with the actual remaining amount
971 */
972 promise = list_first_entry_or_null(&contract->demands,
973 struct sun4i_dma_promise, list);
974 if (promise && pchan) {
975 bytes -= promise->len;
976 if (pchan->is_dedicated)
977 bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
978 else
979 bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
980 }
981
982exit:
983
984 dma_set_residue(state, bytes);
985 spin_unlock_irqrestore(&vchan->vc.lock, flags);
986
987 return ret;
988}
989
990static void sun4i_dma_issue_pending(struct dma_chan *chan)
991{
992 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
993 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
994 unsigned long flags;
995
996 spin_lock_irqsave(&vchan->vc.lock, flags);
997
998 /*
999 * If there are pending transactions for this vchan, push one of
1000 * them into the engine to get the ball rolling.
1001 */
1002 if (vchan_issue_pending(&vchan->vc))
1003 __execute_vchan_pending(priv, vchan);
1004
1005 spin_unlock_irqrestore(&vchan->vc.lock, flags);
1006}
1007
1008static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1009{
1010 struct sun4i_dma_dev *priv = dev_id;
1011 struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1012 struct sun4i_dma_vchan *vchan;
1013 struct sun4i_dma_contract *contract;
1014 struct sun4i_dma_promise *promise;
1015 unsigned long pendirq, irqs, disableirqs;
1016 int bit, i, free_room, allow_mitigation = 1;
1017
1018 pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1019
1020handle_pending:
1021
1022 disableirqs = 0;
1023 free_room = 0;
1024
1025 for_each_set_bit(bit, &pendirq, 32) {
1026 pchan = &pchans[bit >> 1];
1027 vchan = pchan->vchan;
1028 if (!vchan) /* a terminated channel may still interrupt */
1029 continue;
1030 contract = vchan->contract;
1031
1032 /*
1033 * Disable the IRQ and free the pchan if it's an end
1034 * interrupt (odd bit)
1035 */
1036 if (bit & 1) {
1037 spin_lock(&vchan->vc.lock);
1038
1039 /*
1040 * Move the promise into the completed list now that
1041 * we're done with it
1042 */
1043 list_del(&vchan->processing->list);
1044 list_add_tail(&vchan->processing->list,
1045 &contract->completed_demands);
1046
1047 /*
1048 * Cyclic DMA transfers are special:
1049 * - There's always something we can dispatch
1050 * - We need to run the callback
1051 * - Latency is very important, as this is used by audio
1052 * We therefore just cycle through the list and dispatch
1053 * whatever we have here, reusing the pchan. There's
1054 * no need to run the thread after this.
1055 *
1056 * For non-cyclic transfers we need to look around,
1057 * so we can program some more work, or notify the
1058 * client that their transfers have been completed.
1059 */
1060 if (contract->is_cyclic) {
1061 promise = get_next_cyclic_promise(contract);
1062 vchan->processing = promise;
1063 configure_pchan(pchan, promise);
1064 vchan_cyclic_callback(&contract->vd);
1065 } else {
1066 vchan->processing = NULL;
1067 vchan->pchan = NULL;
1068
1069 free_room = 1;
1070 disableirqs |= BIT(bit);
1071 release_pchan(priv, pchan);
1072 }
1073
1074 spin_unlock(&vchan->vc.lock);
1075 } else {
1076 /* Half done interrupt */
1077 if (contract->is_cyclic)
1078 vchan_cyclic_callback(&contract->vd);
1079 else
1080 disableirqs |= BIT(bit);
1081 }
1082 }
1083
1084 /* Disable the IRQs for events we handled */
1085 spin_lock(&priv->lock);
1086 irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1087 writel_relaxed(irqs & ~disableirqs,
1088 priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1089 spin_unlock(&priv->lock);
1090
1091 /* Writing 1 to the pending field will clear the pending interrupt */
1092 writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1093
1094 /*
1095 * If a pchan was freed, we may be able to schedule something else,
1096 * so have a look around
1097 */
1098 if (free_room) {
1099 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1100 vchan = &priv->vchans[i];
1101 spin_lock(&vchan->vc.lock);
1102 __execute_vchan_pending(priv, vchan);
1103 spin_unlock(&vchan->vc.lock);
1104 }
1105 }
1106
1107 /*
1108 * Handle newer interrupts if some showed up, but only do it once
1109 * to avoid a too long a loop
1110 */
1111 if (allow_mitigation) {
1112 pendirq = readl_relaxed(priv->base +
1113 SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1114 if (pendirq) {
1115 allow_mitigation = 0;
1116 goto handle_pending;
1117 }
1118 }
1119
1120 return IRQ_HANDLED;
1121}
1122
1123static int sun4i_dma_probe(struct platform_device *pdev)
1124{
1125 struct sun4i_dma_dev *priv;
1126 struct resource *res;
1127 int i, j, ret;
1128
1129 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1130 if (!priv)
1131 return -ENOMEM;
1132
1133 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1134 priv->base = devm_ioremap_resource(&pdev->dev, res);
1135 if (IS_ERR(priv->base))
1136 return PTR_ERR(priv->base);
1137
1138 priv->irq = platform_get_irq(pdev, 0);
1139 if (priv->irq < 0) {
1140 dev_err(&pdev->dev, "Cannot claim IRQ\n");
1141 return priv->irq;
1142 }
1143
1144 priv->clk = devm_clk_get(&pdev->dev, NULL);
1145 if (IS_ERR(priv->clk)) {
1146 dev_err(&pdev->dev, "No clock specified\n");
1147 return PTR_ERR(priv->clk);
1148 }
1149
1150 platform_set_drvdata(pdev, priv);
1151 spin_lock_init(&priv->lock);
1152
1153 dma_cap_zero(priv->slave.cap_mask);
1154 dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1155 dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1156 dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1157 dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1158
1159 INIT_LIST_HEAD(&priv->slave.channels);
1160 priv->slave.device_free_chan_resources = sun4i_dma_free_chan_resources;
1161 priv->slave.device_tx_status = sun4i_dma_tx_status;
1162 priv->slave.device_issue_pending = sun4i_dma_issue_pending;
1163 priv->slave.device_prep_slave_sg = sun4i_dma_prep_slave_sg;
1164 priv->slave.device_prep_dma_memcpy = sun4i_dma_prep_dma_memcpy;
1165 priv->slave.device_prep_dma_cyclic = sun4i_dma_prep_dma_cyclic;
1166 priv->slave.device_config = sun4i_dma_config;
1167 priv->slave.device_terminate_all = sun4i_dma_terminate_all;
1168 priv->slave.copy_align = 2;
1169 priv->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1170 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1171 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1172 priv->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1173 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1174 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1175 priv->slave.directions = BIT(DMA_DEV_TO_MEM) |
1176 BIT(DMA_MEM_TO_DEV);
1177 priv->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1178
1179 priv->slave.dev = &pdev->dev;
1180
1181 priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS,
1182 sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1183 priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1184 sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1185 if (!priv->vchans || !priv->pchans)
1186 return -ENOMEM;
1187
1188 /*
1189 * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and
1190 * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are
1191 * dedicated ones
1192 */
1193 for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++)
1194 priv->pchans[i].base = priv->base +
1195 SUN4I_NDMA_CHANNEL_REG_BASE(i);
1196
1197 for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) {
1198 priv->pchans[i].base = priv->base +
1199 SUN4I_DDMA_CHANNEL_REG_BASE(j);
1200 priv->pchans[i].is_dedicated = 1;
1201 }
1202
1203 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1204 struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1205
1206 spin_lock_init(&vchan->vc.lock);
1207 vchan->vc.desc_free = sun4i_dma_free_contract;
1208 vchan_init(&vchan->vc, &priv->slave);
1209 }
1210
1211 ret = clk_prepare_enable(priv->clk);
1212 if (ret) {
1213 dev_err(&pdev->dev, "Couldn't enable the clock\n");
1214 return ret;
1215 }
1216
1217 /*
1218 * Make sure the IRQs are all disabled and accounted for. The bootloader
1219 * likes to leave these dirty
1220 */
1221 writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1222 writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1223
1224 ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1225 0, dev_name(&pdev->dev), priv);
1226 if (ret) {
1227 dev_err(&pdev->dev, "Cannot request IRQ\n");
1228 goto err_clk_disable;
1229 }
1230
1231 ret = dma_async_device_register(&priv->slave);
1232 if (ret) {
1233 dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1234 goto err_clk_disable;
1235 }
1236
1237 ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1238 priv);
1239 if (ret) {
1240 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1241 goto err_dma_unregister;
1242 }
1243
1244 dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1245
1246 return 0;
1247
1248err_dma_unregister:
1249 dma_async_device_unregister(&priv->slave);
1250err_clk_disable:
1251 clk_disable_unprepare(priv->clk);
1252 return ret;
1253}
1254
1255static int sun4i_dma_remove(struct platform_device *pdev)
1256{
1257 struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1258
1259 /* Disable IRQ so no more work is scheduled */
1260 disable_irq(priv->irq);
1261
1262 of_dma_controller_free(pdev->dev.of_node);
1263 dma_async_device_unregister(&priv->slave);
1264
1265 clk_disable_unprepare(priv->clk);
1266
1267 return 0;
1268}
1269
1270static const struct of_device_id sun4i_dma_match[] = {
1271 { .compatible = "allwinner,sun4i-a10-dma" },
1272 { /* sentinel */ },
1273};
1274MODULE_DEVICE_TABLE(of, sun4i_dma_match);
1275
1276static struct platform_driver sun4i_dma_driver = {
1277 .probe = sun4i_dma_probe,
1278 .remove = sun4i_dma_remove,
1279 .driver = {
1280 .name = "sun4i-dma",
1281 .of_match_table = sun4i_dma_match,
1282 },
1283};
1284
1285module_platform_driver(sun4i_dma_driver);
1286
1287MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1288MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1289MODULE_LICENSE("GPL");
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * Copyright (C) 2014 Emilio López
4 * Emilio López <emilio@elopez.com.ar>
5 */
6
7#include <linux/bitmap.h>
8#include <linux/bitops.h>
9#include <linux/clk.h>
10#include <linux/dma-mapping.h>
11#include <linux/dmaengine.h>
12#include <linux/dmapool.h>
13#include <linux/interrupt.h>
14#include <linux/module.h>
15#include <linux/of_dma.h>
16#include <linux/platform_device.h>
17#include <linux/slab.h>
18#include <linux/spinlock.h>
19
20#include "virt-dma.h"
21
22/** Common macros to normal and dedicated DMA registers **/
23
24#define SUN4I_DMA_CFG_LOADING BIT(31)
25#define SUN4I_DMA_CFG_DST_DATA_WIDTH(width) ((width) << 25)
26#define SUN4I_DMA_CFG_DST_BURST_LENGTH(len) ((len) << 23)
27#define SUN4I_DMA_CFG_DST_ADDR_MODE(mode) ((mode) << 21)
28#define SUN4I_DMA_CFG_DST_DRQ_TYPE(type) ((type) << 16)
29#define SUN4I_DMA_CFG_SRC_DATA_WIDTH(width) ((width) << 9)
30#define SUN4I_DMA_CFG_SRC_BURST_LENGTH(len) ((len) << 7)
31#define SUN4I_DMA_CFG_SRC_ADDR_MODE(mode) ((mode) << 5)
32#define SUN4I_DMA_CFG_SRC_DRQ_TYPE(type) (type)
33
34/** Normal DMA register values **/
35
36/* Normal DMA source/destination data request type values */
37#define SUN4I_NDMA_DRQ_TYPE_SDRAM 0x16
38#define SUN4I_NDMA_DRQ_TYPE_LIMIT (0x1F + 1)
39
40/** Normal DMA register layout **/
41
42/* Dedicated DMA source/destination address mode values */
43#define SUN4I_NDMA_ADDR_MODE_LINEAR 0
44#define SUN4I_NDMA_ADDR_MODE_IO 1
45
46/* Normal DMA configuration register layout */
47#define SUN4I_NDMA_CFG_CONT_MODE BIT(30)
48#define SUN4I_NDMA_CFG_WAIT_STATE(n) ((n) << 27)
49#define SUN4I_NDMA_CFG_DST_NON_SECURE BIT(22)
50#define SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
51#define SUN4I_NDMA_CFG_SRC_NON_SECURE BIT(6)
52
53/** Dedicated DMA register values **/
54
55/* Dedicated DMA source/destination address mode values */
56#define SUN4I_DDMA_ADDR_MODE_LINEAR 0
57#define SUN4I_DDMA_ADDR_MODE_IO 1
58#define SUN4I_DDMA_ADDR_MODE_HORIZONTAL_PAGE 2
59#define SUN4I_DDMA_ADDR_MODE_VERTICAL_PAGE 3
60
61/* Dedicated DMA source/destination data request type values */
62#define SUN4I_DDMA_DRQ_TYPE_SDRAM 0x1
63#define SUN4I_DDMA_DRQ_TYPE_LIMIT (0x1F + 1)
64
65/** Dedicated DMA register layout **/
66
67/* Dedicated DMA configuration register layout */
68#define SUN4I_DDMA_CFG_BUSY BIT(30)
69#define SUN4I_DDMA_CFG_CONT_MODE BIT(29)
70#define SUN4I_DDMA_CFG_DST_NON_SECURE BIT(28)
71#define SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN BIT(15)
72#define SUN4I_DDMA_CFG_SRC_NON_SECURE BIT(12)
73
74/* Dedicated DMA parameter register layout */
75#define SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(n) (((n) - 1) << 24)
76#define SUN4I_DDMA_PARA_DST_WAIT_CYCLES(n) (((n) - 1) << 16)
77#define SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(n) (((n) - 1) << 8)
78#define SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(n) (((n) - 1) << 0)
79
80/** DMA register offsets **/
81
82/* General register offsets */
83#define SUN4I_DMA_IRQ_ENABLE_REG 0x0
84#define SUN4I_DMA_IRQ_PENDING_STATUS_REG 0x4
85
86/* Normal DMA register offsets */
87#define SUN4I_NDMA_CHANNEL_REG_BASE(n) (0x100 + (n) * 0x20)
88#define SUN4I_NDMA_CFG_REG 0x0
89#define SUN4I_NDMA_SRC_ADDR_REG 0x4
90#define SUN4I_NDMA_DST_ADDR_REG 0x8
91#define SUN4I_NDMA_BYTE_COUNT_REG 0xC
92
93/* Dedicated DMA register offsets */
94#define SUN4I_DDMA_CHANNEL_REG_BASE(n) (0x300 + (n) * 0x20)
95#define SUN4I_DDMA_CFG_REG 0x0
96#define SUN4I_DDMA_SRC_ADDR_REG 0x4
97#define SUN4I_DDMA_DST_ADDR_REG 0x8
98#define SUN4I_DDMA_BYTE_COUNT_REG 0xC
99#define SUN4I_DDMA_PARA_REG 0x18
100
101/** DMA Driver **/
102
103/*
104 * Normal DMA has 8 channels, and Dedicated DMA has another 8, so
105 * that's 16 channels. As for endpoints, there's 29 and 21
106 * respectively. Given that the Normal DMA endpoints (other than
107 * SDRAM) can be used as tx/rx, we need 78 vchans in total
108 */
109#define SUN4I_NDMA_NR_MAX_CHANNELS 8
110#define SUN4I_DDMA_NR_MAX_CHANNELS 8
111#define SUN4I_DMA_NR_MAX_CHANNELS \
112 (SUN4I_NDMA_NR_MAX_CHANNELS + SUN4I_DDMA_NR_MAX_CHANNELS)
113#define SUN4I_NDMA_NR_MAX_VCHANS (29 * 2 - 1)
114#define SUN4I_DDMA_NR_MAX_VCHANS 21
115#define SUN4I_DMA_NR_MAX_VCHANS \
116 (SUN4I_NDMA_NR_MAX_VCHANS + SUN4I_DDMA_NR_MAX_VCHANS)
117
118/* This set of SUN4I_DDMA timing parameters were found experimentally while
119 * working with the SPI driver and seem to make it behave correctly */
120#define SUN4I_DDMA_MAGIC_SPI_PARAMETERS \
121 (SUN4I_DDMA_PARA_DST_DATA_BLK_SIZE(1) | \
122 SUN4I_DDMA_PARA_SRC_DATA_BLK_SIZE(1) | \
123 SUN4I_DDMA_PARA_DST_WAIT_CYCLES(2) | \
124 SUN4I_DDMA_PARA_SRC_WAIT_CYCLES(2))
125
126/*
127 * Normal DMA supports individual transfers (segments) up to 128k.
128 * Dedicated DMA supports transfers up to 16M. We can only report
129 * one size limit, so we have to use the smaller value.
130 */
131#define SUN4I_NDMA_MAX_SEG_SIZE SZ_128K
132#define SUN4I_DDMA_MAX_SEG_SIZE SZ_16M
133#define SUN4I_DMA_MAX_SEG_SIZE SUN4I_NDMA_MAX_SEG_SIZE
134
135struct sun4i_dma_pchan {
136 /* Register base of channel */
137 void __iomem *base;
138 /* vchan currently being serviced */
139 struct sun4i_dma_vchan *vchan;
140 /* Is this a dedicated pchan? */
141 int is_dedicated;
142};
143
144struct sun4i_dma_vchan {
145 struct virt_dma_chan vc;
146 struct dma_slave_config cfg;
147 struct sun4i_dma_pchan *pchan;
148 struct sun4i_dma_promise *processing;
149 struct sun4i_dma_contract *contract;
150 u8 endpoint;
151 int is_dedicated;
152};
153
154struct sun4i_dma_promise {
155 u32 cfg;
156 u32 para;
157 dma_addr_t src;
158 dma_addr_t dst;
159 size_t len;
160 struct list_head list;
161};
162
163/* A contract is a set of promises */
164struct sun4i_dma_contract {
165 struct virt_dma_desc vd;
166 struct list_head demands;
167 struct list_head completed_demands;
168 bool is_cyclic : 1;
169 bool use_half_int : 1;
170};
171
172struct sun4i_dma_dev {
173 DECLARE_BITMAP(pchans_used, SUN4I_DMA_NR_MAX_CHANNELS);
174 struct dma_device slave;
175 struct sun4i_dma_pchan *pchans;
176 struct sun4i_dma_vchan *vchans;
177 void __iomem *base;
178 struct clk *clk;
179 int irq;
180 spinlock_t lock;
181};
182
183static struct sun4i_dma_dev *to_sun4i_dma_dev(struct dma_device *dev)
184{
185 return container_of(dev, struct sun4i_dma_dev, slave);
186}
187
188static struct sun4i_dma_vchan *to_sun4i_dma_vchan(struct dma_chan *chan)
189{
190 return container_of(chan, struct sun4i_dma_vchan, vc.chan);
191}
192
193static struct sun4i_dma_contract *to_sun4i_dma_contract(struct virt_dma_desc *vd)
194{
195 return container_of(vd, struct sun4i_dma_contract, vd);
196}
197
198static struct device *chan2dev(struct dma_chan *chan)
199{
200 return &chan->dev->device;
201}
202
203static int convert_burst(u32 maxburst)
204{
205 if (maxburst > 8)
206 return -EINVAL;
207
208 /* 1 -> 0, 4 -> 1, 8 -> 2 */
209 return (maxburst >> 2);
210}
211
212static int convert_buswidth(enum dma_slave_buswidth addr_width)
213{
214 if (addr_width > DMA_SLAVE_BUSWIDTH_4_BYTES)
215 return -EINVAL;
216
217 /* 8 (1 byte) -> 0, 16 (2 bytes) -> 1, 32 (4 bytes) -> 2 */
218 return (addr_width >> 1);
219}
220
221static void sun4i_dma_free_chan_resources(struct dma_chan *chan)
222{
223 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
224
225 vchan_free_chan_resources(&vchan->vc);
226}
227
228static struct sun4i_dma_pchan *find_and_use_pchan(struct sun4i_dma_dev *priv,
229 struct sun4i_dma_vchan *vchan)
230{
231 struct sun4i_dma_pchan *pchan = NULL, *pchans = priv->pchans;
232 unsigned long flags;
233 int i, max;
234
235 /*
236 * pchans 0-SUN4I_NDMA_NR_MAX_CHANNELS are normal, and
237 * SUN4I_NDMA_NR_MAX_CHANNELS+ are dedicated ones
238 */
239 if (vchan->is_dedicated) {
240 i = SUN4I_NDMA_NR_MAX_CHANNELS;
241 max = SUN4I_DMA_NR_MAX_CHANNELS;
242 } else {
243 i = 0;
244 max = SUN4I_NDMA_NR_MAX_CHANNELS;
245 }
246
247 spin_lock_irqsave(&priv->lock, flags);
248 for_each_clear_bit_from(i, priv->pchans_used, max) {
249 pchan = &pchans[i];
250 pchan->vchan = vchan;
251 set_bit(i, priv->pchans_used);
252 break;
253 }
254 spin_unlock_irqrestore(&priv->lock, flags);
255
256 return pchan;
257}
258
259static void release_pchan(struct sun4i_dma_dev *priv,
260 struct sun4i_dma_pchan *pchan)
261{
262 unsigned long flags;
263 int nr = pchan - priv->pchans;
264
265 spin_lock_irqsave(&priv->lock, flags);
266
267 pchan->vchan = NULL;
268 clear_bit(nr, priv->pchans_used);
269
270 spin_unlock_irqrestore(&priv->lock, flags);
271}
272
273static void configure_pchan(struct sun4i_dma_pchan *pchan,
274 struct sun4i_dma_promise *d)
275{
276 /*
277 * Configure addresses and misc parameters depending on type
278 * SUN4I_DDMA has an extra field with timing parameters
279 */
280 if (pchan->is_dedicated) {
281 writel_relaxed(d->src, pchan->base + SUN4I_DDMA_SRC_ADDR_REG);
282 writel_relaxed(d->dst, pchan->base + SUN4I_DDMA_DST_ADDR_REG);
283 writel_relaxed(d->len, pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
284 writel_relaxed(d->para, pchan->base + SUN4I_DDMA_PARA_REG);
285 writel_relaxed(d->cfg, pchan->base + SUN4I_DDMA_CFG_REG);
286 } else {
287 writel_relaxed(d->src, pchan->base + SUN4I_NDMA_SRC_ADDR_REG);
288 writel_relaxed(d->dst, pchan->base + SUN4I_NDMA_DST_ADDR_REG);
289 writel_relaxed(d->len, pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
290 writel_relaxed(d->cfg, pchan->base + SUN4I_NDMA_CFG_REG);
291 }
292}
293
294static void set_pchan_interrupt(struct sun4i_dma_dev *priv,
295 struct sun4i_dma_pchan *pchan,
296 int half, int end)
297{
298 u32 reg;
299 int pchan_number = pchan - priv->pchans;
300 unsigned long flags;
301
302 spin_lock_irqsave(&priv->lock, flags);
303
304 reg = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
305
306 if (half)
307 reg |= BIT(pchan_number * 2);
308 else
309 reg &= ~BIT(pchan_number * 2);
310
311 if (end)
312 reg |= BIT(pchan_number * 2 + 1);
313 else
314 reg &= ~BIT(pchan_number * 2 + 1);
315
316 writel_relaxed(reg, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
317
318 spin_unlock_irqrestore(&priv->lock, flags);
319}
320
321/*
322 * Execute pending operations on a vchan
323 *
324 * When given a vchan, this function will try to acquire a suitable
325 * pchan and, if successful, will configure it to fulfill a promise
326 * from the next pending contract.
327 *
328 * This function must be called with &vchan->vc.lock held.
329 */
330static int __execute_vchan_pending(struct sun4i_dma_dev *priv,
331 struct sun4i_dma_vchan *vchan)
332{
333 struct sun4i_dma_promise *promise = NULL;
334 struct sun4i_dma_contract *contract = NULL;
335 struct sun4i_dma_pchan *pchan;
336 struct virt_dma_desc *vd;
337 int ret;
338
339 lockdep_assert_held(&vchan->vc.lock);
340
341 /* We need a pchan to do anything, so secure one if available */
342 pchan = find_and_use_pchan(priv, vchan);
343 if (!pchan)
344 return -EBUSY;
345
346 /*
347 * Channel endpoints must not be repeated, so if this vchan
348 * has already submitted some work, we can't do anything else
349 */
350 if (vchan->processing) {
351 dev_dbg(chan2dev(&vchan->vc.chan),
352 "processing something to this endpoint already\n");
353 ret = -EBUSY;
354 goto release_pchan;
355 }
356
357 do {
358 /* Figure out which contract we're working with today */
359 vd = vchan_next_desc(&vchan->vc);
360 if (!vd) {
361 dev_dbg(chan2dev(&vchan->vc.chan),
362 "No pending contract found");
363 ret = 0;
364 goto release_pchan;
365 }
366
367 contract = to_sun4i_dma_contract(vd);
368 if (list_empty(&contract->demands)) {
369 /* The contract has been completed so mark it as such */
370 list_del(&contract->vd.node);
371 vchan_cookie_complete(&contract->vd);
372 dev_dbg(chan2dev(&vchan->vc.chan),
373 "Empty contract found and marked complete");
374 }
375 } while (list_empty(&contract->demands));
376
377 /* Now find out what we need to do */
378 promise = list_first_entry(&contract->demands,
379 struct sun4i_dma_promise, list);
380 vchan->processing = promise;
381
382 /* ... and make it reality */
383 if (promise) {
384 vchan->contract = contract;
385 vchan->pchan = pchan;
386 set_pchan_interrupt(priv, pchan, contract->use_half_int, 1);
387 configure_pchan(pchan, promise);
388 }
389
390 return 0;
391
392release_pchan:
393 release_pchan(priv, pchan);
394 return ret;
395}
396
397static int sanitize_config(struct dma_slave_config *sconfig,
398 enum dma_transfer_direction direction)
399{
400 switch (direction) {
401 case DMA_MEM_TO_DEV:
402 if ((sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
403 !sconfig->dst_maxburst)
404 return -EINVAL;
405
406 if (sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
407 sconfig->src_addr_width = sconfig->dst_addr_width;
408
409 if (!sconfig->src_maxburst)
410 sconfig->src_maxburst = sconfig->dst_maxburst;
411
412 break;
413
414 case DMA_DEV_TO_MEM:
415 if ((sconfig->src_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED) ||
416 !sconfig->src_maxburst)
417 return -EINVAL;
418
419 if (sconfig->dst_addr_width == DMA_SLAVE_BUSWIDTH_UNDEFINED)
420 sconfig->dst_addr_width = sconfig->src_addr_width;
421
422 if (!sconfig->dst_maxburst)
423 sconfig->dst_maxburst = sconfig->src_maxburst;
424
425 break;
426 default:
427 return 0;
428 }
429
430 return 0;
431}
432
433/*
434 * Generate a promise, to be used in a normal DMA contract.
435 *
436 * A NDMA promise contains all the information required to program the
437 * normal part of the DMA Engine and get data copied. A non-executed
438 * promise will live in the demands list on a contract. Once it has been
439 * completed, it will be moved to the completed demands list for later freeing.
440 * All linked promises will be freed when the corresponding contract is freed
441 */
442static struct sun4i_dma_promise *
443generate_ndma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
444 size_t len, struct dma_slave_config *sconfig,
445 enum dma_transfer_direction direction)
446{
447 struct sun4i_dma_promise *promise;
448 int ret;
449
450 ret = sanitize_config(sconfig, direction);
451 if (ret)
452 return NULL;
453
454 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
455 if (!promise)
456 return NULL;
457
458 promise->src = src;
459 promise->dst = dest;
460 promise->len = len;
461 promise->cfg = SUN4I_DMA_CFG_LOADING |
462 SUN4I_NDMA_CFG_BYTE_COUNT_MODE_REMAIN;
463
464 dev_dbg(chan2dev(chan),
465 "src burst %d, dst burst %d, src buswidth %d, dst buswidth %d",
466 sconfig->src_maxburst, sconfig->dst_maxburst,
467 sconfig->src_addr_width, sconfig->dst_addr_width);
468
469 /* Source burst */
470 ret = convert_burst(sconfig->src_maxburst);
471 if (ret < 0)
472 goto fail;
473 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
474
475 /* Destination burst */
476 ret = convert_burst(sconfig->dst_maxburst);
477 if (ret < 0)
478 goto fail;
479 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
480
481 /* Source bus width */
482 ret = convert_buswidth(sconfig->src_addr_width);
483 if (ret < 0)
484 goto fail;
485 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
486
487 /* Destination bus width */
488 ret = convert_buswidth(sconfig->dst_addr_width);
489 if (ret < 0)
490 goto fail;
491 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
492
493 return promise;
494
495fail:
496 kfree(promise);
497 return NULL;
498}
499
500/*
501 * Generate a promise, to be used in a dedicated DMA contract.
502 *
503 * A DDMA promise contains all the information required to program the
504 * Dedicated part of the DMA Engine and get data copied. A non-executed
505 * promise will live in the demands list on a contract. Once it has been
506 * completed, it will be moved to the completed demands list for later freeing.
507 * All linked promises will be freed when the corresponding contract is freed
508 */
509static struct sun4i_dma_promise *
510generate_ddma_promise(struct dma_chan *chan, dma_addr_t src, dma_addr_t dest,
511 size_t len, struct dma_slave_config *sconfig)
512{
513 struct sun4i_dma_promise *promise;
514 int ret;
515
516 promise = kzalloc(sizeof(*promise), GFP_NOWAIT);
517 if (!promise)
518 return NULL;
519
520 promise->src = src;
521 promise->dst = dest;
522 promise->len = len;
523 promise->cfg = SUN4I_DMA_CFG_LOADING |
524 SUN4I_DDMA_CFG_BYTE_COUNT_MODE_REMAIN;
525
526 /* Source burst */
527 ret = convert_burst(sconfig->src_maxburst);
528 if (ret < 0)
529 goto fail;
530 promise->cfg |= SUN4I_DMA_CFG_SRC_BURST_LENGTH(ret);
531
532 /* Destination burst */
533 ret = convert_burst(sconfig->dst_maxburst);
534 if (ret < 0)
535 goto fail;
536 promise->cfg |= SUN4I_DMA_CFG_DST_BURST_LENGTH(ret);
537
538 /* Source bus width */
539 ret = convert_buswidth(sconfig->src_addr_width);
540 if (ret < 0)
541 goto fail;
542 promise->cfg |= SUN4I_DMA_CFG_SRC_DATA_WIDTH(ret);
543
544 /* Destination bus width */
545 ret = convert_buswidth(sconfig->dst_addr_width);
546 if (ret < 0)
547 goto fail;
548 promise->cfg |= SUN4I_DMA_CFG_DST_DATA_WIDTH(ret);
549
550 return promise;
551
552fail:
553 kfree(promise);
554 return NULL;
555}
556
557/*
558 * Generate a contract
559 *
560 * Contracts function as DMA descriptors. As our hardware does not support
561 * linked lists, we need to implement SG via software. We use a contract
562 * to hold all the pieces of the request and process them serially one
563 * after another. Each piece is represented as a promise.
564 */
565static struct sun4i_dma_contract *generate_dma_contract(void)
566{
567 struct sun4i_dma_contract *contract;
568
569 contract = kzalloc(sizeof(*contract), GFP_NOWAIT);
570 if (!contract)
571 return NULL;
572
573 INIT_LIST_HEAD(&contract->demands);
574 INIT_LIST_HEAD(&contract->completed_demands);
575
576 return contract;
577}
578
579/*
580 * Get next promise on a cyclic transfer
581 *
582 * Cyclic contracts contain a series of promises which are executed on a
583 * loop. This function returns the next promise from a cyclic contract,
584 * so it can be programmed into the hardware.
585 */
586static struct sun4i_dma_promise *
587get_next_cyclic_promise(struct sun4i_dma_contract *contract)
588{
589 struct sun4i_dma_promise *promise;
590
591 promise = list_first_entry_or_null(&contract->demands,
592 struct sun4i_dma_promise, list);
593 if (!promise) {
594 list_splice_init(&contract->completed_demands,
595 &contract->demands);
596 promise = list_first_entry(&contract->demands,
597 struct sun4i_dma_promise, list);
598 }
599
600 return promise;
601}
602
603/*
604 * Free a contract and all its associated promises
605 */
606static void sun4i_dma_free_contract(struct virt_dma_desc *vd)
607{
608 struct sun4i_dma_contract *contract = to_sun4i_dma_contract(vd);
609 struct sun4i_dma_promise *promise, *tmp;
610
611 /* Free all the demands and completed demands */
612 list_for_each_entry_safe(promise, tmp, &contract->demands, list)
613 kfree(promise);
614
615 list_for_each_entry_safe(promise, tmp, &contract->completed_demands, list)
616 kfree(promise);
617
618 kfree(contract);
619}
620
621static struct dma_async_tx_descriptor *
622sun4i_dma_prep_dma_memcpy(struct dma_chan *chan, dma_addr_t dest,
623 dma_addr_t src, size_t len, unsigned long flags)
624{
625 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
626 struct dma_slave_config *sconfig = &vchan->cfg;
627 struct sun4i_dma_promise *promise;
628 struct sun4i_dma_contract *contract;
629
630 contract = generate_dma_contract();
631 if (!contract)
632 return NULL;
633
634 /*
635 * We can only do the copy to bus aligned addresses, so
636 * choose the best one so we get decent performance. We also
637 * maximize the burst size for this same reason.
638 */
639 sconfig->src_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
640 sconfig->dst_addr_width = DMA_SLAVE_BUSWIDTH_4_BYTES;
641 sconfig->src_maxburst = 8;
642 sconfig->dst_maxburst = 8;
643
644 if (vchan->is_dedicated)
645 promise = generate_ddma_promise(chan, src, dest, len, sconfig);
646 else
647 promise = generate_ndma_promise(chan, src, dest, len, sconfig,
648 DMA_MEM_TO_MEM);
649
650 if (!promise) {
651 kfree(contract);
652 return NULL;
653 }
654
655 /* Configure memcpy mode */
656 if (vchan->is_dedicated) {
657 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM) |
658 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_DDMA_DRQ_TYPE_SDRAM);
659 } else {
660 promise->cfg |= SUN4I_DMA_CFG_SRC_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM) |
661 SUN4I_DMA_CFG_DST_DRQ_TYPE(SUN4I_NDMA_DRQ_TYPE_SDRAM);
662 }
663
664 /* Fill the contract with our only promise */
665 list_add_tail(&promise->list, &contract->demands);
666
667 /* And add it to the vchan */
668 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
669}
670
671static struct dma_async_tx_descriptor *
672sun4i_dma_prep_dma_cyclic(struct dma_chan *chan, dma_addr_t buf, size_t len,
673 size_t period_len, enum dma_transfer_direction dir,
674 unsigned long flags)
675{
676 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
677 struct dma_slave_config *sconfig = &vchan->cfg;
678 struct sun4i_dma_promise *promise;
679 struct sun4i_dma_contract *contract;
680 dma_addr_t src, dest;
681 u32 endpoints;
682 int nr_periods, offset, plength, i;
683 u8 ram_type, io_mode, linear_mode;
684
685 if (!is_slave_direction(dir)) {
686 dev_err(chan2dev(chan), "Invalid DMA direction\n");
687 return NULL;
688 }
689
690 contract = generate_dma_contract();
691 if (!contract)
692 return NULL;
693
694 contract->is_cyclic = 1;
695
696 if (vchan->is_dedicated) {
697 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
698 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
699 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
700 } else {
701 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
702 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
703 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
704 }
705
706 if (dir == DMA_MEM_TO_DEV) {
707 src = buf;
708 dest = sconfig->dst_addr;
709 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
710 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
711 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
712 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
713 } else {
714 src = sconfig->src_addr;
715 dest = buf;
716 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
717 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
718 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
719 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
720 }
721
722 /*
723 * We will be using half done interrupts to make two periods
724 * out of a promise, so we need to program the DMA engine less
725 * often
726 */
727
728 /*
729 * The engine can interrupt on half-transfer, so we can use
730 * this feature to program the engine half as often as if we
731 * didn't use it (keep in mind the hardware doesn't support
732 * linked lists).
733 *
734 * Say you have a set of periods (| marks the start/end, I for
735 * interrupt, P for programming the engine to do a new
736 * transfer), the easy but slow way would be to do
737 *
738 * |---|---|---|---| (periods / promises)
739 * P I,P I,P I,P I
740 *
741 * Using half transfer interrupts you can do
742 *
743 * |-------|-------| (promises as configured on hw)
744 * |---|---|---|---| (periods)
745 * P I I,P I I
746 *
747 * Which requires half the engine programming for the same
748 * functionality.
749 *
750 * This only works if two periods fit in a single promise. That will
751 * always be the case for dedicated DMA, where the hardware has a much
752 * larger maximum transfer size than advertised to clients.
753 */
754 if (vchan->is_dedicated || period_len <= SUN4I_NDMA_MAX_SEG_SIZE / 2) {
755 period_len *= 2;
756 contract->use_half_int = 1;
757 }
758
759 nr_periods = DIV_ROUND_UP(len, period_len);
760 for (i = 0; i < nr_periods; i++) {
761 /* Calculate the offset in the buffer and the length needed */
762 offset = i * period_len;
763 plength = min((len - offset), period_len);
764 if (dir == DMA_MEM_TO_DEV)
765 src = buf + offset;
766 else
767 dest = buf + offset;
768
769 /* Make the promise */
770 if (vchan->is_dedicated)
771 promise = generate_ddma_promise(chan, src, dest,
772 plength, sconfig);
773 else
774 promise = generate_ndma_promise(chan, src, dest,
775 plength, sconfig, dir);
776
777 if (!promise) {
778 /* TODO: should we free everything? */
779 return NULL;
780 }
781 promise->cfg |= endpoints;
782
783 /* Then add it to the contract */
784 list_add_tail(&promise->list, &contract->demands);
785 }
786
787 /* And add it to the vchan */
788 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
789}
790
791static struct dma_async_tx_descriptor *
792sun4i_dma_prep_slave_sg(struct dma_chan *chan, struct scatterlist *sgl,
793 unsigned int sg_len, enum dma_transfer_direction dir,
794 unsigned long flags, void *context)
795{
796 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
797 struct dma_slave_config *sconfig = &vchan->cfg;
798 struct sun4i_dma_promise *promise;
799 struct sun4i_dma_contract *contract;
800 u8 ram_type, io_mode, linear_mode;
801 struct scatterlist *sg;
802 dma_addr_t srcaddr, dstaddr;
803 u32 endpoints, para;
804 int i;
805
806 if (!sgl)
807 return NULL;
808
809 if (!is_slave_direction(dir)) {
810 dev_err(chan2dev(chan), "Invalid DMA direction\n");
811 return NULL;
812 }
813
814 contract = generate_dma_contract();
815 if (!contract)
816 return NULL;
817
818 if (vchan->is_dedicated) {
819 io_mode = SUN4I_DDMA_ADDR_MODE_IO;
820 linear_mode = SUN4I_DDMA_ADDR_MODE_LINEAR;
821 ram_type = SUN4I_DDMA_DRQ_TYPE_SDRAM;
822 } else {
823 io_mode = SUN4I_NDMA_ADDR_MODE_IO;
824 linear_mode = SUN4I_NDMA_ADDR_MODE_LINEAR;
825 ram_type = SUN4I_NDMA_DRQ_TYPE_SDRAM;
826 }
827
828 if (dir == DMA_MEM_TO_DEV)
829 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(vchan->endpoint) |
830 SUN4I_DMA_CFG_DST_ADDR_MODE(io_mode) |
831 SUN4I_DMA_CFG_SRC_DRQ_TYPE(ram_type) |
832 SUN4I_DMA_CFG_SRC_ADDR_MODE(linear_mode);
833 else
834 endpoints = SUN4I_DMA_CFG_DST_DRQ_TYPE(ram_type) |
835 SUN4I_DMA_CFG_DST_ADDR_MODE(linear_mode) |
836 SUN4I_DMA_CFG_SRC_DRQ_TYPE(vchan->endpoint) |
837 SUN4I_DMA_CFG_SRC_ADDR_MODE(io_mode);
838
839 for_each_sg(sgl, sg, sg_len, i) {
840 /* Figure out addresses */
841 if (dir == DMA_MEM_TO_DEV) {
842 srcaddr = sg_dma_address(sg);
843 dstaddr = sconfig->dst_addr;
844 } else {
845 srcaddr = sconfig->src_addr;
846 dstaddr = sg_dma_address(sg);
847 }
848
849 /*
850 * These are the magic DMA engine timings that keep SPI going.
851 * I haven't seen any interface on DMAEngine to configure
852 * timings, and so far they seem to work for everything we
853 * support, so I've kept them here. I don't know if other
854 * devices need different timings because, as usual, we only
855 * have the "para" bitfield meanings, but no comment on what
856 * the values should be when doing a certain operation :|
857 */
858 para = SUN4I_DDMA_MAGIC_SPI_PARAMETERS;
859
860 /* And make a suitable promise */
861 if (vchan->is_dedicated)
862 promise = generate_ddma_promise(chan, srcaddr, dstaddr,
863 sg_dma_len(sg),
864 sconfig);
865 else
866 promise = generate_ndma_promise(chan, srcaddr, dstaddr,
867 sg_dma_len(sg),
868 sconfig, dir);
869
870 if (!promise)
871 return NULL; /* TODO: should we free everything? */
872
873 promise->cfg |= endpoints;
874 promise->para = para;
875
876 /* Then add it to the contract */
877 list_add_tail(&promise->list, &contract->demands);
878 }
879
880 /*
881 * Once we've got all the promises ready, add the contract
882 * to the pending list on the vchan
883 */
884 return vchan_tx_prep(&vchan->vc, &contract->vd, flags);
885}
886
887static int sun4i_dma_terminate_all(struct dma_chan *chan)
888{
889 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
890 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
891 struct sun4i_dma_pchan *pchan = vchan->pchan;
892 LIST_HEAD(head);
893 unsigned long flags;
894
895 spin_lock_irqsave(&vchan->vc.lock, flags);
896 vchan_get_all_descriptors(&vchan->vc, &head);
897 spin_unlock_irqrestore(&vchan->vc.lock, flags);
898
899 /*
900 * Clearing the configuration register will halt the pchan. Interrupts
901 * may still trigger, so don't forget to disable them.
902 */
903 if (pchan) {
904 if (pchan->is_dedicated)
905 writel(0, pchan->base + SUN4I_DDMA_CFG_REG);
906 else
907 writel(0, pchan->base + SUN4I_NDMA_CFG_REG);
908 set_pchan_interrupt(priv, pchan, 0, 0);
909 release_pchan(priv, pchan);
910 }
911
912 spin_lock_irqsave(&vchan->vc.lock, flags);
913 /* Clear these so the vchan is usable again */
914 vchan->processing = NULL;
915 vchan->pchan = NULL;
916 spin_unlock_irqrestore(&vchan->vc.lock, flags);
917
918 vchan_dma_desc_free_list(&vchan->vc, &head);
919
920 return 0;
921}
922
923static int sun4i_dma_config(struct dma_chan *chan,
924 struct dma_slave_config *config)
925{
926 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
927
928 memcpy(&vchan->cfg, config, sizeof(*config));
929
930 return 0;
931}
932
933static struct dma_chan *sun4i_dma_of_xlate(struct of_phandle_args *dma_spec,
934 struct of_dma *ofdma)
935{
936 struct sun4i_dma_dev *priv = ofdma->of_dma_data;
937 struct sun4i_dma_vchan *vchan;
938 struct dma_chan *chan;
939 u8 is_dedicated = dma_spec->args[0];
940 u8 endpoint = dma_spec->args[1];
941
942 /* Check if type is Normal or Dedicated */
943 if (is_dedicated != 0 && is_dedicated != 1)
944 return NULL;
945
946 /* Make sure the endpoint looks sane */
947 if ((is_dedicated && endpoint >= SUN4I_DDMA_DRQ_TYPE_LIMIT) ||
948 (!is_dedicated && endpoint >= SUN4I_NDMA_DRQ_TYPE_LIMIT))
949 return NULL;
950
951 chan = dma_get_any_slave_channel(&priv->slave);
952 if (!chan)
953 return NULL;
954
955 /* Assign the endpoint to the vchan */
956 vchan = to_sun4i_dma_vchan(chan);
957 vchan->is_dedicated = is_dedicated;
958 vchan->endpoint = endpoint;
959
960 return chan;
961}
962
963static enum dma_status sun4i_dma_tx_status(struct dma_chan *chan,
964 dma_cookie_t cookie,
965 struct dma_tx_state *state)
966{
967 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
968 struct sun4i_dma_pchan *pchan = vchan->pchan;
969 struct sun4i_dma_contract *contract;
970 struct sun4i_dma_promise *promise;
971 struct virt_dma_desc *vd;
972 unsigned long flags;
973 enum dma_status ret;
974 size_t bytes = 0;
975
976 ret = dma_cookie_status(chan, cookie, state);
977 if (!state || (ret == DMA_COMPLETE))
978 return ret;
979
980 spin_lock_irqsave(&vchan->vc.lock, flags);
981 vd = vchan_find_desc(&vchan->vc, cookie);
982 if (!vd)
983 goto exit;
984 contract = to_sun4i_dma_contract(vd);
985
986 list_for_each_entry(promise, &contract->demands, list)
987 bytes += promise->len;
988
989 /*
990 * The hardware is configured to return the remaining byte
991 * quantity. If possible, replace the first listed element's
992 * full size with the actual remaining amount
993 */
994 promise = list_first_entry_or_null(&contract->demands,
995 struct sun4i_dma_promise, list);
996 if (promise && pchan) {
997 bytes -= promise->len;
998 if (pchan->is_dedicated)
999 bytes += readl(pchan->base + SUN4I_DDMA_BYTE_COUNT_REG);
1000 else
1001 bytes += readl(pchan->base + SUN4I_NDMA_BYTE_COUNT_REG);
1002 }
1003
1004exit:
1005
1006 dma_set_residue(state, bytes);
1007 spin_unlock_irqrestore(&vchan->vc.lock, flags);
1008
1009 return ret;
1010}
1011
1012static void sun4i_dma_issue_pending(struct dma_chan *chan)
1013{
1014 struct sun4i_dma_dev *priv = to_sun4i_dma_dev(chan->device);
1015 struct sun4i_dma_vchan *vchan = to_sun4i_dma_vchan(chan);
1016 unsigned long flags;
1017
1018 spin_lock_irqsave(&vchan->vc.lock, flags);
1019
1020 /*
1021 * If there are pending transactions for this vchan, push one of
1022 * them into the engine to get the ball rolling.
1023 */
1024 if (vchan_issue_pending(&vchan->vc))
1025 __execute_vchan_pending(priv, vchan);
1026
1027 spin_unlock_irqrestore(&vchan->vc.lock, flags);
1028}
1029
1030static irqreturn_t sun4i_dma_interrupt(int irq, void *dev_id)
1031{
1032 struct sun4i_dma_dev *priv = dev_id;
1033 struct sun4i_dma_pchan *pchans = priv->pchans, *pchan;
1034 struct sun4i_dma_vchan *vchan;
1035 struct sun4i_dma_contract *contract;
1036 struct sun4i_dma_promise *promise;
1037 unsigned long pendirq, irqs, disableirqs;
1038 int bit, i, free_room, allow_mitigation = 1;
1039
1040 pendirq = readl_relaxed(priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1041
1042handle_pending:
1043
1044 disableirqs = 0;
1045 free_room = 0;
1046
1047 for_each_set_bit(bit, &pendirq, 32) {
1048 pchan = &pchans[bit >> 1];
1049 vchan = pchan->vchan;
1050 if (!vchan) /* a terminated channel may still interrupt */
1051 continue;
1052 contract = vchan->contract;
1053
1054 /*
1055 * Disable the IRQ and free the pchan if it's an end
1056 * interrupt (odd bit)
1057 */
1058 if (bit & 1) {
1059 spin_lock(&vchan->vc.lock);
1060
1061 /*
1062 * Move the promise into the completed list now that
1063 * we're done with it
1064 */
1065 list_move_tail(&vchan->processing->list,
1066 &contract->completed_demands);
1067
1068 /*
1069 * Cyclic DMA transfers are special:
1070 * - There's always something we can dispatch
1071 * - We need to run the callback
1072 * - Latency is very important, as this is used by audio
1073 * We therefore just cycle through the list and dispatch
1074 * whatever we have here, reusing the pchan. There's
1075 * no need to run the thread after this.
1076 *
1077 * For non-cyclic transfers we need to look around,
1078 * so we can program some more work, or notify the
1079 * client that their transfers have been completed.
1080 */
1081 if (contract->is_cyclic) {
1082 promise = get_next_cyclic_promise(contract);
1083 vchan->processing = promise;
1084 configure_pchan(pchan, promise);
1085 vchan_cyclic_callback(&contract->vd);
1086 } else {
1087 vchan->processing = NULL;
1088 vchan->pchan = NULL;
1089
1090 free_room = 1;
1091 disableirqs |= BIT(bit);
1092 release_pchan(priv, pchan);
1093 }
1094
1095 spin_unlock(&vchan->vc.lock);
1096 } else {
1097 /* Half done interrupt */
1098 if (contract->is_cyclic)
1099 vchan_cyclic_callback(&contract->vd);
1100 else
1101 disableirqs |= BIT(bit);
1102 }
1103 }
1104
1105 /* Disable the IRQs for events we handled */
1106 spin_lock(&priv->lock);
1107 irqs = readl_relaxed(priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1108 writel_relaxed(irqs & ~disableirqs,
1109 priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1110 spin_unlock(&priv->lock);
1111
1112 /* Writing 1 to the pending field will clear the pending interrupt */
1113 writel_relaxed(pendirq, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1114
1115 /*
1116 * If a pchan was freed, we may be able to schedule something else,
1117 * so have a look around
1118 */
1119 if (free_room) {
1120 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1121 vchan = &priv->vchans[i];
1122 spin_lock(&vchan->vc.lock);
1123 __execute_vchan_pending(priv, vchan);
1124 spin_unlock(&vchan->vc.lock);
1125 }
1126 }
1127
1128 /*
1129 * Handle newer interrupts if some showed up, but only do it once
1130 * to avoid a too long a loop
1131 */
1132 if (allow_mitigation) {
1133 pendirq = readl_relaxed(priv->base +
1134 SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1135 if (pendirq) {
1136 allow_mitigation = 0;
1137 goto handle_pending;
1138 }
1139 }
1140
1141 return IRQ_HANDLED;
1142}
1143
1144static int sun4i_dma_probe(struct platform_device *pdev)
1145{
1146 struct sun4i_dma_dev *priv;
1147 struct resource *res;
1148 int i, j, ret;
1149
1150 priv = devm_kzalloc(&pdev->dev, sizeof(*priv), GFP_KERNEL);
1151 if (!priv)
1152 return -ENOMEM;
1153
1154 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
1155 priv->base = devm_ioremap_resource(&pdev->dev, res);
1156 if (IS_ERR(priv->base))
1157 return PTR_ERR(priv->base);
1158
1159 priv->irq = platform_get_irq(pdev, 0);
1160 if (priv->irq < 0)
1161 return priv->irq;
1162
1163 priv->clk = devm_clk_get(&pdev->dev, NULL);
1164 if (IS_ERR(priv->clk)) {
1165 dev_err(&pdev->dev, "No clock specified\n");
1166 return PTR_ERR(priv->clk);
1167 }
1168
1169 platform_set_drvdata(pdev, priv);
1170 spin_lock_init(&priv->lock);
1171
1172 dma_set_max_seg_size(&pdev->dev, SUN4I_DMA_MAX_SEG_SIZE);
1173
1174 dma_cap_zero(priv->slave.cap_mask);
1175 dma_cap_set(DMA_PRIVATE, priv->slave.cap_mask);
1176 dma_cap_set(DMA_MEMCPY, priv->slave.cap_mask);
1177 dma_cap_set(DMA_CYCLIC, priv->slave.cap_mask);
1178 dma_cap_set(DMA_SLAVE, priv->slave.cap_mask);
1179
1180 INIT_LIST_HEAD(&priv->slave.channels);
1181 priv->slave.device_free_chan_resources = sun4i_dma_free_chan_resources;
1182 priv->slave.device_tx_status = sun4i_dma_tx_status;
1183 priv->slave.device_issue_pending = sun4i_dma_issue_pending;
1184 priv->slave.device_prep_slave_sg = sun4i_dma_prep_slave_sg;
1185 priv->slave.device_prep_dma_memcpy = sun4i_dma_prep_dma_memcpy;
1186 priv->slave.device_prep_dma_cyclic = sun4i_dma_prep_dma_cyclic;
1187 priv->slave.device_config = sun4i_dma_config;
1188 priv->slave.device_terminate_all = sun4i_dma_terminate_all;
1189 priv->slave.copy_align = 2;
1190 priv->slave.src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1191 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1192 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1193 priv->slave.dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) |
1194 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) |
1195 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
1196 priv->slave.directions = BIT(DMA_DEV_TO_MEM) |
1197 BIT(DMA_MEM_TO_DEV);
1198 priv->slave.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1199
1200 priv->slave.dev = &pdev->dev;
1201
1202 priv->pchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_CHANNELS,
1203 sizeof(struct sun4i_dma_pchan), GFP_KERNEL);
1204 priv->vchans = devm_kcalloc(&pdev->dev, SUN4I_DMA_NR_MAX_VCHANS,
1205 sizeof(struct sun4i_dma_vchan), GFP_KERNEL);
1206 if (!priv->vchans || !priv->pchans)
1207 return -ENOMEM;
1208
1209 /*
1210 * [0..SUN4I_NDMA_NR_MAX_CHANNELS) are normal pchans, and
1211 * [SUN4I_NDMA_NR_MAX_CHANNELS..SUN4I_DMA_NR_MAX_CHANNELS) are
1212 * dedicated ones
1213 */
1214 for (i = 0; i < SUN4I_NDMA_NR_MAX_CHANNELS; i++)
1215 priv->pchans[i].base = priv->base +
1216 SUN4I_NDMA_CHANNEL_REG_BASE(i);
1217
1218 for (j = 0; i < SUN4I_DMA_NR_MAX_CHANNELS; i++, j++) {
1219 priv->pchans[i].base = priv->base +
1220 SUN4I_DDMA_CHANNEL_REG_BASE(j);
1221 priv->pchans[i].is_dedicated = 1;
1222 }
1223
1224 for (i = 0; i < SUN4I_DMA_NR_MAX_VCHANS; i++) {
1225 struct sun4i_dma_vchan *vchan = &priv->vchans[i];
1226
1227 spin_lock_init(&vchan->vc.lock);
1228 vchan->vc.desc_free = sun4i_dma_free_contract;
1229 vchan_init(&vchan->vc, &priv->slave);
1230 }
1231
1232 ret = clk_prepare_enable(priv->clk);
1233 if (ret) {
1234 dev_err(&pdev->dev, "Couldn't enable the clock\n");
1235 return ret;
1236 }
1237
1238 /*
1239 * Make sure the IRQs are all disabled and accounted for. The bootloader
1240 * likes to leave these dirty
1241 */
1242 writel(0, priv->base + SUN4I_DMA_IRQ_ENABLE_REG);
1243 writel(0xFFFFFFFF, priv->base + SUN4I_DMA_IRQ_PENDING_STATUS_REG);
1244
1245 ret = devm_request_irq(&pdev->dev, priv->irq, sun4i_dma_interrupt,
1246 0, dev_name(&pdev->dev), priv);
1247 if (ret) {
1248 dev_err(&pdev->dev, "Cannot request IRQ\n");
1249 goto err_clk_disable;
1250 }
1251
1252 ret = dma_async_device_register(&priv->slave);
1253 if (ret) {
1254 dev_warn(&pdev->dev, "Failed to register DMA engine device\n");
1255 goto err_clk_disable;
1256 }
1257
1258 ret = of_dma_controller_register(pdev->dev.of_node, sun4i_dma_of_xlate,
1259 priv);
1260 if (ret) {
1261 dev_err(&pdev->dev, "of_dma_controller_register failed\n");
1262 goto err_dma_unregister;
1263 }
1264
1265 dev_dbg(&pdev->dev, "Successfully probed SUN4I_DMA\n");
1266
1267 return 0;
1268
1269err_dma_unregister:
1270 dma_async_device_unregister(&priv->slave);
1271err_clk_disable:
1272 clk_disable_unprepare(priv->clk);
1273 return ret;
1274}
1275
1276static int sun4i_dma_remove(struct platform_device *pdev)
1277{
1278 struct sun4i_dma_dev *priv = platform_get_drvdata(pdev);
1279
1280 /* Disable IRQ so no more work is scheduled */
1281 disable_irq(priv->irq);
1282
1283 of_dma_controller_free(pdev->dev.of_node);
1284 dma_async_device_unregister(&priv->slave);
1285
1286 clk_disable_unprepare(priv->clk);
1287
1288 return 0;
1289}
1290
1291static const struct of_device_id sun4i_dma_match[] = {
1292 { .compatible = "allwinner,sun4i-a10-dma" },
1293 { /* sentinel */ },
1294};
1295MODULE_DEVICE_TABLE(of, sun4i_dma_match);
1296
1297static struct platform_driver sun4i_dma_driver = {
1298 .probe = sun4i_dma_probe,
1299 .remove = sun4i_dma_remove,
1300 .driver = {
1301 .name = "sun4i-dma",
1302 .of_match_table = sun4i_dma_match,
1303 },
1304};
1305
1306module_platform_driver(sun4i_dma_driver);
1307
1308MODULE_DESCRIPTION("Allwinner A10 Dedicated DMA Controller Driver");
1309MODULE_AUTHOR("Emilio López <emilio@elopez.com.ar>");
1310MODULE_LICENSE("GPL");