Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * DMA driver for Altera mSGDMA IP core
4 *
5 * Copyright (C) 2017 Stefan Roese <sr@denx.de>
6 *
7 * Based on drivers/dma/xilinx/zynqmp_dma.c, which is:
8 * Copyright (C) 2016 Xilinx, Inc. All rights reserved.
9 */
10
11#include <linux/bitops.h>
12#include <linux/delay.h>
13#include <linux/dma-mapping.h>
14#include <linux/dmapool.h>
15#include <linux/init.h>
16#include <linux/interrupt.h>
17#include <linux/io.h>
18#include <linux/iopoll.h>
19#include <linux/module.h>
20#include <linux/platform_device.h>
21#include <linux/slab.h>
22#include <linux/of_dma.h>
23
24#include "dmaengine.h"
25
26#define MSGDMA_MAX_TRANS_LEN U32_MAX
27#define MSGDMA_DESC_NUM 1024
28
29/**
30 * struct msgdma_extended_desc - implements an extended descriptor
31 * @read_addr_lo: data buffer source address low bits
32 * @write_addr_lo: data buffer destination address low bits
33 * @len: the number of bytes to transfer per descriptor
34 * @burst_seq_num: bit 31:24 write burst
35 * bit 23:16 read burst
36 * bit 15:00 sequence number
37 * @stride: bit 31:16 write stride
38 * bit 15:00 read stride
39 * @read_addr_hi: data buffer source address high bits
40 * @write_addr_hi: data buffer destination address high bits
41 * @control: characteristics of the transfer
42 */
43struct msgdma_extended_desc {
44 u32 read_addr_lo;
45 u32 write_addr_lo;
46 u32 len;
47 u32 burst_seq_num;
48 u32 stride;
49 u32 read_addr_hi;
50 u32 write_addr_hi;
51 u32 control;
52};
53
54/* mSGDMA descriptor control field bit definitions */
55#define MSGDMA_DESC_CTL_SET_CH(x) ((x) & 0xff)
56#define MSGDMA_DESC_CTL_GEN_SOP BIT(8)
57#define MSGDMA_DESC_CTL_GEN_EOP BIT(9)
58#define MSGDMA_DESC_CTL_PARK_READS BIT(10)
59#define MSGDMA_DESC_CTL_PARK_WRITES BIT(11)
60#define MSGDMA_DESC_CTL_END_ON_EOP BIT(12)
61#define MSGDMA_DESC_CTL_END_ON_LEN BIT(13)
62#define MSGDMA_DESC_CTL_TR_COMP_IRQ BIT(14)
63#define MSGDMA_DESC_CTL_EARLY_IRQ BIT(15)
64#define MSGDMA_DESC_CTL_TR_ERR_IRQ GENMASK(23, 16)
65#define MSGDMA_DESC_CTL_EARLY_DONE BIT(24)
66
67/*
68 * Writing "1" the "go" bit commits the entire descriptor into the
69 * descriptor FIFO(s)
70 */
71#define MSGDMA_DESC_CTL_GO BIT(31)
72
73/* Tx buffer control flags */
74#define MSGDMA_DESC_CTL_TX_FIRST (MSGDMA_DESC_CTL_GEN_SOP | \
75 MSGDMA_DESC_CTL_TR_ERR_IRQ | \
76 MSGDMA_DESC_CTL_GO)
77
78#define MSGDMA_DESC_CTL_TX_MIDDLE (MSGDMA_DESC_CTL_TR_ERR_IRQ | \
79 MSGDMA_DESC_CTL_GO)
80
81#define MSGDMA_DESC_CTL_TX_LAST (MSGDMA_DESC_CTL_GEN_EOP | \
82 MSGDMA_DESC_CTL_TR_COMP_IRQ | \
83 MSGDMA_DESC_CTL_TR_ERR_IRQ | \
84 MSGDMA_DESC_CTL_GO)
85
86#define MSGDMA_DESC_CTL_TX_SINGLE (MSGDMA_DESC_CTL_GEN_SOP | \
87 MSGDMA_DESC_CTL_GEN_EOP | \
88 MSGDMA_DESC_CTL_TR_COMP_IRQ | \
89 MSGDMA_DESC_CTL_TR_ERR_IRQ | \
90 MSGDMA_DESC_CTL_GO)
91
92#define MSGDMA_DESC_CTL_RX_SINGLE (MSGDMA_DESC_CTL_END_ON_EOP | \
93 MSGDMA_DESC_CTL_END_ON_LEN | \
94 MSGDMA_DESC_CTL_TR_COMP_IRQ | \
95 MSGDMA_DESC_CTL_EARLY_IRQ | \
96 MSGDMA_DESC_CTL_TR_ERR_IRQ | \
97 MSGDMA_DESC_CTL_GO)
98
99/* mSGDMA extended descriptor stride definitions */
100#define MSGDMA_DESC_STRIDE_RD 0x00000001
101#define MSGDMA_DESC_STRIDE_WR 0x00010000
102#define MSGDMA_DESC_STRIDE_RW 0x00010001
103
104/* mSGDMA dispatcher control and status register map */
105#define MSGDMA_CSR_STATUS 0x00 /* Read / Clear */
106#define MSGDMA_CSR_CONTROL 0x04 /* Read / Write */
107#define MSGDMA_CSR_RW_FILL_LEVEL 0x08 /* 31:16 - write fill level */
108 /* 15:00 - read fill level */
109#define MSGDMA_CSR_RESP_FILL_LEVEL 0x0c /* response FIFO fill level */
110#define MSGDMA_CSR_RW_SEQ_NUM 0x10 /* 31:16 - write seq number */
111 /* 15:00 - read seq number */
112
113/* mSGDMA CSR status register bit definitions */
114#define MSGDMA_CSR_STAT_BUSY BIT(0)
115#define MSGDMA_CSR_STAT_DESC_BUF_EMPTY BIT(1)
116#define MSGDMA_CSR_STAT_DESC_BUF_FULL BIT(2)
117#define MSGDMA_CSR_STAT_RESP_BUF_EMPTY BIT(3)
118#define MSGDMA_CSR_STAT_RESP_BUF_FULL BIT(4)
119#define MSGDMA_CSR_STAT_STOPPED BIT(5)
120#define MSGDMA_CSR_STAT_RESETTING BIT(6)
121#define MSGDMA_CSR_STAT_STOPPED_ON_ERR BIT(7)
122#define MSGDMA_CSR_STAT_STOPPED_ON_EARLY BIT(8)
123#define MSGDMA_CSR_STAT_IRQ BIT(9)
124#define MSGDMA_CSR_STAT_MASK GENMASK(9, 0)
125#define MSGDMA_CSR_STAT_MASK_WITHOUT_IRQ GENMASK(8, 0)
126
127#define DESC_EMPTY (MSGDMA_CSR_STAT_DESC_BUF_EMPTY | \
128 MSGDMA_CSR_STAT_RESP_BUF_EMPTY)
129
130/* mSGDMA CSR control register bit definitions */
131#define MSGDMA_CSR_CTL_STOP BIT(0)
132#define MSGDMA_CSR_CTL_RESET BIT(1)
133#define MSGDMA_CSR_CTL_STOP_ON_ERR BIT(2)
134#define MSGDMA_CSR_CTL_STOP_ON_EARLY BIT(3)
135#define MSGDMA_CSR_CTL_GLOBAL_INTR BIT(4)
136#define MSGDMA_CSR_CTL_STOP_DESCS BIT(5)
137
138/* mSGDMA CSR fill level bits */
139#define MSGDMA_CSR_WR_FILL_LEVEL_GET(v) (((v) & 0xffff0000) >> 16)
140#define MSGDMA_CSR_RD_FILL_LEVEL_GET(v) ((v) & 0x0000ffff)
141#define MSGDMA_CSR_RESP_FILL_LEVEL_GET(v) ((v) & 0x0000ffff)
142
143#define MSGDMA_CSR_SEQ_NUM_GET(v) (((v) & 0xffff0000) >> 16)
144
145/* mSGDMA response register map */
146#define MSGDMA_RESP_BYTES_TRANSFERRED 0x00
147#define MSGDMA_RESP_STATUS 0x04
148
149/* mSGDMA response register bit definitions */
150#define MSGDMA_RESP_EARLY_TERM BIT(8)
151#define MSGDMA_RESP_ERR_MASK 0xff
152
153/**
154 * struct msgdma_sw_desc - implements a sw descriptor
155 * @async_tx: support for the async_tx api
156 * @hw_desc: assosiated HW descriptor
157 * @node: node to move from the free list to the tx list
158 * @tx_list: transmit list node
159 */
160struct msgdma_sw_desc {
161 struct dma_async_tx_descriptor async_tx;
162 struct msgdma_extended_desc hw_desc;
163 struct list_head node;
164 struct list_head tx_list;
165};
166
167/*
168 * struct msgdma_device - DMA device structure
169 */
170struct msgdma_device {
171 spinlock_t lock;
172 struct device *dev;
173 struct tasklet_struct irq_tasklet;
174 struct list_head pending_list;
175 struct list_head free_list;
176 struct list_head active_list;
177 struct list_head done_list;
178 u32 desc_free_cnt;
179 bool idle;
180
181 struct dma_device dmadev;
182 struct dma_chan dmachan;
183 dma_addr_t hw_desq;
184 struct msgdma_sw_desc *sw_desq;
185 unsigned int npendings;
186
187 struct dma_slave_config slave_cfg;
188
189 int irq;
190
191 /* mSGDMA controller */
192 void __iomem *csr;
193
194 /* mSGDMA descriptors */
195 void __iomem *desc;
196
197 /* mSGDMA response */
198 void __iomem *resp;
199};
200
201#define to_mdev(chan) container_of(chan, struct msgdma_device, dmachan)
202#define tx_to_desc(tx) container_of(tx, struct msgdma_sw_desc, async_tx)
203
204/**
205 * msgdma_get_descriptor - Get the sw descriptor from the pool
206 * @mdev: Pointer to the Altera mSGDMA device structure
207 *
208 * Return: The sw descriptor
209 */
210static struct msgdma_sw_desc *msgdma_get_descriptor(struct msgdma_device *mdev)
211{
212 struct msgdma_sw_desc *desc;
213 unsigned long flags;
214
215 spin_lock_irqsave(&mdev->lock, flags);
216 desc = list_first_entry(&mdev->free_list, struct msgdma_sw_desc, node);
217 list_del(&desc->node);
218 spin_unlock_irqrestore(&mdev->lock, flags);
219
220 INIT_LIST_HEAD(&desc->tx_list);
221
222 return desc;
223}
224
225/**
226 * msgdma_free_descriptor - Issue pending transactions
227 * @mdev: Pointer to the Altera mSGDMA device structure
228 * @desc: Transaction descriptor pointer
229 */
230static void msgdma_free_descriptor(struct msgdma_device *mdev,
231 struct msgdma_sw_desc *desc)
232{
233 struct msgdma_sw_desc *child, *next;
234
235 mdev->desc_free_cnt++;
236 list_add_tail(&desc->node, &mdev->free_list);
237 list_for_each_entry_safe(child, next, &desc->tx_list, node) {
238 mdev->desc_free_cnt++;
239 list_move_tail(&child->node, &mdev->free_list);
240 }
241}
242
243/**
244 * msgdma_free_desc_list - Free descriptors list
245 * @mdev: Pointer to the Altera mSGDMA device structure
246 * @list: List to parse and delete the descriptor
247 */
248static void msgdma_free_desc_list(struct msgdma_device *mdev,
249 struct list_head *list)
250{
251 struct msgdma_sw_desc *desc, *next;
252
253 list_for_each_entry_safe(desc, next, list, node)
254 msgdma_free_descriptor(mdev, desc);
255}
256
257/**
258 * msgdma_desc_config - Configure the descriptor
259 * @desc: Hw descriptor pointer
260 * @dst: Destination buffer address
261 * @src: Source buffer address
262 * @len: Transfer length
263 * @stride: Read/write stride value to set
264 */
265static void msgdma_desc_config(struct msgdma_extended_desc *desc,
266 dma_addr_t dst, dma_addr_t src, size_t len,
267 u32 stride)
268{
269 /* Set lower 32bits of src & dst addresses in the descriptor */
270 desc->read_addr_lo = lower_32_bits(src);
271 desc->write_addr_lo = lower_32_bits(dst);
272
273 /* Set upper 32bits of src & dst addresses in the descriptor */
274 desc->read_addr_hi = upper_32_bits(src);
275 desc->write_addr_hi = upper_32_bits(dst);
276
277 desc->len = len;
278 desc->stride = stride;
279 desc->burst_seq_num = 0; /* 0 will result in max burst length */
280
281 /*
282 * Don't set interrupt on xfer end yet, this will be done later
283 * for the "last" descriptor
284 */
285 desc->control = MSGDMA_DESC_CTL_TR_ERR_IRQ | MSGDMA_DESC_CTL_GO |
286 MSGDMA_DESC_CTL_END_ON_LEN;
287}
288
289/**
290 * msgdma_desc_config_eod - Mark the descriptor as end descriptor
291 * @desc: Hw descriptor pointer
292 */
293static void msgdma_desc_config_eod(struct msgdma_extended_desc *desc)
294{
295 desc->control |= MSGDMA_DESC_CTL_TR_COMP_IRQ;
296}
297
298/**
299 * msgdma_tx_submit - Submit DMA transaction
300 * @tx: Async transaction descriptor pointer
301 *
302 * Return: cookie value
303 */
304static dma_cookie_t msgdma_tx_submit(struct dma_async_tx_descriptor *tx)
305{
306 struct msgdma_device *mdev = to_mdev(tx->chan);
307 struct msgdma_sw_desc *new;
308 dma_cookie_t cookie;
309 unsigned long flags;
310
311 new = tx_to_desc(tx);
312 spin_lock_irqsave(&mdev->lock, flags);
313 cookie = dma_cookie_assign(tx);
314
315 list_add_tail(&new->node, &mdev->pending_list);
316 spin_unlock_irqrestore(&mdev->lock, flags);
317
318 return cookie;
319}
320
321/**
322 * msgdma_prep_memcpy - prepare descriptors for memcpy transaction
323 * @dchan: DMA channel
324 * @dma_dst: Destination buffer address
325 * @dma_src: Source buffer address
326 * @len: Transfer length
327 * @flags: transfer ack flags
328 *
329 * Return: Async transaction descriptor on success and NULL on failure
330 */
331static struct dma_async_tx_descriptor *
332msgdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst,
333 dma_addr_t dma_src, size_t len, ulong flags)
334{
335 struct msgdma_device *mdev = to_mdev(dchan);
336 struct msgdma_sw_desc *new, *first = NULL;
337 struct msgdma_extended_desc *desc;
338 size_t copy;
339 u32 desc_cnt;
340 unsigned long irqflags;
341
342 desc_cnt = DIV_ROUND_UP(len, MSGDMA_MAX_TRANS_LEN);
343
344 spin_lock_irqsave(&mdev->lock, irqflags);
345 if (desc_cnt > mdev->desc_free_cnt) {
346 spin_unlock_irqrestore(&mdev->lock, irqflags);
347 dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev);
348 return NULL;
349 }
350 mdev->desc_free_cnt -= desc_cnt;
351 spin_unlock_irqrestore(&mdev->lock, irqflags);
352
353 do {
354 /* Allocate and populate the descriptor */
355 new = msgdma_get_descriptor(mdev);
356
357 copy = min_t(size_t, len, MSGDMA_MAX_TRANS_LEN);
358 desc = &new->hw_desc;
359 msgdma_desc_config(desc, dma_dst, dma_src, copy,
360 MSGDMA_DESC_STRIDE_RW);
361 len -= copy;
362 dma_src += copy;
363 dma_dst += copy;
364 if (!first)
365 first = new;
366 else
367 list_add_tail(&new->node, &first->tx_list);
368 } while (len);
369
370 msgdma_desc_config_eod(desc);
371 async_tx_ack(&first->async_tx);
372 first->async_tx.flags = flags;
373
374 return &first->async_tx;
375}
376
377/**
378 * msgdma_prep_slave_sg - prepare descriptors for a slave sg transaction
379 *
380 * @dchan: DMA channel
381 * @sgl: Destination scatter list
382 * @sg_len: Number of entries in destination scatter list
383 * @dir: DMA transfer direction
384 * @flags: transfer ack flags
385 * @context: transfer context (unused)
386 */
387static struct dma_async_tx_descriptor *
388msgdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
389 unsigned int sg_len, enum dma_transfer_direction dir,
390 unsigned long flags, void *context)
391
392{
393 struct msgdma_device *mdev = to_mdev(dchan);
394 struct dma_slave_config *cfg = &mdev->slave_cfg;
395 struct msgdma_sw_desc *new, *first = NULL;
396 void *desc = NULL;
397 size_t len, avail;
398 dma_addr_t dma_dst, dma_src;
399 u32 desc_cnt = 0, i;
400 struct scatterlist *sg;
401 u32 stride;
402 unsigned long irqflags;
403
404 for_each_sg(sgl, sg, sg_len, i)
405 desc_cnt += DIV_ROUND_UP(sg_dma_len(sg), MSGDMA_MAX_TRANS_LEN);
406
407 spin_lock_irqsave(&mdev->lock, irqflags);
408 if (desc_cnt > mdev->desc_free_cnt) {
409 spin_unlock_irqrestore(&mdev->lock, irqflags);
410 dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev);
411 return NULL;
412 }
413 mdev->desc_free_cnt -= desc_cnt;
414 spin_unlock_irqrestore(&mdev->lock, irqflags);
415
416 avail = sg_dma_len(sgl);
417
418 /* Run until we are out of scatterlist entries */
419 while (true) {
420 /* Allocate and populate the descriptor */
421 new = msgdma_get_descriptor(mdev);
422
423 desc = &new->hw_desc;
424 len = min_t(size_t, avail, MSGDMA_MAX_TRANS_LEN);
425
426 if (dir == DMA_MEM_TO_DEV) {
427 dma_src = sg_dma_address(sgl) + sg_dma_len(sgl) - avail;
428 dma_dst = cfg->dst_addr;
429 stride = MSGDMA_DESC_STRIDE_RD;
430 } else {
431 dma_src = cfg->src_addr;
432 dma_dst = sg_dma_address(sgl) + sg_dma_len(sgl) - avail;
433 stride = MSGDMA_DESC_STRIDE_WR;
434 }
435 msgdma_desc_config(desc, dma_dst, dma_src, len, stride);
436 avail -= len;
437
438 if (!first)
439 first = new;
440 else
441 list_add_tail(&new->node, &first->tx_list);
442
443 /* Fetch the next scatterlist entry */
444 if (avail == 0) {
445 if (sg_len == 0)
446 break;
447 sgl = sg_next(sgl);
448 if (sgl == NULL)
449 break;
450 sg_len--;
451 avail = sg_dma_len(sgl);
452 }
453 }
454
455 msgdma_desc_config_eod(desc);
456 first->async_tx.flags = flags;
457
458 return &first->async_tx;
459}
460
461static int msgdma_dma_config(struct dma_chan *dchan,
462 struct dma_slave_config *config)
463{
464 struct msgdma_device *mdev = to_mdev(dchan);
465
466 memcpy(&mdev->slave_cfg, config, sizeof(*config));
467
468 return 0;
469}
470
471static void msgdma_reset(struct msgdma_device *mdev)
472{
473 u32 val;
474 int ret;
475
476 /* Reset mSGDMA */
477 iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS);
478 iowrite32(MSGDMA_CSR_CTL_RESET, mdev->csr + MSGDMA_CSR_CONTROL);
479
480 ret = readl_poll_timeout(mdev->csr + MSGDMA_CSR_STATUS, val,
481 (val & MSGDMA_CSR_STAT_RESETTING) == 0,
482 1, 10000);
483 if (ret)
484 dev_err(mdev->dev, "DMA channel did not reset\n");
485
486 /* Clear all status bits */
487 iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS);
488
489 /* Enable the DMA controller including interrupts */
490 iowrite32(MSGDMA_CSR_CTL_STOP_ON_ERR | MSGDMA_CSR_CTL_STOP_ON_EARLY |
491 MSGDMA_CSR_CTL_GLOBAL_INTR, mdev->csr + MSGDMA_CSR_CONTROL);
492
493 mdev->idle = true;
494};
495
496static void msgdma_copy_one(struct msgdma_device *mdev,
497 struct msgdma_sw_desc *desc)
498{
499 void __iomem *hw_desc = mdev->desc;
500
501 /*
502 * Check if the DESC FIFO it not full. If its full, we need to wait
503 * for at least one entry to become free again
504 */
505 while (ioread32(mdev->csr + MSGDMA_CSR_STATUS) &
506 MSGDMA_CSR_STAT_DESC_BUF_FULL)
507 mdelay(1);
508
509 /*
510 * The descriptor needs to get copied into the descriptor FIFO
511 * of the DMA controller. The descriptor will get flushed to the
512 * FIFO, once the last word (control word) is written. Since we
513 * are not 100% sure that memcpy() writes all word in the "correct"
514 * oder (address from low to high) on all architectures, we make
515 * sure this control word is written last by single coding it and
516 * adding some write-barriers here.
517 */
518 memcpy((void __force *)hw_desc, &desc->hw_desc,
519 sizeof(desc->hw_desc) - sizeof(u32));
520
521 /* Write control word last to flush this descriptor into the FIFO */
522 mdev->idle = false;
523 wmb();
524 iowrite32(desc->hw_desc.control, hw_desc +
525 offsetof(struct msgdma_extended_desc, control));
526 wmb();
527}
528
529/**
530 * msgdma_copy_desc_to_fifo - copy descriptor(s) into controller FIFO
531 * @mdev: Pointer to the Altera mSGDMA device structure
532 * @desc: Transaction descriptor pointer
533 */
534static void msgdma_copy_desc_to_fifo(struct msgdma_device *mdev,
535 struct msgdma_sw_desc *desc)
536{
537 struct msgdma_sw_desc *sdesc, *next;
538
539 msgdma_copy_one(mdev, desc);
540
541 list_for_each_entry_safe(sdesc, next, &desc->tx_list, node)
542 msgdma_copy_one(mdev, sdesc);
543}
544
545/**
546 * msgdma_start_transfer - Initiate the new transfer
547 * @mdev: Pointer to the Altera mSGDMA device structure
548 */
549static void msgdma_start_transfer(struct msgdma_device *mdev)
550{
551 struct msgdma_sw_desc *desc;
552
553 if (!mdev->idle)
554 return;
555
556 desc = list_first_entry_or_null(&mdev->pending_list,
557 struct msgdma_sw_desc, node);
558 if (!desc)
559 return;
560
561 list_splice_tail_init(&mdev->pending_list, &mdev->active_list);
562 msgdma_copy_desc_to_fifo(mdev, desc);
563}
564
565/**
566 * msgdma_issue_pending - Issue pending transactions
567 * @chan: DMA channel pointer
568 */
569static void msgdma_issue_pending(struct dma_chan *chan)
570{
571 struct msgdma_device *mdev = to_mdev(chan);
572 unsigned long flags;
573
574 spin_lock_irqsave(&mdev->lock, flags);
575 msgdma_start_transfer(mdev);
576 spin_unlock_irqrestore(&mdev->lock, flags);
577}
578
579/**
580 * msgdma_chan_desc_cleanup - Cleanup the completed descriptors
581 * @mdev: Pointer to the Altera mSGDMA device structure
582 */
583static void msgdma_chan_desc_cleanup(struct msgdma_device *mdev)
584{
585 struct msgdma_sw_desc *desc, *next;
586
587 list_for_each_entry_safe(desc, next, &mdev->done_list, node) {
588 struct dmaengine_desc_callback cb;
589
590 list_del(&desc->node);
591
592 dmaengine_desc_get_callback(&desc->async_tx, &cb);
593 if (dmaengine_desc_callback_valid(&cb)) {
594 spin_unlock(&mdev->lock);
595 dmaengine_desc_callback_invoke(&cb, NULL);
596 spin_lock(&mdev->lock);
597 }
598
599 /* Run any dependencies, then free the descriptor */
600 msgdma_free_descriptor(mdev, desc);
601 }
602}
603
604/**
605 * msgdma_complete_descriptor - Mark the active descriptor as complete
606 * @mdev: Pointer to the Altera mSGDMA device structure
607 */
608static void msgdma_complete_descriptor(struct msgdma_device *mdev)
609{
610 struct msgdma_sw_desc *desc;
611
612 desc = list_first_entry_or_null(&mdev->active_list,
613 struct msgdma_sw_desc, node);
614 if (!desc)
615 return;
616 list_del(&desc->node);
617 dma_cookie_complete(&desc->async_tx);
618 list_add_tail(&desc->node, &mdev->done_list);
619}
620
621/**
622 * msgdma_free_descriptors - Free channel descriptors
623 * @mdev: Pointer to the Altera mSGDMA device structure
624 */
625static void msgdma_free_descriptors(struct msgdma_device *mdev)
626{
627 msgdma_free_desc_list(mdev, &mdev->active_list);
628 msgdma_free_desc_list(mdev, &mdev->pending_list);
629 msgdma_free_desc_list(mdev, &mdev->done_list);
630}
631
632/**
633 * msgdma_free_chan_resources - Free channel resources
634 * @dchan: DMA channel pointer
635 */
636static void msgdma_free_chan_resources(struct dma_chan *dchan)
637{
638 struct msgdma_device *mdev = to_mdev(dchan);
639 unsigned long flags;
640
641 spin_lock_irqsave(&mdev->lock, flags);
642 msgdma_free_descriptors(mdev);
643 spin_unlock_irqrestore(&mdev->lock, flags);
644 kfree(mdev->sw_desq);
645}
646
647/**
648 * msgdma_alloc_chan_resources - Allocate channel resources
649 * @dchan: DMA channel
650 *
651 * Return: Number of descriptors on success and failure value on error
652 */
653static int msgdma_alloc_chan_resources(struct dma_chan *dchan)
654{
655 struct msgdma_device *mdev = to_mdev(dchan);
656 struct msgdma_sw_desc *desc;
657 int i;
658
659 mdev->sw_desq = kcalloc(MSGDMA_DESC_NUM, sizeof(*desc), GFP_NOWAIT);
660 if (!mdev->sw_desq)
661 return -ENOMEM;
662
663 mdev->idle = true;
664 mdev->desc_free_cnt = MSGDMA_DESC_NUM;
665
666 INIT_LIST_HEAD(&mdev->free_list);
667
668 for (i = 0; i < MSGDMA_DESC_NUM; i++) {
669 desc = mdev->sw_desq + i;
670 dma_async_tx_descriptor_init(&desc->async_tx, &mdev->dmachan);
671 desc->async_tx.tx_submit = msgdma_tx_submit;
672 list_add_tail(&desc->node, &mdev->free_list);
673 }
674
675 return MSGDMA_DESC_NUM;
676}
677
678/**
679 * msgdma_tasklet - Schedule completion tasklet
680 * @t: Pointer to the Altera sSGDMA channel structure
681 */
682static void msgdma_tasklet(struct tasklet_struct *t)
683{
684 struct msgdma_device *mdev = from_tasklet(mdev, t, irq_tasklet);
685 u32 count;
686 u32 __maybe_unused size;
687 u32 __maybe_unused status;
688 unsigned long flags;
689
690 spin_lock_irqsave(&mdev->lock, flags);
691
692 if (mdev->resp) {
693 /* Read number of responses that are available */
694 count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL);
695 dev_dbg(mdev->dev, "%s (%d): response count=%d\n",
696 __func__, __LINE__, count);
697 } else {
698 count = 1;
699 }
700
701 while (count--) {
702 /*
703 * Read both longwords to purge this response from the FIFO
704 * On Avalon-MM implementations, size and status do not
705 * have any real values, like transferred bytes or error
706 * bits. So we need to just drop these values.
707 */
708 if (mdev->resp) {
709 size = ioread32(mdev->resp +
710 MSGDMA_RESP_BYTES_TRANSFERRED);
711 status = ioread32(mdev->resp +
712 MSGDMA_RESP_STATUS);
713 }
714
715 msgdma_complete_descriptor(mdev);
716 msgdma_chan_desc_cleanup(mdev);
717 }
718
719 spin_unlock_irqrestore(&mdev->lock, flags);
720}
721
722/**
723 * msgdma_irq_handler - Altera mSGDMA Interrupt handler
724 * @irq: IRQ number
725 * @data: Pointer to the Altera mSGDMA device structure
726 *
727 * Return: IRQ_HANDLED/IRQ_NONE
728 */
729static irqreturn_t msgdma_irq_handler(int irq, void *data)
730{
731 struct msgdma_device *mdev = data;
732 u32 status;
733
734 status = ioread32(mdev->csr + MSGDMA_CSR_STATUS);
735 if ((status & MSGDMA_CSR_STAT_BUSY) == 0) {
736 /* Start next transfer if the DMA controller is idle */
737 spin_lock(&mdev->lock);
738 mdev->idle = true;
739 msgdma_start_transfer(mdev);
740 spin_unlock(&mdev->lock);
741 }
742
743 tasklet_schedule(&mdev->irq_tasklet);
744
745 /* Clear interrupt in mSGDMA controller */
746 iowrite32(MSGDMA_CSR_STAT_IRQ, mdev->csr + MSGDMA_CSR_STATUS);
747
748 return IRQ_HANDLED;
749}
750
751/**
752 * msgdma_dev_remove() - Device remove function
753 * @mdev: Pointer to the Altera mSGDMA device structure
754 */
755static void msgdma_dev_remove(struct msgdma_device *mdev)
756{
757 if (!mdev)
758 return;
759
760 devm_free_irq(mdev->dev, mdev->irq, mdev);
761 tasklet_kill(&mdev->irq_tasklet);
762 list_del(&mdev->dmachan.device_node);
763}
764
765static int request_and_map(struct platform_device *pdev, const char *name,
766 struct resource **res, void __iomem **ptr,
767 bool optional)
768{
769 struct resource *region;
770 struct device *device = &pdev->dev;
771
772 *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
773 if (*res == NULL) {
774 if (optional) {
775 *ptr = NULL;
776 dev_info(device, "optional resource %s not defined\n",
777 name);
778 return 0;
779 }
780 dev_err(device, "mandatory resource %s not defined\n", name);
781 return -ENODEV;
782 }
783
784 region = devm_request_mem_region(device, (*res)->start,
785 resource_size(*res), dev_name(device));
786 if (region == NULL) {
787 dev_err(device, "unable to request %s\n", name);
788 return -EBUSY;
789 }
790
791 *ptr = devm_ioremap(device, region->start,
792 resource_size(region));
793 if (*ptr == NULL) {
794 dev_err(device, "ioremap of %s failed!", name);
795 return -ENOMEM;
796 }
797
798 return 0;
799}
800
801/**
802 * msgdma_probe - Driver probe function
803 * @pdev: Pointer to the platform_device structure
804 *
805 * Return: '0' on success and failure value on error
806 */
807static int msgdma_probe(struct platform_device *pdev)
808{
809 struct msgdma_device *mdev;
810 struct dma_device *dma_dev;
811 struct resource *dma_res;
812 int ret;
813
814 mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_NOWAIT);
815 if (!mdev)
816 return -ENOMEM;
817
818 mdev->dev = &pdev->dev;
819
820 /* Map CSR space */
821 ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr, false);
822 if (ret)
823 return ret;
824
825 /* Map (extended) descriptor space */
826 ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc, false);
827 if (ret)
828 return ret;
829
830 /* Map response space */
831 ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp, true);
832 if (ret)
833 return ret;
834
835 platform_set_drvdata(pdev, mdev);
836
837 /* Get interrupt nr from platform data */
838 mdev->irq = platform_get_irq(pdev, 0);
839 if (mdev->irq < 0)
840 return -ENXIO;
841
842 ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler,
843 0, dev_name(&pdev->dev), mdev);
844 if (ret)
845 return ret;
846
847 tasklet_setup(&mdev->irq_tasklet, msgdma_tasklet);
848
849 dma_cookie_init(&mdev->dmachan);
850
851 spin_lock_init(&mdev->lock);
852
853 INIT_LIST_HEAD(&mdev->active_list);
854 INIT_LIST_HEAD(&mdev->pending_list);
855 INIT_LIST_HEAD(&mdev->done_list);
856 INIT_LIST_HEAD(&mdev->free_list);
857
858 dma_dev = &mdev->dmadev;
859
860 /* Set DMA capabilities */
861 dma_cap_zero(dma_dev->cap_mask);
862 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
863 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
864
865 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
866 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
867 dma_dev->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) |
868 BIT(DMA_MEM_TO_MEM);
869 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
870
871 /* Init DMA link list */
872 INIT_LIST_HEAD(&dma_dev->channels);
873
874 /* Set base routines */
875 dma_dev->device_tx_status = dma_cookie_status;
876 dma_dev->device_issue_pending = msgdma_issue_pending;
877 dma_dev->dev = &pdev->dev;
878
879 dma_dev->copy_align = DMAENGINE_ALIGN_4_BYTES;
880 dma_dev->device_prep_dma_memcpy = msgdma_prep_memcpy;
881 dma_dev->device_prep_slave_sg = msgdma_prep_slave_sg;
882 dma_dev->device_config = msgdma_dma_config;
883
884 dma_dev->device_alloc_chan_resources = msgdma_alloc_chan_resources;
885 dma_dev->device_free_chan_resources = msgdma_free_chan_resources;
886
887 mdev->dmachan.device = dma_dev;
888 list_add_tail(&mdev->dmachan.device_node, &dma_dev->channels);
889
890 /* Set DMA mask to 64 bits */
891 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
892 if (ret) {
893 dev_warn(&pdev->dev, "unable to set coherent mask to 64");
894 goto fail;
895 }
896
897 msgdma_reset(mdev);
898
899 ret = dma_async_device_register(dma_dev);
900 if (ret)
901 goto fail;
902
903 ret = of_dma_controller_register(pdev->dev.of_node,
904 of_dma_xlate_by_chan_id, dma_dev);
905 if (ret == -EINVAL)
906 dev_warn(&pdev->dev, "device was not probed from DT");
907 else if (ret && ret != -ENODEV)
908 goto fail;
909
910 dev_notice(&pdev->dev, "Altera mSGDMA driver probe success\n");
911
912 return 0;
913
914fail:
915 msgdma_dev_remove(mdev);
916
917 return ret;
918}
919
920/**
921 * msgdma_remove() - Driver remove function
922 * @pdev: Pointer to the platform_device structure
923 *
924 * Return: Always '0'
925 */
926static void msgdma_remove(struct platform_device *pdev)
927{
928 struct msgdma_device *mdev = platform_get_drvdata(pdev);
929
930 if (pdev->dev.of_node)
931 of_dma_controller_free(pdev->dev.of_node);
932 dma_async_device_unregister(&mdev->dmadev);
933 msgdma_dev_remove(mdev);
934
935 dev_notice(&pdev->dev, "Altera mSGDMA driver removed\n");
936}
937
938#ifdef CONFIG_OF
939static const struct of_device_id msgdma_match[] = {
940 { .compatible = "altr,socfpga-msgdma", },
941 { }
942};
943
944MODULE_DEVICE_TABLE(of, msgdma_match);
945#endif
946
947static struct platform_driver msgdma_driver = {
948 .driver = {
949 .name = "altera-msgdma",
950 .of_match_table = of_match_ptr(msgdma_match),
951 },
952 .probe = msgdma_probe,
953 .remove_new = msgdma_remove,
954};
955
956module_platform_driver(msgdma_driver);
957
958MODULE_ALIAS("platform:altera-msgdma");
959MODULE_DESCRIPTION("Altera mSGDMA driver");
960MODULE_AUTHOR("Stefan Roese <sr@denx.de>");
961MODULE_LICENSE("GPL");
1/*
2 * DMA driver for Altera mSGDMA IP core
3 *
4 * Copyright (C) 2017 Stefan Roese <sr@denx.de>
5 *
6 * Based on drivers/dma/xilinx/zynqmp_dma.c, which is:
7 * Copyright (C) 2016 Xilinx, Inc. All rights reserved.
8 *
9 * This program is free software: you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation, either version 2 of the License, or
12 * (at your option) any later version.
13 */
14
15#include <linux/bitops.h>
16#include <linux/delay.h>
17#include <linux/dma-mapping.h>
18#include <linux/dmapool.h>
19#include <linux/init.h>
20#include <linux/interrupt.h>
21#include <linux/io.h>
22#include <linux/iopoll.h>
23#include <linux/module.h>
24#include <linux/platform_device.h>
25#include <linux/slab.h>
26
27#include "dmaengine.h"
28
29#define MSGDMA_MAX_TRANS_LEN U32_MAX
30#define MSGDMA_DESC_NUM 1024
31
32/**
33 * struct msgdma_extended_desc - implements an extended descriptor
34 * @read_addr_lo: data buffer source address low bits
35 * @write_addr_lo: data buffer destination address low bits
36 * @len: the number of bytes to transfer per descriptor
37 * @burst_seq_num: bit 31:24 write burst
38 * bit 23:16 read burst
39 * bit 15:00 sequence number
40 * @stride: bit 31:16 write stride
41 * bit 15:00 read stride
42 * @read_addr_hi: data buffer source address high bits
43 * @write_addr_hi: data buffer destination address high bits
44 * @control: characteristics of the transfer
45 */
46struct msgdma_extended_desc {
47 u32 read_addr_lo;
48 u32 write_addr_lo;
49 u32 len;
50 u32 burst_seq_num;
51 u32 stride;
52 u32 read_addr_hi;
53 u32 write_addr_hi;
54 u32 control;
55};
56
57/* mSGDMA descriptor control field bit definitions */
58#define MSGDMA_DESC_CTL_SET_CH(x) ((x) & 0xff)
59#define MSGDMA_DESC_CTL_GEN_SOP BIT(8)
60#define MSGDMA_DESC_CTL_GEN_EOP BIT(9)
61#define MSGDMA_DESC_CTL_PARK_READS BIT(10)
62#define MSGDMA_DESC_CTL_PARK_WRITES BIT(11)
63#define MSGDMA_DESC_CTL_END_ON_EOP BIT(12)
64#define MSGDMA_DESC_CTL_END_ON_LEN BIT(13)
65#define MSGDMA_DESC_CTL_TR_COMP_IRQ BIT(14)
66#define MSGDMA_DESC_CTL_EARLY_IRQ BIT(15)
67#define MSGDMA_DESC_CTL_TR_ERR_IRQ GENMASK(23, 16)
68#define MSGDMA_DESC_CTL_EARLY_DONE BIT(24)
69
70/*
71 * Writing "1" the "go" bit commits the entire descriptor into the
72 * descriptor FIFO(s)
73 */
74#define MSGDMA_DESC_CTL_GO BIT(31)
75
76/* Tx buffer control flags */
77#define MSGDMA_DESC_CTL_TX_FIRST (MSGDMA_DESC_CTL_GEN_SOP | \
78 MSGDMA_DESC_CTL_TR_ERR_IRQ | \
79 MSGDMA_DESC_CTL_GO)
80
81#define MSGDMA_DESC_CTL_TX_MIDDLE (MSGDMA_DESC_CTL_TR_ERR_IRQ | \
82 MSGDMA_DESC_CTL_GO)
83
84#define MSGDMA_DESC_CTL_TX_LAST (MSGDMA_DESC_CTL_GEN_EOP | \
85 MSGDMA_DESC_CTL_TR_COMP_IRQ | \
86 MSGDMA_DESC_CTL_TR_ERR_IRQ | \
87 MSGDMA_DESC_CTL_GO)
88
89#define MSGDMA_DESC_CTL_TX_SINGLE (MSGDMA_DESC_CTL_GEN_SOP | \
90 MSGDMA_DESC_CTL_GEN_EOP | \
91 MSGDMA_DESC_CTL_TR_COMP_IRQ | \
92 MSGDMA_DESC_CTL_TR_ERR_IRQ | \
93 MSGDMA_DESC_CTL_GO)
94
95#define MSGDMA_DESC_CTL_RX_SINGLE (MSGDMA_DESC_CTL_END_ON_EOP | \
96 MSGDMA_DESC_CTL_END_ON_LEN | \
97 MSGDMA_DESC_CTL_TR_COMP_IRQ | \
98 MSGDMA_DESC_CTL_EARLY_IRQ | \
99 MSGDMA_DESC_CTL_TR_ERR_IRQ | \
100 MSGDMA_DESC_CTL_GO)
101
102/* mSGDMA extended descriptor stride definitions */
103#define MSGDMA_DESC_STRIDE_RD 0x00000001
104#define MSGDMA_DESC_STRIDE_WR 0x00010000
105#define MSGDMA_DESC_STRIDE_RW 0x00010001
106
107/* mSGDMA dispatcher control and status register map */
108#define MSGDMA_CSR_STATUS 0x00 /* Read / Clear */
109#define MSGDMA_CSR_CONTROL 0x04 /* Read / Write */
110#define MSGDMA_CSR_RW_FILL_LEVEL 0x08 /* 31:16 - write fill level */
111 /* 15:00 - read fill level */
112#define MSGDMA_CSR_RESP_FILL_LEVEL 0x0c /* response FIFO fill level */
113#define MSGDMA_CSR_RW_SEQ_NUM 0x10 /* 31:16 - write seq number */
114 /* 15:00 - read seq number */
115
116/* mSGDMA CSR status register bit definitions */
117#define MSGDMA_CSR_STAT_BUSY BIT(0)
118#define MSGDMA_CSR_STAT_DESC_BUF_EMPTY BIT(1)
119#define MSGDMA_CSR_STAT_DESC_BUF_FULL BIT(2)
120#define MSGDMA_CSR_STAT_RESP_BUF_EMPTY BIT(3)
121#define MSGDMA_CSR_STAT_RESP_BUF_FULL BIT(4)
122#define MSGDMA_CSR_STAT_STOPPED BIT(5)
123#define MSGDMA_CSR_STAT_RESETTING BIT(6)
124#define MSGDMA_CSR_STAT_STOPPED_ON_ERR BIT(7)
125#define MSGDMA_CSR_STAT_STOPPED_ON_EARLY BIT(8)
126#define MSGDMA_CSR_STAT_IRQ BIT(9)
127#define MSGDMA_CSR_STAT_MASK GENMASK(9, 0)
128#define MSGDMA_CSR_STAT_MASK_WITHOUT_IRQ GENMASK(8, 0)
129
130#define DESC_EMPTY (MSGDMA_CSR_STAT_DESC_BUF_EMPTY | \
131 MSGDMA_CSR_STAT_RESP_BUF_EMPTY)
132
133/* mSGDMA CSR control register bit definitions */
134#define MSGDMA_CSR_CTL_STOP BIT(0)
135#define MSGDMA_CSR_CTL_RESET BIT(1)
136#define MSGDMA_CSR_CTL_STOP_ON_ERR BIT(2)
137#define MSGDMA_CSR_CTL_STOP_ON_EARLY BIT(3)
138#define MSGDMA_CSR_CTL_GLOBAL_INTR BIT(4)
139#define MSGDMA_CSR_CTL_STOP_DESCS BIT(5)
140
141/* mSGDMA CSR fill level bits */
142#define MSGDMA_CSR_WR_FILL_LEVEL_GET(v) (((v) & 0xffff0000) >> 16)
143#define MSGDMA_CSR_RD_FILL_LEVEL_GET(v) ((v) & 0x0000ffff)
144#define MSGDMA_CSR_RESP_FILL_LEVEL_GET(v) ((v) & 0x0000ffff)
145
146#define MSGDMA_CSR_SEQ_NUM_GET(v) (((v) & 0xffff0000) >> 16)
147
148/* mSGDMA response register map */
149#define MSGDMA_RESP_BYTES_TRANSFERRED 0x00
150#define MSGDMA_RESP_STATUS 0x04
151
152/* mSGDMA response register bit definitions */
153#define MSGDMA_RESP_EARLY_TERM BIT(8)
154#define MSGDMA_RESP_ERR_MASK 0xff
155
156/**
157 * struct msgdma_sw_desc - implements a sw descriptor
158 * @async_tx: support for the async_tx api
159 * @hw_desc: assosiated HW descriptor
160 * @free_list: node of the free SW descriprots list
161 */
162struct msgdma_sw_desc {
163 struct dma_async_tx_descriptor async_tx;
164 struct msgdma_extended_desc hw_desc;
165 struct list_head node;
166 struct list_head tx_list;
167};
168
169/**
170 * struct msgdma_device - DMA device structure
171 */
172struct msgdma_device {
173 spinlock_t lock;
174 struct device *dev;
175 struct tasklet_struct irq_tasklet;
176 struct list_head pending_list;
177 struct list_head free_list;
178 struct list_head active_list;
179 struct list_head done_list;
180 u32 desc_free_cnt;
181 bool idle;
182
183 struct dma_device dmadev;
184 struct dma_chan dmachan;
185 dma_addr_t hw_desq;
186 struct msgdma_sw_desc *sw_desq;
187 unsigned int npendings;
188
189 struct dma_slave_config slave_cfg;
190
191 int irq;
192
193 /* mSGDMA controller */
194 void __iomem *csr;
195
196 /* mSGDMA descriptors */
197 void __iomem *desc;
198
199 /* mSGDMA response */
200 void __iomem *resp;
201};
202
203#define to_mdev(chan) container_of(chan, struct msgdma_device, dmachan)
204#define tx_to_desc(tx) container_of(tx, struct msgdma_sw_desc, async_tx)
205
206/**
207 * msgdma_get_descriptor - Get the sw descriptor from the pool
208 * @mdev: Pointer to the Altera mSGDMA device structure
209 *
210 * Return: The sw descriptor
211 */
212static struct msgdma_sw_desc *msgdma_get_descriptor(struct msgdma_device *mdev)
213{
214 struct msgdma_sw_desc *desc;
215 unsigned long flags;
216
217 spin_lock_irqsave(&mdev->lock, flags);
218 desc = list_first_entry(&mdev->free_list, struct msgdma_sw_desc, node);
219 list_del(&desc->node);
220 spin_unlock_irqrestore(&mdev->lock, flags);
221
222 INIT_LIST_HEAD(&desc->tx_list);
223
224 return desc;
225}
226
227/**
228 * msgdma_free_descriptor - Issue pending transactions
229 * @mdev: Pointer to the Altera mSGDMA device structure
230 * @desc: Transaction descriptor pointer
231 */
232static void msgdma_free_descriptor(struct msgdma_device *mdev,
233 struct msgdma_sw_desc *desc)
234{
235 struct msgdma_sw_desc *child, *next;
236
237 mdev->desc_free_cnt++;
238 list_add_tail(&desc->node, &mdev->free_list);
239 list_for_each_entry_safe(child, next, &desc->tx_list, node) {
240 mdev->desc_free_cnt++;
241 list_move_tail(&child->node, &mdev->free_list);
242 }
243}
244
245/**
246 * msgdma_free_desc_list - Free descriptors list
247 * @mdev: Pointer to the Altera mSGDMA device structure
248 * @list: List to parse and delete the descriptor
249 */
250static void msgdma_free_desc_list(struct msgdma_device *mdev,
251 struct list_head *list)
252{
253 struct msgdma_sw_desc *desc, *next;
254
255 list_for_each_entry_safe(desc, next, list, node)
256 msgdma_free_descriptor(mdev, desc);
257}
258
259/**
260 * msgdma_desc_config - Configure the descriptor
261 * @desc: Hw descriptor pointer
262 * @dst: Destination buffer address
263 * @src: Source buffer address
264 * @len: Transfer length
265 */
266static void msgdma_desc_config(struct msgdma_extended_desc *desc,
267 dma_addr_t dst, dma_addr_t src, size_t len,
268 u32 stride)
269{
270 /* Set lower 32bits of src & dst addresses in the descriptor */
271 desc->read_addr_lo = lower_32_bits(src);
272 desc->write_addr_lo = lower_32_bits(dst);
273
274 /* Set upper 32bits of src & dst addresses in the descriptor */
275 desc->read_addr_hi = upper_32_bits(src);
276 desc->write_addr_hi = upper_32_bits(dst);
277
278 desc->len = len;
279 desc->stride = stride;
280 desc->burst_seq_num = 0; /* 0 will result in max burst length */
281
282 /*
283 * Don't set interrupt on xfer end yet, this will be done later
284 * for the "last" descriptor
285 */
286 desc->control = MSGDMA_DESC_CTL_TR_ERR_IRQ | MSGDMA_DESC_CTL_GO |
287 MSGDMA_DESC_CTL_END_ON_LEN;
288}
289
290/**
291 * msgdma_desc_config_eod - Mark the descriptor as end descriptor
292 * @desc: Hw descriptor pointer
293 */
294static void msgdma_desc_config_eod(struct msgdma_extended_desc *desc)
295{
296 desc->control |= MSGDMA_DESC_CTL_TR_COMP_IRQ;
297}
298
299/**
300 * msgdma_tx_submit - Submit DMA transaction
301 * @tx: Async transaction descriptor pointer
302 *
303 * Return: cookie value
304 */
305static dma_cookie_t msgdma_tx_submit(struct dma_async_tx_descriptor *tx)
306{
307 struct msgdma_device *mdev = to_mdev(tx->chan);
308 struct msgdma_sw_desc *new;
309 dma_cookie_t cookie;
310 unsigned long flags;
311
312 new = tx_to_desc(tx);
313 spin_lock_irqsave(&mdev->lock, flags);
314 cookie = dma_cookie_assign(tx);
315
316 list_add_tail(&new->node, &mdev->pending_list);
317 spin_unlock_irqrestore(&mdev->lock, flags);
318
319 return cookie;
320}
321
322/**
323 * msgdma_prep_memcpy - prepare descriptors for memcpy transaction
324 * @dchan: DMA channel
325 * @dma_dst: Destination buffer address
326 * @dma_src: Source buffer address
327 * @len: Transfer length
328 * @flags: transfer ack flags
329 *
330 * Return: Async transaction descriptor on success and NULL on failure
331 */
332static struct dma_async_tx_descriptor *
333msgdma_prep_memcpy(struct dma_chan *dchan, dma_addr_t dma_dst,
334 dma_addr_t dma_src, size_t len, ulong flags)
335{
336 struct msgdma_device *mdev = to_mdev(dchan);
337 struct msgdma_sw_desc *new, *first = NULL;
338 struct msgdma_extended_desc *desc;
339 size_t copy;
340 u32 desc_cnt;
341 unsigned long irqflags;
342
343 desc_cnt = DIV_ROUND_UP(len, MSGDMA_MAX_TRANS_LEN);
344
345 spin_lock_irqsave(&mdev->lock, irqflags);
346 if (desc_cnt > mdev->desc_free_cnt) {
347 spin_unlock_irqrestore(&mdev->lock, irqflags);
348 dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev);
349 return NULL;
350 }
351 mdev->desc_free_cnt -= desc_cnt;
352 spin_unlock_irqrestore(&mdev->lock, irqflags);
353
354 do {
355 /* Allocate and populate the descriptor */
356 new = msgdma_get_descriptor(mdev);
357
358 copy = min_t(size_t, len, MSGDMA_MAX_TRANS_LEN);
359 desc = &new->hw_desc;
360 msgdma_desc_config(desc, dma_dst, dma_src, copy,
361 MSGDMA_DESC_STRIDE_RW);
362 len -= copy;
363 dma_src += copy;
364 dma_dst += copy;
365 if (!first)
366 first = new;
367 else
368 list_add_tail(&new->node, &first->tx_list);
369 } while (len);
370
371 msgdma_desc_config_eod(desc);
372 async_tx_ack(&first->async_tx);
373 first->async_tx.flags = flags;
374
375 return &first->async_tx;
376}
377
378/**
379 * msgdma_prep_slave_sg - prepare descriptors for a slave sg transaction
380 *
381 * @dchan: DMA channel
382 * @sgl: Destination scatter list
383 * @sg_len: Number of entries in destination scatter list
384 * @dir: DMA transfer direction
385 * @flags: transfer ack flags
386 * @context: transfer context (unused)
387 */
388static struct dma_async_tx_descriptor *
389msgdma_prep_slave_sg(struct dma_chan *dchan, struct scatterlist *sgl,
390 unsigned int sg_len, enum dma_transfer_direction dir,
391 unsigned long flags, void *context)
392
393{
394 struct msgdma_device *mdev = to_mdev(dchan);
395 struct dma_slave_config *cfg = &mdev->slave_cfg;
396 struct msgdma_sw_desc *new, *first = NULL;
397 void *desc = NULL;
398 size_t len, avail;
399 dma_addr_t dma_dst, dma_src;
400 u32 desc_cnt = 0, i;
401 struct scatterlist *sg;
402 u32 stride;
403 unsigned long irqflags;
404
405 for_each_sg(sgl, sg, sg_len, i)
406 desc_cnt += DIV_ROUND_UP(sg_dma_len(sg), MSGDMA_MAX_TRANS_LEN);
407
408 spin_lock_irqsave(&mdev->lock, irqflags);
409 if (desc_cnt > mdev->desc_free_cnt) {
410 spin_unlock_irqrestore(&mdev->lock, irqflags);
411 dev_dbg(mdev->dev, "mdev %p descs are not available\n", mdev);
412 return NULL;
413 }
414 mdev->desc_free_cnt -= desc_cnt;
415 spin_unlock_irqrestore(&mdev->lock, irqflags);
416
417 avail = sg_dma_len(sgl);
418
419 /* Run until we are out of scatterlist entries */
420 while (true) {
421 /* Allocate and populate the descriptor */
422 new = msgdma_get_descriptor(mdev);
423
424 desc = &new->hw_desc;
425 len = min_t(size_t, avail, MSGDMA_MAX_TRANS_LEN);
426
427 if (dir == DMA_MEM_TO_DEV) {
428 dma_src = sg_dma_address(sgl) + sg_dma_len(sgl) - avail;
429 dma_dst = cfg->dst_addr;
430 stride = MSGDMA_DESC_STRIDE_RD;
431 } else {
432 dma_src = cfg->src_addr;
433 dma_dst = sg_dma_address(sgl) + sg_dma_len(sgl) - avail;
434 stride = MSGDMA_DESC_STRIDE_WR;
435 }
436 msgdma_desc_config(desc, dma_dst, dma_src, len, stride);
437 avail -= len;
438
439 if (!first)
440 first = new;
441 else
442 list_add_tail(&new->node, &first->tx_list);
443
444 /* Fetch the next scatterlist entry */
445 if (avail == 0) {
446 if (sg_len == 0)
447 break;
448 sgl = sg_next(sgl);
449 if (sgl == NULL)
450 break;
451 sg_len--;
452 avail = sg_dma_len(sgl);
453 }
454 }
455
456 msgdma_desc_config_eod(desc);
457 first->async_tx.flags = flags;
458
459 return &first->async_tx;
460}
461
462static int msgdma_dma_config(struct dma_chan *dchan,
463 struct dma_slave_config *config)
464{
465 struct msgdma_device *mdev = to_mdev(dchan);
466
467 memcpy(&mdev->slave_cfg, config, sizeof(*config));
468
469 return 0;
470}
471
472static void msgdma_reset(struct msgdma_device *mdev)
473{
474 u32 val;
475 int ret;
476
477 /* Reset mSGDMA */
478 iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS);
479 iowrite32(MSGDMA_CSR_CTL_RESET, mdev->csr + MSGDMA_CSR_CONTROL);
480
481 ret = readl_poll_timeout(mdev->csr + MSGDMA_CSR_STATUS, val,
482 (val & MSGDMA_CSR_STAT_RESETTING) == 0,
483 1, 10000);
484 if (ret)
485 dev_err(mdev->dev, "DMA channel did not reset\n");
486
487 /* Clear all status bits */
488 iowrite32(MSGDMA_CSR_STAT_MASK, mdev->csr + MSGDMA_CSR_STATUS);
489
490 /* Enable the DMA controller including interrupts */
491 iowrite32(MSGDMA_CSR_CTL_STOP_ON_ERR | MSGDMA_CSR_CTL_STOP_ON_EARLY |
492 MSGDMA_CSR_CTL_GLOBAL_INTR, mdev->csr + MSGDMA_CSR_CONTROL);
493
494 mdev->idle = true;
495};
496
497static void msgdma_copy_one(struct msgdma_device *mdev,
498 struct msgdma_sw_desc *desc)
499{
500 void __iomem *hw_desc = mdev->desc;
501
502 /*
503 * Check if the DESC FIFO it not full. If its full, we need to wait
504 * for at least one entry to become free again
505 */
506 while (ioread32(mdev->csr + MSGDMA_CSR_STATUS) &
507 MSGDMA_CSR_STAT_DESC_BUF_FULL)
508 mdelay(1);
509
510 /*
511 * The descriptor needs to get copied into the descriptor FIFO
512 * of the DMA controller. The descriptor will get flushed to the
513 * FIFO, once the last word (control word) is written. Since we
514 * are not 100% sure that memcpy() writes all word in the "correct"
515 * oder (address from low to high) on all architectures, we make
516 * sure this control word is written last by single coding it and
517 * adding some write-barriers here.
518 */
519 memcpy((void __force *)hw_desc, &desc->hw_desc,
520 sizeof(desc->hw_desc) - sizeof(u32));
521
522 /* Write control word last to flush this descriptor into the FIFO */
523 mdev->idle = false;
524 wmb();
525 iowrite32(desc->hw_desc.control, hw_desc +
526 offsetof(struct msgdma_extended_desc, control));
527 wmb();
528}
529
530/**
531 * msgdma_copy_desc_to_fifo - copy descriptor(s) into controller FIFO
532 * @mdev: Pointer to the Altera mSGDMA device structure
533 * @desc: Transaction descriptor pointer
534 */
535static void msgdma_copy_desc_to_fifo(struct msgdma_device *mdev,
536 struct msgdma_sw_desc *desc)
537{
538 struct msgdma_sw_desc *sdesc, *next;
539
540 msgdma_copy_one(mdev, desc);
541
542 list_for_each_entry_safe(sdesc, next, &desc->tx_list, node)
543 msgdma_copy_one(mdev, sdesc);
544}
545
546/**
547 * msgdma_start_transfer - Initiate the new transfer
548 * @mdev: Pointer to the Altera mSGDMA device structure
549 */
550static void msgdma_start_transfer(struct msgdma_device *mdev)
551{
552 struct msgdma_sw_desc *desc;
553
554 if (!mdev->idle)
555 return;
556
557 desc = list_first_entry_or_null(&mdev->pending_list,
558 struct msgdma_sw_desc, node);
559 if (!desc)
560 return;
561
562 list_splice_tail_init(&mdev->pending_list, &mdev->active_list);
563 msgdma_copy_desc_to_fifo(mdev, desc);
564}
565
566/**
567 * msgdma_issue_pending - Issue pending transactions
568 * @chan: DMA channel pointer
569 */
570static void msgdma_issue_pending(struct dma_chan *chan)
571{
572 struct msgdma_device *mdev = to_mdev(chan);
573 unsigned long flags;
574
575 spin_lock_irqsave(&mdev->lock, flags);
576 msgdma_start_transfer(mdev);
577 spin_unlock_irqrestore(&mdev->lock, flags);
578}
579
580/**
581 * msgdma_chan_desc_cleanup - Cleanup the completed descriptors
582 * @mdev: Pointer to the Altera mSGDMA device structure
583 */
584static void msgdma_chan_desc_cleanup(struct msgdma_device *mdev)
585{
586 struct msgdma_sw_desc *desc, *next;
587
588 list_for_each_entry_safe(desc, next, &mdev->done_list, node) {
589 dma_async_tx_callback callback;
590 void *callback_param;
591
592 list_del(&desc->node);
593
594 callback = desc->async_tx.callback;
595 callback_param = desc->async_tx.callback_param;
596 if (callback) {
597 spin_unlock(&mdev->lock);
598 callback(callback_param);
599 spin_lock(&mdev->lock);
600 }
601
602 /* Run any dependencies, then free the descriptor */
603 msgdma_free_descriptor(mdev, desc);
604 }
605}
606
607/**
608 * msgdma_complete_descriptor - Mark the active descriptor as complete
609 * @mdev: Pointer to the Altera mSGDMA device structure
610 */
611static void msgdma_complete_descriptor(struct msgdma_device *mdev)
612{
613 struct msgdma_sw_desc *desc;
614
615 desc = list_first_entry_or_null(&mdev->active_list,
616 struct msgdma_sw_desc, node);
617 if (!desc)
618 return;
619 list_del(&desc->node);
620 dma_cookie_complete(&desc->async_tx);
621 list_add_tail(&desc->node, &mdev->done_list);
622}
623
624/**
625 * msgdma_free_descriptors - Free channel descriptors
626 * @mdev: Pointer to the Altera mSGDMA device structure
627 */
628static void msgdma_free_descriptors(struct msgdma_device *mdev)
629{
630 msgdma_free_desc_list(mdev, &mdev->active_list);
631 msgdma_free_desc_list(mdev, &mdev->pending_list);
632 msgdma_free_desc_list(mdev, &mdev->done_list);
633}
634
635/**
636 * msgdma_free_chan_resources - Free channel resources
637 * @dchan: DMA channel pointer
638 */
639static void msgdma_free_chan_resources(struct dma_chan *dchan)
640{
641 struct msgdma_device *mdev = to_mdev(dchan);
642 unsigned long flags;
643
644 spin_lock_irqsave(&mdev->lock, flags);
645 msgdma_free_descriptors(mdev);
646 spin_unlock_irqrestore(&mdev->lock, flags);
647 kfree(mdev->sw_desq);
648}
649
650/**
651 * msgdma_alloc_chan_resources - Allocate channel resources
652 * @dchan: DMA channel
653 *
654 * Return: Number of descriptors on success and failure value on error
655 */
656static int msgdma_alloc_chan_resources(struct dma_chan *dchan)
657{
658 struct msgdma_device *mdev = to_mdev(dchan);
659 struct msgdma_sw_desc *desc;
660 int i;
661
662 mdev->sw_desq = kcalloc(MSGDMA_DESC_NUM, sizeof(*desc), GFP_NOWAIT);
663 if (!mdev->sw_desq)
664 return -ENOMEM;
665
666 mdev->idle = true;
667 mdev->desc_free_cnt = MSGDMA_DESC_NUM;
668
669 INIT_LIST_HEAD(&mdev->free_list);
670
671 for (i = 0; i < MSGDMA_DESC_NUM; i++) {
672 desc = mdev->sw_desq + i;
673 dma_async_tx_descriptor_init(&desc->async_tx, &mdev->dmachan);
674 desc->async_tx.tx_submit = msgdma_tx_submit;
675 list_add_tail(&desc->node, &mdev->free_list);
676 }
677
678 return MSGDMA_DESC_NUM;
679}
680
681/**
682 * msgdma_tasklet - Schedule completion tasklet
683 * @data: Pointer to the Altera sSGDMA channel structure
684 */
685static void msgdma_tasklet(unsigned long data)
686{
687 struct msgdma_device *mdev = (struct msgdma_device *)data;
688 u32 count;
689 u32 __maybe_unused size;
690 u32 __maybe_unused status;
691 unsigned long flags;
692
693 spin_lock_irqsave(&mdev->lock, flags);
694
695 /* Read number of responses that are available */
696 count = ioread32(mdev->csr + MSGDMA_CSR_RESP_FILL_LEVEL);
697 dev_dbg(mdev->dev, "%s (%d): response count=%d\n",
698 __func__, __LINE__, count);
699
700 while (count--) {
701 /*
702 * Read both longwords to purge this response from the FIFO
703 * On Avalon-MM implementations, size and status do not
704 * have any real values, like transferred bytes or error
705 * bits. So we need to just drop these values.
706 */
707 size = ioread32(mdev->resp + MSGDMA_RESP_BYTES_TRANSFERRED);
708 status = ioread32(mdev->resp + MSGDMA_RESP_STATUS);
709
710 msgdma_complete_descriptor(mdev);
711 msgdma_chan_desc_cleanup(mdev);
712 }
713
714 spin_unlock_irqrestore(&mdev->lock, flags);
715}
716
717/**
718 * msgdma_irq_handler - Altera mSGDMA Interrupt handler
719 * @irq: IRQ number
720 * @data: Pointer to the Altera mSGDMA device structure
721 *
722 * Return: IRQ_HANDLED/IRQ_NONE
723 */
724static irqreturn_t msgdma_irq_handler(int irq, void *data)
725{
726 struct msgdma_device *mdev = data;
727 u32 status;
728
729 status = ioread32(mdev->csr + MSGDMA_CSR_STATUS);
730 if ((status & MSGDMA_CSR_STAT_BUSY) == 0) {
731 /* Start next transfer if the DMA controller is idle */
732 spin_lock(&mdev->lock);
733 mdev->idle = true;
734 msgdma_start_transfer(mdev);
735 spin_unlock(&mdev->lock);
736 }
737
738 tasklet_schedule(&mdev->irq_tasklet);
739
740 /* Clear interrupt in mSGDMA controller */
741 iowrite32(MSGDMA_CSR_STAT_IRQ, mdev->csr + MSGDMA_CSR_STATUS);
742
743 return IRQ_HANDLED;
744}
745
746/**
747 * msgdma_chan_remove - Channel remove function
748 * @mdev: Pointer to the Altera mSGDMA device structure
749 */
750static void msgdma_dev_remove(struct msgdma_device *mdev)
751{
752 if (!mdev)
753 return;
754
755 devm_free_irq(mdev->dev, mdev->irq, mdev);
756 tasklet_kill(&mdev->irq_tasklet);
757 list_del(&mdev->dmachan.device_node);
758}
759
760static int request_and_map(struct platform_device *pdev, const char *name,
761 struct resource **res, void __iomem **ptr)
762{
763 struct resource *region;
764 struct device *device = &pdev->dev;
765
766 *res = platform_get_resource_byname(pdev, IORESOURCE_MEM, name);
767 if (*res == NULL) {
768 dev_err(device, "resource %s not defined\n", name);
769 return -ENODEV;
770 }
771
772 region = devm_request_mem_region(device, (*res)->start,
773 resource_size(*res), dev_name(device));
774 if (region == NULL) {
775 dev_err(device, "unable to request %s\n", name);
776 return -EBUSY;
777 }
778
779 *ptr = devm_ioremap_nocache(device, region->start,
780 resource_size(region));
781 if (*ptr == NULL) {
782 dev_err(device, "ioremap_nocache of %s failed!", name);
783 return -ENOMEM;
784 }
785
786 return 0;
787}
788
789/**
790 * msgdma_probe - Driver probe function
791 * @pdev: Pointer to the platform_device structure
792 *
793 * Return: '0' on success and failure value on error
794 */
795static int msgdma_probe(struct platform_device *pdev)
796{
797 struct msgdma_device *mdev;
798 struct dma_device *dma_dev;
799 struct resource *dma_res;
800 int ret;
801
802 mdev = devm_kzalloc(&pdev->dev, sizeof(*mdev), GFP_NOWAIT);
803 if (!mdev)
804 return -ENOMEM;
805
806 mdev->dev = &pdev->dev;
807
808 /* Map CSR space */
809 ret = request_and_map(pdev, "csr", &dma_res, &mdev->csr);
810 if (ret)
811 return ret;
812
813 /* Map (extended) descriptor space */
814 ret = request_and_map(pdev, "desc", &dma_res, &mdev->desc);
815 if (ret)
816 return ret;
817
818 /* Map response space */
819 ret = request_and_map(pdev, "resp", &dma_res, &mdev->resp);
820 if (ret)
821 return ret;
822
823 platform_set_drvdata(pdev, mdev);
824
825 /* Get interrupt nr from platform data */
826 mdev->irq = platform_get_irq(pdev, 0);
827 if (mdev->irq < 0)
828 return -ENXIO;
829
830 ret = devm_request_irq(&pdev->dev, mdev->irq, msgdma_irq_handler,
831 0, dev_name(&pdev->dev), mdev);
832 if (ret)
833 return ret;
834
835 tasklet_init(&mdev->irq_tasklet, msgdma_tasklet, (unsigned long)mdev);
836
837 dma_cookie_init(&mdev->dmachan);
838
839 spin_lock_init(&mdev->lock);
840
841 INIT_LIST_HEAD(&mdev->active_list);
842 INIT_LIST_HEAD(&mdev->pending_list);
843 INIT_LIST_HEAD(&mdev->done_list);
844 INIT_LIST_HEAD(&mdev->free_list);
845
846 dma_dev = &mdev->dmadev;
847
848 /* Set DMA capabilities */
849 dma_cap_zero(dma_dev->cap_mask);
850 dma_cap_set(DMA_MEMCPY, dma_dev->cap_mask);
851 dma_cap_set(DMA_SLAVE, dma_dev->cap_mask);
852
853 dma_dev->src_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
854 dma_dev->dst_addr_widths = BIT(DMA_SLAVE_BUSWIDTH_4_BYTES);
855 dma_dev->directions = BIT(DMA_MEM_TO_DEV) | BIT(DMA_DEV_TO_MEM) |
856 BIT(DMA_MEM_TO_MEM);
857 dma_dev->residue_granularity = DMA_RESIDUE_GRANULARITY_DESCRIPTOR;
858
859 /* Init DMA link list */
860 INIT_LIST_HEAD(&dma_dev->channels);
861
862 /* Set base routines */
863 dma_dev->device_tx_status = dma_cookie_status;
864 dma_dev->device_issue_pending = msgdma_issue_pending;
865 dma_dev->dev = &pdev->dev;
866
867 dma_dev->copy_align = DMAENGINE_ALIGN_4_BYTES;
868 dma_dev->device_prep_dma_memcpy = msgdma_prep_memcpy;
869 dma_dev->device_prep_slave_sg = msgdma_prep_slave_sg;
870 dma_dev->device_config = msgdma_dma_config;
871
872 dma_dev->device_alloc_chan_resources = msgdma_alloc_chan_resources;
873 dma_dev->device_free_chan_resources = msgdma_free_chan_resources;
874
875 mdev->dmachan.device = dma_dev;
876 list_add_tail(&mdev->dmachan.device_node, &dma_dev->channels);
877
878 /* Set DMA mask to 64 bits */
879 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
880 if (ret) {
881 dev_warn(&pdev->dev, "unable to set coherent mask to 64");
882 ret = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
883 if (ret)
884 goto fail;
885 }
886
887 msgdma_reset(mdev);
888
889 ret = dma_async_device_register(dma_dev);
890 if (ret)
891 goto fail;
892
893 dev_notice(&pdev->dev, "Altera mSGDMA driver probe success\n");
894
895 return 0;
896
897fail:
898 msgdma_dev_remove(mdev);
899
900 return ret;
901}
902
903/**
904 * msgdma_dma_remove - Driver remove function
905 * @pdev: Pointer to the platform_device structure
906 *
907 * Return: Always '0'
908 */
909static int msgdma_remove(struct platform_device *pdev)
910{
911 struct msgdma_device *mdev = platform_get_drvdata(pdev);
912
913 dma_async_device_unregister(&mdev->dmadev);
914 msgdma_dev_remove(mdev);
915
916 dev_notice(&pdev->dev, "Altera mSGDMA driver removed\n");
917
918 return 0;
919}
920
921static struct platform_driver msgdma_driver = {
922 .driver = {
923 .name = "altera-msgdma",
924 },
925 .probe = msgdma_probe,
926 .remove = msgdma_remove,
927};
928
929module_platform_driver(msgdma_driver);
930
931MODULE_ALIAS("platform:altera-msgdma");
932MODULE_DESCRIPTION("Altera mSGDMA driver");
933MODULE_AUTHOR("Stefan Roese <sr@denx.de>");
934MODULE_LICENSE("GPL");