Loading...
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * DMA driver for STMicroelectronics STi FDMA controller
4 *
5 * Copyright (C) 2014 STMicroelectronics
6 *
7 * Author: Ludovic Barre <Ludovic.barre@st.com>
8 * Peter Griffin <peter.griffin@linaro.org>
9 */
10
11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/of_device.h>
14#include <linux/of_dma.h>
15#include <linux/platform_device.h>
16#include <linux/interrupt.h>
17#include <linux/remoteproc.h>
18#include <linux/slab.h>
19
20#include "st_fdma.h"
21
22static inline struct st_fdma_chan *to_st_fdma_chan(struct dma_chan *c)
23{
24 return container_of(c, struct st_fdma_chan, vchan.chan);
25}
26
27static struct st_fdma_desc *to_st_fdma_desc(struct virt_dma_desc *vd)
28{
29 return container_of(vd, struct st_fdma_desc, vdesc);
30}
31
32static int st_fdma_dreq_get(struct st_fdma_chan *fchan)
33{
34 struct st_fdma_dev *fdev = fchan->fdev;
35 u32 req_line_cfg = fchan->cfg.req_line;
36 u32 dreq_line;
37 int try = 0;
38
39 /*
40 * dreq_mask is shared for n channels of fdma, so all accesses must be
41 * atomic. if the dreq_mask is changed between ffz and set_bit,
42 * we retry
43 */
44 do {
45 if (fdev->dreq_mask == ~0L) {
46 dev_err(fdev->dev, "No req lines available\n");
47 return -EINVAL;
48 }
49
50 if (try || req_line_cfg >= ST_FDMA_NR_DREQS) {
51 dev_err(fdev->dev, "Invalid or used req line\n");
52 return -EINVAL;
53 } else {
54 dreq_line = req_line_cfg;
55 }
56
57 try++;
58 } while (test_and_set_bit(dreq_line, &fdev->dreq_mask));
59
60 dev_dbg(fdev->dev, "get dreq_line:%d mask:%#lx\n",
61 dreq_line, fdev->dreq_mask);
62
63 return dreq_line;
64}
65
66static void st_fdma_dreq_put(struct st_fdma_chan *fchan)
67{
68 struct st_fdma_dev *fdev = fchan->fdev;
69
70 dev_dbg(fdev->dev, "put dreq_line:%#x\n", fchan->dreq_line);
71 clear_bit(fchan->dreq_line, &fdev->dreq_mask);
72}
73
74static void st_fdma_xfer_desc(struct st_fdma_chan *fchan)
75{
76 struct virt_dma_desc *vdesc;
77 unsigned long nbytes, ch_cmd, cmd;
78
79 vdesc = vchan_next_desc(&fchan->vchan);
80 if (!vdesc)
81 return;
82
83 fchan->fdesc = to_st_fdma_desc(vdesc);
84 nbytes = fchan->fdesc->node[0].desc->nbytes;
85 cmd = FDMA_CMD_START(fchan->vchan.chan.chan_id);
86 ch_cmd = fchan->fdesc->node[0].pdesc | FDMA_CH_CMD_STA_START;
87
88 /* start the channel for the descriptor */
89 fnode_write(fchan, nbytes, FDMA_CNTN_OFST);
90 fchan_write(fchan, ch_cmd, FDMA_CH_CMD_OFST);
91 writel(cmd,
92 fchan->fdev->slim_rproc->peri + FDMA_CMD_SET_OFST);
93
94 dev_dbg(fchan->fdev->dev, "start chan:%d\n", fchan->vchan.chan.chan_id);
95}
96
97static void st_fdma_ch_sta_update(struct st_fdma_chan *fchan,
98 unsigned long int_sta)
99{
100 unsigned long ch_sta, ch_err;
101 int ch_id = fchan->vchan.chan.chan_id;
102 struct st_fdma_dev *fdev = fchan->fdev;
103
104 ch_sta = fchan_read(fchan, FDMA_CH_CMD_OFST);
105 ch_err = ch_sta & FDMA_CH_CMD_ERR_MASK;
106 ch_sta &= FDMA_CH_CMD_STA_MASK;
107
108 if (int_sta & FDMA_INT_STA_ERR) {
109 dev_warn(fdev->dev, "chan:%d, error:%ld\n", ch_id, ch_err);
110 fchan->status = DMA_ERROR;
111 return;
112 }
113
114 switch (ch_sta) {
115 case FDMA_CH_CMD_STA_PAUSED:
116 fchan->status = DMA_PAUSED;
117 break;
118
119 case FDMA_CH_CMD_STA_RUNNING:
120 fchan->status = DMA_IN_PROGRESS;
121 break;
122 }
123}
124
125static irqreturn_t st_fdma_irq_handler(int irq, void *dev_id)
126{
127 struct st_fdma_dev *fdev = dev_id;
128 irqreturn_t ret = IRQ_NONE;
129 struct st_fdma_chan *fchan = &fdev->chans[0];
130 unsigned long int_sta, clr;
131
132 int_sta = fdma_read(fdev, FDMA_INT_STA_OFST);
133 clr = int_sta;
134
135 for (; int_sta != 0 ; int_sta >>= 2, fchan++) {
136 if (!(int_sta & (FDMA_INT_STA_CH | FDMA_INT_STA_ERR)))
137 continue;
138
139 spin_lock(&fchan->vchan.lock);
140 st_fdma_ch_sta_update(fchan, int_sta);
141
142 if (fchan->fdesc) {
143 if (!fchan->fdesc->iscyclic) {
144 list_del(&fchan->fdesc->vdesc.node);
145 vchan_cookie_complete(&fchan->fdesc->vdesc);
146 fchan->fdesc = NULL;
147 fchan->status = DMA_COMPLETE;
148 } else {
149 vchan_cyclic_callback(&fchan->fdesc->vdesc);
150 }
151
152 /* Start the next descriptor (if available) */
153 if (!fchan->fdesc)
154 st_fdma_xfer_desc(fchan);
155 }
156
157 spin_unlock(&fchan->vchan.lock);
158 ret = IRQ_HANDLED;
159 }
160
161 fdma_write(fdev, clr, FDMA_INT_CLR_OFST);
162
163 return ret;
164}
165
166static struct dma_chan *st_fdma_of_xlate(struct of_phandle_args *dma_spec,
167 struct of_dma *ofdma)
168{
169 struct st_fdma_dev *fdev = ofdma->of_dma_data;
170 struct dma_chan *chan;
171 struct st_fdma_chan *fchan;
172 int ret;
173
174 if (dma_spec->args_count < 1)
175 return ERR_PTR(-EINVAL);
176
177 if (fdev->dma_device.dev->of_node != dma_spec->np)
178 return ERR_PTR(-EINVAL);
179
180 ret = rproc_boot(fdev->slim_rproc->rproc);
181 if (ret == -ENOENT)
182 return ERR_PTR(-EPROBE_DEFER);
183 else if (ret)
184 return ERR_PTR(ret);
185
186 chan = dma_get_any_slave_channel(&fdev->dma_device);
187 if (!chan)
188 goto err_chan;
189
190 fchan = to_st_fdma_chan(chan);
191
192 fchan->cfg.of_node = dma_spec->np;
193 fchan->cfg.req_line = dma_spec->args[0];
194 fchan->cfg.req_ctrl = 0;
195 fchan->cfg.type = ST_FDMA_TYPE_FREE_RUN;
196
197 if (dma_spec->args_count > 1)
198 fchan->cfg.req_ctrl = dma_spec->args[1]
199 & FDMA_REQ_CTRL_CFG_MASK;
200
201 if (dma_spec->args_count > 2)
202 fchan->cfg.type = dma_spec->args[2];
203
204 if (fchan->cfg.type == ST_FDMA_TYPE_FREE_RUN) {
205 fchan->dreq_line = 0;
206 } else {
207 fchan->dreq_line = st_fdma_dreq_get(fchan);
208 if (IS_ERR_VALUE(fchan->dreq_line)) {
209 chan = ERR_PTR(fchan->dreq_line);
210 goto err_chan;
211 }
212 }
213
214 dev_dbg(fdev->dev, "xlate req_line:%d type:%d req_ctrl:%#lx\n",
215 fchan->cfg.req_line, fchan->cfg.type, fchan->cfg.req_ctrl);
216
217 return chan;
218
219err_chan:
220 rproc_shutdown(fdev->slim_rproc->rproc);
221 return chan;
222
223}
224
225static void st_fdma_free_desc(struct virt_dma_desc *vdesc)
226{
227 struct st_fdma_desc *fdesc;
228 int i;
229
230 fdesc = to_st_fdma_desc(vdesc);
231 for (i = 0; i < fdesc->n_nodes; i++)
232 dma_pool_free(fdesc->fchan->node_pool, fdesc->node[i].desc,
233 fdesc->node[i].pdesc);
234 kfree(fdesc);
235}
236
237static struct st_fdma_desc *st_fdma_alloc_desc(struct st_fdma_chan *fchan,
238 int sg_len)
239{
240 struct st_fdma_desc *fdesc;
241 int i;
242
243 fdesc = kzalloc(struct_size(fdesc, node, sg_len), GFP_NOWAIT);
244 if (!fdesc)
245 return NULL;
246
247 fdesc->fchan = fchan;
248 fdesc->n_nodes = sg_len;
249 for (i = 0; i < sg_len; i++) {
250 fdesc->node[i].desc = dma_pool_alloc(fchan->node_pool,
251 GFP_NOWAIT, &fdesc->node[i].pdesc);
252 if (!fdesc->node[i].desc)
253 goto err;
254 }
255 return fdesc;
256
257err:
258 while (--i >= 0)
259 dma_pool_free(fchan->node_pool, fdesc->node[i].desc,
260 fdesc->node[i].pdesc);
261 kfree(fdesc);
262 return NULL;
263}
264
265static int st_fdma_alloc_chan_res(struct dma_chan *chan)
266{
267 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
268
269 /* Create the dma pool for descriptor allocation */
270 fchan->node_pool = dma_pool_create(dev_name(&chan->dev->device),
271 fchan->fdev->dev,
272 sizeof(struct st_fdma_hw_node),
273 __alignof__(struct st_fdma_hw_node),
274 0);
275
276 if (!fchan->node_pool) {
277 dev_err(fchan->fdev->dev, "unable to allocate desc pool\n");
278 return -ENOMEM;
279 }
280
281 dev_dbg(fchan->fdev->dev, "alloc ch_id:%d type:%d\n",
282 fchan->vchan.chan.chan_id, fchan->cfg.type);
283
284 return 0;
285}
286
287static void st_fdma_free_chan_res(struct dma_chan *chan)
288{
289 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
290 struct rproc *rproc = fchan->fdev->slim_rproc->rproc;
291 unsigned long flags;
292
293 dev_dbg(fchan->fdev->dev, "%s: freeing chan:%d\n",
294 __func__, fchan->vchan.chan.chan_id);
295
296 if (fchan->cfg.type != ST_FDMA_TYPE_FREE_RUN)
297 st_fdma_dreq_put(fchan);
298
299 spin_lock_irqsave(&fchan->vchan.lock, flags);
300 fchan->fdesc = NULL;
301 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
302
303 dma_pool_destroy(fchan->node_pool);
304 fchan->node_pool = NULL;
305 memset(&fchan->cfg, 0, sizeof(struct st_fdma_cfg));
306
307 rproc_shutdown(rproc);
308}
309
310static struct dma_async_tx_descriptor *st_fdma_prep_dma_memcpy(
311 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
312 size_t len, unsigned long flags)
313{
314 struct st_fdma_chan *fchan;
315 struct st_fdma_desc *fdesc;
316 struct st_fdma_hw_node *hw_node;
317
318 if (!len)
319 return NULL;
320
321 fchan = to_st_fdma_chan(chan);
322
323 /* We only require a single descriptor */
324 fdesc = st_fdma_alloc_desc(fchan, 1);
325 if (!fdesc) {
326 dev_err(fchan->fdev->dev, "no memory for desc\n");
327 return NULL;
328 }
329
330 hw_node = fdesc->node[0].desc;
331 hw_node->next = 0;
332 hw_node->control = FDMA_NODE_CTRL_REQ_MAP_FREE_RUN;
333 hw_node->control |= FDMA_NODE_CTRL_SRC_INCR;
334 hw_node->control |= FDMA_NODE_CTRL_DST_INCR;
335 hw_node->control |= FDMA_NODE_CTRL_INT_EON;
336 hw_node->nbytes = len;
337 hw_node->saddr = src;
338 hw_node->daddr = dst;
339 hw_node->generic.length = len;
340 hw_node->generic.sstride = 0;
341 hw_node->generic.dstride = 0;
342
343 return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
344}
345
346static int config_reqctrl(struct st_fdma_chan *fchan,
347 enum dma_transfer_direction direction)
348{
349 u32 maxburst = 0, addr = 0;
350 enum dma_slave_buswidth width;
351 int ch_id = fchan->vchan.chan.chan_id;
352 struct st_fdma_dev *fdev = fchan->fdev;
353
354 switch (direction) {
355
356 case DMA_DEV_TO_MEM:
357 fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_WNR;
358 maxburst = fchan->scfg.src_maxburst;
359 width = fchan->scfg.src_addr_width;
360 addr = fchan->scfg.src_addr;
361 break;
362
363 case DMA_MEM_TO_DEV:
364 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_WNR;
365 maxburst = fchan->scfg.dst_maxburst;
366 width = fchan->scfg.dst_addr_width;
367 addr = fchan->scfg.dst_addr;
368 break;
369
370 default:
371 return -EINVAL;
372 }
373
374 fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_OPCODE_MASK;
375
376 switch (width) {
377
378 case DMA_SLAVE_BUSWIDTH_1_BYTE:
379 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST1;
380 break;
381
382 case DMA_SLAVE_BUSWIDTH_2_BYTES:
383 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST2;
384 break;
385
386 case DMA_SLAVE_BUSWIDTH_4_BYTES:
387 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST4;
388 break;
389
390 case DMA_SLAVE_BUSWIDTH_8_BYTES:
391 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST8;
392 break;
393
394 default:
395 return -EINVAL;
396 }
397
398 fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_NUM_OPS_MASK;
399 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_NUM_OPS(maxburst-1);
400 dreq_write(fchan, fchan->cfg.req_ctrl, FDMA_REQ_CTRL_OFST);
401
402 fchan->cfg.dev_addr = addr;
403 fchan->cfg.dir = direction;
404
405 dev_dbg(fdev->dev, "chan:%d config_reqctrl:%#x req_ctrl:%#lx\n",
406 ch_id, addr, fchan->cfg.req_ctrl);
407
408 return 0;
409}
410
411static void fill_hw_node(struct st_fdma_hw_node *hw_node,
412 struct st_fdma_chan *fchan,
413 enum dma_transfer_direction direction)
414{
415 if (direction == DMA_MEM_TO_DEV) {
416 hw_node->control |= FDMA_NODE_CTRL_SRC_INCR;
417 hw_node->control |= FDMA_NODE_CTRL_DST_STATIC;
418 hw_node->daddr = fchan->cfg.dev_addr;
419 } else {
420 hw_node->control |= FDMA_NODE_CTRL_SRC_STATIC;
421 hw_node->control |= FDMA_NODE_CTRL_DST_INCR;
422 hw_node->saddr = fchan->cfg.dev_addr;
423 }
424
425 hw_node->generic.sstride = 0;
426 hw_node->generic.dstride = 0;
427}
428
429static inline struct st_fdma_chan *st_fdma_prep_common(struct dma_chan *chan,
430 size_t len, enum dma_transfer_direction direction)
431{
432 struct st_fdma_chan *fchan;
433
434 if (!chan || !len)
435 return NULL;
436
437 fchan = to_st_fdma_chan(chan);
438
439 if (!is_slave_direction(direction)) {
440 dev_err(fchan->fdev->dev, "bad direction?\n");
441 return NULL;
442 }
443
444 return fchan;
445}
446
447static struct dma_async_tx_descriptor *st_fdma_prep_dma_cyclic(
448 struct dma_chan *chan, dma_addr_t buf_addr, size_t len,
449 size_t period_len, enum dma_transfer_direction direction,
450 unsigned long flags)
451{
452 struct st_fdma_chan *fchan;
453 struct st_fdma_desc *fdesc;
454 int sg_len, i;
455
456 fchan = st_fdma_prep_common(chan, len, direction);
457 if (!fchan)
458 return NULL;
459
460 if (!period_len)
461 return NULL;
462
463 if (config_reqctrl(fchan, direction)) {
464 dev_err(fchan->fdev->dev, "bad width or direction\n");
465 return NULL;
466 }
467
468 /* the buffer length must be a multiple of period_len */
469 if (len % period_len != 0) {
470 dev_err(fchan->fdev->dev, "len is not multiple of period\n");
471 return NULL;
472 }
473
474 sg_len = len / period_len;
475 fdesc = st_fdma_alloc_desc(fchan, sg_len);
476 if (!fdesc) {
477 dev_err(fchan->fdev->dev, "no memory for desc\n");
478 return NULL;
479 }
480
481 fdesc->iscyclic = true;
482
483 for (i = 0; i < sg_len; i++) {
484 struct st_fdma_hw_node *hw_node = fdesc->node[i].desc;
485
486 hw_node->next = fdesc->node[(i + 1) % sg_len].pdesc;
487
488 hw_node->control =
489 FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line);
490 hw_node->control |= FDMA_NODE_CTRL_INT_EON;
491
492 fill_hw_node(hw_node, fchan, direction);
493
494 if (direction == DMA_MEM_TO_DEV)
495 hw_node->saddr = buf_addr + (i * period_len);
496 else
497 hw_node->daddr = buf_addr + (i * period_len);
498
499 hw_node->nbytes = period_len;
500 hw_node->generic.length = period_len;
501 }
502
503 return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
504}
505
506static struct dma_async_tx_descriptor *st_fdma_prep_slave_sg(
507 struct dma_chan *chan, struct scatterlist *sgl,
508 unsigned int sg_len, enum dma_transfer_direction direction,
509 unsigned long flags, void *context)
510{
511 struct st_fdma_chan *fchan;
512 struct st_fdma_desc *fdesc;
513 struct st_fdma_hw_node *hw_node;
514 struct scatterlist *sg;
515 int i;
516
517 fchan = st_fdma_prep_common(chan, sg_len, direction);
518 if (!fchan)
519 return NULL;
520
521 if (!sgl)
522 return NULL;
523
524 fdesc = st_fdma_alloc_desc(fchan, sg_len);
525 if (!fdesc) {
526 dev_err(fchan->fdev->dev, "no memory for desc\n");
527 return NULL;
528 }
529
530 fdesc->iscyclic = false;
531
532 for_each_sg(sgl, sg, sg_len, i) {
533 hw_node = fdesc->node[i].desc;
534
535 hw_node->next = fdesc->node[(i + 1) % sg_len].pdesc;
536 hw_node->control = FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line);
537
538 fill_hw_node(hw_node, fchan, direction);
539
540 if (direction == DMA_MEM_TO_DEV)
541 hw_node->saddr = sg_dma_address(sg);
542 else
543 hw_node->daddr = sg_dma_address(sg);
544
545 hw_node->nbytes = sg_dma_len(sg);
546 hw_node->generic.length = sg_dma_len(sg);
547 }
548
549 /* interrupt at end of last node */
550 hw_node->control |= FDMA_NODE_CTRL_INT_EON;
551
552 return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
553}
554
555static size_t st_fdma_desc_residue(struct st_fdma_chan *fchan,
556 struct virt_dma_desc *vdesc,
557 bool in_progress)
558{
559 struct st_fdma_desc *fdesc = fchan->fdesc;
560 size_t residue = 0;
561 dma_addr_t cur_addr = 0;
562 int i;
563
564 if (in_progress) {
565 cur_addr = fchan_read(fchan, FDMA_CH_CMD_OFST);
566 cur_addr &= FDMA_CH_CMD_DATA_MASK;
567 }
568
569 for (i = fchan->fdesc->n_nodes - 1 ; i >= 0; i--) {
570 if (cur_addr == fdesc->node[i].pdesc) {
571 residue += fnode_read(fchan, FDMA_CNTN_OFST);
572 break;
573 }
574 residue += fdesc->node[i].desc->nbytes;
575 }
576
577 return residue;
578}
579
580static enum dma_status st_fdma_tx_status(struct dma_chan *chan,
581 dma_cookie_t cookie,
582 struct dma_tx_state *txstate)
583{
584 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
585 struct virt_dma_desc *vd;
586 enum dma_status ret;
587 unsigned long flags;
588
589 ret = dma_cookie_status(chan, cookie, txstate);
590 if (ret == DMA_COMPLETE || !txstate)
591 return ret;
592
593 spin_lock_irqsave(&fchan->vchan.lock, flags);
594 vd = vchan_find_desc(&fchan->vchan, cookie);
595 if (fchan->fdesc && cookie == fchan->fdesc->vdesc.tx.cookie)
596 txstate->residue = st_fdma_desc_residue(fchan, vd, true);
597 else if (vd)
598 txstate->residue = st_fdma_desc_residue(fchan, vd, false);
599 else
600 txstate->residue = 0;
601
602 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
603
604 return ret;
605}
606
607static void st_fdma_issue_pending(struct dma_chan *chan)
608{
609 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
610 unsigned long flags;
611
612 spin_lock_irqsave(&fchan->vchan.lock, flags);
613
614 if (vchan_issue_pending(&fchan->vchan) && !fchan->fdesc)
615 st_fdma_xfer_desc(fchan);
616
617 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
618}
619
620static int st_fdma_pause(struct dma_chan *chan)
621{
622 unsigned long flags;
623 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
624 int ch_id = fchan->vchan.chan.chan_id;
625 unsigned long cmd = FDMA_CMD_PAUSE(ch_id);
626
627 dev_dbg(fchan->fdev->dev, "pause chan:%d\n", ch_id);
628
629 spin_lock_irqsave(&fchan->vchan.lock, flags);
630 if (fchan->fdesc)
631 fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST);
632 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
633
634 return 0;
635}
636
637static int st_fdma_resume(struct dma_chan *chan)
638{
639 unsigned long flags;
640 unsigned long val;
641 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
642 int ch_id = fchan->vchan.chan.chan_id;
643
644 dev_dbg(fchan->fdev->dev, "resume chan:%d\n", ch_id);
645
646 spin_lock_irqsave(&fchan->vchan.lock, flags);
647 if (fchan->fdesc) {
648 val = fchan_read(fchan, FDMA_CH_CMD_OFST);
649 val &= FDMA_CH_CMD_DATA_MASK;
650 fchan_write(fchan, val, FDMA_CH_CMD_OFST);
651 }
652 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
653
654 return 0;
655}
656
657static int st_fdma_terminate_all(struct dma_chan *chan)
658{
659 unsigned long flags;
660 LIST_HEAD(head);
661 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
662 int ch_id = fchan->vchan.chan.chan_id;
663 unsigned long cmd = FDMA_CMD_PAUSE(ch_id);
664
665 dev_dbg(fchan->fdev->dev, "terminate chan:%d\n", ch_id);
666
667 spin_lock_irqsave(&fchan->vchan.lock, flags);
668 fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST);
669 fchan->fdesc = NULL;
670 vchan_get_all_descriptors(&fchan->vchan, &head);
671 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
672 vchan_dma_desc_free_list(&fchan->vchan, &head);
673
674 return 0;
675}
676
677static int st_fdma_slave_config(struct dma_chan *chan,
678 struct dma_slave_config *slave_cfg)
679{
680 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
681
682 memcpy(&fchan->scfg, slave_cfg, sizeof(fchan->scfg));
683 return 0;
684}
685
686static const struct st_fdma_driverdata fdma_mpe31_stih407_11 = {
687 .name = "STiH407",
688 .id = 0,
689};
690
691static const struct st_fdma_driverdata fdma_mpe31_stih407_12 = {
692 .name = "STiH407",
693 .id = 1,
694};
695
696static const struct st_fdma_driverdata fdma_mpe31_stih407_13 = {
697 .name = "STiH407",
698 .id = 2,
699};
700
701static const struct of_device_id st_fdma_match[] = {
702 { .compatible = "st,stih407-fdma-mpe31-11"
703 , .data = &fdma_mpe31_stih407_11 },
704 { .compatible = "st,stih407-fdma-mpe31-12"
705 , .data = &fdma_mpe31_stih407_12 },
706 { .compatible = "st,stih407-fdma-mpe31-13"
707 , .data = &fdma_mpe31_stih407_13 },
708 {},
709};
710MODULE_DEVICE_TABLE(of, st_fdma_match);
711
712static int st_fdma_parse_dt(struct platform_device *pdev,
713 const struct st_fdma_driverdata *drvdata,
714 struct st_fdma_dev *fdev)
715{
716 snprintf(fdev->fw_name, FW_NAME_SIZE, "fdma_%s_%d.elf",
717 drvdata->name, drvdata->id);
718
719 return of_property_read_u32(pdev->dev.of_node, "dma-channels",
720 &fdev->nr_channels);
721}
722#define FDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
723 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
724 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
725 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
726
727static void st_fdma_free(struct st_fdma_dev *fdev)
728{
729 struct st_fdma_chan *fchan;
730 int i;
731
732 for (i = 0; i < fdev->nr_channels; i++) {
733 fchan = &fdev->chans[i];
734 list_del(&fchan->vchan.chan.device_node);
735 tasklet_kill(&fchan->vchan.task);
736 }
737}
738
739static int st_fdma_probe(struct platform_device *pdev)
740{
741 struct st_fdma_dev *fdev;
742 const struct of_device_id *match;
743 struct device_node *np = pdev->dev.of_node;
744 const struct st_fdma_driverdata *drvdata;
745 int ret, i;
746
747 match = of_match_device((st_fdma_match), &pdev->dev);
748 if (!match || !match->data) {
749 dev_err(&pdev->dev, "No device match found\n");
750 return -ENODEV;
751 }
752
753 drvdata = match->data;
754
755 fdev = devm_kzalloc(&pdev->dev, sizeof(*fdev), GFP_KERNEL);
756 if (!fdev)
757 return -ENOMEM;
758
759 ret = st_fdma_parse_dt(pdev, drvdata, fdev);
760 if (ret) {
761 dev_err(&pdev->dev, "unable to find platform data\n");
762 goto err;
763 }
764
765 fdev->chans = devm_kcalloc(&pdev->dev, fdev->nr_channels,
766 sizeof(struct st_fdma_chan), GFP_KERNEL);
767 if (!fdev->chans)
768 return -ENOMEM;
769
770 fdev->dev = &pdev->dev;
771 fdev->drvdata = drvdata;
772 platform_set_drvdata(pdev, fdev);
773
774 fdev->irq = platform_get_irq(pdev, 0);
775 if (fdev->irq < 0)
776 return -EINVAL;
777
778 ret = devm_request_irq(&pdev->dev, fdev->irq, st_fdma_irq_handler, 0,
779 dev_name(&pdev->dev), fdev);
780 if (ret) {
781 dev_err(&pdev->dev, "Failed to request irq (%d)\n", ret);
782 goto err;
783 }
784
785 fdev->slim_rproc = st_slim_rproc_alloc(pdev, fdev->fw_name);
786 if (IS_ERR(fdev->slim_rproc)) {
787 ret = PTR_ERR(fdev->slim_rproc);
788 dev_err(&pdev->dev, "slim_rproc_alloc failed (%d)\n", ret);
789 goto err;
790 }
791
792 /* Initialise list of FDMA channels */
793 INIT_LIST_HEAD(&fdev->dma_device.channels);
794 for (i = 0; i < fdev->nr_channels; i++) {
795 struct st_fdma_chan *fchan = &fdev->chans[i];
796
797 fchan->fdev = fdev;
798 fchan->vchan.desc_free = st_fdma_free_desc;
799 vchan_init(&fchan->vchan, &fdev->dma_device);
800 }
801
802 /* Initialise the FDMA dreq (reserve 0 & 31 for FDMA use) */
803 fdev->dreq_mask = BIT(0) | BIT(31);
804
805 dma_cap_set(DMA_SLAVE, fdev->dma_device.cap_mask);
806 dma_cap_set(DMA_CYCLIC, fdev->dma_device.cap_mask);
807 dma_cap_set(DMA_MEMCPY, fdev->dma_device.cap_mask);
808
809 fdev->dma_device.dev = &pdev->dev;
810 fdev->dma_device.device_alloc_chan_resources = st_fdma_alloc_chan_res;
811 fdev->dma_device.device_free_chan_resources = st_fdma_free_chan_res;
812 fdev->dma_device.device_prep_dma_cyclic = st_fdma_prep_dma_cyclic;
813 fdev->dma_device.device_prep_slave_sg = st_fdma_prep_slave_sg;
814 fdev->dma_device.device_prep_dma_memcpy = st_fdma_prep_dma_memcpy;
815 fdev->dma_device.device_tx_status = st_fdma_tx_status;
816 fdev->dma_device.device_issue_pending = st_fdma_issue_pending;
817 fdev->dma_device.device_terminate_all = st_fdma_terminate_all;
818 fdev->dma_device.device_config = st_fdma_slave_config;
819 fdev->dma_device.device_pause = st_fdma_pause;
820 fdev->dma_device.device_resume = st_fdma_resume;
821
822 fdev->dma_device.src_addr_widths = FDMA_DMA_BUSWIDTHS;
823 fdev->dma_device.dst_addr_widths = FDMA_DMA_BUSWIDTHS;
824 fdev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
825 fdev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
826
827 ret = dmaenginem_async_device_register(&fdev->dma_device);
828 if (ret) {
829 dev_err(&pdev->dev,
830 "Failed to register DMA device (%d)\n", ret);
831 goto err_rproc;
832 }
833
834 ret = of_dma_controller_register(np, st_fdma_of_xlate, fdev);
835 if (ret) {
836 dev_err(&pdev->dev,
837 "Failed to register controller (%d)\n", ret);
838 goto err_rproc;
839 }
840
841 dev_info(&pdev->dev, "ST FDMA engine driver, irq:%d\n", fdev->irq);
842
843 return 0;
844
845err_rproc:
846 st_fdma_free(fdev);
847 st_slim_rproc_put(fdev->slim_rproc);
848err:
849 return ret;
850}
851
852static int st_fdma_remove(struct platform_device *pdev)
853{
854 struct st_fdma_dev *fdev = platform_get_drvdata(pdev);
855
856 devm_free_irq(&pdev->dev, fdev->irq, fdev);
857 st_slim_rproc_put(fdev->slim_rproc);
858 of_dma_controller_free(pdev->dev.of_node);
859
860 return 0;
861}
862
863static struct platform_driver st_fdma_platform_driver = {
864 .driver = {
865 .name = DRIVER_NAME,
866 .of_match_table = st_fdma_match,
867 },
868 .probe = st_fdma_probe,
869 .remove = st_fdma_remove,
870};
871module_platform_driver(st_fdma_platform_driver);
872
873MODULE_LICENSE("GPL v2");
874MODULE_DESCRIPTION("STMicroelectronics FDMA engine driver");
875MODULE_AUTHOR("Ludovic.barre <Ludovic.barre@st.com>");
876MODULE_AUTHOR("Peter Griffin <peter.griffin@linaro.org>");
877MODULE_ALIAS("platform: " DRIVER_NAME);
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * DMA driver for STMicroelectronics STi FDMA controller
4 *
5 * Copyright (C) 2014 STMicroelectronics
6 *
7 * Author: Ludovic Barre <Ludovic.barre@st.com>
8 * Peter Griffin <peter.griffin@linaro.org>
9 */
10
11#include <linux/init.h>
12#include <linux/module.h>
13#include <linux/of.h>
14#include <linux/of_dma.h>
15#include <linux/platform_device.h>
16#include <linux/property.h>
17#include <linux/interrupt.h>
18#include <linux/remoteproc.h>
19#include <linux/slab.h>
20
21#include "st_fdma.h"
22
23static inline struct st_fdma_chan *to_st_fdma_chan(struct dma_chan *c)
24{
25 return container_of(c, struct st_fdma_chan, vchan.chan);
26}
27
28static struct st_fdma_desc *to_st_fdma_desc(struct virt_dma_desc *vd)
29{
30 return container_of(vd, struct st_fdma_desc, vdesc);
31}
32
33static int st_fdma_dreq_get(struct st_fdma_chan *fchan)
34{
35 struct st_fdma_dev *fdev = fchan->fdev;
36 u32 req_line_cfg = fchan->cfg.req_line;
37 u32 dreq_line;
38 int try = 0;
39
40 /*
41 * dreq_mask is shared for n channels of fdma, so all accesses must be
42 * atomic. if the dreq_mask is changed between ffz and set_bit,
43 * we retry
44 */
45 do {
46 if (fdev->dreq_mask == ~0L) {
47 dev_err(fdev->dev, "No req lines available\n");
48 return -EINVAL;
49 }
50
51 if (try || req_line_cfg >= ST_FDMA_NR_DREQS) {
52 dev_err(fdev->dev, "Invalid or used req line\n");
53 return -EINVAL;
54 } else {
55 dreq_line = req_line_cfg;
56 }
57
58 try++;
59 } while (test_and_set_bit(dreq_line, &fdev->dreq_mask));
60
61 dev_dbg(fdev->dev, "get dreq_line:%d mask:%#lx\n",
62 dreq_line, fdev->dreq_mask);
63
64 return dreq_line;
65}
66
67static void st_fdma_dreq_put(struct st_fdma_chan *fchan)
68{
69 struct st_fdma_dev *fdev = fchan->fdev;
70
71 dev_dbg(fdev->dev, "put dreq_line:%#x\n", fchan->dreq_line);
72 clear_bit(fchan->dreq_line, &fdev->dreq_mask);
73}
74
75static void st_fdma_xfer_desc(struct st_fdma_chan *fchan)
76{
77 struct virt_dma_desc *vdesc;
78 unsigned long nbytes, ch_cmd, cmd;
79
80 vdesc = vchan_next_desc(&fchan->vchan);
81 if (!vdesc)
82 return;
83
84 fchan->fdesc = to_st_fdma_desc(vdesc);
85 nbytes = fchan->fdesc->node[0].desc->nbytes;
86 cmd = FDMA_CMD_START(fchan->vchan.chan.chan_id);
87 ch_cmd = fchan->fdesc->node[0].pdesc | FDMA_CH_CMD_STA_START;
88
89 /* start the channel for the descriptor */
90 fnode_write(fchan, nbytes, FDMA_CNTN_OFST);
91 fchan_write(fchan, ch_cmd, FDMA_CH_CMD_OFST);
92 writel(cmd,
93 fchan->fdev->slim_rproc->peri + FDMA_CMD_SET_OFST);
94
95 dev_dbg(fchan->fdev->dev, "start chan:%d\n", fchan->vchan.chan.chan_id);
96}
97
98static void st_fdma_ch_sta_update(struct st_fdma_chan *fchan,
99 unsigned long int_sta)
100{
101 unsigned long ch_sta, ch_err;
102 int ch_id = fchan->vchan.chan.chan_id;
103 struct st_fdma_dev *fdev = fchan->fdev;
104
105 ch_sta = fchan_read(fchan, FDMA_CH_CMD_OFST);
106 ch_err = ch_sta & FDMA_CH_CMD_ERR_MASK;
107 ch_sta &= FDMA_CH_CMD_STA_MASK;
108
109 if (int_sta & FDMA_INT_STA_ERR) {
110 dev_warn(fdev->dev, "chan:%d, error:%ld\n", ch_id, ch_err);
111 fchan->status = DMA_ERROR;
112 return;
113 }
114
115 switch (ch_sta) {
116 case FDMA_CH_CMD_STA_PAUSED:
117 fchan->status = DMA_PAUSED;
118 break;
119
120 case FDMA_CH_CMD_STA_RUNNING:
121 fchan->status = DMA_IN_PROGRESS;
122 break;
123 }
124}
125
126static irqreturn_t st_fdma_irq_handler(int irq, void *dev_id)
127{
128 struct st_fdma_dev *fdev = dev_id;
129 irqreturn_t ret = IRQ_NONE;
130 struct st_fdma_chan *fchan = &fdev->chans[0];
131 unsigned long int_sta, clr;
132
133 int_sta = fdma_read(fdev, FDMA_INT_STA_OFST);
134 clr = int_sta;
135
136 for (; int_sta != 0 ; int_sta >>= 2, fchan++) {
137 if (!(int_sta & (FDMA_INT_STA_CH | FDMA_INT_STA_ERR)))
138 continue;
139
140 spin_lock(&fchan->vchan.lock);
141 st_fdma_ch_sta_update(fchan, int_sta);
142
143 if (fchan->fdesc) {
144 if (!fchan->fdesc->iscyclic) {
145 list_del(&fchan->fdesc->vdesc.node);
146 vchan_cookie_complete(&fchan->fdesc->vdesc);
147 fchan->fdesc = NULL;
148 fchan->status = DMA_COMPLETE;
149 } else {
150 vchan_cyclic_callback(&fchan->fdesc->vdesc);
151 }
152
153 /* Start the next descriptor (if available) */
154 if (!fchan->fdesc)
155 st_fdma_xfer_desc(fchan);
156 }
157
158 spin_unlock(&fchan->vchan.lock);
159 ret = IRQ_HANDLED;
160 }
161
162 fdma_write(fdev, clr, FDMA_INT_CLR_OFST);
163
164 return ret;
165}
166
167static struct dma_chan *st_fdma_of_xlate(struct of_phandle_args *dma_spec,
168 struct of_dma *ofdma)
169{
170 struct st_fdma_dev *fdev = ofdma->of_dma_data;
171 struct dma_chan *chan;
172 struct st_fdma_chan *fchan;
173 int ret;
174
175 if (dma_spec->args_count < 1)
176 return ERR_PTR(-EINVAL);
177
178 if (fdev->dma_device.dev->of_node != dma_spec->np)
179 return ERR_PTR(-EINVAL);
180
181 ret = rproc_boot(fdev->slim_rproc->rproc);
182 if (ret == -ENOENT)
183 return ERR_PTR(-EPROBE_DEFER);
184 else if (ret)
185 return ERR_PTR(ret);
186
187 chan = dma_get_any_slave_channel(&fdev->dma_device);
188 if (!chan)
189 goto err_chan;
190
191 fchan = to_st_fdma_chan(chan);
192
193 fchan->cfg.of_node = dma_spec->np;
194 fchan->cfg.req_line = dma_spec->args[0];
195 fchan->cfg.req_ctrl = 0;
196 fchan->cfg.type = ST_FDMA_TYPE_FREE_RUN;
197
198 if (dma_spec->args_count > 1)
199 fchan->cfg.req_ctrl = dma_spec->args[1]
200 & FDMA_REQ_CTRL_CFG_MASK;
201
202 if (dma_spec->args_count > 2)
203 fchan->cfg.type = dma_spec->args[2];
204
205 if (fchan->cfg.type == ST_FDMA_TYPE_FREE_RUN) {
206 fchan->dreq_line = 0;
207 } else {
208 fchan->dreq_line = st_fdma_dreq_get(fchan);
209 if (IS_ERR_VALUE(fchan->dreq_line)) {
210 chan = ERR_PTR(fchan->dreq_line);
211 goto err_chan;
212 }
213 }
214
215 dev_dbg(fdev->dev, "xlate req_line:%d type:%d req_ctrl:%#lx\n",
216 fchan->cfg.req_line, fchan->cfg.type, fchan->cfg.req_ctrl);
217
218 return chan;
219
220err_chan:
221 rproc_shutdown(fdev->slim_rproc->rproc);
222 return chan;
223
224}
225
226static void st_fdma_free_desc(struct virt_dma_desc *vdesc)
227{
228 struct st_fdma_desc *fdesc;
229 int i;
230
231 fdesc = to_st_fdma_desc(vdesc);
232 for (i = 0; i < fdesc->n_nodes; i++)
233 dma_pool_free(fdesc->fchan->node_pool, fdesc->node[i].desc,
234 fdesc->node[i].pdesc);
235 kfree(fdesc);
236}
237
238static struct st_fdma_desc *st_fdma_alloc_desc(struct st_fdma_chan *fchan,
239 int sg_len)
240{
241 struct st_fdma_desc *fdesc;
242 int i;
243
244 fdesc = kzalloc(struct_size(fdesc, node, sg_len), GFP_NOWAIT);
245 if (!fdesc)
246 return NULL;
247
248 fdesc->fchan = fchan;
249 fdesc->n_nodes = sg_len;
250 for (i = 0; i < sg_len; i++) {
251 fdesc->node[i].desc = dma_pool_alloc(fchan->node_pool,
252 GFP_NOWAIT, &fdesc->node[i].pdesc);
253 if (!fdesc->node[i].desc)
254 goto err;
255 }
256 return fdesc;
257
258err:
259 while (--i >= 0)
260 dma_pool_free(fchan->node_pool, fdesc->node[i].desc,
261 fdesc->node[i].pdesc);
262 kfree(fdesc);
263 return NULL;
264}
265
266static int st_fdma_alloc_chan_res(struct dma_chan *chan)
267{
268 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
269
270 /* Create the dma pool for descriptor allocation */
271 fchan->node_pool = dma_pool_create(dev_name(&chan->dev->device),
272 fchan->fdev->dev,
273 sizeof(struct st_fdma_hw_node),
274 __alignof__(struct st_fdma_hw_node),
275 0);
276
277 if (!fchan->node_pool) {
278 dev_err(fchan->fdev->dev, "unable to allocate desc pool\n");
279 return -ENOMEM;
280 }
281
282 dev_dbg(fchan->fdev->dev, "alloc ch_id:%d type:%d\n",
283 fchan->vchan.chan.chan_id, fchan->cfg.type);
284
285 return 0;
286}
287
288static void st_fdma_free_chan_res(struct dma_chan *chan)
289{
290 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
291 struct rproc *rproc = fchan->fdev->slim_rproc->rproc;
292 unsigned long flags;
293
294 dev_dbg(fchan->fdev->dev, "%s: freeing chan:%d\n",
295 __func__, fchan->vchan.chan.chan_id);
296
297 if (fchan->cfg.type != ST_FDMA_TYPE_FREE_RUN)
298 st_fdma_dreq_put(fchan);
299
300 spin_lock_irqsave(&fchan->vchan.lock, flags);
301 fchan->fdesc = NULL;
302 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
303
304 dma_pool_destroy(fchan->node_pool);
305 fchan->node_pool = NULL;
306 memset(&fchan->cfg, 0, sizeof(struct st_fdma_cfg));
307
308 rproc_shutdown(rproc);
309}
310
311static struct dma_async_tx_descriptor *st_fdma_prep_dma_memcpy(
312 struct dma_chan *chan, dma_addr_t dst, dma_addr_t src,
313 size_t len, unsigned long flags)
314{
315 struct st_fdma_chan *fchan;
316 struct st_fdma_desc *fdesc;
317 struct st_fdma_hw_node *hw_node;
318
319 if (!len)
320 return NULL;
321
322 fchan = to_st_fdma_chan(chan);
323
324 /* We only require a single descriptor */
325 fdesc = st_fdma_alloc_desc(fchan, 1);
326 if (!fdesc) {
327 dev_err(fchan->fdev->dev, "no memory for desc\n");
328 return NULL;
329 }
330
331 hw_node = fdesc->node[0].desc;
332 hw_node->next = 0;
333 hw_node->control = FDMA_NODE_CTRL_REQ_MAP_FREE_RUN;
334 hw_node->control |= FDMA_NODE_CTRL_SRC_INCR;
335 hw_node->control |= FDMA_NODE_CTRL_DST_INCR;
336 hw_node->control |= FDMA_NODE_CTRL_INT_EON;
337 hw_node->nbytes = len;
338 hw_node->saddr = src;
339 hw_node->daddr = dst;
340 hw_node->generic.length = len;
341 hw_node->generic.sstride = 0;
342 hw_node->generic.dstride = 0;
343
344 return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
345}
346
347static int config_reqctrl(struct st_fdma_chan *fchan,
348 enum dma_transfer_direction direction)
349{
350 u32 maxburst = 0, addr = 0;
351 enum dma_slave_buswidth width;
352 int ch_id = fchan->vchan.chan.chan_id;
353 struct st_fdma_dev *fdev = fchan->fdev;
354
355 switch (direction) {
356
357 case DMA_DEV_TO_MEM:
358 fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_WNR;
359 maxburst = fchan->scfg.src_maxburst;
360 width = fchan->scfg.src_addr_width;
361 addr = fchan->scfg.src_addr;
362 break;
363
364 case DMA_MEM_TO_DEV:
365 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_WNR;
366 maxburst = fchan->scfg.dst_maxburst;
367 width = fchan->scfg.dst_addr_width;
368 addr = fchan->scfg.dst_addr;
369 break;
370
371 default:
372 return -EINVAL;
373 }
374
375 fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_OPCODE_MASK;
376
377 switch (width) {
378
379 case DMA_SLAVE_BUSWIDTH_1_BYTE:
380 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST1;
381 break;
382
383 case DMA_SLAVE_BUSWIDTH_2_BYTES:
384 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST2;
385 break;
386
387 case DMA_SLAVE_BUSWIDTH_4_BYTES:
388 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST4;
389 break;
390
391 case DMA_SLAVE_BUSWIDTH_8_BYTES:
392 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_OPCODE_LD_ST8;
393 break;
394
395 default:
396 return -EINVAL;
397 }
398
399 fchan->cfg.req_ctrl &= ~FDMA_REQ_CTRL_NUM_OPS_MASK;
400 fchan->cfg.req_ctrl |= FDMA_REQ_CTRL_NUM_OPS(maxburst-1);
401 dreq_write(fchan, fchan->cfg.req_ctrl, FDMA_REQ_CTRL_OFST);
402
403 fchan->cfg.dev_addr = addr;
404 fchan->cfg.dir = direction;
405
406 dev_dbg(fdev->dev, "chan:%d config_reqctrl:%#x req_ctrl:%#lx\n",
407 ch_id, addr, fchan->cfg.req_ctrl);
408
409 return 0;
410}
411
412static void fill_hw_node(struct st_fdma_hw_node *hw_node,
413 struct st_fdma_chan *fchan,
414 enum dma_transfer_direction direction)
415{
416 if (direction == DMA_MEM_TO_DEV) {
417 hw_node->control |= FDMA_NODE_CTRL_SRC_INCR;
418 hw_node->control |= FDMA_NODE_CTRL_DST_STATIC;
419 hw_node->daddr = fchan->cfg.dev_addr;
420 } else {
421 hw_node->control |= FDMA_NODE_CTRL_SRC_STATIC;
422 hw_node->control |= FDMA_NODE_CTRL_DST_INCR;
423 hw_node->saddr = fchan->cfg.dev_addr;
424 }
425
426 hw_node->generic.sstride = 0;
427 hw_node->generic.dstride = 0;
428}
429
430static inline struct st_fdma_chan *st_fdma_prep_common(struct dma_chan *chan,
431 size_t len, enum dma_transfer_direction direction)
432{
433 struct st_fdma_chan *fchan;
434
435 if (!chan || !len)
436 return NULL;
437
438 fchan = to_st_fdma_chan(chan);
439
440 if (!is_slave_direction(direction)) {
441 dev_err(fchan->fdev->dev, "bad direction?\n");
442 return NULL;
443 }
444
445 return fchan;
446}
447
448static struct dma_async_tx_descriptor *st_fdma_prep_dma_cyclic(
449 struct dma_chan *chan, dma_addr_t buf_addr, size_t len,
450 size_t period_len, enum dma_transfer_direction direction,
451 unsigned long flags)
452{
453 struct st_fdma_chan *fchan;
454 struct st_fdma_desc *fdesc;
455 int sg_len, i;
456
457 fchan = st_fdma_prep_common(chan, len, direction);
458 if (!fchan)
459 return NULL;
460
461 if (!period_len)
462 return NULL;
463
464 if (config_reqctrl(fchan, direction)) {
465 dev_err(fchan->fdev->dev, "bad width or direction\n");
466 return NULL;
467 }
468
469 /* the buffer length must be a multiple of period_len */
470 if (len % period_len != 0) {
471 dev_err(fchan->fdev->dev, "len is not multiple of period\n");
472 return NULL;
473 }
474
475 sg_len = len / period_len;
476 fdesc = st_fdma_alloc_desc(fchan, sg_len);
477 if (!fdesc) {
478 dev_err(fchan->fdev->dev, "no memory for desc\n");
479 return NULL;
480 }
481
482 fdesc->iscyclic = true;
483
484 for (i = 0; i < sg_len; i++) {
485 struct st_fdma_hw_node *hw_node = fdesc->node[i].desc;
486
487 hw_node->next = fdesc->node[(i + 1) % sg_len].pdesc;
488
489 hw_node->control =
490 FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line);
491 hw_node->control |= FDMA_NODE_CTRL_INT_EON;
492
493 fill_hw_node(hw_node, fchan, direction);
494
495 if (direction == DMA_MEM_TO_DEV)
496 hw_node->saddr = buf_addr + (i * period_len);
497 else
498 hw_node->daddr = buf_addr + (i * period_len);
499
500 hw_node->nbytes = period_len;
501 hw_node->generic.length = period_len;
502 }
503
504 return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
505}
506
507static struct dma_async_tx_descriptor *st_fdma_prep_slave_sg(
508 struct dma_chan *chan, struct scatterlist *sgl,
509 unsigned int sg_len, enum dma_transfer_direction direction,
510 unsigned long flags, void *context)
511{
512 struct st_fdma_chan *fchan;
513 struct st_fdma_desc *fdesc;
514 struct st_fdma_hw_node *hw_node;
515 struct scatterlist *sg;
516 int i;
517
518 fchan = st_fdma_prep_common(chan, sg_len, direction);
519 if (!fchan)
520 return NULL;
521
522 if (!sgl)
523 return NULL;
524
525 fdesc = st_fdma_alloc_desc(fchan, sg_len);
526 if (!fdesc) {
527 dev_err(fchan->fdev->dev, "no memory for desc\n");
528 return NULL;
529 }
530
531 fdesc->iscyclic = false;
532
533 for_each_sg(sgl, sg, sg_len, i) {
534 hw_node = fdesc->node[i].desc;
535
536 hw_node->next = fdesc->node[(i + 1) % sg_len].pdesc;
537 hw_node->control = FDMA_NODE_CTRL_REQ_MAP_DREQ(fchan->dreq_line);
538
539 fill_hw_node(hw_node, fchan, direction);
540
541 if (direction == DMA_MEM_TO_DEV)
542 hw_node->saddr = sg_dma_address(sg);
543 else
544 hw_node->daddr = sg_dma_address(sg);
545
546 hw_node->nbytes = sg_dma_len(sg);
547 hw_node->generic.length = sg_dma_len(sg);
548 }
549
550 /* interrupt at end of last node */
551 hw_node->control |= FDMA_NODE_CTRL_INT_EON;
552
553 return vchan_tx_prep(&fchan->vchan, &fdesc->vdesc, flags);
554}
555
556static size_t st_fdma_desc_residue(struct st_fdma_chan *fchan,
557 struct virt_dma_desc *vdesc,
558 bool in_progress)
559{
560 struct st_fdma_desc *fdesc = fchan->fdesc;
561 size_t residue = 0;
562 dma_addr_t cur_addr = 0;
563 int i;
564
565 if (in_progress) {
566 cur_addr = fchan_read(fchan, FDMA_CH_CMD_OFST);
567 cur_addr &= FDMA_CH_CMD_DATA_MASK;
568 }
569
570 for (i = fchan->fdesc->n_nodes - 1 ; i >= 0; i--) {
571 if (cur_addr == fdesc->node[i].pdesc) {
572 residue += fnode_read(fchan, FDMA_CNTN_OFST);
573 break;
574 }
575 residue += fdesc->node[i].desc->nbytes;
576 }
577
578 return residue;
579}
580
581static enum dma_status st_fdma_tx_status(struct dma_chan *chan,
582 dma_cookie_t cookie,
583 struct dma_tx_state *txstate)
584{
585 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
586 struct virt_dma_desc *vd;
587 enum dma_status ret;
588 unsigned long flags;
589
590 ret = dma_cookie_status(chan, cookie, txstate);
591 if (ret == DMA_COMPLETE || !txstate)
592 return ret;
593
594 spin_lock_irqsave(&fchan->vchan.lock, flags);
595 vd = vchan_find_desc(&fchan->vchan, cookie);
596 if (fchan->fdesc && cookie == fchan->fdesc->vdesc.tx.cookie)
597 txstate->residue = st_fdma_desc_residue(fchan, vd, true);
598 else if (vd)
599 txstate->residue = st_fdma_desc_residue(fchan, vd, false);
600 else
601 txstate->residue = 0;
602
603 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
604
605 return ret;
606}
607
608static void st_fdma_issue_pending(struct dma_chan *chan)
609{
610 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
611 unsigned long flags;
612
613 spin_lock_irqsave(&fchan->vchan.lock, flags);
614
615 if (vchan_issue_pending(&fchan->vchan) && !fchan->fdesc)
616 st_fdma_xfer_desc(fchan);
617
618 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
619}
620
621static int st_fdma_pause(struct dma_chan *chan)
622{
623 unsigned long flags;
624 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
625 int ch_id = fchan->vchan.chan.chan_id;
626 unsigned long cmd = FDMA_CMD_PAUSE(ch_id);
627
628 dev_dbg(fchan->fdev->dev, "pause chan:%d\n", ch_id);
629
630 spin_lock_irqsave(&fchan->vchan.lock, flags);
631 if (fchan->fdesc)
632 fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST);
633 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
634
635 return 0;
636}
637
638static int st_fdma_resume(struct dma_chan *chan)
639{
640 unsigned long flags;
641 unsigned long val;
642 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
643 int ch_id = fchan->vchan.chan.chan_id;
644
645 dev_dbg(fchan->fdev->dev, "resume chan:%d\n", ch_id);
646
647 spin_lock_irqsave(&fchan->vchan.lock, flags);
648 if (fchan->fdesc) {
649 val = fchan_read(fchan, FDMA_CH_CMD_OFST);
650 val &= FDMA_CH_CMD_DATA_MASK;
651 fchan_write(fchan, val, FDMA_CH_CMD_OFST);
652 }
653 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
654
655 return 0;
656}
657
658static int st_fdma_terminate_all(struct dma_chan *chan)
659{
660 unsigned long flags;
661 LIST_HEAD(head);
662 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
663 int ch_id = fchan->vchan.chan.chan_id;
664 unsigned long cmd = FDMA_CMD_PAUSE(ch_id);
665
666 dev_dbg(fchan->fdev->dev, "terminate chan:%d\n", ch_id);
667
668 spin_lock_irqsave(&fchan->vchan.lock, flags);
669 fdma_write(fchan->fdev, cmd, FDMA_CMD_SET_OFST);
670 fchan->fdesc = NULL;
671 vchan_get_all_descriptors(&fchan->vchan, &head);
672 spin_unlock_irqrestore(&fchan->vchan.lock, flags);
673 vchan_dma_desc_free_list(&fchan->vchan, &head);
674
675 return 0;
676}
677
678static int st_fdma_slave_config(struct dma_chan *chan,
679 struct dma_slave_config *slave_cfg)
680{
681 struct st_fdma_chan *fchan = to_st_fdma_chan(chan);
682
683 memcpy(&fchan->scfg, slave_cfg, sizeof(fchan->scfg));
684 return 0;
685}
686
687static const struct st_fdma_driverdata fdma_mpe31_stih407_11 = {
688 .name = "STiH407",
689 .id = 0,
690};
691
692static const struct st_fdma_driverdata fdma_mpe31_stih407_12 = {
693 .name = "STiH407",
694 .id = 1,
695};
696
697static const struct st_fdma_driverdata fdma_mpe31_stih407_13 = {
698 .name = "STiH407",
699 .id = 2,
700};
701
702static const struct of_device_id st_fdma_match[] = {
703 { .compatible = "st,stih407-fdma-mpe31-11"
704 , .data = &fdma_mpe31_stih407_11 },
705 { .compatible = "st,stih407-fdma-mpe31-12"
706 , .data = &fdma_mpe31_stih407_12 },
707 { .compatible = "st,stih407-fdma-mpe31-13"
708 , .data = &fdma_mpe31_stih407_13 },
709 {},
710};
711MODULE_DEVICE_TABLE(of, st_fdma_match);
712
713static int st_fdma_parse_dt(struct platform_device *pdev,
714 const struct st_fdma_driverdata *drvdata,
715 struct st_fdma_dev *fdev)
716{
717 snprintf(fdev->fw_name, FW_NAME_SIZE, "fdma_%s_%d.elf",
718 drvdata->name, drvdata->id);
719
720 return of_property_read_u32(pdev->dev.of_node, "dma-channels",
721 &fdev->nr_channels);
722}
723#define FDMA_DMA_BUSWIDTHS (BIT(DMA_SLAVE_BUSWIDTH_1_BYTE) | \
724 BIT(DMA_SLAVE_BUSWIDTH_2_BYTES) | \
725 BIT(DMA_SLAVE_BUSWIDTH_3_BYTES) | \
726 BIT(DMA_SLAVE_BUSWIDTH_4_BYTES))
727
728static void st_fdma_free(struct st_fdma_dev *fdev)
729{
730 struct st_fdma_chan *fchan;
731 int i;
732
733 for (i = 0; i < fdev->nr_channels; i++) {
734 fchan = &fdev->chans[i];
735 list_del(&fchan->vchan.chan.device_node);
736 tasklet_kill(&fchan->vchan.task);
737 }
738}
739
740static int st_fdma_probe(struct platform_device *pdev)
741{
742 struct st_fdma_dev *fdev;
743 struct device_node *np = pdev->dev.of_node;
744 const struct st_fdma_driverdata *drvdata;
745 int ret, i;
746
747 drvdata = device_get_match_data(&pdev->dev);
748
749 fdev = devm_kzalloc(&pdev->dev, sizeof(*fdev), GFP_KERNEL);
750 if (!fdev)
751 return -ENOMEM;
752
753 ret = st_fdma_parse_dt(pdev, drvdata, fdev);
754 if (ret) {
755 dev_err(&pdev->dev, "unable to find platform data\n");
756 goto err;
757 }
758
759 fdev->chans = devm_kcalloc(&pdev->dev, fdev->nr_channels,
760 sizeof(struct st_fdma_chan), GFP_KERNEL);
761 if (!fdev->chans)
762 return -ENOMEM;
763
764 fdev->dev = &pdev->dev;
765 fdev->drvdata = drvdata;
766 platform_set_drvdata(pdev, fdev);
767
768 fdev->irq = platform_get_irq(pdev, 0);
769 if (fdev->irq < 0)
770 return -EINVAL;
771
772 ret = devm_request_irq(&pdev->dev, fdev->irq, st_fdma_irq_handler, 0,
773 dev_name(&pdev->dev), fdev);
774 if (ret) {
775 dev_err(&pdev->dev, "Failed to request irq (%d)\n", ret);
776 goto err;
777 }
778
779 fdev->slim_rproc = st_slim_rproc_alloc(pdev, fdev->fw_name);
780 if (IS_ERR(fdev->slim_rproc)) {
781 ret = PTR_ERR(fdev->slim_rproc);
782 dev_err(&pdev->dev, "slim_rproc_alloc failed (%d)\n", ret);
783 goto err;
784 }
785
786 /* Initialise list of FDMA channels */
787 INIT_LIST_HEAD(&fdev->dma_device.channels);
788 for (i = 0; i < fdev->nr_channels; i++) {
789 struct st_fdma_chan *fchan = &fdev->chans[i];
790
791 fchan->fdev = fdev;
792 fchan->vchan.desc_free = st_fdma_free_desc;
793 vchan_init(&fchan->vchan, &fdev->dma_device);
794 }
795
796 /* Initialise the FDMA dreq (reserve 0 & 31 for FDMA use) */
797 fdev->dreq_mask = BIT(0) | BIT(31);
798
799 dma_cap_set(DMA_SLAVE, fdev->dma_device.cap_mask);
800 dma_cap_set(DMA_CYCLIC, fdev->dma_device.cap_mask);
801 dma_cap_set(DMA_MEMCPY, fdev->dma_device.cap_mask);
802
803 fdev->dma_device.dev = &pdev->dev;
804 fdev->dma_device.device_alloc_chan_resources = st_fdma_alloc_chan_res;
805 fdev->dma_device.device_free_chan_resources = st_fdma_free_chan_res;
806 fdev->dma_device.device_prep_dma_cyclic = st_fdma_prep_dma_cyclic;
807 fdev->dma_device.device_prep_slave_sg = st_fdma_prep_slave_sg;
808 fdev->dma_device.device_prep_dma_memcpy = st_fdma_prep_dma_memcpy;
809 fdev->dma_device.device_tx_status = st_fdma_tx_status;
810 fdev->dma_device.device_issue_pending = st_fdma_issue_pending;
811 fdev->dma_device.device_terminate_all = st_fdma_terminate_all;
812 fdev->dma_device.device_config = st_fdma_slave_config;
813 fdev->dma_device.device_pause = st_fdma_pause;
814 fdev->dma_device.device_resume = st_fdma_resume;
815
816 fdev->dma_device.src_addr_widths = FDMA_DMA_BUSWIDTHS;
817 fdev->dma_device.dst_addr_widths = FDMA_DMA_BUSWIDTHS;
818 fdev->dma_device.directions = BIT(DMA_DEV_TO_MEM) | BIT(DMA_MEM_TO_DEV);
819 fdev->dma_device.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
820
821 ret = dmaenginem_async_device_register(&fdev->dma_device);
822 if (ret) {
823 dev_err(&pdev->dev,
824 "Failed to register DMA device (%d)\n", ret);
825 goto err_rproc;
826 }
827
828 ret = of_dma_controller_register(np, st_fdma_of_xlate, fdev);
829 if (ret) {
830 dev_err(&pdev->dev,
831 "Failed to register controller (%d)\n", ret);
832 goto err_rproc;
833 }
834
835 dev_info(&pdev->dev, "ST FDMA engine driver, irq:%d\n", fdev->irq);
836
837 return 0;
838
839err_rproc:
840 st_fdma_free(fdev);
841 st_slim_rproc_put(fdev->slim_rproc);
842err:
843 return ret;
844}
845
846static void st_fdma_remove(struct platform_device *pdev)
847{
848 struct st_fdma_dev *fdev = platform_get_drvdata(pdev);
849
850 devm_free_irq(&pdev->dev, fdev->irq, fdev);
851 st_slim_rproc_put(fdev->slim_rproc);
852 of_dma_controller_free(pdev->dev.of_node);
853}
854
855static struct platform_driver st_fdma_platform_driver = {
856 .driver = {
857 .name = DRIVER_NAME,
858 .of_match_table = st_fdma_match,
859 },
860 .probe = st_fdma_probe,
861 .remove_new = st_fdma_remove,
862};
863module_platform_driver(st_fdma_platform_driver);
864
865MODULE_LICENSE("GPL v2");
866MODULE_DESCRIPTION("STMicroelectronics FDMA engine driver");
867MODULE_AUTHOR("Ludovic.barre <Ludovic.barre@st.com>");
868MODULE_AUTHOR("Peter Griffin <peter.griffin@linaro.org>");
869MODULE_ALIAS("platform:" DRIVER_NAME);