Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * DMA driver for NVIDIA Tegra GPC DMA controller.
4 *
5 * Copyright (c) 2014-2022, NVIDIA CORPORATION. All rights reserved.
6 */
7
8#include <linux/bitfield.h>
9#include <linux/dmaengine.h>
10#include <linux/dma-mapping.h>
11#include <linux/interrupt.h>
12#include <linux/iommu.h>
13#include <linux/iopoll.h>
14#include <linux/minmax.h>
15#include <linux/module.h>
16#include <linux/of.h>
17#include <linux/of_dma.h>
18#include <linux/platform_device.h>
19#include <linux/reset.h>
20#include <linux/slab.h>
21#include <dt-bindings/memory/tegra186-mc.h>
22#include "virt-dma.h"
23
24/* CSR register */
25#define TEGRA_GPCDMA_CHAN_CSR 0x00
26#define TEGRA_GPCDMA_CSR_ENB BIT(31)
27#define TEGRA_GPCDMA_CSR_IE_EOC BIT(30)
28#define TEGRA_GPCDMA_CSR_ONCE BIT(27)
29
30#define TEGRA_GPCDMA_CSR_FC_MODE GENMASK(25, 24)
31#define TEGRA_GPCDMA_CSR_FC_MODE_NO_MMIO \
32 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 0)
33#define TEGRA_GPCDMA_CSR_FC_MODE_ONE_MMIO \
34 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 1)
35#define TEGRA_GPCDMA_CSR_FC_MODE_TWO_MMIO \
36 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 2)
37#define TEGRA_GPCDMA_CSR_FC_MODE_FOUR_MMIO \
38 FIELD_PREP(TEGRA_GPCDMA_CSR_FC_MODE, 3)
39
40#define TEGRA_GPCDMA_CSR_DMA GENMASK(23, 21)
41#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_NO_FC \
42 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 0)
43#define TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC \
44 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 1)
45#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_NO_FC \
46 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 2)
47#define TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC \
48 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 3)
49#define TEGRA_GPCDMA_CSR_DMA_MEM2MEM \
50 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 4)
51#define TEGRA_GPCDMA_CSR_DMA_FIXED_PAT \
52 FIELD_PREP(TEGRA_GPCDMA_CSR_DMA, 6)
53
54#define TEGRA_GPCDMA_CSR_REQ_SEL_MASK GENMASK(20, 16)
55#define TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED \
56 FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, 4)
57#define TEGRA_GPCDMA_CSR_IRQ_MASK BIT(15)
58#define TEGRA_GPCDMA_CSR_WEIGHT GENMASK(13, 10)
59
60/* STATUS register */
61#define TEGRA_GPCDMA_CHAN_STATUS 0x004
62#define TEGRA_GPCDMA_STATUS_BUSY BIT(31)
63#define TEGRA_GPCDMA_STATUS_ISE_EOC BIT(30)
64#define TEGRA_GPCDMA_STATUS_PING_PONG BIT(28)
65#define TEGRA_GPCDMA_STATUS_DMA_ACTIVITY BIT(27)
66#define TEGRA_GPCDMA_STATUS_CHANNEL_PAUSE BIT(26)
67#define TEGRA_GPCDMA_STATUS_CHANNEL_RX BIT(25)
68#define TEGRA_GPCDMA_STATUS_CHANNEL_TX BIT(24)
69#define TEGRA_GPCDMA_STATUS_IRQ_INTR_STA BIT(23)
70#define TEGRA_GPCDMA_STATUS_IRQ_STA BIT(21)
71#define TEGRA_GPCDMA_STATUS_IRQ_TRIG_STA BIT(20)
72
73#define TEGRA_GPCDMA_CHAN_CSRE 0x008
74#define TEGRA_GPCDMA_CHAN_CSRE_PAUSE BIT(31)
75
76/* Source address */
77#define TEGRA_GPCDMA_CHAN_SRC_PTR 0x00C
78
79/* Destination address */
80#define TEGRA_GPCDMA_CHAN_DST_PTR 0x010
81
82/* High address pointer */
83#define TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR 0x014
84#define TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR GENMASK(7, 0)
85#define TEGRA_GPCDMA_HIGH_ADDR_DST_PTR GENMASK(23, 16)
86
87/* MC sequence register */
88#define TEGRA_GPCDMA_CHAN_MCSEQ 0x18
89#define TEGRA_GPCDMA_MCSEQ_DATA_SWAP BIT(31)
90#define TEGRA_GPCDMA_MCSEQ_REQ_COUNT GENMASK(30, 25)
91#define TEGRA_GPCDMA_MCSEQ_BURST GENMASK(24, 23)
92#define TEGRA_GPCDMA_MCSEQ_BURST_2 \
93 FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 0)
94#define TEGRA_GPCDMA_MCSEQ_BURST_16 \
95 FIELD_PREP(TEGRA_GPCDMA_MCSEQ_BURST, 3)
96#define TEGRA_GPCDMA_MCSEQ_WRAP1 GENMASK(22, 20)
97#define TEGRA_GPCDMA_MCSEQ_WRAP0 GENMASK(19, 17)
98#define TEGRA_GPCDMA_MCSEQ_WRAP_NONE 0
99
100#define TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK GENMASK(13, 7)
101#define TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK GENMASK(6, 0)
102
103/* MMIO sequence register */
104#define TEGRA_GPCDMA_CHAN_MMIOSEQ 0x01c
105#define TEGRA_GPCDMA_MMIOSEQ_DBL_BUF BIT(31)
106#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH GENMASK(30, 28)
107#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8 \
108 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 0)
109#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16 \
110 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 1)
111#define TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32 \
112 FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH, 2)
113#define TEGRA_GPCDMA_MMIOSEQ_DATA_SWAP BIT(27)
114#define TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT 23
115#define TEGRA_GPCDMA_MMIOSEQ_BURST_MIN 2U
116#define TEGRA_GPCDMA_MMIOSEQ_BURST_MAX 32U
117#define TEGRA_GPCDMA_MMIOSEQ_BURST(bs) \
118 (GENMASK((fls(bs) - 2), 0) << TEGRA_GPCDMA_MMIOSEQ_BURST_SHIFT)
119#define TEGRA_GPCDMA_MMIOSEQ_MASTER_ID GENMASK(22, 19)
120#define TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD GENMASK(18, 16)
121#define TEGRA_GPCDMA_MMIOSEQ_MMIO_PROT GENMASK(8, 7)
122
123/* Channel WCOUNT */
124#define TEGRA_GPCDMA_CHAN_WCOUNT 0x20
125
126/* Transfer count */
127#define TEGRA_GPCDMA_CHAN_XFER_COUNT 0x24
128
129/* DMA byte count status */
130#define TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS 0x28
131
132/* Error Status Register */
133#define TEGRA_GPCDMA_CHAN_ERR_STATUS 0x30
134#define TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT 8
135#define TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK 0xF
136#define TEGRA_GPCDMA_CHAN_ERR_TYPE(err) ( \
137 ((err) >> TEGRA_GPCDMA_CHAN_ERR_TYPE_SHIFT) & \
138 TEGRA_GPCDMA_CHAN_ERR_TYPE_MASK)
139#define TEGRA_DMA_BM_FIFO_FULL_ERR 0xF
140#define TEGRA_DMA_PERIPH_FIFO_FULL_ERR 0xE
141#define TEGRA_DMA_PERIPH_ID_ERR 0xD
142#define TEGRA_DMA_STREAM_ID_ERR 0xC
143#define TEGRA_DMA_MC_SLAVE_ERR 0xB
144#define TEGRA_DMA_MMIO_SLAVE_ERR 0xA
145
146/* Fixed Pattern */
147#define TEGRA_GPCDMA_CHAN_FIXED_PATTERN 0x34
148
149#define TEGRA_GPCDMA_CHAN_TZ 0x38
150#define TEGRA_GPCDMA_CHAN_TZ_MMIO_PROT_1 BIT(0)
151#define TEGRA_GPCDMA_CHAN_TZ_MC_PROT_1 BIT(1)
152
153#define TEGRA_GPCDMA_CHAN_SPARE 0x3c
154#define TEGRA_GPCDMA_CHAN_SPARE_EN_LEGACY_FC BIT(16)
155
156/*
157 * If any burst is in flight and DMA paused then this is the time to complete
158 * on-flight burst and update DMA status register.
159 */
160#define TEGRA_GPCDMA_BURST_COMPLETE_TIME 10
161#define TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT 5000 /* 5 msec */
162
163/* Channel base address offset from GPCDMA base address */
164#define TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET 0x10000
165
166/* Default channel mask reserving channel0 */
167#define TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK 0xfffffffe
168
169struct tegra_dma;
170struct tegra_dma_channel;
171
172/*
173 * tegra_dma_chip_data Tegra chip specific DMA data
174 * @nr_channels: Number of channels available in the controller.
175 * @channel_reg_size: Channel register size.
176 * @max_dma_count: Maximum DMA transfer count supported by DMA controller.
177 * @hw_support_pause: DMA HW engine support pause of the channel.
178 */
179struct tegra_dma_chip_data {
180 bool hw_support_pause;
181 unsigned int nr_channels;
182 unsigned int channel_reg_size;
183 unsigned int max_dma_count;
184 int (*terminate)(struct tegra_dma_channel *tdc);
185};
186
187/* DMA channel registers */
188struct tegra_dma_channel_regs {
189 u32 csr;
190 u32 src_ptr;
191 u32 dst_ptr;
192 u32 high_addr_ptr;
193 u32 mc_seq;
194 u32 mmio_seq;
195 u32 wcount;
196 u32 fixed_pattern;
197};
198
199/*
200 * tegra_dma_sg_req: DMA request details to configure hardware. This
201 * contains the details for one transfer to configure DMA hw.
202 * The client's request for data transfer can be broken into multiple
203 * sub-transfer as per requester details and hw support. This sub transfer
204 * get added as an array in Tegra DMA desc which manages the transfer details.
205 */
206struct tegra_dma_sg_req {
207 unsigned int len;
208 struct tegra_dma_channel_regs ch_regs;
209};
210
211/*
212 * tegra_dma_desc: Tegra DMA descriptors which uses virt_dma_desc to
213 * manage client request and keep track of transfer status, callbacks
214 * and request counts etc.
215 */
216struct tegra_dma_desc {
217 bool cyclic;
218 unsigned int bytes_req;
219 unsigned int bytes_xfer;
220 unsigned int sg_idx;
221 unsigned int sg_count;
222 struct virt_dma_desc vd;
223 struct tegra_dma_channel *tdc;
224 struct tegra_dma_sg_req sg_req[] __counted_by(sg_count);
225};
226
227/*
228 * tegra_dma_channel: Channel specific information
229 */
230struct tegra_dma_channel {
231 bool config_init;
232 char name[30];
233 enum dma_transfer_direction sid_dir;
234 int id;
235 int irq;
236 int slave_id;
237 struct tegra_dma *tdma;
238 struct virt_dma_chan vc;
239 struct tegra_dma_desc *dma_desc;
240 struct dma_slave_config dma_sconfig;
241 unsigned int stream_id;
242 unsigned long chan_base_offset;
243};
244
245/*
246 * tegra_dma: Tegra DMA specific information
247 */
248struct tegra_dma {
249 const struct tegra_dma_chip_data *chip_data;
250 unsigned long sid_m2d_reserved;
251 unsigned long sid_d2m_reserved;
252 u32 chan_mask;
253 void __iomem *base_addr;
254 struct device *dev;
255 struct dma_device dma_dev;
256 struct reset_control *rst;
257 struct tegra_dma_channel channels[];
258};
259
260static inline void tdc_write(struct tegra_dma_channel *tdc,
261 u32 reg, u32 val)
262{
263 writel_relaxed(val, tdc->tdma->base_addr + tdc->chan_base_offset + reg);
264}
265
266static inline u32 tdc_read(struct tegra_dma_channel *tdc, u32 reg)
267{
268 return readl_relaxed(tdc->tdma->base_addr + tdc->chan_base_offset + reg);
269}
270
271static inline struct tegra_dma_channel *to_tegra_dma_chan(struct dma_chan *dc)
272{
273 return container_of(dc, struct tegra_dma_channel, vc.chan);
274}
275
276static inline struct tegra_dma_desc *vd_to_tegra_dma_desc(struct virt_dma_desc *vd)
277{
278 return container_of(vd, struct tegra_dma_desc, vd);
279}
280
281static inline struct device *tdc2dev(struct tegra_dma_channel *tdc)
282{
283 return tdc->vc.chan.device->dev;
284}
285
286static void tegra_dma_dump_chan_regs(struct tegra_dma_channel *tdc)
287{
288 dev_dbg(tdc2dev(tdc), "DMA Channel %d name %s register dump:\n",
289 tdc->id, tdc->name);
290 dev_dbg(tdc2dev(tdc), "CSR %x STA %x CSRE %x SRC %x DST %x\n",
291 tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR),
292 tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS),
293 tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE),
294 tdc_read(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR),
295 tdc_read(tdc, TEGRA_GPCDMA_CHAN_DST_PTR)
296 );
297 dev_dbg(tdc2dev(tdc), "MCSEQ %x IOSEQ %x WCNT %x XFER %x BSTA %x\n",
298 tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ),
299 tdc_read(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ),
300 tdc_read(tdc, TEGRA_GPCDMA_CHAN_WCOUNT),
301 tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT),
302 tdc_read(tdc, TEGRA_GPCDMA_CHAN_DMA_BYTE_STATUS)
303 );
304 dev_dbg(tdc2dev(tdc), "DMA ERR_STA %x\n",
305 tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS));
306}
307
308static int tegra_dma_sid_reserve(struct tegra_dma_channel *tdc,
309 enum dma_transfer_direction direction)
310{
311 struct tegra_dma *tdma = tdc->tdma;
312 int sid = tdc->slave_id;
313
314 if (!is_slave_direction(direction))
315 return 0;
316
317 switch (direction) {
318 case DMA_MEM_TO_DEV:
319 if (test_and_set_bit(sid, &tdma->sid_m2d_reserved)) {
320 dev_err(tdma->dev, "slave id already in use\n");
321 return -EINVAL;
322 }
323 break;
324 case DMA_DEV_TO_MEM:
325 if (test_and_set_bit(sid, &tdma->sid_d2m_reserved)) {
326 dev_err(tdma->dev, "slave id already in use\n");
327 return -EINVAL;
328 }
329 break;
330 default:
331 break;
332 }
333
334 tdc->sid_dir = direction;
335
336 return 0;
337}
338
339static void tegra_dma_sid_free(struct tegra_dma_channel *tdc)
340{
341 struct tegra_dma *tdma = tdc->tdma;
342 int sid = tdc->slave_id;
343
344 switch (tdc->sid_dir) {
345 case DMA_MEM_TO_DEV:
346 clear_bit(sid, &tdma->sid_m2d_reserved);
347 break;
348 case DMA_DEV_TO_MEM:
349 clear_bit(sid, &tdma->sid_d2m_reserved);
350 break;
351 default:
352 break;
353 }
354
355 tdc->sid_dir = DMA_TRANS_NONE;
356}
357
358static void tegra_dma_desc_free(struct virt_dma_desc *vd)
359{
360 kfree(container_of(vd, struct tegra_dma_desc, vd));
361}
362
363static int tegra_dma_slave_config(struct dma_chan *dc,
364 struct dma_slave_config *sconfig)
365{
366 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
367
368 memcpy(&tdc->dma_sconfig, sconfig, sizeof(*sconfig));
369 tdc->config_init = true;
370
371 return 0;
372}
373
374static int tegra_dma_pause(struct tegra_dma_channel *tdc)
375{
376 int ret;
377 u32 val;
378
379 val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
380 val |= TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
381 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
382
383 /* Wait until busy bit is de-asserted */
384 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
385 tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
386 val,
387 !(val & TEGRA_GPCDMA_STATUS_BUSY),
388 TEGRA_GPCDMA_BURST_COMPLETE_TIME,
389 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
390
391 if (ret) {
392 dev_err(tdc2dev(tdc), "DMA pause timed out\n");
393 tegra_dma_dump_chan_regs(tdc);
394 }
395
396 return ret;
397}
398
399static int tegra_dma_device_pause(struct dma_chan *dc)
400{
401 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
402 unsigned long flags;
403 int ret;
404
405 if (!tdc->tdma->chip_data->hw_support_pause)
406 return -ENOSYS;
407
408 spin_lock_irqsave(&tdc->vc.lock, flags);
409 ret = tegra_dma_pause(tdc);
410 spin_unlock_irqrestore(&tdc->vc.lock, flags);
411
412 return ret;
413}
414
415static void tegra_dma_resume(struct tegra_dma_channel *tdc)
416{
417 u32 val;
418
419 val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSRE);
420 val &= ~TEGRA_GPCDMA_CHAN_CSRE_PAUSE;
421 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSRE, val);
422}
423
424static int tegra_dma_device_resume(struct dma_chan *dc)
425{
426 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
427 unsigned long flags;
428
429 if (!tdc->tdma->chip_data->hw_support_pause)
430 return -ENOSYS;
431
432 spin_lock_irqsave(&tdc->vc.lock, flags);
433 tegra_dma_resume(tdc);
434 spin_unlock_irqrestore(&tdc->vc.lock, flags);
435
436 return 0;
437}
438
439static inline int tegra_dma_pause_noerr(struct tegra_dma_channel *tdc)
440{
441 /* Return 0 irrespective of PAUSE status.
442 * This is useful to recover channels that can exit out of flush
443 * state when the channel is disabled.
444 */
445
446 tegra_dma_pause(tdc);
447 return 0;
448}
449
450static void tegra_dma_disable(struct tegra_dma_channel *tdc)
451{
452 u32 csr, status;
453
454 csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
455
456 /* Disable interrupts */
457 csr &= ~TEGRA_GPCDMA_CSR_IE_EOC;
458
459 /* Disable DMA */
460 csr &= ~TEGRA_GPCDMA_CSR_ENB;
461 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
462
463 /* Clear interrupt status if it is there */
464 status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
465 if (status & TEGRA_GPCDMA_STATUS_ISE_EOC) {
466 dev_dbg(tdc2dev(tdc), "%s():clearing interrupt\n", __func__);
467 tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS, status);
468 }
469}
470
471static void tegra_dma_configure_next_sg(struct tegra_dma_channel *tdc)
472{
473 struct tegra_dma_desc *dma_desc = tdc->dma_desc;
474 struct tegra_dma_channel_regs *ch_regs;
475 int ret;
476 u32 val;
477
478 dma_desc->sg_idx++;
479
480 /* Reset the sg index for cyclic transfers */
481 if (dma_desc->sg_idx == dma_desc->sg_count)
482 dma_desc->sg_idx = 0;
483
484 /* Configure next transfer immediately after DMA is busy */
485 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
486 tdc->chan_base_offset + TEGRA_GPCDMA_CHAN_STATUS,
487 val,
488 (val & TEGRA_GPCDMA_STATUS_BUSY), 0,
489 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
490 if (ret)
491 return;
492
493 ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
494
495 tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
496 tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
497 tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
498 tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
499
500 /* Start DMA */
501 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
502 ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
503}
504
505static void tegra_dma_start(struct tegra_dma_channel *tdc)
506{
507 struct tegra_dma_desc *dma_desc = tdc->dma_desc;
508 struct tegra_dma_channel_regs *ch_regs;
509 struct virt_dma_desc *vdesc;
510
511 if (!dma_desc) {
512 vdesc = vchan_next_desc(&tdc->vc);
513 if (!vdesc)
514 return;
515
516 dma_desc = vd_to_tegra_dma_desc(vdesc);
517 list_del(&vdesc->node);
518 dma_desc->tdc = tdc;
519 tdc->dma_desc = dma_desc;
520
521 tegra_dma_resume(tdc);
522 }
523
524 ch_regs = &dma_desc->sg_req[dma_desc->sg_idx].ch_regs;
525
526 tdc_write(tdc, TEGRA_GPCDMA_CHAN_WCOUNT, ch_regs->wcount);
527 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, 0);
528 tdc_write(tdc, TEGRA_GPCDMA_CHAN_SRC_PTR, ch_regs->src_ptr);
529 tdc_write(tdc, TEGRA_GPCDMA_CHAN_DST_PTR, ch_regs->dst_ptr);
530 tdc_write(tdc, TEGRA_GPCDMA_CHAN_HIGH_ADDR_PTR, ch_regs->high_addr_ptr);
531 tdc_write(tdc, TEGRA_GPCDMA_CHAN_FIXED_PATTERN, ch_regs->fixed_pattern);
532 tdc_write(tdc, TEGRA_GPCDMA_CHAN_MMIOSEQ, ch_regs->mmio_seq);
533 tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, ch_regs->mc_seq);
534 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, ch_regs->csr);
535
536 /* Start DMA */
537 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR,
538 ch_regs->csr | TEGRA_GPCDMA_CSR_ENB);
539}
540
541static void tegra_dma_xfer_complete(struct tegra_dma_channel *tdc)
542{
543 vchan_cookie_complete(&tdc->dma_desc->vd);
544
545 tegra_dma_sid_free(tdc);
546 tdc->dma_desc = NULL;
547}
548
549static void tegra_dma_chan_decode_error(struct tegra_dma_channel *tdc,
550 unsigned int err_status)
551{
552 switch (TEGRA_GPCDMA_CHAN_ERR_TYPE(err_status)) {
553 case TEGRA_DMA_BM_FIFO_FULL_ERR:
554 dev_err(tdc->tdma->dev,
555 "GPCDMA CH%d bm fifo full\n", tdc->id);
556 break;
557
558 case TEGRA_DMA_PERIPH_FIFO_FULL_ERR:
559 dev_err(tdc->tdma->dev,
560 "GPCDMA CH%d peripheral fifo full\n", tdc->id);
561 break;
562
563 case TEGRA_DMA_PERIPH_ID_ERR:
564 dev_err(tdc->tdma->dev,
565 "GPCDMA CH%d illegal peripheral id\n", tdc->id);
566 break;
567
568 case TEGRA_DMA_STREAM_ID_ERR:
569 dev_err(tdc->tdma->dev,
570 "GPCDMA CH%d illegal stream id\n", tdc->id);
571 break;
572
573 case TEGRA_DMA_MC_SLAVE_ERR:
574 dev_err(tdc->tdma->dev,
575 "GPCDMA CH%d mc slave error\n", tdc->id);
576 break;
577
578 case TEGRA_DMA_MMIO_SLAVE_ERR:
579 dev_err(tdc->tdma->dev,
580 "GPCDMA CH%d mmio slave error\n", tdc->id);
581 break;
582
583 default:
584 dev_err(tdc->tdma->dev,
585 "GPCDMA CH%d security violation %x\n", tdc->id,
586 err_status);
587 }
588}
589
590static irqreturn_t tegra_dma_isr(int irq, void *dev_id)
591{
592 struct tegra_dma_channel *tdc = dev_id;
593 struct tegra_dma_desc *dma_desc = tdc->dma_desc;
594 struct tegra_dma_sg_req *sg_req;
595 u32 status;
596
597 /* Check channel error status register */
598 status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS);
599 if (status) {
600 tegra_dma_chan_decode_error(tdc, status);
601 tegra_dma_dump_chan_regs(tdc);
602 tdc_write(tdc, TEGRA_GPCDMA_CHAN_ERR_STATUS, 0xFFFFFFFF);
603 }
604
605 spin_lock(&tdc->vc.lock);
606 status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
607 if (!(status & TEGRA_GPCDMA_STATUS_ISE_EOC))
608 goto irq_done;
609
610 tdc_write(tdc, TEGRA_GPCDMA_CHAN_STATUS,
611 TEGRA_GPCDMA_STATUS_ISE_EOC);
612
613 if (!dma_desc)
614 goto irq_done;
615
616 sg_req = dma_desc->sg_req;
617 dma_desc->bytes_xfer += sg_req[dma_desc->sg_idx].len;
618
619 if (dma_desc->cyclic) {
620 vchan_cyclic_callback(&dma_desc->vd);
621 tegra_dma_configure_next_sg(tdc);
622 } else {
623 dma_desc->sg_idx++;
624 if (dma_desc->sg_idx == dma_desc->sg_count)
625 tegra_dma_xfer_complete(tdc);
626 else
627 tegra_dma_start(tdc);
628 }
629
630irq_done:
631 spin_unlock(&tdc->vc.lock);
632 return IRQ_HANDLED;
633}
634
635static void tegra_dma_issue_pending(struct dma_chan *dc)
636{
637 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
638 unsigned long flags;
639
640 if (tdc->dma_desc)
641 return;
642
643 spin_lock_irqsave(&tdc->vc.lock, flags);
644 if (vchan_issue_pending(&tdc->vc))
645 tegra_dma_start(tdc);
646
647 /*
648 * For cyclic DMA transfers, program the second
649 * transfer parameters as soon as the first DMA
650 * transfer is started inorder for the DMA
651 * controller to trigger the second transfer
652 * with the correct parameters.
653 */
654 if (tdc->dma_desc && tdc->dma_desc->cyclic)
655 tegra_dma_configure_next_sg(tdc);
656
657 spin_unlock_irqrestore(&tdc->vc.lock, flags);
658}
659
660static int tegra_dma_stop_client(struct tegra_dma_channel *tdc)
661{
662 int ret;
663 u32 status, csr;
664
665 /*
666 * Change the client associated with the DMA channel
667 * to stop DMA engine from starting any more bursts for
668 * the given client and wait for in flight bursts to complete
669 */
670 csr = tdc_read(tdc, TEGRA_GPCDMA_CHAN_CSR);
671 csr &= ~(TEGRA_GPCDMA_CSR_REQ_SEL_MASK);
672 csr |= TEGRA_GPCDMA_CSR_REQ_SEL_UNUSED;
673 tdc_write(tdc, TEGRA_GPCDMA_CHAN_CSR, csr);
674
675 /* Wait for in flight data transfer to finish */
676 udelay(TEGRA_GPCDMA_BURST_COMPLETE_TIME);
677
678 /* If TX/RX path is still active wait till it becomes
679 * inactive
680 */
681
682 ret = readl_relaxed_poll_timeout_atomic(tdc->tdma->base_addr +
683 tdc->chan_base_offset +
684 TEGRA_GPCDMA_CHAN_STATUS,
685 status,
686 !(status & (TEGRA_GPCDMA_STATUS_CHANNEL_TX |
687 TEGRA_GPCDMA_STATUS_CHANNEL_RX)),
688 5,
689 TEGRA_GPCDMA_BURST_COMPLETION_TIMEOUT);
690 if (ret) {
691 dev_err(tdc2dev(tdc), "Timeout waiting for DMA burst completion!\n");
692 tegra_dma_dump_chan_regs(tdc);
693 }
694
695 return ret;
696}
697
698static int tegra_dma_terminate_all(struct dma_chan *dc)
699{
700 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
701 unsigned long flags;
702 LIST_HEAD(head);
703 int err;
704
705 spin_lock_irqsave(&tdc->vc.lock, flags);
706
707 if (tdc->dma_desc) {
708 err = tdc->tdma->chip_data->terminate(tdc);
709 if (err) {
710 spin_unlock_irqrestore(&tdc->vc.lock, flags);
711 return err;
712 }
713
714 vchan_terminate_vdesc(&tdc->dma_desc->vd);
715 tegra_dma_disable(tdc);
716 tdc->dma_desc = NULL;
717 }
718
719 tegra_dma_sid_free(tdc);
720 vchan_get_all_descriptors(&tdc->vc, &head);
721 spin_unlock_irqrestore(&tdc->vc.lock, flags);
722
723 vchan_dma_desc_free_list(&tdc->vc, &head);
724
725 return 0;
726}
727
728static int tegra_dma_get_residual(struct tegra_dma_channel *tdc)
729{
730 struct tegra_dma_desc *dma_desc = tdc->dma_desc;
731 struct tegra_dma_sg_req *sg_req = dma_desc->sg_req;
732 unsigned int bytes_xfer, residual;
733 u32 wcount = 0, status;
734
735 wcount = tdc_read(tdc, TEGRA_GPCDMA_CHAN_XFER_COUNT);
736
737 /*
738 * Set wcount = 0 if EOC bit is set. The transfer would have
739 * already completed and the CHAN_XFER_COUNT could have updated
740 * for the next transfer, specifically in case of cyclic transfers.
741 */
742 status = tdc_read(tdc, TEGRA_GPCDMA_CHAN_STATUS);
743 if (status & TEGRA_GPCDMA_STATUS_ISE_EOC)
744 wcount = 0;
745
746 bytes_xfer = dma_desc->bytes_xfer +
747 sg_req[dma_desc->sg_idx].len - (wcount * 4);
748
749 if (dma_desc->bytes_req == bytes_xfer)
750 return 0;
751
752 residual = dma_desc->bytes_req - (bytes_xfer % dma_desc->bytes_req);
753
754 return residual;
755}
756
757static enum dma_status tegra_dma_tx_status(struct dma_chan *dc,
758 dma_cookie_t cookie,
759 struct dma_tx_state *txstate)
760{
761 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
762 struct tegra_dma_desc *dma_desc;
763 struct virt_dma_desc *vd;
764 unsigned int residual;
765 unsigned long flags;
766 enum dma_status ret;
767
768 ret = dma_cookie_status(dc, cookie, txstate);
769 if (ret == DMA_COMPLETE)
770 return ret;
771
772 spin_lock_irqsave(&tdc->vc.lock, flags);
773 vd = vchan_find_desc(&tdc->vc, cookie);
774 if (vd) {
775 dma_desc = vd_to_tegra_dma_desc(vd);
776 residual = dma_desc->bytes_req;
777 dma_set_residue(txstate, residual);
778 } else if (tdc->dma_desc && tdc->dma_desc->vd.tx.cookie == cookie) {
779 residual = tegra_dma_get_residual(tdc);
780 dma_set_residue(txstate, residual);
781 } else {
782 dev_err(tdc2dev(tdc), "cookie %d is not found\n", cookie);
783 }
784 spin_unlock_irqrestore(&tdc->vc.lock, flags);
785
786 return ret;
787}
788
789static inline int get_bus_width(struct tegra_dma_channel *tdc,
790 enum dma_slave_buswidth slave_bw)
791{
792 switch (slave_bw) {
793 case DMA_SLAVE_BUSWIDTH_1_BYTE:
794 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_8;
795 case DMA_SLAVE_BUSWIDTH_2_BYTES:
796 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_16;
797 case DMA_SLAVE_BUSWIDTH_4_BYTES:
798 return TEGRA_GPCDMA_MMIOSEQ_BUS_WIDTH_32;
799 default:
800 dev_err(tdc2dev(tdc), "given slave bus width is not supported\n");
801 return -EINVAL;
802 }
803}
804
805static unsigned int get_burst_size(struct tegra_dma_channel *tdc,
806 u32 burst_size, enum dma_slave_buswidth slave_bw,
807 int len)
808{
809 unsigned int burst_mmio_width, burst_byte;
810
811 /*
812 * burst_size from client is in terms of the bus_width.
813 * convert that into words.
814 * If burst_size is not specified from client, then use
815 * len to calculate the optimum burst size
816 */
817 burst_byte = burst_size ? burst_size * slave_bw : len;
818 burst_mmio_width = burst_byte / 4;
819
820 if (burst_mmio_width < TEGRA_GPCDMA_MMIOSEQ_BURST_MIN)
821 return 0;
822
823 burst_mmio_width = min(burst_mmio_width, TEGRA_GPCDMA_MMIOSEQ_BURST_MAX);
824
825 return TEGRA_GPCDMA_MMIOSEQ_BURST(burst_mmio_width);
826}
827
828static int get_transfer_param(struct tegra_dma_channel *tdc,
829 enum dma_transfer_direction direction,
830 u32 *apb_addr,
831 u32 *mmio_seq,
832 u32 *csr,
833 unsigned int *burst_size,
834 enum dma_slave_buswidth *slave_bw)
835{
836 switch (direction) {
837 case DMA_MEM_TO_DEV:
838 *apb_addr = tdc->dma_sconfig.dst_addr;
839 *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.dst_addr_width);
840 *burst_size = tdc->dma_sconfig.dst_maxburst;
841 *slave_bw = tdc->dma_sconfig.dst_addr_width;
842 *csr = TEGRA_GPCDMA_CSR_DMA_MEM2IO_FC;
843 return 0;
844 case DMA_DEV_TO_MEM:
845 *apb_addr = tdc->dma_sconfig.src_addr;
846 *mmio_seq = get_bus_width(tdc, tdc->dma_sconfig.src_addr_width);
847 *burst_size = tdc->dma_sconfig.src_maxburst;
848 *slave_bw = tdc->dma_sconfig.src_addr_width;
849 *csr = TEGRA_GPCDMA_CSR_DMA_IO2MEM_FC;
850 return 0;
851 default:
852 dev_err(tdc2dev(tdc), "DMA direction is not supported\n");
853 }
854
855 return -EINVAL;
856}
857
858static struct dma_async_tx_descriptor *
859tegra_dma_prep_dma_memset(struct dma_chan *dc, dma_addr_t dest, int value,
860 size_t len, unsigned long flags)
861{
862 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
863 unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
864 struct tegra_dma_sg_req *sg_req;
865 struct tegra_dma_desc *dma_desc;
866 u32 csr, mc_seq;
867
868 if ((len & 3) || (dest & 3) || len > max_dma_count) {
869 dev_err(tdc2dev(tdc),
870 "DMA length/memory address is not supported\n");
871 return NULL;
872 }
873
874 /* Set DMA mode to fixed pattern */
875 csr = TEGRA_GPCDMA_CSR_DMA_FIXED_PAT;
876 /* Enable once or continuous mode */
877 csr |= TEGRA_GPCDMA_CSR_ONCE;
878 /* Enable IRQ mask */
879 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
880 /* Enable the DMA interrupt */
881 if (flags & DMA_PREP_INTERRUPT)
882 csr |= TEGRA_GPCDMA_CSR_IE_EOC;
883 /* Configure default priority weight for the channel */
884 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
885
886 mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
887 /* retain stream-id and clean rest */
888 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
889
890 /* Set the address wrapping */
891 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
892 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
893 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
894 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
895
896 /* Program outstanding MC requests */
897 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
898 /* Set burst size */
899 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
900
901 dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);
902 if (!dma_desc)
903 return NULL;
904
905 dma_desc->bytes_req = len;
906 dma_desc->sg_count = 1;
907 sg_req = dma_desc->sg_req;
908
909 sg_req[0].ch_regs.src_ptr = 0;
910 sg_req[0].ch_regs.dst_ptr = dest;
911 sg_req[0].ch_regs.high_addr_ptr =
912 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
913 sg_req[0].ch_regs.fixed_pattern = value;
914 /* Word count reg takes value as (N +1) words */
915 sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
916 sg_req[0].ch_regs.csr = csr;
917 sg_req[0].ch_regs.mmio_seq = 0;
918 sg_req[0].ch_regs.mc_seq = mc_seq;
919 sg_req[0].len = len;
920
921 dma_desc->cyclic = false;
922 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
923}
924
925static struct dma_async_tx_descriptor *
926tegra_dma_prep_dma_memcpy(struct dma_chan *dc, dma_addr_t dest,
927 dma_addr_t src, size_t len, unsigned long flags)
928{
929 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
930 struct tegra_dma_sg_req *sg_req;
931 struct tegra_dma_desc *dma_desc;
932 unsigned int max_dma_count;
933 u32 csr, mc_seq;
934
935 max_dma_count = tdc->tdma->chip_data->max_dma_count;
936 if ((len & 3) || (src & 3) || (dest & 3) || len > max_dma_count) {
937 dev_err(tdc2dev(tdc),
938 "DMA length/memory address is not supported\n");
939 return NULL;
940 }
941
942 /* Set DMA mode to memory to memory transfer */
943 csr = TEGRA_GPCDMA_CSR_DMA_MEM2MEM;
944 /* Enable once or continuous mode */
945 csr |= TEGRA_GPCDMA_CSR_ONCE;
946 /* Enable IRQ mask */
947 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
948 /* Enable the DMA interrupt */
949 if (flags & DMA_PREP_INTERRUPT)
950 csr |= TEGRA_GPCDMA_CSR_IE_EOC;
951 /* Configure default priority weight for the channel */
952 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
953
954 mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
955 /* retain stream-id and clean rest */
956 mc_seq &= (TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK) |
957 (TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
958
959 /* Set the address wrapping */
960 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
961 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
962 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
963 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
964
965 /* Program outstanding MC requests */
966 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
967 /* Set burst size */
968 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
969
970 dma_desc = kzalloc(struct_size(dma_desc, sg_req, 1), GFP_NOWAIT);
971 if (!dma_desc)
972 return NULL;
973
974 dma_desc->bytes_req = len;
975 dma_desc->sg_count = 1;
976 sg_req = dma_desc->sg_req;
977
978 sg_req[0].ch_regs.src_ptr = src;
979 sg_req[0].ch_regs.dst_ptr = dest;
980 sg_req[0].ch_regs.high_addr_ptr =
981 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (src >> 32));
982 sg_req[0].ch_regs.high_addr_ptr |=
983 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (dest >> 32));
984 /* Word count reg takes value as (N +1) words */
985 sg_req[0].ch_regs.wcount = ((len - 4) >> 2);
986 sg_req[0].ch_regs.csr = csr;
987 sg_req[0].ch_regs.mmio_seq = 0;
988 sg_req[0].ch_regs.mc_seq = mc_seq;
989 sg_req[0].len = len;
990
991 dma_desc->cyclic = false;
992 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
993}
994
995static struct dma_async_tx_descriptor *
996tegra_dma_prep_slave_sg(struct dma_chan *dc, struct scatterlist *sgl,
997 unsigned int sg_len, enum dma_transfer_direction direction,
998 unsigned long flags, void *context)
999{
1000 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1001 unsigned int max_dma_count = tdc->tdma->chip_data->max_dma_count;
1002 enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
1003 u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0;
1004 struct tegra_dma_sg_req *sg_req;
1005 struct tegra_dma_desc *dma_desc;
1006 struct scatterlist *sg;
1007 u32 burst_size;
1008 unsigned int i;
1009 int ret;
1010
1011 if (!tdc->config_init) {
1012 dev_err(tdc2dev(tdc), "DMA channel is not configured\n");
1013 return NULL;
1014 }
1015 if (sg_len < 1) {
1016 dev_err(tdc2dev(tdc), "Invalid segment length %d\n", sg_len);
1017 return NULL;
1018 }
1019
1020 ret = tegra_dma_sid_reserve(tdc, direction);
1021 if (ret)
1022 return NULL;
1023
1024 ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,
1025 &burst_size, &slave_bw);
1026 if (ret < 0)
1027 return NULL;
1028
1029 /* Enable once or continuous mode */
1030 csr |= TEGRA_GPCDMA_CSR_ONCE;
1031 /* Program the slave id in requestor select */
1032 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);
1033 /* Enable IRQ mask */
1034 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
1035 /* Configure default priority weight for the channel*/
1036 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
1037
1038 /* Enable the DMA interrupt */
1039 if (flags & DMA_PREP_INTERRUPT)
1040 csr |= TEGRA_GPCDMA_CSR_IE_EOC;
1041
1042 mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1043 /* retain stream-id and clean rest */
1044 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
1045
1046 /* Set the address wrapping on both MC and MMIO side */
1047
1048 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
1049 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1050 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
1051 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1052 mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
1053
1054 /* Program 2 MC outstanding requests by default. */
1055 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
1056
1057 /* Setting MC burst size depending on MMIO burst size */
1058 if (burst_size == 64)
1059 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
1060 else
1061 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;
1062
1063 dma_desc = kzalloc(struct_size(dma_desc, sg_req, sg_len), GFP_NOWAIT);
1064 if (!dma_desc)
1065 return NULL;
1066
1067 dma_desc->sg_count = sg_len;
1068 sg_req = dma_desc->sg_req;
1069
1070 /* Make transfer requests */
1071 for_each_sg(sgl, sg, sg_len, i) {
1072 u32 len;
1073 dma_addr_t mem;
1074
1075 mem = sg_dma_address(sg);
1076 len = sg_dma_len(sg);
1077
1078 if ((len & 3) || (mem & 3) || len > max_dma_count) {
1079 dev_err(tdc2dev(tdc),
1080 "DMA length/memory address is not supported\n");
1081 kfree(dma_desc);
1082 return NULL;
1083 }
1084
1085 mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1086 dma_desc->bytes_req += len;
1087
1088 if (direction == DMA_MEM_TO_DEV) {
1089 sg_req[i].ch_regs.src_ptr = mem;
1090 sg_req[i].ch_regs.dst_ptr = apb_ptr;
1091 sg_req[i].ch_regs.high_addr_ptr =
1092 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
1093 } else if (direction == DMA_DEV_TO_MEM) {
1094 sg_req[i].ch_regs.src_ptr = apb_ptr;
1095 sg_req[i].ch_regs.dst_ptr = mem;
1096 sg_req[i].ch_regs.high_addr_ptr =
1097 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
1098 }
1099
1100 /*
1101 * Word count register takes input in words. Writing a value
1102 * of N into word count register means a req of (N+1) words.
1103 */
1104 sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
1105 sg_req[i].ch_regs.csr = csr;
1106 sg_req[i].ch_regs.mmio_seq = mmio_seq;
1107 sg_req[i].ch_regs.mc_seq = mc_seq;
1108 sg_req[i].len = len;
1109 }
1110
1111 dma_desc->cyclic = false;
1112 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
1113}
1114
1115static struct dma_async_tx_descriptor *
1116tegra_dma_prep_dma_cyclic(struct dma_chan *dc, dma_addr_t buf_addr, size_t buf_len,
1117 size_t period_len, enum dma_transfer_direction direction,
1118 unsigned long flags)
1119{
1120 enum dma_slave_buswidth slave_bw = DMA_SLAVE_BUSWIDTH_UNDEFINED;
1121 u32 csr, mc_seq, apb_ptr = 0, mmio_seq = 0, burst_size;
1122 unsigned int max_dma_count, len, period_count, i;
1123 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1124 struct tegra_dma_desc *dma_desc;
1125 struct tegra_dma_sg_req *sg_req;
1126 dma_addr_t mem = buf_addr;
1127 int ret;
1128
1129 if (!buf_len || !period_len) {
1130 dev_err(tdc2dev(tdc), "Invalid buffer/period len\n");
1131 return NULL;
1132 }
1133
1134 if (!tdc->config_init) {
1135 dev_err(tdc2dev(tdc), "DMA slave is not configured\n");
1136 return NULL;
1137 }
1138
1139 ret = tegra_dma_sid_reserve(tdc, direction);
1140 if (ret)
1141 return NULL;
1142
1143 /*
1144 * We only support cycle transfer when buf_len is multiple of
1145 * period_len.
1146 */
1147 if (buf_len % period_len) {
1148 dev_err(tdc2dev(tdc), "buf_len is not multiple of period_len\n");
1149 return NULL;
1150 }
1151
1152 len = period_len;
1153 max_dma_count = tdc->tdma->chip_data->max_dma_count;
1154 if ((len & 3) || (buf_addr & 3) || len > max_dma_count) {
1155 dev_err(tdc2dev(tdc), "Req len/mem address is not correct\n");
1156 return NULL;
1157 }
1158
1159 ret = get_transfer_param(tdc, direction, &apb_ptr, &mmio_seq, &csr,
1160 &burst_size, &slave_bw);
1161 if (ret < 0)
1162 return NULL;
1163
1164 /* Enable once or continuous mode */
1165 csr &= ~TEGRA_GPCDMA_CSR_ONCE;
1166 /* Program the slave id in requestor select */
1167 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_REQ_SEL_MASK, tdc->slave_id);
1168 /* Enable IRQ mask */
1169 csr |= TEGRA_GPCDMA_CSR_IRQ_MASK;
1170 /* Configure default priority weight for the channel*/
1171 csr |= FIELD_PREP(TEGRA_GPCDMA_CSR_WEIGHT, 1);
1172
1173 /* Enable the DMA interrupt */
1174 if (flags & DMA_PREP_INTERRUPT)
1175 csr |= TEGRA_GPCDMA_CSR_IE_EOC;
1176
1177 mmio_seq |= FIELD_PREP(TEGRA_GPCDMA_MMIOSEQ_WRAP_WORD, 1);
1178
1179 mc_seq = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1180 /* retain stream-id and clean rest */
1181 mc_seq &= TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK;
1182
1183 /* Set the address wrapping on both MC and MMIO side */
1184 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP0,
1185 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1186 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_WRAP1,
1187 TEGRA_GPCDMA_MCSEQ_WRAP_NONE);
1188
1189 /* Program 2 MC outstanding requests by default. */
1190 mc_seq |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_REQ_COUNT, 1);
1191 /* Setting MC burst size depending on MMIO burst size */
1192 if (burst_size == 64)
1193 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_16;
1194 else
1195 mc_seq |= TEGRA_GPCDMA_MCSEQ_BURST_2;
1196
1197 period_count = buf_len / period_len;
1198 dma_desc = kzalloc(struct_size(dma_desc, sg_req, period_count),
1199 GFP_NOWAIT);
1200 if (!dma_desc)
1201 return NULL;
1202
1203 dma_desc->bytes_req = buf_len;
1204 dma_desc->sg_count = period_count;
1205 sg_req = dma_desc->sg_req;
1206
1207 /* Split transfer equal to period size */
1208 for (i = 0; i < period_count; i++) {
1209 mmio_seq |= get_burst_size(tdc, burst_size, slave_bw, len);
1210 if (direction == DMA_MEM_TO_DEV) {
1211 sg_req[i].ch_regs.src_ptr = mem;
1212 sg_req[i].ch_regs.dst_ptr = apb_ptr;
1213 sg_req[i].ch_regs.high_addr_ptr =
1214 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_SRC_PTR, (mem >> 32));
1215 } else if (direction == DMA_DEV_TO_MEM) {
1216 sg_req[i].ch_regs.src_ptr = apb_ptr;
1217 sg_req[i].ch_regs.dst_ptr = mem;
1218 sg_req[i].ch_regs.high_addr_ptr =
1219 FIELD_PREP(TEGRA_GPCDMA_HIGH_ADDR_DST_PTR, (mem >> 32));
1220 }
1221 /*
1222 * Word count register takes input in words. Writing a value
1223 * of N into word count register means a req of (N+1) words.
1224 */
1225 sg_req[i].ch_regs.wcount = ((len - 4) >> 2);
1226 sg_req[i].ch_regs.csr = csr;
1227 sg_req[i].ch_regs.mmio_seq = mmio_seq;
1228 sg_req[i].ch_regs.mc_seq = mc_seq;
1229 sg_req[i].len = len;
1230
1231 mem += len;
1232 }
1233
1234 dma_desc->cyclic = true;
1235
1236 return vchan_tx_prep(&tdc->vc, &dma_desc->vd, flags);
1237}
1238
1239static int tegra_dma_alloc_chan_resources(struct dma_chan *dc)
1240{
1241 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1242 int ret;
1243
1244 ret = request_irq(tdc->irq, tegra_dma_isr, 0, tdc->name, tdc);
1245 if (ret) {
1246 dev_err(tdc2dev(tdc), "request_irq failed for %s\n", tdc->name);
1247 return ret;
1248 }
1249
1250 dma_cookie_init(&tdc->vc.chan);
1251 tdc->config_init = false;
1252 return 0;
1253}
1254
1255static void tegra_dma_chan_synchronize(struct dma_chan *dc)
1256{
1257 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1258
1259 synchronize_irq(tdc->irq);
1260 vchan_synchronize(&tdc->vc);
1261}
1262
1263static void tegra_dma_free_chan_resources(struct dma_chan *dc)
1264{
1265 struct tegra_dma_channel *tdc = to_tegra_dma_chan(dc);
1266
1267 dev_dbg(tdc2dev(tdc), "Freeing channel %d\n", tdc->id);
1268
1269 tegra_dma_terminate_all(dc);
1270 synchronize_irq(tdc->irq);
1271
1272 tasklet_kill(&tdc->vc.task);
1273 tdc->config_init = false;
1274 tdc->slave_id = -1;
1275 tdc->sid_dir = DMA_TRANS_NONE;
1276 free_irq(tdc->irq, tdc);
1277
1278 vchan_free_chan_resources(&tdc->vc);
1279}
1280
1281static struct dma_chan *tegra_dma_of_xlate(struct of_phandle_args *dma_spec,
1282 struct of_dma *ofdma)
1283{
1284 struct tegra_dma *tdma = ofdma->of_dma_data;
1285 struct tegra_dma_channel *tdc;
1286 struct dma_chan *chan;
1287
1288 chan = dma_get_any_slave_channel(&tdma->dma_dev);
1289 if (!chan)
1290 return NULL;
1291
1292 tdc = to_tegra_dma_chan(chan);
1293 tdc->slave_id = dma_spec->args[0];
1294
1295 return chan;
1296}
1297
1298static const struct tegra_dma_chip_data tegra186_dma_chip_data = {
1299 .nr_channels = 32,
1300 .channel_reg_size = SZ_64K,
1301 .max_dma_count = SZ_1G,
1302 .hw_support_pause = false,
1303 .terminate = tegra_dma_stop_client,
1304};
1305
1306static const struct tegra_dma_chip_data tegra194_dma_chip_data = {
1307 .nr_channels = 32,
1308 .channel_reg_size = SZ_64K,
1309 .max_dma_count = SZ_1G,
1310 .hw_support_pause = true,
1311 .terminate = tegra_dma_pause,
1312};
1313
1314static const struct tegra_dma_chip_data tegra234_dma_chip_data = {
1315 .nr_channels = 32,
1316 .channel_reg_size = SZ_64K,
1317 .max_dma_count = SZ_1G,
1318 .hw_support_pause = true,
1319 .terminate = tegra_dma_pause_noerr,
1320};
1321
1322static const struct of_device_id tegra_dma_of_match[] = {
1323 {
1324 .compatible = "nvidia,tegra186-gpcdma",
1325 .data = &tegra186_dma_chip_data,
1326 }, {
1327 .compatible = "nvidia,tegra194-gpcdma",
1328 .data = &tegra194_dma_chip_data,
1329 }, {
1330 .compatible = "nvidia,tegra234-gpcdma",
1331 .data = &tegra234_dma_chip_data,
1332 }, {
1333 },
1334};
1335MODULE_DEVICE_TABLE(of, tegra_dma_of_match);
1336
1337static int tegra_dma_program_sid(struct tegra_dma_channel *tdc, int stream_id)
1338{
1339 unsigned int reg_val = tdc_read(tdc, TEGRA_GPCDMA_CHAN_MCSEQ);
1340
1341 reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK);
1342 reg_val &= ~(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK);
1343
1344 reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID0_MASK, stream_id);
1345 reg_val |= FIELD_PREP(TEGRA_GPCDMA_MCSEQ_STREAM_ID1_MASK, stream_id);
1346
1347 tdc_write(tdc, TEGRA_GPCDMA_CHAN_MCSEQ, reg_val);
1348 return 0;
1349}
1350
1351static int tegra_dma_probe(struct platform_device *pdev)
1352{
1353 const struct tegra_dma_chip_data *cdata = NULL;
1354 unsigned int i;
1355 u32 stream_id;
1356 struct tegra_dma *tdma;
1357 int ret;
1358
1359 cdata = of_device_get_match_data(&pdev->dev);
1360
1361 tdma = devm_kzalloc(&pdev->dev,
1362 struct_size(tdma, channels, cdata->nr_channels),
1363 GFP_KERNEL);
1364 if (!tdma)
1365 return -ENOMEM;
1366
1367 tdma->dev = &pdev->dev;
1368 tdma->chip_data = cdata;
1369 platform_set_drvdata(pdev, tdma);
1370
1371 tdma->base_addr = devm_platform_ioremap_resource(pdev, 0);
1372 if (IS_ERR(tdma->base_addr))
1373 return PTR_ERR(tdma->base_addr);
1374
1375 tdma->rst = devm_reset_control_get_exclusive(&pdev->dev, "gpcdma");
1376 if (IS_ERR(tdma->rst)) {
1377 return dev_err_probe(&pdev->dev, PTR_ERR(tdma->rst),
1378 "Missing controller reset\n");
1379 }
1380 reset_control_reset(tdma->rst);
1381
1382 tdma->dma_dev.dev = &pdev->dev;
1383
1384 if (!tegra_dev_iommu_get_stream_id(&pdev->dev, &stream_id)) {
1385 dev_err(&pdev->dev, "Missing iommu stream-id\n");
1386 return -EINVAL;
1387 }
1388
1389 ret = device_property_read_u32(&pdev->dev, "dma-channel-mask",
1390 &tdma->chan_mask);
1391 if (ret) {
1392 dev_warn(&pdev->dev,
1393 "Missing dma-channel-mask property, using default channel mask %#x\n",
1394 TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK);
1395 tdma->chan_mask = TEGRA_GPCDMA_DEFAULT_CHANNEL_MASK;
1396 }
1397
1398 INIT_LIST_HEAD(&tdma->dma_dev.channels);
1399 for (i = 0; i < cdata->nr_channels; i++) {
1400 struct tegra_dma_channel *tdc = &tdma->channels[i];
1401
1402 /* Check for channel mask */
1403 if (!(tdma->chan_mask & BIT(i)))
1404 continue;
1405
1406 tdc->irq = platform_get_irq(pdev, i);
1407 if (tdc->irq < 0)
1408 return tdc->irq;
1409
1410 tdc->chan_base_offset = TEGRA_GPCDMA_CHANNEL_BASE_ADDR_OFFSET +
1411 i * cdata->channel_reg_size;
1412 snprintf(tdc->name, sizeof(tdc->name), "gpcdma.%d", i);
1413 tdc->tdma = tdma;
1414 tdc->id = i;
1415 tdc->slave_id = -1;
1416
1417 vchan_init(&tdc->vc, &tdma->dma_dev);
1418 tdc->vc.desc_free = tegra_dma_desc_free;
1419
1420 /* program stream-id for this channel */
1421 tegra_dma_program_sid(tdc, stream_id);
1422 tdc->stream_id = stream_id;
1423 }
1424
1425 dma_cap_set(DMA_SLAVE, tdma->dma_dev.cap_mask);
1426 dma_cap_set(DMA_PRIVATE, tdma->dma_dev.cap_mask);
1427 dma_cap_set(DMA_MEMCPY, tdma->dma_dev.cap_mask);
1428 dma_cap_set(DMA_MEMSET, tdma->dma_dev.cap_mask);
1429 dma_cap_set(DMA_CYCLIC, tdma->dma_dev.cap_mask);
1430
1431 /*
1432 * Only word aligned transfers are supported. Set the copy
1433 * alignment shift.
1434 */
1435 tdma->dma_dev.copy_align = 2;
1436 tdma->dma_dev.fill_align = 2;
1437 tdma->dma_dev.device_alloc_chan_resources =
1438 tegra_dma_alloc_chan_resources;
1439 tdma->dma_dev.device_free_chan_resources =
1440 tegra_dma_free_chan_resources;
1441 tdma->dma_dev.device_prep_slave_sg = tegra_dma_prep_slave_sg;
1442 tdma->dma_dev.device_prep_dma_memcpy = tegra_dma_prep_dma_memcpy;
1443 tdma->dma_dev.device_prep_dma_memset = tegra_dma_prep_dma_memset;
1444 tdma->dma_dev.device_prep_dma_cyclic = tegra_dma_prep_dma_cyclic;
1445 tdma->dma_dev.device_config = tegra_dma_slave_config;
1446 tdma->dma_dev.device_terminate_all = tegra_dma_terminate_all;
1447 tdma->dma_dev.device_tx_status = tegra_dma_tx_status;
1448 tdma->dma_dev.device_issue_pending = tegra_dma_issue_pending;
1449 tdma->dma_dev.device_pause = tegra_dma_device_pause;
1450 tdma->dma_dev.device_resume = tegra_dma_device_resume;
1451 tdma->dma_dev.device_synchronize = tegra_dma_chan_synchronize;
1452 tdma->dma_dev.residue_granularity = DMA_RESIDUE_GRANULARITY_BURST;
1453
1454 ret = dma_async_device_register(&tdma->dma_dev);
1455 if (ret < 0) {
1456 dev_err_probe(&pdev->dev, ret,
1457 "GPC DMA driver registration failed\n");
1458 return ret;
1459 }
1460
1461 ret = of_dma_controller_register(pdev->dev.of_node,
1462 tegra_dma_of_xlate, tdma);
1463 if (ret < 0) {
1464 dev_err_probe(&pdev->dev, ret,
1465 "GPC DMA OF registration failed\n");
1466
1467 dma_async_device_unregister(&tdma->dma_dev);
1468 return ret;
1469 }
1470
1471 dev_info(&pdev->dev, "GPC DMA driver register %lu channels\n",
1472 hweight_long(tdma->chan_mask));
1473
1474 return 0;
1475}
1476
1477static void tegra_dma_remove(struct platform_device *pdev)
1478{
1479 struct tegra_dma *tdma = platform_get_drvdata(pdev);
1480
1481 of_dma_controller_free(pdev->dev.of_node);
1482 dma_async_device_unregister(&tdma->dma_dev);
1483}
1484
1485static int __maybe_unused tegra_dma_pm_suspend(struct device *dev)
1486{
1487 struct tegra_dma *tdma = dev_get_drvdata(dev);
1488 unsigned int i;
1489
1490 for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1491 struct tegra_dma_channel *tdc = &tdma->channels[i];
1492
1493 if (!(tdma->chan_mask & BIT(i)))
1494 continue;
1495
1496 if (tdc->dma_desc) {
1497 dev_err(tdma->dev, "channel %u busy\n", i);
1498 return -EBUSY;
1499 }
1500 }
1501
1502 return 0;
1503}
1504
1505static int __maybe_unused tegra_dma_pm_resume(struct device *dev)
1506{
1507 struct tegra_dma *tdma = dev_get_drvdata(dev);
1508 unsigned int i;
1509
1510 reset_control_reset(tdma->rst);
1511
1512 for (i = 0; i < tdma->chip_data->nr_channels; i++) {
1513 struct tegra_dma_channel *tdc = &tdma->channels[i];
1514
1515 if (!(tdma->chan_mask & BIT(i)))
1516 continue;
1517
1518 tegra_dma_program_sid(tdc, tdc->stream_id);
1519 }
1520
1521 return 0;
1522}
1523
1524static const struct dev_pm_ops tegra_dma_dev_pm_ops = {
1525 SET_SYSTEM_SLEEP_PM_OPS(tegra_dma_pm_suspend, tegra_dma_pm_resume)
1526};
1527
1528static struct platform_driver tegra_dma_driver = {
1529 .driver = {
1530 .name = "tegra-gpcdma",
1531 .pm = &tegra_dma_dev_pm_ops,
1532 .of_match_table = tegra_dma_of_match,
1533 },
1534 .probe = tegra_dma_probe,
1535 .remove_new = tegra_dma_remove,
1536};
1537
1538module_platform_driver(tegra_dma_driver);
1539
1540MODULE_DESCRIPTION("NVIDIA Tegra GPC DMA Controller driver");
1541MODULE_AUTHOR("Pavan Kunapuli <pkunapuli@nvidia.com>");
1542MODULE_AUTHOR("Rajesh Gumasta <rgumasta@nvidia.com>");
1543MODULE_LICENSE("GPL");