1/* SPDX-License-Identifier: GPL-2.0-only */
  2/*
  3 * Intel MIC Platform Software Stack (MPSS)
  4 *
  5 * Copyright(c) 2014 Intel Corporation.
  6 *
  7 * Intel MIC X100 DMA Driver.
  8 *
  9 * Adapted from IOAT dma driver.
 10 */
 11#ifndef _MIC_X100_DMA_H_
 12#define _MIC_X100_DMA_H_
 13
 14#include <linux/kernel.h>
 15#include <linux/delay.h>
 16#include <linux/sched.h>
 17#include <linux/debugfs.h>
 18#include <linux/slab.h>
 19#include <linux/interrupt.h>
 20#include <linux/mic_bus.h>
 21
 22#include "dmaengine.h"
 23
 24/*
 25 * MIC has a total of 8 dma channels.
 26 * Four channels are assigned for host SW use & the remaining for MIC SW.
 27 * MIC DMA transfer size & addresses need to be 64 byte aligned.
 28 */
 29#define MIC_DMA_MAX_NUM_CHAN	8
 30#define MIC_DMA_NUM_CHAN	4
 31#define MIC_DMA_ALIGN_SHIFT	DMAENGINE_ALIGN_64_BYTES
 32#define MIC_DMA_ALIGN_BYTES	(1 << MIC_DMA_ALIGN_SHIFT)
 33#define MIC_DMA_DESC_RX_SIZE	(128 * 1024 - 4)
 34
 35/*
 36 * Register descriptions
 37 * All the registers are 32 bit registers.
 38 * DCR is a global register and all others are per-channel.
 39 * DCR - bits 0, 2, 4, 6, 8, 10, 12, 14 - enable bits for channels 0 to 7
 40 *	 bits 1, 3, 5, 7, 9, 11, 13, 15 - owner bits for channels 0 to 7
 41 * DCAR - bit 24 & 25 interrupt masks for mic owned & host owned channels
 42 * DHPR - head of the descriptor ring updated by s/w
 43 * DTPR - tail of the descriptor ring updated by h/w
 44 * DRAR_LO - lower 32 bits of descriptor ring's mic address
 45 * DRAR_HI - 3:0 - remaining 4 bits of descriptor ring's mic address
 46 *	     20:4 descriptor ring size
 47 *	     25:21 mic smpt entry number
 48 * DSTAT - 16:0 h/w completion count; 31:28 dma engine status
 49 * DCHERR - this register is non-zero on error
 50 * DCHERRMSK - interrupt mask register
 51 */
 52#define MIC_DMA_HW_CMP_CNT_MASK		0x1ffff
 53#define MIC_DMA_CHAN_QUIESCE		0x20000000
 54#define MIC_DMA_SBOX_BASE		0x00010000
 55#define MIC_DMA_SBOX_DCR		0x0000A280
 56#define MIC_DMA_SBOX_CH_BASE		0x0001A000
 57#define MIC_DMA_SBOX_CHAN_OFF		0x40
 58#define MIC_DMA_SBOX_DCAR_IM0		(0x1 << 24)
 59#define MIC_DMA_SBOX_DCAR_IM1		(0x1 << 25)
 60#define MIC_DMA_SBOX_DRARHI_SYS_MASK	(0x1 << 26)
 61#define MIC_DMA_REG_DCAR		0
 62#define MIC_DMA_REG_DHPR		4
 63#define MIC_DMA_REG_DTPR		8
 64#define MIC_DMA_REG_DRAR_LO		20
 65#define MIC_DMA_REG_DRAR_HI		24
 66#define MIC_DMA_REG_DSTAT		32
 67#define MIC_DMA_REG_DCHERR		44
 68#define MIC_DMA_REG_DCHERRMSK		48
 69
 70/* HW dma desc */
 71struct mic_dma_desc {
 72	u64 qw0;
 73	u64 qw1;
 74};
 75
 76enum mic_dma_chan_owner {
 77	MIC_DMA_CHAN_MIC = 0,
 78	MIC_DMA_CHAN_HOST
 79};
 80
 81/*
 82 * mic_dma_chan - channel specific information
 83 * @ch_num: channel number
 84 * @owner: owner of this channel
 85 * @last_tail: cached value of descriptor ring tail
 86 * @head: index of next descriptor in desc_ring
 87 * @issued: hardware notification point
 88 * @submitted: index that will be used to submit descriptors to h/w
 89 * @api_ch: dma engine api channel
 90 * @desc_ring: dma descriptor ring
 91 * @desc_ring_micpa: mic physical address of desc_ring
 92 * @status_dest: destination for status (fence) descriptor
 93 * @status_dest_micpa: mic address for status_dest,
 94 *		       DMA controller uses this address
 95 * @tx_array: array of async_tx
 96 * @cleanup_lock: lock held when processing completed tx
 97 * @prep_lock: lock held in prep_memcpy & released in tx_submit
 98 * @issue_lock: lock used to synchronize writes to head
 99 * @cookie: mic_irq cookie used with mic irq request
100 */
101struct mic_dma_chan {
102	int ch_num;
103	enum mic_dma_chan_owner owner;
104	u32 last_tail;
105	u32 head;
106	u32 issued;
107	u32 submitted;
108	struct dma_chan api_ch;
109	struct mic_dma_desc *desc_ring;
110	dma_addr_t desc_ring_micpa;
111	u64 *status_dest;
112	dma_addr_t status_dest_micpa;
113	struct dma_async_tx_descriptor *tx_array;
114	spinlock_t cleanup_lock;
115	spinlock_t prep_lock;
116	spinlock_t issue_lock;
117	struct mic_irq *cookie;
118};
119
120/*
121 * struct mic_dma_device - per mic device
122 * @mic_ch: dma channels
123 * @dma_dev: underlying dma device
124 * @mbdev: mic bus dma device
125 * @mmio: virtual address of the mmio space
126 * @dbg_dir: debugfs directory
127 * @start_ch: first channel number that can be used
128 * @max_xfer_size: maximum transfer size per dma descriptor
129 */
130struct mic_dma_device {
131	struct mic_dma_chan mic_ch[MIC_DMA_MAX_NUM_CHAN];
132	struct dma_device dma_dev;
133	struct mbus_device *mbdev;
134	void __iomem *mmio;
135	struct dentry *dbg_dir;
136	int start_ch;
137	size_t max_xfer_size;
138};
139
140static inline struct mic_dma_chan *to_mic_dma_chan(struct dma_chan *ch)
141{
142	return container_of(ch, struct mic_dma_chan, api_ch);
143}
144
145static inline struct mic_dma_device *to_mic_dma_dev(struct mic_dma_chan *ch)
146{
147	return
148	container_of((const typeof(((struct mic_dma_device *)0)->mic_ch)*)
149		     (ch - ch->ch_num), struct mic_dma_device, mic_ch);
150}
151
152static inline struct mbus_device *to_mbus_device(struct mic_dma_chan *ch)
153{
154	return to_mic_dma_dev(ch)->mbdev;
155}
156
157static inline struct mbus_hw_ops *to_mbus_hw_ops(struct mic_dma_chan *ch)
158{
159	return to_mbus_device(ch)->hw_ops;
160}
161
162static inline struct device *mic_dma_ch_to_device(struct mic_dma_chan *ch)
163{
164	return to_mic_dma_dev(ch)->dma_dev.dev;
165}
166
167static inline void __iomem *mic_dma_chan_to_mmio(struct mic_dma_chan *ch)
168{
169	return to_mic_dma_dev(ch)->mmio;
170}
171
172static inline u32 mic_dma_read_reg(struct mic_dma_chan *ch, u32 reg)
173{
174	return ioread32(mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
175			ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
176}
177
178static inline void mic_dma_write_reg(struct mic_dma_chan *ch, u32 reg, u32 val)
179{
180	iowrite32(val, mic_dma_chan_to_mmio(ch) + MIC_DMA_SBOX_CH_BASE +
181		  ch->ch_num * MIC_DMA_SBOX_CHAN_OFF + reg);
182}
183
184static inline u32 mic_dma_mmio_read(struct mic_dma_chan *ch, u32 offset)
185{
186	return ioread32(mic_dma_chan_to_mmio(ch) + offset);
187}
188
189static inline void mic_dma_mmio_write(struct mic_dma_chan *ch, u32 val,
190				      u32 offset)
191{
192	iowrite32(val, mic_dma_chan_to_mmio(ch) + offset);
193}
194
195static inline u32 mic_dma_read_cmp_cnt(struct mic_dma_chan *ch)
196{
197	return mic_dma_read_reg(ch, MIC_DMA_REG_DSTAT) &
198	       MIC_DMA_HW_CMP_CNT_MASK;
199}
200
201static inline void mic_dma_chan_set_owner(struct mic_dma_chan *ch)
202{
203	u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
204	u32 chan_num = ch->ch_num;
205
206	dcr = (dcr & ~(0x1 << (chan_num * 2))) | (ch->owner << (chan_num * 2));
207	mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
208}
209
210static inline void mic_dma_enable_chan(struct mic_dma_chan *ch)
211{
212	u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
213
214	dcr |= 2 << (ch->ch_num << 1);
215	mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
216}
217
218static inline void mic_dma_disable_chan(struct mic_dma_chan *ch)
219{
220	u32 dcr = mic_dma_mmio_read(ch, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
221
222	dcr &= ~(2 << (ch->ch_num << 1));
223	mic_dma_mmio_write(ch, dcr, MIC_DMA_SBOX_BASE + MIC_DMA_SBOX_DCR);
224}
225
226static void mic_dma_chan_set_desc_ring(struct mic_dma_chan *ch)
227{
228	u32 drar_hi;
229	dma_addr_t desc_ring_micpa = ch->desc_ring_micpa;
230
231	drar_hi = (MIC_DMA_DESC_RX_SIZE & 0x1ffff) << 4;
232	if (MIC_DMA_CHAN_MIC == ch->owner) {
233		drar_hi |= (desc_ring_micpa >> 32) & 0xf;
234	} else {
235		drar_hi |= MIC_DMA_SBOX_DRARHI_SYS_MASK;
236		drar_hi |= ((desc_ring_micpa >> 34)
237			    & 0x1f) << 21;
238		drar_hi |= (desc_ring_micpa >> 32) & 0x3;
239	}
240	mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_LO, (u32) desc_ring_micpa);
241	mic_dma_write_reg(ch, MIC_DMA_REG_DRAR_HI, drar_hi);
242}
243
244static inline void mic_dma_chan_mask_intr(struct mic_dma_chan *ch)
245{
246	u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
247
248	if (MIC_DMA_CHAN_MIC == ch->owner)
249		dcar |= MIC_DMA_SBOX_DCAR_IM0;
250	else
251		dcar |= MIC_DMA_SBOX_DCAR_IM1;
252	mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
253}
254
255static inline void mic_dma_chan_unmask_intr(struct mic_dma_chan *ch)
256{
257	u32 dcar = mic_dma_read_reg(ch, MIC_DMA_REG_DCAR);
258
259	if (MIC_DMA_CHAN_MIC == ch->owner)
260		dcar &= ~MIC_DMA_SBOX_DCAR_IM0;
261	else
262		dcar &= ~MIC_DMA_SBOX_DCAR_IM1;
263	mic_dma_write_reg(ch, MIC_DMA_REG_DCAR, dcar);
264}
265
266static void mic_dma_ack_interrupt(struct mic_dma_chan *ch)
267{
268	if (MIC_DMA_CHAN_MIC == ch->owner) {
269		/* HW errata */
270		mic_dma_chan_mask_intr(ch);
271		mic_dma_chan_unmask_intr(ch);
272	}
273	to_mbus_hw_ops(ch)->ack_interrupt(to_mbus_device(ch), ch->ch_num);
274}
275#endif