1// SPDX-License-Identifier: GPL-2.0-only
  2/*
  3 * Tegra host1x Command DMA
  4 *
  5 * Copyright (c) 2010-2013, NVIDIA Corporation.
  6 */
  7
  8#include <linux/slab.h>
  9#include <linux/scatterlist.h>
 10#include <linux/dma-mapping.h>
 11
 12#include "../cdma.h"
 13#include "../channel.h"
 14#include "../dev.h"
 15#include "../debug.h"
 16
 17/*
 18 * Put the restart at the end of pushbuffer memory
 19 */
 20static void push_buffer_init(struct push_buffer *pb)
 21{
 22	*(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
 23}
 24
 25/*
 26 * Increment timedout buffer's syncpt via CPU.
 27 */
 28static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
 29				u32 syncpt_incrs, u32 syncval, u32 nr_slots)
 30{
 31	unsigned int i;
 32
 33	for (i = 0; i < syncpt_incrs; i++)
 34		host1x_syncpt_incr(cdma->timeout.syncpt);
 35
 36	/* after CPU incr, ensure shadow is up to date */
 37	host1x_syncpt_load(cdma->timeout.syncpt);
 38}
 39
 40/*
 41 * Start channel DMA
 42 */
 43static void cdma_start(struct host1x_cdma *cdma)
 44{
 45	struct host1x_channel *ch = cdma_to_channel(cdma);
 46	u64 start, end;
 47
 48	if (cdma->running)
 49		return;
 50
 51	cdma->last_pos = cdma->push_buffer.pos;
 52	start = cdma->push_buffer.dma;
 53	end = cdma->push_buffer.size + 4;
 54
 55	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
 56			 HOST1X_CHANNEL_DMACTRL);
 57
 58	/* set base, put and end pointer */
 59	host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
 60#if HOST1X_HW >= 6
 61	host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
 62#endif
 63	host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
 64#if HOST1X_HW >= 6
 65	host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMAPUT_HI);
 66#endif
 67	host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
 68#if HOST1X_HW >= 6
 69	host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
 70#endif
 71
 72	/* reset GET */
 73	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
 74			 HOST1X_CHANNEL_DMACTRL_DMAGETRST |
 75			 HOST1X_CHANNEL_DMACTRL_DMAINITGET,
 76			 HOST1X_CHANNEL_DMACTRL);
 77
 78	/* start the command DMA */
 79	host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
 80
 81	cdma->running = true;
 82}
 83
 84/*
 85 * Similar to cdma_start(), but rather than starting from an idle
 86 * state (where DMA GET is set to DMA PUT), on a timeout we restore
 87 * DMA GET from an explicit value (so DMA may again be pending).
 88 */
 89static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
 90{
 91	struct host1x *host1x = cdma_to_host1x(cdma);
 92	struct host1x_channel *ch = cdma_to_channel(cdma);
 93	u64 start, end;
 94
 95	if (cdma->running)
 96		return;
 97
 98	cdma->last_pos = cdma->push_buffer.pos;
 99
100	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
101			 HOST1X_CHANNEL_DMACTRL);
102
103	start = cdma->push_buffer.dma;
104	end = cdma->push_buffer.size + 4;
105
106	/* set base, end pointer (all of memory) */
107	host1x_ch_writel(ch, lower_32_bits(start), HOST1X_CHANNEL_DMASTART);
108#if HOST1X_HW >= 6
109	host1x_ch_writel(ch, upper_32_bits(start), HOST1X_CHANNEL_DMASTART_HI);
110#endif
111	host1x_ch_writel(ch, lower_32_bits(end), HOST1X_CHANNEL_DMAEND);
112#if HOST1X_HW >= 6
113	host1x_ch_writel(ch, upper_32_bits(end), HOST1X_CHANNEL_DMAEND_HI);
114#endif
115
116	/* set GET, by loading the value in PUT (then reset GET) */
117	host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
118	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
119			 HOST1X_CHANNEL_DMACTRL_DMAGETRST |
120			 HOST1X_CHANNEL_DMACTRL_DMAINITGET,
121			 HOST1X_CHANNEL_DMACTRL);
122
123	dev_dbg(host1x->dev,
124		"%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
125		host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
126		host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
127		cdma->last_pos);
128
129	/* deassert GET reset and set PUT */
130	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
131			 HOST1X_CHANNEL_DMACTRL);
132	host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
133
134	/* start the command DMA */
135	host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
136
137	cdma->running = true;
138}
139
140/*
141 * Kick channel DMA into action by writing its PUT offset (if it has changed)
142 */
143static void cdma_flush(struct host1x_cdma *cdma)
144{
145	struct host1x_channel *ch = cdma_to_channel(cdma);
146
147	if (cdma->push_buffer.pos != cdma->last_pos) {
148		host1x_ch_writel(ch, cdma->push_buffer.pos,
149				 HOST1X_CHANNEL_DMAPUT);
150		cdma->last_pos = cdma->push_buffer.pos;
151	}
152}
153
154static void cdma_stop(struct host1x_cdma *cdma)
155{
156	struct host1x_channel *ch = cdma_to_channel(cdma);
157
158	mutex_lock(&cdma->lock);
159
160	if (cdma->running) {
161		host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
162		host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
163				 HOST1X_CHANNEL_DMACTRL);
164		cdma->running = false;
165	}
166
167	mutex_unlock(&cdma->lock);
168}
169
170static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
171				 bool stop)
172{
173#if HOST1X_HW >= 6
174	host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
175#else
176	u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
177	if (stop)
178		cmdproc_stop |= BIT(ch->id);
179	else
180		cmdproc_stop &= ~BIT(ch->id);
181	host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
182#endif
183}
184
185static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
186{
187#if HOST1X_HW >= 6
188	host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
189#else
190	host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
191#endif
192}
193
194/*
195 * Stops both channel's command processor and CDMA immediately.
196 * Also, tears down the channel and resets corresponding module.
197 */
198static void cdma_freeze(struct host1x_cdma *cdma)
199{
200	struct host1x *host = cdma_to_host1x(cdma);
201	struct host1x_channel *ch = cdma_to_channel(cdma);
202
203	if (cdma->torndown && !cdma->running) {
204		dev_warn(host->dev, "Already torn down\n");
205		return;
206	}
207
208	dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
209
210	cdma_hw_cmdproc_stop(host, ch, true);
211
212	dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
213		__func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
214		host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
215		cdma->last_pos);
216
217	host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
218			 HOST1X_CHANNEL_DMACTRL);
219
220	cdma_hw_teardown(host, ch);
221
222	cdma->running = false;
223	cdma->torndown = true;
224}
225
226static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
227{
228	struct host1x *host1x = cdma_to_host1x(cdma);
229	struct host1x_channel *ch = cdma_to_channel(cdma);
230
231	dev_dbg(host1x->dev,
232		"resuming channel (id %u, DMAGET restart = 0x%x)\n",
233		ch->id, getptr);
234
235	cdma_hw_cmdproc_stop(host1x, ch, false);
236
237	cdma->torndown = false;
238	cdma_timeout_restart(cdma, getptr);
239}
240
241/*
242 * If this timeout fires, it indicates the current sync_queue entry has
243 * exceeded its TTL and the userctx should be timed out and remaining
244 * submits already issued cleaned up (future submits return an error).
245 */
246static void cdma_timeout_handler(struct work_struct *work)
247{
248	u32 syncpt_val;
249	struct host1x_cdma *cdma;
250	struct host1x *host1x;
251	struct host1x_channel *ch;
252
253	cdma = container_of(to_delayed_work(work), struct host1x_cdma,
254			    timeout.wq);
255	host1x = cdma_to_host1x(cdma);
256	ch = cdma_to_channel(cdma);
257
258	host1x_debug_dump(cdma_to_host1x(cdma));
259
260	mutex_lock(&cdma->lock);
261
262	if (!cdma->timeout.client) {
263		dev_dbg(host1x->dev,
264			"cdma_timeout: expired, but has no clientid\n");
265		mutex_unlock(&cdma->lock);
266		return;
267	}
268
269	/* stop processing to get a clean snapshot */
270	cdma_hw_cmdproc_stop(host1x, ch, true);
271
272	syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
273
274	/* has buffer actually completed? */
275	if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
276		dev_dbg(host1x->dev,
277			"cdma_timeout: expired, but buffer had completed\n");
278		/* restore */
279		cdma_hw_cmdproc_stop(host1x, ch, false);
280		mutex_unlock(&cdma->lock);
281		return;
282	}
283
284	dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
285		 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
286		 syncpt_val, cdma->timeout.syncpt_val);
287
288	/* stop HW, resetting channel/module */
289	host1x_hw_cdma_freeze(host1x, cdma);
290
291	host1x_cdma_update_sync_queue(cdma, ch->dev);
292	mutex_unlock(&cdma->lock);
293}
294
295/*
296 * Init timeout resources
297 */
298static int cdma_timeout_init(struct host1x_cdma *cdma)
299{
300	INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
301	cdma->timeout.initialized = true;
302
303	return 0;
304}
305
306/*
307 * Clean up timeout resources
308 */
309static void cdma_timeout_destroy(struct host1x_cdma *cdma)
310{
311	if (cdma->timeout.initialized)
312		cancel_delayed_work(&cdma->timeout.wq);
313
314	cdma->timeout.initialized = false;
315}
316
317static const struct host1x_cdma_ops host1x_cdma_ops = {
318	.start = cdma_start,
319	.stop = cdma_stop,
320	.flush = cdma_flush,
321
322	.timeout_init = cdma_timeout_init,
323	.timeout_destroy = cdma_timeout_destroy,
324	.freeze = cdma_freeze,
325	.resume = cdma_resume,
326	.timeout_cpu_incr = cdma_timeout_cpu_incr,
327};
328
329static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
330	.init = push_buffer_init,
331};