Loading...
Note: File does not exist in v3.1.
1/*
2 * Tegra host1x Command DMA
3 *
4 * Copyright (c) 2010-2013, NVIDIA Corporation.
5 *
6 * This program is free software; you can redistribute it and/or modify it
7 * under the terms and conditions of the GNU General Public License,
8 * version 2, as published by the Free Software Foundation.
9 *
10 * This program is distributed in the hope it will be useful, but WITHOUT
11 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
12 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
13 * more details.
14 *
15 * You should have received a copy of the GNU General Public License
16 * along with this program. If not, see <http://www.gnu.org/licenses/>.
17 */
18
19#include <linux/slab.h>
20#include <linux/scatterlist.h>
21#include <linux/dma-mapping.h>
22
23#include "../cdma.h"
24#include "../channel.h"
25#include "../dev.h"
26#include "../debug.h"
27
28/*
29 * Put the restart at the end of pushbuffer memory
30 */
31static void push_buffer_init(struct push_buffer *pb)
32{
33 *(u32 *)(pb->mapped + pb->size) = host1x_opcode_restart(0);
34}
35
36/*
37 * Increment timedout buffer's syncpt via CPU.
38 */
39static void cdma_timeout_cpu_incr(struct host1x_cdma *cdma, u32 getptr,
40 u32 syncpt_incrs, u32 syncval, u32 nr_slots)
41{
42 struct host1x *host1x = cdma_to_host1x(cdma);
43 struct push_buffer *pb = &cdma->push_buffer;
44 unsigned int i;
45
46 for (i = 0; i < syncpt_incrs; i++)
47 host1x_syncpt_incr(cdma->timeout.syncpt);
48
49 /* after CPU incr, ensure shadow is up to date */
50 host1x_syncpt_load(cdma->timeout.syncpt);
51
52 /* NOP all the PB slots */
53 while (nr_slots--) {
54 u32 *p = (u32 *)(pb->mapped + getptr);
55 *(p++) = HOST1X_OPCODE_NOP;
56 *(p++) = HOST1X_OPCODE_NOP;
57 dev_dbg(host1x->dev, "%s: NOP at %pad+%#x\n", __func__,
58 &pb->dma, getptr);
59 getptr = (getptr + 8) & (pb->size - 1);
60 }
61
62 wmb();
63}
64
65/*
66 * Start channel DMA
67 */
68static void cdma_start(struct host1x_cdma *cdma)
69{
70 struct host1x_channel *ch = cdma_to_channel(cdma);
71
72 if (cdma->running)
73 return;
74
75 cdma->last_pos = cdma->push_buffer.pos;
76
77 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
78 HOST1X_CHANNEL_DMACTRL);
79
80 /* set base, put and end pointer */
81 host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART);
82 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
83 host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size + 4,
84 HOST1X_CHANNEL_DMAEND);
85
86 /* reset GET */
87 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
88 HOST1X_CHANNEL_DMACTRL_DMAGETRST |
89 HOST1X_CHANNEL_DMACTRL_DMAINITGET,
90 HOST1X_CHANNEL_DMACTRL);
91
92 /* start the command DMA */
93 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
94
95 cdma->running = true;
96}
97
98/*
99 * Similar to cdma_start(), but rather than starting from an idle
100 * state (where DMA GET is set to DMA PUT), on a timeout we restore
101 * DMA GET from an explicit value (so DMA may again be pending).
102 */
103static void cdma_timeout_restart(struct host1x_cdma *cdma, u32 getptr)
104{
105 struct host1x *host1x = cdma_to_host1x(cdma);
106 struct host1x_channel *ch = cdma_to_channel(cdma);
107
108 if (cdma->running)
109 return;
110
111 cdma->last_pos = cdma->push_buffer.pos;
112
113 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
114 HOST1X_CHANNEL_DMACTRL);
115
116 /* set base, end pointer (all of memory) */
117 host1x_ch_writel(ch, cdma->push_buffer.dma, HOST1X_CHANNEL_DMASTART);
118 host1x_ch_writel(ch, cdma->push_buffer.dma + cdma->push_buffer.size,
119 HOST1X_CHANNEL_DMAEND);
120
121 /* set GET, by loading the value in PUT (then reset GET) */
122 host1x_ch_writel(ch, getptr, HOST1X_CHANNEL_DMAPUT);
123 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP |
124 HOST1X_CHANNEL_DMACTRL_DMAGETRST |
125 HOST1X_CHANNEL_DMACTRL_DMAINITGET,
126 HOST1X_CHANNEL_DMACTRL);
127
128 dev_dbg(host1x->dev,
129 "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n", __func__,
130 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
131 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
132 cdma->last_pos);
133
134 /* deassert GET reset and set PUT */
135 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
136 HOST1X_CHANNEL_DMACTRL);
137 host1x_ch_writel(ch, cdma->push_buffer.pos, HOST1X_CHANNEL_DMAPUT);
138
139 /* start the command DMA */
140 host1x_ch_writel(ch, 0, HOST1X_CHANNEL_DMACTRL);
141
142 cdma->running = true;
143}
144
145/*
146 * Kick channel DMA into action by writing its PUT offset (if it has changed)
147 */
148static void cdma_flush(struct host1x_cdma *cdma)
149{
150 struct host1x_channel *ch = cdma_to_channel(cdma);
151
152 if (cdma->push_buffer.pos != cdma->last_pos) {
153 host1x_ch_writel(ch, cdma->push_buffer.pos,
154 HOST1X_CHANNEL_DMAPUT);
155 cdma->last_pos = cdma->push_buffer.pos;
156 }
157}
158
159static void cdma_stop(struct host1x_cdma *cdma)
160{
161 struct host1x_channel *ch = cdma_to_channel(cdma);
162
163 mutex_lock(&cdma->lock);
164
165 if (cdma->running) {
166 host1x_cdma_wait_locked(cdma, CDMA_EVENT_SYNC_QUEUE_EMPTY);
167 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
168 HOST1X_CHANNEL_DMACTRL);
169 cdma->running = false;
170 }
171
172 mutex_unlock(&cdma->lock);
173}
174
175static void cdma_hw_cmdproc_stop(struct host1x *host, struct host1x_channel *ch,
176 bool stop)
177{
178#if HOST1X_HW >= 6
179 host1x_ch_writel(ch, stop ? 0x1 : 0x0, HOST1X_CHANNEL_CMDPROC_STOP);
180#else
181 u32 cmdproc_stop = host1x_sync_readl(host, HOST1X_SYNC_CMDPROC_STOP);
182 if (stop)
183 cmdproc_stop |= BIT(ch->id);
184 else
185 cmdproc_stop &= ~BIT(ch->id);
186 host1x_sync_writel(host, cmdproc_stop, HOST1X_SYNC_CMDPROC_STOP);
187#endif
188}
189
190static void cdma_hw_teardown(struct host1x *host, struct host1x_channel *ch)
191{
192#if HOST1X_HW >= 6
193 host1x_ch_writel(ch, 0x1, HOST1X_CHANNEL_TEARDOWN);
194#else
195 host1x_sync_writel(host, BIT(ch->id), HOST1X_SYNC_CH_TEARDOWN);
196#endif
197}
198
199/*
200 * Stops both channel's command processor and CDMA immediately.
201 * Also, tears down the channel and resets corresponding module.
202 */
203static void cdma_freeze(struct host1x_cdma *cdma)
204{
205 struct host1x *host = cdma_to_host1x(cdma);
206 struct host1x_channel *ch = cdma_to_channel(cdma);
207
208 if (cdma->torndown && !cdma->running) {
209 dev_warn(host->dev, "Already torn down\n");
210 return;
211 }
212
213 dev_dbg(host->dev, "freezing channel (id %d)\n", ch->id);
214
215 cdma_hw_cmdproc_stop(host, ch, true);
216
217 dev_dbg(host->dev, "%s: DMA GET 0x%x, PUT HW 0x%x / shadow 0x%x\n",
218 __func__, host1x_ch_readl(ch, HOST1X_CHANNEL_DMAGET),
219 host1x_ch_readl(ch, HOST1X_CHANNEL_DMAPUT),
220 cdma->last_pos);
221
222 host1x_ch_writel(ch, HOST1X_CHANNEL_DMACTRL_DMASTOP,
223 HOST1X_CHANNEL_DMACTRL);
224
225 cdma_hw_teardown(host, ch);
226
227 cdma->running = false;
228 cdma->torndown = true;
229}
230
231static void cdma_resume(struct host1x_cdma *cdma, u32 getptr)
232{
233 struct host1x *host1x = cdma_to_host1x(cdma);
234 struct host1x_channel *ch = cdma_to_channel(cdma);
235
236 dev_dbg(host1x->dev,
237 "resuming channel (id %u, DMAGET restart = 0x%x)\n",
238 ch->id, getptr);
239
240 cdma_hw_cmdproc_stop(host1x, ch, false);
241
242 cdma->torndown = false;
243 cdma_timeout_restart(cdma, getptr);
244}
245
246/*
247 * If this timeout fires, it indicates the current sync_queue entry has
248 * exceeded its TTL and the userctx should be timed out and remaining
249 * submits already issued cleaned up (future submits return an error).
250 */
251static void cdma_timeout_handler(struct work_struct *work)
252{
253 u32 syncpt_val;
254 struct host1x_cdma *cdma;
255 struct host1x *host1x;
256 struct host1x_channel *ch;
257
258 cdma = container_of(to_delayed_work(work), struct host1x_cdma,
259 timeout.wq);
260 host1x = cdma_to_host1x(cdma);
261 ch = cdma_to_channel(cdma);
262
263 host1x_debug_dump(cdma_to_host1x(cdma));
264
265 mutex_lock(&cdma->lock);
266
267 if (!cdma->timeout.client) {
268 dev_dbg(host1x->dev,
269 "cdma_timeout: expired, but has no clientid\n");
270 mutex_unlock(&cdma->lock);
271 return;
272 }
273
274 /* stop processing to get a clean snapshot */
275 cdma_hw_cmdproc_stop(host1x, ch, true);
276
277 syncpt_val = host1x_syncpt_load(cdma->timeout.syncpt);
278
279 /* has buffer actually completed? */
280 if ((s32)(syncpt_val - cdma->timeout.syncpt_val) >= 0) {
281 dev_dbg(host1x->dev,
282 "cdma_timeout: expired, but buffer had completed\n");
283 /* restore */
284 cdma_hw_cmdproc_stop(host1x, ch, false);
285 mutex_unlock(&cdma->lock);
286 return;
287 }
288
289 dev_warn(host1x->dev, "%s: timeout: %u (%s), HW thresh %d, done %d\n",
290 __func__, cdma->timeout.syncpt->id, cdma->timeout.syncpt->name,
291 syncpt_val, cdma->timeout.syncpt_val);
292
293 /* stop HW, resetting channel/module */
294 host1x_hw_cdma_freeze(host1x, cdma);
295
296 host1x_cdma_update_sync_queue(cdma, ch->dev);
297 mutex_unlock(&cdma->lock);
298}
299
300/*
301 * Init timeout resources
302 */
303static int cdma_timeout_init(struct host1x_cdma *cdma, unsigned int syncpt)
304{
305 INIT_DELAYED_WORK(&cdma->timeout.wq, cdma_timeout_handler);
306 cdma->timeout.initialized = true;
307
308 return 0;
309}
310
311/*
312 * Clean up timeout resources
313 */
314static void cdma_timeout_destroy(struct host1x_cdma *cdma)
315{
316 if (cdma->timeout.initialized)
317 cancel_delayed_work(&cdma->timeout.wq);
318
319 cdma->timeout.initialized = false;
320}
321
322static const struct host1x_cdma_ops host1x_cdma_ops = {
323 .start = cdma_start,
324 .stop = cdma_stop,
325 .flush = cdma_flush,
326
327 .timeout_init = cdma_timeout_init,
328 .timeout_destroy = cdma_timeout_destroy,
329 .freeze = cdma_freeze,
330 .resume = cdma_resume,
331 .timeout_cpu_incr = cdma_timeout_cpu_incr,
332};
333
334static const struct host1x_pushbuffer_ops host1x_pushbuffer_ops = {
335 .init = push_buffer_init,
336};