Loading...
1/*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Dave Airlie
30 */
31#include <linux/seq_file.h>
32#include <linux/atomic.h>
33#include <linux/wait.h>
34#include <linux/list.h>
35#include <linux/kref.h>
36#include <linux/slab.h>
37#include "drmP.h"
38#include "drm.h"
39#include "radeon_reg.h"
40#include "radeon.h"
41#include "radeon_trace.h"
42
43static void radeon_fence_write(struct radeon_device *rdev, u32 seq)
44{
45 if (rdev->wb.enabled) {
46 u32 scratch_index;
47 if (rdev->wb.use_event)
48 scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
49 else
50 scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
51 rdev->wb.wb[scratch_index/4] = cpu_to_le32(seq);;
52 } else
53 WREG32(rdev->fence_drv.scratch_reg, seq);
54}
55
56static u32 radeon_fence_read(struct radeon_device *rdev)
57{
58 u32 seq;
59
60 if (rdev->wb.enabled) {
61 u32 scratch_index;
62 if (rdev->wb.use_event)
63 scratch_index = R600_WB_EVENT_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
64 else
65 scratch_index = RADEON_WB_SCRATCH_OFFSET + rdev->fence_drv.scratch_reg - rdev->scratch.reg_base;
66 seq = le32_to_cpu(rdev->wb.wb[scratch_index/4]);
67 } else
68 seq = RREG32(rdev->fence_drv.scratch_reg);
69 return seq;
70}
71
72int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
73{
74 unsigned long irq_flags;
75
76 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
77 if (fence->emited) {
78 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
79 return 0;
80 }
81 fence->seq = atomic_add_return(1, &rdev->fence_drv.seq);
82 if (!rdev->cp.ready)
83 /* FIXME: cp is not running assume everythings is done right
84 * away
85 */
86 radeon_fence_write(rdev, fence->seq);
87 else
88 radeon_fence_ring_emit(rdev, fence);
89
90 trace_radeon_fence_emit(rdev->ddev, fence->seq);
91 fence->emited = true;
92 list_move_tail(&fence->list, &rdev->fence_drv.emited);
93 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
94 return 0;
95}
96
97static bool radeon_fence_poll_locked(struct radeon_device *rdev)
98{
99 struct radeon_fence *fence;
100 struct list_head *i, *n;
101 uint32_t seq;
102 bool wake = false;
103 unsigned long cjiffies;
104
105 seq = radeon_fence_read(rdev);
106 if (seq != rdev->fence_drv.last_seq) {
107 rdev->fence_drv.last_seq = seq;
108 rdev->fence_drv.last_jiffies = jiffies;
109 rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
110 } else {
111 cjiffies = jiffies;
112 if (time_after(cjiffies, rdev->fence_drv.last_jiffies)) {
113 cjiffies -= rdev->fence_drv.last_jiffies;
114 if (time_after(rdev->fence_drv.last_timeout, cjiffies)) {
115 /* update the timeout */
116 rdev->fence_drv.last_timeout -= cjiffies;
117 } else {
118 /* the 500ms timeout is elapsed we should test
119 * for GPU lockup
120 */
121 rdev->fence_drv.last_timeout = 1;
122 }
123 } else {
124 /* wrap around update last jiffies, we will just wait
125 * a little longer
126 */
127 rdev->fence_drv.last_jiffies = cjiffies;
128 }
129 return false;
130 }
131 n = NULL;
132 list_for_each(i, &rdev->fence_drv.emited) {
133 fence = list_entry(i, struct radeon_fence, list);
134 if (fence->seq == seq) {
135 n = i;
136 break;
137 }
138 }
139 /* all fence previous to this one are considered as signaled */
140 if (n) {
141 i = n;
142 do {
143 n = i->prev;
144 list_move_tail(i, &rdev->fence_drv.signaled);
145 fence = list_entry(i, struct radeon_fence, list);
146 fence->signaled = true;
147 i = n;
148 } while (i != &rdev->fence_drv.emited);
149 wake = true;
150 }
151 return wake;
152}
153
154static void radeon_fence_destroy(struct kref *kref)
155{
156 unsigned long irq_flags;
157 struct radeon_fence *fence;
158
159 fence = container_of(kref, struct radeon_fence, kref);
160 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags);
161 list_del(&fence->list);
162 fence->emited = false;
163 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags);
164 kfree(fence);
165}
166
167int radeon_fence_create(struct radeon_device *rdev, struct radeon_fence **fence)
168{
169 unsigned long irq_flags;
170
171 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
172 if ((*fence) == NULL) {
173 return -ENOMEM;
174 }
175 kref_init(&((*fence)->kref));
176 (*fence)->rdev = rdev;
177 (*fence)->emited = false;
178 (*fence)->signaled = false;
179 (*fence)->seq = 0;
180 INIT_LIST_HEAD(&(*fence)->list);
181
182 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
183 list_add_tail(&(*fence)->list, &rdev->fence_drv.created);
184 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
185 return 0;
186}
187
188
189bool radeon_fence_signaled(struct radeon_fence *fence)
190{
191 unsigned long irq_flags;
192 bool signaled = false;
193
194 if (!fence)
195 return true;
196
197 if (fence->rdev->gpu_lockup)
198 return true;
199
200 write_lock_irqsave(&fence->rdev->fence_drv.lock, irq_flags);
201 signaled = fence->signaled;
202 /* if we are shuting down report all fence as signaled */
203 if (fence->rdev->shutdown) {
204 signaled = true;
205 }
206 if (!fence->emited) {
207 WARN(1, "Querying an unemited fence : %p !\n", fence);
208 signaled = true;
209 }
210 if (!signaled) {
211 radeon_fence_poll_locked(fence->rdev);
212 signaled = fence->signaled;
213 }
214 write_unlock_irqrestore(&fence->rdev->fence_drv.lock, irq_flags);
215 return signaled;
216}
217
218int radeon_fence_wait(struct radeon_fence *fence, bool intr)
219{
220 struct radeon_device *rdev;
221 unsigned long irq_flags, timeout;
222 u32 seq;
223 int r;
224
225 if (fence == NULL) {
226 WARN(1, "Querying an invalid fence : %p !\n", fence);
227 return 0;
228 }
229 rdev = fence->rdev;
230 if (radeon_fence_signaled(fence)) {
231 return 0;
232 }
233 timeout = rdev->fence_drv.last_timeout;
234retry:
235 /* save current sequence used to check for GPU lockup */
236 seq = rdev->fence_drv.last_seq;
237 trace_radeon_fence_wait_begin(rdev->ddev, seq);
238 if (intr) {
239 radeon_irq_kms_sw_irq_get(rdev);
240 r = wait_event_interruptible_timeout(rdev->fence_drv.queue,
241 radeon_fence_signaled(fence), timeout);
242 radeon_irq_kms_sw_irq_put(rdev);
243 if (unlikely(r < 0)) {
244 return r;
245 }
246 } else {
247 radeon_irq_kms_sw_irq_get(rdev);
248 r = wait_event_timeout(rdev->fence_drv.queue,
249 radeon_fence_signaled(fence), timeout);
250 radeon_irq_kms_sw_irq_put(rdev);
251 }
252 trace_radeon_fence_wait_end(rdev->ddev, seq);
253 if (unlikely(!radeon_fence_signaled(fence))) {
254 /* we were interrupted for some reason and fence isn't
255 * isn't signaled yet, resume wait
256 */
257 if (r) {
258 timeout = r;
259 goto retry;
260 }
261 /* don't protect read access to rdev->fence_drv.last_seq
262 * if we experiencing a lockup the value doesn't change
263 */
264 if (seq == rdev->fence_drv.last_seq && radeon_gpu_is_lockup(rdev)) {
265 /* good news we believe it's a lockup */
266 WARN(1, "GPU lockup (waiting for 0x%08X last fence id 0x%08X)\n",
267 fence->seq, seq);
268 /* FIXME: what should we do ? marking everyone
269 * as signaled for now
270 */
271 rdev->gpu_lockup = true;
272 r = radeon_gpu_reset(rdev);
273 if (r)
274 return r;
275 radeon_fence_write(rdev, fence->seq);
276 rdev->gpu_lockup = false;
277 }
278 timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
279 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
280 rdev->fence_drv.last_timeout = RADEON_FENCE_JIFFIES_TIMEOUT;
281 rdev->fence_drv.last_jiffies = jiffies;
282 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
283 goto retry;
284 }
285 return 0;
286}
287
288int radeon_fence_wait_next(struct radeon_device *rdev)
289{
290 unsigned long irq_flags;
291 struct radeon_fence *fence;
292 int r;
293
294 if (rdev->gpu_lockup) {
295 return 0;
296 }
297 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
298 if (list_empty(&rdev->fence_drv.emited)) {
299 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
300 return 0;
301 }
302 fence = list_entry(rdev->fence_drv.emited.next,
303 struct radeon_fence, list);
304 radeon_fence_ref(fence);
305 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
306 r = radeon_fence_wait(fence, false);
307 radeon_fence_unref(&fence);
308 return r;
309}
310
311int radeon_fence_wait_last(struct radeon_device *rdev)
312{
313 unsigned long irq_flags;
314 struct radeon_fence *fence;
315 int r;
316
317 if (rdev->gpu_lockup) {
318 return 0;
319 }
320 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
321 if (list_empty(&rdev->fence_drv.emited)) {
322 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
323 return 0;
324 }
325 fence = list_entry(rdev->fence_drv.emited.prev,
326 struct radeon_fence, list);
327 radeon_fence_ref(fence);
328 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
329 r = radeon_fence_wait(fence, false);
330 radeon_fence_unref(&fence);
331 return r;
332}
333
334struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
335{
336 kref_get(&fence->kref);
337 return fence;
338}
339
340void radeon_fence_unref(struct radeon_fence **fence)
341{
342 struct radeon_fence *tmp = *fence;
343
344 *fence = NULL;
345 if (tmp) {
346 kref_put(&tmp->kref, radeon_fence_destroy);
347 }
348}
349
350void radeon_fence_process(struct radeon_device *rdev)
351{
352 unsigned long irq_flags;
353 bool wake;
354
355 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
356 wake = radeon_fence_poll_locked(rdev);
357 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
358 if (wake) {
359 wake_up_all(&rdev->fence_drv.queue);
360 }
361}
362
363int radeon_fence_driver_init(struct radeon_device *rdev)
364{
365 unsigned long irq_flags;
366 int r;
367
368 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
369 r = radeon_scratch_get(rdev, &rdev->fence_drv.scratch_reg);
370 if (r) {
371 dev_err(rdev->dev, "fence failed to get scratch register\n");
372 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
373 return r;
374 }
375 radeon_fence_write(rdev, 0);
376 atomic_set(&rdev->fence_drv.seq, 0);
377 INIT_LIST_HEAD(&rdev->fence_drv.created);
378 INIT_LIST_HEAD(&rdev->fence_drv.emited);
379 INIT_LIST_HEAD(&rdev->fence_drv.signaled);
380 init_waitqueue_head(&rdev->fence_drv.queue);
381 rdev->fence_drv.initialized = true;
382 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
383 if (radeon_debugfs_fence_init(rdev)) {
384 dev_err(rdev->dev, "fence debugfs file creation failed\n");
385 }
386 return 0;
387}
388
389void radeon_fence_driver_fini(struct radeon_device *rdev)
390{
391 unsigned long irq_flags;
392
393 if (!rdev->fence_drv.initialized)
394 return;
395 wake_up_all(&rdev->fence_drv.queue);
396 write_lock_irqsave(&rdev->fence_drv.lock, irq_flags);
397 radeon_scratch_free(rdev, rdev->fence_drv.scratch_reg);
398 write_unlock_irqrestore(&rdev->fence_drv.lock, irq_flags);
399 rdev->fence_drv.initialized = false;
400}
401
402
403/*
404 * Fence debugfs
405 */
406#if defined(CONFIG_DEBUG_FS)
407static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
408{
409 struct drm_info_node *node = (struct drm_info_node *)m->private;
410 struct drm_device *dev = node->minor->dev;
411 struct radeon_device *rdev = dev->dev_private;
412 struct radeon_fence *fence;
413
414 seq_printf(m, "Last signaled fence 0x%08X\n",
415 radeon_fence_read(rdev));
416 if (!list_empty(&rdev->fence_drv.emited)) {
417 fence = list_entry(rdev->fence_drv.emited.prev,
418 struct radeon_fence, list);
419 seq_printf(m, "Last emited fence %p with 0x%08X\n",
420 fence, fence->seq);
421 }
422 return 0;
423}
424
425static struct drm_info_list radeon_debugfs_fence_list[] = {
426 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
427};
428#endif
429
430int radeon_debugfs_fence_init(struct radeon_device *rdev)
431{
432#if defined(CONFIG_DEBUG_FS)
433 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
434#else
435 return 0;
436#endif
437}
1/*
2 * Copyright 2009 Jerome Glisse.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the
7 * "Software"), to deal in the Software without restriction, including
8 * without limitation the rights to use, copy, modify, merge, publish,
9 * distribute, sub license, and/or sell copies of the Software, and to
10 * permit persons to whom the Software is furnished to do so, subject to
11 * the following conditions:
12 *
13 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
16 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
17 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
18 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
19 * USE OR OTHER DEALINGS IN THE SOFTWARE.
20 *
21 * The above copyright notice and this permission notice (including the
22 * next paragraph) shall be included in all copies or substantial portions
23 * of the Software.
24 *
25 */
26/*
27 * Authors:
28 * Jerome Glisse <glisse@freedesktop.org>
29 * Dave Airlie
30 */
31#include <linux/seq_file.h>
32#include <linux/atomic.h>
33#include <linux/wait.h>
34#include <linux/list.h>
35#include <linux/kref.h>
36#include <linux/slab.h>
37#include "drmP.h"
38#include "drm.h"
39#include "radeon_reg.h"
40#include "radeon.h"
41#include "radeon_trace.h"
42
43static void radeon_fence_write(struct radeon_device *rdev, u32 seq, int ring)
44{
45 if (rdev->wb.enabled) {
46 *rdev->fence_drv[ring].cpu_addr = cpu_to_le32(seq);
47 } else {
48 WREG32(rdev->fence_drv[ring].scratch_reg, seq);
49 }
50}
51
52static u32 radeon_fence_read(struct radeon_device *rdev, int ring)
53{
54 u32 seq = 0;
55
56 if (rdev->wb.enabled) {
57 seq = le32_to_cpu(*rdev->fence_drv[ring].cpu_addr);
58 } else {
59 seq = RREG32(rdev->fence_drv[ring].scratch_reg);
60 }
61 return seq;
62}
63
64int radeon_fence_emit(struct radeon_device *rdev, struct radeon_fence *fence)
65{
66 /* we are protected by the ring emission mutex */
67 if (fence->seq && fence->seq < RADEON_FENCE_NOTEMITED_SEQ) {
68 return 0;
69 }
70 fence->seq = ++rdev->fence_drv[fence->ring].seq;
71 radeon_fence_ring_emit(rdev, fence->ring, fence);
72 trace_radeon_fence_emit(rdev->ddev, fence->seq);
73 return 0;
74}
75
76void radeon_fence_process(struct radeon_device *rdev, int ring)
77{
78 uint64_t seq, last_seq, last_emitted;
79 unsigned count_loop = 0;
80 bool wake = false;
81
82 /* Note there is a scenario here for an infinite loop but it's
83 * very unlikely to happen. For it to happen, the current polling
84 * process need to be interrupted by another process and another
85 * process needs to update the last_seq btw the atomic read and
86 * xchg of the current process.
87 *
88 * More over for this to go in infinite loop there need to be
89 * continuously new fence signaled ie radeon_fence_read needs
90 * to return a different value each time for both the currently
91 * polling process and the other process that xchg the last_seq
92 * btw atomic read and xchg of the current process. And the
93 * value the other process set as last seq must be higher than
94 * the seq value we just read. Which means that current process
95 * need to be interrupted after radeon_fence_read and before
96 * atomic xchg.
97 *
98 * To be even more safe we count the number of time we loop and
99 * we bail after 10 loop just accepting the fact that we might
100 * have temporarly set the last_seq not to the true real last
101 * seq but to an older one.
102 */
103 last_seq = atomic64_read(&rdev->fence_drv[ring].last_seq);
104 do {
105 last_emitted = rdev->fence_drv[ring].seq;
106 seq = radeon_fence_read(rdev, ring);
107 seq |= last_seq & 0xffffffff00000000LL;
108 if (seq < last_seq) {
109 seq &= 0xffffffff;
110 seq |= last_emitted & 0xffffffff00000000LL;
111 }
112
113 if (seq <= last_seq || seq > last_emitted) {
114 break;
115 }
116 /* If we loop over we don't want to return without
117 * checking if a fence is signaled as it means that the
118 * seq we just read is different from the previous on.
119 */
120 wake = true;
121 last_seq = seq;
122 if ((count_loop++) > 10) {
123 /* We looped over too many time leave with the
124 * fact that we might have set an older fence
125 * seq then the current real last seq as signaled
126 * by the hw.
127 */
128 break;
129 }
130 } while (atomic64_xchg(&rdev->fence_drv[ring].last_seq, seq) > seq);
131
132 if (wake) {
133 rdev->fence_drv[ring].last_activity = jiffies;
134 wake_up_all(&rdev->fence_queue);
135 }
136}
137
138static void radeon_fence_destroy(struct kref *kref)
139{
140 struct radeon_fence *fence;
141
142 fence = container_of(kref, struct radeon_fence, kref);
143 fence->seq = RADEON_FENCE_NOTEMITED_SEQ;
144 kfree(fence);
145}
146
147int radeon_fence_create(struct radeon_device *rdev,
148 struct radeon_fence **fence,
149 int ring)
150{
151 *fence = kmalloc(sizeof(struct radeon_fence), GFP_KERNEL);
152 if ((*fence) == NULL) {
153 return -ENOMEM;
154 }
155 kref_init(&((*fence)->kref));
156 (*fence)->rdev = rdev;
157 (*fence)->seq = RADEON_FENCE_NOTEMITED_SEQ;
158 (*fence)->ring = ring;
159 return 0;
160}
161
162static bool radeon_fence_seq_signaled(struct radeon_device *rdev,
163 u64 seq, unsigned ring)
164{
165 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {
166 return true;
167 }
168 /* poll new last sequence at least once */
169 radeon_fence_process(rdev, ring);
170 if (atomic64_read(&rdev->fence_drv[ring].last_seq) >= seq) {
171 return true;
172 }
173 return false;
174}
175
176bool radeon_fence_signaled(struct radeon_fence *fence)
177{
178 if (!fence) {
179 return true;
180 }
181 if (fence->seq == RADEON_FENCE_NOTEMITED_SEQ) {
182 WARN(1, "Querying an unemitted fence : %p !\n", fence);
183 return true;
184 }
185 if (fence->seq == RADEON_FENCE_SIGNALED_SEQ) {
186 return true;
187 }
188 if (radeon_fence_seq_signaled(fence->rdev, fence->seq, fence->ring)) {
189 fence->seq = RADEON_FENCE_SIGNALED_SEQ;
190 return true;
191 }
192 return false;
193}
194
195static int radeon_fence_wait_seq(struct radeon_device *rdev, u64 target_seq,
196 unsigned ring, bool intr, bool lock_ring)
197{
198 unsigned long timeout, last_activity;
199 uint64_t seq;
200 unsigned i;
201 bool signaled;
202 int r;
203
204 while (target_seq > atomic64_read(&rdev->fence_drv[ring].last_seq)) {
205 if (!rdev->ring[ring].ready) {
206 return -EBUSY;
207 }
208
209 timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
210 if (time_after(rdev->fence_drv[ring].last_activity, timeout)) {
211 /* the normal case, timeout is somewhere before last_activity */
212 timeout = rdev->fence_drv[ring].last_activity - timeout;
213 } else {
214 /* either jiffies wrapped around, or no fence was signaled in the last 500ms
215 * anyway we will just wait for the minimum amount and then check for a lockup
216 */
217 timeout = 1;
218 }
219 seq = atomic64_read(&rdev->fence_drv[ring].last_seq);
220 /* Save current last activity valuee, used to check for GPU lockups */
221 last_activity = rdev->fence_drv[ring].last_activity;
222
223 trace_radeon_fence_wait_begin(rdev->ddev, seq);
224 radeon_irq_kms_sw_irq_get(rdev, ring);
225 if (intr) {
226 r = wait_event_interruptible_timeout(rdev->fence_queue,
227 (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)),
228 timeout);
229 } else {
230 r = wait_event_timeout(rdev->fence_queue,
231 (signaled = radeon_fence_seq_signaled(rdev, target_seq, ring)),
232 timeout);
233 }
234 radeon_irq_kms_sw_irq_put(rdev, ring);
235 if (unlikely(r < 0)) {
236 return r;
237 }
238 trace_radeon_fence_wait_end(rdev->ddev, seq);
239
240 if (unlikely(!signaled)) {
241 /* we were interrupted for some reason and fence
242 * isn't signaled yet, resume waiting */
243 if (r) {
244 continue;
245 }
246
247 /* check if sequence value has changed since last_activity */
248 if (seq != atomic64_read(&rdev->fence_drv[ring].last_seq)) {
249 continue;
250 }
251
252 if (lock_ring) {
253 mutex_lock(&rdev->ring_lock);
254 }
255
256 /* test if somebody else has already decided that this is a lockup */
257 if (last_activity != rdev->fence_drv[ring].last_activity) {
258 if (lock_ring) {
259 mutex_unlock(&rdev->ring_lock);
260 }
261 continue;
262 }
263
264 if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
265 /* good news we believe it's a lockup */
266 dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx last fence id 0x%016llx)\n",
267 target_seq, seq);
268
269 /* change last activity so nobody else think there is a lockup */
270 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
271 rdev->fence_drv[i].last_activity = jiffies;
272 }
273
274 /* mark the ring as not ready any more */
275 rdev->ring[ring].ready = false;
276 if (lock_ring) {
277 mutex_unlock(&rdev->ring_lock);
278 }
279 return -EDEADLK;
280 }
281
282 if (lock_ring) {
283 mutex_unlock(&rdev->ring_lock);
284 }
285 }
286 }
287 return 0;
288}
289
290int radeon_fence_wait(struct radeon_fence *fence, bool intr)
291{
292 int r;
293
294 if (fence == NULL) {
295 WARN(1, "Querying an invalid fence : %p !\n", fence);
296 return -EINVAL;
297 }
298
299 r = radeon_fence_wait_seq(fence->rdev, fence->seq,
300 fence->ring, intr, true);
301 if (r) {
302 return r;
303 }
304 fence->seq = RADEON_FENCE_SIGNALED_SEQ;
305 return 0;
306}
307
308bool radeon_fence_any_seq_signaled(struct radeon_device *rdev, u64 *seq)
309{
310 unsigned i;
311
312 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
313 if (seq[i] && radeon_fence_seq_signaled(rdev, seq[i], i)) {
314 return true;
315 }
316 }
317 return false;
318}
319
320static int radeon_fence_wait_any_seq(struct radeon_device *rdev,
321 u64 *target_seq, bool intr)
322{
323 unsigned long timeout, last_activity, tmp;
324 unsigned i, ring = RADEON_NUM_RINGS;
325 bool signaled;
326 int r;
327
328 for (i = 0, last_activity = 0; i < RADEON_NUM_RINGS; ++i) {
329 if (!target_seq[i]) {
330 continue;
331 }
332
333 /* use the most recent one as indicator */
334 if (time_after(rdev->fence_drv[i].last_activity, last_activity)) {
335 last_activity = rdev->fence_drv[i].last_activity;
336 }
337
338 /* For lockup detection just pick the lowest ring we are
339 * actively waiting for
340 */
341 if (i < ring) {
342 ring = i;
343 }
344 }
345
346 /* nothing to wait for ? */
347 if (ring == RADEON_NUM_RINGS) {
348 return 0;
349 }
350
351 while (!radeon_fence_any_seq_signaled(rdev, target_seq)) {
352 timeout = jiffies - RADEON_FENCE_JIFFIES_TIMEOUT;
353 if (time_after(last_activity, timeout)) {
354 /* the normal case, timeout is somewhere before last_activity */
355 timeout = last_activity - timeout;
356 } else {
357 /* either jiffies wrapped around, or no fence was signaled in the last 500ms
358 * anyway we will just wait for the minimum amount and then check for a lockup
359 */
360 timeout = 1;
361 }
362
363 trace_radeon_fence_wait_begin(rdev->ddev, target_seq[ring]);
364 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
365 if (target_seq[i]) {
366 radeon_irq_kms_sw_irq_get(rdev, i);
367 }
368 }
369 if (intr) {
370 r = wait_event_interruptible_timeout(rdev->fence_queue,
371 (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)),
372 timeout);
373 } else {
374 r = wait_event_timeout(rdev->fence_queue,
375 (signaled = radeon_fence_any_seq_signaled(rdev, target_seq)),
376 timeout);
377 }
378 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
379 if (target_seq[i]) {
380 radeon_irq_kms_sw_irq_put(rdev, i);
381 }
382 }
383 if (unlikely(r < 0)) {
384 return r;
385 }
386 trace_radeon_fence_wait_end(rdev->ddev, target_seq[ring]);
387
388 if (unlikely(!signaled)) {
389 /* we were interrupted for some reason and fence
390 * isn't signaled yet, resume waiting */
391 if (r) {
392 continue;
393 }
394
395 mutex_lock(&rdev->ring_lock);
396 for (i = 0, tmp = 0; i < RADEON_NUM_RINGS; ++i) {
397 if (time_after(rdev->fence_drv[i].last_activity, tmp)) {
398 tmp = rdev->fence_drv[i].last_activity;
399 }
400 }
401 /* test if somebody else has already decided that this is a lockup */
402 if (last_activity != tmp) {
403 last_activity = tmp;
404 mutex_unlock(&rdev->ring_lock);
405 continue;
406 }
407
408 if (radeon_ring_is_lockup(rdev, ring, &rdev->ring[ring])) {
409 /* good news we believe it's a lockup */
410 dev_warn(rdev->dev, "GPU lockup (waiting for 0x%016llx)\n",
411 target_seq[ring]);
412
413 /* change last activity so nobody else think there is a lockup */
414 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
415 rdev->fence_drv[i].last_activity = jiffies;
416 }
417
418 /* mark the ring as not ready any more */
419 rdev->ring[ring].ready = false;
420 mutex_unlock(&rdev->ring_lock);
421 return -EDEADLK;
422 }
423 mutex_unlock(&rdev->ring_lock);
424 }
425 }
426 return 0;
427}
428
429int radeon_fence_wait_any(struct radeon_device *rdev,
430 struct radeon_fence **fences,
431 bool intr)
432{
433 uint64_t seq[RADEON_NUM_RINGS];
434 unsigned i;
435 int r;
436
437 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
438 seq[i] = 0;
439
440 if (!fences[i]) {
441 continue;
442 }
443
444 if (fences[i]->seq == RADEON_FENCE_SIGNALED_SEQ) {
445 /* something was allready signaled */
446 return 0;
447 }
448
449 if (fences[i]->seq < RADEON_FENCE_NOTEMITED_SEQ) {
450 seq[i] = fences[i]->seq;
451 }
452 }
453
454 r = radeon_fence_wait_any_seq(rdev, seq, intr);
455 if (r) {
456 return r;
457 }
458 return 0;
459}
460
461int radeon_fence_wait_next_locked(struct radeon_device *rdev, int ring)
462{
463 uint64_t seq;
464
465 /* We are not protected by ring lock when reading current seq but
466 * it's ok as worst case is we return to early while we could have
467 * wait.
468 */
469 seq = atomic64_read(&rdev->fence_drv[ring].last_seq) + 1ULL;
470 if (seq >= rdev->fence_drv[ring].seq) {
471 /* nothing to wait for, last_seq is
472 already the last emited fence */
473 return -ENOENT;
474 }
475 return radeon_fence_wait_seq(rdev, seq, ring, false, false);
476}
477
478int radeon_fence_wait_empty_locked(struct radeon_device *rdev, int ring)
479{
480 /* We are not protected by ring lock when reading current seq
481 * but it's ok as wait empty is call from place where no more
482 * activity can be scheduled so there won't be concurrent access
483 * to seq value.
484 */
485 return radeon_fence_wait_seq(rdev, rdev->fence_drv[ring].seq,
486 ring, false, false);
487}
488
489struct radeon_fence *radeon_fence_ref(struct radeon_fence *fence)
490{
491 kref_get(&fence->kref);
492 return fence;
493}
494
495void radeon_fence_unref(struct radeon_fence **fence)
496{
497 struct radeon_fence *tmp = *fence;
498
499 *fence = NULL;
500 if (tmp) {
501 kref_put(&tmp->kref, radeon_fence_destroy);
502 }
503}
504
505unsigned radeon_fence_count_emitted(struct radeon_device *rdev, int ring)
506{
507 uint64_t emitted;
508
509 /* We are not protected by ring lock when reading the last sequence
510 * but it's ok to report slightly wrong fence count here.
511 */
512 radeon_fence_process(rdev, ring);
513 emitted = rdev->fence_drv[ring].seq - atomic64_read(&rdev->fence_drv[ring].last_seq);
514 /* to avoid 32bits warp around */
515 if (emitted > 0x10000000) {
516 emitted = 0x10000000;
517 }
518 return (unsigned)emitted;
519}
520
521int radeon_fence_driver_start_ring(struct radeon_device *rdev, int ring)
522{
523 uint64_t index;
524 int r;
525
526 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
527 if (rdev->wb.use_event) {
528 rdev->fence_drv[ring].scratch_reg = 0;
529 index = R600_WB_EVENT_OFFSET + ring * 4;
530 } else {
531 r = radeon_scratch_get(rdev, &rdev->fence_drv[ring].scratch_reg);
532 if (r) {
533 dev_err(rdev->dev, "fence failed to get scratch register\n");
534 return r;
535 }
536 index = RADEON_WB_SCRATCH_OFFSET +
537 rdev->fence_drv[ring].scratch_reg -
538 rdev->scratch.reg_base;
539 }
540 rdev->fence_drv[ring].cpu_addr = &rdev->wb.wb[index/4];
541 rdev->fence_drv[ring].gpu_addr = rdev->wb.gpu_addr + index;
542 radeon_fence_write(rdev, rdev->fence_drv[ring].seq, ring);
543 rdev->fence_drv[ring].initialized = true;
544 dev_info(rdev->dev, "fence driver on ring %d use gpu addr 0x%016llx and cpu addr 0x%p\n",
545 ring, rdev->fence_drv[ring].gpu_addr, rdev->fence_drv[ring].cpu_addr);
546 return 0;
547}
548
549static void radeon_fence_driver_init_ring(struct radeon_device *rdev, int ring)
550{
551 rdev->fence_drv[ring].scratch_reg = -1;
552 rdev->fence_drv[ring].cpu_addr = NULL;
553 rdev->fence_drv[ring].gpu_addr = 0;
554 rdev->fence_drv[ring].seq = 0;
555 atomic64_set(&rdev->fence_drv[ring].last_seq, 0);
556 rdev->fence_drv[ring].last_activity = jiffies;
557 rdev->fence_drv[ring].initialized = false;
558}
559
560int radeon_fence_driver_init(struct radeon_device *rdev)
561{
562 int ring;
563
564 init_waitqueue_head(&rdev->fence_queue);
565 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
566 radeon_fence_driver_init_ring(rdev, ring);
567 }
568 if (radeon_debugfs_fence_init(rdev)) {
569 dev_err(rdev->dev, "fence debugfs file creation failed\n");
570 }
571 return 0;
572}
573
574void radeon_fence_driver_fini(struct radeon_device *rdev)
575{
576 int ring;
577
578 mutex_lock(&rdev->ring_lock);
579 for (ring = 0; ring < RADEON_NUM_RINGS; ring++) {
580 if (!rdev->fence_drv[ring].initialized)
581 continue;
582 radeon_fence_wait_empty_locked(rdev, ring);
583 wake_up_all(&rdev->fence_queue);
584 radeon_scratch_free(rdev, rdev->fence_drv[ring].scratch_reg);
585 rdev->fence_drv[ring].initialized = false;
586 }
587 mutex_unlock(&rdev->ring_lock);
588}
589
590
591/*
592 * Fence debugfs
593 */
594#if defined(CONFIG_DEBUG_FS)
595static int radeon_debugfs_fence_info(struct seq_file *m, void *data)
596{
597 struct drm_info_node *node = (struct drm_info_node *)m->private;
598 struct drm_device *dev = node->minor->dev;
599 struct radeon_device *rdev = dev->dev_private;
600 int i;
601
602 for (i = 0; i < RADEON_NUM_RINGS; ++i) {
603 if (!rdev->fence_drv[i].initialized)
604 continue;
605
606 seq_printf(m, "--- ring %d ---\n", i);
607 seq_printf(m, "Last signaled fence 0x%016llx\n",
608 (unsigned long long)atomic64_read(&rdev->fence_drv[i].last_seq));
609 seq_printf(m, "Last emitted 0x%016llx\n",
610 rdev->fence_drv[i].seq);
611 }
612 return 0;
613}
614
615static struct drm_info_list radeon_debugfs_fence_list[] = {
616 {"radeon_fence_info", &radeon_debugfs_fence_info, 0, NULL},
617};
618#endif
619
620int radeon_debugfs_fence_init(struct radeon_device *rdev)
621{
622#if defined(CONFIG_DEBUG_FS)
623 return radeon_debugfs_add_files(rdev, radeon_debugfs_fence_list, 1);
624#else
625 return 0;
626#endif
627}