Loading...
1/**************************************************************************
2 *
3 * Copyright (C) 2010 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28
29#include "vmwgfx_drv.h"
30
31struct vmw_fence {
32 struct list_head head;
33 uint32_t sequence;
34 struct timespec submitted;
35};
36
37void vmw_fence_queue_init(struct vmw_fence_queue *queue)
38{
39 INIT_LIST_HEAD(&queue->head);
40 queue->lag = ns_to_timespec(0);
41 getrawmonotonic(&queue->lag_time);
42 spin_lock_init(&queue->lock);
43}
44
45void vmw_fence_queue_takedown(struct vmw_fence_queue *queue)
46{
47 struct vmw_fence *fence, *next;
48
49 spin_lock(&queue->lock);
50 list_for_each_entry_safe(fence, next, &queue->head, head) {
51 kfree(fence);
52 }
53 spin_unlock(&queue->lock);
54}
55
56int vmw_fence_push(struct vmw_fence_queue *queue,
57 uint32_t sequence)
58{
59 struct vmw_fence *fence = kmalloc(sizeof(*fence), GFP_KERNEL);
60
61 if (unlikely(!fence))
62 return -ENOMEM;
63
64 fence->sequence = sequence;
65 getrawmonotonic(&fence->submitted);
66 spin_lock(&queue->lock);
67 list_add_tail(&fence->head, &queue->head);
68 spin_unlock(&queue->lock);
69
70 return 0;
71}
72
73int vmw_fence_pull(struct vmw_fence_queue *queue,
74 uint32_t signaled_sequence)
75{
76 struct vmw_fence *fence, *next;
77 struct timespec now;
78 bool updated = false;
79
80 spin_lock(&queue->lock);
81 getrawmonotonic(&now);
82
83 if (list_empty(&queue->head)) {
84 queue->lag = ns_to_timespec(0);
85 queue->lag_time = now;
86 updated = true;
87 goto out_unlock;
88 }
89
90 list_for_each_entry_safe(fence, next, &queue->head, head) {
91 if (signaled_sequence - fence->sequence > (1 << 30))
92 continue;
93
94 queue->lag = timespec_sub(now, fence->submitted);
95 queue->lag_time = now;
96 updated = true;
97 list_del(&fence->head);
98 kfree(fence);
99 }
100
101out_unlock:
102 spin_unlock(&queue->lock);
103
104 return (updated) ? 0 : -EBUSY;
105}
106
107static struct timespec vmw_timespec_add(struct timespec t1,
108 struct timespec t2)
109{
110 t1.tv_sec += t2.tv_sec;
111 t1.tv_nsec += t2.tv_nsec;
112 if (t1.tv_nsec >= 1000000000L) {
113 t1.tv_sec += 1;
114 t1.tv_nsec -= 1000000000L;
115 }
116
117 return t1;
118}
119
120static struct timespec vmw_fifo_lag(struct vmw_fence_queue *queue)
121{
122 struct timespec now;
123
124 spin_lock(&queue->lock);
125 getrawmonotonic(&now);
126 queue->lag = vmw_timespec_add(queue->lag,
127 timespec_sub(now, queue->lag_time));
128 queue->lag_time = now;
129 spin_unlock(&queue->lock);
130 return queue->lag;
131}
132
133
134static bool vmw_lag_lt(struct vmw_fence_queue *queue,
135 uint32_t us)
136{
137 struct timespec lag, cond;
138
139 cond = ns_to_timespec((s64) us * 1000);
140 lag = vmw_fifo_lag(queue);
141 return (timespec_compare(&lag, &cond) < 1);
142}
143
144int vmw_wait_lag(struct vmw_private *dev_priv,
145 struct vmw_fence_queue *queue, uint32_t us)
146{
147 struct vmw_fence *fence;
148 uint32_t sequence;
149 int ret;
150
151 while (!vmw_lag_lt(queue, us)) {
152 spin_lock(&queue->lock);
153 if (list_empty(&queue->head))
154 sequence = atomic_read(&dev_priv->fence_seq);
155 else {
156 fence = list_first_entry(&queue->head,
157 struct vmw_fence, head);
158 sequence = fence->sequence;
159 }
160 spin_unlock(&queue->lock);
161
162 ret = vmw_wait_fence(dev_priv, false, sequence, true,
163 3*HZ);
164
165 if (unlikely(ret != 0))
166 return ret;
167
168 (void) vmw_fence_pull(queue, sequence);
169 }
170 return 0;
171}
172
173
1/**************************************************************************
2 *
3 * Copyright © 2011 VMware, Inc., Palo Alto, CA., USA
4 * All Rights Reserved.
5 *
6 * Permission is hereby granted, free of charge, to any person obtaining a
7 * copy of this software and associated documentation files (the
8 * "Software"), to deal in the Software without restriction, including
9 * without limitation the rights to use, copy, modify, merge, publish,
10 * distribute, sub license, and/or sell copies of the Software, and to
11 * permit persons to whom the Software is furnished to do so, subject to
12 * the following conditions:
13 *
14 * The above copyright notice and this permission notice (including the
15 * next paragraph) shall be included in all copies or substantial portions
16 * of the Software.
17 *
18 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
21 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
22 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
23 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
24 * USE OR OTHER DEALINGS IN THE SOFTWARE.
25 *
26 **************************************************************************/
27
28#include <drm/drmP.h>
29#include "vmwgfx_drv.h"
30
31#define VMW_FENCE_WRAP (1 << 31)
32
33struct vmw_fence_manager {
34 int num_fence_objects;
35 struct vmw_private *dev_priv;
36 spinlock_t lock;
37 struct list_head fence_list;
38 struct work_struct work;
39 u32 user_fence_size;
40 u32 fence_size;
41 u32 event_fence_action_size;
42 bool fifo_down;
43 struct list_head cleanup_list;
44 uint32_t pending_actions[VMW_ACTION_MAX];
45 struct mutex goal_irq_mutex;
46 bool goal_irq_on; /* Protected by @goal_irq_mutex */
47 bool seqno_valid; /* Protected by @lock, and may not be set to true
48 without the @goal_irq_mutex held. */
49};
50
51struct vmw_user_fence {
52 struct ttm_base_object base;
53 struct vmw_fence_obj fence;
54};
55
56/**
57 * struct vmw_event_fence_action - fence action that delivers a drm event.
58 *
59 * @e: A struct drm_pending_event that controls the event delivery.
60 * @action: A struct vmw_fence_action to hook up to a fence.
61 * @fence: A referenced pointer to the fence to keep it alive while @action
62 * hangs on it.
63 * @dev: Pointer to a struct drm_device so we can access the event stuff.
64 * @kref: Both @e and @action has destructors, so we need to refcount.
65 * @size: Size accounted for this object.
66 * @tv_sec: If non-null, the variable pointed to will be assigned
67 * current time tv_sec val when the fence signals.
68 * @tv_usec: Must be set if @tv_sec is set, and the variable pointed to will
69 * be assigned the current time tv_usec val when the fence signals.
70 */
71struct vmw_event_fence_action {
72 struct vmw_fence_action action;
73 struct list_head fpriv_head;
74
75 struct drm_pending_event *event;
76 struct vmw_fence_obj *fence;
77 struct drm_device *dev;
78
79 uint32_t *tv_sec;
80 uint32_t *tv_usec;
81};
82
83/**
84 * Note on fencing subsystem usage of irqs:
85 * Typically the vmw_fences_update function is called
86 *
87 * a) When a new fence seqno has been submitted by the fifo code.
88 * b) On-demand when we have waiters. Sleeping waiters will switch on the
89 * ANY_FENCE irq and call vmw_fences_update function each time an ANY_FENCE
90 * irq is received. When the last fence waiter is gone, that IRQ is masked
91 * away.
92 *
93 * In situations where there are no waiters and we don't submit any new fences,
94 * fence objects may not be signaled. This is perfectly OK, since there are
95 * no consumers of the signaled data, but that is NOT ok when there are fence
96 * actions attached to a fence. The fencing subsystem then makes use of the
97 * FENCE_GOAL irq and sets the fence goal seqno to that of the next fence
98 * which has an action attached, and each time vmw_fences_update is called,
99 * the subsystem makes sure the fence goal seqno is updated.
100 *
101 * The fence goal seqno irq is on as long as there are unsignaled fence
102 * objects with actions attached to them.
103 */
104
105static void vmw_fence_obj_destroy_locked(struct kref *kref)
106{
107 struct vmw_fence_obj *fence =
108 container_of(kref, struct vmw_fence_obj, kref);
109
110 struct vmw_fence_manager *fman = fence->fman;
111 unsigned int num_fences;
112
113 list_del_init(&fence->head);
114 num_fences = --fman->num_fence_objects;
115 spin_unlock_irq(&fman->lock);
116 if (fence->destroy)
117 fence->destroy(fence);
118 else
119 kfree(fence);
120
121 spin_lock_irq(&fman->lock);
122}
123
124
125/**
126 * Execute signal actions on fences recently signaled.
127 * This is done from a workqueue so we don't have to execute
128 * signal actions from atomic context.
129 */
130
131static void vmw_fence_work_func(struct work_struct *work)
132{
133 struct vmw_fence_manager *fman =
134 container_of(work, struct vmw_fence_manager, work);
135 struct list_head list;
136 struct vmw_fence_action *action, *next_action;
137 bool seqno_valid;
138
139 do {
140 INIT_LIST_HEAD(&list);
141 mutex_lock(&fman->goal_irq_mutex);
142
143 spin_lock_irq(&fman->lock);
144 list_splice_init(&fman->cleanup_list, &list);
145 seqno_valid = fman->seqno_valid;
146 spin_unlock_irq(&fman->lock);
147
148 if (!seqno_valid && fman->goal_irq_on) {
149 fman->goal_irq_on = false;
150 vmw_goal_waiter_remove(fman->dev_priv);
151 }
152 mutex_unlock(&fman->goal_irq_mutex);
153
154 if (list_empty(&list))
155 return;
156
157 /*
158 * At this point, only we should be able to manipulate the
159 * list heads of the actions we have on the private list.
160 * hence fman::lock not held.
161 */
162
163 list_for_each_entry_safe(action, next_action, &list, head) {
164 list_del_init(&action->head);
165 if (action->cleanup)
166 action->cleanup(action);
167 }
168 } while (1);
169}
170
171struct vmw_fence_manager *vmw_fence_manager_init(struct vmw_private *dev_priv)
172{
173 struct vmw_fence_manager *fman = kzalloc(sizeof(*fman), GFP_KERNEL);
174
175 if (unlikely(fman == NULL))
176 return NULL;
177
178 fman->dev_priv = dev_priv;
179 spin_lock_init(&fman->lock);
180 INIT_LIST_HEAD(&fman->fence_list);
181 INIT_LIST_HEAD(&fman->cleanup_list);
182 INIT_WORK(&fman->work, &vmw_fence_work_func);
183 fman->fifo_down = true;
184 fman->user_fence_size = ttm_round_pot(sizeof(struct vmw_user_fence));
185 fman->fence_size = ttm_round_pot(sizeof(struct vmw_fence_obj));
186 fman->event_fence_action_size =
187 ttm_round_pot(sizeof(struct vmw_event_fence_action));
188 mutex_init(&fman->goal_irq_mutex);
189
190 return fman;
191}
192
193void vmw_fence_manager_takedown(struct vmw_fence_manager *fman)
194{
195 unsigned long irq_flags;
196 bool lists_empty;
197
198 (void) cancel_work_sync(&fman->work);
199
200 spin_lock_irqsave(&fman->lock, irq_flags);
201 lists_empty = list_empty(&fman->fence_list) &&
202 list_empty(&fman->cleanup_list);
203 spin_unlock_irqrestore(&fman->lock, irq_flags);
204
205 BUG_ON(!lists_empty);
206 kfree(fman);
207}
208
209static int vmw_fence_obj_init(struct vmw_fence_manager *fman,
210 struct vmw_fence_obj *fence,
211 u32 seqno,
212 uint32_t mask,
213 void (*destroy) (struct vmw_fence_obj *fence))
214{
215 unsigned long irq_flags;
216 unsigned int num_fences;
217 int ret = 0;
218
219 fence->seqno = seqno;
220 INIT_LIST_HEAD(&fence->seq_passed_actions);
221 fence->fman = fman;
222 fence->signaled = 0;
223 fence->signal_mask = mask;
224 kref_init(&fence->kref);
225 fence->destroy = destroy;
226 init_waitqueue_head(&fence->queue);
227
228 spin_lock_irqsave(&fman->lock, irq_flags);
229 if (unlikely(fman->fifo_down)) {
230 ret = -EBUSY;
231 goto out_unlock;
232 }
233 list_add_tail(&fence->head, &fman->fence_list);
234 num_fences = ++fman->num_fence_objects;
235
236out_unlock:
237 spin_unlock_irqrestore(&fman->lock, irq_flags);
238 return ret;
239
240}
241
242struct vmw_fence_obj *vmw_fence_obj_reference(struct vmw_fence_obj *fence)
243{
244 if (unlikely(fence == NULL))
245 return NULL;
246
247 kref_get(&fence->kref);
248 return fence;
249}
250
251/**
252 * vmw_fence_obj_unreference
253 *
254 * Note that this function may not be entered with disabled irqs since
255 * it may re-enable them in the destroy function.
256 *
257 */
258void vmw_fence_obj_unreference(struct vmw_fence_obj **fence_p)
259{
260 struct vmw_fence_obj *fence = *fence_p;
261 struct vmw_fence_manager *fman;
262
263 if (unlikely(fence == NULL))
264 return;
265
266 fman = fence->fman;
267 *fence_p = NULL;
268 spin_lock_irq(&fman->lock);
269 BUG_ON(atomic_read(&fence->kref.refcount) == 0);
270 kref_put(&fence->kref, vmw_fence_obj_destroy_locked);
271 spin_unlock_irq(&fman->lock);
272}
273
274static void vmw_fences_perform_actions(struct vmw_fence_manager *fman,
275 struct list_head *list)
276{
277 struct vmw_fence_action *action, *next_action;
278
279 list_for_each_entry_safe(action, next_action, list, head) {
280 list_del_init(&action->head);
281 fman->pending_actions[action->type]--;
282 if (action->seq_passed != NULL)
283 action->seq_passed(action);
284
285 /*
286 * Add the cleanup action to the cleanup list so that
287 * it will be performed by a worker task.
288 */
289
290 list_add_tail(&action->head, &fman->cleanup_list);
291 }
292}
293
294/**
295 * vmw_fence_goal_new_locked - Figure out a new device fence goal
296 * seqno if needed.
297 *
298 * @fman: Pointer to a fence manager.
299 * @passed_seqno: The seqno the device currently signals as passed.
300 *
301 * This function should be called with the fence manager lock held.
302 * It is typically called when we have a new passed_seqno, and
303 * we might need to update the fence goal. It checks to see whether
304 * the current fence goal has already passed, and, in that case,
305 * scans through all unsignaled fences to get the next fence object with an
306 * action attached, and sets the seqno of that fence as a new fence goal.
307 *
308 * returns true if the device goal seqno was updated. False otherwise.
309 */
310static bool vmw_fence_goal_new_locked(struct vmw_fence_manager *fman,
311 u32 passed_seqno)
312{
313 u32 goal_seqno;
314 __le32 __iomem *fifo_mem;
315 struct vmw_fence_obj *fence;
316
317 if (likely(!fman->seqno_valid))
318 return false;
319
320 fifo_mem = fman->dev_priv->mmio_virt;
321 goal_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE_GOAL);
322 if (likely(passed_seqno - goal_seqno >= VMW_FENCE_WRAP))
323 return false;
324
325 fman->seqno_valid = false;
326 list_for_each_entry(fence, &fman->fence_list, head) {
327 if (!list_empty(&fence->seq_passed_actions)) {
328 fman->seqno_valid = true;
329 iowrite32(fence->seqno,
330 fifo_mem + SVGA_FIFO_FENCE_GOAL);
331 break;
332 }
333 }
334
335 return true;
336}
337
338
339/**
340 * vmw_fence_goal_check_locked - Replace the device fence goal seqno if
341 * needed.
342 *
343 * @fence: Pointer to a struct vmw_fence_obj the seqno of which should be
344 * considered as a device fence goal.
345 *
346 * This function should be called with the fence manager lock held.
347 * It is typically called when an action has been attached to a fence to
348 * check whether the seqno of that fence should be used for a fence
349 * goal interrupt. This is typically needed if the current fence goal is
350 * invalid, or has a higher seqno than that of the current fence object.
351 *
352 * returns true if the device goal seqno was updated. False otherwise.
353 */
354static bool vmw_fence_goal_check_locked(struct vmw_fence_obj *fence)
355{
356 u32 goal_seqno;
357 __le32 __iomem *fifo_mem;
358
359 if (fence->signaled & DRM_VMW_FENCE_FLAG_EXEC)
360 return false;
361
362 fifo_mem = fence->fman->dev_priv->mmio_virt;
363 goal_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE_GOAL);
364 if (likely(fence->fman->seqno_valid &&
365 goal_seqno - fence->seqno < VMW_FENCE_WRAP))
366 return false;
367
368 iowrite32(fence->seqno, fifo_mem + SVGA_FIFO_FENCE_GOAL);
369 fence->fman->seqno_valid = true;
370
371 return true;
372}
373
374void vmw_fences_update(struct vmw_fence_manager *fman)
375{
376 unsigned long flags;
377 struct vmw_fence_obj *fence, *next_fence;
378 struct list_head action_list;
379 bool needs_rerun;
380 uint32_t seqno, new_seqno;
381 __le32 __iomem *fifo_mem = fman->dev_priv->mmio_virt;
382
383 seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
384rerun:
385 spin_lock_irqsave(&fman->lock, flags);
386 list_for_each_entry_safe(fence, next_fence, &fman->fence_list, head) {
387 if (seqno - fence->seqno < VMW_FENCE_WRAP) {
388 list_del_init(&fence->head);
389 fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC;
390 INIT_LIST_HEAD(&action_list);
391 list_splice_init(&fence->seq_passed_actions,
392 &action_list);
393 vmw_fences_perform_actions(fman, &action_list);
394 wake_up_all(&fence->queue);
395 } else
396 break;
397 }
398
399 needs_rerun = vmw_fence_goal_new_locked(fman, seqno);
400
401 if (!list_empty(&fman->cleanup_list))
402 (void) schedule_work(&fman->work);
403 spin_unlock_irqrestore(&fman->lock, flags);
404
405 /*
406 * Rerun if the fence goal seqno was updated, and the
407 * hardware might have raced with that update, so that
408 * we missed a fence_goal irq.
409 */
410
411 if (unlikely(needs_rerun)) {
412 new_seqno = ioread32(fifo_mem + SVGA_FIFO_FENCE);
413 if (new_seqno != seqno) {
414 seqno = new_seqno;
415 goto rerun;
416 }
417 }
418}
419
420bool vmw_fence_obj_signaled(struct vmw_fence_obj *fence,
421 uint32_t flags)
422{
423 struct vmw_fence_manager *fman = fence->fman;
424 unsigned long irq_flags;
425 uint32_t signaled;
426
427 spin_lock_irqsave(&fman->lock, irq_flags);
428 signaled = fence->signaled;
429 spin_unlock_irqrestore(&fman->lock, irq_flags);
430
431 flags &= fence->signal_mask;
432 if ((signaled & flags) == flags)
433 return 1;
434
435 if ((signaled & DRM_VMW_FENCE_FLAG_EXEC) == 0)
436 vmw_fences_update(fman);
437
438 spin_lock_irqsave(&fman->lock, irq_flags);
439 signaled = fence->signaled;
440 spin_unlock_irqrestore(&fman->lock, irq_flags);
441
442 return ((signaled & flags) == flags);
443}
444
445int vmw_fence_obj_wait(struct vmw_fence_obj *fence,
446 uint32_t flags, bool lazy,
447 bool interruptible, unsigned long timeout)
448{
449 struct vmw_private *dev_priv = fence->fman->dev_priv;
450 long ret;
451
452 if (likely(vmw_fence_obj_signaled(fence, flags)))
453 return 0;
454
455 vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC);
456 vmw_seqno_waiter_add(dev_priv);
457
458 if (interruptible)
459 ret = wait_event_interruptible_timeout
460 (fence->queue,
461 vmw_fence_obj_signaled(fence, flags),
462 timeout);
463 else
464 ret = wait_event_timeout
465 (fence->queue,
466 vmw_fence_obj_signaled(fence, flags),
467 timeout);
468
469 vmw_seqno_waiter_remove(dev_priv);
470
471 if (unlikely(ret == 0))
472 ret = -EBUSY;
473 else if (likely(ret > 0))
474 ret = 0;
475
476 return ret;
477}
478
479void vmw_fence_obj_flush(struct vmw_fence_obj *fence)
480{
481 struct vmw_private *dev_priv = fence->fman->dev_priv;
482
483 vmw_fifo_ping_host(dev_priv, SVGA_SYNC_GENERIC);
484}
485
486static void vmw_fence_destroy(struct vmw_fence_obj *fence)
487{
488 struct vmw_fence_manager *fman = fence->fman;
489
490 kfree(fence);
491 /*
492 * Free kernel space accounting.
493 */
494 ttm_mem_global_free(vmw_mem_glob(fman->dev_priv),
495 fman->fence_size);
496}
497
498int vmw_fence_create(struct vmw_fence_manager *fman,
499 uint32_t seqno,
500 uint32_t mask,
501 struct vmw_fence_obj **p_fence)
502{
503 struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv);
504 struct vmw_fence_obj *fence;
505 int ret;
506
507 ret = ttm_mem_global_alloc(mem_glob, fman->fence_size,
508 false, false);
509 if (unlikely(ret != 0))
510 return ret;
511
512 fence = kzalloc(sizeof(*fence), GFP_KERNEL);
513 if (unlikely(fence == NULL)) {
514 ret = -ENOMEM;
515 goto out_no_object;
516 }
517
518 ret = vmw_fence_obj_init(fman, fence, seqno, mask,
519 vmw_fence_destroy);
520 if (unlikely(ret != 0))
521 goto out_err_init;
522
523 *p_fence = fence;
524 return 0;
525
526out_err_init:
527 kfree(fence);
528out_no_object:
529 ttm_mem_global_free(mem_glob, fman->fence_size);
530 return ret;
531}
532
533
534static void vmw_user_fence_destroy(struct vmw_fence_obj *fence)
535{
536 struct vmw_user_fence *ufence =
537 container_of(fence, struct vmw_user_fence, fence);
538 struct vmw_fence_manager *fman = fence->fman;
539
540 ttm_base_object_kfree(ufence, base);
541 /*
542 * Free kernel space accounting.
543 */
544 ttm_mem_global_free(vmw_mem_glob(fman->dev_priv),
545 fman->user_fence_size);
546}
547
548static void vmw_user_fence_base_release(struct ttm_base_object **p_base)
549{
550 struct ttm_base_object *base = *p_base;
551 struct vmw_user_fence *ufence =
552 container_of(base, struct vmw_user_fence, base);
553 struct vmw_fence_obj *fence = &ufence->fence;
554
555 *p_base = NULL;
556 vmw_fence_obj_unreference(&fence);
557}
558
559int vmw_user_fence_create(struct drm_file *file_priv,
560 struct vmw_fence_manager *fman,
561 uint32_t seqno,
562 uint32_t mask,
563 struct vmw_fence_obj **p_fence,
564 uint32_t *p_handle)
565{
566 struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
567 struct vmw_user_fence *ufence;
568 struct vmw_fence_obj *tmp;
569 struct ttm_mem_global *mem_glob = vmw_mem_glob(fman->dev_priv);
570 int ret;
571
572 /*
573 * Kernel memory space accounting, since this object may
574 * be created by a user-space request.
575 */
576
577 ret = ttm_mem_global_alloc(mem_glob, fman->user_fence_size,
578 false, false);
579 if (unlikely(ret != 0))
580 return ret;
581
582 ufence = kzalloc(sizeof(*ufence), GFP_KERNEL);
583 if (unlikely(ufence == NULL)) {
584 ret = -ENOMEM;
585 goto out_no_object;
586 }
587
588 ret = vmw_fence_obj_init(fman, &ufence->fence, seqno,
589 mask, vmw_user_fence_destroy);
590 if (unlikely(ret != 0)) {
591 kfree(ufence);
592 goto out_no_object;
593 }
594
595 /*
596 * The base object holds a reference which is freed in
597 * vmw_user_fence_base_release.
598 */
599 tmp = vmw_fence_obj_reference(&ufence->fence);
600 ret = ttm_base_object_init(tfile, &ufence->base, false,
601 VMW_RES_FENCE,
602 &vmw_user_fence_base_release, NULL);
603
604
605 if (unlikely(ret != 0)) {
606 /*
607 * Free the base object's reference
608 */
609 vmw_fence_obj_unreference(&tmp);
610 goto out_err;
611 }
612
613 *p_fence = &ufence->fence;
614 *p_handle = ufence->base.hash.key;
615
616 return 0;
617out_err:
618 tmp = &ufence->fence;
619 vmw_fence_obj_unreference(&tmp);
620out_no_object:
621 ttm_mem_global_free(mem_glob, fman->user_fence_size);
622 return ret;
623}
624
625
626/**
627 * vmw_fence_fifo_down - signal all unsignaled fence objects.
628 */
629
630void vmw_fence_fifo_down(struct vmw_fence_manager *fman)
631{
632 unsigned long irq_flags;
633 struct list_head action_list;
634 int ret;
635
636 /*
637 * The list may be altered while we traverse it, so always
638 * restart when we've released the fman->lock.
639 */
640
641 spin_lock_irqsave(&fman->lock, irq_flags);
642 fman->fifo_down = true;
643 while (!list_empty(&fman->fence_list)) {
644 struct vmw_fence_obj *fence =
645 list_entry(fman->fence_list.prev, struct vmw_fence_obj,
646 head);
647 kref_get(&fence->kref);
648 spin_unlock_irq(&fman->lock);
649
650 ret = vmw_fence_obj_wait(fence, fence->signal_mask,
651 false, false,
652 VMW_FENCE_WAIT_TIMEOUT);
653
654 if (unlikely(ret != 0)) {
655 list_del_init(&fence->head);
656 fence->signaled |= DRM_VMW_FENCE_FLAG_EXEC;
657 INIT_LIST_HEAD(&action_list);
658 list_splice_init(&fence->seq_passed_actions,
659 &action_list);
660 vmw_fences_perform_actions(fman, &action_list);
661 wake_up_all(&fence->queue);
662 }
663
664 spin_lock_irq(&fman->lock);
665
666 BUG_ON(!list_empty(&fence->head));
667 kref_put(&fence->kref, vmw_fence_obj_destroy_locked);
668 }
669 spin_unlock_irqrestore(&fman->lock, irq_flags);
670}
671
672void vmw_fence_fifo_up(struct vmw_fence_manager *fman)
673{
674 unsigned long irq_flags;
675
676 spin_lock_irqsave(&fman->lock, irq_flags);
677 fman->fifo_down = false;
678 spin_unlock_irqrestore(&fman->lock, irq_flags);
679}
680
681
682int vmw_fence_obj_wait_ioctl(struct drm_device *dev, void *data,
683 struct drm_file *file_priv)
684{
685 struct drm_vmw_fence_wait_arg *arg =
686 (struct drm_vmw_fence_wait_arg *)data;
687 unsigned long timeout;
688 struct ttm_base_object *base;
689 struct vmw_fence_obj *fence;
690 struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
691 int ret;
692 uint64_t wait_timeout = ((uint64_t)arg->timeout_us * HZ);
693
694 /*
695 * 64-bit division not present on 32-bit systems, so do an
696 * approximation. (Divide by 1000000).
697 */
698
699 wait_timeout = (wait_timeout >> 20) + (wait_timeout >> 24) -
700 (wait_timeout >> 26);
701
702 if (!arg->cookie_valid) {
703 arg->cookie_valid = 1;
704 arg->kernel_cookie = jiffies + wait_timeout;
705 }
706
707 base = ttm_base_object_lookup(tfile, arg->handle);
708 if (unlikely(base == NULL)) {
709 printk(KERN_ERR "Wait invalid fence object handle "
710 "0x%08lx.\n",
711 (unsigned long)arg->handle);
712 return -EINVAL;
713 }
714
715 fence = &(container_of(base, struct vmw_user_fence, base)->fence);
716
717 timeout = jiffies;
718 if (time_after_eq(timeout, (unsigned long)arg->kernel_cookie)) {
719 ret = ((vmw_fence_obj_signaled(fence, arg->flags)) ?
720 0 : -EBUSY);
721 goto out;
722 }
723
724 timeout = (unsigned long)arg->kernel_cookie - timeout;
725
726 ret = vmw_fence_obj_wait(fence, arg->flags, arg->lazy, true, timeout);
727
728out:
729 ttm_base_object_unref(&base);
730
731 /*
732 * Optionally unref the fence object.
733 */
734
735 if (ret == 0 && (arg->wait_options & DRM_VMW_WAIT_OPTION_UNREF))
736 return ttm_ref_object_base_unref(tfile, arg->handle,
737 TTM_REF_USAGE);
738 return ret;
739}
740
741int vmw_fence_obj_signaled_ioctl(struct drm_device *dev, void *data,
742 struct drm_file *file_priv)
743{
744 struct drm_vmw_fence_signaled_arg *arg =
745 (struct drm_vmw_fence_signaled_arg *) data;
746 struct ttm_base_object *base;
747 struct vmw_fence_obj *fence;
748 struct vmw_fence_manager *fman;
749 struct ttm_object_file *tfile = vmw_fpriv(file_priv)->tfile;
750 struct vmw_private *dev_priv = vmw_priv(dev);
751
752 base = ttm_base_object_lookup(tfile, arg->handle);
753 if (unlikely(base == NULL)) {
754 printk(KERN_ERR "Fence signaled invalid fence object handle "
755 "0x%08lx.\n",
756 (unsigned long)arg->handle);
757 return -EINVAL;
758 }
759
760 fence = &(container_of(base, struct vmw_user_fence, base)->fence);
761 fman = fence->fman;
762
763 arg->signaled = vmw_fence_obj_signaled(fence, arg->flags);
764 spin_lock_irq(&fman->lock);
765
766 arg->signaled_flags = fence->signaled;
767 arg->passed_seqno = dev_priv->last_read_seqno;
768 spin_unlock_irq(&fman->lock);
769
770 ttm_base_object_unref(&base);
771
772 return 0;
773}
774
775
776int vmw_fence_obj_unref_ioctl(struct drm_device *dev, void *data,
777 struct drm_file *file_priv)
778{
779 struct drm_vmw_fence_arg *arg =
780 (struct drm_vmw_fence_arg *) data;
781
782 return ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
783 arg->handle,
784 TTM_REF_USAGE);
785}
786
787/**
788 * vmw_event_fence_fpriv_gone - Remove references to struct drm_file objects
789 *
790 * @fman: Pointer to a struct vmw_fence_manager
791 * @event_list: Pointer to linked list of struct vmw_event_fence_action objects
792 * with pointers to a struct drm_file object about to be closed.
793 *
794 * This function removes all pending fence events with references to a
795 * specific struct drm_file object about to be closed. The caller is required
796 * to pass a list of all struct vmw_event_fence_action objects with such
797 * events attached. This function is typically called before the
798 * struct drm_file object's event management is taken down.
799 */
800void vmw_event_fence_fpriv_gone(struct vmw_fence_manager *fman,
801 struct list_head *event_list)
802{
803 struct vmw_event_fence_action *eaction;
804 struct drm_pending_event *event;
805 unsigned long irq_flags;
806
807 while (1) {
808 spin_lock_irqsave(&fman->lock, irq_flags);
809 if (list_empty(event_list))
810 goto out_unlock;
811 eaction = list_first_entry(event_list,
812 struct vmw_event_fence_action,
813 fpriv_head);
814 list_del_init(&eaction->fpriv_head);
815 event = eaction->event;
816 eaction->event = NULL;
817 spin_unlock_irqrestore(&fman->lock, irq_flags);
818 event->destroy(event);
819 }
820out_unlock:
821 spin_unlock_irqrestore(&fman->lock, irq_flags);
822}
823
824
825/**
826 * vmw_event_fence_action_seq_passed
827 *
828 * @action: The struct vmw_fence_action embedded in a struct
829 * vmw_event_fence_action.
830 *
831 * This function is called when the seqno of the fence where @action is
832 * attached has passed. It queues the event on the submitter's event list.
833 * This function is always called from atomic context, and may be called
834 * from irq context.
835 */
836static void vmw_event_fence_action_seq_passed(struct vmw_fence_action *action)
837{
838 struct vmw_event_fence_action *eaction =
839 container_of(action, struct vmw_event_fence_action, action);
840 struct drm_device *dev = eaction->dev;
841 struct drm_pending_event *event = eaction->event;
842 struct drm_file *file_priv;
843 unsigned long irq_flags;
844
845 if (unlikely(event == NULL))
846 return;
847
848 file_priv = event->file_priv;
849 spin_lock_irqsave(&dev->event_lock, irq_flags);
850
851 if (likely(eaction->tv_sec != NULL)) {
852 struct timeval tv;
853
854 do_gettimeofday(&tv);
855 *eaction->tv_sec = tv.tv_sec;
856 *eaction->tv_usec = tv.tv_usec;
857 }
858
859 list_del_init(&eaction->fpriv_head);
860 list_add_tail(&eaction->event->link, &file_priv->event_list);
861 eaction->event = NULL;
862 wake_up_all(&file_priv->event_wait);
863 spin_unlock_irqrestore(&dev->event_lock, irq_flags);
864}
865
866/**
867 * vmw_event_fence_action_cleanup
868 *
869 * @action: The struct vmw_fence_action embedded in a struct
870 * vmw_event_fence_action.
871 *
872 * This function is the struct vmw_fence_action destructor. It's typically
873 * called from a workqueue.
874 */
875static void vmw_event_fence_action_cleanup(struct vmw_fence_action *action)
876{
877 struct vmw_event_fence_action *eaction =
878 container_of(action, struct vmw_event_fence_action, action);
879 struct vmw_fence_manager *fman = eaction->fence->fman;
880 unsigned long irq_flags;
881
882 spin_lock_irqsave(&fman->lock, irq_flags);
883 list_del(&eaction->fpriv_head);
884 spin_unlock_irqrestore(&fman->lock, irq_flags);
885
886 vmw_fence_obj_unreference(&eaction->fence);
887 kfree(eaction);
888}
889
890
891/**
892 * vmw_fence_obj_add_action - Add an action to a fence object.
893 *
894 * @fence - The fence object.
895 * @action - The action to add.
896 *
897 * Note that the action callbacks may be executed before this function
898 * returns.
899 */
900static void vmw_fence_obj_add_action(struct vmw_fence_obj *fence,
901 struct vmw_fence_action *action)
902{
903 struct vmw_fence_manager *fman = fence->fman;
904 unsigned long irq_flags;
905 bool run_update = false;
906
907 mutex_lock(&fman->goal_irq_mutex);
908 spin_lock_irqsave(&fman->lock, irq_flags);
909
910 fman->pending_actions[action->type]++;
911 if (fence->signaled & DRM_VMW_FENCE_FLAG_EXEC) {
912 struct list_head action_list;
913
914 INIT_LIST_HEAD(&action_list);
915 list_add_tail(&action->head, &action_list);
916 vmw_fences_perform_actions(fman, &action_list);
917 } else {
918 list_add_tail(&action->head, &fence->seq_passed_actions);
919
920 /*
921 * This function may set fman::seqno_valid, so it must
922 * be run with the goal_irq_mutex held.
923 */
924 run_update = vmw_fence_goal_check_locked(fence);
925 }
926
927 spin_unlock_irqrestore(&fman->lock, irq_flags);
928
929 if (run_update) {
930 if (!fman->goal_irq_on) {
931 fman->goal_irq_on = true;
932 vmw_goal_waiter_add(fman->dev_priv);
933 }
934 vmw_fences_update(fman);
935 }
936 mutex_unlock(&fman->goal_irq_mutex);
937
938}
939
940/**
941 * vmw_event_fence_action_create - Post an event for sending when a fence
942 * object seqno has passed.
943 *
944 * @file_priv: The file connection on which the event should be posted.
945 * @fence: The fence object on which to post the event.
946 * @event: Event to be posted. This event should've been alloced
947 * using k[mz]alloc, and should've been completely initialized.
948 * @interruptible: Interruptible waits if possible.
949 *
950 * As a side effect, the object pointed to by @event may have been
951 * freed when this function returns. If this function returns with
952 * an error code, the caller needs to free that object.
953 */
954
955int vmw_event_fence_action_queue(struct drm_file *file_priv,
956 struct vmw_fence_obj *fence,
957 struct drm_pending_event *event,
958 uint32_t *tv_sec,
959 uint32_t *tv_usec,
960 bool interruptible)
961{
962 struct vmw_event_fence_action *eaction;
963 struct vmw_fence_manager *fman = fence->fman;
964 struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
965 unsigned long irq_flags;
966
967 eaction = kzalloc(sizeof(*eaction), GFP_KERNEL);
968 if (unlikely(eaction == NULL))
969 return -ENOMEM;
970
971 eaction->event = event;
972
973 eaction->action.seq_passed = vmw_event_fence_action_seq_passed;
974 eaction->action.cleanup = vmw_event_fence_action_cleanup;
975 eaction->action.type = VMW_ACTION_EVENT;
976
977 eaction->fence = vmw_fence_obj_reference(fence);
978 eaction->dev = fman->dev_priv->dev;
979 eaction->tv_sec = tv_sec;
980 eaction->tv_usec = tv_usec;
981
982 spin_lock_irqsave(&fman->lock, irq_flags);
983 list_add_tail(&eaction->fpriv_head, &vmw_fp->fence_events);
984 spin_unlock_irqrestore(&fman->lock, irq_flags);
985
986 vmw_fence_obj_add_action(fence, &eaction->action);
987
988 return 0;
989}
990
991struct vmw_event_fence_pending {
992 struct drm_pending_event base;
993 struct drm_vmw_event_fence event;
994};
995
996static int vmw_event_fence_action_create(struct drm_file *file_priv,
997 struct vmw_fence_obj *fence,
998 uint32_t flags,
999 uint64_t user_data,
1000 bool interruptible)
1001{
1002 struct vmw_event_fence_pending *event;
1003 struct drm_device *dev = fence->fman->dev_priv->dev;
1004 unsigned long irq_flags;
1005 int ret;
1006
1007 spin_lock_irqsave(&dev->event_lock, irq_flags);
1008
1009 ret = (file_priv->event_space < sizeof(event->event)) ? -EBUSY : 0;
1010 if (likely(ret == 0))
1011 file_priv->event_space -= sizeof(event->event);
1012
1013 spin_unlock_irqrestore(&dev->event_lock, irq_flags);
1014
1015 if (unlikely(ret != 0)) {
1016 DRM_ERROR("Failed to allocate event space for this file.\n");
1017 goto out_no_space;
1018 }
1019
1020
1021 event = kzalloc(sizeof(*event), GFP_KERNEL);
1022 if (unlikely(event == NULL)) {
1023 DRM_ERROR("Failed to allocate an event.\n");
1024 ret = -ENOMEM;
1025 goto out_no_event;
1026 }
1027
1028 event->event.base.type = DRM_VMW_EVENT_FENCE_SIGNALED;
1029 event->event.base.length = sizeof(*event);
1030 event->event.user_data = user_data;
1031
1032 event->base.event = &event->event.base;
1033 event->base.file_priv = file_priv;
1034 event->base.destroy = (void (*) (struct drm_pending_event *)) kfree;
1035
1036
1037 if (flags & DRM_VMW_FE_FLAG_REQ_TIME)
1038 ret = vmw_event_fence_action_queue(file_priv, fence,
1039 &event->base,
1040 &event->event.tv_sec,
1041 &event->event.tv_usec,
1042 interruptible);
1043 else
1044 ret = vmw_event_fence_action_queue(file_priv, fence,
1045 &event->base,
1046 NULL,
1047 NULL,
1048 interruptible);
1049 if (ret != 0)
1050 goto out_no_queue;
1051
1052out_no_queue:
1053 event->base.destroy(&event->base);
1054out_no_event:
1055 spin_lock_irqsave(&dev->event_lock, irq_flags);
1056 file_priv->event_space += sizeof(*event);
1057 spin_unlock_irqrestore(&dev->event_lock, irq_flags);
1058out_no_space:
1059 return ret;
1060}
1061
1062int vmw_fence_event_ioctl(struct drm_device *dev, void *data,
1063 struct drm_file *file_priv)
1064{
1065 struct vmw_private *dev_priv = vmw_priv(dev);
1066 struct drm_vmw_fence_event_arg *arg =
1067 (struct drm_vmw_fence_event_arg *) data;
1068 struct vmw_fence_obj *fence = NULL;
1069 struct vmw_fpriv *vmw_fp = vmw_fpriv(file_priv);
1070 struct drm_vmw_fence_rep __user *user_fence_rep =
1071 (struct drm_vmw_fence_rep __user *)(unsigned long)
1072 arg->fence_rep;
1073 uint32_t handle;
1074 int ret;
1075
1076 /*
1077 * Look up an existing fence object,
1078 * and if user-space wants a new reference,
1079 * add one.
1080 */
1081 if (arg->handle) {
1082 struct ttm_base_object *base =
1083 ttm_base_object_lookup_for_ref(dev_priv->tdev,
1084 arg->handle);
1085
1086 if (unlikely(base == NULL)) {
1087 DRM_ERROR("Fence event invalid fence object handle "
1088 "0x%08lx.\n",
1089 (unsigned long)arg->handle);
1090 return -EINVAL;
1091 }
1092 fence = &(container_of(base, struct vmw_user_fence,
1093 base)->fence);
1094 (void) vmw_fence_obj_reference(fence);
1095
1096 if (user_fence_rep != NULL) {
1097 bool existed;
1098
1099 ret = ttm_ref_object_add(vmw_fp->tfile, base,
1100 TTM_REF_USAGE, &existed);
1101 if (unlikely(ret != 0)) {
1102 DRM_ERROR("Failed to reference a fence "
1103 "object.\n");
1104 goto out_no_ref_obj;
1105 }
1106 handle = base->hash.key;
1107 }
1108 ttm_base_object_unref(&base);
1109 }
1110
1111 /*
1112 * Create a new fence object.
1113 */
1114 if (!fence) {
1115 ret = vmw_execbuf_fence_commands(file_priv, dev_priv,
1116 &fence,
1117 (user_fence_rep) ?
1118 &handle : NULL);
1119 if (unlikely(ret != 0)) {
1120 DRM_ERROR("Fence event failed to create fence.\n");
1121 return ret;
1122 }
1123 }
1124
1125 BUG_ON(fence == NULL);
1126
1127 if (arg->flags & DRM_VMW_FE_FLAG_REQ_TIME)
1128 ret = vmw_event_fence_action_create(file_priv, fence,
1129 arg->flags,
1130 arg->user_data,
1131 true);
1132 else
1133 ret = vmw_event_fence_action_create(file_priv, fence,
1134 arg->flags,
1135 arg->user_data,
1136 true);
1137
1138 if (unlikely(ret != 0)) {
1139 if (ret != -ERESTARTSYS)
1140 DRM_ERROR("Failed to attach event to fence.\n");
1141 goto out_no_create;
1142 }
1143
1144 vmw_execbuf_copy_fence_user(dev_priv, vmw_fp, 0, user_fence_rep, fence,
1145 handle);
1146 vmw_fence_obj_unreference(&fence);
1147 return 0;
1148out_no_create:
1149 if (user_fence_rep != NULL)
1150 ttm_ref_object_base_unref(vmw_fpriv(file_priv)->tfile,
1151 handle, TTM_REF_USAGE);
1152out_no_ref_obj:
1153 vmw_fence_obj_unreference(&fence);
1154 return ret;
1155}