Loading...
Note: File does not exist in v4.6.
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include "i915_vma.h"
26
27#include "i915_drv.h"
28#include "intel_ringbuffer.h"
29#include "intel_frontbuffer.h"
30
31#include <drm/drm_gem.h>
32
33static void
34i915_vma_retire(struct i915_gem_active *active, struct i915_request *rq)
35{
36 const unsigned int idx = rq->engine->id;
37 struct i915_vma *vma =
38 container_of(active, struct i915_vma, last_read[idx]);
39 struct drm_i915_gem_object *obj = vma->obj;
40
41 GEM_BUG_ON(!i915_vma_has_active_engine(vma, idx));
42
43 i915_vma_clear_active(vma, idx);
44 if (i915_vma_is_active(vma))
45 return;
46
47 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
48 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
49 if (unlikely(i915_vma_is_closed(vma) && !i915_vma_is_pinned(vma)))
50 WARN_ON(i915_vma_unbind(vma));
51
52 GEM_BUG_ON(!i915_gem_object_is_active(obj));
53 if (--obj->active_count)
54 return;
55
56 /* Prune the shared fence arrays iff completely idle (inc. external) */
57 if (reservation_object_trylock(obj->resv)) {
58 if (reservation_object_test_signaled_rcu(obj->resv, true))
59 reservation_object_add_excl_fence(obj->resv, NULL);
60 reservation_object_unlock(obj->resv);
61 }
62
63 /* Bump our place on the bound list to keep it roughly in LRU order
64 * so that we don't steal from recently used but inactive objects
65 * (unless we are forced to ofc!)
66 */
67 spin_lock(&rq->i915->mm.obj_lock);
68 if (obj->bind_count)
69 list_move_tail(&obj->mm.link, &rq->i915->mm.bound_list);
70 spin_unlock(&rq->i915->mm.obj_lock);
71
72 obj->mm.dirty = true; /* be paranoid */
73
74 if (i915_gem_object_has_active_reference(obj)) {
75 i915_gem_object_clear_active_reference(obj);
76 i915_gem_object_put(obj);
77 }
78}
79
80static struct i915_vma *
81vma_create(struct drm_i915_gem_object *obj,
82 struct i915_address_space *vm,
83 const struct i915_ggtt_view *view)
84{
85 struct i915_vma *vma;
86 struct rb_node *rb, **p;
87 int i;
88
89 /* The aliasing_ppgtt should never be used directly! */
90 GEM_BUG_ON(vm == &vm->i915->mm.aliasing_ppgtt->base);
91
92 vma = kmem_cache_zalloc(vm->i915->vmas, GFP_KERNEL);
93 if (vma == NULL)
94 return ERR_PTR(-ENOMEM);
95
96 for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
97 init_request_active(&vma->last_read[i], i915_vma_retire);
98 init_request_active(&vma->last_fence, NULL);
99 vma->vm = vm;
100 vma->obj = obj;
101 vma->resv = obj->resv;
102 vma->size = obj->base.size;
103 vma->display_alignment = I915_GTT_MIN_ALIGNMENT;
104
105 if (view && view->type != I915_GGTT_VIEW_NORMAL) {
106 vma->ggtt_view = *view;
107 if (view->type == I915_GGTT_VIEW_PARTIAL) {
108 GEM_BUG_ON(range_overflows_t(u64,
109 view->partial.offset,
110 view->partial.size,
111 obj->base.size >> PAGE_SHIFT));
112 vma->size = view->partial.size;
113 vma->size <<= PAGE_SHIFT;
114 GEM_BUG_ON(vma->size >= obj->base.size);
115 } else if (view->type == I915_GGTT_VIEW_ROTATED) {
116 vma->size = intel_rotation_info_size(&view->rotated);
117 vma->size <<= PAGE_SHIFT;
118 }
119 }
120
121 if (unlikely(vma->size > vm->total))
122 goto err_vma;
123
124 GEM_BUG_ON(!IS_ALIGNED(vma->size, I915_GTT_PAGE_SIZE));
125
126 if (i915_is_ggtt(vm)) {
127 if (unlikely(overflows_type(vma->size, u32)))
128 goto err_vma;
129
130 vma->fence_size = i915_gem_fence_size(vm->i915, vma->size,
131 i915_gem_object_get_tiling(obj),
132 i915_gem_object_get_stride(obj));
133 if (unlikely(vma->fence_size < vma->size || /* overflow */
134 vma->fence_size > vm->total))
135 goto err_vma;
136
137 GEM_BUG_ON(!IS_ALIGNED(vma->fence_size, I915_GTT_MIN_ALIGNMENT));
138
139 vma->fence_alignment = i915_gem_fence_alignment(vm->i915, vma->size,
140 i915_gem_object_get_tiling(obj),
141 i915_gem_object_get_stride(obj));
142 GEM_BUG_ON(!is_power_of_2(vma->fence_alignment));
143
144 /*
145 * We put the GGTT vma at the start of the vma-list, followed
146 * by the ppGGTT vma. This allows us to break early when
147 * iterating over only the GGTT vma for an object, see
148 * for_each_ggtt_vma()
149 */
150 vma->flags |= I915_VMA_GGTT;
151 list_add(&vma->obj_link, &obj->vma_list);
152 } else {
153 i915_ppgtt_get(i915_vm_to_ppgtt(vm));
154 list_add_tail(&vma->obj_link, &obj->vma_list);
155 }
156
157 rb = NULL;
158 p = &obj->vma_tree.rb_node;
159 while (*p) {
160 struct i915_vma *pos;
161
162 rb = *p;
163 pos = rb_entry(rb, struct i915_vma, obj_node);
164 if (i915_vma_compare(pos, vm, view) < 0)
165 p = &rb->rb_right;
166 else
167 p = &rb->rb_left;
168 }
169 rb_link_node(&vma->obj_node, rb, p);
170 rb_insert_color(&vma->obj_node, &obj->vma_tree);
171 list_add(&vma->vm_link, &vm->unbound_list);
172
173 return vma;
174
175err_vma:
176 kmem_cache_free(vm->i915->vmas, vma);
177 return ERR_PTR(-E2BIG);
178}
179
180static struct i915_vma *
181vma_lookup(struct drm_i915_gem_object *obj,
182 struct i915_address_space *vm,
183 const struct i915_ggtt_view *view)
184{
185 struct rb_node *rb;
186
187 rb = obj->vma_tree.rb_node;
188 while (rb) {
189 struct i915_vma *vma = rb_entry(rb, struct i915_vma, obj_node);
190 long cmp;
191
192 cmp = i915_vma_compare(vma, vm, view);
193 if (cmp == 0)
194 return vma;
195
196 if (cmp < 0)
197 rb = rb->rb_right;
198 else
199 rb = rb->rb_left;
200 }
201
202 return NULL;
203}
204
205/**
206 * i915_vma_instance - return the singleton instance of the VMA
207 * @obj: parent &struct drm_i915_gem_object to be mapped
208 * @vm: address space in which the mapping is located
209 * @view: additional mapping requirements
210 *
211 * i915_vma_instance() looks up an existing VMA of the @obj in the @vm with
212 * the same @view characteristics. If a match is not found, one is created.
213 * Once created, the VMA is kept until either the object is freed, or the
214 * address space is closed.
215 *
216 * Must be called with struct_mutex held.
217 *
218 * Returns the vma, or an error pointer.
219 */
220struct i915_vma *
221i915_vma_instance(struct drm_i915_gem_object *obj,
222 struct i915_address_space *vm,
223 const struct i915_ggtt_view *view)
224{
225 struct i915_vma *vma;
226
227 lockdep_assert_held(&obj->base.dev->struct_mutex);
228 GEM_BUG_ON(view && !i915_is_ggtt(vm));
229 GEM_BUG_ON(vm->closed);
230
231 vma = vma_lookup(obj, vm, view);
232 if (!vma)
233 vma = vma_create(obj, vm, view);
234
235 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_is_closed(vma));
236 GEM_BUG_ON(!IS_ERR(vma) && i915_vma_compare(vma, vm, view));
237 GEM_BUG_ON(!IS_ERR(vma) && vma_lookup(obj, vm, view) != vma);
238 return vma;
239}
240
241/**
242 * i915_vma_bind - Sets up PTEs for an VMA in it's corresponding address space.
243 * @vma: VMA to map
244 * @cache_level: mapping cache level
245 * @flags: flags like global or local mapping
246 *
247 * DMA addresses are taken from the scatter-gather table of this object (or of
248 * this VMA in case of non-default GGTT views) and PTE entries set up.
249 * Note that DMA addresses are also the only part of the SG table we care about.
250 */
251int i915_vma_bind(struct i915_vma *vma, enum i915_cache_level cache_level,
252 u32 flags)
253{
254 u32 bind_flags;
255 u32 vma_flags;
256 int ret;
257
258 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
259 GEM_BUG_ON(vma->size > vma->node.size);
260
261 if (GEM_WARN_ON(range_overflows(vma->node.start,
262 vma->node.size,
263 vma->vm->total)))
264 return -ENODEV;
265
266 if (GEM_WARN_ON(!flags))
267 return -EINVAL;
268
269 bind_flags = 0;
270 if (flags & PIN_GLOBAL)
271 bind_flags |= I915_VMA_GLOBAL_BIND;
272 if (flags & PIN_USER)
273 bind_flags |= I915_VMA_LOCAL_BIND;
274
275 vma_flags = vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
276 if (flags & PIN_UPDATE)
277 bind_flags |= vma_flags;
278 else
279 bind_flags &= ~vma_flags;
280 if (bind_flags == 0)
281 return 0;
282
283 GEM_BUG_ON(!vma->pages);
284
285 trace_i915_vma_bind(vma, bind_flags);
286 ret = vma->vm->bind_vma(vma, cache_level, bind_flags);
287 if (ret)
288 return ret;
289
290 vma->flags |= bind_flags;
291 return 0;
292}
293
294void __iomem *i915_vma_pin_iomap(struct i915_vma *vma)
295{
296 void __iomem *ptr;
297 int err;
298
299 /* Access through the GTT requires the device to be awake. */
300 assert_rpm_wakelock_held(vma->vm->i915);
301
302 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
303 if (WARN_ON(!i915_vma_is_map_and_fenceable(vma))) {
304 err = -ENODEV;
305 goto err;
306 }
307
308 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
309 GEM_BUG_ON((vma->flags & I915_VMA_GLOBAL_BIND) == 0);
310
311 ptr = vma->iomap;
312 if (ptr == NULL) {
313 ptr = io_mapping_map_wc(&i915_vm_to_ggtt(vma->vm)->iomap,
314 vma->node.start,
315 vma->node.size);
316 if (ptr == NULL) {
317 err = -ENOMEM;
318 goto err;
319 }
320
321 vma->iomap = ptr;
322 }
323
324 __i915_vma_pin(vma);
325
326 err = i915_vma_pin_fence(vma);
327 if (err)
328 goto err_unpin;
329
330 i915_vma_set_ggtt_write(vma);
331 return ptr;
332
333err_unpin:
334 __i915_vma_unpin(vma);
335err:
336 return IO_ERR_PTR(err);
337}
338
339void i915_vma_flush_writes(struct i915_vma *vma)
340{
341 if (!i915_vma_has_ggtt_write(vma))
342 return;
343
344 i915_gem_flush_ggtt_writes(vma->vm->i915);
345
346 i915_vma_unset_ggtt_write(vma);
347}
348
349void i915_vma_unpin_iomap(struct i915_vma *vma)
350{
351 lockdep_assert_held(&vma->obj->base.dev->struct_mutex);
352
353 GEM_BUG_ON(vma->iomap == NULL);
354
355 i915_vma_flush_writes(vma);
356
357 i915_vma_unpin_fence(vma);
358 i915_vma_unpin(vma);
359}
360
361void i915_vma_unpin_and_release(struct i915_vma **p_vma)
362{
363 struct i915_vma *vma;
364 struct drm_i915_gem_object *obj;
365
366 vma = fetch_and_zero(p_vma);
367 if (!vma)
368 return;
369
370 obj = vma->obj;
371
372 i915_vma_unpin(vma);
373 i915_vma_close(vma);
374
375 __i915_gem_object_release_unless_active(obj);
376}
377
378bool i915_vma_misplaced(const struct i915_vma *vma,
379 u64 size, u64 alignment, u64 flags)
380{
381 if (!drm_mm_node_allocated(&vma->node))
382 return false;
383
384 if (vma->node.size < size)
385 return true;
386
387 GEM_BUG_ON(alignment && !is_power_of_2(alignment));
388 if (alignment && !IS_ALIGNED(vma->node.start, alignment))
389 return true;
390
391 if (flags & PIN_MAPPABLE && !i915_vma_is_map_and_fenceable(vma))
392 return true;
393
394 if (flags & PIN_OFFSET_BIAS &&
395 vma->node.start < (flags & PIN_OFFSET_MASK))
396 return true;
397
398 if (flags & PIN_OFFSET_FIXED &&
399 vma->node.start != (flags & PIN_OFFSET_MASK))
400 return true;
401
402 return false;
403}
404
405void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
406{
407 bool mappable, fenceable;
408
409 GEM_BUG_ON(!i915_vma_is_ggtt(vma));
410 GEM_BUG_ON(!vma->fence_size);
411
412 /*
413 * Explicitly disable for rotated VMA since the display does not
414 * need the fence and the VMA is not accessible to other users.
415 */
416 if (vma->ggtt_view.type == I915_GGTT_VIEW_ROTATED)
417 return;
418
419 fenceable = (vma->node.size >= vma->fence_size &&
420 IS_ALIGNED(vma->node.start, vma->fence_alignment));
421
422 mappable = vma->node.start + vma->fence_size <= i915_vm_to_ggtt(vma->vm)->mappable_end;
423
424 if (mappable && fenceable)
425 vma->flags |= I915_VMA_CAN_FENCE;
426 else
427 vma->flags &= ~I915_VMA_CAN_FENCE;
428}
429
430static bool color_differs(struct drm_mm_node *node, unsigned long color)
431{
432 return node->allocated && node->color != color;
433}
434
435bool i915_gem_valid_gtt_space(struct i915_vma *vma, unsigned long cache_level)
436{
437 struct drm_mm_node *node = &vma->node;
438 struct drm_mm_node *other;
439
440 /*
441 * On some machines we have to be careful when putting differing types
442 * of snoopable memory together to avoid the prefetcher crossing memory
443 * domains and dying. During vm initialisation, we decide whether or not
444 * these constraints apply and set the drm_mm.color_adjust
445 * appropriately.
446 */
447 if (vma->vm->mm.color_adjust == NULL)
448 return true;
449
450 /* Only valid to be called on an already inserted vma */
451 GEM_BUG_ON(!drm_mm_node_allocated(node));
452 GEM_BUG_ON(list_empty(&node->node_list));
453
454 other = list_prev_entry(node, node_list);
455 if (color_differs(other, cache_level) && !drm_mm_hole_follows(other))
456 return false;
457
458 other = list_next_entry(node, node_list);
459 if (color_differs(other, cache_level) && !drm_mm_hole_follows(node))
460 return false;
461
462 return true;
463}
464
465/**
466 * i915_vma_insert - finds a slot for the vma in its address space
467 * @vma: the vma
468 * @size: requested size in bytes (can be larger than the VMA)
469 * @alignment: required alignment
470 * @flags: mask of PIN_* flags to use
471 *
472 * First we try to allocate some free space that meets the requirements for
473 * the VMA. Failiing that, if the flags permit, it will evict an old VMA,
474 * preferrably the oldest idle entry to make room for the new VMA.
475 *
476 * Returns:
477 * 0 on success, negative error code otherwise.
478 */
479static int
480i915_vma_insert(struct i915_vma *vma, u64 size, u64 alignment, u64 flags)
481{
482 struct drm_i915_private *dev_priv = vma->vm->i915;
483 struct drm_i915_gem_object *obj = vma->obj;
484 u64 start, end;
485 int ret;
486
487 GEM_BUG_ON(i915_vma_is_closed(vma));
488 GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
489 GEM_BUG_ON(drm_mm_node_allocated(&vma->node));
490
491 size = max(size, vma->size);
492 alignment = max(alignment, vma->display_alignment);
493 if (flags & PIN_MAPPABLE) {
494 size = max_t(typeof(size), size, vma->fence_size);
495 alignment = max_t(typeof(alignment),
496 alignment, vma->fence_alignment);
497 }
498
499 GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
500 GEM_BUG_ON(!IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
501 GEM_BUG_ON(!is_power_of_2(alignment));
502
503 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
504 GEM_BUG_ON(!IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
505
506 end = vma->vm->total;
507 if (flags & PIN_MAPPABLE)
508 end = min_t(u64, end, dev_priv->ggtt.mappable_end);
509 if (flags & PIN_ZONE_4G)
510 end = min_t(u64, end, (1ULL << 32) - I915_GTT_PAGE_SIZE);
511 GEM_BUG_ON(!IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
512
513 /* If binding the object/GGTT view requires more space than the entire
514 * aperture has, reject it early before evicting everything in a vain
515 * attempt to find space.
516 */
517 if (size > end) {
518 DRM_DEBUG("Attempting to bind an object larger than the aperture: request=%llu [object=%zd] > %s aperture=%llu\n",
519 size, obj->base.size,
520 flags & PIN_MAPPABLE ? "mappable" : "total",
521 end);
522 return -ENOSPC;
523 }
524
525 ret = i915_gem_object_pin_pages(obj);
526 if (ret)
527 return ret;
528
529 GEM_BUG_ON(vma->pages);
530
531 ret = vma->vm->set_pages(vma);
532 if (ret)
533 goto err_unpin;
534
535 if (flags & PIN_OFFSET_FIXED) {
536 u64 offset = flags & PIN_OFFSET_MASK;
537 if (!IS_ALIGNED(offset, alignment) ||
538 range_overflows(offset, size, end)) {
539 ret = -EINVAL;
540 goto err_clear;
541 }
542
543 ret = i915_gem_gtt_reserve(vma->vm, &vma->node,
544 size, offset, obj->cache_level,
545 flags);
546 if (ret)
547 goto err_clear;
548 } else {
549 /*
550 * We only support huge gtt pages through the 48b PPGTT,
551 * however we also don't want to force any alignment for
552 * objects which need to be tightly packed into the low 32bits.
553 *
554 * Note that we assume that GGTT are limited to 4GiB for the
555 * forseeable future. See also i915_ggtt_offset().
556 */
557 if (upper_32_bits(end - 1) &&
558 vma->page_sizes.sg > I915_GTT_PAGE_SIZE) {
559 /*
560 * We can't mix 64K and 4K PTEs in the same page-table
561 * (2M block), and so to avoid the ugliness and
562 * complexity of coloring we opt for just aligning 64K
563 * objects to 2M.
564 */
565 u64 page_alignment =
566 rounddown_pow_of_two(vma->page_sizes.sg |
567 I915_GTT_PAGE_SIZE_2M);
568
569 /*
570 * Check we don't expand for the limited Global GTT
571 * (mappable aperture is even more precious!). This
572 * also checks that we exclude the aliasing-ppgtt.
573 */
574 GEM_BUG_ON(i915_vma_is_ggtt(vma));
575
576 alignment = max(alignment, page_alignment);
577
578 if (vma->page_sizes.sg & I915_GTT_PAGE_SIZE_64K)
579 size = round_up(size, I915_GTT_PAGE_SIZE_2M);
580 }
581
582 ret = i915_gem_gtt_insert(vma->vm, &vma->node,
583 size, alignment, obj->cache_level,
584 start, end, flags);
585 if (ret)
586 goto err_clear;
587
588 GEM_BUG_ON(vma->node.start < start);
589 GEM_BUG_ON(vma->node.start + vma->node.size > end);
590 }
591 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
592 GEM_BUG_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level));
593
594 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
595
596 spin_lock(&dev_priv->mm.obj_lock);
597 list_move_tail(&obj->mm.link, &dev_priv->mm.bound_list);
598 obj->bind_count++;
599 spin_unlock(&dev_priv->mm.obj_lock);
600
601 GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count);
602
603 return 0;
604
605err_clear:
606 vma->vm->clear_pages(vma);
607err_unpin:
608 i915_gem_object_unpin_pages(obj);
609 return ret;
610}
611
612static void
613i915_vma_remove(struct i915_vma *vma)
614{
615 struct drm_i915_private *i915 = vma->vm->i915;
616 struct drm_i915_gem_object *obj = vma->obj;
617
618 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
619 GEM_BUG_ON(vma->flags & (I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND));
620
621 vma->vm->clear_pages(vma);
622
623 drm_mm_remove_node(&vma->node);
624 list_move_tail(&vma->vm_link, &vma->vm->unbound_list);
625
626 /* Since the unbound list is global, only move to that list if
627 * no more VMAs exist.
628 */
629 spin_lock(&i915->mm.obj_lock);
630 if (--obj->bind_count == 0)
631 list_move_tail(&obj->mm.link, &i915->mm.unbound_list);
632 spin_unlock(&i915->mm.obj_lock);
633
634 /* And finally now the object is completely decoupled from this vma,
635 * we can drop its hold on the backing storage and allow it to be
636 * reaped by the shrinker.
637 */
638 i915_gem_object_unpin_pages(obj);
639 GEM_BUG_ON(atomic_read(&obj->mm.pages_pin_count) < obj->bind_count);
640}
641
642int __i915_vma_do_pin(struct i915_vma *vma,
643 u64 size, u64 alignment, u64 flags)
644{
645 const unsigned int bound = vma->flags;
646 int ret;
647
648 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
649 GEM_BUG_ON((flags & (PIN_GLOBAL | PIN_USER)) == 0);
650 GEM_BUG_ON((flags & PIN_GLOBAL) && !i915_vma_is_ggtt(vma));
651
652 if (WARN_ON(bound & I915_VMA_PIN_OVERFLOW)) {
653 ret = -EBUSY;
654 goto err_unpin;
655 }
656
657 if ((bound & I915_VMA_BIND_MASK) == 0) {
658 ret = i915_vma_insert(vma, size, alignment, flags);
659 if (ret)
660 goto err_unpin;
661 }
662 GEM_BUG_ON(!drm_mm_node_allocated(&vma->node));
663
664 ret = i915_vma_bind(vma, vma->obj->cache_level, flags);
665 if (ret)
666 goto err_remove;
667
668 GEM_BUG_ON((vma->flags & I915_VMA_BIND_MASK) == 0);
669
670 if ((bound ^ vma->flags) & I915_VMA_GLOBAL_BIND)
671 __i915_vma_set_map_and_fenceable(vma);
672
673 GEM_BUG_ON(i915_vma_misplaced(vma, size, alignment, flags));
674 return 0;
675
676err_remove:
677 if ((bound & I915_VMA_BIND_MASK) == 0) {
678 i915_vma_remove(vma);
679 GEM_BUG_ON(vma->pages);
680 GEM_BUG_ON(vma->flags & I915_VMA_BIND_MASK);
681 }
682err_unpin:
683 __i915_vma_unpin(vma);
684 return ret;
685}
686
687static void i915_vma_destroy(struct i915_vma *vma)
688{
689 int i;
690
691 GEM_BUG_ON(vma->node.allocated);
692 GEM_BUG_ON(i915_vma_is_active(vma));
693 GEM_BUG_ON(!i915_vma_is_closed(vma));
694 GEM_BUG_ON(vma->fence);
695
696 for (i = 0; i < ARRAY_SIZE(vma->last_read); i++)
697 GEM_BUG_ON(i915_gem_active_isset(&vma->last_read[i]));
698 GEM_BUG_ON(i915_gem_active_isset(&vma->last_fence));
699
700 list_del(&vma->obj_link);
701 list_del(&vma->vm_link);
702
703 if (!i915_vma_is_ggtt(vma))
704 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
705
706 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
707}
708
709void i915_vma_close(struct i915_vma *vma)
710{
711 GEM_BUG_ON(i915_vma_is_closed(vma));
712 vma->flags |= I915_VMA_CLOSED;
713
714 rb_erase(&vma->obj_node, &vma->obj->vma_tree);
715
716 if (!i915_vma_is_active(vma) && !i915_vma_is_pinned(vma))
717 WARN_ON(i915_vma_unbind(vma));
718}
719
720static void __i915_vma_iounmap(struct i915_vma *vma)
721{
722 GEM_BUG_ON(i915_vma_is_pinned(vma));
723
724 if (vma->iomap == NULL)
725 return;
726
727 io_mapping_unmap(vma->iomap);
728 vma->iomap = NULL;
729}
730
731void i915_vma_revoke_mmap(struct i915_vma *vma)
732{
733 struct drm_vma_offset_node *node = &vma->obj->base.vma_node;
734 u64 vma_offset;
735
736 lockdep_assert_held(&vma->vm->i915->drm.struct_mutex);
737
738 if (!i915_vma_has_userfault(vma))
739 return;
740
741 GEM_BUG_ON(!i915_vma_is_map_and_fenceable(vma));
742 GEM_BUG_ON(!vma->obj->userfault_count);
743
744 vma_offset = vma->ggtt_view.partial.offset << PAGE_SHIFT;
745 unmap_mapping_range(vma->vm->i915->drm.anon_inode->i_mapping,
746 drm_vma_node_offset_addr(node) + vma_offset,
747 vma->size,
748 1);
749
750 i915_vma_unset_userfault(vma);
751 if (!--vma->obj->userfault_count)
752 list_del(&vma->obj->userfault_link);
753}
754
755int i915_vma_unbind(struct i915_vma *vma)
756{
757 struct drm_i915_gem_object *obj = vma->obj;
758 unsigned long active;
759 int ret;
760
761 lockdep_assert_held(&obj->base.dev->struct_mutex);
762
763 /* First wait upon any activity as retiring the request may
764 * have side-effects such as unpinning or even unbinding this vma.
765 */
766 might_sleep();
767 active = i915_vma_get_active(vma);
768 if (active) {
769 int idx;
770
771 /* When a closed VMA is retired, it is unbound - eek.
772 * In order to prevent it from being recursively closed,
773 * take a pin on the vma so that the second unbind is
774 * aborted.
775 *
776 * Even more scary is that the retire callback may free
777 * the object (last active vma). To prevent the explosion
778 * we defer the actual object free to a worker that can
779 * only proceed once it acquires the struct_mutex (which
780 * we currently hold, therefore it cannot free this object
781 * before we are finished).
782 */
783 __i915_vma_pin(vma);
784
785 for_each_active(active, idx) {
786 ret = i915_gem_active_retire(&vma->last_read[idx],
787 &vma->vm->i915->drm.struct_mutex);
788 if (ret)
789 break;
790 }
791
792 if (!ret) {
793 ret = i915_gem_active_retire(&vma->last_fence,
794 &vma->vm->i915->drm.struct_mutex);
795 }
796
797 __i915_vma_unpin(vma);
798 if (ret)
799 return ret;
800 }
801 GEM_BUG_ON(i915_vma_is_active(vma));
802
803 if (i915_vma_is_pinned(vma))
804 return -EBUSY;
805
806 if (!drm_mm_node_allocated(&vma->node))
807 goto destroy;
808
809 GEM_BUG_ON(obj->bind_count == 0);
810 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
811
812 if (i915_vma_is_map_and_fenceable(vma)) {
813 /*
814 * Check that we have flushed all writes through the GGTT
815 * before the unbind, other due to non-strict nature of those
816 * indirect writes they may end up referencing the GGTT PTE
817 * after the unbind.
818 */
819 i915_vma_flush_writes(vma);
820 GEM_BUG_ON(i915_vma_has_ggtt_write(vma));
821
822 /* release the fence reg _after_ flushing */
823 ret = i915_vma_put_fence(vma);
824 if (ret)
825 return ret;
826
827 /* Force a pagefault for domain tracking on next user access */
828 i915_vma_revoke_mmap(vma);
829
830 __i915_vma_iounmap(vma);
831 vma->flags &= ~I915_VMA_CAN_FENCE;
832 }
833 GEM_BUG_ON(vma->fence);
834 GEM_BUG_ON(i915_vma_has_userfault(vma));
835
836 if (likely(!vma->vm->closed)) {
837 trace_i915_vma_unbind(vma);
838 vma->vm->unbind_vma(vma);
839 }
840 vma->flags &= ~(I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND);
841
842 i915_vma_remove(vma);
843
844destroy:
845 if (unlikely(i915_vma_is_closed(vma)))
846 i915_vma_destroy(vma);
847
848 return 0;
849}
850
851#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
852#include "selftests/i915_vma.c"
853#endif