Loading...
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include <drm/drmP.h>
29#include <drm/drm_vma_manager.h>
30#include <drm/i915_drm.h>
31#include "i915_drv.h"
32#include "i915_vgpu.h"
33#include "i915_trace.h"
34#include "intel_drv.h"
35#include <linux/shmem_fs.h>
36#include <linux/slab.h>
37#include <linux/swap.h>
38#include <linux/pci.h>
39#include <linux/dma-buf.h>
40
41#define RQ_BUG_ON(expr)
42
43static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
45static void
46i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
47static void
48i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
49
50static bool cpu_cache_is_coherent(struct drm_device *dev,
51 enum i915_cache_level level)
52{
53 return HAS_LLC(dev) || level != I915_CACHE_NONE;
54}
55
56static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
57{
58 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
59 return true;
60
61 return obj->pin_display;
62}
63
64/* some bookkeeping */
65static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
66 size_t size)
67{
68 spin_lock(&dev_priv->mm.object_stat_lock);
69 dev_priv->mm.object_count++;
70 dev_priv->mm.object_memory += size;
71 spin_unlock(&dev_priv->mm.object_stat_lock);
72}
73
74static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
75 size_t size)
76{
77 spin_lock(&dev_priv->mm.object_stat_lock);
78 dev_priv->mm.object_count--;
79 dev_priv->mm.object_memory -= size;
80 spin_unlock(&dev_priv->mm.object_stat_lock);
81}
82
83static int
84i915_gem_wait_for_error(struct i915_gpu_error *error)
85{
86 int ret;
87
88#define EXIT_COND (!i915_reset_in_progress(error) || \
89 i915_terminally_wedged(error))
90 if (EXIT_COND)
91 return 0;
92
93 /*
94 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
95 * userspace. If it takes that long something really bad is going on and
96 * we should simply try to bail out and fail as gracefully as possible.
97 */
98 ret = wait_event_interruptible_timeout(error->reset_queue,
99 EXIT_COND,
100 10*HZ);
101 if (ret == 0) {
102 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
103 return -EIO;
104 } else if (ret < 0) {
105 return ret;
106 }
107#undef EXIT_COND
108
109 return 0;
110}
111
112int i915_mutex_lock_interruptible(struct drm_device *dev)
113{
114 struct drm_i915_private *dev_priv = dev->dev_private;
115 int ret;
116
117 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
118 if (ret)
119 return ret;
120
121 ret = mutex_lock_interruptible(&dev->struct_mutex);
122 if (ret)
123 return ret;
124
125 WARN_ON(i915_verify_lists(dev));
126 return 0;
127}
128
129int
130i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
131 struct drm_file *file)
132{
133 struct drm_i915_private *dev_priv = dev->dev_private;
134 struct drm_i915_gem_get_aperture *args = data;
135 struct i915_gtt *ggtt = &dev_priv->gtt;
136 struct i915_vma *vma;
137 size_t pinned;
138
139 pinned = 0;
140 mutex_lock(&dev->struct_mutex);
141 list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
142 if (vma->pin_count)
143 pinned += vma->node.size;
144 list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
145 if (vma->pin_count)
146 pinned += vma->node.size;
147 mutex_unlock(&dev->struct_mutex);
148
149 args->aper_size = dev_priv->gtt.base.total;
150 args->aper_available_size = args->aper_size - pinned;
151
152 return 0;
153}
154
155static int
156i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
157{
158 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
159 char *vaddr = obj->phys_handle->vaddr;
160 struct sg_table *st;
161 struct scatterlist *sg;
162 int i;
163
164 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
165 return -EINVAL;
166
167 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
168 struct page *page;
169 char *src;
170
171 page = shmem_read_mapping_page(mapping, i);
172 if (IS_ERR(page))
173 return PTR_ERR(page);
174
175 src = kmap_atomic(page);
176 memcpy(vaddr, src, PAGE_SIZE);
177 drm_clflush_virt_range(vaddr, PAGE_SIZE);
178 kunmap_atomic(src);
179
180 put_page(page);
181 vaddr += PAGE_SIZE;
182 }
183
184 i915_gem_chipset_flush(obj->base.dev);
185
186 st = kmalloc(sizeof(*st), GFP_KERNEL);
187 if (st == NULL)
188 return -ENOMEM;
189
190 if (sg_alloc_table(st, 1, GFP_KERNEL)) {
191 kfree(st);
192 return -ENOMEM;
193 }
194
195 sg = st->sgl;
196 sg->offset = 0;
197 sg->length = obj->base.size;
198
199 sg_dma_address(sg) = obj->phys_handle->busaddr;
200 sg_dma_len(sg) = obj->base.size;
201
202 obj->pages = st;
203 return 0;
204}
205
206static void
207i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
208{
209 int ret;
210
211 BUG_ON(obj->madv == __I915_MADV_PURGED);
212
213 ret = i915_gem_object_set_to_cpu_domain(obj, true);
214 if (ret) {
215 /* In the event of a disaster, abandon all caches and
216 * hope for the best.
217 */
218 WARN_ON(ret != -EIO);
219 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
220 }
221
222 if (obj->madv == I915_MADV_DONTNEED)
223 obj->dirty = 0;
224
225 if (obj->dirty) {
226 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
227 char *vaddr = obj->phys_handle->vaddr;
228 int i;
229
230 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
231 struct page *page;
232 char *dst;
233
234 page = shmem_read_mapping_page(mapping, i);
235 if (IS_ERR(page))
236 continue;
237
238 dst = kmap_atomic(page);
239 drm_clflush_virt_range(vaddr, PAGE_SIZE);
240 memcpy(dst, vaddr, PAGE_SIZE);
241 kunmap_atomic(dst);
242
243 set_page_dirty(page);
244 if (obj->madv == I915_MADV_WILLNEED)
245 mark_page_accessed(page);
246 put_page(page);
247 vaddr += PAGE_SIZE;
248 }
249 obj->dirty = 0;
250 }
251
252 sg_free_table(obj->pages);
253 kfree(obj->pages);
254}
255
256static void
257i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
258{
259 drm_pci_free(obj->base.dev, obj->phys_handle);
260}
261
262static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
263 .get_pages = i915_gem_object_get_pages_phys,
264 .put_pages = i915_gem_object_put_pages_phys,
265 .release = i915_gem_object_release_phys,
266};
267
268static int
269drop_pages(struct drm_i915_gem_object *obj)
270{
271 struct i915_vma *vma, *next;
272 int ret;
273
274 drm_gem_object_reference(&obj->base);
275 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
276 if (i915_vma_unbind(vma))
277 break;
278
279 ret = i915_gem_object_put_pages(obj);
280 drm_gem_object_unreference(&obj->base);
281
282 return ret;
283}
284
285int
286i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
287 int align)
288{
289 drm_dma_handle_t *phys;
290 int ret;
291
292 if (obj->phys_handle) {
293 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
294 return -EBUSY;
295
296 return 0;
297 }
298
299 if (obj->madv != I915_MADV_WILLNEED)
300 return -EFAULT;
301
302 if (obj->base.filp == NULL)
303 return -EINVAL;
304
305 ret = drop_pages(obj);
306 if (ret)
307 return ret;
308
309 /* create a new object */
310 phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
311 if (!phys)
312 return -ENOMEM;
313
314 obj->phys_handle = phys;
315 obj->ops = &i915_gem_phys_ops;
316
317 return i915_gem_object_get_pages(obj);
318}
319
320static int
321i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
322 struct drm_i915_gem_pwrite *args,
323 struct drm_file *file_priv)
324{
325 struct drm_device *dev = obj->base.dev;
326 void *vaddr = obj->phys_handle->vaddr + args->offset;
327 char __user *user_data = to_user_ptr(args->data_ptr);
328 int ret = 0;
329
330 /* We manually control the domain here and pretend that it
331 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
332 */
333 ret = i915_gem_object_wait_rendering(obj, false);
334 if (ret)
335 return ret;
336
337 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
338 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
339 unsigned long unwritten;
340
341 /* The physical object once assigned is fixed for the lifetime
342 * of the obj, so we can safely drop the lock and continue
343 * to access vaddr.
344 */
345 mutex_unlock(&dev->struct_mutex);
346 unwritten = copy_from_user(vaddr, user_data, args->size);
347 mutex_lock(&dev->struct_mutex);
348 if (unwritten) {
349 ret = -EFAULT;
350 goto out;
351 }
352 }
353
354 drm_clflush_virt_range(vaddr, args->size);
355 i915_gem_chipset_flush(dev);
356
357out:
358 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
359 return ret;
360}
361
362void *i915_gem_object_alloc(struct drm_device *dev)
363{
364 struct drm_i915_private *dev_priv = dev->dev_private;
365 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
366}
367
368void i915_gem_object_free(struct drm_i915_gem_object *obj)
369{
370 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
371 kmem_cache_free(dev_priv->objects, obj);
372}
373
374static int
375i915_gem_create(struct drm_file *file,
376 struct drm_device *dev,
377 uint64_t size,
378 uint32_t *handle_p)
379{
380 struct drm_i915_gem_object *obj;
381 int ret;
382 u32 handle;
383
384 size = roundup(size, PAGE_SIZE);
385 if (size == 0)
386 return -EINVAL;
387
388 /* Allocate the new object */
389 obj = i915_gem_alloc_object(dev, size);
390 if (obj == NULL)
391 return -ENOMEM;
392
393 ret = drm_gem_handle_create(file, &obj->base, &handle);
394 /* drop reference from allocate - handle holds it now */
395 drm_gem_object_unreference_unlocked(&obj->base);
396 if (ret)
397 return ret;
398
399 *handle_p = handle;
400 return 0;
401}
402
403int
404i915_gem_dumb_create(struct drm_file *file,
405 struct drm_device *dev,
406 struct drm_mode_create_dumb *args)
407{
408 /* have to work out size/pitch and return them */
409 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
410 args->size = args->pitch * args->height;
411 return i915_gem_create(file, dev,
412 args->size, &args->handle);
413}
414
415/**
416 * Creates a new mm object and returns a handle to it.
417 */
418int
419i915_gem_create_ioctl(struct drm_device *dev, void *data,
420 struct drm_file *file)
421{
422 struct drm_i915_gem_create *args = data;
423
424 return i915_gem_create(file, dev,
425 args->size, &args->handle);
426}
427
428static inline int
429__copy_to_user_swizzled(char __user *cpu_vaddr,
430 const char *gpu_vaddr, int gpu_offset,
431 int length)
432{
433 int ret, cpu_offset = 0;
434
435 while (length > 0) {
436 int cacheline_end = ALIGN(gpu_offset + 1, 64);
437 int this_length = min(cacheline_end - gpu_offset, length);
438 int swizzled_gpu_offset = gpu_offset ^ 64;
439
440 ret = __copy_to_user(cpu_vaddr + cpu_offset,
441 gpu_vaddr + swizzled_gpu_offset,
442 this_length);
443 if (ret)
444 return ret + length;
445
446 cpu_offset += this_length;
447 gpu_offset += this_length;
448 length -= this_length;
449 }
450
451 return 0;
452}
453
454static inline int
455__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
456 const char __user *cpu_vaddr,
457 int length)
458{
459 int ret, cpu_offset = 0;
460
461 while (length > 0) {
462 int cacheline_end = ALIGN(gpu_offset + 1, 64);
463 int this_length = min(cacheline_end - gpu_offset, length);
464 int swizzled_gpu_offset = gpu_offset ^ 64;
465
466 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
467 cpu_vaddr + cpu_offset,
468 this_length);
469 if (ret)
470 return ret + length;
471
472 cpu_offset += this_length;
473 gpu_offset += this_length;
474 length -= this_length;
475 }
476
477 return 0;
478}
479
480/*
481 * Pins the specified object's pages and synchronizes the object with
482 * GPU accesses. Sets needs_clflush to non-zero if the caller should
483 * flush the object from the CPU cache.
484 */
485int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
486 int *needs_clflush)
487{
488 int ret;
489
490 *needs_clflush = 0;
491
492 if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
493 return -EINVAL;
494
495 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
496 /* If we're not in the cpu read domain, set ourself into the gtt
497 * read domain and manually flush cachelines (if required). This
498 * optimizes for the case when the gpu will dirty the data
499 * anyway again before the next pread happens. */
500 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
501 obj->cache_level);
502 ret = i915_gem_object_wait_rendering(obj, true);
503 if (ret)
504 return ret;
505 }
506
507 ret = i915_gem_object_get_pages(obj);
508 if (ret)
509 return ret;
510
511 i915_gem_object_pin_pages(obj);
512
513 return ret;
514}
515
516/* Per-page copy function for the shmem pread fastpath.
517 * Flushes invalid cachelines before reading the target if
518 * needs_clflush is set. */
519static int
520shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
521 char __user *user_data,
522 bool page_do_bit17_swizzling, bool needs_clflush)
523{
524 char *vaddr;
525 int ret;
526
527 if (unlikely(page_do_bit17_swizzling))
528 return -EINVAL;
529
530 vaddr = kmap_atomic(page);
531 if (needs_clflush)
532 drm_clflush_virt_range(vaddr + shmem_page_offset,
533 page_length);
534 ret = __copy_to_user_inatomic(user_data,
535 vaddr + shmem_page_offset,
536 page_length);
537 kunmap_atomic(vaddr);
538
539 return ret ? -EFAULT : 0;
540}
541
542static void
543shmem_clflush_swizzled_range(char *addr, unsigned long length,
544 bool swizzled)
545{
546 if (unlikely(swizzled)) {
547 unsigned long start = (unsigned long) addr;
548 unsigned long end = (unsigned long) addr + length;
549
550 /* For swizzling simply ensure that we always flush both
551 * channels. Lame, but simple and it works. Swizzled
552 * pwrite/pread is far from a hotpath - current userspace
553 * doesn't use it at all. */
554 start = round_down(start, 128);
555 end = round_up(end, 128);
556
557 drm_clflush_virt_range((void *)start, end - start);
558 } else {
559 drm_clflush_virt_range(addr, length);
560 }
561
562}
563
564/* Only difference to the fast-path function is that this can handle bit17
565 * and uses non-atomic copy and kmap functions. */
566static int
567shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
568 char __user *user_data,
569 bool page_do_bit17_swizzling, bool needs_clflush)
570{
571 char *vaddr;
572 int ret;
573
574 vaddr = kmap(page);
575 if (needs_clflush)
576 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
577 page_length,
578 page_do_bit17_swizzling);
579
580 if (page_do_bit17_swizzling)
581 ret = __copy_to_user_swizzled(user_data,
582 vaddr, shmem_page_offset,
583 page_length);
584 else
585 ret = __copy_to_user(user_data,
586 vaddr + shmem_page_offset,
587 page_length);
588 kunmap(page);
589
590 return ret ? - EFAULT : 0;
591}
592
593static int
594i915_gem_shmem_pread(struct drm_device *dev,
595 struct drm_i915_gem_object *obj,
596 struct drm_i915_gem_pread *args,
597 struct drm_file *file)
598{
599 char __user *user_data;
600 ssize_t remain;
601 loff_t offset;
602 int shmem_page_offset, page_length, ret = 0;
603 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
604 int prefaulted = 0;
605 int needs_clflush = 0;
606 struct sg_page_iter sg_iter;
607
608 user_data = to_user_ptr(args->data_ptr);
609 remain = args->size;
610
611 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
612
613 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
614 if (ret)
615 return ret;
616
617 offset = args->offset;
618
619 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
620 offset >> PAGE_SHIFT) {
621 struct page *page = sg_page_iter_page(&sg_iter);
622
623 if (remain <= 0)
624 break;
625
626 /* Operation in this page
627 *
628 * shmem_page_offset = offset within page in shmem file
629 * page_length = bytes to copy for this page
630 */
631 shmem_page_offset = offset_in_page(offset);
632 page_length = remain;
633 if ((shmem_page_offset + page_length) > PAGE_SIZE)
634 page_length = PAGE_SIZE - shmem_page_offset;
635
636 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
637 (page_to_phys(page) & (1 << 17)) != 0;
638
639 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
640 user_data, page_do_bit17_swizzling,
641 needs_clflush);
642 if (ret == 0)
643 goto next_page;
644
645 mutex_unlock(&dev->struct_mutex);
646
647 if (likely(!i915.prefault_disable) && !prefaulted) {
648 ret = fault_in_multipages_writeable(user_data, remain);
649 /* Userspace is tricking us, but we've already clobbered
650 * its pages with the prefault and promised to write the
651 * data up to the first fault. Hence ignore any errors
652 * and just continue. */
653 (void)ret;
654 prefaulted = 1;
655 }
656
657 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
658 user_data, page_do_bit17_swizzling,
659 needs_clflush);
660
661 mutex_lock(&dev->struct_mutex);
662
663 if (ret)
664 goto out;
665
666next_page:
667 remain -= page_length;
668 user_data += page_length;
669 offset += page_length;
670 }
671
672out:
673 i915_gem_object_unpin_pages(obj);
674
675 return ret;
676}
677
678/**
679 * Reads data from the object referenced by handle.
680 *
681 * On error, the contents of *data are undefined.
682 */
683int
684i915_gem_pread_ioctl(struct drm_device *dev, void *data,
685 struct drm_file *file)
686{
687 struct drm_i915_gem_pread *args = data;
688 struct drm_i915_gem_object *obj;
689 int ret = 0;
690
691 if (args->size == 0)
692 return 0;
693
694 if (!access_ok(VERIFY_WRITE,
695 to_user_ptr(args->data_ptr),
696 args->size))
697 return -EFAULT;
698
699 ret = i915_mutex_lock_interruptible(dev);
700 if (ret)
701 return ret;
702
703 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
704 if (&obj->base == NULL) {
705 ret = -ENOENT;
706 goto unlock;
707 }
708
709 /* Bounds check source. */
710 if (args->offset > obj->base.size ||
711 args->size > obj->base.size - args->offset) {
712 ret = -EINVAL;
713 goto out;
714 }
715
716 /* prime objects have no backing filp to GEM pread/pwrite
717 * pages from.
718 */
719 if (!obj->base.filp) {
720 ret = -EINVAL;
721 goto out;
722 }
723
724 trace_i915_gem_object_pread(obj, args->offset, args->size);
725
726 ret = i915_gem_shmem_pread(dev, obj, args, file);
727
728out:
729 drm_gem_object_unreference(&obj->base);
730unlock:
731 mutex_unlock(&dev->struct_mutex);
732 return ret;
733}
734
735/* This is the fast write path which cannot handle
736 * page faults in the source data
737 */
738
739static inline int
740fast_user_write(struct io_mapping *mapping,
741 loff_t page_base, int page_offset,
742 char __user *user_data,
743 int length)
744{
745 void __iomem *vaddr_atomic;
746 void *vaddr;
747 unsigned long unwritten;
748
749 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
750 /* We can use the cpu mem copy function because this is X86. */
751 vaddr = (void __force*)vaddr_atomic + page_offset;
752 unwritten = __copy_from_user_inatomic_nocache(vaddr,
753 user_data, length);
754 io_mapping_unmap_atomic(vaddr_atomic);
755 return unwritten;
756}
757
758/**
759 * This is the fast pwrite path, where we copy the data directly from the
760 * user into the GTT, uncached.
761 */
762static int
763i915_gem_gtt_pwrite_fast(struct drm_device *dev,
764 struct drm_i915_gem_object *obj,
765 struct drm_i915_gem_pwrite *args,
766 struct drm_file *file)
767{
768 struct drm_i915_private *dev_priv = dev->dev_private;
769 ssize_t remain;
770 loff_t offset, page_base;
771 char __user *user_data;
772 int page_offset, page_length, ret;
773
774 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
775 if (ret)
776 goto out;
777
778 ret = i915_gem_object_set_to_gtt_domain(obj, true);
779 if (ret)
780 goto out_unpin;
781
782 ret = i915_gem_object_put_fence(obj);
783 if (ret)
784 goto out_unpin;
785
786 user_data = to_user_ptr(args->data_ptr);
787 remain = args->size;
788
789 offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
790
791 intel_fb_obj_invalidate(obj, ORIGIN_GTT);
792
793 while (remain > 0) {
794 /* Operation in this page
795 *
796 * page_base = page offset within aperture
797 * page_offset = offset within page
798 * page_length = bytes to copy for this page
799 */
800 page_base = offset & PAGE_MASK;
801 page_offset = offset_in_page(offset);
802 page_length = remain;
803 if ((page_offset + remain) > PAGE_SIZE)
804 page_length = PAGE_SIZE - page_offset;
805
806 /* If we get a fault while copying data, then (presumably) our
807 * source page isn't available. Return the error and we'll
808 * retry in the slow path.
809 */
810 if (fast_user_write(dev_priv->gtt.mappable, page_base,
811 page_offset, user_data, page_length)) {
812 ret = -EFAULT;
813 goto out_flush;
814 }
815
816 remain -= page_length;
817 user_data += page_length;
818 offset += page_length;
819 }
820
821out_flush:
822 intel_fb_obj_flush(obj, false, ORIGIN_GTT);
823out_unpin:
824 i915_gem_object_ggtt_unpin(obj);
825out:
826 return ret;
827}
828
829/* Per-page copy function for the shmem pwrite fastpath.
830 * Flushes invalid cachelines before writing to the target if
831 * needs_clflush_before is set and flushes out any written cachelines after
832 * writing if needs_clflush is set. */
833static int
834shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
835 char __user *user_data,
836 bool page_do_bit17_swizzling,
837 bool needs_clflush_before,
838 bool needs_clflush_after)
839{
840 char *vaddr;
841 int ret;
842
843 if (unlikely(page_do_bit17_swizzling))
844 return -EINVAL;
845
846 vaddr = kmap_atomic(page);
847 if (needs_clflush_before)
848 drm_clflush_virt_range(vaddr + shmem_page_offset,
849 page_length);
850 ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
851 user_data, page_length);
852 if (needs_clflush_after)
853 drm_clflush_virt_range(vaddr + shmem_page_offset,
854 page_length);
855 kunmap_atomic(vaddr);
856
857 return ret ? -EFAULT : 0;
858}
859
860/* Only difference to the fast-path function is that this can handle bit17
861 * and uses non-atomic copy and kmap functions. */
862static int
863shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
864 char __user *user_data,
865 bool page_do_bit17_swizzling,
866 bool needs_clflush_before,
867 bool needs_clflush_after)
868{
869 char *vaddr;
870 int ret;
871
872 vaddr = kmap(page);
873 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
874 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
875 page_length,
876 page_do_bit17_swizzling);
877 if (page_do_bit17_swizzling)
878 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
879 user_data,
880 page_length);
881 else
882 ret = __copy_from_user(vaddr + shmem_page_offset,
883 user_data,
884 page_length);
885 if (needs_clflush_after)
886 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
887 page_length,
888 page_do_bit17_swizzling);
889 kunmap(page);
890
891 return ret ? -EFAULT : 0;
892}
893
894static int
895i915_gem_shmem_pwrite(struct drm_device *dev,
896 struct drm_i915_gem_object *obj,
897 struct drm_i915_gem_pwrite *args,
898 struct drm_file *file)
899{
900 ssize_t remain;
901 loff_t offset;
902 char __user *user_data;
903 int shmem_page_offset, page_length, ret = 0;
904 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
905 int hit_slowpath = 0;
906 int needs_clflush_after = 0;
907 int needs_clflush_before = 0;
908 struct sg_page_iter sg_iter;
909
910 user_data = to_user_ptr(args->data_ptr);
911 remain = args->size;
912
913 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
914
915 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
916 /* If we're not in the cpu write domain, set ourself into the gtt
917 * write domain and manually flush cachelines (if required). This
918 * optimizes for the case when the gpu will use the data
919 * right away and we therefore have to clflush anyway. */
920 needs_clflush_after = cpu_write_needs_clflush(obj);
921 ret = i915_gem_object_wait_rendering(obj, false);
922 if (ret)
923 return ret;
924 }
925 /* Same trick applies to invalidate partially written cachelines read
926 * before writing. */
927 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
928 needs_clflush_before =
929 !cpu_cache_is_coherent(dev, obj->cache_level);
930
931 ret = i915_gem_object_get_pages(obj);
932 if (ret)
933 return ret;
934
935 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
936
937 i915_gem_object_pin_pages(obj);
938
939 offset = args->offset;
940 obj->dirty = 1;
941
942 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
943 offset >> PAGE_SHIFT) {
944 struct page *page = sg_page_iter_page(&sg_iter);
945 int partial_cacheline_write;
946
947 if (remain <= 0)
948 break;
949
950 /* Operation in this page
951 *
952 * shmem_page_offset = offset within page in shmem file
953 * page_length = bytes to copy for this page
954 */
955 shmem_page_offset = offset_in_page(offset);
956
957 page_length = remain;
958 if ((shmem_page_offset + page_length) > PAGE_SIZE)
959 page_length = PAGE_SIZE - shmem_page_offset;
960
961 /* If we don't overwrite a cacheline completely we need to be
962 * careful to have up-to-date data by first clflushing. Don't
963 * overcomplicate things and flush the entire patch. */
964 partial_cacheline_write = needs_clflush_before &&
965 ((shmem_page_offset | page_length)
966 & (boot_cpu_data.x86_clflush_size - 1));
967
968 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
969 (page_to_phys(page) & (1 << 17)) != 0;
970
971 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
972 user_data, page_do_bit17_swizzling,
973 partial_cacheline_write,
974 needs_clflush_after);
975 if (ret == 0)
976 goto next_page;
977
978 hit_slowpath = 1;
979 mutex_unlock(&dev->struct_mutex);
980 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
981 user_data, page_do_bit17_swizzling,
982 partial_cacheline_write,
983 needs_clflush_after);
984
985 mutex_lock(&dev->struct_mutex);
986
987 if (ret)
988 goto out;
989
990next_page:
991 remain -= page_length;
992 user_data += page_length;
993 offset += page_length;
994 }
995
996out:
997 i915_gem_object_unpin_pages(obj);
998
999 if (hit_slowpath) {
1000 /*
1001 * Fixup: Flush cpu caches in case we didn't flush the dirty
1002 * cachelines in-line while writing and the object moved
1003 * out of the cpu write domain while we've dropped the lock.
1004 */
1005 if (!needs_clflush_after &&
1006 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1007 if (i915_gem_clflush_object(obj, obj->pin_display))
1008 needs_clflush_after = true;
1009 }
1010 }
1011
1012 if (needs_clflush_after)
1013 i915_gem_chipset_flush(dev);
1014 else
1015 obj->cache_dirty = true;
1016
1017 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1018 return ret;
1019}
1020
1021/**
1022 * Writes data to the object referenced by handle.
1023 *
1024 * On error, the contents of the buffer that were to be modified are undefined.
1025 */
1026int
1027i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1028 struct drm_file *file)
1029{
1030 struct drm_i915_private *dev_priv = dev->dev_private;
1031 struct drm_i915_gem_pwrite *args = data;
1032 struct drm_i915_gem_object *obj;
1033 int ret;
1034
1035 if (args->size == 0)
1036 return 0;
1037
1038 if (!access_ok(VERIFY_READ,
1039 to_user_ptr(args->data_ptr),
1040 args->size))
1041 return -EFAULT;
1042
1043 if (likely(!i915.prefault_disable)) {
1044 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1045 args->size);
1046 if (ret)
1047 return -EFAULT;
1048 }
1049
1050 intel_runtime_pm_get(dev_priv);
1051
1052 ret = i915_mutex_lock_interruptible(dev);
1053 if (ret)
1054 goto put_rpm;
1055
1056 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1057 if (&obj->base == NULL) {
1058 ret = -ENOENT;
1059 goto unlock;
1060 }
1061
1062 /* Bounds check destination. */
1063 if (args->offset > obj->base.size ||
1064 args->size > obj->base.size - args->offset) {
1065 ret = -EINVAL;
1066 goto out;
1067 }
1068
1069 /* prime objects have no backing filp to GEM pread/pwrite
1070 * pages from.
1071 */
1072 if (!obj->base.filp) {
1073 ret = -EINVAL;
1074 goto out;
1075 }
1076
1077 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1078
1079 ret = -EFAULT;
1080 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1081 * it would end up going through the fenced access, and we'll get
1082 * different detiling behavior between reading and writing.
1083 * pread/pwrite currently are reading and writing from the CPU
1084 * perspective, requiring manual detiling by the client.
1085 */
1086 if (obj->tiling_mode == I915_TILING_NONE &&
1087 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1088 cpu_write_needs_clflush(obj)) {
1089 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1090 /* Note that the gtt paths might fail with non-page-backed user
1091 * pointers (e.g. gtt mappings when moving data between
1092 * textures). Fallback to the shmem path in that case. */
1093 }
1094
1095 if (ret == -EFAULT || ret == -ENOSPC) {
1096 if (obj->phys_handle)
1097 ret = i915_gem_phys_pwrite(obj, args, file);
1098 else
1099 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1100 }
1101
1102out:
1103 drm_gem_object_unreference(&obj->base);
1104unlock:
1105 mutex_unlock(&dev->struct_mutex);
1106put_rpm:
1107 intel_runtime_pm_put(dev_priv);
1108
1109 return ret;
1110}
1111
1112int
1113i915_gem_check_wedge(struct i915_gpu_error *error,
1114 bool interruptible)
1115{
1116 if (i915_reset_in_progress(error)) {
1117 /* Non-interruptible callers can't handle -EAGAIN, hence return
1118 * -EIO unconditionally for these. */
1119 if (!interruptible)
1120 return -EIO;
1121
1122 /* Recovery complete, but the reset failed ... */
1123 if (i915_terminally_wedged(error))
1124 return -EIO;
1125
1126 /*
1127 * Check if GPU Reset is in progress - we need intel_ring_begin
1128 * to work properly to reinit the hw state while the gpu is
1129 * still marked as reset-in-progress. Handle this with a flag.
1130 */
1131 if (!error->reload_in_reset)
1132 return -EAGAIN;
1133 }
1134
1135 return 0;
1136}
1137
1138static void fake_irq(unsigned long data)
1139{
1140 wake_up_process((struct task_struct *)data);
1141}
1142
1143static bool missed_irq(struct drm_i915_private *dev_priv,
1144 struct intel_engine_cs *ring)
1145{
1146 return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
1147}
1148
1149static unsigned long local_clock_us(unsigned *cpu)
1150{
1151 unsigned long t;
1152
1153 /* Cheaply and approximately convert from nanoseconds to microseconds.
1154 * The result and subsequent calculations are also defined in the same
1155 * approximate microseconds units. The principal source of timing
1156 * error here is from the simple truncation.
1157 *
1158 * Note that local_clock() is only defined wrt to the current CPU;
1159 * the comparisons are no longer valid if we switch CPUs. Instead of
1160 * blocking preemption for the entire busywait, we can detect the CPU
1161 * switch and use that as indicator of system load and a reason to
1162 * stop busywaiting, see busywait_stop().
1163 */
1164 *cpu = get_cpu();
1165 t = local_clock() >> 10;
1166 put_cpu();
1167
1168 return t;
1169}
1170
1171static bool busywait_stop(unsigned long timeout, unsigned cpu)
1172{
1173 unsigned this_cpu;
1174
1175 if (time_after(local_clock_us(&this_cpu), timeout))
1176 return true;
1177
1178 return this_cpu != cpu;
1179}
1180
1181static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
1182{
1183 unsigned long timeout;
1184 unsigned cpu;
1185
1186 /* When waiting for high frequency requests, e.g. during synchronous
1187 * rendering split between the CPU and GPU, the finite amount of time
1188 * required to set up the irq and wait upon it limits the response
1189 * rate. By busywaiting on the request completion for a short while we
1190 * can service the high frequency waits as quick as possible. However,
1191 * if it is a slow request, we want to sleep as quickly as possible.
1192 * The tradeoff between waiting and sleeping is roughly the time it
1193 * takes to sleep on a request, on the order of a microsecond.
1194 */
1195
1196 if (req->ring->irq_refcount)
1197 return -EBUSY;
1198
1199 /* Only spin if we know the GPU is processing this request */
1200 if (!i915_gem_request_started(req, true))
1201 return -EAGAIN;
1202
1203 timeout = local_clock_us(&cpu) + 5;
1204 while (!need_resched()) {
1205 if (i915_gem_request_completed(req, true))
1206 return 0;
1207
1208 if (signal_pending_state(state, current))
1209 break;
1210
1211 if (busywait_stop(timeout, cpu))
1212 break;
1213
1214 cpu_relax_lowlatency();
1215 }
1216
1217 if (i915_gem_request_completed(req, false))
1218 return 0;
1219
1220 return -EAGAIN;
1221}
1222
1223/**
1224 * __i915_wait_request - wait until execution of request has finished
1225 * @req: duh!
1226 * @reset_counter: reset sequence associated with the given request
1227 * @interruptible: do an interruptible wait (normally yes)
1228 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1229 *
1230 * Note: It is of utmost importance that the passed in seqno and reset_counter
1231 * values have been read by the caller in an smp safe manner. Where read-side
1232 * locks are involved, it is sufficient to read the reset_counter before
1233 * unlocking the lock that protects the seqno. For lockless tricks, the
1234 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1235 * inserted.
1236 *
1237 * Returns 0 if the request was found within the alloted time. Else returns the
1238 * errno with remaining time filled in timeout argument.
1239 */
1240int __i915_wait_request(struct drm_i915_gem_request *req,
1241 unsigned reset_counter,
1242 bool interruptible,
1243 s64 *timeout,
1244 struct intel_rps_client *rps)
1245{
1246 struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
1247 struct drm_device *dev = ring->dev;
1248 struct drm_i915_private *dev_priv = dev->dev_private;
1249 const bool irq_test_in_progress =
1250 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1251 int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1252 DEFINE_WAIT(wait);
1253 unsigned long timeout_expire;
1254 s64 before = 0; /* Only to silence a compiler warning. */
1255 int ret;
1256
1257 WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1258
1259 if (list_empty(&req->list))
1260 return 0;
1261
1262 if (i915_gem_request_completed(req, true))
1263 return 0;
1264
1265 timeout_expire = 0;
1266 if (timeout) {
1267 if (WARN_ON(*timeout < 0))
1268 return -EINVAL;
1269
1270 if (*timeout == 0)
1271 return -ETIME;
1272
1273 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
1274
1275 /*
1276 * Record current time in case interrupted by signal, or wedged.
1277 */
1278 before = ktime_get_raw_ns();
1279 }
1280
1281 if (INTEL_INFO(dev_priv)->gen >= 6)
1282 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
1283
1284 trace_i915_gem_request_wait_begin(req);
1285
1286 /* Optimistic spin for the next jiffie before touching IRQs */
1287 ret = __i915_spin_request(req, state);
1288 if (ret == 0)
1289 goto out;
1290
1291 if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
1292 ret = -ENODEV;
1293 goto out;
1294 }
1295
1296 for (;;) {
1297 struct timer_list timer;
1298
1299 prepare_to_wait(&ring->irq_queue, &wait, state);
1300
1301 /* We need to check whether any gpu reset happened in between
1302 * the caller grabbing the seqno and now ... */
1303 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1304 /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1305 * is truely gone. */
1306 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1307 if (ret == 0)
1308 ret = -EAGAIN;
1309 break;
1310 }
1311
1312 if (i915_gem_request_completed(req, false)) {
1313 ret = 0;
1314 break;
1315 }
1316
1317 if (signal_pending_state(state, current)) {
1318 ret = -ERESTARTSYS;
1319 break;
1320 }
1321
1322 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1323 ret = -ETIME;
1324 break;
1325 }
1326
1327 timer.function = NULL;
1328 if (timeout || missed_irq(dev_priv, ring)) {
1329 unsigned long expire;
1330
1331 setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1332 expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1333 mod_timer(&timer, expire);
1334 }
1335
1336 io_schedule();
1337
1338 if (timer.function) {
1339 del_singleshot_timer_sync(&timer);
1340 destroy_timer_on_stack(&timer);
1341 }
1342 }
1343 if (!irq_test_in_progress)
1344 ring->irq_put(ring);
1345
1346 finish_wait(&ring->irq_queue, &wait);
1347
1348out:
1349 trace_i915_gem_request_wait_end(req);
1350
1351 if (timeout) {
1352 s64 tres = *timeout - (ktime_get_raw_ns() - before);
1353
1354 *timeout = tres < 0 ? 0 : tres;
1355
1356 /*
1357 * Apparently ktime isn't accurate enough and occasionally has a
1358 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1359 * things up to make the test happy. We allow up to 1 jiffy.
1360 *
1361 * This is a regrssion from the timespec->ktime conversion.
1362 */
1363 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1364 *timeout = 0;
1365 }
1366
1367 return ret;
1368}
1369
1370int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
1371 struct drm_file *file)
1372{
1373 struct drm_i915_private *dev_private;
1374 struct drm_i915_file_private *file_priv;
1375
1376 WARN_ON(!req || !file || req->file_priv);
1377
1378 if (!req || !file)
1379 return -EINVAL;
1380
1381 if (req->file_priv)
1382 return -EINVAL;
1383
1384 dev_private = req->ring->dev->dev_private;
1385 file_priv = file->driver_priv;
1386
1387 spin_lock(&file_priv->mm.lock);
1388 req->file_priv = file_priv;
1389 list_add_tail(&req->client_list, &file_priv->mm.request_list);
1390 spin_unlock(&file_priv->mm.lock);
1391
1392 req->pid = get_pid(task_pid(current));
1393
1394 return 0;
1395}
1396
1397static inline void
1398i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1399{
1400 struct drm_i915_file_private *file_priv = request->file_priv;
1401
1402 if (!file_priv)
1403 return;
1404
1405 spin_lock(&file_priv->mm.lock);
1406 list_del(&request->client_list);
1407 request->file_priv = NULL;
1408 spin_unlock(&file_priv->mm.lock);
1409
1410 put_pid(request->pid);
1411 request->pid = NULL;
1412}
1413
1414static void i915_gem_request_retire(struct drm_i915_gem_request *request)
1415{
1416 trace_i915_gem_request_retire(request);
1417
1418 /* We know the GPU must have read the request to have
1419 * sent us the seqno + interrupt, so use the position
1420 * of tail of the request to update the last known position
1421 * of the GPU head.
1422 *
1423 * Note this requires that we are always called in request
1424 * completion order.
1425 */
1426 request->ringbuf->last_retired_head = request->postfix;
1427
1428 list_del_init(&request->list);
1429 i915_gem_request_remove_from_client(request);
1430
1431 i915_gem_request_unreference(request);
1432}
1433
1434static void
1435__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
1436{
1437 struct intel_engine_cs *engine = req->ring;
1438 struct drm_i915_gem_request *tmp;
1439
1440 lockdep_assert_held(&engine->dev->struct_mutex);
1441
1442 if (list_empty(&req->list))
1443 return;
1444
1445 do {
1446 tmp = list_first_entry(&engine->request_list,
1447 typeof(*tmp), list);
1448
1449 i915_gem_request_retire(tmp);
1450 } while (tmp != req);
1451
1452 WARN_ON(i915_verify_lists(engine->dev));
1453}
1454
1455/**
1456 * Waits for a request to be signaled, and cleans up the
1457 * request and object lists appropriately for that event.
1458 */
1459int
1460i915_wait_request(struct drm_i915_gem_request *req)
1461{
1462 struct drm_device *dev;
1463 struct drm_i915_private *dev_priv;
1464 bool interruptible;
1465 int ret;
1466
1467 BUG_ON(req == NULL);
1468
1469 dev = req->ring->dev;
1470 dev_priv = dev->dev_private;
1471 interruptible = dev_priv->mm.interruptible;
1472
1473 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1474
1475 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1476 if (ret)
1477 return ret;
1478
1479 ret = __i915_wait_request(req,
1480 atomic_read(&dev_priv->gpu_error.reset_counter),
1481 interruptible, NULL, NULL);
1482 if (ret)
1483 return ret;
1484
1485 __i915_gem_request_retire__upto(req);
1486 return 0;
1487}
1488
1489/**
1490 * Ensures that all rendering to the object has completed and the object is
1491 * safe to unbind from the GTT or access from the CPU.
1492 */
1493int
1494i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1495 bool readonly)
1496{
1497 int ret, i;
1498
1499 if (!obj->active)
1500 return 0;
1501
1502 if (readonly) {
1503 if (obj->last_write_req != NULL) {
1504 ret = i915_wait_request(obj->last_write_req);
1505 if (ret)
1506 return ret;
1507
1508 i = obj->last_write_req->ring->id;
1509 if (obj->last_read_req[i] == obj->last_write_req)
1510 i915_gem_object_retire__read(obj, i);
1511 else
1512 i915_gem_object_retire__write(obj);
1513 }
1514 } else {
1515 for (i = 0; i < I915_NUM_RINGS; i++) {
1516 if (obj->last_read_req[i] == NULL)
1517 continue;
1518
1519 ret = i915_wait_request(obj->last_read_req[i]);
1520 if (ret)
1521 return ret;
1522
1523 i915_gem_object_retire__read(obj, i);
1524 }
1525 RQ_BUG_ON(obj->active);
1526 }
1527
1528 return 0;
1529}
1530
1531static void
1532i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1533 struct drm_i915_gem_request *req)
1534{
1535 int ring = req->ring->id;
1536
1537 if (obj->last_read_req[ring] == req)
1538 i915_gem_object_retire__read(obj, ring);
1539 else if (obj->last_write_req == req)
1540 i915_gem_object_retire__write(obj);
1541
1542 __i915_gem_request_retire__upto(req);
1543}
1544
1545/* A nonblocking variant of the above wait. This is a highly dangerous routine
1546 * as the object state may change during this call.
1547 */
1548static __must_check int
1549i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1550 struct intel_rps_client *rps,
1551 bool readonly)
1552{
1553 struct drm_device *dev = obj->base.dev;
1554 struct drm_i915_private *dev_priv = dev->dev_private;
1555 struct drm_i915_gem_request *requests[I915_NUM_RINGS];
1556 unsigned reset_counter;
1557 int ret, i, n = 0;
1558
1559 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1560 BUG_ON(!dev_priv->mm.interruptible);
1561
1562 if (!obj->active)
1563 return 0;
1564
1565 ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1566 if (ret)
1567 return ret;
1568
1569 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1570
1571 if (readonly) {
1572 struct drm_i915_gem_request *req;
1573
1574 req = obj->last_write_req;
1575 if (req == NULL)
1576 return 0;
1577
1578 requests[n++] = i915_gem_request_reference(req);
1579 } else {
1580 for (i = 0; i < I915_NUM_RINGS; i++) {
1581 struct drm_i915_gem_request *req;
1582
1583 req = obj->last_read_req[i];
1584 if (req == NULL)
1585 continue;
1586
1587 requests[n++] = i915_gem_request_reference(req);
1588 }
1589 }
1590
1591 mutex_unlock(&dev->struct_mutex);
1592 for (i = 0; ret == 0 && i < n; i++)
1593 ret = __i915_wait_request(requests[i], reset_counter, true,
1594 NULL, rps);
1595 mutex_lock(&dev->struct_mutex);
1596
1597 for (i = 0; i < n; i++) {
1598 if (ret == 0)
1599 i915_gem_object_retire_request(obj, requests[i]);
1600 i915_gem_request_unreference(requests[i]);
1601 }
1602
1603 return ret;
1604}
1605
1606static struct intel_rps_client *to_rps_client(struct drm_file *file)
1607{
1608 struct drm_i915_file_private *fpriv = file->driver_priv;
1609 return &fpriv->rps;
1610}
1611
1612/**
1613 * Called when user space prepares to use an object with the CPU, either
1614 * through the mmap ioctl's mapping or a GTT mapping.
1615 */
1616int
1617i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1618 struct drm_file *file)
1619{
1620 struct drm_i915_gem_set_domain *args = data;
1621 struct drm_i915_gem_object *obj;
1622 uint32_t read_domains = args->read_domains;
1623 uint32_t write_domain = args->write_domain;
1624 int ret;
1625
1626 /* Only handle setting domains to types used by the CPU. */
1627 if (write_domain & I915_GEM_GPU_DOMAINS)
1628 return -EINVAL;
1629
1630 if (read_domains & I915_GEM_GPU_DOMAINS)
1631 return -EINVAL;
1632
1633 /* Having something in the write domain implies it's in the read
1634 * domain, and only that read domain. Enforce that in the request.
1635 */
1636 if (write_domain != 0 && read_domains != write_domain)
1637 return -EINVAL;
1638
1639 ret = i915_mutex_lock_interruptible(dev);
1640 if (ret)
1641 return ret;
1642
1643 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1644 if (&obj->base == NULL) {
1645 ret = -ENOENT;
1646 goto unlock;
1647 }
1648
1649 /* Try to flush the object off the GPU without holding the lock.
1650 * We will repeat the flush holding the lock in the normal manner
1651 * to catch cases where we are gazumped.
1652 */
1653 ret = i915_gem_object_wait_rendering__nonblocking(obj,
1654 to_rps_client(file),
1655 !write_domain);
1656 if (ret)
1657 goto unref;
1658
1659 if (read_domains & I915_GEM_DOMAIN_GTT)
1660 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1661 else
1662 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1663
1664 if (write_domain != 0)
1665 intel_fb_obj_invalidate(obj,
1666 write_domain == I915_GEM_DOMAIN_GTT ?
1667 ORIGIN_GTT : ORIGIN_CPU);
1668
1669unref:
1670 drm_gem_object_unreference(&obj->base);
1671unlock:
1672 mutex_unlock(&dev->struct_mutex);
1673 return ret;
1674}
1675
1676/**
1677 * Called when user space has done writes to this buffer
1678 */
1679int
1680i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1681 struct drm_file *file)
1682{
1683 struct drm_i915_gem_sw_finish *args = data;
1684 struct drm_i915_gem_object *obj;
1685 int ret = 0;
1686
1687 ret = i915_mutex_lock_interruptible(dev);
1688 if (ret)
1689 return ret;
1690
1691 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1692 if (&obj->base == NULL) {
1693 ret = -ENOENT;
1694 goto unlock;
1695 }
1696
1697 /* Pinned buffers may be scanout, so flush the cache */
1698 if (obj->pin_display)
1699 i915_gem_object_flush_cpu_write_domain(obj);
1700
1701 drm_gem_object_unreference(&obj->base);
1702unlock:
1703 mutex_unlock(&dev->struct_mutex);
1704 return ret;
1705}
1706
1707/**
1708 * Maps the contents of an object, returning the address it is mapped
1709 * into.
1710 *
1711 * While the mapping holds a reference on the contents of the object, it doesn't
1712 * imply a ref on the object itself.
1713 *
1714 * IMPORTANT:
1715 *
1716 * DRM driver writers who look a this function as an example for how to do GEM
1717 * mmap support, please don't implement mmap support like here. The modern way
1718 * to implement DRM mmap support is with an mmap offset ioctl (like
1719 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1720 * That way debug tooling like valgrind will understand what's going on, hiding
1721 * the mmap call in a driver private ioctl will break that. The i915 driver only
1722 * does cpu mmaps this way because we didn't know better.
1723 */
1724int
1725i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1726 struct drm_file *file)
1727{
1728 struct drm_i915_gem_mmap *args = data;
1729 struct drm_gem_object *obj;
1730 unsigned long addr;
1731
1732 if (args->flags & ~(I915_MMAP_WC))
1733 return -EINVAL;
1734
1735 if (args->flags & I915_MMAP_WC && !cpu_has_pat)
1736 return -ENODEV;
1737
1738 obj = drm_gem_object_lookup(dev, file, args->handle);
1739 if (obj == NULL)
1740 return -ENOENT;
1741
1742 /* prime objects have no backing filp to GEM mmap
1743 * pages from.
1744 */
1745 if (!obj->filp) {
1746 drm_gem_object_unreference_unlocked(obj);
1747 return -EINVAL;
1748 }
1749
1750 addr = vm_mmap(obj->filp, 0, args->size,
1751 PROT_READ | PROT_WRITE, MAP_SHARED,
1752 args->offset);
1753 if (args->flags & I915_MMAP_WC) {
1754 struct mm_struct *mm = current->mm;
1755 struct vm_area_struct *vma;
1756
1757 down_write(&mm->mmap_sem);
1758 vma = find_vma(mm, addr);
1759 if (vma)
1760 vma->vm_page_prot =
1761 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1762 else
1763 addr = -ENOMEM;
1764 up_write(&mm->mmap_sem);
1765 }
1766 drm_gem_object_unreference_unlocked(obj);
1767 if (IS_ERR((void *)addr))
1768 return addr;
1769
1770 args->addr_ptr = (uint64_t) addr;
1771
1772 return 0;
1773}
1774
1775/**
1776 * i915_gem_fault - fault a page into the GTT
1777 * @vma: VMA in question
1778 * @vmf: fault info
1779 *
1780 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1781 * from userspace. The fault handler takes care of binding the object to
1782 * the GTT (if needed), allocating and programming a fence register (again,
1783 * only if needed based on whether the old reg is still valid or the object
1784 * is tiled) and inserting a new PTE into the faulting process.
1785 *
1786 * Note that the faulting process may involve evicting existing objects
1787 * from the GTT and/or fence registers to make room. So performance may
1788 * suffer if the GTT working set is large or there are few fence registers
1789 * left.
1790 */
1791int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1792{
1793 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1794 struct drm_device *dev = obj->base.dev;
1795 struct drm_i915_private *dev_priv = dev->dev_private;
1796 struct i915_ggtt_view view = i915_ggtt_view_normal;
1797 pgoff_t page_offset;
1798 unsigned long pfn;
1799 int ret = 0;
1800 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1801
1802 intel_runtime_pm_get(dev_priv);
1803
1804 /* We don't use vmf->pgoff since that has the fake offset */
1805 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1806 PAGE_SHIFT;
1807
1808 ret = i915_mutex_lock_interruptible(dev);
1809 if (ret)
1810 goto out;
1811
1812 trace_i915_gem_object_fault(obj, page_offset, true, write);
1813
1814 /* Try to flush the object off the GPU first without holding the lock.
1815 * Upon reacquiring the lock, we will perform our sanity checks and then
1816 * repeat the flush holding the lock in the normal manner to catch cases
1817 * where we are gazumped.
1818 */
1819 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1820 if (ret)
1821 goto unlock;
1822
1823 /* Access to snoopable pages through the GTT is incoherent. */
1824 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1825 ret = -EFAULT;
1826 goto unlock;
1827 }
1828
1829 /* Use a partial view if the object is bigger than the aperture. */
1830 if (obj->base.size >= dev_priv->gtt.mappable_end &&
1831 obj->tiling_mode == I915_TILING_NONE) {
1832 static const unsigned int chunk_size = 256; // 1 MiB
1833
1834 memset(&view, 0, sizeof(view));
1835 view.type = I915_GGTT_VIEW_PARTIAL;
1836 view.params.partial.offset = rounddown(page_offset, chunk_size);
1837 view.params.partial.size =
1838 min_t(unsigned int,
1839 chunk_size,
1840 (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1841 view.params.partial.offset);
1842 }
1843
1844 /* Now pin it into the GTT if needed */
1845 ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
1846 if (ret)
1847 goto unlock;
1848
1849 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1850 if (ret)
1851 goto unpin;
1852
1853 ret = i915_gem_object_get_fence(obj);
1854 if (ret)
1855 goto unpin;
1856
1857 /* Finally, remap it using the new GTT offset */
1858 pfn = dev_priv->gtt.mappable_base +
1859 i915_gem_obj_ggtt_offset_view(obj, &view);
1860 pfn >>= PAGE_SHIFT;
1861
1862 if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1863 /* Overriding existing pages in partial view does not cause
1864 * us any trouble as TLBs are still valid because the fault
1865 * is due to userspace losing part of the mapping or never
1866 * having accessed it before (at this partials' range).
1867 */
1868 unsigned long base = vma->vm_start +
1869 (view.params.partial.offset << PAGE_SHIFT);
1870 unsigned int i;
1871
1872 for (i = 0; i < view.params.partial.size; i++) {
1873 ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1874 if (ret)
1875 break;
1876 }
1877
1878 obj->fault_mappable = true;
1879 } else {
1880 if (!obj->fault_mappable) {
1881 unsigned long size = min_t(unsigned long,
1882 vma->vm_end - vma->vm_start,
1883 obj->base.size);
1884 int i;
1885
1886 for (i = 0; i < size >> PAGE_SHIFT; i++) {
1887 ret = vm_insert_pfn(vma,
1888 (unsigned long)vma->vm_start + i * PAGE_SIZE,
1889 pfn + i);
1890 if (ret)
1891 break;
1892 }
1893
1894 obj->fault_mappable = true;
1895 } else
1896 ret = vm_insert_pfn(vma,
1897 (unsigned long)vmf->virtual_address,
1898 pfn + page_offset);
1899 }
1900unpin:
1901 i915_gem_object_ggtt_unpin_view(obj, &view);
1902unlock:
1903 mutex_unlock(&dev->struct_mutex);
1904out:
1905 switch (ret) {
1906 case -EIO:
1907 /*
1908 * We eat errors when the gpu is terminally wedged to avoid
1909 * userspace unduly crashing (gl has no provisions for mmaps to
1910 * fail). But any other -EIO isn't ours (e.g. swap in failure)
1911 * and so needs to be reported.
1912 */
1913 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1914 ret = VM_FAULT_SIGBUS;
1915 break;
1916 }
1917 case -EAGAIN:
1918 /*
1919 * EAGAIN means the gpu is hung and we'll wait for the error
1920 * handler to reset everything when re-faulting in
1921 * i915_mutex_lock_interruptible.
1922 */
1923 case 0:
1924 case -ERESTARTSYS:
1925 case -EINTR:
1926 case -EBUSY:
1927 /*
1928 * EBUSY is ok: this just means that another thread
1929 * already did the job.
1930 */
1931 ret = VM_FAULT_NOPAGE;
1932 break;
1933 case -ENOMEM:
1934 ret = VM_FAULT_OOM;
1935 break;
1936 case -ENOSPC:
1937 case -EFAULT:
1938 ret = VM_FAULT_SIGBUS;
1939 break;
1940 default:
1941 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1942 ret = VM_FAULT_SIGBUS;
1943 break;
1944 }
1945
1946 intel_runtime_pm_put(dev_priv);
1947 return ret;
1948}
1949
1950/**
1951 * i915_gem_release_mmap - remove physical page mappings
1952 * @obj: obj in question
1953 *
1954 * Preserve the reservation of the mmapping with the DRM core code, but
1955 * relinquish ownership of the pages back to the system.
1956 *
1957 * It is vital that we remove the page mapping if we have mapped a tiled
1958 * object through the GTT and then lose the fence register due to
1959 * resource pressure. Similarly if the object has been moved out of the
1960 * aperture, than pages mapped into userspace must be revoked. Removing the
1961 * mapping will then trigger a page fault on the next user access, allowing
1962 * fixup by i915_gem_fault().
1963 */
1964void
1965i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1966{
1967 if (!obj->fault_mappable)
1968 return;
1969
1970 drm_vma_node_unmap(&obj->base.vma_node,
1971 obj->base.dev->anon_inode->i_mapping);
1972 obj->fault_mappable = false;
1973}
1974
1975void
1976i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1977{
1978 struct drm_i915_gem_object *obj;
1979
1980 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1981 i915_gem_release_mmap(obj);
1982}
1983
1984uint32_t
1985i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1986{
1987 uint32_t gtt_size;
1988
1989 if (INTEL_INFO(dev)->gen >= 4 ||
1990 tiling_mode == I915_TILING_NONE)
1991 return size;
1992
1993 /* Previous chips need a power-of-two fence region when tiling */
1994 if (INTEL_INFO(dev)->gen == 3)
1995 gtt_size = 1024*1024;
1996 else
1997 gtt_size = 512*1024;
1998
1999 while (gtt_size < size)
2000 gtt_size <<= 1;
2001
2002 return gtt_size;
2003}
2004
2005/**
2006 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
2007 * @obj: object to check
2008 *
2009 * Return the required GTT alignment for an object, taking into account
2010 * potential fence register mapping.
2011 */
2012uint32_t
2013i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
2014 int tiling_mode, bool fenced)
2015{
2016 /*
2017 * Minimum alignment is 4k (GTT page size), but might be greater
2018 * if a fence register is needed for the object.
2019 */
2020 if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
2021 tiling_mode == I915_TILING_NONE)
2022 return 4096;
2023
2024 /*
2025 * Previous chips need to be aligned to the size of the smallest
2026 * fence register that can contain the object.
2027 */
2028 return i915_gem_get_gtt_size(dev, size, tiling_mode);
2029}
2030
2031static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2032{
2033 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2034 int ret;
2035
2036 if (drm_vma_node_has_offset(&obj->base.vma_node))
2037 return 0;
2038
2039 dev_priv->mm.shrinker_no_lock_stealing = true;
2040
2041 ret = drm_gem_create_mmap_offset(&obj->base);
2042 if (ret != -ENOSPC)
2043 goto out;
2044
2045 /* Badly fragmented mmap space? The only way we can recover
2046 * space is by destroying unwanted objects. We can't randomly release
2047 * mmap_offsets as userspace expects them to be persistent for the
2048 * lifetime of the objects. The closest we can is to release the
2049 * offsets on purgeable objects by truncating it and marking it purged,
2050 * which prevents userspace from ever using that object again.
2051 */
2052 i915_gem_shrink(dev_priv,
2053 obj->base.size >> PAGE_SHIFT,
2054 I915_SHRINK_BOUND |
2055 I915_SHRINK_UNBOUND |
2056 I915_SHRINK_PURGEABLE);
2057 ret = drm_gem_create_mmap_offset(&obj->base);
2058 if (ret != -ENOSPC)
2059 goto out;
2060
2061 i915_gem_shrink_all(dev_priv);
2062 ret = drm_gem_create_mmap_offset(&obj->base);
2063out:
2064 dev_priv->mm.shrinker_no_lock_stealing = false;
2065
2066 return ret;
2067}
2068
2069static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2070{
2071 drm_gem_free_mmap_offset(&obj->base);
2072}
2073
2074int
2075i915_gem_mmap_gtt(struct drm_file *file,
2076 struct drm_device *dev,
2077 uint32_t handle,
2078 uint64_t *offset)
2079{
2080 struct drm_i915_gem_object *obj;
2081 int ret;
2082
2083 ret = i915_mutex_lock_interruptible(dev);
2084 if (ret)
2085 return ret;
2086
2087 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
2088 if (&obj->base == NULL) {
2089 ret = -ENOENT;
2090 goto unlock;
2091 }
2092
2093 if (obj->madv != I915_MADV_WILLNEED) {
2094 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
2095 ret = -EFAULT;
2096 goto out;
2097 }
2098
2099 ret = i915_gem_object_create_mmap_offset(obj);
2100 if (ret)
2101 goto out;
2102
2103 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2104
2105out:
2106 drm_gem_object_unreference(&obj->base);
2107unlock:
2108 mutex_unlock(&dev->struct_mutex);
2109 return ret;
2110}
2111
2112/**
2113 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2114 * @dev: DRM device
2115 * @data: GTT mapping ioctl data
2116 * @file: GEM object info
2117 *
2118 * Simply returns the fake offset to userspace so it can mmap it.
2119 * The mmap call will end up in drm_gem_mmap(), which will set things
2120 * up so we can get faults in the handler above.
2121 *
2122 * The fault handler will take care of binding the object into the GTT
2123 * (since it may have been evicted to make room for something), allocating
2124 * a fence register, and mapping the appropriate aperture address into
2125 * userspace.
2126 */
2127int
2128i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2129 struct drm_file *file)
2130{
2131 struct drm_i915_gem_mmap_gtt *args = data;
2132
2133 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2134}
2135
2136/* Immediately discard the backing storage */
2137static void
2138i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2139{
2140 i915_gem_object_free_mmap_offset(obj);
2141
2142 if (obj->base.filp == NULL)
2143 return;
2144
2145 /* Our goal here is to return as much of the memory as
2146 * is possible back to the system as we are called from OOM.
2147 * To do this we must instruct the shmfs to drop all of its
2148 * backing pages, *now*.
2149 */
2150 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2151 obj->madv = __I915_MADV_PURGED;
2152}
2153
2154/* Try to discard unwanted pages */
2155static void
2156i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2157{
2158 struct address_space *mapping;
2159
2160 switch (obj->madv) {
2161 case I915_MADV_DONTNEED:
2162 i915_gem_object_truncate(obj);
2163 case __I915_MADV_PURGED:
2164 return;
2165 }
2166
2167 if (obj->base.filp == NULL)
2168 return;
2169
2170 mapping = file_inode(obj->base.filp)->i_mapping,
2171 invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2172}
2173
2174static void
2175i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2176{
2177 struct sg_page_iter sg_iter;
2178 int ret;
2179
2180 BUG_ON(obj->madv == __I915_MADV_PURGED);
2181
2182 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2183 if (ret) {
2184 /* In the event of a disaster, abandon all caches and
2185 * hope for the best.
2186 */
2187 WARN_ON(ret != -EIO);
2188 i915_gem_clflush_object(obj, true);
2189 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2190 }
2191
2192 i915_gem_gtt_finish_object(obj);
2193
2194 if (i915_gem_object_needs_bit17_swizzle(obj))
2195 i915_gem_object_save_bit_17_swizzle(obj);
2196
2197 if (obj->madv == I915_MADV_DONTNEED)
2198 obj->dirty = 0;
2199
2200 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
2201 struct page *page = sg_page_iter_page(&sg_iter);
2202
2203 if (obj->dirty)
2204 set_page_dirty(page);
2205
2206 if (obj->madv == I915_MADV_WILLNEED)
2207 mark_page_accessed(page);
2208
2209 put_page(page);
2210 }
2211 obj->dirty = 0;
2212
2213 sg_free_table(obj->pages);
2214 kfree(obj->pages);
2215}
2216
2217int
2218i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2219{
2220 const struct drm_i915_gem_object_ops *ops = obj->ops;
2221
2222 if (obj->pages == NULL)
2223 return 0;
2224
2225 if (obj->pages_pin_count)
2226 return -EBUSY;
2227
2228 BUG_ON(i915_gem_obj_bound_any(obj));
2229
2230 /* ->put_pages might need to allocate memory for the bit17 swizzle
2231 * array, hence protect them from being reaped by removing them from gtt
2232 * lists early. */
2233 list_del(&obj->global_list);
2234
2235 ops->put_pages(obj);
2236 obj->pages = NULL;
2237
2238 i915_gem_object_invalidate(obj);
2239
2240 return 0;
2241}
2242
2243static int
2244i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2245{
2246 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2247 int page_count, i;
2248 struct address_space *mapping;
2249 struct sg_table *st;
2250 struct scatterlist *sg;
2251 struct sg_page_iter sg_iter;
2252 struct page *page;
2253 unsigned long last_pfn = 0; /* suppress gcc warning */
2254 int ret;
2255 gfp_t gfp;
2256
2257 /* Assert that the object is not currently in any GPU domain. As it
2258 * wasn't in the GTT, there shouldn't be any way it could have been in
2259 * a GPU cache
2260 */
2261 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2262 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2263
2264 st = kmalloc(sizeof(*st), GFP_KERNEL);
2265 if (st == NULL)
2266 return -ENOMEM;
2267
2268 page_count = obj->base.size / PAGE_SIZE;
2269 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2270 kfree(st);
2271 return -ENOMEM;
2272 }
2273
2274 /* Get the list of pages out of our struct file. They'll be pinned
2275 * at this point until we release them.
2276 *
2277 * Fail silently without starting the shrinker
2278 */
2279 mapping = file_inode(obj->base.filp)->i_mapping;
2280 gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2281 gfp |= __GFP_NORETRY | __GFP_NOWARN;
2282 sg = st->sgl;
2283 st->nents = 0;
2284 for (i = 0; i < page_count; i++) {
2285 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2286 if (IS_ERR(page)) {
2287 i915_gem_shrink(dev_priv,
2288 page_count,
2289 I915_SHRINK_BOUND |
2290 I915_SHRINK_UNBOUND |
2291 I915_SHRINK_PURGEABLE);
2292 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2293 }
2294 if (IS_ERR(page)) {
2295 /* We've tried hard to allocate the memory by reaping
2296 * our own buffer, now let the real VM do its job and
2297 * go down in flames if truly OOM.
2298 */
2299 i915_gem_shrink_all(dev_priv);
2300 page = shmem_read_mapping_page(mapping, i);
2301 if (IS_ERR(page)) {
2302 ret = PTR_ERR(page);
2303 goto err_pages;
2304 }
2305 }
2306#ifdef CONFIG_SWIOTLB
2307 if (swiotlb_nr_tbl()) {
2308 st->nents++;
2309 sg_set_page(sg, page, PAGE_SIZE, 0);
2310 sg = sg_next(sg);
2311 continue;
2312 }
2313#endif
2314 if (!i || page_to_pfn(page) != last_pfn + 1) {
2315 if (i)
2316 sg = sg_next(sg);
2317 st->nents++;
2318 sg_set_page(sg, page, PAGE_SIZE, 0);
2319 } else {
2320 sg->length += PAGE_SIZE;
2321 }
2322 last_pfn = page_to_pfn(page);
2323
2324 /* Check that the i965g/gm workaround works. */
2325 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2326 }
2327#ifdef CONFIG_SWIOTLB
2328 if (!swiotlb_nr_tbl())
2329#endif
2330 sg_mark_end(sg);
2331 obj->pages = st;
2332
2333 ret = i915_gem_gtt_prepare_object(obj);
2334 if (ret)
2335 goto err_pages;
2336
2337 if (i915_gem_object_needs_bit17_swizzle(obj))
2338 i915_gem_object_do_bit_17_swizzle(obj);
2339
2340 if (obj->tiling_mode != I915_TILING_NONE &&
2341 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2342 i915_gem_object_pin_pages(obj);
2343
2344 return 0;
2345
2346err_pages:
2347 sg_mark_end(sg);
2348 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2349 put_page(sg_page_iter_page(&sg_iter));
2350 sg_free_table(st);
2351 kfree(st);
2352
2353 /* shmemfs first checks if there is enough memory to allocate the page
2354 * and reports ENOSPC should there be insufficient, along with the usual
2355 * ENOMEM for a genuine allocation failure.
2356 *
2357 * We use ENOSPC in our driver to mean that we have run out of aperture
2358 * space and so want to translate the error from shmemfs back to our
2359 * usual understanding of ENOMEM.
2360 */
2361 if (ret == -ENOSPC)
2362 ret = -ENOMEM;
2363
2364 return ret;
2365}
2366
2367/* Ensure that the associated pages are gathered from the backing storage
2368 * and pinned into our object. i915_gem_object_get_pages() may be called
2369 * multiple times before they are released by a single call to
2370 * i915_gem_object_put_pages() - once the pages are no longer referenced
2371 * either as a result of memory pressure (reaping pages under the shrinker)
2372 * or as the object is itself released.
2373 */
2374int
2375i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2376{
2377 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2378 const struct drm_i915_gem_object_ops *ops = obj->ops;
2379 int ret;
2380
2381 if (obj->pages)
2382 return 0;
2383
2384 if (obj->madv != I915_MADV_WILLNEED) {
2385 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2386 return -EFAULT;
2387 }
2388
2389 BUG_ON(obj->pages_pin_count);
2390
2391 ret = ops->get_pages(obj);
2392 if (ret)
2393 return ret;
2394
2395 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2396
2397 obj->get_page.sg = obj->pages->sgl;
2398 obj->get_page.last = 0;
2399
2400 return 0;
2401}
2402
2403void i915_vma_move_to_active(struct i915_vma *vma,
2404 struct drm_i915_gem_request *req)
2405{
2406 struct drm_i915_gem_object *obj = vma->obj;
2407 struct intel_engine_cs *ring;
2408
2409 ring = i915_gem_request_get_ring(req);
2410
2411 /* Add a reference if we're newly entering the active list. */
2412 if (obj->active == 0)
2413 drm_gem_object_reference(&obj->base);
2414 obj->active |= intel_ring_flag(ring);
2415
2416 list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
2417 i915_gem_request_assign(&obj->last_read_req[ring->id], req);
2418
2419 list_move_tail(&vma->vm_link, &vma->vm->active_list);
2420}
2421
2422static void
2423i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2424{
2425 RQ_BUG_ON(obj->last_write_req == NULL);
2426 RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
2427
2428 i915_gem_request_assign(&obj->last_write_req, NULL);
2429 intel_fb_obj_flush(obj, true, ORIGIN_CS);
2430}
2431
2432static void
2433i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
2434{
2435 struct i915_vma *vma;
2436
2437 RQ_BUG_ON(obj->last_read_req[ring] == NULL);
2438 RQ_BUG_ON(!(obj->active & (1 << ring)));
2439
2440 list_del_init(&obj->ring_list[ring]);
2441 i915_gem_request_assign(&obj->last_read_req[ring], NULL);
2442
2443 if (obj->last_write_req && obj->last_write_req->ring->id == ring)
2444 i915_gem_object_retire__write(obj);
2445
2446 obj->active &= ~(1 << ring);
2447 if (obj->active)
2448 return;
2449
2450 /* Bump our place on the bound list to keep it roughly in LRU order
2451 * so that we don't steal from recently used but inactive objects
2452 * (unless we are forced to ofc!)
2453 */
2454 list_move_tail(&obj->global_list,
2455 &to_i915(obj->base.dev)->mm.bound_list);
2456
2457 list_for_each_entry(vma, &obj->vma_list, obj_link) {
2458 if (!list_empty(&vma->vm_link))
2459 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
2460 }
2461
2462 i915_gem_request_assign(&obj->last_fenced_req, NULL);
2463 drm_gem_object_unreference(&obj->base);
2464}
2465
2466static int
2467i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2468{
2469 struct drm_i915_private *dev_priv = dev->dev_private;
2470 struct intel_engine_cs *ring;
2471 int ret, i, j;
2472
2473 /* Carefully retire all requests without writing to the rings */
2474 for_each_ring(ring, dev_priv, i) {
2475 ret = intel_ring_idle(ring);
2476 if (ret)
2477 return ret;
2478 }
2479 i915_gem_retire_requests(dev);
2480
2481 /* Finally reset hw state */
2482 for_each_ring(ring, dev_priv, i) {
2483 intel_ring_init_seqno(ring, seqno);
2484
2485 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
2486 ring->semaphore.sync_seqno[j] = 0;
2487 }
2488
2489 return 0;
2490}
2491
2492int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2493{
2494 struct drm_i915_private *dev_priv = dev->dev_private;
2495 int ret;
2496
2497 if (seqno == 0)
2498 return -EINVAL;
2499
2500 /* HWS page needs to be set less than what we
2501 * will inject to ring
2502 */
2503 ret = i915_gem_init_seqno(dev, seqno - 1);
2504 if (ret)
2505 return ret;
2506
2507 /* Carefully set the last_seqno value so that wrap
2508 * detection still works
2509 */
2510 dev_priv->next_seqno = seqno;
2511 dev_priv->last_seqno = seqno - 1;
2512 if (dev_priv->last_seqno == 0)
2513 dev_priv->last_seqno--;
2514
2515 return 0;
2516}
2517
2518int
2519i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2520{
2521 struct drm_i915_private *dev_priv = dev->dev_private;
2522
2523 /* reserve 0 for non-seqno */
2524 if (dev_priv->next_seqno == 0) {
2525 int ret = i915_gem_init_seqno(dev, 0);
2526 if (ret)
2527 return ret;
2528
2529 dev_priv->next_seqno = 1;
2530 }
2531
2532 *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2533 return 0;
2534}
2535
2536/*
2537 * NB: This function is not allowed to fail. Doing so would mean the the
2538 * request is not being tracked for completion but the work itself is
2539 * going to happen on the hardware. This would be a Bad Thing(tm).
2540 */
2541void __i915_add_request(struct drm_i915_gem_request *request,
2542 struct drm_i915_gem_object *obj,
2543 bool flush_caches)
2544{
2545 struct intel_engine_cs *ring;
2546 struct drm_i915_private *dev_priv;
2547 struct intel_ringbuffer *ringbuf;
2548 u32 request_start;
2549 int ret;
2550
2551 if (WARN_ON(request == NULL))
2552 return;
2553
2554 ring = request->ring;
2555 dev_priv = ring->dev->dev_private;
2556 ringbuf = request->ringbuf;
2557
2558 /*
2559 * To ensure that this call will not fail, space for its emissions
2560 * should already have been reserved in the ring buffer. Let the ring
2561 * know that it is time to use that space up.
2562 */
2563 intel_ring_reserved_space_use(ringbuf);
2564
2565 request_start = intel_ring_get_tail(ringbuf);
2566 /*
2567 * Emit any outstanding flushes - execbuf can fail to emit the flush
2568 * after having emitted the batchbuffer command. Hence we need to fix
2569 * things up similar to emitting the lazy request. The difference here
2570 * is that the flush _must_ happen before the next request, no matter
2571 * what.
2572 */
2573 if (flush_caches) {
2574 if (i915.enable_execlists)
2575 ret = logical_ring_flush_all_caches(request);
2576 else
2577 ret = intel_ring_flush_all_caches(request);
2578 /* Not allowed to fail! */
2579 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
2580 }
2581
2582 /* Record the position of the start of the request so that
2583 * should we detect the updated seqno part-way through the
2584 * GPU processing the request, we never over-estimate the
2585 * position of the head.
2586 */
2587 request->postfix = intel_ring_get_tail(ringbuf);
2588
2589 if (i915.enable_execlists)
2590 ret = ring->emit_request(request);
2591 else {
2592 ret = ring->add_request(request);
2593
2594 request->tail = intel_ring_get_tail(ringbuf);
2595 }
2596 /* Not allowed to fail! */
2597 WARN(ret, "emit|add_request failed: %d!\n", ret);
2598
2599 request->head = request_start;
2600
2601 /* Whilst this request exists, batch_obj will be on the
2602 * active_list, and so will hold the active reference. Only when this
2603 * request is retired will the the batch_obj be moved onto the
2604 * inactive_list and lose its active reference. Hence we do not need
2605 * to explicitly hold another reference here.
2606 */
2607 request->batch_obj = obj;
2608
2609 request->emitted_jiffies = jiffies;
2610 request->previous_seqno = ring->last_submitted_seqno;
2611 ring->last_submitted_seqno = request->seqno;
2612 list_add_tail(&request->list, &ring->request_list);
2613
2614 trace_i915_gem_request_add(request);
2615
2616 i915_queue_hangcheck(ring->dev);
2617
2618 queue_delayed_work(dev_priv->wq,
2619 &dev_priv->mm.retire_work,
2620 round_jiffies_up_relative(HZ));
2621 intel_mark_busy(dev_priv->dev);
2622
2623 /* Sanity check that the reserved size was large enough. */
2624 intel_ring_reserved_space_end(ringbuf);
2625}
2626
2627static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2628 const struct intel_context *ctx)
2629{
2630 unsigned long elapsed;
2631
2632 elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2633
2634 if (ctx->hang_stats.banned)
2635 return true;
2636
2637 if (ctx->hang_stats.ban_period_seconds &&
2638 elapsed <= ctx->hang_stats.ban_period_seconds) {
2639 if (!i915_gem_context_is_default(ctx)) {
2640 DRM_DEBUG("context hanging too fast, banning!\n");
2641 return true;
2642 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2643 if (i915_stop_ring_allow_warn(dev_priv))
2644 DRM_ERROR("gpu hanging too fast, banning!\n");
2645 return true;
2646 }
2647 }
2648
2649 return false;
2650}
2651
2652static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2653 struct intel_context *ctx,
2654 const bool guilty)
2655{
2656 struct i915_ctx_hang_stats *hs;
2657
2658 if (WARN_ON(!ctx))
2659 return;
2660
2661 hs = &ctx->hang_stats;
2662
2663 if (guilty) {
2664 hs->banned = i915_context_is_banned(dev_priv, ctx);
2665 hs->batch_active++;
2666 hs->guilty_ts = get_seconds();
2667 } else {
2668 hs->batch_pending++;
2669 }
2670}
2671
2672void i915_gem_request_free(struct kref *req_ref)
2673{
2674 struct drm_i915_gem_request *req = container_of(req_ref,
2675 typeof(*req), ref);
2676 struct intel_context *ctx = req->ctx;
2677
2678 if (req->file_priv)
2679 i915_gem_request_remove_from_client(req);
2680
2681 if (ctx) {
2682 if (i915.enable_execlists && ctx != req->i915->kernel_context)
2683 intel_lr_context_unpin(ctx, req->ring);
2684
2685 i915_gem_context_unreference(ctx);
2686 }
2687
2688 kmem_cache_free(req->i915->requests, req);
2689}
2690
2691static inline int
2692__i915_gem_request_alloc(struct intel_engine_cs *ring,
2693 struct intel_context *ctx,
2694 struct drm_i915_gem_request **req_out)
2695{
2696 struct drm_i915_private *dev_priv = to_i915(ring->dev);
2697 struct drm_i915_gem_request *req;
2698 int ret;
2699
2700 if (!req_out)
2701 return -EINVAL;
2702
2703 *req_out = NULL;
2704
2705 req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
2706 if (req == NULL)
2707 return -ENOMEM;
2708
2709 ret = i915_gem_get_seqno(ring->dev, &req->seqno);
2710 if (ret)
2711 goto err;
2712
2713 kref_init(&req->ref);
2714 req->i915 = dev_priv;
2715 req->ring = ring;
2716 req->ctx = ctx;
2717 i915_gem_context_reference(req->ctx);
2718
2719 if (i915.enable_execlists)
2720 ret = intel_logical_ring_alloc_request_extras(req);
2721 else
2722 ret = intel_ring_alloc_request_extras(req);
2723 if (ret) {
2724 i915_gem_context_unreference(req->ctx);
2725 goto err;
2726 }
2727
2728 /*
2729 * Reserve space in the ring buffer for all the commands required to
2730 * eventually emit this request. This is to guarantee that the
2731 * i915_add_request() call can't fail. Note that the reserve may need
2732 * to be redone if the request is not actually submitted straight
2733 * away, e.g. because a GPU scheduler has deferred it.
2734 */
2735 if (i915.enable_execlists)
2736 ret = intel_logical_ring_reserve_space(req);
2737 else
2738 ret = intel_ring_reserve_space(req);
2739 if (ret) {
2740 /*
2741 * At this point, the request is fully allocated even if not
2742 * fully prepared. Thus it can be cleaned up using the proper
2743 * free code.
2744 */
2745 i915_gem_request_cancel(req);
2746 return ret;
2747 }
2748
2749 *req_out = req;
2750 return 0;
2751
2752err:
2753 kmem_cache_free(dev_priv->requests, req);
2754 return ret;
2755}
2756
2757/**
2758 * i915_gem_request_alloc - allocate a request structure
2759 *
2760 * @engine: engine that we wish to issue the request on.
2761 * @ctx: context that the request will be associated with.
2762 * This can be NULL if the request is not directly related to
2763 * any specific user context, in which case this function will
2764 * choose an appropriate context to use.
2765 *
2766 * Returns a pointer to the allocated request if successful,
2767 * or an error code if not.
2768 */
2769struct drm_i915_gem_request *
2770i915_gem_request_alloc(struct intel_engine_cs *engine,
2771 struct intel_context *ctx)
2772{
2773 struct drm_i915_gem_request *req;
2774 int err;
2775
2776 if (ctx == NULL)
2777 ctx = to_i915(engine->dev)->kernel_context;
2778 err = __i915_gem_request_alloc(engine, ctx, &req);
2779 return err ? ERR_PTR(err) : req;
2780}
2781
2782void i915_gem_request_cancel(struct drm_i915_gem_request *req)
2783{
2784 intel_ring_reserved_space_cancel(req->ringbuf);
2785
2786 i915_gem_request_unreference(req);
2787}
2788
2789struct drm_i915_gem_request *
2790i915_gem_find_active_request(struct intel_engine_cs *ring)
2791{
2792 struct drm_i915_gem_request *request;
2793
2794 list_for_each_entry(request, &ring->request_list, list) {
2795 if (i915_gem_request_completed(request, false))
2796 continue;
2797
2798 return request;
2799 }
2800
2801 return NULL;
2802}
2803
2804static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2805 struct intel_engine_cs *ring)
2806{
2807 struct drm_i915_gem_request *request;
2808 bool ring_hung;
2809
2810 request = i915_gem_find_active_request(ring);
2811
2812 if (request == NULL)
2813 return;
2814
2815 ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2816
2817 i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2818
2819 list_for_each_entry_continue(request, &ring->request_list, list)
2820 i915_set_reset_status(dev_priv, request->ctx, false);
2821}
2822
2823static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2824 struct intel_engine_cs *ring)
2825{
2826 struct intel_ringbuffer *buffer;
2827
2828 while (!list_empty(&ring->active_list)) {
2829 struct drm_i915_gem_object *obj;
2830
2831 obj = list_first_entry(&ring->active_list,
2832 struct drm_i915_gem_object,
2833 ring_list[ring->id]);
2834
2835 i915_gem_object_retire__read(obj, ring->id);
2836 }
2837
2838 /*
2839 * Clear the execlists queue up before freeing the requests, as those
2840 * are the ones that keep the context and ringbuffer backing objects
2841 * pinned in place.
2842 */
2843
2844 if (i915.enable_execlists) {
2845 spin_lock_irq(&ring->execlist_lock);
2846
2847 /* list_splice_tail_init checks for empty lists */
2848 list_splice_tail_init(&ring->execlist_queue,
2849 &ring->execlist_retired_req_list);
2850
2851 spin_unlock_irq(&ring->execlist_lock);
2852 intel_execlists_retire_requests(ring);
2853 }
2854
2855 /*
2856 * We must free the requests after all the corresponding objects have
2857 * been moved off active lists. Which is the same order as the normal
2858 * retire_requests function does. This is important if object hold
2859 * implicit references on things like e.g. ppgtt address spaces through
2860 * the request.
2861 */
2862 while (!list_empty(&ring->request_list)) {
2863 struct drm_i915_gem_request *request;
2864
2865 request = list_first_entry(&ring->request_list,
2866 struct drm_i915_gem_request,
2867 list);
2868
2869 i915_gem_request_retire(request);
2870 }
2871
2872 /* Having flushed all requests from all queues, we know that all
2873 * ringbuffers must now be empty. However, since we do not reclaim
2874 * all space when retiring the request (to prevent HEADs colliding
2875 * with rapid ringbuffer wraparound) the amount of available space
2876 * upon reset is less than when we start. Do one more pass over
2877 * all the ringbuffers to reset last_retired_head.
2878 */
2879 list_for_each_entry(buffer, &ring->buffers, link) {
2880 buffer->last_retired_head = buffer->tail;
2881 intel_ring_update_space(buffer);
2882 }
2883}
2884
2885void i915_gem_reset(struct drm_device *dev)
2886{
2887 struct drm_i915_private *dev_priv = dev->dev_private;
2888 struct intel_engine_cs *ring;
2889 int i;
2890
2891 /*
2892 * Before we free the objects from the requests, we need to inspect
2893 * them for finding the guilty party. As the requests only borrow
2894 * their reference to the objects, the inspection must be done first.
2895 */
2896 for_each_ring(ring, dev_priv, i)
2897 i915_gem_reset_ring_status(dev_priv, ring);
2898
2899 for_each_ring(ring, dev_priv, i)
2900 i915_gem_reset_ring_cleanup(dev_priv, ring);
2901
2902 i915_gem_context_reset(dev);
2903
2904 i915_gem_restore_fences(dev);
2905
2906 WARN_ON(i915_verify_lists(dev));
2907}
2908
2909/**
2910 * This function clears the request list as sequence numbers are passed.
2911 */
2912void
2913i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2914{
2915 WARN_ON(i915_verify_lists(ring->dev));
2916
2917 /* Retire requests first as we use it above for the early return.
2918 * If we retire requests last, we may use a later seqno and so clear
2919 * the requests lists without clearing the active list, leading to
2920 * confusion.
2921 */
2922 while (!list_empty(&ring->request_list)) {
2923 struct drm_i915_gem_request *request;
2924
2925 request = list_first_entry(&ring->request_list,
2926 struct drm_i915_gem_request,
2927 list);
2928
2929 if (!i915_gem_request_completed(request, true))
2930 break;
2931
2932 i915_gem_request_retire(request);
2933 }
2934
2935 /* Move any buffers on the active list that are no longer referenced
2936 * by the ringbuffer to the flushing/inactive lists as appropriate,
2937 * before we free the context associated with the requests.
2938 */
2939 while (!list_empty(&ring->active_list)) {
2940 struct drm_i915_gem_object *obj;
2941
2942 obj = list_first_entry(&ring->active_list,
2943 struct drm_i915_gem_object,
2944 ring_list[ring->id]);
2945
2946 if (!list_empty(&obj->last_read_req[ring->id]->list))
2947 break;
2948
2949 i915_gem_object_retire__read(obj, ring->id);
2950 }
2951
2952 if (unlikely(ring->trace_irq_req &&
2953 i915_gem_request_completed(ring->trace_irq_req, true))) {
2954 ring->irq_put(ring);
2955 i915_gem_request_assign(&ring->trace_irq_req, NULL);
2956 }
2957
2958 WARN_ON(i915_verify_lists(ring->dev));
2959}
2960
2961bool
2962i915_gem_retire_requests(struct drm_device *dev)
2963{
2964 struct drm_i915_private *dev_priv = dev->dev_private;
2965 struct intel_engine_cs *ring;
2966 bool idle = true;
2967 int i;
2968
2969 for_each_ring(ring, dev_priv, i) {
2970 i915_gem_retire_requests_ring(ring);
2971 idle &= list_empty(&ring->request_list);
2972 if (i915.enable_execlists) {
2973 spin_lock_irq(&ring->execlist_lock);
2974 idle &= list_empty(&ring->execlist_queue);
2975 spin_unlock_irq(&ring->execlist_lock);
2976
2977 intel_execlists_retire_requests(ring);
2978 }
2979 }
2980
2981 if (idle)
2982 mod_delayed_work(dev_priv->wq,
2983 &dev_priv->mm.idle_work,
2984 msecs_to_jiffies(100));
2985
2986 return idle;
2987}
2988
2989static void
2990i915_gem_retire_work_handler(struct work_struct *work)
2991{
2992 struct drm_i915_private *dev_priv =
2993 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2994 struct drm_device *dev = dev_priv->dev;
2995 bool idle;
2996
2997 /* Come back later if the device is busy... */
2998 idle = false;
2999 if (mutex_trylock(&dev->struct_mutex)) {
3000 idle = i915_gem_retire_requests(dev);
3001 mutex_unlock(&dev->struct_mutex);
3002 }
3003 if (!idle)
3004 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3005 round_jiffies_up_relative(HZ));
3006}
3007
3008static void
3009i915_gem_idle_work_handler(struct work_struct *work)
3010{
3011 struct drm_i915_private *dev_priv =
3012 container_of(work, typeof(*dev_priv), mm.idle_work.work);
3013 struct drm_device *dev = dev_priv->dev;
3014 struct intel_engine_cs *ring;
3015 int i;
3016
3017 for_each_ring(ring, dev_priv, i)
3018 if (!list_empty(&ring->request_list))
3019 return;
3020
3021 /* we probably should sync with hangcheck here, using cancel_work_sync.
3022 * Also locking seems to be fubar here, ring->request_list is protected
3023 * by dev->struct_mutex. */
3024
3025 intel_mark_idle(dev);
3026
3027 if (mutex_trylock(&dev->struct_mutex)) {
3028 struct intel_engine_cs *ring;
3029 int i;
3030
3031 for_each_ring(ring, dev_priv, i)
3032 i915_gem_batch_pool_fini(&ring->batch_pool);
3033
3034 mutex_unlock(&dev->struct_mutex);
3035 }
3036}
3037
3038/**
3039 * Ensures that an object will eventually get non-busy by flushing any required
3040 * write domains, emitting any outstanding lazy request and retiring and
3041 * completed requests.
3042 */
3043static int
3044i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
3045{
3046 int i;
3047
3048 if (!obj->active)
3049 return 0;
3050
3051 for (i = 0; i < I915_NUM_RINGS; i++) {
3052 struct drm_i915_gem_request *req;
3053
3054 req = obj->last_read_req[i];
3055 if (req == NULL)
3056 continue;
3057
3058 if (list_empty(&req->list))
3059 goto retire;
3060
3061 if (i915_gem_request_completed(req, true)) {
3062 __i915_gem_request_retire__upto(req);
3063retire:
3064 i915_gem_object_retire__read(obj, i);
3065 }
3066 }
3067
3068 return 0;
3069}
3070
3071/**
3072 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3073 * @DRM_IOCTL_ARGS: standard ioctl arguments
3074 *
3075 * Returns 0 if successful, else an error is returned with the remaining time in
3076 * the timeout parameter.
3077 * -ETIME: object is still busy after timeout
3078 * -ERESTARTSYS: signal interrupted the wait
3079 * -ENONENT: object doesn't exist
3080 * Also possible, but rare:
3081 * -EAGAIN: GPU wedged
3082 * -ENOMEM: damn
3083 * -ENODEV: Internal IRQ fail
3084 * -E?: The add request failed
3085 *
3086 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3087 * non-zero timeout parameter the wait ioctl will wait for the given number of
3088 * nanoseconds on an object becoming unbusy. Since the wait itself does so
3089 * without holding struct_mutex the object may become re-busied before this
3090 * function completes. A similar but shorter * race condition exists in the busy
3091 * ioctl
3092 */
3093int
3094i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3095{
3096 struct drm_i915_private *dev_priv = dev->dev_private;
3097 struct drm_i915_gem_wait *args = data;
3098 struct drm_i915_gem_object *obj;
3099 struct drm_i915_gem_request *req[I915_NUM_RINGS];
3100 unsigned reset_counter;
3101 int i, n = 0;
3102 int ret;
3103
3104 if (args->flags != 0)
3105 return -EINVAL;
3106
3107 ret = i915_mutex_lock_interruptible(dev);
3108 if (ret)
3109 return ret;
3110
3111 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
3112 if (&obj->base == NULL) {
3113 mutex_unlock(&dev->struct_mutex);
3114 return -ENOENT;
3115 }
3116
3117 /* Need to make sure the object gets inactive eventually. */
3118 ret = i915_gem_object_flush_active(obj);
3119 if (ret)
3120 goto out;
3121
3122 if (!obj->active)
3123 goto out;
3124
3125 /* Do this after OLR check to make sure we make forward progress polling
3126 * on this IOCTL with a timeout == 0 (like busy ioctl)
3127 */
3128 if (args->timeout_ns == 0) {
3129 ret = -ETIME;
3130 goto out;
3131 }
3132
3133 drm_gem_object_unreference(&obj->base);
3134 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3135
3136 for (i = 0; i < I915_NUM_RINGS; i++) {
3137 if (obj->last_read_req[i] == NULL)
3138 continue;
3139
3140 req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
3141 }
3142
3143 mutex_unlock(&dev->struct_mutex);
3144
3145 for (i = 0; i < n; i++) {
3146 if (ret == 0)
3147 ret = __i915_wait_request(req[i], reset_counter, true,
3148 args->timeout_ns > 0 ? &args->timeout_ns : NULL,
3149 to_rps_client(file));
3150 i915_gem_request_unreference__unlocked(req[i]);
3151 }
3152 return ret;
3153
3154out:
3155 drm_gem_object_unreference(&obj->base);
3156 mutex_unlock(&dev->struct_mutex);
3157 return ret;
3158}
3159
3160static int
3161__i915_gem_object_sync(struct drm_i915_gem_object *obj,
3162 struct intel_engine_cs *to,
3163 struct drm_i915_gem_request *from_req,
3164 struct drm_i915_gem_request **to_req)
3165{
3166 struct intel_engine_cs *from;
3167 int ret;
3168
3169 from = i915_gem_request_get_ring(from_req);
3170 if (to == from)
3171 return 0;
3172
3173 if (i915_gem_request_completed(from_req, true))
3174 return 0;
3175
3176 if (!i915_semaphore_is_enabled(obj->base.dev)) {
3177 struct drm_i915_private *i915 = to_i915(obj->base.dev);
3178 ret = __i915_wait_request(from_req,
3179 atomic_read(&i915->gpu_error.reset_counter),
3180 i915->mm.interruptible,
3181 NULL,
3182 &i915->rps.semaphores);
3183 if (ret)
3184 return ret;
3185
3186 i915_gem_object_retire_request(obj, from_req);
3187 } else {
3188 int idx = intel_ring_sync_index(from, to);
3189 u32 seqno = i915_gem_request_get_seqno(from_req);
3190
3191 WARN_ON(!to_req);
3192
3193 if (seqno <= from->semaphore.sync_seqno[idx])
3194 return 0;
3195
3196 if (*to_req == NULL) {
3197 struct drm_i915_gem_request *req;
3198
3199 req = i915_gem_request_alloc(to, NULL);
3200 if (IS_ERR(req))
3201 return PTR_ERR(req);
3202
3203 *to_req = req;
3204 }
3205
3206 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
3207 ret = to->semaphore.sync_to(*to_req, from, seqno);
3208 if (ret)
3209 return ret;
3210
3211 /* We use last_read_req because sync_to()
3212 * might have just caused seqno wrap under
3213 * the radar.
3214 */
3215 from->semaphore.sync_seqno[idx] =
3216 i915_gem_request_get_seqno(obj->last_read_req[from->id]);
3217 }
3218
3219 return 0;
3220}
3221
3222/**
3223 * i915_gem_object_sync - sync an object to a ring.
3224 *
3225 * @obj: object which may be in use on another ring.
3226 * @to: ring we wish to use the object on. May be NULL.
3227 * @to_req: request we wish to use the object for. See below.
3228 * This will be allocated and returned if a request is
3229 * required but not passed in.
3230 *
3231 * This code is meant to abstract object synchronization with the GPU.
3232 * Calling with NULL implies synchronizing the object with the CPU
3233 * rather than a particular GPU ring. Conceptually we serialise writes
3234 * between engines inside the GPU. We only allow one engine to write
3235 * into a buffer at any time, but multiple readers. To ensure each has
3236 * a coherent view of memory, we must:
3237 *
3238 * - If there is an outstanding write request to the object, the new
3239 * request must wait for it to complete (either CPU or in hw, requests
3240 * on the same ring will be naturally ordered).
3241 *
3242 * - If we are a write request (pending_write_domain is set), the new
3243 * request must wait for outstanding read requests to complete.
3244 *
3245 * For CPU synchronisation (NULL to) no request is required. For syncing with
3246 * rings to_req must be non-NULL. However, a request does not have to be
3247 * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
3248 * request will be allocated automatically and returned through *to_req. Note
3249 * that it is not guaranteed that commands will be emitted (because the system
3250 * might already be idle). Hence there is no need to create a request that
3251 * might never have any work submitted. Note further that if a request is
3252 * returned in *to_req, it is the responsibility of the caller to submit
3253 * that request (after potentially adding more work to it).
3254 *
3255 * Returns 0 if successful, else propagates up the lower layer error.
3256 */
3257int
3258i915_gem_object_sync(struct drm_i915_gem_object *obj,
3259 struct intel_engine_cs *to,
3260 struct drm_i915_gem_request **to_req)
3261{
3262 const bool readonly = obj->base.pending_write_domain == 0;
3263 struct drm_i915_gem_request *req[I915_NUM_RINGS];
3264 int ret, i, n;
3265
3266 if (!obj->active)
3267 return 0;
3268
3269 if (to == NULL)
3270 return i915_gem_object_wait_rendering(obj, readonly);
3271
3272 n = 0;
3273 if (readonly) {
3274 if (obj->last_write_req)
3275 req[n++] = obj->last_write_req;
3276 } else {
3277 for (i = 0; i < I915_NUM_RINGS; i++)
3278 if (obj->last_read_req[i])
3279 req[n++] = obj->last_read_req[i];
3280 }
3281 for (i = 0; i < n; i++) {
3282 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
3283 if (ret)
3284 return ret;
3285 }
3286
3287 return 0;
3288}
3289
3290static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3291{
3292 u32 old_write_domain, old_read_domains;
3293
3294 /* Force a pagefault for domain tracking on next user access */
3295 i915_gem_release_mmap(obj);
3296
3297 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3298 return;
3299
3300 /* Wait for any direct GTT access to complete */
3301 mb();
3302
3303 old_read_domains = obj->base.read_domains;
3304 old_write_domain = obj->base.write_domain;
3305
3306 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3307 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3308
3309 trace_i915_gem_object_change_domain(obj,
3310 old_read_domains,
3311 old_write_domain);
3312}
3313
3314static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
3315{
3316 struct drm_i915_gem_object *obj = vma->obj;
3317 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3318 int ret;
3319
3320 if (list_empty(&vma->obj_link))
3321 return 0;
3322
3323 if (!drm_mm_node_allocated(&vma->node)) {
3324 i915_gem_vma_destroy(vma);
3325 return 0;
3326 }
3327
3328 if (vma->pin_count)
3329 return -EBUSY;
3330
3331 BUG_ON(obj->pages == NULL);
3332
3333 if (wait) {
3334 ret = i915_gem_object_wait_rendering(obj, false);
3335 if (ret)
3336 return ret;
3337 }
3338
3339 if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3340 i915_gem_object_finish_gtt(obj);
3341
3342 /* release the fence reg _after_ flushing */
3343 ret = i915_gem_object_put_fence(obj);
3344 if (ret)
3345 return ret;
3346 }
3347
3348 trace_i915_vma_unbind(vma);
3349
3350 vma->vm->unbind_vma(vma);
3351 vma->bound = 0;
3352
3353 list_del_init(&vma->vm_link);
3354 if (vma->is_ggtt) {
3355 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3356 obj->map_and_fenceable = false;
3357 } else if (vma->ggtt_view.pages) {
3358 sg_free_table(vma->ggtt_view.pages);
3359 kfree(vma->ggtt_view.pages);
3360 }
3361 vma->ggtt_view.pages = NULL;
3362 }
3363
3364 drm_mm_remove_node(&vma->node);
3365 i915_gem_vma_destroy(vma);
3366
3367 /* Since the unbound list is global, only move to that list if
3368 * no more VMAs exist. */
3369 if (list_empty(&obj->vma_list))
3370 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3371
3372 /* And finally now the object is completely decoupled from this vma,
3373 * we can drop its hold on the backing storage and allow it to be
3374 * reaped by the shrinker.
3375 */
3376 i915_gem_object_unpin_pages(obj);
3377
3378 return 0;
3379}
3380
3381int i915_vma_unbind(struct i915_vma *vma)
3382{
3383 return __i915_vma_unbind(vma, true);
3384}
3385
3386int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3387{
3388 return __i915_vma_unbind(vma, false);
3389}
3390
3391int i915_gpu_idle(struct drm_device *dev)
3392{
3393 struct drm_i915_private *dev_priv = dev->dev_private;
3394 struct intel_engine_cs *ring;
3395 int ret, i;
3396
3397 /* Flush everything onto the inactive list. */
3398 for_each_ring(ring, dev_priv, i) {
3399 if (!i915.enable_execlists) {
3400 struct drm_i915_gem_request *req;
3401
3402 req = i915_gem_request_alloc(ring, NULL);
3403 if (IS_ERR(req))
3404 return PTR_ERR(req);
3405
3406 ret = i915_switch_context(req);
3407 if (ret) {
3408 i915_gem_request_cancel(req);
3409 return ret;
3410 }
3411
3412 i915_add_request_no_flush(req);
3413 }
3414
3415 ret = intel_ring_idle(ring);
3416 if (ret)
3417 return ret;
3418 }
3419
3420 WARN_ON(i915_verify_lists(dev));
3421 return 0;
3422}
3423
3424static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3425 unsigned long cache_level)
3426{
3427 struct drm_mm_node *gtt_space = &vma->node;
3428 struct drm_mm_node *other;
3429
3430 /*
3431 * On some machines we have to be careful when putting differing types
3432 * of snoopable memory together to avoid the prefetcher crossing memory
3433 * domains and dying. During vm initialisation, we decide whether or not
3434 * these constraints apply and set the drm_mm.color_adjust
3435 * appropriately.
3436 */
3437 if (vma->vm->mm.color_adjust == NULL)
3438 return true;
3439
3440 if (!drm_mm_node_allocated(gtt_space))
3441 return true;
3442
3443 if (list_empty(>t_space->node_list))
3444 return true;
3445
3446 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3447 if (other->allocated && !other->hole_follows && other->color != cache_level)
3448 return false;
3449
3450 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3451 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3452 return false;
3453
3454 return true;
3455}
3456
3457/**
3458 * Finds free space in the GTT aperture and binds the object or a view of it
3459 * there.
3460 */
3461static struct i915_vma *
3462i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3463 struct i915_address_space *vm,
3464 const struct i915_ggtt_view *ggtt_view,
3465 unsigned alignment,
3466 uint64_t flags)
3467{
3468 struct drm_device *dev = obj->base.dev;
3469 struct drm_i915_private *dev_priv = dev->dev_private;
3470 u32 fence_alignment, unfenced_alignment;
3471 u32 search_flag, alloc_flag;
3472 u64 start, end;
3473 u64 size, fence_size;
3474 struct i915_vma *vma;
3475 int ret;
3476
3477 if (i915_is_ggtt(vm)) {
3478 u32 view_size;
3479
3480 if (WARN_ON(!ggtt_view))
3481 return ERR_PTR(-EINVAL);
3482
3483 view_size = i915_ggtt_view_size(obj, ggtt_view);
3484
3485 fence_size = i915_gem_get_gtt_size(dev,
3486 view_size,
3487 obj->tiling_mode);
3488 fence_alignment = i915_gem_get_gtt_alignment(dev,
3489 view_size,
3490 obj->tiling_mode,
3491 true);
3492 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3493 view_size,
3494 obj->tiling_mode,
3495 false);
3496 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3497 } else {
3498 fence_size = i915_gem_get_gtt_size(dev,
3499 obj->base.size,
3500 obj->tiling_mode);
3501 fence_alignment = i915_gem_get_gtt_alignment(dev,
3502 obj->base.size,
3503 obj->tiling_mode,
3504 true);
3505 unfenced_alignment =
3506 i915_gem_get_gtt_alignment(dev,
3507 obj->base.size,
3508 obj->tiling_mode,
3509 false);
3510 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3511 }
3512
3513 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3514 end = vm->total;
3515 if (flags & PIN_MAPPABLE)
3516 end = min_t(u64, end, dev_priv->gtt.mappable_end);
3517 if (flags & PIN_ZONE_4G)
3518 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3519
3520 if (alignment == 0)
3521 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3522 unfenced_alignment;
3523 if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3524 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3525 ggtt_view ? ggtt_view->type : 0,
3526 alignment);
3527 return ERR_PTR(-EINVAL);
3528 }
3529
3530 /* If binding the object/GGTT view requires more space than the entire
3531 * aperture has, reject it early before evicting everything in a vain
3532 * attempt to find space.
3533 */
3534 if (size > end) {
3535 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
3536 ggtt_view ? ggtt_view->type : 0,
3537 size,
3538 flags & PIN_MAPPABLE ? "mappable" : "total",
3539 end);
3540 return ERR_PTR(-E2BIG);
3541 }
3542
3543 ret = i915_gem_object_get_pages(obj);
3544 if (ret)
3545 return ERR_PTR(ret);
3546
3547 i915_gem_object_pin_pages(obj);
3548
3549 vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3550 i915_gem_obj_lookup_or_create_vma(obj, vm);
3551
3552 if (IS_ERR(vma))
3553 goto err_unpin;
3554
3555 if (flags & PIN_OFFSET_FIXED) {
3556 uint64_t offset = flags & PIN_OFFSET_MASK;
3557
3558 if (offset & (alignment - 1) || offset + size > end) {
3559 ret = -EINVAL;
3560 goto err_free_vma;
3561 }
3562 vma->node.start = offset;
3563 vma->node.size = size;
3564 vma->node.color = obj->cache_level;
3565 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3566 if (ret) {
3567 ret = i915_gem_evict_for_vma(vma);
3568 if (ret == 0)
3569 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3570 }
3571 if (ret)
3572 goto err_free_vma;
3573 } else {
3574 if (flags & PIN_HIGH) {
3575 search_flag = DRM_MM_SEARCH_BELOW;
3576 alloc_flag = DRM_MM_CREATE_TOP;
3577 } else {
3578 search_flag = DRM_MM_SEARCH_DEFAULT;
3579 alloc_flag = DRM_MM_CREATE_DEFAULT;
3580 }
3581
3582search_free:
3583 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3584 size, alignment,
3585 obj->cache_level,
3586 start, end,
3587 search_flag,
3588 alloc_flag);
3589 if (ret) {
3590 ret = i915_gem_evict_something(dev, vm, size, alignment,
3591 obj->cache_level,
3592 start, end,
3593 flags);
3594 if (ret == 0)
3595 goto search_free;
3596
3597 goto err_free_vma;
3598 }
3599 }
3600 if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3601 ret = -EINVAL;
3602 goto err_remove_node;
3603 }
3604
3605 trace_i915_vma_bind(vma, flags);
3606 ret = i915_vma_bind(vma, obj->cache_level, flags);
3607 if (ret)
3608 goto err_remove_node;
3609
3610 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3611 list_add_tail(&vma->vm_link, &vm->inactive_list);
3612
3613 return vma;
3614
3615err_remove_node:
3616 drm_mm_remove_node(&vma->node);
3617err_free_vma:
3618 i915_gem_vma_destroy(vma);
3619 vma = ERR_PTR(ret);
3620err_unpin:
3621 i915_gem_object_unpin_pages(obj);
3622 return vma;
3623}
3624
3625bool
3626i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3627 bool force)
3628{
3629 /* If we don't have a page list set up, then we're not pinned
3630 * to GPU, and we can ignore the cache flush because it'll happen
3631 * again at bind time.
3632 */
3633 if (obj->pages == NULL)
3634 return false;
3635
3636 /*
3637 * Stolen memory is always coherent with the GPU as it is explicitly
3638 * marked as wc by the system, or the system is cache-coherent.
3639 */
3640 if (obj->stolen || obj->phys_handle)
3641 return false;
3642
3643 /* If the GPU is snooping the contents of the CPU cache,
3644 * we do not need to manually clear the CPU cache lines. However,
3645 * the caches are only snooped when the render cache is
3646 * flushed/invalidated. As we always have to emit invalidations
3647 * and flushes when moving into and out of the RENDER domain, correct
3648 * snooping behaviour occurs naturally as the result of our domain
3649 * tracking.
3650 */
3651 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3652 obj->cache_dirty = true;
3653 return false;
3654 }
3655
3656 trace_i915_gem_object_clflush(obj);
3657 drm_clflush_sg(obj->pages);
3658 obj->cache_dirty = false;
3659
3660 return true;
3661}
3662
3663/** Flushes the GTT write domain for the object if it's dirty. */
3664static void
3665i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3666{
3667 uint32_t old_write_domain;
3668
3669 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3670 return;
3671
3672 /* No actual flushing is required for the GTT write domain. Writes
3673 * to it immediately go to main memory as far as we know, so there's
3674 * no chipset flush. It also doesn't land in render cache.
3675 *
3676 * However, we do have to enforce the order so that all writes through
3677 * the GTT land before any writes to the device, such as updates to
3678 * the GATT itself.
3679 */
3680 wmb();
3681
3682 old_write_domain = obj->base.write_domain;
3683 obj->base.write_domain = 0;
3684
3685 intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3686
3687 trace_i915_gem_object_change_domain(obj,
3688 obj->base.read_domains,
3689 old_write_domain);
3690}
3691
3692/** Flushes the CPU write domain for the object if it's dirty. */
3693static void
3694i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3695{
3696 uint32_t old_write_domain;
3697
3698 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3699 return;
3700
3701 if (i915_gem_clflush_object(obj, obj->pin_display))
3702 i915_gem_chipset_flush(obj->base.dev);
3703
3704 old_write_domain = obj->base.write_domain;
3705 obj->base.write_domain = 0;
3706
3707 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3708
3709 trace_i915_gem_object_change_domain(obj,
3710 obj->base.read_domains,
3711 old_write_domain);
3712}
3713
3714/**
3715 * Moves a single object to the GTT read, and possibly write domain.
3716 *
3717 * This function returns when the move is complete, including waiting on
3718 * flushes to occur.
3719 */
3720int
3721i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3722{
3723 uint32_t old_write_domain, old_read_domains;
3724 struct i915_vma *vma;
3725 int ret;
3726
3727 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3728 return 0;
3729
3730 ret = i915_gem_object_wait_rendering(obj, !write);
3731 if (ret)
3732 return ret;
3733
3734 /* Flush and acquire obj->pages so that we are coherent through
3735 * direct access in memory with previous cached writes through
3736 * shmemfs and that our cache domain tracking remains valid.
3737 * For example, if the obj->filp was moved to swap without us
3738 * being notified and releasing the pages, we would mistakenly
3739 * continue to assume that the obj remained out of the CPU cached
3740 * domain.
3741 */
3742 ret = i915_gem_object_get_pages(obj);
3743 if (ret)
3744 return ret;
3745
3746 i915_gem_object_flush_cpu_write_domain(obj);
3747
3748 /* Serialise direct access to this object with the barriers for
3749 * coherent writes from the GPU, by effectively invalidating the
3750 * GTT domain upon first access.
3751 */
3752 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3753 mb();
3754
3755 old_write_domain = obj->base.write_domain;
3756 old_read_domains = obj->base.read_domains;
3757
3758 /* It should now be out of any other write domains, and we can update
3759 * the domain values for our changes.
3760 */
3761 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3762 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3763 if (write) {
3764 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3765 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3766 obj->dirty = 1;
3767 }
3768
3769 trace_i915_gem_object_change_domain(obj,
3770 old_read_domains,
3771 old_write_domain);
3772
3773 /* And bump the LRU for this access */
3774 vma = i915_gem_obj_to_ggtt(obj);
3775 if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
3776 list_move_tail(&vma->vm_link,
3777 &to_i915(obj->base.dev)->gtt.base.inactive_list);
3778
3779 return 0;
3780}
3781
3782/**
3783 * Changes the cache-level of an object across all VMA.
3784 *
3785 * After this function returns, the object will be in the new cache-level
3786 * across all GTT and the contents of the backing storage will be coherent,
3787 * with respect to the new cache-level. In order to keep the backing storage
3788 * coherent for all users, we only allow a single cache level to be set
3789 * globally on the object and prevent it from being changed whilst the
3790 * hardware is reading from the object. That is if the object is currently
3791 * on the scanout it will be set to uncached (or equivalent display
3792 * cache coherency) and all non-MOCS GPU access will also be uncached so
3793 * that all direct access to the scanout remains coherent.
3794 */
3795int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3796 enum i915_cache_level cache_level)
3797{
3798 struct drm_device *dev = obj->base.dev;
3799 struct i915_vma *vma, *next;
3800 bool bound = false;
3801 int ret = 0;
3802
3803 if (obj->cache_level == cache_level)
3804 goto out;
3805
3806 /* Inspect the list of currently bound VMA and unbind any that would
3807 * be invalid given the new cache-level. This is principally to
3808 * catch the issue of the CS prefetch crossing page boundaries and
3809 * reading an invalid PTE on older architectures.
3810 */
3811 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
3812 if (!drm_mm_node_allocated(&vma->node))
3813 continue;
3814
3815 if (vma->pin_count) {
3816 DRM_DEBUG("can not change the cache level of pinned objects\n");
3817 return -EBUSY;
3818 }
3819
3820 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3821 ret = i915_vma_unbind(vma);
3822 if (ret)
3823 return ret;
3824 } else
3825 bound = true;
3826 }
3827
3828 /* We can reuse the existing drm_mm nodes but need to change the
3829 * cache-level on the PTE. We could simply unbind them all and
3830 * rebind with the correct cache-level on next use. However since
3831 * we already have a valid slot, dma mapping, pages etc, we may as
3832 * rewrite the PTE in the belief that doing so tramples upon less
3833 * state and so involves less work.
3834 */
3835 if (bound) {
3836 /* Before we change the PTE, the GPU must not be accessing it.
3837 * If we wait upon the object, we know that all the bound
3838 * VMA are no longer active.
3839 */
3840 ret = i915_gem_object_wait_rendering(obj, false);
3841 if (ret)
3842 return ret;
3843
3844 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
3845 /* Access to snoopable pages through the GTT is
3846 * incoherent and on some machines causes a hard
3847 * lockup. Relinquish the CPU mmaping to force
3848 * userspace to refault in the pages and we can
3849 * then double check if the GTT mapping is still
3850 * valid for that pointer access.
3851 */
3852 i915_gem_release_mmap(obj);
3853
3854 /* As we no longer need a fence for GTT access,
3855 * we can relinquish it now (and so prevent having
3856 * to steal a fence from someone else on the next
3857 * fence request). Note GPU activity would have
3858 * dropped the fence as all snoopable access is
3859 * supposed to be linear.
3860 */
3861 ret = i915_gem_object_put_fence(obj);
3862 if (ret)
3863 return ret;
3864 } else {
3865 /* We either have incoherent backing store and
3866 * so no GTT access or the architecture is fully
3867 * coherent. In such cases, existing GTT mmaps
3868 * ignore the cache bit in the PTE and we can
3869 * rewrite it without confusing the GPU or having
3870 * to force userspace to fault back in its mmaps.
3871 */
3872 }
3873
3874 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3875 if (!drm_mm_node_allocated(&vma->node))
3876 continue;
3877
3878 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3879 if (ret)
3880 return ret;
3881 }
3882 }
3883
3884 list_for_each_entry(vma, &obj->vma_list, obj_link)
3885 vma->node.color = cache_level;
3886 obj->cache_level = cache_level;
3887
3888out:
3889 /* Flush the dirty CPU caches to the backing storage so that the
3890 * object is now coherent at its new cache level (with respect
3891 * to the access domain).
3892 */
3893 if (obj->cache_dirty &&
3894 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
3895 cpu_write_needs_clflush(obj)) {
3896 if (i915_gem_clflush_object(obj, true))
3897 i915_gem_chipset_flush(obj->base.dev);
3898 }
3899
3900 return 0;
3901}
3902
3903int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3904 struct drm_file *file)
3905{
3906 struct drm_i915_gem_caching *args = data;
3907 struct drm_i915_gem_object *obj;
3908
3909 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3910 if (&obj->base == NULL)
3911 return -ENOENT;
3912
3913 switch (obj->cache_level) {
3914 case I915_CACHE_LLC:
3915 case I915_CACHE_L3_LLC:
3916 args->caching = I915_CACHING_CACHED;
3917 break;
3918
3919 case I915_CACHE_WT:
3920 args->caching = I915_CACHING_DISPLAY;
3921 break;
3922
3923 default:
3924 args->caching = I915_CACHING_NONE;
3925 break;
3926 }
3927
3928 drm_gem_object_unreference_unlocked(&obj->base);
3929 return 0;
3930}
3931
3932int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3933 struct drm_file *file)
3934{
3935 struct drm_i915_private *dev_priv = dev->dev_private;
3936 struct drm_i915_gem_caching *args = data;
3937 struct drm_i915_gem_object *obj;
3938 enum i915_cache_level level;
3939 int ret;
3940
3941 switch (args->caching) {
3942 case I915_CACHING_NONE:
3943 level = I915_CACHE_NONE;
3944 break;
3945 case I915_CACHING_CACHED:
3946 /*
3947 * Due to a HW issue on BXT A stepping, GPU stores via a
3948 * snooped mapping may leave stale data in a corresponding CPU
3949 * cacheline, whereas normally such cachelines would get
3950 * invalidated.
3951 */
3952 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
3953 return -ENODEV;
3954
3955 level = I915_CACHE_LLC;
3956 break;
3957 case I915_CACHING_DISPLAY:
3958 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3959 break;
3960 default:
3961 return -EINVAL;
3962 }
3963
3964 intel_runtime_pm_get(dev_priv);
3965
3966 ret = i915_mutex_lock_interruptible(dev);
3967 if (ret)
3968 goto rpm_put;
3969
3970 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3971 if (&obj->base == NULL) {
3972 ret = -ENOENT;
3973 goto unlock;
3974 }
3975
3976 ret = i915_gem_object_set_cache_level(obj, level);
3977
3978 drm_gem_object_unreference(&obj->base);
3979unlock:
3980 mutex_unlock(&dev->struct_mutex);
3981rpm_put:
3982 intel_runtime_pm_put(dev_priv);
3983
3984 return ret;
3985}
3986
3987/*
3988 * Prepare buffer for display plane (scanout, cursors, etc).
3989 * Can be called from an uninterruptible phase (modesetting) and allows
3990 * any flushes to be pipelined (for pageflips).
3991 */
3992int
3993i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3994 u32 alignment,
3995 const struct i915_ggtt_view *view)
3996{
3997 u32 old_read_domains, old_write_domain;
3998 int ret;
3999
4000 /* Mark the pin_display early so that we account for the
4001 * display coherency whilst setting up the cache domains.
4002 */
4003 obj->pin_display++;
4004
4005 /* The display engine is not coherent with the LLC cache on gen6. As
4006 * a result, we make sure that the pinning that is about to occur is
4007 * done with uncached PTEs. This is lowest common denominator for all
4008 * chipsets.
4009 *
4010 * However for gen6+, we could do better by using the GFDT bit instead
4011 * of uncaching, which would allow us to flush all the LLC-cached data
4012 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4013 */
4014 ret = i915_gem_object_set_cache_level(obj,
4015 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
4016 if (ret)
4017 goto err_unpin_display;
4018
4019 /* As the user may map the buffer once pinned in the display plane
4020 * (e.g. libkms for the bootup splash), we have to ensure that we
4021 * always use map_and_fenceable for all scanout buffers.
4022 */
4023 ret = i915_gem_object_ggtt_pin(obj, view, alignment,
4024 view->type == I915_GGTT_VIEW_NORMAL ?
4025 PIN_MAPPABLE : 0);
4026 if (ret)
4027 goto err_unpin_display;
4028
4029 i915_gem_object_flush_cpu_write_domain(obj);
4030
4031 old_write_domain = obj->base.write_domain;
4032 old_read_domains = obj->base.read_domains;
4033
4034 /* It should now be out of any other write domains, and we can update
4035 * the domain values for our changes.
4036 */
4037 obj->base.write_domain = 0;
4038 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
4039
4040 trace_i915_gem_object_change_domain(obj,
4041 old_read_domains,
4042 old_write_domain);
4043
4044 return 0;
4045
4046err_unpin_display:
4047 obj->pin_display--;
4048 return ret;
4049}
4050
4051void
4052i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
4053 const struct i915_ggtt_view *view)
4054{
4055 if (WARN_ON(obj->pin_display == 0))
4056 return;
4057
4058 i915_gem_object_ggtt_unpin_view(obj, view);
4059
4060 obj->pin_display--;
4061}
4062
4063/**
4064 * Moves a single object to the CPU read, and possibly write domain.
4065 *
4066 * This function returns when the move is complete, including waiting on
4067 * flushes to occur.
4068 */
4069int
4070i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4071{
4072 uint32_t old_write_domain, old_read_domains;
4073 int ret;
4074
4075 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4076 return 0;
4077
4078 ret = i915_gem_object_wait_rendering(obj, !write);
4079 if (ret)
4080 return ret;
4081
4082 i915_gem_object_flush_gtt_write_domain(obj);
4083
4084 old_write_domain = obj->base.write_domain;
4085 old_read_domains = obj->base.read_domains;
4086
4087 /* Flush the CPU cache if it's still invalid. */
4088 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4089 i915_gem_clflush_object(obj, false);
4090
4091 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
4092 }
4093
4094 /* It should now be out of any other write domains, and we can update
4095 * the domain values for our changes.
4096 */
4097 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
4098
4099 /* If we're writing through the CPU, then the GPU read domains will
4100 * need to be invalidated at next use.
4101 */
4102 if (write) {
4103 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4104 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4105 }
4106
4107 trace_i915_gem_object_change_domain(obj,
4108 old_read_domains,
4109 old_write_domain);
4110
4111 return 0;
4112}
4113
4114/* Throttle our rendering by waiting until the ring has completed our requests
4115 * emitted over 20 msec ago.
4116 *
4117 * Note that if we were to use the current jiffies each time around the loop,
4118 * we wouldn't escape the function with any frames outstanding if the time to
4119 * render a frame was over 20ms.
4120 *
4121 * This should get us reasonable parallelism between CPU and GPU but also
4122 * relatively low latency when blocking on a particular request to finish.
4123 */
4124static int
4125i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4126{
4127 struct drm_i915_private *dev_priv = dev->dev_private;
4128 struct drm_i915_file_private *file_priv = file->driver_priv;
4129 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4130 struct drm_i915_gem_request *request, *target = NULL;
4131 unsigned reset_counter;
4132 int ret;
4133
4134 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4135 if (ret)
4136 return ret;
4137
4138 ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
4139 if (ret)
4140 return ret;
4141
4142 spin_lock(&file_priv->mm.lock);
4143 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
4144 if (time_after_eq(request->emitted_jiffies, recent_enough))
4145 break;
4146
4147 /*
4148 * Note that the request might not have been submitted yet.
4149 * In which case emitted_jiffies will be zero.
4150 */
4151 if (!request->emitted_jiffies)
4152 continue;
4153
4154 target = request;
4155 }
4156 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
4157 if (target)
4158 i915_gem_request_reference(target);
4159 spin_unlock(&file_priv->mm.lock);
4160
4161 if (target == NULL)
4162 return 0;
4163
4164 ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
4165 if (ret == 0)
4166 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4167
4168 i915_gem_request_unreference__unlocked(target);
4169
4170 return ret;
4171}
4172
4173static bool
4174i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4175{
4176 struct drm_i915_gem_object *obj = vma->obj;
4177
4178 if (alignment &&
4179 vma->node.start & (alignment - 1))
4180 return true;
4181
4182 if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4183 return true;
4184
4185 if (flags & PIN_OFFSET_BIAS &&
4186 vma->node.start < (flags & PIN_OFFSET_MASK))
4187 return true;
4188
4189 if (flags & PIN_OFFSET_FIXED &&
4190 vma->node.start != (flags & PIN_OFFSET_MASK))
4191 return true;
4192
4193 return false;
4194}
4195
4196void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
4197{
4198 struct drm_i915_gem_object *obj = vma->obj;
4199 bool mappable, fenceable;
4200 u32 fence_size, fence_alignment;
4201
4202 fence_size = i915_gem_get_gtt_size(obj->base.dev,
4203 obj->base.size,
4204 obj->tiling_mode);
4205 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4206 obj->base.size,
4207 obj->tiling_mode,
4208 true);
4209
4210 fenceable = (vma->node.size == fence_size &&
4211 (vma->node.start & (fence_alignment - 1)) == 0);
4212
4213 mappable = (vma->node.start + fence_size <=
4214 to_i915(obj->base.dev)->gtt.mappable_end);
4215
4216 obj->map_and_fenceable = mappable && fenceable;
4217}
4218
4219static int
4220i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
4221 struct i915_address_space *vm,
4222 const struct i915_ggtt_view *ggtt_view,
4223 uint32_t alignment,
4224 uint64_t flags)
4225{
4226 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4227 struct i915_vma *vma;
4228 unsigned bound;
4229 int ret;
4230
4231 if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4232 return -ENODEV;
4233
4234 if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4235 return -EINVAL;
4236
4237 if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4238 return -EINVAL;
4239
4240 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
4241 return -EINVAL;
4242
4243 vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
4244 i915_gem_obj_to_vma(obj, vm);
4245
4246 if (IS_ERR(vma))
4247 return PTR_ERR(vma);
4248
4249 if (vma) {
4250 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4251 return -EBUSY;
4252
4253 if (i915_vma_misplaced(vma, alignment, flags)) {
4254 WARN(vma->pin_count,
4255 "bo is already pinned in %s with incorrect alignment:"
4256 " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
4257 " obj->map_and_fenceable=%d\n",
4258 ggtt_view ? "ggtt" : "ppgtt",
4259 upper_32_bits(vma->node.start),
4260 lower_32_bits(vma->node.start),
4261 alignment,
4262 !!(flags & PIN_MAPPABLE),
4263 obj->map_and_fenceable);
4264 ret = i915_vma_unbind(vma);
4265 if (ret)
4266 return ret;
4267
4268 vma = NULL;
4269 }
4270 }
4271
4272 bound = vma ? vma->bound : 0;
4273 if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4274 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
4275 flags);
4276 if (IS_ERR(vma))
4277 return PTR_ERR(vma);
4278 } else {
4279 ret = i915_vma_bind(vma, obj->cache_level, flags);
4280 if (ret)
4281 return ret;
4282 }
4283
4284 if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
4285 (bound ^ vma->bound) & GLOBAL_BIND) {
4286 __i915_vma_set_map_and_fenceable(vma);
4287 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4288 }
4289
4290 vma->pin_count++;
4291 return 0;
4292}
4293
4294int
4295i915_gem_object_pin(struct drm_i915_gem_object *obj,
4296 struct i915_address_space *vm,
4297 uint32_t alignment,
4298 uint64_t flags)
4299{
4300 return i915_gem_object_do_pin(obj, vm,
4301 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
4302 alignment, flags);
4303}
4304
4305int
4306i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4307 const struct i915_ggtt_view *view,
4308 uint32_t alignment,
4309 uint64_t flags)
4310{
4311 if (WARN_ONCE(!view, "no view specified"))
4312 return -EINVAL;
4313
4314 return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
4315 alignment, flags | PIN_GLOBAL);
4316}
4317
4318void
4319i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
4320 const struct i915_ggtt_view *view)
4321{
4322 struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
4323
4324 BUG_ON(!vma);
4325 WARN_ON(vma->pin_count == 0);
4326 WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
4327
4328 --vma->pin_count;
4329}
4330
4331int
4332i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4333 struct drm_file *file)
4334{
4335 struct drm_i915_gem_busy *args = data;
4336 struct drm_i915_gem_object *obj;
4337 int ret;
4338
4339 ret = i915_mutex_lock_interruptible(dev);
4340 if (ret)
4341 return ret;
4342
4343 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4344 if (&obj->base == NULL) {
4345 ret = -ENOENT;
4346 goto unlock;
4347 }
4348
4349 /* Count all active objects as busy, even if they are currently not used
4350 * by the gpu. Users of this interface expect objects to eventually
4351 * become non-busy without any further actions, therefore emit any
4352 * necessary flushes here.
4353 */
4354 ret = i915_gem_object_flush_active(obj);
4355 if (ret)
4356 goto unref;
4357
4358 args->busy = 0;
4359 if (obj->active) {
4360 int i;
4361
4362 for (i = 0; i < I915_NUM_RINGS; i++) {
4363 struct drm_i915_gem_request *req;
4364
4365 req = obj->last_read_req[i];
4366 if (req)
4367 args->busy |= 1 << (16 + req->ring->exec_id);
4368 }
4369 if (obj->last_write_req)
4370 args->busy |= obj->last_write_req->ring->exec_id;
4371 }
4372
4373unref:
4374 drm_gem_object_unreference(&obj->base);
4375unlock:
4376 mutex_unlock(&dev->struct_mutex);
4377 return ret;
4378}
4379
4380int
4381i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4382 struct drm_file *file_priv)
4383{
4384 return i915_gem_ring_throttle(dev, file_priv);
4385}
4386
4387int
4388i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4389 struct drm_file *file_priv)
4390{
4391 struct drm_i915_private *dev_priv = dev->dev_private;
4392 struct drm_i915_gem_madvise *args = data;
4393 struct drm_i915_gem_object *obj;
4394 int ret;
4395
4396 switch (args->madv) {
4397 case I915_MADV_DONTNEED:
4398 case I915_MADV_WILLNEED:
4399 break;
4400 default:
4401 return -EINVAL;
4402 }
4403
4404 ret = i915_mutex_lock_interruptible(dev);
4405 if (ret)
4406 return ret;
4407
4408 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4409 if (&obj->base == NULL) {
4410 ret = -ENOENT;
4411 goto unlock;
4412 }
4413
4414 if (i915_gem_obj_is_pinned(obj)) {
4415 ret = -EINVAL;
4416 goto out;
4417 }
4418
4419 if (obj->pages &&
4420 obj->tiling_mode != I915_TILING_NONE &&
4421 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4422 if (obj->madv == I915_MADV_WILLNEED)
4423 i915_gem_object_unpin_pages(obj);
4424 if (args->madv == I915_MADV_WILLNEED)
4425 i915_gem_object_pin_pages(obj);
4426 }
4427
4428 if (obj->madv != __I915_MADV_PURGED)
4429 obj->madv = args->madv;
4430
4431 /* if the object is no longer attached, discard its backing storage */
4432 if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4433 i915_gem_object_truncate(obj);
4434
4435 args->retained = obj->madv != __I915_MADV_PURGED;
4436
4437out:
4438 drm_gem_object_unreference(&obj->base);
4439unlock:
4440 mutex_unlock(&dev->struct_mutex);
4441 return ret;
4442}
4443
4444void i915_gem_object_init(struct drm_i915_gem_object *obj,
4445 const struct drm_i915_gem_object_ops *ops)
4446{
4447 int i;
4448
4449 INIT_LIST_HEAD(&obj->global_list);
4450 for (i = 0; i < I915_NUM_RINGS; i++)
4451 INIT_LIST_HEAD(&obj->ring_list[i]);
4452 INIT_LIST_HEAD(&obj->obj_exec_link);
4453 INIT_LIST_HEAD(&obj->vma_list);
4454 INIT_LIST_HEAD(&obj->batch_pool_link);
4455
4456 obj->ops = ops;
4457
4458 obj->fence_reg = I915_FENCE_REG_NONE;
4459 obj->madv = I915_MADV_WILLNEED;
4460
4461 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4462}
4463
4464static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4465 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4466 .get_pages = i915_gem_object_get_pages_gtt,
4467 .put_pages = i915_gem_object_put_pages_gtt,
4468};
4469
4470struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4471 size_t size)
4472{
4473 struct drm_i915_gem_object *obj;
4474 struct address_space *mapping;
4475 gfp_t mask;
4476
4477 obj = i915_gem_object_alloc(dev);
4478 if (obj == NULL)
4479 return NULL;
4480
4481 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4482 i915_gem_object_free(obj);
4483 return NULL;
4484 }
4485
4486 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4487 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4488 /* 965gm cannot relocate objects above 4GiB. */
4489 mask &= ~__GFP_HIGHMEM;
4490 mask |= __GFP_DMA32;
4491 }
4492
4493 mapping = file_inode(obj->base.filp)->i_mapping;
4494 mapping_set_gfp_mask(mapping, mask);
4495
4496 i915_gem_object_init(obj, &i915_gem_object_ops);
4497
4498 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4499 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4500
4501 if (HAS_LLC(dev)) {
4502 /* On some devices, we can have the GPU use the LLC (the CPU
4503 * cache) for about a 10% performance improvement
4504 * compared to uncached. Graphics requests other than
4505 * display scanout are coherent with the CPU in
4506 * accessing this cache. This means in this mode we
4507 * don't need to clflush on the CPU side, and on the
4508 * GPU side we only need to flush internal caches to
4509 * get data visible to the CPU.
4510 *
4511 * However, we maintain the display planes as UC, and so
4512 * need to rebind when first used as such.
4513 */
4514 obj->cache_level = I915_CACHE_LLC;
4515 } else
4516 obj->cache_level = I915_CACHE_NONE;
4517
4518 trace_i915_gem_object_create(obj);
4519
4520 return obj;
4521}
4522
4523static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4524{
4525 /* If we are the last user of the backing storage (be it shmemfs
4526 * pages or stolen etc), we know that the pages are going to be
4527 * immediately released. In this case, we can then skip copying
4528 * back the contents from the GPU.
4529 */
4530
4531 if (obj->madv != I915_MADV_WILLNEED)
4532 return false;
4533
4534 if (obj->base.filp == NULL)
4535 return true;
4536
4537 /* At first glance, this looks racy, but then again so would be
4538 * userspace racing mmap against close. However, the first external
4539 * reference to the filp can only be obtained through the
4540 * i915_gem_mmap_ioctl() which safeguards us against the user
4541 * acquiring such a reference whilst we are in the middle of
4542 * freeing the object.
4543 */
4544 return atomic_long_read(&obj->base.filp->f_count) == 1;
4545}
4546
4547void i915_gem_free_object(struct drm_gem_object *gem_obj)
4548{
4549 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4550 struct drm_device *dev = obj->base.dev;
4551 struct drm_i915_private *dev_priv = dev->dev_private;
4552 struct i915_vma *vma, *next;
4553
4554 intel_runtime_pm_get(dev_priv);
4555
4556 trace_i915_gem_object_destroy(obj);
4557
4558 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4559 int ret;
4560
4561 vma->pin_count = 0;
4562 ret = i915_vma_unbind(vma);
4563 if (WARN_ON(ret == -ERESTARTSYS)) {
4564 bool was_interruptible;
4565
4566 was_interruptible = dev_priv->mm.interruptible;
4567 dev_priv->mm.interruptible = false;
4568
4569 WARN_ON(i915_vma_unbind(vma));
4570
4571 dev_priv->mm.interruptible = was_interruptible;
4572 }
4573 }
4574
4575 /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4576 * before progressing. */
4577 if (obj->stolen)
4578 i915_gem_object_unpin_pages(obj);
4579
4580 WARN_ON(obj->frontbuffer_bits);
4581
4582 if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4583 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4584 obj->tiling_mode != I915_TILING_NONE)
4585 i915_gem_object_unpin_pages(obj);
4586
4587 if (WARN_ON(obj->pages_pin_count))
4588 obj->pages_pin_count = 0;
4589 if (discard_backing_storage(obj))
4590 obj->madv = I915_MADV_DONTNEED;
4591 i915_gem_object_put_pages(obj);
4592 i915_gem_object_free_mmap_offset(obj);
4593
4594 BUG_ON(obj->pages);
4595
4596 if (obj->base.import_attach)
4597 drm_prime_gem_destroy(&obj->base, NULL);
4598
4599 if (obj->ops->release)
4600 obj->ops->release(obj);
4601
4602 drm_gem_object_release(&obj->base);
4603 i915_gem_info_remove_obj(dev_priv, obj->base.size);
4604
4605 kfree(obj->bit_17);
4606 i915_gem_object_free(obj);
4607
4608 intel_runtime_pm_put(dev_priv);
4609}
4610
4611struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4612 struct i915_address_space *vm)
4613{
4614 struct i915_vma *vma;
4615 list_for_each_entry(vma, &obj->vma_list, obj_link) {
4616 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4617 vma->vm == vm)
4618 return vma;
4619 }
4620 return NULL;
4621}
4622
4623struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4624 const struct i915_ggtt_view *view)
4625{
4626 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
4627 struct i915_vma *vma;
4628
4629 if (WARN_ONCE(!view, "no view specified"))
4630 return ERR_PTR(-EINVAL);
4631
4632 list_for_each_entry(vma, &obj->vma_list, obj_link)
4633 if (vma->vm == ggtt &&
4634 i915_ggtt_view_equal(&vma->ggtt_view, view))
4635 return vma;
4636 return NULL;
4637}
4638
4639void i915_gem_vma_destroy(struct i915_vma *vma)
4640{
4641 WARN_ON(vma->node.allocated);
4642
4643 /* Keep the vma as a placeholder in the execbuffer reservation lists */
4644 if (!list_empty(&vma->exec_list))
4645 return;
4646
4647 if (!vma->is_ggtt)
4648 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
4649
4650 list_del(&vma->obj_link);
4651
4652 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
4653}
4654
4655static void
4656i915_gem_stop_ringbuffers(struct drm_device *dev)
4657{
4658 struct drm_i915_private *dev_priv = dev->dev_private;
4659 struct intel_engine_cs *ring;
4660 int i;
4661
4662 for_each_ring(ring, dev_priv, i)
4663 dev_priv->gt.stop_ring(ring);
4664}
4665
4666int
4667i915_gem_suspend(struct drm_device *dev)
4668{
4669 struct drm_i915_private *dev_priv = dev->dev_private;
4670 int ret = 0;
4671
4672 mutex_lock(&dev->struct_mutex);
4673 ret = i915_gpu_idle(dev);
4674 if (ret)
4675 goto err;
4676
4677 i915_gem_retire_requests(dev);
4678
4679 i915_gem_stop_ringbuffers(dev);
4680 mutex_unlock(&dev->struct_mutex);
4681
4682 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4683 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4684 flush_delayed_work(&dev_priv->mm.idle_work);
4685
4686 /* Assert that we sucessfully flushed all the work and
4687 * reset the GPU back to its idle, low power state.
4688 */
4689 WARN_ON(dev_priv->mm.busy);
4690
4691 return 0;
4692
4693err:
4694 mutex_unlock(&dev->struct_mutex);
4695 return ret;
4696}
4697
4698int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
4699{
4700 struct intel_engine_cs *ring = req->ring;
4701 struct drm_device *dev = ring->dev;
4702 struct drm_i915_private *dev_priv = dev->dev_private;
4703 u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4704 int i, ret;
4705
4706 if (!HAS_L3_DPF(dev) || !remap_info)
4707 return 0;
4708
4709 ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
4710 if (ret)
4711 return ret;
4712
4713 /*
4714 * Note: We do not worry about the concurrent register cacheline hang
4715 * here because no other code should access these registers other than
4716 * at initialization time.
4717 */
4718 for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
4719 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
4720 intel_ring_emit_reg(ring, GEN7_L3LOG(slice, i));
4721 intel_ring_emit(ring, remap_info[i]);
4722 }
4723
4724 intel_ring_advance(ring);
4725
4726 return ret;
4727}
4728
4729void i915_gem_init_swizzling(struct drm_device *dev)
4730{
4731 struct drm_i915_private *dev_priv = dev->dev_private;
4732
4733 if (INTEL_INFO(dev)->gen < 5 ||
4734 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4735 return;
4736
4737 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4738 DISP_TILE_SURFACE_SWIZZLING);
4739
4740 if (IS_GEN5(dev))
4741 return;
4742
4743 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4744 if (IS_GEN6(dev))
4745 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4746 else if (IS_GEN7(dev))
4747 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4748 else if (IS_GEN8(dev))
4749 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4750 else
4751 BUG();
4752}
4753
4754static void init_unused_ring(struct drm_device *dev, u32 base)
4755{
4756 struct drm_i915_private *dev_priv = dev->dev_private;
4757
4758 I915_WRITE(RING_CTL(base), 0);
4759 I915_WRITE(RING_HEAD(base), 0);
4760 I915_WRITE(RING_TAIL(base), 0);
4761 I915_WRITE(RING_START(base), 0);
4762}
4763
4764static void init_unused_rings(struct drm_device *dev)
4765{
4766 if (IS_I830(dev)) {
4767 init_unused_ring(dev, PRB1_BASE);
4768 init_unused_ring(dev, SRB0_BASE);
4769 init_unused_ring(dev, SRB1_BASE);
4770 init_unused_ring(dev, SRB2_BASE);
4771 init_unused_ring(dev, SRB3_BASE);
4772 } else if (IS_GEN2(dev)) {
4773 init_unused_ring(dev, SRB0_BASE);
4774 init_unused_ring(dev, SRB1_BASE);
4775 } else if (IS_GEN3(dev)) {
4776 init_unused_ring(dev, PRB1_BASE);
4777 init_unused_ring(dev, PRB2_BASE);
4778 }
4779}
4780
4781int i915_gem_init_rings(struct drm_device *dev)
4782{
4783 struct drm_i915_private *dev_priv = dev->dev_private;
4784 int ret;
4785
4786 ret = intel_init_render_ring_buffer(dev);
4787 if (ret)
4788 return ret;
4789
4790 if (HAS_BSD(dev)) {
4791 ret = intel_init_bsd_ring_buffer(dev);
4792 if (ret)
4793 goto cleanup_render_ring;
4794 }
4795
4796 if (HAS_BLT(dev)) {
4797 ret = intel_init_blt_ring_buffer(dev);
4798 if (ret)
4799 goto cleanup_bsd_ring;
4800 }
4801
4802 if (HAS_VEBOX(dev)) {
4803 ret = intel_init_vebox_ring_buffer(dev);
4804 if (ret)
4805 goto cleanup_blt_ring;
4806 }
4807
4808 if (HAS_BSD2(dev)) {
4809 ret = intel_init_bsd2_ring_buffer(dev);
4810 if (ret)
4811 goto cleanup_vebox_ring;
4812 }
4813
4814 return 0;
4815
4816cleanup_vebox_ring:
4817 intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4818cleanup_blt_ring:
4819 intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4820cleanup_bsd_ring:
4821 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4822cleanup_render_ring:
4823 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4824
4825 return ret;
4826}
4827
4828int
4829i915_gem_init_hw(struct drm_device *dev)
4830{
4831 struct drm_i915_private *dev_priv = dev->dev_private;
4832 struct intel_engine_cs *ring;
4833 int ret, i, j;
4834
4835 if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4836 return -EIO;
4837
4838 /* Double layer security blanket, see i915_gem_init() */
4839 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4840
4841 if (dev_priv->ellc_size)
4842 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4843
4844 if (IS_HASWELL(dev))
4845 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4846 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4847
4848 if (HAS_PCH_NOP(dev)) {
4849 if (IS_IVYBRIDGE(dev)) {
4850 u32 temp = I915_READ(GEN7_MSG_CTL);
4851 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4852 I915_WRITE(GEN7_MSG_CTL, temp);
4853 } else if (INTEL_INFO(dev)->gen >= 7) {
4854 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4855 temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4856 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4857 }
4858 }
4859
4860 i915_gem_init_swizzling(dev);
4861
4862 /*
4863 * At least 830 can leave some of the unused rings
4864 * "active" (ie. head != tail) after resume which
4865 * will prevent c3 entry. Makes sure all unused rings
4866 * are totally idle.
4867 */
4868 init_unused_rings(dev);
4869
4870 BUG_ON(!dev_priv->kernel_context);
4871
4872 ret = i915_ppgtt_init_hw(dev);
4873 if (ret) {
4874 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4875 goto out;
4876 }
4877
4878 /* Need to do basic initialisation of all rings first: */
4879 for_each_ring(ring, dev_priv, i) {
4880 ret = ring->init_hw(ring);
4881 if (ret)
4882 goto out;
4883 }
4884
4885 /* We can't enable contexts until all firmware is loaded */
4886 if (HAS_GUC_UCODE(dev)) {
4887 ret = intel_guc_ucode_load(dev);
4888 if (ret) {
4889 DRM_ERROR("Failed to initialize GuC, error %d\n", ret);
4890 ret = -EIO;
4891 goto out;
4892 }
4893 }
4894
4895 /*
4896 * Increment the next seqno by 0x100 so we have a visible break
4897 * on re-initialisation
4898 */
4899 ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
4900 if (ret)
4901 goto out;
4902
4903 /* Now it is safe to go back round and do everything else: */
4904 for_each_ring(ring, dev_priv, i) {
4905 struct drm_i915_gem_request *req;
4906
4907 req = i915_gem_request_alloc(ring, NULL);
4908 if (IS_ERR(req)) {
4909 ret = PTR_ERR(req);
4910 i915_gem_cleanup_ringbuffer(dev);
4911 goto out;
4912 }
4913
4914 if (ring->id == RCS) {
4915 for (j = 0; j < NUM_L3_SLICES(dev); j++)
4916 i915_gem_l3_remap(req, j);
4917 }
4918
4919 ret = i915_ppgtt_init_ring(req);
4920 if (ret && ret != -EIO) {
4921 DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
4922 i915_gem_request_cancel(req);
4923 i915_gem_cleanup_ringbuffer(dev);
4924 goto out;
4925 }
4926
4927 ret = i915_gem_context_enable(req);
4928 if (ret && ret != -EIO) {
4929 DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
4930 i915_gem_request_cancel(req);
4931 i915_gem_cleanup_ringbuffer(dev);
4932 goto out;
4933 }
4934
4935 i915_add_request_no_flush(req);
4936 }
4937
4938out:
4939 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4940 return ret;
4941}
4942
4943int i915_gem_init(struct drm_device *dev)
4944{
4945 struct drm_i915_private *dev_priv = dev->dev_private;
4946 int ret;
4947
4948 i915.enable_execlists = intel_sanitize_enable_execlists(dev,
4949 i915.enable_execlists);
4950
4951 mutex_lock(&dev->struct_mutex);
4952
4953 if (!i915.enable_execlists) {
4954 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
4955 dev_priv->gt.init_rings = i915_gem_init_rings;
4956 dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
4957 dev_priv->gt.stop_ring = intel_stop_ring_buffer;
4958 } else {
4959 dev_priv->gt.execbuf_submit = intel_execlists_submission;
4960 dev_priv->gt.init_rings = intel_logical_rings_init;
4961 dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
4962 dev_priv->gt.stop_ring = intel_logical_ring_stop;
4963 }
4964
4965 /* This is just a security blanket to placate dragons.
4966 * On some systems, we very sporadically observe that the first TLBs
4967 * used by the CS may be stale, despite us poking the TLB reset. If
4968 * we hold the forcewake during initialisation these problems
4969 * just magically go away.
4970 */
4971 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4972
4973 ret = i915_gem_init_userptr(dev);
4974 if (ret)
4975 goto out_unlock;
4976
4977 i915_gem_init_global_gtt(dev);
4978
4979 ret = i915_gem_context_init(dev);
4980 if (ret)
4981 goto out_unlock;
4982
4983 ret = dev_priv->gt.init_rings(dev);
4984 if (ret)
4985 goto out_unlock;
4986
4987 ret = i915_gem_init_hw(dev);
4988 if (ret == -EIO) {
4989 /* Allow ring initialisation to fail by marking the GPU as
4990 * wedged. But we only want to do this where the GPU is angry,
4991 * for all other failure, such as an allocation failure, bail.
4992 */
4993 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4994 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4995 ret = 0;
4996 }
4997
4998out_unlock:
4999 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5000 mutex_unlock(&dev->struct_mutex);
5001
5002 return ret;
5003}
5004
5005void
5006i915_gem_cleanup_ringbuffer(struct drm_device *dev)
5007{
5008 struct drm_i915_private *dev_priv = dev->dev_private;
5009 struct intel_engine_cs *ring;
5010 int i;
5011
5012 for_each_ring(ring, dev_priv, i)
5013 dev_priv->gt.cleanup_ring(ring);
5014
5015 if (i915.enable_execlists)
5016 /*
5017 * Neither the BIOS, ourselves or any other kernel
5018 * expects the system to be in execlists mode on startup,
5019 * so we need to reset the GPU back to legacy mode.
5020 */
5021 intel_gpu_reset(dev);
5022}
5023
5024static void
5025init_ring_lists(struct intel_engine_cs *ring)
5026{
5027 INIT_LIST_HEAD(&ring->active_list);
5028 INIT_LIST_HEAD(&ring->request_list);
5029}
5030
5031void
5032i915_gem_load_init(struct drm_device *dev)
5033{
5034 struct drm_i915_private *dev_priv = dev->dev_private;
5035 int i;
5036
5037 dev_priv->objects =
5038 kmem_cache_create("i915_gem_object",
5039 sizeof(struct drm_i915_gem_object), 0,
5040 SLAB_HWCACHE_ALIGN,
5041 NULL);
5042 dev_priv->vmas =
5043 kmem_cache_create("i915_gem_vma",
5044 sizeof(struct i915_vma), 0,
5045 SLAB_HWCACHE_ALIGN,
5046 NULL);
5047 dev_priv->requests =
5048 kmem_cache_create("i915_gem_request",
5049 sizeof(struct drm_i915_gem_request), 0,
5050 SLAB_HWCACHE_ALIGN,
5051 NULL);
5052
5053 INIT_LIST_HEAD(&dev_priv->vm_list);
5054 INIT_LIST_HEAD(&dev_priv->context_list);
5055 INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5056 INIT_LIST_HEAD(&dev_priv->mm.bound_list);
5057 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5058 for (i = 0; i < I915_NUM_RINGS; i++)
5059 init_ring_lists(&dev_priv->ring[i]);
5060 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
5061 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
5062 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5063 i915_gem_retire_work_handler);
5064 INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5065 i915_gem_idle_work_handler);
5066 init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5067
5068 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5069
5070 if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
5071 dev_priv->num_fence_regs = 32;
5072 else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
5073 dev_priv->num_fence_regs = 16;
5074 else
5075 dev_priv->num_fence_regs = 8;
5076
5077 if (intel_vgpu_active(dev))
5078 dev_priv->num_fence_regs =
5079 I915_READ(vgtif_reg(avail_rs.fence_num));
5080
5081 /*
5082 * Set initial sequence number for requests.
5083 * Using this number allows the wraparound to happen early,
5084 * catching any obvious problems.
5085 */
5086 dev_priv->next_seqno = ((u32)~0 - 0x1100);
5087 dev_priv->last_seqno = ((u32)~0 - 0x1101);
5088
5089 /* Initialize fence registers to zero */
5090 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5091 i915_gem_restore_fences(dev);
5092
5093 i915_gem_detect_bit_6_swizzle(dev);
5094 init_waitqueue_head(&dev_priv->pending_flip_queue);
5095
5096 dev_priv->mm.interruptible = true;
5097
5098 mutex_init(&dev_priv->fb_tracking.lock);
5099}
5100
5101void i915_gem_load_cleanup(struct drm_device *dev)
5102{
5103 struct drm_i915_private *dev_priv = to_i915(dev);
5104
5105 kmem_cache_destroy(dev_priv->requests);
5106 kmem_cache_destroy(dev_priv->vmas);
5107 kmem_cache_destroy(dev_priv->objects);
5108}
5109
5110void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5111{
5112 struct drm_i915_file_private *file_priv = file->driver_priv;
5113
5114 /* Clean up our request list when the client is going away, so that
5115 * later retire_requests won't dereference our soon-to-be-gone
5116 * file_priv.
5117 */
5118 spin_lock(&file_priv->mm.lock);
5119 while (!list_empty(&file_priv->mm.request_list)) {
5120 struct drm_i915_gem_request *request;
5121
5122 request = list_first_entry(&file_priv->mm.request_list,
5123 struct drm_i915_gem_request,
5124 client_list);
5125 list_del(&request->client_list);
5126 request->file_priv = NULL;
5127 }
5128 spin_unlock(&file_priv->mm.lock);
5129
5130 if (!list_empty(&file_priv->rps.link)) {
5131 spin_lock(&to_i915(dev)->rps.client_lock);
5132 list_del(&file_priv->rps.link);
5133 spin_unlock(&to_i915(dev)->rps.client_lock);
5134 }
5135}
5136
5137int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5138{
5139 struct drm_i915_file_private *file_priv;
5140 int ret;
5141
5142 DRM_DEBUG_DRIVER("\n");
5143
5144 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5145 if (!file_priv)
5146 return -ENOMEM;
5147
5148 file->driver_priv = file_priv;
5149 file_priv->dev_priv = dev->dev_private;
5150 file_priv->file = file;
5151 INIT_LIST_HEAD(&file_priv->rps.link);
5152
5153 spin_lock_init(&file_priv->mm.lock);
5154 INIT_LIST_HEAD(&file_priv->mm.request_list);
5155
5156 file_priv->bsd_ring = -1;
5157
5158 ret = i915_gem_context_open(dev, file);
5159 if (ret)
5160 kfree(file_priv);
5161
5162 return ret;
5163}
5164
5165/**
5166 * i915_gem_track_fb - update frontbuffer tracking
5167 * @old: current GEM buffer for the frontbuffer slots
5168 * @new: new GEM buffer for the frontbuffer slots
5169 * @frontbuffer_bits: bitmask of frontbuffer slots
5170 *
5171 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5172 * from @old and setting them in @new. Both @old and @new can be NULL.
5173 */
5174void i915_gem_track_fb(struct drm_i915_gem_object *old,
5175 struct drm_i915_gem_object *new,
5176 unsigned frontbuffer_bits)
5177{
5178 if (old) {
5179 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5180 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5181 old->frontbuffer_bits &= ~frontbuffer_bits;
5182 }
5183
5184 if (new) {
5185 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5186 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5187 new->frontbuffer_bits |= frontbuffer_bits;
5188 }
5189}
5190
5191/* All the new VM stuff */
5192u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
5193 struct i915_address_space *vm)
5194{
5195 struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5196 struct i915_vma *vma;
5197
5198 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5199
5200 list_for_each_entry(vma, &o->vma_list, obj_link) {
5201 if (vma->is_ggtt &&
5202 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5203 continue;
5204 if (vma->vm == vm)
5205 return vma->node.start;
5206 }
5207
5208 WARN(1, "%s vma for this object not found.\n",
5209 i915_is_ggtt(vm) ? "global" : "ppgtt");
5210 return -1;
5211}
5212
5213u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
5214 const struct i915_ggtt_view *view)
5215{
5216 struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5217 struct i915_vma *vma;
5218
5219 list_for_each_entry(vma, &o->vma_list, obj_link)
5220 if (vma->vm == ggtt &&
5221 i915_ggtt_view_equal(&vma->ggtt_view, view))
5222 return vma->node.start;
5223
5224 WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
5225 return -1;
5226}
5227
5228bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5229 struct i915_address_space *vm)
5230{
5231 struct i915_vma *vma;
5232
5233 list_for_each_entry(vma, &o->vma_list, obj_link) {
5234 if (vma->is_ggtt &&
5235 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5236 continue;
5237 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5238 return true;
5239 }
5240
5241 return false;
5242}
5243
5244bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
5245 const struct i915_ggtt_view *view)
5246{
5247 struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5248 struct i915_vma *vma;
5249
5250 list_for_each_entry(vma, &o->vma_list, obj_link)
5251 if (vma->vm == ggtt &&
5252 i915_ggtt_view_equal(&vma->ggtt_view, view) &&
5253 drm_mm_node_allocated(&vma->node))
5254 return true;
5255
5256 return false;
5257}
5258
5259bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5260{
5261 struct i915_vma *vma;
5262
5263 list_for_each_entry(vma, &o->vma_list, obj_link)
5264 if (drm_mm_node_allocated(&vma->node))
5265 return true;
5266
5267 return false;
5268}
5269
5270unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5271 struct i915_address_space *vm)
5272{
5273 struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5274 struct i915_vma *vma;
5275
5276 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5277
5278 BUG_ON(list_empty(&o->vma_list));
5279
5280 list_for_each_entry(vma, &o->vma_list, obj_link) {
5281 if (vma->is_ggtt &&
5282 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5283 continue;
5284 if (vma->vm == vm)
5285 return vma->node.size;
5286 }
5287 return 0;
5288}
5289
5290bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
5291{
5292 struct i915_vma *vma;
5293 list_for_each_entry(vma, &obj->vma_list, obj_link)
5294 if (vma->pin_count > 0)
5295 return true;
5296
5297 return false;
5298}
5299
5300/* Like i915_gem_object_get_page(), but mark the returned page dirty */
5301struct page *
5302i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
5303{
5304 struct page *page;
5305
5306 /* Only default objects have per-page dirty tracking */
5307 if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
5308 return NULL;
5309
5310 page = i915_gem_object_get_page(obj, n);
5311 set_page_dirty(page);
5312 return page;
5313}
5314
5315/* Allocate a new GEM object and fill it with the supplied data */
5316struct drm_i915_gem_object *
5317i915_gem_object_create_from_data(struct drm_device *dev,
5318 const void *data, size_t size)
5319{
5320 struct drm_i915_gem_object *obj;
5321 struct sg_table *sg;
5322 size_t bytes;
5323 int ret;
5324
5325 obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
5326 if (IS_ERR_OR_NULL(obj))
5327 return obj;
5328
5329 ret = i915_gem_object_set_to_cpu_domain(obj, true);
5330 if (ret)
5331 goto fail;
5332
5333 ret = i915_gem_object_get_pages(obj);
5334 if (ret)
5335 goto fail;
5336
5337 i915_gem_object_pin_pages(obj);
5338 sg = obj->pages;
5339 bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
5340 obj->dirty = 1; /* Backing store is now out of date */
5341 i915_gem_object_unpin_pages(obj);
5342
5343 if (WARN_ON(bytes != size)) {
5344 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
5345 ret = -EFAULT;
5346 goto fail;
5347 }
5348
5349 return obj;
5350
5351fail:
5352 drm_gem_object_unreference(&obj->base);
5353 return ERR_PTR(ret);
5354}
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include <drm/drm_vma_manager.h>
29#include <drm/i915_drm.h>
30#include <linux/dma-fence-array.h>
31#include <linux/kthread.h>
32#include <linux/dma-resv.h>
33#include <linux/shmem_fs.h>
34#include <linux/slab.h>
35#include <linux/stop_machine.h>
36#include <linux/swap.h>
37#include <linux/pci.h>
38#include <linux/dma-buf.h>
39#include <linux/mman.h>
40
41#include "display/intel_display.h"
42#include "display/intel_frontbuffer.h"
43
44#include "gem/i915_gem_clflush.h"
45#include "gem/i915_gem_context.h"
46#include "gem/i915_gem_ioctls.h"
47#include "gem/i915_gem_pm.h"
48#include "gem/i915_gemfs.h"
49#include "gt/intel_engine_user.h"
50#include "gt/intel_gt.h"
51#include "gt/intel_gt_pm.h"
52#include "gt/intel_mocs.h"
53#include "gt/intel_reset.h"
54#include "gt/intel_renderstate.h"
55#include "gt/intel_workarounds.h"
56
57#include "i915_drv.h"
58#include "i915_scatterlist.h"
59#include "i915_trace.h"
60#include "i915_vgpu.h"
61
62#include "intel_pm.h"
63
64static int
65insert_mappable_node(struct i915_ggtt *ggtt,
66 struct drm_mm_node *node, u32 size)
67{
68 memset(node, 0, sizeof(*node));
69 return drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
70 size, 0, I915_COLOR_UNEVICTABLE,
71 0, ggtt->mappable_end,
72 DRM_MM_INSERT_LOW);
73}
74
75static void
76remove_mappable_node(struct drm_mm_node *node)
77{
78 drm_mm_remove_node(node);
79}
80
81int
82i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
83 struct drm_file *file)
84{
85 struct i915_ggtt *ggtt = &to_i915(dev)->ggtt;
86 struct drm_i915_gem_get_aperture *args = data;
87 struct i915_vma *vma;
88 u64 pinned;
89
90 mutex_lock(&ggtt->vm.mutex);
91
92 pinned = ggtt->vm.reserved;
93 list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
94 if (i915_vma_is_pinned(vma))
95 pinned += vma->node.size;
96
97 mutex_unlock(&ggtt->vm.mutex);
98
99 args->aper_size = ggtt->vm.total;
100 args->aper_available_size = args->aper_size - pinned;
101
102 return 0;
103}
104
105int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
106 unsigned long flags)
107{
108 struct i915_vma *vma;
109 LIST_HEAD(still_in_list);
110 int ret = 0;
111
112 lockdep_assert_held(&obj->base.dev->struct_mutex);
113
114 spin_lock(&obj->vma.lock);
115 while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
116 struct i915_vma,
117 obj_link))) {
118 list_move_tail(&vma->obj_link, &still_in_list);
119 spin_unlock(&obj->vma.lock);
120
121 ret = -EBUSY;
122 if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
123 !i915_vma_is_active(vma))
124 ret = i915_vma_unbind(vma);
125
126 spin_lock(&obj->vma.lock);
127 }
128 list_splice(&still_in_list, &obj->vma.list);
129 spin_unlock(&obj->vma.lock);
130
131 return ret;
132}
133
134static int
135i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
136 struct drm_i915_gem_pwrite *args,
137 struct drm_file *file)
138{
139 void *vaddr = obj->phys_handle->vaddr + args->offset;
140 char __user *user_data = u64_to_user_ptr(args->data_ptr);
141
142 /*
143 * We manually control the domain here and pretend that it
144 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
145 */
146 intel_frontbuffer_invalidate(obj->frontbuffer, ORIGIN_CPU);
147
148 if (copy_from_user(vaddr, user_data, args->size))
149 return -EFAULT;
150
151 drm_clflush_virt_range(vaddr, args->size);
152 intel_gt_chipset_flush(&to_i915(obj->base.dev)->gt);
153
154 intel_frontbuffer_flush(obj->frontbuffer, ORIGIN_CPU);
155 return 0;
156}
157
158static int
159i915_gem_create(struct drm_file *file,
160 struct drm_i915_private *dev_priv,
161 u64 *size_p,
162 u32 *handle_p)
163{
164 struct drm_i915_gem_object *obj;
165 u32 handle;
166 u64 size;
167 int ret;
168
169 size = round_up(*size_p, PAGE_SIZE);
170 if (size == 0)
171 return -EINVAL;
172
173 /* Allocate the new object */
174 obj = i915_gem_object_create_shmem(dev_priv, size);
175 if (IS_ERR(obj))
176 return PTR_ERR(obj);
177
178 ret = drm_gem_handle_create(file, &obj->base, &handle);
179 /* drop reference from allocate - handle holds it now */
180 i915_gem_object_put(obj);
181 if (ret)
182 return ret;
183
184 *handle_p = handle;
185 *size_p = size;
186 return 0;
187}
188
189int
190i915_gem_dumb_create(struct drm_file *file,
191 struct drm_device *dev,
192 struct drm_mode_create_dumb *args)
193{
194 int cpp = DIV_ROUND_UP(args->bpp, 8);
195 u32 format;
196
197 switch (cpp) {
198 case 1:
199 format = DRM_FORMAT_C8;
200 break;
201 case 2:
202 format = DRM_FORMAT_RGB565;
203 break;
204 case 4:
205 format = DRM_FORMAT_XRGB8888;
206 break;
207 default:
208 return -EINVAL;
209 }
210
211 /* have to work out size/pitch and return them */
212 args->pitch = ALIGN(args->width * cpp, 64);
213
214 /* align stride to page size so that we can remap */
215 if (args->pitch > intel_plane_fb_max_stride(to_i915(dev), format,
216 DRM_FORMAT_MOD_LINEAR))
217 args->pitch = ALIGN(args->pitch, 4096);
218
219 args->size = args->pitch * args->height;
220 return i915_gem_create(file, to_i915(dev),
221 &args->size, &args->handle);
222}
223
224/**
225 * Creates a new mm object and returns a handle to it.
226 * @dev: drm device pointer
227 * @data: ioctl data blob
228 * @file: drm file pointer
229 */
230int
231i915_gem_create_ioctl(struct drm_device *dev, void *data,
232 struct drm_file *file)
233{
234 struct drm_i915_private *dev_priv = to_i915(dev);
235 struct drm_i915_gem_create *args = data;
236
237 i915_gem_flush_free_objects(dev_priv);
238
239 return i915_gem_create(file, dev_priv,
240 &args->size, &args->handle);
241}
242
243static int
244shmem_pread(struct page *page, int offset, int len, char __user *user_data,
245 bool needs_clflush)
246{
247 char *vaddr;
248 int ret;
249
250 vaddr = kmap(page);
251
252 if (needs_clflush)
253 drm_clflush_virt_range(vaddr + offset, len);
254
255 ret = __copy_to_user(user_data, vaddr + offset, len);
256
257 kunmap(page);
258
259 return ret ? -EFAULT : 0;
260}
261
262static int
263i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
264 struct drm_i915_gem_pread *args)
265{
266 unsigned int needs_clflush;
267 unsigned int idx, offset;
268 struct dma_fence *fence;
269 char __user *user_data;
270 u64 remain;
271 int ret;
272
273 ret = i915_gem_object_prepare_read(obj, &needs_clflush);
274 if (ret)
275 return ret;
276
277 fence = i915_gem_object_lock_fence(obj);
278 i915_gem_object_finish_access(obj);
279 if (!fence)
280 return -ENOMEM;
281
282 remain = args->size;
283 user_data = u64_to_user_ptr(args->data_ptr);
284 offset = offset_in_page(args->offset);
285 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
286 struct page *page = i915_gem_object_get_page(obj, idx);
287 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
288
289 ret = shmem_pread(page, offset, length, user_data,
290 needs_clflush);
291 if (ret)
292 break;
293
294 remain -= length;
295 user_data += length;
296 offset = 0;
297 }
298
299 i915_gem_object_unlock_fence(obj, fence);
300 return ret;
301}
302
303static inline bool
304gtt_user_read(struct io_mapping *mapping,
305 loff_t base, int offset,
306 char __user *user_data, int length)
307{
308 void __iomem *vaddr;
309 unsigned long unwritten;
310
311 /* We can use the cpu mem copy function because this is X86. */
312 vaddr = io_mapping_map_atomic_wc(mapping, base);
313 unwritten = __copy_to_user_inatomic(user_data,
314 (void __force *)vaddr + offset,
315 length);
316 io_mapping_unmap_atomic(vaddr);
317 if (unwritten) {
318 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
319 unwritten = copy_to_user(user_data,
320 (void __force *)vaddr + offset,
321 length);
322 io_mapping_unmap(vaddr);
323 }
324 return unwritten;
325}
326
327static int
328i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
329 const struct drm_i915_gem_pread *args)
330{
331 struct drm_i915_private *i915 = to_i915(obj->base.dev);
332 struct i915_ggtt *ggtt = &i915->ggtt;
333 intel_wakeref_t wakeref;
334 struct drm_mm_node node;
335 struct dma_fence *fence;
336 void __user *user_data;
337 struct i915_vma *vma;
338 u64 remain, offset;
339 int ret;
340
341 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
342 if (ret)
343 return ret;
344
345 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
346 vma = ERR_PTR(-ENODEV);
347 if (!i915_gem_object_is_tiled(obj))
348 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
349 PIN_MAPPABLE |
350 PIN_NONBLOCK /* NOWARN */ |
351 PIN_NOEVICT);
352 if (!IS_ERR(vma)) {
353 node.start = i915_ggtt_offset(vma);
354 node.allocated = false;
355 } else {
356 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
357 if (ret)
358 goto out_unlock;
359 GEM_BUG_ON(!node.allocated);
360 }
361
362 mutex_unlock(&i915->drm.struct_mutex);
363
364 ret = i915_gem_object_lock_interruptible(obj);
365 if (ret)
366 goto out_unpin;
367
368 ret = i915_gem_object_set_to_gtt_domain(obj, false);
369 if (ret) {
370 i915_gem_object_unlock(obj);
371 goto out_unpin;
372 }
373
374 fence = i915_gem_object_lock_fence(obj);
375 i915_gem_object_unlock(obj);
376 if (!fence) {
377 ret = -ENOMEM;
378 goto out_unpin;
379 }
380
381 user_data = u64_to_user_ptr(args->data_ptr);
382 remain = args->size;
383 offset = args->offset;
384
385 while (remain > 0) {
386 /* Operation in this page
387 *
388 * page_base = page offset within aperture
389 * page_offset = offset within page
390 * page_length = bytes to copy for this page
391 */
392 u32 page_base = node.start;
393 unsigned page_offset = offset_in_page(offset);
394 unsigned page_length = PAGE_SIZE - page_offset;
395 page_length = remain < page_length ? remain : page_length;
396 if (node.allocated) {
397 ggtt->vm.insert_page(&ggtt->vm,
398 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
399 node.start, I915_CACHE_NONE, 0);
400 } else {
401 page_base += offset & PAGE_MASK;
402 }
403
404 if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
405 user_data, page_length)) {
406 ret = -EFAULT;
407 break;
408 }
409
410 remain -= page_length;
411 user_data += page_length;
412 offset += page_length;
413 }
414
415 i915_gem_object_unlock_fence(obj, fence);
416out_unpin:
417 mutex_lock(&i915->drm.struct_mutex);
418 if (node.allocated) {
419 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
420 remove_mappable_node(&node);
421 } else {
422 i915_vma_unpin(vma);
423 }
424out_unlock:
425 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
426 mutex_unlock(&i915->drm.struct_mutex);
427
428 return ret;
429}
430
431/**
432 * Reads data from the object referenced by handle.
433 * @dev: drm device pointer
434 * @data: ioctl data blob
435 * @file: drm file pointer
436 *
437 * On error, the contents of *data are undefined.
438 */
439int
440i915_gem_pread_ioctl(struct drm_device *dev, void *data,
441 struct drm_file *file)
442{
443 struct drm_i915_gem_pread *args = data;
444 struct drm_i915_gem_object *obj;
445 int ret;
446
447 if (args->size == 0)
448 return 0;
449
450 if (!access_ok(u64_to_user_ptr(args->data_ptr),
451 args->size))
452 return -EFAULT;
453
454 obj = i915_gem_object_lookup(file, args->handle);
455 if (!obj)
456 return -ENOENT;
457
458 /* Bounds check source. */
459 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
460 ret = -EINVAL;
461 goto out;
462 }
463
464 trace_i915_gem_object_pread(obj, args->offset, args->size);
465
466 ret = i915_gem_object_wait(obj,
467 I915_WAIT_INTERRUPTIBLE,
468 MAX_SCHEDULE_TIMEOUT);
469 if (ret)
470 goto out;
471
472 ret = i915_gem_object_pin_pages(obj);
473 if (ret)
474 goto out;
475
476 ret = i915_gem_shmem_pread(obj, args);
477 if (ret == -EFAULT || ret == -ENODEV)
478 ret = i915_gem_gtt_pread(obj, args);
479
480 i915_gem_object_unpin_pages(obj);
481out:
482 i915_gem_object_put(obj);
483 return ret;
484}
485
486/* This is the fast write path which cannot handle
487 * page faults in the source data
488 */
489
490static inline bool
491ggtt_write(struct io_mapping *mapping,
492 loff_t base, int offset,
493 char __user *user_data, int length)
494{
495 void __iomem *vaddr;
496 unsigned long unwritten;
497
498 /* We can use the cpu mem copy function because this is X86. */
499 vaddr = io_mapping_map_atomic_wc(mapping, base);
500 unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
501 user_data, length);
502 io_mapping_unmap_atomic(vaddr);
503 if (unwritten) {
504 vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
505 unwritten = copy_from_user((void __force *)vaddr + offset,
506 user_data, length);
507 io_mapping_unmap(vaddr);
508 }
509
510 return unwritten;
511}
512
513/**
514 * This is the fast pwrite path, where we copy the data directly from the
515 * user into the GTT, uncached.
516 * @obj: i915 GEM object
517 * @args: pwrite arguments structure
518 */
519static int
520i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
521 const struct drm_i915_gem_pwrite *args)
522{
523 struct drm_i915_private *i915 = to_i915(obj->base.dev);
524 struct i915_ggtt *ggtt = &i915->ggtt;
525 struct intel_runtime_pm *rpm = &i915->runtime_pm;
526 intel_wakeref_t wakeref;
527 struct drm_mm_node node;
528 struct dma_fence *fence;
529 struct i915_vma *vma;
530 u64 remain, offset;
531 void __user *user_data;
532 int ret;
533
534 ret = mutex_lock_interruptible(&i915->drm.struct_mutex);
535 if (ret)
536 return ret;
537
538 if (i915_gem_object_has_struct_page(obj)) {
539 /*
540 * Avoid waking the device up if we can fallback, as
541 * waking/resuming is very slow (worst-case 10-100 ms
542 * depending on PCI sleeps and our own resume time).
543 * This easily dwarfs any performance advantage from
544 * using the cache bypass of indirect GGTT access.
545 */
546 wakeref = intel_runtime_pm_get_if_in_use(rpm);
547 if (!wakeref) {
548 ret = -EFAULT;
549 goto out_unlock;
550 }
551 } else {
552 /* No backing pages, no fallback, we must force GGTT access */
553 wakeref = intel_runtime_pm_get(rpm);
554 }
555
556 vma = ERR_PTR(-ENODEV);
557 if (!i915_gem_object_is_tiled(obj))
558 vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
559 PIN_MAPPABLE |
560 PIN_NONBLOCK /* NOWARN */ |
561 PIN_NOEVICT);
562 if (!IS_ERR(vma)) {
563 node.start = i915_ggtt_offset(vma);
564 node.allocated = false;
565 } else {
566 ret = insert_mappable_node(ggtt, &node, PAGE_SIZE);
567 if (ret)
568 goto out_rpm;
569 GEM_BUG_ON(!node.allocated);
570 }
571
572 mutex_unlock(&i915->drm.struct_mutex);
573
574 ret = i915_gem_object_lock_interruptible(obj);
575 if (ret)
576 goto out_unpin;
577
578 ret = i915_gem_object_set_to_gtt_domain(obj, true);
579 if (ret) {
580 i915_gem_object_unlock(obj);
581 goto out_unpin;
582 }
583
584 fence = i915_gem_object_lock_fence(obj);
585 i915_gem_object_unlock(obj);
586 if (!fence) {
587 ret = -ENOMEM;
588 goto out_unpin;
589 }
590
591 intel_frontbuffer_invalidate(obj->frontbuffer, ORIGIN_CPU);
592
593 user_data = u64_to_user_ptr(args->data_ptr);
594 offset = args->offset;
595 remain = args->size;
596 while (remain) {
597 /* Operation in this page
598 *
599 * page_base = page offset within aperture
600 * page_offset = offset within page
601 * page_length = bytes to copy for this page
602 */
603 u32 page_base = node.start;
604 unsigned int page_offset = offset_in_page(offset);
605 unsigned int page_length = PAGE_SIZE - page_offset;
606 page_length = remain < page_length ? remain : page_length;
607 if (node.allocated) {
608 /* flush the write before we modify the GGTT */
609 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
610 ggtt->vm.insert_page(&ggtt->vm,
611 i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
612 node.start, I915_CACHE_NONE, 0);
613 wmb(); /* flush modifications to the GGTT (insert_page) */
614 } else {
615 page_base += offset & PAGE_MASK;
616 }
617 /* If we get a fault while copying data, then (presumably) our
618 * source page isn't available. Return the error and we'll
619 * retry in the slow path.
620 * If the object is non-shmem backed, we retry again with the
621 * path that handles page fault.
622 */
623 if (ggtt_write(&ggtt->iomap, page_base, page_offset,
624 user_data, page_length)) {
625 ret = -EFAULT;
626 break;
627 }
628
629 remain -= page_length;
630 user_data += page_length;
631 offset += page_length;
632 }
633 intel_frontbuffer_flush(obj->frontbuffer, ORIGIN_CPU);
634
635 i915_gem_object_unlock_fence(obj, fence);
636out_unpin:
637 mutex_lock(&i915->drm.struct_mutex);
638 intel_gt_flush_ggtt_writes(ggtt->vm.gt);
639 if (node.allocated) {
640 ggtt->vm.clear_range(&ggtt->vm, node.start, node.size);
641 remove_mappable_node(&node);
642 } else {
643 i915_vma_unpin(vma);
644 }
645out_rpm:
646 intel_runtime_pm_put(rpm, wakeref);
647out_unlock:
648 mutex_unlock(&i915->drm.struct_mutex);
649 return ret;
650}
651
652/* Per-page copy function for the shmem pwrite fastpath.
653 * Flushes invalid cachelines before writing to the target if
654 * needs_clflush_before is set and flushes out any written cachelines after
655 * writing if needs_clflush is set.
656 */
657static int
658shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
659 bool needs_clflush_before,
660 bool needs_clflush_after)
661{
662 char *vaddr;
663 int ret;
664
665 vaddr = kmap(page);
666
667 if (needs_clflush_before)
668 drm_clflush_virt_range(vaddr + offset, len);
669
670 ret = __copy_from_user(vaddr + offset, user_data, len);
671 if (!ret && needs_clflush_after)
672 drm_clflush_virt_range(vaddr + offset, len);
673
674 kunmap(page);
675
676 return ret ? -EFAULT : 0;
677}
678
679static int
680i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
681 const struct drm_i915_gem_pwrite *args)
682{
683 unsigned int partial_cacheline_write;
684 unsigned int needs_clflush;
685 unsigned int offset, idx;
686 struct dma_fence *fence;
687 void __user *user_data;
688 u64 remain;
689 int ret;
690
691 ret = i915_gem_object_prepare_write(obj, &needs_clflush);
692 if (ret)
693 return ret;
694
695 fence = i915_gem_object_lock_fence(obj);
696 i915_gem_object_finish_access(obj);
697 if (!fence)
698 return -ENOMEM;
699
700 /* If we don't overwrite a cacheline completely we need to be
701 * careful to have up-to-date data by first clflushing. Don't
702 * overcomplicate things and flush the entire patch.
703 */
704 partial_cacheline_write = 0;
705 if (needs_clflush & CLFLUSH_BEFORE)
706 partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
707
708 user_data = u64_to_user_ptr(args->data_ptr);
709 remain = args->size;
710 offset = offset_in_page(args->offset);
711 for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
712 struct page *page = i915_gem_object_get_page(obj, idx);
713 unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
714
715 ret = shmem_pwrite(page, offset, length, user_data,
716 (offset | length) & partial_cacheline_write,
717 needs_clflush & CLFLUSH_AFTER);
718 if (ret)
719 break;
720
721 remain -= length;
722 user_data += length;
723 offset = 0;
724 }
725
726 intel_frontbuffer_flush(obj->frontbuffer, ORIGIN_CPU);
727 i915_gem_object_unlock_fence(obj, fence);
728
729 return ret;
730}
731
732/**
733 * Writes data to the object referenced by handle.
734 * @dev: drm device
735 * @data: ioctl data blob
736 * @file: drm file
737 *
738 * On error, the contents of the buffer that were to be modified are undefined.
739 */
740int
741i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
742 struct drm_file *file)
743{
744 struct drm_i915_gem_pwrite *args = data;
745 struct drm_i915_gem_object *obj;
746 int ret;
747
748 if (args->size == 0)
749 return 0;
750
751 if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
752 return -EFAULT;
753
754 obj = i915_gem_object_lookup(file, args->handle);
755 if (!obj)
756 return -ENOENT;
757
758 /* Bounds check destination. */
759 if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
760 ret = -EINVAL;
761 goto err;
762 }
763
764 /* Writes not allowed into this read-only object */
765 if (i915_gem_object_is_readonly(obj)) {
766 ret = -EINVAL;
767 goto err;
768 }
769
770 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
771
772 ret = -ENODEV;
773 if (obj->ops->pwrite)
774 ret = obj->ops->pwrite(obj, args);
775 if (ret != -ENODEV)
776 goto err;
777
778 ret = i915_gem_object_wait(obj,
779 I915_WAIT_INTERRUPTIBLE |
780 I915_WAIT_ALL,
781 MAX_SCHEDULE_TIMEOUT);
782 if (ret)
783 goto err;
784
785 ret = i915_gem_object_pin_pages(obj);
786 if (ret)
787 goto err;
788
789 ret = -EFAULT;
790 /* We can only do the GTT pwrite on untiled buffers, as otherwise
791 * it would end up going through the fenced access, and we'll get
792 * different detiling behavior between reading and writing.
793 * pread/pwrite currently are reading and writing from the CPU
794 * perspective, requiring manual detiling by the client.
795 */
796 if (!i915_gem_object_has_struct_page(obj) ||
797 cpu_write_needs_clflush(obj))
798 /* Note that the gtt paths might fail with non-page-backed user
799 * pointers (e.g. gtt mappings when moving data between
800 * textures). Fallback to the shmem path in that case.
801 */
802 ret = i915_gem_gtt_pwrite_fast(obj, args);
803
804 if (ret == -EFAULT || ret == -ENOSPC) {
805 if (obj->phys_handle)
806 ret = i915_gem_phys_pwrite(obj, args, file);
807 else
808 ret = i915_gem_shmem_pwrite(obj, args);
809 }
810
811 i915_gem_object_unpin_pages(obj);
812err:
813 i915_gem_object_put(obj);
814 return ret;
815}
816
817/**
818 * Called when user space has done writes to this buffer
819 * @dev: drm device
820 * @data: ioctl data blob
821 * @file: drm file
822 */
823int
824i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
825 struct drm_file *file)
826{
827 struct drm_i915_gem_sw_finish *args = data;
828 struct drm_i915_gem_object *obj;
829
830 obj = i915_gem_object_lookup(file, args->handle);
831 if (!obj)
832 return -ENOENT;
833
834 /*
835 * Proxy objects are barred from CPU access, so there is no
836 * need to ban sw_finish as it is a nop.
837 */
838
839 /* Pinned buffers may be scanout, so flush the cache */
840 i915_gem_object_flush_if_display(obj);
841 i915_gem_object_put(obj);
842
843 return 0;
844}
845
846void i915_gem_runtime_suspend(struct drm_i915_private *i915)
847{
848 struct drm_i915_gem_object *obj, *on;
849 int i;
850
851 /*
852 * Only called during RPM suspend. All users of the userfault_list
853 * must be holding an RPM wakeref to ensure that this can not
854 * run concurrently with themselves (and use the struct_mutex for
855 * protection between themselves).
856 */
857
858 list_for_each_entry_safe(obj, on,
859 &i915->ggtt.userfault_list, userfault_link)
860 __i915_gem_object_release_mmap(obj);
861
862 /*
863 * The fence will be lost when the device powers down. If any were
864 * in use by hardware (i.e. they are pinned), we should not be powering
865 * down! All other fences will be reacquired by the user upon waking.
866 */
867 for (i = 0; i < i915->ggtt.num_fences; i++) {
868 struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i];
869
870 /*
871 * Ideally we want to assert that the fence register is not
872 * live at this point (i.e. that no piece of code will be
873 * trying to write through fence + GTT, as that both violates
874 * our tracking of activity and associated locking/barriers,
875 * but also is illegal given that the hw is powered down).
876 *
877 * Previously we used reg->pin_count as a "liveness" indicator.
878 * That is not sufficient, and we need a more fine-grained
879 * tool if we want to have a sanity check here.
880 */
881
882 if (!reg->vma)
883 continue;
884
885 GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
886 reg->dirty = true;
887 }
888}
889
890static long
891wait_for_timelines(struct drm_i915_private *i915,
892 unsigned int wait, long timeout)
893{
894 struct intel_gt_timelines *timelines = &i915->gt.timelines;
895 struct intel_timeline *tl;
896 unsigned long flags;
897
898 spin_lock_irqsave(&timelines->lock, flags);
899 list_for_each_entry(tl, &timelines->active_list, link) {
900 struct i915_request *rq;
901
902 rq = i915_active_request_get_unlocked(&tl->last_request);
903 if (!rq)
904 continue;
905
906 spin_unlock_irqrestore(&timelines->lock, flags);
907
908 /*
909 * "Race-to-idle".
910 *
911 * Switching to the kernel context is often used a synchronous
912 * step prior to idling, e.g. in suspend for flushing all
913 * current operations to memory before sleeping. These we
914 * want to complete as quickly as possible to avoid prolonged
915 * stalls, so allow the gpu to boost to maximum clocks.
916 */
917 if (wait & I915_WAIT_FOR_IDLE_BOOST)
918 gen6_rps_boost(rq);
919
920 timeout = i915_request_wait(rq, wait, timeout);
921 i915_request_put(rq);
922 if (timeout < 0)
923 return timeout;
924
925 /* restart after reacquiring the lock */
926 spin_lock_irqsave(&timelines->lock, flags);
927 tl = list_entry(&timelines->active_list, typeof(*tl), link);
928 }
929 spin_unlock_irqrestore(&timelines->lock, flags);
930
931 return timeout;
932}
933
934int i915_gem_wait_for_idle(struct drm_i915_private *i915,
935 unsigned int flags, long timeout)
936{
937 /* If the device is asleep, we have no requests outstanding */
938 if (!intel_gt_pm_is_awake(&i915->gt))
939 return 0;
940
941 GEM_TRACE("flags=%x (%s), timeout=%ld%s\n",
942 flags, flags & I915_WAIT_LOCKED ? "locked" : "unlocked",
943 timeout, timeout == MAX_SCHEDULE_TIMEOUT ? " (forever)" : "");
944
945 timeout = wait_for_timelines(i915, flags, timeout);
946 if (timeout < 0)
947 return timeout;
948
949 if (flags & I915_WAIT_LOCKED) {
950 lockdep_assert_held(&i915->drm.struct_mutex);
951
952 i915_retire_requests(i915);
953 }
954
955 return 0;
956}
957
958struct i915_vma *
959i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
960 const struct i915_ggtt_view *view,
961 u64 size,
962 u64 alignment,
963 u64 flags)
964{
965 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
966 struct i915_address_space *vm = &dev_priv->ggtt.vm;
967
968 return i915_gem_object_pin(obj, vm, view, size, alignment,
969 flags | PIN_GLOBAL);
970}
971
972struct i915_vma *
973i915_gem_object_pin(struct drm_i915_gem_object *obj,
974 struct i915_address_space *vm,
975 const struct i915_ggtt_view *view,
976 u64 size,
977 u64 alignment,
978 u64 flags)
979{
980 struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
981 struct i915_vma *vma;
982 int ret;
983
984 lockdep_assert_held(&obj->base.dev->struct_mutex);
985
986 if (i915_gem_object_never_bind_ggtt(obj))
987 return ERR_PTR(-ENODEV);
988
989 if (flags & PIN_MAPPABLE &&
990 (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
991 /* If the required space is larger than the available
992 * aperture, we will not able to find a slot for the
993 * object and unbinding the object now will be in
994 * vain. Worse, doing so may cause us to ping-pong
995 * the object in and out of the Global GTT and
996 * waste a lot of cycles under the mutex.
997 */
998 if (obj->base.size > dev_priv->ggtt.mappable_end)
999 return ERR_PTR(-E2BIG);
1000
1001 /* If NONBLOCK is set the caller is optimistically
1002 * trying to cache the full object within the mappable
1003 * aperture, and *must* have a fallback in place for
1004 * situations where we cannot bind the object. We
1005 * can be a little more lax here and use the fallback
1006 * more often to avoid costly migrations of ourselves
1007 * and other objects within the aperture.
1008 *
1009 * Half-the-aperture is used as a simple heuristic.
1010 * More interesting would to do search for a free
1011 * block prior to making the commitment to unbind.
1012 * That caters for the self-harm case, and with a
1013 * little more heuristics (e.g. NOFAULT, NOEVICT)
1014 * we could try to minimise harm to others.
1015 */
1016 if (flags & PIN_NONBLOCK &&
1017 obj->base.size > dev_priv->ggtt.mappable_end / 2)
1018 return ERR_PTR(-ENOSPC);
1019 }
1020
1021 vma = i915_vma_instance(obj, vm, view);
1022 if (IS_ERR(vma))
1023 return vma;
1024
1025 if (i915_vma_misplaced(vma, size, alignment, flags)) {
1026 if (flags & PIN_NONBLOCK) {
1027 if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
1028 return ERR_PTR(-ENOSPC);
1029
1030 if (flags & PIN_MAPPABLE &&
1031 vma->fence_size > dev_priv->ggtt.mappable_end / 2)
1032 return ERR_PTR(-ENOSPC);
1033 }
1034
1035 WARN(i915_vma_is_pinned(vma),
1036 "bo is already pinned in ggtt with incorrect alignment:"
1037 " offset=%08x, req.alignment=%llx,"
1038 " req.map_and_fenceable=%d, vma->map_and_fenceable=%d\n",
1039 i915_ggtt_offset(vma), alignment,
1040 !!(flags & PIN_MAPPABLE),
1041 i915_vma_is_map_and_fenceable(vma));
1042 ret = i915_vma_unbind(vma);
1043 if (ret)
1044 return ERR_PTR(ret);
1045 }
1046
1047 if (vma->fence && !i915_gem_object_is_tiled(obj)) {
1048 mutex_lock(&vma->vm->mutex);
1049 ret = i915_vma_revoke_fence(vma);
1050 mutex_unlock(&vma->vm->mutex);
1051 if (ret)
1052 return ERR_PTR(ret);
1053 }
1054
1055 ret = i915_vma_pin(vma, size, alignment, flags);
1056 if (ret)
1057 return ERR_PTR(ret);
1058
1059 return vma;
1060}
1061
1062int
1063i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
1064 struct drm_file *file_priv)
1065{
1066 struct drm_i915_private *i915 = to_i915(dev);
1067 struct drm_i915_gem_madvise *args = data;
1068 struct drm_i915_gem_object *obj;
1069 int err;
1070
1071 switch (args->madv) {
1072 case I915_MADV_DONTNEED:
1073 case I915_MADV_WILLNEED:
1074 break;
1075 default:
1076 return -EINVAL;
1077 }
1078
1079 obj = i915_gem_object_lookup(file_priv, args->handle);
1080 if (!obj)
1081 return -ENOENT;
1082
1083 err = mutex_lock_interruptible(&obj->mm.lock);
1084 if (err)
1085 goto out;
1086
1087 if (i915_gem_object_has_pages(obj) &&
1088 i915_gem_object_is_tiled(obj) &&
1089 i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
1090 if (obj->mm.madv == I915_MADV_WILLNEED) {
1091 GEM_BUG_ON(!obj->mm.quirked);
1092 __i915_gem_object_unpin_pages(obj);
1093 obj->mm.quirked = false;
1094 }
1095 if (args->madv == I915_MADV_WILLNEED) {
1096 GEM_BUG_ON(obj->mm.quirked);
1097 __i915_gem_object_pin_pages(obj);
1098 obj->mm.quirked = true;
1099 }
1100 }
1101
1102 if (obj->mm.madv != __I915_MADV_PURGED)
1103 obj->mm.madv = args->madv;
1104
1105 if (i915_gem_object_has_pages(obj)) {
1106 struct list_head *list;
1107
1108 if (i915_gem_object_is_shrinkable(obj)) {
1109 unsigned long flags;
1110
1111 spin_lock_irqsave(&i915->mm.obj_lock, flags);
1112
1113 if (obj->mm.madv != I915_MADV_WILLNEED)
1114 list = &i915->mm.purge_list;
1115 else
1116 list = &i915->mm.shrink_list;
1117 list_move_tail(&obj->mm.link, list);
1118
1119 spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1120 }
1121 }
1122
1123 /* if the object is no longer attached, discard its backing storage */
1124 if (obj->mm.madv == I915_MADV_DONTNEED &&
1125 !i915_gem_object_has_pages(obj))
1126 i915_gem_object_truncate(obj);
1127
1128 args->retained = obj->mm.madv != __I915_MADV_PURGED;
1129 mutex_unlock(&obj->mm.lock);
1130
1131out:
1132 i915_gem_object_put(obj);
1133 return err;
1134}
1135
1136void i915_gem_sanitize(struct drm_i915_private *i915)
1137{
1138 intel_wakeref_t wakeref;
1139
1140 GEM_TRACE("\n");
1141
1142 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1143 intel_uncore_forcewake_get(&i915->uncore, FORCEWAKE_ALL);
1144
1145 /*
1146 * As we have just resumed the machine and woken the device up from
1147 * deep PCI sleep (presumably D3_cold), assume the HW has been reset
1148 * back to defaults, recovering from whatever wedged state we left it
1149 * in and so worth trying to use the device once more.
1150 */
1151 if (intel_gt_is_wedged(&i915->gt))
1152 intel_gt_unset_wedged(&i915->gt);
1153
1154 /*
1155 * If we inherit context state from the BIOS or earlier occupants
1156 * of the GPU, the GPU may be in an inconsistent state when we
1157 * try to take over. The only way to remove the earlier state
1158 * is by resetting. However, resetting on earlier gen is tricky as
1159 * it may impact the display and we are uncertain about the stability
1160 * of the reset, so this could be applied to even earlier gen.
1161 */
1162 intel_gt_sanitize(&i915->gt, false);
1163
1164 intel_uncore_forcewake_put(&i915->uncore, FORCEWAKE_ALL);
1165 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1166}
1167
1168static void init_unused_ring(struct intel_gt *gt, u32 base)
1169{
1170 struct intel_uncore *uncore = gt->uncore;
1171
1172 intel_uncore_write(uncore, RING_CTL(base), 0);
1173 intel_uncore_write(uncore, RING_HEAD(base), 0);
1174 intel_uncore_write(uncore, RING_TAIL(base), 0);
1175 intel_uncore_write(uncore, RING_START(base), 0);
1176}
1177
1178static void init_unused_rings(struct intel_gt *gt)
1179{
1180 struct drm_i915_private *i915 = gt->i915;
1181
1182 if (IS_I830(i915)) {
1183 init_unused_ring(gt, PRB1_BASE);
1184 init_unused_ring(gt, SRB0_BASE);
1185 init_unused_ring(gt, SRB1_BASE);
1186 init_unused_ring(gt, SRB2_BASE);
1187 init_unused_ring(gt, SRB3_BASE);
1188 } else if (IS_GEN(i915, 2)) {
1189 init_unused_ring(gt, SRB0_BASE);
1190 init_unused_ring(gt, SRB1_BASE);
1191 } else if (IS_GEN(i915, 3)) {
1192 init_unused_ring(gt, PRB1_BASE);
1193 init_unused_ring(gt, PRB2_BASE);
1194 }
1195}
1196
1197int i915_gem_init_hw(struct drm_i915_private *i915)
1198{
1199 struct intel_uncore *uncore = &i915->uncore;
1200 struct intel_gt *gt = &i915->gt;
1201 int ret;
1202
1203 BUG_ON(!i915->kernel_context);
1204 ret = intel_gt_terminally_wedged(gt);
1205 if (ret)
1206 return ret;
1207
1208 gt->last_init_time = ktime_get();
1209
1210 /* Double layer security blanket, see i915_gem_init() */
1211 intel_uncore_forcewake_get(uncore, FORCEWAKE_ALL);
1212
1213 if (HAS_EDRAM(i915) && INTEL_GEN(i915) < 9)
1214 intel_uncore_rmw(uncore, HSW_IDICR, 0, IDIHASHMSK(0xf));
1215
1216 if (IS_HASWELL(i915))
1217 intel_uncore_write(uncore,
1218 MI_PREDICATE_RESULT_2,
1219 IS_HSW_GT3(i915) ?
1220 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
1221
1222 /* Apply the GT workarounds... */
1223 intel_gt_apply_workarounds(gt);
1224 /* ...and determine whether they are sticking. */
1225 intel_gt_verify_workarounds(gt, "init");
1226
1227 intel_gt_init_swizzling(gt);
1228
1229 /*
1230 * At least 830 can leave some of the unused rings
1231 * "active" (ie. head != tail) after resume which
1232 * will prevent c3 entry. Makes sure all unused rings
1233 * are totally idle.
1234 */
1235 init_unused_rings(gt);
1236
1237 ret = i915_ppgtt_init_hw(gt);
1238 if (ret) {
1239 DRM_ERROR("Enabling PPGTT failed (%d)\n", ret);
1240 goto out;
1241 }
1242
1243 /* We can't enable contexts until all firmware is loaded */
1244 ret = intel_uc_init_hw(>->uc);
1245 if (ret) {
1246 i915_probe_error(i915, "Enabling uc failed (%d)\n", ret);
1247 goto out;
1248 }
1249
1250 intel_mocs_init(gt);
1251
1252out:
1253 intel_uncore_forcewake_put(uncore, FORCEWAKE_ALL);
1254 return ret;
1255}
1256
1257static int __intel_engines_record_defaults(struct drm_i915_private *i915)
1258{
1259 struct i915_request *requests[I915_NUM_ENGINES] = {};
1260 struct intel_engine_cs *engine;
1261 enum intel_engine_id id;
1262 int err = 0;
1263
1264 /*
1265 * As we reset the gpu during very early sanitisation, the current
1266 * register state on the GPU should reflect its defaults values.
1267 * We load a context onto the hw (with restore-inhibit), then switch
1268 * over to a second context to save that default register state. We
1269 * can then prime every new context with that state so they all start
1270 * from the same default HW values.
1271 */
1272
1273 for_each_engine(engine, i915, id) {
1274 struct intel_context *ce;
1275 struct i915_request *rq;
1276
1277 /* We must be able to switch to something! */
1278 GEM_BUG_ON(!engine->kernel_context);
1279 engine->serial++; /* force the kernel context switch */
1280
1281 ce = intel_context_create(i915->kernel_context, engine);
1282 if (IS_ERR(ce)) {
1283 err = PTR_ERR(ce);
1284 goto out;
1285 }
1286
1287 rq = intel_context_create_request(ce);
1288 if (IS_ERR(rq)) {
1289 err = PTR_ERR(rq);
1290 intel_context_put(ce);
1291 goto out;
1292 }
1293
1294 err = intel_engine_emit_ctx_wa(rq);
1295 if (err)
1296 goto err_rq;
1297
1298 /*
1299 * Failing to program the MOCS is non-fatal.The system will not
1300 * run at peak performance. So warn the user and carry on.
1301 */
1302 err = intel_mocs_emit(rq);
1303 if (err)
1304 dev_notice(i915->drm.dev,
1305 "Failed to program MOCS registers; expect performance issues.\n");
1306
1307 err = intel_renderstate_emit(rq);
1308 if (err)
1309 goto err_rq;
1310
1311err_rq:
1312 requests[id] = i915_request_get(rq);
1313 i915_request_add(rq);
1314 if (err)
1315 goto out;
1316 }
1317
1318 /* Flush the default context image to memory, and enable powersaving. */
1319 if (!i915_gem_load_power_context(i915)) {
1320 err = -EIO;
1321 goto out;
1322 }
1323
1324 for (id = 0; id < ARRAY_SIZE(requests); id++) {
1325 struct i915_request *rq;
1326 struct i915_vma *state;
1327 void *vaddr;
1328
1329 rq = requests[id];
1330 if (!rq)
1331 continue;
1332
1333 /* We want to be able to unbind the state from the GGTT */
1334 GEM_BUG_ON(intel_context_is_pinned(rq->hw_context));
1335
1336 state = rq->hw_context->state;
1337 if (!state)
1338 continue;
1339
1340 /*
1341 * As we will hold a reference to the logical state, it will
1342 * not be torn down with the context, and importantly the
1343 * object will hold onto its vma (making it possible for a
1344 * stray GTT write to corrupt our defaults). Unmap the vma
1345 * from the GTT to prevent such accidents and reclaim the
1346 * space.
1347 */
1348 err = i915_vma_unbind(state);
1349 if (err)
1350 goto out;
1351
1352 i915_gem_object_lock(state->obj);
1353 err = i915_gem_object_set_to_cpu_domain(state->obj, false);
1354 i915_gem_object_unlock(state->obj);
1355 if (err)
1356 goto out;
1357
1358 i915_gem_object_set_cache_coherency(state->obj, I915_CACHE_LLC);
1359
1360 /* Check we can acquire the image of the context state */
1361 vaddr = i915_gem_object_pin_map(state->obj, I915_MAP_FORCE_WB);
1362 if (IS_ERR(vaddr)) {
1363 err = PTR_ERR(vaddr);
1364 goto out;
1365 }
1366
1367 rq->engine->default_state = i915_gem_object_get(state->obj);
1368 i915_gem_object_unpin_map(state->obj);
1369 }
1370
1371out:
1372 /*
1373 * If we have to abandon now, we expect the engines to be idle
1374 * and ready to be torn-down. The quickest way we can accomplish
1375 * this is by declaring ourselves wedged.
1376 */
1377 if (err)
1378 intel_gt_set_wedged(&i915->gt);
1379
1380 for (id = 0; id < ARRAY_SIZE(requests); id++) {
1381 struct intel_context *ce;
1382 struct i915_request *rq;
1383
1384 rq = requests[id];
1385 if (!rq)
1386 continue;
1387
1388 ce = rq->hw_context;
1389 i915_request_put(rq);
1390 intel_context_put(ce);
1391 }
1392 return err;
1393}
1394
1395static int
1396i915_gem_init_scratch(struct drm_i915_private *i915, unsigned int size)
1397{
1398 return intel_gt_init_scratch(&i915->gt, size);
1399}
1400
1401static void i915_gem_fini_scratch(struct drm_i915_private *i915)
1402{
1403 intel_gt_fini_scratch(&i915->gt);
1404}
1405
1406static int intel_engines_verify_workarounds(struct drm_i915_private *i915)
1407{
1408 struct intel_engine_cs *engine;
1409 enum intel_engine_id id;
1410 int err = 0;
1411
1412 if (!IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM))
1413 return 0;
1414
1415 for_each_engine(engine, i915, id) {
1416 if (intel_engine_verify_workarounds(engine, "load"))
1417 err = -EIO;
1418 }
1419
1420 return err;
1421}
1422
1423int i915_gem_init(struct drm_i915_private *dev_priv)
1424{
1425 int ret;
1426
1427 /* We need to fallback to 4K pages if host doesn't support huge gtt. */
1428 if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1429 mkwrite_device_info(dev_priv)->page_sizes =
1430 I915_GTT_PAGE_SIZE_4K;
1431
1432 intel_timelines_init(dev_priv);
1433
1434 ret = i915_gem_init_userptr(dev_priv);
1435 if (ret)
1436 return ret;
1437
1438 intel_uc_fetch_firmwares(&dev_priv->gt.uc);
1439 intel_wopcm_init(&dev_priv->wopcm);
1440
1441 /* This is just a security blanket to placate dragons.
1442 * On some systems, we very sporadically observe that the first TLBs
1443 * used by the CS may be stale, despite us poking the TLB reset. If
1444 * we hold the forcewake during initialisation these problems
1445 * just magically go away.
1446 */
1447 mutex_lock(&dev_priv->drm.struct_mutex);
1448 intel_uncore_forcewake_get(&dev_priv->uncore, FORCEWAKE_ALL);
1449
1450 ret = i915_init_ggtt(dev_priv);
1451 if (ret) {
1452 GEM_BUG_ON(ret == -EIO);
1453 goto err_unlock;
1454 }
1455
1456 ret = i915_gem_init_scratch(dev_priv,
1457 IS_GEN(dev_priv, 2) ? SZ_256K : PAGE_SIZE);
1458 if (ret) {
1459 GEM_BUG_ON(ret == -EIO);
1460 goto err_ggtt;
1461 }
1462
1463 ret = intel_engines_setup(dev_priv);
1464 if (ret) {
1465 GEM_BUG_ON(ret == -EIO);
1466 goto err_unlock;
1467 }
1468
1469 ret = i915_gem_contexts_init(dev_priv);
1470 if (ret) {
1471 GEM_BUG_ON(ret == -EIO);
1472 goto err_scratch;
1473 }
1474
1475 ret = intel_engines_init(dev_priv);
1476 if (ret) {
1477 GEM_BUG_ON(ret == -EIO);
1478 goto err_context;
1479 }
1480
1481 intel_init_gt_powersave(dev_priv);
1482
1483 intel_uc_init(&dev_priv->gt.uc);
1484
1485 ret = i915_gem_init_hw(dev_priv);
1486 if (ret)
1487 goto err_uc_init;
1488
1489 /* Only when the HW is re-initialised, can we replay the requests */
1490 ret = intel_gt_resume(&dev_priv->gt);
1491 if (ret)
1492 goto err_init_hw;
1493
1494 /*
1495 * Despite its name intel_init_clock_gating applies both display
1496 * clock gating workarounds; GT mmio workarounds and the occasional
1497 * GT power context workaround. Worse, sometimes it includes a context
1498 * register workaround which we need to apply before we record the
1499 * default HW state for all contexts.
1500 *
1501 * FIXME: break up the workarounds and apply them at the right time!
1502 */
1503 intel_init_clock_gating(dev_priv);
1504
1505 ret = intel_engines_verify_workarounds(dev_priv);
1506 if (ret)
1507 goto err_gt;
1508
1509 ret = __intel_engines_record_defaults(dev_priv);
1510 if (ret)
1511 goto err_gt;
1512
1513 ret = i915_inject_load_error(dev_priv, -ENODEV);
1514 if (ret)
1515 goto err_gt;
1516
1517 ret = i915_inject_load_error(dev_priv, -EIO);
1518 if (ret)
1519 goto err_gt;
1520
1521 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1522 mutex_unlock(&dev_priv->drm.struct_mutex);
1523
1524 return 0;
1525
1526 /*
1527 * Unwinding is complicated by that we want to handle -EIO to mean
1528 * disable GPU submission but keep KMS alive. We want to mark the
1529 * HW as irrevisibly wedged, but keep enough state around that the
1530 * driver doesn't explode during runtime.
1531 */
1532err_gt:
1533 mutex_unlock(&dev_priv->drm.struct_mutex);
1534
1535 intel_gt_set_wedged(&dev_priv->gt);
1536 i915_gem_suspend(dev_priv);
1537 i915_gem_suspend_late(dev_priv);
1538
1539 i915_gem_drain_workqueue(dev_priv);
1540
1541 mutex_lock(&dev_priv->drm.struct_mutex);
1542err_init_hw:
1543 intel_uc_fini_hw(&dev_priv->gt.uc);
1544err_uc_init:
1545 if (ret != -EIO) {
1546 intel_uc_fini(&dev_priv->gt.uc);
1547 intel_cleanup_gt_powersave(dev_priv);
1548 intel_engines_cleanup(dev_priv);
1549 }
1550err_context:
1551 if (ret != -EIO)
1552 i915_gem_contexts_fini(dev_priv);
1553err_scratch:
1554 i915_gem_fini_scratch(dev_priv);
1555err_ggtt:
1556err_unlock:
1557 intel_uncore_forcewake_put(&dev_priv->uncore, FORCEWAKE_ALL);
1558 mutex_unlock(&dev_priv->drm.struct_mutex);
1559
1560 if (ret != -EIO) {
1561 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1562 i915_gem_cleanup_userptr(dev_priv);
1563 intel_timelines_fini(dev_priv);
1564 }
1565
1566 if (ret == -EIO) {
1567 mutex_lock(&dev_priv->drm.struct_mutex);
1568
1569 /*
1570 * Allow engines or uC initialisation to fail by marking the GPU
1571 * as wedged. But we only want to do this when the GPU is angry,
1572 * for all other failure, such as an allocation failure, bail.
1573 */
1574 if (!intel_gt_is_wedged(&dev_priv->gt)) {
1575 i915_probe_error(dev_priv,
1576 "Failed to initialize GPU, declaring it wedged!\n");
1577 intel_gt_set_wedged(&dev_priv->gt);
1578 }
1579
1580 /* Minimal basic recovery for KMS */
1581 ret = i915_ggtt_enable_hw(dev_priv);
1582 i915_gem_restore_gtt_mappings(dev_priv);
1583 i915_gem_restore_fences(dev_priv);
1584 intel_init_clock_gating(dev_priv);
1585
1586 mutex_unlock(&dev_priv->drm.struct_mutex);
1587 }
1588
1589 i915_gem_drain_freed_objects(dev_priv);
1590 return ret;
1591}
1592
1593void i915_gem_driver_register(struct drm_i915_private *i915)
1594{
1595 i915_gem_driver_register__shrinker(i915);
1596
1597 intel_engines_driver_register(i915);
1598}
1599
1600void i915_gem_driver_unregister(struct drm_i915_private *i915)
1601{
1602 i915_gem_driver_unregister__shrinker(i915);
1603}
1604
1605void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1606{
1607 GEM_BUG_ON(dev_priv->gt.awake);
1608
1609 intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref);
1610
1611 i915_gem_suspend_late(dev_priv);
1612 intel_disable_gt_powersave(dev_priv);
1613
1614 /* Flush any outstanding unpin_work. */
1615 i915_gem_drain_workqueue(dev_priv);
1616
1617 mutex_lock(&dev_priv->drm.struct_mutex);
1618 intel_uc_fini_hw(&dev_priv->gt.uc);
1619 intel_uc_fini(&dev_priv->gt.uc);
1620 mutex_unlock(&dev_priv->drm.struct_mutex);
1621
1622 i915_gem_drain_freed_objects(dev_priv);
1623}
1624
1625void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1626{
1627 mutex_lock(&dev_priv->drm.struct_mutex);
1628 intel_engines_cleanup(dev_priv);
1629 i915_gem_contexts_fini(dev_priv);
1630 i915_gem_fini_scratch(dev_priv);
1631 mutex_unlock(&dev_priv->drm.struct_mutex);
1632
1633 intel_wa_list_free(&dev_priv->gt_wa_list);
1634
1635 intel_cleanup_gt_powersave(dev_priv);
1636
1637 intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1638 i915_gem_cleanup_userptr(dev_priv);
1639 intel_timelines_fini(dev_priv);
1640
1641 i915_gem_drain_freed_objects(dev_priv);
1642
1643 WARN_ON(!list_empty(&dev_priv->contexts.list));
1644}
1645
1646void i915_gem_init_mmio(struct drm_i915_private *i915)
1647{
1648 i915_gem_sanitize(i915);
1649}
1650
1651static void i915_gem_init__mm(struct drm_i915_private *i915)
1652{
1653 spin_lock_init(&i915->mm.obj_lock);
1654
1655 init_llist_head(&i915->mm.free_list);
1656
1657 INIT_LIST_HEAD(&i915->mm.purge_list);
1658 INIT_LIST_HEAD(&i915->mm.shrink_list);
1659
1660 i915_gem_init__objects(i915);
1661}
1662
1663int i915_gem_init_early(struct drm_i915_private *dev_priv)
1664{
1665 int err;
1666
1667 i915_gem_init__mm(dev_priv);
1668 i915_gem_init__pm(dev_priv);
1669
1670 spin_lock_init(&dev_priv->fb_tracking.lock);
1671
1672 err = i915_gemfs_init(dev_priv);
1673 if (err)
1674 DRM_NOTE("Unable to create a private tmpfs mount, hugepage support will be disabled(%d).\n", err);
1675
1676 return 0;
1677}
1678
1679void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1680{
1681 i915_gem_drain_freed_objects(dev_priv);
1682 GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1683 GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1684 WARN_ON(dev_priv->mm.shrink_count);
1685
1686 i915_gemfs_fini(dev_priv);
1687}
1688
1689int i915_gem_freeze(struct drm_i915_private *dev_priv)
1690{
1691 /* Discard all purgeable objects, let userspace recover those as
1692 * required after resuming.
1693 */
1694 i915_gem_shrink_all(dev_priv);
1695
1696 return 0;
1697}
1698
1699int i915_gem_freeze_late(struct drm_i915_private *i915)
1700{
1701 struct drm_i915_gem_object *obj;
1702 intel_wakeref_t wakeref;
1703
1704 /*
1705 * Called just before we write the hibernation image.
1706 *
1707 * We need to update the domain tracking to reflect that the CPU
1708 * will be accessing all the pages to create and restore from the
1709 * hibernation, and so upon restoration those pages will be in the
1710 * CPU domain.
1711 *
1712 * To make sure the hibernation image contains the latest state,
1713 * we update that state just before writing out the image.
1714 *
1715 * To try and reduce the hibernation image, we manually shrink
1716 * the objects as well, see i915_gem_freeze()
1717 */
1718
1719 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1720
1721 i915_gem_shrink(i915, -1UL, NULL, ~0);
1722 i915_gem_drain_freed_objects(i915);
1723
1724 list_for_each_entry(obj, &i915->mm.shrink_list, mm.link) {
1725 i915_gem_object_lock(obj);
1726 WARN_ON(i915_gem_object_set_to_cpu_domain(obj, true));
1727 i915_gem_object_unlock(obj);
1728 }
1729
1730 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1731
1732 return 0;
1733}
1734
1735void i915_gem_release(struct drm_device *dev, struct drm_file *file)
1736{
1737 struct drm_i915_file_private *file_priv = file->driver_priv;
1738 struct i915_request *request;
1739
1740 /* Clean up our request list when the client is going away, so that
1741 * later retire_requests won't dereference our soon-to-be-gone
1742 * file_priv.
1743 */
1744 spin_lock(&file_priv->mm.lock);
1745 list_for_each_entry(request, &file_priv->mm.request_list, client_link)
1746 request->file_priv = NULL;
1747 spin_unlock(&file_priv->mm.lock);
1748}
1749
1750int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1751{
1752 struct drm_i915_file_private *file_priv;
1753 int ret;
1754
1755 DRM_DEBUG("\n");
1756
1757 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1758 if (!file_priv)
1759 return -ENOMEM;
1760
1761 file->driver_priv = file_priv;
1762 file_priv->dev_priv = i915;
1763 file_priv->file = file;
1764
1765 spin_lock_init(&file_priv->mm.lock);
1766 INIT_LIST_HEAD(&file_priv->mm.request_list);
1767
1768 file_priv->bsd_engine = -1;
1769 file_priv->hang_timestamp = jiffies;
1770
1771 ret = i915_gem_context_open(i915, file);
1772 if (ret)
1773 kfree(file_priv);
1774
1775 return ret;
1776}
1777
1778#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1779#include "selftests/mock_gem_device.c"
1780#include "selftests/i915_gem.c"
1781#endif