Loading...
1/*
2 * Copyright © 2008 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include "drmP.h"
29#include "drm.h"
30#include "i915_drm.h"
31#include "i915_drv.h"
32#include "i915_trace.h"
33#include "intel_drv.h"
34#include <linux/shmem_fs.h>
35#include <linux/slab.h>
36#include <linux/swap.h>
37#include <linux/pci.h>
38
39static __must_check int i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj);
40static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
41static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
42static __must_check int i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj,
43 bool write);
44static __must_check int i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
45 uint64_t offset,
46 uint64_t size);
47static void i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj);
48static __must_check int i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
49 unsigned alignment,
50 bool map_and_fenceable);
51static void i915_gem_clear_fence_reg(struct drm_device *dev,
52 struct drm_i915_fence_reg *reg);
53static int i915_gem_phys_pwrite(struct drm_device *dev,
54 struct drm_i915_gem_object *obj,
55 struct drm_i915_gem_pwrite *args,
56 struct drm_file *file);
57static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj);
58
59static int i915_gem_inactive_shrink(struct shrinker *shrinker,
60 struct shrink_control *sc);
61
62/* some bookkeeping */
63static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
64 size_t size)
65{
66 dev_priv->mm.object_count++;
67 dev_priv->mm.object_memory += size;
68}
69
70static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
71 size_t size)
72{
73 dev_priv->mm.object_count--;
74 dev_priv->mm.object_memory -= size;
75}
76
77static int
78i915_gem_wait_for_error(struct drm_device *dev)
79{
80 struct drm_i915_private *dev_priv = dev->dev_private;
81 struct completion *x = &dev_priv->error_completion;
82 unsigned long flags;
83 int ret;
84
85 if (!atomic_read(&dev_priv->mm.wedged))
86 return 0;
87
88 ret = wait_for_completion_interruptible(x);
89 if (ret)
90 return ret;
91
92 if (atomic_read(&dev_priv->mm.wedged)) {
93 /* GPU is hung, bump the completion count to account for
94 * the token we just consumed so that we never hit zero and
95 * end up waiting upon a subsequent completion event that
96 * will never happen.
97 */
98 spin_lock_irqsave(&x->wait.lock, flags);
99 x->done++;
100 spin_unlock_irqrestore(&x->wait.lock, flags);
101 }
102 return 0;
103}
104
105int i915_mutex_lock_interruptible(struct drm_device *dev)
106{
107 int ret;
108
109 ret = i915_gem_wait_for_error(dev);
110 if (ret)
111 return ret;
112
113 ret = mutex_lock_interruptible(&dev->struct_mutex);
114 if (ret)
115 return ret;
116
117 WARN_ON(i915_verify_lists(dev));
118 return 0;
119}
120
121static inline bool
122i915_gem_object_is_inactive(struct drm_i915_gem_object *obj)
123{
124 return obj->gtt_space && !obj->active && obj->pin_count == 0;
125}
126
127void i915_gem_do_init(struct drm_device *dev,
128 unsigned long start,
129 unsigned long mappable_end,
130 unsigned long end)
131{
132 drm_i915_private_t *dev_priv = dev->dev_private;
133
134 drm_mm_init(&dev_priv->mm.gtt_space, start, end - start);
135
136 dev_priv->mm.gtt_start = start;
137 dev_priv->mm.gtt_mappable_end = mappable_end;
138 dev_priv->mm.gtt_end = end;
139 dev_priv->mm.gtt_total = end - start;
140 dev_priv->mm.mappable_gtt_total = min(end, mappable_end) - start;
141
142 /* Take over this portion of the GTT */
143 intel_gtt_clear_range(start / PAGE_SIZE, (end-start) / PAGE_SIZE);
144}
145
146int
147i915_gem_init_ioctl(struct drm_device *dev, void *data,
148 struct drm_file *file)
149{
150 struct drm_i915_gem_init *args = data;
151
152 if (args->gtt_start >= args->gtt_end ||
153 (args->gtt_end | args->gtt_start) & (PAGE_SIZE - 1))
154 return -EINVAL;
155
156 mutex_lock(&dev->struct_mutex);
157 i915_gem_do_init(dev, args->gtt_start, args->gtt_end, args->gtt_end);
158 mutex_unlock(&dev->struct_mutex);
159
160 return 0;
161}
162
163int
164i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
165 struct drm_file *file)
166{
167 struct drm_i915_private *dev_priv = dev->dev_private;
168 struct drm_i915_gem_get_aperture *args = data;
169 struct drm_i915_gem_object *obj;
170 size_t pinned;
171
172 if (!(dev->driver->driver_features & DRIVER_GEM))
173 return -ENODEV;
174
175 pinned = 0;
176 mutex_lock(&dev->struct_mutex);
177 list_for_each_entry(obj, &dev_priv->mm.pinned_list, mm_list)
178 pinned += obj->gtt_space->size;
179 mutex_unlock(&dev->struct_mutex);
180
181 args->aper_size = dev_priv->mm.gtt_total;
182 args->aper_available_size = args->aper_size -pinned;
183
184 return 0;
185}
186
187static int
188i915_gem_create(struct drm_file *file,
189 struct drm_device *dev,
190 uint64_t size,
191 uint32_t *handle_p)
192{
193 struct drm_i915_gem_object *obj;
194 int ret;
195 u32 handle;
196
197 size = roundup(size, PAGE_SIZE);
198
199 /* Allocate the new object */
200 obj = i915_gem_alloc_object(dev, size);
201 if (obj == NULL)
202 return -ENOMEM;
203
204 ret = drm_gem_handle_create(file, &obj->base, &handle);
205 if (ret) {
206 drm_gem_object_release(&obj->base);
207 i915_gem_info_remove_obj(dev->dev_private, obj->base.size);
208 kfree(obj);
209 return ret;
210 }
211
212 /* drop reference from allocate - handle holds it now */
213 drm_gem_object_unreference(&obj->base);
214 trace_i915_gem_object_create(obj);
215
216 *handle_p = handle;
217 return 0;
218}
219
220int
221i915_gem_dumb_create(struct drm_file *file,
222 struct drm_device *dev,
223 struct drm_mode_create_dumb *args)
224{
225 /* have to work out size/pitch and return them */
226 args->pitch = ALIGN(args->width * ((args->bpp + 7) / 8), 64);
227 args->size = args->pitch * args->height;
228 return i915_gem_create(file, dev,
229 args->size, &args->handle);
230}
231
232int i915_gem_dumb_destroy(struct drm_file *file,
233 struct drm_device *dev,
234 uint32_t handle)
235{
236 return drm_gem_handle_delete(file, handle);
237}
238
239/**
240 * Creates a new mm object and returns a handle to it.
241 */
242int
243i915_gem_create_ioctl(struct drm_device *dev, void *data,
244 struct drm_file *file)
245{
246 struct drm_i915_gem_create *args = data;
247 return i915_gem_create(file, dev,
248 args->size, &args->handle);
249}
250
251static int i915_gem_object_needs_bit17_swizzle(struct drm_i915_gem_object *obj)
252{
253 drm_i915_private_t *dev_priv = obj->base.dev->dev_private;
254
255 return dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_9_10_17 &&
256 obj->tiling_mode != I915_TILING_NONE;
257}
258
259static inline void
260slow_shmem_copy(struct page *dst_page,
261 int dst_offset,
262 struct page *src_page,
263 int src_offset,
264 int length)
265{
266 char *dst_vaddr, *src_vaddr;
267
268 dst_vaddr = kmap(dst_page);
269 src_vaddr = kmap(src_page);
270
271 memcpy(dst_vaddr + dst_offset, src_vaddr + src_offset, length);
272
273 kunmap(src_page);
274 kunmap(dst_page);
275}
276
277static inline void
278slow_shmem_bit17_copy(struct page *gpu_page,
279 int gpu_offset,
280 struct page *cpu_page,
281 int cpu_offset,
282 int length,
283 int is_read)
284{
285 char *gpu_vaddr, *cpu_vaddr;
286
287 /* Use the unswizzled path if this page isn't affected. */
288 if ((page_to_phys(gpu_page) & (1 << 17)) == 0) {
289 if (is_read)
290 return slow_shmem_copy(cpu_page, cpu_offset,
291 gpu_page, gpu_offset, length);
292 else
293 return slow_shmem_copy(gpu_page, gpu_offset,
294 cpu_page, cpu_offset, length);
295 }
296
297 gpu_vaddr = kmap(gpu_page);
298 cpu_vaddr = kmap(cpu_page);
299
300 /* Copy the data, XORing A6 with A17 (1). The user already knows he's
301 * XORing with the other bits (A9 for Y, A9 and A10 for X)
302 */
303 while (length > 0) {
304 int cacheline_end = ALIGN(gpu_offset + 1, 64);
305 int this_length = min(cacheline_end - gpu_offset, length);
306 int swizzled_gpu_offset = gpu_offset ^ 64;
307
308 if (is_read) {
309 memcpy(cpu_vaddr + cpu_offset,
310 gpu_vaddr + swizzled_gpu_offset,
311 this_length);
312 } else {
313 memcpy(gpu_vaddr + swizzled_gpu_offset,
314 cpu_vaddr + cpu_offset,
315 this_length);
316 }
317 cpu_offset += this_length;
318 gpu_offset += this_length;
319 length -= this_length;
320 }
321
322 kunmap(cpu_page);
323 kunmap(gpu_page);
324}
325
326/**
327 * This is the fast shmem pread path, which attempts to copy_from_user directly
328 * from the backing pages of the object to the user's address space. On a
329 * fault, it fails so we can fall back to i915_gem_shmem_pwrite_slow().
330 */
331static int
332i915_gem_shmem_pread_fast(struct drm_device *dev,
333 struct drm_i915_gem_object *obj,
334 struct drm_i915_gem_pread *args,
335 struct drm_file *file)
336{
337 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
338 ssize_t remain;
339 loff_t offset;
340 char __user *user_data;
341 int page_offset, page_length;
342
343 user_data = (char __user *) (uintptr_t) args->data_ptr;
344 remain = args->size;
345
346 offset = args->offset;
347
348 while (remain > 0) {
349 struct page *page;
350 char *vaddr;
351 int ret;
352
353 /* Operation in this page
354 *
355 * page_offset = offset within page
356 * page_length = bytes to copy for this page
357 */
358 page_offset = offset_in_page(offset);
359 page_length = remain;
360 if ((page_offset + remain) > PAGE_SIZE)
361 page_length = PAGE_SIZE - page_offset;
362
363 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
364 if (IS_ERR(page))
365 return PTR_ERR(page);
366
367 vaddr = kmap_atomic(page);
368 ret = __copy_to_user_inatomic(user_data,
369 vaddr + page_offset,
370 page_length);
371 kunmap_atomic(vaddr);
372
373 mark_page_accessed(page);
374 page_cache_release(page);
375 if (ret)
376 return -EFAULT;
377
378 remain -= page_length;
379 user_data += page_length;
380 offset += page_length;
381 }
382
383 return 0;
384}
385
386/**
387 * This is the fallback shmem pread path, which allocates temporary storage
388 * in kernel space to copy_to_user into outside of the struct_mutex, so we
389 * can copy out of the object's backing pages while holding the struct mutex
390 * and not take page faults.
391 */
392static int
393i915_gem_shmem_pread_slow(struct drm_device *dev,
394 struct drm_i915_gem_object *obj,
395 struct drm_i915_gem_pread *args,
396 struct drm_file *file)
397{
398 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
399 struct mm_struct *mm = current->mm;
400 struct page **user_pages;
401 ssize_t remain;
402 loff_t offset, pinned_pages, i;
403 loff_t first_data_page, last_data_page, num_pages;
404 int shmem_page_offset;
405 int data_page_index, data_page_offset;
406 int page_length;
407 int ret;
408 uint64_t data_ptr = args->data_ptr;
409 int do_bit17_swizzling;
410
411 remain = args->size;
412
413 /* Pin the user pages containing the data. We can't fault while
414 * holding the struct mutex, yet we want to hold it while
415 * dereferencing the user data.
416 */
417 first_data_page = data_ptr / PAGE_SIZE;
418 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
419 num_pages = last_data_page - first_data_page + 1;
420
421 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
422 if (user_pages == NULL)
423 return -ENOMEM;
424
425 mutex_unlock(&dev->struct_mutex);
426 down_read(&mm->mmap_sem);
427 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
428 num_pages, 1, 0, user_pages, NULL);
429 up_read(&mm->mmap_sem);
430 mutex_lock(&dev->struct_mutex);
431 if (pinned_pages < num_pages) {
432 ret = -EFAULT;
433 goto out;
434 }
435
436 ret = i915_gem_object_set_cpu_read_domain_range(obj,
437 args->offset,
438 args->size);
439 if (ret)
440 goto out;
441
442 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
443
444 offset = args->offset;
445
446 while (remain > 0) {
447 struct page *page;
448
449 /* Operation in this page
450 *
451 * shmem_page_offset = offset within page in shmem file
452 * data_page_index = page number in get_user_pages return
453 * data_page_offset = offset with data_page_index page.
454 * page_length = bytes to copy for this page
455 */
456 shmem_page_offset = offset_in_page(offset);
457 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
458 data_page_offset = offset_in_page(data_ptr);
459
460 page_length = remain;
461 if ((shmem_page_offset + page_length) > PAGE_SIZE)
462 page_length = PAGE_SIZE - shmem_page_offset;
463 if ((data_page_offset + page_length) > PAGE_SIZE)
464 page_length = PAGE_SIZE - data_page_offset;
465
466 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
467 if (IS_ERR(page)) {
468 ret = PTR_ERR(page);
469 goto out;
470 }
471
472 if (do_bit17_swizzling) {
473 slow_shmem_bit17_copy(page,
474 shmem_page_offset,
475 user_pages[data_page_index],
476 data_page_offset,
477 page_length,
478 1);
479 } else {
480 slow_shmem_copy(user_pages[data_page_index],
481 data_page_offset,
482 page,
483 shmem_page_offset,
484 page_length);
485 }
486
487 mark_page_accessed(page);
488 page_cache_release(page);
489
490 remain -= page_length;
491 data_ptr += page_length;
492 offset += page_length;
493 }
494
495out:
496 for (i = 0; i < pinned_pages; i++) {
497 SetPageDirty(user_pages[i]);
498 mark_page_accessed(user_pages[i]);
499 page_cache_release(user_pages[i]);
500 }
501 drm_free_large(user_pages);
502
503 return ret;
504}
505
506/**
507 * Reads data from the object referenced by handle.
508 *
509 * On error, the contents of *data are undefined.
510 */
511int
512i915_gem_pread_ioctl(struct drm_device *dev, void *data,
513 struct drm_file *file)
514{
515 struct drm_i915_gem_pread *args = data;
516 struct drm_i915_gem_object *obj;
517 int ret = 0;
518
519 if (args->size == 0)
520 return 0;
521
522 if (!access_ok(VERIFY_WRITE,
523 (char __user *)(uintptr_t)args->data_ptr,
524 args->size))
525 return -EFAULT;
526
527 ret = fault_in_pages_writeable((char __user *)(uintptr_t)args->data_ptr,
528 args->size);
529 if (ret)
530 return -EFAULT;
531
532 ret = i915_mutex_lock_interruptible(dev);
533 if (ret)
534 return ret;
535
536 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
537 if (&obj->base == NULL) {
538 ret = -ENOENT;
539 goto unlock;
540 }
541
542 /* Bounds check source. */
543 if (args->offset > obj->base.size ||
544 args->size > obj->base.size - args->offset) {
545 ret = -EINVAL;
546 goto out;
547 }
548
549 trace_i915_gem_object_pread(obj, args->offset, args->size);
550
551 ret = i915_gem_object_set_cpu_read_domain_range(obj,
552 args->offset,
553 args->size);
554 if (ret)
555 goto out;
556
557 ret = -EFAULT;
558 if (!i915_gem_object_needs_bit17_swizzle(obj))
559 ret = i915_gem_shmem_pread_fast(dev, obj, args, file);
560 if (ret == -EFAULT)
561 ret = i915_gem_shmem_pread_slow(dev, obj, args, file);
562
563out:
564 drm_gem_object_unreference(&obj->base);
565unlock:
566 mutex_unlock(&dev->struct_mutex);
567 return ret;
568}
569
570/* This is the fast write path which cannot handle
571 * page faults in the source data
572 */
573
574static inline int
575fast_user_write(struct io_mapping *mapping,
576 loff_t page_base, int page_offset,
577 char __user *user_data,
578 int length)
579{
580 char *vaddr_atomic;
581 unsigned long unwritten;
582
583 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
584 unwritten = __copy_from_user_inatomic_nocache(vaddr_atomic + page_offset,
585 user_data, length);
586 io_mapping_unmap_atomic(vaddr_atomic);
587 return unwritten;
588}
589
590/* Here's the write path which can sleep for
591 * page faults
592 */
593
594static inline void
595slow_kernel_write(struct io_mapping *mapping,
596 loff_t gtt_base, int gtt_offset,
597 struct page *user_page, int user_offset,
598 int length)
599{
600 char __iomem *dst_vaddr;
601 char *src_vaddr;
602
603 dst_vaddr = io_mapping_map_wc(mapping, gtt_base);
604 src_vaddr = kmap(user_page);
605
606 memcpy_toio(dst_vaddr + gtt_offset,
607 src_vaddr + user_offset,
608 length);
609
610 kunmap(user_page);
611 io_mapping_unmap(dst_vaddr);
612}
613
614/**
615 * This is the fast pwrite path, where we copy the data directly from the
616 * user into the GTT, uncached.
617 */
618static int
619i915_gem_gtt_pwrite_fast(struct drm_device *dev,
620 struct drm_i915_gem_object *obj,
621 struct drm_i915_gem_pwrite *args,
622 struct drm_file *file)
623{
624 drm_i915_private_t *dev_priv = dev->dev_private;
625 ssize_t remain;
626 loff_t offset, page_base;
627 char __user *user_data;
628 int page_offset, page_length;
629
630 user_data = (char __user *) (uintptr_t) args->data_ptr;
631 remain = args->size;
632
633 offset = obj->gtt_offset + args->offset;
634
635 while (remain > 0) {
636 /* Operation in this page
637 *
638 * page_base = page offset within aperture
639 * page_offset = offset within page
640 * page_length = bytes to copy for this page
641 */
642 page_base = offset & PAGE_MASK;
643 page_offset = offset_in_page(offset);
644 page_length = remain;
645 if ((page_offset + remain) > PAGE_SIZE)
646 page_length = PAGE_SIZE - page_offset;
647
648 /* If we get a fault while copying data, then (presumably) our
649 * source page isn't available. Return the error and we'll
650 * retry in the slow path.
651 */
652 if (fast_user_write(dev_priv->mm.gtt_mapping, page_base,
653 page_offset, user_data, page_length))
654 return -EFAULT;
655
656 remain -= page_length;
657 user_data += page_length;
658 offset += page_length;
659 }
660
661 return 0;
662}
663
664/**
665 * This is the fallback GTT pwrite path, which uses get_user_pages to pin
666 * the memory and maps it using kmap_atomic for copying.
667 *
668 * This code resulted in x11perf -rgb10text consuming about 10% more CPU
669 * than using i915_gem_gtt_pwrite_fast on a G45 (32-bit).
670 */
671static int
672i915_gem_gtt_pwrite_slow(struct drm_device *dev,
673 struct drm_i915_gem_object *obj,
674 struct drm_i915_gem_pwrite *args,
675 struct drm_file *file)
676{
677 drm_i915_private_t *dev_priv = dev->dev_private;
678 ssize_t remain;
679 loff_t gtt_page_base, offset;
680 loff_t first_data_page, last_data_page, num_pages;
681 loff_t pinned_pages, i;
682 struct page **user_pages;
683 struct mm_struct *mm = current->mm;
684 int gtt_page_offset, data_page_offset, data_page_index, page_length;
685 int ret;
686 uint64_t data_ptr = args->data_ptr;
687
688 remain = args->size;
689
690 /* Pin the user pages containing the data. We can't fault while
691 * holding the struct mutex, and all of the pwrite implementations
692 * want to hold it while dereferencing the user data.
693 */
694 first_data_page = data_ptr / PAGE_SIZE;
695 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
696 num_pages = last_data_page - first_data_page + 1;
697
698 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
699 if (user_pages == NULL)
700 return -ENOMEM;
701
702 mutex_unlock(&dev->struct_mutex);
703 down_read(&mm->mmap_sem);
704 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
705 num_pages, 0, 0, user_pages, NULL);
706 up_read(&mm->mmap_sem);
707 mutex_lock(&dev->struct_mutex);
708 if (pinned_pages < num_pages) {
709 ret = -EFAULT;
710 goto out_unpin_pages;
711 }
712
713 ret = i915_gem_object_set_to_gtt_domain(obj, true);
714 if (ret)
715 goto out_unpin_pages;
716
717 ret = i915_gem_object_put_fence(obj);
718 if (ret)
719 goto out_unpin_pages;
720
721 offset = obj->gtt_offset + args->offset;
722
723 while (remain > 0) {
724 /* Operation in this page
725 *
726 * gtt_page_base = page offset within aperture
727 * gtt_page_offset = offset within page in aperture
728 * data_page_index = page number in get_user_pages return
729 * data_page_offset = offset with data_page_index page.
730 * page_length = bytes to copy for this page
731 */
732 gtt_page_base = offset & PAGE_MASK;
733 gtt_page_offset = offset_in_page(offset);
734 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
735 data_page_offset = offset_in_page(data_ptr);
736
737 page_length = remain;
738 if ((gtt_page_offset + page_length) > PAGE_SIZE)
739 page_length = PAGE_SIZE - gtt_page_offset;
740 if ((data_page_offset + page_length) > PAGE_SIZE)
741 page_length = PAGE_SIZE - data_page_offset;
742
743 slow_kernel_write(dev_priv->mm.gtt_mapping,
744 gtt_page_base, gtt_page_offset,
745 user_pages[data_page_index],
746 data_page_offset,
747 page_length);
748
749 remain -= page_length;
750 offset += page_length;
751 data_ptr += page_length;
752 }
753
754out_unpin_pages:
755 for (i = 0; i < pinned_pages; i++)
756 page_cache_release(user_pages[i]);
757 drm_free_large(user_pages);
758
759 return ret;
760}
761
762/**
763 * This is the fast shmem pwrite path, which attempts to directly
764 * copy_from_user into the kmapped pages backing the object.
765 */
766static int
767i915_gem_shmem_pwrite_fast(struct drm_device *dev,
768 struct drm_i915_gem_object *obj,
769 struct drm_i915_gem_pwrite *args,
770 struct drm_file *file)
771{
772 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
773 ssize_t remain;
774 loff_t offset;
775 char __user *user_data;
776 int page_offset, page_length;
777
778 user_data = (char __user *) (uintptr_t) args->data_ptr;
779 remain = args->size;
780
781 offset = args->offset;
782 obj->dirty = 1;
783
784 while (remain > 0) {
785 struct page *page;
786 char *vaddr;
787 int ret;
788
789 /* Operation in this page
790 *
791 * page_offset = offset within page
792 * page_length = bytes to copy for this page
793 */
794 page_offset = offset_in_page(offset);
795 page_length = remain;
796 if ((page_offset + remain) > PAGE_SIZE)
797 page_length = PAGE_SIZE - page_offset;
798
799 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
800 if (IS_ERR(page))
801 return PTR_ERR(page);
802
803 vaddr = kmap_atomic(page, KM_USER0);
804 ret = __copy_from_user_inatomic(vaddr + page_offset,
805 user_data,
806 page_length);
807 kunmap_atomic(vaddr, KM_USER0);
808
809 set_page_dirty(page);
810 mark_page_accessed(page);
811 page_cache_release(page);
812
813 /* If we get a fault while copying data, then (presumably) our
814 * source page isn't available. Return the error and we'll
815 * retry in the slow path.
816 */
817 if (ret)
818 return -EFAULT;
819
820 remain -= page_length;
821 user_data += page_length;
822 offset += page_length;
823 }
824
825 return 0;
826}
827
828/**
829 * This is the fallback shmem pwrite path, which uses get_user_pages to pin
830 * the memory and maps it using kmap_atomic for copying.
831 *
832 * This avoids taking mmap_sem for faulting on the user's address while the
833 * struct_mutex is held.
834 */
835static int
836i915_gem_shmem_pwrite_slow(struct drm_device *dev,
837 struct drm_i915_gem_object *obj,
838 struct drm_i915_gem_pwrite *args,
839 struct drm_file *file)
840{
841 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
842 struct mm_struct *mm = current->mm;
843 struct page **user_pages;
844 ssize_t remain;
845 loff_t offset, pinned_pages, i;
846 loff_t first_data_page, last_data_page, num_pages;
847 int shmem_page_offset;
848 int data_page_index, data_page_offset;
849 int page_length;
850 int ret;
851 uint64_t data_ptr = args->data_ptr;
852 int do_bit17_swizzling;
853
854 remain = args->size;
855
856 /* Pin the user pages containing the data. We can't fault while
857 * holding the struct mutex, and all of the pwrite implementations
858 * want to hold it while dereferencing the user data.
859 */
860 first_data_page = data_ptr / PAGE_SIZE;
861 last_data_page = (data_ptr + args->size - 1) / PAGE_SIZE;
862 num_pages = last_data_page - first_data_page + 1;
863
864 user_pages = drm_malloc_ab(num_pages, sizeof(struct page *));
865 if (user_pages == NULL)
866 return -ENOMEM;
867
868 mutex_unlock(&dev->struct_mutex);
869 down_read(&mm->mmap_sem);
870 pinned_pages = get_user_pages(current, mm, (uintptr_t)args->data_ptr,
871 num_pages, 0, 0, user_pages, NULL);
872 up_read(&mm->mmap_sem);
873 mutex_lock(&dev->struct_mutex);
874 if (pinned_pages < num_pages) {
875 ret = -EFAULT;
876 goto out;
877 }
878
879 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
880 if (ret)
881 goto out;
882
883 do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
884
885 offset = args->offset;
886 obj->dirty = 1;
887
888 while (remain > 0) {
889 struct page *page;
890
891 /* Operation in this page
892 *
893 * shmem_page_offset = offset within page in shmem file
894 * data_page_index = page number in get_user_pages return
895 * data_page_offset = offset with data_page_index page.
896 * page_length = bytes to copy for this page
897 */
898 shmem_page_offset = offset_in_page(offset);
899 data_page_index = data_ptr / PAGE_SIZE - first_data_page;
900 data_page_offset = offset_in_page(data_ptr);
901
902 page_length = remain;
903 if ((shmem_page_offset + page_length) > PAGE_SIZE)
904 page_length = PAGE_SIZE - shmem_page_offset;
905 if ((data_page_offset + page_length) > PAGE_SIZE)
906 page_length = PAGE_SIZE - data_page_offset;
907
908 page = shmem_read_mapping_page(mapping, offset >> PAGE_SHIFT);
909 if (IS_ERR(page)) {
910 ret = PTR_ERR(page);
911 goto out;
912 }
913
914 if (do_bit17_swizzling) {
915 slow_shmem_bit17_copy(page,
916 shmem_page_offset,
917 user_pages[data_page_index],
918 data_page_offset,
919 page_length,
920 0);
921 } else {
922 slow_shmem_copy(page,
923 shmem_page_offset,
924 user_pages[data_page_index],
925 data_page_offset,
926 page_length);
927 }
928
929 set_page_dirty(page);
930 mark_page_accessed(page);
931 page_cache_release(page);
932
933 remain -= page_length;
934 data_ptr += page_length;
935 offset += page_length;
936 }
937
938out:
939 for (i = 0; i < pinned_pages; i++)
940 page_cache_release(user_pages[i]);
941 drm_free_large(user_pages);
942
943 return ret;
944}
945
946/**
947 * Writes data to the object referenced by handle.
948 *
949 * On error, the contents of the buffer that were to be modified are undefined.
950 */
951int
952i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
953 struct drm_file *file)
954{
955 struct drm_i915_gem_pwrite *args = data;
956 struct drm_i915_gem_object *obj;
957 int ret;
958
959 if (args->size == 0)
960 return 0;
961
962 if (!access_ok(VERIFY_READ,
963 (char __user *)(uintptr_t)args->data_ptr,
964 args->size))
965 return -EFAULT;
966
967 ret = fault_in_pages_readable((char __user *)(uintptr_t)args->data_ptr,
968 args->size);
969 if (ret)
970 return -EFAULT;
971
972 ret = i915_mutex_lock_interruptible(dev);
973 if (ret)
974 return ret;
975
976 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
977 if (&obj->base == NULL) {
978 ret = -ENOENT;
979 goto unlock;
980 }
981
982 /* Bounds check destination. */
983 if (args->offset > obj->base.size ||
984 args->size > obj->base.size - args->offset) {
985 ret = -EINVAL;
986 goto out;
987 }
988
989 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
990
991 /* We can only do the GTT pwrite on untiled buffers, as otherwise
992 * it would end up going through the fenced access, and we'll get
993 * different detiling behavior between reading and writing.
994 * pread/pwrite currently are reading and writing from the CPU
995 * perspective, requiring manual detiling by the client.
996 */
997 if (obj->phys_obj)
998 ret = i915_gem_phys_pwrite(dev, obj, args, file);
999 else if (obj->gtt_space &&
1000 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1001 ret = i915_gem_object_pin(obj, 0, true);
1002 if (ret)
1003 goto out;
1004
1005 ret = i915_gem_object_set_to_gtt_domain(obj, true);
1006 if (ret)
1007 goto out_unpin;
1008
1009 ret = i915_gem_object_put_fence(obj);
1010 if (ret)
1011 goto out_unpin;
1012
1013 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1014 if (ret == -EFAULT)
1015 ret = i915_gem_gtt_pwrite_slow(dev, obj, args, file);
1016
1017out_unpin:
1018 i915_gem_object_unpin(obj);
1019 } else {
1020 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
1021 if (ret)
1022 goto out;
1023
1024 ret = -EFAULT;
1025 if (!i915_gem_object_needs_bit17_swizzle(obj))
1026 ret = i915_gem_shmem_pwrite_fast(dev, obj, args, file);
1027 if (ret == -EFAULT)
1028 ret = i915_gem_shmem_pwrite_slow(dev, obj, args, file);
1029 }
1030
1031out:
1032 drm_gem_object_unreference(&obj->base);
1033unlock:
1034 mutex_unlock(&dev->struct_mutex);
1035 return ret;
1036}
1037
1038/**
1039 * Called when user space prepares to use an object with the CPU, either
1040 * through the mmap ioctl's mapping or a GTT mapping.
1041 */
1042int
1043i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1044 struct drm_file *file)
1045{
1046 struct drm_i915_gem_set_domain *args = data;
1047 struct drm_i915_gem_object *obj;
1048 uint32_t read_domains = args->read_domains;
1049 uint32_t write_domain = args->write_domain;
1050 int ret;
1051
1052 if (!(dev->driver->driver_features & DRIVER_GEM))
1053 return -ENODEV;
1054
1055 /* Only handle setting domains to types used by the CPU. */
1056 if (write_domain & I915_GEM_GPU_DOMAINS)
1057 return -EINVAL;
1058
1059 if (read_domains & I915_GEM_GPU_DOMAINS)
1060 return -EINVAL;
1061
1062 /* Having something in the write domain implies it's in the read
1063 * domain, and only that read domain. Enforce that in the request.
1064 */
1065 if (write_domain != 0 && read_domains != write_domain)
1066 return -EINVAL;
1067
1068 ret = i915_mutex_lock_interruptible(dev);
1069 if (ret)
1070 return ret;
1071
1072 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1073 if (&obj->base == NULL) {
1074 ret = -ENOENT;
1075 goto unlock;
1076 }
1077
1078 if (read_domains & I915_GEM_DOMAIN_GTT) {
1079 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1080
1081 /* Silently promote "you're not bound, there was nothing to do"
1082 * to success, since the client was just asking us to
1083 * make sure everything was done.
1084 */
1085 if (ret == -EINVAL)
1086 ret = 0;
1087 } else {
1088 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1089 }
1090
1091 drm_gem_object_unreference(&obj->base);
1092unlock:
1093 mutex_unlock(&dev->struct_mutex);
1094 return ret;
1095}
1096
1097/**
1098 * Called when user space has done writes to this buffer
1099 */
1100int
1101i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1102 struct drm_file *file)
1103{
1104 struct drm_i915_gem_sw_finish *args = data;
1105 struct drm_i915_gem_object *obj;
1106 int ret = 0;
1107
1108 if (!(dev->driver->driver_features & DRIVER_GEM))
1109 return -ENODEV;
1110
1111 ret = i915_mutex_lock_interruptible(dev);
1112 if (ret)
1113 return ret;
1114
1115 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1116 if (&obj->base == NULL) {
1117 ret = -ENOENT;
1118 goto unlock;
1119 }
1120
1121 /* Pinned buffers may be scanout, so flush the cache */
1122 if (obj->pin_count)
1123 i915_gem_object_flush_cpu_write_domain(obj);
1124
1125 drm_gem_object_unreference(&obj->base);
1126unlock:
1127 mutex_unlock(&dev->struct_mutex);
1128 return ret;
1129}
1130
1131/**
1132 * Maps the contents of an object, returning the address it is mapped
1133 * into.
1134 *
1135 * While the mapping holds a reference on the contents of the object, it doesn't
1136 * imply a ref on the object itself.
1137 */
1138int
1139i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1140 struct drm_file *file)
1141{
1142 struct drm_i915_private *dev_priv = dev->dev_private;
1143 struct drm_i915_gem_mmap *args = data;
1144 struct drm_gem_object *obj;
1145 unsigned long addr;
1146
1147 if (!(dev->driver->driver_features & DRIVER_GEM))
1148 return -ENODEV;
1149
1150 obj = drm_gem_object_lookup(dev, file, args->handle);
1151 if (obj == NULL)
1152 return -ENOENT;
1153
1154 if (obj->size > dev_priv->mm.gtt_mappable_end) {
1155 drm_gem_object_unreference_unlocked(obj);
1156 return -E2BIG;
1157 }
1158
1159 down_write(¤t->mm->mmap_sem);
1160 addr = do_mmap(obj->filp, 0, args->size,
1161 PROT_READ | PROT_WRITE, MAP_SHARED,
1162 args->offset);
1163 up_write(¤t->mm->mmap_sem);
1164 drm_gem_object_unreference_unlocked(obj);
1165 if (IS_ERR((void *)addr))
1166 return addr;
1167
1168 args->addr_ptr = (uint64_t) addr;
1169
1170 return 0;
1171}
1172
1173/**
1174 * i915_gem_fault - fault a page into the GTT
1175 * vma: VMA in question
1176 * vmf: fault info
1177 *
1178 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1179 * from userspace. The fault handler takes care of binding the object to
1180 * the GTT (if needed), allocating and programming a fence register (again,
1181 * only if needed based on whether the old reg is still valid or the object
1182 * is tiled) and inserting a new PTE into the faulting process.
1183 *
1184 * Note that the faulting process may involve evicting existing objects
1185 * from the GTT and/or fence registers to make room. So performance may
1186 * suffer if the GTT working set is large or there are few fence registers
1187 * left.
1188 */
1189int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1190{
1191 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1192 struct drm_device *dev = obj->base.dev;
1193 drm_i915_private_t *dev_priv = dev->dev_private;
1194 pgoff_t page_offset;
1195 unsigned long pfn;
1196 int ret = 0;
1197 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1198
1199 /* We don't use vmf->pgoff since that has the fake offset */
1200 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1201 PAGE_SHIFT;
1202
1203 ret = i915_mutex_lock_interruptible(dev);
1204 if (ret)
1205 goto out;
1206
1207 trace_i915_gem_object_fault(obj, page_offset, true, write);
1208
1209 /* Now bind it into the GTT if needed */
1210 if (!obj->map_and_fenceable) {
1211 ret = i915_gem_object_unbind(obj);
1212 if (ret)
1213 goto unlock;
1214 }
1215 if (!obj->gtt_space) {
1216 ret = i915_gem_object_bind_to_gtt(obj, 0, true);
1217 if (ret)
1218 goto unlock;
1219
1220 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1221 if (ret)
1222 goto unlock;
1223 }
1224
1225 if (obj->tiling_mode == I915_TILING_NONE)
1226 ret = i915_gem_object_put_fence(obj);
1227 else
1228 ret = i915_gem_object_get_fence(obj, NULL);
1229 if (ret)
1230 goto unlock;
1231
1232 if (i915_gem_object_is_inactive(obj))
1233 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1234
1235 obj->fault_mappable = true;
1236
1237 pfn = ((dev->agp->base + obj->gtt_offset) >> PAGE_SHIFT) +
1238 page_offset;
1239
1240 /* Finally, remap it using the new GTT offset */
1241 ret = vm_insert_pfn(vma, (unsigned long)vmf->virtual_address, pfn);
1242unlock:
1243 mutex_unlock(&dev->struct_mutex);
1244out:
1245 switch (ret) {
1246 case -EIO:
1247 case -EAGAIN:
1248 /* Give the error handler a chance to run and move the
1249 * objects off the GPU active list. Next time we service the
1250 * fault, we should be able to transition the page into the
1251 * GTT without touching the GPU (and so avoid further
1252 * EIO/EGAIN). If the GPU is wedged, then there is no issue
1253 * with coherency, just lost writes.
1254 */
1255 set_need_resched();
1256 case 0:
1257 case -ERESTARTSYS:
1258 case -EINTR:
1259 return VM_FAULT_NOPAGE;
1260 case -ENOMEM:
1261 return VM_FAULT_OOM;
1262 default:
1263 return VM_FAULT_SIGBUS;
1264 }
1265}
1266
1267/**
1268 * i915_gem_create_mmap_offset - create a fake mmap offset for an object
1269 * @obj: obj in question
1270 *
1271 * GEM memory mapping works by handing back to userspace a fake mmap offset
1272 * it can use in a subsequent mmap(2) call. The DRM core code then looks
1273 * up the object based on the offset and sets up the various memory mapping
1274 * structures.
1275 *
1276 * This routine allocates and attaches a fake offset for @obj.
1277 */
1278static int
1279i915_gem_create_mmap_offset(struct drm_i915_gem_object *obj)
1280{
1281 struct drm_device *dev = obj->base.dev;
1282 struct drm_gem_mm *mm = dev->mm_private;
1283 struct drm_map_list *list;
1284 struct drm_local_map *map;
1285 int ret = 0;
1286
1287 /* Set the object up for mmap'ing */
1288 list = &obj->base.map_list;
1289 list->map = kzalloc(sizeof(struct drm_map_list), GFP_KERNEL);
1290 if (!list->map)
1291 return -ENOMEM;
1292
1293 map = list->map;
1294 map->type = _DRM_GEM;
1295 map->size = obj->base.size;
1296 map->handle = obj;
1297
1298 /* Get a DRM GEM mmap offset allocated... */
1299 list->file_offset_node = drm_mm_search_free(&mm->offset_manager,
1300 obj->base.size / PAGE_SIZE,
1301 0, 0);
1302 if (!list->file_offset_node) {
1303 DRM_ERROR("failed to allocate offset for bo %d\n",
1304 obj->base.name);
1305 ret = -ENOSPC;
1306 goto out_free_list;
1307 }
1308
1309 list->file_offset_node = drm_mm_get_block(list->file_offset_node,
1310 obj->base.size / PAGE_SIZE,
1311 0);
1312 if (!list->file_offset_node) {
1313 ret = -ENOMEM;
1314 goto out_free_list;
1315 }
1316
1317 list->hash.key = list->file_offset_node->start;
1318 ret = drm_ht_insert_item(&mm->offset_hash, &list->hash);
1319 if (ret) {
1320 DRM_ERROR("failed to add to map hash\n");
1321 goto out_free_mm;
1322 }
1323
1324 return 0;
1325
1326out_free_mm:
1327 drm_mm_put_block(list->file_offset_node);
1328out_free_list:
1329 kfree(list->map);
1330 list->map = NULL;
1331
1332 return ret;
1333}
1334
1335/**
1336 * i915_gem_release_mmap - remove physical page mappings
1337 * @obj: obj in question
1338 *
1339 * Preserve the reservation of the mmapping with the DRM core code, but
1340 * relinquish ownership of the pages back to the system.
1341 *
1342 * It is vital that we remove the page mapping if we have mapped a tiled
1343 * object through the GTT and then lose the fence register due to
1344 * resource pressure. Similarly if the object has been moved out of the
1345 * aperture, than pages mapped into userspace must be revoked. Removing the
1346 * mapping will then trigger a page fault on the next user access, allowing
1347 * fixup by i915_gem_fault().
1348 */
1349void
1350i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1351{
1352 if (!obj->fault_mappable)
1353 return;
1354
1355 if (obj->base.dev->dev_mapping)
1356 unmap_mapping_range(obj->base.dev->dev_mapping,
1357 (loff_t)obj->base.map_list.hash.key<<PAGE_SHIFT,
1358 obj->base.size, 1);
1359
1360 obj->fault_mappable = false;
1361}
1362
1363static void
1364i915_gem_free_mmap_offset(struct drm_i915_gem_object *obj)
1365{
1366 struct drm_device *dev = obj->base.dev;
1367 struct drm_gem_mm *mm = dev->mm_private;
1368 struct drm_map_list *list = &obj->base.map_list;
1369
1370 drm_ht_remove_item(&mm->offset_hash, &list->hash);
1371 drm_mm_put_block(list->file_offset_node);
1372 kfree(list->map);
1373 list->map = NULL;
1374}
1375
1376static uint32_t
1377i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1378{
1379 uint32_t gtt_size;
1380
1381 if (INTEL_INFO(dev)->gen >= 4 ||
1382 tiling_mode == I915_TILING_NONE)
1383 return size;
1384
1385 /* Previous chips need a power-of-two fence region when tiling */
1386 if (INTEL_INFO(dev)->gen == 3)
1387 gtt_size = 1024*1024;
1388 else
1389 gtt_size = 512*1024;
1390
1391 while (gtt_size < size)
1392 gtt_size <<= 1;
1393
1394 return gtt_size;
1395}
1396
1397/**
1398 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
1399 * @obj: object to check
1400 *
1401 * Return the required GTT alignment for an object, taking into account
1402 * potential fence register mapping.
1403 */
1404static uint32_t
1405i915_gem_get_gtt_alignment(struct drm_device *dev,
1406 uint32_t size,
1407 int tiling_mode)
1408{
1409 /*
1410 * Minimum alignment is 4k (GTT page size), but might be greater
1411 * if a fence register is needed for the object.
1412 */
1413 if (INTEL_INFO(dev)->gen >= 4 ||
1414 tiling_mode == I915_TILING_NONE)
1415 return 4096;
1416
1417 /*
1418 * Previous chips need to be aligned to the size of the smallest
1419 * fence register that can contain the object.
1420 */
1421 return i915_gem_get_gtt_size(dev, size, tiling_mode);
1422}
1423
1424/**
1425 * i915_gem_get_unfenced_gtt_alignment - return required GTT alignment for an
1426 * unfenced object
1427 * @dev: the device
1428 * @size: size of the object
1429 * @tiling_mode: tiling mode of the object
1430 *
1431 * Return the required GTT alignment for an object, only taking into account
1432 * unfenced tiled surface requirements.
1433 */
1434uint32_t
1435i915_gem_get_unfenced_gtt_alignment(struct drm_device *dev,
1436 uint32_t size,
1437 int tiling_mode)
1438{
1439 /*
1440 * Minimum alignment is 4k (GTT page size) for sane hw.
1441 */
1442 if (INTEL_INFO(dev)->gen >= 4 || IS_G33(dev) ||
1443 tiling_mode == I915_TILING_NONE)
1444 return 4096;
1445
1446 /* Previous hardware however needs to be aligned to a power-of-two
1447 * tile height. The simplest method for determining this is to reuse
1448 * the power-of-tile object size.
1449 */
1450 return i915_gem_get_gtt_size(dev, size, tiling_mode);
1451}
1452
1453int
1454i915_gem_mmap_gtt(struct drm_file *file,
1455 struct drm_device *dev,
1456 uint32_t handle,
1457 uint64_t *offset)
1458{
1459 struct drm_i915_private *dev_priv = dev->dev_private;
1460 struct drm_i915_gem_object *obj;
1461 int ret;
1462
1463 if (!(dev->driver->driver_features & DRIVER_GEM))
1464 return -ENODEV;
1465
1466 ret = i915_mutex_lock_interruptible(dev);
1467 if (ret)
1468 return ret;
1469
1470 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
1471 if (&obj->base == NULL) {
1472 ret = -ENOENT;
1473 goto unlock;
1474 }
1475
1476 if (obj->base.size > dev_priv->mm.gtt_mappable_end) {
1477 ret = -E2BIG;
1478 goto unlock;
1479 }
1480
1481 if (obj->madv != I915_MADV_WILLNEED) {
1482 DRM_ERROR("Attempting to mmap a purgeable buffer\n");
1483 ret = -EINVAL;
1484 goto out;
1485 }
1486
1487 if (!obj->base.map_list.map) {
1488 ret = i915_gem_create_mmap_offset(obj);
1489 if (ret)
1490 goto out;
1491 }
1492
1493 *offset = (u64)obj->base.map_list.hash.key << PAGE_SHIFT;
1494
1495out:
1496 drm_gem_object_unreference(&obj->base);
1497unlock:
1498 mutex_unlock(&dev->struct_mutex);
1499 return ret;
1500}
1501
1502/**
1503 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
1504 * @dev: DRM device
1505 * @data: GTT mapping ioctl data
1506 * @file: GEM object info
1507 *
1508 * Simply returns the fake offset to userspace so it can mmap it.
1509 * The mmap call will end up in drm_gem_mmap(), which will set things
1510 * up so we can get faults in the handler above.
1511 *
1512 * The fault handler will take care of binding the object into the GTT
1513 * (since it may have been evicted to make room for something), allocating
1514 * a fence register, and mapping the appropriate aperture address into
1515 * userspace.
1516 */
1517int
1518i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
1519 struct drm_file *file)
1520{
1521 struct drm_i915_gem_mmap_gtt *args = data;
1522
1523 if (!(dev->driver->driver_features & DRIVER_GEM))
1524 return -ENODEV;
1525
1526 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
1527}
1528
1529
1530static int
1531i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj,
1532 gfp_t gfpmask)
1533{
1534 int page_count, i;
1535 struct address_space *mapping;
1536 struct inode *inode;
1537 struct page *page;
1538
1539 /* Get the list of pages out of our struct file. They'll be pinned
1540 * at this point until we release them.
1541 */
1542 page_count = obj->base.size / PAGE_SIZE;
1543 BUG_ON(obj->pages != NULL);
1544 obj->pages = drm_malloc_ab(page_count, sizeof(struct page *));
1545 if (obj->pages == NULL)
1546 return -ENOMEM;
1547
1548 inode = obj->base.filp->f_path.dentry->d_inode;
1549 mapping = inode->i_mapping;
1550 gfpmask |= mapping_gfp_mask(mapping);
1551
1552 for (i = 0; i < page_count; i++) {
1553 page = shmem_read_mapping_page_gfp(mapping, i, gfpmask);
1554 if (IS_ERR(page))
1555 goto err_pages;
1556
1557 obj->pages[i] = page;
1558 }
1559
1560 if (obj->tiling_mode != I915_TILING_NONE)
1561 i915_gem_object_do_bit_17_swizzle(obj);
1562
1563 return 0;
1564
1565err_pages:
1566 while (i--)
1567 page_cache_release(obj->pages[i]);
1568
1569 drm_free_large(obj->pages);
1570 obj->pages = NULL;
1571 return PTR_ERR(page);
1572}
1573
1574static void
1575i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
1576{
1577 int page_count = obj->base.size / PAGE_SIZE;
1578 int i;
1579
1580 BUG_ON(obj->madv == __I915_MADV_PURGED);
1581
1582 if (obj->tiling_mode != I915_TILING_NONE)
1583 i915_gem_object_save_bit_17_swizzle(obj);
1584
1585 if (obj->madv == I915_MADV_DONTNEED)
1586 obj->dirty = 0;
1587
1588 for (i = 0; i < page_count; i++) {
1589 if (obj->dirty)
1590 set_page_dirty(obj->pages[i]);
1591
1592 if (obj->madv == I915_MADV_WILLNEED)
1593 mark_page_accessed(obj->pages[i]);
1594
1595 page_cache_release(obj->pages[i]);
1596 }
1597 obj->dirty = 0;
1598
1599 drm_free_large(obj->pages);
1600 obj->pages = NULL;
1601}
1602
1603void
1604i915_gem_object_move_to_active(struct drm_i915_gem_object *obj,
1605 struct intel_ring_buffer *ring,
1606 u32 seqno)
1607{
1608 struct drm_device *dev = obj->base.dev;
1609 struct drm_i915_private *dev_priv = dev->dev_private;
1610
1611 BUG_ON(ring == NULL);
1612 obj->ring = ring;
1613
1614 /* Add a reference if we're newly entering the active list. */
1615 if (!obj->active) {
1616 drm_gem_object_reference(&obj->base);
1617 obj->active = 1;
1618 }
1619
1620 /* Move from whatever list we were on to the tail of execution. */
1621 list_move_tail(&obj->mm_list, &dev_priv->mm.active_list);
1622 list_move_tail(&obj->ring_list, &ring->active_list);
1623
1624 obj->last_rendering_seqno = seqno;
1625 if (obj->fenced_gpu_access) {
1626 struct drm_i915_fence_reg *reg;
1627
1628 BUG_ON(obj->fence_reg == I915_FENCE_REG_NONE);
1629
1630 obj->last_fenced_seqno = seqno;
1631 obj->last_fenced_ring = ring;
1632
1633 reg = &dev_priv->fence_regs[obj->fence_reg];
1634 list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
1635 }
1636}
1637
1638static void
1639i915_gem_object_move_off_active(struct drm_i915_gem_object *obj)
1640{
1641 list_del_init(&obj->ring_list);
1642 obj->last_rendering_seqno = 0;
1643}
1644
1645static void
1646i915_gem_object_move_to_flushing(struct drm_i915_gem_object *obj)
1647{
1648 struct drm_device *dev = obj->base.dev;
1649 drm_i915_private_t *dev_priv = dev->dev_private;
1650
1651 BUG_ON(!obj->active);
1652 list_move_tail(&obj->mm_list, &dev_priv->mm.flushing_list);
1653
1654 i915_gem_object_move_off_active(obj);
1655}
1656
1657static void
1658i915_gem_object_move_to_inactive(struct drm_i915_gem_object *obj)
1659{
1660 struct drm_device *dev = obj->base.dev;
1661 struct drm_i915_private *dev_priv = dev->dev_private;
1662
1663 if (obj->pin_count != 0)
1664 list_move_tail(&obj->mm_list, &dev_priv->mm.pinned_list);
1665 else
1666 list_move_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
1667
1668 BUG_ON(!list_empty(&obj->gpu_write_list));
1669 BUG_ON(!obj->active);
1670 obj->ring = NULL;
1671
1672 i915_gem_object_move_off_active(obj);
1673 obj->fenced_gpu_access = false;
1674
1675 obj->active = 0;
1676 obj->pending_gpu_write = false;
1677 drm_gem_object_unreference(&obj->base);
1678
1679 WARN_ON(i915_verify_lists(dev));
1680}
1681
1682/* Immediately discard the backing storage */
1683static void
1684i915_gem_object_truncate(struct drm_i915_gem_object *obj)
1685{
1686 struct inode *inode;
1687
1688 /* Our goal here is to return as much of the memory as
1689 * is possible back to the system as we are called from OOM.
1690 * To do this we must instruct the shmfs to drop all of its
1691 * backing pages, *now*.
1692 */
1693 inode = obj->base.filp->f_path.dentry->d_inode;
1694 shmem_truncate_range(inode, 0, (loff_t)-1);
1695
1696 obj->madv = __I915_MADV_PURGED;
1697}
1698
1699static inline int
1700i915_gem_object_is_purgeable(struct drm_i915_gem_object *obj)
1701{
1702 return obj->madv == I915_MADV_DONTNEED;
1703}
1704
1705static void
1706i915_gem_process_flushing_list(struct intel_ring_buffer *ring,
1707 uint32_t flush_domains)
1708{
1709 struct drm_i915_gem_object *obj, *next;
1710
1711 list_for_each_entry_safe(obj, next,
1712 &ring->gpu_write_list,
1713 gpu_write_list) {
1714 if (obj->base.write_domain & flush_domains) {
1715 uint32_t old_write_domain = obj->base.write_domain;
1716
1717 obj->base.write_domain = 0;
1718 list_del_init(&obj->gpu_write_list);
1719 i915_gem_object_move_to_active(obj, ring,
1720 i915_gem_next_request_seqno(ring));
1721
1722 trace_i915_gem_object_change_domain(obj,
1723 obj->base.read_domains,
1724 old_write_domain);
1725 }
1726 }
1727}
1728
1729int
1730i915_add_request(struct intel_ring_buffer *ring,
1731 struct drm_file *file,
1732 struct drm_i915_gem_request *request)
1733{
1734 drm_i915_private_t *dev_priv = ring->dev->dev_private;
1735 uint32_t seqno;
1736 int was_empty;
1737 int ret;
1738
1739 BUG_ON(request == NULL);
1740
1741 ret = ring->add_request(ring, &seqno);
1742 if (ret)
1743 return ret;
1744
1745 trace_i915_gem_request_add(ring, seqno);
1746
1747 request->seqno = seqno;
1748 request->ring = ring;
1749 request->emitted_jiffies = jiffies;
1750 was_empty = list_empty(&ring->request_list);
1751 list_add_tail(&request->list, &ring->request_list);
1752
1753 if (file) {
1754 struct drm_i915_file_private *file_priv = file->driver_priv;
1755
1756 spin_lock(&file_priv->mm.lock);
1757 request->file_priv = file_priv;
1758 list_add_tail(&request->client_list,
1759 &file_priv->mm.request_list);
1760 spin_unlock(&file_priv->mm.lock);
1761 }
1762
1763 ring->outstanding_lazy_request = false;
1764
1765 if (!dev_priv->mm.suspended) {
1766 if (i915_enable_hangcheck) {
1767 mod_timer(&dev_priv->hangcheck_timer,
1768 jiffies +
1769 msecs_to_jiffies(DRM_I915_HANGCHECK_PERIOD));
1770 }
1771 if (was_empty)
1772 queue_delayed_work(dev_priv->wq,
1773 &dev_priv->mm.retire_work, HZ);
1774 }
1775 return 0;
1776}
1777
1778static inline void
1779i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1780{
1781 struct drm_i915_file_private *file_priv = request->file_priv;
1782
1783 if (!file_priv)
1784 return;
1785
1786 spin_lock(&file_priv->mm.lock);
1787 if (request->file_priv) {
1788 list_del(&request->client_list);
1789 request->file_priv = NULL;
1790 }
1791 spin_unlock(&file_priv->mm.lock);
1792}
1793
1794static void i915_gem_reset_ring_lists(struct drm_i915_private *dev_priv,
1795 struct intel_ring_buffer *ring)
1796{
1797 while (!list_empty(&ring->request_list)) {
1798 struct drm_i915_gem_request *request;
1799
1800 request = list_first_entry(&ring->request_list,
1801 struct drm_i915_gem_request,
1802 list);
1803
1804 list_del(&request->list);
1805 i915_gem_request_remove_from_client(request);
1806 kfree(request);
1807 }
1808
1809 while (!list_empty(&ring->active_list)) {
1810 struct drm_i915_gem_object *obj;
1811
1812 obj = list_first_entry(&ring->active_list,
1813 struct drm_i915_gem_object,
1814 ring_list);
1815
1816 obj->base.write_domain = 0;
1817 list_del_init(&obj->gpu_write_list);
1818 i915_gem_object_move_to_inactive(obj);
1819 }
1820}
1821
1822static void i915_gem_reset_fences(struct drm_device *dev)
1823{
1824 struct drm_i915_private *dev_priv = dev->dev_private;
1825 int i;
1826
1827 for (i = 0; i < 16; i++) {
1828 struct drm_i915_fence_reg *reg = &dev_priv->fence_regs[i];
1829 struct drm_i915_gem_object *obj = reg->obj;
1830
1831 if (!obj)
1832 continue;
1833
1834 if (obj->tiling_mode)
1835 i915_gem_release_mmap(obj);
1836
1837 reg->obj->fence_reg = I915_FENCE_REG_NONE;
1838 reg->obj->fenced_gpu_access = false;
1839 reg->obj->last_fenced_seqno = 0;
1840 reg->obj->last_fenced_ring = NULL;
1841 i915_gem_clear_fence_reg(dev, reg);
1842 }
1843}
1844
1845void i915_gem_reset(struct drm_device *dev)
1846{
1847 struct drm_i915_private *dev_priv = dev->dev_private;
1848 struct drm_i915_gem_object *obj;
1849 int i;
1850
1851 for (i = 0; i < I915_NUM_RINGS; i++)
1852 i915_gem_reset_ring_lists(dev_priv, &dev_priv->ring[i]);
1853
1854 /* Remove anything from the flushing lists. The GPU cache is likely
1855 * to be lost on reset along with the data, so simply move the
1856 * lost bo to the inactive list.
1857 */
1858 while (!list_empty(&dev_priv->mm.flushing_list)) {
1859 obj= list_first_entry(&dev_priv->mm.flushing_list,
1860 struct drm_i915_gem_object,
1861 mm_list);
1862
1863 obj->base.write_domain = 0;
1864 list_del_init(&obj->gpu_write_list);
1865 i915_gem_object_move_to_inactive(obj);
1866 }
1867
1868 /* Move everything out of the GPU domains to ensure we do any
1869 * necessary invalidation upon reuse.
1870 */
1871 list_for_each_entry(obj,
1872 &dev_priv->mm.inactive_list,
1873 mm_list)
1874 {
1875 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
1876 }
1877
1878 /* The fence registers are invalidated so clear them out */
1879 i915_gem_reset_fences(dev);
1880}
1881
1882/**
1883 * This function clears the request list as sequence numbers are passed.
1884 */
1885static void
1886i915_gem_retire_requests_ring(struct intel_ring_buffer *ring)
1887{
1888 uint32_t seqno;
1889 int i;
1890
1891 if (list_empty(&ring->request_list))
1892 return;
1893
1894 WARN_ON(i915_verify_lists(ring->dev));
1895
1896 seqno = ring->get_seqno(ring);
1897
1898 for (i = 0; i < ARRAY_SIZE(ring->sync_seqno); i++)
1899 if (seqno >= ring->sync_seqno[i])
1900 ring->sync_seqno[i] = 0;
1901
1902 while (!list_empty(&ring->request_list)) {
1903 struct drm_i915_gem_request *request;
1904
1905 request = list_first_entry(&ring->request_list,
1906 struct drm_i915_gem_request,
1907 list);
1908
1909 if (!i915_seqno_passed(seqno, request->seqno))
1910 break;
1911
1912 trace_i915_gem_request_retire(ring, request->seqno);
1913
1914 list_del(&request->list);
1915 i915_gem_request_remove_from_client(request);
1916 kfree(request);
1917 }
1918
1919 /* Move any buffers on the active list that are no longer referenced
1920 * by the ringbuffer to the flushing/inactive lists as appropriate.
1921 */
1922 while (!list_empty(&ring->active_list)) {
1923 struct drm_i915_gem_object *obj;
1924
1925 obj= list_first_entry(&ring->active_list,
1926 struct drm_i915_gem_object,
1927 ring_list);
1928
1929 if (!i915_seqno_passed(seqno, obj->last_rendering_seqno))
1930 break;
1931
1932 if (obj->base.write_domain != 0)
1933 i915_gem_object_move_to_flushing(obj);
1934 else
1935 i915_gem_object_move_to_inactive(obj);
1936 }
1937
1938 if (unlikely(ring->trace_irq_seqno &&
1939 i915_seqno_passed(seqno, ring->trace_irq_seqno))) {
1940 ring->irq_put(ring);
1941 ring->trace_irq_seqno = 0;
1942 }
1943
1944 WARN_ON(i915_verify_lists(ring->dev));
1945}
1946
1947void
1948i915_gem_retire_requests(struct drm_device *dev)
1949{
1950 drm_i915_private_t *dev_priv = dev->dev_private;
1951 int i;
1952
1953 if (!list_empty(&dev_priv->mm.deferred_free_list)) {
1954 struct drm_i915_gem_object *obj, *next;
1955
1956 /* We must be careful that during unbind() we do not
1957 * accidentally infinitely recurse into retire requests.
1958 * Currently:
1959 * retire -> free -> unbind -> wait -> retire_ring
1960 */
1961 list_for_each_entry_safe(obj, next,
1962 &dev_priv->mm.deferred_free_list,
1963 mm_list)
1964 i915_gem_free_object_tail(obj);
1965 }
1966
1967 for (i = 0; i < I915_NUM_RINGS; i++)
1968 i915_gem_retire_requests_ring(&dev_priv->ring[i]);
1969}
1970
1971static void
1972i915_gem_retire_work_handler(struct work_struct *work)
1973{
1974 drm_i915_private_t *dev_priv;
1975 struct drm_device *dev;
1976 bool idle;
1977 int i;
1978
1979 dev_priv = container_of(work, drm_i915_private_t,
1980 mm.retire_work.work);
1981 dev = dev_priv->dev;
1982
1983 /* Come back later if the device is busy... */
1984 if (!mutex_trylock(&dev->struct_mutex)) {
1985 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
1986 return;
1987 }
1988
1989 i915_gem_retire_requests(dev);
1990
1991 /* Send a periodic flush down the ring so we don't hold onto GEM
1992 * objects indefinitely.
1993 */
1994 idle = true;
1995 for (i = 0; i < I915_NUM_RINGS; i++) {
1996 struct intel_ring_buffer *ring = &dev_priv->ring[i];
1997
1998 if (!list_empty(&ring->gpu_write_list)) {
1999 struct drm_i915_gem_request *request;
2000 int ret;
2001
2002 ret = i915_gem_flush_ring(ring,
2003 0, I915_GEM_GPU_DOMAINS);
2004 request = kzalloc(sizeof(*request), GFP_KERNEL);
2005 if (ret || request == NULL ||
2006 i915_add_request(ring, NULL, request))
2007 kfree(request);
2008 }
2009
2010 idle &= list_empty(&ring->request_list);
2011 }
2012
2013 if (!dev_priv->mm.suspended && !idle)
2014 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, HZ);
2015
2016 mutex_unlock(&dev->struct_mutex);
2017}
2018
2019/**
2020 * Waits for a sequence number to be signaled, and cleans up the
2021 * request and object lists appropriately for that event.
2022 */
2023int
2024i915_wait_request(struct intel_ring_buffer *ring,
2025 uint32_t seqno)
2026{
2027 drm_i915_private_t *dev_priv = ring->dev->dev_private;
2028 u32 ier;
2029 int ret = 0;
2030
2031 BUG_ON(seqno == 0);
2032
2033 if (atomic_read(&dev_priv->mm.wedged)) {
2034 struct completion *x = &dev_priv->error_completion;
2035 bool recovery_complete;
2036 unsigned long flags;
2037
2038 /* Give the error handler a chance to run. */
2039 spin_lock_irqsave(&x->wait.lock, flags);
2040 recovery_complete = x->done > 0;
2041 spin_unlock_irqrestore(&x->wait.lock, flags);
2042
2043 return recovery_complete ? -EIO : -EAGAIN;
2044 }
2045
2046 if (seqno == ring->outstanding_lazy_request) {
2047 struct drm_i915_gem_request *request;
2048
2049 request = kzalloc(sizeof(*request), GFP_KERNEL);
2050 if (request == NULL)
2051 return -ENOMEM;
2052
2053 ret = i915_add_request(ring, NULL, request);
2054 if (ret) {
2055 kfree(request);
2056 return ret;
2057 }
2058
2059 seqno = request->seqno;
2060 }
2061
2062 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
2063 if (HAS_PCH_SPLIT(ring->dev))
2064 ier = I915_READ(DEIER) | I915_READ(GTIER);
2065 else
2066 ier = I915_READ(IER);
2067 if (!ier) {
2068 DRM_ERROR("something (likely vbetool) disabled "
2069 "interrupts, re-enabling\n");
2070 ring->dev->driver->irq_preinstall(ring->dev);
2071 ring->dev->driver->irq_postinstall(ring->dev);
2072 }
2073
2074 trace_i915_gem_request_wait_begin(ring, seqno);
2075
2076 ring->waiting_seqno = seqno;
2077 if (ring->irq_get(ring)) {
2078 if (dev_priv->mm.interruptible)
2079 ret = wait_event_interruptible(ring->irq_queue,
2080 i915_seqno_passed(ring->get_seqno(ring), seqno)
2081 || atomic_read(&dev_priv->mm.wedged));
2082 else
2083 wait_event(ring->irq_queue,
2084 i915_seqno_passed(ring->get_seqno(ring), seqno)
2085 || atomic_read(&dev_priv->mm.wedged));
2086
2087 ring->irq_put(ring);
2088 } else if (wait_for(i915_seqno_passed(ring->get_seqno(ring),
2089 seqno) ||
2090 atomic_read(&dev_priv->mm.wedged), 3000))
2091 ret = -EBUSY;
2092 ring->waiting_seqno = 0;
2093
2094 trace_i915_gem_request_wait_end(ring, seqno);
2095 }
2096 if (atomic_read(&dev_priv->mm.wedged))
2097 ret = -EAGAIN;
2098
2099 if (ret && ret != -ERESTARTSYS)
2100 DRM_ERROR("%s returns %d (awaiting %d at %d, next %d)\n",
2101 __func__, ret, seqno, ring->get_seqno(ring),
2102 dev_priv->next_seqno);
2103
2104 /* Directly dispatch request retiring. While we have the work queue
2105 * to handle this, the waiter on a request often wants an associated
2106 * buffer to have made it to the inactive list, and we would need
2107 * a separate wait queue to handle that.
2108 */
2109 if (ret == 0)
2110 i915_gem_retire_requests_ring(ring);
2111
2112 return ret;
2113}
2114
2115/**
2116 * Ensures that all rendering to the object has completed and the object is
2117 * safe to unbind from the GTT or access from the CPU.
2118 */
2119int
2120i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj)
2121{
2122 int ret;
2123
2124 /* This function only exists to support waiting for existing rendering,
2125 * not for emitting required flushes.
2126 */
2127 BUG_ON((obj->base.write_domain & I915_GEM_GPU_DOMAINS) != 0);
2128
2129 /* If there is rendering queued on the buffer being evicted, wait for
2130 * it.
2131 */
2132 if (obj->active) {
2133 ret = i915_wait_request(obj->ring, obj->last_rendering_seqno);
2134 if (ret)
2135 return ret;
2136 }
2137
2138 return 0;
2139}
2140
2141static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
2142{
2143 u32 old_write_domain, old_read_domains;
2144
2145 /* Act a barrier for all accesses through the GTT */
2146 mb();
2147
2148 /* Force a pagefault for domain tracking on next user access */
2149 i915_gem_release_mmap(obj);
2150
2151 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
2152 return;
2153
2154 old_read_domains = obj->base.read_domains;
2155 old_write_domain = obj->base.write_domain;
2156
2157 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
2158 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
2159
2160 trace_i915_gem_object_change_domain(obj,
2161 old_read_domains,
2162 old_write_domain);
2163}
2164
2165/**
2166 * Unbinds an object from the GTT aperture.
2167 */
2168int
2169i915_gem_object_unbind(struct drm_i915_gem_object *obj)
2170{
2171 int ret = 0;
2172
2173 if (obj->gtt_space == NULL)
2174 return 0;
2175
2176 if (obj->pin_count != 0) {
2177 DRM_ERROR("Attempting to unbind pinned buffer\n");
2178 return -EINVAL;
2179 }
2180
2181 ret = i915_gem_object_finish_gpu(obj);
2182 if (ret == -ERESTARTSYS)
2183 return ret;
2184 /* Continue on if we fail due to EIO, the GPU is hung so we
2185 * should be safe and we need to cleanup or else we might
2186 * cause memory corruption through use-after-free.
2187 */
2188
2189 i915_gem_object_finish_gtt(obj);
2190
2191 /* Move the object to the CPU domain to ensure that
2192 * any possible CPU writes while it's not in the GTT
2193 * are flushed when we go to remap it.
2194 */
2195 if (ret == 0)
2196 ret = i915_gem_object_set_to_cpu_domain(obj, 1);
2197 if (ret == -ERESTARTSYS)
2198 return ret;
2199 if (ret) {
2200 /* In the event of a disaster, abandon all caches and
2201 * hope for the best.
2202 */
2203 i915_gem_clflush_object(obj);
2204 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2205 }
2206
2207 /* release the fence reg _after_ flushing */
2208 ret = i915_gem_object_put_fence(obj);
2209 if (ret == -ERESTARTSYS)
2210 return ret;
2211
2212 trace_i915_gem_object_unbind(obj);
2213
2214 i915_gem_gtt_unbind_object(obj);
2215 i915_gem_object_put_pages_gtt(obj);
2216
2217 list_del_init(&obj->gtt_list);
2218 list_del_init(&obj->mm_list);
2219 /* Avoid an unnecessary call to unbind on rebind. */
2220 obj->map_and_fenceable = true;
2221
2222 drm_mm_put_block(obj->gtt_space);
2223 obj->gtt_space = NULL;
2224 obj->gtt_offset = 0;
2225
2226 if (i915_gem_object_is_purgeable(obj))
2227 i915_gem_object_truncate(obj);
2228
2229 return ret;
2230}
2231
2232int
2233i915_gem_flush_ring(struct intel_ring_buffer *ring,
2234 uint32_t invalidate_domains,
2235 uint32_t flush_domains)
2236{
2237 int ret;
2238
2239 if (((invalidate_domains | flush_domains) & I915_GEM_GPU_DOMAINS) == 0)
2240 return 0;
2241
2242 trace_i915_gem_ring_flush(ring, invalidate_domains, flush_domains);
2243
2244 ret = ring->flush(ring, invalidate_domains, flush_domains);
2245 if (ret)
2246 return ret;
2247
2248 if (flush_domains & I915_GEM_GPU_DOMAINS)
2249 i915_gem_process_flushing_list(ring, flush_domains);
2250
2251 return 0;
2252}
2253
2254static int i915_ring_idle(struct intel_ring_buffer *ring)
2255{
2256 int ret;
2257
2258 if (list_empty(&ring->gpu_write_list) && list_empty(&ring->active_list))
2259 return 0;
2260
2261 if (!list_empty(&ring->gpu_write_list)) {
2262 ret = i915_gem_flush_ring(ring,
2263 I915_GEM_GPU_DOMAINS, I915_GEM_GPU_DOMAINS);
2264 if (ret)
2265 return ret;
2266 }
2267
2268 return i915_wait_request(ring, i915_gem_next_request_seqno(ring));
2269}
2270
2271int
2272i915_gpu_idle(struct drm_device *dev)
2273{
2274 drm_i915_private_t *dev_priv = dev->dev_private;
2275 bool lists_empty;
2276 int ret, i;
2277
2278 lists_empty = (list_empty(&dev_priv->mm.flushing_list) &&
2279 list_empty(&dev_priv->mm.active_list));
2280 if (lists_empty)
2281 return 0;
2282
2283 /* Flush everything onto the inactive list. */
2284 for (i = 0; i < I915_NUM_RINGS; i++) {
2285 ret = i915_ring_idle(&dev_priv->ring[i]);
2286 if (ret)
2287 return ret;
2288 }
2289
2290 return 0;
2291}
2292
2293static int sandybridge_write_fence_reg(struct drm_i915_gem_object *obj,
2294 struct intel_ring_buffer *pipelined)
2295{
2296 struct drm_device *dev = obj->base.dev;
2297 drm_i915_private_t *dev_priv = dev->dev_private;
2298 u32 size = obj->gtt_space->size;
2299 int regnum = obj->fence_reg;
2300 uint64_t val;
2301
2302 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2303 0xfffff000) << 32;
2304 val |= obj->gtt_offset & 0xfffff000;
2305 val |= (uint64_t)((obj->stride / 128) - 1) <<
2306 SANDYBRIDGE_FENCE_PITCH_SHIFT;
2307
2308 if (obj->tiling_mode == I915_TILING_Y)
2309 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2310 val |= I965_FENCE_REG_VALID;
2311
2312 if (pipelined) {
2313 int ret = intel_ring_begin(pipelined, 6);
2314 if (ret)
2315 return ret;
2316
2317 intel_ring_emit(pipelined, MI_NOOP);
2318 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2319 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8);
2320 intel_ring_emit(pipelined, (u32)val);
2321 intel_ring_emit(pipelined, FENCE_REG_SANDYBRIDGE_0 + regnum*8 + 4);
2322 intel_ring_emit(pipelined, (u32)(val >> 32));
2323 intel_ring_advance(pipelined);
2324 } else
2325 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + regnum * 8, val);
2326
2327 return 0;
2328}
2329
2330static int i965_write_fence_reg(struct drm_i915_gem_object *obj,
2331 struct intel_ring_buffer *pipelined)
2332{
2333 struct drm_device *dev = obj->base.dev;
2334 drm_i915_private_t *dev_priv = dev->dev_private;
2335 u32 size = obj->gtt_space->size;
2336 int regnum = obj->fence_reg;
2337 uint64_t val;
2338
2339 val = (uint64_t)((obj->gtt_offset + size - 4096) &
2340 0xfffff000) << 32;
2341 val |= obj->gtt_offset & 0xfffff000;
2342 val |= ((obj->stride / 128) - 1) << I965_FENCE_PITCH_SHIFT;
2343 if (obj->tiling_mode == I915_TILING_Y)
2344 val |= 1 << I965_FENCE_TILING_Y_SHIFT;
2345 val |= I965_FENCE_REG_VALID;
2346
2347 if (pipelined) {
2348 int ret = intel_ring_begin(pipelined, 6);
2349 if (ret)
2350 return ret;
2351
2352 intel_ring_emit(pipelined, MI_NOOP);
2353 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(2));
2354 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8);
2355 intel_ring_emit(pipelined, (u32)val);
2356 intel_ring_emit(pipelined, FENCE_REG_965_0 + regnum*8 + 4);
2357 intel_ring_emit(pipelined, (u32)(val >> 32));
2358 intel_ring_advance(pipelined);
2359 } else
2360 I915_WRITE64(FENCE_REG_965_0 + regnum * 8, val);
2361
2362 return 0;
2363}
2364
2365static int i915_write_fence_reg(struct drm_i915_gem_object *obj,
2366 struct intel_ring_buffer *pipelined)
2367{
2368 struct drm_device *dev = obj->base.dev;
2369 drm_i915_private_t *dev_priv = dev->dev_private;
2370 u32 size = obj->gtt_space->size;
2371 u32 fence_reg, val, pitch_val;
2372 int tile_width;
2373
2374 if (WARN((obj->gtt_offset & ~I915_FENCE_START_MASK) ||
2375 (size & -size) != size ||
2376 (obj->gtt_offset & (size - 1)),
2377 "object 0x%08x [fenceable? %d] not 1M or pot-size (0x%08x) aligned\n",
2378 obj->gtt_offset, obj->map_and_fenceable, size))
2379 return -EINVAL;
2380
2381 if (obj->tiling_mode == I915_TILING_Y && HAS_128_BYTE_Y_TILING(dev))
2382 tile_width = 128;
2383 else
2384 tile_width = 512;
2385
2386 /* Note: pitch better be a power of two tile widths */
2387 pitch_val = obj->stride / tile_width;
2388 pitch_val = ffs(pitch_val) - 1;
2389
2390 val = obj->gtt_offset;
2391 if (obj->tiling_mode == I915_TILING_Y)
2392 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2393 val |= I915_FENCE_SIZE_BITS(size);
2394 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2395 val |= I830_FENCE_REG_VALID;
2396
2397 fence_reg = obj->fence_reg;
2398 if (fence_reg < 8)
2399 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2400 else
2401 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2402
2403 if (pipelined) {
2404 int ret = intel_ring_begin(pipelined, 4);
2405 if (ret)
2406 return ret;
2407
2408 intel_ring_emit(pipelined, MI_NOOP);
2409 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2410 intel_ring_emit(pipelined, fence_reg);
2411 intel_ring_emit(pipelined, val);
2412 intel_ring_advance(pipelined);
2413 } else
2414 I915_WRITE(fence_reg, val);
2415
2416 return 0;
2417}
2418
2419static int i830_write_fence_reg(struct drm_i915_gem_object *obj,
2420 struct intel_ring_buffer *pipelined)
2421{
2422 struct drm_device *dev = obj->base.dev;
2423 drm_i915_private_t *dev_priv = dev->dev_private;
2424 u32 size = obj->gtt_space->size;
2425 int regnum = obj->fence_reg;
2426 uint32_t val;
2427 uint32_t pitch_val;
2428
2429 if (WARN((obj->gtt_offset & ~I830_FENCE_START_MASK) ||
2430 (size & -size) != size ||
2431 (obj->gtt_offset & (size - 1)),
2432 "object 0x%08x not 512K or pot-size 0x%08x aligned\n",
2433 obj->gtt_offset, size))
2434 return -EINVAL;
2435
2436 pitch_val = obj->stride / 128;
2437 pitch_val = ffs(pitch_val) - 1;
2438
2439 val = obj->gtt_offset;
2440 if (obj->tiling_mode == I915_TILING_Y)
2441 val |= 1 << I830_FENCE_TILING_Y_SHIFT;
2442 val |= I830_FENCE_SIZE_BITS(size);
2443 val |= pitch_val << I830_FENCE_PITCH_SHIFT;
2444 val |= I830_FENCE_REG_VALID;
2445
2446 if (pipelined) {
2447 int ret = intel_ring_begin(pipelined, 4);
2448 if (ret)
2449 return ret;
2450
2451 intel_ring_emit(pipelined, MI_NOOP);
2452 intel_ring_emit(pipelined, MI_LOAD_REGISTER_IMM(1));
2453 intel_ring_emit(pipelined, FENCE_REG_830_0 + regnum*4);
2454 intel_ring_emit(pipelined, val);
2455 intel_ring_advance(pipelined);
2456 } else
2457 I915_WRITE(FENCE_REG_830_0 + regnum * 4, val);
2458
2459 return 0;
2460}
2461
2462static bool ring_passed_seqno(struct intel_ring_buffer *ring, u32 seqno)
2463{
2464 return i915_seqno_passed(ring->get_seqno(ring), seqno);
2465}
2466
2467static int
2468i915_gem_object_flush_fence(struct drm_i915_gem_object *obj,
2469 struct intel_ring_buffer *pipelined)
2470{
2471 int ret;
2472
2473 if (obj->fenced_gpu_access) {
2474 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
2475 ret = i915_gem_flush_ring(obj->last_fenced_ring,
2476 0, obj->base.write_domain);
2477 if (ret)
2478 return ret;
2479 }
2480
2481 obj->fenced_gpu_access = false;
2482 }
2483
2484 if (obj->last_fenced_seqno && pipelined != obj->last_fenced_ring) {
2485 if (!ring_passed_seqno(obj->last_fenced_ring,
2486 obj->last_fenced_seqno)) {
2487 ret = i915_wait_request(obj->last_fenced_ring,
2488 obj->last_fenced_seqno);
2489 if (ret)
2490 return ret;
2491 }
2492
2493 obj->last_fenced_seqno = 0;
2494 obj->last_fenced_ring = NULL;
2495 }
2496
2497 /* Ensure that all CPU reads are completed before installing a fence
2498 * and all writes before removing the fence.
2499 */
2500 if (obj->base.read_domains & I915_GEM_DOMAIN_GTT)
2501 mb();
2502
2503 return 0;
2504}
2505
2506int
2507i915_gem_object_put_fence(struct drm_i915_gem_object *obj)
2508{
2509 int ret;
2510
2511 if (obj->tiling_mode)
2512 i915_gem_release_mmap(obj);
2513
2514 ret = i915_gem_object_flush_fence(obj, NULL);
2515 if (ret)
2516 return ret;
2517
2518 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2519 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2520 i915_gem_clear_fence_reg(obj->base.dev,
2521 &dev_priv->fence_regs[obj->fence_reg]);
2522
2523 obj->fence_reg = I915_FENCE_REG_NONE;
2524 }
2525
2526 return 0;
2527}
2528
2529static struct drm_i915_fence_reg *
2530i915_find_fence_reg(struct drm_device *dev,
2531 struct intel_ring_buffer *pipelined)
2532{
2533 struct drm_i915_private *dev_priv = dev->dev_private;
2534 struct drm_i915_fence_reg *reg, *first, *avail;
2535 int i;
2536
2537 /* First try to find a free reg */
2538 avail = NULL;
2539 for (i = dev_priv->fence_reg_start; i < dev_priv->num_fence_regs; i++) {
2540 reg = &dev_priv->fence_regs[i];
2541 if (!reg->obj)
2542 return reg;
2543
2544 if (!reg->obj->pin_count)
2545 avail = reg;
2546 }
2547
2548 if (avail == NULL)
2549 return NULL;
2550
2551 /* None available, try to steal one or wait for a user to finish */
2552 avail = first = NULL;
2553 list_for_each_entry(reg, &dev_priv->mm.fence_list, lru_list) {
2554 if (reg->obj->pin_count)
2555 continue;
2556
2557 if (first == NULL)
2558 first = reg;
2559
2560 if (!pipelined ||
2561 !reg->obj->last_fenced_ring ||
2562 reg->obj->last_fenced_ring == pipelined) {
2563 avail = reg;
2564 break;
2565 }
2566 }
2567
2568 if (avail == NULL)
2569 avail = first;
2570
2571 return avail;
2572}
2573
2574/**
2575 * i915_gem_object_get_fence - set up a fence reg for an object
2576 * @obj: object to map through a fence reg
2577 * @pipelined: ring on which to queue the change, or NULL for CPU access
2578 * @interruptible: must we wait uninterruptibly for the register to retire?
2579 *
2580 * When mapping objects through the GTT, userspace wants to be able to write
2581 * to them without having to worry about swizzling if the object is tiled.
2582 *
2583 * This function walks the fence regs looking for a free one for @obj,
2584 * stealing one if it can't find any.
2585 *
2586 * It then sets up the reg based on the object's properties: address, pitch
2587 * and tiling format.
2588 */
2589int
2590i915_gem_object_get_fence(struct drm_i915_gem_object *obj,
2591 struct intel_ring_buffer *pipelined)
2592{
2593 struct drm_device *dev = obj->base.dev;
2594 struct drm_i915_private *dev_priv = dev->dev_private;
2595 struct drm_i915_fence_reg *reg;
2596 int ret;
2597
2598 /* XXX disable pipelining. There are bugs. Shocking. */
2599 pipelined = NULL;
2600
2601 /* Just update our place in the LRU if our fence is getting reused. */
2602 if (obj->fence_reg != I915_FENCE_REG_NONE) {
2603 reg = &dev_priv->fence_regs[obj->fence_reg];
2604 list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
2605
2606 if (obj->tiling_changed) {
2607 ret = i915_gem_object_flush_fence(obj, pipelined);
2608 if (ret)
2609 return ret;
2610
2611 if (!obj->fenced_gpu_access && !obj->last_fenced_seqno)
2612 pipelined = NULL;
2613
2614 if (pipelined) {
2615 reg->setup_seqno =
2616 i915_gem_next_request_seqno(pipelined);
2617 obj->last_fenced_seqno = reg->setup_seqno;
2618 obj->last_fenced_ring = pipelined;
2619 }
2620
2621 goto update;
2622 }
2623
2624 if (!pipelined) {
2625 if (reg->setup_seqno) {
2626 if (!ring_passed_seqno(obj->last_fenced_ring,
2627 reg->setup_seqno)) {
2628 ret = i915_wait_request(obj->last_fenced_ring,
2629 reg->setup_seqno);
2630 if (ret)
2631 return ret;
2632 }
2633
2634 reg->setup_seqno = 0;
2635 }
2636 } else if (obj->last_fenced_ring &&
2637 obj->last_fenced_ring != pipelined) {
2638 ret = i915_gem_object_flush_fence(obj, pipelined);
2639 if (ret)
2640 return ret;
2641 }
2642
2643 return 0;
2644 }
2645
2646 reg = i915_find_fence_reg(dev, pipelined);
2647 if (reg == NULL)
2648 return -ENOSPC;
2649
2650 ret = i915_gem_object_flush_fence(obj, pipelined);
2651 if (ret)
2652 return ret;
2653
2654 if (reg->obj) {
2655 struct drm_i915_gem_object *old = reg->obj;
2656
2657 drm_gem_object_reference(&old->base);
2658
2659 if (old->tiling_mode)
2660 i915_gem_release_mmap(old);
2661
2662 ret = i915_gem_object_flush_fence(old, pipelined);
2663 if (ret) {
2664 drm_gem_object_unreference(&old->base);
2665 return ret;
2666 }
2667
2668 if (old->last_fenced_seqno == 0 && obj->last_fenced_seqno == 0)
2669 pipelined = NULL;
2670
2671 old->fence_reg = I915_FENCE_REG_NONE;
2672 old->last_fenced_ring = pipelined;
2673 old->last_fenced_seqno =
2674 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2675
2676 drm_gem_object_unreference(&old->base);
2677 } else if (obj->last_fenced_seqno == 0)
2678 pipelined = NULL;
2679
2680 reg->obj = obj;
2681 list_move_tail(®->lru_list, &dev_priv->mm.fence_list);
2682 obj->fence_reg = reg - dev_priv->fence_regs;
2683 obj->last_fenced_ring = pipelined;
2684
2685 reg->setup_seqno =
2686 pipelined ? i915_gem_next_request_seqno(pipelined) : 0;
2687 obj->last_fenced_seqno = reg->setup_seqno;
2688
2689update:
2690 obj->tiling_changed = false;
2691 switch (INTEL_INFO(dev)->gen) {
2692 case 7:
2693 case 6:
2694 ret = sandybridge_write_fence_reg(obj, pipelined);
2695 break;
2696 case 5:
2697 case 4:
2698 ret = i965_write_fence_reg(obj, pipelined);
2699 break;
2700 case 3:
2701 ret = i915_write_fence_reg(obj, pipelined);
2702 break;
2703 case 2:
2704 ret = i830_write_fence_reg(obj, pipelined);
2705 break;
2706 }
2707
2708 return ret;
2709}
2710
2711/**
2712 * i915_gem_clear_fence_reg - clear out fence register info
2713 * @obj: object to clear
2714 *
2715 * Zeroes out the fence register itself and clears out the associated
2716 * data structures in dev_priv and obj.
2717 */
2718static void
2719i915_gem_clear_fence_reg(struct drm_device *dev,
2720 struct drm_i915_fence_reg *reg)
2721{
2722 drm_i915_private_t *dev_priv = dev->dev_private;
2723 uint32_t fence_reg = reg - dev_priv->fence_regs;
2724
2725 switch (INTEL_INFO(dev)->gen) {
2726 case 7:
2727 case 6:
2728 I915_WRITE64(FENCE_REG_SANDYBRIDGE_0 + fence_reg*8, 0);
2729 break;
2730 case 5:
2731 case 4:
2732 I915_WRITE64(FENCE_REG_965_0 + fence_reg*8, 0);
2733 break;
2734 case 3:
2735 if (fence_reg >= 8)
2736 fence_reg = FENCE_REG_945_8 + (fence_reg - 8) * 4;
2737 else
2738 case 2:
2739 fence_reg = FENCE_REG_830_0 + fence_reg * 4;
2740
2741 I915_WRITE(fence_reg, 0);
2742 break;
2743 }
2744
2745 list_del_init(®->lru_list);
2746 reg->obj = NULL;
2747 reg->setup_seqno = 0;
2748}
2749
2750/**
2751 * Finds free space in the GTT aperture and binds the object there.
2752 */
2753static int
2754i915_gem_object_bind_to_gtt(struct drm_i915_gem_object *obj,
2755 unsigned alignment,
2756 bool map_and_fenceable)
2757{
2758 struct drm_device *dev = obj->base.dev;
2759 drm_i915_private_t *dev_priv = dev->dev_private;
2760 struct drm_mm_node *free_space;
2761 gfp_t gfpmask = __GFP_NORETRY | __GFP_NOWARN;
2762 u32 size, fence_size, fence_alignment, unfenced_alignment;
2763 bool mappable, fenceable;
2764 int ret;
2765
2766 if (obj->madv != I915_MADV_WILLNEED) {
2767 DRM_ERROR("Attempting to bind a purgeable object\n");
2768 return -EINVAL;
2769 }
2770
2771 fence_size = i915_gem_get_gtt_size(dev,
2772 obj->base.size,
2773 obj->tiling_mode);
2774 fence_alignment = i915_gem_get_gtt_alignment(dev,
2775 obj->base.size,
2776 obj->tiling_mode);
2777 unfenced_alignment =
2778 i915_gem_get_unfenced_gtt_alignment(dev,
2779 obj->base.size,
2780 obj->tiling_mode);
2781
2782 if (alignment == 0)
2783 alignment = map_and_fenceable ? fence_alignment :
2784 unfenced_alignment;
2785 if (map_and_fenceable && alignment & (fence_alignment - 1)) {
2786 DRM_ERROR("Invalid object alignment requested %u\n", alignment);
2787 return -EINVAL;
2788 }
2789
2790 size = map_and_fenceable ? fence_size : obj->base.size;
2791
2792 /* If the object is bigger than the entire aperture, reject it early
2793 * before evicting everything in a vain attempt to find space.
2794 */
2795 if (obj->base.size >
2796 (map_and_fenceable ? dev_priv->mm.gtt_mappable_end : dev_priv->mm.gtt_total)) {
2797 DRM_ERROR("Attempting to bind an object larger than the aperture\n");
2798 return -E2BIG;
2799 }
2800
2801 search_free:
2802 if (map_and_fenceable)
2803 free_space =
2804 drm_mm_search_free_in_range(&dev_priv->mm.gtt_space,
2805 size, alignment, 0,
2806 dev_priv->mm.gtt_mappable_end,
2807 0);
2808 else
2809 free_space = drm_mm_search_free(&dev_priv->mm.gtt_space,
2810 size, alignment, 0);
2811
2812 if (free_space != NULL) {
2813 if (map_and_fenceable)
2814 obj->gtt_space =
2815 drm_mm_get_block_range_generic(free_space,
2816 size, alignment, 0,
2817 dev_priv->mm.gtt_mappable_end,
2818 0);
2819 else
2820 obj->gtt_space =
2821 drm_mm_get_block(free_space, size, alignment);
2822 }
2823 if (obj->gtt_space == NULL) {
2824 /* If the gtt is empty and we're still having trouble
2825 * fitting our object in, we're out of memory.
2826 */
2827 ret = i915_gem_evict_something(dev, size, alignment,
2828 map_and_fenceable);
2829 if (ret)
2830 return ret;
2831
2832 goto search_free;
2833 }
2834
2835 ret = i915_gem_object_get_pages_gtt(obj, gfpmask);
2836 if (ret) {
2837 drm_mm_put_block(obj->gtt_space);
2838 obj->gtt_space = NULL;
2839
2840 if (ret == -ENOMEM) {
2841 /* first try to reclaim some memory by clearing the GTT */
2842 ret = i915_gem_evict_everything(dev, false);
2843 if (ret) {
2844 /* now try to shrink everyone else */
2845 if (gfpmask) {
2846 gfpmask = 0;
2847 goto search_free;
2848 }
2849
2850 return -ENOMEM;
2851 }
2852
2853 goto search_free;
2854 }
2855
2856 return ret;
2857 }
2858
2859 ret = i915_gem_gtt_bind_object(obj);
2860 if (ret) {
2861 i915_gem_object_put_pages_gtt(obj);
2862 drm_mm_put_block(obj->gtt_space);
2863 obj->gtt_space = NULL;
2864
2865 if (i915_gem_evict_everything(dev, false))
2866 return ret;
2867
2868 goto search_free;
2869 }
2870
2871 list_add_tail(&obj->gtt_list, &dev_priv->mm.gtt_list);
2872 list_add_tail(&obj->mm_list, &dev_priv->mm.inactive_list);
2873
2874 /* Assert that the object is not currently in any GPU domain. As it
2875 * wasn't in the GTT, there shouldn't be any way it could have been in
2876 * a GPU cache
2877 */
2878 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2879 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2880
2881 obj->gtt_offset = obj->gtt_space->start;
2882
2883 fenceable =
2884 obj->gtt_space->size == fence_size &&
2885 (obj->gtt_space->start & (fence_alignment -1)) == 0;
2886
2887 mappable =
2888 obj->gtt_offset + obj->base.size <= dev_priv->mm.gtt_mappable_end;
2889
2890 obj->map_and_fenceable = mappable && fenceable;
2891
2892 trace_i915_gem_object_bind(obj, map_and_fenceable);
2893 return 0;
2894}
2895
2896void
2897i915_gem_clflush_object(struct drm_i915_gem_object *obj)
2898{
2899 /* If we don't have a page list set up, then we're not pinned
2900 * to GPU, and we can ignore the cache flush because it'll happen
2901 * again at bind time.
2902 */
2903 if (obj->pages == NULL)
2904 return;
2905
2906 /* If the GPU is snooping the contents of the CPU cache,
2907 * we do not need to manually clear the CPU cache lines. However,
2908 * the caches are only snooped when the render cache is
2909 * flushed/invalidated. As we always have to emit invalidations
2910 * and flushes when moving into and out of the RENDER domain, correct
2911 * snooping behaviour occurs naturally as the result of our domain
2912 * tracking.
2913 */
2914 if (obj->cache_level != I915_CACHE_NONE)
2915 return;
2916
2917 trace_i915_gem_object_clflush(obj);
2918
2919 drm_clflush_pages(obj->pages, obj->base.size / PAGE_SIZE);
2920}
2921
2922/** Flushes any GPU write domain for the object if it's dirty. */
2923static int
2924i915_gem_object_flush_gpu_write_domain(struct drm_i915_gem_object *obj)
2925{
2926 if ((obj->base.write_domain & I915_GEM_GPU_DOMAINS) == 0)
2927 return 0;
2928
2929 /* Queue the GPU write cache flushing we need. */
2930 return i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
2931}
2932
2933/** Flushes the GTT write domain for the object if it's dirty. */
2934static void
2935i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
2936{
2937 uint32_t old_write_domain;
2938
2939 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
2940 return;
2941
2942 /* No actual flushing is required for the GTT write domain. Writes
2943 * to it immediately go to main memory as far as we know, so there's
2944 * no chipset flush. It also doesn't land in render cache.
2945 *
2946 * However, we do have to enforce the order so that all writes through
2947 * the GTT land before any writes to the device, such as updates to
2948 * the GATT itself.
2949 */
2950 wmb();
2951
2952 old_write_domain = obj->base.write_domain;
2953 obj->base.write_domain = 0;
2954
2955 trace_i915_gem_object_change_domain(obj,
2956 obj->base.read_domains,
2957 old_write_domain);
2958}
2959
2960/** Flushes the CPU write domain for the object if it's dirty. */
2961static void
2962i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
2963{
2964 uint32_t old_write_domain;
2965
2966 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
2967 return;
2968
2969 i915_gem_clflush_object(obj);
2970 intel_gtt_chipset_flush();
2971 old_write_domain = obj->base.write_domain;
2972 obj->base.write_domain = 0;
2973
2974 trace_i915_gem_object_change_domain(obj,
2975 obj->base.read_domains,
2976 old_write_domain);
2977}
2978
2979/**
2980 * Moves a single object to the GTT read, and possibly write domain.
2981 *
2982 * This function returns when the move is complete, including waiting on
2983 * flushes to occur.
2984 */
2985int
2986i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
2987{
2988 uint32_t old_write_domain, old_read_domains;
2989 int ret;
2990
2991 /* Not valid to be called on unbound objects. */
2992 if (obj->gtt_space == NULL)
2993 return -EINVAL;
2994
2995 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
2996 return 0;
2997
2998 ret = i915_gem_object_flush_gpu_write_domain(obj);
2999 if (ret)
3000 return ret;
3001
3002 if (obj->pending_gpu_write || write) {
3003 ret = i915_gem_object_wait_rendering(obj);
3004 if (ret)
3005 return ret;
3006 }
3007
3008 i915_gem_object_flush_cpu_write_domain(obj);
3009
3010 old_write_domain = obj->base.write_domain;
3011 old_read_domains = obj->base.read_domains;
3012
3013 /* It should now be out of any other write domains, and we can update
3014 * the domain values for our changes.
3015 */
3016 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3017 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3018 if (write) {
3019 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3020 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3021 obj->dirty = 1;
3022 }
3023
3024 trace_i915_gem_object_change_domain(obj,
3025 old_read_domains,
3026 old_write_domain);
3027
3028 return 0;
3029}
3030
3031int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3032 enum i915_cache_level cache_level)
3033{
3034 int ret;
3035
3036 if (obj->cache_level == cache_level)
3037 return 0;
3038
3039 if (obj->pin_count) {
3040 DRM_DEBUG("can not change the cache level of pinned objects\n");
3041 return -EBUSY;
3042 }
3043
3044 if (obj->gtt_space) {
3045 ret = i915_gem_object_finish_gpu(obj);
3046 if (ret)
3047 return ret;
3048
3049 i915_gem_object_finish_gtt(obj);
3050
3051 /* Before SandyBridge, you could not use tiling or fence
3052 * registers with snooped memory, so relinquish any fences
3053 * currently pointing to our region in the aperture.
3054 */
3055 if (INTEL_INFO(obj->base.dev)->gen < 6) {
3056 ret = i915_gem_object_put_fence(obj);
3057 if (ret)
3058 return ret;
3059 }
3060
3061 i915_gem_gtt_rebind_object(obj, cache_level);
3062 }
3063
3064 if (cache_level == I915_CACHE_NONE) {
3065 u32 old_read_domains, old_write_domain;
3066
3067 /* If we're coming from LLC cached, then we haven't
3068 * actually been tracking whether the data is in the
3069 * CPU cache or not, since we only allow one bit set
3070 * in obj->write_domain and have been skipping the clflushes.
3071 * Just set it to the CPU cache for now.
3072 */
3073 WARN_ON(obj->base.write_domain & ~I915_GEM_DOMAIN_CPU);
3074 WARN_ON(obj->base.read_domains & ~I915_GEM_DOMAIN_CPU);
3075
3076 old_read_domains = obj->base.read_domains;
3077 old_write_domain = obj->base.write_domain;
3078
3079 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3080 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3081
3082 trace_i915_gem_object_change_domain(obj,
3083 old_read_domains,
3084 old_write_domain);
3085 }
3086
3087 obj->cache_level = cache_level;
3088 return 0;
3089}
3090
3091/*
3092 * Prepare buffer for display plane (scanout, cursors, etc).
3093 * Can be called from an uninterruptible phase (modesetting) and allows
3094 * any flushes to be pipelined (for pageflips).
3095 *
3096 * For the display plane, we want to be in the GTT but out of any write
3097 * domains. So in many ways this looks like set_to_gtt_domain() apart from the
3098 * ability to pipeline the waits, pinning and any additional subtleties
3099 * that may differentiate the display plane from ordinary buffers.
3100 */
3101int
3102i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3103 u32 alignment,
3104 struct intel_ring_buffer *pipelined)
3105{
3106 u32 old_read_domains, old_write_domain;
3107 int ret;
3108
3109 ret = i915_gem_object_flush_gpu_write_domain(obj);
3110 if (ret)
3111 return ret;
3112
3113 if (pipelined != obj->ring) {
3114 ret = i915_gem_object_wait_rendering(obj);
3115 if (ret == -ERESTARTSYS)
3116 return ret;
3117 }
3118
3119 /* The display engine is not coherent with the LLC cache on gen6. As
3120 * a result, we make sure that the pinning that is about to occur is
3121 * done with uncached PTEs. This is lowest common denominator for all
3122 * chipsets.
3123 *
3124 * However for gen6+, we could do better by using the GFDT bit instead
3125 * of uncaching, which would allow us to flush all the LLC-cached data
3126 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
3127 */
3128 ret = i915_gem_object_set_cache_level(obj, I915_CACHE_NONE);
3129 if (ret)
3130 return ret;
3131
3132 /* As the user may map the buffer once pinned in the display plane
3133 * (e.g. libkms for the bootup splash), we have to ensure that we
3134 * always use map_and_fenceable for all scanout buffers.
3135 */
3136 ret = i915_gem_object_pin(obj, alignment, true);
3137 if (ret)
3138 return ret;
3139
3140 i915_gem_object_flush_cpu_write_domain(obj);
3141
3142 old_write_domain = obj->base.write_domain;
3143 old_read_domains = obj->base.read_domains;
3144
3145 /* It should now be out of any other write domains, and we can update
3146 * the domain values for our changes.
3147 */
3148 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3149 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3150
3151 trace_i915_gem_object_change_domain(obj,
3152 old_read_domains,
3153 old_write_domain);
3154
3155 return 0;
3156}
3157
3158int
3159i915_gem_object_finish_gpu(struct drm_i915_gem_object *obj)
3160{
3161 int ret;
3162
3163 if ((obj->base.read_domains & I915_GEM_GPU_DOMAINS) == 0)
3164 return 0;
3165
3166 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3167 ret = i915_gem_flush_ring(obj->ring, 0, obj->base.write_domain);
3168 if (ret)
3169 return ret;
3170 }
3171
3172 /* Ensure that we invalidate the GPU's caches and TLBs. */
3173 obj->base.read_domains &= ~I915_GEM_GPU_DOMAINS;
3174
3175 return i915_gem_object_wait_rendering(obj);
3176}
3177
3178/**
3179 * Moves a single object to the CPU read, and possibly write domain.
3180 *
3181 * This function returns when the move is complete, including waiting on
3182 * flushes to occur.
3183 */
3184static int
3185i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
3186{
3187 uint32_t old_write_domain, old_read_domains;
3188 int ret;
3189
3190 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
3191 return 0;
3192
3193 ret = i915_gem_object_flush_gpu_write_domain(obj);
3194 if (ret)
3195 return ret;
3196
3197 ret = i915_gem_object_wait_rendering(obj);
3198 if (ret)
3199 return ret;
3200
3201 i915_gem_object_flush_gtt_write_domain(obj);
3202
3203 /* If we have a partially-valid cache of the object in the CPU,
3204 * finish invalidating it and free the per-page flags.
3205 */
3206 i915_gem_object_set_to_full_cpu_read_domain(obj);
3207
3208 old_write_domain = obj->base.write_domain;
3209 old_read_domains = obj->base.read_domains;
3210
3211 /* Flush the CPU cache if it's still invalid. */
3212 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
3213 i915_gem_clflush_object(obj);
3214
3215 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3216 }
3217
3218 /* It should now be out of any other write domains, and we can update
3219 * the domain values for our changes.
3220 */
3221 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3222
3223 /* If we're writing through the CPU, then the GPU read domains will
3224 * need to be invalidated at next use.
3225 */
3226 if (write) {
3227 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3228 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3229 }
3230
3231 trace_i915_gem_object_change_domain(obj,
3232 old_read_domains,
3233 old_write_domain);
3234
3235 return 0;
3236}
3237
3238/**
3239 * Moves the object from a partially CPU read to a full one.
3240 *
3241 * Note that this only resolves i915_gem_object_set_cpu_read_domain_range(),
3242 * and doesn't handle transitioning from !(read_domains & I915_GEM_DOMAIN_CPU).
3243 */
3244static void
3245i915_gem_object_set_to_full_cpu_read_domain(struct drm_i915_gem_object *obj)
3246{
3247 if (!obj->page_cpu_valid)
3248 return;
3249
3250 /* If we're partially in the CPU read domain, finish moving it in.
3251 */
3252 if (obj->base.read_domains & I915_GEM_DOMAIN_CPU) {
3253 int i;
3254
3255 for (i = 0; i <= (obj->base.size - 1) / PAGE_SIZE; i++) {
3256 if (obj->page_cpu_valid[i])
3257 continue;
3258 drm_clflush_pages(obj->pages + i, 1);
3259 }
3260 }
3261
3262 /* Free the page_cpu_valid mappings which are now stale, whether
3263 * or not we've got I915_GEM_DOMAIN_CPU.
3264 */
3265 kfree(obj->page_cpu_valid);
3266 obj->page_cpu_valid = NULL;
3267}
3268
3269/**
3270 * Set the CPU read domain on a range of the object.
3271 *
3272 * The object ends up with I915_GEM_DOMAIN_CPU in its read flags although it's
3273 * not entirely valid. The page_cpu_valid member of the object flags which
3274 * pages have been flushed, and will be respected by
3275 * i915_gem_object_set_to_cpu_domain() if it's called on to get a valid mapping
3276 * of the whole object.
3277 *
3278 * This function returns when the move is complete, including waiting on
3279 * flushes to occur.
3280 */
3281static int
3282i915_gem_object_set_cpu_read_domain_range(struct drm_i915_gem_object *obj,
3283 uint64_t offset, uint64_t size)
3284{
3285 uint32_t old_read_domains;
3286 int i, ret;
3287
3288 if (offset == 0 && size == obj->base.size)
3289 return i915_gem_object_set_to_cpu_domain(obj, 0);
3290
3291 ret = i915_gem_object_flush_gpu_write_domain(obj);
3292 if (ret)
3293 return ret;
3294
3295 ret = i915_gem_object_wait_rendering(obj);
3296 if (ret)
3297 return ret;
3298
3299 i915_gem_object_flush_gtt_write_domain(obj);
3300
3301 /* If we're already fully in the CPU read domain, we're done. */
3302 if (obj->page_cpu_valid == NULL &&
3303 (obj->base.read_domains & I915_GEM_DOMAIN_CPU) != 0)
3304 return 0;
3305
3306 /* Otherwise, create/clear the per-page CPU read domain flag if we're
3307 * newly adding I915_GEM_DOMAIN_CPU
3308 */
3309 if (obj->page_cpu_valid == NULL) {
3310 obj->page_cpu_valid = kzalloc(obj->base.size / PAGE_SIZE,
3311 GFP_KERNEL);
3312 if (obj->page_cpu_valid == NULL)
3313 return -ENOMEM;
3314 } else if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
3315 memset(obj->page_cpu_valid, 0, obj->base.size / PAGE_SIZE);
3316
3317 /* Flush the cache on any pages that are still invalid from the CPU's
3318 * perspective.
3319 */
3320 for (i = offset / PAGE_SIZE; i <= (offset + size - 1) / PAGE_SIZE;
3321 i++) {
3322 if (obj->page_cpu_valid[i])
3323 continue;
3324
3325 drm_clflush_pages(obj->pages + i, 1);
3326
3327 obj->page_cpu_valid[i] = 1;
3328 }
3329
3330 /* It should now be out of any other write domains, and we can update
3331 * the domain values for our changes.
3332 */
3333 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
3334
3335 old_read_domains = obj->base.read_domains;
3336 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
3337
3338 trace_i915_gem_object_change_domain(obj,
3339 old_read_domains,
3340 obj->base.write_domain);
3341
3342 return 0;
3343}
3344
3345/* Throttle our rendering by waiting until the ring has completed our requests
3346 * emitted over 20 msec ago.
3347 *
3348 * Note that if we were to use the current jiffies each time around the loop,
3349 * we wouldn't escape the function with any frames outstanding if the time to
3350 * render a frame was over 20ms.
3351 *
3352 * This should get us reasonable parallelism between CPU and GPU but also
3353 * relatively low latency when blocking on a particular request to finish.
3354 */
3355static int
3356i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
3357{
3358 struct drm_i915_private *dev_priv = dev->dev_private;
3359 struct drm_i915_file_private *file_priv = file->driver_priv;
3360 unsigned long recent_enough = jiffies - msecs_to_jiffies(20);
3361 struct drm_i915_gem_request *request;
3362 struct intel_ring_buffer *ring = NULL;
3363 u32 seqno = 0;
3364 int ret;
3365
3366 if (atomic_read(&dev_priv->mm.wedged))
3367 return -EIO;
3368
3369 spin_lock(&file_priv->mm.lock);
3370 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
3371 if (time_after_eq(request->emitted_jiffies, recent_enough))
3372 break;
3373
3374 ring = request->ring;
3375 seqno = request->seqno;
3376 }
3377 spin_unlock(&file_priv->mm.lock);
3378
3379 if (seqno == 0)
3380 return 0;
3381
3382 ret = 0;
3383 if (!i915_seqno_passed(ring->get_seqno(ring), seqno)) {
3384 /* And wait for the seqno passing without holding any locks and
3385 * causing extra latency for others. This is safe as the irq
3386 * generation is designed to be run atomically and so is
3387 * lockless.
3388 */
3389 if (ring->irq_get(ring)) {
3390 ret = wait_event_interruptible(ring->irq_queue,
3391 i915_seqno_passed(ring->get_seqno(ring), seqno)
3392 || atomic_read(&dev_priv->mm.wedged));
3393 ring->irq_put(ring);
3394
3395 if (ret == 0 && atomic_read(&dev_priv->mm.wedged))
3396 ret = -EIO;
3397 }
3398 }
3399
3400 if (ret == 0)
3401 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
3402
3403 return ret;
3404}
3405
3406int
3407i915_gem_object_pin(struct drm_i915_gem_object *obj,
3408 uint32_t alignment,
3409 bool map_and_fenceable)
3410{
3411 struct drm_device *dev = obj->base.dev;
3412 struct drm_i915_private *dev_priv = dev->dev_private;
3413 int ret;
3414
3415 BUG_ON(obj->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT);
3416 WARN_ON(i915_verify_lists(dev));
3417
3418 if (obj->gtt_space != NULL) {
3419 if ((alignment && obj->gtt_offset & (alignment - 1)) ||
3420 (map_and_fenceable && !obj->map_and_fenceable)) {
3421 WARN(obj->pin_count,
3422 "bo is already pinned with incorrect alignment:"
3423 " offset=%x, req.alignment=%x, req.map_and_fenceable=%d,"
3424 " obj->map_and_fenceable=%d\n",
3425 obj->gtt_offset, alignment,
3426 map_and_fenceable,
3427 obj->map_and_fenceable);
3428 ret = i915_gem_object_unbind(obj);
3429 if (ret)
3430 return ret;
3431 }
3432 }
3433
3434 if (obj->gtt_space == NULL) {
3435 ret = i915_gem_object_bind_to_gtt(obj, alignment,
3436 map_and_fenceable);
3437 if (ret)
3438 return ret;
3439 }
3440
3441 if (obj->pin_count++ == 0) {
3442 if (!obj->active)
3443 list_move_tail(&obj->mm_list,
3444 &dev_priv->mm.pinned_list);
3445 }
3446 obj->pin_mappable |= map_and_fenceable;
3447
3448 WARN_ON(i915_verify_lists(dev));
3449 return 0;
3450}
3451
3452void
3453i915_gem_object_unpin(struct drm_i915_gem_object *obj)
3454{
3455 struct drm_device *dev = obj->base.dev;
3456 drm_i915_private_t *dev_priv = dev->dev_private;
3457
3458 WARN_ON(i915_verify_lists(dev));
3459 BUG_ON(obj->pin_count == 0);
3460 BUG_ON(obj->gtt_space == NULL);
3461
3462 if (--obj->pin_count == 0) {
3463 if (!obj->active)
3464 list_move_tail(&obj->mm_list,
3465 &dev_priv->mm.inactive_list);
3466 obj->pin_mappable = false;
3467 }
3468 WARN_ON(i915_verify_lists(dev));
3469}
3470
3471int
3472i915_gem_pin_ioctl(struct drm_device *dev, void *data,
3473 struct drm_file *file)
3474{
3475 struct drm_i915_gem_pin *args = data;
3476 struct drm_i915_gem_object *obj;
3477 int ret;
3478
3479 ret = i915_mutex_lock_interruptible(dev);
3480 if (ret)
3481 return ret;
3482
3483 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3484 if (&obj->base == NULL) {
3485 ret = -ENOENT;
3486 goto unlock;
3487 }
3488
3489 if (obj->madv != I915_MADV_WILLNEED) {
3490 DRM_ERROR("Attempting to pin a purgeable buffer\n");
3491 ret = -EINVAL;
3492 goto out;
3493 }
3494
3495 if (obj->pin_filp != NULL && obj->pin_filp != file) {
3496 DRM_ERROR("Already pinned in i915_gem_pin_ioctl(): %d\n",
3497 args->handle);
3498 ret = -EINVAL;
3499 goto out;
3500 }
3501
3502 obj->user_pin_count++;
3503 obj->pin_filp = file;
3504 if (obj->user_pin_count == 1) {
3505 ret = i915_gem_object_pin(obj, args->alignment, true);
3506 if (ret)
3507 goto out;
3508 }
3509
3510 /* XXX - flush the CPU caches for pinned objects
3511 * as the X server doesn't manage domains yet
3512 */
3513 i915_gem_object_flush_cpu_write_domain(obj);
3514 args->offset = obj->gtt_offset;
3515out:
3516 drm_gem_object_unreference(&obj->base);
3517unlock:
3518 mutex_unlock(&dev->struct_mutex);
3519 return ret;
3520}
3521
3522int
3523i915_gem_unpin_ioctl(struct drm_device *dev, void *data,
3524 struct drm_file *file)
3525{
3526 struct drm_i915_gem_pin *args = data;
3527 struct drm_i915_gem_object *obj;
3528 int ret;
3529
3530 ret = i915_mutex_lock_interruptible(dev);
3531 if (ret)
3532 return ret;
3533
3534 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3535 if (&obj->base == NULL) {
3536 ret = -ENOENT;
3537 goto unlock;
3538 }
3539
3540 if (obj->pin_filp != file) {
3541 DRM_ERROR("Not pinned by caller in i915_gem_pin_ioctl(): %d\n",
3542 args->handle);
3543 ret = -EINVAL;
3544 goto out;
3545 }
3546 obj->user_pin_count--;
3547 if (obj->user_pin_count == 0) {
3548 obj->pin_filp = NULL;
3549 i915_gem_object_unpin(obj);
3550 }
3551
3552out:
3553 drm_gem_object_unreference(&obj->base);
3554unlock:
3555 mutex_unlock(&dev->struct_mutex);
3556 return ret;
3557}
3558
3559int
3560i915_gem_busy_ioctl(struct drm_device *dev, void *data,
3561 struct drm_file *file)
3562{
3563 struct drm_i915_gem_busy *args = data;
3564 struct drm_i915_gem_object *obj;
3565 int ret;
3566
3567 ret = i915_mutex_lock_interruptible(dev);
3568 if (ret)
3569 return ret;
3570
3571 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3572 if (&obj->base == NULL) {
3573 ret = -ENOENT;
3574 goto unlock;
3575 }
3576
3577 /* Count all active objects as busy, even if they are currently not used
3578 * by the gpu. Users of this interface expect objects to eventually
3579 * become non-busy without any further actions, therefore emit any
3580 * necessary flushes here.
3581 */
3582 args->busy = obj->active;
3583 if (args->busy) {
3584 /* Unconditionally flush objects, even when the gpu still uses this
3585 * object. Userspace calling this function indicates that it wants to
3586 * use this buffer rather sooner than later, so issuing the required
3587 * flush earlier is beneficial.
3588 */
3589 if (obj->base.write_domain & I915_GEM_GPU_DOMAINS) {
3590 ret = i915_gem_flush_ring(obj->ring,
3591 0, obj->base.write_domain);
3592 } else if (obj->ring->outstanding_lazy_request ==
3593 obj->last_rendering_seqno) {
3594 struct drm_i915_gem_request *request;
3595
3596 /* This ring is not being cleared by active usage,
3597 * so emit a request to do so.
3598 */
3599 request = kzalloc(sizeof(*request), GFP_KERNEL);
3600 if (request)
3601 ret = i915_add_request(obj->ring, NULL,request);
3602 else
3603 ret = -ENOMEM;
3604 }
3605
3606 /* Update the active list for the hardware's current position.
3607 * Otherwise this only updates on a delayed timer or when irqs
3608 * are actually unmasked, and our working set ends up being
3609 * larger than required.
3610 */
3611 i915_gem_retire_requests_ring(obj->ring);
3612
3613 args->busy = obj->active;
3614 }
3615
3616 drm_gem_object_unreference(&obj->base);
3617unlock:
3618 mutex_unlock(&dev->struct_mutex);
3619 return ret;
3620}
3621
3622int
3623i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
3624 struct drm_file *file_priv)
3625{
3626 return i915_gem_ring_throttle(dev, file_priv);
3627}
3628
3629int
3630i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
3631 struct drm_file *file_priv)
3632{
3633 struct drm_i915_gem_madvise *args = data;
3634 struct drm_i915_gem_object *obj;
3635 int ret;
3636
3637 switch (args->madv) {
3638 case I915_MADV_DONTNEED:
3639 case I915_MADV_WILLNEED:
3640 break;
3641 default:
3642 return -EINVAL;
3643 }
3644
3645 ret = i915_mutex_lock_interruptible(dev);
3646 if (ret)
3647 return ret;
3648
3649 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
3650 if (&obj->base == NULL) {
3651 ret = -ENOENT;
3652 goto unlock;
3653 }
3654
3655 if (obj->pin_count) {
3656 ret = -EINVAL;
3657 goto out;
3658 }
3659
3660 if (obj->madv != __I915_MADV_PURGED)
3661 obj->madv = args->madv;
3662
3663 /* if the object is no longer bound, discard its backing storage */
3664 if (i915_gem_object_is_purgeable(obj) &&
3665 obj->gtt_space == NULL)
3666 i915_gem_object_truncate(obj);
3667
3668 args->retained = obj->madv != __I915_MADV_PURGED;
3669
3670out:
3671 drm_gem_object_unreference(&obj->base);
3672unlock:
3673 mutex_unlock(&dev->struct_mutex);
3674 return ret;
3675}
3676
3677struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
3678 size_t size)
3679{
3680 struct drm_i915_private *dev_priv = dev->dev_private;
3681 struct drm_i915_gem_object *obj;
3682 struct address_space *mapping;
3683
3684 obj = kzalloc(sizeof(*obj), GFP_KERNEL);
3685 if (obj == NULL)
3686 return NULL;
3687
3688 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
3689 kfree(obj);
3690 return NULL;
3691 }
3692
3693 mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
3694 mapping_set_gfp_mask(mapping, GFP_HIGHUSER | __GFP_RECLAIMABLE);
3695
3696 i915_gem_info_add_obj(dev_priv, size);
3697
3698 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
3699 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
3700
3701 if (IS_GEN6(dev)) {
3702 /* On Gen6, we can have the GPU use the LLC (the CPU
3703 * cache) for about a 10% performance improvement
3704 * compared to uncached. Graphics requests other than
3705 * display scanout are coherent with the CPU in
3706 * accessing this cache. This means in this mode we
3707 * don't need to clflush on the CPU side, and on the
3708 * GPU side we only need to flush internal caches to
3709 * get data visible to the CPU.
3710 *
3711 * However, we maintain the display planes as UC, and so
3712 * need to rebind when first used as such.
3713 */
3714 obj->cache_level = I915_CACHE_LLC;
3715 } else
3716 obj->cache_level = I915_CACHE_NONE;
3717
3718 obj->base.driver_private = NULL;
3719 obj->fence_reg = I915_FENCE_REG_NONE;
3720 INIT_LIST_HEAD(&obj->mm_list);
3721 INIT_LIST_HEAD(&obj->gtt_list);
3722 INIT_LIST_HEAD(&obj->ring_list);
3723 INIT_LIST_HEAD(&obj->exec_list);
3724 INIT_LIST_HEAD(&obj->gpu_write_list);
3725 obj->madv = I915_MADV_WILLNEED;
3726 /* Avoid an unnecessary call to unbind on the first bind. */
3727 obj->map_and_fenceable = true;
3728
3729 return obj;
3730}
3731
3732int i915_gem_init_object(struct drm_gem_object *obj)
3733{
3734 BUG();
3735
3736 return 0;
3737}
3738
3739static void i915_gem_free_object_tail(struct drm_i915_gem_object *obj)
3740{
3741 struct drm_device *dev = obj->base.dev;
3742 drm_i915_private_t *dev_priv = dev->dev_private;
3743 int ret;
3744
3745 ret = i915_gem_object_unbind(obj);
3746 if (ret == -ERESTARTSYS) {
3747 list_move(&obj->mm_list,
3748 &dev_priv->mm.deferred_free_list);
3749 return;
3750 }
3751
3752 trace_i915_gem_object_destroy(obj);
3753
3754 if (obj->base.map_list.map)
3755 i915_gem_free_mmap_offset(obj);
3756
3757 drm_gem_object_release(&obj->base);
3758 i915_gem_info_remove_obj(dev_priv, obj->base.size);
3759
3760 kfree(obj->page_cpu_valid);
3761 kfree(obj->bit_17);
3762 kfree(obj);
3763}
3764
3765void i915_gem_free_object(struct drm_gem_object *gem_obj)
3766{
3767 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
3768 struct drm_device *dev = obj->base.dev;
3769
3770 while (obj->pin_count > 0)
3771 i915_gem_object_unpin(obj);
3772
3773 if (obj->phys_obj)
3774 i915_gem_detach_phys_object(dev, obj);
3775
3776 i915_gem_free_object_tail(obj);
3777}
3778
3779int
3780i915_gem_idle(struct drm_device *dev)
3781{
3782 drm_i915_private_t *dev_priv = dev->dev_private;
3783 int ret;
3784
3785 mutex_lock(&dev->struct_mutex);
3786
3787 if (dev_priv->mm.suspended) {
3788 mutex_unlock(&dev->struct_mutex);
3789 return 0;
3790 }
3791
3792 ret = i915_gpu_idle(dev);
3793 if (ret) {
3794 mutex_unlock(&dev->struct_mutex);
3795 return ret;
3796 }
3797
3798 /* Under UMS, be paranoid and evict. */
3799 if (!drm_core_check_feature(dev, DRIVER_MODESET)) {
3800 ret = i915_gem_evict_inactive(dev, false);
3801 if (ret) {
3802 mutex_unlock(&dev->struct_mutex);
3803 return ret;
3804 }
3805 }
3806
3807 i915_gem_reset_fences(dev);
3808
3809 /* Hack! Don't let anybody do execbuf while we don't control the chip.
3810 * We need to replace this with a semaphore, or something.
3811 * And not confound mm.suspended!
3812 */
3813 dev_priv->mm.suspended = 1;
3814 del_timer_sync(&dev_priv->hangcheck_timer);
3815
3816 i915_kernel_lost_context(dev);
3817 i915_gem_cleanup_ringbuffer(dev);
3818
3819 mutex_unlock(&dev->struct_mutex);
3820
3821 /* Cancel the retire work handler, which should be idle now. */
3822 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
3823
3824 return 0;
3825}
3826
3827int
3828i915_gem_init_ringbuffer(struct drm_device *dev)
3829{
3830 drm_i915_private_t *dev_priv = dev->dev_private;
3831 int ret;
3832
3833 ret = intel_init_render_ring_buffer(dev);
3834 if (ret)
3835 return ret;
3836
3837 if (HAS_BSD(dev)) {
3838 ret = intel_init_bsd_ring_buffer(dev);
3839 if (ret)
3840 goto cleanup_render_ring;
3841 }
3842
3843 if (HAS_BLT(dev)) {
3844 ret = intel_init_blt_ring_buffer(dev);
3845 if (ret)
3846 goto cleanup_bsd_ring;
3847 }
3848
3849 dev_priv->next_seqno = 1;
3850
3851 return 0;
3852
3853cleanup_bsd_ring:
3854 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
3855cleanup_render_ring:
3856 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
3857 return ret;
3858}
3859
3860void
3861i915_gem_cleanup_ringbuffer(struct drm_device *dev)
3862{
3863 drm_i915_private_t *dev_priv = dev->dev_private;
3864 int i;
3865
3866 for (i = 0; i < I915_NUM_RINGS; i++)
3867 intel_cleanup_ring_buffer(&dev_priv->ring[i]);
3868}
3869
3870int
3871i915_gem_entervt_ioctl(struct drm_device *dev, void *data,
3872 struct drm_file *file_priv)
3873{
3874 drm_i915_private_t *dev_priv = dev->dev_private;
3875 int ret, i;
3876
3877 if (drm_core_check_feature(dev, DRIVER_MODESET))
3878 return 0;
3879
3880 if (atomic_read(&dev_priv->mm.wedged)) {
3881 DRM_ERROR("Reenabling wedged hardware, good luck\n");
3882 atomic_set(&dev_priv->mm.wedged, 0);
3883 }
3884
3885 mutex_lock(&dev->struct_mutex);
3886 dev_priv->mm.suspended = 0;
3887
3888 ret = i915_gem_init_ringbuffer(dev);
3889 if (ret != 0) {
3890 mutex_unlock(&dev->struct_mutex);
3891 return ret;
3892 }
3893
3894 BUG_ON(!list_empty(&dev_priv->mm.active_list));
3895 BUG_ON(!list_empty(&dev_priv->mm.flushing_list));
3896 BUG_ON(!list_empty(&dev_priv->mm.inactive_list));
3897 for (i = 0; i < I915_NUM_RINGS; i++) {
3898 BUG_ON(!list_empty(&dev_priv->ring[i].active_list));
3899 BUG_ON(!list_empty(&dev_priv->ring[i].request_list));
3900 }
3901 mutex_unlock(&dev->struct_mutex);
3902
3903 ret = drm_irq_install(dev);
3904 if (ret)
3905 goto cleanup_ringbuffer;
3906
3907 return 0;
3908
3909cleanup_ringbuffer:
3910 mutex_lock(&dev->struct_mutex);
3911 i915_gem_cleanup_ringbuffer(dev);
3912 dev_priv->mm.suspended = 1;
3913 mutex_unlock(&dev->struct_mutex);
3914
3915 return ret;
3916}
3917
3918int
3919i915_gem_leavevt_ioctl(struct drm_device *dev, void *data,
3920 struct drm_file *file_priv)
3921{
3922 if (drm_core_check_feature(dev, DRIVER_MODESET))
3923 return 0;
3924
3925 drm_irq_uninstall(dev);
3926 return i915_gem_idle(dev);
3927}
3928
3929void
3930i915_gem_lastclose(struct drm_device *dev)
3931{
3932 int ret;
3933
3934 if (drm_core_check_feature(dev, DRIVER_MODESET))
3935 return;
3936
3937 ret = i915_gem_idle(dev);
3938 if (ret)
3939 DRM_ERROR("failed to idle hardware: %d\n", ret);
3940}
3941
3942static void
3943init_ring_lists(struct intel_ring_buffer *ring)
3944{
3945 INIT_LIST_HEAD(&ring->active_list);
3946 INIT_LIST_HEAD(&ring->request_list);
3947 INIT_LIST_HEAD(&ring->gpu_write_list);
3948}
3949
3950void
3951i915_gem_load(struct drm_device *dev)
3952{
3953 int i;
3954 drm_i915_private_t *dev_priv = dev->dev_private;
3955
3956 INIT_LIST_HEAD(&dev_priv->mm.active_list);
3957 INIT_LIST_HEAD(&dev_priv->mm.flushing_list);
3958 INIT_LIST_HEAD(&dev_priv->mm.inactive_list);
3959 INIT_LIST_HEAD(&dev_priv->mm.pinned_list);
3960 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
3961 INIT_LIST_HEAD(&dev_priv->mm.deferred_free_list);
3962 INIT_LIST_HEAD(&dev_priv->mm.gtt_list);
3963 for (i = 0; i < I915_NUM_RINGS; i++)
3964 init_ring_lists(&dev_priv->ring[i]);
3965 for (i = 0; i < 16; i++)
3966 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
3967 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
3968 i915_gem_retire_work_handler);
3969 init_completion(&dev_priv->error_completion);
3970
3971 /* On GEN3 we really need to make sure the ARB C3 LP bit is set */
3972 if (IS_GEN3(dev)) {
3973 u32 tmp = I915_READ(MI_ARB_STATE);
3974 if (!(tmp & MI_ARB_C3_LP_WRITE_ENABLE)) {
3975 /* arb state is a masked write, so set bit + bit in mask */
3976 tmp = MI_ARB_C3_LP_WRITE_ENABLE | (MI_ARB_C3_LP_WRITE_ENABLE << MI_ARB_MASK_SHIFT);
3977 I915_WRITE(MI_ARB_STATE, tmp);
3978 }
3979 }
3980
3981 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
3982
3983 /* Old X drivers will take 0-2 for front, back, depth buffers */
3984 if (!drm_core_check_feature(dev, DRIVER_MODESET))
3985 dev_priv->fence_reg_start = 3;
3986
3987 if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
3988 dev_priv->num_fence_regs = 16;
3989 else
3990 dev_priv->num_fence_regs = 8;
3991
3992 /* Initialize fence registers to zero */
3993 for (i = 0; i < dev_priv->num_fence_regs; i++) {
3994 i915_gem_clear_fence_reg(dev, &dev_priv->fence_regs[i]);
3995 }
3996
3997 i915_gem_detect_bit_6_swizzle(dev);
3998 init_waitqueue_head(&dev_priv->pending_flip_queue);
3999
4000 dev_priv->mm.interruptible = true;
4001
4002 dev_priv->mm.inactive_shrinker.shrink = i915_gem_inactive_shrink;
4003 dev_priv->mm.inactive_shrinker.seeks = DEFAULT_SEEKS;
4004 register_shrinker(&dev_priv->mm.inactive_shrinker);
4005}
4006
4007/*
4008 * Create a physically contiguous memory object for this object
4009 * e.g. for cursor + overlay regs
4010 */
4011static int i915_gem_init_phys_object(struct drm_device *dev,
4012 int id, int size, int align)
4013{
4014 drm_i915_private_t *dev_priv = dev->dev_private;
4015 struct drm_i915_gem_phys_object *phys_obj;
4016 int ret;
4017
4018 if (dev_priv->mm.phys_objs[id - 1] || !size)
4019 return 0;
4020
4021 phys_obj = kzalloc(sizeof(struct drm_i915_gem_phys_object), GFP_KERNEL);
4022 if (!phys_obj)
4023 return -ENOMEM;
4024
4025 phys_obj->id = id;
4026
4027 phys_obj->handle = drm_pci_alloc(dev, size, align);
4028 if (!phys_obj->handle) {
4029 ret = -ENOMEM;
4030 goto kfree_obj;
4031 }
4032#ifdef CONFIG_X86
4033 set_memory_wc((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4034#endif
4035
4036 dev_priv->mm.phys_objs[id - 1] = phys_obj;
4037
4038 return 0;
4039kfree_obj:
4040 kfree(phys_obj);
4041 return ret;
4042}
4043
4044static void i915_gem_free_phys_object(struct drm_device *dev, int id)
4045{
4046 drm_i915_private_t *dev_priv = dev->dev_private;
4047 struct drm_i915_gem_phys_object *phys_obj;
4048
4049 if (!dev_priv->mm.phys_objs[id - 1])
4050 return;
4051
4052 phys_obj = dev_priv->mm.phys_objs[id - 1];
4053 if (phys_obj->cur_obj) {
4054 i915_gem_detach_phys_object(dev, phys_obj->cur_obj);
4055 }
4056
4057#ifdef CONFIG_X86
4058 set_memory_wb((unsigned long)phys_obj->handle->vaddr, phys_obj->handle->size / PAGE_SIZE);
4059#endif
4060 drm_pci_free(dev, phys_obj->handle);
4061 kfree(phys_obj);
4062 dev_priv->mm.phys_objs[id - 1] = NULL;
4063}
4064
4065void i915_gem_free_all_phys_object(struct drm_device *dev)
4066{
4067 int i;
4068
4069 for (i = I915_GEM_PHYS_CURSOR_0; i <= I915_MAX_PHYS_OBJECT; i++)
4070 i915_gem_free_phys_object(dev, i);
4071}
4072
4073void i915_gem_detach_phys_object(struct drm_device *dev,
4074 struct drm_i915_gem_object *obj)
4075{
4076 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4077 char *vaddr;
4078 int i;
4079 int page_count;
4080
4081 if (!obj->phys_obj)
4082 return;
4083 vaddr = obj->phys_obj->handle->vaddr;
4084
4085 page_count = obj->base.size / PAGE_SIZE;
4086 for (i = 0; i < page_count; i++) {
4087 struct page *page = shmem_read_mapping_page(mapping, i);
4088 if (!IS_ERR(page)) {
4089 char *dst = kmap_atomic(page);
4090 memcpy(dst, vaddr + i*PAGE_SIZE, PAGE_SIZE);
4091 kunmap_atomic(dst);
4092
4093 drm_clflush_pages(&page, 1);
4094
4095 set_page_dirty(page);
4096 mark_page_accessed(page);
4097 page_cache_release(page);
4098 }
4099 }
4100 intel_gtt_chipset_flush();
4101
4102 obj->phys_obj->cur_obj = NULL;
4103 obj->phys_obj = NULL;
4104}
4105
4106int
4107i915_gem_attach_phys_object(struct drm_device *dev,
4108 struct drm_i915_gem_object *obj,
4109 int id,
4110 int align)
4111{
4112 struct address_space *mapping = obj->base.filp->f_path.dentry->d_inode->i_mapping;
4113 drm_i915_private_t *dev_priv = dev->dev_private;
4114 int ret = 0;
4115 int page_count;
4116 int i;
4117
4118 if (id > I915_MAX_PHYS_OBJECT)
4119 return -EINVAL;
4120
4121 if (obj->phys_obj) {
4122 if (obj->phys_obj->id == id)
4123 return 0;
4124 i915_gem_detach_phys_object(dev, obj);
4125 }
4126
4127 /* create a new object */
4128 if (!dev_priv->mm.phys_objs[id - 1]) {
4129 ret = i915_gem_init_phys_object(dev, id,
4130 obj->base.size, align);
4131 if (ret) {
4132 DRM_ERROR("failed to init phys object %d size: %zu\n",
4133 id, obj->base.size);
4134 return ret;
4135 }
4136 }
4137
4138 /* bind to the object */
4139 obj->phys_obj = dev_priv->mm.phys_objs[id - 1];
4140 obj->phys_obj->cur_obj = obj;
4141
4142 page_count = obj->base.size / PAGE_SIZE;
4143
4144 for (i = 0; i < page_count; i++) {
4145 struct page *page;
4146 char *dst, *src;
4147
4148 page = shmem_read_mapping_page(mapping, i);
4149 if (IS_ERR(page))
4150 return PTR_ERR(page);
4151
4152 src = kmap_atomic(page);
4153 dst = obj->phys_obj->handle->vaddr + (i * PAGE_SIZE);
4154 memcpy(dst, src, PAGE_SIZE);
4155 kunmap_atomic(src);
4156
4157 mark_page_accessed(page);
4158 page_cache_release(page);
4159 }
4160
4161 return 0;
4162}
4163
4164static int
4165i915_gem_phys_pwrite(struct drm_device *dev,
4166 struct drm_i915_gem_object *obj,
4167 struct drm_i915_gem_pwrite *args,
4168 struct drm_file *file_priv)
4169{
4170 void *vaddr = obj->phys_obj->handle->vaddr + args->offset;
4171 char __user *user_data = (char __user *) (uintptr_t) args->data_ptr;
4172
4173 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
4174 unsigned long unwritten;
4175
4176 /* The physical object once assigned is fixed for the lifetime
4177 * of the obj, so we can safely drop the lock and continue
4178 * to access vaddr.
4179 */
4180 mutex_unlock(&dev->struct_mutex);
4181 unwritten = copy_from_user(vaddr, user_data, args->size);
4182 mutex_lock(&dev->struct_mutex);
4183 if (unwritten)
4184 return -EFAULT;
4185 }
4186
4187 intel_gtt_chipset_flush();
4188 return 0;
4189}
4190
4191void i915_gem_release(struct drm_device *dev, struct drm_file *file)
4192{
4193 struct drm_i915_file_private *file_priv = file->driver_priv;
4194
4195 /* Clean up our request list when the client is going away, so that
4196 * later retire_requests won't dereference our soon-to-be-gone
4197 * file_priv.
4198 */
4199 spin_lock(&file_priv->mm.lock);
4200 while (!list_empty(&file_priv->mm.request_list)) {
4201 struct drm_i915_gem_request *request;
4202
4203 request = list_first_entry(&file_priv->mm.request_list,
4204 struct drm_i915_gem_request,
4205 client_list);
4206 list_del(&request->client_list);
4207 request->file_priv = NULL;
4208 }
4209 spin_unlock(&file_priv->mm.lock);
4210}
4211
4212static int
4213i915_gpu_is_active(struct drm_device *dev)
4214{
4215 drm_i915_private_t *dev_priv = dev->dev_private;
4216 int lists_empty;
4217
4218 lists_empty = list_empty(&dev_priv->mm.flushing_list) &&
4219 list_empty(&dev_priv->mm.active_list);
4220
4221 return !lists_empty;
4222}
4223
4224static int
4225i915_gem_inactive_shrink(struct shrinker *shrinker, struct shrink_control *sc)
4226{
4227 struct drm_i915_private *dev_priv =
4228 container_of(shrinker,
4229 struct drm_i915_private,
4230 mm.inactive_shrinker);
4231 struct drm_device *dev = dev_priv->dev;
4232 struct drm_i915_gem_object *obj, *next;
4233 int nr_to_scan = sc->nr_to_scan;
4234 int cnt;
4235
4236 if (!mutex_trylock(&dev->struct_mutex))
4237 return 0;
4238
4239 /* "fast-path" to count number of available objects */
4240 if (nr_to_scan == 0) {
4241 cnt = 0;
4242 list_for_each_entry(obj,
4243 &dev_priv->mm.inactive_list,
4244 mm_list)
4245 cnt++;
4246 mutex_unlock(&dev->struct_mutex);
4247 return cnt / 100 * sysctl_vfs_cache_pressure;
4248 }
4249
4250rescan:
4251 /* first scan for clean buffers */
4252 i915_gem_retire_requests(dev);
4253
4254 list_for_each_entry_safe(obj, next,
4255 &dev_priv->mm.inactive_list,
4256 mm_list) {
4257 if (i915_gem_object_is_purgeable(obj)) {
4258 if (i915_gem_object_unbind(obj) == 0 &&
4259 --nr_to_scan == 0)
4260 break;
4261 }
4262 }
4263
4264 /* second pass, evict/count anything still on the inactive list */
4265 cnt = 0;
4266 list_for_each_entry_safe(obj, next,
4267 &dev_priv->mm.inactive_list,
4268 mm_list) {
4269 if (nr_to_scan &&
4270 i915_gem_object_unbind(obj) == 0)
4271 nr_to_scan--;
4272 else
4273 cnt++;
4274 }
4275
4276 if (nr_to_scan && i915_gpu_is_active(dev)) {
4277 /*
4278 * We are desperate for pages, so as a last resort, wait
4279 * for the GPU to finish and discard whatever we can.
4280 * This has a dramatic impact to reduce the number of
4281 * OOM-killer events whilst running the GPU aggressively.
4282 */
4283 if (i915_gpu_idle(dev) == 0)
4284 goto rescan;
4285 }
4286 mutex_unlock(&dev->struct_mutex);
4287 return cnt / 100 * sysctl_vfs_cache_pressure;
4288}
1/*
2 * Copyright © 2008-2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Eric Anholt <eric@anholt.net>
25 *
26 */
27
28#include <drm/drmP.h>
29#include <drm/drm_vma_manager.h>
30#include <drm/i915_drm.h>
31#include "i915_drv.h"
32#include "i915_vgpu.h"
33#include "i915_trace.h"
34#include "intel_drv.h"
35#include <linux/shmem_fs.h>
36#include <linux/slab.h>
37#include <linux/swap.h>
38#include <linux/pci.h>
39#include <linux/dma-buf.h>
40
41#define RQ_BUG_ON(expr)
42
43static void i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj);
44static void i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj);
45static void
46i915_gem_object_retire__write(struct drm_i915_gem_object *obj);
47static void
48i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring);
49
50static bool cpu_cache_is_coherent(struct drm_device *dev,
51 enum i915_cache_level level)
52{
53 return HAS_LLC(dev) || level != I915_CACHE_NONE;
54}
55
56static bool cpu_write_needs_clflush(struct drm_i915_gem_object *obj)
57{
58 if (!cpu_cache_is_coherent(obj->base.dev, obj->cache_level))
59 return true;
60
61 return obj->pin_display;
62}
63
64/* some bookkeeping */
65static void i915_gem_info_add_obj(struct drm_i915_private *dev_priv,
66 size_t size)
67{
68 spin_lock(&dev_priv->mm.object_stat_lock);
69 dev_priv->mm.object_count++;
70 dev_priv->mm.object_memory += size;
71 spin_unlock(&dev_priv->mm.object_stat_lock);
72}
73
74static void i915_gem_info_remove_obj(struct drm_i915_private *dev_priv,
75 size_t size)
76{
77 spin_lock(&dev_priv->mm.object_stat_lock);
78 dev_priv->mm.object_count--;
79 dev_priv->mm.object_memory -= size;
80 spin_unlock(&dev_priv->mm.object_stat_lock);
81}
82
83static int
84i915_gem_wait_for_error(struct i915_gpu_error *error)
85{
86 int ret;
87
88#define EXIT_COND (!i915_reset_in_progress(error) || \
89 i915_terminally_wedged(error))
90 if (EXIT_COND)
91 return 0;
92
93 /*
94 * Only wait 10 seconds for the gpu reset to complete to avoid hanging
95 * userspace. If it takes that long something really bad is going on and
96 * we should simply try to bail out and fail as gracefully as possible.
97 */
98 ret = wait_event_interruptible_timeout(error->reset_queue,
99 EXIT_COND,
100 10*HZ);
101 if (ret == 0) {
102 DRM_ERROR("Timed out waiting for the gpu reset to complete\n");
103 return -EIO;
104 } else if (ret < 0) {
105 return ret;
106 }
107#undef EXIT_COND
108
109 return 0;
110}
111
112int i915_mutex_lock_interruptible(struct drm_device *dev)
113{
114 struct drm_i915_private *dev_priv = dev->dev_private;
115 int ret;
116
117 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
118 if (ret)
119 return ret;
120
121 ret = mutex_lock_interruptible(&dev->struct_mutex);
122 if (ret)
123 return ret;
124
125 WARN_ON(i915_verify_lists(dev));
126 return 0;
127}
128
129int
130i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
131 struct drm_file *file)
132{
133 struct drm_i915_private *dev_priv = dev->dev_private;
134 struct drm_i915_gem_get_aperture *args = data;
135 struct i915_gtt *ggtt = &dev_priv->gtt;
136 struct i915_vma *vma;
137 size_t pinned;
138
139 pinned = 0;
140 mutex_lock(&dev->struct_mutex);
141 list_for_each_entry(vma, &ggtt->base.active_list, vm_link)
142 if (vma->pin_count)
143 pinned += vma->node.size;
144 list_for_each_entry(vma, &ggtt->base.inactive_list, vm_link)
145 if (vma->pin_count)
146 pinned += vma->node.size;
147 mutex_unlock(&dev->struct_mutex);
148
149 args->aper_size = dev_priv->gtt.base.total;
150 args->aper_available_size = args->aper_size - pinned;
151
152 return 0;
153}
154
155static int
156i915_gem_object_get_pages_phys(struct drm_i915_gem_object *obj)
157{
158 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
159 char *vaddr = obj->phys_handle->vaddr;
160 struct sg_table *st;
161 struct scatterlist *sg;
162 int i;
163
164 if (WARN_ON(i915_gem_object_needs_bit17_swizzle(obj)))
165 return -EINVAL;
166
167 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
168 struct page *page;
169 char *src;
170
171 page = shmem_read_mapping_page(mapping, i);
172 if (IS_ERR(page))
173 return PTR_ERR(page);
174
175 src = kmap_atomic(page);
176 memcpy(vaddr, src, PAGE_SIZE);
177 drm_clflush_virt_range(vaddr, PAGE_SIZE);
178 kunmap_atomic(src);
179
180 put_page(page);
181 vaddr += PAGE_SIZE;
182 }
183
184 i915_gem_chipset_flush(obj->base.dev);
185
186 st = kmalloc(sizeof(*st), GFP_KERNEL);
187 if (st == NULL)
188 return -ENOMEM;
189
190 if (sg_alloc_table(st, 1, GFP_KERNEL)) {
191 kfree(st);
192 return -ENOMEM;
193 }
194
195 sg = st->sgl;
196 sg->offset = 0;
197 sg->length = obj->base.size;
198
199 sg_dma_address(sg) = obj->phys_handle->busaddr;
200 sg_dma_len(sg) = obj->base.size;
201
202 obj->pages = st;
203 return 0;
204}
205
206static void
207i915_gem_object_put_pages_phys(struct drm_i915_gem_object *obj)
208{
209 int ret;
210
211 BUG_ON(obj->madv == __I915_MADV_PURGED);
212
213 ret = i915_gem_object_set_to_cpu_domain(obj, true);
214 if (ret) {
215 /* In the event of a disaster, abandon all caches and
216 * hope for the best.
217 */
218 WARN_ON(ret != -EIO);
219 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
220 }
221
222 if (obj->madv == I915_MADV_DONTNEED)
223 obj->dirty = 0;
224
225 if (obj->dirty) {
226 struct address_space *mapping = file_inode(obj->base.filp)->i_mapping;
227 char *vaddr = obj->phys_handle->vaddr;
228 int i;
229
230 for (i = 0; i < obj->base.size / PAGE_SIZE; i++) {
231 struct page *page;
232 char *dst;
233
234 page = shmem_read_mapping_page(mapping, i);
235 if (IS_ERR(page))
236 continue;
237
238 dst = kmap_atomic(page);
239 drm_clflush_virt_range(vaddr, PAGE_SIZE);
240 memcpy(dst, vaddr, PAGE_SIZE);
241 kunmap_atomic(dst);
242
243 set_page_dirty(page);
244 if (obj->madv == I915_MADV_WILLNEED)
245 mark_page_accessed(page);
246 put_page(page);
247 vaddr += PAGE_SIZE;
248 }
249 obj->dirty = 0;
250 }
251
252 sg_free_table(obj->pages);
253 kfree(obj->pages);
254}
255
256static void
257i915_gem_object_release_phys(struct drm_i915_gem_object *obj)
258{
259 drm_pci_free(obj->base.dev, obj->phys_handle);
260}
261
262static const struct drm_i915_gem_object_ops i915_gem_phys_ops = {
263 .get_pages = i915_gem_object_get_pages_phys,
264 .put_pages = i915_gem_object_put_pages_phys,
265 .release = i915_gem_object_release_phys,
266};
267
268static int
269drop_pages(struct drm_i915_gem_object *obj)
270{
271 struct i915_vma *vma, *next;
272 int ret;
273
274 drm_gem_object_reference(&obj->base);
275 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link)
276 if (i915_vma_unbind(vma))
277 break;
278
279 ret = i915_gem_object_put_pages(obj);
280 drm_gem_object_unreference(&obj->base);
281
282 return ret;
283}
284
285int
286i915_gem_object_attach_phys(struct drm_i915_gem_object *obj,
287 int align)
288{
289 drm_dma_handle_t *phys;
290 int ret;
291
292 if (obj->phys_handle) {
293 if ((unsigned long)obj->phys_handle->vaddr & (align -1))
294 return -EBUSY;
295
296 return 0;
297 }
298
299 if (obj->madv != I915_MADV_WILLNEED)
300 return -EFAULT;
301
302 if (obj->base.filp == NULL)
303 return -EINVAL;
304
305 ret = drop_pages(obj);
306 if (ret)
307 return ret;
308
309 /* create a new object */
310 phys = drm_pci_alloc(obj->base.dev, obj->base.size, align);
311 if (!phys)
312 return -ENOMEM;
313
314 obj->phys_handle = phys;
315 obj->ops = &i915_gem_phys_ops;
316
317 return i915_gem_object_get_pages(obj);
318}
319
320static int
321i915_gem_phys_pwrite(struct drm_i915_gem_object *obj,
322 struct drm_i915_gem_pwrite *args,
323 struct drm_file *file_priv)
324{
325 struct drm_device *dev = obj->base.dev;
326 void *vaddr = obj->phys_handle->vaddr + args->offset;
327 char __user *user_data = to_user_ptr(args->data_ptr);
328 int ret = 0;
329
330 /* We manually control the domain here and pretend that it
331 * remains coherent i.e. in the GTT domain, like shmem_pwrite.
332 */
333 ret = i915_gem_object_wait_rendering(obj, false);
334 if (ret)
335 return ret;
336
337 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
338 if (__copy_from_user_inatomic_nocache(vaddr, user_data, args->size)) {
339 unsigned long unwritten;
340
341 /* The physical object once assigned is fixed for the lifetime
342 * of the obj, so we can safely drop the lock and continue
343 * to access vaddr.
344 */
345 mutex_unlock(&dev->struct_mutex);
346 unwritten = copy_from_user(vaddr, user_data, args->size);
347 mutex_lock(&dev->struct_mutex);
348 if (unwritten) {
349 ret = -EFAULT;
350 goto out;
351 }
352 }
353
354 drm_clflush_virt_range(vaddr, args->size);
355 i915_gem_chipset_flush(dev);
356
357out:
358 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
359 return ret;
360}
361
362void *i915_gem_object_alloc(struct drm_device *dev)
363{
364 struct drm_i915_private *dev_priv = dev->dev_private;
365 return kmem_cache_zalloc(dev_priv->objects, GFP_KERNEL);
366}
367
368void i915_gem_object_free(struct drm_i915_gem_object *obj)
369{
370 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
371 kmem_cache_free(dev_priv->objects, obj);
372}
373
374static int
375i915_gem_create(struct drm_file *file,
376 struct drm_device *dev,
377 uint64_t size,
378 uint32_t *handle_p)
379{
380 struct drm_i915_gem_object *obj;
381 int ret;
382 u32 handle;
383
384 size = roundup(size, PAGE_SIZE);
385 if (size == 0)
386 return -EINVAL;
387
388 /* Allocate the new object */
389 obj = i915_gem_alloc_object(dev, size);
390 if (obj == NULL)
391 return -ENOMEM;
392
393 ret = drm_gem_handle_create(file, &obj->base, &handle);
394 /* drop reference from allocate - handle holds it now */
395 drm_gem_object_unreference_unlocked(&obj->base);
396 if (ret)
397 return ret;
398
399 *handle_p = handle;
400 return 0;
401}
402
403int
404i915_gem_dumb_create(struct drm_file *file,
405 struct drm_device *dev,
406 struct drm_mode_create_dumb *args)
407{
408 /* have to work out size/pitch and return them */
409 args->pitch = ALIGN(args->width * DIV_ROUND_UP(args->bpp, 8), 64);
410 args->size = args->pitch * args->height;
411 return i915_gem_create(file, dev,
412 args->size, &args->handle);
413}
414
415/**
416 * Creates a new mm object and returns a handle to it.
417 */
418int
419i915_gem_create_ioctl(struct drm_device *dev, void *data,
420 struct drm_file *file)
421{
422 struct drm_i915_gem_create *args = data;
423
424 return i915_gem_create(file, dev,
425 args->size, &args->handle);
426}
427
428static inline int
429__copy_to_user_swizzled(char __user *cpu_vaddr,
430 const char *gpu_vaddr, int gpu_offset,
431 int length)
432{
433 int ret, cpu_offset = 0;
434
435 while (length > 0) {
436 int cacheline_end = ALIGN(gpu_offset + 1, 64);
437 int this_length = min(cacheline_end - gpu_offset, length);
438 int swizzled_gpu_offset = gpu_offset ^ 64;
439
440 ret = __copy_to_user(cpu_vaddr + cpu_offset,
441 gpu_vaddr + swizzled_gpu_offset,
442 this_length);
443 if (ret)
444 return ret + length;
445
446 cpu_offset += this_length;
447 gpu_offset += this_length;
448 length -= this_length;
449 }
450
451 return 0;
452}
453
454static inline int
455__copy_from_user_swizzled(char *gpu_vaddr, int gpu_offset,
456 const char __user *cpu_vaddr,
457 int length)
458{
459 int ret, cpu_offset = 0;
460
461 while (length > 0) {
462 int cacheline_end = ALIGN(gpu_offset + 1, 64);
463 int this_length = min(cacheline_end - gpu_offset, length);
464 int swizzled_gpu_offset = gpu_offset ^ 64;
465
466 ret = __copy_from_user(gpu_vaddr + swizzled_gpu_offset,
467 cpu_vaddr + cpu_offset,
468 this_length);
469 if (ret)
470 return ret + length;
471
472 cpu_offset += this_length;
473 gpu_offset += this_length;
474 length -= this_length;
475 }
476
477 return 0;
478}
479
480/*
481 * Pins the specified object's pages and synchronizes the object with
482 * GPU accesses. Sets needs_clflush to non-zero if the caller should
483 * flush the object from the CPU cache.
484 */
485int i915_gem_obj_prepare_shmem_read(struct drm_i915_gem_object *obj,
486 int *needs_clflush)
487{
488 int ret;
489
490 *needs_clflush = 0;
491
492 if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
493 return -EINVAL;
494
495 if (!(obj->base.read_domains & I915_GEM_DOMAIN_CPU)) {
496 /* If we're not in the cpu read domain, set ourself into the gtt
497 * read domain and manually flush cachelines (if required). This
498 * optimizes for the case when the gpu will dirty the data
499 * anyway again before the next pread happens. */
500 *needs_clflush = !cpu_cache_is_coherent(obj->base.dev,
501 obj->cache_level);
502 ret = i915_gem_object_wait_rendering(obj, true);
503 if (ret)
504 return ret;
505 }
506
507 ret = i915_gem_object_get_pages(obj);
508 if (ret)
509 return ret;
510
511 i915_gem_object_pin_pages(obj);
512
513 return ret;
514}
515
516/* Per-page copy function for the shmem pread fastpath.
517 * Flushes invalid cachelines before reading the target if
518 * needs_clflush is set. */
519static int
520shmem_pread_fast(struct page *page, int shmem_page_offset, int page_length,
521 char __user *user_data,
522 bool page_do_bit17_swizzling, bool needs_clflush)
523{
524 char *vaddr;
525 int ret;
526
527 if (unlikely(page_do_bit17_swizzling))
528 return -EINVAL;
529
530 vaddr = kmap_atomic(page);
531 if (needs_clflush)
532 drm_clflush_virt_range(vaddr + shmem_page_offset,
533 page_length);
534 ret = __copy_to_user_inatomic(user_data,
535 vaddr + shmem_page_offset,
536 page_length);
537 kunmap_atomic(vaddr);
538
539 return ret ? -EFAULT : 0;
540}
541
542static void
543shmem_clflush_swizzled_range(char *addr, unsigned long length,
544 bool swizzled)
545{
546 if (unlikely(swizzled)) {
547 unsigned long start = (unsigned long) addr;
548 unsigned long end = (unsigned long) addr + length;
549
550 /* For swizzling simply ensure that we always flush both
551 * channels. Lame, but simple and it works. Swizzled
552 * pwrite/pread is far from a hotpath - current userspace
553 * doesn't use it at all. */
554 start = round_down(start, 128);
555 end = round_up(end, 128);
556
557 drm_clflush_virt_range((void *)start, end - start);
558 } else {
559 drm_clflush_virt_range(addr, length);
560 }
561
562}
563
564/* Only difference to the fast-path function is that this can handle bit17
565 * and uses non-atomic copy and kmap functions. */
566static int
567shmem_pread_slow(struct page *page, int shmem_page_offset, int page_length,
568 char __user *user_data,
569 bool page_do_bit17_swizzling, bool needs_clflush)
570{
571 char *vaddr;
572 int ret;
573
574 vaddr = kmap(page);
575 if (needs_clflush)
576 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
577 page_length,
578 page_do_bit17_swizzling);
579
580 if (page_do_bit17_swizzling)
581 ret = __copy_to_user_swizzled(user_data,
582 vaddr, shmem_page_offset,
583 page_length);
584 else
585 ret = __copy_to_user(user_data,
586 vaddr + shmem_page_offset,
587 page_length);
588 kunmap(page);
589
590 return ret ? - EFAULT : 0;
591}
592
593static int
594i915_gem_shmem_pread(struct drm_device *dev,
595 struct drm_i915_gem_object *obj,
596 struct drm_i915_gem_pread *args,
597 struct drm_file *file)
598{
599 char __user *user_data;
600 ssize_t remain;
601 loff_t offset;
602 int shmem_page_offset, page_length, ret = 0;
603 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
604 int prefaulted = 0;
605 int needs_clflush = 0;
606 struct sg_page_iter sg_iter;
607
608 user_data = to_user_ptr(args->data_ptr);
609 remain = args->size;
610
611 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
612
613 ret = i915_gem_obj_prepare_shmem_read(obj, &needs_clflush);
614 if (ret)
615 return ret;
616
617 offset = args->offset;
618
619 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
620 offset >> PAGE_SHIFT) {
621 struct page *page = sg_page_iter_page(&sg_iter);
622
623 if (remain <= 0)
624 break;
625
626 /* Operation in this page
627 *
628 * shmem_page_offset = offset within page in shmem file
629 * page_length = bytes to copy for this page
630 */
631 shmem_page_offset = offset_in_page(offset);
632 page_length = remain;
633 if ((shmem_page_offset + page_length) > PAGE_SIZE)
634 page_length = PAGE_SIZE - shmem_page_offset;
635
636 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
637 (page_to_phys(page) & (1 << 17)) != 0;
638
639 ret = shmem_pread_fast(page, shmem_page_offset, page_length,
640 user_data, page_do_bit17_swizzling,
641 needs_clflush);
642 if (ret == 0)
643 goto next_page;
644
645 mutex_unlock(&dev->struct_mutex);
646
647 if (likely(!i915.prefault_disable) && !prefaulted) {
648 ret = fault_in_multipages_writeable(user_data, remain);
649 /* Userspace is tricking us, but we've already clobbered
650 * its pages with the prefault and promised to write the
651 * data up to the first fault. Hence ignore any errors
652 * and just continue. */
653 (void)ret;
654 prefaulted = 1;
655 }
656
657 ret = shmem_pread_slow(page, shmem_page_offset, page_length,
658 user_data, page_do_bit17_swizzling,
659 needs_clflush);
660
661 mutex_lock(&dev->struct_mutex);
662
663 if (ret)
664 goto out;
665
666next_page:
667 remain -= page_length;
668 user_data += page_length;
669 offset += page_length;
670 }
671
672out:
673 i915_gem_object_unpin_pages(obj);
674
675 return ret;
676}
677
678/**
679 * Reads data from the object referenced by handle.
680 *
681 * On error, the contents of *data are undefined.
682 */
683int
684i915_gem_pread_ioctl(struct drm_device *dev, void *data,
685 struct drm_file *file)
686{
687 struct drm_i915_gem_pread *args = data;
688 struct drm_i915_gem_object *obj;
689 int ret = 0;
690
691 if (args->size == 0)
692 return 0;
693
694 if (!access_ok(VERIFY_WRITE,
695 to_user_ptr(args->data_ptr),
696 args->size))
697 return -EFAULT;
698
699 ret = i915_mutex_lock_interruptible(dev);
700 if (ret)
701 return ret;
702
703 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
704 if (&obj->base == NULL) {
705 ret = -ENOENT;
706 goto unlock;
707 }
708
709 /* Bounds check source. */
710 if (args->offset > obj->base.size ||
711 args->size > obj->base.size - args->offset) {
712 ret = -EINVAL;
713 goto out;
714 }
715
716 /* prime objects have no backing filp to GEM pread/pwrite
717 * pages from.
718 */
719 if (!obj->base.filp) {
720 ret = -EINVAL;
721 goto out;
722 }
723
724 trace_i915_gem_object_pread(obj, args->offset, args->size);
725
726 ret = i915_gem_shmem_pread(dev, obj, args, file);
727
728out:
729 drm_gem_object_unreference(&obj->base);
730unlock:
731 mutex_unlock(&dev->struct_mutex);
732 return ret;
733}
734
735/* This is the fast write path which cannot handle
736 * page faults in the source data
737 */
738
739static inline int
740fast_user_write(struct io_mapping *mapping,
741 loff_t page_base, int page_offset,
742 char __user *user_data,
743 int length)
744{
745 void __iomem *vaddr_atomic;
746 void *vaddr;
747 unsigned long unwritten;
748
749 vaddr_atomic = io_mapping_map_atomic_wc(mapping, page_base);
750 /* We can use the cpu mem copy function because this is X86. */
751 vaddr = (void __force*)vaddr_atomic + page_offset;
752 unwritten = __copy_from_user_inatomic_nocache(vaddr,
753 user_data, length);
754 io_mapping_unmap_atomic(vaddr_atomic);
755 return unwritten;
756}
757
758/**
759 * This is the fast pwrite path, where we copy the data directly from the
760 * user into the GTT, uncached.
761 */
762static int
763i915_gem_gtt_pwrite_fast(struct drm_device *dev,
764 struct drm_i915_gem_object *obj,
765 struct drm_i915_gem_pwrite *args,
766 struct drm_file *file)
767{
768 struct drm_i915_private *dev_priv = dev->dev_private;
769 ssize_t remain;
770 loff_t offset, page_base;
771 char __user *user_data;
772 int page_offset, page_length, ret;
773
774 ret = i915_gem_obj_ggtt_pin(obj, 0, PIN_MAPPABLE | PIN_NONBLOCK);
775 if (ret)
776 goto out;
777
778 ret = i915_gem_object_set_to_gtt_domain(obj, true);
779 if (ret)
780 goto out_unpin;
781
782 ret = i915_gem_object_put_fence(obj);
783 if (ret)
784 goto out_unpin;
785
786 user_data = to_user_ptr(args->data_ptr);
787 remain = args->size;
788
789 offset = i915_gem_obj_ggtt_offset(obj) + args->offset;
790
791 intel_fb_obj_invalidate(obj, ORIGIN_GTT);
792
793 while (remain > 0) {
794 /* Operation in this page
795 *
796 * page_base = page offset within aperture
797 * page_offset = offset within page
798 * page_length = bytes to copy for this page
799 */
800 page_base = offset & PAGE_MASK;
801 page_offset = offset_in_page(offset);
802 page_length = remain;
803 if ((page_offset + remain) > PAGE_SIZE)
804 page_length = PAGE_SIZE - page_offset;
805
806 /* If we get a fault while copying data, then (presumably) our
807 * source page isn't available. Return the error and we'll
808 * retry in the slow path.
809 */
810 if (fast_user_write(dev_priv->gtt.mappable, page_base,
811 page_offset, user_data, page_length)) {
812 ret = -EFAULT;
813 goto out_flush;
814 }
815
816 remain -= page_length;
817 user_data += page_length;
818 offset += page_length;
819 }
820
821out_flush:
822 intel_fb_obj_flush(obj, false, ORIGIN_GTT);
823out_unpin:
824 i915_gem_object_ggtt_unpin(obj);
825out:
826 return ret;
827}
828
829/* Per-page copy function for the shmem pwrite fastpath.
830 * Flushes invalid cachelines before writing to the target if
831 * needs_clflush_before is set and flushes out any written cachelines after
832 * writing if needs_clflush is set. */
833static int
834shmem_pwrite_fast(struct page *page, int shmem_page_offset, int page_length,
835 char __user *user_data,
836 bool page_do_bit17_swizzling,
837 bool needs_clflush_before,
838 bool needs_clflush_after)
839{
840 char *vaddr;
841 int ret;
842
843 if (unlikely(page_do_bit17_swizzling))
844 return -EINVAL;
845
846 vaddr = kmap_atomic(page);
847 if (needs_clflush_before)
848 drm_clflush_virt_range(vaddr + shmem_page_offset,
849 page_length);
850 ret = __copy_from_user_inatomic(vaddr + shmem_page_offset,
851 user_data, page_length);
852 if (needs_clflush_after)
853 drm_clflush_virt_range(vaddr + shmem_page_offset,
854 page_length);
855 kunmap_atomic(vaddr);
856
857 return ret ? -EFAULT : 0;
858}
859
860/* Only difference to the fast-path function is that this can handle bit17
861 * and uses non-atomic copy and kmap functions. */
862static int
863shmem_pwrite_slow(struct page *page, int shmem_page_offset, int page_length,
864 char __user *user_data,
865 bool page_do_bit17_swizzling,
866 bool needs_clflush_before,
867 bool needs_clflush_after)
868{
869 char *vaddr;
870 int ret;
871
872 vaddr = kmap(page);
873 if (unlikely(needs_clflush_before || page_do_bit17_swizzling))
874 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
875 page_length,
876 page_do_bit17_swizzling);
877 if (page_do_bit17_swizzling)
878 ret = __copy_from_user_swizzled(vaddr, shmem_page_offset,
879 user_data,
880 page_length);
881 else
882 ret = __copy_from_user(vaddr + shmem_page_offset,
883 user_data,
884 page_length);
885 if (needs_clflush_after)
886 shmem_clflush_swizzled_range(vaddr + shmem_page_offset,
887 page_length,
888 page_do_bit17_swizzling);
889 kunmap(page);
890
891 return ret ? -EFAULT : 0;
892}
893
894static int
895i915_gem_shmem_pwrite(struct drm_device *dev,
896 struct drm_i915_gem_object *obj,
897 struct drm_i915_gem_pwrite *args,
898 struct drm_file *file)
899{
900 ssize_t remain;
901 loff_t offset;
902 char __user *user_data;
903 int shmem_page_offset, page_length, ret = 0;
904 int obj_do_bit17_swizzling, page_do_bit17_swizzling;
905 int hit_slowpath = 0;
906 int needs_clflush_after = 0;
907 int needs_clflush_before = 0;
908 struct sg_page_iter sg_iter;
909
910 user_data = to_user_ptr(args->data_ptr);
911 remain = args->size;
912
913 obj_do_bit17_swizzling = i915_gem_object_needs_bit17_swizzle(obj);
914
915 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
916 /* If we're not in the cpu write domain, set ourself into the gtt
917 * write domain and manually flush cachelines (if required). This
918 * optimizes for the case when the gpu will use the data
919 * right away and we therefore have to clflush anyway. */
920 needs_clflush_after = cpu_write_needs_clflush(obj);
921 ret = i915_gem_object_wait_rendering(obj, false);
922 if (ret)
923 return ret;
924 }
925 /* Same trick applies to invalidate partially written cachelines read
926 * before writing. */
927 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0)
928 needs_clflush_before =
929 !cpu_cache_is_coherent(dev, obj->cache_level);
930
931 ret = i915_gem_object_get_pages(obj);
932 if (ret)
933 return ret;
934
935 intel_fb_obj_invalidate(obj, ORIGIN_CPU);
936
937 i915_gem_object_pin_pages(obj);
938
939 offset = args->offset;
940 obj->dirty = 1;
941
942 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents,
943 offset >> PAGE_SHIFT) {
944 struct page *page = sg_page_iter_page(&sg_iter);
945 int partial_cacheline_write;
946
947 if (remain <= 0)
948 break;
949
950 /* Operation in this page
951 *
952 * shmem_page_offset = offset within page in shmem file
953 * page_length = bytes to copy for this page
954 */
955 shmem_page_offset = offset_in_page(offset);
956
957 page_length = remain;
958 if ((shmem_page_offset + page_length) > PAGE_SIZE)
959 page_length = PAGE_SIZE - shmem_page_offset;
960
961 /* If we don't overwrite a cacheline completely we need to be
962 * careful to have up-to-date data by first clflushing. Don't
963 * overcomplicate things and flush the entire patch. */
964 partial_cacheline_write = needs_clflush_before &&
965 ((shmem_page_offset | page_length)
966 & (boot_cpu_data.x86_clflush_size - 1));
967
968 page_do_bit17_swizzling = obj_do_bit17_swizzling &&
969 (page_to_phys(page) & (1 << 17)) != 0;
970
971 ret = shmem_pwrite_fast(page, shmem_page_offset, page_length,
972 user_data, page_do_bit17_swizzling,
973 partial_cacheline_write,
974 needs_clflush_after);
975 if (ret == 0)
976 goto next_page;
977
978 hit_slowpath = 1;
979 mutex_unlock(&dev->struct_mutex);
980 ret = shmem_pwrite_slow(page, shmem_page_offset, page_length,
981 user_data, page_do_bit17_swizzling,
982 partial_cacheline_write,
983 needs_clflush_after);
984
985 mutex_lock(&dev->struct_mutex);
986
987 if (ret)
988 goto out;
989
990next_page:
991 remain -= page_length;
992 user_data += page_length;
993 offset += page_length;
994 }
995
996out:
997 i915_gem_object_unpin_pages(obj);
998
999 if (hit_slowpath) {
1000 /*
1001 * Fixup: Flush cpu caches in case we didn't flush the dirty
1002 * cachelines in-line while writing and the object moved
1003 * out of the cpu write domain while we've dropped the lock.
1004 */
1005 if (!needs_clflush_after &&
1006 obj->base.write_domain != I915_GEM_DOMAIN_CPU) {
1007 if (i915_gem_clflush_object(obj, obj->pin_display))
1008 needs_clflush_after = true;
1009 }
1010 }
1011
1012 if (needs_clflush_after)
1013 i915_gem_chipset_flush(dev);
1014 else
1015 obj->cache_dirty = true;
1016
1017 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
1018 return ret;
1019}
1020
1021/**
1022 * Writes data to the object referenced by handle.
1023 *
1024 * On error, the contents of the buffer that were to be modified are undefined.
1025 */
1026int
1027i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
1028 struct drm_file *file)
1029{
1030 struct drm_i915_private *dev_priv = dev->dev_private;
1031 struct drm_i915_gem_pwrite *args = data;
1032 struct drm_i915_gem_object *obj;
1033 int ret;
1034
1035 if (args->size == 0)
1036 return 0;
1037
1038 if (!access_ok(VERIFY_READ,
1039 to_user_ptr(args->data_ptr),
1040 args->size))
1041 return -EFAULT;
1042
1043 if (likely(!i915.prefault_disable)) {
1044 ret = fault_in_multipages_readable(to_user_ptr(args->data_ptr),
1045 args->size);
1046 if (ret)
1047 return -EFAULT;
1048 }
1049
1050 intel_runtime_pm_get(dev_priv);
1051
1052 ret = i915_mutex_lock_interruptible(dev);
1053 if (ret)
1054 goto put_rpm;
1055
1056 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1057 if (&obj->base == NULL) {
1058 ret = -ENOENT;
1059 goto unlock;
1060 }
1061
1062 /* Bounds check destination. */
1063 if (args->offset > obj->base.size ||
1064 args->size > obj->base.size - args->offset) {
1065 ret = -EINVAL;
1066 goto out;
1067 }
1068
1069 /* prime objects have no backing filp to GEM pread/pwrite
1070 * pages from.
1071 */
1072 if (!obj->base.filp) {
1073 ret = -EINVAL;
1074 goto out;
1075 }
1076
1077 trace_i915_gem_object_pwrite(obj, args->offset, args->size);
1078
1079 ret = -EFAULT;
1080 /* We can only do the GTT pwrite on untiled buffers, as otherwise
1081 * it would end up going through the fenced access, and we'll get
1082 * different detiling behavior between reading and writing.
1083 * pread/pwrite currently are reading and writing from the CPU
1084 * perspective, requiring manual detiling by the client.
1085 */
1086 if (obj->tiling_mode == I915_TILING_NONE &&
1087 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
1088 cpu_write_needs_clflush(obj)) {
1089 ret = i915_gem_gtt_pwrite_fast(dev, obj, args, file);
1090 /* Note that the gtt paths might fail with non-page-backed user
1091 * pointers (e.g. gtt mappings when moving data between
1092 * textures). Fallback to the shmem path in that case. */
1093 }
1094
1095 if (ret == -EFAULT || ret == -ENOSPC) {
1096 if (obj->phys_handle)
1097 ret = i915_gem_phys_pwrite(obj, args, file);
1098 else
1099 ret = i915_gem_shmem_pwrite(dev, obj, args, file);
1100 }
1101
1102out:
1103 drm_gem_object_unreference(&obj->base);
1104unlock:
1105 mutex_unlock(&dev->struct_mutex);
1106put_rpm:
1107 intel_runtime_pm_put(dev_priv);
1108
1109 return ret;
1110}
1111
1112int
1113i915_gem_check_wedge(struct i915_gpu_error *error,
1114 bool interruptible)
1115{
1116 if (i915_reset_in_progress(error)) {
1117 /* Non-interruptible callers can't handle -EAGAIN, hence return
1118 * -EIO unconditionally for these. */
1119 if (!interruptible)
1120 return -EIO;
1121
1122 /* Recovery complete, but the reset failed ... */
1123 if (i915_terminally_wedged(error))
1124 return -EIO;
1125
1126 /*
1127 * Check if GPU Reset is in progress - we need intel_ring_begin
1128 * to work properly to reinit the hw state while the gpu is
1129 * still marked as reset-in-progress. Handle this with a flag.
1130 */
1131 if (!error->reload_in_reset)
1132 return -EAGAIN;
1133 }
1134
1135 return 0;
1136}
1137
1138static void fake_irq(unsigned long data)
1139{
1140 wake_up_process((struct task_struct *)data);
1141}
1142
1143static bool missed_irq(struct drm_i915_private *dev_priv,
1144 struct intel_engine_cs *ring)
1145{
1146 return test_bit(ring->id, &dev_priv->gpu_error.missed_irq_rings);
1147}
1148
1149static unsigned long local_clock_us(unsigned *cpu)
1150{
1151 unsigned long t;
1152
1153 /* Cheaply and approximately convert from nanoseconds to microseconds.
1154 * The result and subsequent calculations are also defined in the same
1155 * approximate microseconds units. The principal source of timing
1156 * error here is from the simple truncation.
1157 *
1158 * Note that local_clock() is only defined wrt to the current CPU;
1159 * the comparisons are no longer valid if we switch CPUs. Instead of
1160 * blocking preemption for the entire busywait, we can detect the CPU
1161 * switch and use that as indicator of system load and a reason to
1162 * stop busywaiting, see busywait_stop().
1163 */
1164 *cpu = get_cpu();
1165 t = local_clock() >> 10;
1166 put_cpu();
1167
1168 return t;
1169}
1170
1171static bool busywait_stop(unsigned long timeout, unsigned cpu)
1172{
1173 unsigned this_cpu;
1174
1175 if (time_after(local_clock_us(&this_cpu), timeout))
1176 return true;
1177
1178 return this_cpu != cpu;
1179}
1180
1181static int __i915_spin_request(struct drm_i915_gem_request *req, int state)
1182{
1183 unsigned long timeout;
1184 unsigned cpu;
1185
1186 /* When waiting for high frequency requests, e.g. during synchronous
1187 * rendering split between the CPU and GPU, the finite amount of time
1188 * required to set up the irq and wait upon it limits the response
1189 * rate. By busywaiting on the request completion for a short while we
1190 * can service the high frequency waits as quick as possible. However,
1191 * if it is a slow request, we want to sleep as quickly as possible.
1192 * The tradeoff between waiting and sleeping is roughly the time it
1193 * takes to sleep on a request, on the order of a microsecond.
1194 */
1195
1196 if (req->ring->irq_refcount)
1197 return -EBUSY;
1198
1199 /* Only spin if we know the GPU is processing this request */
1200 if (!i915_gem_request_started(req, true))
1201 return -EAGAIN;
1202
1203 timeout = local_clock_us(&cpu) + 5;
1204 while (!need_resched()) {
1205 if (i915_gem_request_completed(req, true))
1206 return 0;
1207
1208 if (signal_pending_state(state, current))
1209 break;
1210
1211 if (busywait_stop(timeout, cpu))
1212 break;
1213
1214 cpu_relax_lowlatency();
1215 }
1216
1217 if (i915_gem_request_completed(req, false))
1218 return 0;
1219
1220 return -EAGAIN;
1221}
1222
1223/**
1224 * __i915_wait_request - wait until execution of request has finished
1225 * @req: duh!
1226 * @reset_counter: reset sequence associated with the given request
1227 * @interruptible: do an interruptible wait (normally yes)
1228 * @timeout: in - how long to wait (NULL forever); out - how much time remaining
1229 *
1230 * Note: It is of utmost importance that the passed in seqno and reset_counter
1231 * values have been read by the caller in an smp safe manner. Where read-side
1232 * locks are involved, it is sufficient to read the reset_counter before
1233 * unlocking the lock that protects the seqno. For lockless tricks, the
1234 * reset_counter _must_ be read before, and an appropriate smp_rmb must be
1235 * inserted.
1236 *
1237 * Returns 0 if the request was found within the alloted time. Else returns the
1238 * errno with remaining time filled in timeout argument.
1239 */
1240int __i915_wait_request(struct drm_i915_gem_request *req,
1241 unsigned reset_counter,
1242 bool interruptible,
1243 s64 *timeout,
1244 struct intel_rps_client *rps)
1245{
1246 struct intel_engine_cs *ring = i915_gem_request_get_ring(req);
1247 struct drm_device *dev = ring->dev;
1248 struct drm_i915_private *dev_priv = dev->dev_private;
1249 const bool irq_test_in_progress =
1250 ACCESS_ONCE(dev_priv->gpu_error.test_irq_rings) & intel_ring_flag(ring);
1251 int state = interruptible ? TASK_INTERRUPTIBLE : TASK_UNINTERRUPTIBLE;
1252 DEFINE_WAIT(wait);
1253 unsigned long timeout_expire;
1254 s64 before = 0; /* Only to silence a compiler warning. */
1255 int ret;
1256
1257 WARN(!intel_irqs_enabled(dev_priv), "IRQs disabled");
1258
1259 if (list_empty(&req->list))
1260 return 0;
1261
1262 if (i915_gem_request_completed(req, true))
1263 return 0;
1264
1265 timeout_expire = 0;
1266 if (timeout) {
1267 if (WARN_ON(*timeout < 0))
1268 return -EINVAL;
1269
1270 if (*timeout == 0)
1271 return -ETIME;
1272
1273 timeout_expire = jiffies + nsecs_to_jiffies_timeout(*timeout);
1274
1275 /*
1276 * Record current time in case interrupted by signal, or wedged.
1277 */
1278 before = ktime_get_raw_ns();
1279 }
1280
1281 if (INTEL_INFO(dev_priv)->gen >= 6)
1282 gen6_rps_boost(dev_priv, rps, req->emitted_jiffies);
1283
1284 trace_i915_gem_request_wait_begin(req);
1285
1286 /* Optimistic spin for the next jiffie before touching IRQs */
1287 ret = __i915_spin_request(req, state);
1288 if (ret == 0)
1289 goto out;
1290
1291 if (!irq_test_in_progress && WARN_ON(!ring->irq_get(ring))) {
1292 ret = -ENODEV;
1293 goto out;
1294 }
1295
1296 for (;;) {
1297 struct timer_list timer;
1298
1299 prepare_to_wait(&ring->irq_queue, &wait, state);
1300
1301 /* We need to check whether any gpu reset happened in between
1302 * the caller grabbing the seqno and now ... */
1303 if (reset_counter != atomic_read(&dev_priv->gpu_error.reset_counter)) {
1304 /* ... but upgrade the -EAGAIN to an -EIO if the gpu
1305 * is truely gone. */
1306 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1307 if (ret == 0)
1308 ret = -EAGAIN;
1309 break;
1310 }
1311
1312 if (i915_gem_request_completed(req, false)) {
1313 ret = 0;
1314 break;
1315 }
1316
1317 if (signal_pending_state(state, current)) {
1318 ret = -ERESTARTSYS;
1319 break;
1320 }
1321
1322 if (timeout && time_after_eq(jiffies, timeout_expire)) {
1323 ret = -ETIME;
1324 break;
1325 }
1326
1327 timer.function = NULL;
1328 if (timeout || missed_irq(dev_priv, ring)) {
1329 unsigned long expire;
1330
1331 setup_timer_on_stack(&timer, fake_irq, (unsigned long)current);
1332 expire = missed_irq(dev_priv, ring) ? jiffies + 1 : timeout_expire;
1333 mod_timer(&timer, expire);
1334 }
1335
1336 io_schedule();
1337
1338 if (timer.function) {
1339 del_singleshot_timer_sync(&timer);
1340 destroy_timer_on_stack(&timer);
1341 }
1342 }
1343 if (!irq_test_in_progress)
1344 ring->irq_put(ring);
1345
1346 finish_wait(&ring->irq_queue, &wait);
1347
1348out:
1349 trace_i915_gem_request_wait_end(req);
1350
1351 if (timeout) {
1352 s64 tres = *timeout - (ktime_get_raw_ns() - before);
1353
1354 *timeout = tres < 0 ? 0 : tres;
1355
1356 /*
1357 * Apparently ktime isn't accurate enough and occasionally has a
1358 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
1359 * things up to make the test happy. We allow up to 1 jiffy.
1360 *
1361 * This is a regrssion from the timespec->ktime conversion.
1362 */
1363 if (ret == -ETIME && *timeout < jiffies_to_usecs(1)*1000)
1364 *timeout = 0;
1365 }
1366
1367 return ret;
1368}
1369
1370int i915_gem_request_add_to_client(struct drm_i915_gem_request *req,
1371 struct drm_file *file)
1372{
1373 struct drm_i915_private *dev_private;
1374 struct drm_i915_file_private *file_priv;
1375
1376 WARN_ON(!req || !file || req->file_priv);
1377
1378 if (!req || !file)
1379 return -EINVAL;
1380
1381 if (req->file_priv)
1382 return -EINVAL;
1383
1384 dev_private = req->ring->dev->dev_private;
1385 file_priv = file->driver_priv;
1386
1387 spin_lock(&file_priv->mm.lock);
1388 req->file_priv = file_priv;
1389 list_add_tail(&req->client_list, &file_priv->mm.request_list);
1390 spin_unlock(&file_priv->mm.lock);
1391
1392 req->pid = get_pid(task_pid(current));
1393
1394 return 0;
1395}
1396
1397static inline void
1398i915_gem_request_remove_from_client(struct drm_i915_gem_request *request)
1399{
1400 struct drm_i915_file_private *file_priv = request->file_priv;
1401
1402 if (!file_priv)
1403 return;
1404
1405 spin_lock(&file_priv->mm.lock);
1406 list_del(&request->client_list);
1407 request->file_priv = NULL;
1408 spin_unlock(&file_priv->mm.lock);
1409
1410 put_pid(request->pid);
1411 request->pid = NULL;
1412}
1413
1414static void i915_gem_request_retire(struct drm_i915_gem_request *request)
1415{
1416 trace_i915_gem_request_retire(request);
1417
1418 /* We know the GPU must have read the request to have
1419 * sent us the seqno + interrupt, so use the position
1420 * of tail of the request to update the last known position
1421 * of the GPU head.
1422 *
1423 * Note this requires that we are always called in request
1424 * completion order.
1425 */
1426 request->ringbuf->last_retired_head = request->postfix;
1427
1428 list_del_init(&request->list);
1429 i915_gem_request_remove_from_client(request);
1430
1431 i915_gem_request_unreference(request);
1432}
1433
1434static void
1435__i915_gem_request_retire__upto(struct drm_i915_gem_request *req)
1436{
1437 struct intel_engine_cs *engine = req->ring;
1438 struct drm_i915_gem_request *tmp;
1439
1440 lockdep_assert_held(&engine->dev->struct_mutex);
1441
1442 if (list_empty(&req->list))
1443 return;
1444
1445 do {
1446 tmp = list_first_entry(&engine->request_list,
1447 typeof(*tmp), list);
1448
1449 i915_gem_request_retire(tmp);
1450 } while (tmp != req);
1451
1452 WARN_ON(i915_verify_lists(engine->dev));
1453}
1454
1455/**
1456 * Waits for a request to be signaled, and cleans up the
1457 * request and object lists appropriately for that event.
1458 */
1459int
1460i915_wait_request(struct drm_i915_gem_request *req)
1461{
1462 struct drm_device *dev;
1463 struct drm_i915_private *dev_priv;
1464 bool interruptible;
1465 int ret;
1466
1467 BUG_ON(req == NULL);
1468
1469 dev = req->ring->dev;
1470 dev_priv = dev->dev_private;
1471 interruptible = dev_priv->mm.interruptible;
1472
1473 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1474
1475 ret = i915_gem_check_wedge(&dev_priv->gpu_error, interruptible);
1476 if (ret)
1477 return ret;
1478
1479 ret = __i915_wait_request(req,
1480 atomic_read(&dev_priv->gpu_error.reset_counter),
1481 interruptible, NULL, NULL);
1482 if (ret)
1483 return ret;
1484
1485 __i915_gem_request_retire__upto(req);
1486 return 0;
1487}
1488
1489/**
1490 * Ensures that all rendering to the object has completed and the object is
1491 * safe to unbind from the GTT or access from the CPU.
1492 */
1493int
1494i915_gem_object_wait_rendering(struct drm_i915_gem_object *obj,
1495 bool readonly)
1496{
1497 int ret, i;
1498
1499 if (!obj->active)
1500 return 0;
1501
1502 if (readonly) {
1503 if (obj->last_write_req != NULL) {
1504 ret = i915_wait_request(obj->last_write_req);
1505 if (ret)
1506 return ret;
1507
1508 i = obj->last_write_req->ring->id;
1509 if (obj->last_read_req[i] == obj->last_write_req)
1510 i915_gem_object_retire__read(obj, i);
1511 else
1512 i915_gem_object_retire__write(obj);
1513 }
1514 } else {
1515 for (i = 0; i < I915_NUM_RINGS; i++) {
1516 if (obj->last_read_req[i] == NULL)
1517 continue;
1518
1519 ret = i915_wait_request(obj->last_read_req[i]);
1520 if (ret)
1521 return ret;
1522
1523 i915_gem_object_retire__read(obj, i);
1524 }
1525 RQ_BUG_ON(obj->active);
1526 }
1527
1528 return 0;
1529}
1530
1531static void
1532i915_gem_object_retire_request(struct drm_i915_gem_object *obj,
1533 struct drm_i915_gem_request *req)
1534{
1535 int ring = req->ring->id;
1536
1537 if (obj->last_read_req[ring] == req)
1538 i915_gem_object_retire__read(obj, ring);
1539 else if (obj->last_write_req == req)
1540 i915_gem_object_retire__write(obj);
1541
1542 __i915_gem_request_retire__upto(req);
1543}
1544
1545/* A nonblocking variant of the above wait. This is a highly dangerous routine
1546 * as the object state may change during this call.
1547 */
1548static __must_check int
1549i915_gem_object_wait_rendering__nonblocking(struct drm_i915_gem_object *obj,
1550 struct intel_rps_client *rps,
1551 bool readonly)
1552{
1553 struct drm_device *dev = obj->base.dev;
1554 struct drm_i915_private *dev_priv = dev->dev_private;
1555 struct drm_i915_gem_request *requests[I915_NUM_RINGS];
1556 unsigned reset_counter;
1557 int ret, i, n = 0;
1558
1559 BUG_ON(!mutex_is_locked(&dev->struct_mutex));
1560 BUG_ON(!dev_priv->mm.interruptible);
1561
1562 if (!obj->active)
1563 return 0;
1564
1565 ret = i915_gem_check_wedge(&dev_priv->gpu_error, true);
1566 if (ret)
1567 return ret;
1568
1569 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
1570
1571 if (readonly) {
1572 struct drm_i915_gem_request *req;
1573
1574 req = obj->last_write_req;
1575 if (req == NULL)
1576 return 0;
1577
1578 requests[n++] = i915_gem_request_reference(req);
1579 } else {
1580 for (i = 0; i < I915_NUM_RINGS; i++) {
1581 struct drm_i915_gem_request *req;
1582
1583 req = obj->last_read_req[i];
1584 if (req == NULL)
1585 continue;
1586
1587 requests[n++] = i915_gem_request_reference(req);
1588 }
1589 }
1590
1591 mutex_unlock(&dev->struct_mutex);
1592 for (i = 0; ret == 0 && i < n; i++)
1593 ret = __i915_wait_request(requests[i], reset_counter, true,
1594 NULL, rps);
1595 mutex_lock(&dev->struct_mutex);
1596
1597 for (i = 0; i < n; i++) {
1598 if (ret == 0)
1599 i915_gem_object_retire_request(obj, requests[i]);
1600 i915_gem_request_unreference(requests[i]);
1601 }
1602
1603 return ret;
1604}
1605
1606static struct intel_rps_client *to_rps_client(struct drm_file *file)
1607{
1608 struct drm_i915_file_private *fpriv = file->driver_priv;
1609 return &fpriv->rps;
1610}
1611
1612/**
1613 * Called when user space prepares to use an object with the CPU, either
1614 * through the mmap ioctl's mapping or a GTT mapping.
1615 */
1616int
1617i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
1618 struct drm_file *file)
1619{
1620 struct drm_i915_gem_set_domain *args = data;
1621 struct drm_i915_gem_object *obj;
1622 uint32_t read_domains = args->read_domains;
1623 uint32_t write_domain = args->write_domain;
1624 int ret;
1625
1626 /* Only handle setting domains to types used by the CPU. */
1627 if (write_domain & I915_GEM_GPU_DOMAINS)
1628 return -EINVAL;
1629
1630 if (read_domains & I915_GEM_GPU_DOMAINS)
1631 return -EINVAL;
1632
1633 /* Having something in the write domain implies it's in the read
1634 * domain, and only that read domain. Enforce that in the request.
1635 */
1636 if (write_domain != 0 && read_domains != write_domain)
1637 return -EINVAL;
1638
1639 ret = i915_mutex_lock_interruptible(dev);
1640 if (ret)
1641 return ret;
1642
1643 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1644 if (&obj->base == NULL) {
1645 ret = -ENOENT;
1646 goto unlock;
1647 }
1648
1649 /* Try to flush the object off the GPU without holding the lock.
1650 * We will repeat the flush holding the lock in the normal manner
1651 * to catch cases where we are gazumped.
1652 */
1653 ret = i915_gem_object_wait_rendering__nonblocking(obj,
1654 to_rps_client(file),
1655 !write_domain);
1656 if (ret)
1657 goto unref;
1658
1659 if (read_domains & I915_GEM_DOMAIN_GTT)
1660 ret = i915_gem_object_set_to_gtt_domain(obj, write_domain != 0);
1661 else
1662 ret = i915_gem_object_set_to_cpu_domain(obj, write_domain != 0);
1663
1664 if (write_domain != 0)
1665 intel_fb_obj_invalidate(obj,
1666 write_domain == I915_GEM_DOMAIN_GTT ?
1667 ORIGIN_GTT : ORIGIN_CPU);
1668
1669unref:
1670 drm_gem_object_unreference(&obj->base);
1671unlock:
1672 mutex_unlock(&dev->struct_mutex);
1673 return ret;
1674}
1675
1676/**
1677 * Called when user space has done writes to this buffer
1678 */
1679int
1680i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
1681 struct drm_file *file)
1682{
1683 struct drm_i915_gem_sw_finish *args = data;
1684 struct drm_i915_gem_object *obj;
1685 int ret = 0;
1686
1687 ret = i915_mutex_lock_interruptible(dev);
1688 if (ret)
1689 return ret;
1690
1691 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
1692 if (&obj->base == NULL) {
1693 ret = -ENOENT;
1694 goto unlock;
1695 }
1696
1697 /* Pinned buffers may be scanout, so flush the cache */
1698 if (obj->pin_display)
1699 i915_gem_object_flush_cpu_write_domain(obj);
1700
1701 drm_gem_object_unreference(&obj->base);
1702unlock:
1703 mutex_unlock(&dev->struct_mutex);
1704 return ret;
1705}
1706
1707/**
1708 * Maps the contents of an object, returning the address it is mapped
1709 * into.
1710 *
1711 * While the mapping holds a reference on the contents of the object, it doesn't
1712 * imply a ref on the object itself.
1713 *
1714 * IMPORTANT:
1715 *
1716 * DRM driver writers who look a this function as an example for how to do GEM
1717 * mmap support, please don't implement mmap support like here. The modern way
1718 * to implement DRM mmap support is with an mmap offset ioctl (like
1719 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
1720 * That way debug tooling like valgrind will understand what's going on, hiding
1721 * the mmap call in a driver private ioctl will break that. The i915 driver only
1722 * does cpu mmaps this way because we didn't know better.
1723 */
1724int
1725i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
1726 struct drm_file *file)
1727{
1728 struct drm_i915_gem_mmap *args = data;
1729 struct drm_gem_object *obj;
1730 unsigned long addr;
1731
1732 if (args->flags & ~(I915_MMAP_WC))
1733 return -EINVAL;
1734
1735 if (args->flags & I915_MMAP_WC && !cpu_has_pat)
1736 return -ENODEV;
1737
1738 obj = drm_gem_object_lookup(dev, file, args->handle);
1739 if (obj == NULL)
1740 return -ENOENT;
1741
1742 /* prime objects have no backing filp to GEM mmap
1743 * pages from.
1744 */
1745 if (!obj->filp) {
1746 drm_gem_object_unreference_unlocked(obj);
1747 return -EINVAL;
1748 }
1749
1750 addr = vm_mmap(obj->filp, 0, args->size,
1751 PROT_READ | PROT_WRITE, MAP_SHARED,
1752 args->offset);
1753 if (args->flags & I915_MMAP_WC) {
1754 struct mm_struct *mm = current->mm;
1755 struct vm_area_struct *vma;
1756
1757 down_write(&mm->mmap_sem);
1758 vma = find_vma(mm, addr);
1759 if (vma)
1760 vma->vm_page_prot =
1761 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1762 else
1763 addr = -ENOMEM;
1764 up_write(&mm->mmap_sem);
1765 }
1766 drm_gem_object_unreference_unlocked(obj);
1767 if (IS_ERR((void *)addr))
1768 return addr;
1769
1770 args->addr_ptr = (uint64_t) addr;
1771
1772 return 0;
1773}
1774
1775/**
1776 * i915_gem_fault - fault a page into the GTT
1777 * @vma: VMA in question
1778 * @vmf: fault info
1779 *
1780 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
1781 * from userspace. The fault handler takes care of binding the object to
1782 * the GTT (if needed), allocating and programming a fence register (again,
1783 * only if needed based on whether the old reg is still valid or the object
1784 * is tiled) and inserting a new PTE into the faulting process.
1785 *
1786 * Note that the faulting process may involve evicting existing objects
1787 * from the GTT and/or fence registers to make room. So performance may
1788 * suffer if the GTT working set is large or there are few fence registers
1789 * left.
1790 */
1791int i915_gem_fault(struct vm_area_struct *vma, struct vm_fault *vmf)
1792{
1793 struct drm_i915_gem_object *obj = to_intel_bo(vma->vm_private_data);
1794 struct drm_device *dev = obj->base.dev;
1795 struct drm_i915_private *dev_priv = dev->dev_private;
1796 struct i915_ggtt_view view = i915_ggtt_view_normal;
1797 pgoff_t page_offset;
1798 unsigned long pfn;
1799 int ret = 0;
1800 bool write = !!(vmf->flags & FAULT_FLAG_WRITE);
1801
1802 intel_runtime_pm_get(dev_priv);
1803
1804 /* We don't use vmf->pgoff since that has the fake offset */
1805 page_offset = ((unsigned long)vmf->virtual_address - vma->vm_start) >>
1806 PAGE_SHIFT;
1807
1808 ret = i915_mutex_lock_interruptible(dev);
1809 if (ret)
1810 goto out;
1811
1812 trace_i915_gem_object_fault(obj, page_offset, true, write);
1813
1814 /* Try to flush the object off the GPU first without holding the lock.
1815 * Upon reacquiring the lock, we will perform our sanity checks and then
1816 * repeat the flush holding the lock in the normal manner to catch cases
1817 * where we are gazumped.
1818 */
1819 ret = i915_gem_object_wait_rendering__nonblocking(obj, NULL, !write);
1820 if (ret)
1821 goto unlock;
1822
1823 /* Access to snoopable pages through the GTT is incoherent. */
1824 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(dev)) {
1825 ret = -EFAULT;
1826 goto unlock;
1827 }
1828
1829 /* Use a partial view if the object is bigger than the aperture. */
1830 if (obj->base.size >= dev_priv->gtt.mappable_end &&
1831 obj->tiling_mode == I915_TILING_NONE) {
1832 static const unsigned int chunk_size = 256; // 1 MiB
1833
1834 memset(&view, 0, sizeof(view));
1835 view.type = I915_GGTT_VIEW_PARTIAL;
1836 view.params.partial.offset = rounddown(page_offset, chunk_size);
1837 view.params.partial.size =
1838 min_t(unsigned int,
1839 chunk_size,
1840 (vma->vm_end - vma->vm_start)/PAGE_SIZE -
1841 view.params.partial.offset);
1842 }
1843
1844 /* Now pin it into the GTT if needed */
1845 ret = i915_gem_object_ggtt_pin(obj, &view, 0, PIN_MAPPABLE);
1846 if (ret)
1847 goto unlock;
1848
1849 ret = i915_gem_object_set_to_gtt_domain(obj, write);
1850 if (ret)
1851 goto unpin;
1852
1853 ret = i915_gem_object_get_fence(obj);
1854 if (ret)
1855 goto unpin;
1856
1857 /* Finally, remap it using the new GTT offset */
1858 pfn = dev_priv->gtt.mappable_base +
1859 i915_gem_obj_ggtt_offset_view(obj, &view);
1860 pfn >>= PAGE_SHIFT;
1861
1862 if (unlikely(view.type == I915_GGTT_VIEW_PARTIAL)) {
1863 /* Overriding existing pages in partial view does not cause
1864 * us any trouble as TLBs are still valid because the fault
1865 * is due to userspace losing part of the mapping or never
1866 * having accessed it before (at this partials' range).
1867 */
1868 unsigned long base = vma->vm_start +
1869 (view.params.partial.offset << PAGE_SHIFT);
1870 unsigned int i;
1871
1872 for (i = 0; i < view.params.partial.size; i++) {
1873 ret = vm_insert_pfn(vma, base + i * PAGE_SIZE, pfn + i);
1874 if (ret)
1875 break;
1876 }
1877
1878 obj->fault_mappable = true;
1879 } else {
1880 if (!obj->fault_mappable) {
1881 unsigned long size = min_t(unsigned long,
1882 vma->vm_end - vma->vm_start,
1883 obj->base.size);
1884 int i;
1885
1886 for (i = 0; i < size >> PAGE_SHIFT; i++) {
1887 ret = vm_insert_pfn(vma,
1888 (unsigned long)vma->vm_start + i * PAGE_SIZE,
1889 pfn + i);
1890 if (ret)
1891 break;
1892 }
1893
1894 obj->fault_mappable = true;
1895 } else
1896 ret = vm_insert_pfn(vma,
1897 (unsigned long)vmf->virtual_address,
1898 pfn + page_offset);
1899 }
1900unpin:
1901 i915_gem_object_ggtt_unpin_view(obj, &view);
1902unlock:
1903 mutex_unlock(&dev->struct_mutex);
1904out:
1905 switch (ret) {
1906 case -EIO:
1907 /*
1908 * We eat errors when the gpu is terminally wedged to avoid
1909 * userspace unduly crashing (gl has no provisions for mmaps to
1910 * fail). But any other -EIO isn't ours (e.g. swap in failure)
1911 * and so needs to be reported.
1912 */
1913 if (!i915_terminally_wedged(&dev_priv->gpu_error)) {
1914 ret = VM_FAULT_SIGBUS;
1915 break;
1916 }
1917 case -EAGAIN:
1918 /*
1919 * EAGAIN means the gpu is hung and we'll wait for the error
1920 * handler to reset everything when re-faulting in
1921 * i915_mutex_lock_interruptible.
1922 */
1923 case 0:
1924 case -ERESTARTSYS:
1925 case -EINTR:
1926 case -EBUSY:
1927 /*
1928 * EBUSY is ok: this just means that another thread
1929 * already did the job.
1930 */
1931 ret = VM_FAULT_NOPAGE;
1932 break;
1933 case -ENOMEM:
1934 ret = VM_FAULT_OOM;
1935 break;
1936 case -ENOSPC:
1937 case -EFAULT:
1938 ret = VM_FAULT_SIGBUS;
1939 break;
1940 default:
1941 WARN_ONCE(ret, "unhandled error in i915_gem_fault: %i\n", ret);
1942 ret = VM_FAULT_SIGBUS;
1943 break;
1944 }
1945
1946 intel_runtime_pm_put(dev_priv);
1947 return ret;
1948}
1949
1950/**
1951 * i915_gem_release_mmap - remove physical page mappings
1952 * @obj: obj in question
1953 *
1954 * Preserve the reservation of the mmapping with the DRM core code, but
1955 * relinquish ownership of the pages back to the system.
1956 *
1957 * It is vital that we remove the page mapping if we have mapped a tiled
1958 * object through the GTT and then lose the fence register due to
1959 * resource pressure. Similarly if the object has been moved out of the
1960 * aperture, than pages mapped into userspace must be revoked. Removing the
1961 * mapping will then trigger a page fault on the next user access, allowing
1962 * fixup by i915_gem_fault().
1963 */
1964void
1965i915_gem_release_mmap(struct drm_i915_gem_object *obj)
1966{
1967 if (!obj->fault_mappable)
1968 return;
1969
1970 drm_vma_node_unmap(&obj->base.vma_node,
1971 obj->base.dev->anon_inode->i_mapping);
1972 obj->fault_mappable = false;
1973}
1974
1975void
1976i915_gem_release_all_mmaps(struct drm_i915_private *dev_priv)
1977{
1978 struct drm_i915_gem_object *obj;
1979
1980 list_for_each_entry(obj, &dev_priv->mm.bound_list, global_list)
1981 i915_gem_release_mmap(obj);
1982}
1983
1984uint32_t
1985i915_gem_get_gtt_size(struct drm_device *dev, uint32_t size, int tiling_mode)
1986{
1987 uint32_t gtt_size;
1988
1989 if (INTEL_INFO(dev)->gen >= 4 ||
1990 tiling_mode == I915_TILING_NONE)
1991 return size;
1992
1993 /* Previous chips need a power-of-two fence region when tiling */
1994 if (INTEL_INFO(dev)->gen == 3)
1995 gtt_size = 1024*1024;
1996 else
1997 gtt_size = 512*1024;
1998
1999 while (gtt_size < size)
2000 gtt_size <<= 1;
2001
2002 return gtt_size;
2003}
2004
2005/**
2006 * i915_gem_get_gtt_alignment - return required GTT alignment for an object
2007 * @obj: object to check
2008 *
2009 * Return the required GTT alignment for an object, taking into account
2010 * potential fence register mapping.
2011 */
2012uint32_t
2013i915_gem_get_gtt_alignment(struct drm_device *dev, uint32_t size,
2014 int tiling_mode, bool fenced)
2015{
2016 /*
2017 * Minimum alignment is 4k (GTT page size), but might be greater
2018 * if a fence register is needed for the object.
2019 */
2020 if (INTEL_INFO(dev)->gen >= 4 || (!fenced && IS_G33(dev)) ||
2021 tiling_mode == I915_TILING_NONE)
2022 return 4096;
2023
2024 /*
2025 * Previous chips need to be aligned to the size of the smallest
2026 * fence register that can contain the object.
2027 */
2028 return i915_gem_get_gtt_size(dev, size, tiling_mode);
2029}
2030
2031static int i915_gem_object_create_mmap_offset(struct drm_i915_gem_object *obj)
2032{
2033 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2034 int ret;
2035
2036 if (drm_vma_node_has_offset(&obj->base.vma_node))
2037 return 0;
2038
2039 dev_priv->mm.shrinker_no_lock_stealing = true;
2040
2041 ret = drm_gem_create_mmap_offset(&obj->base);
2042 if (ret != -ENOSPC)
2043 goto out;
2044
2045 /* Badly fragmented mmap space? The only way we can recover
2046 * space is by destroying unwanted objects. We can't randomly release
2047 * mmap_offsets as userspace expects them to be persistent for the
2048 * lifetime of the objects. The closest we can is to release the
2049 * offsets on purgeable objects by truncating it and marking it purged,
2050 * which prevents userspace from ever using that object again.
2051 */
2052 i915_gem_shrink(dev_priv,
2053 obj->base.size >> PAGE_SHIFT,
2054 I915_SHRINK_BOUND |
2055 I915_SHRINK_UNBOUND |
2056 I915_SHRINK_PURGEABLE);
2057 ret = drm_gem_create_mmap_offset(&obj->base);
2058 if (ret != -ENOSPC)
2059 goto out;
2060
2061 i915_gem_shrink_all(dev_priv);
2062 ret = drm_gem_create_mmap_offset(&obj->base);
2063out:
2064 dev_priv->mm.shrinker_no_lock_stealing = false;
2065
2066 return ret;
2067}
2068
2069static void i915_gem_object_free_mmap_offset(struct drm_i915_gem_object *obj)
2070{
2071 drm_gem_free_mmap_offset(&obj->base);
2072}
2073
2074int
2075i915_gem_mmap_gtt(struct drm_file *file,
2076 struct drm_device *dev,
2077 uint32_t handle,
2078 uint64_t *offset)
2079{
2080 struct drm_i915_gem_object *obj;
2081 int ret;
2082
2083 ret = i915_mutex_lock_interruptible(dev);
2084 if (ret)
2085 return ret;
2086
2087 obj = to_intel_bo(drm_gem_object_lookup(dev, file, handle));
2088 if (&obj->base == NULL) {
2089 ret = -ENOENT;
2090 goto unlock;
2091 }
2092
2093 if (obj->madv != I915_MADV_WILLNEED) {
2094 DRM_DEBUG("Attempting to mmap a purgeable buffer\n");
2095 ret = -EFAULT;
2096 goto out;
2097 }
2098
2099 ret = i915_gem_object_create_mmap_offset(obj);
2100 if (ret)
2101 goto out;
2102
2103 *offset = drm_vma_node_offset_addr(&obj->base.vma_node);
2104
2105out:
2106 drm_gem_object_unreference(&obj->base);
2107unlock:
2108 mutex_unlock(&dev->struct_mutex);
2109 return ret;
2110}
2111
2112/**
2113 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
2114 * @dev: DRM device
2115 * @data: GTT mapping ioctl data
2116 * @file: GEM object info
2117 *
2118 * Simply returns the fake offset to userspace so it can mmap it.
2119 * The mmap call will end up in drm_gem_mmap(), which will set things
2120 * up so we can get faults in the handler above.
2121 *
2122 * The fault handler will take care of binding the object into the GTT
2123 * (since it may have been evicted to make room for something), allocating
2124 * a fence register, and mapping the appropriate aperture address into
2125 * userspace.
2126 */
2127int
2128i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
2129 struct drm_file *file)
2130{
2131 struct drm_i915_gem_mmap_gtt *args = data;
2132
2133 return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
2134}
2135
2136/* Immediately discard the backing storage */
2137static void
2138i915_gem_object_truncate(struct drm_i915_gem_object *obj)
2139{
2140 i915_gem_object_free_mmap_offset(obj);
2141
2142 if (obj->base.filp == NULL)
2143 return;
2144
2145 /* Our goal here is to return as much of the memory as
2146 * is possible back to the system as we are called from OOM.
2147 * To do this we must instruct the shmfs to drop all of its
2148 * backing pages, *now*.
2149 */
2150 shmem_truncate_range(file_inode(obj->base.filp), 0, (loff_t)-1);
2151 obj->madv = __I915_MADV_PURGED;
2152}
2153
2154/* Try to discard unwanted pages */
2155static void
2156i915_gem_object_invalidate(struct drm_i915_gem_object *obj)
2157{
2158 struct address_space *mapping;
2159
2160 switch (obj->madv) {
2161 case I915_MADV_DONTNEED:
2162 i915_gem_object_truncate(obj);
2163 case __I915_MADV_PURGED:
2164 return;
2165 }
2166
2167 if (obj->base.filp == NULL)
2168 return;
2169
2170 mapping = file_inode(obj->base.filp)->i_mapping,
2171 invalidate_mapping_pages(mapping, 0, (loff_t)-1);
2172}
2173
2174static void
2175i915_gem_object_put_pages_gtt(struct drm_i915_gem_object *obj)
2176{
2177 struct sg_page_iter sg_iter;
2178 int ret;
2179
2180 BUG_ON(obj->madv == __I915_MADV_PURGED);
2181
2182 ret = i915_gem_object_set_to_cpu_domain(obj, true);
2183 if (ret) {
2184 /* In the event of a disaster, abandon all caches and
2185 * hope for the best.
2186 */
2187 WARN_ON(ret != -EIO);
2188 i915_gem_clflush_object(obj, true);
2189 obj->base.read_domains = obj->base.write_domain = I915_GEM_DOMAIN_CPU;
2190 }
2191
2192 i915_gem_gtt_finish_object(obj);
2193
2194 if (i915_gem_object_needs_bit17_swizzle(obj))
2195 i915_gem_object_save_bit_17_swizzle(obj);
2196
2197 if (obj->madv == I915_MADV_DONTNEED)
2198 obj->dirty = 0;
2199
2200 for_each_sg_page(obj->pages->sgl, &sg_iter, obj->pages->nents, 0) {
2201 struct page *page = sg_page_iter_page(&sg_iter);
2202
2203 if (obj->dirty)
2204 set_page_dirty(page);
2205
2206 if (obj->madv == I915_MADV_WILLNEED)
2207 mark_page_accessed(page);
2208
2209 put_page(page);
2210 }
2211 obj->dirty = 0;
2212
2213 sg_free_table(obj->pages);
2214 kfree(obj->pages);
2215}
2216
2217int
2218i915_gem_object_put_pages(struct drm_i915_gem_object *obj)
2219{
2220 const struct drm_i915_gem_object_ops *ops = obj->ops;
2221
2222 if (obj->pages == NULL)
2223 return 0;
2224
2225 if (obj->pages_pin_count)
2226 return -EBUSY;
2227
2228 BUG_ON(i915_gem_obj_bound_any(obj));
2229
2230 /* ->put_pages might need to allocate memory for the bit17 swizzle
2231 * array, hence protect them from being reaped by removing them from gtt
2232 * lists early. */
2233 list_del(&obj->global_list);
2234
2235 ops->put_pages(obj);
2236 obj->pages = NULL;
2237
2238 i915_gem_object_invalidate(obj);
2239
2240 return 0;
2241}
2242
2243static int
2244i915_gem_object_get_pages_gtt(struct drm_i915_gem_object *obj)
2245{
2246 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2247 int page_count, i;
2248 struct address_space *mapping;
2249 struct sg_table *st;
2250 struct scatterlist *sg;
2251 struct sg_page_iter sg_iter;
2252 struct page *page;
2253 unsigned long last_pfn = 0; /* suppress gcc warning */
2254 int ret;
2255 gfp_t gfp;
2256
2257 /* Assert that the object is not currently in any GPU domain. As it
2258 * wasn't in the GTT, there shouldn't be any way it could have been in
2259 * a GPU cache
2260 */
2261 BUG_ON(obj->base.read_domains & I915_GEM_GPU_DOMAINS);
2262 BUG_ON(obj->base.write_domain & I915_GEM_GPU_DOMAINS);
2263
2264 st = kmalloc(sizeof(*st), GFP_KERNEL);
2265 if (st == NULL)
2266 return -ENOMEM;
2267
2268 page_count = obj->base.size / PAGE_SIZE;
2269 if (sg_alloc_table(st, page_count, GFP_KERNEL)) {
2270 kfree(st);
2271 return -ENOMEM;
2272 }
2273
2274 /* Get the list of pages out of our struct file. They'll be pinned
2275 * at this point until we release them.
2276 *
2277 * Fail silently without starting the shrinker
2278 */
2279 mapping = file_inode(obj->base.filp)->i_mapping;
2280 gfp = mapping_gfp_constraint(mapping, ~(__GFP_IO | __GFP_RECLAIM));
2281 gfp |= __GFP_NORETRY | __GFP_NOWARN;
2282 sg = st->sgl;
2283 st->nents = 0;
2284 for (i = 0; i < page_count; i++) {
2285 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2286 if (IS_ERR(page)) {
2287 i915_gem_shrink(dev_priv,
2288 page_count,
2289 I915_SHRINK_BOUND |
2290 I915_SHRINK_UNBOUND |
2291 I915_SHRINK_PURGEABLE);
2292 page = shmem_read_mapping_page_gfp(mapping, i, gfp);
2293 }
2294 if (IS_ERR(page)) {
2295 /* We've tried hard to allocate the memory by reaping
2296 * our own buffer, now let the real VM do its job and
2297 * go down in flames if truly OOM.
2298 */
2299 i915_gem_shrink_all(dev_priv);
2300 page = shmem_read_mapping_page(mapping, i);
2301 if (IS_ERR(page)) {
2302 ret = PTR_ERR(page);
2303 goto err_pages;
2304 }
2305 }
2306#ifdef CONFIG_SWIOTLB
2307 if (swiotlb_nr_tbl()) {
2308 st->nents++;
2309 sg_set_page(sg, page, PAGE_SIZE, 0);
2310 sg = sg_next(sg);
2311 continue;
2312 }
2313#endif
2314 if (!i || page_to_pfn(page) != last_pfn + 1) {
2315 if (i)
2316 sg = sg_next(sg);
2317 st->nents++;
2318 sg_set_page(sg, page, PAGE_SIZE, 0);
2319 } else {
2320 sg->length += PAGE_SIZE;
2321 }
2322 last_pfn = page_to_pfn(page);
2323
2324 /* Check that the i965g/gm workaround works. */
2325 WARN_ON((gfp & __GFP_DMA32) && (last_pfn >= 0x00100000UL));
2326 }
2327#ifdef CONFIG_SWIOTLB
2328 if (!swiotlb_nr_tbl())
2329#endif
2330 sg_mark_end(sg);
2331 obj->pages = st;
2332
2333 ret = i915_gem_gtt_prepare_object(obj);
2334 if (ret)
2335 goto err_pages;
2336
2337 if (i915_gem_object_needs_bit17_swizzle(obj))
2338 i915_gem_object_do_bit_17_swizzle(obj);
2339
2340 if (obj->tiling_mode != I915_TILING_NONE &&
2341 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
2342 i915_gem_object_pin_pages(obj);
2343
2344 return 0;
2345
2346err_pages:
2347 sg_mark_end(sg);
2348 for_each_sg_page(st->sgl, &sg_iter, st->nents, 0)
2349 put_page(sg_page_iter_page(&sg_iter));
2350 sg_free_table(st);
2351 kfree(st);
2352
2353 /* shmemfs first checks if there is enough memory to allocate the page
2354 * and reports ENOSPC should there be insufficient, along with the usual
2355 * ENOMEM for a genuine allocation failure.
2356 *
2357 * We use ENOSPC in our driver to mean that we have run out of aperture
2358 * space and so want to translate the error from shmemfs back to our
2359 * usual understanding of ENOMEM.
2360 */
2361 if (ret == -ENOSPC)
2362 ret = -ENOMEM;
2363
2364 return ret;
2365}
2366
2367/* Ensure that the associated pages are gathered from the backing storage
2368 * and pinned into our object. i915_gem_object_get_pages() may be called
2369 * multiple times before they are released by a single call to
2370 * i915_gem_object_put_pages() - once the pages are no longer referenced
2371 * either as a result of memory pressure (reaping pages under the shrinker)
2372 * or as the object is itself released.
2373 */
2374int
2375i915_gem_object_get_pages(struct drm_i915_gem_object *obj)
2376{
2377 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
2378 const struct drm_i915_gem_object_ops *ops = obj->ops;
2379 int ret;
2380
2381 if (obj->pages)
2382 return 0;
2383
2384 if (obj->madv != I915_MADV_WILLNEED) {
2385 DRM_DEBUG("Attempting to obtain a purgeable object\n");
2386 return -EFAULT;
2387 }
2388
2389 BUG_ON(obj->pages_pin_count);
2390
2391 ret = ops->get_pages(obj);
2392 if (ret)
2393 return ret;
2394
2395 list_add_tail(&obj->global_list, &dev_priv->mm.unbound_list);
2396
2397 obj->get_page.sg = obj->pages->sgl;
2398 obj->get_page.last = 0;
2399
2400 return 0;
2401}
2402
2403void i915_vma_move_to_active(struct i915_vma *vma,
2404 struct drm_i915_gem_request *req)
2405{
2406 struct drm_i915_gem_object *obj = vma->obj;
2407 struct intel_engine_cs *ring;
2408
2409 ring = i915_gem_request_get_ring(req);
2410
2411 /* Add a reference if we're newly entering the active list. */
2412 if (obj->active == 0)
2413 drm_gem_object_reference(&obj->base);
2414 obj->active |= intel_ring_flag(ring);
2415
2416 list_move_tail(&obj->ring_list[ring->id], &ring->active_list);
2417 i915_gem_request_assign(&obj->last_read_req[ring->id], req);
2418
2419 list_move_tail(&vma->vm_link, &vma->vm->active_list);
2420}
2421
2422static void
2423i915_gem_object_retire__write(struct drm_i915_gem_object *obj)
2424{
2425 RQ_BUG_ON(obj->last_write_req == NULL);
2426 RQ_BUG_ON(!(obj->active & intel_ring_flag(obj->last_write_req->ring)));
2427
2428 i915_gem_request_assign(&obj->last_write_req, NULL);
2429 intel_fb_obj_flush(obj, true, ORIGIN_CS);
2430}
2431
2432static void
2433i915_gem_object_retire__read(struct drm_i915_gem_object *obj, int ring)
2434{
2435 struct i915_vma *vma;
2436
2437 RQ_BUG_ON(obj->last_read_req[ring] == NULL);
2438 RQ_BUG_ON(!(obj->active & (1 << ring)));
2439
2440 list_del_init(&obj->ring_list[ring]);
2441 i915_gem_request_assign(&obj->last_read_req[ring], NULL);
2442
2443 if (obj->last_write_req && obj->last_write_req->ring->id == ring)
2444 i915_gem_object_retire__write(obj);
2445
2446 obj->active &= ~(1 << ring);
2447 if (obj->active)
2448 return;
2449
2450 /* Bump our place on the bound list to keep it roughly in LRU order
2451 * so that we don't steal from recently used but inactive objects
2452 * (unless we are forced to ofc!)
2453 */
2454 list_move_tail(&obj->global_list,
2455 &to_i915(obj->base.dev)->mm.bound_list);
2456
2457 list_for_each_entry(vma, &obj->vma_list, obj_link) {
2458 if (!list_empty(&vma->vm_link))
2459 list_move_tail(&vma->vm_link, &vma->vm->inactive_list);
2460 }
2461
2462 i915_gem_request_assign(&obj->last_fenced_req, NULL);
2463 drm_gem_object_unreference(&obj->base);
2464}
2465
2466static int
2467i915_gem_init_seqno(struct drm_device *dev, u32 seqno)
2468{
2469 struct drm_i915_private *dev_priv = dev->dev_private;
2470 struct intel_engine_cs *ring;
2471 int ret, i, j;
2472
2473 /* Carefully retire all requests without writing to the rings */
2474 for_each_ring(ring, dev_priv, i) {
2475 ret = intel_ring_idle(ring);
2476 if (ret)
2477 return ret;
2478 }
2479 i915_gem_retire_requests(dev);
2480
2481 /* Finally reset hw state */
2482 for_each_ring(ring, dev_priv, i) {
2483 intel_ring_init_seqno(ring, seqno);
2484
2485 for (j = 0; j < ARRAY_SIZE(ring->semaphore.sync_seqno); j++)
2486 ring->semaphore.sync_seqno[j] = 0;
2487 }
2488
2489 return 0;
2490}
2491
2492int i915_gem_set_seqno(struct drm_device *dev, u32 seqno)
2493{
2494 struct drm_i915_private *dev_priv = dev->dev_private;
2495 int ret;
2496
2497 if (seqno == 0)
2498 return -EINVAL;
2499
2500 /* HWS page needs to be set less than what we
2501 * will inject to ring
2502 */
2503 ret = i915_gem_init_seqno(dev, seqno - 1);
2504 if (ret)
2505 return ret;
2506
2507 /* Carefully set the last_seqno value so that wrap
2508 * detection still works
2509 */
2510 dev_priv->next_seqno = seqno;
2511 dev_priv->last_seqno = seqno - 1;
2512 if (dev_priv->last_seqno == 0)
2513 dev_priv->last_seqno--;
2514
2515 return 0;
2516}
2517
2518int
2519i915_gem_get_seqno(struct drm_device *dev, u32 *seqno)
2520{
2521 struct drm_i915_private *dev_priv = dev->dev_private;
2522
2523 /* reserve 0 for non-seqno */
2524 if (dev_priv->next_seqno == 0) {
2525 int ret = i915_gem_init_seqno(dev, 0);
2526 if (ret)
2527 return ret;
2528
2529 dev_priv->next_seqno = 1;
2530 }
2531
2532 *seqno = dev_priv->last_seqno = dev_priv->next_seqno++;
2533 return 0;
2534}
2535
2536/*
2537 * NB: This function is not allowed to fail. Doing so would mean the the
2538 * request is not being tracked for completion but the work itself is
2539 * going to happen on the hardware. This would be a Bad Thing(tm).
2540 */
2541void __i915_add_request(struct drm_i915_gem_request *request,
2542 struct drm_i915_gem_object *obj,
2543 bool flush_caches)
2544{
2545 struct intel_engine_cs *ring;
2546 struct drm_i915_private *dev_priv;
2547 struct intel_ringbuffer *ringbuf;
2548 u32 request_start;
2549 int ret;
2550
2551 if (WARN_ON(request == NULL))
2552 return;
2553
2554 ring = request->ring;
2555 dev_priv = ring->dev->dev_private;
2556 ringbuf = request->ringbuf;
2557
2558 /*
2559 * To ensure that this call will not fail, space for its emissions
2560 * should already have been reserved in the ring buffer. Let the ring
2561 * know that it is time to use that space up.
2562 */
2563 intel_ring_reserved_space_use(ringbuf);
2564
2565 request_start = intel_ring_get_tail(ringbuf);
2566 /*
2567 * Emit any outstanding flushes - execbuf can fail to emit the flush
2568 * after having emitted the batchbuffer command. Hence we need to fix
2569 * things up similar to emitting the lazy request. The difference here
2570 * is that the flush _must_ happen before the next request, no matter
2571 * what.
2572 */
2573 if (flush_caches) {
2574 if (i915.enable_execlists)
2575 ret = logical_ring_flush_all_caches(request);
2576 else
2577 ret = intel_ring_flush_all_caches(request);
2578 /* Not allowed to fail! */
2579 WARN(ret, "*_ring_flush_all_caches failed: %d!\n", ret);
2580 }
2581
2582 /* Record the position of the start of the request so that
2583 * should we detect the updated seqno part-way through the
2584 * GPU processing the request, we never over-estimate the
2585 * position of the head.
2586 */
2587 request->postfix = intel_ring_get_tail(ringbuf);
2588
2589 if (i915.enable_execlists)
2590 ret = ring->emit_request(request);
2591 else {
2592 ret = ring->add_request(request);
2593
2594 request->tail = intel_ring_get_tail(ringbuf);
2595 }
2596 /* Not allowed to fail! */
2597 WARN(ret, "emit|add_request failed: %d!\n", ret);
2598
2599 request->head = request_start;
2600
2601 /* Whilst this request exists, batch_obj will be on the
2602 * active_list, and so will hold the active reference. Only when this
2603 * request is retired will the the batch_obj be moved onto the
2604 * inactive_list and lose its active reference. Hence we do not need
2605 * to explicitly hold another reference here.
2606 */
2607 request->batch_obj = obj;
2608
2609 request->emitted_jiffies = jiffies;
2610 request->previous_seqno = ring->last_submitted_seqno;
2611 ring->last_submitted_seqno = request->seqno;
2612 list_add_tail(&request->list, &ring->request_list);
2613
2614 trace_i915_gem_request_add(request);
2615
2616 i915_queue_hangcheck(ring->dev);
2617
2618 queue_delayed_work(dev_priv->wq,
2619 &dev_priv->mm.retire_work,
2620 round_jiffies_up_relative(HZ));
2621 intel_mark_busy(dev_priv->dev);
2622
2623 /* Sanity check that the reserved size was large enough. */
2624 intel_ring_reserved_space_end(ringbuf);
2625}
2626
2627static bool i915_context_is_banned(struct drm_i915_private *dev_priv,
2628 const struct intel_context *ctx)
2629{
2630 unsigned long elapsed;
2631
2632 elapsed = get_seconds() - ctx->hang_stats.guilty_ts;
2633
2634 if (ctx->hang_stats.banned)
2635 return true;
2636
2637 if (ctx->hang_stats.ban_period_seconds &&
2638 elapsed <= ctx->hang_stats.ban_period_seconds) {
2639 if (!i915_gem_context_is_default(ctx)) {
2640 DRM_DEBUG("context hanging too fast, banning!\n");
2641 return true;
2642 } else if (i915_stop_ring_allow_ban(dev_priv)) {
2643 if (i915_stop_ring_allow_warn(dev_priv))
2644 DRM_ERROR("gpu hanging too fast, banning!\n");
2645 return true;
2646 }
2647 }
2648
2649 return false;
2650}
2651
2652static void i915_set_reset_status(struct drm_i915_private *dev_priv,
2653 struct intel_context *ctx,
2654 const bool guilty)
2655{
2656 struct i915_ctx_hang_stats *hs;
2657
2658 if (WARN_ON(!ctx))
2659 return;
2660
2661 hs = &ctx->hang_stats;
2662
2663 if (guilty) {
2664 hs->banned = i915_context_is_banned(dev_priv, ctx);
2665 hs->batch_active++;
2666 hs->guilty_ts = get_seconds();
2667 } else {
2668 hs->batch_pending++;
2669 }
2670}
2671
2672void i915_gem_request_free(struct kref *req_ref)
2673{
2674 struct drm_i915_gem_request *req = container_of(req_ref,
2675 typeof(*req), ref);
2676 struct intel_context *ctx = req->ctx;
2677
2678 if (req->file_priv)
2679 i915_gem_request_remove_from_client(req);
2680
2681 if (ctx) {
2682 if (i915.enable_execlists && ctx != req->i915->kernel_context)
2683 intel_lr_context_unpin(ctx, req->ring);
2684
2685 i915_gem_context_unreference(ctx);
2686 }
2687
2688 kmem_cache_free(req->i915->requests, req);
2689}
2690
2691static inline int
2692__i915_gem_request_alloc(struct intel_engine_cs *ring,
2693 struct intel_context *ctx,
2694 struct drm_i915_gem_request **req_out)
2695{
2696 struct drm_i915_private *dev_priv = to_i915(ring->dev);
2697 struct drm_i915_gem_request *req;
2698 int ret;
2699
2700 if (!req_out)
2701 return -EINVAL;
2702
2703 *req_out = NULL;
2704
2705 req = kmem_cache_zalloc(dev_priv->requests, GFP_KERNEL);
2706 if (req == NULL)
2707 return -ENOMEM;
2708
2709 ret = i915_gem_get_seqno(ring->dev, &req->seqno);
2710 if (ret)
2711 goto err;
2712
2713 kref_init(&req->ref);
2714 req->i915 = dev_priv;
2715 req->ring = ring;
2716 req->ctx = ctx;
2717 i915_gem_context_reference(req->ctx);
2718
2719 if (i915.enable_execlists)
2720 ret = intel_logical_ring_alloc_request_extras(req);
2721 else
2722 ret = intel_ring_alloc_request_extras(req);
2723 if (ret) {
2724 i915_gem_context_unreference(req->ctx);
2725 goto err;
2726 }
2727
2728 /*
2729 * Reserve space in the ring buffer for all the commands required to
2730 * eventually emit this request. This is to guarantee that the
2731 * i915_add_request() call can't fail. Note that the reserve may need
2732 * to be redone if the request is not actually submitted straight
2733 * away, e.g. because a GPU scheduler has deferred it.
2734 */
2735 if (i915.enable_execlists)
2736 ret = intel_logical_ring_reserve_space(req);
2737 else
2738 ret = intel_ring_reserve_space(req);
2739 if (ret) {
2740 /*
2741 * At this point, the request is fully allocated even if not
2742 * fully prepared. Thus it can be cleaned up using the proper
2743 * free code.
2744 */
2745 i915_gem_request_cancel(req);
2746 return ret;
2747 }
2748
2749 *req_out = req;
2750 return 0;
2751
2752err:
2753 kmem_cache_free(dev_priv->requests, req);
2754 return ret;
2755}
2756
2757/**
2758 * i915_gem_request_alloc - allocate a request structure
2759 *
2760 * @engine: engine that we wish to issue the request on.
2761 * @ctx: context that the request will be associated with.
2762 * This can be NULL if the request is not directly related to
2763 * any specific user context, in which case this function will
2764 * choose an appropriate context to use.
2765 *
2766 * Returns a pointer to the allocated request if successful,
2767 * or an error code if not.
2768 */
2769struct drm_i915_gem_request *
2770i915_gem_request_alloc(struct intel_engine_cs *engine,
2771 struct intel_context *ctx)
2772{
2773 struct drm_i915_gem_request *req;
2774 int err;
2775
2776 if (ctx == NULL)
2777 ctx = to_i915(engine->dev)->kernel_context;
2778 err = __i915_gem_request_alloc(engine, ctx, &req);
2779 return err ? ERR_PTR(err) : req;
2780}
2781
2782void i915_gem_request_cancel(struct drm_i915_gem_request *req)
2783{
2784 intel_ring_reserved_space_cancel(req->ringbuf);
2785
2786 i915_gem_request_unreference(req);
2787}
2788
2789struct drm_i915_gem_request *
2790i915_gem_find_active_request(struct intel_engine_cs *ring)
2791{
2792 struct drm_i915_gem_request *request;
2793
2794 list_for_each_entry(request, &ring->request_list, list) {
2795 if (i915_gem_request_completed(request, false))
2796 continue;
2797
2798 return request;
2799 }
2800
2801 return NULL;
2802}
2803
2804static void i915_gem_reset_ring_status(struct drm_i915_private *dev_priv,
2805 struct intel_engine_cs *ring)
2806{
2807 struct drm_i915_gem_request *request;
2808 bool ring_hung;
2809
2810 request = i915_gem_find_active_request(ring);
2811
2812 if (request == NULL)
2813 return;
2814
2815 ring_hung = ring->hangcheck.score >= HANGCHECK_SCORE_RING_HUNG;
2816
2817 i915_set_reset_status(dev_priv, request->ctx, ring_hung);
2818
2819 list_for_each_entry_continue(request, &ring->request_list, list)
2820 i915_set_reset_status(dev_priv, request->ctx, false);
2821}
2822
2823static void i915_gem_reset_ring_cleanup(struct drm_i915_private *dev_priv,
2824 struct intel_engine_cs *ring)
2825{
2826 struct intel_ringbuffer *buffer;
2827
2828 while (!list_empty(&ring->active_list)) {
2829 struct drm_i915_gem_object *obj;
2830
2831 obj = list_first_entry(&ring->active_list,
2832 struct drm_i915_gem_object,
2833 ring_list[ring->id]);
2834
2835 i915_gem_object_retire__read(obj, ring->id);
2836 }
2837
2838 /*
2839 * Clear the execlists queue up before freeing the requests, as those
2840 * are the ones that keep the context and ringbuffer backing objects
2841 * pinned in place.
2842 */
2843
2844 if (i915.enable_execlists) {
2845 spin_lock_irq(&ring->execlist_lock);
2846
2847 /* list_splice_tail_init checks for empty lists */
2848 list_splice_tail_init(&ring->execlist_queue,
2849 &ring->execlist_retired_req_list);
2850
2851 spin_unlock_irq(&ring->execlist_lock);
2852 intel_execlists_retire_requests(ring);
2853 }
2854
2855 /*
2856 * We must free the requests after all the corresponding objects have
2857 * been moved off active lists. Which is the same order as the normal
2858 * retire_requests function does. This is important if object hold
2859 * implicit references on things like e.g. ppgtt address spaces through
2860 * the request.
2861 */
2862 while (!list_empty(&ring->request_list)) {
2863 struct drm_i915_gem_request *request;
2864
2865 request = list_first_entry(&ring->request_list,
2866 struct drm_i915_gem_request,
2867 list);
2868
2869 i915_gem_request_retire(request);
2870 }
2871
2872 /* Having flushed all requests from all queues, we know that all
2873 * ringbuffers must now be empty. However, since we do not reclaim
2874 * all space when retiring the request (to prevent HEADs colliding
2875 * with rapid ringbuffer wraparound) the amount of available space
2876 * upon reset is less than when we start. Do one more pass over
2877 * all the ringbuffers to reset last_retired_head.
2878 */
2879 list_for_each_entry(buffer, &ring->buffers, link) {
2880 buffer->last_retired_head = buffer->tail;
2881 intel_ring_update_space(buffer);
2882 }
2883}
2884
2885void i915_gem_reset(struct drm_device *dev)
2886{
2887 struct drm_i915_private *dev_priv = dev->dev_private;
2888 struct intel_engine_cs *ring;
2889 int i;
2890
2891 /*
2892 * Before we free the objects from the requests, we need to inspect
2893 * them for finding the guilty party. As the requests only borrow
2894 * their reference to the objects, the inspection must be done first.
2895 */
2896 for_each_ring(ring, dev_priv, i)
2897 i915_gem_reset_ring_status(dev_priv, ring);
2898
2899 for_each_ring(ring, dev_priv, i)
2900 i915_gem_reset_ring_cleanup(dev_priv, ring);
2901
2902 i915_gem_context_reset(dev);
2903
2904 i915_gem_restore_fences(dev);
2905
2906 WARN_ON(i915_verify_lists(dev));
2907}
2908
2909/**
2910 * This function clears the request list as sequence numbers are passed.
2911 */
2912void
2913i915_gem_retire_requests_ring(struct intel_engine_cs *ring)
2914{
2915 WARN_ON(i915_verify_lists(ring->dev));
2916
2917 /* Retire requests first as we use it above for the early return.
2918 * If we retire requests last, we may use a later seqno and so clear
2919 * the requests lists without clearing the active list, leading to
2920 * confusion.
2921 */
2922 while (!list_empty(&ring->request_list)) {
2923 struct drm_i915_gem_request *request;
2924
2925 request = list_first_entry(&ring->request_list,
2926 struct drm_i915_gem_request,
2927 list);
2928
2929 if (!i915_gem_request_completed(request, true))
2930 break;
2931
2932 i915_gem_request_retire(request);
2933 }
2934
2935 /* Move any buffers on the active list that are no longer referenced
2936 * by the ringbuffer to the flushing/inactive lists as appropriate,
2937 * before we free the context associated with the requests.
2938 */
2939 while (!list_empty(&ring->active_list)) {
2940 struct drm_i915_gem_object *obj;
2941
2942 obj = list_first_entry(&ring->active_list,
2943 struct drm_i915_gem_object,
2944 ring_list[ring->id]);
2945
2946 if (!list_empty(&obj->last_read_req[ring->id]->list))
2947 break;
2948
2949 i915_gem_object_retire__read(obj, ring->id);
2950 }
2951
2952 if (unlikely(ring->trace_irq_req &&
2953 i915_gem_request_completed(ring->trace_irq_req, true))) {
2954 ring->irq_put(ring);
2955 i915_gem_request_assign(&ring->trace_irq_req, NULL);
2956 }
2957
2958 WARN_ON(i915_verify_lists(ring->dev));
2959}
2960
2961bool
2962i915_gem_retire_requests(struct drm_device *dev)
2963{
2964 struct drm_i915_private *dev_priv = dev->dev_private;
2965 struct intel_engine_cs *ring;
2966 bool idle = true;
2967 int i;
2968
2969 for_each_ring(ring, dev_priv, i) {
2970 i915_gem_retire_requests_ring(ring);
2971 idle &= list_empty(&ring->request_list);
2972 if (i915.enable_execlists) {
2973 spin_lock_irq(&ring->execlist_lock);
2974 idle &= list_empty(&ring->execlist_queue);
2975 spin_unlock_irq(&ring->execlist_lock);
2976
2977 intel_execlists_retire_requests(ring);
2978 }
2979 }
2980
2981 if (idle)
2982 mod_delayed_work(dev_priv->wq,
2983 &dev_priv->mm.idle_work,
2984 msecs_to_jiffies(100));
2985
2986 return idle;
2987}
2988
2989static void
2990i915_gem_retire_work_handler(struct work_struct *work)
2991{
2992 struct drm_i915_private *dev_priv =
2993 container_of(work, typeof(*dev_priv), mm.retire_work.work);
2994 struct drm_device *dev = dev_priv->dev;
2995 bool idle;
2996
2997 /* Come back later if the device is busy... */
2998 idle = false;
2999 if (mutex_trylock(&dev->struct_mutex)) {
3000 idle = i915_gem_retire_requests(dev);
3001 mutex_unlock(&dev->struct_mutex);
3002 }
3003 if (!idle)
3004 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work,
3005 round_jiffies_up_relative(HZ));
3006}
3007
3008static void
3009i915_gem_idle_work_handler(struct work_struct *work)
3010{
3011 struct drm_i915_private *dev_priv =
3012 container_of(work, typeof(*dev_priv), mm.idle_work.work);
3013 struct drm_device *dev = dev_priv->dev;
3014 struct intel_engine_cs *ring;
3015 int i;
3016
3017 for_each_ring(ring, dev_priv, i)
3018 if (!list_empty(&ring->request_list))
3019 return;
3020
3021 /* we probably should sync with hangcheck here, using cancel_work_sync.
3022 * Also locking seems to be fubar here, ring->request_list is protected
3023 * by dev->struct_mutex. */
3024
3025 intel_mark_idle(dev);
3026
3027 if (mutex_trylock(&dev->struct_mutex)) {
3028 struct intel_engine_cs *ring;
3029 int i;
3030
3031 for_each_ring(ring, dev_priv, i)
3032 i915_gem_batch_pool_fini(&ring->batch_pool);
3033
3034 mutex_unlock(&dev->struct_mutex);
3035 }
3036}
3037
3038/**
3039 * Ensures that an object will eventually get non-busy by flushing any required
3040 * write domains, emitting any outstanding lazy request and retiring and
3041 * completed requests.
3042 */
3043static int
3044i915_gem_object_flush_active(struct drm_i915_gem_object *obj)
3045{
3046 int i;
3047
3048 if (!obj->active)
3049 return 0;
3050
3051 for (i = 0; i < I915_NUM_RINGS; i++) {
3052 struct drm_i915_gem_request *req;
3053
3054 req = obj->last_read_req[i];
3055 if (req == NULL)
3056 continue;
3057
3058 if (list_empty(&req->list))
3059 goto retire;
3060
3061 if (i915_gem_request_completed(req, true)) {
3062 __i915_gem_request_retire__upto(req);
3063retire:
3064 i915_gem_object_retire__read(obj, i);
3065 }
3066 }
3067
3068 return 0;
3069}
3070
3071/**
3072 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
3073 * @DRM_IOCTL_ARGS: standard ioctl arguments
3074 *
3075 * Returns 0 if successful, else an error is returned with the remaining time in
3076 * the timeout parameter.
3077 * -ETIME: object is still busy after timeout
3078 * -ERESTARTSYS: signal interrupted the wait
3079 * -ENONENT: object doesn't exist
3080 * Also possible, but rare:
3081 * -EAGAIN: GPU wedged
3082 * -ENOMEM: damn
3083 * -ENODEV: Internal IRQ fail
3084 * -E?: The add request failed
3085 *
3086 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
3087 * non-zero timeout parameter the wait ioctl will wait for the given number of
3088 * nanoseconds on an object becoming unbusy. Since the wait itself does so
3089 * without holding struct_mutex the object may become re-busied before this
3090 * function completes. A similar but shorter * race condition exists in the busy
3091 * ioctl
3092 */
3093int
3094i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
3095{
3096 struct drm_i915_private *dev_priv = dev->dev_private;
3097 struct drm_i915_gem_wait *args = data;
3098 struct drm_i915_gem_object *obj;
3099 struct drm_i915_gem_request *req[I915_NUM_RINGS];
3100 unsigned reset_counter;
3101 int i, n = 0;
3102 int ret;
3103
3104 if (args->flags != 0)
3105 return -EINVAL;
3106
3107 ret = i915_mutex_lock_interruptible(dev);
3108 if (ret)
3109 return ret;
3110
3111 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->bo_handle));
3112 if (&obj->base == NULL) {
3113 mutex_unlock(&dev->struct_mutex);
3114 return -ENOENT;
3115 }
3116
3117 /* Need to make sure the object gets inactive eventually. */
3118 ret = i915_gem_object_flush_active(obj);
3119 if (ret)
3120 goto out;
3121
3122 if (!obj->active)
3123 goto out;
3124
3125 /* Do this after OLR check to make sure we make forward progress polling
3126 * on this IOCTL with a timeout == 0 (like busy ioctl)
3127 */
3128 if (args->timeout_ns == 0) {
3129 ret = -ETIME;
3130 goto out;
3131 }
3132
3133 drm_gem_object_unreference(&obj->base);
3134 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
3135
3136 for (i = 0; i < I915_NUM_RINGS; i++) {
3137 if (obj->last_read_req[i] == NULL)
3138 continue;
3139
3140 req[n++] = i915_gem_request_reference(obj->last_read_req[i]);
3141 }
3142
3143 mutex_unlock(&dev->struct_mutex);
3144
3145 for (i = 0; i < n; i++) {
3146 if (ret == 0)
3147 ret = __i915_wait_request(req[i], reset_counter, true,
3148 args->timeout_ns > 0 ? &args->timeout_ns : NULL,
3149 to_rps_client(file));
3150 i915_gem_request_unreference__unlocked(req[i]);
3151 }
3152 return ret;
3153
3154out:
3155 drm_gem_object_unreference(&obj->base);
3156 mutex_unlock(&dev->struct_mutex);
3157 return ret;
3158}
3159
3160static int
3161__i915_gem_object_sync(struct drm_i915_gem_object *obj,
3162 struct intel_engine_cs *to,
3163 struct drm_i915_gem_request *from_req,
3164 struct drm_i915_gem_request **to_req)
3165{
3166 struct intel_engine_cs *from;
3167 int ret;
3168
3169 from = i915_gem_request_get_ring(from_req);
3170 if (to == from)
3171 return 0;
3172
3173 if (i915_gem_request_completed(from_req, true))
3174 return 0;
3175
3176 if (!i915_semaphore_is_enabled(obj->base.dev)) {
3177 struct drm_i915_private *i915 = to_i915(obj->base.dev);
3178 ret = __i915_wait_request(from_req,
3179 atomic_read(&i915->gpu_error.reset_counter),
3180 i915->mm.interruptible,
3181 NULL,
3182 &i915->rps.semaphores);
3183 if (ret)
3184 return ret;
3185
3186 i915_gem_object_retire_request(obj, from_req);
3187 } else {
3188 int idx = intel_ring_sync_index(from, to);
3189 u32 seqno = i915_gem_request_get_seqno(from_req);
3190
3191 WARN_ON(!to_req);
3192
3193 if (seqno <= from->semaphore.sync_seqno[idx])
3194 return 0;
3195
3196 if (*to_req == NULL) {
3197 struct drm_i915_gem_request *req;
3198
3199 req = i915_gem_request_alloc(to, NULL);
3200 if (IS_ERR(req))
3201 return PTR_ERR(req);
3202
3203 *to_req = req;
3204 }
3205
3206 trace_i915_gem_ring_sync_to(*to_req, from, from_req);
3207 ret = to->semaphore.sync_to(*to_req, from, seqno);
3208 if (ret)
3209 return ret;
3210
3211 /* We use last_read_req because sync_to()
3212 * might have just caused seqno wrap under
3213 * the radar.
3214 */
3215 from->semaphore.sync_seqno[idx] =
3216 i915_gem_request_get_seqno(obj->last_read_req[from->id]);
3217 }
3218
3219 return 0;
3220}
3221
3222/**
3223 * i915_gem_object_sync - sync an object to a ring.
3224 *
3225 * @obj: object which may be in use on another ring.
3226 * @to: ring we wish to use the object on. May be NULL.
3227 * @to_req: request we wish to use the object for. See below.
3228 * This will be allocated and returned if a request is
3229 * required but not passed in.
3230 *
3231 * This code is meant to abstract object synchronization with the GPU.
3232 * Calling with NULL implies synchronizing the object with the CPU
3233 * rather than a particular GPU ring. Conceptually we serialise writes
3234 * between engines inside the GPU. We only allow one engine to write
3235 * into a buffer at any time, but multiple readers. To ensure each has
3236 * a coherent view of memory, we must:
3237 *
3238 * - If there is an outstanding write request to the object, the new
3239 * request must wait for it to complete (either CPU or in hw, requests
3240 * on the same ring will be naturally ordered).
3241 *
3242 * - If we are a write request (pending_write_domain is set), the new
3243 * request must wait for outstanding read requests to complete.
3244 *
3245 * For CPU synchronisation (NULL to) no request is required. For syncing with
3246 * rings to_req must be non-NULL. However, a request does not have to be
3247 * pre-allocated. If *to_req is NULL and sync commands will be emitted then a
3248 * request will be allocated automatically and returned through *to_req. Note
3249 * that it is not guaranteed that commands will be emitted (because the system
3250 * might already be idle). Hence there is no need to create a request that
3251 * might never have any work submitted. Note further that if a request is
3252 * returned in *to_req, it is the responsibility of the caller to submit
3253 * that request (after potentially adding more work to it).
3254 *
3255 * Returns 0 if successful, else propagates up the lower layer error.
3256 */
3257int
3258i915_gem_object_sync(struct drm_i915_gem_object *obj,
3259 struct intel_engine_cs *to,
3260 struct drm_i915_gem_request **to_req)
3261{
3262 const bool readonly = obj->base.pending_write_domain == 0;
3263 struct drm_i915_gem_request *req[I915_NUM_RINGS];
3264 int ret, i, n;
3265
3266 if (!obj->active)
3267 return 0;
3268
3269 if (to == NULL)
3270 return i915_gem_object_wait_rendering(obj, readonly);
3271
3272 n = 0;
3273 if (readonly) {
3274 if (obj->last_write_req)
3275 req[n++] = obj->last_write_req;
3276 } else {
3277 for (i = 0; i < I915_NUM_RINGS; i++)
3278 if (obj->last_read_req[i])
3279 req[n++] = obj->last_read_req[i];
3280 }
3281 for (i = 0; i < n; i++) {
3282 ret = __i915_gem_object_sync(obj, to, req[i], to_req);
3283 if (ret)
3284 return ret;
3285 }
3286
3287 return 0;
3288}
3289
3290static void i915_gem_object_finish_gtt(struct drm_i915_gem_object *obj)
3291{
3292 u32 old_write_domain, old_read_domains;
3293
3294 /* Force a pagefault for domain tracking on next user access */
3295 i915_gem_release_mmap(obj);
3296
3297 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3298 return;
3299
3300 /* Wait for any direct GTT access to complete */
3301 mb();
3302
3303 old_read_domains = obj->base.read_domains;
3304 old_write_domain = obj->base.write_domain;
3305
3306 obj->base.read_domains &= ~I915_GEM_DOMAIN_GTT;
3307 obj->base.write_domain &= ~I915_GEM_DOMAIN_GTT;
3308
3309 trace_i915_gem_object_change_domain(obj,
3310 old_read_domains,
3311 old_write_domain);
3312}
3313
3314static int __i915_vma_unbind(struct i915_vma *vma, bool wait)
3315{
3316 struct drm_i915_gem_object *obj = vma->obj;
3317 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
3318 int ret;
3319
3320 if (list_empty(&vma->obj_link))
3321 return 0;
3322
3323 if (!drm_mm_node_allocated(&vma->node)) {
3324 i915_gem_vma_destroy(vma);
3325 return 0;
3326 }
3327
3328 if (vma->pin_count)
3329 return -EBUSY;
3330
3331 BUG_ON(obj->pages == NULL);
3332
3333 if (wait) {
3334 ret = i915_gem_object_wait_rendering(obj, false);
3335 if (ret)
3336 return ret;
3337 }
3338
3339 if (vma->is_ggtt && vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3340 i915_gem_object_finish_gtt(obj);
3341
3342 /* release the fence reg _after_ flushing */
3343 ret = i915_gem_object_put_fence(obj);
3344 if (ret)
3345 return ret;
3346 }
3347
3348 trace_i915_vma_unbind(vma);
3349
3350 vma->vm->unbind_vma(vma);
3351 vma->bound = 0;
3352
3353 list_del_init(&vma->vm_link);
3354 if (vma->is_ggtt) {
3355 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL) {
3356 obj->map_and_fenceable = false;
3357 } else if (vma->ggtt_view.pages) {
3358 sg_free_table(vma->ggtt_view.pages);
3359 kfree(vma->ggtt_view.pages);
3360 }
3361 vma->ggtt_view.pages = NULL;
3362 }
3363
3364 drm_mm_remove_node(&vma->node);
3365 i915_gem_vma_destroy(vma);
3366
3367 /* Since the unbound list is global, only move to that list if
3368 * no more VMAs exist. */
3369 if (list_empty(&obj->vma_list))
3370 list_move_tail(&obj->global_list, &dev_priv->mm.unbound_list);
3371
3372 /* And finally now the object is completely decoupled from this vma,
3373 * we can drop its hold on the backing storage and allow it to be
3374 * reaped by the shrinker.
3375 */
3376 i915_gem_object_unpin_pages(obj);
3377
3378 return 0;
3379}
3380
3381int i915_vma_unbind(struct i915_vma *vma)
3382{
3383 return __i915_vma_unbind(vma, true);
3384}
3385
3386int __i915_vma_unbind_no_wait(struct i915_vma *vma)
3387{
3388 return __i915_vma_unbind(vma, false);
3389}
3390
3391int i915_gpu_idle(struct drm_device *dev)
3392{
3393 struct drm_i915_private *dev_priv = dev->dev_private;
3394 struct intel_engine_cs *ring;
3395 int ret, i;
3396
3397 /* Flush everything onto the inactive list. */
3398 for_each_ring(ring, dev_priv, i) {
3399 if (!i915.enable_execlists) {
3400 struct drm_i915_gem_request *req;
3401
3402 req = i915_gem_request_alloc(ring, NULL);
3403 if (IS_ERR(req))
3404 return PTR_ERR(req);
3405
3406 ret = i915_switch_context(req);
3407 if (ret) {
3408 i915_gem_request_cancel(req);
3409 return ret;
3410 }
3411
3412 i915_add_request_no_flush(req);
3413 }
3414
3415 ret = intel_ring_idle(ring);
3416 if (ret)
3417 return ret;
3418 }
3419
3420 WARN_ON(i915_verify_lists(dev));
3421 return 0;
3422}
3423
3424static bool i915_gem_valid_gtt_space(struct i915_vma *vma,
3425 unsigned long cache_level)
3426{
3427 struct drm_mm_node *gtt_space = &vma->node;
3428 struct drm_mm_node *other;
3429
3430 /*
3431 * On some machines we have to be careful when putting differing types
3432 * of snoopable memory together to avoid the prefetcher crossing memory
3433 * domains and dying. During vm initialisation, we decide whether or not
3434 * these constraints apply and set the drm_mm.color_adjust
3435 * appropriately.
3436 */
3437 if (vma->vm->mm.color_adjust == NULL)
3438 return true;
3439
3440 if (!drm_mm_node_allocated(gtt_space))
3441 return true;
3442
3443 if (list_empty(>t_space->node_list))
3444 return true;
3445
3446 other = list_entry(gtt_space->node_list.prev, struct drm_mm_node, node_list);
3447 if (other->allocated && !other->hole_follows && other->color != cache_level)
3448 return false;
3449
3450 other = list_entry(gtt_space->node_list.next, struct drm_mm_node, node_list);
3451 if (other->allocated && !gtt_space->hole_follows && other->color != cache_level)
3452 return false;
3453
3454 return true;
3455}
3456
3457/**
3458 * Finds free space in the GTT aperture and binds the object or a view of it
3459 * there.
3460 */
3461static struct i915_vma *
3462i915_gem_object_bind_to_vm(struct drm_i915_gem_object *obj,
3463 struct i915_address_space *vm,
3464 const struct i915_ggtt_view *ggtt_view,
3465 unsigned alignment,
3466 uint64_t flags)
3467{
3468 struct drm_device *dev = obj->base.dev;
3469 struct drm_i915_private *dev_priv = dev->dev_private;
3470 u32 fence_alignment, unfenced_alignment;
3471 u32 search_flag, alloc_flag;
3472 u64 start, end;
3473 u64 size, fence_size;
3474 struct i915_vma *vma;
3475 int ret;
3476
3477 if (i915_is_ggtt(vm)) {
3478 u32 view_size;
3479
3480 if (WARN_ON(!ggtt_view))
3481 return ERR_PTR(-EINVAL);
3482
3483 view_size = i915_ggtt_view_size(obj, ggtt_view);
3484
3485 fence_size = i915_gem_get_gtt_size(dev,
3486 view_size,
3487 obj->tiling_mode);
3488 fence_alignment = i915_gem_get_gtt_alignment(dev,
3489 view_size,
3490 obj->tiling_mode,
3491 true);
3492 unfenced_alignment = i915_gem_get_gtt_alignment(dev,
3493 view_size,
3494 obj->tiling_mode,
3495 false);
3496 size = flags & PIN_MAPPABLE ? fence_size : view_size;
3497 } else {
3498 fence_size = i915_gem_get_gtt_size(dev,
3499 obj->base.size,
3500 obj->tiling_mode);
3501 fence_alignment = i915_gem_get_gtt_alignment(dev,
3502 obj->base.size,
3503 obj->tiling_mode,
3504 true);
3505 unfenced_alignment =
3506 i915_gem_get_gtt_alignment(dev,
3507 obj->base.size,
3508 obj->tiling_mode,
3509 false);
3510 size = flags & PIN_MAPPABLE ? fence_size : obj->base.size;
3511 }
3512
3513 start = flags & PIN_OFFSET_BIAS ? flags & PIN_OFFSET_MASK : 0;
3514 end = vm->total;
3515 if (flags & PIN_MAPPABLE)
3516 end = min_t(u64, end, dev_priv->gtt.mappable_end);
3517 if (flags & PIN_ZONE_4G)
3518 end = min_t(u64, end, (1ULL << 32) - PAGE_SIZE);
3519
3520 if (alignment == 0)
3521 alignment = flags & PIN_MAPPABLE ? fence_alignment :
3522 unfenced_alignment;
3523 if (flags & PIN_MAPPABLE && alignment & (fence_alignment - 1)) {
3524 DRM_DEBUG("Invalid object (view type=%u) alignment requested %u\n",
3525 ggtt_view ? ggtt_view->type : 0,
3526 alignment);
3527 return ERR_PTR(-EINVAL);
3528 }
3529
3530 /* If binding the object/GGTT view requires more space than the entire
3531 * aperture has, reject it early before evicting everything in a vain
3532 * attempt to find space.
3533 */
3534 if (size > end) {
3535 DRM_DEBUG("Attempting to bind an object (view type=%u) larger than the aperture: size=%llu > %s aperture=%llu\n",
3536 ggtt_view ? ggtt_view->type : 0,
3537 size,
3538 flags & PIN_MAPPABLE ? "mappable" : "total",
3539 end);
3540 return ERR_PTR(-E2BIG);
3541 }
3542
3543 ret = i915_gem_object_get_pages(obj);
3544 if (ret)
3545 return ERR_PTR(ret);
3546
3547 i915_gem_object_pin_pages(obj);
3548
3549 vma = ggtt_view ? i915_gem_obj_lookup_or_create_ggtt_vma(obj, ggtt_view) :
3550 i915_gem_obj_lookup_or_create_vma(obj, vm);
3551
3552 if (IS_ERR(vma))
3553 goto err_unpin;
3554
3555 if (flags & PIN_OFFSET_FIXED) {
3556 uint64_t offset = flags & PIN_OFFSET_MASK;
3557
3558 if (offset & (alignment - 1) || offset + size > end) {
3559 ret = -EINVAL;
3560 goto err_free_vma;
3561 }
3562 vma->node.start = offset;
3563 vma->node.size = size;
3564 vma->node.color = obj->cache_level;
3565 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3566 if (ret) {
3567 ret = i915_gem_evict_for_vma(vma);
3568 if (ret == 0)
3569 ret = drm_mm_reserve_node(&vm->mm, &vma->node);
3570 }
3571 if (ret)
3572 goto err_free_vma;
3573 } else {
3574 if (flags & PIN_HIGH) {
3575 search_flag = DRM_MM_SEARCH_BELOW;
3576 alloc_flag = DRM_MM_CREATE_TOP;
3577 } else {
3578 search_flag = DRM_MM_SEARCH_DEFAULT;
3579 alloc_flag = DRM_MM_CREATE_DEFAULT;
3580 }
3581
3582search_free:
3583 ret = drm_mm_insert_node_in_range_generic(&vm->mm, &vma->node,
3584 size, alignment,
3585 obj->cache_level,
3586 start, end,
3587 search_flag,
3588 alloc_flag);
3589 if (ret) {
3590 ret = i915_gem_evict_something(dev, vm, size, alignment,
3591 obj->cache_level,
3592 start, end,
3593 flags);
3594 if (ret == 0)
3595 goto search_free;
3596
3597 goto err_free_vma;
3598 }
3599 }
3600 if (WARN_ON(!i915_gem_valid_gtt_space(vma, obj->cache_level))) {
3601 ret = -EINVAL;
3602 goto err_remove_node;
3603 }
3604
3605 trace_i915_vma_bind(vma, flags);
3606 ret = i915_vma_bind(vma, obj->cache_level, flags);
3607 if (ret)
3608 goto err_remove_node;
3609
3610 list_move_tail(&obj->global_list, &dev_priv->mm.bound_list);
3611 list_add_tail(&vma->vm_link, &vm->inactive_list);
3612
3613 return vma;
3614
3615err_remove_node:
3616 drm_mm_remove_node(&vma->node);
3617err_free_vma:
3618 i915_gem_vma_destroy(vma);
3619 vma = ERR_PTR(ret);
3620err_unpin:
3621 i915_gem_object_unpin_pages(obj);
3622 return vma;
3623}
3624
3625bool
3626i915_gem_clflush_object(struct drm_i915_gem_object *obj,
3627 bool force)
3628{
3629 /* If we don't have a page list set up, then we're not pinned
3630 * to GPU, and we can ignore the cache flush because it'll happen
3631 * again at bind time.
3632 */
3633 if (obj->pages == NULL)
3634 return false;
3635
3636 /*
3637 * Stolen memory is always coherent with the GPU as it is explicitly
3638 * marked as wc by the system, or the system is cache-coherent.
3639 */
3640 if (obj->stolen || obj->phys_handle)
3641 return false;
3642
3643 /* If the GPU is snooping the contents of the CPU cache,
3644 * we do not need to manually clear the CPU cache lines. However,
3645 * the caches are only snooped when the render cache is
3646 * flushed/invalidated. As we always have to emit invalidations
3647 * and flushes when moving into and out of the RENDER domain, correct
3648 * snooping behaviour occurs naturally as the result of our domain
3649 * tracking.
3650 */
3651 if (!force && cpu_cache_is_coherent(obj->base.dev, obj->cache_level)) {
3652 obj->cache_dirty = true;
3653 return false;
3654 }
3655
3656 trace_i915_gem_object_clflush(obj);
3657 drm_clflush_sg(obj->pages);
3658 obj->cache_dirty = false;
3659
3660 return true;
3661}
3662
3663/** Flushes the GTT write domain for the object if it's dirty. */
3664static void
3665i915_gem_object_flush_gtt_write_domain(struct drm_i915_gem_object *obj)
3666{
3667 uint32_t old_write_domain;
3668
3669 if (obj->base.write_domain != I915_GEM_DOMAIN_GTT)
3670 return;
3671
3672 /* No actual flushing is required for the GTT write domain. Writes
3673 * to it immediately go to main memory as far as we know, so there's
3674 * no chipset flush. It also doesn't land in render cache.
3675 *
3676 * However, we do have to enforce the order so that all writes through
3677 * the GTT land before any writes to the device, such as updates to
3678 * the GATT itself.
3679 */
3680 wmb();
3681
3682 old_write_domain = obj->base.write_domain;
3683 obj->base.write_domain = 0;
3684
3685 intel_fb_obj_flush(obj, false, ORIGIN_GTT);
3686
3687 trace_i915_gem_object_change_domain(obj,
3688 obj->base.read_domains,
3689 old_write_domain);
3690}
3691
3692/** Flushes the CPU write domain for the object if it's dirty. */
3693static void
3694i915_gem_object_flush_cpu_write_domain(struct drm_i915_gem_object *obj)
3695{
3696 uint32_t old_write_domain;
3697
3698 if (obj->base.write_domain != I915_GEM_DOMAIN_CPU)
3699 return;
3700
3701 if (i915_gem_clflush_object(obj, obj->pin_display))
3702 i915_gem_chipset_flush(obj->base.dev);
3703
3704 old_write_domain = obj->base.write_domain;
3705 obj->base.write_domain = 0;
3706
3707 intel_fb_obj_flush(obj, false, ORIGIN_CPU);
3708
3709 trace_i915_gem_object_change_domain(obj,
3710 obj->base.read_domains,
3711 old_write_domain);
3712}
3713
3714/**
3715 * Moves a single object to the GTT read, and possibly write domain.
3716 *
3717 * This function returns when the move is complete, including waiting on
3718 * flushes to occur.
3719 */
3720int
3721i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
3722{
3723 uint32_t old_write_domain, old_read_domains;
3724 struct i915_vma *vma;
3725 int ret;
3726
3727 if (obj->base.write_domain == I915_GEM_DOMAIN_GTT)
3728 return 0;
3729
3730 ret = i915_gem_object_wait_rendering(obj, !write);
3731 if (ret)
3732 return ret;
3733
3734 /* Flush and acquire obj->pages so that we are coherent through
3735 * direct access in memory with previous cached writes through
3736 * shmemfs and that our cache domain tracking remains valid.
3737 * For example, if the obj->filp was moved to swap without us
3738 * being notified and releasing the pages, we would mistakenly
3739 * continue to assume that the obj remained out of the CPU cached
3740 * domain.
3741 */
3742 ret = i915_gem_object_get_pages(obj);
3743 if (ret)
3744 return ret;
3745
3746 i915_gem_object_flush_cpu_write_domain(obj);
3747
3748 /* Serialise direct access to this object with the barriers for
3749 * coherent writes from the GPU, by effectively invalidating the
3750 * GTT domain upon first access.
3751 */
3752 if ((obj->base.read_domains & I915_GEM_DOMAIN_GTT) == 0)
3753 mb();
3754
3755 old_write_domain = obj->base.write_domain;
3756 old_read_domains = obj->base.read_domains;
3757
3758 /* It should now be out of any other write domains, and we can update
3759 * the domain values for our changes.
3760 */
3761 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
3762 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
3763 if (write) {
3764 obj->base.read_domains = I915_GEM_DOMAIN_GTT;
3765 obj->base.write_domain = I915_GEM_DOMAIN_GTT;
3766 obj->dirty = 1;
3767 }
3768
3769 trace_i915_gem_object_change_domain(obj,
3770 old_read_domains,
3771 old_write_domain);
3772
3773 /* And bump the LRU for this access */
3774 vma = i915_gem_obj_to_ggtt(obj);
3775 if (vma && drm_mm_node_allocated(&vma->node) && !obj->active)
3776 list_move_tail(&vma->vm_link,
3777 &to_i915(obj->base.dev)->gtt.base.inactive_list);
3778
3779 return 0;
3780}
3781
3782/**
3783 * Changes the cache-level of an object across all VMA.
3784 *
3785 * After this function returns, the object will be in the new cache-level
3786 * across all GTT and the contents of the backing storage will be coherent,
3787 * with respect to the new cache-level. In order to keep the backing storage
3788 * coherent for all users, we only allow a single cache level to be set
3789 * globally on the object and prevent it from being changed whilst the
3790 * hardware is reading from the object. That is if the object is currently
3791 * on the scanout it will be set to uncached (or equivalent display
3792 * cache coherency) and all non-MOCS GPU access will also be uncached so
3793 * that all direct access to the scanout remains coherent.
3794 */
3795int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
3796 enum i915_cache_level cache_level)
3797{
3798 struct drm_device *dev = obj->base.dev;
3799 struct i915_vma *vma, *next;
3800 bool bound = false;
3801 int ret = 0;
3802
3803 if (obj->cache_level == cache_level)
3804 goto out;
3805
3806 /* Inspect the list of currently bound VMA and unbind any that would
3807 * be invalid given the new cache-level. This is principally to
3808 * catch the issue of the CS prefetch crossing page boundaries and
3809 * reading an invalid PTE on older architectures.
3810 */
3811 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
3812 if (!drm_mm_node_allocated(&vma->node))
3813 continue;
3814
3815 if (vma->pin_count) {
3816 DRM_DEBUG("can not change the cache level of pinned objects\n");
3817 return -EBUSY;
3818 }
3819
3820 if (!i915_gem_valid_gtt_space(vma, cache_level)) {
3821 ret = i915_vma_unbind(vma);
3822 if (ret)
3823 return ret;
3824 } else
3825 bound = true;
3826 }
3827
3828 /* We can reuse the existing drm_mm nodes but need to change the
3829 * cache-level on the PTE. We could simply unbind them all and
3830 * rebind with the correct cache-level on next use. However since
3831 * we already have a valid slot, dma mapping, pages etc, we may as
3832 * rewrite the PTE in the belief that doing so tramples upon less
3833 * state and so involves less work.
3834 */
3835 if (bound) {
3836 /* Before we change the PTE, the GPU must not be accessing it.
3837 * If we wait upon the object, we know that all the bound
3838 * VMA are no longer active.
3839 */
3840 ret = i915_gem_object_wait_rendering(obj, false);
3841 if (ret)
3842 return ret;
3843
3844 if (!HAS_LLC(dev) && cache_level != I915_CACHE_NONE) {
3845 /* Access to snoopable pages through the GTT is
3846 * incoherent and on some machines causes a hard
3847 * lockup. Relinquish the CPU mmaping to force
3848 * userspace to refault in the pages and we can
3849 * then double check if the GTT mapping is still
3850 * valid for that pointer access.
3851 */
3852 i915_gem_release_mmap(obj);
3853
3854 /* As we no longer need a fence for GTT access,
3855 * we can relinquish it now (and so prevent having
3856 * to steal a fence from someone else on the next
3857 * fence request). Note GPU activity would have
3858 * dropped the fence as all snoopable access is
3859 * supposed to be linear.
3860 */
3861 ret = i915_gem_object_put_fence(obj);
3862 if (ret)
3863 return ret;
3864 } else {
3865 /* We either have incoherent backing store and
3866 * so no GTT access or the architecture is fully
3867 * coherent. In such cases, existing GTT mmaps
3868 * ignore the cache bit in the PTE and we can
3869 * rewrite it without confusing the GPU or having
3870 * to force userspace to fault back in its mmaps.
3871 */
3872 }
3873
3874 list_for_each_entry(vma, &obj->vma_list, obj_link) {
3875 if (!drm_mm_node_allocated(&vma->node))
3876 continue;
3877
3878 ret = i915_vma_bind(vma, cache_level, PIN_UPDATE);
3879 if (ret)
3880 return ret;
3881 }
3882 }
3883
3884 list_for_each_entry(vma, &obj->vma_list, obj_link)
3885 vma->node.color = cache_level;
3886 obj->cache_level = cache_level;
3887
3888out:
3889 /* Flush the dirty CPU caches to the backing storage so that the
3890 * object is now coherent at its new cache level (with respect
3891 * to the access domain).
3892 */
3893 if (obj->cache_dirty &&
3894 obj->base.write_domain != I915_GEM_DOMAIN_CPU &&
3895 cpu_write_needs_clflush(obj)) {
3896 if (i915_gem_clflush_object(obj, true))
3897 i915_gem_chipset_flush(obj->base.dev);
3898 }
3899
3900 return 0;
3901}
3902
3903int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
3904 struct drm_file *file)
3905{
3906 struct drm_i915_gem_caching *args = data;
3907 struct drm_i915_gem_object *obj;
3908
3909 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3910 if (&obj->base == NULL)
3911 return -ENOENT;
3912
3913 switch (obj->cache_level) {
3914 case I915_CACHE_LLC:
3915 case I915_CACHE_L3_LLC:
3916 args->caching = I915_CACHING_CACHED;
3917 break;
3918
3919 case I915_CACHE_WT:
3920 args->caching = I915_CACHING_DISPLAY;
3921 break;
3922
3923 default:
3924 args->caching = I915_CACHING_NONE;
3925 break;
3926 }
3927
3928 drm_gem_object_unreference_unlocked(&obj->base);
3929 return 0;
3930}
3931
3932int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
3933 struct drm_file *file)
3934{
3935 struct drm_i915_private *dev_priv = dev->dev_private;
3936 struct drm_i915_gem_caching *args = data;
3937 struct drm_i915_gem_object *obj;
3938 enum i915_cache_level level;
3939 int ret;
3940
3941 switch (args->caching) {
3942 case I915_CACHING_NONE:
3943 level = I915_CACHE_NONE;
3944 break;
3945 case I915_CACHING_CACHED:
3946 /*
3947 * Due to a HW issue on BXT A stepping, GPU stores via a
3948 * snooped mapping may leave stale data in a corresponding CPU
3949 * cacheline, whereas normally such cachelines would get
3950 * invalidated.
3951 */
3952 if (IS_BXT_REVID(dev, 0, BXT_REVID_A1))
3953 return -ENODEV;
3954
3955 level = I915_CACHE_LLC;
3956 break;
3957 case I915_CACHING_DISPLAY:
3958 level = HAS_WT(dev) ? I915_CACHE_WT : I915_CACHE_NONE;
3959 break;
3960 default:
3961 return -EINVAL;
3962 }
3963
3964 intel_runtime_pm_get(dev_priv);
3965
3966 ret = i915_mutex_lock_interruptible(dev);
3967 if (ret)
3968 goto rpm_put;
3969
3970 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
3971 if (&obj->base == NULL) {
3972 ret = -ENOENT;
3973 goto unlock;
3974 }
3975
3976 ret = i915_gem_object_set_cache_level(obj, level);
3977
3978 drm_gem_object_unreference(&obj->base);
3979unlock:
3980 mutex_unlock(&dev->struct_mutex);
3981rpm_put:
3982 intel_runtime_pm_put(dev_priv);
3983
3984 return ret;
3985}
3986
3987/*
3988 * Prepare buffer for display plane (scanout, cursors, etc).
3989 * Can be called from an uninterruptible phase (modesetting) and allows
3990 * any flushes to be pipelined (for pageflips).
3991 */
3992int
3993i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
3994 u32 alignment,
3995 const struct i915_ggtt_view *view)
3996{
3997 u32 old_read_domains, old_write_domain;
3998 int ret;
3999
4000 /* Mark the pin_display early so that we account for the
4001 * display coherency whilst setting up the cache domains.
4002 */
4003 obj->pin_display++;
4004
4005 /* The display engine is not coherent with the LLC cache on gen6. As
4006 * a result, we make sure that the pinning that is about to occur is
4007 * done with uncached PTEs. This is lowest common denominator for all
4008 * chipsets.
4009 *
4010 * However for gen6+, we could do better by using the GFDT bit instead
4011 * of uncaching, which would allow us to flush all the LLC-cached data
4012 * with that bit in the PTE to main memory with just one PIPE_CONTROL.
4013 */
4014 ret = i915_gem_object_set_cache_level(obj,
4015 HAS_WT(obj->base.dev) ? I915_CACHE_WT : I915_CACHE_NONE);
4016 if (ret)
4017 goto err_unpin_display;
4018
4019 /* As the user may map the buffer once pinned in the display plane
4020 * (e.g. libkms for the bootup splash), we have to ensure that we
4021 * always use map_and_fenceable for all scanout buffers.
4022 */
4023 ret = i915_gem_object_ggtt_pin(obj, view, alignment,
4024 view->type == I915_GGTT_VIEW_NORMAL ?
4025 PIN_MAPPABLE : 0);
4026 if (ret)
4027 goto err_unpin_display;
4028
4029 i915_gem_object_flush_cpu_write_domain(obj);
4030
4031 old_write_domain = obj->base.write_domain;
4032 old_read_domains = obj->base.read_domains;
4033
4034 /* It should now be out of any other write domains, and we can update
4035 * the domain values for our changes.
4036 */
4037 obj->base.write_domain = 0;
4038 obj->base.read_domains |= I915_GEM_DOMAIN_GTT;
4039
4040 trace_i915_gem_object_change_domain(obj,
4041 old_read_domains,
4042 old_write_domain);
4043
4044 return 0;
4045
4046err_unpin_display:
4047 obj->pin_display--;
4048 return ret;
4049}
4050
4051void
4052i915_gem_object_unpin_from_display_plane(struct drm_i915_gem_object *obj,
4053 const struct i915_ggtt_view *view)
4054{
4055 if (WARN_ON(obj->pin_display == 0))
4056 return;
4057
4058 i915_gem_object_ggtt_unpin_view(obj, view);
4059
4060 obj->pin_display--;
4061}
4062
4063/**
4064 * Moves a single object to the CPU read, and possibly write domain.
4065 *
4066 * This function returns when the move is complete, including waiting on
4067 * flushes to occur.
4068 */
4069int
4070i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
4071{
4072 uint32_t old_write_domain, old_read_domains;
4073 int ret;
4074
4075 if (obj->base.write_domain == I915_GEM_DOMAIN_CPU)
4076 return 0;
4077
4078 ret = i915_gem_object_wait_rendering(obj, !write);
4079 if (ret)
4080 return ret;
4081
4082 i915_gem_object_flush_gtt_write_domain(obj);
4083
4084 old_write_domain = obj->base.write_domain;
4085 old_read_domains = obj->base.read_domains;
4086
4087 /* Flush the CPU cache if it's still invalid. */
4088 if ((obj->base.read_domains & I915_GEM_DOMAIN_CPU) == 0) {
4089 i915_gem_clflush_object(obj, false);
4090
4091 obj->base.read_domains |= I915_GEM_DOMAIN_CPU;
4092 }
4093
4094 /* It should now be out of any other write domains, and we can update
4095 * the domain values for our changes.
4096 */
4097 BUG_ON((obj->base.write_domain & ~I915_GEM_DOMAIN_CPU) != 0);
4098
4099 /* If we're writing through the CPU, then the GPU read domains will
4100 * need to be invalidated at next use.
4101 */
4102 if (write) {
4103 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4104 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4105 }
4106
4107 trace_i915_gem_object_change_domain(obj,
4108 old_read_domains,
4109 old_write_domain);
4110
4111 return 0;
4112}
4113
4114/* Throttle our rendering by waiting until the ring has completed our requests
4115 * emitted over 20 msec ago.
4116 *
4117 * Note that if we were to use the current jiffies each time around the loop,
4118 * we wouldn't escape the function with any frames outstanding if the time to
4119 * render a frame was over 20ms.
4120 *
4121 * This should get us reasonable parallelism between CPU and GPU but also
4122 * relatively low latency when blocking on a particular request to finish.
4123 */
4124static int
4125i915_gem_ring_throttle(struct drm_device *dev, struct drm_file *file)
4126{
4127 struct drm_i915_private *dev_priv = dev->dev_private;
4128 struct drm_i915_file_private *file_priv = file->driver_priv;
4129 unsigned long recent_enough = jiffies - DRM_I915_THROTTLE_JIFFIES;
4130 struct drm_i915_gem_request *request, *target = NULL;
4131 unsigned reset_counter;
4132 int ret;
4133
4134 ret = i915_gem_wait_for_error(&dev_priv->gpu_error);
4135 if (ret)
4136 return ret;
4137
4138 ret = i915_gem_check_wedge(&dev_priv->gpu_error, false);
4139 if (ret)
4140 return ret;
4141
4142 spin_lock(&file_priv->mm.lock);
4143 list_for_each_entry(request, &file_priv->mm.request_list, client_list) {
4144 if (time_after_eq(request->emitted_jiffies, recent_enough))
4145 break;
4146
4147 /*
4148 * Note that the request might not have been submitted yet.
4149 * In which case emitted_jiffies will be zero.
4150 */
4151 if (!request->emitted_jiffies)
4152 continue;
4153
4154 target = request;
4155 }
4156 reset_counter = atomic_read(&dev_priv->gpu_error.reset_counter);
4157 if (target)
4158 i915_gem_request_reference(target);
4159 spin_unlock(&file_priv->mm.lock);
4160
4161 if (target == NULL)
4162 return 0;
4163
4164 ret = __i915_wait_request(target, reset_counter, true, NULL, NULL);
4165 if (ret == 0)
4166 queue_delayed_work(dev_priv->wq, &dev_priv->mm.retire_work, 0);
4167
4168 i915_gem_request_unreference__unlocked(target);
4169
4170 return ret;
4171}
4172
4173static bool
4174i915_vma_misplaced(struct i915_vma *vma, uint32_t alignment, uint64_t flags)
4175{
4176 struct drm_i915_gem_object *obj = vma->obj;
4177
4178 if (alignment &&
4179 vma->node.start & (alignment - 1))
4180 return true;
4181
4182 if (flags & PIN_MAPPABLE && !obj->map_and_fenceable)
4183 return true;
4184
4185 if (flags & PIN_OFFSET_BIAS &&
4186 vma->node.start < (flags & PIN_OFFSET_MASK))
4187 return true;
4188
4189 if (flags & PIN_OFFSET_FIXED &&
4190 vma->node.start != (flags & PIN_OFFSET_MASK))
4191 return true;
4192
4193 return false;
4194}
4195
4196void __i915_vma_set_map_and_fenceable(struct i915_vma *vma)
4197{
4198 struct drm_i915_gem_object *obj = vma->obj;
4199 bool mappable, fenceable;
4200 u32 fence_size, fence_alignment;
4201
4202 fence_size = i915_gem_get_gtt_size(obj->base.dev,
4203 obj->base.size,
4204 obj->tiling_mode);
4205 fence_alignment = i915_gem_get_gtt_alignment(obj->base.dev,
4206 obj->base.size,
4207 obj->tiling_mode,
4208 true);
4209
4210 fenceable = (vma->node.size == fence_size &&
4211 (vma->node.start & (fence_alignment - 1)) == 0);
4212
4213 mappable = (vma->node.start + fence_size <=
4214 to_i915(obj->base.dev)->gtt.mappable_end);
4215
4216 obj->map_and_fenceable = mappable && fenceable;
4217}
4218
4219static int
4220i915_gem_object_do_pin(struct drm_i915_gem_object *obj,
4221 struct i915_address_space *vm,
4222 const struct i915_ggtt_view *ggtt_view,
4223 uint32_t alignment,
4224 uint64_t flags)
4225{
4226 struct drm_i915_private *dev_priv = obj->base.dev->dev_private;
4227 struct i915_vma *vma;
4228 unsigned bound;
4229 int ret;
4230
4231 if (WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base))
4232 return -ENODEV;
4233
4234 if (WARN_ON(flags & (PIN_GLOBAL | PIN_MAPPABLE) && !i915_is_ggtt(vm)))
4235 return -EINVAL;
4236
4237 if (WARN_ON((flags & (PIN_MAPPABLE | PIN_GLOBAL)) == PIN_MAPPABLE))
4238 return -EINVAL;
4239
4240 if (WARN_ON(i915_is_ggtt(vm) != !!ggtt_view))
4241 return -EINVAL;
4242
4243 vma = ggtt_view ? i915_gem_obj_to_ggtt_view(obj, ggtt_view) :
4244 i915_gem_obj_to_vma(obj, vm);
4245
4246 if (IS_ERR(vma))
4247 return PTR_ERR(vma);
4248
4249 if (vma) {
4250 if (WARN_ON(vma->pin_count == DRM_I915_GEM_OBJECT_MAX_PIN_COUNT))
4251 return -EBUSY;
4252
4253 if (i915_vma_misplaced(vma, alignment, flags)) {
4254 WARN(vma->pin_count,
4255 "bo is already pinned in %s with incorrect alignment:"
4256 " offset=%08x %08x, req.alignment=%x, req.map_and_fenceable=%d,"
4257 " obj->map_and_fenceable=%d\n",
4258 ggtt_view ? "ggtt" : "ppgtt",
4259 upper_32_bits(vma->node.start),
4260 lower_32_bits(vma->node.start),
4261 alignment,
4262 !!(flags & PIN_MAPPABLE),
4263 obj->map_and_fenceable);
4264 ret = i915_vma_unbind(vma);
4265 if (ret)
4266 return ret;
4267
4268 vma = NULL;
4269 }
4270 }
4271
4272 bound = vma ? vma->bound : 0;
4273 if (vma == NULL || !drm_mm_node_allocated(&vma->node)) {
4274 vma = i915_gem_object_bind_to_vm(obj, vm, ggtt_view, alignment,
4275 flags);
4276 if (IS_ERR(vma))
4277 return PTR_ERR(vma);
4278 } else {
4279 ret = i915_vma_bind(vma, obj->cache_level, flags);
4280 if (ret)
4281 return ret;
4282 }
4283
4284 if (ggtt_view && ggtt_view->type == I915_GGTT_VIEW_NORMAL &&
4285 (bound ^ vma->bound) & GLOBAL_BIND) {
4286 __i915_vma_set_map_and_fenceable(vma);
4287 WARN_ON(flags & PIN_MAPPABLE && !obj->map_and_fenceable);
4288 }
4289
4290 vma->pin_count++;
4291 return 0;
4292}
4293
4294int
4295i915_gem_object_pin(struct drm_i915_gem_object *obj,
4296 struct i915_address_space *vm,
4297 uint32_t alignment,
4298 uint64_t flags)
4299{
4300 return i915_gem_object_do_pin(obj, vm,
4301 i915_is_ggtt(vm) ? &i915_ggtt_view_normal : NULL,
4302 alignment, flags);
4303}
4304
4305int
4306i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
4307 const struct i915_ggtt_view *view,
4308 uint32_t alignment,
4309 uint64_t flags)
4310{
4311 if (WARN_ONCE(!view, "no view specified"))
4312 return -EINVAL;
4313
4314 return i915_gem_object_do_pin(obj, i915_obj_to_ggtt(obj), view,
4315 alignment, flags | PIN_GLOBAL);
4316}
4317
4318void
4319i915_gem_object_ggtt_unpin_view(struct drm_i915_gem_object *obj,
4320 const struct i915_ggtt_view *view)
4321{
4322 struct i915_vma *vma = i915_gem_obj_to_ggtt_view(obj, view);
4323
4324 BUG_ON(!vma);
4325 WARN_ON(vma->pin_count == 0);
4326 WARN_ON(!i915_gem_obj_ggtt_bound_view(obj, view));
4327
4328 --vma->pin_count;
4329}
4330
4331int
4332i915_gem_busy_ioctl(struct drm_device *dev, void *data,
4333 struct drm_file *file)
4334{
4335 struct drm_i915_gem_busy *args = data;
4336 struct drm_i915_gem_object *obj;
4337 int ret;
4338
4339 ret = i915_mutex_lock_interruptible(dev);
4340 if (ret)
4341 return ret;
4342
4343 obj = to_intel_bo(drm_gem_object_lookup(dev, file, args->handle));
4344 if (&obj->base == NULL) {
4345 ret = -ENOENT;
4346 goto unlock;
4347 }
4348
4349 /* Count all active objects as busy, even if they are currently not used
4350 * by the gpu. Users of this interface expect objects to eventually
4351 * become non-busy without any further actions, therefore emit any
4352 * necessary flushes here.
4353 */
4354 ret = i915_gem_object_flush_active(obj);
4355 if (ret)
4356 goto unref;
4357
4358 args->busy = 0;
4359 if (obj->active) {
4360 int i;
4361
4362 for (i = 0; i < I915_NUM_RINGS; i++) {
4363 struct drm_i915_gem_request *req;
4364
4365 req = obj->last_read_req[i];
4366 if (req)
4367 args->busy |= 1 << (16 + req->ring->exec_id);
4368 }
4369 if (obj->last_write_req)
4370 args->busy |= obj->last_write_req->ring->exec_id;
4371 }
4372
4373unref:
4374 drm_gem_object_unreference(&obj->base);
4375unlock:
4376 mutex_unlock(&dev->struct_mutex);
4377 return ret;
4378}
4379
4380int
4381i915_gem_throttle_ioctl(struct drm_device *dev, void *data,
4382 struct drm_file *file_priv)
4383{
4384 return i915_gem_ring_throttle(dev, file_priv);
4385}
4386
4387int
4388i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
4389 struct drm_file *file_priv)
4390{
4391 struct drm_i915_private *dev_priv = dev->dev_private;
4392 struct drm_i915_gem_madvise *args = data;
4393 struct drm_i915_gem_object *obj;
4394 int ret;
4395
4396 switch (args->madv) {
4397 case I915_MADV_DONTNEED:
4398 case I915_MADV_WILLNEED:
4399 break;
4400 default:
4401 return -EINVAL;
4402 }
4403
4404 ret = i915_mutex_lock_interruptible(dev);
4405 if (ret)
4406 return ret;
4407
4408 obj = to_intel_bo(drm_gem_object_lookup(dev, file_priv, args->handle));
4409 if (&obj->base == NULL) {
4410 ret = -ENOENT;
4411 goto unlock;
4412 }
4413
4414 if (i915_gem_obj_is_pinned(obj)) {
4415 ret = -EINVAL;
4416 goto out;
4417 }
4418
4419 if (obj->pages &&
4420 obj->tiling_mode != I915_TILING_NONE &&
4421 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
4422 if (obj->madv == I915_MADV_WILLNEED)
4423 i915_gem_object_unpin_pages(obj);
4424 if (args->madv == I915_MADV_WILLNEED)
4425 i915_gem_object_pin_pages(obj);
4426 }
4427
4428 if (obj->madv != __I915_MADV_PURGED)
4429 obj->madv = args->madv;
4430
4431 /* if the object is no longer attached, discard its backing storage */
4432 if (obj->madv == I915_MADV_DONTNEED && obj->pages == NULL)
4433 i915_gem_object_truncate(obj);
4434
4435 args->retained = obj->madv != __I915_MADV_PURGED;
4436
4437out:
4438 drm_gem_object_unreference(&obj->base);
4439unlock:
4440 mutex_unlock(&dev->struct_mutex);
4441 return ret;
4442}
4443
4444void i915_gem_object_init(struct drm_i915_gem_object *obj,
4445 const struct drm_i915_gem_object_ops *ops)
4446{
4447 int i;
4448
4449 INIT_LIST_HEAD(&obj->global_list);
4450 for (i = 0; i < I915_NUM_RINGS; i++)
4451 INIT_LIST_HEAD(&obj->ring_list[i]);
4452 INIT_LIST_HEAD(&obj->obj_exec_link);
4453 INIT_LIST_HEAD(&obj->vma_list);
4454 INIT_LIST_HEAD(&obj->batch_pool_link);
4455
4456 obj->ops = ops;
4457
4458 obj->fence_reg = I915_FENCE_REG_NONE;
4459 obj->madv = I915_MADV_WILLNEED;
4460
4461 i915_gem_info_add_obj(obj->base.dev->dev_private, obj->base.size);
4462}
4463
4464static const struct drm_i915_gem_object_ops i915_gem_object_ops = {
4465 .flags = I915_GEM_OBJECT_HAS_STRUCT_PAGE,
4466 .get_pages = i915_gem_object_get_pages_gtt,
4467 .put_pages = i915_gem_object_put_pages_gtt,
4468};
4469
4470struct drm_i915_gem_object *i915_gem_alloc_object(struct drm_device *dev,
4471 size_t size)
4472{
4473 struct drm_i915_gem_object *obj;
4474 struct address_space *mapping;
4475 gfp_t mask;
4476
4477 obj = i915_gem_object_alloc(dev);
4478 if (obj == NULL)
4479 return NULL;
4480
4481 if (drm_gem_object_init(dev, &obj->base, size) != 0) {
4482 i915_gem_object_free(obj);
4483 return NULL;
4484 }
4485
4486 mask = GFP_HIGHUSER | __GFP_RECLAIMABLE;
4487 if (IS_CRESTLINE(dev) || IS_BROADWATER(dev)) {
4488 /* 965gm cannot relocate objects above 4GiB. */
4489 mask &= ~__GFP_HIGHMEM;
4490 mask |= __GFP_DMA32;
4491 }
4492
4493 mapping = file_inode(obj->base.filp)->i_mapping;
4494 mapping_set_gfp_mask(mapping, mask);
4495
4496 i915_gem_object_init(obj, &i915_gem_object_ops);
4497
4498 obj->base.write_domain = I915_GEM_DOMAIN_CPU;
4499 obj->base.read_domains = I915_GEM_DOMAIN_CPU;
4500
4501 if (HAS_LLC(dev)) {
4502 /* On some devices, we can have the GPU use the LLC (the CPU
4503 * cache) for about a 10% performance improvement
4504 * compared to uncached. Graphics requests other than
4505 * display scanout are coherent with the CPU in
4506 * accessing this cache. This means in this mode we
4507 * don't need to clflush on the CPU side, and on the
4508 * GPU side we only need to flush internal caches to
4509 * get data visible to the CPU.
4510 *
4511 * However, we maintain the display planes as UC, and so
4512 * need to rebind when first used as such.
4513 */
4514 obj->cache_level = I915_CACHE_LLC;
4515 } else
4516 obj->cache_level = I915_CACHE_NONE;
4517
4518 trace_i915_gem_object_create(obj);
4519
4520 return obj;
4521}
4522
4523static bool discard_backing_storage(struct drm_i915_gem_object *obj)
4524{
4525 /* If we are the last user of the backing storage (be it shmemfs
4526 * pages or stolen etc), we know that the pages are going to be
4527 * immediately released. In this case, we can then skip copying
4528 * back the contents from the GPU.
4529 */
4530
4531 if (obj->madv != I915_MADV_WILLNEED)
4532 return false;
4533
4534 if (obj->base.filp == NULL)
4535 return true;
4536
4537 /* At first glance, this looks racy, but then again so would be
4538 * userspace racing mmap against close. However, the first external
4539 * reference to the filp can only be obtained through the
4540 * i915_gem_mmap_ioctl() which safeguards us against the user
4541 * acquiring such a reference whilst we are in the middle of
4542 * freeing the object.
4543 */
4544 return atomic_long_read(&obj->base.filp->f_count) == 1;
4545}
4546
4547void i915_gem_free_object(struct drm_gem_object *gem_obj)
4548{
4549 struct drm_i915_gem_object *obj = to_intel_bo(gem_obj);
4550 struct drm_device *dev = obj->base.dev;
4551 struct drm_i915_private *dev_priv = dev->dev_private;
4552 struct i915_vma *vma, *next;
4553
4554 intel_runtime_pm_get(dev_priv);
4555
4556 trace_i915_gem_object_destroy(obj);
4557
4558 list_for_each_entry_safe(vma, next, &obj->vma_list, obj_link) {
4559 int ret;
4560
4561 vma->pin_count = 0;
4562 ret = i915_vma_unbind(vma);
4563 if (WARN_ON(ret == -ERESTARTSYS)) {
4564 bool was_interruptible;
4565
4566 was_interruptible = dev_priv->mm.interruptible;
4567 dev_priv->mm.interruptible = false;
4568
4569 WARN_ON(i915_vma_unbind(vma));
4570
4571 dev_priv->mm.interruptible = was_interruptible;
4572 }
4573 }
4574
4575 /* Stolen objects don't hold a ref, but do hold pin count. Fix that up
4576 * before progressing. */
4577 if (obj->stolen)
4578 i915_gem_object_unpin_pages(obj);
4579
4580 WARN_ON(obj->frontbuffer_bits);
4581
4582 if (obj->pages && obj->madv == I915_MADV_WILLNEED &&
4583 dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES &&
4584 obj->tiling_mode != I915_TILING_NONE)
4585 i915_gem_object_unpin_pages(obj);
4586
4587 if (WARN_ON(obj->pages_pin_count))
4588 obj->pages_pin_count = 0;
4589 if (discard_backing_storage(obj))
4590 obj->madv = I915_MADV_DONTNEED;
4591 i915_gem_object_put_pages(obj);
4592 i915_gem_object_free_mmap_offset(obj);
4593
4594 BUG_ON(obj->pages);
4595
4596 if (obj->base.import_attach)
4597 drm_prime_gem_destroy(&obj->base, NULL);
4598
4599 if (obj->ops->release)
4600 obj->ops->release(obj);
4601
4602 drm_gem_object_release(&obj->base);
4603 i915_gem_info_remove_obj(dev_priv, obj->base.size);
4604
4605 kfree(obj->bit_17);
4606 i915_gem_object_free(obj);
4607
4608 intel_runtime_pm_put(dev_priv);
4609}
4610
4611struct i915_vma *i915_gem_obj_to_vma(struct drm_i915_gem_object *obj,
4612 struct i915_address_space *vm)
4613{
4614 struct i915_vma *vma;
4615 list_for_each_entry(vma, &obj->vma_list, obj_link) {
4616 if (vma->ggtt_view.type == I915_GGTT_VIEW_NORMAL &&
4617 vma->vm == vm)
4618 return vma;
4619 }
4620 return NULL;
4621}
4622
4623struct i915_vma *i915_gem_obj_to_ggtt_view(struct drm_i915_gem_object *obj,
4624 const struct i915_ggtt_view *view)
4625{
4626 struct i915_address_space *ggtt = i915_obj_to_ggtt(obj);
4627 struct i915_vma *vma;
4628
4629 if (WARN_ONCE(!view, "no view specified"))
4630 return ERR_PTR(-EINVAL);
4631
4632 list_for_each_entry(vma, &obj->vma_list, obj_link)
4633 if (vma->vm == ggtt &&
4634 i915_ggtt_view_equal(&vma->ggtt_view, view))
4635 return vma;
4636 return NULL;
4637}
4638
4639void i915_gem_vma_destroy(struct i915_vma *vma)
4640{
4641 WARN_ON(vma->node.allocated);
4642
4643 /* Keep the vma as a placeholder in the execbuffer reservation lists */
4644 if (!list_empty(&vma->exec_list))
4645 return;
4646
4647 if (!vma->is_ggtt)
4648 i915_ppgtt_put(i915_vm_to_ppgtt(vma->vm));
4649
4650 list_del(&vma->obj_link);
4651
4652 kmem_cache_free(to_i915(vma->obj->base.dev)->vmas, vma);
4653}
4654
4655static void
4656i915_gem_stop_ringbuffers(struct drm_device *dev)
4657{
4658 struct drm_i915_private *dev_priv = dev->dev_private;
4659 struct intel_engine_cs *ring;
4660 int i;
4661
4662 for_each_ring(ring, dev_priv, i)
4663 dev_priv->gt.stop_ring(ring);
4664}
4665
4666int
4667i915_gem_suspend(struct drm_device *dev)
4668{
4669 struct drm_i915_private *dev_priv = dev->dev_private;
4670 int ret = 0;
4671
4672 mutex_lock(&dev->struct_mutex);
4673 ret = i915_gpu_idle(dev);
4674 if (ret)
4675 goto err;
4676
4677 i915_gem_retire_requests(dev);
4678
4679 i915_gem_stop_ringbuffers(dev);
4680 mutex_unlock(&dev->struct_mutex);
4681
4682 cancel_delayed_work_sync(&dev_priv->gpu_error.hangcheck_work);
4683 cancel_delayed_work_sync(&dev_priv->mm.retire_work);
4684 flush_delayed_work(&dev_priv->mm.idle_work);
4685
4686 /* Assert that we sucessfully flushed all the work and
4687 * reset the GPU back to its idle, low power state.
4688 */
4689 WARN_ON(dev_priv->mm.busy);
4690
4691 return 0;
4692
4693err:
4694 mutex_unlock(&dev->struct_mutex);
4695 return ret;
4696}
4697
4698int i915_gem_l3_remap(struct drm_i915_gem_request *req, int slice)
4699{
4700 struct intel_engine_cs *ring = req->ring;
4701 struct drm_device *dev = ring->dev;
4702 struct drm_i915_private *dev_priv = dev->dev_private;
4703 u32 *remap_info = dev_priv->l3_parity.remap_info[slice];
4704 int i, ret;
4705
4706 if (!HAS_L3_DPF(dev) || !remap_info)
4707 return 0;
4708
4709 ret = intel_ring_begin(req, GEN7_L3LOG_SIZE / 4 * 3);
4710 if (ret)
4711 return ret;
4712
4713 /*
4714 * Note: We do not worry about the concurrent register cacheline hang
4715 * here because no other code should access these registers other than
4716 * at initialization time.
4717 */
4718 for (i = 0; i < GEN7_L3LOG_SIZE / 4; i++) {
4719 intel_ring_emit(ring, MI_LOAD_REGISTER_IMM(1));
4720 intel_ring_emit_reg(ring, GEN7_L3LOG(slice, i));
4721 intel_ring_emit(ring, remap_info[i]);
4722 }
4723
4724 intel_ring_advance(ring);
4725
4726 return ret;
4727}
4728
4729void i915_gem_init_swizzling(struct drm_device *dev)
4730{
4731 struct drm_i915_private *dev_priv = dev->dev_private;
4732
4733 if (INTEL_INFO(dev)->gen < 5 ||
4734 dev_priv->mm.bit_6_swizzle_x == I915_BIT_6_SWIZZLE_NONE)
4735 return;
4736
4737 I915_WRITE(DISP_ARB_CTL, I915_READ(DISP_ARB_CTL) |
4738 DISP_TILE_SURFACE_SWIZZLING);
4739
4740 if (IS_GEN5(dev))
4741 return;
4742
4743 I915_WRITE(TILECTL, I915_READ(TILECTL) | TILECTL_SWZCTL);
4744 if (IS_GEN6(dev))
4745 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_SNB));
4746 else if (IS_GEN7(dev))
4747 I915_WRITE(ARB_MODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_IVB));
4748 else if (IS_GEN8(dev))
4749 I915_WRITE(GAMTARBMODE, _MASKED_BIT_ENABLE(ARB_MODE_SWIZZLE_BDW));
4750 else
4751 BUG();
4752}
4753
4754static void init_unused_ring(struct drm_device *dev, u32 base)
4755{
4756 struct drm_i915_private *dev_priv = dev->dev_private;
4757
4758 I915_WRITE(RING_CTL(base), 0);
4759 I915_WRITE(RING_HEAD(base), 0);
4760 I915_WRITE(RING_TAIL(base), 0);
4761 I915_WRITE(RING_START(base), 0);
4762}
4763
4764static void init_unused_rings(struct drm_device *dev)
4765{
4766 if (IS_I830(dev)) {
4767 init_unused_ring(dev, PRB1_BASE);
4768 init_unused_ring(dev, SRB0_BASE);
4769 init_unused_ring(dev, SRB1_BASE);
4770 init_unused_ring(dev, SRB2_BASE);
4771 init_unused_ring(dev, SRB3_BASE);
4772 } else if (IS_GEN2(dev)) {
4773 init_unused_ring(dev, SRB0_BASE);
4774 init_unused_ring(dev, SRB1_BASE);
4775 } else if (IS_GEN3(dev)) {
4776 init_unused_ring(dev, PRB1_BASE);
4777 init_unused_ring(dev, PRB2_BASE);
4778 }
4779}
4780
4781int i915_gem_init_rings(struct drm_device *dev)
4782{
4783 struct drm_i915_private *dev_priv = dev->dev_private;
4784 int ret;
4785
4786 ret = intel_init_render_ring_buffer(dev);
4787 if (ret)
4788 return ret;
4789
4790 if (HAS_BSD(dev)) {
4791 ret = intel_init_bsd_ring_buffer(dev);
4792 if (ret)
4793 goto cleanup_render_ring;
4794 }
4795
4796 if (HAS_BLT(dev)) {
4797 ret = intel_init_blt_ring_buffer(dev);
4798 if (ret)
4799 goto cleanup_bsd_ring;
4800 }
4801
4802 if (HAS_VEBOX(dev)) {
4803 ret = intel_init_vebox_ring_buffer(dev);
4804 if (ret)
4805 goto cleanup_blt_ring;
4806 }
4807
4808 if (HAS_BSD2(dev)) {
4809 ret = intel_init_bsd2_ring_buffer(dev);
4810 if (ret)
4811 goto cleanup_vebox_ring;
4812 }
4813
4814 return 0;
4815
4816cleanup_vebox_ring:
4817 intel_cleanup_ring_buffer(&dev_priv->ring[VECS]);
4818cleanup_blt_ring:
4819 intel_cleanup_ring_buffer(&dev_priv->ring[BCS]);
4820cleanup_bsd_ring:
4821 intel_cleanup_ring_buffer(&dev_priv->ring[VCS]);
4822cleanup_render_ring:
4823 intel_cleanup_ring_buffer(&dev_priv->ring[RCS]);
4824
4825 return ret;
4826}
4827
4828int
4829i915_gem_init_hw(struct drm_device *dev)
4830{
4831 struct drm_i915_private *dev_priv = dev->dev_private;
4832 struct intel_engine_cs *ring;
4833 int ret, i, j;
4834
4835 if (INTEL_INFO(dev)->gen < 6 && !intel_enable_gtt())
4836 return -EIO;
4837
4838 /* Double layer security blanket, see i915_gem_init() */
4839 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4840
4841 if (dev_priv->ellc_size)
4842 I915_WRITE(HSW_IDICR, I915_READ(HSW_IDICR) | IDIHASHMSK(0xf));
4843
4844 if (IS_HASWELL(dev))
4845 I915_WRITE(MI_PREDICATE_RESULT_2, IS_HSW_GT3(dev) ?
4846 LOWER_SLICE_ENABLED : LOWER_SLICE_DISABLED);
4847
4848 if (HAS_PCH_NOP(dev)) {
4849 if (IS_IVYBRIDGE(dev)) {
4850 u32 temp = I915_READ(GEN7_MSG_CTL);
4851 temp &= ~(WAIT_FOR_PCH_FLR_ACK | WAIT_FOR_PCH_RESET_ACK);
4852 I915_WRITE(GEN7_MSG_CTL, temp);
4853 } else if (INTEL_INFO(dev)->gen >= 7) {
4854 u32 temp = I915_READ(HSW_NDE_RSTWRN_OPT);
4855 temp &= ~RESET_PCH_HANDSHAKE_ENABLE;
4856 I915_WRITE(HSW_NDE_RSTWRN_OPT, temp);
4857 }
4858 }
4859
4860 i915_gem_init_swizzling(dev);
4861
4862 /*
4863 * At least 830 can leave some of the unused rings
4864 * "active" (ie. head != tail) after resume which
4865 * will prevent c3 entry. Makes sure all unused rings
4866 * are totally idle.
4867 */
4868 init_unused_rings(dev);
4869
4870 BUG_ON(!dev_priv->kernel_context);
4871
4872 ret = i915_ppgtt_init_hw(dev);
4873 if (ret) {
4874 DRM_ERROR("PPGTT enable HW failed %d\n", ret);
4875 goto out;
4876 }
4877
4878 /* Need to do basic initialisation of all rings first: */
4879 for_each_ring(ring, dev_priv, i) {
4880 ret = ring->init_hw(ring);
4881 if (ret)
4882 goto out;
4883 }
4884
4885 /* We can't enable contexts until all firmware is loaded */
4886 if (HAS_GUC_UCODE(dev)) {
4887 ret = intel_guc_ucode_load(dev);
4888 if (ret) {
4889 DRM_ERROR("Failed to initialize GuC, error %d\n", ret);
4890 ret = -EIO;
4891 goto out;
4892 }
4893 }
4894
4895 /*
4896 * Increment the next seqno by 0x100 so we have a visible break
4897 * on re-initialisation
4898 */
4899 ret = i915_gem_set_seqno(dev, dev_priv->next_seqno+0x100);
4900 if (ret)
4901 goto out;
4902
4903 /* Now it is safe to go back round and do everything else: */
4904 for_each_ring(ring, dev_priv, i) {
4905 struct drm_i915_gem_request *req;
4906
4907 req = i915_gem_request_alloc(ring, NULL);
4908 if (IS_ERR(req)) {
4909 ret = PTR_ERR(req);
4910 i915_gem_cleanup_ringbuffer(dev);
4911 goto out;
4912 }
4913
4914 if (ring->id == RCS) {
4915 for (j = 0; j < NUM_L3_SLICES(dev); j++)
4916 i915_gem_l3_remap(req, j);
4917 }
4918
4919 ret = i915_ppgtt_init_ring(req);
4920 if (ret && ret != -EIO) {
4921 DRM_ERROR("PPGTT enable ring #%d failed %d\n", i, ret);
4922 i915_gem_request_cancel(req);
4923 i915_gem_cleanup_ringbuffer(dev);
4924 goto out;
4925 }
4926
4927 ret = i915_gem_context_enable(req);
4928 if (ret && ret != -EIO) {
4929 DRM_ERROR("Context enable ring #%d failed %d\n", i, ret);
4930 i915_gem_request_cancel(req);
4931 i915_gem_cleanup_ringbuffer(dev);
4932 goto out;
4933 }
4934
4935 i915_add_request_no_flush(req);
4936 }
4937
4938out:
4939 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
4940 return ret;
4941}
4942
4943int i915_gem_init(struct drm_device *dev)
4944{
4945 struct drm_i915_private *dev_priv = dev->dev_private;
4946 int ret;
4947
4948 i915.enable_execlists = intel_sanitize_enable_execlists(dev,
4949 i915.enable_execlists);
4950
4951 mutex_lock(&dev->struct_mutex);
4952
4953 if (!i915.enable_execlists) {
4954 dev_priv->gt.execbuf_submit = i915_gem_ringbuffer_submission;
4955 dev_priv->gt.init_rings = i915_gem_init_rings;
4956 dev_priv->gt.cleanup_ring = intel_cleanup_ring_buffer;
4957 dev_priv->gt.stop_ring = intel_stop_ring_buffer;
4958 } else {
4959 dev_priv->gt.execbuf_submit = intel_execlists_submission;
4960 dev_priv->gt.init_rings = intel_logical_rings_init;
4961 dev_priv->gt.cleanup_ring = intel_logical_ring_cleanup;
4962 dev_priv->gt.stop_ring = intel_logical_ring_stop;
4963 }
4964
4965 /* This is just a security blanket to placate dragons.
4966 * On some systems, we very sporadically observe that the first TLBs
4967 * used by the CS may be stale, despite us poking the TLB reset. If
4968 * we hold the forcewake during initialisation these problems
4969 * just magically go away.
4970 */
4971 intel_uncore_forcewake_get(dev_priv, FORCEWAKE_ALL);
4972
4973 ret = i915_gem_init_userptr(dev);
4974 if (ret)
4975 goto out_unlock;
4976
4977 i915_gem_init_global_gtt(dev);
4978
4979 ret = i915_gem_context_init(dev);
4980 if (ret)
4981 goto out_unlock;
4982
4983 ret = dev_priv->gt.init_rings(dev);
4984 if (ret)
4985 goto out_unlock;
4986
4987 ret = i915_gem_init_hw(dev);
4988 if (ret == -EIO) {
4989 /* Allow ring initialisation to fail by marking the GPU as
4990 * wedged. But we only want to do this where the GPU is angry,
4991 * for all other failure, such as an allocation failure, bail.
4992 */
4993 DRM_ERROR("Failed to initialize GPU, declaring it wedged\n");
4994 atomic_or(I915_WEDGED, &dev_priv->gpu_error.reset_counter);
4995 ret = 0;
4996 }
4997
4998out_unlock:
4999 intel_uncore_forcewake_put(dev_priv, FORCEWAKE_ALL);
5000 mutex_unlock(&dev->struct_mutex);
5001
5002 return ret;
5003}
5004
5005void
5006i915_gem_cleanup_ringbuffer(struct drm_device *dev)
5007{
5008 struct drm_i915_private *dev_priv = dev->dev_private;
5009 struct intel_engine_cs *ring;
5010 int i;
5011
5012 for_each_ring(ring, dev_priv, i)
5013 dev_priv->gt.cleanup_ring(ring);
5014
5015 if (i915.enable_execlists)
5016 /*
5017 * Neither the BIOS, ourselves or any other kernel
5018 * expects the system to be in execlists mode on startup,
5019 * so we need to reset the GPU back to legacy mode.
5020 */
5021 intel_gpu_reset(dev);
5022}
5023
5024static void
5025init_ring_lists(struct intel_engine_cs *ring)
5026{
5027 INIT_LIST_HEAD(&ring->active_list);
5028 INIT_LIST_HEAD(&ring->request_list);
5029}
5030
5031void
5032i915_gem_load_init(struct drm_device *dev)
5033{
5034 struct drm_i915_private *dev_priv = dev->dev_private;
5035 int i;
5036
5037 dev_priv->objects =
5038 kmem_cache_create("i915_gem_object",
5039 sizeof(struct drm_i915_gem_object), 0,
5040 SLAB_HWCACHE_ALIGN,
5041 NULL);
5042 dev_priv->vmas =
5043 kmem_cache_create("i915_gem_vma",
5044 sizeof(struct i915_vma), 0,
5045 SLAB_HWCACHE_ALIGN,
5046 NULL);
5047 dev_priv->requests =
5048 kmem_cache_create("i915_gem_request",
5049 sizeof(struct drm_i915_gem_request), 0,
5050 SLAB_HWCACHE_ALIGN,
5051 NULL);
5052
5053 INIT_LIST_HEAD(&dev_priv->vm_list);
5054 INIT_LIST_HEAD(&dev_priv->context_list);
5055 INIT_LIST_HEAD(&dev_priv->mm.unbound_list);
5056 INIT_LIST_HEAD(&dev_priv->mm.bound_list);
5057 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5058 for (i = 0; i < I915_NUM_RINGS; i++)
5059 init_ring_lists(&dev_priv->ring[i]);
5060 for (i = 0; i < I915_MAX_NUM_FENCES; i++)
5061 INIT_LIST_HEAD(&dev_priv->fence_regs[i].lru_list);
5062 INIT_DELAYED_WORK(&dev_priv->mm.retire_work,
5063 i915_gem_retire_work_handler);
5064 INIT_DELAYED_WORK(&dev_priv->mm.idle_work,
5065 i915_gem_idle_work_handler);
5066 init_waitqueue_head(&dev_priv->gpu_error.reset_queue);
5067
5068 dev_priv->relative_constants_mode = I915_EXEC_CONSTANTS_REL_GENERAL;
5069
5070 if (INTEL_INFO(dev)->gen >= 7 && !IS_VALLEYVIEW(dev) && !IS_CHERRYVIEW(dev))
5071 dev_priv->num_fence_regs = 32;
5072 else if (INTEL_INFO(dev)->gen >= 4 || IS_I945G(dev) || IS_I945GM(dev) || IS_G33(dev))
5073 dev_priv->num_fence_regs = 16;
5074 else
5075 dev_priv->num_fence_regs = 8;
5076
5077 if (intel_vgpu_active(dev))
5078 dev_priv->num_fence_regs =
5079 I915_READ(vgtif_reg(avail_rs.fence_num));
5080
5081 /*
5082 * Set initial sequence number for requests.
5083 * Using this number allows the wraparound to happen early,
5084 * catching any obvious problems.
5085 */
5086 dev_priv->next_seqno = ((u32)~0 - 0x1100);
5087 dev_priv->last_seqno = ((u32)~0 - 0x1101);
5088
5089 /* Initialize fence registers to zero */
5090 INIT_LIST_HEAD(&dev_priv->mm.fence_list);
5091 i915_gem_restore_fences(dev);
5092
5093 i915_gem_detect_bit_6_swizzle(dev);
5094 init_waitqueue_head(&dev_priv->pending_flip_queue);
5095
5096 dev_priv->mm.interruptible = true;
5097
5098 mutex_init(&dev_priv->fb_tracking.lock);
5099}
5100
5101void i915_gem_load_cleanup(struct drm_device *dev)
5102{
5103 struct drm_i915_private *dev_priv = to_i915(dev);
5104
5105 kmem_cache_destroy(dev_priv->requests);
5106 kmem_cache_destroy(dev_priv->vmas);
5107 kmem_cache_destroy(dev_priv->objects);
5108}
5109
5110void i915_gem_release(struct drm_device *dev, struct drm_file *file)
5111{
5112 struct drm_i915_file_private *file_priv = file->driver_priv;
5113
5114 /* Clean up our request list when the client is going away, so that
5115 * later retire_requests won't dereference our soon-to-be-gone
5116 * file_priv.
5117 */
5118 spin_lock(&file_priv->mm.lock);
5119 while (!list_empty(&file_priv->mm.request_list)) {
5120 struct drm_i915_gem_request *request;
5121
5122 request = list_first_entry(&file_priv->mm.request_list,
5123 struct drm_i915_gem_request,
5124 client_list);
5125 list_del(&request->client_list);
5126 request->file_priv = NULL;
5127 }
5128 spin_unlock(&file_priv->mm.lock);
5129
5130 if (!list_empty(&file_priv->rps.link)) {
5131 spin_lock(&to_i915(dev)->rps.client_lock);
5132 list_del(&file_priv->rps.link);
5133 spin_unlock(&to_i915(dev)->rps.client_lock);
5134 }
5135}
5136
5137int i915_gem_open(struct drm_device *dev, struct drm_file *file)
5138{
5139 struct drm_i915_file_private *file_priv;
5140 int ret;
5141
5142 DRM_DEBUG_DRIVER("\n");
5143
5144 file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
5145 if (!file_priv)
5146 return -ENOMEM;
5147
5148 file->driver_priv = file_priv;
5149 file_priv->dev_priv = dev->dev_private;
5150 file_priv->file = file;
5151 INIT_LIST_HEAD(&file_priv->rps.link);
5152
5153 spin_lock_init(&file_priv->mm.lock);
5154 INIT_LIST_HEAD(&file_priv->mm.request_list);
5155
5156 file_priv->bsd_ring = -1;
5157
5158 ret = i915_gem_context_open(dev, file);
5159 if (ret)
5160 kfree(file_priv);
5161
5162 return ret;
5163}
5164
5165/**
5166 * i915_gem_track_fb - update frontbuffer tracking
5167 * @old: current GEM buffer for the frontbuffer slots
5168 * @new: new GEM buffer for the frontbuffer slots
5169 * @frontbuffer_bits: bitmask of frontbuffer slots
5170 *
5171 * This updates the frontbuffer tracking bits @frontbuffer_bits by clearing them
5172 * from @old and setting them in @new. Both @old and @new can be NULL.
5173 */
5174void i915_gem_track_fb(struct drm_i915_gem_object *old,
5175 struct drm_i915_gem_object *new,
5176 unsigned frontbuffer_bits)
5177{
5178 if (old) {
5179 WARN_ON(!mutex_is_locked(&old->base.dev->struct_mutex));
5180 WARN_ON(!(old->frontbuffer_bits & frontbuffer_bits));
5181 old->frontbuffer_bits &= ~frontbuffer_bits;
5182 }
5183
5184 if (new) {
5185 WARN_ON(!mutex_is_locked(&new->base.dev->struct_mutex));
5186 WARN_ON(new->frontbuffer_bits & frontbuffer_bits);
5187 new->frontbuffer_bits |= frontbuffer_bits;
5188 }
5189}
5190
5191/* All the new VM stuff */
5192u64 i915_gem_obj_offset(struct drm_i915_gem_object *o,
5193 struct i915_address_space *vm)
5194{
5195 struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5196 struct i915_vma *vma;
5197
5198 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5199
5200 list_for_each_entry(vma, &o->vma_list, obj_link) {
5201 if (vma->is_ggtt &&
5202 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5203 continue;
5204 if (vma->vm == vm)
5205 return vma->node.start;
5206 }
5207
5208 WARN(1, "%s vma for this object not found.\n",
5209 i915_is_ggtt(vm) ? "global" : "ppgtt");
5210 return -1;
5211}
5212
5213u64 i915_gem_obj_ggtt_offset_view(struct drm_i915_gem_object *o,
5214 const struct i915_ggtt_view *view)
5215{
5216 struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5217 struct i915_vma *vma;
5218
5219 list_for_each_entry(vma, &o->vma_list, obj_link)
5220 if (vma->vm == ggtt &&
5221 i915_ggtt_view_equal(&vma->ggtt_view, view))
5222 return vma->node.start;
5223
5224 WARN(1, "global vma for this object not found. (view=%u)\n", view->type);
5225 return -1;
5226}
5227
5228bool i915_gem_obj_bound(struct drm_i915_gem_object *o,
5229 struct i915_address_space *vm)
5230{
5231 struct i915_vma *vma;
5232
5233 list_for_each_entry(vma, &o->vma_list, obj_link) {
5234 if (vma->is_ggtt &&
5235 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5236 continue;
5237 if (vma->vm == vm && drm_mm_node_allocated(&vma->node))
5238 return true;
5239 }
5240
5241 return false;
5242}
5243
5244bool i915_gem_obj_ggtt_bound_view(struct drm_i915_gem_object *o,
5245 const struct i915_ggtt_view *view)
5246{
5247 struct i915_address_space *ggtt = i915_obj_to_ggtt(o);
5248 struct i915_vma *vma;
5249
5250 list_for_each_entry(vma, &o->vma_list, obj_link)
5251 if (vma->vm == ggtt &&
5252 i915_ggtt_view_equal(&vma->ggtt_view, view) &&
5253 drm_mm_node_allocated(&vma->node))
5254 return true;
5255
5256 return false;
5257}
5258
5259bool i915_gem_obj_bound_any(struct drm_i915_gem_object *o)
5260{
5261 struct i915_vma *vma;
5262
5263 list_for_each_entry(vma, &o->vma_list, obj_link)
5264 if (drm_mm_node_allocated(&vma->node))
5265 return true;
5266
5267 return false;
5268}
5269
5270unsigned long i915_gem_obj_size(struct drm_i915_gem_object *o,
5271 struct i915_address_space *vm)
5272{
5273 struct drm_i915_private *dev_priv = o->base.dev->dev_private;
5274 struct i915_vma *vma;
5275
5276 WARN_ON(vm == &dev_priv->mm.aliasing_ppgtt->base);
5277
5278 BUG_ON(list_empty(&o->vma_list));
5279
5280 list_for_each_entry(vma, &o->vma_list, obj_link) {
5281 if (vma->is_ggtt &&
5282 vma->ggtt_view.type != I915_GGTT_VIEW_NORMAL)
5283 continue;
5284 if (vma->vm == vm)
5285 return vma->node.size;
5286 }
5287 return 0;
5288}
5289
5290bool i915_gem_obj_is_pinned(struct drm_i915_gem_object *obj)
5291{
5292 struct i915_vma *vma;
5293 list_for_each_entry(vma, &obj->vma_list, obj_link)
5294 if (vma->pin_count > 0)
5295 return true;
5296
5297 return false;
5298}
5299
5300/* Like i915_gem_object_get_page(), but mark the returned page dirty */
5301struct page *
5302i915_gem_object_get_dirty_page(struct drm_i915_gem_object *obj, int n)
5303{
5304 struct page *page;
5305
5306 /* Only default objects have per-page dirty tracking */
5307 if (WARN_ON((obj->ops->flags & I915_GEM_OBJECT_HAS_STRUCT_PAGE) == 0))
5308 return NULL;
5309
5310 page = i915_gem_object_get_page(obj, n);
5311 set_page_dirty(page);
5312 return page;
5313}
5314
5315/* Allocate a new GEM object and fill it with the supplied data */
5316struct drm_i915_gem_object *
5317i915_gem_object_create_from_data(struct drm_device *dev,
5318 const void *data, size_t size)
5319{
5320 struct drm_i915_gem_object *obj;
5321 struct sg_table *sg;
5322 size_t bytes;
5323 int ret;
5324
5325 obj = i915_gem_alloc_object(dev, round_up(size, PAGE_SIZE));
5326 if (IS_ERR_OR_NULL(obj))
5327 return obj;
5328
5329 ret = i915_gem_object_set_to_cpu_domain(obj, true);
5330 if (ret)
5331 goto fail;
5332
5333 ret = i915_gem_object_get_pages(obj);
5334 if (ret)
5335 goto fail;
5336
5337 i915_gem_object_pin_pages(obj);
5338 sg = obj->pages;
5339 bytes = sg_copy_from_buffer(sg->sgl, sg->nents, (void *)data, size);
5340 obj->dirty = 1; /* Backing store is now out of date */
5341 i915_gem_object_unpin_pages(obj);
5342
5343 if (WARN_ON(bytes != size)) {
5344 DRM_ERROR("Incomplete copy, wrote %zu of %zu", bytes, size);
5345 ret = -EFAULT;
5346 goto fail;
5347 }
5348
5349 return obj;
5350
5351fail:
5352 drm_gem_object_unreference(&obj->base);
5353 return ERR_PTR(ret);
5354}