Loading...
Note: File does not exist in v3.1.
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2016 Intel Corporation
5 */
6
7#include <linux/anon_inodes.h>
8#include <linux/mman.h>
9#include <linux/pfn_t.h>
10#include <linux/sizes.h>
11
12#include <drm/drm_cache.h>
13
14#include "gt/intel_gt.h"
15#include "gt/intel_gt_requests.h"
16
17#include "i915_drv.h"
18#include "i915_gem_evict.h"
19#include "i915_gem_gtt.h"
20#include "i915_gem_ioctls.h"
21#include "i915_gem_object.h"
22#include "i915_gem_mman.h"
23#include "i915_mm.h"
24#include "i915_trace.h"
25#include "i915_user_extensions.h"
26#include "i915_gem_ttm.h"
27#include "i915_vma.h"
28
29static inline bool
30__vma_matches(struct vm_area_struct *vma, struct file *filp,
31 unsigned long addr, unsigned long size)
32{
33 if (vma->vm_file != filp)
34 return false;
35
36 return vma->vm_start == addr &&
37 (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);
38}
39
40/**
41 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
42 * it is mapped to.
43 * @dev: drm device
44 * @data: ioctl data blob
45 * @file: drm file
46 *
47 * While the mapping holds a reference on the contents of the object, it doesn't
48 * imply a ref on the object itself.
49 *
50 * IMPORTANT:
51 *
52 * DRM driver writers who look a this function as an example for how to do GEM
53 * mmap support, please don't implement mmap support like here. The modern way
54 * to implement DRM mmap support is with an mmap offset ioctl (like
55 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
56 * That way debug tooling like valgrind will understand what's going on, hiding
57 * the mmap call in a driver private ioctl will break that. The i915 driver only
58 * does cpu mmaps this way because we didn't know better.
59 */
60int
61i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
62 struct drm_file *file)
63{
64 struct drm_i915_private *i915 = to_i915(dev);
65 struct drm_i915_gem_mmap *args = data;
66 struct drm_i915_gem_object *obj;
67 unsigned long addr;
68
69 /*
70 * mmap ioctl is disallowed for all discrete platforms,
71 * and for all platforms with GRAPHICS_VER > 12.
72 */
73 if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0))
74 return -EOPNOTSUPP;
75
76 if (args->flags & ~(I915_MMAP_WC))
77 return -EINVAL;
78
79 if (args->flags & I915_MMAP_WC && !pat_enabled())
80 return -ENODEV;
81
82 obj = i915_gem_object_lookup(file, args->handle);
83 if (!obj)
84 return -ENOENT;
85
86 /* prime objects have no backing filp to GEM mmap
87 * pages from.
88 */
89 if (!obj->base.filp) {
90 addr = -ENXIO;
91 goto err;
92 }
93
94 if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {
95 addr = -EINVAL;
96 goto err;
97 }
98
99 addr = vm_mmap(obj->base.filp, 0, args->size,
100 PROT_READ | PROT_WRITE, MAP_SHARED,
101 args->offset);
102 if (IS_ERR_VALUE(addr))
103 goto err;
104
105 if (args->flags & I915_MMAP_WC) {
106 struct mm_struct *mm = current->mm;
107 struct vm_area_struct *vma;
108
109 if (mmap_write_lock_killable(mm)) {
110 addr = -EINTR;
111 goto err;
112 }
113 vma = find_vma(mm, addr);
114 if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
115 vma->vm_page_prot =
116 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
117 else
118 addr = -ENOMEM;
119 mmap_write_unlock(mm);
120 if (IS_ERR_VALUE(addr))
121 goto err;
122 }
123 i915_gem_object_put(obj);
124
125 args->addr_ptr = (u64)addr;
126 return 0;
127
128err:
129 i915_gem_object_put(obj);
130 return addr;
131}
132
133static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
134{
135 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
136}
137
138/**
139 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
140 *
141 * A history of the GTT mmap interface:
142 *
143 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
144 * aligned and suitable for fencing, and still fit into the available
145 * mappable space left by the pinned display objects. A classic problem
146 * we called the page-fault-of-doom where we would ping-pong between
147 * two objects that could not fit inside the GTT and so the memcpy
148 * would page one object in at the expense of the other between every
149 * single byte.
150 *
151 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
152 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
153 * object is too large for the available space (or simply too large
154 * for the mappable aperture!), a view is created instead and faulted
155 * into userspace. (This view is aligned and sized appropriately for
156 * fenced access.)
157 *
158 * 2 - Recognise WC as a separate cache domain so that we can flush the
159 * delayed writes via GTT before performing direct access via WC.
160 *
161 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
162 * pagefault; swapin remains transparent.
163 *
164 * 4 - Support multiple fault handlers per object depending on object's
165 * backing storage (a.k.a. MMAP_OFFSET).
166 *
167 * Restrictions:
168 *
169 * * snoopable objects cannot be accessed via the GTT. It can cause machine
170 * hangs on some architectures, corruption on others. An attempt to service
171 * a GTT page fault from a snoopable object will generate a SIGBUS.
172 *
173 * * the object must be able to fit into RAM (physical memory, though no
174 * limited to the mappable aperture).
175 *
176 *
177 * Caveats:
178 *
179 * * a new GTT page fault will synchronize rendering from the GPU and flush
180 * all data to system memory. Subsequent access will not be synchronized.
181 *
182 * * all mappings are revoked on runtime device suspend.
183 *
184 * * there are only 8, 16 or 32 fence registers to share between all users
185 * (older machines require fence register for display and blitter access
186 * as well). Contention of the fence registers will cause the previous users
187 * to be unmapped and any new access will generate new page faults.
188 *
189 * * running out of memory while servicing a fault may generate a SIGBUS,
190 * rather than the expected SIGSEGV.
191 */
192int i915_gem_mmap_gtt_version(void)
193{
194 return 4;
195}
196
197static inline struct i915_gtt_view
198compute_partial_view(const struct drm_i915_gem_object *obj,
199 pgoff_t page_offset,
200 unsigned int chunk)
201{
202 struct i915_gtt_view view;
203
204 if (i915_gem_object_is_tiled(obj))
205 chunk = roundup(chunk, tile_row_pages(obj) ?: 1);
206
207 view.type = I915_GTT_VIEW_PARTIAL;
208 view.partial.offset = rounddown(page_offset, chunk);
209 view.partial.size =
210 min_t(unsigned int, chunk,
211 (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
212
213 /* If the partial covers the entire object, just create a normal VMA. */
214 if (chunk >= obj->base.size >> PAGE_SHIFT)
215 view.type = I915_GTT_VIEW_NORMAL;
216
217 return view;
218}
219
220static vm_fault_t i915_error_to_vmf_fault(int err)
221{
222 switch (err) {
223 default:
224 WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err);
225 fallthrough;
226 case -EIO: /* shmemfs failure from swap device */
227 case -EFAULT: /* purged object */
228 case -ENODEV: /* bad object, how did you get here! */
229 case -ENXIO: /* unable to access backing store (on device) */
230 return VM_FAULT_SIGBUS;
231
232 case -ENOMEM: /* our allocation failure */
233 return VM_FAULT_OOM;
234
235 case 0:
236 case -EAGAIN:
237 case -ENOSPC: /* transient failure to evict? */
238 case -ERESTARTSYS:
239 case -EINTR:
240 case -EBUSY:
241 /*
242 * EBUSY is ok: this just means that another thread
243 * already did the job.
244 */
245 return VM_FAULT_NOPAGE;
246 }
247}
248
249static vm_fault_t vm_fault_cpu(struct vm_fault *vmf)
250{
251 struct vm_area_struct *area = vmf->vma;
252 struct i915_mmap_offset *mmo = area->vm_private_data;
253 struct drm_i915_gem_object *obj = mmo->obj;
254 resource_size_t iomap;
255 int err;
256
257 /* Sanity check that we allow writing into this object */
258 if (unlikely(i915_gem_object_is_readonly(obj) &&
259 area->vm_flags & VM_WRITE))
260 return VM_FAULT_SIGBUS;
261
262 if (i915_gem_object_lock_interruptible(obj, NULL))
263 return VM_FAULT_NOPAGE;
264
265 err = i915_gem_object_pin_pages(obj);
266 if (err)
267 goto out;
268
269 iomap = -1;
270 if (!i915_gem_object_has_struct_page(obj)) {
271 iomap = obj->mm.region->iomap.base;
272 iomap -= obj->mm.region->region.start;
273 }
274
275 /* PTEs are revoked in obj->ops->put_pages() */
276 err = remap_io_sg(area,
277 area->vm_start, area->vm_end - area->vm_start,
278 obj->mm.pages->sgl, iomap);
279
280 if (area->vm_flags & VM_WRITE) {
281 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
282 obj->mm.dirty = true;
283 }
284
285 i915_gem_object_unpin_pages(obj);
286
287out:
288 i915_gem_object_unlock(obj);
289 return i915_error_to_vmf_fault(err);
290}
291
292static vm_fault_t vm_fault_gtt(struct vm_fault *vmf)
293{
294#define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
295 struct vm_area_struct *area = vmf->vma;
296 struct i915_mmap_offset *mmo = area->vm_private_data;
297 struct drm_i915_gem_object *obj = mmo->obj;
298 struct drm_device *dev = obj->base.dev;
299 struct drm_i915_private *i915 = to_i915(dev);
300 struct intel_runtime_pm *rpm = &i915->runtime_pm;
301 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
302 bool write = area->vm_flags & VM_WRITE;
303 struct i915_gem_ww_ctx ww;
304 intel_wakeref_t wakeref;
305 struct i915_vma *vma;
306 pgoff_t page_offset;
307 int srcu;
308 int ret;
309
310 /* We don't use vmf->pgoff since that has the fake offset */
311 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
312
313 trace_i915_gem_object_fault(obj, page_offset, true, write);
314
315 wakeref = intel_runtime_pm_get(rpm);
316
317 i915_gem_ww_ctx_init(&ww, true);
318retry:
319 ret = i915_gem_object_lock(obj, &ww);
320 if (ret)
321 goto err_rpm;
322
323 /* Sanity check that we allow writing into this object */
324 if (i915_gem_object_is_readonly(obj) && write) {
325 ret = -EFAULT;
326 goto err_rpm;
327 }
328
329 ret = i915_gem_object_pin_pages(obj);
330 if (ret)
331 goto err_rpm;
332
333 ret = intel_gt_reset_lock_interruptible(ggtt->vm.gt, &srcu);
334 if (ret)
335 goto err_pages;
336
337 /* Now pin it into the GTT as needed */
338 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
339 PIN_MAPPABLE |
340 PIN_NONBLOCK /* NOWARN */ |
341 PIN_NOEVICT);
342 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
343 /* Use a partial view if it is bigger than available space */
344 struct i915_gtt_view view =
345 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
346 unsigned int flags;
347
348 flags = PIN_MAPPABLE | PIN_NOSEARCH;
349 if (view.type == I915_GTT_VIEW_NORMAL)
350 flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
351
352 /*
353 * Userspace is now writing through an untracked VMA, abandon
354 * all hope that the hardware is able to track future writes.
355 */
356
357 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
358 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
359 flags = PIN_MAPPABLE;
360 view.type = I915_GTT_VIEW_PARTIAL;
361 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
362 }
363
364 /*
365 * The entire mappable GGTT is pinned? Unexpected!
366 * Try to evict the object we locked too, as normally we skip it
367 * due to lack of short term pinning inside execbuf.
368 */
369 if (vma == ERR_PTR(-ENOSPC)) {
370 ret = mutex_lock_interruptible(&ggtt->vm.mutex);
371 if (!ret) {
372 ret = i915_gem_evict_vm(&ggtt->vm, &ww, NULL);
373 mutex_unlock(&ggtt->vm.mutex);
374 }
375 if (ret)
376 goto err_reset;
377 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
378 }
379 }
380 if (IS_ERR(vma)) {
381 ret = PTR_ERR(vma);
382 goto err_reset;
383 }
384
385 /* Access to snoopable pages through the GTT is incoherent. */
386 if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) {
387 ret = -EFAULT;
388 goto err_unpin;
389 }
390
391 ret = i915_vma_pin_fence(vma);
392 if (ret)
393 goto err_unpin;
394
395 /* Finally, remap it using the new GTT offset */
396 ret = remap_io_mapping(area,
397 area->vm_start + (vma->gtt_view.partial.offset << PAGE_SHIFT),
398 (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
399 min_t(u64, vma->size, area->vm_end - area->vm_start),
400 &ggtt->iomap);
401 if (ret)
402 goto err_fence;
403
404 assert_rpm_wakelock_held(rpm);
405
406 /* Mark as being mmapped into userspace for later revocation */
407 mutex_lock(&to_gt(i915)->ggtt->vm.mutex);
408 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
409 list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list);
410 mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);
411
412 /* Track the mmo associated with the fenced vma */
413 vma->mmo = mmo;
414
415 if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
416 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref,
417 msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
418
419 if (write) {
420 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
421 i915_vma_set_ggtt_write(vma);
422 obj->mm.dirty = true;
423 }
424
425err_fence:
426 i915_vma_unpin_fence(vma);
427err_unpin:
428 __i915_vma_unpin(vma);
429err_reset:
430 intel_gt_reset_unlock(ggtt->vm.gt, srcu);
431err_pages:
432 i915_gem_object_unpin_pages(obj);
433err_rpm:
434 if (ret == -EDEADLK) {
435 ret = i915_gem_ww_ctx_backoff(&ww);
436 if (!ret)
437 goto retry;
438 }
439 i915_gem_ww_ctx_fini(&ww);
440 intel_runtime_pm_put(rpm, wakeref);
441 return i915_error_to_vmf_fault(ret);
442}
443
444static int
445vm_access(struct vm_area_struct *area, unsigned long addr,
446 void *buf, int len, int write)
447{
448 struct i915_mmap_offset *mmo = area->vm_private_data;
449 struct drm_i915_gem_object *obj = mmo->obj;
450 struct i915_gem_ww_ctx ww;
451 void *vaddr;
452 int err = 0;
453
454 if (i915_gem_object_is_readonly(obj) && write)
455 return -EACCES;
456
457 addr -= area->vm_start;
458 if (range_overflows_t(u64, addr, len, obj->base.size))
459 return -EINVAL;
460
461 i915_gem_ww_ctx_init(&ww, true);
462retry:
463 err = i915_gem_object_lock(obj, &ww);
464 if (err)
465 goto out;
466
467 /* As this is primarily for debugging, let's focus on simplicity */
468 vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC);
469 if (IS_ERR(vaddr)) {
470 err = PTR_ERR(vaddr);
471 goto out;
472 }
473
474 if (write) {
475 memcpy(vaddr + addr, buf, len);
476 __i915_gem_object_flush_map(obj, addr, len);
477 } else {
478 memcpy(buf, vaddr + addr, len);
479 }
480
481 i915_gem_object_unpin_map(obj);
482out:
483 if (err == -EDEADLK) {
484 err = i915_gem_ww_ctx_backoff(&ww);
485 if (!err)
486 goto retry;
487 }
488 i915_gem_ww_ctx_fini(&ww);
489
490 if (err)
491 return err;
492
493 return len;
494}
495
496void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
497{
498 struct i915_vma *vma;
499
500 GEM_BUG_ON(!obj->userfault_count);
501
502 for_each_ggtt_vma(vma, obj)
503 i915_vma_revoke_mmap(vma);
504
505 GEM_BUG_ON(obj->userfault_count);
506}
507
508/*
509 * It is vital that we remove the page mapping if we have mapped a tiled
510 * object through the GTT and then lose the fence register due to
511 * resource pressure. Similarly if the object has been moved out of the
512 * aperture, than pages mapped into userspace must be revoked. Removing the
513 * mapping will then trigger a page fault on the next user access, allowing
514 * fixup by vm_fault_gtt().
515 */
516void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
517{
518 struct drm_i915_private *i915 = to_i915(obj->base.dev);
519 intel_wakeref_t wakeref;
520
521 /*
522 * Serialisation between user GTT access and our code depends upon
523 * revoking the CPU's PTE whilst the mutex is held. The next user
524 * pagefault then has to wait until we release the mutex.
525 *
526 * Note that RPM complicates somewhat by adding an additional
527 * requirement that operations to the GGTT be made holding the RPM
528 * wakeref.
529 */
530 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
531 mutex_lock(&to_gt(i915)->ggtt->vm.mutex);
532
533 if (!obj->userfault_count)
534 goto out;
535
536 __i915_gem_object_release_mmap_gtt(obj);
537
538 /*
539 * Ensure that the CPU's PTE are revoked and there are not outstanding
540 * memory transactions from userspace before we return. The TLB
541 * flushing implied above by changing the PTE above *should* be
542 * sufficient, an extra barrier here just provides us with a bit
543 * of paranoid documentation about our requirement to serialise
544 * memory writes before touching registers / GSM.
545 */
546 wmb();
547
548out:
549 mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);
550 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
551}
552
553void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj)
554{
555 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
556 struct ttm_device *bdev = bo->bdev;
557
558 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
559
560 /*
561 * We have exclusive access here via runtime suspend. All other callers
562 * must first grab the rpm wakeref.
563 */
564 GEM_BUG_ON(!obj->userfault_count);
565 list_del(&obj->userfault_link);
566 obj->userfault_count = 0;
567}
568
569void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj)
570{
571 struct i915_mmap_offset *mmo, *mn;
572
573 if (obj->ops->unmap_virtual)
574 obj->ops->unmap_virtual(obj);
575
576 spin_lock(&obj->mmo.lock);
577 rbtree_postorder_for_each_entry_safe(mmo, mn,
578 &obj->mmo.offsets, offset) {
579 /*
580 * vma_node_unmap for GTT mmaps handled already in
581 * __i915_gem_object_release_mmap_gtt
582 */
583 if (mmo->mmap_type == I915_MMAP_TYPE_GTT)
584 continue;
585
586 spin_unlock(&obj->mmo.lock);
587 drm_vma_node_unmap(&mmo->vma_node,
588 obj->base.dev->anon_inode->i_mapping);
589 spin_lock(&obj->mmo.lock);
590 }
591 spin_unlock(&obj->mmo.lock);
592}
593
594static struct i915_mmap_offset *
595lookup_mmo(struct drm_i915_gem_object *obj,
596 enum i915_mmap_type mmap_type)
597{
598 struct rb_node *rb;
599
600 spin_lock(&obj->mmo.lock);
601 rb = obj->mmo.offsets.rb_node;
602 while (rb) {
603 struct i915_mmap_offset *mmo =
604 rb_entry(rb, typeof(*mmo), offset);
605
606 if (mmo->mmap_type == mmap_type) {
607 spin_unlock(&obj->mmo.lock);
608 return mmo;
609 }
610
611 if (mmo->mmap_type < mmap_type)
612 rb = rb->rb_right;
613 else
614 rb = rb->rb_left;
615 }
616 spin_unlock(&obj->mmo.lock);
617
618 return NULL;
619}
620
621static struct i915_mmap_offset *
622insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo)
623{
624 struct rb_node *rb, **p;
625
626 spin_lock(&obj->mmo.lock);
627 rb = NULL;
628 p = &obj->mmo.offsets.rb_node;
629 while (*p) {
630 struct i915_mmap_offset *pos;
631
632 rb = *p;
633 pos = rb_entry(rb, typeof(*pos), offset);
634
635 if (pos->mmap_type == mmo->mmap_type) {
636 spin_unlock(&obj->mmo.lock);
637 drm_vma_offset_remove(obj->base.dev->vma_offset_manager,
638 &mmo->vma_node);
639 kfree(mmo);
640 return pos;
641 }
642
643 if (pos->mmap_type < mmo->mmap_type)
644 p = &rb->rb_right;
645 else
646 p = &rb->rb_left;
647 }
648 rb_link_node(&mmo->offset, rb, p);
649 rb_insert_color(&mmo->offset, &obj->mmo.offsets);
650 spin_unlock(&obj->mmo.lock);
651
652 return mmo;
653}
654
655static struct i915_mmap_offset *
656mmap_offset_attach(struct drm_i915_gem_object *obj,
657 enum i915_mmap_type mmap_type,
658 struct drm_file *file)
659{
660 struct drm_i915_private *i915 = to_i915(obj->base.dev);
661 struct i915_mmap_offset *mmo;
662 int err;
663
664 GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops);
665
666 mmo = lookup_mmo(obj, mmap_type);
667 if (mmo)
668 goto out;
669
670 mmo = kmalloc(sizeof(*mmo), GFP_KERNEL);
671 if (!mmo)
672 return ERR_PTR(-ENOMEM);
673
674 mmo->obj = obj;
675 mmo->mmap_type = mmap_type;
676 drm_vma_node_reset(&mmo->vma_node);
677
678 err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,
679 &mmo->vma_node, obj->base.size / PAGE_SIZE);
680 if (likely(!err))
681 goto insert;
682
683 /* Attempt to reap some mmap space from dead objects */
684 err = intel_gt_retire_requests_timeout(to_gt(i915), MAX_SCHEDULE_TIMEOUT,
685 NULL);
686 if (err)
687 goto err;
688
689 i915_gem_drain_freed_objects(i915);
690 err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,
691 &mmo->vma_node, obj->base.size / PAGE_SIZE);
692 if (err)
693 goto err;
694
695insert:
696 mmo = insert_mmo(obj, mmo);
697 GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo);
698out:
699 if (file)
700 drm_vma_node_allow_once(&mmo->vma_node, file);
701 return mmo;
702
703err:
704 kfree(mmo);
705 return ERR_PTR(err);
706}
707
708static int
709__assign_mmap_offset(struct drm_i915_gem_object *obj,
710 enum i915_mmap_type mmap_type,
711 u64 *offset, struct drm_file *file)
712{
713 struct i915_mmap_offset *mmo;
714
715 if (i915_gem_object_never_mmap(obj))
716 return -ENODEV;
717
718 if (obj->ops->mmap_offset) {
719 if (mmap_type != I915_MMAP_TYPE_FIXED)
720 return -ENODEV;
721
722 *offset = obj->ops->mmap_offset(obj);
723 return 0;
724 }
725
726 if (mmap_type == I915_MMAP_TYPE_FIXED)
727 return -ENODEV;
728
729 if (mmap_type != I915_MMAP_TYPE_GTT &&
730 !i915_gem_object_has_struct_page(obj) &&
731 !i915_gem_object_has_iomem(obj))
732 return -ENODEV;
733
734 mmo = mmap_offset_attach(obj, mmap_type, file);
735 if (IS_ERR(mmo))
736 return PTR_ERR(mmo);
737
738 *offset = drm_vma_node_offset_addr(&mmo->vma_node);
739 return 0;
740}
741
742static int
743__assign_mmap_offset_handle(struct drm_file *file,
744 u32 handle,
745 enum i915_mmap_type mmap_type,
746 u64 *offset)
747{
748 struct drm_i915_gem_object *obj;
749 int err;
750
751 obj = i915_gem_object_lookup(file, handle);
752 if (!obj)
753 return -ENOENT;
754
755 err = i915_gem_object_lock_interruptible(obj, NULL);
756 if (err)
757 goto out_put;
758 err = __assign_mmap_offset(obj, mmap_type, offset, file);
759 i915_gem_object_unlock(obj);
760out_put:
761 i915_gem_object_put(obj);
762 return err;
763}
764
765int
766i915_gem_dumb_mmap_offset(struct drm_file *file,
767 struct drm_device *dev,
768 u32 handle,
769 u64 *offset)
770{
771 struct drm_i915_private *i915 = to_i915(dev);
772 enum i915_mmap_type mmap_type;
773
774 if (HAS_LMEM(to_i915(dev)))
775 mmap_type = I915_MMAP_TYPE_FIXED;
776 else if (pat_enabled())
777 mmap_type = I915_MMAP_TYPE_WC;
778 else if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
779 return -ENODEV;
780 else
781 mmap_type = I915_MMAP_TYPE_GTT;
782
783 return __assign_mmap_offset_handle(file, handle, mmap_type, offset);
784}
785
786/**
787 * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing
788 * @dev: DRM device
789 * @data: GTT mapping ioctl data
790 * @file: GEM object info
791 *
792 * Simply returns the fake offset to userspace so it can mmap it.
793 * The mmap call will end up in drm_gem_mmap(), which will set things
794 * up so we can get faults in the handler above.
795 *
796 * The fault handler will take care of binding the object into the GTT
797 * (since it may have been evicted to make room for something), allocating
798 * a fence register, and mapping the appropriate aperture address into
799 * userspace.
800 */
801int
802i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
803 struct drm_file *file)
804{
805 struct drm_i915_private *i915 = to_i915(dev);
806 struct drm_i915_gem_mmap_offset *args = data;
807 enum i915_mmap_type type;
808 int err;
809
810 /*
811 * Historically we failed to check args.pad and args.offset
812 * and so we cannot use those fields for user input and we cannot
813 * add -EINVAL for them as the ABI is fixed, i.e. old userspace
814 * may be feeding in garbage in those fields.
815 *
816 * if (args->pad) return -EINVAL; is verbotten!
817 */
818
819 err = i915_user_extensions(u64_to_user_ptr(args->extensions),
820 NULL, 0, NULL);
821 if (err)
822 return err;
823
824 switch (args->flags) {
825 case I915_MMAP_OFFSET_GTT:
826 if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
827 return -ENODEV;
828 type = I915_MMAP_TYPE_GTT;
829 break;
830
831 case I915_MMAP_OFFSET_WC:
832 if (!pat_enabled())
833 return -ENODEV;
834 type = I915_MMAP_TYPE_WC;
835 break;
836
837 case I915_MMAP_OFFSET_WB:
838 type = I915_MMAP_TYPE_WB;
839 break;
840
841 case I915_MMAP_OFFSET_UC:
842 if (!pat_enabled())
843 return -ENODEV;
844 type = I915_MMAP_TYPE_UC;
845 break;
846
847 case I915_MMAP_OFFSET_FIXED:
848 type = I915_MMAP_TYPE_FIXED;
849 break;
850
851 default:
852 return -EINVAL;
853 }
854
855 return __assign_mmap_offset_handle(file, args->handle, type, &args->offset);
856}
857
858static void vm_open(struct vm_area_struct *vma)
859{
860 struct i915_mmap_offset *mmo = vma->vm_private_data;
861 struct drm_i915_gem_object *obj = mmo->obj;
862
863 GEM_BUG_ON(!obj);
864 i915_gem_object_get(obj);
865}
866
867static void vm_close(struct vm_area_struct *vma)
868{
869 struct i915_mmap_offset *mmo = vma->vm_private_data;
870 struct drm_i915_gem_object *obj = mmo->obj;
871
872 GEM_BUG_ON(!obj);
873 i915_gem_object_put(obj);
874}
875
876static const struct vm_operations_struct vm_ops_gtt = {
877 .fault = vm_fault_gtt,
878 .access = vm_access,
879 .open = vm_open,
880 .close = vm_close,
881};
882
883static const struct vm_operations_struct vm_ops_cpu = {
884 .fault = vm_fault_cpu,
885 .access = vm_access,
886 .open = vm_open,
887 .close = vm_close,
888};
889
890static int singleton_release(struct inode *inode, struct file *file)
891{
892 struct drm_i915_private *i915 = file->private_data;
893
894 cmpxchg(&i915->gem.mmap_singleton, file, NULL);
895 drm_dev_put(&i915->drm);
896
897 return 0;
898}
899
900static const struct file_operations singleton_fops = {
901 .owner = THIS_MODULE,
902 .release = singleton_release,
903};
904
905static struct file *mmap_singleton(struct drm_i915_private *i915)
906{
907 struct file *file;
908
909 rcu_read_lock();
910 file = READ_ONCE(i915->gem.mmap_singleton);
911 if (file && !get_file_rcu(file))
912 file = NULL;
913 rcu_read_unlock();
914 if (file)
915 return file;
916
917 file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR);
918 if (IS_ERR(file))
919 return file;
920
921 /* Everyone shares a single global address space */
922 file->f_mapping = i915->drm.anon_inode->i_mapping;
923
924 smp_store_mb(i915->gem.mmap_singleton, file);
925 drm_dev_get(&i915->drm);
926
927 return file;
928}
929
930/*
931 * This overcomes the limitation in drm_gem_mmap's assignment of a
932 * drm_gem_object as the vma->vm_private_data. Since we need to
933 * be able to resolve multiple mmap offsets which could be tied
934 * to a single gem object.
935 */
936int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma)
937{
938 struct drm_vma_offset_node *node;
939 struct drm_file *priv = filp->private_data;
940 struct drm_device *dev = priv->minor->dev;
941 struct drm_i915_gem_object *obj = NULL;
942 struct i915_mmap_offset *mmo = NULL;
943 struct file *anon;
944
945 if (drm_dev_is_unplugged(dev))
946 return -ENODEV;
947
948 rcu_read_lock();
949 drm_vma_offset_lock_lookup(dev->vma_offset_manager);
950 node = drm_vma_offset_exact_lookup_locked(dev->vma_offset_manager,
951 vma->vm_pgoff,
952 vma_pages(vma));
953 if (node && drm_vma_node_is_allowed(node, priv)) {
954 /*
955 * Skip 0-refcnted objects as it is in the process of being
956 * destroyed and will be invalid when the vma manager lock
957 * is released.
958 */
959 if (!node->driver_private) {
960 mmo = container_of(node, struct i915_mmap_offset, vma_node);
961 obj = i915_gem_object_get_rcu(mmo->obj);
962
963 GEM_BUG_ON(obj && obj->ops->mmap_ops);
964 } else {
965 obj = i915_gem_object_get_rcu
966 (container_of(node, struct drm_i915_gem_object,
967 base.vma_node));
968
969 GEM_BUG_ON(obj && !obj->ops->mmap_ops);
970 }
971 }
972 drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
973 rcu_read_unlock();
974 if (!obj)
975 return node ? -EACCES : -EINVAL;
976
977 if (i915_gem_object_is_readonly(obj)) {
978 if (vma->vm_flags & VM_WRITE) {
979 i915_gem_object_put(obj);
980 return -EINVAL;
981 }
982 vma->vm_flags &= ~VM_MAYWRITE;
983 }
984
985 anon = mmap_singleton(to_i915(dev));
986 if (IS_ERR(anon)) {
987 i915_gem_object_put(obj);
988 return PTR_ERR(anon);
989 }
990
991 vma->vm_flags |= VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO;
992
993 /*
994 * We keep the ref on mmo->obj, not vm_file, but we require
995 * vma->vm_file->f_mapping, see vma_link(), for later revocation.
996 * Our userspace is accustomed to having per-file resource cleanup
997 * (i.e. contexts, objects and requests) on their close(fd), which
998 * requires avoiding extraneous references to their filp, hence why
999 * we prefer to use an anonymous file for their mmaps.
1000 */
1001 vma_set_file(vma, anon);
1002 /* Drop the initial creation reference, the vma is now holding one. */
1003 fput(anon);
1004
1005 if (obj->ops->mmap_ops) {
1006 vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags));
1007 vma->vm_ops = obj->ops->mmap_ops;
1008 vma->vm_private_data = node->driver_private;
1009 return 0;
1010 }
1011
1012 vma->vm_private_data = mmo;
1013
1014 switch (mmo->mmap_type) {
1015 case I915_MMAP_TYPE_WC:
1016 vma->vm_page_prot =
1017 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1018 vma->vm_ops = &vm_ops_cpu;
1019 break;
1020
1021 case I915_MMAP_TYPE_FIXED:
1022 GEM_WARN_ON(1);
1023 fallthrough;
1024 case I915_MMAP_TYPE_WB:
1025 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
1026 vma->vm_ops = &vm_ops_cpu;
1027 break;
1028
1029 case I915_MMAP_TYPE_UC:
1030 vma->vm_page_prot =
1031 pgprot_noncached(vm_get_page_prot(vma->vm_flags));
1032 vma->vm_ops = &vm_ops_cpu;
1033 break;
1034
1035 case I915_MMAP_TYPE_GTT:
1036 vma->vm_page_prot =
1037 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1038 vma->vm_ops = &vm_ops_gtt;
1039 break;
1040 }
1041 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1042
1043 return 0;
1044}
1045
1046#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1047#include "selftests/i915_gem_mman.c"
1048#endif