Loading...
Note: File does not exist in v3.1.
1/*
2 * SPDX-License-Identifier: MIT
3 *
4 * Copyright © 2014-2016 Intel Corporation
5 */
6
7#include <linux/anon_inodes.h>
8#include <linux/mman.h>
9#include <linux/pfn_t.h>
10#include <linux/sizes.h>
11
12#include <drm/drm_cache.h>
13
14#include "gt/intel_gt.h"
15#include "gt/intel_gt_requests.h"
16
17#include "i915_drv.h"
18#include "i915_gem_evict.h"
19#include "i915_gem_gtt.h"
20#include "i915_gem_ioctls.h"
21#include "i915_gem_object.h"
22#include "i915_gem_mman.h"
23#include "i915_mm.h"
24#include "i915_trace.h"
25#include "i915_user_extensions.h"
26#include "i915_gem_ttm.h"
27#include "i915_vma.h"
28
29static inline bool
30__vma_matches(struct vm_area_struct *vma, struct file *filp,
31 unsigned long addr, unsigned long size)
32{
33 if (vma->vm_file != filp)
34 return false;
35
36 return vma->vm_start == addr &&
37 (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);
38}
39
40/**
41 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
42 * it is mapped to.
43 * @dev: drm device
44 * @data: ioctl data blob
45 * @file: drm file
46 *
47 * While the mapping holds a reference on the contents of the object, it doesn't
48 * imply a ref on the object itself.
49 *
50 * IMPORTANT:
51 *
52 * DRM driver writers who look a this function as an example for how to do GEM
53 * mmap support, please don't implement mmap support like here. The modern way
54 * to implement DRM mmap support is with an mmap offset ioctl (like
55 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
56 * That way debug tooling like valgrind will understand what's going on, hiding
57 * the mmap call in a driver private ioctl will break that. The i915 driver only
58 * does cpu mmaps this way because we didn't know better.
59 */
60int
61i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
62 struct drm_file *file)
63{
64 struct drm_i915_private *i915 = to_i915(dev);
65 struct drm_i915_gem_mmap *args = data;
66 struct drm_i915_gem_object *obj;
67 unsigned long addr;
68
69 /*
70 * mmap ioctl is disallowed for all discrete platforms,
71 * and for all platforms with GRAPHICS_VER > 12.
72 */
73 if (IS_DGFX(i915) || GRAPHICS_VER_FULL(i915) > IP_VER(12, 0))
74 return -EOPNOTSUPP;
75
76 if (args->flags & ~(I915_MMAP_WC))
77 return -EINVAL;
78
79 if (args->flags & I915_MMAP_WC && !pat_enabled())
80 return -ENODEV;
81
82 obj = i915_gem_object_lookup(file, args->handle);
83 if (!obj)
84 return -ENOENT;
85
86 /* prime objects have no backing filp to GEM mmap
87 * pages from.
88 */
89 if (!obj->base.filp) {
90 addr = -ENXIO;
91 goto err;
92 }
93
94 if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {
95 addr = -EINVAL;
96 goto err;
97 }
98
99 addr = vm_mmap(obj->base.filp, 0, args->size,
100 PROT_READ | PROT_WRITE, MAP_SHARED,
101 args->offset);
102 if (IS_ERR_VALUE(addr))
103 goto err;
104
105 if (args->flags & I915_MMAP_WC) {
106 struct mm_struct *mm = current->mm;
107 struct vm_area_struct *vma;
108
109 if (mmap_write_lock_killable(mm)) {
110 addr = -EINTR;
111 goto err;
112 }
113 vma = find_vma(mm, addr);
114 if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
115 vma->vm_page_prot =
116 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
117 else
118 addr = -ENOMEM;
119 mmap_write_unlock(mm);
120 if (IS_ERR_VALUE(addr))
121 goto err;
122 }
123 i915_gem_object_put(obj);
124
125 args->addr_ptr = (u64)addr;
126 return 0;
127
128err:
129 i915_gem_object_put(obj);
130 return addr;
131}
132
133static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
134{
135 return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
136}
137
138/**
139 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
140 *
141 * A history of the GTT mmap interface:
142 *
143 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
144 * aligned and suitable for fencing, and still fit into the available
145 * mappable space left by the pinned display objects. A classic problem
146 * we called the page-fault-of-doom where we would ping-pong between
147 * two objects that could not fit inside the GTT and so the memcpy
148 * would page one object in at the expense of the other between every
149 * single byte.
150 *
151 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
152 * as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
153 * object is too large for the available space (or simply too large
154 * for the mappable aperture!), a view is created instead and faulted
155 * into userspace. (This view is aligned and sized appropriately for
156 * fenced access.)
157 *
158 * 2 - Recognise WC as a separate cache domain so that we can flush the
159 * delayed writes via GTT before performing direct access via WC.
160 *
161 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
162 * pagefault; swapin remains transparent.
163 *
164 * 4 - Support multiple fault handlers per object depending on object's
165 * backing storage (a.k.a. MMAP_OFFSET).
166 *
167 * Restrictions:
168 *
169 * * snoopable objects cannot be accessed via the GTT. It can cause machine
170 * hangs on some architectures, corruption on others. An attempt to service
171 * a GTT page fault from a snoopable object will generate a SIGBUS.
172 *
173 * * the object must be able to fit into RAM (physical memory, though no
174 * limited to the mappable aperture).
175 *
176 *
177 * Caveats:
178 *
179 * * a new GTT page fault will synchronize rendering from the GPU and flush
180 * all data to system memory. Subsequent access will not be synchronized.
181 *
182 * * all mappings are revoked on runtime device suspend.
183 *
184 * * there are only 8, 16 or 32 fence registers to share between all users
185 * (older machines require fence register for display and blitter access
186 * as well). Contention of the fence registers will cause the previous users
187 * to be unmapped and any new access will generate new page faults.
188 *
189 * * running out of memory while servicing a fault may generate a SIGBUS,
190 * rather than the expected SIGSEGV.
191 */
192int i915_gem_mmap_gtt_version(void)
193{
194 return 4;
195}
196
197static inline struct i915_gtt_view
198compute_partial_view(const struct drm_i915_gem_object *obj,
199 pgoff_t page_offset,
200 unsigned int chunk)
201{
202 struct i915_gtt_view view;
203
204 if (i915_gem_object_is_tiled(obj))
205 chunk = roundup(chunk, tile_row_pages(obj) ?: 1);
206
207 view.type = I915_GTT_VIEW_PARTIAL;
208 view.partial.offset = rounddown(page_offset, chunk);
209 view.partial.size =
210 min_t(unsigned int, chunk,
211 (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
212
213 /* If the partial covers the entire object, just create a normal VMA. */
214 if (chunk >= obj->base.size >> PAGE_SHIFT)
215 view.type = I915_GTT_VIEW_NORMAL;
216
217 return view;
218}
219
220static vm_fault_t i915_error_to_vmf_fault(int err)
221{
222 switch (err) {
223 default:
224 WARN_ONCE(err, "unhandled error in %s: %i\n", __func__, err);
225 fallthrough;
226 case -EIO: /* shmemfs failure from swap device */
227 case -EFAULT: /* purged object */
228 case -ENODEV: /* bad object, how did you get here! */
229 case -ENXIO: /* unable to access backing store (on device) */
230 return VM_FAULT_SIGBUS;
231
232 case -ENOMEM: /* our allocation failure */
233 return VM_FAULT_OOM;
234
235 case 0:
236 case -EAGAIN:
237 case -ENOSPC: /* transient failure to evict? */
238 case -ENOBUFS: /* temporarily out of fences? */
239 case -ERESTARTSYS:
240 case -EINTR:
241 case -EBUSY:
242 /*
243 * EBUSY is ok: this just means that another thread
244 * already did the job.
245 */
246 return VM_FAULT_NOPAGE;
247 }
248}
249
250static vm_fault_t vm_fault_cpu(struct vm_fault *vmf)
251{
252 struct vm_area_struct *area = vmf->vma;
253 struct i915_mmap_offset *mmo = area->vm_private_data;
254 struct drm_i915_gem_object *obj = mmo->obj;
255 unsigned long obj_offset;
256 resource_size_t iomap;
257 int err;
258
259 /* Sanity check that we allow writing into this object */
260 if (unlikely(i915_gem_object_is_readonly(obj) &&
261 area->vm_flags & VM_WRITE))
262 return VM_FAULT_SIGBUS;
263
264 if (i915_gem_object_lock_interruptible(obj, NULL))
265 return VM_FAULT_NOPAGE;
266
267 err = i915_gem_object_pin_pages(obj);
268 if (err)
269 goto out;
270
271 iomap = -1;
272 if (!i915_gem_object_has_struct_page(obj)) {
273 iomap = obj->mm.region->iomap.base;
274 iomap -= obj->mm.region->region.start;
275 }
276
277 obj_offset = area->vm_pgoff - drm_vma_node_start(&mmo->vma_node);
278 /* PTEs are revoked in obj->ops->put_pages() */
279 err = remap_io_sg(area,
280 area->vm_start, area->vm_end - area->vm_start,
281 obj->mm.pages->sgl, obj_offset, iomap);
282
283 if (area->vm_flags & VM_WRITE) {
284 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
285 obj->mm.dirty = true;
286 }
287
288 i915_gem_object_unpin_pages(obj);
289
290out:
291 i915_gem_object_unlock(obj);
292 return i915_error_to_vmf_fault(err);
293}
294
295static void set_address_limits(struct vm_area_struct *area,
296 struct i915_vma *vma,
297 unsigned long obj_offset,
298 resource_size_t gmadr_start,
299 unsigned long *start_vaddr,
300 unsigned long *end_vaddr,
301 unsigned long *pfn)
302{
303 unsigned long vm_start, vm_end, vma_size; /* user's memory parameters */
304 long start, end; /* memory boundaries */
305
306 /*
307 * Let's move into the ">> PAGE_SHIFT"
308 * domain to be sure not to lose bits
309 */
310 vm_start = area->vm_start >> PAGE_SHIFT;
311 vm_end = area->vm_end >> PAGE_SHIFT;
312 vma_size = vma->size >> PAGE_SHIFT;
313
314 /*
315 * Calculate the memory boundaries by considering the offset
316 * provided by the user during memory mapping and the offset
317 * provided for the partial mapping.
318 */
319 start = vm_start;
320 start -= obj_offset;
321 start += vma->gtt_view.partial.offset;
322 end = start + vma_size;
323
324 start = max_t(long, start, vm_start);
325 end = min_t(long, end, vm_end);
326
327 /* Let's move back into the "<< PAGE_SHIFT" domain */
328 *start_vaddr = (unsigned long)start << PAGE_SHIFT;
329 *end_vaddr = (unsigned long)end << PAGE_SHIFT;
330
331 *pfn = (gmadr_start + i915_ggtt_offset(vma)) >> PAGE_SHIFT;
332 *pfn += (*start_vaddr - area->vm_start) >> PAGE_SHIFT;
333 *pfn += obj_offset - vma->gtt_view.partial.offset;
334}
335
336static vm_fault_t vm_fault_gtt(struct vm_fault *vmf)
337{
338#define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
339 struct vm_area_struct *area = vmf->vma;
340 struct i915_mmap_offset *mmo = area->vm_private_data;
341 struct drm_i915_gem_object *obj = mmo->obj;
342 struct drm_device *dev = obj->base.dev;
343 struct drm_i915_private *i915 = to_i915(dev);
344 struct intel_runtime_pm *rpm = &i915->runtime_pm;
345 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
346 bool write = area->vm_flags & VM_WRITE;
347 struct i915_gem_ww_ctx ww;
348 unsigned long obj_offset;
349 unsigned long start, end; /* memory boundaries */
350 intel_wakeref_t wakeref;
351 struct i915_vma *vma;
352 pgoff_t page_offset;
353 unsigned long pfn;
354 int srcu;
355 int ret;
356
357 obj_offset = area->vm_pgoff - drm_vma_node_start(&mmo->vma_node);
358 page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
359 page_offset += obj_offset;
360
361 trace_i915_gem_object_fault(obj, page_offset, true, write);
362
363 wakeref = intel_runtime_pm_get(rpm);
364
365 i915_gem_ww_ctx_init(&ww, true);
366retry:
367 ret = i915_gem_object_lock(obj, &ww);
368 if (ret)
369 goto err_rpm;
370
371 /* Sanity check that we allow writing into this object */
372 if (i915_gem_object_is_readonly(obj) && write) {
373 ret = -EFAULT;
374 goto err_rpm;
375 }
376
377 ret = i915_gem_object_pin_pages(obj);
378 if (ret)
379 goto err_rpm;
380
381 ret = intel_gt_reset_lock_interruptible(ggtt->vm.gt, &srcu);
382 if (ret)
383 goto err_pages;
384
385 /* Now pin it into the GTT as needed */
386 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
387 PIN_MAPPABLE |
388 PIN_NONBLOCK /* NOWARN */ |
389 PIN_NOEVICT);
390 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
391 /* Use a partial view if it is bigger than available space */
392 struct i915_gtt_view view =
393 compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
394 unsigned int flags;
395
396 flags = PIN_MAPPABLE | PIN_NOSEARCH;
397 if (view.type == I915_GTT_VIEW_NORMAL)
398 flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
399
400 /*
401 * Userspace is now writing through an untracked VMA, abandon
402 * all hope that the hardware is able to track future writes.
403 */
404
405 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
406 if (IS_ERR(vma) && vma != ERR_PTR(-EDEADLK)) {
407 flags = PIN_MAPPABLE;
408 view.type = I915_GTT_VIEW_PARTIAL;
409 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
410 }
411
412 /*
413 * The entire mappable GGTT is pinned? Unexpected!
414 * Try to evict the object we locked too, as normally we skip it
415 * due to lack of short term pinning inside execbuf.
416 */
417 if (vma == ERR_PTR(-ENOSPC)) {
418 ret = mutex_lock_interruptible(&ggtt->vm.mutex);
419 if (!ret) {
420 ret = i915_gem_evict_vm(&ggtt->vm, &ww, NULL);
421 mutex_unlock(&ggtt->vm.mutex);
422 }
423 if (ret)
424 goto err_reset;
425 vma = i915_gem_object_ggtt_pin_ww(obj, &ww, &view, 0, 0, flags);
426 }
427 }
428 if (IS_ERR(vma)) {
429 ret = PTR_ERR(vma);
430 goto err_reset;
431 }
432
433 /* Access to snoopable pages through the GTT is incoherent. */
434 /*
435 * For objects created by userspace through GEM_CREATE with pat_index
436 * set by set_pat extension, coherency is managed by userspace, make
437 * sure we don't fail handling the vm fault by calling
438 * i915_gem_object_has_cache_level() which always return true for such
439 * objects. Otherwise this helper function would fall back to checking
440 * whether the object is un-cached.
441 */
442 if (!(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) ||
443 HAS_LLC(i915))) {
444 ret = -EFAULT;
445 goto err_unpin;
446 }
447
448 ret = i915_vma_pin_fence(vma);
449 if (ret)
450 goto err_unpin;
451
452 /*
453 * Dump all the necessary parameters in this function to perform the
454 * arithmetic calculation for the virtual address start and end and
455 * the PFN (Page Frame Number).
456 */
457 set_address_limits(area, vma, obj_offset, ggtt->gmadr.start,
458 &start, &end, &pfn);
459
460 /* Finally, remap it using the new GTT offset */
461 ret = remap_io_mapping(area, start, pfn, end - start, &ggtt->iomap);
462 if (ret)
463 goto err_fence;
464
465 assert_rpm_wakelock_held(rpm);
466
467 /* Mark as being mmapped into userspace for later revocation */
468 mutex_lock(&to_gt(i915)->ggtt->vm.mutex);
469 if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
470 list_add(&obj->userfault_link, &to_gt(i915)->ggtt->userfault_list);
471 mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);
472
473 /* Track the mmo associated with the fenced vma */
474 vma->mmo = mmo;
475
476 if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
477 intel_wakeref_auto(&i915->runtime_pm.userfault_wakeref,
478 msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
479
480 if (write) {
481 GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
482 i915_vma_set_ggtt_write(vma);
483 obj->mm.dirty = true;
484 }
485
486err_fence:
487 i915_vma_unpin_fence(vma);
488err_unpin:
489 __i915_vma_unpin(vma);
490err_reset:
491 intel_gt_reset_unlock(ggtt->vm.gt, srcu);
492err_pages:
493 i915_gem_object_unpin_pages(obj);
494err_rpm:
495 if (ret == -EDEADLK) {
496 ret = i915_gem_ww_ctx_backoff(&ww);
497 if (!ret)
498 goto retry;
499 }
500 i915_gem_ww_ctx_fini(&ww);
501 intel_runtime_pm_put(rpm, wakeref);
502 return i915_error_to_vmf_fault(ret);
503}
504
505static int
506vm_access(struct vm_area_struct *area, unsigned long addr,
507 void *buf, int len, int write)
508{
509 struct i915_mmap_offset *mmo = area->vm_private_data;
510 struct drm_i915_gem_object *obj = mmo->obj;
511 struct i915_gem_ww_ctx ww;
512 void *vaddr;
513 int err = 0;
514
515 if (i915_gem_object_is_readonly(obj) && write)
516 return -EACCES;
517
518 addr -= area->vm_start;
519 if (range_overflows_t(u64, addr, len, obj->base.size))
520 return -EINVAL;
521
522 i915_gem_ww_ctx_init(&ww, true);
523retry:
524 err = i915_gem_object_lock(obj, &ww);
525 if (err)
526 goto out;
527
528 /* As this is primarily for debugging, let's focus on simplicity */
529 vaddr = i915_gem_object_pin_map(obj, I915_MAP_FORCE_WC);
530 if (IS_ERR(vaddr)) {
531 err = PTR_ERR(vaddr);
532 goto out;
533 }
534
535 if (write) {
536 memcpy(vaddr + addr, buf, len);
537 __i915_gem_object_flush_map(obj, addr, len);
538 } else {
539 memcpy(buf, vaddr + addr, len);
540 }
541
542 i915_gem_object_unpin_map(obj);
543out:
544 if (err == -EDEADLK) {
545 err = i915_gem_ww_ctx_backoff(&ww);
546 if (!err)
547 goto retry;
548 }
549 i915_gem_ww_ctx_fini(&ww);
550
551 if (err)
552 return err;
553
554 return len;
555}
556
557void __i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
558{
559 struct i915_vma *vma;
560
561 GEM_BUG_ON(!obj->userfault_count);
562
563 for_each_ggtt_vma(vma, obj)
564 i915_vma_revoke_mmap(vma);
565
566 GEM_BUG_ON(obj->userfault_count);
567}
568
569/*
570 * It is vital that we remove the page mapping if we have mapped a tiled
571 * object through the GTT and then lose the fence register due to
572 * resource pressure. Similarly if the object has been moved out of the
573 * aperture, than pages mapped into userspace must be revoked. Removing the
574 * mapping will then trigger a page fault on the next user access, allowing
575 * fixup by vm_fault_gtt().
576 */
577void i915_gem_object_release_mmap_gtt(struct drm_i915_gem_object *obj)
578{
579 struct drm_i915_private *i915 = to_i915(obj->base.dev);
580 intel_wakeref_t wakeref;
581
582 /*
583 * Serialisation between user GTT access and our code depends upon
584 * revoking the CPU's PTE whilst the mutex is held. The next user
585 * pagefault then has to wait until we release the mutex.
586 *
587 * Note that RPM complicates somewhat by adding an additional
588 * requirement that operations to the GGTT be made holding the RPM
589 * wakeref.
590 */
591 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
592 mutex_lock(&to_gt(i915)->ggtt->vm.mutex);
593
594 if (!obj->userfault_count)
595 goto out;
596
597 __i915_gem_object_release_mmap_gtt(obj);
598
599 /*
600 * Ensure that the CPU's PTE are revoked and there are not outstanding
601 * memory transactions from userspace before we return. The TLB
602 * flushing implied above by changing the PTE above *should* be
603 * sufficient, an extra barrier here just provides us with a bit
604 * of paranoid documentation about our requirement to serialise
605 * memory writes before touching registers / GSM.
606 */
607 wmb();
608
609out:
610 mutex_unlock(&to_gt(i915)->ggtt->vm.mutex);
611 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
612}
613
614void i915_gem_object_runtime_pm_release_mmap_offset(struct drm_i915_gem_object *obj)
615{
616 struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
617 struct ttm_device *bdev = bo->bdev;
618
619 drm_vma_node_unmap(&bo->base.vma_node, bdev->dev_mapping);
620
621 /*
622 * We have exclusive access here via runtime suspend. All other callers
623 * must first grab the rpm wakeref.
624 */
625 GEM_BUG_ON(!obj->userfault_count);
626 list_del(&obj->userfault_link);
627 obj->userfault_count = 0;
628}
629
630void i915_gem_object_release_mmap_offset(struct drm_i915_gem_object *obj)
631{
632 struct i915_mmap_offset *mmo, *mn;
633
634 if (obj->ops->unmap_virtual)
635 obj->ops->unmap_virtual(obj);
636
637 spin_lock(&obj->mmo.lock);
638 rbtree_postorder_for_each_entry_safe(mmo, mn,
639 &obj->mmo.offsets, offset) {
640 /*
641 * vma_node_unmap for GTT mmaps handled already in
642 * __i915_gem_object_release_mmap_gtt
643 */
644 if (mmo->mmap_type == I915_MMAP_TYPE_GTT)
645 continue;
646
647 spin_unlock(&obj->mmo.lock);
648 drm_vma_node_unmap(&mmo->vma_node,
649 obj->base.dev->anon_inode->i_mapping);
650 spin_lock(&obj->mmo.lock);
651 }
652 spin_unlock(&obj->mmo.lock);
653}
654
655static struct i915_mmap_offset *
656lookup_mmo(struct drm_i915_gem_object *obj,
657 enum i915_mmap_type mmap_type)
658{
659 struct rb_node *rb;
660
661 spin_lock(&obj->mmo.lock);
662 rb = obj->mmo.offsets.rb_node;
663 while (rb) {
664 struct i915_mmap_offset *mmo =
665 rb_entry(rb, typeof(*mmo), offset);
666
667 if (mmo->mmap_type == mmap_type) {
668 spin_unlock(&obj->mmo.lock);
669 return mmo;
670 }
671
672 if (mmo->mmap_type < mmap_type)
673 rb = rb->rb_right;
674 else
675 rb = rb->rb_left;
676 }
677 spin_unlock(&obj->mmo.lock);
678
679 return NULL;
680}
681
682static struct i915_mmap_offset *
683insert_mmo(struct drm_i915_gem_object *obj, struct i915_mmap_offset *mmo)
684{
685 struct rb_node *rb, **p;
686
687 spin_lock(&obj->mmo.lock);
688 rb = NULL;
689 p = &obj->mmo.offsets.rb_node;
690 while (*p) {
691 struct i915_mmap_offset *pos;
692
693 rb = *p;
694 pos = rb_entry(rb, typeof(*pos), offset);
695
696 if (pos->mmap_type == mmo->mmap_type) {
697 spin_unlock(&obj->mmo.lock);
698 drm_vma_offset_remove(obj->base.dev->vma_offset_manager,
699 &mmo->vma_node);
700 kfree(mmo);
701 return pos;
702 }
703
704 if (pos->mmap_type < mmo->mmap_type)
705 p = &rb->rb_right;
706 else
707 p = &rb->rb_left;
708 }
709 rb_link_node(&mmo->offset, rb, p);
710 rb_insert_color(&mmo->offset, &obj->mmo.offsets);
711 spin_unlock(&obj->mmo.lock);
712
713 return mmo;
714}
715
716static struct i915_mmap_offset *
717mmap_offset_attach(struct drm_i915_gem_object *obj,
718 enum i915_mmap_type mmap_type,
719 struct drm_file *file)
720{
721 struct drm_i915_private *i915 = to_i915(obj->base.dev);
722 struct i915_mmap_offset *mmo;
723 int err;
724
725 GEM_BUG_ON(obj->ops->mmap_offset || obj->ops->mmap_ops);
726
727 mmo = lookup_mmo(obj, mmap_type);
728 if (mmo)
729 goto out;
730
731 mmo = kmalloc(sizeof(*mmo), GFP_KERNEL);
732 if (!mmo)
733 return ERR_PTR(-ENOMEM);
734
735 mmo->obj = obj;
736 mmo->mmap_type = mmap_type;
737 drm_vma_node_reset(&mmo->vma_node);
738
739 err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,
740 &mmo->vma_node, obj->base.size / PAGE_SIZE);
741 if (likely(!err))
742 goto insert;
743
744 /* Attempt to reap some mmap space from dead objects */
745 err = intel_gt_retire_requests_timeout(to_gt(i915), MAX_SCHEDULE_TIMEOUT,
746 NULL);
747 if (err)
748 goto err;
749
750 i915_gem_drain_freed_objects(i915);
751 err = drm_vma_offset_add(obj->base.dev->vma_offset_manager,
752 &mmo->vma_node, obj->base.size / PAGE_SIZE);
753 if (err)
754 goto err;
755
756insert:
757 mmo = insert_mmo(obj, mmo);
758 GEM_BUG_ON(lookup_mmo(obj, mmap_type) != mmo);
759out:
760 if (file)
761 drm_vma_node_allow_once(&mmo->vma_node, file);
762 return mmo;
763
764err:
765 kfree(mmo);
766 return ERR_PTR(err);
767}
768
769static int
770__assign_mmap_offset(struct drm_i915_gem_object *obj,
771 enum i915_mmap_type mmap_type,
772 u64 *offset, struct drm_file *file)
773{
774 struct i915_mmap_offset *mmo;
775
776 if (i915_gem_object_never_mmap(obj))
777 return -ENODEV;
778
779 if (obj->ops->mmap_offset) {
780 if (mmap_type != I915_MMAP_TYPE_FIXED)
781 return -ENODEV;
782
783 *offset = obj->ops->mmap_offset(obj);
784 return 0;
785 }
786
787 if (mmap_type == I915_MMAP_TYPE_FIXED)
788 return -ENODEV;
789
790 if (mmap_type != I915_MMAP_TYPE_GTT &&
791 !i915_gem_object_has_struct_page(obj) &&
792 !i915_gem_object_has_iomem(obj))
793 return -ENODEV;
794
795 mmo = mmap_offset_attach(obj, mmap_type, file);
796 if (IS_ERR(mmo))
797 return PTR_ERR(mmo);
798
799 *offset = drm_vma_node_offset_addr(&mmo->vma_node);
800 return 0;
801}
802
803static int
804__assign_mmap_offset_handle(struct drm_file *file,
805 u32 handle,
806 enum i915_mmap_type mmap_type,
807 u64 *offset)
808{
809 struct drm_i915_gem_object *obj;
810 int err;
811
812 obj = i915_gem_object_lookup(file, handle);
813 if (!obj)
814 return -ENOENT;
815
816 err = i915_gem_object_lock_interruptible(obj, NULL);
817 if (err)
818 goto out_put;
819 err = __assign_mmap_offset(obj, mmap_type, offset, file);
820 i915_gem_object_unlock(obj);
821out_put:
822 i915_gem_object_put(obj);
823 return err;
824}
825
826int
827i915_gem_dumb_mmap_offset(struct drm_file *file,
828 struct drm_device *dev,
829 u32 handle,
830 u64 *offset)
831{
832 struct drm_i915_private *i915 = to_i915(dev);
833 enum i915_mmap_type mmap_type;
834
835 if (HAS_LMEM(to_i915(dev)))
836 mmap_type = I915_MMAP_TYPE_FIXED;
837 else if (pat_enabled())
838 mmap_type = I915_MMAP_TYPE_WC;
839 else if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
840 return -ENODEV;
841 else
842 mmap_type = I915_MMAP_TYPE_GTT;
843
844 return __assign_mmap_offset_handle(file, handle, mmap_type, offset);
845}
846
847/**
848 * i915_gem_mmap_offset_ioctl - prepare an object for GTT mmap'ing
849 * @dev: DRM device
850 * @data: GTT mapping ioctl data
851 * @file: GEM object info
852 *
853 * Simply returns the fake offset to userspace so it can mmap it.
854 * The mmap call will end up in drm_gem_mmap(), which will set things
855 * up so we can get faults in the handler above.
856 *
857 * The fault handler will take care of binding the object into the GTT
858 * (since it may have been evicted to make room for something), allocating
859 * a fence register, and mapping the appropriate aperture address into
860 * userspace.
861 */
862int
863i915_gem_mmap_offset_ioctl(struct drm_device *dev, void *data,
864 struct drm_file *file)
865{
866 struct drm_i915_private *i915 = to_i915(dev);
867 struct drm_i915_gem_mmap_offset *args = data;
868 enum i915_mmap_type type;
869 int err;
870
871 /*
872 * Historically we failed to check args.pad and args.offset
873 * and so we cannot use those fields for user input and we cannot
874 * add -EINVAL for them as the ABI is fixed, i.e. old userspace
875 * may be feeding in garbage in those fields.
876 *
877 * if (args->pad) return -EINVAL; is verbotten!
878 */
879
880 err = i915_user_extensions(u64_to_user_ptr(args->extensions),
881 NULL, 0, NULL);
882 if (err)
883 return err;
884
885 switch (args->flags) {
886 case I915_MMAP_OFFSET_GTT:
887 if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
888 return -ENODEV;
889 type = I915_MMAP_TYPE_GTT;
890 break;
891
892 case I915_MMAP_OFFSET_WC:
893 if (!pat_enabled())
894 return -ENODEV;
895 type = I915_MMAP_TYPE_WC;
896 break;
897
898 case I915_MMAP_OFFSET_WB:
899 type = I915_MMAP_TYPE_WB;
900 break;
901
902 case I915_MMAP_OFFSET_UC:
903 if (!pat_enabled())
904 return -ENODEV;
905 type = I915_MMAP_TYPE_UC;
906 break;
907
908 case I915_MMAP_OFFSET_FIXED:
909 type = I915_MMAP_TYPE_FIXED;
910 break;
911
912 default:
913 return -EINVAL;
914 }
915
916 return __assign_mmap_offset_handle(file, args->handle, type, &args->offset);
917}
918
919static void vm_open(struct vm_area_struct *vma)
920{
921 struct i915_mmap_offset *mmo = vma->vm_private_data;
922 struct drm_i915_gem_object *obj = mmo->obj;
923
924 GEM_BUG_ON(!obj);
925 i915_gem_object_get(obj);
926}
927
928static void vm_close(struct vm_area_struct *vma)
929{
930 struct i915_mmap_offset *mmo = vma->vm_private_data;
931 struct drm_i915_gem_object *obj = mmo->obj;
932
933 GEM_BUG_ON(!obj);
934 i915_gem_object_put(obj);
935}
936
937static const struct vm_operations_struct vm_ops_gtt = {
938 .fault = vm_fault_gtt,
939 .access = vm_access,
940 .open = vm_open,
941 .close = vm_close,
942};
943
944static const struct vm_operations_struct vm_ops_cpu = {
945 .fault = vm_fault_cpu,
946 .access = vm_access,
947 .open = vm_open,
948 .close = vm_close,
949};
950
951static int singleton_release(struct inode *inode, struct file *file)
952{
953 struct drm_i915_private *i915 = file->private_data;
954
955 cmpxchg(&i915->gem.mmap_singleton, file, NULL);
956 drm_dev_put(&i915->drm);
957
958 return 0;
959}
960
961static const struct file_operations singleton_fops = {
962 .owner = THIS_MODULE,
963 .release = singleton_release,
964};
965
966static struct file *mmap_singleton(struct drm_i915_private *i915)
967{
968 struct file *file;
969
970 file = get_file_active(&i915->gem.mmap_singleton);
971 if (file)
972 return file;
973
974 file = anon_inode_getfile("i915.gem", &singleton_fops, i915, O_RDWR);
975 if (IS_ERR(file))
976 return file;
977
978 /* Everyone shares a single global address space */
979 file->f_mapping = i915->drm.anon_inode->i_mapping;
980
981 smp_store_mb(i915->gem.mmap_singleton, file);
982 drm_dev_get(&i915->drm);
983
984 return file;
985}
986
987static int
988i915_gem_object_mmap(struct drm_i915_gem_object *obj,
989 struct i915_mmap_offset *mmo,
990 struct vm_area_struct *vma)
991{
992 struct drm_i915_private *i915 = to_i915(obj->base.dev);
993 struct drm_device *dev = &i915->drm;
994 struct file *anon;
995
996 if (i915_gem_object_is_readonly(obj)) {
997 if (vma->vm_flags & VM_WRITE) {
998 i915_gem_object_put(obj);
999 return -EINVAL;
1000 }
1001 vm_flags_clear(vma, VM_MAYWRITE);
1002 }
1003
1004 anon = mmap_singleton(to_i915(dev));
1005 if (IS_ERR(anon)) {
1006 i915_gem_object_put(obj);
1007 return PTR_ERR(anon);
1008 }
1009
1010 vm_flags_set(vma, VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP | VM_IO);
1011
1012 /*
1013 * We keep the ref on mmo->obj, not vm_file, but we require
1014 * vma->vm_file->f_mapping, see vma_link(), for later revocation.
1015 * Our userspace is accustomed to having per-file resource cleanup
1016 * (i.e. contexts, objects and requests) on their close(fd), which
1017 * requires avoiding extraneous references to their filp, hence why
1018 * we prefer to use an anonymous file for their mmaps.
1019 */
1020 vma_set_file(vma, anon);
1021 /* Drop the initial creation reference, the vma is now holding one. */
1022 fput(anon);
1023
1024 if (obj->ops->mmap_ops) {
1025 vma->vm_page_prot = pgprot_decrypted(vm_get_page_prot(vma->vm_flags));
1026 vma->vm_ops = obj->ops->mmap_ops;
1027 vma->vm_private_data = obj->base.vma_node.driver_private;
1028 return 0;
1029 }
1030
1031 vma->vm_private_data = mmo;
1032
1033 switch (mmo->mmap_type) {
1034 case I915_MMAP_TYPE_WC:
1035 vma->vm_page_prot =
1036 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1037 vma->vm_ops = &vm_ops_cpu;
1038 break;
1039
1040 case I915_MMAP_TYPE_FIXED:
1041 GEM_WARN_ON(1);
1042 fallthrough;
1043 case I915_MMAP_TYPE_WB:
1044 vma->vm_page_prot = vm_get_page_prot(vma->vm_flags);
1045 vma->vm_ops = &vm_ops_cpu;
1046 break;
1047
1048 case I915_MMAP_TYPE_UC:
1049 vma->vm_page_prot =
1050 pgprot_noncached(vm_get_page_prot(vma->vm_flags));
1051 vma->vm_ops = &vm_ops_cpu;
1052 break;
1053
1054 case I915_MMAP_TYPE_GTT:
1055 vma->vm_page_prot =
1056 pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
1057 vma->vm_ops = &vm_ops_gtt;
1058 break;
1059 }
1060 vma->vm_page_prot = pgprot_decrypted(vma->vm_page_prot);
1061
1062 return 0;
1063}
1064
1065/*
1066 * This overcomes the limitation in drm_gem_mmap's assignment of a
1067 * drm_gem_object as the vma->vm_private_data. Since we need to
1068 * be able to resolve multiple mmap offsets which could be tied
1069 * to a single gem object.
1070 */
1071int i915_gem_mmap(struct file *filp, struct vm_area_struct *vma)
1072{
1073 struct drm_vma_offset_node *node;
1074 struct drm_file *priv = filp->private_data;
1075 struct drm_device *dev = priv->minor->dev;
1076 struct drm_i915_gem_object *obj = NULL;
1077 struct i915_mmap_offset *mmo = NULL;
1078
1079 if (drm_dev_is_unplugged(dev))
1080 return -ENODEV;
1081
1082 rcu_read_lock();
1083 drm_vma_offset_lock_lookup(dev->vma_offset_manager);
1084 node = drm_vma_offset_lookup_locked(dev->vma_offset_manager,
1085 vma->vm_pgoff,
1086 vma_pages(vma));
1087 if (node && drm_vma_node_is_allowed(node, priv)) {
1088 /*
1089 * Skip 0-refcnted objects as it is in the process of being
1090 * destroyed and will be invalid when the vma manager lock
1091 * is released.
1092 */
1093 if (!node->driver_private) {
1094 mmo = container_of(node, struct i915_mmap_offset, vma_node);
1095 obj = i915_gem_object_get_rcu(mmo->obj);
1096
1097 GEM_BUG_ON(obj && obj->ops->mmap_ops);
1098 } else {
1099 obj = i915_gem_object_get_rcu
1100 (container_of(node, struct drm_i915_gem_object,
1101 base.vma_node));
1102
1103 GEM_BUG_ON(obj && !obj->ops->mmap_ops);
1104 }
1105 }
1106 drm_vma_offset_unlock_lookup(dev->vma_offset_manager);
1107 rcu_read_unlock();
1108 if (!obj)
1109 return node ? -EACCES : -EINVAL;
1110
1111 return i915_gem_object_mmap(obj, mmo, vma);
1112}
1113
1114int i915_gem_fb_mmap(struct drm_i915_gem_object *obj, struct vm_area_struct *vma)
1115{
1116 struct drm_i915_private *i915 = to_i915(obj->base.dev);
1117 struct drm_device *dev = &i915->drm;
1118 struct i915_mmap_offset *mmo = NULL;
1119 enum i915_mmap_type mmap_type;
1120 struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
1121
1122 if (drm_dev_is_unplugged(dev))
1123 return -ENODEV;
1124
1125 /* handle ttm object */
1126 if (obj->ops->mmap_ops) {
1127 /*
1128 * ttm fault handler, ttm_bo_vm_fault_reserved() uses fake offset
1129 * to calculate page offset so set that up.
1130 */
1131 vma->vm_pgoff += drm_vma_node_start(&obj->base.vma_node);
1132 } else {
1133 /* handle stolen and smem objects */
1134 mmap_type = i915_ggtt_has_aperture(ggtt) ? I915_MMAP_TYPE_GTT : I915_MMAP_TYPE_WC;
1135 mmo = mmap_offset_attach(obj, mmap_type, NULL);
1136 if (IS_ERR(mmo))
1137 return PTR_ERR(mmo);
1138
1139 vma->vm_pgoff += drm_vma_node_start(&mmo->vma_node);
1140 }
1141
1142 /*
1143 * When we install vm_ops for mmap we are too late for
1144 * the vm_ops->open() which increases the ref_count of
1145 * this obj and then it gets decreased by the vm_ops->close().
1146 * To balance this increase the obj ref_count here.
1147 */
1148 obj = i915_gem_object_get(obj);
1149 return i915_gem_object_mmap(obj, mmo, vma);
1150}
1151
1152#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1153#include "selftests/i915_gem_mman.c"
1154#endif