1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2014-2016 Intel Corporation
  5 */
  6
  7#include <linux/mman.h>
  8#include <linux/sizes.h>
  9
 10#include "gt/intel_gt.h"
 11
 12#include "i915_drv.h"
 13#include "i915_gem_gtt.h"
 14#include "i915_gem_ioctls.h"
 15#include "i915_gem_object.h"
 16#include "i915_trace.h"
 17#include "i915_vma.h"
 18
 19static inline bool
 20__vma_matches(struct vm_area_struct *vma, struct file *filp,
 21	      unsigned long addr, unsigned long size)
 22{
 23	if (vma->vm_file != filp)
 24		return false;
 25
 26	return vma->vm_start == addr &&
 27	       (vma->vm_end - vma->vm_start) == PAGE_ALIGN(size);
 28}
 29
 30/**
 31 * i915_gem_mmap_ioctl - Maps the contents of an object, returning the address
 32 *			 it is mapped to.
 33 * @dev: drm device
 34 * @data: ioctl data blob
 35 * @file: drm file
 36 *
 37 * While the mapping holds a reference on the contents of the object, it doesn't
 38 * imply a ref on the object itself.
 39 *
 40 * IMPORTANT:
 41 *
 42 * DRM driver writers who look a this function as an example for how to do GEM
 43 * mmap support, please don't implement mmap support like here. The modern way
 44 * to implement DRM mmap support is with an mmap offset ioctl (like
 45 * i915_gem_mmap_gtt) and then using the mmap syscall on the DRM fd directly.
 46 * That way debug tooling like valgrind will understand what's going on, hiding
 47 * the mmap call in a driver private ioctl will break that. The i915 driver only
 48 * does cpu mmaps this way because we didn't know better.
 49 */
 50int
 51i915_gem_mmap_ioctl(struct drm_device *dev, void *data,
 52		    struct drm_file *file)
 53{
 54	struct drm_i915_gem_mmap *args = data;
 55	struct drm_i915_gem_object *obj;
 56	unsigned long addr;
 57
 58	if (args->flags & ~(I915_MMAP_WC))
 59		return -EINVAL;
 60
 61	if (args->flags & I915_MMAP_WC && !boot_cpu_has(X86_FEATURE_PAT))
 62		return -ENODEV;
 63
 64	obj = i915_gem_object_lookup(file, args->handle);
 65	if (!obj)
 66		return -ENOENT;
 67
 68	/* prime objects have no backing filp to GEM mmap
 69	 * pages from.
 70	 */
 71	if (!obj->base.filp) {
 72		addr = -ENXIO;
 73		goto err;
 74	}
 75
 76	if (range_overflows(args->offset, args->size, (u64)obj->base.size)) {
 77		addr = -EINVAL;
 78		goto err;
 79	}
 80
 81	addr = vm_mmap(obj->base.filp, 0, args->size,
 82		       PROT_READ | PROT_WRITE, MAP_SHARED,
 83		       args->offset);
 84	if (IS_ERR_VALUE(addr))
 85		goto err;
 86
 87	if (args->flags & I915_MMAP_WC) {
 88		struct mm_struct *mm = current->mm;
 89		struct vm_area_struct *vma;
 90
 91		if (down_write_killable(&mm->mmap_sem)) {
 92			addr = -EINTR;
 93			goto err;
 94		}
 95		vma = find_vma(mm, addr);
 96		if (vma && __vma_matches(vma, obj->base.filp, addr, args->size))
 97			vma->vm_page_prot =
 98				pgprot_writecombine(vm_get_page_prot(vma->vm_flags));
 99		else
100			addr = -ENOMEM;
101		up_write(&mm->mmap_sem);
102		if (IS_ERR_VALUE(addr))
103			goto err;
104	}
105	i915_gem_object_put(obj);
106
107	args->addr_ptr = (u64)addr;
108	return 0;
109
110err:
111	i915_gem_object_put(obj);
112	return addr;
113}
114
115static unsigned int tile_row_pages(const struct drm_i915_gem_object *obj)
116{
117	return i915_gem_object_get_tile_row_size(obj) >> PAGE_SHIFT;
118}
119
120/**
121 * i915_gem_mmap_gtt_version - report the current feature set for GTT mmaps
122 *
123 * A history of the GTT mmap interface:
124 *
125 * 0 - Everything had to fit into the GTT. Both parties of a memcpy had to
126 *     aligned and suitable for fencing, and still fit into the available
127 *     mappable space left by the pinned display objects. A classic problem
128 *     we called the page-fault-of-doom where we would ping-pong between
129 *     two objects that could not fit inside the GTT and so the memcpy
130 *     would page one object in at the expense of the other between every
131 *     single byte.
132 *
133 * 1 - Objects can be any size, and have any compatible fencing (X Y, or none
134 *     as set via i915_gem_set_tiling() [DRM_I915_GEM_SET_TILING]). If the
135 *     object is too large for the available space (or simply too large
136 *     for the mappable aperture!), a view is created instead and faulted
137 *     into userspace. (This view is aligned and sized appropriately for
138 *     fenced access.)
139 *
140 * 2 - Recognise WC as a separate cache domain so that we can flush the
141 *     delayed writes via GTT before performing direct access via WC.
142 *
143 * 3 - Remove implicit set-domain(GTT) and synchronisation on initial
144 *     pagefault; swapin remains transparent.
145 *
146 * Restrictions:
147 *
148 *  * snoopable objects cannot be accessed via the GTT. It can cause machine
149 *    hangs on some architectures, corruption on others. An attempt to service
150 *    a GTT page fault from a snoopable object will generate a SIGBUS.
151 *
152 *  * the object must be able to fit into RAM (physical memory, though no
153 *    limited to the mappable aperture).
154 *
155 *
156 * Caveats:
157 *
158 *  * a new GTT page fault will synchronize rendering from the GPU and flush
159 *    all data to system memory. Subsequent access will not be synchronized.
160 *
161 *  * all mappings are revoked on runtime device suspend.
162 *
163 *  * there are only 8, 16 or 32 fence registers to share between all users
164 *    (older machines require fence register for display and blitter access
165 *    as well). Contention of the fence registers will cause the previous users
166 *    to be unmapped and any new access will generate new page faults.
167 *
168 *  * running out of memory while servicing a fault may generate a SIGBUS,
169 *    rather than the expected SIGSEGV.
170 */
171int i915_gem_mmap_gtt_version(void)
172{
173	return 3;
174}
175
176static inline struct i915_ggtt_view
177compute_partial_view(const struct drm_i915_gem_object *obj,
178		     pgoff_t page_offset,
179		     unsigned int chunk)
180{
181	struct i915_ggtt_view view;
182
183	if (i915_gem_object_is_tiled(obj))
184		chunk = roundup(chunk, tile_row_pages(obj));
185
186	view.type = I915_GGTT_VIEW_PARTIAL;
187	view.partial.offset = rounddown(page_offset, chunk);
188	view.partial.size =
189		min_t(unsigned int, chunk,
190		      (obj->base.size >> PAGE_SHIFT) - view.partial.offset);
191
192	/* If the partial covers the entire object, just create a normal VMA. */
193	if (chunk >= obj->base.size >> PAGE_SHIFT)
194		view.type = I915_GGTT_VIEW_NORMAL;
195
196	return view;
197}
198
199/**
200 * i915_gem_fault - fault a page into the GTT
201 * @vmf: fault info
202 *
203 * The fault handler is set up by drm_gem_mmap() when a object is GTT mapped
204 * from userspace.  The fault handler takes care of binding the object to
205 * the GTT (if needed), allocating and programming a fence register (again,
206 * only if needed based on whether the old reg is still valid or the object
207 * is tiled) and inserting a new PTE into the faulting process.
208 *
209 * Note that the faulting process may involve evicting existing objects
210 * from the GTT and/or fence registers to make room.  So performance may
211 * suffer if the GTT working set is large or there are few fence registers
212 * left.
213 *
214 * The current feature set supported by i915_gem_fault() and thus GTT mmaps
215 * is exposed via I915_PARAM_MMAP_GTT_VERSION (see i915_gem_mmap_gtt_version).
216 */
217vm_fault_t i915_gem_fault(struct vm_fault *vmf)
218{
219#define MIN_CHUNK_PAGES (SZ_1M >> PAGE_SHIFT)
220	struct vm_area_struct *area = vmf->vma;
221	struct drm_i915_gem_object *obj = to_intel_bo(area->vm_private_data);
222	struct drm_device *dev = obj->base.dev;
223	struct drm_i915_private *i915 = to_i915(dev);
224	struct intel_runtime_pm *rpm = &i915->runtime_pm;
225	struct i915_ggtt *ggtt = &i915->ggtt;
226	bool write = area->vm_flags & VM_WRITE;
227	intel_wakeref_t wakeref;
228	struct i915_vma *vma;
229	pgoff_t page_offset;
230	int srcu;
231	int ret;
232
233	/* Sanity check that we allow writing into this object */
234	if (i915_gem_object_is_readonly(obj) && write)
235		return VM_FAULT_SIGBUS;
236
237	/* We don't use vmf->pgoff since that has the fake offset */
238	page_offset = (vmf->address - area->vm_start) >> PAGE_SHIFT;
239
240	trace_i915_gem_object_fault(obj, page_offset, true, write);
241
242	ret = i915_gem_object_pin_pages(obj);
243	if (ret)
244		goto err;
245
246	wakeref = intel_runtime_pm_get(rpm);
247
248	ret = intel_gt_reset_trylock(ggtt->vm.gt, &srcu);
249	if (ret)
250		goto err_rpm;
251
252	ret = i915_mutex_lock_interruptible(dev);
253	if (ret)
254		goto err_reset;
255
256	/* Access to snoopable pages through the GTT is incoherent. */
257	if (obj->cache_level != I915_CACHE_NONE && !HAS_LLC(i915)) {
258		ret = -EFAULT;
259		goto err_unlock;
260	}
261
262	/* Now pin it into the GTT as needed */
263	vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0,
264				       PIN_MAPPABLE |
265				       PIN_NONBLOCK /* NOWARN */ |
266				       PIN_NOEVICT);
267	if (IS_ERR(vma)) {
268		/* Use a partial view if it is bigger than available space */
269		struct i915_ggtt_view view =
270			compute_partial_view(obj, page_offset, MIN_CHUNK_PAGES);
271		unsigned int flags;
272
273		flags = PIN_MAPPABLE | PIN_NOSEARCH;
274		if (view.type == I915_GGTT_VIEW_NORMAL)
275			flags |= PIN_NONBLOCK; /* avoid warnings for pinned */
276
277		/*
278		 * Userspace is now writing through an untracked VMA, abandon
279		 * all hope that the hardware is able to track future writes.
280		 */
281
282		vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
283		if (IS_ERR(vma)) {
284			flags = PIN_MAPPABLE;
285			view.type = I915_GGTT_VIEW_PARTIAL;
286			vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, flags);
287		}
288	}
289	if (IS_ERR(vma)) {
290		ret = PTR_ERR(vma);
291		goto err_unlock;
292	}
293
294	ret = i915_vma_pin_fence(vma);
295	if (ret)
296		goto err_unpin;
297
298	/* Finally, remap it using the new GTT offset */
299	ret = remap_io_mapping(area,
300			       area->vm_start + (vma->ggtt_view.partial.offset << PAGE_SHIFT),
301			       (ggtt->gmadr.start + vma->node.start) >> PAGE_SHIFT,
302			       min_t(u64, vma->size, area->vm_end - area->vm_start),
303			       &ggtt->iomap);
304	if (ret)
305		goto err_fence;
306
307	assert_rpm_wakelock_held(rpm);
308
309	/* Mark as being mmapped into userspace for later revocation */
310	mutex_lock(&i915->ggtt.vm.mutex);
311	if (!i915_vma_set_userfault(vma) && !obj->userfault_count++)
312		list_add(&obj->userfault_link, &i915->ggtt.userfault_list);
313	mutex_unlock(&i915->ggtt.vm.mutex);
314
315	if (CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND)
316		intel_wakeref_auto(&i915->ggtt.userfault_wakeref,
317				   msecs_to_jiffies_timeout(CONFIG_DRM_I915_USERFAULT_AUTOSUSPEND));
318
319	if (write) {
320		GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
321		i915_vma_set_ggtt_write(vma);
322		obj->mm.dirty = true;
323	}
324
325err_fence:
326	i915_vma_unpin_fence(vma);
327err_unpin:
328	__i915_vma_unpin(vma);
329err_unlock:
330	mutex_unlock(&dev->struct_mutex);
331err_reset:
332	intel_gt_reset_unlock(ggtt->vm.gt, srcu);
333err_rpm:
334	intel_runtime_pm_put(rpm, wakeref);
335	i915_gem_object_unpin_pages(obj);
336err:
337	switch (ret) {
338	case -EIO:
339		/*
340		 * We eat errors when the gpu is terminally wedged to avoid
341		 * userspace unduly crashing (gl has no provisions for mmaps to
342		 * fail). But any other -EIO isn't ours (e.g. swap in failure)
343		 * and so needs to be reported.
344		 */
345		if (!intel_gt_is_wedged(ggtt->vm.gt))
346			return VM_FAULT_SIGBUS;
347		/* else, fall through */
348	case -EAGAIN:
349		/*
350		 * EAGAIN means the gpu is hung and we'll wait for the error
351		 * handler to reset everything when re-faulting in
352		 * i915_mutex_lock_interruptible.
353		 */
354	case 0:
355	case -ERESTARTSYS:
356	case -EINTR:
357	case -EBUSY:
358		/*
359		 * EBUSY is ok: this just means that another thread
360		 * already did the job.
361		 */
362		return VM_FAULT_NOPAGE;
363	case -ENOMEM:
364		return VM_FAULT_OOM;
365	case -ENOSPC:
366	case -EFAULT:
367	case -ENODEV: /* bad object, how did you get here! */
368		return VM_FAULT_SIGBUS;
369	default:
370		WARN_ONCE(ret, "unhandled error in %s: %i\n", __func__, ret);
371		return VM_FAULT_SIGBUS;
372	}
373}
374
375void __i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
376{
377	struct i915_vma *vma;
378
379	GEM_BUG_ON(!obj->userfault_count);
380
381	obj->userfault_count = 0;
382	list_del(&obj->userfault_link);
383	drm_vma_node_unmap(&obj->base.vma_node,
384			   obj->base.dev->anon_inode->i_mapping);
385
386	for_each_ggtt_vma(vma, obj)
387		i915_vma_unset_userfault(vma);
388}
389
390/**
391 * i915_gem_object_release_mmap - remove physical page mappings
392 * @obj: obj in question
393 *
394 * Preserve the reservation of the mmapping with the DRM core code, but
395 * relinquish ownership of the pages back to the system.
396 *
397 * It is vital that we remove the page mapping if we have mapped a tiled
398 * object through the GTT and then lose the fence register due to
399 * resource pressure. Similarly if the object has been moved out of the
400 * aperture, than pages mapped into userspace must be revoked. Removing the
401 * mapping will then trigger a page fault on the next user access, allowing
402 * fixup by i915_gem_fault().
403 */
404void i915_gem_object_release_mmap(struct drm_i915_gem_object *obj)
405{
406	struct drm_i915_private *i915 = to_i915(obj->base.dev);
407	intel_wakeref_t wakeref;
408
409	/* Serialisation between user GTT access and our code depends upon
410	 * revoking the CPU's PTE whilst the mutex is held. The next user
411	 * pagefault then has to wait until we release the mutex.
412	 *
413	 * Note that RPM complicates somewhat by adding an additional
414	 * requirement that operations to the GGTT be made holding the RPM
415	 * wakeref.
416	 */
417	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
418	mutex_lock(&i915->ggtt.vm.mutex);
419
420	if (!obj->userfault_count)
421		goto out;
422
423	__i915_gem_object_release_mmap(obj);
424
425	/* Ensure that the CPU's PTE are revoked and there are not outstanding
426	 * memory transactions from userspace before we return. The TLB
427	 * flushing implied above by changing the PTE above *should* be
428	 * sufficient, an extra barrier here just provides us with a bit
429	 * of paranoid documentation about our requirement to serialise
430	 * memory writes before touching registers / GSM.
431	 */
432	wmb();
433
434out:
435	mutex_unlock(&i915->ggtt.vm.mutex);
436	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
437}
438
439static int create_mmap_offset(struct drm_i915_gem_object *obj)
440{
441	struct drm_i915_private *i915 = to_i915(obj->base.dev);
442	int err;
443
444	err = drm_gem_create_mmap_offset(&obj->base);
445	if (likely(!err))
446		return 0;
447
448	/* Attempt to reap some mmap space from dead objects */
449	do {
450		err = i915_gem_wait_for_idle(i915,
451					     I915_WAIT_INTERRUPTIBLE,
452					     MAX_SCHEDULE_TIMEOUT);
453		if (err)
454			break;
455
456		i915_gem_drain_freed_objects(i915);
457		err = drm_gem_create_mmap_offset(&obj->base);
458		if (!err)
459			break;
460
461	} while (flush_delayed_work(&i915->gem.retire_work));
462
463	return err;
464}
465
466int
467i915_gem_mmap_gtt(struct drm_file *file,
468		  struct drm_device *dev,
469		  u32 handle,
470		  u64 *offset)
471{
472	struct drm_i915_gem_object *obj;
473	int ret;
474
475	obj = i915_gem_object_lookup(file, handle);
476	if (!obj)
477		return -ENOENT;
478
479	if (i915_gem_object_never_bind_ggtt(obj)) {
480		ret = -ENODEV;
481		goto out;
482	}
483
484	ret = create_mmap_offset(obj);
485	if (ret == 0)
486		*offset = drm_vma_node_offset_addr(&obj->base.vma_node);
487
488out:
489	i915_gem_object_put(obj);
490	return ret;
491}
492
493/**
494 * i915_gem_mmap_gtt_ioctl - prepare an object for GTT mmap'ing
495 * @dev: DRM device
496 * @data: GTT mapping ioctl data
497 * @file: GEM object info
498 *
499 * Simply returns the fake offset to userspace so it can mmap it.
500 * The mmap call will end up in drm_gem_mmap(), which will set things
501 * up so we can get faults in the handler above.
502 *
503 * The fault handler will take care of binding the object into the GTT
504 * (since it may have been evicted to make room for something), allocating
505 * a fence register, and mapping the appropriate aperture address into
506 * userspace.
507 */
508int
509i915_gem_mmap_gtt_ioctl(struct drm_device *dev, void *data,
510			struct drm_file *file)
511{
512	struct drm_i915_gem_mmap_gtt *args = data;
513
514	return i915_gem_mmap_gtt(file, dev, args->handle, &args->offset);
515}
516
517#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
518#include "selftests/i915_gem_mman.c"
519#endif