1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2012-2014 Intel Corporation
  5 *
  6  * Based on amdgpu_mn, which bears the following notice:
  7 *
  8 * Copyright 2014 Advanced Micro Devices, Inc.
  9 * All Rights Reserved.
 10 *
 11 * Permission is hereby granted, free of charge, to any person obtaining a
 12 * copy of this software and associated documentation files (the
 13 * "Software"), to deal in the Software without restriction, including
 14 * without limitation the rights to use, copy, modify, merge, publish,
 15 * distribute, sub license, and/or sell copies of the Software, and to
 16 * permit persons to whom the Software is furnished to do so, subject to
 17 * the following conditions:
 18 *
 19 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 20 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 21 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
 22 * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
 23 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
 24 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
 25 * USE OR OTHER DEALINGS IN THE SOFTWARE.
 26 *
 27 * The above copyright notice and this permission notice (including the
 28 * next paragraph) shall be included in all copies or substantial portions
 29 * of the Software.
 30 *
 31 */
 32/*
 33 * Authors:
 34 *    Christian König <christian.koenig@amd.com>
 35 */
 36
 37#include <linux/mmu_context.h>
 38#include <linux/mempolicy.h>
 39#include <linux/swap.h>
 40#include <linux/sched/mm.h>
 41
 42#include "i915_drv.h"
 43#include "i915_gem_ioctls.h"
 44#include "i915_gem_object.h"
 45#include "i915_scatterlist.h"
 46
 47#ifdef CONFIG_MMU_NOTIFIER
 48
 49/**
 50 * i915_gem_userptr_invalidate - callback to notify about mm change
 51 *
 52 * @mni: the range (mm) is about to update
 53 * @range: details on the invalidation
 54 * @cur_seq: Value to pass to mmu_interval_set_seq()
 55 *
 56 * Block for operations on BOs to finish and mark pages as accessed and
 57 * potentially dirty.
 58 */
 59static bool i915_gem_userptr_invalidate(struct mmu_interval_notifier *mni,
 60					const struct mmu_notifier_range *range,
 61					unsigned long cur_seq)
 62{
 63	struct drm_i915_gem_object *obj = container_of(mni, struct drm_i915_gem_object, userptr.notifier);
 64	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 65	long r;
 66
 67	if (!mmu_notifier_range_blockable(range))
 68		return false;
 69
 70	spin_lock(&i915->mm.notifier_lock);
 71
 72	mmu_interval_set_seq(mni, cur_seq);
 73
 74	spin_unlock(&i915->mm.notifier_lock);
 75
 76	/*
 77	 * We don't wait when the process is exiting. This is valid
 78	 * because the object will be cleaned up anyway.
 79	 *
 80	 * This is also temporarily required as a hack, because we
 81	 * cannot currently force non-consistent batch buffers to preempt
 82	 * and reschedule by waiting on it, hanging processes on exit.
 83	 */
 84	if (current->flags & PF_EXITING)
 85		return true;
 86
 87	/* we will unbind on next submission, still have userptr pins */
 88	r = dma_resv_wait_timeout(obj->base.resv, true, false,
 89				  MAX_SCHEDULE_TIMEOUT);
 90	if (r <= 0)
 91		drm_err(&i915->drm, "(%ld) failed to wait for idle\n", r);
 92
 93	return true;
 94}
 95
 96static const struct mmu_interval_notifier_ops i915_gem_userptr_notifier_ops = {
 97	.invalidate = i915_gem_userptr_invalidate,
 98};
 99
100static int
101i915_gem_userptr_init__mmu_notifier(struct drm_i915_gem_object *obj)
102{
103	return mmu_interval_notifier_insert(&obj->userptr.notifier, current->mm,
104					    obj->userptr.ptr, obj->base.size,
105					    &i915_gem_userptr_notifier_ops);
106}
107
108static void i915_gem_object_userptr_drop_ref(struct drm_i915_gem_object *obj)
109{
110	struct drm_i915_private *i915 = to_i915(obj->base.dev);
111	struct page **pvec = NULL;
112
113	spin_lock(&i915->mm.notifier_lock);
114	if (!--obj->userptr.page_ref) {
115		pvec = obj->userptr.pvec;
116		obj->userptr.pvec = NULL;
117	}
118	GEM_BUG_ON(obj->userptr.page_ref < 0);
119	spin_unlock(&i915->mm.notifier_lock);
120
121	if (pvec) {
122		const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
123
124		unpin_user_pages(pvec, num_pages);
125		kvfree(pvec);
126	}
127}
128
129static int i915_gem_userptr_get_pages(struct drm_i915_gem_object *obj)
130{
131	struct drm_i915_private *i915 = to_i915(obj->base.dev);
132	const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
133	unsigned int max_segment = i915_sg_segment_size();
134	struct sg_table *st;
135	unsigned int sg_page_sizes;
136	struct scatterlist *sg;
137	struct page **pvec;
138	int ret;
139
140	st = kmalloc(sizeof(*st), GFP_KERNEL);
141	if (!st)
142		return -ENOMEM;
143
144	spin_lock(&i915->mm.notifier_lock);
145	if (GEM_WARN_ON(!obj->userptr.page_ref)) {
146		spin_unlock(&i915->mm.notifier_lock);
147		ret = -EFAULT;
148		goto err_free;
149	}
150
151	obj->userptr.page_ref++;
152	pvec = obj->userptr.pvec;
153	spin_unlock(&i915->mm.notifier_lock);
154
155alloc_table:
156	sg = __sg_alloc_table_from_pages(st, pvec, num_pages, 0,
157					 num_pages << PAGE_SHIFT, max_segment,
158					 NULL, 0, GFP_KERNEL);
159	if (IS_ERR(sg)) {
160		ret = PTR_ERR(sg);
161		goto err;
162	}
163
164	ret = i915_gem_gtt_prepare_pages(obj, st);
165	if (ret) {
166		sg_free_table(st);
167
168		if (max_segment > PAGE_SIZE) {
169			max_segment = PAGE_SIZE;
170			goto alloc_table;
171		}
172
173		goto err;
174	}
175
176	sg_page_sizes = i915_sg_dma_sizes(st->sgl);
177
178	__i915_gem_object_set_pages(obj, st, sg_page_sizes);
179
180	return 0;
181
182err:
183	i915_gem_object_userptr_drop_ref(obj);
184err_free:
185	kfree(st);
186	return ret;
187}
188
189static void
190i915_gem_userptr_put_pages(struct drm_i915_gem_object *obj,
191			   struct sg_table *pages)
192{
193	struct sgt_iter sgt_iter;
194	struct page *page;
195
196	if (!pages)
197		return;
198
199	__i915_gem_object_release_shmem(obj, pages, true);
200	i915_gem_gtt_finish_pages(obj, pages);
201
202	/*
203	 * We always mark objects as dirty when they are used by the GPU,
204	 * just in case. However, if we set the vma as being read-only we know
205	 * that the object will never have been written to.
206	 */
207	if (i915_gem_object_is_readonly(obj))
208		obj->mm.dirty = false;
209
210	for_each_sgt_page(page, sgt_iter, pages) {
211		if (obj->mm.dirty && trylock_page(page)) {
212			/*
213			 * As this may not be anonymous memory (e.g. shmem)
214			 * but exist on a real mapping, we have to lock
215			 * the page in order to dirty it -- holding
216			 * the page reference is not sufficient to
217			 * prevent the inode from being truncated.
218			 * Play safe and take the lock.
219			 *
220			 * However...!
221			 *
222			 * The mmu-notifier can be invalidated for a
223			 * migrate_page, that is alreadying holding the lock
224			 * on the page. Such a try_to_unmap() will result
225			 * in us calling put_pages() and so recursively try
226			 * to lock the page. We avoid that deadlock with
227			 * a trylock_page() and in exchange we risk missing
228			 * some page dirtying.
229			 */
230			set_page_dirty(page);
231			unlock_page(page);
232		}
233
234		mark_page_accessed(page);
235	}
236	obj->mm.dirty = false;
237
238	sg_free_table(pages);
239	kfree(pages);
240
241	i915_gem_object_userptr_drop_ref(obj);
242}
243
244static int i915_gem_object_userptr_unbind(struct drm_i915_gem_object *obj, bool get_pages)
245{
246	struct sg_table *pages;
247	int err;
248
249	err = i915_gem_object_unbind(obj, I915_GEM_OBJECT_UNBIND_ACTIVE);
250	if (err)
251		return err;
252
253	if (GEM_WARN_ON(i915_gem_object_has_pinned_pages(obj)))
254		return -EBUSY;
255
256	assert_object_held(obj);
257
258	pages = __i915_gem_object_unset_pages(obj);
259	if (!IS_ERR_OR_NULL(pages))
260		i915_gem_userptr_put_pages(obj, pages);
261
262	if (get_pages)
263		err = ____i915_gem_object_get_pages(obj);
264
265	return err;
266}
267
268int i915_gem_object_userptr_submit_init(struct drm_i915_gem_object *obj)
269{
270	struct drm_i915_private *i915 = to_i915(obj->base.dev);
271	const unsigned long num_pages = obj->base.size >> PAGE_SHIFT;
272	struct page **pvec;
273	unsigned int gup_flags = 0;
274	unsigned long notifier_seq;
275	int pinned, ret;
276
277	if (obj->userptr.notifier.mm != current->mm)
278		return -EFAULT;
279
280	ret = i915_gem_object_lock_interruptible(obj, NULL);
281	if (ret)
282		return ret;
283
284	/* optimistically try to preserve current pages while unlocked */
285	if (i915_gem_object_has_pages(obj) &&
286	    !mmu_interval_check_retry(&obj->userptr.notifier,
287				      obj->userptr.notifier_seq)) {
288		spin_lock(&i915->mm.notifier_lock);
289		if (obj->userptr.pvec &&
290		    !mmu_interval_read_retry(&obj->userptr.notifier,
291					     obj->userptr.notifier_seq)) {
292			obj->userptr.page_ref++;
293
294			/* We can keep using the current binding, this is the fastpath */
295			ret = 1;
296		}
297		spin_unlock(&i915->mm.notifier_lock);
298	}
299
300	if (!ret) {
301		/* Make sure userptr is unbound for next attempt, so we don't use stale pages. */
302		ret = i915_gem_object_userptr_unbind(obj, false);
303	}
304	i915_gem_object_unlock(obj);
305	if (ret < 0)
306		return ret;
307
308	if (ret > 0)
309		return 0;
310
311	notifier_seq = mmu_interval_read_begin(&obj->userptr.notifier);
312
313	pvec = kvmalloc_array(num_pages, sizeof(struct page *), GFP_KERNEL);
314	if (!pvec)
315		return -ENOMEM;
316
317	if (!i915_gem_object_is_readonly(obj))
318		gup_flags |= FOLL_WRITE;
319
320	pinned = ret = 0;
321	while (pinned < num_pages) {
322		ret = pin_user_pages_fast(obj->userptr.ptr + pinned * PAGE_SIZE,
323					  num_pages - pinned, gup_flags,
324					  &pvec[pinned]);
325		if (ret < 0)
326			goto out;
327
328		pinned += ret;
329	}
330	ret = 0;
331
332	spin_lock(&i915->mm.notifier_lock);
333
334	if (mmu_interval_read_retry(&obj->userptr.notifier,
335		!obj->userptr.page_ref ? notifier_seq :
336		obj->userptr.notifier_seq)) {
337		ret = -EAGAIN;
338		goto out_unlock;
339	}
340
341	if (!obj->userptr.page_ref++) {
342		obj->userptr.pvec = pvec;
343		obj->userptr.notifier_seq = notifier_seq;
344
345		pvec = NULL;
346	}
347
348out_unlock:
349	spin_unlock(&i915->mm.notifier_lock);
350
351out:
352	if (pvec) {
353		unpin_user_pages(pvec, pinned);
354		kvfree(pvec);
355	}
356
357	return ret;
358}
359
360int i915_gem_object_userptr_submit_done(struct drm_i915_gem_object *obj)
361{
362	if (mmu_interval_read_retry(&obj->userptr.notifier,
363				    obj->userptr.notifier_seq)) {
364		/* We collided with the mmu notifier, need to retry */
365
366		return -EAGAIN;
367	}
368
369	return 0;
370}
371
372void i915_gem_object_userptr_submit_fini(struct drm_i915_gem_object *obj)
373{
374	i915_gem_object_userptr_drop_ref(obj);
375}
376
377int i915_gem_object_userptr_validate(struct drm_i915_gem_object *obj)
378{
379	int err;
380
381	err = i915_gem_object_userptr_submit_init(obj);
382	if (err)
383		return err;
384
385	err = i915_gem_object_lock_interruptible(obj, NULL);
386	if (!err) {
387		/*
388		 * Since we only check validity, not use the pages,
389		 * it doesn't matter if we collide with the mmu notifier,
390		 * and -EAGAIN handling is not required.
391		 */
392		err = i915_gem_object_pin_pages(obj);
393		if (!err)
394			i915_gem_object_unpin_pages(obj);
395
396		i915_gem_object_unlock(obj);
397	}
398
399	i915_gem_object_userptr_submit_fini(obj);
400	return err;
401}
402
403static void
404i915_gem_userptr_release(struct drm_i915_gem_object *obj)
405{
406	GEM_WARN_ON(obj->userptr.page_ref);
407
408	mmu_interval_notifier_remove(&obj->userptr.notifier);
409	obj->userptr.notifier.mm = NULL;
410}
411
412static int
413i915_gem_userptr_dmabuf_export(struct drm_i915_gem_object *obj)
414{
415	drm_dbg(obj->base.dev, "Exporting userptr no longer allowed\n");
416
417	return -EINVAL;
418}
419
420static int
421i915_gem_userptr_pwrite(struct drm_i915_gem_object *obj,
422			const struct drm_i915_gem_pwrite *args)
423{
424	drm_dbg(obj->base.dev, "pwrite to userptr no longer allowed\n");
425
426	return -EINVAL;
427}
428
429static int
430i915_gem_userptr_pread(struct drm_i915_gem_object *obj,
431		       const struct drm_i915_gem_pread *args)
432{
433	drm_dbg(obj->base.dev, "pread from userptr no longer allowed\n");
434
435	return -EINVAL;
436}
437
438static const struct drm_i915_gem_object_ops i915_gem_userptr_ops = {
439	.name = "i915_gem_object_userptr",
440	.flags = I915_GEM_OBJECT_IS_SHRINKABLE |
441		 I915_GEM_OBJECT_NO_MMAP |
442		 I915_GEM_OBJECT_IS_PROXY,
443	.get_pages = i915_gem_userptr_get_pages,
444	.put_pages = i915_gem_userptr_put_pages,
445	.dmabuf_export = i915_gem_userptr_dmabuf_export,
446	.pwrite = i915_gem_userptr_pwrite,
447	.pread = i915_gem_userptr_pread,
448	.release = i915_gem_userptr_release,
449};
450
451#endif
452
453/*
454 * Creates a new mm object that wraps some normal memory from the process
455 * context - user memory.
456 *
457 * We impose several restrictions upon the memory being mapped
458 * into the GPU.
459 * 1. It must be page aligned (both start/end addresses, i.e ptr and size).
460 * 2. It must be normal system memory, not a pointer into another map of IO
461 *    space (e.g. it must not be a GTT mmapping of another object).
462 * 3. We only allow a bo as large as we could in theory map into the GTT,
463 *    that is we limit the size to the total size of the GTT.
464 * 4. The bo is marked as being snoopable. The backing pages are left
465 *    accessible directly by the CPU, but reads and writes by the GPU may
466 *    incur the cost of a snoop (unless you have an LLC architecture).
467 *
468 * Synchronisation between multiple users and the GPU is left to userspace
469 * through the normal set-domain-ioctl. The kernel will enforce that the
470 * GPU relinquishes the VMA before it is returned back to the system
471 * i.e. upon free(), munmap() or process termination. However, the userspace
472 * malloc() library may not immediately relinquish the VMA after free() and
473 * instead reuse it whilst the GPU is still reading and writing to the VMA.
474 * Caveat emptor.
475 *
476 * Also note, that the object created here is not currently a "first class"
477 * object, in that several ioctls are banned. These are the CPU access
478 * ioctls: mmap(), pwrite and pread. In practice, you are expected to use
479 * direct access via your pointer rather than use those ioctls. Another
480 * restriction is that we do not allow userptr surfaces to be pinned to the
481 * hardware and so we reject any attempt to create a framebuffer out of a
482 * userptr.
483 *
484 * If you think this is a good interface to use to pass GPU memory between
485 * drivers, please use dma-buf instead. In fact, wherever possible use
486 * dma-buf instead.
487 */
488int
489i915_gem_userptr_ioctl(struct drm_device *dev,
490		       void *data,
491		       struct drm_file *file)
492{
493	static struct lock_class_key __maybe_unused lock_class;
494	struct drm_i915_private *dev_priv = to_i915(dev);
495	struct drm_i915_gem_userptr *args = data;
496	struct drm_i915_gem_object __maybe_unused *obj;
497	int __maybe_unused ret;
498	u32 __maybe_unused handle;
499
500	if (!HAS_LLC(dev_priv) && !HAS_SNOOP(dev_priv)) {
501		/* We cannot support coherent userptr objects on hw without
502		 * LLC and broken snooping.
503		 */
504		return -ENODEV;
505	}
506
507	if (args->flags & ~(I915_USERPTR_READ_ONLY |
508			    I915_USERPTR_UNSYNCHRONIZED))
509		return -EINVAL;
510
511	if (i915_gem_object_size_2big(args->user_size))
512		return -E2BIG;
513
514	if (!args->user_size)
515		return -EINVAL;
516
517	if (offset_in_page(args->user_ptr | args->user_size))
518		return -EINVAL;
519
520	if (!access_ok((char __user *)(unsigned long)args->user_ptr, args->user_size))
521		return -EFAULT;
522
523	if (args->flags & I915_USERPTR_UNSYNCHRONIZED)
524		return -ENODEV;
525
526	if (args->flags & I915_USERPTR_READ_ONLY) {
527		/*
528		 * On almost all of the older hw, we cannot tell the GPU that
529		 * a page is readonly.
530		 */
531		if (!dev_priv->gt.vm->has_read_only)
532			return -ENODEV;
533	}
534
535#ifdef CONFIG_MMU_NOTIFIER
536	obj = i915_gem_object_alloc();
537	if (obj == NULL)
538		return -ENOMEM;
539
540	drm_gem_private_object_init(dev, &obj->base, args->user_size);
541	i915_gem_object_init(obj, &i915_gem_userptr_ops, &lock_class,
542			     I915_BO_ALLOC_STRUCT_PAGE);
543	obj->read_domains = I915_GEM_DOMAIN_CPU;
544	obj->write_domain = I915_GEM_DOMAIN_CPU;
545	i915_gem_object_set_cache_coherency(obj, I915_CACHE_LLC);
546
547	obj->userptr.ptr = args->user_ptr;
548	obj->userptr.notifier_seq = ULONG_MAX;
549	if (args->flags & I915_USERPTR_READ_ONLY)
550		i915_gem_object_set_readonly(obj);
551
552	/* And keep a pointer to the current->mm for resolving the user pages
553	 * at binding. This means that we need to hook into the mmu_notifier
554	 * in order to detect if the mmu is destroyed.
555	 */
556	ret = i915_gem_userptr_init__mmu_notifier(obj);
557	if (ret == 0)
558		ret = drm_gem_handle_create(file, &obj->base, &handle);
559
560	/* drop reference from allocate - handle holds it now */
561	i915_gem_object_put(obj);
562	if (ret)
563		return ret;
564
565	args->handle = handle;
566	return 0;
567#else
568	return -ENODEV;
569#endif
570}
571
572int i915_gem_init_userptr(struct drm_i915_private *dev_priv)
573{
574#ifdef CONFIG_MMU_NOTIFIER
575	spin_lock_init(&dev_priv->mm.notifier_lock);
576#endif
577
578	return 0;
579}
580
581void i915_gem_cleanup_userptr(struct drm_i915_private *dev_priv)
582{
583}