Linux Audio

Check our new training course

Loading...
v6.2
   1/*
   2 * Copyright © 2008-2015 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * Authors:
  24 *    Eric Anholt <eric@anholt.net>
  25 *
  26 */
  27
 
  28#include <linux/dma-fence-array.h>
  29#include <linux/kthread.h>
  30#include <linux/dma-resv.h>
  31#include <linux/shmem_fs.h>
  32#include <linux/slab.h>
  33#include <linux/stop_machine.h>
  34#include <linux/swap.h>
  35#include <linux/pci.h>
  36#include <linux/dma-buf.h>
  37#include <linux/mman.h>
  38
  39#include <drm/drm_cache.h>
  40#include <drm/drm_vma_manager.h>
  41
  42#include "display/intel_display.h"
  43#include "display/intel_frontbuffer.h"
  44
  45#include "gem/i915_gem_clflush.h"
  46#include "gem/i915_gem_context.h"
  47#include "gem/i915_gem_ioctls.h"
  48#include "gem/i915_gem_mman.h"
  49#include "gem/i915_gem_pm.h"
  50#include "gem/i915_gem_region.h"
  51#include "gem/i915_gem_userptr.h"
  52#include "gt/intel_engine_user.h"
  53#include "gt/intel_gt.h"
  54#include "gt/intel_gt_pm.h"
  55#include "gt/intel_workarounds.h"
  56
  57#include "i915_drv.h"
  58#include "i915_file_private.h"
  59#include "i915_trace.h"
  60#include "i915_vgpu.h"
 
  61#include "intel_pm.h"
  62
  63static int
  64insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  65{
  66	int err;
  67
  68	err = mutex_lock_interruptible(&ggtt->vm.mutex);
  69	if (err)
  70		return err;
  71
  72	memset(node, 0, sizeof(*node));
  73	err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  74					  size, 0, I915_COLOR_UNEVICTABLE,
  75					  0, ggtt->mappable_end,
  76					  DRM_MM_INSERT_LOW);
  77
  78	mutex_unlock(&ggtt->vm.mutex);
  79
  80	return err;
  81}
  82
  83static void
  84remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  85{
  86	mutex_lock(&ggtt->vm.mutex);
  87	drm_mm_remove_node(node);
  88	mutex_unlock(&ggtt->vm.mutex);
  89}
  90
  91int
  92i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  93			    struct drm_file *file)
  94{
  95	struct drm_i915_private *i915 = to_i915(dev);
  96	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
  97	struct drm_i915_gem_get_aperture *args = data;
  98	struct i915_vma *vma;
  99	u64 pinned;
 100
 101	if (mutex_lock_interruptible(&ggtt->vm.mutex))
 102		return -EINTR;
 103
 104	pinned = ggtt->vm.reserved;
 105	list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 106		if (i915_vma_is_pinned(vma))
 107			pinned += vma->node.size;
 108
 109	mutex_unlock(&ggtt->vm.mutex);
 110
 111	args->aper_size = ggtt->vm.total;
 112	args->aper_available_size = args->aper_size - pinned;
 113
 114	return 0;
 115}
 116
 117int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 118			   unsigned long flags)
 119{
 120	struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
 121	bool vm_trylock = !!(flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK);
 122	LIST_HEAD(still_in_list);
 123	intel_wakeref_t wakeref;
 124	struct i915_vma *vma;
 125	int ret;
 126
 127	assert_object_held(obj);
 128
 129	if (list_empty(&obj->vma.list))
 130		return 0;
 131
 132	/*
 133	 * As some machines use ACPI to handle runtime-resume callbacks, and
 134	 * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
 135	 * as they are required by the shrinker. Ergo, we wake the device up
 136	 * first just in case.
 137	 */
 138	wakeref = intel_runtime_pm_get(rpm);
 139
 140try_again:
 141	ret = 0;
 142	spin_lock(&obj->vma.lock);
 143	while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 144						       struct i915_vma,
 145						       obj_link))) {
 
 
 146		list_move_tail(&vma->obj_link, &still_in_list);
 147		if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
 148			continue;
 149
 150		if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
 151			ret = -EBUSY;
 152			break;
 153		}
 154
 155		/*
 156		 * Requiring the vm destructor to take the object lock
 157		 * before destroying a vma would help us eliminate the
 158		 * i915_vm_tryget() here, AND thus also the barrier stuff
 159		 * at the end. That's an easy fix, but sleeping locks in
 160		 * a kthread should generally be avoided.
 161		 */
 162		ret = -EAGAIN;
 163		if (!i915_vm_tryget(vma->vm))
 164			break;
 165
 
 
 166		spin_unlock(&obj->vma.lock);
 167
 168		/*
 169		 * Since i915_vma_parked() takes the object lock
 170		 * before vma destruction, it won't race us here,
 171		 * and destroy the vma from under us.
 172		 */
 173
 174		ret = -EBUSY;
 175		if (flags & I915_GEM_OBJECT_UNBIND_ASYNC) {
 176			assert_object_held(vma->obj);
 177			ret = i915_vma_unbind_async(vma, vm_trylock);
 178		}
 179
 180		if (ret == -EBUSY && (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 181				      !i915_vma_is_active(vma))) {
 182			if (vm_trylock) {
 183				if (mutex_trylock(&vma->vm->mutex)) {
 184					ret = __i915_vma_unbind(vma);
 185					mutex_unlock(&vma->vm->mutex);
 186				}
 187			} else {
 188				ret = i915_vma_unbind(vma);
 189			}
 
 
 190		}
 191
 192		i915_vm_put(vma->vm);
 193		spin_lock(&obj->vma.lock);
 194	}
 195	list_splice_init(&still_in_list, &obj->vma.list);
 196	spin_unlock(&obj->vma.lock);
 197
 198	if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
 199		rcu_barrier(); /* flush the i915_vm_release() */
 200		goto try_again;
 201	}
 202
 203	intel_runtime_pm_put(rpm, wakeref);
 204
 205	return ret;
 206}
 207
 208static int
 209shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 210	    bool needs_clflush)
 211{
 212	char *vaddr;
 213	int ret;
 214
 215	vaddr = kmap(page);
 216
 217	if (needs_clflush)
 218		drm_clflush_virt_range(vaddr + offset, len);
 219
 220	ret = __copy_to_user(user_data, vaddr + offset, len);
 221
 222	kunmap(page);
 223
 224	return ret ? -EFAULT : 0;
 225}
 226
 227static int
 228i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 229		     struct drm_i915_gem_pread *args)
 230{
 231	unsigned int needs_clflush;
 232	unsigned int idx, offset;
 233	char __user *user_data;
 234	u64 remain;
 235	int ret;
 236
 237	ret = i915_gem_object_lock_interruptible(obj, NULL);
 238	if (ret)
 239		return ret;
 240
 241	ret = i915_gem_object_pin_pages(obj);
 242	if (ret)
 243		goto err_unlock;
 244
 245	ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 246	if (ret)
 247		goto err_unpin;
 248
 249	i915_gem_object_finish_access(obj);
 250	i915_gem_object_unlock(obj);
 251
 252	remain = args->size;
 253	user_data = u64_to_user_ptr(args->data_ptr);
 254	offset = offset_in_page(args->offset);
 255	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 256		struct page *page = i915_gem_object_get_page(obj, idx);
 257		unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 258
 259		ret = shmem_pread(page, offset, length, user_data,
 260				  needs_clflush);
 261		if (ret)
 262			break;
 263
 264		remain -= length;
 265		user_data += length;
 266		offset = 0;
 267	}
 268
 269	i915_gem_object_unpin_pages(obj);
 270	return ret;
 271
 272err_unpin:
 273	i915_gem_object_unpin_pages(obj);
 274err_unlock:
 275	i915_gem_object_unlock(obj);
 276	return ret;
 277}
 278
 279static inline bool
 280gtt_user_read(struct io_mapping *mapping,
 281	      loff_t base, int offset,
 282	      char __user *user_data, int length)
 283{
 284	void __iomem *vaddr;
 285	unsigned long unwritten;
 286
 287	/* We can use the cpu mem copy function because this is X86. */
 288	vaddr = io_mapping_map_atomic_wc(mapping, base);
 289	unwritten = __copy_to_user_inatomic(user_data,
 290					    (void __force *)vaddr + offset,
 291					    length);
 292	io_mapping_unmap_atomic(vaddr);
 293	if (unwritten) {
 294		vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 295		unwritten = copy_to_user(user_data,
 296					 (void __force *)vaddr + offset,
 297					 length);
 298		io_mapping_unmap(vaddr);
 299	}
 300	return unwritten;
 301}
 302
 303static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
 304					     struct drm_mm_node *node,
 305					     bool write)
 306{
 307	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 308	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 309	struct i915_vma *vma;
 310	struct i915_gem_ww_ctx ww;
 311	int ret;
 312
 313	i915_gem_ww_ctx_init(&ww, true);
 314retry:
 315	vma = ERR_PTR(-ENODEV);
 316	ret = i915_gem_object_lock(obj, &ww);
 317	if (ret)
 318		goto err_ww;
 319
 320	ret = i915_gem_object_set_to_gtt_domain(obj, write);
 321	if (ret)
 322		goto err_ww;
 323
 324	if (!i915_gem_object_is_tiled(obj))
 325		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
 326						  PIN_MAPPABLE |
 327						  PIN_NONBLOCK /* NOWARN */ |
 328						  PIN_NOEVICT);
 329	if (vma == ERR_PTR(-EDEADLK)) {
 330		ret = -EDEADLK;
 331		goto err_ww;
 332	} else if (!IS_ERR(vma)) {
 333		node->start = i915_ggtt_offset(vma);
 334		node->flags = 0;
 335	} else {
 336		ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
 337		if (ret)
 338			goto err_ww;
 339		GEM_BUG_ON(!drm_mm_node_allocated(node));
 340		vma = NULL;
 341	}
 342
 343	ret = i915_gem_object_pin_pages(obj);
 344	if (ret) {
 345		if (drm_mm_node_allocated(node)) {
 346			ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 347			remove_mappable_node(ggtt, node);
 348		} else {
 349			i915_vma_unpin(vma);
 350		}
 351	}
 352
 353err_ww:
 354	if (ret == -EDEADLK) {
 355		ret = i915_gem_ww_ctx_backoff(&ww);
 356		if (!ret)
 357			goto retry;
 358	}
 359	i915_gem_ww_ctx_fini(&ww);
 360
 361	return ret ? ERR_PTR(ret) : vma;
 362}
 363
 364static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
 365				 struct drm_mm_node *node,
 366				 struct i915_vma *vma)
 367{
 368	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 369	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 370
 371	i915_gem_object_unpin_pages(obj);
 372	if (drm_mm_node_allocated(node)) {
 373		ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 374		remove_mappable_node(ggtt, node);
 375	} else {
 376		i915_vma_unpin(vma);
 377	}
 378}
 379
 380static int
 381i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 382		   const struct drm_i915_gem_pread *args)
 383{
 384	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 385	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 386	intel_wakeref_t wakeref;
 387	struct drm_mm_node node;
 388	void __user *user_data;
 389	struct i915_vma *vma;
 390	u64 remain, offset;
 391	int ret = 0;
 392
 393	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 394
 395	vma = i915_gem_gtt_prepare(obj, &node, false);
 396	if (IS_ERR(vma)) {
 397		ret = PTR_ERR(vma);
 398		goto out_rpm;
 399	}
 400
 401	user_data = u64_to_user_ptr(args->data_ptr);
 402	remain = args->size;
 403	offset = args->offset;
 404
 405	while (remain > 0) {
 406		/* Operation in this page
 407		 *
 408		 * page_base = page offset within aperture
 409		 * page_offset = offset within page
 410		 * page_length = bytes to copy for this page
 411		 */
 412		u32 page_base = node.start;
 413		unsigned page_offset = offset_in_page(offset);
 414		unsigned page_length = PAGE_SIZE - page_offset;
 415		page_length = remain < page_length ? remain : page_length;
 416		if (drm_mm_node_allocated(&node)) {
 417			ggtt->vm.insert_page(&ggtt->vm,
 418					     i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 419					     node.start, I915_CACHE_NONE, 0);
 420		} else {
 421			page_base += offset & PAGE_MASK;
 422		}
 423
 424		if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 425				  user_data, page_length)) {
 426			ret = -EFAULT;
 427			break;
 428		}
 429
 430		remain -= page_length;
 431		user_data += page_length;
 432		offset += page_length;
 433	}
 434
 435	i915_gem_gtt_cleanup(obj, &node, vma);
 436out_rpm:
 437	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 438	return ret;
 439}
 440
 441/**
 442 * Reads data from the object referenced by handle.
 443 * @dev: drm device pointer
 444 * @data: ioctl data blob
 445 * @file: drm file pointer
 446 *
 447 * On error, the contents of *data are undefined.
 448 */
 449int
 450i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 451		     struct drm_file *file)
 452{
 453	struct drm_i915_private *i915 = to_i915(dev);
 454	struct drm_i915_gem_pread *args = data;
 455	struct drm_i915_gem_object *obj;
 456	int ret;
 457
 458	/* PREAD is disallowed for all platforms after TGL-LP.  This also
 459	 * covers all platforms with local memory.
 460	 */
 461	if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 462		return -EOPNOTSUPP;
 463
 464	if (args->size == 0)
 465		return 0;
 466
 467	if (!access_ok(u64_to_user_ptr(args->data_ptr),
 468		       args->size))
 469		return -EFAULT;
 470
 471	obj = i915_gem_object_lookup(file, args->handle);
 472	if (!obj)
 473		return -ENOENT;
 474
 475	/* Bounds check source.  */
 476	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 477		ret = -EINVAL;
 478		goto out;
 479	}
 480
 481	trace_i915_gem_object_pread(obj, args->offset, args->size);
 482	ret = -ENODEV;
 483	if (obj->ops->pread)
 484		ret = obj->ops->pread(obj, args);
 485	if (ret != -ENODEV)
 486		goto out;
 487
 
 
 
 
 
 
 488	ret = i915_gem_object_wait(obj,
 489				   I915_WAIT_INTERRUPTIBLE,
 490				   MAX_SCHEDULE_TIMEOUT);
 491	if (ret)
 492		goto out;
 493
 494	ret = i915_gem_shmem_pread(obj, args);
 495	if (ret == -EFAULT || ret == -ENODEV)
 496		ret = i915_gem_gtt_pread(obj, args);
 497
 498out:
 499	i915_gem_object_put(obj);
 500	return ret;
 501}
 502
 503/* This is the fast write path which cannot handle
 504 * page faults in the source data
 505 */
 506
 507static inline bool
 508ggtt_write(struct io_mapping *mapping,
 509	   loff_t base, int offset,
 510	   char __user *user_data, int length)
 511{
 512	void __iomem *vaddr;
 513	unsigned long unwritten;
 514
 515	/* We can use the cpu mem copy function because this is X86. */
 516	vaddr = io_mapping_map_atomic_wc(mapping, base);
 517	unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
 518						      user_data, length);
 519	io_mapping_unmap_atomic(vaddr);
 520	if (unwritten) {
 521		vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 522		unwritten = copy_from_user((void __force *)vaddr + offset,
 523					   user_data, length);
 524		io_mapping_unmap(vaddr);
 525	}
 526
 527	return unwritten;
 528}
 529
 530/**
 531 * This is the fast pwrite path, where we copy the data directly from the
 532 * user into the GTT, uncached.
 533 * @obj: i915 GEM object
 534 * @args: pwrite arguments structure
 535 */
 536static int
 537i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 538			 const struct drm_i915_gem_pwrite *args)
 539{
 540	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 541	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 542	struct intel_runtime_pm *rpm = &i915->runtime_pm;
 543	intel_wakeref_t wakeref;
 544	struct drm_mm_node node;
 545	struct i915_vma *vma;
 546	u64 remain, offset;
 547	void __user *user_data;
 548	int ret = 0;
 549
 550	if (i915_gem_object_has_struct_page(obj)) {
 551		/*
 552		 * Avoid waking the device up if we can fallback, as
 553		 * waking/resuming is very slow (worst-case 10-100 ms
 554		 * depending on PCI sleeps and our own resume time).
 555		 * This easily dwarfs any performance advantage from
 556		 * using the cache bypass of indirect GGTT access.
 557		 */
 558		wakeref = intel_runtime_pm_get_if_in_use(rpm);
 559		if (!wakeref)
 560			return -EFAULT;
 561	} else {
 562		/* No backing pages, no fallback, we must force GGTT access */
 563		wakeref = intel_runtime_pm_get(rpm);
 564	}
 565
 566	vma = i915_gem_gtt_prepare(obj, &node, true);
 567	if (IS_ERR(vma)) {
 568		ret = PTR_ERR(vma);
 569		goto out_rpm;
 570	}
 571
 572	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 573
 574	user_data = u64_to_user_ptr(args->data_ptr);
 575	offset = args->offset;
 576	remain = args->size;
 577	while (remain) {
 578		/* Operation in this page
 579		 *
 580		 * page_base = page offset within aperture
 581		 * page_offset = offset within page
 582		 * page_length = bytes to copy for this page
 583		 */
 584		u32 page_base = node.start;
 585		unsigned int page_offset = offset_in_page(offset);
 586		unsigned int page_length = PAGE_SIZE - page_offset;
 587		page_length = remain < page_length ? remain : page_length;
 588		if (drm_mm_node_allocated(&node)) {
 589			/* flush the write before we modify the GGTT */
 590			intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 591			ggtt->vm.insert_page(&ggtt->vm,
 592					     i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 593					     node.start, I915_CACHE_NONE, 0);
 594			wmb(); /* flush modifications to the GGTT (insert_page) */
 595		} else {
 596			page_base += offset & PAGE_MASK;
 597		}
 598		/* If we get a fault while copying data, then (presumably) our
 599		 * source page isn't available.  Return the error and we'll
 600		 * retry in the slow path.
 601		 * If the object is non-shmem backed, we retry again with the
 602		 * path that handles page fault.
 603		 */
 604		if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 605			       user_data, page_length)) {
 606			ret = -EFAULT;
 607			break;
 608		}
 609
 610		remain -= page_length;
 611		user_data += page_length;
 612		offset += page_length;
 613	}
 614
 615	intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 616	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 617
 618	i915_gem_gtt_cleanup(obj, &node, vma);
 619out_rpm:
 620	intel_runtime_pm_put(rpm, wakeref);
 621	return ret;
 622}
 623
 624/* Per-page copy function for the shmem pwrite fastpath.
 625 * Flushes invalid cachelines before writing to the target if
 626 * needs_clflush_before is set and flushes out any written cachelines after
 627 * writing if needs_clflush is set.
 628 */
 629static int
 630shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 631	     bool needs_clflush_before,
 632	     bool needs_clflush_after)
 633{
 634	char *vaddr;
 635	int ret;
 636
 637	vaddr = kmap(page);
 638
 639	if (needs_clflush_before)
 640		drm_clflush_virt_range(vaddr + offset, len);
 641
 642	ret = __copy_from_user(vaddr + offset, user_data, len);
 643	if (!ret && needs_clflush_after)
 644		drm_clflush_virt_range(vaddr + offset, len);
 645
 646	kunmap(page);
 647
 648	return ret ? -EFAULT : 0;
 649}
 650
 651static int
 652i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 653		      const struct drm_i915_gem_pwrite *args)
 654{
 655	unsigned int partial_cacheline_write;
 656	unsigned int needs_clflush;
 657	unsigned int offset, idx;
 658	void __user *user_data;
 659	u64 remain;
 660	int ret;
 661
 662	ret = i915_gem_object_lock_interruptible(obj, NULL);
 663	if (ret)
 664		return ret;
 665
 666	ret = i915_gem_object_pin_pages(obj);
 667	if (ret)
 668		goto err_unlock;
 669
 670	ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 671	if (ret)
 672		goto err_unpin;
 673
 674	i915_gem_object_finish_access(obj);
 675	i915_gem_object_unlock(obj);
 676
 677	/* If we don't overwrite a cacheline completely we need to be
 678	 * careful to have up-to-date data by first clflushing. Don't
 679	 * overcomplicate things and flush the entire patch.
 680	 */
 681	partial_cacheline_write = 0;
 682	if (needs_clflush & CLFLUSH_BEFORE)
 683		partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 684
 685	user_data = u64_to_user_ptr(args->data_ptr);
 686	remain = args->size;
 687	offset = offset_in_page(args->offset);
 688	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 689		struct page *page = i915_gem_object_get_page(obj, idx);
 690		unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 691
 692		ret = shmem_pwrite(page, offset, length, user_data,
 693				   (offset | length) & partial_cacheline_write,
 694				   needs_clflush & CLFLUSH_AFTER);
 695		if (ret)
 696			break;
 697
 698		remain -= length;
 699		user_data += length;
 700		offset = 0;
 701	}
 702
 703	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 704
 705	i915_gem_object_unpin_pages(obj);
 706	return ret;
 707
 708err_unpin:
 709	i915_gem_object_unpin_pages(obj);
 710err_unlock:
 711	i915_gem_object_unlock(obj);
 712	return ret;
 713}
 714
 715/**
 716 * Writes data to the object referenced by handle.
 717 * @dev: drm device
 718 * @data: ioctl data blob
 719 * @file: drm file
 720 *
 721 * On error, the contents of the buffer that were to be modified are undefined.
 722 */
 723int
 724i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 725		      struct drm_file *file)
 726{
 727	struct drm_i915_private *i915 = to_i915(dev);
 728	struct drm_i915_gem_pwrite *args = data;
 729	struct drm_i915_gem_object *obj;
 730	int ret;
 731
 732	/* PWRITE is disallowed for all platforms after TGL-LP.  This also
 733	 * covers all platforms with local memory.
 734	 */
 735	if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 736		return -EOPNOTSUPP;
 737
 738	if (args->size == 0)
 739		return 0;
 740
 741	if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 742		return -EFAULT;
 743
 744	obj = i915_gem_object_lookup(file, args->handle);
 745	if (!obj)
 746		return -ENOENT;
 747
 748	/* Bounds check destination. */
 749	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 750		ret = -EINVAL;
 751		goto err;
 752	}
 753
 754	/* Writes not allowed into this read-only object */
 755	if (i915_gem_object_is_readonly(obj)) {
 756		ret = -EINVAL;
 757		goto err;
 758	}
 759
 760	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 761
 762	ret = -ENODEV;
 763	if (obj->ops->pwrite)
 764		ret = obj->ops->pwrite(obj, args);
 765	if (ret != -ENODEV)
 766		goto err;
 767
 768	ret = i915_gem_object_wait(obj,
 769				   I915_WAIT_INTERRUPTIBLE |
 770				   I915_WAIT_ALL,
 771				   MAX_SCHEDULE_TIMEOUT);
 772	if (ret)
 773		goto err;
 774
 775	ret = -EFAULT;
 776	/* We can only do the GTT pwrite on untiled buffers, as otherwise
 777	 * it would end up going through the fenced access, and we'll get
 778	 * different detiling behavior between reading and writing.
 779	 * pread/pwrite currently are reading and writing from the CPU
 780	 * perspective, requiring manual detiling by the client.
 781	 */
 782	if (!i915_gem_object_has_struct_page(obj) ||
 783	    i915_gem_cpu_write_needs_clflush(obj))
 784		/* Note that the gtt paths might fail with non-page-backed user
 785		 * pointers (e.g. gtt mappings when moving data between
 786		 * textures). Fallback to the shmem path in that case.
 787		 */
 788		ret = i915_gem_gtt_pwrite_fast(obj, args);
 789
 790	if (ret == -EFAULT || ret == -ENOSPC) {
 791		if (i915_gem_object_has_struct_page(obj))
 792			ret = i915_gem_shmem_pwrite(obj, args);
 793	}
 794
 795err:
 796	i915_gem_object_put(obj);
 797	return ret;
 798}
 799
 800/**
 801 * Called when user space has done writes to this buffer
 802 * @dev: drm device
 803 * @data: ioctl data blob
 804 * @file: drm file
 805 */
 806int
 807i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 808			 struct drm_file *file)
 809{
 810	struct drm_i915_gem_sw_finish *args = data;
 811	struct drm_i915_gem_object *obj;
 812
 813	obj = i915_gem_object_lookup(file, args->handle);
 814	if (!obj)
 815		return -ENOENT;
 816
 817	/*
 818	 * Proxy objects are barred from CPU access, so there is no
 819	 * need to ban sw_finish as it is a nop.
 820	 */
 821
 822	/* Pinned buffers may be scanout, so flush the cache */
 823	i915_gem_object_flush_if_display(obj);
 824	i915_gem_object_put(obj);
 825
 826	return 0;
 827}
 828
 829void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 830{
 831	struct drm_i915_gem_object *obj, *on;
 832	int i;
 833
 834	/*
 835	 * Only called during RPM suspend. All users of the userfault_list
 836	 * must be holding an RPM wakeref to ensure that this can not
 837	 * run concurrently with themselves (and use the struct_mutex for
 838	 * protection between themselves).
 839	 */
 840
 841	list_for_each_entry_safe(obj, on,
 842				 &to_gt(i915)->ggtt->userfault_list, userfault_link)
 843		__i915_gem_object_release_mmap_gtt(obj);
 844
 845	list_for_each_entry_safe(obj, on,
 846				 &i915->runtime_pm.lmem_userfault_list, userfault_link)
 847		i915_gem_object_runtime_pm_release_mmap_offset(obj);
 848
 849	/*
 850	 * The fence will be lost when the device powers down. If any were
 851	 * in use by hardware (i.e. they are pinned), we should not be powering
 852	 * down! All other fences will be reacquired by the user upon waking.
 853	 */
 854	for (i = 0; i < to_gt(i915)->ggtt->num_fences; i++) {
 855		struct i915_fence_reg *reg = &to_gt(i915)->ggtt->fence_regs[i];
 856
 857		/*
 858		 * Ideally we want to assert that the fence register is not
 859		 * live at this point (i.e. that no piece of code will be
 860		 * trying to write through fence + GTT, as that both violates
 861		 * our tracking of activity and associated locking/barriers,
 862		 * but also is illegal given that the hw is powered down).
 863		 *
 864		 * Previously we used reg->pin_count as a "liveness" indicator.
 865		 * That is not sufficient, and we need a more fine-grained
 866		 * tool if we want to have a sanity check here.
 867		 */
 868
 869		if (!reg->vma)
 870			continue;
 871
 872		GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 873		reg->dirty = true;
 874	}
 875}
 876
 877static void discard_ggtt_vma(struct i915_vma *vma)
 878{
 879	struct drm_i915_gem_object *obj = vma->obj;
 880
 881	spin_lock(&obj->vma.lock);
 882	if (!RB_EMPTY_NODE(&vma->obj_node)) {
 883		rb_erase(&vma->obj_node, &obj->vma.tree);
 884		RB_CLEAR_NODE(&vma->obj_node);
 885	}
 886	spin_unlock(&obj->vma.lock);
 887}
 888
 889struct i915_vma *
 890i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 891			    struct i915_gem_ww_ctx *ww,
 892			    const struct i915_gtt_view *view,
 893			    u64 size, u64 alignment, u64 flags)
 894{
 895	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 896	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;
 897	struct i915_vma *vma;
 898	int ret;
 899
 900	GEM_WARN_ON(!ww);
 901
 902	if (flags & PIN_MAPPABLE &&
 903	    (!view || view->type == I915_GTT_VIEW_NORMAL)) {
 904		/*
 905		 * If the required space is larger than the available
 906		 * aperture, we will not able to find a slot for the
 907		 * object and unbinding the object now will be in
 908		 * vain. Worse, doing so may cause us to ping-pong
 909		 * the object in and out of the Global GTT and
 910		 * waste a lot of cycles under the mutex.
 911		 */
 912		if (obj->base.size > ggtt->mappable_end)
 913			return ERR_PTR(-E2BIG);
 914
 915		/*
 916		 * If NONBLOCK is set the caller is optimistically
 917		 * trying to cache the full object within the mappable
 918		 * aperture, and *must* have a fallback in place for
 919		 * situations where we cannot bind the object. We
 920		 * can be a little more lax here and use the fallback
 921		 * more often to avoid costly migrations of ourselves
 922		 * and other objects within the aperture.
 923		 *
 924		 * Half-the-aperture is used as a simple heuristic.
 925		 * More interesting would to do search for a free
 926		 * block prior to making the commitment to unbind.
 927		 * That caters for the self-harm case, and with a
 928		 * little more heuristics (e.g. NOFAULT, NOEVICT)
 929		 * we could try to minimise harm to others.
 930		 */
 931		if (flags & PIN_NONBLOCK &&
 932		    obj->base.size > ggtt->mappable_end / 2)
 933			return ERR_PTR(-ENOSPC);
 934	}
 935
 936new_vma:
 937	vma = i915_vma_instance(obj, &ggtt->vm, view);
 938	if (IS_ERR(vma))
 939		return vma;
 940
 941	if (i915_vma_misplaced(vma, size, alignment, flags)) {
 942		if (flags & PIN_NONBLOCK) {
 943			if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 944				return ERR_PTR(-ENOSPC);
 945
 946			/*
 947			 * If this misplaced vma is too big (i.e, at-least
 948			 * half the size of aperture) or hasn't been pinned
 949			 * mappable before, we ignore the misplacement when
 950			 * PIN_NONBLOCK is set in order to avoid the ping-pong
 951			 * issue described above. In other words, we try to
 952			 * avoid the costly operation of unbinding this vma
 953			 * from the GGTT and rebinding it back because there
 954			 * may not be enough space for this vma in the aperture.
 955			 */
 956			if (flags & PIN_MAPPABLE &&
 957			    (vma->fence_size > ggtt->mappable_end / 2 ||
 958			    !i915_vma_is_map_and_fenceable(vma)))
 959				return ERR_PTR(-ENOSPC);
 960		}
 961
 962		if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
 963			discard_ggtt_vma(vma);
 964			goto new_vma;
 965		}
 966
 967		ret = i915_vma_unbind(vma);
 968		if (ret)
 969			return ERR_PTR(ret);
 970	}
 971
 972	ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
 
 
 
 973
 974	if (ret)
 975		return ERR_PTR(ret);
 976
 977	if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 978		mutex_lock(&ggtt->vm.mutex);
 979		i915_vma_revoke_fence(vma);
 980		mutex_unlock(&ggtt->vm.mutex);
 981	}
 982
 983	ret = i915_vma_wait_for_bind(vma);
 984	if (ret) {
 985		i915_vma_unpin(vma);
 986		return ERR_PTR(ret);
 987	}
 988
 989	return vma;
 990}
 991
 992struct i915_vma * __must_check
 993i915_gem_object_ggtt_pin(struct drm_i915_gem_object *obj,
 994			 const struct i915_gtt_view *view,
 995			 u64 size, u64 alignment, u64 flags)
 996{
 997	struct i915_gem_ww_ctx ww;
 998	struct i915_vma *ret;
 999	int err;
1000
1001	for_i915_gem_ww(&ww, err, true) {
1002		err = i915_gem_object_lock(obj, &ww);
1003		if (err)
1004			continue;
1005
1006		ret = i915_gem_object_ggtt_pin_ww(obj, &ww, view, size,
1007						  alignment, flags);
1008		if (IS_ERR(ret))
1009			err = PTR_ERR(ret);
1010	}
1011
1012	return err ? ERR_PTR(err) : ret;
1013}
1014
1015int
1016i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
1017		       struct drm_file *file_priv)
1018{
1019	struct drm_i915_private *i915 = to_i915(dev);
1020	struct drm_i915_gem_madvise *args = data;
1021	struct drm_i915_gem_object *obj;
1022	int err;
1023
1024	switch (args->madv) {
1025	case I915_MADV_DONTNEED:
1026	case I915_MADV_WILLNEED:
1027	    break;
1028	default:
1029	    return -EINVAL;
1030	}
1031
1032	obj = i915_gem_object_lookup(file_priv, args->handle);
1033	if (!obj)
1034		return -ENOENT;
1035
1036	err = i915_gem_object_lock_interruptible(obj, NULL);
1037	if (err)
1038		goto out;
1039
1040	if (i915_gem_object_has_pages(obj) &&
1041	    i915_gem_object_is_tiled(obj) &&
1042	    i915->gem_quirks & GEM_QUIRK_PIN_SWIZZLED_PAGES) {
1043		if (obj->mm.madv == I915_MADV_WILLNEED) {
1044			GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
1045			i915_gem_object_clear_tiling_quirk(obj);
1046			i915_gem_object_make_shrinkable(obj);
1047		}
1048		if (args->madv == I915_MADV_WILLNEED) {
1049			GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
1050			i915_gem_object_make_unshrinkable(obj);
1051			i915_gem_object_set_tiling_quirk(obj);
1052		}
1053	}
1054
1055	if (obj->mm.madv != __I915_MADV_PURGED) {
1056		obj->mm.madv = args->madv;
1057		if (obj->ops->adjust_lru)
1058			obj->ops->adjust_lru(obj);
1059	}
1060
1061	if (i915_gem_object_has_pages(obj) ||
1062	    i915_gem_object_has_self_managed_shrink_list(obj)) {
1063		unsigned long flags;
1064
1065		spin_lock_irqsave(&i915->mm.obj_lock, flags);
1066		if (!list_empty(&obj->mm.link)) {
1067			struct list_head *list;
1068
1069			if (obj->mm.madv != I915_MADV_WILLNEED)
1070				list = &i915->mm.purge_list;
1071			else
1072				list = &i915->mm.shrink_list;
1073			list_move_tail(&obj->mm.link, list);
1074
1075		}
1076		spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1077	}
1078
1079	/* if the object is no longer attached, discard its backing storage */
1080	if (obj->mm.madv == I915_MADV_DONTNEED &&
1081	    !i915_gem_object_has_pages(obj))
1082		i915_gem_object_truncate(obj);
1083
1084	args->retained = obj->mm.madv != __I915_MADV_PURGED;
1085
1086	i915_gem_object_unlock(obj);
1087out:
1088	i915_gem_object_put(obj);
1089	return err;
1090}
1091
1092/*
1093 * A single pass should suffice to release all the freed objects (along most
1094 * call paths), but be a little more paranoid in that freeing the objects does
1095 * take a little amount of time, during which the rcu callbacks could have added
1096 * new objects into the freed list, and armed the work again.
1097 */
1098void i915_gem_drain_freed_objects(struct drm_i915_private *i915)
1099{
1100	while (atomic_read(&i915->mm.free_count)) {
1101		flush_work(&i915->mm.free_work);
1102		flush_delayed_work(&i915->bdev.wq);
1103		rcu_barrier();
1104	}
1105}
1106
1107/*
1108 * Similar to objects above (see i915_gem_drain_freed-objects), in general we
1109 * have workers that are armed by RCU and then rearm themselves in their
1110 * callbacks. To be paranoid, we need to drain the workqueue a second time after
1111 * waiting for the RCU grace period so that we catch work queued via RCU from
1112 * the first pass. As neither drain_workqueue() nor flush_workqueue() report a
1113 * result, we make an assumption that we only don't require more than 3 passes
1114 * to catch all _recursive_ RCU delayed work.
1115 */
1116void i915_gem_drain_workqueue(struct drm_i915_private *i915)
1117{
1118	int i;
1119
1120	for (i = 0; i < 3; i++) {
1121		flush_workqueue(i915->wq);
1122		rcu_barrier();
1123		i915_gem_drain_freed_objects(i915);
1124	}
1125
1126	drain_workqueue(i915->wq);
1127}
1128
1129int i915_gem_init(struct drm_i915_private *dev_priv)
1130{
1131	struct intel_gt *gt;
1132	unsigned int i;
1133	int ret;
1134
1135	/* We need to fallback to 4K pages if host doesn't support huge gtt. */
1136	if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1137		RUNTIME_INFO(dev_priv)->page_sizes = I915_GTT_PAGE_SIZE_4K;
 
1138
1139	ret = i915_gem_init_userptr(dev_priv);
1140	if (ret)
1141		return ret;
1142
1143	for_each_gt(gt, dev_priv, i) {
1144		intel_uc_fetch_firmwares(&gt->uc);
1145		intel_wopcm_init(&gt->wopcm);
1146	}
1147
1148	ret = i915_init_ggtt(dev_priv);
1149	if (ret) {
1150		GEM_BUG_ON(ret == -EIO);
1151		goto err_unlock;
1152	}
1153
1154	/*
1155	 * Despite its name intel_init_clock_gating applies both display
1156	 * clock gating workarounds; GT mmio workarounds and the occasional
1157	 * GT power context workaround. Worse, sometimes it includes a context
1158	 * register workaround which we need to apply before we record the
1159	 * default HW state for all contexts.
1160	 *
1161	 * FIXME: break up the workarounds and apply them at the right time!
1162	 */
1163	intel_init_clock_gating(dev_priv);
1164
1165	for_each_gt(gt, dev_priv, i) {
1166		ret = intel_gt_init(gt);
1167		if (ret)
1168			goto err_unlock;
1169	}
1170
1171	return 0;
1172
1173	/*
1174	 * Unwinding is complicated by that we want to handle -EIO to mean
1175	 * disable GPU submission but keep KMS alive. We want to mark the
1176	 * HW as irrevisibly wedged, but keep enough state around that the
1177	 * driver doesn't explode during runtime.
1178	 */
1179err_unlock:
1180	i915_gem_drain_workqueue(dev_priv);
1181
1182	if (ret != -EIO) {
1183		for_each_gt(gt, dev_priv, i) {
1184			intel_gt_driver_remove(gt);
1185			intel_gt_driver_release(gt);
1186			intel_uc_cleanup_firmwares(&gt->uc);
1187		}
1188	}
1189
1190	if (ret == -EIO) {
1191		/*
1192		 * Allow engines or uC initialisation to fail by marking the GPU
1193		 * as wedged. But we only want to do this when the GPU is angry,
1194		 * for all other failure, such as an allocation failure, bail.
1195		 */
1196		for_each_gt(gt, dev_priv, i) {
1197			if (!intel_gt_is_wedged(gt)) {
1198				i915_probe_error(dev_priv,
1199						 "Failed to initialize GPU, declaring it wedged!\n");
1200				intel_gt_set_wedged(gt);
1201			}
1202		}
1203
1204		/* Minimal basic recovery for KMS */
1205		ret = i915_ggtt_enable_hw(dev_priv);
1206		i915_ggtt_resume(to_gt(dev_priv)->ggtt);
1207		intel_init_clock_gating(dev_priv);
1208	}
1209
1210	i915_gem_drain_freed_objects(dev_priv);
1211
1212	return ret;
1213}
1214
1215void i915_gem_driver_register(struct drm_i915_private *i915)
1216{
1217	i915_gem_driver_register__shrinker(i915);
1218
1219	intel_engines_driver_register(i915);
1220}
1221
1222void i915_gem_driver_unregister(struct drm_i915_private *i915)
1223{
1224	i915_gem_driver_unregister__shrinker(i915);
1225}
1226
1227void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1228{
1229	struct intel_gt *gt;
1230	unsigned int i;
1231
1232	i915_gem_suspend_late(dev_priv);
1233	for_each_gt(gt, dev_priv, i)
1234		intel_gt_driver_remove(gt);
1235	dev_priv->uabi_engines = RB_ROOT;
1236
1237	/* Flush any outstanding unpin_work. */
1238	i915_gem_drain_workqueue(dev_priv);
 
 
1239}
1240
1241void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1242{
1243	struct intel_gt *gt;
1244	unsigned int i;
1245
1246	for_each_gt(gt, dev_priv, i) {
1247		intel_gt_driver_release(gt);
1248		intel_uc_cleanup_firmwares(&gt->uc);
1249	}
1250
1251	/* Flush any outstanding work, including i915_gem_context.release_work. */
1252	i915_gem_drain_workqueue(dev_priv);
 
1253
1254	drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
1255}
1256
1257static void i915_gem_init__mm(struct drm_i915_private *i915)
1258{
1259	spin_lock_init(&i915->mm.obj_lock);
1260
1261	init_llist_head(&i915->mm.free_list);
1262
1263	INIT_LIST_HEAD(&i915->mm.purge_list);
1264	INIT_LIST_HEAD(&i915->mm.shrink_list);
1265
1266	i915_gem_init__objects(i915);
1267}
1268
1269void i915_gem_init_early(struct drm_i915_private *dev_priv)
1270{
1271	i915_gem_init__mm(dev_priv);
1272	i915_gem_init__contexts(dev_priv);
1273
1274	spin_lock_init(&dev_priv->display.fb_tracking.lock);
1275}
1276
1277void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1278{
1279	i915_gem_drain_workqueue(dev_priv);
1280	GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1281	GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1282	drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
1283}
1284
1285int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1286{
1287	struct drm_i915_file_private *file_priv;
1288	struct i915_drm_client *client;
1289	int ret = -ENOMEM;
1290
1291	drm_dbg(&i915->drm, "\n");
1292
1293	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1294	if (!file_priv)
1295		goto err_alloc;
1296
1297	client = i915_drm_client_add(&i915->clients);
1298	if (IS_ERR(client)) {
1299		ret = PTR_ERR(client);
1300		goto err_client;
1301	}
1302
1303	file->driver_priv = file_priv;
1304	file_priv->dev_priv = i915;
1305	file_priv->file = file;
1306	file_priv->client = client;
1307
1308	file_priv->bsd_engine = -1;
1309	file_priv->hang_timestamp = jiffies;
1310
1311	ret = i915_gem_context_open(i915, file);
1312	if (ret)
1313		goto err_context;
1314
1315	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1316
1317err_context:
1318	i915_drm_client_put(client);
1319err_client:
1320	kfree(file_priv);
1321err_alloc:
1322	return ret;
1323}
1324
1325#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1326#include "selftests/mock_gem_device.c"
1327#include "selftests/i915_gem.c"
1328#endif
v5.14.15
   1/*
   2 * Copyright © 2008-2015 Intel Corporation
   3 *
   4 * Permission is hereby granted, free of charge, to any person obtaining a
   5 * copy of this software and associated documentation files (the "Software"),
   6 * to deal in the Software without restriction, including without limitation
   7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
   8 * and/or sell copies of the Software, and to permit persons to whom the
   9 * Software is furnished to do so, subject to the following conditions:
  10 *
  11 * The above copyright notice and this permission notice (including the next
  12 * paragraph) shall be included in all copies or substantial portions of the
  13 * Software.
  14 *
  15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
  18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
  19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
  20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
  21 * IN THE SOFTWARE.
  22 *
  23 * Authors:
  24 *    Eric Anholt <eric@anholt.net>
  25 *
  26 */
  27
  28#include <drm/drm_vma_manager.h>
  29#include <linux/dma-fence-array.h>
  30#include <linux/kthread.h>
  31#include <linux/dma-resv.h>
  32#include <linux/shmem_fs.h>
  33#include <linux/slab.h>
  34#include <linux/stop_machine.h>
  35#include <linux/swap.h>
  36#include <linux/pci.h>
  37#include <linux/dma-buf.h>
  38#include <linux/mman.h>
  39
 
 
 
  40#include "display/intel_display.h"
  41#include "display/intel_frontbuffer.h"
  42
  43#include "gem/i915_gem_clflush.h"
  44#include "gem/i915_gem_context.h"
  45#include "gem/i915_gem_ioctls.h"
  46#include "gem/i915_gem_mman.h"
 
  47#include "gem/i915_gem_region.h"
 
  48#include "gt/intel_engine_user.h"
  49#include "gt/intel_gt.h"
  50#include "gt/intel_gt_pm.h"
  51#include "gt/intel_workarounds.h"
  52
  53#include "i915_drv.h"
 
  54#include "i915_trace.h"
  55#include "i915_vgpu.h"
  56
  57#include "intel_pm.h"
  58
  59static int
  60insert_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node, u32 size)
  61{
  62	int err;
  63
  64	err = mutex_lock_interruptible(&ggtt->vm.mutex);
  65	if (err)
  66		return err;
  67
  68	memset(node, 0, sizeof(*node));
  69	err = drm_mm_insert_node_in_range(&ggtt->vm.mm, node,
  70					  size, 0, I915_COLOR_UNEVICTABLE,
  71					  0, ggtt->mappable_end,
  72					  DRM_MM_INSERT_LOW);
  73
  74	mutex_unlock(&ggtt->vm.mutex);
  75
  76	return err;
  77}
  78
  79static void
  80remove_mappable_node(struct i915_ggtt *ggtt, struct drm_mm_node *node)
  81{
  82	mutex_lock(&ggtt->vm.mutex);
  83	drm_mm_remove_node(node);
  84	mutex_unlock(&ggtt->vm.mutex);
  85}
  86
  87int
  88i915_gem_get_aperture_ioctl(struct drm_device *dev, void *data,
  89			    struct drm_file *file)
  90{
  91	struct i915_ggtt *ggtt = &to_i915(dev)->ggtt;
 
  92	struct drm_i915_gem_get_aperture *args = data;
  93	struct i915_vma *vma;
  94	u64 pinned;
  95
  96	if (mutex_lock_interruptible(&ggtt->vm.mutex))
  97		return -EINTR;
  98
  99	pinned = ggtt->vm.reserved;
 100	list_for_each_entry(vma, &ggtt->vm.bound_list, vm_link)
 101		if (i915_vma_is_pinned(vma))
 102			pinned += vma->node.size;
 103
 104	mutex_unlock(&ggtt->vm.mutex);
 105
 106	args->aper_size = ggtt->vm.total;
 107	args->aper_available_size = args->aper_size - pinned;
 108
 109	return 0;
 110}
 111
 112int i915_gem_object_unbind(struct drm_i915_gem_object *obj,
 113			   unsigned long flags)
 114{
 115	struct intel_runtime_pm *rpm = &to_i915(obj->base.dev)->runtime_pm;
 
 116	LIST_HEAD(still_in_list);
 117	intel_wakeref_t wakeref;
 118	struct i915_vma *vma;
 119	int ret;
 120
 
 
 121	if (list_empty(&obj->vma.list))
 122		return 0;
 123
 124	/*
 125	 * As some machines use ACPI to handle runtime-resume callbacks, and
 126	 * ACPI is quite kmalloc happy, we cannot resume beneath the vm->mutex
 127	 * as they are required by the shrinker. Ergo, we wake the device up
 128	 * first just in case.
 129	 */
 130	wakeref = intel_runtime_pm_get(rpm);
 131
 132try_again:
 133	ret = 0;
 134	spin_lock(&obj->vma.lock);
 135	while (!ret && (vma = list_first_entry_or_null(&obj->vma.list,
 136						       struct i915_vma,
 137						       obj_link))) {
 138		struct i915_address_space *vm = vma->vm;
 139
 140		list_move_tail(&vma->obj_link, &still_in_list);
 141		if (!i915_vma_is_bound(vma, I915_VMA_BIND_MASK))
 142			continue;
 143
 144		if (flags & I915_GEM_OBJECT_UNBIND_TEST) {
 145			ret = -EBUSY;
 146			break;
 147		}
 148
 
 
 
 
 
 
 
 149		ret = -EAGAIN;
 150		if (!i915_vm_tryopen(vm))
 151			break;
 152
 153		/* Prevent vma being freed by i915_vma_parked as we unbind */
 154		vma = __i915_vma_get(vma);
 155		spin_unlock(&obj->vma.lock);
 156
 157		if (vma) {
 158			ret = -EBUSY;
 159			if (flags & I915_GEM_OBJECT_UNBIND_ACTIVE ||
 160			    !i915_vma_is_active(vma)) {
 161				if (flags & I915_GEM_OBJECT_UNBIND_VM_TRYLOCK) {
 162					if (mutex_trylock(&vma->vm->mutex)) {
 163						ret = __i915_vma_unbind(vma);
 164						mutex_unlock(&vma->vm->mutex);
 165					} else {
 166						ret = -EBUSY;
 167					}
 168				} else {
 169					ret = i915_vma_unbind(vma);
 
 
 
 
 
 170				}
 
 
 171			}
 172
 173			__i915_vma_put(vma);
 174		}
 175
 176		i915_vm_close(vm);
 177		spin_lock(&obj->vma.lock);
 178	}
 179	list_splice_init(&still_in_list, &obj->vma.list);
 180	spin_unlock(&obj->vma.lock);
 181
 182	if (ret == -EAGAIN && flags & I915_GEM_OBJECT_UNBIND_BARRIER) {
 183		rcu_barrier(); /* flush the i915_vm_release() */
 184		goto try_again;
 185	}
 186
 187	intel_runtime_pm_put(rpm, wakeref);
 188
 189	return ret;
 190}
 191
 192static int
 193shmem_pread(struct page *page, int offset, int len, char __user *user_data,
 194	    bool needs_clflush)
 195{
 196	char *vaddr;
 197	int ret;
 198
 199	vaddr = kmap(page);
 200
 201	if (needs_clflush)
 202		drm_clflush_virt_range(vaddr + offset, len);
 203
 204	ret = __copy_to_user(user_data, vaddr + offset, len);
 205
 206	kunmap(page);
 207
 208	return ret ? -EFAULT : 0;
 209}
 210
 211static int
 212i915_gem_shmem_pread(struct drm_i915_gem_object *obj,
 213		     struct drm_i915_gem_pread *args)
 214{
 215	unsigned int needs_clflush;
 216	unsigned int idx, offset;
 217	char __user *user_data;
 218	u64 remain;
 219	int ret;
 220
 221	ret = i915_gem_object_lock_interruptible(obj, NULL);
 222	if (ret)
 223		return ret;
 224
 225	ret = i915_gem_object_pin_pages(obj);
 226	if (ret)
 227		goto err_unlock;
 228
 229	ret = i915_gem_object_prepare_read(obj, &needs_clflush);
 230	if (ret)
 231		goto err_unpin;
 232
 233	i915_gem_object_finish_access(obj);
 234	i915_gem_object_unlock(obj);
 235
 236	remain = args->size;
 237	user_data = u64_to_user_ptr(args->data_ptr);
 238	offset = offset_in_page(args->offset);
 239	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 240		struct page *page = i915_gem_object_get_page(obj, idx);
 241		unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 242
 243		ret = shmem_pread(page, offset, length, user_data,
 244				  needs_clflush);
 245		if (ret)
 246			break;
 247
 248		remain -= length;
 249		user_data += length;
 250		offset = 0;
 251	}
 252
 253	i915_gem_object_unpin_pages(obj);
 254	return ret;
 255
 256err_unpin:
 257	i915_gem_object_unpin_pages(obj);
 258err_unlock:
 259	i915_gem_object_unlock(obj);
 260	return ret;
 261}
 262
 263static inline bool
 264gtt_user_read(struct io_mapping *mapping,
 265	      loff_t base, int offset,
 266	      char __user *user_data, int length)
 267{
 268	void __iomem *vaddr;
 269	unsigned long unwritten;
 270
 271	/* We can use the cpu mem copy function because this is X86. */
 272	vaddr = io_mapping_map_atomic_wc(mapping, base);
 273	unwritten = __copy_to_user_inatomic(user_data,
 274					    (void __force *)vaddr + offset,
 275					    length);
 276	io_mapping_unmap_atomic(vaddr);
 277	if (unwritten) {
 278		vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 279		unwritten = copy_to_user(user_data,
 280					 (void __force *)vaddr + offset,
 281					 length);
 282		io_mapping_unmap(vaddr);
 283	}
 284	return unwritten;
 285}
 286
 287static struct i915_vma *i915_gem_gtt_prepare(struct drm_i915_gem_object *obj,
 288					     struct drm_mm_node *node,
 289					     bool write)
 290{
 291	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 292	struct i915_ggtt *ggtt = &i915->ggtt;
 293	struct i915_vma *vma;
 294	struct i915_gem_ww_ctx ww;
 295	int ret;
 296
 297	i915_gem_ww_ctx_init(&ww, true);
 298retry:
 299	vma = ERR_PTR(-ENODEV);
 300	ret = i915_gem_object_lock(obj, &ww);
 301	if (ret)
 302		goto err_ww;
 303
 304	ret = i915_gem_object_set_to_gtt_domain(obj, write);
 305	if (ret)
 306		goto err_ww;
 307
 308	if (!i915_gem_object_is_tiled(obj))
 309		vma = i915_gem_object_ggtt_pin_ww(obj, &ww, NULL, 0, 0,
 310						  PIN_MAPPABLE |
 311						  PIN_NONBLOCK /* NOWARN */ |
 312						  PIN_NOEVICT);
 313	if (vma == ERR_PTR(-EDEADLK)) {
 314		ret = -EDEADLK;
 315		goto err_ww;
 316	} else if (!IS_ERR(vma)) {
 317		node->start = i915_ggtt_offset(vma);
 318		node->flags = 0;
 319	} else {
 320		ret = insert_mappable_node(ggtt, node, PAGE_SIZE);
 321		if (ret)
 322			goto err_ww;
 323		GEM_BUG_ON(!drm_mm_node_allocated(node));
 324		vma = NULL;
 325	}
 326
 327	ret = i915_gem_object_pin_pages(obj);
 328	if (ret) {
 329		if (drm_mm_node_allocated(node)) {
 330			ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 331			remove_mappable_node(ggtt, node);
 332		} else {
 333			i915_vma_unpin(vma);
 334		}
 335	}
 336
 337err_ww:
 338	if (ret == -EDEADLK) {
 339		ret = i915_gem_ww_ctx_backoff(&ww);
 340		if (!ret)
 341			goto retry;
 342	}
 343	i915_gem_ww_ctx_fini(&ww);
 344
 345	return ret ? ERR_PTR(ret) : vma;
 346}
 347
 348static void i915_gem_gtt_cleanup(struct drm_i915_gem_object *obj,
 349				 struct drm_mm_node *node,
 350				 struct i915_vma *vma)
 351{
 352	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 353	struct i915_ggtt *ggtt = &i915->ggtt;
 354
 355	i915_gem_object_unpin_pages(obj);
 356	if (drm_mm_node_allocated(node)) {
 357		ggtt->vm.clear_range(&ggtt->vm, node->start, node->size);
 358		remove_mappable_node(ggtt, node);
 359	} else {
 360		i915_vma_unpin(vma);
 361	}
 362}
 363
 364static int
 365i915_gem_gtt_pread(struct drm_i915_gem_object *obj,
 366		   const struct drm_i915_gem_pread *args)
 367{
 368	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 369	struct i915_ggtt *ggtt = &i915->ggtt;
 370	intel_wakeref_t wakeref;
 371	struct drm_mm_node node;
 372	void __user *user_data;
 373	struct i915_vma *vma;
 374	u64 remain, offset;
 375	int ret = 0;
 376
 377	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
 378
 379	vma = i915_gem_gtt_prepare(obj, &node, false);
 380	if (IS_ERR(vma)) {
 381		ret = PTR_ERR(vma);
 382		goto out_rpm;
 383	}
 384
 385	user_data = u64_to_user_ptr(args->data_ptr);
 386	remain = args->size;
 387	offset = args->offset;
 388
 389	while (remain > 0) {
 390		/* Operation in this page
 391		 *
 392		 * page_base = page offset within aperture
 393		 * page_offset = offset within page
 394		 * page_length = bytes to copy for this page
 395		 */
 396		u32 page_base = node.start;
 397		unsigned page_offset = offset_in_page(offset);
 398		unsigned page_length = PAGE_SIZE - page_offset;
 399		page_length = remain < page_length ? remain : page_length;
 400		if (drm_mm_node_allocated(&node)) {
 401			ggtt->vm.insert_page(&ggtt->vm,
 402					     i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 403					     node.start, I915_CACHE_NONE, 0);
 404		} else {
 405			page_base += offset & PAGE_MASK;
 406		}
 407
 408		if (gtt_user_read(&ggtt->iomap, page_base, page_offset,
 409				  user_data, page_length)) {
 410			ret = -EFAULT;
 411			break;
 412		}
 413
 414		remain -= page_length;
 415		user_data += page_length;
 416		offset += page_length;
 417	}
 418
 419	i915_gem_gtt_cleanup(obj, &node, vma);
 420out_rpm:
 421	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
 422	return ret;
 423}
 424
 425/**
 426 * Reads data from the object referenced by handle.
 427 * @dev: drm device pointer
 428 * @data: ioctl data blob
 429 * @file: drm file pointer
 430 *
 431 * On error, the contents of *data are undefined.
 432 */
 433int
 434i915_gem_pread_ioctl(struct drm_device *dev, void *data,
 435		     struct drm_file *file)
 436{
 437	struct drm_i915_private *i915 = to_i915(dev);
 438	struct drm_i915_gem_pread *args = data;
 439	struct drm_i915_gem_object *obj;
 440	int ret;
 441
 442	/* PREAD is disallowed for all platforms after TGL-LP.  This also
 443	 * covers all platforms with local memory.
 444	 */
 445	if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 446		return -EOPNOTSUPP;
 447
 448	if (args->size == 0)
 449		return 0;
 450
 451	if (!access_ok(u64_to_user_ptr(args->data_ptr),
 452		       args->size))
 453		return -EFAULT;
 454
 455	obj = i915_gem_object_lookup(file, args->handle);
 456	if (!obj)
 457		return -ENOENT;
 458
 459	/* Bounds check source.  */
 460	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 461		ret = -EINVAL;
 462		goto out;
 463	}
 464
 465	trace_i915_gem_object_pread(obj, args->offset, args->size);
 466	ret = -ENODEV;
 467	if (obj->ops->pread)
 468		ret = obj->ops->pread(obj, args);
 469	if (ret != -ENODEV)
 470		goto out;
 471
 472	ret = -ENODEV;
 473	if (obj->ops->pread)
 474		ret = obj->ops->pread(obj, args);
 475	if (ret != -ENODEV)
 476		goto out;
 477
 478	ret = i915_gem_object_wait(obj,
 479				   I915_WAIT_INTERRUPTIBLE,
 480				   MAX_SCHEDULE_TIMEOUT);
 481	if (ret)
 482		goto out;
 483
 484	ret = i915_gem_shmem_pread(obj, args);
 485	if (ret == -EFAULT || ret == -ENODEV)
 486		ret = i915_gem_gtt_pread(obj, args);
 487
 488out:
 489	i915_gem_object_put(obj);
 490	return ret;
 491}
 492
 493/* This is the fast write path which cannot handle
 494 * page faults in the source data
 495 */
 496
 497static inline bool
 498ggtt_write(struct io_mapping *mapping,
 499	   loff_t base, int offset,
 500	   char __user *user_data, int length)
 501{
 502	void __iomem *vaddr;
 503	unsigned long unwritten;
 504
 505	/* We can use the cpu mem copy function because this is X86. */
 506	vaddr = io_mapping_map_atomic_wc(mapping, base);
 507	unwritten = __copy_from_user_inatomic_nocache((void __force *)vaddr + offset,
 508						      user_data, length);
 509	io_mapping_unmap_atomic(vaddr);
 510	if (unwritten) {
 511		vaddr = io_mapping_map_wc(mapping, base, PAGE_SIZE);
 512		unwritten = copy_from_user((void __force *)vaddr + offset,
 513					   user_data, length);
 514		io_mapping_unmap(vaddr);
 515	}
 516
 517	return unwritten;
 518}
 519
 520/**
 521 * This is the fast pwrite path, where we copy the data directly from the
 522 * user into the GTT, uncached.
 523 * @obj: i915 GEM object
 524 * @args: pwrite arguments structure
 525 */
 526static int
 527i915_gem_gtt_pwrite_fast(struct drm_i915_gem_object *obj,
 528			 const struct drm_i915_gem_pwrite *args)
 529{
 530	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 531	struct i915_ggtt *ggtt = &i915->ggtt;
 532	struct intel_runtime_pm *rpm = &i915->runtime_pm;
 533	intel_wakeref_t wakeref;
 534	struct drm_mm_node node;
 535	struct i915_vma *vma;
 536	u64 remain, offset;
 537	void __user *user_data;
 538	int ret = 0;
 539
 540	if (i915_gem_object_has_struct_page(obj)) {
 541		/*
 542		 * Avoid waking the device up if we can fallback, as
 543		 * waking/resuming is very slow (worst-case 10-100 ms
 544		 * depending on PCI sleeps and our own resume time).
 545		 * This easily dwarfs any performance advantage from
 546		 * using the cache bypass of indirect GGTT access.
 547		 */
 548		wakeref = intel_runtime_pm_get_if_in_use(rpm);
 549		if (!wakeref)
 550			return -EFAULT;
 551	} else {
 552		/* No backing pages, no fallback, we must force GGTT access */
 553		wakeref = intel_runtime_pm_get(rpm);
 554	}
 555
 556	vma = i915_gem_gtt_prepare(obj, &node, true);
 557	if (IS_ERR(vma)) {
 558		ret = PTR_ERR(vma);
 559		goto out_rpm;
 560	}
 561
 562	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
 563
 564	user_data = u64_to_user_ptr(args->data_ptr);
 565	offset = args->offset;
 566	remain = args->size;
 567	while (remain) {
 568		/* Operation in this page
 569		 *
 570		 * page_base = page offset within aperture
 571		 * page_offset = offset within page
 572		 * page_length = bytes to copy for this page
 573		 */
 574		u32 page_base = node.start;
 575		unsigned int page_offset = offset_in_page(offset);
 576		unsigned int page_length = PAGE_SIZE - page_offset;
 577		page_length = remain < page_length ? remain : page_length;
 578		if (drm_mm_node_allocated(&node)) {
 579			/* flush the write before we modify the GGTT */
 580			intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 581			ggtt->vm.insert_page(&ggtt->vm,
 582					     i915_gem_object_get_dma_address(obj, offset >> PAGE_SHIFT),
 583					     node.start, I915_CACHE_NONE, 0);
 584			wmb(); /* flush modifications to the GGTT (insert_page) */
 585		} else {
 586			page_base += offset & PAGE_MASK;
 587		}
 588		/* If we get a fault while copying data, then (presumably) our
 589		 * source page isn't available.  Return the error and we'll
 590		 * retry in the slow path.
 591		 * If the object is non-shmem backed, we retry again with the
 592		 * path that handles page fault.
 593		 */
 594		if (ggtt_write(&ggtt->iomap, page_base, page_offset,
 595			       user_data, page_length)) {
 596			ret = -EFAULT;
 597			break;
 598		}
 599
 600		remain -= page_length;
 601		user_data += page_length;
 602		offset += page_length;
 603	}
 604
 605	intel_gt_flush_ggtt_writes(ggtt->vm.gt);
 606	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 607
 608	i915_gem_gtt_cleanup(obj, &node, vma);
 609out_rpm:
 610	intel_runtime_pm_put(rpm, wakeref);
 611	return ret;
 612}
 613
 614/* Per-page copy function for the shmem pwrite fastpath.
 615 * Flushes invalid cachelines before writing to the target if
 616 * needs_clflush_before is set and flushes out any written cachelines after
 617 * writing if needs_clflush is set.
 618 */
 619static int
 620shmem_pwrite(struct page *page, int offset, int len, char __user *user_data,
 621	     bool needs_clflush_before,
 622	     bool needs_clflush_after)
 623{
 624	char *vaddr;
 625	int ret;
 626
 627	vaddr = kmap(page);
 628
 629	if (needs_clflush_before)
 630		drm_clflush_virt_range(vaddr + offset, len);
 631
 632	ret = __copy_from_user(vaddr + offset, user_data, len);
 633	if (!ret && needs_clflush_after)
 634		drm_clflush_virt_range(vaddr + offset, len);
 635
 636	kunmap(page);
 637
 638	return ret ? -EFAULT : 0;
 639}
 640
 641static int
 642i915_gem_shmem_pwrite(struct drm_i915_gem_object *obj,
 643		      const struct drm_i915_gem_pwrite *args)
 644{
 645	unsigned int partial_cacheline_write;
 646	unsigned int needs_clflush;
 647	unsigned int offset, idx;
 648	void __user *user_data;
 649	u64 remain;
 650	int ret;
 651
 652	ret = i915_gem_object_lock_interruptible(obj, NULL);
 653	if (ret)
 654		return ret;
 655
 656	ret = i915_gem_object_pin_pages(obj);
 657	if (ret)
 658		goto err_unlock;
 659
 660	ret = i915_gem_object_prepare_write(obj, &needs_clflush);
 661	if (ret)
 662		goto err_unpin;
 663
 664	i915_gem_object_finish_access(obj);
 665	i915_gem_object_unlock(obj);
 666
 667	/* If we don't overwrite a cacheline completely we need to be
 668	 * careful to have up-to-date data by first clflushing. Don't
 669	 * overcomplicate things and flush the entire patch.
 670	 */
 671	partial_cacheline_write = 0;
 672	if (needs_clflush & CLFLUSH_BEFORE)
 673		partial_cacheline_write = boot_cpu_data.x86_clflush_size - 1;
 674
 675	user_data = u64_to_user_ptr(args->data_ptr);
 676	remain = args->size;
 677	offset = offset_in_page(args->offset);
 678	for (idx = args->offset >> PAGE_SHIFT; remain; idx++) {
 679		struct page *page = i915_gem_object_get_page(obj, idx);
 680		unsigned int length = min_t(u64, remain, PAGE_SIZE - offset);
 681
 682		ret = shmem_pwrite(page, offset, length, user_data,
 683				   (offset | length) & partial_cacheline_write,
 684				   needs_clflush & CLFLUSH_AFTER);
 685		if (ret)
 686			break;
 687
 688		remain -= length;
 689		user_data += length;
 690		offset = 0;
 691	}
 692
 693	i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);
 694
 695	i915_gem_object_unpin_pages(obj);
 696	return ret;
 697
 698err_unpin:
 699	i915_gem_object_unpin_pages(obj);
 700err_unlock:
 701	i915_gem_object_unlock(obj);
 702	return ret;
 703}
 704
 705/**
 706 * Writes data to the object referenced by handle.
 707 * @dev: drm device
 708 * @data: ioctl data blob
 709 * @file: drm file
 710 *
 711 * On error, the contents of the buffer that were to be modified are undefined.
 712 */
 713int
 714i915_gem_pwrite_ioctl(struct drm_device *dev, void *data,
 715		      struct drm_file *file)
 716{
 717	struct drm_i915_private *i915 = to_i915(dev);
 718	struct drm_i915_gem_pwrite *args = data;
 719	struct drm_i915_gem_object *obj;
 720	int ret;
 721
 722	/* PWRITE is disallowed for all platforms after TGL-LP.  This also
 723	 * covers all platforms with local memory.
 724	 */
 725	if (GRAPHICS_VER(i915) >= 12 && !IS_TIGERLAKE(i915))
 726		return -EOPNOTSUPP;
 727
 728	if (args->size == 0)
 729		return 0;
 730
 731	if (!access_ok(u64_to_user_ptr(args->data_ptr), args->size))
 732		return -EFAULT;
 733
 734	obj = i915_gem_object_lookup(file, args->handle);
 735	if (!obj)
 736		return -ENOENT;
 737
 738	/* Bounds check destination. */
 739	if (range_overflows_t(u64, args->offset, args->size, obj->base.size)) {
 740		ret = -EINVAL;
 741		goto err;
 742	}
 743
 744	/* Writes not allowed into this read-only object */
 745	if (i915_gem_object_is_readonly(obj)) {
 746		ret = -EINVAL;
 747		goto err;
 748	}
 749
 750	trace_i915_gem_object_pwrite(obj, args->offset, args->size);
 751
 752	ret = -ENODEV;
 753	if (obj->ops->pwrite)
 754		ret = obj->ops->pwrite(obj, args);
 755	if (ret != -ENODEV)
 756		goto err;
 757
 758	ret = i915_gem_object_wait(obj,
 759				   I915_WAIT_INTERRUPTIBLE |
 760				   I915_WAIT_ALL,
 761				   MAX_SCHEDULE_TIMEOUT);
 762	if (ret)
 763		goto err;
 764
 765	ret = -EFAULT;
 766	/* We can only do the GTT pwrite on untiled buffers, as otherwise
 767	 * it would end up going through the fenced access, and we'll get
 768	 * different detiling behavior between reading and writing.
 769	 * pread/pwrite currently are reading and writing from the CPU
 770	 * perspective, requiring manual detiling by the client.
 771	 */
 772	if (!i915_gem_object_has_struct_page(obj) ||
 773	    cpu_write_needs_clflush(obj))
 774		/* Note that the gtt paths might fail with non-page-backed user
 775		 * pointers (e.g. gtt mappings when moving data between
 776		 * textures). Fallback to the shmem path in that case.
 777		 */
 778		ret = i915_gem_gtt_pwrite_fast(obj, args);
 779
 780	if (ret == -EFAULT || ret == -ENOSPC) {
 781		if (i915_gem_object_has_struct_page(obj))
 782			ret = i915_gem_shmem_pwrite(obj, args);
 783	}
 784
 785err:
 786	i915_gem_object_put(obj);
 787	return ret;
 788}
 789
 790/**
 791 * Called when user space has done writes to this buffer
 792 * @dev: drm device
 793 * @data: ioctl data blob
 794 * @file: drm file
 795 */
 796int
 797i915_gem_sw_finish_ioctl(struct drm_device *dev, void *data,
 798			 struct drm_file *file)
 799{
 800	struct drm_i915_gem_sw_finish *args = data;
 801	struct drm_i915_gem_object *obj;
 802
 803	obj = i915_gem_object_lookup(file, args->handle);
 804	if (!obj)
 805		return -ENOENT;
 806
 807	/*
 808	 * Proxy objects are barred from CPU access, so there is no
 809	 * need to ban sw_finish as it is a nop.
 810	 */
 811
 812	/* Pinned buffers may be scanout, so flush the cache */
 813	i915_gem_object_flush_if_display(obj);
 814	i915_gem_object_put(obj);
 815
 816	return 0;
 817}
 818
 819void i915_gem_runtime_suspend(struct drm_i915_private *i915)
 820{
 821	struct drm_i915_gem_object *obj, *on;
 822	int i;
 823
 824	/*
 825	 * Only called during RPM suspend. All users of the userfault_list
 826	 * must be holding an RPM wakeref to ensure that this can not
 827	 * run concurrently with themselves (and use the struct_mutex for
 828	 * protection between themselves).
 829	 */
 830
 831	list_for_each_entry_safe(obj, on,
 832				 &i915->ggtt.userfault_list, userfault_link)
 833		__i915_gem_object_release_mmap_gtt(obj);
 834
 
 
 
 
 835	/*
 836	 * The fence will be lost when the device powers down. If any were
 837	 * in use by hardware (i.e. they are pinned), we should not be powering
 838	 * down! All other fences will be reacquired by the user upon waking.
 839	 */
 840	for (i = 0; i < i915->ggtt.num_fences; i++) {
 841		struct i915_fence_reg *reg = &i915->ggtt.fence_regs[i];
 842
 843		/*
 844		 * Ideally we want to assert that the fence register is not
 845		 * live at this point (i.e. that no piece of code will be
 846		 * trying to write through fence + GTT, as that both violates
 847		 * our tracking of activity and associated locking/barriers,
 848		 * but also is illegal given that the hw is powered down).
 849		 *
 850		 * Previously we used reg->pin_count as a "liveness" indicator.
 851		 * That is not sufficient, and we need a more fine-grained
 852		 * tool if we want to have a sanity check here.
 853		 */
 854
 855		if (!reg->vma)
 856			continue;
 857
 858		GEM_BUG_ON(i915_vma_has_userfault(reg->vma));
 859		reg->dirty = true;
 860	}
 861}
 862
 863static void discard_ggtt_vma(struct i915_vma *vma)
 864{
 865	struct drm_i915_gem_object *obj = vma->obj;
 866
 867	spin_lock(&obj->vma.lock);
 868	if (!RB_EMPTY_NODE(&vma->obj_node)) {
 869		rb_erase(&vma->obj_node, &obj->vma.tree);
 870		RB_CLEAR_NODE(&vma->obj_node);
 871	}
 872	spin_unlock(&obj->vma.lock);
 873}
 874
 875struct i915_vma *
 876i915_gem_object_ggtt_pin_ww(struct drm_i915_gem_object *obj,
 877			    struct i915_gem_ww_ctx *ww,
 878			    const struct i915_ggtt_view *view,
 879			    u64 size, u64 alignment, u64 flags)
 880{
 881	struct drm_i915_private *i915 = to_i915(obj->base.dev);
 882	struct i915_ggtt *ggtt = &i915->ggtt;
 883	struct i915_vma *vma;
 884	int ret;
 885
 
 
 886	if (flags & PIN_MAPPABLE &&
 887	    (!view || view->type == I915_GGTT_VIEW_NORMAL)) {
 888		/*
 889		 * If the required space is larger than the available
 890		 * aperture, we will not able to find a slot for the
 891		 * object and unbinding the object now will be in
 892		 * vain. Worse, doing so may cause us to ping-pong
 893		 * the object in and out of the Global GTT and
 894		 * waste a lot of cycles under the mutex.
 895		 */
 896		if (obj->base.size > ggtt->mappable_end)
 897			return ERR_PTR(-E2BIG);
 898
 899		/*
 900		 * If NONBLOCK is set the caller is optimistically
 901		 * trying to cache the full object within the mappable
 902		 * aperture, and *must* have a fallback in place for
 903		 * situations where we cannot bind the object. We
 904		 * can be a little more lax here and use the fallback
 905		 * more often to avoid costly migrations of ourselves
 906		 * and other objects within the aperture.
 907		 *
 908		 * Half-the-aperture is used as a simple heuristic.
 909		 * More interesting would to do search for a free
 910		 * block prior to making the commitment to unbind.
 911		 * That caters for the self-harm case, and with a
 912		 * little more heuristics (e.g. NOFAULT, NOEVICT)
 913		 * we could try to minimise harm to others.
 914		 */
 915		if (flags & PIN_NONBLOCK &&
 916		    obj->base.size > ggtt->mappable_end / 2)
 917			return ERR_PTR(-ENOSPC);
 918	}
 919
 920new_vma:
 921	vma = i915_vma_instance(obj, &ggtt->vm, view);
 922	if (IS_ERR(vma))
 923		return vma;
 924
 925	if (i915_vma_misplaced(vma, size, alignment, flags)) {
 926		if (flags & PIN_NONBLOCK) {
 927			if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma))
 928				return ERR_PTR(-ENOSPC);
 929
 
 
 
 
 
 
 
 
 
 
 930			if (flags & PIN_MAPPABLE &&
 931			    vma->fence_size > ggtt->mappable_end / 2)
 
 932				return ERR_PTR(-ENOSPC);
 933		}
 934
 935		if (i915_vma_is_pinned(vma) || i915_vma_is_active(vma)) {
 936			discard_ggtt_vma(vma);
 937			goto new_vma;
 938		}
 939
 940		ret = i915_vma_unbind(vma);
 941		if (ret)
 942			return ERR_PTR(ret);
 943	}
 944
 945	if (ww)
 946		ret = i915_vma_pin_ww(vma, ww, size, alignment, flags | PIN_GLOBAL);
 947	else
 948		ret = i915_vma_pin(vma, size, alignment, flags | PIN_GLOBAL);
 949
 950	if (ret)
 951		return ERR_PTR(ret);
 952
 953	if (vma->fence && !i915_gem_object_is_tiled(obj)) {
 954		mutex_lock(&ggtt->vm.mutex);
 955		i915_vma_revoke_fence(vma);
 956		mutex_unlock(&ggtt->vm.mutex);
 957	}
 958
 959	ret = i915_vma_wait_for_bind(vma);
 960	if (ret) {
 961		i915_vma_unpin(vma);
 962		return ERR_PTR(ret);
 963	}
 964
 965	return vma;
 966}
 967
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 968int
 969i915_gem_madvise_ioctl(struct drm_device *dev, void *data,
 970		       struct drm_file *file_priv)
 971{
 972	struct drm_i915_private *i915 = to_i915(dev);
 973	struct drm_i915_gem_madvise *args = data;
 974	struct drm_i915_gem_object *obj;
 975	int err;
 976
 977	switch (args->madv) {
 978	case I915_MADV_DONTNEED:
 979	case I915_MADV_WILLNEED:
 980	    break;
 981	default:
 982	    return -EINVAL;
 983	}
 984
 985	obj = i915_gem_object_lookup(file_priv, args->handle);
 986	if (!obj)
 987		return -ENOENT;
 988
 989	err = i915_gem_object_lock_interruptible(obj, NULL);
 990	if (err)
 991		goto out;
 992
 993	if (i915_gem_object_has_pages(obj) &&
 994	    i915_gem_object_is_tiled(obj) &&
 995	    i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
 996		if (obj->mm.madv == I915_MADV_WILLNEED) {
 997			GEM_BUG_ON(!i915_gem_object_has_tiling_quirk(obj));
 998			i915_gem_object_clear_tiling_quirk(obj);
 999			i915_gem_object_make_shrinkable(obj);
1000		}
1001		if (args->madv == I915_MADV_WILLNEED) {
1002			GEM_BUG_ON(i915_gem_object_has_tiling_quirk(obj));
1003			i915_gem_object_make_unshrinkable(obj);
1004			i915_gem_object_set_tiling_quirk(obj);
1005		}
1006	}
1007
1008	if (obj->mm.madv != __I915_MADV_PURGED)
1009		obj->mm.madv = args->madv;
 
 
 
1010
1011	if (i915_gem_object_has_pages(obj)) {
 
1012		unsigned long flags;
1013
1014		spin_lock_irqsave(&i915->mm.obj_lock, flags);
1015		if (!list_empty(&obj->mm.link)) {
1016			struct list_head *list;
1017
1018			if (obj->mm.madv != I915_MADV_WILLNEED)
1019				list = &i915->mm.purge_list;
1020			else
1021				list = &i915->mm.shrink_list;
1022			list_move_tail(&obj->mm.link, list);
1023
1024		}
1025		spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
1026	}
1027
1028	/* if the object is no longer attached, discard its backing storage */
1029	if (obj->mm.madv == I915_MADV_DONTNEED &&
1030	    !i915_gem_object_has_pages(obj))
1031		i915_gem_object_truncate(obj);
1032
1033	args->retained = obj->mm.madv != __I915_MADV_PURGED;
1034
1035	i915_gem_object_unlock(obj);
1036out:
1037	i915_gem_object_put(obj);
1038	return err;
1039}
1040
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1041int i915_gem_init(struct drm_i915_private *dev_priv)
1042{
 
 
1043	int ret;
1044
1045	/* We need to fallback to 4K pages if host doesn't support huge gtt. */
1046	if (intel_vgpu_active(dev_priv) && !intel_vgpu_has_huge_gtt(dev_priv))
1047		mkwrite_device_info(dev_priv)->page_sizes =
1048			I915_GTT_PAGE_SIZE_4K;
1049
1050	ret = i915_gem_init_userptr(dev_priv);
1051	if (ret)
1052		return ret;
1053
1054	intel_uc_fetch_firmwares(&dev_priv->gt.uc);
1055	intel_wopcm_init(&dev_priv->wopcm);
 
 
1056
1057	ret = i915_init_ggtt(dev_priv);
1058	if (ret) {
1059		GEM_BUG_ON(ret == -EIO);
1060		goto err_unlock;
1061	}
1062
1063	/*
1064	 * Despite its name intel_init_clock_gating applies both display
1065	 * clock gating workarounds; GT mmio workarounds and the occasional
1066	 * GT power context workaround. Worse, sometimes it includes a context
1067	 * register workaround which we need to apply before we record the
1068	 * default HW state for all contexts.
1069	 *
1070	 * FIXME: break up the workarounds and apply them at the right time!
1071	 */
1072	intel_init_clock_gating(dev_priv);
1073
1074	ret = intel_gt_init(&dev_priv->gt);
1075	if (ret)
1076		goto err_unlock;
 
 
1077
1078	return 0;
1079
1080	/*
1081	 * Unwinding is complicated by that we want to handle -EIO to mean
1082	 * disable GPU submission but keep KMS alive. We want to mark the
1083	 * HW as irrevisibly wedged, but keep enough state around that the
1084	 * driver doesn't explode during runtime.
1085	 */
1086err_unlock:
1087	i915_gem_drain_workqueue(dev_priv);
1088
1089	if (ret != -EIO)
1090		intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
 
 
 
 
 
1091
1092	if (ret == -EIO) {
1093		/*
1094		 * Allow engines or uC initialisation to fail by marking the GPU
1095		 * as wedged. But we only want to do this when the GPU is angry,
1096		 * for all other failure, such as an allocation failure, bail.
1097		 */
1098		if (!intel_gt_is_wedged(&dev_priv->gt)) {
1099			i915_probe_error(dev_priv,
1100					 "Failed to initialize GPU, declaring it wedged!\n");
1101			intel_gt_set_wedged(&dev_priv->gt);
 
 
1102		}
1103
1104		/* Minimal basic recovery for KMS */
1105		ret = i915_ggtt_enable_hw(dev_priv);
1106		i915_ggtt_resume(&dev_priv->ggtt);
1107		intel_init_clock_gating(dev_priv);
1108	}
1109
1110	i915_gem_drain_freed_objects(dev_priv);
1111
1112	return ret;
1113}
1114
1115void i915_gem_driver_register(struct drm_i915_private *i915)
1116{
1117	i915_gem_driver_register__shrinker(i915);
1118
1119	intel_engines_driver_register(i915);
1120}
1121
1122void i915_gem_driver_unregister(struct drm_i915_private *i915)
1123{
1124	i915_gem_driver_unregister__shrinker(i915);
1125}
1126
1127void i915_gem_driver_remove(struct drm_i915_private *dev_priv)
1128{
1129	intel_wakeref_auto_fini(&dev_priv->ggtt.userfault_wakeref);
 
1130
1131	i915_gem_suspend_late(dev_priv);
1132	intel_gt_driver_remove(&dev_priv->gt);
 
1133	dev_priv->uabi_engines = RB_ROOT;
1134
1135	/* Flush any outstanding unpin_work. */
1136	i915_gem_drain_workqueue(dev_priv);
1137
1138	i915_gem_drain_freed_objects(dev_priv);
1139}
1140
1141void i915_gem_driver_release(struct drm_i915_private *dev_priv)
1142{
1143	intel_gt_driver_release(&dev_priv->gt);
 
1144
1145	intel_wa_list_free(&dev_priv->gt_wa_list);
 
 
 
1146
1147	intel_uc_cleanup_firmwares(&dev_priv->gt.uc);
1148
1149	i915_gem_drain_freed_objects(dev_priv);
1150
1151	drm_WARN_ON(&dev_priv->drm, !list_empty(&dev_priv->gem.contexts.list));
1152}
1153
1154static void i915_gem_init__mm(struct drm_i915_private *i915)
1155{
1156	spin_lock_init(&i915->mm.obj_lock);
1157
1158	init_llist_head(&i915->mm.free_list);
1159
1160	INIT_LIST_HEAD(&i915->mm.purge_list);
1161	INIT_LIST_HEAD(&i915->mm.shrink_list);
1162
1163	i915_gem_init__objects(i915);
1164}
1165
1166void i915_gem_init_early(struct drm_i915_private *dev_priv)
1167{
1168	i915_gem_init__mm(dev_priv);
1169	i915_gem_init__contexts(dev_priv);
1170
1171	spin_lock_init(&dev_priv->fb_tracking.lock);
1172}
1173
1174void i915_gem_cleanup_early(struct drm_i915_private *dev_priv)
1175{
1176	i915_gem_drain_freed_objects(dev_priv);
1177	GEM_BUG_ON(!llist_empty(&dev_priv->mm.free_list));
1178	GEM_BUG_ON(atomic_read(&dev_priv->mm.free_count));
1179	drm_WARN_ON(&dev_priv->drm, dev_priv->mm.shrink_count);
1180}
1181
1182int i915_gem_open(struct drm_i915_private *i915, struct drm_file *file)
1183{
1184	struct drm_i915_file_private *file_priv;
1185	int ret;
 
1186
1187	DRM_DEBUG("\n");
1188
1189	file_priv = kzalloc(sizeof(*file_priv), GFP_KERNEL);
1190	if (!file_priv)
1191		return -ENOMEM;
 
 
 
 
 
 
1192
1193	file->driver_priv = file_priv;
1194	file_priv->dev_priv = i915;
1195	file_priv->file = file;
 
1196
1197	file_priv->bsd_engine = -1;
1198	file_priv->hang_timestamp = jiffies;
1199
1200	ret = i915_gem_context_open(i915, file);
1201	if (ret)
1202		kfree(file_priv);
1203
1204	return ret;
1205}
1206
1207void i915_gem_ww_ctx_init(struct i915_gem_ww_ctx *ww, bool intr)
1208{
1209	ww_acquire_init(&ww->ctx, &reservation_ww_class);
1210	INIT_LIST_HEAD(&ww->obj_list);
1211	ww->intr = intr;
1212	ww->contended = NULL;
1213}
1214
1215static void i915_gem_ww_ctx_unlock_all(struct i915_gem_ww_ctx *ww)
1216{
1217	struct drm_i915_gem_object *obj;
1218
1219	while ((obj = list_first_entry_or_null(&ww->obj_list, struct drm_i915_gem_object, obj_link))) {
1220		list_del(&obj->obj_link);
1221		i915_gem_object_unlock(obj);
1222	}
1223}
1224
1225void i915_gem_ww_unlock_single(struct drm_i915_gem_object *obj)
1226{
1227	list_del(&obj->obj_link);
1228	i915_gem_object_unlock(obj);
1229}
1230
1231void i915_gem_ww_ctx_fini(struct i915_gem_ww_ctx *ww)
1232{
1233	i915_gem_ww_ctx_unlock_all(ww);
1234	WARN_ON(ww->contended);
1235	ww_acquire_fini(&ww->ctx);
1236}
1237
1238int __must_check i915_gem_ww_ctx_backoff(struct i915_gem_ww_ctx *ww)
1239{
1240	int ret = 0;
1241
1242	if (WARN_ON(!ww->contended))
1243		return -EINVAL;
1244
1245	i915_gem_ww_ctx_unlock_all(ww);
1246	if (ww->intr)
1247		ret = dma_resv_lock_slow_interruptible(ww->contended->base.resv, &ww->ctx);
1248	else
1249		dma_resv_lock_slow(ww->contended->base.resv, &ww->ctx);
1250
1251	if (!ret)
1252		list_add_tail(&ww->contended->obj_link, &ww->obj_list);
1253
1254	ww->contended = NULL;
1255
 
 
 
 
 
1256	return ret;
1257}
1258
1259#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
1260#include "selftests/mock_gem_device.c"
1261#include "selftests/i915_gem.c"
1262#endif