1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2019 Intel Corporation
  4 */
  5
  6#include <uapi/drm/i915_drm.h>
  7
  8#include "intel_memory_region.h"
  9#include "i915_gem_region.h"
 10#include "i915_drv.h"
 11#include "i915_trace.h"
 12
 13void i915_gem_object_init_memory_region(struct drm_i915_gem_object *obj,
 14					struct intel_memory_region *mem)
 15{
 16	obj->mm.region = mem;
 17
 18	mutex_lock(&mem->objects.lock);
 19	list_add(&obj->mm.region_link, &mem->objects.list);
 20	mutex_unlock(&mem->objects.lock);
 21}
 22
 23void i915_gem_object_release_memory_region(struct drm_i915_gem_object *obj)
 24{
 25	struct intel_memory_region *mem = obj->mm.region;
 26
 27	mutex_lock(&mem->objects.lock);
 28	list_del(&obj->mm.region_link);
 29	mutex_unlock(&mem->objects.lock);
 30}
 31
 32static struct drm_i915_gem_object *
 33__i915_gem_object_create_region(struct intel_memory_region *mem,
 34				resource_size_t offset,
 35				resource_size_t size,
 36				resource_size_t page_size,
 37				unsigned int flags)
 38{
 39	struct drm_i915_gem_object *obj;
 40	resource_size_t default_page_size;
 41	int err;
 42
 43	/*
 44	 * NB: Our use of resource_size_t for the size stems from using struct
 45	 * resource for the mem->region. We might need to revisit this in the
 46	 * future.
 47	 */
 48
 49	GEM_BUG_ON(flags & ~I915_BO_ALLOC_FLAGS);
 50
 51	if (WARN_ON_ONCE(flags & I915_BO_ALLOC_GPU_ONLY &&
 52			 (flags & I915_BO_ALLOC_CPU_CLEAR ||
 53			  flags & I915_BO_ALLOC_PM_EARLY)))
 54		return ERR_PTR(-EINVAL);
 55
 56	if (!mem)
 57		return ERR_PTR(-ENODEV);
 58
 59	default_page_size = mem->min_page_size;
 60	if (page_size)
 61		default_page_size = page_size;
 62
 63	/* We should be able to fit a page within an sg entry */
 64	GEM_BUG_ON(overflows_type(default_page_size, u32));
 65	GEM_BUG_ON(!is_power_of_2_u64(default_page_size));
 66	GEM_BUG_ON(default_page_size < PAGE_SIZE);
 67
 68	size = round_up(size, default_page_size);
 69
 70	if (default_page_size == size)
 71		flags |= I915_BO_ALLOC_CONTIGUOUS;
 72
 73	GEM_BUG_ON(!size);
 74	GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_MIN_ALIGNMENT));
 75
 76	if (i915_gem_object_size_2big(size))
 77		return ERR_PTR(-E2BIG);
 78
 79	obj = i915_gem_object_alloc();
 80	if (!obj)
 81		return ERR_PTR(-ENOMEM);
 82
 83	/*
 84	 * Anything smaller than the min_page_size can't be freely inserted into
 85	 * the GTT, due to alignemnt restrictions. For such special objects,
 86	 * make sure we force memcpy based suspend-resume. In the future we can
 87	 * revisit this, either by allowing special mis-aligned objects in the
 88	 * migration path, or by mapping all of LMEM upfront using cheap 1G
 89	 * GTT entries.
 90	 */
 91	if (default_page_size < mem->min_page_size)
 92		flags |= I915_BO_ALLOC_PM_EARLY;
 93
 94	err = mem->ops->init_object(mem, obj, offset, size, page_size, flags);
 95	if (err)
 96		goto err_object_free;
 97
 98	trace_i915_gem_object_create(obj);
 99	return obj;
100
101err_object_free:
102	i915_gem_object_free(obj);
103	return ERR_PTR(err);
104}
105
106struct drm_i915_gem_object *
107i915_gem_object_create_region(struct intel_memory_region *mem,
108			      resource_size_t size,
109			      resource_size_t page_size,
110			      unsigned int flags)
111{
112	return __i915_gem_object_create_region(mem, I915_BO_INVALID_OFFSET,
113					       size, page_size, flags);
114}
115
116struct drm_i915_gem_object *
117i915_gem_object_create_region_at(struct intel_memory_region *mem,
118				 resource_size_t offset,
119				 resource_size_t size,
120				 unsigned int flags)
121{
122	GEM_BUG_ON(offset == I915_BO_INVALID_OFFSET);
123
124	if (GEM_WARN_ON(!IS_ALIGNED(size, mem->min_page_size)) ||
125	    GEM_WARN_ON(!IS_ALIGNED(offset, mem->min_page_size)))
126		return ERR_PTR(-EINVAL);
127
128	if (range_overflows(offset, size, resource_size(&mem->region)))
129		return ERR_PTR(-EINVAL);
130
131	if (!(flags & I915_BO_ALLOC_GPU_ONLY) &&
132	    offset + size > resource_size(&mem->io) &&
133	    !i915_ggtt_has_aperture(to_gt(mem->i915)->ggtt))
134		return ERR_PTR(-ENOSPC);
135
136	return __i915_gem_object_create_region(mem, offset, size, 0,
137					       flags | I915_BO_ALLOC_CONTIGUOUS);
138}
139
140/**
141 * i915_gem_process_region - Iterate over all objects of a region using ops
142 * to process and optionally skip objects
143 * @mr: The memory region
144 * @apply: ops and private data
145 *
146 * This function can be used to iterate over the regions object list,
147 * checking whether to skip objects, and, if not, lock the objects and
148 * process them using the supplied ops. Note that this function temporarily
149 * removes objects from the region list while iterating, so that if run
150 * concurrently with itself may not iterate over all objects.
151 *
152 * Return: 0 if successful, negative error code on failure.
153 */
154int i915_gem_process_region(struct intel_memory_region *mr,
155			    struct i915_gem_apply_to_region *apply)
156{
157	const struct i915_gem_apply_to_region_ops *ops = apply->ops;
158	struct drm_i915_gem_object *obj;
159	struct list_head still_in_list;
160	int ret = 0;
161
162	/*
163	 * In the future, a non-NULL apply->ww could mean the caller is
164	 * already in a locking transaction and provides its own context.
165	 */
166	GEM_WARN_ON(apply->ww);
167
168	INIT_LIST_HEAD(&still_in_list);
169	mutex_lock(&mr->objects.lock);
170	for (;;) {
171		struct i915_gem_ww_ctx ww;
172
173		obj = list_first_entry_or_null(&mr->objects.list, typeof(*obj),
174					       mm.region_link);
175		if (!obj)
176			break;
177
178		list_move_tail(&obj->mm.region_link, &still_in_list);
179		if (!kref_get_unless_zero(&obj->base.refcount))
180			continue;
181
182		/*
183		 * Note: Someone else might be migrating the object at this
184		 * point. The object's region is not stable until we lock
185		 * the object.
186		 */
187		mutex_unlock(&mr->objects.lock);
188		apply->ww = &ww;
189		for_i915_gem_ww(&ww, ret, apply->interruptible) {
190			ret = i915_gem_object_lock(obj, apply->ww);
191			if (ret)
192				continue;
193
194			if (obj->mm.region == mr)
195				ret = ops->process_obj(apply, obj);
196			/* Implicit object unlock */
197		}
198
199		i915_gem_object_put(obj);
200		mutex_lock(&mr->objects.lock);
201		if (ret)
202			break;
203	}
204	list_splice_tail(&still_in_list, &mr->objects.list);
205	mutex_unlock(&mr->objects.lock);
206
207	return ret;
208}