1// SPDX-License-Identifier: GPL-2.0 OR MIT
  2
  3#include <drm/drm_exec.h>
  4#include <drm/drm_gem.h>
  5#include <linux/dma-resv.h>
  6
  7/**
  8 * DOC: Overview
  9 *
 10 * This component mainly abstracts the retry loop necessary for locking
 11 * multiple GEM objects while preparing hardware operations (e.g. command
 12 * submissions, page table updates etc..).
 13 *
 14 * If a contention is detected while locking a GEM object the cleanup procedure
 15 * unlocks all previously locked GEM objects and locks the contended one first
 16 * before locking any further objects.
 17 *
 18 * After an object is locked fences slots can optionally be reserved on the
 19 * dma_resv object inside the GEM object.
 20 *
 21 * A typical usage pattern should look like this::
 22 *
 23 *	struct drm_gem_object *obj;
 24 *	struct drm_exec exec;
 25 *	unsigned long index;
 26 *	int ret;
 27 *
 28 *	drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
 29 *	drm_exec_until_all_locked(&exec) {
 30 *		ret = drm_exec_prepare_obj(&exec, boA, 1);
 31 *		drm_exec_retry_on_contention(&exec);
 32 *		if (ret)
 33 *			goto error;
 34 *
 35 *		ret = drm_exec_prepare_obj(&exec, boB, 1);
 36 *		drm_exec_retry_on_contention(&exec);
 37 *		if (ret)
 38 *			goto error;
 39 *	}
 40 *
 41 *	drm_exec_for_each_locked_object(&exec, index, obj) {
 42 *		dma_resv_add_fence(obj->resv, fence, DMA_RESV_USAGE_READ);
 43 *		...
 44 *	}
 45 *	drm_exec_fini(&exec);
 46 *
 47 * See struct dma_exec for more details.
 48 */
 49
 50/* Dummy value used to initially enter the retry loop */
 51#define DRM_EXEC_DUMMY ((void *)~0)
 52
 53/* Unlock all objects and drop references */
 54static void drm_exec_unlock_all(struct drm_exec *exec)
 55{
 56	struct drm_gem_object *obj;
 57	unsigned long index;
 58
 59	drm_exec_for_each_locked_object_reverse(exec, index, obj) {
 60		dma_resv_unlock(obj->resv);
 61		drm_gem_object_put(obj);
 62	}
 63
 64	drm_gem_object_put(exec->prelocked);
 65	exec->prelocked = NULL;
 66}
 67
 68/**
 69 * drm_exec_init - initialize a drm_exec object
 70 * @exec: the drm_exec object to initialize
 71 * @flags: controls locking behavior, see DRM_EXEC_* defines
 72 * @nr: the initial # of objects
 73 *
 74 * Initialize the object and make sure that we can track locked objects.
 75 *
 76 * If nr is non-zero then it is used as the initial objects table size.
 77 * In either case, the table will grow (be re-allocated) on demand.
 78 */
 79void drm_exec_init(struct drm_exec *exec, u32 flags, unsigned nr)
 80{
 81	if (!nr)
 82		nr = PAGE_SIZE / sizeof(void *);
 83
 84	exec->flags = flags;
 85	exec->objects = kvmalloc_array(nr, sizeof(void *), GFP_KERNEL);
 86
 87	/* If allocation here fails, just delay that till the first use */
 88	exec->max_objects = exec->objects ? nr : 0;
 89	exec->num_objects = 0;
 90	exec->contended = DRM_EXEC_DUMMY;
 91	exec->prelocked = NULL;
 92}
 93EXPORT_SYMBOL(drm_exec_init);
 94
 95/**
 96 * drm_exec_fini - finalize a drm_exec object
 97 * @exec: the drm_exec object to finalize
 98 *
 99 * Unlock all locked objects, drop the references to objects and free all memory
100 * used for tracking the state.
101 */
102void drm_exec_fini(struct drm_exec *exec)
103{
104	drm_exec_unlock_all(exec);
105	kvfree(exec->objects);
106	if (exec->contended != DRM_EXEC_DUMMY) {
107		drm_gem_object_put(exec->contended);
108		ww_acquire_fini(&exec->ticket);
109	}
110}
111EXPORT_SYMBOL(drm_exec_fini);
112
113/**
114 * drm_exec_cleanup - cleanup when contention is detected
115 * @exec: the drm_exec object to cleanup
116 *
117 * Cleanup the current state and return true if we should stay inside the retry
118 * loop, false if there wasn't any contention detected and we can keep the
119 * objects locked.
120 */
121bool drm_exec_cleanup(struct drm_exec *exec)
122{
123	if (likely(!exec->contended)) {
124		ww_acquire_done(&exec->ticket);
125		return false;
126	}
127
128	if (likely(exec->contended == DRM_EXEC_DUMMY)) {
129		exec->contended = NULL;
130		ww_acquire_init(&exec->ticket, &reservation_ww_class);
131		return true;
132	}
133
134	drm_exec_unlock_all(exec);
135	exec->num_objects = 0;
136	return true;
137}
138EXPORT_SYMBOL(drm_exec_cleanup);
139
140/* Track the locked object in the array */
141static int drm_exec_obj_locked(struct drm_exec *exec,
142			       struct drm_gem_object *obj)
143{
144	if (unlikely(exec->num_objects == exec->max_objects)) {
145		size_t size = exec->max_objects * sizeof(void *);
146		void *tmp;
147
148		tmp = kvrealloc(exec->objects, size + PAGE_SIZE, GFP_KERNEL);
149		if (!tmp)
150			return -ENOMEM;
151
152		exec->objects = tmp;
153		exec->max_objects += PAGE_SIZE / sizeof(void *);
154	}
155	drm_gem_object_get(obj);
156	exec->objects[exec->num_objects++] = obj;
157
158	return 0;
159}
160
161/* Make sure the contended object is locked first */
162static int drm_exec_lock_contended(struct drm_exec *exec)
163{
164	struct drm_gem_object *obj = exec->contended;
165	int ret;
166
167	if (likely(!obj))
168		return 0;
169
170	/* Always cleanup the contention so that error handling can kick in */
171	exec->contended = NULL;
172	if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) {
173		ret = dma_resv_lock_slow_interruptible(obj->resv,
174						       &exec->ticket);
175		if (unlikely(ret))
176			goto error_dropref;
177	} else {
178		dma_resv_lock_slow(obj->resv, &exec->ticket);
179	}
180
181	ret = drm_exec_obj_locked(exec, obj);
182	if (unlikely(ret))
183		goto error_unlock;
184
185	exec->prelocked = obj;
186	return 0;
187
188error_unlock:
189	dma_resv_unlock(obj->resv);
190
191error_dropref:
192	drm_gem_object_put(obj);
193	return ret;
194}
195
196/**
197 * drm_exec_lock_obj - lock a GEM object for use
198 * @exec: the drm_exec object with the state
199 * @obj: the GEM object to lock
200 *
201 * Lock a GEM object for use and grab a reference to it.
202 *
203 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
204 * already locked (can be suppressed by setting the DRM_EXEC_IGNORE_DUPLICATES
205 * flag), -ENOMEM when memory allocation failed and zero for success.
206 */
207int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
208{
209	int ret;
210
211	ret = drm_exec_lock_contended(exec);
212	if (unlikely(ret))
213		return ret;
214
215	if (exec->prelocked == obj) {
216		drm_gem_object_put(exec->prelocked);
217		exec->prelocked = NULL;
218		return 0;
219	}
220
221	if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT)
222		ret = dma_resv_lock_interruptible(obj->resv, &exec->ticket);
223	else
224		ret = dma_resv_lock(obj->resv, &exec->ticket);
225
226	if (unlikely(ret == -EDEADLK)) {
227		drm_gem_object_get(obj);
228		exec->contended = obj;
229		return -EDEADLK;
230	}
231
232	if (unlikely(ret == -EALREADY) &&
233	    exec->flags & DRM_EXEC_IGNORE_DUPLICATES)
234		return 0;
235
236	if (unlikely(ret))
237		return ret;
238
239	ret = drm_exec_obj_locked(exec, obj);
240	if (ret)
241		goto error_unlock;
242
243	return 0;
244
245error_unlock:
246	dma_resv_unlock(obj->resv);
247	return ret;
248}
249EXPORT_SYMBOL(drm_exec_lock_obj);
250
251/**
252 * drm_exec_unlock_obj - unlock a GEM object in this exec context
253 * @exec: the drm_exec object with the state
254 * @obj: the GEM object to unlock
255 *
256 * Unlock the GEM object and remove it from the collection of locked objects.
257 * Should only be used to unlock the most recently locked objects. It's not time
258 * efficient to unlock objects locked long ago.
259 */
260void drm_exec_unlock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
261{
262	unsigned int i;
263
264	for (i = exec->num_objects; i--;) {
265		if (exec->objects[i] == obj) {
266			dma_resv_unlock(obj->resv);
267			for (++i; i < exec->num_objects; ++i)
268				exec->objects[i - 1] = exec->objects[i];
269			--exec->num_objects;
270			drm_gem_object_put(obj);
271			return;
272		}
273
274	}
275}
276EXPORT_SYMBOL(drm_exec_unlock_obj);
277
278/**
279 * drm_exec_prepare_obj - prepare a GEM object for use
280 * @exec: the drm_exec object with the state
281 * @obj: the GEM object to prepare
282 * @num_fences: how many fences to reserve
283 *
284 * Prepare a GEM object for use by locking it and reserving fence slots.
285 *
286 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
287 * already locked, -ENOMEM when memory allocation failed and zero for success.
288 */
289int drm_exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,
290			 unsigned int num_fences)
291{
292	int ret;
293
294	ret = drm_exec_lock_obj(exec, obj);
295	if (ret)
296		return ret;
297
298	ret = dma_resv_reserve_fences(obj->resv, num_fences);
299	if (ret) {
300		drm_exec_unlock_obj(exec, obj);
301		return ret;
302	}
303
304	return 0;
305}
306EXPORT_SYMBOL(drm_exec_prepare_obj);
307
308/**
309 * drm_exec_prepare_array - helper to prepare an array of objects
310 * @exec: the drm_exec object with the state
311 * @objects: array of GEM object to prepare
312 * @num_objects: number of GEM objects in the array
313 * @num_fences: number of fences to reserve on each GEM object
314 *
315 * Prepares all GEM objects in an array, aborts on first error.
316 * Reserves @num_fences on each GEM object after locking it.
317 *
318 * Returns: -EDEADLOCK on contention, -EALREADY when object is already locked,
319 * -ENOMEM when memory allocation failed and zero for success.
320 */
321int drm_exec_prepare_array(struct drm_exec *exec,
322			   struct drm_gem_object **objects,
323			   unsigned int num_objects,
324			   unsigned int num_fences)
325{
326	int ret;
327
328	for (unsigned int i = 0; i < num_objects; ++i) {
329		ret = drm_exec_prepare_obj(exec, objects[i], num_fences);
330		if (unlikely(ret))
331			return ret;
332	}
333
334	return 0;
335}
336EXPORT_SYMBOL(drm_exec_prepare_array);
337
338MODULE_DESCRIPTION("DRM execution context");
339MODULE_LICENSE("Dual MIT/GPL");