1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2016 Intel Corporation
  5 */
  6
  7#include <linux/dma-fence-array.h>
  8#include <linux/jiffies.h>
  9
 10#include "gt/intel_engine.h"
 11
 12#include "i915_gem_ioctls.h"
 13#include "i915_gem_object.h"
 14
 15static long
 16i915_gem_object_wait_fence(struct dma_fence *fence,
 17			   unsigned int flags,
 18			   long timeout)
 19{
 20	BUILD_BUG_ON(I915_WAIT_INTERRUPTIBLE != 0x1);
 21
 22	if (test_bit(DMA_FENCE_FLAG_SIGNALED_BIT, &fence->flags))
 23		return timeout;
 24
 25	if (dma_fence_is_i915(fence))
 26		return i915_request_wait(to_request(fence), flags, timeout);
 27
 28	return dma_fence_wait_timeout(fence,
 29				      flags & I915_WAIT_INTERRUPTIBLE,
 30				      timeout);
 31}
 32
 33static long
 34i915_gem_object_wait_reservation(struct dma_resv *resv,
 35				 unsigned int flags,
 36				 long timeout)
 37{
 38	struct dma_fence *excl;
 39	bool prune_fences = false;
 40
 41	if (flags & I915_WAIT_ALL) {
 42		struct dma_fence **shared;
 43		unsigned int count, i;
 44		int ret;
 45
 46		ret = dma_resv_get_fences_rcu(resv,
 47							&excl, &count, &shared);
 48		if (ret)
 49			return ret;
 50
 51		for (i = 0; i < count; i++) {
 52			timeout = i915_gem_object_wait_fence(shared[i],
 53							     flags, timeout);
 54			if (timeout < 0)
 55				break;
 56
 57			dma_fence_put(shared[i]);
 58		}
 59
 60		for (; i < count; i++)
 61			dma_fence_put(shared[i]);
 62		kfree(shared);
 63
 64		/*
 65		 * If both shared fences and an exclusive fence exist,
 66		 * then by construction the shared fences must be later
 67		 * than the exclusive fence. If we successfully wait for
 68		 * all the shared fences, we know that the exclusive fence
 69		 * must all be signaled. If all the shared fences are
 70		 * signaled, we can prune the array and recover the
 71		 * floating references on the fences/requests.
 72		 */
 73		prune_fences = count && timeout >= 0;
 74	} else {
 75		excl = dma_resv_get_excl_rcu(resv);
 76	}
 77
 78	if (excl && timeout >= 0)
 79		timeout = i915_gem_object_wait_fence(excl, flags, timeout);
 80
 81	dma_fence_put(excl);
 82
 83	/*
 84	 * Opportunistically prune the fences iff we know they have *all* been
 85	 * signaled.
 86	 */
 87	if (prune_fences && dma_resv_trylock(resv)) {
 88		if (dma_resv_test_signaled_rcu(resv, true))
 89			dma_resv_add_excl_fence(resv, NULL);
 90		dma_resv_unlock(resv);
 91	}
 92
 93	return timeout;
 94}
 95
 96static void __fence_set_priority(struct dma_fence *fence,
 97				 const struct i915_sched_attr *attr)
 98{
 99	struct i915_request *rq;
100	struct intel_engine_cs *engine;
101
102	if (dma_fence_is_signaled(fence) || !dma_fence_is_i915(fence))
103		return;
104
105	rq = to_request(fence);
106	engine = rq->engine;
107
108	local_bh_disable();
109	rcu_read_lock(); /* RCU serialisation for set-wedged protection */
110	if (engine->schedule)
111		engine->schedule(rq, attr);
112	rcu_read_unlock();
113	local_bh_enable(); /* kick the tasklets if queues were reprioritised */
114}
115
116static void fence_set_priority(struct dma_fence *fence,
117			       const struct i915_sched_attr *attr)
118{
119	/* Recurse once into a fence-array */
120	if (dma_fence_is_array(fence)) {
121		struct dma_fence_array *array = to_dma_fence_array(fence);
122		int i;
123
124		for (i = 0; i < array->num_fences; i++)
125			__fence_set_priority(array->fences[i], attr);
126	} else {
127		__fence_set_priority(fence, attr);
128	}
129}
130
131int
132i915_gem_object_wait_priority(struct drm_i915_gem_object *obj,
133			      unsigned int flags,
134			      const struct i915_sched_attr *attr)
135{
136	struct dma_fence *excl;
137
138	if (flags & I915_WAIT_ALL) {
139		struct dma_fence **shared;
140		unsigned int count, i;
141		int ret;
142
143		ret = dma_resv_get_fences_rcu(obj->base.resv,
144							&excl, &count, &shared);
145		if (ret)
146			return ret;
147
148		for (i = 0; i < count; i++) {
149			fence_set_priority(shared[i], attr);
150			dma_fence_put(shared[i]);
151		}
152
153		kfree(shared);
154	} else {
155		excl = dma_resv_get_excl_rcu(obj->base.resv);
156	}
157
158	if (excl) {
159		fence_set_priority(excl, attr);
160		dma_fence_put(excl);
161	}
162	return 0;
163}
164
165/**
166 * Waits for rendering to the object to be completed
167 * @obj: i915 gem object
168 * @flags: how to wait (under a lock, for all rendering or just for writes etc)
169 * @timeout: how long to wait
170 */
171int
172i915_gem_object_wait(struct drm_i915_gem_object *obj,
173		     unsigned int flags,
174		     long timeout)
175{
176	might_sleep();
177	GEM_BUG_ON(timeout < 0);
178
179	timeout = i915_gem_object_wait_reservation(obj->base.resv,
180						   flags, timeout);
181	return timeout < 0 ? timeout : 0;
182}
183
184static inline unsigned long nsecs_to_jiffies_timeout(const u64 n)
185{
186	/* nsecs_to_jiffies64() does not guard against overflow */
187	if (NSEC_PER_SEC % HZ &&
188	    div_u64(n, NSEC_PER_SEC) >= MAX_JIFFY_OFFSET / HZ)
189		return MAX_JIFFY_OFFSET;
190
191	return min_t(u64, MAX_JIFFY_OFFSET, nsecs_to_jiffies64(n) + 1);
192}
193
194static unsigned long to_wait_timeout(s64 timeout_ns)
195{
196	if (timeout_ns < 0)
197		return MAX_SCHEDULE_TIMEOUT;
198
199	if (timeout_ns == 0)
200		return 0;
201
202	return nsecs_to_jiffies_timeout(timeout_ns);
203}
204
205/**
206 * i915_gem_wait_ioctl - implements DRM_IOCTL_I915_GEM_WAIT
207 * @dev: drm device pointer
208 * @data: ioctl data blob
209 * @file: drm file pointer
210 *
211 * Returns 0 if successful, else an error is returned with the remaining time in
212 * the timeout parameter.
213 *  -ETIME: object is still busy after timeout
214 *  -ERESTARTSYS: signal interrupted the wait
215 *  -ENONENT: object doesn't exist
216 * Also possible, but rare:
217 *  -EAGAIN: incomplete, restart syscall
218 *  -ENOMEM: damn
219 *  -ENODEV: Internal IRQ fail
220 *  -E?: The add request failed
221 *
222 * The wait ioctl with a timeout of 0 reimplements the busy ioctl. With any
223 * non-zero timeout parameter the wait ioctl will wait for the given number of
224 * nanoseconds on an object becoming unbusy. Since the wait itself does so
225 * without holding struct_mutex the object may become re-busied before this
226 * function completes. A similar but shorter * race condition exists in the busy
227 * ioctl
228 */
229int
230i915_gem_wait_ioctl(struct drm_device *dev, void *data, struct drm_file *file)
231{
232	struct drm_i915_gem_wait *args = data;
233	struct drm_i915_gem_object *obj;
234	ktime_t start;
235	long ret;
236
237	if (args->flags != 0)
238		return -EINVAL;
239
240	obj = i915_gem_object_lookup(file, args->bo_handle);
241	if (!obj)
242		return -ENOENT;
243
244	start = ktime_get();
245
246	ret = i915_gem_object_wait(obj,
247				   I915_WAIT_INTERRUPTIBLE |
248				   I915_WAIT_PRIORITY |
249				   I915_WAIT_ALL,
250				   to_wait_timeout(args->timeout_ns));
251
252	if (args->timeout_ns > 0) {
253		args->timeout_ns -= ktime_to_ns(ktime_sub(ktime_get(), start));
254		if (args->timeout_ns < 0)
255			args->timeout_ns = 0;
256
257		/*
258		 * Apparently ktime isn't accurate enough and occasionally has a
259		 * bit of mismatch in the jiffies<->nsecs<->ktime loop. So patch
260		 * things up to make the test happy. We allow up to 1 jiffy.
261		 *
262		 * This is a regression from the timespec->ktime conversion.
263		 */
264		if (ret == -ETIME && !nsecs_to_jiffies(args->timeout_ns))
265			args->timeout_ns = 0;
266
267		/* Asked to wait beyond the jiffie/scheduler precision? */
268		if (ret == -ETIME && args->timeout_ns)
269			ret = -EAGAIN;
270	}
271
272	i915_gem_object_put(obj);
273	return ret;
274}