1// SPDX-License-Identifier: MIT
  2/*
  3 * Copyright © 2019 Intel Corporation
  4 */
  5
  6#include "gem/i915_gem_lmem.h"
  7#include "gem/i915_gem_object.h"
  8
  9#include "i915_drv.h"
 10#include "i915_vma.h"
 11#include "intel_engine.h"
 12#include "intel_gpu_commands.h"
 13#include "intel_ring.h"
 14#include "intel_timeline.h"
 15
 16unsigned int intel_ring_update_space(struct intel_ring *ring)
 17{
 18	unsigned int space;
 19
 20	space = __intel_ring_space(ring->head, ring->emit, ring->size);
 21
 22	ring->space = space;
 23	return space;
 24}
 25
 26void __intel_ring_pin(struct intel_ring *ring)
 27{
 28	GEM_BUG_ON(!atomic_read(&ring->pin_count));
 29	atomic_inc(&ring->pin_count);
 30}
 31
 32int intel_ring_pin(struct intel_ring *ring, struct i915_gem_ww_ctx *ww)
 33{
 34	struct i915_vma *vma = ring->vma;
 35	unsigned int flags;
 36	void *addr;
 37	int ret;
 38
 39	if (atomic_fetch_inc(&ring->pin_count))
 40		return 0;
 41
 42	/* Ring wraparound at offset 0 sometimes hangs. No idea why. */
 43	flags = PIN_OFFSET_BIAS | i915_ggtt_pin_bias(vma);
 44
 45	if (i915_gem_object_is_stolen(vma->obj))
 46		flags |= PIN_MAPPABLE;
 47	else
 48		flags |= PIN_HIGH;
 49
 50	ret = i915_ggtt_pin(vma, ww, 0, flags);
 51	if (unlikely(ret))
 52		goto err_unpin;
 53
 54	if (i915_vma_is_map_and_fenceable(vma)) {
 55		addr = (void __force *)i915_vma_pin_iomap(vma);
 56	} else {
 57		int type = i915_coherent_map_type(vma->vm->i915, vma->obj, false);
 58
 59		addr = i915_gem_object_pin_map(vma->obj, type);
 60	}
 61
 62	if (IS_ERR(addr)) {
 63		ret = PTR_ERR(addr);
 64		goto err_ring;
 65	}
 66
 67	i915_vma_make_unshrinkable(vma);
 68
 69	/* Discard any unused bytes beyond that submitted to hw. */
 70	intel_ring_reset(ring, ring->emit);
 71
 72	ring->vaddr = addr;
 73	return 0;
 74
 75err_ring:
 76	i915_vma_unpin(vma);
 77err_unpin:
 78	atomic_dec(&ring->pin_count);
 79	return ret;
 80}
 81
 82void intel_ring_reset(struct intel_ring *ring, u32 tail)
 83{
 84	tail = intel_ring_wrap(ring, tail);
 85	ring->tail = tail;
 86	ring->head = tail;
 87	ring->emit = tail;
 88	intel_ring_update_space(ring);
 89}
 90
 91void intel_ring_unpin(struct intel_ring *ring)
 92{
 93	struct i915_vma *vma = ring->vma;
 94
 95	if (!atomic_dec_and_test(&ring->pin_count))
 96		return;
 97
 98	i915_vma_unset_ggtt_write(vma);
 99	if (i915_vma_is_map_and_fenceable(vma))
100		i915_vma_unpin_iomap(vma);
101	else
102		i915_gem_object_unpin_map(vma->obj);
103
104	i915_vma_make_purgeable(vma);
105	i915_vma_unpin(vma);
106}
107
108static struct i915_vma *create_ring_vma(struct i915_ggtt *ggtt, int size)
109{
110	struct i915_address_space *vm = &ggtt->vm;
111	struct drm_i915_private *i915 = vm->i915;
112	struct drm_i915_gem_object *obj;
113	struct i915_vma *vma;
114
115	obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_VOLATILE);
116	if (IS_ERR(obj) && i915_ggtt_has_aperture(ggtt))
117		obj = i915_gem_object_create_stolen(i915, size);
118	if (IS_ERR(obj))
119		obj = i915_gem_object_create_internal(i915, size);
120	if (IS_ERR(obj))
121		return ERR_CAST(obj);
122
123	/*
124	 * Mark ring buffers as read-only from GPU side (so no stray overwrites)
125	 * if supported by the platform's GGTT.
126	 */
127	if (vm->has_read_only)
128		i915_gem_object_set_readonly(obj);
129
130	vma = i915_vma_instance(obj, vm, NULL);
131	if (IS_ERR(vma))
132		goto err;
133
134	return vma;
135
136err:
137	i915_gem_object_put(obj);
138	return vma;
139}
140
141struct intel_ring *
142intel_engine_create_ring(struct intel_engine_cs *engine, int size)
143{
144	struct drm_i915_private *i915 = engine->i915;
145	struct intel_ring *ring;
146	struct i915_vma *vma;
147
148	GEM_BUG_ON(!is_power_of_2(size));
149	GEM_BUG_ON(RING_CTL_SIZE(size) & ~RING_NR_PAGES);
150
151	ring = kzalloc(sizeof(*ring), GFP_KERNEL);
152	if (!ring)
153		return ERR_PTR(-ENOMEM);
154
155	kref_init(&ring->ref);
156	ring->size = size;
157	ring->wrap = BITS_PER_TYPE(ring->size) - ilog2(size);
158
159	/*
160	 * Workaround an erratum on the i830 which causes a hang if
161	 * the TAIL pointer points to within the last 2 cachelines
162	 * of the buffer.
163	 */
164	ring->effective_size = size;
165	if (IS_I830(i915) || IS_I845G(i915))
166		ring->effective_size -= 2 * CACHELINE_BYTES;
167
168	intel_ring_update_space(ring);
169
170	vma = create_ring_vma(engine->gt->ggtt, size);
171	if (IS_ERR(vma)) {
172		kfree(ring);
173		return ERR_CAST(vma);
174	}
175	ring->vma = vma;
176
177	return ring;
178}
179
180void intel_ring_free(struct kref *ref)
181{
182	struct intel_ring *ring = container_of(ref, typeof(*ring), ref);
183
184	i915_vma_put(ring->vma);
185	kfree(ring);
186}
187
188static noinline int
189wait_for_space(struct intel_ring *ring,
190	       struct intel_timeline *tl,
191	       unsigned int bytes)
192{
193	struct i915_request *target;
194	long timeout;
195
196	if (intel_ring_update_space(ring) >= bytes)
197		return 0;
198
199	GEM_BUG_ON(list_empty(&tl->requests));
200	list_for_each_entry(target, &tl->requests, link) {
201		if (target->ring != ring)
202			continue;
203
204		/* Would completion of this request free enough space? */
205		if (bytes <= __intel_ring_space(target->postfix,
206						ring->emit, ring->size))
207			break;
208	}
209
210	if (GEM_WARN_ON(&target->link == &tl->requests))
211		return -ENOSPC;
212
213	timeout = i915_request_wait(target,
214				    I915_WAIT_INTERRUPTIBLE,
215				    MAX_SCHEDULE_TIMEOUT);
216	if (timeout < 0)
217		return timeout;
218
219	i915_request_retire_upto(target);
220
221	intel_ring_update_space(ring);
222	GEM_BUG_ON(ring->space < bytes);
223	return 0;
224}
225
226u32 *intel_ring_begin(struct i915_request *rq, unsigned int num_dwords)
227{
228	struct intel_ring *ring = rq->ring;
229	const unsigned int remain_usable = ring->effective_size - ring->emit;
230	const unsigned int bytes = num_dwords * sizeof(u32);
231	unsigned int need_wrap = 0;
232	unsigned int total_bytes;
233	u32 *cs;
234
235	/* Packets must be qword aligned. */
236	GEM_BUG_ON(num_dwords & 1);
237
238	total_bytes = bytes + rq->reserved_space;
239	GEM_BUG_ON(total_bytes > ring->effective_size);
240
241	if (unlikely(total_bytes > remain_usable)) {
242		const int remain_actual = ring->size - ring->emit;
243
244		if (bytes > remain_usable) {
245			/*
246			 * Not enough space for the basic request. So need to
247			 * flush out the remainder and then wait for
248			 * base + reserved.
249			 */
250			total_bytes += remain_actual;
251			need_wrap = remain_actual | 1;
252		} else  {
253			/*
254			 * The base request will fit but the reserved space
255			 * falls off the end. So we don't need an immediate
256			 * wrap and only need to effectively wait for the
257			 * reserved size from the start of ringbuffer.
258			 */
259			total_bytes = rq->reserved_space + remain_actual;
260		}
261	}
262
263	if (unlikely(total_bytes > ring->space)) {
264		int ret;
265
266		/*
267		 * Space is reserved in the ringbuffer for finalising the
268		 * request, as that cannot be allowed to fail. During request
269		 * finalisation, reserved_space is set to 0 to stop the
270		 * overallocation and the assumption is that then we never need
271		 * to wait (which has the risk of failing with EINTR).
272		 *
273		 * See also i915_request_alloc() and i915_request_add().
274		 */
275		GEM_BUG_ON(!rq->reserved_space);
276
277		ret = wait_for_space(ring,
278				     i915_request_timeline(rq),
279				     total_bytes);
280		if (unlikely(ret))
281			return ERR_PTR(ret);
282	}
283
284	if (unlikely(need_wrap)) {
285		need_wrap &= ~1;
286		GEM_BUG_ON(need_wrap > ring->space);
287		GEM_BUG_ON(ring->emit + need_wrap > ring->size);
288		GEM_BUG_ON(!IS_ALIGNED(need_wrap, sizeof(u64)));
289
290		/* Fill the tail with MI_NOOP */
291		memset64(ring->vaddr + ring->emit, 0, need_wrap / sizeof(u64));
292		ring->space -= need_wrap;
293		ring->emit = 0;
294	}
295
296	GEM_BUG_ON(ring->emit > ring->size - bytes);
297	GEM_BUG_ON(ring->space < bytes);
298	cs = ring->vaddr + ring->emit;
299	GEM_DEBUG_EXEC(memset32(cs, POISON_INUSE, bytes / sizeof(*cs)));
300	ring->emit += bytes;
301	ring->space -= bytes;
302
303	return cs;
304}
305
306/* Align the ring tail to a cacheline boundary */
307int intel_ring_cacheline_align(struct i915_request *rq)
308{
309	int num_dwords;
310	void *cs;
311
312	num_dwords = (rq->ring->emit & (CACHELINE_BYTES - 1)) / sizeof(u32);
313	if (num_dwords == 0)
314		return 0;
315
316	num_dwords = CACHELINE_DWORDS - num_dwords;
317	GEM_BUG_ON(num_dwords & 1);
318
319	cs = intel_ring_begin(rq, num_dwords);
320	if (IS_ERR(cs))
321		return PTR_ERR(cs);
322
323	memset64(cs, (u64)MI_NOOP << 32 | MI_NOOP, num_dwords / 2);
324	intel_ring_advance(rq, cs + num_dwords);
325
326	GEM_BUG_ON(rq->ring->emit & (CACHELINE_BYTES - 1));
327	return 0;
328}
329
330#if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
331#include "selftest_ring.c"
332#endif