1/*
  2 * SPDX-License-Identifier: MIT
  3 *
  4 * Copyright © 2018 Intel Corporation
  5 */
  6#include "gt/intel_gpu_commands.h"
  7#include "gt/intel_gt.h"
  8
  9#include "gem/i915_gem_internal.h"
 10#include "gem/selftests/igt_gem_utils.h"
 11
 12#include "igt_spinner.h"
 13
 14int igt_spinner_init(struct igt_spinner *spin, struct intel_gt *gt)
 15{
 16	int err;
 17
 18	memset(spin, 0, sizeof(*spin));
 19	spin->gt = gt;
 20
 21	spin->hws = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
 22	if (IS_ERR(spin->hws)) {
 23		err = PTR_ERR(spin->hws);
 24		goto err;
 25	}
 26	i915_gem_object_set_cache_coherency(spin->hws, I915_CACHE_LLC);
 27
 28	spin->obj = i915_gem_object_create_internal(gt->i915, PAGE_SIZE);
 29	if (IS_ERR(spin->obj)) {
 30		err = PTR_ERR(spin->obj);
 31		goto err_hws;
 32	}
 33
 34	return 0;
 35
 36err_hws:
 37	i915_gem_object_put(spin->hws);
 38err:
 39	return err;
 40}
 41
 42static void *igt_spinner_pin_obj(struct intel_context *ce,
 43				 struct i915_gem_ww_ctx *ww,
 44				 struct drm_i915_gem_object *obj,
 45				 unsigned int mode, struct i915_vma **vma)
 46{
 47	void *vaddr;
 48	int ret;
 49
 50	*vma = i915_vma_instance(obj, ce->vm, NULL);
 51	if (IS_ERR(*vma))
 52		return ERR_CAST(*vma);
 53
 54	ret = i915_gem_object_lock(obj, ww);
 55	if (ret)
 56		return ERR_PTR(ret);
 57
 58	vaddr = i915_gem_object_pin_map(obj, mode);
 59
 60	if (!ww)
 61		i915_gem_object_unlock(obj);
 62
 63	if (IS_ERR(vaddr))
 64		return vaddr;
 65
 66	if (ww)
 67		ret = i915_vma_pin_ww(*vma, ww, 0, 0, PIN_USER);
 68	else
 69		ret = i915_vma_pin(*vma, 0, 0, PIN_USER);
 70
 71	if (ret) {
 72		i915_gem_object_unpin_map(obj);
 73		return ERR_PTR(ret);
 74	}
 75
 76	return vaddr;
 77}
 78
 79int igt_spinner_pin(struct igt_spinner *spin,
 80		    struct intel_context *ce,
 81		    struct i915_gem_ww_ctx *ww)
 82{
 83	void *vaddr;
 84
 85	if (spin->ce && WARN_ON(spin->ce != ce))
 86		return -ENODEV;
 87	spin->ce = ce;
 88
 89	if (!spin->seqno) {
 90		vaddr = igt_spinner_pin_obj(ce, ww, spin->hws, I915_MAP_WB, &spin->hws_vma);
 91		if (IS_ERR(vaddr))
 92			return PTR_ERR(vaddr);
 93
 94		spin->seqno = memset(vaddr, 0xff, PAGE_SIZE);
 95	}
 96
 97	if (!spin->batch) {
 98		unsigned int mode;
 99
100		mode = intel_gt_coherent_map_type(spin->gt, spin->obj, false);
101		vaddr = igt_spinner_pin_obj(ce, ww, spin->obj, mode, &spin->batch_vma);
102		if (IS_ERR(vaddr))
103			return PTR_ERR(vaddr);
104
105		spin->batch = vaddr;
106	}
107
108	return 0;
109}
110
111static unsigned int seqno_offset(u64 fence)
112{
113	return offset_in_page(sizeof(u32) * fence);
114}
115
116static u64 hws_address(const struct i915_vma *hws,
117		       const struct i915_request *rq)
118{
119	return i915_vma_offset(hws) + seqno_offset(rq->fence.context);
120}
121
122struct i915_request *
123igt_spinner_create_request(struct igt_spinner *spin,
124			   struct intel_context *ce,
125			   u32 arbitration_command)
126{
127	struct intel_engine_cs *engine = ce->engine;
128	struct i915_request *rq = NULL;
129	struct i915_vma *hws, *vma;
130	unsigned int flags;
131	u32 *batch;
132	int err;
133
134	GEM_BUG_ON(spin->gt != ce->vm->gt);
135
136	if (!intel_engine_can_store_dword(ce->engine))
137		return ERR_PTR(-ENODEV);
138
139	if (!spin->batch) {
140		err = igt_spinner_pin(spin, ce, NULL);
141		if (err)
142			return ERR_PTR(err);
143	}
144
145	hws = spin->hws_vma;
146	vma = spin->batch_vma;
147
148	rq = intel_context_create_request(ce);
149	if (IS_ERR(rq))
150		return ERR_CAST(rq);
151
152	err = igt_vma_move_to_active_unlocked(vma, rq, 0);
153	if (err)
154		goto cancel_rq;
155
156	err = igt_vma_move_to_active_unlocked(hws, rq, 0);
157	if (err)
158		goto cancel_rq;
159
160	batch = spin->batch;
161
162	if (GRAPHICS_VER(rq->i915) >= 8) {
163		*batch++ = MI_STORE_DWORD_IMM_GEN4;
164		*batch++ = lower_32_bits(hws_address(hws, rq));
165		*batch++ = upper_32_bits(hws_address(hws, rq));
166	} else if (GRAPHICS_VER(rq->i915) >= 6) {
167		*batch++ = MI_STORE_DWORD_IMM_GEN4;
168		*batch++ = 0;
169		*batch++ = hws_address(hws, rq);
170	} else if (GRAPHICS_VER(rq->i915) >= 4) {
171		*batch++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
172		*batch++ = 0;
173		*batch++ = hws_address(hws, rq);
174	} else {
175		*batch++ = MI_STORE_DWORD_IMM | MI_MEM_VIRTUAL;
176		*batch++ = hws_address(hws, rq);
177	}
178	*batch++ = rq->fence.seqno;
179
180	*batch++ = arbitration_command;
181
182	memset32(batch, MI_NOOP, 128);
183	batch += 128;
184
185	if (GRAPHICS_VER(rq->i915) >= 8)
186		*batch++ = MI_BATCH_BUFFER_START | BIT(8) | 1;
187	else if (IS_HASWELL(rq->i915))
188		*batch++ = MI_BATCH_BUFFER_START | MI_BATCH_PPGTT_HSW;
189	else if (GRAPHICS_VER(rq->i915) >= 6)
190		*batch++ = MI_BATCH_BUFFER_START;
191	else
192		*batch++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
193	*batch++ = lower_32_bits(i915_vma_offset(vma));
194	*batch++ = upper_32_bits(i915_vma_offset(vma));
195
196	*batch++ = MI_BATCH_BUFFER_END; /* not reached */
197
198	intel_gt_chipset_flush(engine->gt);
199
200	if (engine->emit_init_breadcrumb) {
201		err = engine->emit_init_breadcrumb(rq);
202		if (err)
203			goto cancel_rq;
204	}
205
206	flags = 0;
207	if (GRAPHICS_VER(rq->i915) <= 5)
208		flags |= I915_DISPATCH_SECURE;
209	err = engine->emit_bb_start(rq, i915_vma_offset(vma), PAGE_SIZE, flags);
210
211cancel_rq:
212	if (err) {
213		i915_request_set_error_once(rq, err);
214		i915_request_add(rq);
215	}
216	return err ? ERR_PTR(err) : rq;
217}
218
219static u32
220hws_seqno(const struct igt_spinner *spin, const struct i915_request *rq)
221{
222	u32 *seqno = spin->seqno + seqno_offset(rq->fence.context);
223
224	return READ_ONCE(*seqno);
225}
226
227void igt_spinner_end(struct igt_spinner *spin)
228{
229	if (!spin->batch)
230		return;
231
232	*spin->batch = MI_BATCH_BUFFER_END;
233	intel_gt_chipset_flush(spin->gt);
234}
235
236void igt_spinner_fini(struct igt_spinner *spin)
237{
238	igt_spinner_end(spin);
239
240	if (spin->batch) {
241		i915_vma_unpin(spin->batch_vma);
242		i915_gem_object_unpin_map(spin->obj);
243	}
244	i915_gem_object_put(spin->obj);
245
246	if (spin->seqno) {
247		i915_vma_unpin(spin->hws_vma);
248		i915_gem_object_unpin_map(spin->hws);
249	}
250	i915_gem_object_put(spin->hws);
251}
252
253bool igt_wait_for_spinner(struct igt_spinner *spin, struct i915_request *rq)
254{
255	if (i915_request_is_ready(rq))
256		intel_engine_flush_submission(rq->engine);
257
258	return !(wait_for_us(i915_seqno_passed(hws_seqno(spin, rq),
259					       rq->fence.seqno),
260			     100) &&
261		 wait_for(i915_seqno_passed(hws_seqno(spin, rq),
262					    rq->fence.seqno),
263			  50));
264}