1/*
  2 * Copyright © 2017 Intel Corporation
  3 *
  4 * Permission is hereby granted, free of charge, to any person obtaining a
  5 * copy of this software and associated documentation files (the "Software"),
  6 * to deal in the Software without restriction, including without limitation
  7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8 * and/or sell copies of the Software, and to permit persons to whom the
  9 * Software is furnished to do so, subject to the following conditions:
 10 *
 11 * The above copyright notice and this permission notice (including the next
 12 * paragraph) shall be included in all copies or substantial portions of the
 13 * Software.
 14 *
 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
 21 * IN THE SOFTWARE.
 22 *
 23 */
 24
 25#include "../i915_selftest.h"
 26
 27#include "mock_drm.h"
 28#include "huge_gem_object.h"
 29
 30#define DW_PER_PAGE (PAGE_SIZE / sizeof(u32))
 31
 32static struct i915_vma *
 33gpu_fill_dw(struct i915_vma *vma, u64 offset, unsigned long count, u32 value)
 34{
 35	struct drm_i915_gem_object *obj;
 36	const int gen = INTEL_GEN(vma->vm->i915);
 37	unsigned long n, size;
 38	u32 *cmd;
 39	int err;
 40
 41	size = (4 * count + 1) * sizeof(u32);
 42	size = round_up(size, PAGE_SIZE);
 43	obj = i915_gem_object_create_internal(vma->vm->i915, size);
 44	if (IS_ERR(obj))
 45		return ERR_CAST(obj);
 46
 47	cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
 48	if (IS_ERR(cmd)) {
 49		err = PTR_ERR(cmd);
 50		goto err;
 51	}
 52
 53	GEM_BUG_ON(offset + (count - 1) * PAGE_SIZE > vma->node.size);
 54	offset += vma->node.start;
 55
 56	for (n = 0; n < count; n++) {
 57		if (gen >= 8) {
 58			*cmd++ = MI_STORE_DWORD_IMM_GEN4;
 59			*cmd++ = lower_32_bits(offset);
 60			*cmd++ = upper_32_bits(offset);
 61			*cmd++ = value;
 62		} else if (gen >= 4) {
 63			*cmd++ = MI_STORE_DWORD_IMM_GEN4 |
 64				(gen < 6 ? 1 << 22 : 0);
 65			*cmd++ = 0;
 66			*cmd++ = offset;
 67			*cmd++ = value;
 68		} else {
 69			*cmd++ = MI_STORE_DWORD_IMM | 1 << 22;
 70			*cmd++ = offset;
 71			*cmd++ = value;
 72		}
 73		offset += PAGE_SIZE;
 74	}
 75	*cmd = MI_BATCH_BUFFER_END;
 76	i915_gem_object_unpin_map(obj);
 77
 78	err = i915_gem_object_set_to_gtt_domain(obj, false);
 79	if (err)
 80		goto err;
 81
 82	vma = i915_vma_instance(obj, vma->vm, NULL);
 83	if (IS_ERR(vma)) {
 84		err = PTR_ERR(vma);
 85		goto err;
 86	}
 87
 88	err = i915_vma_pin(vma, 0, 0, PIN_USER);
 89	if (err)
 90		goto err;
 91
 92	return vma;
 93
 94err:
 95	i915_gem_object_put(obj);
 96	return ERR_PTR(err);
 97}
 98
 99static unsigned long real_page_count(struct drm_i915_gem_object *obj)
100{
101	return huge_gem_object_phys_size(obj) >> PAGE_SHIFT;
102}
103
104static unsigned long fake_page_count(struct drm_i915_gem_object *obj)
105{
106	return huge_gem_object_dma_size(obj) >> PAGE_SHIFT;
107}
108
109static int gpu_fill(struct drm_i915_gem_object *obj,
110		    struct i915_gem_context *ctx,
111		    struct intel_engine_cs *engine,
112		    unsigned int dw)
113{
114	struct drm_i915_private *i915 = to_i915(obj->base.dev);
115	struct i915_address_space *vm =
116		ctx->ppgtt ? &ctx->ppgtt->base : &i915->ggtt.base;
117	struct i915_request *rq;
118	struct i915_vma *vma;
119	struct i915_vma *batch;
120	unsigned int flags;
121	int err;
122
123	GEM_BUG_ON(obj->base.size > vm->total);
124	GEM_BUG_ON(!intel_engine_can_store_dword(engine));
125
126	vma = i915_vma_instance(obj, vm, NULL);
127	if (IS_ERR(vma))
128		return PTR_ERR(vma);
129
130	err = i915_gem_object_set_to_gtt_domain(obj, false);
131	if (err)
132		return err;
133
134	err = i915_vma_pin(vma, 0, 0, PIN_HIGH | PIN_USER);
135	if (err)
136		return err;
137
138	/* Within the GTT the huge objects maps every page onto
139	 * its 1024 real pages (using phys_pfn = dma_pfn % 1024).
140	 * We set the nth dword within the page using the nth
141	 * mapping via the GTT - this should exercise the GTT mapping
142	 * whilst checking that each context provides a unique view
143	 * into the object.
144	 */
145	batch = gpu_fill_dw(vma,
146			    (dw * real_page_count(obj)) << PAGE_SHIFT |
147			    (dw * sizeof(u32)),
148			    real_page_count(obj),
149			    dw);
150	if (IS_ERR(batch)) {
151		err = PTR_ERR(batch);
152		goto err_vma;
153	}
154
155	rq = i915_request_alloc(engine, ctx);
156	if (IS_ERR(rq)) {
157		err = PTR_ERR(rq);
158		goto err_batch;
159	}
160
161	flags = 0;
162	if (INTEL_GEN(vm->i915) <= 5)
163		flags |= I915_DISPATCH_SECURE;
164
165	err = engine->emit_bb_start(rq,
166				    batch->node.start, batch->node.size,
167				    flags);
168	if (err)
169		goto err_request;
170
171	i915_vma_move_to_active(batch, rq, 0);
172	i915_gem_object_set_active_reference(batch->obj);
173	i915_vma_unpin(batch);
174	i915_vma_close(batch);
175
176	i915_vma_move_to_active(vma, rq, 0);
177	i915_vma_unpin(vma);
178
179	reservation_object_lock(obj->resv, NULL);
180	reservation_object_add_excl_fence(obj->resv, &rq->fence);
181	reservation_object_unlock(obj->resv);
182
183	__i915_request_add(rq, true);
184
185	return 0;
186
187err_request:
188	__i915_request_add(rq, false);
189err_batch:
190	i915_vma_unpin(batch);
191err_vma:
192	i915_vma_unpin(vma);
193	return err;
194}
195
196static int cpu_fill(struct drm_i915_gem_object *obj, u32 value)
197{
198	const bool has_llc = HAS_LLC(to_i915(obj->base.dev));
199	unsigned int n, m, need_flush;
200	int err;
201
202	err = i915_gem_obj_prepare_shmem_write(obj, &need_flush);
203	if (err)
204		return err;
205
206	for (n = 0; n < real_page_count(obj); n++) {
207		u32 *map;
208
209		map = kmap_atomic(i915_gem_object_get_page(obj, n));
210		for (m = 0; m < DW_PER_PAGE; m++)
211			map[m] = value;
212		if (!has_llc)
213			drm_clflush_virt_range(map, PAGE_SIZE);
214		kunmap_atomic(map);
215	}
216
217	i915_gem_obj_finish_shmem_access(obj);
218	obj->read_domains = I915_GEM_DOMAIN_GTT | I915_GEM_DOMAIN_CPU;
219	obj->write_domain = 0;
220	return 0;
221}
222
223static int cpu_check(struct drm_i915_gem_object *obj, unsigned int max)
224{
225	unsigned int n, m, needs_flush;
226	int err;
227
228	err = i915_gem_obj_prepare_shmem_read(obj, &needs_flush);
229	if (err)
230		return err;
231
232	for (n = 0; n < real_page_count(obj); n++) {
233		u32 *map;
234
235		map = kmap_atomic(i915_gem_object_get_page(obj, n));
236		if (needs_flush & CLFLUSH_BEFORE)
237			drm_clflush_virt_range(map, PAGE_SIZE);
238
239		for (m = 0; m < max; m++) {
240			if (map[m] != m) {
241				pr_err("Invalid value at page %d, offset %d: found %x expected %x\n",
242				       n, m, map[m], m);
243				err = -EINVAL;
244				goto out_unmap;
245			}
246		}
247
248		for (; m < DW_PER_PAGE; m++) {
249			if (map[m] != 0xdeadbeef) {
250				pr_err("Invalid value at page %d, offset %d: found %x expected %x\n",
251				       n, m, map[m], 0xdeadbeef);
252				err = -EINVAL;
253				goto out_unmap;
254			}
255		}
256
257out_unmap:
258		kunmap_atomic(map);
259		if (err)
260			break;
261	}
262
263	i915_gem_obj_finish_shmem_access(obj);
264	return err;
265}
266
267static int file_add_object(struct drm_file *file,
268			    struct drm_i915_gem_object *obj)
269{
270	int err;
271
272	GEM_BUG_ON(obj->base.handle_count);
273
274	/* tie the object to the drm_file for easy reaping */
275	err = idr_alloc(&file->object_idr, &obj->base, 1, 0, GFP_KERNEL);
276	if (err < 0)
277		return  err;
278
279	i915_gem_object_get(obj);
280	obj->base.handle_count++;
281	return 0;
282}
283
284static struct drm_i915_gem_object *
285create_test_object(struct i915_gem_context *ctx,
286		   struct drm_file *file,
287		   struct list_head *objects)
288{
289	struct drm_i915_gem_object *obj;
290	struct i915_address_space *vm =
291		ctx->ppgtt ? &ctx->ppgtt->base : &ctx->i915->ggtt.base;
292	u64 size;
293	int err;
294
295	size = min(vm->total / 2, 1024ull * DW_PER_PAGE * PAGE_SIZE);
296	size = round_down(size, DW_PER_PAGE * PAGE_SIZE);
297
298	obj = huge_gem_object(ctx->i915, DW_PER_PAGE * PAGE_SIZE, size);
299	if (IS_ERR(obj))
300		return obj;
301
302	err = file_add_object(file, obj);
303	i915_gem_object_put(obj);
304	if (err)
305		return ERR_PTR(err);
306
307	err = cpu_fill(obj, 0xdeadbeef);
308	if (err) {
309		pr_err("Failed to fill object with cpu, err=%d\n",
310		       err);
311		return ERR_PTR(err);
312	}
313
314	list_add_tail(&obj->st_link, objects);
315	return obj;
316}
317
318static unsigned long max_dwords(struct drm_i915_gem_object *obj)
319{
320	unsigned long npages = fake_page_count(obj);
321
322	GEM_BUG_ON(!IS_ALIGNED(npages, DW_PER_PAGE));
323	return npages / DW_PER_PAGE;
324}
325
326static int igt_ctx_exec(void *arg)
327{
328	struct drm_i915_private *i915 = arg;
329	struct drm_i915_gem_object *obj = NULL;
330	struct drm_file *file;
331	IGT_TIMEOUT(end_time);
332	LIST_HEAD(objects);
333	unsigned long ncontexts, ndwords, dw;
334	bool first_shared_gtt = true;
335	int err = -ENODEV;
336
337	/* Create a few different contexts (with different mm) and write
338	 * through each ctx/mm using the GPU making sure those writes end
339	 * up in the expected pages of our obj.
340	 */
341
342	file = mock_file(i915);
343	if (IS_ERR(file))
344		return PTR_ERR(file);
345
346	mutex_lock(&i915->drm.struct_mutex);
347
348	ncontexts = 0;
349	ndwords = 0;
350	dw = 0;
351	while (!time_after(jiffies, end_time)) {
352		struct intel_engine_cs *engine;
353		struct i915_gem_context *ctx;
354		unsigned int id;
355
356		if (first_shared_gtt) {
357			ctx = __create_hw_context(i915, file->driver_priv);
358			first_shared_gtt = false;
359		} else {
360			ctx = i915_gem_create_context(i915, file->driver_priv);
361		}
362		if (IS_ERR(ctx)) {
363			err = PTR_ERR(ctx);
364			goto out_unlock;
365		}
366
367		for_each_engine(engine, i915, id) {
368			if (!intel_engine_can_store_dword(engine))
369				continue;
370
371			if (!obj) {
372				obj = create_test_object(ctx, file, &objects);
373				if (IS_ERR(obj)) {
374					err = PTR_ERR(obj);
375					goto out_unlock;
376				}
377			}
378
379			intel_runtime_pm_get(i915);
380			err = gpu_fill(obj, ctx, engine, dw);
381			intel_runtime_pm_put(i915);
382			if (err) {
383				pr_err("Failed to fill dword %lu [%lu/%lu] with gpu (%s) in ctx %u [full-ppgtt? %s], err=%d\n",
384				       ndwords, dw, max_dwords(obj),
385				       engine->name, ctx->hw_id,
386				       yesno(!!ctx->ppgtt), err);
387				goto out_unlock;
388			}
389
390			if (++dw == max_dwords(obj)) {
391				obj = NULL;
392				dw = 0;
393			}
394			ndwords++;
395		}
396		ncontexts++;
397	}
398	pr_info("Submitted %lu contexts (across %u engines), filling %lu dwords\n",
399		ncontexts, INTEL_INFO(i915)->num_rings, ndwords);
400
401	dw = 0;
402	list_for_each_entry(obj, &objects, st_link) {
403		unsigned int rem =
404			min_t(unsigned int, ndwords - dw, max_dwords(obj));
405
406		err = cpu_check(obj, rem);
407		if (err)
408			break;
409
410		dw += rem;
411	}
412
413out_unlock:
414	mutex_unlock(&i915->drm.struct_mutex);
415
416	mock_file_free(i915, file);
417	return err;
418}
419
420static int fake_aliasing_ppgtt_enable(struct drm_i915_private *i915)
421{
422	struct drm_i915_gem_object *obj;
423	int err;
424
425	err = i915_gem_init_aliasing_ppgtt(i915);
426	if (err)
427		return err;
428
429	list_for_each_entry(obj, &i915->mm.bound_list, mm.link) {
430		struct i915_vma *vma;
431
432		vma = i915_vma_instance(obj, &i915->ggtt.base, NULL);
433		if (IS_ERR(vma))
434			continue;
435
436		vma->flags &= ~I915_VMA_LOCAL_BIND;
437	}
438
439	return 0;
440}
441
442static void fake_aliasing_ppgtt_disable(struct drm_i915_private *i915)
443{
444	i915_gem_fini_aliasing_ppgtt(i915);
445}
446
447int i915_gem_context_live_selftests(struct drm_i915_private *dev_priv)
448{
449	static const struct i915_subtest tests[] = {
450		SUBTEST(igt_ctx_exec),
451	};
452	bool fake_alias = false;
453	int err;
454
455	/* Install a fake aliasing gtt for exercise */
456	if (USES_PPGTT(dev_priv) && !dev_priv->mm.aliasing_ppgtt) {
457		mutex_lock(&dev_priv->drm.struct_mutex);
458		err = fake_aliasing_ppgtt_enable(dev_priv);
459		mutex_unlock(&dev_priv->drm.struct_mutex);
460		if (err)
461			return err;
462
463		GEM_BUG_ON(!dev_priv->mm.aliasing_ppgtt);
464		fake_alias = true;
465	}
466
467	err = i915_subtests(tests, dev_priv);
468
469	if (fake_alias) {
470		mutex_lock(&dev_priv->drm.struct_mutex);
471		fake_aliasing_ppgtt_disable(dev_priv);
472		mutex_unlock(&dev_priv->drm.struct_mutex);
473	}
474
475	return err;
476}