Loading...
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/prime_numbers.h>
26
27#include "gem/i915_gem_pm.h"
28#include "gem/selftests/mock_context.h"
29
30#include "gt/intel_gt.h"
31
32#include "i915_random.h"
33#include "i915_selftest.h"
34#include "igt_live_test.h"
35#include "lib_sw_fence.h"
36
37#include "mock_drm.h"
38#include "mock_gem_device.h"
39
40static int igt_add_request(void *arg)
41{
42 struct drm_i915_private *i915 = arg;
43 struct i915_request *request;
44 int err = -ENOMEM;
45
46 /* Basic preliminary test to create a request and let it loose! */
47
48 mutex_lock(&i915->drm.struct_mutex);
49 request = mock_request(i915->engine[RCS0]->kernel_context, HZ / 10);
50 if (!request)
51 goto out_unlock;
52
53 i915_request_add(request);
54
55 err = 0;
56out_unlock:
57 mutex_unlock(&i915->drm.struct_mutex);
58 return err;
59}
60
61static int igt_wait_request(void *arg)
62{
63 const long T = HZ / 4;
64 struct drm_i915_private *i915 = arg;
65 struct i915_request *request;
66 int err = -EINVAL;
67
68 /* Submit a request, then wait upon it */
69
70 mutex_lock(&i915->drm.struct_mutex);
71 request = mock_request(i915->engine[RCS0]->kernel_context, T);
72 if (!request) {
73 err = -ENOMEM;
74 goto out_unlock;
75 }
76 i915_request_get(request);
77
78 if (i915_request_wait(request, 0, 0) != -ETIME) {
79 pr_err("request wait (busy query) succeeded (expected timeout before submit!)\n");
80 goto out_request;
81 }
82
83 if (i915_request_wait(request, 0, T) != -ETIME) {
84 pr_err("request wait succeeded (expected timeout before submit!)\n");
85 goto out_request;
86 }
87
88 if (i915_request_completed(request)) {
89 pr_err("request completed before submit!!\n");
90 goto out_request;
91 }
92
93 i915_request_add(request);
94
95 if (i915_request_wait(request, 0, 0) != -ETIME) {
96 pr_err("request wait (busy query) succeeded (expected timeout after submit!)\n");
97 goto out_request;
98 }
99
100 if (i915_request_completed(request)) {
101 pr_err("request completed immediately!\n");
102 goto out_request;
103 }
104
105 if (i915_request_wait(request, 0, T / 2) != -ETIME) {
106 pr_err("request wait succeeded (expected timeout!)\n");
107 goto out_request;
108 }
109
110 if (i915_request_wait(request, 0, T) == -ETIME) {
111 pr_err("request wait timed out!\n");
112 goto out_request;
113 }
114
115 if (!i915_request_completed(request)) {
116 pr_err("request not complete after waiting!\n");
117 goto out_request;
118 }
119
120 if (i915_request_wait(request, 0, T) == -ETIME) {
121 pr_err("request wait timed out when already complete!\n");
122 goto out_request;
123 }
124
125 err = 0;
126out_request:
127 i915_request_put(request);
128out_unlock:
129 mock_device_flush(i915);
130 mutex_unlock(&i915->drm.struct_mutex);
131 return err;
132}
133
134static int igt_fence_wait(void *arg)
135{
136 const long T = HZ / 4;
137 struct drm_i915_private *i915 = arg;
138 struct i915_request *request;
139 int err = -EINVAL;
140
141 /* Submit a request, treat it as a fence and wait upon it */
142
143 mutex_lock(&i915->drm.struct_mutex);
144 request = mock_request(i915->engine[RCS0]->kernel_context, T);
145 if (!request) {
146 err = -ENOMEM;
147 goto out_locked;
148 }
149
150 if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) {
151 pr_err("fence wait success before submit (expected timeout)!\n");
152 goto out_locked;
153 }
154
155 i915_request_add(request);
156 mutex_unlock(&i915->drm.struct_mutex);
157
158 if (dma_fence_is_signaled(&request->fence)) {
159 pr_err("fence signaled immediately!\n");
160 goto out_device;
161 }
162
163 if (dma_fence_wait_timeout(&request->fence, false, T / 2) != -ETIME) {
164 pr_err("fence wait success after submit (expected timeout)!\n");
165 goto out_device;
166 }
167
168 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) {
169 pr_err("fence wait timed out (expected success)!\n");
170 goto out_device;
171 }
172
173 if (!dma_fence_is_signaled(&request->fence)) {
174 pr_err("fence unsignaled after waiting!\n");
175 goto out_device;
176 }
177
178 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) {
179 pr_err("fence wait timed out when complete (expected success)!\n");
180 goto out_device;
181 }
182
183 err = 0;
184out_device:
185 mutex_lock(&i915->drm.struct_mutex);
186out_locked:
187 mock_device_flush(i915);
188 mutex_unlock(&i915->drm.struct_mutex);
189 return err;
190}
191
192static int igt_request_rewind(void *arg)
193{
194 struct drm_i915_private *i915 = arg;
195 struct i915_request *request, *vip;
196 struct i915_gem_context *ctx[2];
197 struct intel_context *ce;
198 int err = -EINVAL;
199
200 mutex_lock(&i915->drm.struct_mutex);
201 ctx[0] = mock_context(i915, "A");
202 ce = i915_gem_context_get_engine(ctx[0], RCS0);
203 GEM_BUG_ON(IS_ERR(ce));
204 request = mock_request(ce, 2 * HZ);
205 intel_context_put(ce);
206 if (!request) {
207 err = -ENOMEM;
208 goto err_context_0;
209 }
210
211 i915_request_get(request);
212 i915_request_add(request);
213
214 ctx[1] = mock_context(i915, "B");
215 ce = i915_gem_context_get_engine(ctx[1], RCS0);
216 GEM_BUG_ON(IS_ERR(ce));
217 vip = mock_request(ce, 0);
218 intel_context_put(ce);
219 if (!vip) {
220 err = -ENOMEM;
221 goto err_context_1;
222 }
223
224 /* Simulate preemption by manual reordering */
225 if (!mock_cancel_request(request)) {
226 pr_err("failed to cancel request (already executed)!\n");
227 i915_request_add(vip);
228 goto err_context_1;
229 }
230 i915_request_get(vip);
231 i915_request_add(vip);
232 rcu_read_lock();
233 request->engine->submit_request(request);
234 rcu_read_unlock();
235
236 mutex_unlock(&i915->drm.struct_mutex);
237
238 if (i915_request_wait(vip, 0, HZ) == -ETIME) {
239 pr_err("timed out waiting for high priority request\n");
240 goto err;
241 }
242
243 if (i915_request_completed(request)) {
244 pr_err("low priority request already completed\n");
245 goto err;
246 }
247
248 err = 0;
249err:
250 i915_request_put(vip);
251 mutex_lock(&i915->drm.struct_mutex);
252err_context_1:
253 mock_context_close(ctx[1]);
254 i915_request_put(request);
255err_context_0:
256 mock_context_close(ctx[0]);
257 mock_device_flush(i915);
258 mutex_unlock(&i915->drm.struct_mutex);
259 return err;
260}
261
262struct smoketest {
263 struct intel_engine_cs *engine;
264 struct i915_gem_context **contexts;
265 atomic_long_t num_waits, num_fences;
266 int ncontexts, max_batch;
267 struct i915_request *(*request_alloc)(struct intel_context *ce);
268};
269
270static struct i915_request *
271__mock_request_alloc(struct intel_context *ce)
272{
273 return mock_request(ce, 0);
274}
275
276static struct i915_request *
277__live_request_alloc(struct intel_context *ce)
278{
279 return intel_context_create_request(ce);
280}
281
282static int __igt_breadcrumbs_smoketest(void *arg)
283{
284 struct smoketest *t = arg;
285 struct mutex * const BKL = &t->engine->i915->drm.struct_mutex;
286 const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1;
287 const unsigned int total = 4 * t->ncontexts + 1;
288 unsigned int num_waits = 0, num_fences = 0;
289 struct i915_request **requests;
290 I915_RND_STATE(prng);
291 unsigned int *order;
292 int err = 0;
293
294 /*
295 * A very simple test to catch the most egregious of list handling bugs.
296 *
297 * At its heart, we simply create oodles of requests running across
298 * multiple kthreads and enable signaling on them, for the sole purpose
299 * of stressing our breadcrumb handling. The only inspection we do is
300 * that the fences were marked as signaled.
301 */
302
303 requests = kmalloc_array(total, sizeof(*requests), GFP_KERNEL);
304 if (!requests)
305 return -ENOMEM;
306
307 order = i915_random_order(total, &prng);
308 if (!order) {
309 err = -ENOMEM;
310 goto out_requests;
311 }
312
313 while (!kthread_should_stop()) {
314 struct i915_sw_fence *submit, *wait;
315 unsigned int n, count;
316
317 submit = heap_fence_create(GFP_KERNEL);
318 if (!submit) {
319 err = -ENOMEM;
320 break;
321 }
322
323 wait = heap_fence_create(GFP_KERNEL);
324 if (!wait) {
325 i915_sw_fence_commit(submit);
326 heap_fence_put(submit);
327 err = ENOMEM;
328 break;
329 }
330
331 i915_random_reorder(order, total, &prng);
332 count = 1 + i915_prandom_u32_max_state(max_batch, &prng);
333
334 for (n = 0; n < count; n++) {
335 struct i915_gem_context *ctx =
336 t->contexts[order[n] % t->ncontexts];
337 struct i915_request *rq;
338 struct intel_context *ce;
339
340 mutex_lock(BKL);
341
342 ce = i915_gem_context_get_engine(ctx, t->engine->legacy_idx);
343 GEM_BUG_ON(IS_ERR(ce));
344 rq = t->request_alloc(ce);
345 intel_context_put(ce);
346 if (IS_ERR(rq)) {
347 mutex_unlock(BKL);
348 err = PTR_ERR(rq);
349 count = n;
350 break;
351 }
352
353 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
354 submit,
355 GFP_KERNEL);
356
357 requests[n] = i915_request_get(rq);
358 i915_request_add(rq);
359
360 mutex_unlock(BKL);
361
362 if (err >= 0)
363 err = i915_sw_fence_await_dma_fence(wait,
364 &rq->fence,
365 0,
366 GFP_KERNEL);
367
368 if (err < 0) {
369 i915_request_put(rq);
370 count = n;
371 break;
372 }
373 }
374
375 i915_sw_fence_commit(submit);
376 i915_sw_fence_commit(wait);
377
378 if (!wait_event_timeout(wait->wait,
379 i915_sw_fence_done(wait),
380 5 * HZ)) {
381 struct i915_request *rq = requests[count - 1];
382
383 pr_err("waiting for %d/%d fences (last %llx:%lld) on %s timed out!\n",
384 atomic_read(&wait->pending), count,
385 rq->fence.context, rq->fence.seqno,
386 t->engine->name);
387 GEM_TRACE_DUMP();
388
389 intel_gt_set_wedged(t->engine->gt);
390 GEM_BUG_ON(!i915_request_completed(rq));
391 i915_sw_fence_wait(wait);
392 err = -EIO;
393 }
394
395 for (n = 0; n < count; n++) {
396 struct i915_request *rq = requests[n];
397
398 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
399 &rq->fence.flags)) {
400 pr_err("%llu:%llu was not signaled!\n",
401 rq->fence.context, rq->fence.seqno);
402 err = -EINVAL;
403 }
404
405 i915_request_put(rq);
406 }
407
408 heap_fence_put(wait);
409 heap_fence_put(submit);
410
411 if (err < 0)
412 break;
413
414 num_fences += count;
415 num_waits++;
416
417 cond_resched();
418 }
419
420 atomic_long_add(num_fences, &t->num_fences);
421 atomic_long_add(num_waits, &t->num_waits);
422
423 kfree(order);
424out_requests:
425 kfree(requests);
426 return err;
427}
428
429static int mock_breadcrumbs_smoketest(void *arg)
430{
431 struct drm_i915_private *i915 = arg;
432 struct smoketest t = {
433 .engine = i915->engine[RCS0],
434 .ncontexts = 1024,
435 .max_batch = 1024,
436 .request_alloc = __mock_request_alloc
437 };
438 unsigned int ncpus = num_online_cpus();
439 struct task_struct **threads;
440 unsigned int n;
441 int ret = 0;
442
443 /*
444 * Smoketest our breadcrumb/signal handling for requests across multiple
445 * threads. A very simple test to only catch the most egregious of bugs.
446 * See __igt_breadcrumbs_smoketest();
447 */
448
449 threads = kmalloc_array(ncpus, sizeof(*threads), GFP_KERNEL);
450 if (!threads)
451 return -ENOMEM;
452
453 t.contexts =
454 kmalloc_array(t.ncontexts, sizeof(*t.contexts), GFP_KERNEL);
455 if (!t.contexts) {
456 ret = -ENOMEM;
457 goto out_threads;
458 }
459
460 mutex_lock(&t.engine->i915->drm.struct_mutex);
461 for (n = 0; n < t.ncontexts; n++) {
462 t.contexts[n] = mock_context(t.engine->i915, "mock");
463 if (!t.contexts[n]) {
464 ret = -ENOMEM;
465 goto out_contexts;
466 }
467 }
468 mutex_unlock(&t.engine->i915->drm.struct_mutex);
469
470 for (n = 0; n < ncpus; n++) {
471 threads[n] = kthread_run(__igt_breadcrumbs_smoketest,
472 &t, "igt/%d", n);
473 if (IS_ERR(threads[n])) {
474 ret = PTR_ERR(threads[n]);
475 ncpus = n;
476 break;
477 }
478
479 get_task_struct(threads[n]);
480 }
481
482 msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies));
483
484 for (n = 0; n < ncpus; n++) {
485 int err;
486
487 err = kthread_stop(threads[n]);
488 if (err < 0 && !ret)
489 ret = err;
490
491 put_task_struct(threads[n]);
492 }
493 pr_info("Completed %lu waits for %lu fence across %d cpus\n",
494 atomic_long_read(&t.num_waits),
495 atomic_long_read(&t.num_fences),
496 ncpus);
497
498 mutex_lock(&t.engine->i915->drm.struct_mutex);
499out_contexts:
500 for (n = 0; n < t.ncontexts; n++) {
501 if (!t.contexts[n])
502 break;
503 mock_context_close(t.contexts[n]);
504 }
505 mutex_unlock(&t.engine->i915->drm.struct_mutex);
506 kfree(t.contexts);
507out_threads:
508 kfree(threads);
509
510 return ret;
511}
512
513int i915_request_mock_selftests(void)
514{
515 static const struct i915_subtest tests[] = {
516 SUBTEST(igt_add_request),
517 SUBTEST(igt_wait_request),
518 SUBTEST(igt_fence_wait),
519 SUBTEST(igt_request_rewind),
520 SUBTEST(mock_breadcrumbs_smoketest),
521 };
522 struct drm_i915_private *i915;
523 intel_wakeref_t wakeref;
524 int err = 0;
525
526 i915 = mock_gem_device();
527 if (!i915)
528 return -ENOMEM;
529
530 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
531 err = i915_subtests(tests, i915);
532
533 drm_dev_put(&i915->drm);
534
535 return err;
536}
537
538static int live_nop_request(void *arg)
539{
540 struct drm_i915_private *i915 = arg;
541 struct intel_engine_cs *engine;
542 intel_wakeref_t wakeref;
543 struct igt_live_test t;
544 unsigned int id;
545 int err = -ENODEV;
546
547 /* Submit various sized batches of empty requests, to each engine
548 * (individually), and wait for the batch to complete. We can check
549 * the overhead of submitting requests to the hardware.
550 */
551
552 mutex_lock(&i915->drm.struct_mutex);
553 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
554
555 for_each_engine(engine, i915, id) {
556 struct i915_request *request = NULL;
557 unsigned long n, prime;
558 IGT_TIMEOUT(end_time);
559 ktime_t times[2] = {};
560
561 err = igt_live_test_begin(&t, i915, __func__, engine->name);
562 if (err)
563 goto out_unlock;
564
565 for_each_prime_number_from(prime, 1, 8192) {
566 times[1] = ktime_get_raw();
567
568 for (n = 0; n < prime; n++) {
569 request = i915_request_create(engine->kernel_context);
570 if (IS_ERR(request)) {
571 err = PTR_ERR(request);
572 goto out_unlock;
573 }
574
575 /* This space is left intentionally blank.
576 *
577 * We do not actually want to perform any
578 * action with this request, we just want
579 * to measure the latency in allocation
580 * and submission of our breadcrumbs -
581 * ensuring that the bare request is sufficient
582 * for the system to work (i.e. proper HEAD
583 * tracking of the rings, interrupt handling,
584 * etc). It also gives us the lowest bounds
585 * for latency.
586 */
587
588 i915_request_add(request);
589 }
590 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
591
592 times[1] = ktime_sub(ktime_get_raw(), times[1]);
593 if (prime == 1)
594 times[0] = times[1];
595
596 if (__igt_timeout(end_time, NULL))
597 break;
598 }
599
600 err = igt_live_test_end(&t);
601 if (err)
602 goto out_unlock;
603
604 pr_info("Request latencies on %s: 1 = %lluns, %lu = %lluns\n",
605 engine->name,
606 ktime_to_ns(times[0]),
607 prime, div64_u64(ktime_to_ns(times[1]), prime));
608 }
609
610out_unlock:
611 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
612 mutex_unlock(&i915->drm.struct_mutex);
613 return err;
614}
615
616static struct i915_vma *empty_batch(struct drm_i915_private *i915)
617{
618 struct drm_i915_gem_object *obj;
619 struct i915_vma *vma;
620 u32 *cmd;
621 int err;
622
623 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
624 if (IS_ERR(obj))
625 return ERR_CAST(obj);
626
627 cmd = i915_gem_object_pin_map(obj, I915_MAP_WB);
628 if (IS_ERR(cmd)) {
629 err = PTR_ERR(cmd);
630 goto err;
631 }
632
633 *cmd = MI_BATCH_BUFFER_END;
634
635 __i915_gem_object_flush_map(obj, 0, 64);
636 i915_gem_object_unpin_map(obj);
637
638 intel_gt_chipset_flush(&i915->gt);
639
640 vma = i915_vma_instance(obj, &i915->ggtt.vm, NULL);
641 if (IS_ERR(vma)) {
642 err = PTR_ERR(vma);
643 goto err;
644 }
645
646 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_GLOBAL);
647 if (err)
648 goto err;
649
650 return vma;
651
652err:
653 i915_gem_object_put(obj);
654 return ERR_PTR(err);
655}
656
657static struct i915_request *
658empty_request(struct intel_engine_cs *engine,
659 struct i915_vma *batch)
660{
661 struct i915_request *request;
662 int err;
663
664 request = i915_request_create(engine->kernel_context);
665 if (IS_ERR(request))
666 return request;
667
668 err = engine->emit_bb_start(request,
669 batch->node.start,
670 batch->node.size,
671 I915_DISPATCH_SECURE);
672 if (err)
673 goto out_request;
674
675out_request:
676 i915_request_add(request);
677 return err ? ERR_PTR(err) : request;
678}
679
680static int live_empty_request(void *arg)
681{
682 struct drm_i915_private *i915 = arg;
683 struct intel_engine_cs *engine;
684 intel_wakeref_t wakeref;
685 struct igt_live_test t;
686 struct i915_vma *batch;
687 unsigned int id;
688 int err = 0;
689
690 /* Submit various sized batches of empty requests, to each engine
691 * (individually), and wait for the batch to complete. We can check
692 * the overhead of submitting requests to the hardware.
693 */
694
695 mutex_lock(&i915->drm.struct_mutex);
696 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
697
698 batch = empty_batch(i915);
699 if (IS_ERR(batch)) {
700 err = PTR_ERR(batch);
701 goto out_unlock;
702 }
703
704 for_each_engine(engine, i915, id) {
705 IGT_TIMEOUT(end_time);
706 struct i915_request *request;
707 unsigned long n, prime;
708 ktime_t times[2] = {};
709
710 err = igt_live_test_begin(&t, i915, __func__, engine->name);
711 if (err)
712 goto out_batch;
713
714 /* Warmup / preload */
715 request = empty_request(engine, batch);
716 if (IS_ERR(request)) {
717 err = PTR_ERR(request);
718 goto out_batch;
719 }
720 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
721
722 for_each_prime_number_from(prime, 1, 8192) {
723 times[1] = ktime_get_raw();
724
725 for (n = 0; n < prime; n++) {
726 request = empty_request(engine, batch);
727 if (IS_ERR(request)) {
728 err = PTR_ERR(request);
729 goto out_batch;
730 }
731 }
732 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
733
734 times[1] = ktime_sub(ktime_get_raw(), times[1]);
735 if (prime == 1)
736 times[0] = times[1];
737
738 if (__igt_timeout(end_time, NULL))
739 break;
740 }
741
742 err = igt_live_test_end(&t);
743 if (err)
744 goto out_batch;
745
746 pr_info("Batch latencies on %s: 1 = %lluns, %lu = %lluns\n",
747 engine->name,
748 ktime_to_ns(times[0]),
749 prime, div64_u64(ktime_to_ns(times[1]), prime));
750 }
751
752out_batch:
753 i915_vma_unpin(batch);
754 i915_vma_put(batch);
755out_unlock:
756 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
757 mutex_unlock(&i915->drm.struct_mutex);
758 return err;
759}
760
761static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
762{
763 struct i915_gem_context *ctx = i915->kernel_context;
764 struct i915_address_space *vm = ctx->vm ?: &i915->ggtt.vm;
765 struct drm_i915_gem_object *obj;
766 const int gen = INTEL_GEN(i915);
767 struct i915_vma *vma;
768 u32 *cmd;
769 int err;
770
771 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
772 if (IS_ERR(obj))
773 return ERR_CAST(obj);
774
775 vma = i915_vma_instance(obj, vm, NULL);
776 if (IS_ERR(vma)) {
777 err = PTR_ERR(vma);
778 goto err;
779 }
780
781 err = i915_vma_pin(vma, 0, 0, PIN_USER);
782 if (err)
783 goto err;
784
785 cmd = i915_gem_object_pin_map(obj, I915_MAP_WC);
786 if (IS_ERR(cmd)) {
787 err = PTR_ERR(cmd);
788 goto err;
789 }
790
791 if (gen >= 8) {
792 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
793 *cmd++ = lower_32_bits(vma->node.start);
794 *cmd++ = upper_32_bits(vma->node.start);
795 } else if (gen >= 6) {
796 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8;
797 *cmd++ = lower_32_bits(vma->node.start);
798 } else {
799 *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
800 *cmd++ = lower_32_bits(vma->node.start);
801 }
802 *cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */
803
804 __i915_gem_object_flush_map(obj, 0, 64);
805 i915_gem_object_unpin_map(obj);
806
807 intel_gt_chipset_flush(&i915->gt);
808
809 return vma;
810
811err:
812 i915_gem_object_put(obj);
813 return ERR_PTR(err);
814}
815
816static int recursive_batch_resolve(struct i915_vma *batch)
817{
818 u32 *cmd;
819
820 cmd = i915_gem_object_pin_map(batch->obj, I915_MAP_WC);
821 if (IS_ERR(cmd))
822 return PTR_ERR(cmd);
823
824 *cmd = MI_BATCH_BUFFER_END;
825 intel_gt_chipset_flush(batch->vm->gt);
826
827 i915_gem_object_unpin_map(batch->obj);
828
829 return 0;
830}
831
832static int live_all_engines(void *arg)
833{
834 struct drm_i915_private *i915 = arg;
835 struct intel_engine_cs *engine;
836 struct i915_request *request[I915_NUM_ENGINES];
837 intel_wakeref_t wakeref;
838 struct igt_live_test t;
839 struct i915_vma *batch;
840 unsigned int id;
841 int err;
842
843 /* Check we can submit requests to all engines simultaneously. We
844 * send a recursive batch to each engine - checking that we don't
845 * block doing so, and that they don't complete too soon.
846 */
847
848 mutex_lock(&i915->drm.struct_mutex);
849 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
850
851 err = igt_live_test_begin(&t, i915, __func__, "");
852 if (err)
853 goto out_unlock;
854
855 batch = recursive_batch(i915);
856 if (IS_ERR(batch)) {
857 err = PTR_ERR(batch);
858 pr_err("%s: Unable to create batch, err=%d\n", __func__, err);
859 goto out_unlock;
860 }
861
862 for_each_engine(engine, i915, id) {
863 request[id] = i915_request_create(engine->kernel_context);
864 if (IS_ERR(request[id])) {
865 err = PTR_ERR(request[id]);
866 pr_err("%s: Request allocation failed with err=%d\n",
867 __func__, err);
868 goto out_request;
869 }
870
871 err = engine->emit_bb_start(request[id],
872 batch->node.start,
873 batch->node.size,
874 0);
875 GEM_BUG_ON(err);
876 request[id]->batch = batch;
877
878 i915_vma_lock(batch);
879 err = i915_request_await_object(request[id], batch->obj, 0);
880 if (err == 0)
881 err = i915_vma_move_to_active(batch, request[id], 0);
882 i915_vma_unlock(batch);
883 GEM_BUG_ON(err);
884
885 i915_request_get(request[id]);
886 i915_request_add(request[id]);
887 }
888
889 for_each_engine(engine, i915, id) {
890 if (i915_request_completed(request[id])) {
891 pr_err("%s(%s): request completed too early!\n",
892 __func__, engine->name);
893 err = -EINVAL;
894 goto out_request;
895 }
896 }
897
898 err = recursive_batch_resolve(batch);
899 if (err) {
900 pr_err("%s: failed to resolve batch, err=%d\n", __func__, err);
901 goto out_request;
902 }
903
904 for_each_engine(engine, i915, id) {
905 long timeout;
906
907 timeout = i915_request_wait(request[id], 0,
908 MAX_SCHEDULE_TIMEOUT);
909 if (timeout < 0) {
910 err = timeout;
911 pr_err("%s: error waiting for request on %s, err=%d\n",
912 __func__, engine->name, err);
913 goto out_request;
914 }
915
916 GEM_BUG_ON(!i915_request_completed(request[id]));
917 i915_request_put(request[id]);
918 request[id] = NULL;
919 }
920
921 err = igt_live_test_end(&t);
922
923out_request:
924 for_each_engine(engine, i915, id)
925 if (request[id])
926 i915_request_put(request[id]);
927 i915_vma_unpin(batch);
928 i915_vma_put(batch);
929out_unlock:
930 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
931 mutex_unlock(&i915->drm.struct_mutex);
932 return err;
933}
934
935static int live_sequential_engines(void *arg)
936{
937 struct drm_i915_private *i915 = arg;
938 struct i915_request *request[I915_NUM_ENGINES] = {};
939 struct i915_request *prev = NULL;
940 struct intel_engine_cs *engine;
941 intel_wakeref_t wakeref;
942 struct igt_live_test t;
943 unsigned int id;
944 int err;
945
946 /* Check we can submit requests to all engines sequentially, such
947 * that each successive request waits for the earlier ones. This
948 * tests that we don't execute requests out of order, even though
949 * they are running on independent engines.
950 */
951
952 mutex_lock(&i915->drm.struct_mutex);
953 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
954
955 err = igt_live_test_begin(&t, i915, __func__, "");
956 if (err)
957 goto out_unlock;
958
959 for_each_engine(engine, i915, id) {
960 struct i915_vma *batch;
961
962 batch = recursive_batch(i915);
963 if (IS_ERR(batch)) {
964 err = PTR_ERR(batch);
965 pr_err("%s: Unable to create batch for %s, err=%d\n",
966 __func__, engine->name, err);
967 goto out_unlock;
968 }
969
970 request[id] = i915_request_create(engine->kernel_context);
971 if (IS_ERR(request[id])) {
972 err = PTR_ERR(request[id]);
973 pr_err("%s: Request allocation failed for %s with err=%d\n",
974 __func__, engine->name, err);
975 goto out_request;
976 }
977
978 if (prev) {
979 err = i915_request_await_dma_fence(request[id],
980 &prev->fence);
981 if (err) {
982 i915_request_add(request[id]);
983 pr_err("%s: Request await failed for %s with err=%d\n",
984 __func__, engine->name, err);
985 goto out_request;
986 }
987 }
988
989 err = engine->emit_bb_start(request[id],
990 batch->node.start,
991 batch->node.size,
992 0);
993 GEM_BUG_ON(err);
994 request[id]->batch = batch;
995
996 i915_vma_lock(batch);
997 err = i915_request_await_object(request[id], batch->obj, false);
998 if (err == 0)
999 err = i915_vma_move_to_active(batch, request[id], 0);
1000 i915_vma_unlock(batch);
1001 GEM_BUG_ON(err);
1002
1003 i915_request_get(request[id]);
1004 i915_request_add(request[id]);
1005
1006 prev = request[id];
1007 }
1008
1009 for_each_engine(engine, i915, id) {
1010 long timeout;
1011
1012 if (i915_request_completed(request[id])) {
1013 pr_err("%s(%s): request completed too early!\n",
1014 __func__, engine->name);
1015 err = -EINVAL;
1016 goto out_request;
1017 }
1018
1019 err = recursive_batch_resolve(request[id]->batch);
1020 if (err) {
1021 pr_err("%s: failed to resolve batch, err=%d\n",
1022 __func__, err);
1023 goto out_request;
1024 }
1025
1026 timeout = i915_request_wait(request[id], 0,
1027 MAX_SCHEDULE_TIMEOUT);
1028 if (timeout < 0) {
1029 err = timeout;
1030 pr_err("%s: error waiting for request on %s, err=%d\n",
1031 __func__, engine->name, err);
1032 goto out_request;
1033 }
1034
1035 GEM_BUG_ON(!i915_request_completed(request[id]));
1036 }
1037
1038 err = igt_live_test_end(&t);
1039
1040out_request:
1041 for_each_engine(engine, i915, id) {
1042 u32 *cmd;
1043
1044 if (!request[id])
1045 break;
1046
1047 cmd = i915_gem_object_pin_map(request[id]->batch->obj,
1048 I915_MAP_WC);
1049 if (!IS_ERR(cmd)) {
1050 *cmd = MI_BATCH_BUFFER_END;
1051 intel_gt_chipset_flush(engine->gt);
1052
1053 i915_gem_object_unpin_map(request[id]->batch->obj);
1054 }
1055
1056 i915_vma_put(request[id]->batch);
1057 i915_request_put(request[id]);
1058 }
1059out_unlock:
1060 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1061 mutex_unlock(&i915->drm.struct_mutex);
1062 return err;
1063}
1064
1065static int
1066max_batches(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
1067{
1068 struct i915_request *rq;
1069 int ret;
1070
1071 /*
1072 * Before execlists, all contexts share the same ringbuffer. With
1073 * execlists, each context/engine has a separate ringbuffer and
1074 * for the purposes of this test, inexhaustible.
1075 *
1076 * For the global ringbuffer though, we have to be very careful
1077 * that we do not wrap while preventing the execution of requests
1078 * with a unsignaled fence.
1079 */
1080 if (HAS_EXECLISTS(ctx->i915))
1081 return INT_MAX;
1082
1083 rq = igt_request_alloc(ctx, engine);
1084 if (IS_ERR(rq)) {
1085 ret = PTR_ERR(rq);
1086 } else {
1087 int sz;
1088
1089 ret = rq->ring->size - rq->reserved_space;
1090 i915_request_add(rq);
1091
1092 sz = rq->ring->emit - rq->head;
1093 if (sz < 0)
1094 sz += rq->ring->size;
1095 ret /= sz;
1096 ret /= 2; /* leave half spare, in case of emergency! */
1097 }
1098
1099 return ret;
1100}
1101
1102static int live_breadcrumbs_smoketest(void *arg)
1103{
1104 struct drm_i915_private *i915 = arg;
1105 struct smoketest t[I915_NUM_ENGINES];
1106 unsigned int ncpus = num_online_cpus();
1107 unsigned long num_waits, num_fences;
1108 struct intel_engine_cs *engine;
1109 struct task_struct **threads;
1110 struct igt_live_test live;
1111 enum intel_engine_id id;
1112 intel_wakeref_t wakeref;
1113 struct drm_file *file;
1114 unsigned int n;
1115 int ret = 0;
1116
1117 /*
1118 * Smoketest our breadcrumb/signal handling for requests across multiple
1119 * threads. A very simple test to only catch the most egregious of bugs.
1120 * See __igt_breadcrumbs_smoketest();
1121 *
1122 * On real hardware this time.
1123 */
1124
1125 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1126
1127 file = mock_file(i915);
1128 if (IS_ERR(file)) {
1129 ret = PTR_ERR(file);
1130 goto out_rpm;
1131 }
1132
1133 threads = kcalloc(ncpus * I915_NUM_ENGINES,
1134 sizeof(*threads),
1135 GFP_KERNEL);
1136 if (!threads) {
1137 ret = -ENOMEM;
1138 goto out_file;
1139 }
1140
1141 memset(&t[0], 0, sizeof(t[0]));
1142 t[0].request_alloc = __live_request_alloc;
1143 t[0].ncontexts = 64;
1144 t[0].contexts = kmalloc_array(t[0].ncontexts,
1145 sizeof(*t[0].contexts),
1146 GFP_KERNEL);
1147 if (!t[0].contexts) {
1148 ret = -ENOMEM;
1149 goto out_threads;
1150 }
1151
1152 mutex_lock(&i915->drm.struct_mutex);
1153 for (n = 0; n < t[0].ncontexts; n++) {
1154 t[0].contexts[n] = live_context(i915, file);
1155 if (!t[0].contexts[n]) {
1156 ret = -ENOMEM;
1157 goto out_contexts;
1158 }
1159 }
1160
1161 ret = igt_live_test_begin(&live, i915, __func__, "");
1162 if (ret)
1163 goto out_contexts;
1164
1165 for_each_engine(engine, i915, id) {
1166 t[id] = t[0];
1167 t[id].engine = engine;
1168 t[id].max_batch = max_batches(t[0].contexts[0], engine);
1169 if (t[id].max_batch < 0) {
1170 ret = t[id].max_batch;
1171 mutex_unlock(&i915->drm.struct_mutex);
1172 goto out_flush;
1173 }
1174 /* One ring interleaved between requests from all cpus */
1175 t[id].max_batch /= num_online_cpus() + 1;
1176 pr_debug("Limiting batches to %d requests on %s\n",
1177 t[id].max_batch, engine->name);
1178
1179 for (n = 0; n < ncpus; n++) {
1180 struct task_struct *tsk;
1181
1182 tsk = kthread_run(__igt_breadcrumbs_smoketest,
1183 &t[id], "igt/%d.%d", id, n);
1184 if (IS_ERR(tsk)) {
1185 ret = PTR_ERR(tsk);
1186 mutex_unlock(&i915->drm.struct_mutex);
1187 goto out_flush;
1188 }
1189
1190 get_task_struct(tsk);
1191 threads[id * ncpus + n] = tsk;
1192 }
1193 }
1194 mutex_unlock(&i915->drm.struct_mutex);
1195
1196 msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies));
1197
1198out_flush:
1199 num_waits = 0;
1200 num_fences = 0;
1201 for_each_engine(engine, i915, id) {
1202 for (n = 0; n < ncpus; n++) {
1203 struct task_struct *tsk = threads[id * ncpus + n];
1204 int err;
1205
1206 if (!tsk)
1207 continue;
1208
1209 err = kthread_stop(tsk);
1210 if (err < 0 && !ret)
1211 ret = err;
1212
1213 put_task_struct(tsk);
1214 }
1215
1216 num_waits += atomic_long_read(&t[id].num_waits);
1217 num_fences += atomic_long_read(&t[id].num_fences);
1218 }
1219 pr_info("Completed %lu waits for %lu fences across %d engines and %d cpus\n",
1220 num_waits, num_fences, RUNTIME_INFO(i915)->num_engines, ncpus);
1221
1222 mutex_lock(&i915->drm.struct_mutex);
1223 ret = igt_live_test_end(&live) ?: ret;
1224out_contexts:
1225 mutex_unlock(&i915->drm.struct_mutex);
1226 kfree(t[0].contexts);
1227out_threads:
1228 kfree(threads);
1229out_file:
1230 mock_file_free(i915, file);
1231out_rpm:
1232 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1233
1234 return ret;
1235}
1236
1237int i915_request_live_selftests(struct drm_i915_private *i915)
1238{
1239 static const struct i915_subtest tests[] = {
1240 SUBTEST(live_nop_request),
1241 SUBTEST(live_all_engines),
1242 SUBTEST(live_sequential_engines),
1243 SUBTEST(live_empty_request),
1244 SUBTEST(live_breadcrumbs_smoketest),
1245 };
1246
1247 if (intel_gt_is_wedged(&i915->gt))
1248 return 0;
1249
1250 return i915_subtests(tests, i915);
1251}
1/*
2 * Copyright © 2016 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25#include <linux/prime_numbers.h>
26#include <linux/pm_qos.h>
27#include <linux/sort.h>
28
29#include "gem/i915_gem_internal.h"
30#include "gem/i915_gem_pm.h"
31#include "gem/selftests/mock_context.h"
32
33#include "gt/intel_engine_heartbeat.h"
34#include "gt/intel_engine_pm.h"
35#include "gt/intel_engine_user.h"
36#include "gt/intel_gt.h"
37#include "gt/intel_gt_clock_utils.h"
38#include "gt/intel_gt_requests.h"
39#include "gt/selftest_engine_heartbeat.h"
40
41#include "i915_random.h"
42#include "i915_selftest.h"
43#include "igt_flush_test.h"
44#include "igt_live_test.h"
45#include "igt_spinner.h"
46#include "lib_sw_fence.h"
47
48#include "mock_drm.h"
49#include "mock_gem_device.h"
50
51static unsigned int num_uabi_engines(struct drm_i915_private *i915)
52{
53 struct intel_engine_cs *engine;
54 unsigned int count;
55
56 count = 0;
57 for_each_uabi_engine(engine, i915)
58 count++;
59
60 return count;
61}
62
63static struct intel_engine_cs *rcs0(struct drm_i915_private *i915)
64{
65 return intel_engine_lookup_user(i915, I915_ENGINE_CLASS_RENDER, 0);
66}
67
68static int igt_add_request(void *arg)
69{
70 struct drm_i915_private *i915 = arg;
71 struct i915_request *request;
72
73 /* Basic preliminary test to create a request and let it loose! */
74
75 request = mock_request(rcs0(i915)->kernel_context, HZ / 10);
76 if (!request)
77 return -ENOMEM;
78
79 i915_request_add(request);
80
81 return 0;
82}
83
84static int igt_wait_request(void *arg)
85{
86 const long T = HZ / 4;
87 struct drm_i915_private *i915 = arg;
88 struct i915_request *request;
89 int err = -EINVAL;
90
91 /* Submit a request, then wait upon it */
92
93 request = mock_request(rcs0(i915)->kernel_context, T);
94 if (!request)
95 return -ENOMEM;
96
97 i915_request_get(request);
98
99 if (i915_request_wait(request, 0, 0) != -ETIME) {
100 pr_err("request wait (busy query) succeeded (expected timeout before submit!)\n");
101 goto out_request;
102 }
103
104 if (i915_request_wait(request, 0, T) != -ETIME) {
105 pr_err("request wait succeeded (expected timeout before submit!)\n");
106 goto out_request;
107 }
108
109 if (i915_request_completed(request)) {
110 pr_err("request completed before submit!!\n");
111 goto out_request;
112 }
113
114 i915_request_add(request);
115
116 if (i915_request_wait(request, 0, 0) != -ETIME) {
117 pr_err("request wait (busy query) succeeded (expected timeout after submit!)\n");
118 goto out_request;
119 }
120
121 if (i915_request_completed(request)) {
122 pr_err("request completed immediately!\n");
123 goto out_request;
124 }
125
126 if (i915_request_wait(request, 0, T / 2) != -ETIME) {
127 pr_err("request wait succeeded (expected timeout!)\n");
128 goto out_request;
129 }
130
131 if (i915_request_wait(request, 0, T) == -ETIME) {
132 pr_err("request wait timed out!\n");
133 goto out_request;
134 }
135
136 if (!i915_request_completed(request)) {
137 pr_err("request not complete after waiting!\n");
138 goto out_request;
139 }
140
141 if (i915_request_wait(request, 0, T) == -ETIME) {
142 pr_err("request wait timed out when already complete!\n");
143 goto out_request;
144 }
145
146 err = 0;
147out_request:
148 i915_request_put(request);
149 mock_device_flush(i915);
150 return err;
151}
152
153static int igt_fence_wait(void *arg)
154{
155 const long T = HZ / 4;
156 struct drm_i915_private *i915 = arg;
157 struct i915_request *request;
158 int err = -EINVAL;
159
160 /* Submit a request, treat it as a fence and wait upon it */
161
162 request = mock_request(rcs0(i915)->kernel_context, T);
163 if (!request)
164 return -ENOMEM;
165
166 if (dma_fence_wait_timeout(&request->fence, false, T) != -ETIME) {
167 pr_err("fence wait success before submit (expected timeout)!\n");
168 goto out;
169 }
170
171 i915_request_add(request);
172
173 if (dma_fence_is_signaled(&request->fence)) {
174 pr_err("fence signaled immediately!\n");
175 goto out;
176 }
177
178 if (dma_fence_wait_timeout(&request->fence, false, T / 2) != -ETIME) {
179 pr_err("fence wait success after submit (expected timeout)!\n");
180 goto out;
181 }
182
183 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) {
184 pr_err("fence wait timed out (expected success)!\n");
185 goto out;
186 }
187
188 if (!dma_fence_is_signaled(&request->fence)) {
189 pr_err("fence unsignaled after waiting!\n");
190 goto out;
191 }
192
193 if (dma_fence_wait_timeout(&request->fence, false, T) <= 0) {
194 pr_err("fence wait timed out when complete (expected success)!\n");
195 goto out;
196 }
197
198 err = 0;
199out:
200 mock_device_flush(i915);
201 return err;
202}
203
204static int igt_request_rewind(void *arg)
205{
206 struct drm_i915_private *i915 = arg;
207 struct i915_request *request, *vip;
208 struct i915_gem_context *ctx[2];
209 struct intel_context *ce;
210 int err = -EINVAL;
211
212 ctx[0] = mock_context(i915, "A");
213 if (!ctx[0]) {
214 err = -ENOMEM;
215 goto err_ctx_0;
216 }
217
218 ce = i915_gem_context_get_engine(ctx[0], RCS0);
219 GEM_BUG_ON(IS_ERR(ce));
220 request = mock_request(ce, 2 * HZ);
221 intel_context_put(ce);
222 if (!request) {
223 err = -ENOMEM;
224 goto err_context_0;
225 }
226
227 i915_request_get(request);
228 i915_request_add(request);
229
230 ctx[1] = mock_context(i915, "B");
231 if (!ctx[1]) {
232 err = -ENOMEM;
233 goto err_ctx_1;
234 }
235
236 ce = i915_gem_context_get_engine(ctx[1], RCS0);
237 GEM_BUG_ON(IS_ERR(ce));
238 vip = mock_request(ce, 0);
239 intel_context_put(ce);
240 if (!vip) {
241 err = -ENOMEM;
242 goto err_context_1;
243 }
244
245 /* Simulate preemption by manual reordering */
246 if (!mock_cancel_request(request)) {
247 pr_err("failed to cancel request (already executed)!\n");
248 i915_request_add(vip);
249 goto err_context_1;
250 }
251 i915_request_get(vip);
252 i915_request_add(vip);
253 rcu_read_lock();
254 request->engine->submit_request(request);
255 rcu_read_unlock();
256
257
258 if (i915_request_wait(vip, 0, HZ) == -ETIME) {
259 pr_err("timed out waiting for high priority request\n");
260 goto err;
261 }
262
263 if (i915_request_completed(request)) {
264 pr_err("low priority request already completed\n");
265 goto err;
266 }
267
268 err = 0;
269err:
270 i915_request_put(vip);
271err_context_1:
272 mock_context_close(ctx[1]);
273err_ctx_1:
274 i915_request_put(request);
275err_context_0:
276 mock_context_close(ctx[0]);
277err_ctx_0:
278 mock_device_flush(i915);
279 return err;
280}
281
282struct smoketest {
283 struct intel_engine_cs *engine;
284 struct i915_gem_context **contexts;
285 atomic_long_t num_waits, num_fences;
286 int ncontexts, max_batch;
287 struct i915_request *(*request_alloc)(struct intel_context *ce);
288};
289
290static struct i915_request *
291__mock_request_alloc(struct intel_context *ce)
292{
293 return mock_request(ce, 0);
294}
295
296static struct i915_request *
297__live_request_alloc(struct intel_context *ce)
298{
299 return intel_context_create_request(ce);
300}
301
302struct smoke_thread {
303 struct kthread_worker *worker;
304 struct kthread_work work;
305 struct smoketest *t;
306 bool stop;
307 int result;
308};
309
310static void __igt_breadcrumbs_smoketest(struct kthread_work *work)
311{
312 struct smoke_thread *thread = container_of(work, typeof(*thread), work);
313 struct smoketest *t = thread->t;
314 const unsigned int max_batch = min(t->ncontexts, t->max_batch) - 1;
315 const unsigned int total = 4 * t->ncontexts + 1;
316 unsigned int num_waits = 0, num_fences = 0;
317 struct i915_request **requests;
318 I915_RND_STATE(prng);
319 unsigned int *order;
320 int err = 0;
321
322 /*
323 * A very simple test to catch the most egregious of list handling bugs.
324 *
325 * At its heart, we simply create oodles of requests running across
326 * multiple kthreads and enable signaling on them, for the sole purpose
327 * of stressing our breadcrumb handling. The only inspection we do is
328 * that the fences were marked as signaled.
329 */
330
331 requests = kcalloc(total, sizeof(*requests), GFP_KERNEL);
332 if (!requests) {
333 thread->result = -ENOMEM;
334 return;
335 }
336
337 order = i915_random_order(total, &prng);
338 if (!order) {
339 err = -ENOMEM;
340 goto out_requests;
341 }
342
343 while (!READ_ONCE(thread->stop)) {
344 struct i915_sw_fence *submit, *wait;
345 unsigned int n, count;
346
347 submit = heap_fence_create(GFP_KERNEL);
348 if (!submit) {
349 err = -ENOMEM;
350 break;
351 }
352
353 wait = heap_fence_create(GFP_KERNEL);
354 if (!wait) {
355 i915_sw_fence_commit(submit);
356 heap_fence_put(submit);
357 err = -ENOMEM;
358 break;
359 }
360
361 i915_random_reorder(order, total, &prng);
362 count = 1 + i915_prandom_u32_max_state(max_batch, &prng);
363
364 for (n = 0; n < count; n++) {
365 struct i915_gem_context *ctx =
366 t->contexts[order[n] % t->ncontexts];
367 struct i915_request *rq;
368 struct intel_context *ce;
369
370 ce = i915_gem_context_get_engine(ctx, t->engine->legacy_idx);
371 GEM_BUG_ON(IS_ERR(ce));
372 rq = t->request_alloc(ce);
373 intel_context_put(ce);
374 if (IS_ERR(rq)) {
375 err = PTR_ERR(rq);
376 count = n;
377 break;
378 }
379
380 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
381 submit,
382 GFP_KERNEL);
383
384 requests[n] = i915_request_get(rq);
385 i915_request_add(rq);
386
387 if (err >= 0)
388 err = i915_sw_fence_await_dma_fence(wait,
389 &rq->fence,
390 0,
391 GFP_KERNEL);
392
393 if (err < 0) {
394 i915_request_put(rq);
395 count = n;
396 break;
397 }
398 }
399
400 i915_sw_fence_commit(submit);
401 i915_sw_fence_commit(wait);
402
403 if (!wait_event_timeout(wait->wait,
404 i915_sw_fence_done(wait),
405 5 * HZ)) {
406 struct i915_request *rq = requests[count - 1];
407
408 pr_err("waiting for %d/%d fences (last %llx:%lld) on %s timed out!\n",
409 atomic_read(&wait->pending), count,
410 rq->fence.context, rq->fence.seqno,
411 t->engine->name);
412 GEM_TRACE_DUMP();
413
414 intel_gt_set_wedged(t->engine->gt);
415 GEM_BUG_ON(!i915_request_completed(rq));
416 i915_sw_fence_wait(wait);
417 err = -EIO;
418 }
419
420 for (n = 0; n < count; n++) {
421 struct i915_request *rq = requests[n];
422
423 if (!test_bit(DMA_FENCE_FLAG_SIGNALED_BIT,
424 &rq->fence.flags)) {
425 pr_err("%llu:%llu was not signaled!\n",
426 rq->fence.context, rq->fence.seqno);
427 err = -EINVAL;
428 }
429
430 i915_request_put(rq);
431 }
432
433 heap_fence_put(wait);
434 heap_fence_put(submit);
435
436 if (err < 0)
437 break;
438
439 num_fences += count;
440 num_waits++;
441
442 cond_resched();
443 }
444
445 atomic_long_add(num_fences, &t->num_fences);
446 atomic_long_add(num_waits, &t->num_waits);
447
448 kfree(order);
449out_requests:
450 kfree(requests);
451 thread->result = err;
452}
453
454static int mock_breadcrumbs_smoketest(void *arg)
455{
456 struct drm_i915_private *i915 = arg;
457 struct smoketest t = {
458 .engine = rcs0(i915),
459 .ncontexts = 1024,
460 .max_batch = 1024,
461 .request_alloc = __mock_request_alloc
462 };
463 unsigned int ncpus = num_online_cpus();
464 struct smoke_thread *threads;
465 unsigned int n;
466 int ret = 0;
467
468 /*
469 * Smoketest our breadcrumb/signal handling for requests across multiple
470 * threads. A very simple test to only catch the most egregious of bugs.
471 * See __igt_breadcrumbs_smoketest();
472 */
473
474 threads = kcalloc(ncpus, sizeof(*threads), GFP_KERNEL);
475 if (!threads)
476 return -ENOMEM;
477
478 t.contexts = kcalloc(t.ncontexts, sizeof(*t.contexts), GFP_KERNEL);
479 if (!t.contexts) {
480 ret = -ENOMEM;
481 goto out_threads;
482 }
483
484 for (n = 0; n < t.ncontexts; n++) {
485 t.contexts[n] = mock_context(t.engine->i915, "mock");
486 if (!t.contexts[n]) {
487 ret = -ENOMEM;
488 goto out_contexts;
489 }
490 }
491
492 for (n = 0; n < ncpus; n++) {
493 struct kthread_worker *worker;
494
495 worker = kthread_create_worker(0, "igt/%d", n);
496 if (IS_ERR(worker)) {
497 ret = PTR_ERR(worker);
498 ncpus = n;
499 break;
500 }
501
502 threads[n].worker = worker;
503 threads[n].t = &t;
504 threads[n].stop = false;
505 threads[n].result = 0;
506
507 kthread_init_work(&threads[n].work,
508 __igt_breadcrumbs_smoketest);
509 kthread_queue_work(worker, &threads[n].work);
510 }
511
512 msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies));
513
514 for (n = 0; n < ncpus; n++) {
515 int err;
516
517 WRITE_ONCE(threads[n].stop, true);
518 kthread_flush_work(&threads[n].work);
519 err = READ_ONCE(threads[n].result);
520 if (err < 0 && !ret)
521 ret = err;
522
523 kthread_destroy_worker(threads[n].worker);
524 }
525 pr_info("Completed %lu waits for %lu fence across %d cpus\n",
526 atomic_long_read(&t.num_waits),
527 atomic_long_read(&t.num_fences),
528 ncpus);
529
530out_contexts:
531 for (n = 0; n < t.ncontexts; n++) {
532 if (!t.contexts[n])
533 break;
534 mock_context_close(t.contexts[n]);
535 }
536 kfree(t.contexts);
537out_threads:
538 kfree(threads);
539 return ret;
540}
541
542int i915_request_mock_selftests(void)
543{
544 static const struct i915_subtest tests[] = {
545 SUBTEST(igt_add_request),
546 SUBTEST(igt_wait_request),
547 SUBTEST(igt_fence_wait),
548 SUBTEST(igt_request_rewind),
549 SUBTEST(mock_breadcrumbs_smoketest),
550 };
551 struct drm_i915_private *i915;
552 intel_wakeref_t wakeref;
553 int err = 0;
554
555 i915 = mock_gem_device();
556 if (!i915)
557 return -ENOMEM;
558
559 with_intel_runtime_pm(&i915->runtime_pm, wakeref)
560 err = i915_subtests(tests, i915);
561
562 mock_destroy_device(i915);
563
564 return err;
565}
566
567static int live_nop_request(void *arg)
568{
569 struct drm_i915_private *i915 = arg;
570 struct intel_engine_cs *engine;
571 struct igt_live_test t;
572 int err = -ENODEV;
573
574 /*
575 * Submit various sized batches of empty requests, to each engine
576 * (individually), and wait for the batch to complete. We can check
577 * the overhead of submitting requests to the hardware.
578 */
579
580 for_each_uabi_engine(engine, i915) {
581 unsigned long n, prime;
582 IGT_TIMEOUT(end_time);
583 ktime_t times[2] = {};
584
585 err = igt_live_test_begin(&t, i915, __func__, engine->name);
586 if (err)
587 return err;
588
589 intel_engine_pm_get(engine);
590 for_each_prime_number_from(prime, 1, 8192) {
591 struct i915_request *request = NULL;
592
593 times[1] = ktime_get_raw();
594
595 for (n = 0; n < prime; n++) {
596 i915_request_put(request);
597 request = i915_request_create(engine->kernel_context);
598 if (IS_ERR(request))
599 return PTR_ERR(request);
600
601 /*
602 * This space is left intentionally blank.
603 *
604 * We do not actually want to perform any
605 * action with this request, we just want
606 * to measure the latency in allocation
607 * and submission of our breadcrumbs -
608 * ensuring that the bare request is sufficient
609 * for the system to work (i.e. proper HEAD
610 * tracking of the rings, interrupt handling,
611 * etc). It also gives us the lowest bounds
612 * for latency.
613 */
614
615 i915_request_get(request);
616 i915_request_add(request);
617 }
618 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
619 i915_request_put(request);
620
621 times[1] = ktime_sub(ktime_get_raw(), times[1]);
622 if (prime == 1)
623 times[0] = times[1];
624
625 if (__igt_timeout(end_time, NULL))
626 break;
627 }
628 intel_engine_pm_put(engine);
629
630 err = igt_live_test_end(&t);
631 if (err)
632 return err;
633
634 pr_info("Request latencies on %s: 1 = %lluns, %lu = %lluns\n",
635 engine->name,
636 ktime_to_ns(times[0]),
637 prime, div64_u64(ktime_to_ns(times[1]), prime));
638 }
639
640 return err;
641}
642
643static int __cancel_inactive(struct intel_engine_cs *engine)
644{
645 struct intel_context *ce;
646 struct igt_spinner spin;
647 struct i915_request *rq;
648 int err = 0;
649
650 if (igt_spinner_init(&spin, engine->gt))
651 return -ENOMEM;
652
653 ce = intel_context_create(engine);
654 if (IS_ERR(ce)) {
655 err = PTR_ERR(ce);
656 goto out_spin;
657 }
658
659 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
660 if (IS_ERR(rq)) {
661 err = PTR_ERR(rq);
662 goto out_ce;
663 }
664
665 pr_debug("%s: Cancelling inactive request\n", engine->name);
666 i915_request_cancel(rq, -EINTR);
667 i915_request_get(rq);
668 i915_request_add(rq);
669
670 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
671 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
672
673 pr_err("%s: Failed to cancel inactive request\n", engine->name);
674 intel_engine_dump(engine, &p, "%s\n", engine->name);
675 err = -ETIME;
676 goto out_rq;
677 }
678
679 if (rq->fence.error != -EINTR) {
680 pr_err("%s: fence not cancelled (%u)\n",
681 engine->name, rq->fence.error);
682 err = -EINVAL;
683 }
684
685out_rq:
686 i915_request_put(rq);
687out_ce:
688 intel_context_put(ce);
689out_spin:
690 igt_spinner_fini(&spin);
691 if (err)
692 pr_err("%s: %s error %d\n", __func__, engine->name, err);
693 return err;
694}
695
696static int __cancel_active(struct intel_engine_cs *engine)
697{
698 struct intel_context *ce;
699 struct igt_spinner spin;
700 struct i915_request *rq;
701 int err = 0;
702
703 if (igt_spinner_init(&spin, engine->gt))
704 return -ENOMEM;
705
706 ce = intel_context_create(engine);
707 if (IS_ERR(ce)) {
708 err = PTR_ERR(ce);
709 goto out_spin;
710 }
711
712 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
713 if (IS_ERR(rq)) {
714 err = PTR_ERR(rq);
715 goto out_ce;
716 }
717
718 pr_debug("%s: Cancelling active request\n", engine->name);
719 i915_request_get(rq);
720 i915_request_add(rq);
721 if (!igt_wait_for_spinner(&spin, rq)) {
722 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
723
724 pr_err("Failed to start spinner on %s\n", engine->name);
725 intel_engine_dump(engine, &p, "%s\n", engine->name);
726 err = -ETIME;
727 goto out_rq;
728 }
729 i915_request_cancel(rq, -EINTR);
730
731 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
732 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
733
734 pr_err("%s: Failed to cancel active request\n", engine->name);
735 intel_engine_dump(engine, &p, "%s\n", engine->name);
736 err = -ETIME;
737 goto out_rq;
738 }
739
740 if (rq->fence.error != -EINTR) {
741 pr_err("%s: fence not cancelled (%u)\n",
742 engine->name, rq->fence.error);
743 err = -EINVAL;
744 }
745
746out_rq:
747 i915_request_put(rq);
748out_ce:
749 intel_context_put(ce);
750out_spin:
751 igt_spinner_fini(&spin);
752 if (err)
753 pr_err("%s: %s error %d\n", __func__, engine->name, err);
754 return err;
755}
756
757static int __cancel_completed(struct intel_engine_cs *engine)
758{
759 struct intel_context *ce;
760 struct igt_spinner spin;
761 struct i915_request *rq;
762 int err = 0;
763
764 if (igt_spinner_init(&spin, engine->gt))
765 return -ENOMEM;
766
767 ce = intel_context_create(engine);
768 if (IS_ERR(ce)) {
769 err = PTR_ERR(ce);
770 goto out_spin;
771 }
772
773 rq = igt_spinner_create_request(&spin, ce, MI_ARB_CHECK);
774 if (IS_ERR(rq)) {
775 err = PTR_ERR(rq);
776 goto out_ce;
777 }
778 igt_spinner_end(&spin);
779 i915_request_get(rq);
780 i915_request_add(rq);
781
782 if (i915_request_wait(rq, 0, HZ / 5) < 0) {
783 err = -ETIME;
784 goto out_rq;
785 }
786
787 pr_debug("%s: Cancelling completed request\n", engine->name);
788 i915_request_cancel(rq, -EINTR);
789 if (rq->fence.error) {
790 pr_err("%s: fence not cancelled (%u)\n",
791 engine->name, rq->fence.error);
792 err = -EINVAL;
793 }
794
795out_rq:
796 i915_request_put(rq);
797out_ce:
798 intel_context_put(ce);
799out_spin:
800 igt_spinner_fini(&spin);
801 if (err)
802 pr_err("%s: %s error %d\n", __func__, engine->name, err);
803 return err;
804}
805
806/*
807 * Test to prove a non-preemptable request can be cancelled and a subsequent
808 * request on the same context can successfully complete after cancellation.
809 *
810 * Testing methodology is to create a non-preemptible request and submit it,
811 * wait for spinner to start, create a NOP request and submit it, cancel the
812 * spinner, wait for spinner to complete and verify it failed with an error,
813 * finally wait for NOP request to complete verify it succeeded without an
814 * error. Preemption timeout also reduced / restored so test runs in a timely
815 * maner.
816 */
817static int __cancel_reset(struct drm_i915_private *i915,
818 struct intel_engine_cs *engine)
819{
820 struct intel_context *ce;
821 struct igt_spinner spin;
822 struct i915_request *rq, *nop;
823 unsigned long preempt_timeout_ms;
824 int err = 0;
825
826 if (!CONFIG_DRM_I915_PREEMPT_TIMEOUT ||
827 !intel_has_reset_engine(engine->gt))
828 return 0;
829
830 preempt_timeout_ms = engine->props.preempt_timeout_ms;
831 engine->props.preempt_timeout_ms = 100;
832
833 if (igt_spinner_init(&spin, engine->gt))
834 goto out_restore;
835
836 ce = intel_context_create(engine);
837 if (IS_ERR(ce)) {
838 err = PTR_ERR(ce);
839 goto out_spin;
840 }
841
842 rq = igt_spinner_create_request(&spin, ce, MI_NOOP);
843 if (IS_ERR(rq)) {
844 err = PTR_ERR(rq);
845 goto out_ce;
846 }
847
848 pr_debug("%s: Cancelling active non-preemptable request\n",
849 engine->name);
850 i915_request_get(rq);
851 i915_request_add(rq);
852 if (!igt_wait_for_spinner(&spin, rq)) {
853 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
854
855 pr_err("Failed to start spinner on %s\n", engine->name);
856 intel_engine_dump(engine, &p, "%s\n", engine->name);
857 err = -ETIME;
858 goto out_rq;
859 }
860
861 nop = intel_context_create_request(ce);
862 if (IS_ERR(nop))
863 goto out_rq;
864 i915_request_get(nop);
865 i915_request_add(nop);
866
867 i915_request_cancel(rq, -EINTR);
868
869 if (i915_request_wait(rq, 0, HZ) < 0) {
870 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
871
872 pr_err("%s: Failed to cancel hung request\n", engine->name);
873 intel_engine_dump(engine, &p, "%s\n", engine->name);
874 err = -ETIME;
875 goto out_nop;
876 }
877
878 if (rq->fence.error != -EINTR) {
879 pr_err("%s: fence not cancelled (%u)\n",
880 engine->name, rq->fence.error);
881 err = -EINVAL;
882 goto out_nop;
883 }
884
885 if (i915_request_wait(nop, 0, HZ) < 0) {
886 struct drm_printer p = drm_info_printer(engine->i915->drm.dev);
887
888 pr_err("%s: Failed to complete nop request\n", engine->name);
889 intel_engine_dump(engine, &p, "%s\n", engine->name);
890 err = -ETIME;
891 goto out_nop;
892 }
893
894 if (nop->fence.error != 0) {
895 pr_err("%s: Nop request errored (%u)\n",
896 engine->name, nop->fence.error);
897 err = -EINVAL;
898 }
899
900out_nop:
901 i915_request_put(nop);
902out_rq:
903 i915_request_put(rq);
904out_ce:
905 intel_context_put(ce);
906out_spin:
907 igt_spinner_fini(&spin);
908out_restore:
909 engine->props.preempt_timeout_ms = preempt_timeout_ms;
910 if (err)
911 pr_err("%s: %s error %d\n", __func__, engine->name, err);
912 return err;
913}
914
915static int live_cancel_request(void *arg)
916{
917 struct drm_i915_private *i915 = arg;
918 struct intel_engine_cs *engine;
919
920 /*
921 * Check cancellation of requests. We expect to be able to immediately
922 * cancel active requests, even if they are currently on the GPU.
923 */
924
925 for_each_uabi_engine(engine, i915) {
926 struct igt_live_test t;
927 int err, err2;
928
929 if (!intel_engine_has_preemption(engine))
930 continue;
931
932 err = igt_live_test_begin(&t, i915, __func__, engine->name);
933 if (err)
934 return err;
935
936 err = __cancel_inactive(engine);
937 if (err == 0)
938 err = __cancel_active(engine);
939 if (err == 0)
940 err = __cancel_completed(engine);
941
942 err2 = igt_live_test_end(&t);
943 if (err)
944 return err;
945 if (err2)
946 return err2;
947
948 /* Expects reset so call outside of igt_live_test_* */
949 err = __cancel_reset(i915, engine);
950 if (err)
951 return err;
952
953 if (igt_flush_test(i915))
954 return -EIO;
955 }
956
957 return 0;
958}
959
960static struct i915_vma *empty_batch(struct drm_i915_private *i915)
961{
962 struct drm_i915_gem_object *obj;
963 struct i915_vma *vma;
964 u32 *cmd;
965 int err;
966
967 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
968 if (IS_ERR(obj))
969 return ERR_CAST(obj);
970
971 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WB);
972 if (IS_ERR(cmd)) {
973 err = PTR_ERR(cmd);
974 goto err;
975 }
976
977 *cmd = MI_BATCH_BUFFER_END;
978
979 __i915_gem_object_flush_map(obj, 0, 64);
980 i915_gem_object_unpin_map(obj);
981
982 intel_gt_chipset_flush(to_gt(i915));
983
984 vma = i915_vma_instance(obj, &to_gt(i915)->ggtt->vm, NULL);
985 if (IS_ERR(vma)) {
986 err = PTR_ERR(vma);
987 goto err;
988 }
989
990 err = i915_vma_pin(vma, 0, 0, PIN_USER | PIN_GLOBAL);
991 if (err)
992 goto err;
993
994 /* Force the wait now to avoid including it in the benchmark */
995 err = i915_vma_sync(vma);
996 if (err)
997 goto err_pin;
998
999 return vma;
1000
1001err_pin:
1002 i915_vma_unpin(vma);
1003err:
1004 i915_gem_object_put(obj);
1005 return ERR_PTR(err);
1006}
1007
1008static struct i915_request *
1009empty_request(struct intel_engine_cs *engine,
1010 struct i915_vma *batch)
1011{
1012 struct i915_request *request;
1013 int err;
1014
1015 request = i915_request_create(engine->kernel_context);
1016 if (IS_ERR(request))
1017 return request;
1018
1019 err = engine->emit_bb_start(request,
1020 batch->node.start,
1021 batch->node.size,
1022 I915_DISPATCH_SECURE);
1023 if (err)
1024 goto out_request;
1025
1026 i915_request_get(request);
1027out_request:
1028 i915_request_add(request);
1029 return err ? ERR_PTR(err) : request;
1030}
1031
1032static int live_empty_request(void *arg)
1033{
1034 struct drm_i915_private *i915 = arg;
1035 struct intel_engine_cs *engine;
1036 struct igt_live_test t;
1037 struct i915_vma *batch;
1038 int err = 0;
1039
1040 /*
1041 * Submit various sized batches of empty requests, to each engine
1042 * (individually), and wait for the batch to complete. We can check
1043 * the overhead of submitting requests to the hardware.
1044 */
1045
1046 batch = empty_batch(i915);
1047 if (IS_ERR(batch))
1048 return PTR_ERR(batch);
1049
1050 for_each_uabi_engine(engine, i915) {
1051 IGT_TIMEOUT(end_time);
1052 struct i915_request *request;
1053 unsigned long n, prime;
1054 ktime_t times[2] = {};
1055
1056 err = igt_live_test_begin(&t, i915, __func__, engine->name);
1057 if (err)
1058 goto out_batch;
1059
1060 intel_engine_pm_get(engine);
1061
1062 /* Warmup / preload */
1063 request = empty_request(engine, batch);
1064 if (IS_ERR(request)) {
1065 err = PTR_ERR(request);
1066 intel_engine_pm_put(engine);
1067 goto out_batch;
1068 }
1069 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
1070
1071 for_each_prime_number_from(prime, 1, 8192) {
1072 times[1] = ktime_get_raw();
1073
1074 for (n = 0; n < prime; n++) {
1075 i915_request_put(request);
1076 request = empty_request(engine, batch);
1077 if (IS_ERR(request)) {
1078 err = PTR_ERR(request);
1079 intel_engine_pm_put(engine);
1080 goto out_batch;
1081 }
1082 }
1083 i915_request_wait(request, 0, MAX_SCHEDULE_TIMEOUT);
1084
1085 times[1] = ktime_sub(ktime_get_raw(), times[1]);
1086 if (prime == 1)
1087 times[0] = times[1];
1088
1089 if (__igt_timeout(end_time, NULL))
1090 break;
1091 }
1092 i915_request_put(request);
1093 intel_engine_pm_put(engine);
1094
1095 err = igt_live_test_end(&t);
1096 if (err)
1097 goto out_batch;
1098
1099 pr_info("Batch latencies on %s: 1 = %lluns, %lu = %lluns\n",
1100 engine->name,
1101 ktime_to_ns(times[0]),
1102 prime, div64_u64(ktime_to_ns(times[1]), prime));
1103 }
1104
1105out_batch:
1106 i915_vma_unpin(batch);
1107 i915_vma_put(batch);
1108 return err;
1109}
1110
1111static struct i915_vma *recursive_batch(struct drm_i915_private *i915)
1112{
1113 struct drm_i915_gem_object *obj;
1114 const int ver = GRAPHICS_VER(i915);
1115 struct i915_vma *vma;
1116 u32 *cmd;
1117 int err;
1118
1119 obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
1120 if (IS_ERR(obj))
1121 return ERR_CAST(obj);
1122
1123 vma = i915_vma_instance(obj, to_gt(i915)->vm, NULL);
1124 if (IS_ERR(vma)) {
1125 err = PTR_ERR(vma);
1126 goto err;
1127 }
1128
1129 err = i915_vma_pin(vma, 0, 0, PIN_USER);
1130 if (err)
1131 goto err;
1132
1133 cmd = i915_gem_object_pin_map_unlocked(obj, I915_MAP_WC);
1134 if (IS_ERR(cmd)) {
1135 err = PTR_ERR(cmd);
1136 goto err;
1137 }
1138
1139 if (ver >= 8) {
1140 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8 | 1;
1141 *cmd++ = lower_32_bits(vma->node.start);
1142 *cmd++ = upper_32_bits(vma->node.start);
1143 } else if (ver >= 6) {
1144 *cmd++ = MI_BATCH_BUFFER_START | 1 << 8;
1145 *cmd++ = lower_32_bits(vma->node.start);
1146 } else {
1147 *cmd++ = MI_BATCH_BUFFER_START | MI_BATCH_GTT;
1148 *cmd++ = lower_32_bits(vma->node.start);
1149 }
1150 *cmd++ = MI_BATCH_BUFFER_END; /* terminate early in case of error */
1151
1152 __i915_gem_object_flush_map(obj, 0, 64);
1153 i915_gem_object_unpin_map(obj);
1154
1155 intel_gt_chipset_flush(to_gt(i915));
1156
1157 return vma;
1158
1159err:
1160 i915_gem_object_put(obj);
1161 return ERR_PTR(err);
1162}
1163
1164static int recursive_batch_resolve(struct i915_vma *batch)
1165{
1166 u32 *cmd;
1167
1168 cmd = i915_gem_object_pin_map_unlocked(batch->obj, I915_MAP_WC);
1169 if (IS_ERR(cmd))
1170 return PTR_ERR(cmd);
1171
1172 *cmd = MI_BATCH_BUFFER_END;
1173
1174 __i915_gem_object_flush_map(batch->obj, 0, sizeof(*cmd));
1175 i915_gem_object_unpin_map(batch->obj);
1176
1177 intel_gt_chipset_flush(batch->vm->gt);
1178
1179 return 0;
1180}
1181
1182static int live_all_engines(void *arg)
1183{
1184 struct drm_i915_private *i915 = arg;
1185 const unsigned int nengines = num_uabi_engines(i915);
1186 struct intel_engine_cs *engine;
1187 struct i915_request **request;
1188 struct igt_live_test t;
1189 struct i915_vma *batch;
1190 unsigned int idx;
1191 int err;
1192
1193 /*
1194 * Check we can submit requests to all engines simultaneously. We
1195 * send a recursive batch to each engine - checking that we don't
1196 * block doing so, and that they don't complete too soon.
1197 */
1198
1199 request = kcalloc(nengines, sizeof(*request), GFP_KERNEL);
1200 if (!request)
1201 return -ENOMEM;
1202
1203 err = igt_live_test_begin(&t, i915, __func__, "");
1204 if (err)
1205 goto out_free;
1206
1207 batch = recursive_batch(i915);
1208 if (IS_ERR(batch)) {
1209 err = PTR_ERR(batch);
1210 pr_err("%s: Unable to create batch, err=%d\n", __func__, err);
1211 goto out_free;
1212 }
1213
1214 i915_vma_lock(batch);
1215
1216 idx = 0;
1217 for_each_uabi_engine(engine, i915) {
1218 request[idx] = intel_engine_create_kernel_request(engine);
1219 if (IS_ERR(request[idx])) {
1220 err = PTR_ERR(request[idx]);
1221 pr_err("%s: Request allocation failed with err=%d\n",
1222 __func__, err);
1223 goto out_request;
1224 }
1225
1226 err = i915_vma_move_to_active(batch, request[idx], 0);
1227 GEM_BUG_ON(err);
1228
1229 err = engine->emit_bb_start(request[idx],
1230 batch->node.start,
1231 batch->node.size,
1232 0);
1233 GEM_BUG_ON(err);
1234 request[idx]->batch = batch;
1235
1236 i915_request_get(request[idx]);
1237 i915_request_add(request[idx]);
1238 idx++;
1239 }
1240
1241 i915_vma_unlock(batch);
1242
1243 idx = 0;
1244 for_each_uabi_engine(engine, i915) {
1245 if (i915_request_completed(request[idx])) {
1246 pr_err("%s(%s): request completed too early!\n",
1247 __func__, engine->name);
1248 err = -EINVAL;
1249 goto out_request;
1250 }
1251 idx++;
1252 }
1253
1254 err = recursive_batch_resolve(batch);
1255 if (err) {
1256 pr_err("%s: failed to resolve batch, err=%d\n", __func__, err);
1257 goto out_request;
1258 }
1259
1260 idx = 0;
1261 for_each_uabi_engine(engine, i915) {
1262 long timeout;
1263
1264 timeout = i915_request_wait(request[idx], 0,
1265 MAX_SCHEDULE_TIMEOUT);
1266 if (timeout < 0) {
1267 err = timeout;
1268 pr_err("%s: error waiting for request on %s, err=%d\n",
1269 __func__, engine->name, err);
1270 goto out_request;
1271 }
1272
1273 GEM_BUG_ON(!i915_request_completed(request[idx]));
1274 i915_request_put(request[idx]);
1275 request[idx] = NULL;
1276 idx++;
1277 }
1278
1279 err = igt_live_test_end(&t);
1280
1281out_request:
1282 idx = 0;
1283 for_each_uabi_engine(engine, i915) {
1284 if (request[idx])
1285 i915_request_put(request[idx]);
1286 idx++;
1287 }
1288 i915_vma_unpin(batch);
1289 i915_vma_put(batch);
1290out_free:
1291 kfree(request);
1292 return err;
1293}
1294
1295static int live_sequential_engines(void *arg)
1296{
1297 struct drm_i915_private *i915 = arg;
1298 const unsigned int nengines = num_uabi_engines(i915);
1299 struct i915_request **request;
1300 struct i915_request *prev = NULL;
1301 struct intel_engine_cs *engine;
1302 struct igt_live_test t;
1303 unsigned int idx;
1304 int err;
1305
1306 /*
1307 * Check we can submit requests to all engines sequentially, such
1308 * that each successive request waits for the earlier ones. This
1309 * tests that we don't execute requests out of order, even though
1310 * they are running on independent engines.
1311 */
1312
1313 request = kcalloc(nengines, sizeof(*request), GFP_KERNEL);
1314 if (!request)
1315 return -ENOMEM;
1316
1317 err = igt_live_test_begin(&t, i915, __func__, "");
1318 if (err)
1319 goto out_free;
1320
1321 idx = 0;
1322 for_each_uabi_engine(engine, i915) {
1323 struct i915_vma *batch;
1324
1325 batch = recursive_batch(i915);
1326 if (IS_ERR(batch)) {
1327 err = PTR_ERR(batch);
1328 pr_err("%s: Unable to create batch for %s, err=%d\n",
1329 __func__, engine->name, err);
1330 goto out_free;
1331 }
1332
1333 i915_vma_lock(batch);
1334 request[idx] = intel_engine_create_kernel_request(engine);
1335 if (IS_ERR(request[idx])) {
1336 err = PTR_ERR(request[idx]);
1337 pr_err("%s: Request allocation failed for %s with err=%d\n",
1338 __func__, engine->name, err);
1339 goto out_unlock;
1340 }
1341
1342 if (prev) {
1343 err = i915_request_await_dma_fence(request[idx],
1344 &prev->fence);
1345 if (err) {
1346 i915_request_add(request[idx]);
1347 pr_err("%s: Request await failed for %s with err=%d\n",
1348 __func__, engine->name, err);
1349 goto out_unlock;
1350 }
1351 }
1352
1353 err = i915_vma_move_to_active(batch, request[idx], 0);
1354 GEM_BUG_ON(err);
1355
1356 err = engine->emit_bb_start(request[idx],
1357 batch->node.start,
1358 batch->node.size,
1359 0);
1360 GEM_BUG_ON(err);
1361 request[idx]->batch = batch;
1362
1363 i915_request_get(request[idx]);
1364 i915_request_add(request[idx]);
1365
1366 prev = request[idx];
1367 idx++;
1368
1369out_unlock:
1370 i915_vma_unlock(batch);
1371 if (err)
1372 goto out_request;
1373 }
1374
1375 idx = 0;
1376 for_each_uabi_engine(engine, i915) {
1377 long timeout;
1378
1379 if (i915_request_completed(request[idx])) {
1380 pr_err("%s(%s): request completed too early!\n",
1381 __func__, engine->name);
1382 err = -EINVAL;
1383 goto out_request;
1384 }
1385
1386 err = recursive_batch_resolve(request[idx]->batch);
1387 if (err) {
1388 pr_err("%s: failed to resolve batch, err=%d\n",
1389 __func__, err);
1390 goto out_request;
1391 }
1392
1393 timeout = i915_request_wait(request[idx], 0,
1394 MAX_SCHEDULE_TIMEOUT);
1395 if (timeout < 0) {
1396 err = timeout;
1397 pr_err("%s: error waiting for request on %s, err=%d\n",
1398 __func__, engine->name, err);
1399 goto out_request;
1400 }
1401
1402 GEM_BUG_ON(!i915_request_completed(request[idx]));
1403 idx++;
1404 }
1405
1406 err = igt_live_test_end(&t);
1407
1408out_request:
1409 idx = 0;
1410 for_each_uabi_engine(engine, i915) {
1411 u32 *cmd;
1412
1413 if (!request[idx])
1414 break;
1415
1416 cmd = i915_gem_object_pin_map_unlocked(request[idx]->batch->obj,
1417 I915_MAP_WC);
1418 if (!IS_ERR(cmd)) {
1419 *cmd = MI_BATCH_BUFFER_END;
1420
1421 __i915_gem_object_flush_map(request[idx]->batch->obj,
1422 0, sizeof(*cmd));
1423 i915_gem_object_unpin_map(request[idx]->batch->obj);
1424
1425 intel_gt_chipset_flush(engine->gt);
1426 }
1427
1428 i915_vma_put(request[idx]->batch);
1429 i915_request_put(request[idx]);
1430 idx++;
1431 }
1432out_free:
1433 kfree(request);
1434 return err;
1435}
1436
1437struct parallel_thread {
1438 struct kthread_worker *worker;
1439 struct kthread_work work;
1440 struct intel_engine_cs *engine;
1441 int result;
1442};
1443
1444static void __live_parallel_engine1(struct kthread_work *work)
1445{
1446 struct parallel_thread *thread =
1447 container_of(work, typeof(*thread), work);
1448 struct intel_engine_cs *engine = thread->engine;
1449 IGT_TIMEOUT(end_time);
1450 unsigned long count;
1451 int err = 0;
1452
1453 count = 0;
1454 intel_engine_pm_get(engine);
1455 do {
1456 struct i915_request *rq;
1457
1458 rq = i915_request_create(engine->kernel_context);
1459 if (IS_ERR(rq)) {
1460 err = PTR_ERR(rq);
1461 break;
1462 }
1463
1464 i915_request_get(rq);
1465 i915_request_add(rq);
1466
1467 err = 0;
1468 if (i915_request_wait(rq, 0, HZ) < 0)
1469 err = -ETIME;
1470 i915_request_put(rq);
1471 if (err)
1472 break;
1473
1474 count++;
1475 } while (!__igt_timeout(end_time, NULL));
1476 intel_engine_pm_put(engine);
1477
1478 pr_info("%s: %lu request + sync\n", engine->name, count);
1479 thread->result = err;
1480}
1481
1482static void __live_parallel_engineN(struct kthread_work *work)
1483{
1484 struct parallel_thread *thread =
1485 container_of(work, typeof(*thread), work);
1486 struct intel_engine_cs *engine = thread->engine;
1487 IGT_TIMEOUT(end_time);
1488 unsigned long count;
1489 int err = 0;
1490
1491 count = 0;
1492 intel_engine_pm_get(engine);
1493 do {
1494 struct i915_request *rq;
1495
1496 rq = i915_request_create(engine->kernel_context);
1497 if (IS_ERR(rq)) {
1498 err = PTR_ERR(rq);
1499 break;
1500 }
1501
1502 i915_request_add(rq);
1503 count++;
1504 } while (!__igt_timeout(end_time, NULL));
1505 intel_engine_pm_put(engine);
1506
1507 pr_info("%s: %lu requests\n", engine->name, count);
1508 thread->result = err;
1509}
1510
1511static bool wake_all(struct drm_i915_private *i915)
1512{
1513 if (atomic_dec_and_test(&i915->selftest.counter)) {
1514 wake_up_var(&i915->selftest.counter);
1515 return true;
1516 }
1517
1518 return false;
1519}
1520
1521static int wait_for_all(struct drm_i915_private *i915)
1522{
1523 if (wake_all(i915))
1524 return 0;
1525
1526 if (wait_var_event_timeout(&i915->selftest.counter,
1527 !atomic_read(&i915->selftest.counter),
1528 i915_selftest.timeout_jiffies))
1529 return 0;
1530
1531 return -ETIME;
1532}
1533
1534static void __live_parallel_spin(struct kthread_work *work)
1535{
1536 struct parallel_thread *thread =
1537 container_of(work, typeof(*thread), work);
1538 struct intel_engine_cs *engine = thread->engine;
1539 struct igt_spinner spin;
1540 struct i915_request *rq;
1541 int err = 0;
1542
1543 /*
1544 * Create a spinner running for eternity on each engine. If a second
1545 * spinner is incorrectly placed on the same engine, it will not be
1546 * able to start in time.
1547 */
1548
1549 if (igt_spinner_init(&spin, engine->gt)) {
1550 wake_all(engine->i915);
1551 thread->result = -ENOMEM;
1552 return;
1553 }
1554
1555 intel_engine_pm_get(engine);
1556 rq = igt_spinner_create_request(&spin,
1557 engine->kernel_context,
1558 MI_NOOP); /* no preemption */
1559 intel_engine_pm_put(engine);
1560 if (IS_ERR(rq)) {
1561 err = PTR_ERR(rq);
1562 if (err == -ENODEV)
1563 err = 0;
1564 wake_all(engine->i915);
1565 goto out_spin;
1566 }
1567
1568 i915_request_get(rq);
1569 i915_request_add(rq);
1570 if (igt_wait_for_spinner(&spin, rq)) {
1571 /* Occupy this engine for the whole test */
1572 err = wait_for_all(engine->i915);
1573 } else {
1574 pr_err("Failed to start spinner on %s\n", engine->name);
1575 err = -EINVAL;
1576 }
1577 igt_spinner_end(&spin);
1578
1579 if (err == 0 && i915_request_wait(rq, 0, HZ) < 0)
1580 err = -EIO;
1581 i915_request_put(rq);
1582
1583out_spin:
1584 igt_spinner_fini(&spin);
1585 thread->result = err;
1586}
1587
1588static int live_parallel_engines(void *arg)
1589{
1590 struct drm_i915_private *i915 = arg;
1591 static void (* const func[])(struct kthread_work *) = {
1592 __live_parallel_engine1,
1593 __live_parallel_engineN,
1594 __live_parallel_spin,
1595 NULL,
1596 };
1597 const unsigned int nengines = num_uabi_engines(i915);
1598 struct parallel_thread *threads;
1599 struct intel_engine_cs *engine;
1600 void (* const *fn)(struct kthread_work *);
1601 int err = 0;
1602
1603 /*
1604 * Check we can submit requests to all engines concurrently. This
1605 * tests that we load up the system maximally.
1606 */
1607
1608 threads = kcalloc(nengines, sizeof(*threads), GFP_KERNEL);
1609 if (!threads)
1610 return -ENOMEM;
1611
1612 for (fn = func; !err && *fn; fn++) {
1613 char name[KSYM_NAME_LEN];
1614 struct igt_live_test t;
1615 unsigned int idx;
1616
1617 snprintf(name, sizeof(name), "%ps", *fn);
1618 err = igt_live_test_begin(&t, i915, __func__, name);
1619 if (err)
1620 break;
1621
1622 atomic_set(&i915->selftest.counter, nengines);
1623
1624 idx = 0;
1625 for_each_uabi_engine(engine, i915) {
1626 struct kthread_worker *worker;
1627
1628 worker = kthread_create_worker(0, "igt/parallel:%s",
1629 engine->name);
1630 if (IS_ERR(worker)) {
1631 err = PTR_ERR(worker);
1632 break;
1633 }
1634
1635 threads[idx].worker = worker;
1636 threads[idx].result = 0;
1637 threads[idx].engine = engine;
1638
1639 kthread_init_work(&threads[idx].work, *fn);
1640 kthread_queue_work(worker, &threads[idx].work);
1641 idx++;
1642 }
1643
1644 idx = 0;
1645 for_each_uabi_engine(engine, i915) {
1646 int status;
1647
1648 if (!threads[idx].worker)
1649 break;
1650
1651 kthread_flush_work(&threads[idx].work);
1652 status = READ_ONCE(threads[idx].result);
1653 if (status && !err)
1654 err = status;
1655
1656 kthread_destroy_worker(threads[idx++].worker);
1657 }
1658
1659 if (igt_live_test_end(&t))
1660 err = -EIO;
1661 }
1662
1663 kfree(threads);
1664 return err;
1665}
1666
1667static int
1668max_batches(struct i915_gem_context *ctx, struct intel_engine_cs *engine)
1669{
1670 struct i915_request *rq;
1671 int ret;
1672
1673 /*
1674 * Before execlists, all contexts share the same ringbuffer. With
1675 * execlists, each context/engine has a separate ringbuffer and
1676 * for the purposes of this test, inexhaustible.
1677 *
1678 * For the global ringbuffer though, we have to be very careful
1679 * that we do not wrap while preventing the execution of requests
1680 * with a unsignaled fence.
1681 */
1682 if (HAS_EXECLISTS(ctx->i915))
1683 return INT_MAX;
1684
1685 rq = igt_request_alloc(ctx, engine);
1686 if (IS_ERR(rq)) {
1687 ret = PTR_ERR(rq);
1688 } else {
1689 int sz;
1690
1691 ret = rq->ring->size - rq->reserved_space;
1692 i915_request_add(rq);
1693
1694 sz = rq->ring->emit - rq->head;
1695 if (sz < 0)
1696 sz += rq->ring->size;
1697 ret /= sz;
1698 ret /= 2; /* leave half spare, in case of emergency! */
1699 }
1700
1701 return ret;
1702}
1703
1704static int live_breadcrumbs_smoketest(void *arg)
1705{
1706 struct drm_i915_private *i915 = arg;
1707 const unsigned int nengines = num_uabi_engines(i915);
1708 const unsigned int ncpus = /* saturate with nengines * ncpus */
1709 max_t(int, 2, DIV_ROUND_UP(num_online_cpus(), nengines));
1710 unsigned long num_waits, num_fences;
1711 struct intel_engine_cs *engine;
1712 struct smoke_thread *threads;
1713 struct igt_live_test live;
1714 intel_wakeref_t wakeref;
1715 struct smoketest *smoke;
1716 unsigned int n, idx;
1717 struct file *file;
1718 int ret = 0;
1719
1720 /*
1721 * Smoketest our breadcrumb/signal handling for requests across multiple
1722 * threads. A very simple test to only catch the most egregious of bugs.
1723 * See __igt_breadcrumbs_smoketest();
1724 *
1725 * On real hardware this time.
1726 */
1727
1728 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1729
1730 file = mock_file(i915);
1731 if (IS_ERR(file)) {
1732 ret = PTR_ERR(file);
1733 goto out_rpm;
1734 }
1735
1736 smoke = kcalloc(nengines, sizeof(*smoke), GFP_KERNEL);
1737 if (!smoke) {
1738 ret = -ENOMEM;
1739 goto out_file;
1740 }
1741
1742 threads = kcalloc(ncpus * nengines, sizeof(*threads), GFP_KERNEL);
1743 if (!threads) {
1744 ret = -ENOMEM;
1745 goto out_smoke;
1746 }
1747
1748 smoke[0].request_alloc = __live_request_alloc;
1749 smoke[0].ncontexts = 64;
1750 smoke[0].contexts = kcalloc(smoke[0].ncontexts,
1751 sizeof(*smoke[0].contexts),
1752 GFP_KERNEL);
1753 if (!smoke[0].contexts) {
1754 ret = -ENOMEM;
1755 goto out_threads;
1756 }
1757
1758 for (n = 0; n < smoke[0].ncontexts; n++) {
1759 smoke[0].contexts[n] = live_context(i915, file);
1760 if (IS_ERR(smoke[0].contexts[n])) {
1761 ret = PTR_ERR(smoke[0].contexts[n]);
1762 goto out_contexts;
1763 }
1764 }
1765
1766 ret = igt_live_test_begin(&live, i915, __func__, "");
1767 if (ret)
1768 goto out_contexts;
1769
1770 idx = 0;
1771 for_each_uabi_engine(engine, i915) {
1772 smoke[idx] = smoke[0];
1773 smoke[idx].engine = engine;
1774 smoke[idx].max_batch =
1775 max_batches(smoke[0].contexts[0], engine);
1776 if (smoke[idx].max_batch < 0) {
1777 ret = smoke[idx].max_batch;
1778 goto out_flush;
1779 }
1780 /* One ring interleaved between requests from all cpus */
1781 smoke[idx].max_batch /= ncpus + 1;
1782 pr_debug("Limiting batches to %d requests on %s\n",
1783 smoke[idx].max_batch, engine->name);
1784
1785 for (n = 0; n < ncpus; n++) {
1786 unsigned int i = idx * ncpus + n;
1787 struct kthread_worker *worker;
1788
1789 worker = kthread_create_worker(0, "igt/%d.%d", idx, n);
1790 if (IS_ERR(worker)) {
1791 ret = PTR_ERR(worker);
1792 goto out_flush;
1793 }
1794
1795 threads[i].worker = worker;
1796 threads[i].t = &smoke[idx];
1797
1798 kthread_init_work(&threads[i].work,
1799 __igt_breadcrumbs_smoketest);
1800 kthread_queue_work(worker, &threads[i].work);
1801 }
1802
1803 idx++;
1804 }
1805
1806 msleep(jiffies_to_msecs(i915_selftest.timeout_jiffies));
1807
1808out_flush:
1809 idx = 0;
1810 num_waits = 0;
1811 num_fences = 0;
1812 for_each_uabi_engine(engine, i915) {
1813 for (n = 0; n < ncpus; n++) {
1814 unsigned int i = idx * ncpus + n;
1815 int err;
1816
1817 if (!threads[i].worker)
1818 continue;
1819
1820 WRITE_ONCE(threads[i].stop, true);
1821 kthread_flush_work(&threads[i].work);
1822 err = READ_ONCE(threads[i].result);
1823 if (err < 0 && !ret)
1824 ret = err;
1825
1826 kthread_destroy_worker(threads[i].worker);
1827 }
1828
1829 num_waits += atomic_long_read(&smoke[idx].num_waits);
1830 num_fences += atomic_long_read(&smoke[idx].num_fences);
1831 idx++;
1832 }
1833 pr_info("Completed %lu waits for %lu fences across %d engines and %d cpus\n",
1834 num_waits, num_fences, idx, ncpus);
1835
1836 ret = igt_live_test_end(&live) ?: ret;
1837out_contexts:
1838 kfree(smoke[0].contexts);
1839out_threads:
1840 kfree(threads);
1841out_smoke:
1842 kfree(smoke);
1843out_file:
1844 fput(file);
1845out_rpm:
1846 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1847
1848 return ret;
1849}
1850
1851int i915_request_live_selftests(struct drm_i915_private *i915)
1852{
1853 static const struct i915_subtest tests[] = {
1854 SUBTEST(live_nop_request),
1855 SUBTEST(live_all_engines),
1856 SUBTEST(live_sequential_engines),
1857 SUBTEST(live_parallel_engines),
1858 SUBTEST(live_empty_request),
1859 SUBTEST(live_cancel_request),
1860 SUBTEST(live_breadcrumbs_smoketest),
1861 };
1862
1863 if (intel_gt_is_wedged(to_gt(i915)))
1864 return 0;
1865
1866 return i915_live_subtests(tests, i915);
1867}
1868
1869static int switch_to_kernel_sync(struct intel_context *ce, int err)
1870{
1871 struct i915_request *rq;
1872 struct dma_fence *fence;
1873
1874 rq = intel_engine_create_kernel_request(ce->engine);
1875 if (IS_ERR(rq))
1876 return PTR_ERR(rq);
1877
1878 fence = i915_active_fence_get(&ce->timeline->last_request);
1879 if (fence) {
1880 i915_request_await_dma_fence(rq, fence);
1881 dma_fence_put(fence);
1882 }
1883
1884 rq = i915_request_get(rq);
1885 i915_request_add(rq);
1886 if (i915_request_wait(rq, 0, HZ / 2) < 0 && !err)
1887 err = -ETIME;
1888 i915_request_put(rq);
1889
1890 while (!err && !intel_engine_is_idle(ce->engine))
1891 intel_engine_flush_submission(ce->engine);
1892
1893 return err;
1894}
1895
1896struct perf_stats {
1897 struct intel_engine_cs *engine;
1898 unsigned long count;
1899 ktime_t time;
1900 ktime_t busy;
1901 u64 runtime;
1902};
1903
1904struct perf_series {
1905 struct drm_i915_private *i915;
1906 unsigned int nengines;
1907 struct intel_context *ce[];
1908};
1909
1910static int cmp_u32(const void *A, const void *B)
1911{
1912 const u32 *a = A, *b = B;
1913
1914 return *a - *b;
1915}
1916
1917static u32 trifilter(u32 *a)
1918{
1919 u64 sum;
1920
1921#define TF_COUNT 5
1922 sort(a, TF_COUNT, sizeof(*a), cmp_u32, NULL);
1923
1924 sum = mul_u32_u32(a[2], 2);
1925 sum += a[1];
1926 sum += a[3];
1927
1928 GEM_BUG_ON(sum > U32_MAX);
1929 return sum;
1930#define TF_BIAS 2
1931}
1932
1933static u64 cycles_to_ns(struct intel_engine_cs *engine, u32 cycles)
1934{
1935 u64 ns = intel_gt_clock_interval_to_ns(engine->gt, cycles);
1936
1937 return DIV_ROUND_CLOSEST(ns, 1 << TF_BIAS);
1938}
1939
1940static u32 *emit_timestamp_store(u32 *cs, struct intel_context *ce, u32 offset)
1941{
1942 *cs++ = MI_STORE_REGISTER_MEM_GEN8 | MI_USE_GGTT;
1943 *cs++ = i915_mmio_reg_offset(RING_TIMESTAMP((ce->engine->mmio_base)));
1944 *cs++ = offset;
1945 *cs++ = 0;
1946
1947 return cs;
1948}
1949
1950static u32 *emit_store_dw(u32 *cs, u32 offset, u32 value)
1951{
1952 *cs++ = MI_STORE_DWORD_IMM_GEN4 | MI_USE_GGTT;
1953 *cs++ = offset;
1954 *cs++ = 0;
1955 *cs++ = value;
1956
1957 return cs;
1958}
1959
1960static u32 *emit_semaphore_poll(u32 *cs, u32 mode, u32 value, u32 offset)
1961{
1962 *cs++ = MI_SEMAPHORE_WAIT |
1963 MI_SEMAPHORE_GLOBAL_GTT |
1964 MI_SEMAPHORE_POLL |
1965 mode;
1966 *cs++ = value;
1967 *cs++ = offset;
1968 *cs++ = 0;
1969
1970 return cs;
1971}
1972
1973static u32 *emit_semaphore_poll_until(u32 *cs, u32 offset, u32 value)
1974{
1975 return emit_semaphore_poll(cs, MI_SEMAPHORE_SAD_EQ_SDD, value, offset);
1976}
1977
1978static void semaphore_set(u32 *sema, u32 value)
1979{
1980 WRITE_ONCE(*sema, value);
1981 wmb(); /* flush the update to the cache, and beyond */
1982}
1983
1984static u32 *hwsp_scratch(const struct intel_context *ce)
1985{
1986 return memset32(ce->engine->status_page.addr + 1000, 0, 21);
1987}
1988
1989static u32 hwsp_offset(const struct intel_context *ce, u32 *dw)
1990{
1991 return (i915_ggtt_offset(ce->engine->status_page.vma) +
1992 offset_in_page(dw));
1993}
1994
1995static int measure_semaphore_response(struct intel_context *ce)
1996{
1997 u32 *sema = hwsp_scratch(ce);
1998 const u32 offset = hwsp_offset(ce, sema);
1999 u32 elapsed[TF_COUNT], cycles;
2000 struct i915_request *rq;
2001 u32 *cs;
2002 int err;
2003 int i;
2004
2005 /*
2006 * Measure how many cycles it takes for the HW to detect the change
2007 * in a semaphore value.
2008 *
2009 * A: read CS_TIMESTAMP from CPU
2010 * poke semaphore
2011 * B: read CS_TIMESTAMP on GPU
2012 *
2013 * Semaphore latency: B - A
2014 */
2015
2016 semaphore_set(sema, -1);
2017
2018 rq = i915_request_create(ce);
2019 if (IS_ERR(rq))
2020 return PTR_ERR(rq);
2021
2022 cs = intel_ring_begin(rq, 4 + 12 * ARRAY_SIZE(elapsed));
2023 if (IS_ERR(cs)) {
2024 i915_request_add(rq);
2025 err = PTR_ERR(cs);
2026 goto err;
2027 }
2028
2029 cs = emit_store_dw(cs, offset, 0);
2030 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2031 cs = emit_semaphore_poll_until(cs, offset, i);
2032 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2033 cs = emit_store_dw(cs, offset, 0);
2034 }
2035
2036 intel_ring_advance(rq, cs);
2037 i915_request_add(rq);
2038
2039 if (wait_for(READ_ONCE(*sema) == 0, 50)) {
2040 err = -EIO;
2041 goto err;
2042 }
2043
2044 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2045 preempt_disable();
2046 cycles = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2047 semaphore_set(sema, i);
2048 preempt_enable();
2049
2050 if (wait_for(READ_ONCE(*sema) == 0, 50)) {
2051 err = -EIO;
2052 goto err;
2053 }
2054
2055 elapsed[i - 1] = sema[i] - cycles;
2056 }
2057
2058 cycles = trifilter(elapsed);
2059 pr_info("%s: semaphore response %d cycles, %lluns\n",
2060 ce->engine->name, cycles >> TF_BIAS,
2061 cycles_to_ns(ce->engine, cycles));
2062
2063 return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2064
2065err:
2066 intel_gt_set_wedged(ce->engine->gt);
2067 return err;
2068}
2069
2070static int measure_idle_dispatch(struct intel_context *ce)
2071{
2072 u32 *sema = hwsp_scratch(ce);
2073 const u32 offset = hwsp_offset(ce, sema);
2074 u32 elapsed[TF_COUNT], cycles;
2075 u32 *cs;
2076 int err;
2077 int i;
2078
2079 /*
2080 * Measure how long it takes for us to submit a request while the
2081 * engine is idle, but is resting in our context.
2082 *
2083 * A: read CS_TIMESTAMP from CPU
2084 * submit request
2085 * B: read CS_TIMESTAMP on GPU
2086 *
2087 * Submission latency: B - A
2088 */
2089
2090 for (i = 0; i < ARRAY_SIZE(elapsed); i++) {
2091 struct i915_request *rq;
2092
2093 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2094 if (err)
2095 return err;
2096
2097 rq = i915_request_create(ce);
2098 if (IS_ERR(rq)) {
2099 err = PTR_ERR(rq);
2100 goto err;
2101 }
2102
2103 cs = intel_ring_begin(rq, 4);
2104 if (IS_ERR(cs)) {
2105 i915_request_add(rq);
2106 err = PTR_ERR(cs);
2107 goto err;
2108 }
2109
2110 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2111
2112 intel_ring_advance(rq, cs);
2113
2114 preempt_disable();
2115 local_bh_disable();
2116 elapsed[i] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2117 i915_request_add(rq);
2118 local_bh_enable();
2119 preempt_enable();
2120 }
2121
2122 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2123 if (err)
2124 goto err;
2125
2126 for (i = 0; i < ARRAY_SIZE(elapsed); i++)
2127 elapsed[i] = sema[i] - elapsed[i];
2128
2129 cycles = trifilter(elapsed);
2130 pr_info("%s: idle dispatch latency %d cycles, %lluns\n",
2131 ce->engine->name, cycles >> TF_BIAS,
2132 cycles_to_ns(ce->engine, cycles));
2133
2134 return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2135
2136err:
2137 intel_gt_set_wedged(ce->engine->gt);
2138 return err;
2139}
2140
2141static int measure_busy_dispatch(struct intel_context *ce)
2142{
2143 u32 *sema = hwsp_scratch(ce);
2144 const u32 offset = hwsp_offset(ce, sema);
2145 u32 elapsed[TF_COUNT + 1], cycles;
2146 u32 *cs;
2147 int err;
2148 int i;
2149
2150 /*
2151 * Measure how long it takes for us to submit a request while the
2152 * engine is busy, polling on a semaphore in our context. With
2153 * direct submission, this will include the cost of a lite restore.
2154 *
2155 * A: read CS_TIMESTAMP from CPU
2156 * submit request
2157 * B: read CS_TIMESTAMP on GPU
2158 *
2159 * Submission latency: B - A
2160 */
2161
2162 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2163 struct i915_request *rq;
2164
2165 rq = i915_request_create(ce);
2166 if (IS_ERR(rq)) {
2167 err = PTR_ERR(rq);
2168 goto err;
2169 }
2170
2171 cs = intel_ring_begin(rq, 12);
2172 if (IS_ERR(cs)) {
2173 i915_request_add(rq);
2174 err = PTR_ERR(cs);
2175 goto err;
2176 }
2177
2178 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1);
2179 cs = emit_semaphore_poll_until(cs, offset, i);
2180 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2181
2182 intel_ring_advance(rq, cs);
2183
2184 if (i > 1 && wait_for(READ_ONCE(sema[i - 1]), 500)) {
2185 err = -EIO;
2186 goto err;
2187 }
2188
2189 preempt_disable();
2190 local_bh_disable();
2191 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2192 i915_request_add(rq);
2193 local_bh_enable();
2194 semaphore_set(sema, i - 1);
2195 preempt_enable();
2196 }
2197
2198 wait_for(READ_ONCE(sema[i - 1]), 500);
2199 semaphore_set(sema, i - 1);
2200
2201 for (i = 1; i <= TF_COUNT; i++) {
2202 GEM_BUG_ON(sema[i] == -1);
2203 elapsed[i - 1] = sema[i] - elapsed[i];
2204 }
2205
2206 cycles = trifilter(elapsed);
2207 pr_info("%s: busy dispatch latency %d cycles, %lluns\n",
2208 ce->engine->name, cycles >> TF_BIAS,
2209 cycles_to_ns(ce->engine, cycles));
2210
2211 return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2212
2213err:
2214 intel_gt_set_wedged(ce->engine->gt);
2215 return err;
2216}
2217
2218static int plug(struct intel_engine_cs *engine, u32 *sema, u32 mode, int value)
2219{
2220 const u32 offset =
2221 i915_ggtt_offset(engine->status_page.vma) +
2222 offset_in_page(sema);
2223 struct i915_request *rq;
2224 u32 *cs;
2225
2226 rq = i915_request_create(engine->kernel_context);
2227 if (IS_ERR(rq))
2228 return PTR_ERR(rq);
2229
2230 cs = intel_ring_begin(rq, 4);
2231 if (IS_ERR(cs)) {
2232 i915_request_add(rq);
2233 return PTR_ERR(cs);
2234 }
2235
2236 cs = emit_semaphore_poll(cs, mode, value, offset);
2237
2238 intel_ring_advance(rq, cs);
2239 i915_request_add(rq);
2240
2241 return 0;
2242}
2243
2244static int measure_inter_request(struct intel_context *ce)
2245{
2246 u32 *sema = hwsp_scratch(ce);
2247 const u32 offset = hwsp_offset(ce, sema);
2248 u32 elapsed[TF_COUNT + 1], cycles;
2249 struct i915_sw_fence *submit;
2250 int i, err;
2251
2252 /*
2253 * Measure how long it takes to advance from one request into the
2254 * next. Between each request we flush the GPU caches to memory,
2255 * update the breadcrumbs, and then invalidate those caches.
2256 * We queue up all the requests to be submitted in one batch so
2257 * it should be one set of contiguous measurements.
2258 *
2259 * A: read CS_TIMESTAMP on GPU
2260 * advance request
2261 * B: read CS_TIMESTAMP on GPU
2262 *
2263 * Request latency: B - A
2264 */
2265
2266 err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0);
2267 if (err)
2268 return err;
2269
2270 submit = heap_fence_create(GFP_KERNEL);
2271 if (!submit) {
2272 semaphore_set(sema, 1);
2273 return -ENOMEM;
2274 }
2275
2276 intel_engine_flush_submission(ce->engine);
2277 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2278 struct i915_request *rq;
2279 u32 *cs;
2280
2281 rq = i915_request_create(ce);
2282 if (IS_ERR(rq)) {
2283 err = PTR_ERR(rq);
2284 goto err_submit;
2285 }
2286
2287 err = i915_sw_fence_await_sw_fence_gfp(&rq->submit,
2288 submit,
2289 GFP_KERNEL);
2290 if (err < 0) {
2291 i915_request_add(rq);
2292 goto err_submit;
2293 }
2294
2295 cs = intel_ring_begin(rq, 4);
2296 if (IS_ERR(cs)) {
2297 i915_request_add(rq);
2298 err = PTR_ERR(cs);
2299 goto err_submit;
2300 }
2301
2302 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2303
2304 intel_ring_advance(rq, cs);
2305 i915_request_add(rq);
2306 }
2307 i915_sw_fence_commit(submit);
2308 intel_engine_flush_submission(ce->engine);
2309 heap_fence_put(submit);
2310
2311 semaphore_set(sema, 1);
2312 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2313 if (err)
2314 goto err;
2315
2316 for (i = 1; i <= TF_COUNT; i++)
2317 elapsed[i - 1] = sema[i + 1] - sema[i];
2318
2319 cycles = trifilter(elapsed);
2320 pr_info("%s: inter-request latency %d cycles, %lluns\n",
2321 ce->engine->name, cycles >> TF_BIAS,
2322 cycles_to_ns(ce->engine, cycles));
2323
2324 return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2325
2326err_submit:
2327 i915_sw_fence_commit(submit);
2328 heap_fence_put(submit);
2329 semaphore_set(sema, 1);
2330err:
2331 intel_gt_set_wedged(ce->engine->gt);
2332 return err;
2333}
2334
2335static int measure_context_switch(struct intel_context *ce)
2336{
2337 u32 *sema = hwsp_scratch(ce);
2338 const u32 offset = hwsp_offset(ce, sema);
2339 struct i915_request *fence = NULL;
2340 u32 elapsed[TF_COUNT + 1], cycles;
2341 int i, j, err;
2342 u32 *cs;
2343
2344 /*
2345 * Measure how long it takes to advance from one request in one
2346 * context to a request in another context. This allows us to
2347 * measure how long the context save/restore take, along with all
2348 * the inter-context setup we require.
2349 *
2350 * A: read CS_TIMESTAMP on GPU
2351 * switch context
2352 * B: read CS_TIMESTAMP on GPU
2353 *
2354 * Context switch latency: B - A
2355 */
2356
2357 err = plug(ce->engine, sema, MI_SEMAPHORE_SAD_NEQ_SDD, 0);
2358 if (err)
2359 return err;
2360
2361 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2362 struct intel_context *arr[] = {
2363 ce, ce->engine->kernel_context
2364 };
2365 u32 addr = offset + ARRAY_SIZE(arr) * i * sizeof(u32);
2366
2367 for (j = 0; j < ARRAY_SIZE(arr); j++) {
2368 struct i915_request *rq;
2369
2370 rq = i915_request_create(arr[j]);
2371 if (IS_ERR(rq)) {
2372 err = PTR_ERR(rq);
2373 goto err_fence;
2374 }
2375
2376 if (fence) {
2377 err = i915_request_await_dma_fence(rq,
2378 &fence->fence);
2379 if (err) {
2380 i915_request_add(rq);
2381 goto err_fence;
2382 }
2383 }
2384
2385 cs = intel_ring_begin(rq, 4);
2386 if (IS_ERR(cs)) {
2387 i915_request_add(rq);
2388 err = PTR_ERR(cs);
2389 goto err_fence;
2390 }
2391
2392 cs = emit_timestamp_store(cs, ce, addr);
2393 addr += sizeof(u32);
2394
2395 intel_ring_advance(rq, cs);
2396
2397 i915_request_put(fence);
2398 fence = i915_request_get(rq);
2399
2400 i915_request_add(rq);
2401 }
2402 }
2403 i915_request_put(fence);
2404 intel_engine_flush_submission(ce->engine);
2405
2406 semaphore_set(sema, 1);
2407 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2408 if (err)
2409 goto err;
2410
2411 for (i = 1; i <= TF_COUNT; i++)
2412 elapsed[i - 1] = sema[2 * i + 2] - sema[2 * i + 1];
2413
2414 cycles = trifilter(elapsed);
2415 pr_info("%s: context switch latency %d cycles, %lluns\n",
2416 ce->engine->name, cycles >> TF_BIAS,
2417 cycles_to_ns(ce->engine, cycles));
2418
2419 return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2420
2421err_fence:
2422 i915_request_put(fence);
2423 semaphore_set(sema, 1);
2424err:
2425 intel_gt_set_wedged(ce->engine->gt);
2426 return err;
2427}
2428
2429static int measure_preemption(struct intel_context *ce)
2430{
2431 u32 *sema = hwsp_scratch(ce);
2432 const u32 offset = hwsp_offset(ce, sema);
2433 u32 elapsed[TF_COUNT], cycles;
2434 u32 *cs;
2435 int err;
2436 int i;
2437
2438 /*
2439 * We measure two latencies while triggering preemption. The first
2440 * latency is how long it takes for us to submit a preempting request.
2441 * The second latency is how it takes for us to return from the
2442 * preemption back to the original context.
2443 *
2444 * A: read CS_TIMESTAMP from CPU
2445 * submit preemption
2446 * B: read CS_TIMESTAMP on GPU (in preempting context)
2447 * context switch
2448 * C: read CS_TIMESTAMP on GPU (in original context)
2449 *
2450 * Preemption dispatch latency: B - A
2451 * Preemption switch latency: C - B
2452 */
2453
2454 if (!intel_engine_has_preemption(ce->engine))
2455 return 0;
2456
2457 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2458 u32 addr = offset + 2 * i * sizeof(u32);
2459 struct i915_request *rq;
2460
2461 rq = i915_request_create(ce);
2462 if (IS_ERR(rq)) {
2463 err = PTR_ERR(rq);
2464 goto err;
2465 }
2466
2467 cs = intel_ring_begin(rq, 12);
2468 if (IS_ERR(cs)) {
2469 i915_request_add(rq);
2470 err = PTR_ERR(cs);
2471 goto err;
2472 }
2473
2474 cs = emit_store_dw(cs, addr, -1);
2475 cs = emit_semaphore_poll_until(cs, offset, i);
2476 cs = emit_timestamp_store(cs, ce, addr + sizeof(u32));
2477
2478 intel_ring_advance(rq, cs);
2479 i915_request_add(rq);
2480
2481 if (wait_for(READ_ONCE(sema[2 * i]) == -1, 500)) {
2482 err = -EIO;
2483 goto err;
2484 }
2485
2486 rq = i915_request_create(ce->engine->kernel_context);
2487 if (IS_ERR(rq)) {
2488 err = PTR_ERR(rq);
2489 goto err;
2490 }
2491
2492 cs = intel_ring_begin(rq, 8);
2493 if (IS_ERR(cs)) {
2494 i915_request_add(rq);
2495 err = PTR_ERR(cs);
2496 goto err;
2497 }
2498
2499 cs = emit_timestamp_store(cs, ce, addr);
2500 cs = emit_store_dw(cs, offset, i);
2501
2502 intel_ring_advance(rq, cs);
2503 rq->sched.attr.priority = I915_PRIORITY_BARRIER;
2504
2505 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2506 i915_request_add(rq);
2507 }
2508
2509 if (wait_for(READ_ONCE(sema[2 * i - 2]) != -1, 500)) {
2510 err = -EIO;
2511 goto err;
2512 }
2513
2514 for (i = 1; i <= TF_COUNT; i++)
2515 elapsed[i - 1] = sema[2 * i + 0] - elapsed[i - 1];
2516
2517 cycles = trifilter(elapsed);
2518 pr_info("%s: preemption dispatch latency %d cycles, %lluns\n",
2519 ce->engine->name, cycles >> TF_BIAS,
2520 cycles_to_ns(ce->engine, cycles));
2521
2522 for (i = 1; i <= TF_COUNT; i++)
2523 elapsed[i - 1] = sema[2 * i + 1] - sema[2 * i + 0];
2524
2525 cycles = trifilter(elapsed);
2526 pr_info("%s: preemption switch latency %d cycles, %lluns\n",
2527 ce->engine->name, cycles >> TF_BIAS,
2528 cycles_to_ns(ce->engine, cycles));
2529
2530 return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2531
2532err:
2533 intel_gt_set_wedged(ce->engine->gt);
2534 return err;
2535}
2536
2537struct signal_cb {
2538 struct dma_fence_cb base;
2539 bool seen;
2540};
2541
2542static void signal_cb(struct dma_fence *fence, struct dma_fence_cb *cb)
2543{
2544 struct signal_cb *s = container_of(cb, typeof(*s), base);
2545
2546 smp_store_mb(s->seen, true); /* be safe, be strong */
2547}
2548
2549static int measure_completion(struct intel_context *ce)
2550{
2551 u32 *sema = hwsp_scratch(ce);
2552 const u32 offset = hwsp_offset(ce, sema);
2553 u32 elapsed[TF_COUNT], cycles;
2554 u32 *cs;
2555 int err;
2556 int i;
2557
2558 /*
2559 * Measure how long it takes for the signal (interrupt) to be
2560 * sent from the GPU to be processed by the CPU.
2561 *
2562 * A: read CS_TIMESTAMP on GPU
2563 * signal
2564 * B: read CS_TIMESTAMP from CPU
2565 *
2566 * Completion latency: B - A
2567 */
2568
2569 for (i = 1; i <= ARRAY_SIZE(elapsed); i++) {
2570 struct signal_cb cb = { .seen = false };
2571 struct i915_request *rq;
2572
2573 rq = i915_request_create(ce);
2574 if (IS_ERR(rq)) {
2575 err = PTR_ERR(rq);
2576 goto err;
2577 }
2578
2579 cs = intel_ring_begin(rq, 12);
2580 if (IS_ERR(cs)) {
2581 i915_request_add(rq);
2582 err = PTR_ERR(cs);
2583 goto err;
2584 }
2585
2586 cs = emit_store_dw(cs, offset + i * sizeof(u32), -1);
2587 cs = emit_semaphore_poll_until(cs, offset, i);
2588 cs = emit_timestamp_store(cs, ce, offset + i * sizeof(u32));
2589
2590 intel_ring_advance(rq, cs);
2591
2592 dma_fence_add_callback(&rq->fence, &cb.base, signal_cb);
2593 i915_request_add(rq);
2594
2595 intel_engine_flush_submission(ce->engine);
2596 if (wait_for(READ_ONCE(sema[i]) == -1, 50)) {
2597 err = -EIO;
2598 goto err;
2599 }
2600
2601 preempt_disable();
2602 semaphore_set(sema, i);
2603 while (!READ_ONCE(cb.seen))
2604 cpu_relax();
2605
2606 elapsed[i - 1] = ENGINE_READ_FW(ce->engine, RING_TIMESTAMP);
2607 preempt_enable();
2608 }
2609
2610 err = intel_gt_wait_for_idle(ce->engine->gt, HZ / 2);
2611 if (err)
2612 goto err;
2613
2614 for (i = 0; i < ARRAY_SIZE(elapsed); i++) {
2615 GEM_BUG_ON(sema[i + 1] == -1);
2616 elapsed[i] = elapsed[i] - sema[i + 1];
2617 }
2618
2619 cycles = trifilter(elapsed);
2620 pr_info("%s: completion latency %d cycles, %lluns\n",
2621 ce->engine->name, cycles >> TF_BIAS,
2622 cycles_to_ns(ce->engine, cycles));
2623
2624 return intel_gt_wait_for_idle(ce->engine->gt, HZ);
2625
2626err:
2627 intel_gt_set_wedged(ce->engine->gt);
2628 return err;
2629}
2630
2631static void rps_pin(struct intel_gt *gt)
2632{
2633 /* Pin the frequency to max */
2634 atomic_inc(>->rps.num_waiters);
2635 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL);
2636
2637 mutex_lock(>->rps.lock);
2638 intel_rps_set(>->rps, gt->rps.max_freq);
2639 mutex_unlock(>->rps.lock);
2640}
2641
2642static void rps_unpin(struct intel_gt *gt)
2643{
2644 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL);
2645 atomic_dec(>->rps.num_waiters);
2646}
2647
2648static int perf_request_latency(void *arg)
2649{
2650 struct drm_i915_private *i915 = arg;
2651 struct intel_engine_cs *engine;
2652 struct pm_qos_request qos;
2653 int err = 0;
2654
2655 if (GRAPHICS_VER(i915) < 8) /* per-engine CS timestamp, semaphores */
2656 return 0;
2657
2658 cpu_latency_qos_add_request(&qos, 0); /* disable cstates */
2659
2660 for_each_uabi_engine(engine, i915) {
2661 struct intel_context *ce;
2662
2663 ce = intel_context_create(engine);
2664 if (IS_ERR(ce)) {
2665 err = PTR_ERR(ce);
2666 goto out;
2667 }
2668
2669 err = intel_context_pin(ce);
2670 if (err) {
2671 intel_context_put(ce);
2672 goto out;
2673 }
2674
2675 st_engine_heartbeat_disable(engine);
2676 rps_pin(engine->gt);
2677
2678 if (err == 0)
2679 err = measure_semaphore_response(ce);
2680 if (err == 0)
2681 err = measure_idle_dispatch(ce);
2682 if (err == 0)
2683 err = measure_busy_dispatch(ce);
2684 if (err == 0)
2685 err = measure_inter_request(ce);
2686 if (err == 0)
2687 err = measure_context_switch(ce);
2688 if (err == 0)
2689 err = measure_preemption(ce);
2690 if (err == 0)
2691 err = measure_completion(ce);
2692
2693 rps_unpin(engine->gt);
2694 st_engine_heartbeat_enable(engine);
2695
2696 intel_context_unpin(ce);
2697 intel_context_put(ce);
2698 if (err)
2699 goto out;
2700 }
2701
2702out:
2703 if (igt_flush_test(i915))
2704 err = -EIO;
2705
2706 cpu_latency_qos_remove_request(&qos);
2707 return err;
2708}
2709
2710static int s_sync0(void *arg)
2711{
2712 struct perf_series *ps = arg;
2713 IGT_TIMEOUT(end_time);
2714 unsigned int idx = 0;
2715 int err = 0;
2716
2717 GEM_BUG_ON(!ps->nengines);
2718 do {
2719 struct i915_request *rq;
2720
2721 rq = i915_request_create(ps->ce[idx]);
2722 if (IS_ERR(rq)) {
2723 err = PTR_ERR(rq);
2724 break;
2725 }
2726
2727 i915_request_get(rq);
2728 i915_request_add(rq);
2729
2730 if (i915_request_wait(rq, 0, HZ / 5) < 0)
2731 err = -ETIME;
2732 i915_request_put(rq);
2733 if (err)
2734 break;
2735
2736 if (++idx == ps->nengines)
2737 idx = 0;
2738 } while (!__igt_timeout(end_time, NULL));
2739
2740 return err;
2741}
2742
2743static int s_sync1(void *arg)
2744{
2745 struct perf_series *ps = arg;
2746 struct i915_request *prev = NULL;
2747 IGT_TIMEOUT(end_time);
2748 unsigned int idx = 0;
2749 int err = 0;
2750
2751 GEM_BUG_ON(!ps->nengines);
2752 do {
2753 struct i915_request *rq;
2754
2755 rq = i915_request_create(ps->ce[idx]);
2756 if (IS_ERR(rq)) {
2757 err = PTR_ERR(rq);
2758 break;
2759 }
2760
2761 i915_request_get(rq);
2762 i915_request_add(rq);
2763
2764 if (prev && i915_request_wait(prev, 0, HZ / 5) < 0)
2765 err = -ETIME;
2766 i915_request_put(prev);
2767 prev = rq;
2768 if (err)
2769 break;
2770
2771 if (++idx == ps->nengines)
2772 idx = 0;
2773 } while (!__igt_timeout(end_time, NULL));
2774 i915_request_put(prev);
2775
2776 return err;
2777}
2778
2779static int s_many(void *arg)
2780{
2781 struct perf_series *ps = arg;
2782 IGT_TIMEOUT(end_time);
2783 unsigned int idx = 0;
2784
2785 GEM_BUG_ON(!ps->nengines);
2786 do {
2787 struct i915_request *rq;
2788
2789 rq = i915_request_create(ps->ce[idx]);
2790 if (IS_ERR(rq))
2791 return PTR_ERR(rq);
2792
2793 i915_request_add(rq);
2794
2795 if (++idx == ps->nengines)
2796 idx = 0;
2797 } while (!__igt_timeout(end_time, NULL));
2798
2799 return 0;
2800}
2801
2802static int perf_series_engines(void *arg)
2803{
2804 struct drm_i915_private *i915 = arg;
2805 static int (* const func[])(void *arg) = {
2806 s_sync0,
2807 s_sync1,
2808 s_many,
2809 NULL,
2810 };
2811 const unsigned int nengines = num_uabi_engines(i915);
2812 struct intel_engine_cs *engine;
2813 int (* const *fn)(void *arg);
2814 struct pm_qos_request qos;
2815 struct perf_stats *stats;
2816 struct perf_series *ps;
2817 unsigned int idx;
2818 int err = 0;
2819
2820 stats = kcalloc(nengines, sizeof(*stats), GFP_KERNEL);
2821 if (!stats)
2822 return -ENOMEM;
2823
2824 ps = kzalloc(struct_size(ps, ce, nengines), GFP_KERNEL);
2825 if (!ps) {
2826 kfree(stats);
2827 return -ENOMEM;
2828 }
2829
2830 cpu_latency_qos_add_request(&qos, 0); /* disable cstates */
2831
2832 ps->i915 = i915;
2833 ps->nengines = nengines;
2834
2835 idx = 0;
2836 for_each_uabi_engine(engine, i915) {
2837 struct intel_context *ce;
2838
2839 ce = intel_context_create(engine);
2840 if (IS_ERR(ce)) {
2841 err = PTR_ERR(ce);
2842 goto out;
2843 }
2844
2845 err = intel_context_pin(ce);
2846 if (err) {
2847 intel_context_put(ce);
2848 goto out;
2849 }
2850
2851 ps->ce[idx++] = ce;
2852 }
2853 GEM_BUG_ON(idx != ps->nengines);
2854
2855 for (fn = func; *fn && !err; fn++) {
2856 char name[KSYM_NAME_LEN];
2857 struct igt_live_test t;
2858
2859 snprintf(name, sizeof(name), "%ps", *fn);
2860 err = igt_live_test_begin(&t, i915, __func__, name);
2861 if (err)
2862 break;
2863
2864 for (idx = 0; idx < nengines; idx++) {
2865 struct perf_stats *p =
2866 memset(&stats[idx], 0, sizeof(stats[idx]));
2867 struct intel_context *ce = ps->ce[idx];
2868
2869 p->engine = ps->ce[idx]->engine;
2870 intel_engine_pm_get(p->engine);
2871
2872 if (intel_engine_supports_stats(p->engine))
2873 p->busy = intel_engine_get_busy_time(p->engine,
2874 &p->time) + 1;
2875 else
2876 p->time = ktime_get();
2877 p->runtime = -intel_context_get_total_runtime_ns(ce);
2878 }
2879
2880 err = (*fn)(ps);
2881 if (igt_live_test_end(&t))
2882 err = -EIO;
2883
2884 for (idx = 0; idx < nengines; idx++) {
2885 struct perf_stats *p = &stats[idx];
2886 struct intel_context *ce = ps->ce[idx];
2887 int integer, decimal;
2888 u64 busy, dt, now;
2889
2890 if (p->busy)
2891 p->busy = ktime_sub(intel_engine_get_busy_time(p->engine,
2892 &now),
2893 p->busy - 1);
2894 else
2895 now = ktime_get();
2896 p->time = ktime_sub(now, p->time);
2897
2898 err = switch_to_kernel_sync(ce, err);
2899 p->runtime += intel_context_get_total_runtime_ns(ce);
2900 intel_engine_pm_put(p->engine);
2901
2902 busy = 100 * ktime_to_ns(p->busy);
2903 dt = ktime_to_ns(p->time);
2904 if (dt) {
2905 integer = div64_u64(busy, dt);
2906 busy -= integer * dt;
2907 decimal = div64_u64(100 * busy, dt);
2908 } else {
2909 integer = 0;
2910 decimal = 0;
2911 }
2912
2913 pr_info("%s %5s: { seqno:%d, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n",
2914 name, p->engine->name, ce->timeline->seqno,
2915 integer, decimal,
2916 div_u64(p->runtime, 1000 * 1000),
2917 div_u64(ktime_to_ns(p->time), 1000 * 1000));
2918 }
2919 }
2920
2921out:
2922 for (idx = 0; idx < nengines; idx++) {
2923 if (IS_ERR_OR_NULL(ps->ce[idx]))
2924 break;
2925
2926 intel_context_unpin(ps->ce[idx]);
2927 intel_context_put(ps->ce[idx]);
2928 }
2929 kfree(ps);
2930
2931 cpu_latency_qos_remove_request(&qos);
2932 kfree(stats);
2933 return err;
2934}
2935
2936struct p_thread {
2937 struct perf_stats p;
2938 struct kthread_worker *worker;
2939 struct kthread_work work;
2940 struct intel_engine_cs *engine;
2941 int result;
2942};
2943
2944static void p_sync0(struct kthread_work *work)
2945{
2946 struct p_thread *thread = container_of(work, typeof(*thread), work);
2947 struct perf_stats *p = &thread->p;
2948 struct intel_engine_cs *engine = p->engine;
2949 struct intel_context *ce;
2950 IGT_TIMEOUT(end_time);
2951 unsigned long count;
2952 bool busy;
2953 int err = 0;
2954
2955 ce = intel_context_create(engine);
2956 if (IS_ERR(ce)) {
2957 thread->result = PTR_ERR(ce);
2958 return;
2959 }
2960
2961 err = intel_context_pin(ce);
2962 if (err) {
2963 intel_context_put(ce);
2964 thread->result = err;
2965 return;
2966 }
2967
2968 if (intel_engine_supports_stats(engine)) {
2969 p->busy = intel_engine_get_busy_time(engine, &p->time);
2970 busy = true;
2971 } else {
2972 p->time = ktime_get();
2973 busy = false;
2974 }
2975
2976 count = 0;
2977 do {
2978 struct i915_request *rq;
2979
2980 rq = i915_request_create(ce);
2981 if (IS_ERR(rq)) {
2982 err = PTR_ERR(rq);
2983 break;
2984 }
2985
2986 i915_request_get(rq);
2987 i915_request_add(rq);
2988
2989 err = 0;
2990 if (i915_request_wait(rq, 0, HZ) < 0)
2991 err = -ETIME;
2992 i915_request_put(rq);
2993 if (err)
2994 break;
2995
2996 count++;
2997 } while (!__igt_timeout(end_time, NULL));
2998
2999 if (busy) {
3000 ktime_t now;
3001
3002 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
3003 p->busy);
3004 p->time = ktime_sub(now, p->time);
3005 } else {
3006 p->time = ktime_sub(ktime_get(), p->time);
3007 }
3008
3009 err = switch_to_kernel_sync(ce, err);
3010 p->runtime = intel_context_get_total_runtime_ns(ce);
3011 p->count = count;
3012
3013 intel_context_unpin(ce);
3014 intel_context_put(ce);
3015 thread->result = err;
3016}
3017
3018static void p_sync1(struct kthread_work *work)
3019{
3020 struct p_thread *thread = container_of(work, typeof(*thread), work);
3021 struct perf_stats *p = &thread->p;
3022 struct intel_engine_cs *engine = p->engine;
3023 struct i915_request *prev = NULL;
3024 struct intel_context *ce;
3025 IGT_TIMEOUT(end_time);
3026 unsigned long count;
3027 bool busy;
3028 int err = 0;
3029
3030 ce = intel_context_create(engine);
3031 if (IS_ERR(ce)) {
3032 thread->result = PTR_ERR(ce);
3033 return;
3034 }
3035
3036 err = intel_context_pin(ce);
3037 if (err) {
3038 intel_context_put(ce);
3039 thread->result = err;
3040 return;
3041 }
3042
3043 if (intel_engine_supports_stats(engine)) {
3044 p->busy = intel_engine_get_busy_time(engine, &p->time);
3045 busy = true;
3046 } else {
3047 p->time = ktime_get();
3048 busy = false;
3049 }
3050
3051 count = 0;
3052 do {
3053 struct i915_request *rq;
3054
3055 rq = i915_request_create(ce);
3056 if (IS_ERR(rq)) {
3057 err = PTR_ERR(rq);
3058 break;
3059 }
3060
3061 i915_request_get(rq);
3062 i915_request_add(rq);
3063
3064 err = 0;
3065 if (prev && i915_request_wait(prev, 0, HZ) < 0)
3066 err = -ETIME;
3067 i915_request_put(prev);
3068 prev = rq;
3069 if (err)
3070 break;
3071
3072 count++;
3073 } while (!__igt_timeout(end_time, NULL));
3074 i915_request_put(prev);
3075
3076 if (busy) {
3077 ktime_t now;
3078
3079 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
3080 p->busy);
3081 p->time = ktime_sub(now, p->time);
3082 } else {
3083 p->time = ktime_sub(ktime_get(), p->time);
3084 }
3085
3086 err = switch_to_kernel_sync(ce, err);
3087 p->runtime = intel_context_get_total_runtime_ns(ce);
3088 p->count = count;
3089
3090 intel_context_unpin(ce);
3091 intel_context_put(ce);
3092 thread->result = err;
3093}
3094
3095static void p_many(struct kthread_work *work)
3096{
3097 struct p_thread *thread = container_of(work, typeof(*thread), work);
3098 struct perf_stats *p = &thread->p;
3099 struct intel_engine_cs *engine = p->engine;
3100 struct intel_context *ce;
3101 IGT_TIMEOUT(end_time);
3102 unsigned long count;
3103 int err = 0;
3104 bool busy;
3105
3106 ce = intel_context_create(engine);
3107 if (IS_ERR(ce)) {
3108 thread->result = PTR_ERR(ce);
3109 return;
3110 }
3111
3112 err = intel_context_pin(ce);
3113 if (err) {
3114 intel_context_put(ce);
3115 thread->result = err;
3116 return;
3117 }
3118
3119 if (intel_engine_supports_stats(engine)) {
3120 p->busy = intel_engine_get_busy_time(engine, &p->time);
3121 busy = true;
3122 } else {
3123 p->time = ktime_get();
3124 busy = false;
3125 }
3126
3127 count = 0;
3128 do {
3129 struct i915_request *rq;
3130
3131 rq = i915_request_create(ce);
3132 if (IS_ERR(rq)) {
3133 err = PTR_ERR(rq);
3134 break;
3135 }
3136
3137 i915_request_add(rq);
3138 count++;
3139 } while (!__igt_timeout(end_time, NULL));
3140
3141 if (busy) {
3142 ktime_t now;
3143
3144 p->busy = ktime_sub(intel_engine_get_busy_time(engine, &now),
3145 p->busy);
3146 p->time = ktime_sub(now, p->time);
3147 } else {
3148 p->time = ktime_sub(ktime_get(), p->time);
3149 }
3150
3151 err = switch_to_kernel_sync(ce, err);
3152 p->runtime = intel_context_get_total_runtime_ns(ce);
3153 p->count = count;
3154
3155 intel_context_unpin(ce);
3156 intel_context_put(ce);
3157 thread->result = err;
3158}
3159
3160static int perf_parallel_engines(void *arg)
3161{
3162 struct drm_i915_private *i915 = arg;
3163 static void (* const func[])(struct kthread_work *) = {
3164 p_sync0,
3165 p_sync1,
3166 p_many,
3167 NULL,
3168 };
3169 const unsigned int nengines = num_uabi_engines(i915);
3170 void (* const *fn)(struct kthread_work *);
3171 struct intel_engine_cs *engine;
3172 struct pm_qos_request qos;
3173 struct p_thread *engines;
3174 int err = 0;
3175
3176 engines = kcalloc(nengines, sizeof(*engines), GFP_KERNEL);
3177 if (!engines)
3178 return -ENOMEM;
3179
3180 cpu_latency_qos_add_request(&qos, 0);
3181
3182 for (fn = func; *fn; fn++) {
3183 char name[KSYM_NAME_LEN];
3184 struct igt_live_test t;
3185 unsigned int idx;
3186
3187 snprintf(name, sizeof(name), "%ps", *fn);
3188 err = igt_live_test_begin(&t, i915, __func__, name);
3189 if (err)
3190 break;
3191
3192 atomic_set(&i915->selftest.counter, nengines);
3193
3194 idx = 0;
3195 for_each_uabi_engine(engine, i915) {
3196 struct kthread_worker *worker;
3197
3198 intel_engine_pm_get(engine);
3199
3200 memset(&engines[idx].p, 0, sizeof(engines[idx].p));
3201
3202 worker = kthread_create_worker(0, "igt:%s",
3203 engine->name);
3204 if (IS_ERR(worker)) {
3205 err = PTR_ERR(worker);
3206 intel_engine_pm_put(engine);
3207 break;
3208 }
3209 engines[idx].worker = worker;
3210 engines[idx].result = 0;
3211 engines[idx].p.engine = engine;
3212 engines[idx].engine = engine;
3213
3214 kthread_init_work(&engines[idx].work, *fn);
3215 kthread_queue_work(worker, &engines[idx].work);
3216 idx++;
3217 }
3218
3219 idx = 0;
3220 for_each_uabi_engine(engine, i915) {
3221 int status;
3222
3223 if (!engines[idx].worker)
3224 break;
3225
3226 kthread_flush_work(&engines[idx].work);
3227 status = READ_ONCE(engines[idx].result);
3228 if (status && !err)
3229 err = status;
3230
3231 intel_engine_pm_put(engine);
3232
3233 kthread_destroy_worker(engines[idx].worker);
3234 idx++;
3235 }
3236
3237 if (igt_live_test_end(&t))
3238 err = -EIO;
3239 if (err)
3240 break;
3241
3242 idx = 0;
3243 for_each_uabi_engine(engine, i915) {
3244 struct perf_stats *p = &engines[idx].p;
3245 u64 busy = 100 * ktime_to_ns(p->busy);
3246 u64 dt = ktime_to_ns(p->time);
3247 int integer, decimal;
3248
3249 if (dt) {
3250 integer = div64_u64(busy, dt);
3251 busy -= integer * dt;
3252 decimal = div64_u64(100 * busy, dt);
3253 } else {
3254 integer = 0;
3255 decimal = 0;
3256 }
3257
3258 GEM_BUG_ON(engine != p->engine);
3259 pr_info("%s %5s: { count:%lu, busy:%d.%02d%%, runtime:%lldms, walltime:%lldms }\n",
3260 name, engine->name, p->count, integer, decimal,
3261 div_u64(p->runtime, 1000 * 1000),
3262 div_u64(ktime_to_ns(p->time), 1000 * 1000));
3263 idx++;
3264 }
3265 }
3266
3267 cpu_latency_qos_remove_request(&qos);
3268 kfree(engines);
3269 return err;
3270}
3271
3272int i915_request_perf_selftests(struct drm_i915_private *i915)
3273{
3274 static const struct i915_subtest tests[] = {
3275 SUBTEST(perf_request_latency),
3276 SUBTEST(perf_series_engines),
3277 SUBTEST(perf_parallel_engines),
3278 };
3279
3280 if (intel_gt_is_wedged(to_gt(i915)))
3281 return 0;
3282
3283 return i915_subtests(tests, i915);
3284}