Loading...
Note: File does not exist in v4.6.
1// SPDX-License-Identifier: MIT
2/*
3 * Copyright © 2020 Intel Corporation
4 */
5
6#include <linux/sort.h>
7
8#include "gem/i915_gem_internal.h"
9#include "gem/i915_gem_lmem.h"
10
11#include "selftests/i915_random.h"
12
13static const unsigned int sizes[] = {
14 SZ_4K,
15 SZ_64K,
16 SZ_2M,
17 CHUNK_SZ - SZ_4K,
18 CHUNK_SZ,
19 CHUNK_SZ + SZ_4K,
20 SZ_64M,
21};
22
23static struct drm_i915_gem_object *
24create_lmem_or_internal(struct drm_i915_private *i915, size_t size)
25{
26 struct drm_i915_gem_object *obj;
27
28 obj = i915_gem_object_create_lmem(i915, size, 0);
29 if (!IS_ERR(obj))
30 return obj;
31
32 return i915_gem_object_create_internal(i915, size);
33}
34
35static int copy(struct intel_migrate *migrate,
36 int (*fn)(struct intel_migrate *migrate,
37 struct i915_gem_ww_ctx *ww,
38 struct drm_i915_gem_object *src,
39 struct drm_i915_gem_object *dst,
40 struct i915_request **out),
41 u32 sz, struct rnd_state *prng)
42{
43 struct drm_i915_private *i915 = migrate->context->engine->i915;
44 struct drm_i915_gem_object *src, *dst;
45 struct i915_request *rq;
46 struct i915_gem_ww_ctx ww;
47 u32 *vaddr;
48 int err = 0;
49 int i;
50
51 src = create_lmem_or_internal(i915, sz);
52 if (IS_ERR(src))
53 return 0;
54
55 sz = src->base.size;
56 dst = i915_gem_object_create_internal(i915, sz);
57 if (IS_ERR(dst))
58 goto err_free_src;
59
60 for_i915_gem_ww(&ww, err, true) {
61 err = i915_gem_object_lock(src, &ww);
62 if (err)
63 continue;
64
65 err = i915_gem_object_lock(dst, &ww);
66 if (err)
67 continue;
68
69 vaddr = i915_gem_object_pin_map(src, I915_MAP_WC);
70 if (IS_ERR(vaddr)) {
71 err = PTR_ERR(vaddr);
72 continue;
73 }
74
75 for (i = 0; i < sz / sizeof(u32); i++)
76 vaddr[i] = i;
77 i915_gem_object_flush_map(src);
78
79 vaddr = i915_gem_object_pin_map(dst, I915_MAP_WC);
80 if (IS_ERR(vaddr)) {
81 err = PTR_ERR(vaddr);
82 goto unpin_src;
83 }
84
85 for (i = 0; i < sz / sizeof(u32); i++)
86 vaddr[i] = ~i;
87 i915_gem_object_flush_map(dst);
88
89 err = fn(migrate, &ww, src, dst, &rq);
90 if (!err)
91 continue;
92
93 if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS)
94 pr_err("%ps failed, size: %u\n", fn, sz);
95 if (rq) {
96 i915_request_wait(rq, 0, HZ);
97 i915_request_put(rq);
98 }
99 i915_gem_object_unpin_map(dst);
100unpin_src:
101 i915_gem_object_unpin_map(src);
102 }
103 if (err)
104 goto err_out;
105
106 if (rq) {
107 if (i915_request_wait(rq, 0, HZ) < 0) {
108 pr_err("%ps timed out, size: %u\n", fn, sz);
109 err = -ETIME;
110 }
111 i915_request_put(rq);
112 }
113
114 for (i = 0; !err && i < sz / PAGE_SIZE; i++) {
115 int x = i * 1024 + i915_prandom_u32_max_state(1024, prng);
116
117 if (vaddr[x] != x) {
118 pr_err("%ps failed, size: %u, offset: %zu\n",
119 fn, sz, x * sizeof(u32));
120 igt_hexdump(vaddr + i * 1024, 4096);
121 err = -EINVAL;
122 }
123 }
124
125 i915_gem_object_unpin_map(dst);
126 i915_gem_object_unpin_map(src);
127
128err_out:
129 i915_gem_object_put(dst);
130err_free_src:
131 i915_gem_object_put(src);
132
133 return err;
134}
135
136static int intel_context_copy_ccs(struct intel_context *ce,
137 const struct i915_deps *deps,
138 struct scatterlist *sg,
139 enum i915_cache_level cache_level,
140 bool write_to_ccs,
141 struct i915_request **out)
142{
143 u8 src_access = write_to_ccs ? DIRECT_ACCESS : INDIRECT_ACCESS;
144 u8 dst_access = write_to_ccs ? INDIRECT_ACCESS : DIRECT_ACCESS;
145 struct sgt_dma it = sg_sgt(sg);
146 struct i915_request *rq;
147 u32 offset;
148 int err;
149
150 GEM_BUG_ON(ce->vm != ce->engine->gt->migrate.context->vm);
151 *out = NULL;
152
153 GEM_BUG_ON(ce->ring->size < SZ_64K);
154
155 offset = 0;
156 if (HAS_64K_PAGES(ce->engine->i915))
157 offset = CHUNK_SZ;
158
159 do {
160 int len;
161
162 rq = i915_request_create(ce);
163 if (IS_ERR(rq)) {
164 err = PTR_ERR(rq);
165 goto out_ce;
166 }
167
168 if (deps) {
169 err = i915_request_await_deps(rq, deps);
170 if (err)
171 goto out_rq;
172
173 if (rq->engine->emit_init_breadcrumb) {
174 err = rq->engine->emit_init_breadcrumb(rq);
175 if (err)
176 goto out_rq;
177 }
178
179 deps = NULL;
180 }
181
182 /* The PTE updates + clear must not be interrupted. */
183 err = emit_no_arbitration(rq);
184 if (err)
185 goto out_rq;
186
187 len = emit_pte(rq, &it, cache_level, true, offset, CHUNK_SZ);
188 if (len <= 0) {
189 err = len;
190 goto out_rq;
191 }
192
193 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
194 if (err)
195 goto out_rq;
196
197 err = emit_copy_ccs(rq, offset, dst_access,
198 offset, src_access, len);
199 if (err)
200 goto out_rq;
201
202 err = rq->engine->emit_flush(rq, EMIT_INVALIDATE);
203
204 /* Arbitration is re-enabled between requests. */
205out_rq:
206 if (*out)
207 i915_request_put(*out);
208 *out = i915_request_get(rq);
209 i915_request_add(rq);
210 if (err || !it.sg || !sg_dma_len(it.sg))
211 break;
212
213 cond_resched();
214 } while (1);
215
216out_ce:
217 return err;
218}
219
220static int
221intel_migrate_ccs_copy(struct intel_migrate *m,
222 struct i915_gem_ww_ctx *ww,
223 const struct i915_deps *deps,
224 struct scatterlist *sg,
225 enum i915_cache_level cache_level,
226 bool write_to_ccs,
227 struct i915_request **out)
228{
229 struct intel_context *ce;
230 int err;
231
232 *out = NULL;
233 if (!m->context)
234 return -ENODEV;
235
236 ce = intel_migrate_create_context(m);
237 if (IS_ERR(ce))
238 ce = intel_context_get(m->context);
239 GEM_BUG_ON(IS_ERR(ce));
240
241 err = intel_context_pin_ww(ce, ww);
242 if (err)
243 goto out;
244
245 err = intel_context_copy_ccs(ce, deps, sg, cache_level,
246 write_to_ccs, out);
247
248 intel_context_unpin(ce);
249out:
250 intel_context_put(ce);
251 return err;
252}
253
254static int clear(struct intel_migrate *migrate,
255 int (*fn)(struct intel_migrate *migrate,
256 struct i915_gem_ww_ctx *ww,
257 struct drm_i915_gem_object *obj,
258 u32 value,
259 struct i915_request **out),
260 u32 sz, struct rnd_state *prng)
261{
262 struct drm_i915_private *i915 = migrate->context->engine->i915;
263 struct drm_i915_gem_object *obj;
264 struct i915_request *rq;
265 struct i915_gem_ww_ctx ww;
266 u32 *vaddr, val = 0;
267 bool ccs_cap = false;
268 int err = 0;
269 int i;
270
271 obj = create_lmem_or_internal(i915, sz);
272 if (IS_ERR(obj))
273 return 0;
274
275 /* Consider the rounded up memory too */
276 sz = obj->base.size;
277
278 if (HAS_FLAT_CCS(i915) && i915_gem_object_is_lmem(obj))
279 ccs_cap = true;
280
281 for_i915_gem_ww(&ww, err, true) {
282 int ccs_bytes, ccs_bytes_per_chunk;
283
284 err = i915_gem_object_lock(obj, &ww);
285 if (err)
286 continue;
287
288 vaddr = i915_gem_object_pin_map(obj, I915_MAP_WC);
289 if (IS_ERR(vaddr)) {
290 err = PTR_ERR(vaddr);
291 continue;
292 }
293
294 for (i = 0; i < sz / sizeof(u32); i++)
295 vaddr[i] = ~i;
296 i915_gem_object_flush_map(obj);
297
298 if (ccs_cap && !val) {
299 /* Write the obj data into ccs surface */
300 err = intel_migrate_ccs_copy(migrate, &ww, NULL,
301 obj->mm.pages->sgl,
302 obj->cache_level,
303 true, &rq);
304 if (rq && !err) {
305 if (i915_request_wait(rq, 0, HZ) < 0) {
306 pr_err("%ps timed out, size: %u\n",
307 fn, sz);
308 err = -ETIME;
309 }
310 i915_request_put(rq);
311 rq = NULL;
312 }
313 if (err)
314 continue;
315 }
316
317 err = fn(migrate, &ww, obj, val, &rq);
318 if (rq && !err) {
319 if (i915_request_wait(rq, 0, HZ) < 0) {
320 pr_err("%ps timed out, size: %u\n", fn, sz);
321 err = -ETIME;
322 }
323 i915_request_put(rq);
324 rq = NULL;
325 }
326 if (err)
327 continue;
328
329 i915_gem_object_flush_map(obj);
330
331 /* Verify the set/clear of the obj mem */
332 for (i = 0; !err && i < sz / PAGE_SIZE; i++) {
333 int x = i * 1024 +
334 i915_prandom_u32_max_state(1024, prng);
335
336 if (vaddr[x] != val) {
337 pr_err("%ps failed, (%u != %u), offset: %zu\n",
338 fn, vaddr[x], val, x * sizeof(u32));
339 igt_hexdump(vaddr + i * 1024, 4096);
340 err = -EINVAL;
341 }
342 }
343 if (err)
344 continue;
345
346 if (ccs_cap && !val) {
347 for (i = 0; i < sz / sizeof(u32); i++)
348 vaddr[i] = ~i;
349 i915_gem_object_flush_map(obj);
350
351 err = intel_migrate_ccs_copy(migrate, &ww, NULL,
352 obj->mm.pages->sgl,
353 obj->cache_level,
354 false, &rq);
355 if (rq && !err) {
356 if (i915_request_wait(rq, 0, HZ) < 0) {
357 pr_err("%ps timed out, size: %u\n",
358 fn, sz);
359 err = -ETIME;
360 }
361 i915_request_put(rq);
362 rq = NULL;
363 }
364 if (err)
365 continue;
366
367 ccs_bytes = GET_CCS_BYTES(i915, sz);
368 ccs_bytes_per_chunk = GET_CCS_BYTES(i915, CHUNK_SZ);
369 i915_gem_object_flush_map(obj);
370
371 for (i = 0; !err && i < DIV_ROUND_UP(ccs_bytes, PAGE_SIZE); i++) {
372 int offset = ((i * PAGE_SIZE) /
373 ccs_bytes_per_chunk) * CHUNK_SZ / sizeof(u32);
374 int ccs_bytes_left = (ccs_bytes - i * PAGE_SIZE) / sizeof(u32);
375 int x = i915_prandom_u32_max_state(min_t(int, 1024,
376 ccs_bytes_left), prng);
377
378 if (vaddr[offset + x]) {
379 pr_err("%ps ccs clearing failed, offset: %ld/%d\n",
380 fn, i * PAGE_SIZE + x * sizeof(u32), ccs_bytes);
381 igt_hexdump(vaddr + offset,
382 min_t(int, 4096,
383 ccs_bytes_left * sizeof(u32)));
384 err = -EINVAL;
385 }
386 }
387
388 if (err)
389 continue;
390 }
391 i915_gem_object_unpin_map(obj);
392 }
393
394 if (err) {
395 if (err != -EDEADLK && err != -EINTR && err != -ERESTARTSYS)
396 pr_err("%ps failed, size: %u\n", fn, sz);
397 if (rq && err != -EINVAL) {
398 i915_request_wait(rq, 0, HZ);
399 i915_request_put(rq);
400 }
401
402 i915_gem_object_unpin_map(obj);
403 }
404
405 i915_gem_object_put(obj);
406 return err;
407}
408
409static int __migrate_copy(struct intel_migrate *migrate,
410 struct i915_gem_ww_ctx *ww,
411 struct drm_i915_gem_object *src,
412 struct drm_i915_gem_object *dst,
413 struct i915_request **out)
414{
415 return intel_migrate_copy(migrate, ww, NULL,
416 src->mm.pages->sgl, src->cache_level,
417 i915_gem_object_is_lmem(src),
418 dst->mm.pages->sgl, dst->cache_level,
419 i915_gem_object_is_lmem(dst),
420 out);
421}
422
423static int __global_copy(struct intel_migrate *migrate,
424 struct i915_gem_ww_ctx *ww,
425 struct drm_i915_gem_object *src,
426 struct drm_i915_gem_object *dst,
427 struct i915_request **out)
428{
429 return intel_context_migrate_copy(migrate->context, NULL,
430 src->mm.pages->sgl, src->cache_level,
431 i915_gem_object_is_lmem(src),
432 dst->mm.pages->sgl, dst->cache_level,
433 i915_gem_object_is_lmem(dst),
434 out);
435}
436
437static int
438migrate_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
439{
440 return copy(migrate, __migrate_copy, sz, prng);
441}
442
443static int
444global_copy(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
445{
446 return copy(migrate, __global_copy, sz, prng);
447}
448
449static int __migrate_clear(struct intel_migrate *migrate,
450 struct i915_gem_ww_ctx *ww,
451 struct drm_i915_gem_object *obj,
452 u32 value,
453 struct i915_request **out)
454{
455 return intel_migrate_clear(migrate, ww, NULL,
456 obj->mm.pages->sgl,
457 obj->cache_level,
458 i915_gem_object_is_lmem(obj),
459 value, out);
460}
461
462static int __global_clear(struct intel_migrate *migrate,
463 struct i915_gem_ww_ctx *ww,
464 struct drm_i915_gem_object *obj,
465 u32 value,
466 struct i915_request **out)
467{
468 return intel_context_migrate_clear(migrate->context, NULL,
469 obj->mm.pages->sgl,
470 obj->cache_level,
471 i915_gem_object_is_lmem(obj),
472 value, out);
473}
474
475static int
476migrate_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
477{
478 return clear(migrate, __migrate_clear, sz, prng);
479}
480
481static int
482global_clear(struct intel_migrate *migrate, u32 sz, struct rnd_state *prng)
483{
484 return clear(migrate, __global_clear, sz, prng);
485}
486
487static int live_migrate_copy(void *arg)
488{
489 struct intel_migrate *migrate = arg;
490 struct drm_i915_private *i915 = migrate->context->engine->i915;
491 I915_RND_STATE(prng);
492 int i;
493
494 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
495 int err;
496
497 err = migrate_copy(migrate, sizes[i], &prng);
498 if (err == 0)
499 err = global_copy(migrate, sizes[i], &prng);
500 i915_gem_drain_freed_objects(i915);
501 if (err)
502 return err;
503 }
504
505 return 0;
506}
507
508static int live_migrate_clear(void *arg)
509{
510 struct intel_migrate *migrate = arg;
511 struct drm_i915_private *i915 = migrate->context->engine->i915;
512 I915_RND_STATE(prng);
513 int i;
514
515 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
516 int err;
517
518 err = migrate_clear(migrate, sizes[i], &prng);
519 if (err == 0)
520 err = global_clear(migrate, sizes[i], &prng);
521
522 i915_gem_drain_freed_objects(i915);
523 if (err)
524 return err;
525 }
526
527 return 0;
528}
529
530struct threaded_migrate {
531 struct intel_migrate *migrate;
532 struct task_struct *tsk;
533 struct rnd_state prng;
534};
535
536static int threaded_migrate(struct intel_migrate *migrate,
537 int (*fn)(void *arg),
538 unsigned int flags)
539{
540 const unsigned int n_cpus = num_online_cpus() + 1;
541 struct threaded_migrate *thread;
542 I915_RND_STATE(prng);
543 unsigned int i;
544 int err = 0;
545
546 thread = kcalloc(n_cpus, sizeof(*thread), GFP_KERNEL);
547 if (!thread)
548 return 0;
549
550 for (i = 0; i < n_cpus; ++i) {
551 struct task_struct *tsk;
552
553 thread[i].migrate = migrate;
554 thread[i].prng =
555 I915_RND_STATE_INITIALIZER(prandom_u32_state(&prng));
556
557 tsk = kthread_run(fn, &thread[i], "igt-%d", i);
558 if (IS_ERR(tsk)) {
559 err = PTR_ERR(tsk);
560 break;
561 }
562
563 get_task_struct(tsk);
564 thread[i].tsk = tsk;
565 }
566
567 msleep(10); /* start all threads before we kthread_stop() */
568
569 for (i = 0; i < n_cpus; ++i) {
570 struct task_struct *tsk = thread[i].tsk;
571 int status;
572
573 if (IS_ERR_OR_NULL(tsk))
574 continue;
575
576 status = kthread_stop(tsk);
577 if (status && !err)
578 err = status;
579
580 put_task_struct(tsk);
581 }
582
583 kfree(thread);
584 return err;
585}
586
587static int __thread_migrate_copy(void *arg)
588{
589 struct threaded_migrate *tm = arg;
590
591 return migrate_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
592}
593
594static int thread_migrate_copy(void *arg)
595{
596 return threaded_migrate(arg, __thread_migrate_copy, 0);
597}
598
599static int __thread_global_copy(void *arg)
600{
601 struct threaded_migrate *tm = arg;
602
603 return global_copy(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
604}
605
606static int thread_global_copy(void *arg)
607{
608 return threaded_migrate(arg, __thread_global_copy, 0);
609}
610
611static int __thread_migrate_clear(void *arg)
612{
613 struct threaded_migrate *tm = arg;
614
615 return migrate_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
616}
617
618static int __thread_global_clear(void *arg)
619{
620 struct threaded_migrate *tm = arg;
621
622 return global_clear(tm->migrate, 2 * CHUNK_SZ, &tm->prng);
623}
624
625static int thread_migrate_clear(void *arg)
626{
627 return threaded_migrate(arg, __thread_migrate_clear, 0);
628}
629
630static int thread_global_clear(void *arg)
631{
632 return threaded_migrate(arg, __thread_global_clear, 0);
633}
634
635int intel_migrate_live_selftests(struct drm_i915_private *i915)
636{
637 static const struct i915_subtest tests[] = {
638 SUBTEST(live_migrate_copy),
639 SUBTEST(live_migrate_clear),
640 SUBTEST(thread_migrate_copy),
641 SUBTEST(thread_migrate_clear),
642 SUBTEST(thread_global_copy),
643 SUBTEST(thread_global_clear),
644 };
645 struct intel_gt *gt = to_gt(i915);
646
647 if (!gt->migrate.context)
648 return 0;
649
650 return i915_subtests(tests, >->migrate);
651}
652
653static struct drm_i915_gem_object *
654create_init_lmem_internal(struct intel_gt *gt, size_t sz, bool try_lmem)
655{
656 struct drm_i915_gem_object *obj = NULL;
657 int err;
658
659 if (try_lmem)
660 obj = i915_gem_object_create_lmem(gt->i915, sz, 0);
661
662 if (IS_ERR_OR_NULL(obj)) {
663 obj = i915_gem_object_create_internal(gt->i915, sz);
664 if (IS_ERR(obj))
665 return obj;
666 }
667
668 i915_gem_object_trylock(obj, NULL);
669 err = i915_gem_object_pin_pages(obj);
670 if (err) {
671 i915_gem_object_unlock(obj);
672 i915_gem_object_put(obj);
673 return ERR_PTR(err);
674 }
675
676 return obj;
677}
678
679static int wrap_ktime_compare(const void *A, const void *B)
680{
681 const ktime_t *a = A, *b = B;
682
683 return ktime_compare(*a, *b);
684}
685
686static int __perf_clear_blt(struct intel_context *ce,
687 struct scatterlist *sg,
688 enum i915_cache_level cache_level,
689 bool is_lmem,
690 size_t sz)
691{
692 ktime_t t[5];
693 int pass;
694 int err = 0;
695
696 for (pass = 0; pass < ARRAY_SIZE(t); pass++) {
697 struct i915_request *rq;
698 ktime_t t0, t1;
699
700 t0 = ktime_get();
701
702 err = intel_context_migrate_clear(ce, NULL, sg, cache_level,
703 is_lmem, 0, &rq);
704 if (rq) {
705 if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0)
706 err = -EIO;
707 i915_request_put(rq);
708 }
709 if (err)
710 break;
711
712 t1 = ktime_get();
713 t[pass] = ktime_sub(t1, t0);
714 }
715 if (err)
716 return err;
717
718 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL);
719 pr_info("%s: %zd KiB fill: %lld MiB/s\n",
720 ce->engine->name, sz >> 10,
721 div64_u64(mul_u32_u32(4 * sz,
722 1000 * 1000 * 1000),
723 t[1] + 2 * t[2] + t[3]) >> 20);
724 return 0;
725}
726
727static int perf_clear_blt(void *arg)
728{
729 struct intel_gt *gt = arg;
730 static const unsigned long sizes[] = {
731 SZ_4K,
732 SZ_64K,
733 SZ_2M,
734 SZ_64M
735 };
736 int i;
737
738 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
739 struct drm_i915_gem_object *dst;
740 int err;
741
742 dst = create_init_lmem_internal(gt, sizes[i], true);
743 if (IS_ERR(dst))
744 return PTR_ERR(dst);
745
746 err = __perf_clear_blt(gt->migrate.context,
747 dst->mm.pages->sgl,
748 I915_CACHE_NONE,
749 i915_gem_object_is_lmem(dst),
750 sizes[i]);
751
752 i915_gem_object_unlock(dst);
753 i915_gem_object_put(dst);
754 if (err)
755 return err;
756 }
757
758 return 0;
759}
760
761static int __perf_copy_blt(struct intel_context *ce,
762 struct scatterlist *src,
763 enum i915_cache_level src_cache_level,
764 bool src_is_lmem,
765 struct scatterlist *dst,
766 enum i915_cache_level dst_cache_level,
767 bool dst_is_lmem,
768 size_t sz)
769{
770 ktime_t t[5];
771 int pass;
772 int err = 0;
773
774 for (pass = 0; pass < ARRAY_SIZE(t); pass++) {
775 struct i915_request *rq;
776 ktime_t t0, t1;
777
778 t0 = ktime_get();
779
780 err = intel_context_migrate_copy(ce, NULL,
781 src, src_cache_level,
782 src_is_lmem,
783 dst, dst_cache_level,
784 dst_is_lmem,
785 &rq);
786 if (rq) {
787 if (i915_request_wait(rq, 0, MAX_SCHEDULE_TIMEOUT) < 0)
788 err = -EIO;
789 i915_request_put(rq);
790 }
791 if (err)
792 break;
793
794 t1 = ktime_get();
795 t[pass] = ktime_sub(t1, t0);
796 }
797 if (err)
798 return err;
799
800 sort(t, ARRAY_SIZE(t), sizeof(*t), wrap_ktime_compare, NULL);
801 pr_info("%s: %zd KiB copy: %lld MiB/s\n",
802 ce->engine->name, sz >> 10,
803 div64_u64(mul_u32_u32(4 * sz,
804 1000 * 1000 * 1000),
805 t[1] + 2 * t[2] + t[3]) >> 20);
806 return 0;
807}
808
809static int perf_copy_blt(void *arg)
810{
811 struct intel_gt *gt = arg;
812 static const unsigned long sizes[] = {
813 SZ_4K,
814 SZ_64K,
815 SZ_2M,
816 SZ_64M
817 };
818 int i;
819
820 for (i = 0; i < ARRAY_SIZE(sizes); i++) {
821 struct drm_i915_gem_object *src, *dst;
822 size_t sz;
823 int err;
824
825 src = create_init_lmem_internal(gt, sizes[i], true);
826 if (IS_ERR(src))
827 return PTR_ERR(src);
828
829 sz = src->base.size;
830 dst = create_init_lmem_internal(gt, sz, false);
831 if (IS_ERR(dst)) {
832 err = PTR_ERR(dst);
833 goto err_src;
834 }
835
836 err = __perf_copy_blt(gt->migrate.context,
837 src->mm.pages->sgl,
838 I915_CACHE_NONE,
839 i915_gem_object_is_lmem(src),
840 dst->mm.pages->sgl,
841 I915_CACHE_NONE,
842 i915_gem_object_is_lmem(dst),
843 sz);
844
845 i915_gem_object_unlock(dst);
846 i915_gem_object_put(dst);
847err_src:
848 i915_gem_object_unlock(src);
849 i915_gem_object_put(src);
850 if (err)
851 return err;
852 }
853
854 return 0;
855}
856
857int intel_migrate_perf_selftests(struct drm_i915_private *i915)
858{
859 static const struct i915_subtest tests[] = {
860 SUBTEST(perf_clear_blt),
861 SUBTEST(perf_copy_blt),
862 };
863 struct intel_gt *gt = to_gt(i915);
864
865 if (intel_gt_is_wedged(gt))
866 return 0;
867
868 if (!gt->migrate.context)
869 return 0;
870
871 return intel_gt_live_subtests(tests, gt);
872}