Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0 AND MIT
2/*
3 * Copyright © 2022 Intel Corporation
4 */
5
6#include <kunit/test.h>
7#include <kunit/visibility.h>
8
9#include <linux/iosys-map.h>
10#include <linux/math64.h>
11#include <linux/prandom.h>
12#include <linux/swap.h>
13
14#include <uapi/linux/sysinfo.h>
15
16#include "tests/xe_kunit_helpers.h"
17#include "tests/xe_pci_test.h"
18#include "tests/xe_test.h"
19
20#include "xe_bo_evict.h"
21#include "xe_pci.h"
22#include "xe_pm.h"
23
24static int ccs_test_migrate(struct xe_tile *tile, struct xe_bo *bo,
25 bool clear, u64 get_val, u64 assign_val,
26 struct kunit *test)
27{
28 struct dma_fence *fence;
29 struct ttm_tt *ttm;
30 struct page *page;
31 pgoff_t ccs_page;
32 long timeout;
33 u64 *cpu_map;
34 int ret;
35 u32 offset;
36
37 /* Move bo to VRAM if not already there. */
38 ret = xe_bo_validate(bo, NULL, false);
39 if (ret) {
40 KUNIT_FAIL(test, "Failed to validate bo.\n");
41 return ret;
42 }
43
44 /* Optionally clear bo *and* CCS data in VRAM. */
45 if (clear) {
46 fence = xe_migrate_clear(tile->migrate, bo, bo->ttm.resource,
47 XE_MIGRATE_CLEAR_FLAG_FULL);
48 if (IS_ERR(fence)) {
49 KUNIT_FAIL(test, "Failed to submit bo clear.\n");
50 return PTR_ERR(fence);
51 }
52 dma_fence_put(fence);
53 }
54
55 /* Evict to system. CCS data should be copied. */
56 ret = xe_bo_evict(bo, true);
57 if (ret) {
58 KUNIT_FAIL(test, "Failed to evict bo.\n");
59 return ret;
60 }
61
62 /* Sync all migration blits */
63 timeout = dma_resv_wait_timeout(bo->ttm.base.resv,
64 DMA_RESV_USAGE_KERNEL,
65 true,
66 5 * HZ);
67 if (timeout <= 0) {
68 KUNIT_FAIL(test, "Failed to sync bo eviction.\n");
69 return -ETIME;
70 }
71
72 /*
73 * Bo with CCS data is now in system memory. Verify backing store
74 * and data integrity. Then assign for the next testing round while
75 * we still have a CPU map.
76 */
77 ttm = bo->ttm.ttm;
78 if (!ttm || !ttm_tt_is_populated(ttm)) {
79 KUNIT_FAIL(test, "Bo was not in expected placement.\n");
80 return -EINVAL;
81 }
82
83 ccs_page = xe_bo_ccs_pages_start(bo) >> PAGE_SHIFT;
84 if (ccs_page >= ttm->num_pages) {
85 KUNIT_FAIL(test, "No TTM CCS pages present.\n");
86 return -EINVAL;
87 }
88
89 page = ttm->pages[ccs_page];
90 cpu_map = kmap_local_page(page);
91
92 /* Check first CCS value */
93 if (cpu_map[0] != get_val) {
94 KUNIT_FAIL(test,
95 "Expected CCS readout 0x%016llx, got 0x%016llx.\n",
96 (unsigned long long)get_val,
97 (unsigned long long)cpu_map[0]);
98 ret = -EINVAL;
99 }
100
101 /* Check last CCS value, or at least last value in page. */
102 offset = xe_device_ccs_bytes(tile_to_xe(tile), bo->size);
103 offset = min_t(u32, offset, PAGE_SIZE) / sizeof(u64) - 1;
104 if (cpu_map[offset] != get_val) {
105 KUNIT_FAIL(test,
106 "Expected CCS readout 0x%016llx, got 0x%016llx.\n",
107 (unsigned long long)get_val,
108 (unsigned long long)cpu_map[offset]);
109 ret = -EINVAL;
110 }
111
112 cpu_map[0] = assign_val;
113 cpu_map[offset] = assign_val;
114 kunmap_local(cpu_map);
115
116 return ret;
117}
118
119static void ccs_test_run_tile(struct xe_device *xe, struct xe_tile *tile,
120 struct kunit *test)
121{
122 struct xe_bo *bo;
123
124 int ret;
125
126 /* TODO: Sanity check */
127 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
128
129 if (IS_DGFX(xe))
130 kunit_info(test, "Testing vram id %u\n", tile->id);
131 else
132 kunit_info(test, "Testing system memory\n");
133
134 bo = xe_bo_create_user(xe, NULL, NULL, SZ_1M, DRM_XE_GEM_CPU_CACHING_WC,
135 bo_flags);
136 if (IS_ERR(bo)) {
137 KUNIT_FAIL(test, "Failed to create bo.\n");
138 return;
139 }
140
141 xe_bo_lock(bo, false);
142
143 kunit_info(test, "Verifying that CCS data is cleared on creation.\n");
144 ret = ccs_test_migrate(tile, bo, false, 0ULL, 0xdeadbeefdeadbeefULL,
145 test);
146 if (ret)
147 goto out_unlock;
148
149 kunit_info(test, "Verifying that CCS data survives migration.\n");
150 ret = ccs_test_migrate(tile, bo, false, 0xdeadbeefdeadbeefULL,
151 0xdeadbeefdeadbeefULL, test);
152 if (ret)
153 goto out_unlock;
154
155 kunit_info(test, "Verifying that CCS data can be properly cleared.\n");
156 ret = ccs_test_migrate(tile, bo, true, 0ULL, 0ULL, test);
157
158out_unlock:
159 xe_bo_unlock(bo);
160 xe_bo_put(bo);
161}
162
163static int ccs_test_run_device(struct xe_device *xe)
164{
165 struct kunit *test = kunit_get_current_test();
166 struct xe_tile *tile;
167 int id;
168
169 if (!xe_device_has_flat_ccs(xe)) {
170 kunit_skip(test, "non-flat-ccs device\n");
171 return 0;
172 }
173
174 /* For xe2+ dgfx, we don't handle ccs metadata */
175 if (GRAPHICS_VER(xe) >= 20 && IS_DGFX(xe)) {
176 kunit_skip(test, "xe2+ dgfx device\n");
177 return 0;
178 }
179
180 xe_pm_runtime_get(xe);
181
182 for_each_tile(tile, xe, id) {
183 /* For igfx run only for primary tile */
184 if (!IS_DGFX(xe) && id > 0)
185 continue;
186 ccs_test_run_tile(xe, tile, test);
187 }
188
189 xe_pm_runtime_put(xe);
190
191 return 0;
192}
193
194static void xe_ccs_migrate_kunit(struct kunit *test)
195{
196 struct xe_device *xe = test->priv;
197
198 ccs_test_run_device(xe);
199}
200
201static int evict_test_run_tile(struct xe_device *xe, struct xe_tile *tile, struct kunit *test)
202{
203 struct xe_bo *bo, *external;
204 unsigned int bo_flags = XE_BO_FLAG_VRAM_IF_DGFX(tile);
205 struct xe_vm *vm = xe_migrate_get_vm(xe_device_get_root_tile(xe)->migrate);
206 struct xe_gt *__gt;
207 int err, i, id;
208
209 kunit_info(test, "Testing device %s vram id %u\n",
210 dev_name(xe->drm.dev), tile->id);
211
212 for (i = 0; i < 2; ++i) {
213 xe_vm_lock(vm, false);
214 bo = xe_bo_create_user(xe, NULL, vm, 0x10000,
215 DRM_XE_GEM_CPU_CACHING_WC,
216 bo_flags);
217 xe_vm_unlock(vm);
218 if (IS_ERR(bo)) {
219 KUNIT_FAIL(test, "bo create err=%pe\n", bo);
220 break;
221 }
222
223 external = xe_bo_create_user(xe, NULL, NULL, 0x10000,
224 DRM_XE_GEM_CPU_CACHING_WC,
225 bo_flags);
226 if (IS_ERR(external)) {
227 KUNIT_FAIL(test, "external bo create err=%pe\n", external);
228 goto cleanup_bo;
229 }
230
231 xe_bo_lock(external, false);
232 err = xe_bo_pin_external(external);
233 xe_bo_unlock(external);
234 if (err) {
235 KUNIT_FAIL(test, "external bo pin err=%pe\n",
236 ERR_PTR(err));
237 goto cleanup_external;
238 }
239
240 err = xe_bo_evict_all(xe);
241 if (err) {
242 KUNIT_FAIL(test, "evict err=%pe\n", ERR_PTR(err));
243 goto cleanup_all;
244 }
245
246 for_each_gt(__gt, xe, id)
247 xe_gt_sanitize(__gt);
248 err = xe_bo_restore_kernel(xe);
249 /*
250 * Snapshotting the CTB and copying back a potentially old
251 * version seems risky, depending on what might have been
252 * inflight. Also it seems snapshotting the ADS object and
253 * copying back results in serious breakage. Normally when
254 * calling xe_bo_restore_kernel() we always fully restart the
255 * GT, which re-intializes such things. We could potentially
256 * skip saving and restoring such objects in xe_bo_evict_all()
257 * however seems quite fragile not to also restart the GT. Try
258 * to do that here by triggering a GT reset.
259 */
260 for_each_gt(__gt, xe, id)
261 xe_gt_reset(__gt);
262
263 if (err) {
264 KUNIT_FAIL(test, "restore kernel err=%pe\n",
265 ERR_PTR(err));
266 goto cleanup_all;
267 }
268
269 err = xe_bo_restore_user(xe);
270 if (err) {
271 KUNIT_FAIL(test, "restore user err=%pe\n", ERR_PTR(err));
272 goto cleanup_all;
273 }
274
275 if (!xe_bo_is_vram(external)) {
276 KUNIT_FAIL(test, "external bo is not vram\n");
277 err = -EPROTO;
278 goto cleanup_all;
279 }
280
281 if (xe_bo_is_vram(bo)) {
282 KUNIT_FAIL(test, "bo is vram\n");
283 err = -EPROTO;
284 goto cleanup_all;
285 }
286
287 if (i) {
288 down_read(&vm->lock);
289 xe_vm_lock(vm, false);
290 err = xe_bo_validate(bo, bo->vm, false);
291 xe_vm_unlock(vm);
292 up_read(&vm->lock);
293 if (err) {
294 KUNIT_FAIL(test, "bo valid err=%pe\n",
295 ERR_PTR(err));
296 goto cleanup_all;
297 }
298 xe_bo_lock(external, false);
299 err = xe_bo_validate(external, NULL, false);
300 xe_bo_unlock(external);
301 if (err) {
302 KUNIT_FAIL(test, "external bo valid err=%pe\n",
303 ERR_PTR(err));
304 goto cleanup_all;
305 }
306 }
307
308 xe_bo_lock(external, false);
309 xe_bo_unpin_external(external);
310 xe_bo_unlock(external);
311
312 xe_bo_put(external);
313
314 xe_bo_lock(bo, false);
315 __xe_bo_unset_bulk_move(bo);
316 xe_bo_unlock(bo);
317 xe_bo_put(bo);
318 continue;
319
320cleanup_all:
321 xe_bo_lock(external, false);
322 xe_bo_unpin_external(external);
323 xe_bo_unlock(external);
324cleanup_external:
325 xe_bo_put(external);
326cleanup_bo:
327 xe_bo_lock(bo, false);
328 __xe_bo_unset_bulk_move(bo);
329 xe_bo_unlock(bo);
330 xe_bo_put(bo);
331 break;
332 }
333
334 xe_vm_put(vm);
335
336 return 0;
337}
338
339static int evict_test_run_device(struct xe_device *xe)
340{
341 struct kunit *test = kunit_get_current_test();
342 struct xe_tile *tile;
343 int id;
344
345 if (!IS_DGFX(xe)) {
346 kunit_skip(test, "non-discrete device\n");
347 return 0;
348 }
349
350 xe_pm_runtime_get(xe);
351
352 for_each_tile(tile, xe, id)
353 evict_test_run_tile(xe, tile, test);
354
355 xe_pm_runtime_put(xe);
356
357 return 0;
358}
359
360static void xe_bo_evict_kunit(struct kunit *test)
361{
362 struct xe_device *xe = test->priv;
363
364 evict_test_run_device(xe);
365}
366
367struct xe_bo_link {
368 struct list_head link;
369 struct xe_bo *bo;
370 u32 val;
371};
372
373#define XE_BO_SHRINK_SIZE ((unsigned long)SZ_64M)
374
375static int shrink_test_fill_random(struct xe_bo *bo, struct rnd_state *state,
376 struct xe_bo_link *link)
377{
378 struct iosys_map map;
379 int ret = ttm_bo_vmap(&bo->ttm, &map);
380 size_t __maybe_unused i;
381
382 if (ret)
383 return ret;
384
385 for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) {
386 u32 val = prandom_u32_state(state);
387
388 iosys_map_wr(&map, i, u32, val);
389 if (i == 0)
390 link->val = val;
391 }
392
393 ttm_bo_vunmap(&bo->ttm, &map);
394 return 0;
395}
396
397static bool shrink_test_verify(struct kunit *test, struct xe_bo *bo,
398 unsigned int bo_nr, struct rnd_state *state,
399 struct xe_bo_link *link)
400{
401 struct iosys_map map;
402 int ret = ttm_bo_vmap(&bo->ttm, &map);
403 size_t i;
404 bool failed = false;
405
406 if (ret) {
407 KUNIT_FAIL(test, "Error mapping bo %u for content check.\n", bo_nr);
408 return true;
409 }
410
411 for (i = 0; i < bo->ttm.base.size; i += sizeof(u32)) {
412 u32 val = prandom_u32_state(state);
413
414 if (iosys_map_rd(&map, i, u32) != val) {
415 KUNIT_FAIL(test, "Content not preserved, bo %u offset 0x%016llx",
416 bo_nr, (unsigned long long)i);
417 kunit_info(test, "Failed value is 0x%08x, recorded 0x%08x\n",
418 (unsigned int)iosys_map_rd(&map, i, u32), val);
419 if (i == 0 && val != link->val)
420 kunit_info(test, "Looks like PRNG is out of sync.\n");
421 failed = true;
422 break;
423 }
424 }
425
426 ttm_bo_vunmap(&bo->ttm, &map);
427
428 return failed;
429}
430
431/*
432 * Try to create system bos corresponding to twice the amount
433 * of available system memory to test shrinker functionality.
434 * If no swap space is available to accommodate the
435 * memory overcommit, mark bos purgeable.
436 */
437static int shrink_test_run_device(struct xe_device *xe)
438{
439 struct kunit *test = kunit_get_current_test();
440 LIST_HEAD(bos);
441 struct xe_bo_link *link, *next;
442 struct sysinfo si;
443 u64 ram, ram_and_swap, purgeable = 0, alloced, to_alloc, limit;
444 unsigned int interrupted = 0, successful = 0, count = 0;
445 struct rnd_state prng;
446 u64 rand_seed;
447 bool failed = false;
448
449 rand_seed = get_random_u64();
450 prandom_seed_state(&prng, rand_seed);
451 kunit_info(test, "Random seed is 0x%016llx.\n",
452 (unsigned long long)rand_seed);
453
454 /* Skip if execution time is expected to be too long. */
455
456 limit = SZ_32G;
457 /* IGFX with flat CCS needs to copy when swapping / shrinking */
458 if (!IS_DGFX(xe) && xe_device_has_flat_ccs(xe))
459 limit = SZ_16G;
460
461 si_meminfo(&si);
462 ram = (size_t)si.freeram * si.mem_unit;
463 if (ram > limit) {
464 kunit_skip(test, "Too long expected execution time.\n");
465 return 0;
466 }
467 to_alloc = ram * 2;
468
469 ram_and_swap = ram + get_nr_swap_pages() * PAGE_SIZE;
470 if (to_alloc > ram_and_swap)
471 purgeable = to_alloc - ram_and_swap;
472 purgeable += div64_u64(purgeable, 5);
473
474 kunit_info(test, "Free ram is %lu bytes. Will allocate twice of that.\n",
475 (unsigned long)ram);
476 for (alloced = 0; alloced < to_alloc; alloced += XE_BO_SHRINK_SIZE) {
477 struct xe_bo *bo;
478 unsigned int mem_type;
479 struct xe_ttm_tt *xe_tt;
480
481 link = kzalloc(sizeof(*link), GFP_KERNEL);
482 if (!link) {
483 KUNIT_FAIL(test, "Unexpected link allocation failure\n");
484 failed = true;
485 break;
486 }
487
488 INIT_LIST_HEAD(&link->link);
489
490 /* We can create bos using WC caching here. But it is slower. */
491 bo = xe_bo_create_user(xe, NULL, NULL, XE_BO_SHRINK_SIZE,
492 DRM_XE_GEM_CPU_CACHING_WB,
493 XE_BO_FLAG_SYSTEM);
494 if (IS_ERR(bo)) {
495 if (bo != ERR_PTR(-ENOMEM) && bo != ERR_PTR(-ENOSPC) &&
496 bo != ERR_PTR(-EINTR) && bo != ERR_PTR(-ERESTARTSYS))
497 KUNIT_FAIL(test, "Error creating bo: %pe\n", bo);
498 kfree(link);
499 failed = true;
500 break;
501 }
502 xe_bo_lock(bo, false);
503 xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm);
504
505 /*
506 * Allocate purgeable bos first, because if we do it the
507 * other way around, they may not be subject to swapping...
508 */
509 if (alloced < purgeable) {
510 xe_tt->purgeable = true;
511 bo->ttm.priority = 0;
512 } else {
513 int ret = shrink_test_fill_random(bo, &prng, link);
514
515 if (ret) {
516 xe_bo_unlock(bo);
517 xe_bo_put(bo);
518 KUNIT_FAIL(test, "Error filling bo with random data: %pe\n",
519 ERR_PTR(ret));
520 kfree(link);
521 failed = true;
522 break;
523 }
524 }
525
526 mem_type = bo->ttm.resource->mem_type;
527 xe_bo_unlock(bo);
528 link->bo = bo;
529 list_add_tail(&link->link, &bos);
530
531 if (mem_type != XE_PL_TT) {
532 KUNIT_FAIL(test, "Bo in incorrect memory type: %u\n",
533 bo->ttm.resource->mem_type);
534 failed = true;
535 }
536 cond_resched();
537 if (signal_pending(current))
538 break;
539 }
540
541 /*
542 * Read back and destroy bos. Reset the pseudo-random seed to get an
543 * identical pseudo-random number sequence for readback.
544 */
545 prandom_seed_state(&prng, rand_seed);
546 list_for_each_entry_safe(link, next, &bos, link) {
547 static struct ttm_operation_ctx ctx = {.interruptible = true};
548 struct xe_bo *bo = link->bo;
549 struct xe_ttm_tt *xe_tt;
550 int ret;
551
552 count++;
553 if (!signal_pending(current) && !failed) {
554 bool purgeable, intr = false;
555
556 xe_bo_lock(bo, NULL);
557
558 /* xe_tt->purgeable is cleared on validate. */
559 xe_tt = container_of(bo->ttm.ttm, typeof(*xe_tt), ttm);
560 purgeable = xe_tt->purgeable;
561 do {
562 ret = ttm_bo_validate(&bo->ttm, &tt_placement, &ctx);
563 if (ret == -EINTR)
564 intr = true;
565 } while (ret == -EINTR && !signal_pending(current));
566
567 if (!ret && !purgeable)
568 failed = shrink_test_verify(test, bo, count, &prng, link);
569
570 xe_bo_unlock(bo);
571 if (ret) {
572 KUNIT_FAIL(test, "Validation failed: %pe\n",
573 ERR_PTR(ret));
574 failed = true;
575 } else if (intr) {
576 interrupted++;
577 } else {
578 successful++;
579 }
580 }
581 xe_bo_put(link->bo);
582 list_del(&link->link);
583 kfree(link);
584 }
585 kunit_info(test, "Readbacks interrupted: %u successful: %u\n",
586 interrupted, successful);
587
588 return 0;
589}
590
591static void xe_bo_shrink_kunit(struct kunit *test)
592{
593 struct xe_device *xe = test->priv;
594
595 shrink_test_run_device(xe);
596}
597
598static struct kunit_case xe_bo_tests[] = {
599 KUNIT_CASE_PARAM(xe_ccs_migrate_kunit, xe_pci_live_device_gen_param),
600 KUNIT_CASE_PARAM(xe_bo_evict_kunit, xe_pci_live_device_gen_param),
601 KUNIT_CASE_PARAM_ATTR(xe_bo_shrink_kunit, xe_pci_live_device_gen_param,
602 {.speed = KUNIT_SPEED_SLOW}),
603 {}
604};
605
606VISIBLE_IF_KUNIT
607struct kunit_suite xe_bo_test_suite = {
608 .name = "xe_bo",
609 .test_cases = xe_bo_tests,
610 .init = xe_kunit_helper_xe_device_live_test_init,
611};
612EXPORT_SYMBOL_IF_KUNIT(xe_bo_test_suite);